User-agent: * Disallow: Allow: / Sitemap: https://addisoncrump.info/sitemap.xml # Do not call Dark Visitors API when using local server# DarkVisitors discourage bot list# Scraper # https://knownagents.com/agents/008 User-agent: 008 Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/a360-search User-agent: A360-Search Disallow: / # SEO Crawler # https://knownagents.com/agents/aa User-agent: aa Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/activecomply User-agent: ActiveComply Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/adagiobot User-agent: Adagiobot Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/adbeat-bot User-agent: adbeat_bot Disallow: / # AI Search Crawler # https://knownagents.com/agents/addsearchbot User-agent: AddSearchBot Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/adidxbot User-agent: adidxbot Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/adkerneltopiccrawler User-agent: AdkernelTopicCrawler Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/admantx User-agent: admantx Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/admixerbot User-agent: AdmixerBot Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/adsbot-google User-agent: AdsBot-Google Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/adsbot-google-mobile User-agent: AdsBot-Google-Mobile Disallow: / # Scraper # https://knownagents.com/agents/adstxtcrawler User-agent: AdsTxtCrawler Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/affsignalcrawler User-agent: AffsignalCrawler Disallow: / # SEO Crawler # https://knownagents.com/agents/ahrefs-site-audit User-agent: Ahrefs Site Audit Disallow: / # SEO Crawler # https://knownagents.com/agents/ahrefsbot User-agent: AhrefsBot Disallow: / # SEO Crawler # https://knownagents.com/agents/ahrefssiteaudit User-agent: AhrefsSiteAudit Disallow: / # AI Data Scraper # https://knownagents.com/agents/ai2bot User-agent: AI2Bot Disallow: / # AI Assistant # https://knownagents.com/agents/ai2bot-deepresearcheval User-agent: AI2Bot-DeepResearchEval Disallow: / # AI Data Scraper # https://knownagents.com/agents/ai2bot-dolma User-agent: Ai2Bot-Dolma Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/aihitbot User-agent: aiHitBot Disallow: / # AI Search Crawler # https://knownagents.com/agents/aiwebindex User-agent: AIWebIndex Disallow: / # AI Search Crawler # https://knownagents.com/agents/aiwebindex-agent User-agent: AIWebIndex-Agent Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/amazing-searchbot User-agent: Amazing-SearchBot Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/amazingtest-searchbot User-agent: AmazingTest-SearchBot Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/amazon-adbot User-agent: Amazon AdBot Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/amazon-advertising-ad-standards-bot User-agent: Amazon-Advertising-ad-standards-bot Disallow: / # AI Search Crawler # https://knownagents.com/agents/amazon-kendra User-agent: amazon-kendra Disallow: / # AI Assistant # https://knownagents.com/agents/amazon-qbusiness User-agent: amazon-QBusiness Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/amazonadbot User-agent: AmazonAdBot Disallow: / # AI Data Scraper # https://knownagents.com/agents/amazonbot User-agent: Amazonbot Disallow: / # AI Agent # https://knownagents.com/agents/amazonbuyforme User-agent: AmazonBuyForMe Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/amazonproductdiscovery User-agent: AmazonProductDiscovery Disallow: / # AI Search Crawler # https://knownagents.com/agents/amzn-searchbot User-agent: Amzn-SearchBot Disallow: / # AI Assistant # https://knownagents.com/agents/amzn-user User-agent: Amzn-User Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/anderspinkbot User-agent: AndersPinkBot Disallow: / # AI Search Crawler # https://knownagents.com/agents/anomura User-agent: Anomura Disallow: / # Undocumented AI Agent # https://knownagents.com/agents/anthropic-ai User-agent: anthropic-ai Disallow: / # AI Search Crawler # https://knownagents.com/agents/applebot User-agent: Applebot Disallow: / # AI Data Scraper # https://knownagents.com/agents/applebot-extended User-agent: Applebot-Extended Disallow: / # Undocumented AI Agent # https://knownagents.com/agents/aranet-searchbot User-agent: Aranet-SearchBot Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/archivebot User-agent: ArchiveBot Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/assaybot User-agent: Assaybot Disallow: / # AI Search Crawler # https://knownagents.com/agents/atlassian-bot User-agent: atlassian-bot Disallow: / # SEO Crawler # https://knownagents.com/agents/attracta User-agent: Attracta Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/audigentadbot User-agent: AudigentAdBot Disallow: / # SEO Crawler # https://knownagents.com/agents/audisto-crawler User-agent: Audisto Crawler Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/automattic-analytics-crawler User-agent: Automattic Analytics Crawler Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/awario User-agent: Awario Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/awariobot User-agent: AwarioBot Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/awariosmartbot User-agent: AwarioSmartBot Disallow: / # AI Search Crawler # https://knownagents.com/agents/azureai-searchbot User-agent: AzureAI-SearchBot Disallow: / # SEO Crawler # https://knownagents.com/agents/barkrowler User-agent: Barkrowler Disallow: / # AI Data Scraper # https://knownagents.com/agents/bedrockbot User-agent: bedrockbot Disallow: / # AI Assistant # https://knownagents.com/agents/bigsur-ai User-agent: bigsur.ai Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/bigupdatabot User-agent: BigUpDataBot Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/bing-ads User-agent: Bing Ads Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/bitsightbot User-agent: BitSightBot Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/blackboard User-agent: Blackboard Disallow: / # SEO Crawler # https://knownagents.com/agents/blexbot User-agent: BLEXBot Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/bomborabot User-agent: BomboraBot Disallow: / # SEO Crawler # https://knownagents.com/agents/botify User-agent: botify Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/brandverity User-agent: BrandVerity Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/brightedge-bot User-agent: BrightEdge Bot Disallow: / # SEO Crawler # https://knownagents.com/agents/brightedge-crawler User-agent: BrightEdge Crawler Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/buck User-agent: Buck Disallow: / # AI Data Scraper # https://knownagents.com/agents/bytespider User-agent: Bytespider Disallow: / # SEO Crawler # https://knownagents.com/agents/caliberbot User-agent: CaliberBot Disallow: / # SEO Crawler # https://knownagents.com/agents/caliperbot User-agent: Caliperbot Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/captify-crawler User-agent: captify-crawler Disallow: / # AI Data Scraper # https://knownagents.com/agents/ccbot User-agent: CCBot Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/channable User-agent: channable Disallow: / # AI Search Crawler # https://knownagents.com/agents/channel3bot User-agent: Channel3Bot Disallow: / # AI Data Scraper # https://knownagents.com/agents/chatglm-spider User-agent: ChatGLM-Spider Disallow: / # AI Agent # https://knownagents.com/agents/chatgpt-agent User-agent: ChatGPT Agent Disallow: / # AI Assistant # https://knownagents.com/agents/chatgpt-user User-agent: ChatGPT-User Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/checkmarknetwork User-agent: CheckMarkNetwork Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/cincraw User-agent: Cincraw Disallow: / # SEO Crawler # https://knownagents.com/agents/claritybot User-agent: ClarityBot Disallow: / # AI Search Crawler # https://knownagents.com/agents/claude-searchbot User-agent: Claude-SearchBot Disallow: / # AI Assistant # https://knownagents.com/agents/claude-user User-agent: Claude-User Disallow: / # Undocumented AI Agent # https://knownagents.com/agents/claude-web User-agent: Claude-Web Disallow: / # AI Data Scraper # https://knownagents.com/agents/claudebot User-agent: ClaudeBot Disallow: / # SEO Crawler # https://knownagents.com/agents/clearscopebot User-agent: Clearscopebot Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/clickagy-intelligence-bot User-agent: Clickagy Intelligence Bot Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/clickagy-intelligence-bot-v2 User-agent: Clickagy Intelligence Bot v2 Disallow: / # SEO Crawler # https://knownagents.com/agents/cloudflare-custom-hostname-verification User-agent: Cloudflare Custom Hostname Verification Disallow: / # SEO Crawler # https://knownagents.com/agents/cloudflare-stream-webhook User-agent: Cloudflare Stream Webhook Disallow: / # AI Search Crawler # https://knownagents.com/agents/cloudflare-autorag User-agent: Cloudflare-AutoRAG Disallow: / # SEO Crawler # https://knownagents.com/agents/cloudflare-healthchecks User-agent: Cloudflare-Healthchecks Disallow: / # SEO Crawler # https://knownagents.com/agents/cloudflare-prefetch User-agent: CloudFlare-Prefetch Disallow: / # SEO Crawler # https://knownagents.com/agents/cloudflare-ssldetector User-agent: Cloudflare-SSLDetector Disallow: / # SEO Crawler # https://knownagents.com/agents/cloudflare-traffic-manager User-agent: Cloudflare-Traffic-Manager Disallow: / # SEO Crawler # https://knownagents.com/agents/cloudtrellis User-agent: Cloudtrellis Disallow: / # AI Data Scraper # https://knownagents.com/agents/cloudvertexbot User-agent: CloudVertexBot Disallow: / # SEO Crawler # https://knownagents.com/agents/cludo-com User-agent: cludo.com Disallow: / # SEO Crawler # https://knownagents.com/agents/cocolyzebot User-agent: Cocolyzebot Disallow: / # SEO Crawler # https://knownagents.com/agents/cognitiveseo-crawler User-agent: cognitiveSEO Crawler Disallow: / # Undocumented AI Agent # https://knownagents.com/agents/cohere-ai User-agent: cohere-ai Disallow: / # AI Data Scraper # https://knownagents.com/agents/cohere-training-data-crawler User-agent: cohere-training-data-crawler Disallow: / # AI Data Scraper # https://knownagents.com/agents/coherebot User-agent: CohereBot Disallow: / # SEO Crawler # https://knownagents.com/agents/contentking User-agent: contentking Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/contextualbot User-agent: ContextualBot Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/contxbot User-agent: contxbot Disallow: / # SEO Crawler # https://knownagents.com/agents/convermax User-agent: Convermax Disallow: / # AI Data Scraper # https://knownagents.com/agents/cotoyogi User-agent: Cotoyogi Disallow: / # Undocumented AI Agent # https://knownagents.com/agents/crawl4ai User-agent: Crawl4AI Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/criteobot User-agent: CriteoBot Disallow: / # SEO Crawler # https://knownagents.com/agents/cxense User-agent: Cxense Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/cxensebot User-agent: cXensebot Disallow: / # SEO Crawler # https://knownagents.com/agents/dataforseo User-agent: DataForSEO Disallow: / # SEO Crawler # https://knownagents.com/agents/dataforseo-bot User-agent: DataForSEO Bot Disallow: / # SEO Crawler # https://knownagents.com/agents/dataforseobot User-agent: DataForSeoBot Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/dataprovider User-agent: Dataprovider Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/dataprovider-com User-agent: Dataprovider.com Disallow: / # AI Data Scraper # https://knownagents.com/agents/datenbank-crawler User-agent: Datenbank Crawler Disallow: / # Scraper # https://knownagents.com/agents/dcrawl User-agent: dcrawl Disallow: / # SEO Crawler # https://knownagents.com/agents/deepcrawl User-agent: deepcrawl Disallow: / # SEO Crawler # https://knownagents.com/agents/deepcrawl-com User-agent: deepcrawl.com Disallow: / # AI Data Scraper # https://knownagents.com/agents/deepseekbot User-agent: DeepSeekBot Disallow: / # SEO Crawler # https://knownagents.com/agents/domainstatsbot User-agent: DomainStatsBot Disallow: / # SEO Crawler # https://knownagents.com/agents/dotbot User-agent: DotBot Disallow: / # AI Data Scraper # https://knownagents.com/agents/doubaobot User-agent: Doubaobot Disallow: / # SEO Crawler # https://knownagents.com/agents/dragonbot User-agent: Dragonbot Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/dragonfly User-agent: Dragonfly Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/ds9 User-agent: ds9 Disallow: / # AI Assistant # https://knownagents.com/agents/duckassistbot User-agent: DuckAssistBot Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/dvbot User-agent: DVbot Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/echobot-bot User-agent: Echobot Bot Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/ecovadissustainabilitybot User-agent: EcoVadisSustainabilityBot Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/epivozcrawler User-agent: EpivozCrawler Disallow: / # AI Data Scraper # https://knownagents.com/agents/erniebot User-agent: ERNIEBot Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/ev-crawler User-agent: ev-crawler Disallow: / # AI Search Crawler # https://knownagents.com/agents/exasearchbot User-agent: ExaSearchBot Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/ezlynx User-agent: EzLynx Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/ezoicbot User-agent: EzoicBot Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/ezoicbot-nicheiq User-agent: EzoicBot-Nicheiq Disallow: / # AI Data Scraper # https://knownagents.com/agents/facebookbot User-agent: FacebookBot Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/factset-spyderbot User-agent: factset_spyderbot Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/fdl-stats-bot User-agent: FDL Stats Bot Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/fedicabot User-agent: Fedicabot Disallow: / # AI Assistant # https://knownagents.com/agents/geisthaus-pagefetcher User-agent: GeistHaus-PageFetcher Disallow: / # AI Assistant # https://knownagents.com/agents/gemini-deep-research User-agent: Gemini-Deep-Research Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/gnowitnewsbot User-agent: GnowitNewsbot Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/google-adsbot User-agent: Google AdsBot Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/google-adsense User-agent: Google AdSense Disallow: / # SEO Crawler # https://knownagents.com/agents/google-digital-asset-links User-agent: Google Digital Asset Links Disallow: / # SEO Crawler # https://knownagents.com/agents/google-storebot User-agent: Google StoreBot Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/google-adstxt User-agent: Google-adstxt Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/google-adwords-instant User-agent: Google-Adwords-Instant Disallow: / # AI Agent # https://knownagents.com/agents/google-agent User-agent: Google-Agent Disallow: / # AI Search Crawler # https://knownagents.com/agents/google-cloudvertexbot User-agent: Google-CloudVertexBot Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/google-display-ads-bot User-agent: Google-Display-Ads-Bot Disallow: / # AI Data Scraper # https://knownagents.com/agents/google-extended User-agent: Google-Extended Disallow: / # AI Assistant # https://knownagents.com/agents/google-notebooklm User-agent: Google-NotebookLM Disallow: / # SEO Crawler # https://knownagents.com/agents/google-structured-data-testing User-agent: Google-Structured-Data-Testing Disallow: / # AI Agent # https://knownagents.com/agents/googleagent-mariner User-agent: GoogleAgent-Mariner Disallow: / # AI Assistant # https://knownagents.com/agents/googleagent-urlcontext User-agent: GoogleAgent-URLContext Disallow: / # AI Data Scraper # https://knownagents.com/agents/googleother User-agent: GoogleOther Disallow: / # AI Data Scraper # https://knownagents.com/agents/gptbot User-agent: GPTBot Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/grapeshotcrawler User-agent: GrapeshotCrawler Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/groovinaadsbot User-agent: GroovinaAdsbot Disallow: / # Scraper # https://knownagents.com/agents/helloworkjobpostingbot User-agent: HelloworkJobPostingBot Disallow: / # Scraper # https://knownagents.com/agents/httrack User-agent: HTTrack Disallow: / # Scraper # https://knownagents.com/agents/httrack-3-0 User-agent: HTTrack 3.0 Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/hubspot User-agent: HubSpot Disallow: / # SEO Crawler # https://knownagents.com/agents/huckabot User-agent: Huckabot Disallow: / # SEO Crawler # https://knownagents.com/agents/huckabuy-bot User-agent: Huckabuy Bot Disallow: / # SEO Crawler # https://knownagents.com/agents/hypestat User-agent: hypestat Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/ias-crawler User-agent: IAS crawler Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/ias-ie User-agent: ias-ie Disallow: / # Undocumented AI Agent # https://knownagents.com/agents/iaskbot User-agent: iAskBot Disallow: / # Undocumented AI Agent # https://knownagents.com/agents/iaskspider User-agent: iaskspider Disallow: / # AI Data Scraper # https://knownagents.com/agents/icc-crawler User-agent: ICC-Crawler Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/idgit User-agent: IDG/IT Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/imagesiftbot User-agent: ImagesiftBot Disallow: / # AI Data Scraper # https://knownagents.com/agents/imagespider User-agent: imageSpider Disallow: / # Scraper # https://knownagents.com/agents/indeedjobbot User-agent: IndeedJobBot Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/innguma User-agent: Innguma Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/interactiveadvertisingbureau User-agent: InteractiveAdvertisingBureau Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/ioncrawl User-agent: IonCrawl Disallow: / # SEO Crawler # https://knownagents.com/agents/james-bot User-agent: James BOT Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/jugendschutzprogramm-crawler User-agent: Jugendschutzprogramm-Crawler Disallow: / # AI Assistant # https://knownagents.com/agents/kagi-fetcher User-agent: kagi-fetcher Disallow: / # AI Data Scraper # https://knownagents.com/agents/kangaroo-bot User-agent: Kangaroo Bot Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/kargobot User-agent: KargoBot Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/kargobot-artemis-mobile User-agent: KargoBot-Artemis-Mobile Disallow: / # AI Search Crawler # https://knownagents.com/agents/kimi-searchbot User-agent: Kimi-SearchBot Disallow: / # AI Assistant # https://knownagents.com/agents/kimi-user User-agent: Kimi-User Disallow: / # AI Data Scraper # https://knownagents.com/agents/kimibot User-agent: KimiBot Disallow: / # AI Data Scraper # https://knownagents.com/agents/kimicrawler User-agent: KimiCrawler Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/klaviyo User-agent: Klaviyo Disallow: / # AI Assistant # https://knownagents.com/agents/klaviyoaibot User-agent: KlaviyoAIBot Disallow: / # Scraper # https://knownagents.com/agents/krawlerbot User-agent: KrawlerBot Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/kstandbot User-agent: KStandBot Disallow: / # Undocumented AI Agent # https://knownagents.com/agents/kunatocrawler User-agent: KunatoCrawler Disallow: / # AI Data Scraper # https://knownagents.com/agents/laion-huggingface-processor User-agent: laion-huggingface-processor Disallow: / # AI Data Scraper # https://knownagents.com/agents/lcc User-agent: LCC Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/leikibot User-agent: Leikibot Disallow: / # SEO Crawler # https://knownagents.com/agents/librecrawl User-agent: LibreCrawl Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/lightspeedsystemscrawler User-agent: LightspeedSystemsCrawler Disallow: / # AI Assistant # https://knownagents.com/agents/linerbot User-agent: LinerBot Disallow: / # SEO Crawler # https://knownagents.com/agents/linkcheck-by-siteimprove User-agent: LinkCheck by Siteimprove Disallow: / # SEO Crawler # https://knownagents.com/agents/linkchecker-bot User-agent: LinkChecker Bot Disallow: / # SEO Crawler # https://knownagents.com/agents/linkchecker-pro User-agent: linkchecker.pro Disallow: / # SEO Crawler # https://knownagents.com/agents/linkcheckerbot User-agent: LinkCheckerBot Disallow: / # SEO Crawler # https://knownagents.com/agents/linkdexbot User-agent: linkdexbot Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/linkfluence User-agent: linkfluence Disallow: / # SEO Crawler # https://knownagents.com/agents/linksindexerbot User-agent: LinksIndexerBot Disallow: / # AI Search Crawler # https://knownagents.com/agents/linkupbot User-agent: LinkupBot Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/linkwalker User-agent: LinkWalker Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/macrobondbot User-agent: Macrobondbot Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/magpie-crawler User-agent: magpie-crawler Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/mantisbot User-agent: Mantisbot Disallow: / # AI Agent # https://knownagents.com/agents/manus-user User-agent: Manus-User Disallow: / # SEO Crawler # https://knownagents.com/agents/marketgoo User-agent: MarketGoo Disallow: / # SEO Crawler # https://knownagents.com/agents/mbcrawler User-agent: MBCrawler Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/mcontextualbot User-agent: McontextualBot Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/mediaboardbot User-agent: mediaboardbot Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/medialogiabot User-agent: MedialogiaBot Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/mediamonitoringbot User-agent: MediaMonitoringBot Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/mediapartners-google User-agent: Mediapartners-Google Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/mediatoolkitbot User-agent: Mediatoolkitbot Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/mediavine-medatada-parser User-agent: Mediavine Medatada Parser Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/mediavinemetadataparser User-agent: MediavineMetadataParser Disallow: / # SEO Crawler # https://knownagents.com/agents/megaindex-ru User-agent: MegaIndex.ru Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/meta-externalads User-agent: meta-externalads Disallow: / # AI Data Scraper # https://knownagents.com/agents/meta-externalagent User-agent: meta-externalagent Disallow: / # AI Assistant # https://knownagents.com/agents/meta-externalfetcher User-agent: meta-externalfetcher Disallow: / # AI Search Crawler # https://knownagents.com/agents/meta-webindexer User-agent: meta-webindexer Disallow: / # Scraper # https://knownagents.com/agents/metainspector User-agent: MetaInspector Disallow: / # Scraper # https://knownagents.com/agents/metajobbot User-agent: MetaJobBot Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/mgidbot User-agent: MgidBot Disallow: / # AI Data Scraper # https://knownagents.com/agents/micro-crawl User-agent: micro-crawl Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/missinglettr-bot User-agent: Missinglettr Bot Disallow: / # AI Search Crawler # https://knownagents.com/agents/mistralai-index User-agent: MistralAI-Index Disallow: / # AI Assistant # https://knownagents.com/agents/mistralai-user User-agent: MistralAI-User Disallow: / # AI Data Scraper # https://knownagents.com/agents/mistralbot User-agent: MistralBot Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/mixrankbot User-agent: MixrankBot Disallow: / # SEO Crawler # https://knownagents.com/agents/mj12bot User-agent: MJ12bot Disallow: / # SEO Crawler # https://knownagents.com/agents/modernizebot User-agent: ModernizeBot Disallow: / # AI Data Scraper # https://knownagents.com/agents/moonshotbot User-agent: MoonshotBot Disallow: / # AI Data Scraper # https://knownagents.com/agents/moonspider User-agent: MoonSpider Disallow: / # SEO Crawler # https://knownagents.com/agents/moz-dotbot User-agent: Moz dotbot Disallow: / # SEO Crawler # https://knownagents.com/agents/moz-rogerbot User-agent: Moz rogerbot Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/muckrack User-agent: MuckRack Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/nanointeractive User-agent: NanoInteractive Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/netcraft User-agent: Netcraft Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/netcraftsurveyagent User-agent: NetcraftSurveyAgent Disallow: / # AI Data Scraper # https://knownagents.com/agents/netestate-imprint-crawler User-agent: netEstate Imprint Crawler Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/netestate-ne-crawler User-agent: netEstate NE Crawler Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/neticle-crawler User-agent: Neticle Crawler Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/netseer-crawler User-agent: NetSeer crawler Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/netvibes User-agent: Netvibes Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/neuwobot User-agent: NeuwoBot Disallow: / # Scraper # https://knownagents.com/agents/newspaper User-agent: newspaper Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/newsroom-bi User-agent: NewsRoom.BI Disallow: / # SEO Crawler # https://knownagents.com/agents/nitro User-agent: Nitro- Disallow: / # SEO Crawler # https://knownagents.com/agents/nitrobot User-agent: NitroBot Disallow: / # SEO Crawler # https://knownagents.com/agents/noorobot User-agent: noorobot Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/nostocrawlerbot User-agent: NostoCrawlerBot Disallow: / # AI Agent # https://knownagents.com/agents/novaact User-agent: NovaAct Disallow: / # Scraper # https://knownagents.com/agents/nutch User-agent: Nutch Disallow: / # Scraper # https://knownagents.com/agents/nyt-scraping User-agent: nyt_scraping Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/oai-adsbot User-agent: OAI-AdsBot Disallow: / # AI Search Crawler # https://knownagents.com/agents/oai-searchbot User-agent: OAI-SearchBot Disallow: / # Scraper # https://knownagents.com/agents/offline-explorer User-agent: Offline Explorer Disallow: / # AI Data Scraper # https://knownagents.com/agents/omgili User-agent: omgili Disallow: / # SEO Crawler # https://knownagents.com/agents/oncrawl User-agent: OnCrawl Disallow: / # SEO Crawler # https://knownagents.com/agents/oncrawl-mobile User-agent: OnCrawl Mobile Disallow: / # SEO Crawler # https://knownagents.com/agents/online-webceo-bot User-agent: online-webceo-bot Disallow: / # Scraper # https://knownagents.com/agents/openindexspider User-agent: OpenindexSpider Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/outboundbuddy-enrichment User-agent: outboundbuddy-enrichment Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/outbrain User-agent: outbrain Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/owler User-agent: Owler Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/page-preview-tool User-agent: page-preview-tool Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/pandalytics User-agent: Pandalytics Disallow: / # AI Data Scraper # https://knownagents.com/agents/pangubot User-agent: PanguBot Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/panscient-com User-agent: panscient.com Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/parse-ly-scraper User-agent: parse.ly scraper Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/partneroptimizer-bot User-agent: PartnerOptimizer-bot Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/peer39-crawler User-agent: peer39_crawler Disallow: / # AI Assistant # https://knownagents.com/agents/perplexity-user User-agent: Perplexity-User Disallow: / # AI Search Crawler # https://knownagents.com/agents/perplexitybot User-agent: PerplexityBot Disallow: / # AI Search Crawler # https://knownagents.com/agents/petalbot User-agent: PetalBot Disallow: / # AI Assistant # https://knownagents.com/agents/phindbot User-agent: PhindBot Disallow: / # AI Assistant # https://knownagents.com/agents/poggio-citations User-agent: Poggio-Citations Disallow: / # Scraper # https://knownagents.com/agents/potions User-agent: Potions Disallow: / # SEO Crawler # https://knownagents.com/agents/pro-sitemaps User-agent: Pro Sitemaps Disallow: / # SEO Crawler # https://knownagents.com/agents/pro-sitemaps-generator User-agent: Pro Sitemaps Generator Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/productsup-iocrawler User-agent: productsup.io/crawler Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/proximic User-agent: proximic Disallow: / # AI Assistant # https://knownagents.com/agents/qualifiedbot User-agent: QualifiedBot Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/quantcastbot User-agent: Quantcastbot Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/quuubot User-agent: QuuuBot Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/qwarrybot User-agent: Qwarrybot Disallow: / # AI Data Scraper # https://knownagents.com/agents/qwenbot User-agent: QwenBot Disallow: / # SEO Crawler # https://knownagents.com/agents/rankmath-link-checker User-agent: RankMath Link Checker Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/rankurbot User-agent: RankurBot Disallow: / # SEO Crawler # https://knownagents.com/agents/ravencrawler User-agent: RavenCrawler Disallow: / # SEO Crawler # https://knownagents.com/agents/readable User-agent: Readable Disallow: / # AI Data Scraper # https://knownagents.com/agents/reflectionbot User-agent: Reflectionbot Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/retrolistecom User-agent: RetroListeCOM Disallow: / # SEO Crawler # https://knownagents.com/agents/revvimgort User-agent: RevvimGort Disallow: / # SEO Crawler # https://knownagents.com/agents/rogerbot-crawler User-agent: rogerbot-crawler Disallow: / # SEO Crawler # https://knownagents.com/agents/rsiteauditor User-agent: RSiteAuditor Disallow: / # SEO Crawler # https://knownagents.com/agents/rytebot User-agent: RyteBot Disallow: / # AI Data Scraper # https://knownagents.com/agents/salamandravlm User-agent: SalamandraVLM Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/salesviewerbot User-agent: SalesViewerBot Disallow: / # AI Data Scraper # https://knownagents.com/agents/sbintuitionsbot User-agent: SBIntuitionsBot Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/scoop-it User-agent: scoop.it Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/scope3 User-agent: Scope3 Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/scrapingnytimes-com User-agent: scraping@nytimes.com Disallow: / # Scraper # https://knownagents.com/agents/scrapy User-agent: Scrapy Disallow: / # SEO Crawler # https://knownagents.com/agents/screaming-frog-seo-spider User-agent: Screaming Frog SEO Spider Disallow: / # SEO Crawler # https://knownagents.com/agents/searchatlas-bot User-agent: SearchAtlas Bot Disallow: / # SEO Crawler # https://knownagents.com/agents/sebot-wa User-agent: SEBot-WA Disallow: / # SEO Crawler # https://knownagents.com/agents/semrushbot User-agent: SemrushBot Disallow: / # SEO Crawler # https://knownagents.com/agents/semrushbot-ba User-agent: SemrushBot-BA Disallow: / # SEO Crawler # https://knownagents.com/agents/semrushbot-ct User-agent: SemrushBot-CT Disallow: / # SEO Crawler # https://knownagents.com/agents/semrushbot-ocob User-agent: SemrushBot-OCOB Disallow: / # SEO Crawler # https://knownagents.com/agents/semrushbot-sa User-agent: SemrushBot-SA Disallow: / # SEO Crawler # https://knownagents.com/agents/semrushbot-si User-agent: SemrushBot-SI Disallow: / # SEO Crawler # https://knownagents.com/agents/semrushbotbacklinks User-agent: SemrushBotBacklinks Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/sentibot User-agent: SentiBot Disallow: / # SEO Crawler # https://knownagents.com/agents/senutobot User-agent: Senutobot Disallow: / # SEO Crawler # https://knownagents.com/agents/seo-audit-check-bot User-agent: seo-audit-check-bot Disallow: / # SEO Crawler # https://knownagents.com/agents/seo4ajax User-agent: seo4ajax Disallow: / # SEO Crawler # https://knownagents.com/agents/seo4ajax-com User-agent: seo4ajax.com Disallow: / # SEO Crawler # https://knownagents.com/agents/seobilitybot User-agent: SeobilityBot Disallow: / # SEO Crawler # https://knownagents.com/agents/seokicks User-agent: SEOkicks Disallow: / # SEO Crawler # https://knownagents.com/agents/seolizer User-agent: SEOlizer Disallow: / # SEO Crawler # https://knownagents.com/agents/serankingbacklinksbot User-agent: SERankingBacklinksBot Disallow: / # SEO Crawler # https://knownagents.com/agents/serpminerbot User-agent: SerpMinerBot Disallow: / # SEO Crawler # https://knownagents.com/agents/serpstatbot User-agent: serpstatbot Disallow: / # Scraper # https://knownagents.com/agents/serverhunterspider User-agent: ServerHunterSpider Disallow: / # AI Assistant # https://knownagents.com/agents/shap-user User-agent: Shap-User Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/sirdatabot User-agent: SirdataBot Disallow: / # SEO Crawler # https://knownagents.com/agents/siteauditbot User-agent: SiteAuditBot Disallow: / # SEO Crawler # https://knownagents.com/agents/sitebulb User-agent: Sitebulb Disallow: / # SEO Crawler # https://knownagents.com/agents/sitecheck-sitecrawl User-agent: SiteCheck-sitecrawl Disallow: / # SEO Crawler # https://knownagents.com/agents/sitecheckerbotcrawler User-agent: SiteCheckerBotCrawler Disallow: / # SEO Crawler # https://knownagents.com/agents/sitegurucrawler User-agent: SiteGuruCrawler Disallow: / # SEO Crawler # https://knownagents.com/agents/siteimprove-crawl User-agent: Siteimprove Crawl Disallow: / # SEO Crawler # https://knownagents.com/agents/siteliner User-agent: Siteliner Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/slickbot User-agent: SlickBot Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/slickstream User-agent: slickstream Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/smartologybot User-agent: SmartologyBot Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/smtbot User-agent: SMTBot Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/sottopop User-agent: Sottopop Disallow: / # AI Data Scraper # https://knownagents.com/agents/spider User-agent: Spider Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/statistikaustria User-agent: StatistikAustria Disallow: / # Scraper # https://knownagents.com/agents/statsdronebot User-agent: StatsDroneBot Disallow: / # SEO Crawler # https://knownagents.com/agents/statusnestbacklinkspider User-agent: StatusNestBacklinkSpider Disallow: / # SEO Crawler # https://knownagents.com/agents/storebot-google User-agent: Storebot-Google Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/stripebot User-agent: Stripebot Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/taboola User-agent: Taboola Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/taboolabot User-agent: Taboolabot Disallow: / # AI Search Crawler # https://knownagents.com/agents/talarionsearchbot User-agent: TalarionSearchBot Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/techspybot User-agent: TechSpyBot Disallow: / # Scraper # https://knownagents.com/agents/thinkbot User-agent: Thinkbot Disallow: / # AI Data Scraper # https://knownagents.com/agents/timpibot User-agent: Timpibot Disallow: / # AI Assistant # https://knownagents.com/agents/tongyibot User-agent: TongyiBot Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/trendictionbot User-agent: trendictionbot Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/trendsmapresolver User-agent: TrendsmapResolver Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/ttd-content User-agent: TTD-Content Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/turnitinbot User-agent: TurnitinBot Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/tweetmemebot User-agent: TweetmemeBot Disallow: / # AI Agent # https://knownagents.com/agents/twinagent User-agent: TwinAgent Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/twingly User-agent: Twingly Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/twingly-recon-sjostrom User-agent: Twingly Recon-Sjostrom Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/uipbot User-agent: uipbot Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/um-fc User-agent: um-FC Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/um-ic User-agent: um-IC Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/um-ln User-agent: um-LN Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/upcontent User-agent: UpContent Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/upcontent-qa User-agent: UpContent-QA Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/upcontent-test User-agent: UpContent-Test Disallow: / # AI Assistant # https://knownagents.com/agents/useai User-agent: UseAI Disallow: / # AI Data Scraper # https://knownagents.com/agents/velenpublicwebcrawler User-agent: VelenPublicWebCrawler Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/verity User-agent: Verity Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/virustotal User-agent: virustotal Disallow: / # SEO Crawler # https://knownagents.com/agents/w3-validator-services User-agent: W3 Validator Services Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/weborama-fetcher User-agent: weborama-fetcher Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/webspidermount User-agent: webspidermount Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/webtrackrcrawler User-agent: WebTrackrCrawler Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/webzio User-agent: webzio Disallow: / # AI Data Scraper # https://knownagents.com/agents/webzio-extended User-agent: webzio-extended Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/wknd-bot User-agent: wknd-bot Disallow: / # SEO Crawler # https://knownagents.com/agents/woorankreview User-agent: woorankreview Disallow: / # Undocumented AI Agent # https://knownagents.com/agents/wrtnbot User-agent: WRTNBot Disallow: / # AI Search Crawler # https://knownagents.com/agents/xai-searchbot User-agent: xAI-SearchBot Disallow: / # SEO Crawler # https://knownagents.com/agents/xovionpagecrawler User-agent: XoviOnpageCrawler Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/yadirectfetcher User-agent: YaDirectFetcher Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/yahoo-ad-monitoring User-agent: Yahoo Ad monitoring Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/yahoo-japan User-agent: Yahoo Japan Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/yak User-agent: YaK Disallow: / # AI Search Crawler # https://knownagents.com/agents/yandexadditional User-agent: YandexAdditional Disallow: / # AI Search Crawler # https://knownagents.com/agents/yandexadditionalbot User-agent: YandexAdditionalBot Disallow: / # AI Search Crawler # https://knownagents.com/agents/yepbot User-agent: YepBot Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/yext-inc User-agent: Yext Inc Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/yextbot User-agent: YextBot Disallow: / # AI Assistant # https://knownagents.com/agents/yiyanbot User-agent: YiyanBot Disallow: / # AI Search Crawler # https://knownagents.com/agents/zanistabot User-agent: ZanistaBot Disallow: / # Undocumented AI Agent # https://knownagents.com/agents/zenphonybot User-agent: ZenphonyBot Disallow: / # SEO Crawler # https://knownagents.com/agents/zoombot User-agent: Zoombot Disallow: / # Intelligence Gatherer # https://knownagents.com/agents/zoominfobot User-agent: ZoominfoBot Disallow: /