# Generated by AI Bot Manager # AI bots and search engines may crawl and use public content. # Updated: 2025-11-02 07:26:41 # DEFAULT RULE - Allow everything User-agent: * Allow: / # Sitemap Sitemap: https://tao4all.com/sitemap.xml # Search Engines and AI Bots User-agent: ChatGPT Allow: / User-agent: OpenAI GPTBot Allow: / User-agent: Claude Allow: / User-agent: ClaudeBot Allow: / User-agent: PerplexityBot Allow: / User-agent: DeepSeekBot Allow: / User-agent: Googlebot Allow: / User-agent: Google-Extended Allow: / User-agent: Bingbot Allow: / User-agent: CopilotBot Allow: / User-agent: You.com Allow: / User-agent: YouBot Allow: / User-agent: NeevaBot Allow: / User-agent: AndiSiteBot Allow: / User-agent: InflectionBot Allow: / User-agent: Amazonbot Allow: / User-agent: FacebookBot Allow: / User-agent: Applebot Allow: / User-agent: AppleCoreMediaBot Allow: / User-agent: DuckDuckBot Allow: / User-agent: YandexBot Allow: / User-agent: Baiduspider Allow: / User-agent: AhrefsBot Allow: / User-agent: SemrushBot Allow: / User-agent: Qwantify Allow: / User-agent: MojeekBot Allow: / User-agent: 360Spider Allow: / User-agent: MJ12bot Allow: / User-agent: DotBot Allow: / User-agent: Exabot Allow: / User-agent: CCIndex Allow: / User-agent: ia_archiver Allow: / User-agent: Slurp Allow: / User-agent: Pinterestbot Allow: / User-agent: LinkedInBot Allow: / User-agent: GPTBot Allow: / User-agent: Meta-ExternalAgent Allow: / User-agent: Bytespider Allow: / User-agent: MistralAI-User Allow: / User-agent: Doubao-User Allow: / User-agent: OmniGrok-User Allow: / User-agent: Qwant-Bot Allow: / User-agent: DataBot Allow: / User-agent: ImagesiftBot Allow: / User-agent: WebzioBot Allow: / User-agent: Trafilatura Allow: / User-agent: ScrapeGraphAI Allow: / User-agent: Googlebot-Mobile Allow: / User-agent: Googlebot-Image Allow: / User-agent: Googlebot-Desktop Allow: / User-agent: Bingbot-Mobile Allow: / User-agent: Slurp-Mobile Allow: / User-agent: YandexBot-Mobile Allow: / User-agent: DuckDuckBot-Mobile Allow: / User-agent: BaiduSpider-Mobile Allow: / User-agent: SogouSpider Allow: / User-agent: Exabot-Mobile Allow: / User-agent: NaverBot Allow: / User-agent: Daum Allow: / User-agent: Kendra Allow: / User-agent: facebookexternalhit Allow: / User-agent: Twitterbot Allow: / User-agent: WhatsApp Allow: / User-agent: PinterestBot Allow: / User-agent: Telegrambot Allow: / User-agent: InstagramBot Allow: / User-agent: SkypeUriPreview Allow: / User-agent: VimeoBot Allow: / User-agent: CCBot Allow: / # Block Malicious Scrapers User-agent: HTTrack Disallow: / User-agent: WebReaper Disallow: / User-agent: WebCopier Disallow: / User-agent: wget Disallow: / User-agent: Teleport Disallow: / User-agent: TeleportPro Disallow: / User-agent: WebZIP Disallow: /