# corz.org - updated: 2023-8-12 # # Big Bots.. User-agent: Googlebot Disallow: /public/avatars/ Disallow: /public/images/smileys/ Disallow: /corzoogle.php Disallow: /search Disallow: /public/search User-agent: Googlebot-Image Disallow: /public/avatars/ Disallow: /public/images/smileys/ Disallow: /corzoogle.php Disallow: /search Disallow: /public/search User-agent: msnbot Disallow: /public/avatars/ Disallow: /public/images/smileys/ Disallow: /corzoogle.php Disallow: /search Disallow: /public/search Crawl-delay: 1 User-agent: bingbot Disallow: /public/avatars/ Disallow: /public/images/smileys/ Disallow: /corzoogle.php Disallow: /search Disallow: /public/search Crawl-delay: 1 User-agent: Yahoo! Slurp Disallow: /public/avatars/ Disallow: /public/images/smileys/ Disallow: /corzoogle.php Disallow: /search Disallow: /public/search Crawl-delay: 2 # # AI LLM Data-Suckers # User-agent: CCBot # Disallow: / # # User-agent: GPTBot # Disallow: / # # User-agent: ChatGPT-User # Disallow: / User-agent: influencebot Crawl-delay: 5 # Little Bots.. User-agent: Ocelli Crawl-delay: 5 # If Yandex keeps ignoring this it will be banned. User-agent: Yandex Crawl-delay: 5 User-agent: Hatena Antenna Crawl-delay: 5 User-agent: Baiduspider Crawl-delay: 5 # Bad Bots.. # yeah, like they will adhere to this! (Use Anti-Hammer!) # Commercial Bots.. # Most of these bots are also now banned.. User-agent: magpie-crawler Disallow: / User-agent: ShopWiki Disallow: / # ignores multi-line statements. fuxer! User-agent: SemrushBot Disallow: / User-agent: SemrushBot-SA Disallow: / User-agent: SemrushBot-BA Disallow: / User-agent: SemrushBot-SI Disallow: / User-agent: SemrushBot-SWA Disallow: / User-agent: SemrushBot-CT Disallow: / User-agent: SemrushBot-BM Disallow: / User-agent: SemrushBot-SEOAB Disallow: / User-agent: MJ12bot Disallow: / User-agent: dotbot Disallow: / User-agent: AhrefsBot Disallow: / # All Bots.. User-agent: * Disallow: /img/ Disallow: /public/avatars/ Disallow: /public/images/smileys/ Disallow: /corzoogle.php Disallow: /search Disallow: /public/search Crawl-delay: 2