# robots.txt for quiltersguild.org.uk # Updated to manage bot crawling of image-heavy pages # Default rules for all bots User-agent: * # Allow general access Disallow: /admin-area/ Disallow: /member-login/ Disallow: /*.pdf$ # Rate limiting for major search engines # Crawl-delay is in seconds between requests User-agent: bingbot # Slow down Bing (which was most aggressive in your logs) Crawl-delay: 10 # Optionally limit access to image-heavy albums during peak hours # You can uncomment these if the problem persists: # Disallow: /microsites/*/albums/*/album_image_*.jpeg # Disallow: /microsites/special_interest/includes/images/ User-agent: Googlebot Crawl-delay: 5 User-agent: Googlebot-Image Crawl-delay: 10 # Prevent direct crawling of large unoptimized images Disallow: /microsites/*/includes/images/*/albums/*/album_image_*.jpeg Disallow: /microsites/*/includes/images/*/*/special_interest_group_banner_mobile_image_*.jpeg Disallow: /microsites/*/includes/images/*/*/region_banner_mobile_image_*.jpeg # Other common bots User-agent: AhrefsBot Crawl-delay: 30 User-agent: SemrushBot Crawl-delay: 30 User-agent: MJ12bot Crawl-delay: 30 User-agent: DotBot Crawl-delay: 30 # Social media bots (usually fine, but can rate limit if needed) User-agent: facebookexternalhit Crawl-delay: 2 User-agent: Twitterbot Crawl-delay: 2 # Alternative approach: Block specific image directories entirely # Uncomment these if Crawl-delay doesn't help enough: # User-agent: * # Disallow: /microsites/special_interest/includes/images/*/albums/*/album_image_* # Disallow: /microsites/regional/includes/images/*/albums/*/album_image_* # Allow: /microsites/*/albums/*/album_image_*_thumb.jpeg