User-agent: GPTBot Crawl-delay: 45 # www.robotstxt.org/ # Allow crawling of all content User-agent: * Disallow: