# Allow all robots to spider everything by disallowing nothing User-agent: * Disallow: Disallow: /core/media/ Sitemap: http://www.deiequipment.com/sitemap.xml Crawl-Delay: 10