User-Agent: *
Disallow: /sitemap.html
Disallow: /pagespeed/defer/
Disallow: /*.pdf$
Disallow: /feed/
Disallow: /*/feed/

Sitemap: https://www.marketengine.ai/sitemap.xml

# Reference for AI crawlers (LLMs)
llms: /llms.txt