# https://www.robotstxt.org/robotstxt.html
User-agent: *
Allow: /

# Sitemap
Sitemap: https://unschooler.me/sitemap.xml

# Disallow admin pages
Disallow: /admin/
Disallow: /embed/
Disallow: /api/

# Common crawl delays
Crawl-delay: 1

# Allow important pages
Allow: /
Allow: /online-courses/
Allow: /skills/
Allow: /careers/
Allow: /profile/
Allow: /terms/
Allow: /policy/
Allow: /affiliate/

# Social media bots
User-agent: facebookexternalhit
Allow: /

User-agent: Twitterbot
Allow: /

User-agent: LinkedInBot
Allow: /