# robots.txt for https://henitan.com

# Allow all user agents to crawl the entire site
User-agent: *
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
Disallow: /wp-content/cache/
Disallow: /tmp/
Disallow: /private/
Disallow: /backup/
Disallow: /scripts/

# Allow Googlebot and other major search engines to crawl
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

# Block specific bots that may harm your site
User-agent: BadBot
Disallow: /

# Sitemap file for better indexing
Sitemap: https://henitan.com/sitemap_index.xml

# Allow crawling of AdSense-related content
User-agent: AdsBot-Google
Allow: /

# Crawl-delay settings for less aggressive bots (optional)
# User-agent: *
# Crawl-delay:

# Block any URLs that end with .html or .htm
User-agent: *
Disallow: /112924tpgealegtw-.html
Disallow: /145460tpgetokyo/aleqtg-.htm
Disallow: /41446tpgealecgs-el.html
Disallow: /*.html$
Disallow: /*.htm$