# Allow all crawlers access to everything by default
User-agent: *
Disallow:

# Block specific crawlers
User-agent: BadBot
Disallow: /

User-agent: AnotherBadBot
Disallow: /

# Allow Googlebot to crawl everything except private directory
User-agent: Googlebot
Disallow: /private-directory/

# Allow Bingbot to crawl everything
User-agent: Bingbot
Disallow:

# Block all crawlers from specific directories
User-agent: *
Disallow: /no-index/
Disallow: /private/
Disallow: /tmp/
Disallow: /secret.html
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-content/cache/
Disallow: /wp-login*
Disallow: /wp-comments-post.php
Disallow: /wp-json/wp/v2/*
Disallow: /xmlrpc*
Disallow: /cgi-bin/
Disallow: /author/
Disallow: /tag/
Disallow: /replytocom
Disallow: /comments
Disallow: /*?s=
Disallow: /?p=
Disallow: /*page/
Disallow: /feed/
Disallow: /*/feed/
Disallow: //author//feed/
Disallow: /*/category/
Disallow: /wp-cron.php?*

# Allow essential files for functionality
Allow: /wp-admin/admin-ajax.php
Allow: /wp-content/uploads/
Allow: /wp-includes/js/
Allow: /wp-content/themes/
Allow: /wp-content/plugins/

# Block URLs containing specific query parameters
User-agent: *
Disallow: /*?kref=
Disallow: /*?kuid=
Disallow: /*?uuid=
Disallow: /*?*kref=
Disallow: /*?*kuid=
Disallow: /*?*uuid=
Disallow: /*&kref=
Disallow: /*&kuid=
Disallow: /*&uuid=

# Block specific files
User-agent: *
Disallow: /hidden/file.html

# Allow specific pages while blocking others
User-agent: *
Disallow: /search/
Allow: /search/specific-page

# Block access to /page and URLs with query parameters
User-agent: *
Disallow: /page
Disallow: /?*

# Allow all crawlers but disallow access to specific query parameters
User-agent: *
Disallow: /*?private

# Block specific bot (AdsBot-Google)
User-agent: AdsBot-Google
Disallow: /

# Sitemap 
Sitemap: https://www.drtarekaesthetics.com/sitemap_index.xml

# Crawl-delay setting (prevents server overload)
User-agent: *
Crawl-delay: 5

# Specify default crawl delay (not supported by all bots)
Crawl-delay: 5