{"id":52,"date":"2026-05-30T16:46:15","date_gmt":"2026-05-30T11:16:15","guid":{"rendered":"https:\/\/bloggingden.com\/tools\/?p=52"},"modified":"2026-07-11T10:37:23","modified_gmt":"2026-07-11T05:07:23","slug":"free-robots-txt-generator-tool","status":"publish","type":"post","link":"https:\/\/bloggingden.com\/tools\/free-robots-txt-generator-tool\/","title":{"rendered":"Free Robots.txt Generator Tool."},"content":{"rendered":"\n<div class=\"wp-block-group alignwide has-theme-palette-8-background-color has-background is-layout-constrained wp-block-group-is-layout-constrained\" style=\"margin-top:var(--wp--preset--spacing--40);margin-bottom:var(--wp--preset--spacing--40)\">\n<!-- BDTools: Robots.txt Generator | BloggingDen.com -->\n<style>\n.bdtw *,\n.bdtw *::before,\n.bdtw *::after { box-sizing: border-box; margin: 0; padding: 0; }\n\n.bdtw {\n  --bd-bg:        #ffffff;\n  --bd-surface:   #f8f7f4;\n  --bd-border:    #e2ddd6;\n  --bd-accent:    #2563eb;\n  --bd-accent-h:  #1d4ed8;\n  --bd-gold:      #92722a;\n  --bd-text:      #1a1814;\n  --bd-muted:     #6b6459;\n  --bd-success:   #15803d;\n  --bd-warn:      #b45309;\n  --bd-radius:    8px;\n  --bd-font:      'Georgia', 'Times New Roman', serif;\n  --bd-mono:      'Courier New', Courier, monospace;\n  font-family: var(--bd-font);\n  color: var(--bd-text);\n  background: var(--bd-bg);\n  max-width: 860px;\n  margin: 0 auto;\n  padding: 0;\n}\n\n\/* Card wrapper *\/\n.bdt-card {\n  background: var(--bd-bg);\n  border: 1px solid var(--bd-border);\n  border-radius: 12px;\n  padding: 36px 40px;\n  box-shadow: 0 2px 16px rgba(0,0,0,.06);\n}\n\n\/* Header *\/\n.bdt-header { margin-bottom: 28px; border-bottom: 2px solid var(--bd-border); padding-bottom: 20px; }\n.bdt-header h1 {\n  font-size: 1.75rem;\n  font-weight: 700;\n  letter-spacing: -.02em;\n  color: var(--bd-text);\n  margin-bottom: 6px;\n}\n.bdt-header p {\n  font-size: .9rem;\n  color: var(--bd-muted);\n  line-height: 1.6;\n  font-family: Arial, sans-serif;\n}\n\n\/* Form sections *\/\n.bdt-section { margin-bottom: 24px; }\n.bdt-label {\n  display: block;\n  font-size: .85rem;\n  font-weight: 700;\n  text-transform: uppercase;\n  letter-spacing: .06em;\n  color: var(--bd-text);\n  margin-bottom: 8px;\n}\n.bdt-hint {\n  font-size: .78rem;\n  color: var(--bd-muted);\n  margin-top: 5px;\n  font-family: Arial, sans-serif;\n}\n.bdt-hint.bdt-warn { color: var(--bd-warn); }\n\n.bdt-input {\n  width: 100%;\n  padding: 10px 14px;\n  font-size: .9rem;\n  font-family: Arial, sans-serif;\n  border: 1px solid var(--bd-border);\n  border-radius: var(--bd-radius);\n  background: var(--bd-surface);\n  color: var(--bd-text);\n  outline: none;\n  transition: border-color .18s, box-shadow .18s;\n}\n.bdt-input:focus {\n  border-color: var(--bd-accent);\n  box-shadow: 0 0 0 3px rgba(37,99,235,.12);\n  background: #fff;\n}\n\n\/* Divider *\/\n.bdt-divider { border: none; border-top: 1px solid var(--bd-border); margin: 6px 0 18px; }\n\n\/* Section heading *\/\n.bdt-section-title {\n  font-size: 1rem;\n  font-weight: 700;\n  color: var(--bd-text);\n  margin-bottom: 4px;\n}\n.bdt-section-sub {\n  font-size: .78rem;\n  color: var(--bd-gold);\n  font-family: Arial, sans-serif;\n  margin-bottom: 12px;\n}\n\n\/* Checkboxes *\/\n.bdt-check-group { display: flex; flex-direction: column; gap: 9px; }\n.bdt-check-item {\n  display: flex;\n  align-items: center;\n  gap: 10px;\n  font-size: .875rem;\n  font-family: Arial, sans-serif;\n  color: var(--bd-text);\n  cursor: pointer;\n  user-select: none;\n}\n.bdt-check-item input[type=\"checkbox\"] {\n  width: 16px;\n  height: 16px;\n  accent-color: var(--bd-accent);\n  cursor: pointer;\n  flex-shrink: 0;\n}\n\n\/* Toggle master checkbox row *\/\n.bdt-master-check {\n  display: flex;\n  align-items: center;\n  gap: 10px;\n  font-size: .875rem;\n  font-weight: 600;\n  font-family: Arial, sans-serif;\n  color: var(--bd-text);\n  cursor: pointer;\n  margin-bottom: 14px;\n}\n.bdt-master-check input[type=\"checkbox\"] {\n  width: 16px; height: 16px;\n  accent-color: var(--bd-accent);\n  cursor: pointer;\n}\n\n\/* Collapsible sub-section *\/\n.bdt-subsection { padding-left: 26px; }\n.bdt-subsection.bdt-hidden { display: none; }\n\n\/* Inline row: label + select *\/\n.bdt-inline-row {\n  display: flex;\n  align-items: center;\n  gap: 10px;\n  margin-bottom: 14px;\n  flex-wrap: wrap;\n}\n.bdt-inline-label {\n  font-size: .85rem;\n  font-weight: 600;\n  font-family: Arial, sans-serif;\n  color: var(--bd-text);\n}\n.bdt-select {\n  padding: 6px 12px;\n  font-size: .85rem;\n  font-family: Arial, sans-serif;\n  border: 1px solid var(--bd-border);\n  border-radius: var(--bd-radius);\n  background: var(--bd-surface);\n  color: var(--bd-text);\n  outline: none;\n  cursor: pointer;\n  transition: border-color .18s;\n}\n.bdt-select:focus { border-color: var(--bd-accent); }\n\n\/* Textarea *\/\n.bdt-textarea {\n  width: 100%;\n  min-height: 110px;\n  padding: 10px 14px;\n  font-size: .85rem;\n  font-family: Arial, sans-serif;\n  border: 1px solid var(--bd-border);\n  border-radius: var(--bd-radius);\n  background: var(--bd-surface);\n  color: var(--bd-text);\n  resize: vertical;\n  outline: none;\n  transition: border-color .18s, box-shadow .18s;\n}\n.bdt-textarea:focus {\n  border-color: var(--bd-accent);\n  box-shadow: 0 0 0 3px rgba(37,99,235,.12);\n  background: #fff;\n}\n\n\/* Generate button *\/\n.bdt-btn-primary {\n  display: block;\n  width: 100%;\n  padding: 14px;\n  background: var(--bd-accent);\n  color: #fff;\n  font-size: 1rem;\n  font-weight: 700;\n  font-family: Arial, sans-serif;\n  border: none;\n  border-radius: var(--bd-radius);\n  cursor: pointer;\n  letter-spacing: .02em;\n  transition: background .18s, transform .1s;\n  margin-top: 8px;\n}\n.bdt-btn-primary:hover { background: var(--bd-accent-h); }\n.bdt-btn-primary:active { transform: scale(.99); }\n\n\/* Output area *\/\n.bdt-output { display: none; margin-top: 28px; }\n.bdt-output.bdt-visible { display: block; }\n.bdt-output-header {\n  display: flex;\n  align-items: center;\n  justify-content: space-between;\n  margin-bottom: 10px;\n  flex-wrap: wrap;\n  gap: 8px;\n}\n.bdt-output-title {\n  font-size: 1rem;\n  font-weight: 700;\n  color: var(--bd-text);\n}\n.bdt-output-actions { display: flex; gap: 8px; }\n.bdt-btn-sm {\n  padding: 6px 14px;\n  font-size: .8rem;\n  font-weight: 600;\n  font-family: Arial, sans-serif;\n  border-radius: 6px;\n  cursor: pointer;\n  border: 1px solid var(--bd-border);\n  background: var(--bd-surface);\n  color: var(--bd-text);\n  transition: background .15s, border-color .15s;\n}\n.bdt-btn-sm:hover { background: #eee; border-color: #bbb; }\n.bdt-btn-sm.bdt-btn-copy {\n  background: var(--bd-accent);\n  color: #fff;\n  border-color: var(--bd-accent);\n}\n.bdt-btn-sm.bdt-btn-copy:hover { background: var(--bd-accent-h); }\n\n.bdt-code-output {\n  background: #0f172a;\n  color: #e2e8f0;\n  font-family: var(--bd-mono);\n  font-size: .82rem;\n  line-height: 1.7;\n  padding: 20px 22px;\n  border-radius: var(--bd-radius);\n  white-space: pre;\n  overflow-x: auto;\n  border: 1px solid #1e293b;\n  max-height: 420px;\n  overflow-y: auto;\n}\n\n.bdt-copy-toast {\n  display: none;\n  background: var(--bd-success);\n  color: #fff;\n  font-size: .8rem;\n  font-family: Arial, sans-serif;\n  font-weight: 600;\n  padding: 7px 14px;\n  border-radius: 6px;\n  margin-top: 10px;\n  text-align: center;\n}\n.bdt-copy-toast.bdt-show { display: block; }\n\n\/* Responsive *\/\n@media (max-width: 600px) {\n  .bdt-card { padding: 24px 18px; }\n  .bdt-header h1 { font-size: 1.4rem; }\n}\n<\/style>\n\n<div class=\"bdtw\">\n  <div class=\"bdt-card\">\n\n    <!-- Header -->\n    <div class=\"bdt-header\">\n      <p class=\"has-large-font-size\"><strong>Robots.txt Generator<\/strong><\/p>\n      <p>Create a custom robots.txt file for your website. This file tells search engines and AI crawlers which pages they can or cannot crawl on your site.<\/p>\n    <\/div>\n\n    <!-- Website URL -->\n    <div class=\"bdt-section\">\n      <label class=\"bdt-label\" for=\"bdt-site-url\">Your Website URL<\/label>\n      <input class=\"bdt-input\" type=\"url\" id=\"bdt-site-url\" placeholder=\"https:\/\/www.yourwebsite.com\">\n      <p class=\"bdt-hint\">Enter your website&#8217;s full URL including https:\/\/<\/p><div id=\"ez-toc-container\" class=\"ez-toc-v2_0_86 counter-hierarchy ez-toc-counter ez-toc-custom ez-toc-container-direction\">\n<div class=\"ez-toc-title-container\">\n<p class=\"ez-toc-title\" style=\"cursor:inherit\">Table of Contents<\/p>\n<span class=\"ez-toc-title-toggle\"><a href=\"#\" class=\"ez-toc-pull-right ez-toc-btn ez-toc-btn-xs ez-toc-btn-default ez-toc-toggle\" aria-label=\"Toggle Table of Content\"><span class=\"ez-toc-js-icon-con\"><span class=\"\"><span class=\"eztoc-hide\" style=\"display:none;\">Toggle<\/span><span class=\"ez-toc-icon-toggle-span\"><svg style=\"fill: #000000;color:#000000\" xmlns=\"http:\/\/www.w3.org\/2000\/svg\" class=\"list-377408\" width=\"20px\" height=\"20px\" viewBox=\"0 0 24 24\" fill=\"none\"><path d=\"M6 6H4v2h2V6zm14 0H8v2h12V6zM4 11h2v2H4v-2zm16 0H8v2h12v-2zM4 16h2v2H4v-2zm16 0H8v2h12v-2z\" fill=\"currentColor\"><\/path><\/svg><svg style=\"fill: #000000;color:#000000\" class=\"arrow-unsorted-368013\" xmlns=\"http:\/\/www.w3.org\/2000\/svg\" width=\"10px\" height=\"10px\" viewBox=\"0 0 24 24\" version=\"1.2\" baseProfile=\"tiny\"><path d=\"M18.2 9.3l-6.2-6.3-6.2 6.3c-.2.2-.3.4-.3.7s.1.5.3.7c.2.2.4.3.7.3h11c.3 0 .5-.1.7-.3.2-.2.3-.5.3-.7s-.1-.5-.3-.7zM5.8 14.7l6.2 6.3 6.2-6.3c.2-.2.3-.5.3-.7s-.1-.5-.3-.7c-.2-.2-.4-.3-.7-.3h-11c-.3 0-.5.1-.7.3-.2.2-.3.5-.3.7s.1.5.3.7z\"\/><\/svg><\/span><\/span><\/span><\/a><\/span><\/div>\n<nav><ul class='ez-toc-list ez-toc-list-level-1 ez-toc-columns-2 eztoc-toggle-hide-by-default' ><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-1\" href=\"https:\/\/bloggingden.com\/tools\/free-robots-txt-generator-tool\/#How_to_Use_This_Free_Robotstxt_Generator\" >How to Use This Free Robots.txt Generator<\/a><ul class='ez-toc-list-level-3' ><li class='ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-2\" href=\"https:\/\/bloggingden.com\/tools\/free-robots-txt-generator-tool\/#Step_1_Enter_your_website_URL\" >Step 1 Enter your website URL<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-3\" href=\"https:\/\/bloggingden.com\/tools\/free-robots-txt-generator-tool\/#Step_2_Add_your_sitemap_URL_optional_but_recommended\" >Step 2 Add your sitemap URL (optional but recommended)<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-4\" href=\"https:\/\/bloggingden.com\/tools\/free-robots-txt-generator-tool\/#Step_3_Select_your_restrictions\" >Step 3 Select your restrictions<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-5\" href=\"https:\/\/bloggingden.com\/tools\/free-robots-txt-generator-tool\/#Step_4_Click_%E2%80%9CGenerate_Robotstxt%E2%80%9D\" >Step 4 Click &#8220;Generate Robots.txt&#8221;<\/a><\/li><\/ul><\/li><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-6\" href=\"https:\/\/bloggingden.com\/tools\/free-robots-txt-generator-tool\/#Understanding_the_Robotstxt_Syntax\" >Understanding the Robots.txt Syntax<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-7\" href=\"https:\/\/bloggingden.com\/tools\/free-robots-txt-generator-tool\/#Blocking_AI_and_LLM_Crawlers_Should_You_Opt_Out\" >Blocking AI and LLM Crawlers Should You Opt Out?<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-8\" href=\"https:\/\/bloggingden.com\/tools\/free-robots-txt-generator-tool\/#Common_Robotstxt_Mistakes_to_Avoid\" >Common Robots.txt Mistakes to Avoid<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-9\" href=\"https:\/\/bloggingden.com\/tools\/free-robots-txt-generator-tool\/#How_to_Add_Robotstxt_to_a_WordPress_Site\" >How to Add Robots.txt to a WordPress Site<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-10\" href=\"https:\/\/bloggingden.com\/tools\/free-robots-txt-generator-tool\/#Frequently_Asked_Questions_on_Robotstxt_Generator\" >Frequently Asked Questions on Robots.txt Generator<\/a><\/li><\/ul><\/nav><\/div>\n\n    <\/div>\n\n    <!-- Sitemap URL -->\n    <div class=\"bdt-section\">\n      <label class=\"bdt-label\" for=\"bdt-sitemap-url\">Sitemap URL <span style=\"font-weight:400;text-transform:none;letter-spacing:0;\">(Optional)<\/span><\/label>\n      <input class=\"bdt-input\" type=\"url\" id=\"bdt-sitemap-url\" placeholder=\"https:\/\/www.yourwebsite.com\/sitemap.xml\">\n      <p class=\"bdt-hint\">Enter the URL of your XML sitemap<\/p>\n    <\/div>\n\n    <hr class=\"bdt-divider\">\n\n    <!-- Common Restrictions -->\n    <div class=\"bdt-section\">\n      <p class=\"bdt-section-title\">Common Restrictions:<\/p>\n      <div class=\"bdt-check-group\" style=\"margin-top:10px;\">\n        <label class=\"bdt-check-item\">\n          <input type=\"checkbox\" id=\"bdt-block-wpadmin\" checked=\"\">\n          Block admin pages (\/wp-admin\/, \/admin\/)\n        <\/label>\n        <label class=\"bdt-check-item\">\n          <input type=\"checkbox\" id=\"bdt-block-cgi\">\n          Block CGI-BIN directory (\/cgi-bin\/)\n        <\/label>\n        <label class=\"bdt-check-item\">\n          <input type=\"checkbox\" id=\"bdt-block-tmp\">\n          Block temporary files (\/tmp\/)\n        <\/label>\n        <label class=\"bdt-check-item\">\n          <input type=\"checkbox\" id=\"bdt-block-private\">\n          Block private directory (\/private\/)\n        <\/label>\n      <\/div>\n    <\/div>\n\n    <hr class=\"bdt-divider\">\n\n    <!-- AI & LLM Crawlers -->\n    <div class=\"bdt-section\">\n      <p class=\"bdt-section-title\">AI &amp; LLM Crawlers <span style=\"font-weight:400;\">(optional):<\/span><\/p>\n      <p class=\"bdt-section-sub\">Control access for popular AI crawlers like GPTBot, Perplexity, Claude, and Google-Extended.<\/p>\n      <label class=\"bdt-master-check\" id=\"bdt-ai-toggle-label\">\n        <input type=\"checkbox\" id=\"bdt-ai-toggle\" checked=\"\" onchange=\"bdtToggleAI(this)\">\n        Add rules for AI \/ LLM crawlers\n      <\/label>\n      <div class=\"bdt-subsection\" id=\"bdt-ai-sub\">\n        <div class=\"bdt-inline-row\">\n          <span class=\"bdt-inline-label\">Action for selected AI bots:<\/span>\n          <select class=\"bdt-select\" id=\"bdt-ai-action\">\n            <option value=\"Disallow\" selected=\"\">Disallow all content (recommended if you want to opt-out)<\/option>\n            <option value=\"Allow\">Allow all content<\/option>\n          <\/select>\n        <\/div>\n        <div class=\"bdt-check-group\">\n          <label class=\"bdt-check-item\"><input type=\"checkbox\" class=\"bdt-ai-bot\" value=\"GPTBot\" checked=\"\"> GPTBot (OpenAI)<\/label>\n          <label class=\"bdt-check-item\"><input type=\"checkbox\" class=\"bdt-ai-bot\" value=\"OAI-SearchBot\" checked=\"\"> OAI-SearchBot (OpenAI search crawling)<\/label>\n          <label class=\"bdt-check-item\"><input type=\"checkbox\" class=\"bdt-ai-bot\" value=\"ChatGPT-User\"> ChatGPT-User (user-triggered browsing)<\/label>\n          <label class=\"bdt-check-item\"><input type=\"checkbox\" class=\"bdt-ai-bot\" value=\"PerplexityBot\" checked=\"\"> PerplexityBot<\/label>\n          <label class=\"bdt-check-item\"><input type=\"checkbox\" class=\"bdt-ai-bot\" value=\"Perplexity-User\"> Perplexity-User (user-triggered)<\/label>\n          <label class=\"bdt-check-item\"><input type=\"checkbox\" class=\"bdt-ai-bot\" value=\"ClaudeBot\" checked=\"\"> ClaudeBot<\/label>\n          <label class=\"bdt-check-item\"><input type=\"checkbox\" class=\"bdt-ai-bot\" value=\"Google-Extended\" checked=\"\"> Google-Extended (AI training opt-out)<\/label>\n        <\/div>\n      <\/div>\n    <\/div>\n\n    <hr class=\"bdt-divider\">\n\n    <!-- Other Specific Bots -->\n    <div class=\"bdt-section\">\n      <p class=\"bdt-section-title\">Other Specific Bots <span style=\"font-weight:400;\">(optional):<\/span><\/p>\n      <p class=\"bdt-section-sub\">Use carefully. Disallowing Googlebot or Bingbot can remove your site from search results.<\/p>\n      <label class=\"bdt-master-check\">\n        <input type=\"checkbox\" id=\"bdt-seo-toggle\" checked=\"\" onchange=\"bdtToggleSEO(this)\">\n        Add rules for selected SEO \/ crawler bots\n      <\/label>\n      <div class=\"bdt-subsection\" id=\"bdt-seo-sub\">\n        <div class=\"bdt-inline-row\">\n          <span class=\"bdt-inline-label\">Action for selected bots:<\/span>\n          <select class=\"bdt-select\" id=\"bdt-seo-action\">\n            <option value=\"Allow\" selected=\"\">Allow<\/option>\n            <option value=\"Disallow\">Disallow<\/option>\n          <\/select>\n        <\/div>\n        <div class=\"bdt-check-group\">\n          <label class=\"bdt-check-item\"><input type=\"checkbox\" class=\"bdt-seo-bot\" value=\"Googlebot\"> Googlebot<\/label>\n          <label class=\"bdt-check-item\"><input type=\"checkbox\" class=\"bdt-seo-bot\" value=\"Bingbot\"> Bingbot<\/label>\n          <label class=\"bdt-check-item\"><input type=\"checkbox\" class=\"bdt-seo-bot\" value=\"AhrefsBot\"> AhrefsBot<\/label>\n          <label class=\"bdt-check-item\"><input type=\"checkbox\" class=\"bdt-seo-bot\" value=\"SemrushBot\"> SemrushBot<\/label>\n          <label class=\"bdt-check-item\"><input type=\"checkbox\" class=\"bdt-seo-bot\" value=\"MJ12bot\"> MJ12bot<\/label>\n          <label class=\"bdt-check-item\"><input type=\"checkbox\" class=\"bdt-seo-bot\" value=\"Screaming Frog SEO Spider\"> Screaming Frog SEO Spider<\/label>\n        <\/div>\n      <\/div>\n    <\/div>\n\n    <hr class=\"bdt-divider\">\n\n    <!-- Custom Paths -->\n    <div class=\"bdt-section\">\n      <label class=\"bdt-label\" for=\"bdt-custom-paths\">Custom Paths to Block <span style=\"font-weight:400;text-transform:none;letter-spacing:0;\">(Optional)<\/span><\/label>\n      <textarea class=\"bdt-textarea\" id=\"bdt-custom-paths\" placeholder=\"Enter paths to block, one per line. Example:\n\/search\/\n\/cart\/\n\/checkout\/\"><\/textarea>\n      <p class=\"bdt-hint\">Add any additional paths you want to block from search engines<\/p>\n    <\/div>\n\n    <!-- Generate Button -->\n    <button class=\"bdt-btn-primary\" onclick=\"bdtGenerate()\">Generate Robots.txt<\/button>\n\n    <!-- Output -->\n    <div class=\"bdt-output\" id=\"bdt-output\">\n      <div class=\"bdt-output-header\">\n        <span class=\"bdt-output-title\">Your robots.txt file<\/span>\n        <div class=\"bdt-output-actions\">\n          <button class=\"bdt-btn-sm bdt-btn-copy\" onclick=\"bdtCopy()\">Copy to Clipboard<\/button>\n          <button class=\"bdt-btn-sm\" onclick=\"bdtDownload()\">Download .txt<\/button>\n        <\/div>\n      <\/div>\n      <pre class=\"bdt-code-output\" id=\"bdt-code-out\"><\/pre>\n      <div class=\"bdt-copy-toast\" id=\"bdt-toast\">\u2713 Copied to clipboard!<\/div>\n    <\/div>\n\n  <\/div><!-- \/.bdt-card -->\n<\/div><!-- \/.bdtw -->\n\n<script>\n(function(){\n\n  function bdtToggleAI(cb) {\n    document.getElementById('bdt-ai-sub').classList.toggle('bdt-hidden', !cb.checked);\n  }\n\n  function bdtToggleSEO(cb) {\n    document.getElementById('bdt-seo-sub').classList.toggle('bdt-hidden', !cb.checked);\n  }\n\n  function bdtGenerate() {\n    var lines = [];\n    var siteUrl  = (document.getElementById('bdt-site-url').value || '').trim();\n    var sitemapUrl = (document.getElementById('bdt-sitemap-url').value || '').trim();\n\n    \/\/ ---- Default allow all ----\n    lines.push('User-agent: *');\n\n    \/\/ Common restrictions\n    if (document.getElementById('bdt-block-wpadmin').checked) {\n      lines.push('Disallow: \/wp-admin\/');\n      lines.push('Disallow: \/admin\/');\n    }\n    if (document.getElementById('bdt-block-cgi').checked)     lines.push('Disallow: \/cgi-bin\/');\n    if (document.getElementById('bdt-block-tmp').checked)      lines.push('Disallow: \/tmp\/');\n    if (document.getElementById('bdt-block-private').checked)  lines.push('Disallow: \/private\/');\n\n    \/\/ Custom paths\n    var customRaw = document.getElementById('bdt-custom-paths').value || '';\n    customRaw.split('\\n').forEach(function(p) {\n      p = p.trim();\n      if (p) lines.push('Disallow: ' + p);\n    });\n\n    lines.push('Allow: \/');\n    lines.push('');\n\n    \/\/ ---- AI \/ LLM bots ----\n    if (document.getElementById('bdt-ai-toggle').checked) {\n      var aiAction = document.getElementById('bdt-ai-action').value;\n      var aiBots = document.querySelectorAll('.bdt-ai-bot:checked');\n      aiBots.forEach(function(b) {\n        lines.push('User-agent: ' + b.value);\n        lines.push(aiAction + ': \/');\n        lines.push('');\n      });\n    }\n\n    \/\/ ---- SEO \/ crawler bots ----\n    if (document.getElementById('bdt-seo-toggle').checked) {\n      var seoAction = document.getElementById('bdt-seo-action').value;\n      var seoBots = document.querySelectorAll('.bdt-seo-bot:checked');\n      seoBots.forEach(function(b) {\n        lines.push('User-agent: ' + b.value);\n        if (seoAction === 'Allow') {\n          lines.push('Allow: \/');\n        } else {\n          lines.push('Disallow: \/');\n        }\n        lines.push('');\n      });\n    }\n\n    \/\/ ---- Sitemap ----\n    if (sitemapUrl) {\n      lines.push('Sitemap: ' + sitemapUrl);\n    } else if (siteUrl) {\n      var base = siteUrl.replace(\/\\\/$\/, '');\n      lines.push('Sitemap: ' + base + '\/sitemap.xml');\n    }\n\n    \/\/ ---- Host ----\n    if (siteUrl) {\n      lines.push('Host: ' + siteUrl.replace(\/https?:\\\/\\\/\/,'').replace(\/\\\/$',''));\n    }\n\n    var output = lines.join('\\n');\n    document.getElementById('bdt-code-out').textContent = output;\n    var outEl = document.getElementById('bdt-output');\n    outEl.classList.add('bdt-visible');\n    outEl.scrollIntoView({ behavior: 'smooth', block: 'start' });\n  }\n\n  function bdtCopy() {\n    var text = document.getElementById('bdt-code-out').textContent;\n    if (navigator.clipboard && navigator.clipboard.writeText) {\n      navigator.clipboard.writeText(text).then(function(){ bdtShowToast(); });\n    } else {\n      var ta = document.createElement('textarea');\n      ta.value = text;\n      document.body.appendChild(ta);\n      ta.select();\n      document.execCommand('copy');\n      document.body.removeChild(ta);\n      bdtShowToast();\n    }\n  }\n\n  function bdtShowToast() {\n    var t = document.getElementById('bdt-toast');\n    t.classList.add('bdt-show');\n    setTimeout(function(){ t.classList.remove('bdt-show'); }, 2500);\n  }\n\n  function bdtDownload() {\n    var text = document.getElementById('bdt-code-out').textContent;\n    var blob = new Blob([text], { type: 'text\/plain' });\n    var a = document.createElement('a');\n    a.href = URL.createObjectURL(blob);\n    a.download = 'robots.txt';\n    a.click();\n    URL.revokeObjectURL(a.href);\n  }\n\n  \/\/ Expose to global scope for inline onclick handlers\n  window.bdtToggleAI    = bdtToggleAI;\n  window.bdtToggleSEO   = bdtToggleSEO;\n  window.bdtGenerate    = bdtGenerate;\n  window.bdtCopy        = bdtCopy;\n  window.bdtDownload    = bdtDownload;\n\n})();\n<\/script>\n<\/div>\n\n\n\n<p class=\"wp-block-paragraph\">A <strong>robots.txt file<\/strong> is a plain-text file placed at the root of your website\u2014for example, <code>https:\/\/yourwebsite.com\/robots.txt<\/code> \u2014 that instructs web crawlers and bots which pages or directories they are permitted or prohibited from accessing. It forms part of the <em>Robots Exclusion Protocol<\/em>, a widely accepted standard observed by all major search engines, including Google, Bing, and Yahoo.<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">For bloggers, publishers, and website owners, a correctly configured robots.txt file is one of the most cost-effective SEO improvements available. Without it, search engine crawlers may waste their <em>crawl budget<\/em> on low-value pages such as admin panels, tag archives, or internal search results\u2014time and resources that could instead be spent indexing your best content.<\/p>\n\n\n\n<p class=\"wp-block-paragraph\"><strong>Quick Fact:<\/strong> Google has confirmed that crawl budget matters for large websites. Even for smaller sites, unnecessary crawler activity on duplicate or private pages can dilute how efficiently your content is indexed.<\/p>\n\n\n\n<h2 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"How_to_Use_This_Free_Robotstxt_Generator\"><\/span>How to Use This Free Robots.txt Generator<span class=\"ez-toc-section-end\"><\/span><\/h2>\n\n\n<div id=\"rank-math-howto\" class=\"rank-math-block\" >\n<div class=\"rank-math-howto-description\">\n\n<p>Our tool above requires no registration, no installation, and no technical knowledge. Follow these four steps to generate your customized robots.txt file in under two minutes.<\/p>\n\n<\/div>\n\n<div class=\"rank-math-steps \">\n<div id=\"howto-step-1780139091157\" class=\"rank-math-step\">\n<h3 class=\"rank-math-step-title \"><span class=\"ez-toc-section\" id=\"Step_1_Enter_your_website_URL\"><\/span>Step 1: Enter your website URL<span class=\"ez-toc-section-end\"><\/span><\/h3>\n<div class=\"rank-math-step-content \"><p>Provide your full domain, including the <code>https:\/\/<\/code> protocol. This is used to construct the <code>Host<\/code> directive and auto-fill your sitemap path if you leave the sitemap field blank.<\/p>\n<\/div>\n<\/div>\n<div id=\"howto-step-1780139105769\" class=\"rank-math-step\">\n<h3 class=\"rank-math-step-title \"><span class=\"ez-toc-section\" id=\"Step_2_Add_your_sitemap_URL_optional_but_recommended\"><\/span>Step 2: Add your sitemap URL (optional but recommended)<span class=\"ez-toc-section-end\"><\/span><\/h3>\n<div class=\"rank-math-step-content \"><p>If you have an XML sitemap\u2014WordPress users typically find theirs at <code>\/sitemap.xml<\/code> or <code>\/sitemap_index.xml<\/code> \u2014 entering it here ensures search engines can discover it directly from your robots.txt file.<\/p>\n<\/div>\n<\/div>\n<div id=\"howto-step-1780139129322\" class=\"rank-math-step\">\n<h3 class=\"rank-math-step-title \"><span class=\"ez-toc-section\" id=\"Step_3_Select_your_restrictions\"><\/span>Step 3: Select your restrictions<span class=\"ez-toc-section-end\"><\/span><\/h3>\n<div class=\"rank-math-step-content \"><p>Choose from common preset restrictions (admin directories, CGI-BIN, and temp files); configure AI and LLM crawler rules, manage specific SEO bots; and add any custom paths you wish to block.<\/p>\n<\/div>\n<\/div>\n<div id=\"howto-step-1780139146377\" class=\"rank-math-step\">\n<h3 class=\"rank-math-step-title \"><span class=\"ez-toc-section\" id=\"Step_4_Click_%E2%80%9CGenerate_Robotstxt%E2%80%9D\"><\/span>Step 4: Click &#8220;Generate Robots.txt&#8221;<span class=\"ez-toc-section-end\"><\/span><\/h3>\n<div class=\"rank-math-step-content \"><p>Your file is produced instantly. Copy it to your clipboard or download it directly as a <code>robots.txt<\/code> file ready for upload to your server.<\/p>\n<\/div>\n<\/div>\n<\/div>\n<\/div>\n\n\n\n\n\n\n<h2 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"Understanding_the_Robotstxt_Syntax\"><\/span>Understanding the Robots.txt Syntax<span class=\"ez-toc-section-end\"><\/span><\/h2>\n\n\n\n<p class=\"wp-block-paragraph\">The structure of a robots.txt file is straightforward. Each instruction block begins with a <code>User-agent<\/code> directive identifying the crawler, followed by one or more <strong>Allow<\/strong> or <strong>disallow<\/strong> directives specifying which paths the crawler may or may not access. A blank line separates each block.<\/p>\n\n\n\n<pre class=\"wp-block-preformatted\"># Allow all crawlers full access, except specific directories\nUser-agent: *\nDisallow: \/wp-admin\/\nDisallow: \/cgi-bin\/\nAllow: \/\n\n# Block OpenAI's GPTBot from training on your content\nUser-agent: GPTBot\nDisallow: \/\n\n# Block Google's AI training crawler\nUser-agent: Google-Extended\nDisallow: \/\n\n# Sitemap location\nSitemap: https:\/\/www.yourwebsite.com\/sitemap.xml<\/pre>\n\n\n\n<p class=\"wp-block-paragraph\">The <code>*<\/code> wildcard in <code>User-agent: *<\/code> applies the rules to every crawler that has not been given specific instructions elsewhere in the file. Rules listed under a specific user-agent name take precedence over wildcard rules for that particular bot.<\/p>\n\n\n\n<h2 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"Blocking_AI_and_LLM_Crawlers_Should_You_Opt_Out\"><\/span>Blocking AI and LLM Crawlers: Should You Opt Out?<span class=\"ez-toc-section-end\"><\/span><\/h2>\n\n\n\n<p class=\"wp-block-paragraph\">Since 2023, a growing number of AI companies have deployed crawlers to scrape publicly accessible web content for training large language models. This has prompted considerable debate among publishers and content creators about whether \u2014 and how \u2014 to restrict such access.<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">Unlike search engine crawlers, AI training bots typically offer no direct benefit to your site&#8217;s discoverability or traffic. If your content is scraped for model training, you receive no attribution, no backlink, and no visitor. For this reason, many publishers are choosing to opt out by blocking known AI crawlers via robots.txt file.<\/p>\n\n\n\n<figure class=\"wp-block-table\"><table class=\"has-fixed-layout\"><thead><tr><th>User-agent<\/th><th>Company<\/th><th>Purpose<\/th><\/tr><\/thead><tbody><tr><td><code>GPTBot<\/code><\/td><td>OpenAI<\/td><td>Training data for GPT models<\/td><\/tr><tr><td><code>OAI-SearchBot<\/code><\/td><td>OpenAI<\/td><td>ChatGPT search index<\/td><\/tr><tr><td><code>ChatGPT-User<\/code><\/td><td>OpenAI<\/td><td>Real-time browsing by users<\/td><\/tr><tr><td><code>PerplexityBot<\/code><\/td><td>Perplexity AI<\/td><td>AI search engine indexing<\/td><\/tr><tr><td><code>ClaudeBot<\/code><\/td><td>Anthropic<\/td><td>Training data collection<\/td><\/tr><tr><td><code>Google-Extended<\/code><\/td><td>Google<\/td><td>Gemini AI training opt-out<\/td><\/tr><\/tbody><\/table><\/figure>\n\n\n\n<p class=\"wp-block-paragraph\"><strong>Important:<\/strong> Blocking <code>GPTBot<\/code> does <em>not<\/em> block <code>OAI-SearchBot<\/code>, which powers ChatGPT&#8217;s search results. If you wish to exclude your site from both training data and ChatGPT search results, you must list each user-agent separately. Our generator handles this automatically when you select multiple bots.<\/p>\n\n\n\n<h2 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"Common_Robotstxt_Mistakes_to_Avoid\"><\/span>Common Robots.txt Mistakes to Avoid<span class=\"ez-toc-section-end\"><\/span><\/h2>\n\n\n\n<ol class=\"wp-block-list\">\n<li><strong>Blocking Your Entire Site by Accident<\/strong>: The most damaging error is accidentally disallowing all crawlers from all content. The directive <code>Disallow: \/<\/code> under <code>User-agent: *<\/code> will prevent every search engine from indexing your site. Always verify that your global wildcard block includes an explicit <code>Allow: \/<\/code> after any specific disallow rules.<\/li>\n\n\n\n<li><strong>Using Robots.txt to Hide Sensitive Data<\/strong>: Robots.txt is a <em>publicly accessible file<\/em>. Any path you list in a <code>Disallow<\/code> directive is, by definition, visible to anyone who reads your robots.txt. For genuinely sensitive content, use server-side authentication, not robots.txt directives.<\/li>\n\n\n\n<li><strong>Blocking CSS and JavaScript Files:<\/strong> Googlebot renders pages visually and needs access to your CSS and JavaScript in order to accurately assess your page content and layout. Blocking these file types can negatively affect how your pages are understood and ranked.<\/li>\n\n\n\n<li><strong>Placing the File in the Wrong Location:<\/strong> Your robots.txt file must reside at the root of your domain: <code>https:\/\/yourdomain.com\/robots.txt<\/code>.  A file placed in a subdirectory such as this <code>\/blog\/robots.txt<\/code> will not be recognized by crawlers.<\/li>\n\n\n\n<li><strong>Assuming All Bots Comply:<\/strong> Reputable crawlers from Google, Bing, and most legitimate SEO tools respect robots.txt. However, malicious bots and scrapers typically do not. Robots.txt is a directive, not a technical barrier \u2014 use additional security measures for true access control.<\/li>\n<\/ol>\n\n\n\n<h2 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"How_to_Add_Robotstxt_to_a_WordPress_Site\"><\/span>How to Add Robots.txt to a WordPress Site<span class=\"ez-toc-section-end\"><\/span><\/h2>\n\n\n\n<p class=\"wp-block-paragraph\">WordPress 5.3 and later generates a virtual robots.txt file automatically via its built-in virtual file handling. There are two primary approaches for customizing it.<\/p>\n\n\n\n<ul class=\"wp-block-list\">\n<li><strong>Option A \u2014 Via Yoast SEO or Rank Math:<\/strong> Both plugins provide a built-in robots.txt editor under their respective settings menus. Paste the output from our generator directly into the editor and save. This is the recommended method, as it avoids directly editing server files.<\/li>\n\n\n\n<li><strong>Option B \u2014 Upload directly to your server:<\/strong> Download the <code>robots.txt<\/code> file produced by our generator and upload it to the root directory of your hosting account using FTP or your hosting control panel&#8217;s file manager. Ensure it is placed alongside your <code>wp-config.php<\/code> file, not inside the <code>\/wp-content\/<\/code> folder.<\/li>\n<\/ul>\n\n\n\n<p class=\"wp-block-paragraph\"><strong>Tip:<\/strong> After uploading or saving your robots.txt, verify it is accessible by visiting <code>https:\/\/yourdomain.com\/robots.txt<\/code> in your browser. You can also test it using Google Search Console&#8217;s Robots.txt Tester tool.<\/p>\n\n\n\n<h2 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"Frequently_Asked_Questions_on_Robotstxt_Generator\"><\/span>Frequently Asked Questions on Robots.txt Generator<span class=\"ez-toc-section-end\"><\/span><\/h2>\n\n\n<div id=\"rank-math-faq\" class=\"rank-math-block\">\n<div class=\"rank-math-list \">\n<div id=\"faq-question-1780139300660\" class=\"rank-math-list-item\">\n<p class=\"rank-math-question \">Does robots.txt affect my Google rankings?<\/p>\n<div class=\"rank-math-answer \">\n\n<p>Indirectly, yes. Blocking low-value pages from crawling preserves your crawl budget for high-value content and prevents duplicate or thin pages from being indexed. However, robots.txt directives do not directly raise or lower a page&#8217;s ranking signals\u2014they simply control crawler access.<\/p>\n\n<\/div>\n<\/div>\n<div id=\"faq-question-1780139311710\" class=\"rank-math-list-item\">\n<p class=\"rank-math-question \">What is the difference between robots.txt and a noindex tag?<\/p>\n<div class=\"rank-math-answer \">\n\n<p>A <code>Disallow<\/code> directive in robots.txt prevents a crawler from visiting a page. A <code>noindex<\/code> meta tag allows a crawler to visit the page but instructs it not to include the page in search results. Notably, Google cannot see a <code>noindex<\/code> tag on a page it is blocked from crawling, so never use both simultaneously on the same URL.<\/p>\n\n<\/div>\n<\/div>\n<div id=\"faq-question-1780139314877\" class=\"rank-math-list-item\">\n<p class=\"rank-math-question \">How often should I update my robots.txt file?<\/p>\n<div class=\"rank-math-answer \">\n\n<p>Review your robots.txt whenever you add new sections to your website, install new plugins that create new URL structures, or when new significant crawlers emerge (as has been the case with AI bots). An annual audit is a reasonable baseline for most blogs.<\/p>\n\n<\/div>\n<\/div>\n<div id=\"faq-question-1780139325877\" class=\"rank-math-list-item\">\n<p class=\"rank-math-question \">Do I need a robots.txt file if my site is small?<\/p>\n<div class=\"rank-math-answer \">\n\n<p>Even small sites benefit from a robots.txt file that, at minimum, points to the sitemap and restricts admin directory access. It establishes good hygiene and ensures no inadvertent exposure of backend paths occurs as your site grows.<\/p>\n\n<\/div>\n<\/div>\n<div id=\"faq-question-1780139334333\" class=\"rank-math-list-item\">\n<p class=\"rank-math-question \">Is it legal to block AI crawlers?<\/p>\n<div class=\"rank-math-answer \">\n\n<p>Yes. You are fully within your rights to instruct bots not to access your content via robots.txt. Reputable AI companies\u2014including OpenAI, Anthropic, and Google\u2014have publicly committed to honoring these directives.<\/p>\n\n<\/div>\n<\/div>\n<\/div>\n<\/div>\n\n\n<p class=\"wp-block-paragraph\"><\/p>\n\n\n\n<p class=\"wp-block-paragraph\"><\/p>\n\n\n\n<p class=\"wp-block-paragraph\"><\/p>\n\n\n\n<p class=\"wp-block-paragraph\"><\/p>\n\n\n\n<p class=\"wp-block-paragraph\"><\/p>\n\n\n\n<p class=\"wp-block-paragraph\"><\/p>\n\n\n\n<p class=\"wp-block-paragraph\"><\/p>\n","protected":false},"excerpt":{"rendered":"<p>Robots.txt Generator Create a custom robots.txt file for your website. This file tells search engines and AI crawlers which pages they can or cannot crawl on your site. Your Website URL Enter your website&#8217;s full URL including https:\/\/ Sitemap URL (Optional) Enter the URL of your XML sitemap Common Restrictions: Block admin pages (\/wp-admin\/, \/admin\/)&#8230;<\/p>\n","protected":false},"author":1,"featured_media":55,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"_kad_post_transparent":"","_kad_post_title":"","_kad_post_layout":"","_kad_post_sidebar_id":"","_kad_post_content_style":"","_kad_post_vertical_padding":"","_kad_post_feature":"","_kad_post_feature_position":"","_kad_post_header":false,"_kad_post_footer":false,"_kad_post_classname":"","footnotes":""},"categories":[2],"tags":[49,37,35,46,48,50,36,45,42,40,41,34,43,38,33,39,47,44],"class_list":["post-52","post","type-post","status-publish","format-standard","has-post-thumbnail","hentry","category-blogging-tools","tag-bdtools","tag-block-ai-crawlers","tag-block-chatgpt-crawler","tag-block-googlebot","tag-blogging-tools","tag-bloggingden-tools","tag-claudebot-disallow","tag-crawl-budget","tag-create-robots-txt","tag-free-seo-tools","tag-google-extended-opt-out","tag-gptbot-block","tag-robots-exclusion-protocol","tag-robots-txt-file","tag-robots-txt-generator","tag-seo-tools","tag-sitemap-robots-txt","tag-wordpress-robots-txt"],"_links":{"self":[{"href":"https:\/\/bloggingden.com\/tools\/wp-json\/wp\/v2\/posts\/52","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/bloggingden.com\/tools\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/bloggingden.com\/tools\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/bloggingden.com\/tools\/wp-json\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/bloggingden.com\/tools\/wp-json\/wp\/v2\/comments?post=52"}],"version-history":[{"count":0,"href":"https:\/\/bloggingden.com\/tools\/wp-json\/wp\/v2\/posts\/52\/revisions"}],"wp:featuredmedia":[{"embeddable":true,"href":"https:\/\/bloggingden.com\/tools\/wp-json\/wp\/v2\/media\/55"}],"wp:attachment":[{"href":"https:\/\/bloggingden.com\/tools\/wp-json\/wp\/v2\/media?parent=52"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/bloggingden.com\/tools\/wp-json\/wp\/v2\/categories?post=52"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/bloggingden.com\/tools\/wp-json\/wp\/v2\/tags?post=52"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}