fix: disallow /robots.txt when using baseURL

Author: harlan-zw

Diff for: docs/content/3.api/1.config.md

@@ -130,6 +130,13 @@ export default defineNuxtConfig({
 })
 ```
 
+## `robotsTxt`
+
+- Type: `boolean`
+- Default: `true`
+
+Whether to generate a `robots.txt` file. Useful for disabling when using a base URL.
+
 ## `debug`
 
 - Type: `boolean`

Diff for: src/module.ts

@@ -127,6 +127,12 @@ export interface ModuleOptions {
    * @default max-age=14400, must-revalidate
    */
   cacheControl?: string | false
+  /**
+   * Whether the robots.txt file should be generated. Useful to disable when running your app with a base URL.
+   *
+   * @default false
+   */
+  robotsTxt?: boolean
   /**
    * Enables debug logs and a debug endpoint.
    *
@@ -179,6 +185,7 @@ export default defineNuxtModule<ModuleOptions>({
     robotsEnabledValue: 'index, follow, max-image-preview:large, max-snippet:-1, max-video-preview:-1',
     robotsDisabledValue: 'noindex, nofollow',
     disallowNonIndexableRoutes: true,
+    robotsTxt: true,
   },
   async setup(config, nuxt) {
     const { resolve } = createResolver(import.meta.url)
@@ -204,6 +211,11 @@ export default defineNuxtModule<ModuleOptions>({
       return
     }
 
+    if (nuxt.options.app.baseURL?.length > 1 && config.robotsTxt) {
+      logger.error(`You are not allowed to generate a robots.txt with a base URL, please set \`{ robots: { robotsTxt: false } }\` in your nuxt.config.`)
+      config.robotsTxt = false
+    }
+
     // TODO remove with v5
     if (config.rules) {
       // warn v3 usage and convert to v4
@@ -257,7 +269,7 @@ export default defineNuxtModule<ModuleOptions>({
     if (config.metaTag)
       addPlugin({ mode: 'server', src: resolve('./runtime/nuxt/plugins/robot-meta.server') })
 
-    if (config.mergeWithRobotsTxtPath !== false) {
+    if (config.robotsTxt && config.mergeWithRobotsTxtPath !== false) {
       let usingRobotsTxtPath = ''
       let robotsTxt: boolean | string = false
       const publicRobotsTxtPath = resolve(nuxt.options.rootDir, nuxt.options.dir.public, 'robots.txt')
@@ -268,6 +280,8 @@ export default defineNuxtModule<ModuleOptions>({
         resolve(nuxt.options.rootDir, nuxt.options.dir.assets, 'robots.txt'),
         // public/_robots.txt
         resolve(nuxt.options.rootDir, nuxt.options.dir.public, '_robots.txt'),
+        // public/_robots.txt
+        resolve(nuxt.options.rootDir, nuxt.options.dir.public, '_robots.txt'),
         // public/_dir/robots.txt
         resolve(nuxt.options.rootDir, nuxt.options.dir.public, '_dir', 'robots.txt'),
         // pages/_dir/robots.txt
@@ -478,7 +492,7 @@ declare module 'h3' {
     const nitroPreset = resolveNitroPreset(nuxt.options.nitro)
     // only prerender for `nuxi generate`
     const isFirebase = nitroPreset === 'firebase'
-    if (isNuxtGenerate() || (isFirebase && nuxt.options._build)) {
+    if ((isNuxtGenerate() || (isFirebase && nuxt.options._build)) && config.robotsTxt) {
       nuxt.options.generate = nuxt.options.generate || {}
       nuxt.options.generate.routes = asArray(nuxt.options.generate.routes || [])
       nuxt.options.generate.routes.push('/robots.txt')
@@ -504,11 +518,13 @@ declare module 'h3' {
       filePath: resolve('./runtime/nuxt/components/RobotMeta'),
     })
 
-    // add robots.txt server handler
-    addServerHandler({
-      route: '/robots.txt',
-      handler: resolve('./runtime/nitro/server/robots-txt'),
-    })
+    if (config.robotsTxt) {
+      // add robots.txt server handler
+      addServerHandler({
+        route: '/robots.txt',
+        handler: resolve('./runtime/nitro/server/robots-txt'),
+      })
+    }
     // add robots HTTP header handler
     addServerHandler({
       handler: resolve('./runtime/nitro/server/middleware'),