feat: header, disable setting X-Robots-Tag header (#134)

Author: HADB

Diff for: src/module.ts

@@ -29,6 +29,12 @@ export interface ModuleOptions {
    * @default true
    */
   enabled: boolean
+  /**
+   * Should a `X-Robots-Tag` header be added to the response.
+   *
+   * @default true
+   */
+  header: boolean
   /**
    * Should a `<meta name="robots" content="<>">` tag be added to the head.
    *
@@ -161,6 +167,7 @@ export default defineNuxtModule<ModuleOptions>({
     groups: [],
     blockNonSeoBots: false,
     mergeWithRobotsTxtPath: true,
+    header: true,
     metaTag: true,
     cacheControl: 'max-age=14400, must-revalidate',
     robotsEnabledValue: 'index, follow, max-image-preview:large, max-snippet:-1, max-video-preview:-1',
@@ -362,18 +369,20 @@ export default defineNuxtModule<ModuleOptions>({
 
       nuxt.options.routeRules = nuxt.options.routeRules || {}
       // convert robot routeRules to header routeRules for static hosting
-      Object.entries(nuxt.options.routeRules).forEach(([route, rules]) => {
-        const url = route.split('/').map(segment => segment.startsWith(':') ? '*' : segment).join('/')
-        const groupIndexable = indexableFromGroup(config.groups, url)
-        const robotRules = normaliseRobotsRouteRule(rules, groupIndexable, config.robotsDisabledValue, config.robotsEnabledValue)
-        // single * is supported but ignored
-        // @ts-expect-error untyped
-        nuxt.options.routeRules[route] = defu({
-          headers: {
-            'X-Robots-Tag': robotRules.rule,
-          },
-        }, nuxt.options.routeRules?.[route])
-      })
+      if (config.header) {
+        Object.entries(nuxt.options.routeRules).forEach(([route, rules]) => {
+          const url = route.split('/').map(segment => segment.startsWith(':') ? '*' : segment).join('/')
+          const groupIndexable = indexableFromGroup(config.groups, url)
+          const robotRules = normaliseRobotsRouteRule(rules, groupIndexable, config.robotsDisabledValue, config.robotsEnabledValue)
+          // single * is supported but ignored
+          // @ts-expect-error untyped
+          nuxt.options.routeRules[route] = defu({
+            headers: {
+              'X-Robots-Tag': robotRules.rule,
+            },
+          }, nuxt.options.routeRules?.[route])
+        })
+      }
 
       const extraDisallows = new Set<string>()
       if (config.disallowNonIndexableRoutes) {
@@ -408,6 +417,7 @@ export default defineNuxtModule<ModuleOptions>({
         credits: config.credits,
         groups: config.groups,
         sitemap: config.sitemap,
+        header: config.header,
         robotsEnabledValue: config.robotsEnabledValue,
         robotsDisabledValue: config.robotsDisabledValue,
         // @ts-expect-error untyped

Diff for: src/runtime/nitro/server/middleware.ts

@@ -1,10 +1,14 @@
 import { defineEventHandler, setHeader } from 'h3'
 import { getPathRobotConfig } from '#internal/nuxt-robots'
+import { useRuntimeConfig } from '#imports'
 
 export default defineEventHandler(async (e) => {
   if (e.path === '/robots.txt' || e.path.startsWith('/__') || e.path.startsWith('/api') || e.path.startsWith('/_nuxt'))
     return
   const robotConfig = getPathRobotConfig(e)
-  setHeader(e, 'X-Robots-Tag', robotConfig.rule)
+  const { header } = useRuntimeConfig(e)['nuxt-robots']
+  if (header) {
+    setHeader(e, 'X-Robots-Tag', robotConfig.rule)
+  }
   e.context.robots = robotConfig
 })

Diff for: src/runtime/nuxt/composables/useRobotsRule.ts

@@ -37,7 +37,9 @@ export function useRobotsRule(rule?: MaybeRef<boolean | string>) {
       _rule = _rule ? config['nuxt-robots'].robotsEnabledValue : config['nuxt-robots'].robotsDisabledValue
     }
     event.context.robots.rule = _rule
-    setHeader(event, 'X-Robots-Tag', _rule)
+    if (config['nuxt-robots'].header) {
+      setHeader(event, 'X-Robots-Tag', _rule)
+    }
     useServerHead({
       meta: [
         {

Diff for: test/noHeader.test.ts

@@ -0,0 +1,24 @@
+import { describe, expect, it } from 'vitest'
+import { setup } from '@nuxt/test-utils'
+import { createResolver } from '@nuxt/kit'
+
+const { resolve } = createResolver(import.meta.url)
+
+process.env.NODE_ENV = 'production'
+await setup({
+  rootDir: resolve('../.playground'),
+  build: true,
+  nuxtConfig: {
+    robots: {
+      debug: true,
+      header: false,
+    },
+  },
+})
+
+describe('noHeader', () => {
+  it('basic', async () => {
+    const { headers, _data } = await $fetch.raw('/')
+    expect(headers.get('X-Robots-Tag')).toBe(null)
+  })
+})