fix: warn when unknown directives are used

Author: harlan-zw

Diff for: src/module.ts

@@ -1,3 +1,5 @@
+import type { Preset } from 'unimport'
+import type { Arrayable, AutoI18nConfig, Robots3Rules, RobotsGroupInput, RobotsGroupResolved } from './runtime/types'
 import fsp from 'node:fs/promises'
 import {
   addComponent,
@@ -12,21 +14,19 @@ import {
 import { defu } from 'defu'
 import { installNuxtSiteConfig, updateSiteConfig } from 'nuxt-site-config-kit'
 import { relative } from 'pathe'
-import type { Preset } from 'unimport'
 import { readPackageJSON } from 'pkg-types'
+import { NonHelpfulBots } from './const'
+import { setupDevToolsUI } from './devtools'
+import { resolveI18nConfig, splitPathForI18nLocales } from './i18n'
+import { extendTypes, isNuxtGenerate, resolveNitroPreset } from './kit'
+import { logger } from './logger'
 import {
   asArray,
   normaliseRobotsRouteRule,
   normalizeGroup,
   parseRobotsTxt,
   validateRobots,
 } from './runtime/util'
-import { extendTypes, isNuxtGenerate, resolveNitroPreset } from './kit'
-import type { Arrayable, AutoI18nConfig, Robots3Rules, RobotsGroupInput, RobotsGroupResolved } from './runtime/types'
-import { NonHelpfulBots } from './const'
-import { resolveI18nConfig, splitPathForI18nLocales } from './i18n'
-import { setupDevToolsUI } from './devtools'
-import { logger } from './logger'
 
 export interface ModuleOptions {
   /**
@@ -301,7 +301,7 @@ export default defineNuxtModule<ModuleOptions>({
         const path = relative(nuxt.options.rootDir, usingRobotsTxtPath)
         logger.debug(`A robots.txt file was found at \`./${path}\`, merging config.`)
         const parsedRobotsTxt = parseRobotsTxt(robotsTxt)
-        const errors = validateRobots(parsedRobotsTxt)
+        const { errors } = validateRobots(parsedRobotsTxt)
         if (errors.length > 0) {
           logger.error(`The \`./${path}\` file contains errors:`)
           for (const error of errors)

Diff for: src/runtime/types.ts

@@ -5,6 +5,7 @@ export type Arrayable<T> = T | T[]
 export interface ParsedRobotsTxt {
   groups: RobotsGroupResolved[]
   sitemaps: string[]
+  errors: string[]
 }
 
 export type RobotsGroupInput = GoogleInput | YandexInput

Diff for: src/runtime/util.ts

@@ -1,7 +1,7 @@
-import { createDefu } from 'defu'
 import type { NitroRouteConfig } from 'nitropack'
-import { withoutLeadingSlash } from 'ufo'
 import type { ParsedRobotsTxt, RobotsGroupInput, RobotsGroupResolved } from './types'
+import { createDefu } from 'defu'
+import { withoutLeadingSlash } from 'ufo'
 
 /**
  * We're going to read the robots.txt and extract any disallow or sitemaps rules from it.
@@ -15,20 +15,24 @@ import type { ParsedRobotsTxt, RobotsGroupInput, RobotsGroupResolved } from './t
  * - host: the host name of the site, this is optional non-standard directive.
  *
  * @see https://developers.google.com/search/docs/crawling-indexing/robots/robots_txt
+ * @see https://github.com/google/robotstxt/blob/86d5836ba2d5a0b6b938ab49501be0e09d9c276c/robots.cc#L714C1-L720C2
  */
 export function parseRobotsTxt(s: string): ParsedRobotsTxt {
   // then we'll extract the disallow and sitemap rules
   const groups: RobotsGroupResolved[] = []
   const sitemaps: string[] = []
+  const errors: string[] = []
   let createNewGroup = false
   let currentGroup: RobotsGroupResolved = {
     comment: [], // comments are too hard to parse in a logical order, we'll just omit them
     disallow: [],
     allow: [],
     userAgent: [],
   }
+  let ln = -1
   // read the contents
   for (const line of s.split('\n')) {
+    ln++
     const sepIndex = line.indexOf(':')
     // may not exist for comments
     if (sepIndex === -1)
@@ -39,6 +43,8 @@ export function parseRobotsTxt(s: string): ParsedRobotsTxt {
 
     switch (rule) {
       case 'user-agent':
+      case 'useragent':
+      case 'user agent':
         if (createNewGroup) {
           groups.push({
             ...currentGroup,
@@ -58,20 +64,32 @@ export function parseRobotsTxt(s: string): ParsedRobotsTxt {
         createNewGroup = true
         break
       case 'disallow':
+      case 'dissallow':
+      case 'dissalow':
+      case 'disalow':
+      case 'diasllow':
+      case 'disallaw':
         currentGroup.disallow.push(val)
         createNewGroup = true
         break
       case 'sitemap':
+      case 'site-map':
         sitemaps.push(val)
         break
       case 'host':
         currentGroup.host = val
         break
       case 'clean-param':
-        if (currentGroup.userAgent.includes('Yandex')) {
+        if (currentGroup.userAgent.some(u => u.toLowerCase().includes('yandex'))) {
           currentGroup.cleanParam = currentGroup.cleanParam || []
           currentGroup.cleanParam.push(val)
         }
+        else {
+          errors.push(`L${ln}: Clean-param directive is only when targeting Yandex user agent.`)
+        }
+        break
+      default:
+        errors.push(`L${ln}: Unknown directive ${rule} `)
         break
     }
   }
@@ -82,6 +100,7 @@ export function parseRobotsTxt(s: string): ParsedRobotsTxt {
   return {
     groups,
     sitemaps,
+    errors,
   }
 }
 
@@ -167,17 +186,16 @@ export function matchPathToRule(path: string, _rules: RobotsGroupResolved['_rule
 }
 
 export function validateRobots(robotsTxt: ParsedRobotsTxt) {
-  const errors: string[] = []
   // 1. check that the include / exclude is either empty or starts with a slash OR a wildcard
   robotsTxt.groups = robotsTxt.groups.filter((group) => {
     if (!group.allow.length && !group.disallow.length) {
-      errors.push(`Group "${group.userAgent.join(', ')}" has no allow or disallow rules. You must provide one of either.`)
+      robotsTxt.errors.push(`Group "${group.userAgent.join(', ')}" has no allow or disallow rules. You must provide one of either.`)
       return false
     }
-    validateGroupRules(group, errors)
+    validateGroupRules(group, robotsTxt.errors)
     return true
   })
-  return errors
+  return robotsTxt
 }
 
 export function asArray(v: any) {

Diff for: test/unit/robotsTxtParser.test.ts

@@ -8,6 +8,7 @@ describe('robotsTxtParser', () => {
     const robotsTxt = await fsp.readFile('./test/fixtures/yoastRobots.txt', { encoding: 'utf-8' })
     expect(parseRobotsTxt(robotsTxt)).toMatchInlineSnapshot(`
       {
+        "errors": [],
         "groups": [
           {
             "allow": [],
@@ -58,6 +59,7 @@ describe('robotsTxtParser', () => {
     const robotsTxt = await fsp.readFile('./test/fixtures/squareSpace.txt', { encoding: 'utf-8' })
     expect(parseRobotsTxt(robotsTxt)).toMatchInlineSnapshot(`
       {
+        "errors": [],
         "groups": [
           {
             "allow": [
@@ -224,6 +226,7 @@ describe('robotsTxtParser', () => {
     const robotsTxt = await fsp.readFile('./test/fixtures/issue36.txt', { encoding: 'utf-8' })
     expect(parseRobotsTxt(robotsTxt)).toMatchInlineSnapshot(`
       {
+        "errors": [],
         "groups": [
           {
             "allow": [],
@@ -249,6 +252,7 @@ describe('robotsTxtParser', () => {
     const robotsTxt = await fsp.readFile('./test/fixtures/yandex.txt', { encoding: 'utf-8' })
     expect(parseRobotsTxt(robotsTxt)).toMatchInlineSnapshot(`
       {
+        "errors": [],
         "groups": [
           {
             "allow": [],
@@ -287,6 +291,7 @@ describe('robotsTxtParser', () => {
     const robotsTxt = await fsp.readFile('./test/fixtures/startgroupRobots.txt', { encoding: 'utf-8' })
     expect(parseRobotsTxt(robotsTxt)).toMatchInlineSnapshot(`
       {
+        "errors": [],
         "groups": [
           {
             "allow": [
@@ -337,4 +342,17 @@ describe('robotsTxtParser', () => {
       }
     `)
   })
+
+  it('collects errors', async () => {
+    // read fixture startgroupRobots.txt
+    expect(parseRobotsTxt(`
+User-Agent: *
+Disallow: /foo
+Unknown: /bar
+    `).errors).toMatchInlineSnapshot(`
+      [
+        "L3: Unknown directive unknown ",
+      ]
+    `)
+  })
 })

Diff for: test/unit/robotsTxtValidator.test.ts

@@ -3,7 +3,8 @@ import { validateRobots } from '../../src/runtime/util'
 
 describe('robotsTxtValidator', () => {
   it('basic', async () => {
-    const errors = validateRobots({
+    const { errors } = validateRobots({
+      errors: [],
       sitemaps: [],
       groups: [
         {