fix: case insensitive directives when merging robots.txt (#150)

Co-authored-by: Philipp Naderer-Puiu <philipp.naderer-puiu@orf.at>

Author: harlan-zw

Diff for: src/runtime/util.ts

@@ -34,11 +34,11 @@ export function parseRobotsTxt(s: string): ParsedRobotsTxt {
     if (sepIndex === -1)
       continue
     // get the rule, pop before the first :
-    const rule = line.substring(0, sepIndex).trim()
+    const rule = line.substring(0, sepIndex).trim().toLowerCase()
     const val = line.substring(sepIndex + 1).trim()
 
     switch (rule) {
-      case 'User-agent':
+      case 'user-agent':
         if (createNewGroup) {
           groups.push({
             ...currentGroup,
@@ -53,21 +53,21 @@ export function parseRobotsTxt(s: string): ParsedRobotsTxt {
         }
         currentGroup.userAgent.push(val)
         break
-      case 'Allow':
+      case 'allow':
         currentGroup.allow.push(val)
         createNewGroup = true
         break
-      case 'Disallow':
+      case 'disallow':
         currentGroup.disallow.push(val)
         createNewGroup = true
         break
-      case 'Sitemap':
+      case 'sitemap':
         sitemaps.push(val)
         break
-      case 'Host':
+      case 'host':
         currentGroup.host = val
         break
-      case 'Clean-param':
+      case 'clean-param':
         if (currentGroup.userAgent.includes('Yandex')) {
           currentGroup.cleanParam = currentGroup.cleanParam || []
           currentGroup.cleanParam.push(val)

Diff for: test/fixtures/startgroupRobots.txt

@@ -0,0 +1,14 @@
+# see rfc 9309
+user-agent: ExampleBot
+disallow: /foo
+allow: /bar
+
+user-agent: examplebot
+disallow: /baz
+allow: /boo
+
+user-agent:
+disallow: /invalid
+
+user-agent: *
+disallow: /star

Diff for: test/unit/robotsTxtParser.test.ts

@@ -245,7 +245,7 @@ describe('robotsTxtParser', () => {
   })
 
   it('yandex', async () => {
-    // read fixture yoastRobots.txt
+    // read fixture yandex.txt
     const robotsTxt = await fsp.readFile('./test/fixtures/yandex.txt', { encoding: 'utf-8' })
     expect(parseRobotsTxt(robotsTxt)).toMatchInlineSnapshot(`
       {
@@ -281,4 +281,60 @@ describe('robotsTxtParser', () => {
       }
     `)
   })
+
+  it('case-insensitive startgroupline', async () => {
+    // read fixture startgroupRobots.txt
+    const robotsTxt = await fsp.readFile('./test/fixtures/startgroupRobots.txt', { encoding: 'utf-8' })
+    expect(parseRobotsTxt(robotsTxt)).toMatchInlineSnapshot(`
+      {
+        "groups": [
+          {
+            "allow": [
+              "/bar",
+            ],
+            "comment": [],
+            "disallow": [
+              "/foo",
+            ],
+            "userAgent": [
+              "ExampleBot",
+            ],
+          },
+          {
+            "allow": [
+              "/boo",
+            ],
+            "comment": [],
+            "disallow": [
+              "/baz",
+            ],
+            "userAgent": [
+              "examplebot",
+            ],
+          },
+          {
+            "allow": [],
+            "comment": [],
+            "disallow": [
+              "/invalid",
+            ],
+            "userAgent": [
+              "",
+            ],
+          },
+          {
+            "allow": [],
+            "comment": [],
+            "disallow": [
+              "/star",
+            ],
+            "userAgent": [
+              "*",
+            ],
+          },
+        ],
+        "sitemaps": [],
+      }
+    `)
+  })
 })