fix: comments being treated as errors (#152)

Author: hacknug

Diff for: src/runtime/util.ts

@@ -31,8 +31,9 @@ export function parseRobotsTxt(s: string): ParsedRobotsTxt {
   }
   let ln = -1
   // read the contents
-  for (const line of s.split('\n')) {
+  for (const _line of s.split('\n')) {
     ln++
+    const [line, comment] = _line.split('#').map(s => s.trim())
     const sepIndex = line.indexOf(':')
     // may not exist for comments
     if (sepIndex === -1)

Diff for: test/fixtures/comments.txt

@@ -0,0 +1,4 @@
+# line comment
+
+User-agent: *   # 1st inline comment
+Allow: /        # 2nd inline comment

Diff for: test/unit/robotsTxtParser.test.ts

@@ -355,4 +355,26 @@ Unknown: /bar
       ]
     `)
   })
+
+  it('comments do not error', async () => {
+    const robotsTxt = await fsp.readFile('./test/fixtures/comments.txt', { encoding: 'utf-8' })
+    expect(parseRobotsTxt(robotsTxt)).toMatchInlineSnapshot(`
+      {
+        "errors": [],
+        "groups": [
+          {
+            "allow": [
+              "/",
+            ],
+            "comment": [],
+            "disallow": [],
+            "userAgent": [
+              "*",
+            ],
+          },
+        ],
+        "sitemaps": [],
+      }
+    `)
+  })
 })