Merge pull request #348 from Cyriuz/add-ignoreClasses-option

Add ignoreCssSelectors option
easyops-cn · Sep 9, 2023 · 211a817 · 211a817
2 parents 4336500 + 4e2bc60
commit 211a817
Show file tree

Hide file tree

Showing 18 changed files with 233 additions and 56 deletions.
diff --git a/README.md b/README.md
@@ -87,6 +87,7 @@ module.exports = {
 | searchResultContextMaxLength     | number                                   | `50`      | Set the max length of characters of each search result to show.                                                                                                                                                                                                                                                                                                |
 | explicitSearchResultPath         | boolean                                  | `false`   | Whether an explicit path to a heading should be presented on a suggestion template.                                                                                                                                                                                                                                                                            |
 | ignoreFiles                      | string \| RegExp \| (string \| RegExp)[] | `[]`      | Set the match rules to ignore some routes. Put a string if you want an exact match, or put a regex if you want a partial match. Note: without the website base url.                                                                                                                                                                                            |
+| ignoreCssSelectors               | string \| string[]                       | `[]`      | A list of css selectors to ignore when indexing each page.                                                                                                                                                                                                                                                                                                      |
 | searchBarShortcut                | boolean                                  | `true`    | Whether to enable keyboard shortcut to focus in search bar.                                                                                                                                                                                                                                                                                                    |
 | searchBarShortcutHint            | boolean                                  | `true`    | Whether to show keyboard shortcut hint in search bar. Disable it if you need to hide the hint while shortcut is still enabled.                                                                                                                                                                                                                                 |
 | searchBarPosition                | `"auto"` \| `"left"` \| `"right"`        | `"auto"`  | The side of the navbar the search bar should appear on. By default, it will try to autodetect based on your docusaurus config according to [the docs](https://docusaurus.io/docs/api/themes/configuration#navbar-search).                                                                                                                                      |

diff --git a/docusaurus-search-local/src/index.ts b/docusaurus-search-local/src/index.ts
@@ -120,6 +120,13 @@ export interface PluginOptions {
    */
   ignoreFiles?: string | RegExp | (string | RegExp)[];
 
+  /**
+   * A list of css selectors to ignore when indexing each page.
+   *
+   * @default []
+   */
+  ignoreCssSelectors?: string | string[];
+
   /**
    * Whether to enable keyboard shortcut to focus in search bar.
    *

diff --git a/docusaurus-search-local/src/server/utils/generate.spec.ts b/docusaurus-search-local/src/server/utils/generate.spec.ts
@@ -18,7 +18,7 @@ describe("generate", () => {
     [
       ["en"],
       [
-        expect.stringMatching(/^import lunr from ".+\/lunr\/lunr\.js";$/),
+        expect.stringMatching(/^import lunr from ".+\/|\\lunr\/|\\lunr\.js";$/),
         'export const language = ["en"];',
         "export const removeDefaultStopWordFilter = false;",
         "export const removeDefaultStemmer = false;",
@@ -40,9 +40,9 @@ describe("generate", () => {
     [
       ["zh"],
       [
-        expect.stringMatching(/^import lunr from ".+\/lunr\/lunr\.js";$/),
+        expect.stringMatching(/^import lunr from ".+\/|\\lunr\/|\\lunr\.js";$/),
         expect.stringMatching(
-          /^require\(".+\/lunr-languages\/lunr\.stemmer\.support\.js"\)\(lunr\);$/
+          /^require\(".+\/|\\lunr-languages\/|\\lunr\.stemmer\.support\.js"\)\(lunr\);$/
         ),
         'require("@easyops-cn/docusaurus-search-local/dist/client/shared/lunrLanguageZh").lunrLanguageZh(lunr);',
         'export const language = ["zh"];',
@@ -66,12 +66,12 @@ describe("generate", () => {
     [
       ["es"],
       [
-        expect.stringMatching(/^import lunr from ".+\/lunr\/lunr\.js";$/),
+        expect.stringMatching(/^import lunr from ".+\/|\\lunr\/|\\lunr\.js";$/),
         expect.stringMatching(
-          /^require\(".+\/lunr-languages\/lunr\.stemmer\.support\.js"\)\(lunr\);$/
+          /^require\(".+\/|\\lunr-languages\/|\\lunr\.stemmer\.support\.js"\)\(lunr\);$/
         ),
         expect.stringMatching(
-          /^require\(".+\/lunr-languages\/lunr\.es\.js"\)\(lunr\);$/
+          /^require\(".+\/|\\lunr-languages\/|\\lunr\.es\.js"\)\(lunr\);$/
         ),
         'export const language = ["es"];',
         "export const removeDefaultStopWordFilter = false;",
@@ -94,15 +94,15 @@ describe("generate", () => {
     [
       ["ja"],
       [
-        expect.stringMatching(/^import lunr from ".+\/lunr\/lunr\.js";$/),
+        expect.stringMatching(/^import lunr from ".+\/|\\lunr\/|\\lunr\.js";$/),
         expect.stringMatching(
-          /^require\(".+\/lunr-languages\/lunr\.stemmer\.support\.js"\)\(lunr\);$/
+          /^require\(".+\/|\\lunr-languages\/|\\lunr\.stemmer\.support\.js"\)\(lunr\);$/
         ),
         expect.stringMatching(
-          /^require\(".+\/lunr-languages\/tinyseg\.js"\)\(lunr\);$/
+          /^require\(".+\/|\\lunr-languages\/|\\tinyseg\.js"\)\(lunr\);$/
         ),
         expect.stringMatching(
-          /^require\(".+\/lunr-languages\/lunr\.ja\.js"\)\(lunr\);$/
+          /^require\(".+\/|\\lunr-languages\/|\\lunr\.ja\.js"\)\(lunr\);$/
         ),
         'export const language = ["ja"];',
         "export const removeDefaultStopWordFilter = false;",
@@ -125,13 +125,13 @@ describe("generate", () => {
     [
       ["en", "zh"],
       [
-        expect.stringMatching(/^import lunr from ".+\/lunr\/lunr\.js";$/),
+        expect.stringMatching(/^import lunr from ".+\/|\\lunr\/|\\lunr\.js";$/),
         expect.stringMatching(
-          /^require\(".+\/lunr-languages\/lunr\.stemmer\.support\.js"\)\(lunr\);$/
+          /^require\(".+\/|\\lunr-languages\/|\\lunr\.stemmer\.support\.js"\)\(lunr\);$/
         ),
         'require("@easyops-cn/docusaurus-search-local/dist/client/shared/lunrLanguageZh").lunrLanguageZh(lunr);',
         expect.stringMatching(
-          /^require\(".+\/lunr-languages\/lunr\.multi\.js"\)\(lunr\);$/
+          /^require\(".+\/|\\lunr-languages\/|\\lunr\.multi\.js"\)\(lunr\);$/
         ),
         'export const language = ["en","zh"];',
         "export const removeDefaultStopWordFilter = false;",
@@ -154,16 +154,16 @@ describe("generate", () => {
     [
       ["en", "es", "zh"],
       [
-        expect.stringMatching(/^import lunr from ".+\/lunr\/lunr\.js";$/),
+        expect.stringMatching(/^import lunr from ".+\/|\\lunr\/|\\lunr\.js";$/),
         expect.stringMatching(
-          /^require\(".+\/lunr-languages\/lunr\.stemmer\.support\.js"\)\(lunr\);$/
+          /^require\(".+\/|\\lunr-languages\/|\\lunr\.stemmer\.support\.js"\)\(lunr\);$/
         ),
         expect.stringMatching(
-          /^require\(".+\/lunr-languages\/lunr\.es\.js"\)\(lunr\);$/
+          /^require\(".+\/|\\lunr-languages\/|\\lunr\.es\.js"\)\(lunr\);$/
         ),
         'require("@easyops-cn/docusaurus-search-local/dist/client/shared/lunrLanguageZh").lunrLanguageZh(lunr);',
         expect.stringMatching(
-          /^require\(".+\/lunr-languages\/lunr\.multi\.js"\)\(lunr\);$/
+          /^require\(".+\/|\\lunr-languages\/|\\lunr\.multi\.js"\)\(lunr\);$/
         ),
         'export const language = ["en","es","zh"];',
         "export const removeDefaultStopWordFilter = false;",
@@ -200,7 +200,7 @@ describe("generate", () => {
       "/tmp"
     );
     expect(mockWriteFileSync).toBeCalledWith(
-      "/tmp/generated.js",
+      expect.toMatchPath("/tmp/generated.js"),
       expect.any(String)
     );
     const calledContents = (mockWriteFileSync.mock.calls[0][1] as string).split(
@@ -224,7 +224,7 @@ describe("generate", () => {
     );
 
     expect(mockWriteFileSync).toBeCalledWith(
-      "/tmp/generated.js",
+      expect.toMatchPath("/tmp/generated.js"),
       expect.stringContaining("export { default as Mark } from")
     );
   });
@@ -242,7 +242,7 @@ describe("generate", () => {
     );
 
     expect(mockWriteFileSync).toBeCalledWith(
-      "/tmp/generated.js",
+      expect.toMatchPath("/tmp/generated.js"),
       expect.stringContaining("export const searchBarShortcut = false")
     );
   });
@@ -260,7 +260,7 @@ describe("generate", () => {
     );
 
     expect(mockWriteFileSync).toBeCalledWith(
-      "/tmp/generated.js",
+      expect.toMatchPath("/tmp/generated.js"),
       expect.stringContaining("export const searchBarShortcutHint = false")
     );
   });
@@ -278,7 +278,7 @@ describe("generate", () => {
     );
 
     expect(mockWriteFileSync).toBeCalledWith(
-      "/tmp/generated.js",
+      expect.toMatchPath("/tmp/generated.js"),
       expect.stringContaining(
         'export const docsPluginIdForPreferredVersion = "product"'
       )
@@ -298,7 +298,7 @@ describe("generate", () => {
     );
 
     expect(mockWriteFileSync).toBeCalledWith(
-      "/tmp/generated.js",
+      expect.toMatchPath("/tmp/generated.js"),
       expect.stringContaining(
         'export const searchIndexUrl = "search-index{dir}-abc.json"'
       )

diff --git a/docusaurus-search-local/src/server/utils/parse.spec.ts b/docusaurus-search-local/src/server/utils/parse.spec.ts
@@ -1,4 +1,7 @@
-import { ParsedDocument } from "../../shared/interfaces";
+import {
+  ParsedDocument,
+  ProcessedPluginOptions,
+} from "../../shared/interfaces";
 import { parse } from "./parse";
 
 describe("parse", () => {
@@ -56,7 +59,36 @@ describe("parse", () => {
         breadcrumb: [],
       },
     ],
+    [
+      `<body>
+        <article>
+          <header>
+            <h1>Hello World</h1>
+          </header>
+          <main>
+            <span class="ignore">Test</span>
+            Peace.
+          </main>
+        </article>
+      </body>`,
+      "docs",
+      {
+        pageTitle: "Hello World",
+        sections: [
+          {
+            title: "Hello World",
+            hash: "",
+            content: "Peace.",
+          },
+        ],
+        breadcrumb: [],
+      },
+    ],
   ])("parse(...) should work", (html, type, doc) => {
-    expect(parse(html, type, "")).toEqual(doc);
+    expect(
+      parse(html, type, "", {
+        ignoreCssSelectors: [".ignore"],
+      } as ProcessedPluginOptions)
+    ).toEqual(doc);
   });
 });
diff --git a/docusaurus-search-local/src/server/utils/parse.ts b/docusaurus-search-local/src/server/utils/parse.ts
@@ -1,18 +1,27 @@
 import cheerio from "cheerio";
-import { ParsedDocument } from "../../shared/interfaces";
+import {
+  ParsedDocument,
+  ProcessedPluginOptions,
+} from "../../shared/interfaces";
 import { parseDocument } from "./parseDocument";
 import { parsePage } from "./parsePage";
 
 export function parse(
   html: string,
   type: "docs" | "blog" | "page",
-  url: string
+  url: string,
+  { ignoreCssSelectors }: ProcessedPluginOptions
 ): ParsedDocument {
   const $ = cheerio.load(html);
-
   // Remove copy buttons from code boxes
   $('div[class^="mdxCodeBlock_"] button').remove();
 
+  if (ignoreCssSelectors) {
+    for (const ignoreSelector of ignoreCssSelectors) {
+      $(ignoreSelector).remove();
+    }
+  }
+
   if (type === "docs") {
     // Remove version badges
     $("span.badge")

diff --git a/docusaurus-search-local/src/server/utils/postBuildFactory.ts b/docusaurus-search-local/src/server/utils/postBuildFactory.ts
@@ -26,7 +26,7 @@ export function postBuildFactory(
 
     for (const versionData of data) {
       // Give every index entry a unique id so that the index does not need to store long URLs.
-      const allDocuments = await scanDocuments(versionData.paths);
+      const allDocuments = await scanDocuments(versionData.paths, config);
 
       debugInfo("building index");
 

diff --git a/docusaurus-search-local/src/server/utils/processDocInfos.spec.ts b/docusaurus-search-local/src/server/utils/processDocInfos.spec.ts
@@ -50,17 +50,17 @@ describe("processDocInfos", () => {
             outDir: "/build",
             paths: [
               {
-                filePath: "/build/docs/a/index.html",
+                filePath: expect.toMatchPath("/build/docs/a/index.html"),
                 type: "docs",
                 url: "/base/docs/a",
               },
               {
-                filePath: "/build/blog/b/index.html",
+                filePath: expect.toMatchPath("/build/blog/b/index.html"),
                 type: "blog",
                 url: "/base/blog/b",
               },
               {
-                filePath: "/build/page/index.html",
+                filePath: expect.toMatchPath("/build/page/index.html"),
                 type: "page",
                 url: "/base/page",
               },
@@ -120,17 +120,17 @@ describe("processDocInfos", () => {
             outDir: "/build",
             paths: [
               {
-                filePath: "/build/docs/a.html",
+                filePath: expect.toMatchPath("/build/docs/a.html"),
                 type: "docs",
                 url: "/base/docs/a",
               },
               {
-                filePath: "/build/blog/b.html",
+                filePath: expect.toMatchPath("/build/blog/b.html"),
                 type: "blog",
                 url: "/base/blog/b",
               },
               {
-                filePath: "/build/page.html",
+                filePath: expect.toMatchPath("/build/page.html"),
                 type: "page",
                 url: "/base/page",
               },
@@ -190,17 +190,17 @@ describe("processDocInfos", () => {
             outDir: "/build",
             paths: [
               {
-                filePath: "/build/docs/a/index.html",
+                filePath: expect.toMatchPath("/build/docs/a/index.html"),
                 type: "docs",
                 url: "/base/docs/a/",
               },
               {
-                filePath: "/build/blog/b/index.html",
+                filePath: expect.toMatchPath("/build/blog/b/index.html"),
                 type: "blog",
                 url: "/base/blog/b/",
               },
               {
-                filePath: "/build/page/index.html",
+                filePath: expect.toMatchPath("/build/page/index.html"),
                 type: "page",
                 url: "/base/page/",
               },
@@ -244,22 +244,22 @@ describe("processDocInfos", () => {
             outDir: "/build",
             paths: [
               {
-                filePath: "/build/index.html",
+                filePath: expect.toMatchPath("/build/index.html"),
                 type: "docs",
                 url: "/base/",
               },
               {
-                filePath: "/build/docs-a/index.html",
+                filePath: expect.toMatchPath("/build/docs-a/index.html"),
                 type: "docs",
                 url: "/base/docs-a",
               },
               {
-                filePath: "/build/docs-b/c/index.html",
+                filePath: expect.toMatchPath("/build/docs-b/c/index.html"),
                 type: "docs",
                 url: "/base/docs-b/c",
               },
               {
-                filePath: "/build/page/index.html",
+                filePath: expect.toMatchPath("/build/page/index.html"),
                 type: "docs",
                 url: "/base/page",
               },
@@ -303,17 +303,17 @@ describe("processDocInfos", () => {
             outDir: "/build",
             paths: [
               {
-                filePath: "/build/blog-a/index.html",
+                filePath: expect.toMatchPath("/build/blog-a/index.html"),
                 type: "blog",
                 url: "/base/blog-a",
               },
               {
-                filePath: "/build/blog-b/c/index.html",
+                filePath: expect.toMatchPath("/build/blog-b/c/index.html"),
                 type: "blog",
                 url: "/base/blog-b/c",
               },
               {
-                filePath: "/build/page/index.html",
+                filePath: expect.toMatchPath("/build/page/index.html"),
                 type: "blog",
                 url: "/base/page",
               },