feat: added ignoreClasses option to be able to exclude specific conte…

…nt from indexing
easyops-cn · Sep 4, 2023 · eef8a7e · eef8a7e
1 parent 565199a
commit eef8a7e
Show file tree

Hide file tree

Showing 12 changed files with 94 additions and 11 deletions.
diff --git a/README.md b/README.md
@@ -87,6 +87,7 @@ module.exports = {
 | searchResultContextMaxLength     | number                                   | `50`      | Set the max length of characters of each search result to show.                                                                                                                                                                                                                                                                                                |
 | explicitSearchResultPath         | boolean                                  | `false`   | Whether an explicit path to a heading should be presented on a suggestion template.                                                                                                                                                                                                                                                                            |
 | ignoreFiles                      | string \| RegExp \| (string \| RegExp)[] | `[]`      | Set the match rules to ignore some routes. Put a string if you want an exact match, or put a regex if you want a partial match. Note: without the website base url.                                                                                                                                                                                            |
+| ignoreClasses                    | string \| string[]                       | `[]`      | A list of html classes to ignore when indexing each page.                                                                                                                                                                                                                                                                                                      |
 | searchBarShortcut                | boolean                                  | `true`    | Whether to enable keyboard shortcut to focus in search bar.                                                                                                                                                                                                                                                                                                    |
 | searchBarShortcutHint            | boolean                                  | `true`    | Whether to show keyboard shortcut hint in search bar. Disable it if you need to hide the hint while shortcut is still enabled.                                                                                                                                                                                                                                 |
 | searchBarPosition                | `"auto"` \| `"left"` \| `"right"`        | `"auto"`  | The side of the navbar the search bar should appear on. By default, it will try to autodetect based on your docusaurus config according to [the docs](https://docusaurus.io/docs/api/themes/configuration#navbar-search).                                                                                                                                      |

diff --git a/docusaurus-search-local/src/index.ts b/docusaurus-search-local/src/index.ts
@@ -120,6 +120,13 @@ export interface PluginOptions {
    */
   ignoreFiles?: string | RegExp | (string | RegExp)[];
 
+  /**
+   * A list of html classes to ignore when indexing each page.
+   *
+   * @default []
+   */
+  ignoreClasses?: string | string[];
+
   /**
    * Whether to enable keyboard shortcut to focus in search bar.
    *

diff --git a/docusaurus-search-local/src/server/utils/parse.spec.ts b/docusaurus-search-local/src/server/utils/parse.spec.ts
@@ -1,4 +1,7 @@
-import { ParsedDocument } from "../../shared/interfaces";
+import {
+  ParsedDocument,
+  ProcessedPluginOptions,
+} from "../../shared/interfaces";
 import { parse } from "./parse";
 
 describe("parse", () => {
@@ -56,7 +59,36 @@ describe("parse", () => {
         breadcrumb: [],
       },
     ],
+    [
+      `<body>
+        <article>
+          <header>
+            <h1>Hello World</h1>
+          </header>
+          <main>
+            <span class="ignore">Test</span>
+            Peace.
+          </main>
+        </article>
+      </body>`,
+      "docs",
+      {
+        pageTitle: "Hello World",
+        sections: [
+          {
+            title: "Hello World",
+            hash: "",
+            content: "Peace.",
+          },
+        ],
+        breadcrumb: [],
+      },
+    ],
   ])("parse(...) should work", (html, type, doc) => {
-    expect(parse(html, type, "")).toEqual(doc);
+    expect(
+      parse(html, type, "", {
+        ignoreClasses: ["ignore"],
+      } as ProcessedPluginOptions)
+    ).toEqual(doc);
   });
 });
diff --git a/docusaurus-search-local/src/server/utils/parse.ts b/docusaurus-search-local/src/server/utils/parse.ts
@@ -1,18 +1,27 @@
 import cheerio from "cheerio";
-import { ParsedDocument } from "../../shared/interfaces";
+import {
+  ParsedDocument,
+  ProcessedPluginOptions,
+} from "../../shared/interfaces";
 import { parseDocument } from "./parseDocument";
 import { parsePage } from "./parsePage";
 
 export function parse(
   html: string,
   type: "docs" | "blog" | "page",
-  url: string
+  url: string,
+  { ignoreClasses }: ProcessedPluginOptions
 ): ParsedDocument {
   const $ = cheerio.load(html);
-
   // Remove copy buttons from code boxes
   $('div[class^="mdxCodeBlock_"] button').remove();
 
+  if (ignoreClasses) {
+    for (const ignoreClass of ignoreClasses) {
+      $("." + ignoreClass).remove();
+    }
+  }
+
   if (type === "docs") {
     // Remove version badges
     $("span.badge")

diff --git a/docusaurus-search-local/src/server/utils/postBuildFactory.ts b/docusaurus-search-local/src/server/utils/postBuildFactory.ts
@@ -26,7 +26,7 @@ export function postBuildFactory(
 
     for (const versionData of data) {
       // Give every index entry a unique id so that the index does not need to store long URLs.
-      const allDocuments = await scanDocuments(versionData.paths);
+      const allDocuments = await scanDocuments(versionData.paths, config);
 
       debugInfo("building index");
 

diff --git a/docusaurus-search-local/src/server/utils/processPluginOptions.spec.ts b/docusaurus-search-local/src/server/utils/processPluginOptions.spec.ts
@@ -13,6 +13,7 @@ describe("processPluginOptions", () => {
         blogDir: "blog",
         language: "en",
         ignoreFiles: "test",
+        ignoreClasses: [],
         searchBarPosition: "auto",
       },
       {
@@ -22,6 +23,7 @@ describe("processPluginOptions", () => {
         docsDir: [expect.toMatchPath("/tmp/docs")],
         language: ["en"],
         ignoreFiles: ["test"],
+        ignoreClasses: [],
         searchBarPosition: "right",
       },
     ],
@@ -33,6 +35,7 @@ describe("processPluginOptions", () => {
         blogDir: "blog",
         language: ["en", "zh"],
         ignoreFiles: [/__meta__$/],
+        ignoreClasses: [],
         searchBarPosition: "left",
       },
       {
@@ -42,6 +45,7 @@ describe("processPluginOptions", () => {
         docsDir: [expect.toMatchPath("/tmp/docs")],
         language: ["en", "zh"],
         ignoreFiles: [/__meta__$/],
+        ignoreClasses: [],
         searchBarPosition: "left",
       },
     ],
@@ -66,6 +70,7 @@ describe("processPluginOptions", () => {
           blogDir: "blog",
           language: "en",
           ignoreFiles: "test",
+          ignoreClasses: [],
           searchBarPosition: "auto",
         },
         {
@@ -95,6 +100,7 @@ describe("processPluginOptions", () => {
       docsDir: [expect.toMatchPath("/tmp/docs")],
       language: ["en"],
       ignoreFiles: ["test"],
+      ignoreClasses: [],
       searchBarPosition: "left",
     });
   });

diff --git a/docusaurus-search-local/src/server/utils/processPluginOptions.ts b/docusaurus-search-local/src/server/utils/processPluginOptions.ts
@@ -19,6 +19,7 @@ export function processPluginOptions(
   ensureArray(config, "docsDir");
   ensureArray(config, "blogDir");
   ensureArray(config, "ignoreFiles");
+  ensureArray(config, "ignoreClasses");
   config.docsRouteBasePath = config.docsRouteBasePath.map((basePath) =>
     basePath.replace(/^\//, "")
   );

diff --git a/docusaurus-search-local/src/server/utils/scanDocuments.spec.ts b/docusaurus-search-local/src/server/utils/scanDocuments.spec.ts
@@ -1,6 +1,9 @@
 import fs from "fs";
 import { parse } from "./parse";
-import { DocInfoWithFilePath } from "../../shared/interfaces";
+import {
+  DocInfoWithFilePath,
+  ProcessedPluginOptions,
+} from "../../shared/interfaces";
 
 jest.mock("./parse");
 jest.spyOn(fs, "readFile").mockImplementation(((
@@ -63,7 +66,10 @@ describe("scanDocuments", () => {
         };
       }
     });
-    const allDocuments = await scanDocuments(DocInfoWithFilePathList);
+    const allDocuments = await scanDocuments(
+      DocInfoWithFilePathList,
+      {} as ProcessedPluginOptions
+    );
     expect(allDocuments).toMatchInlineSnapshot(`
       Array [
         Array [

diff --git a/docusaurus-search-local/src/server/utils/scanDocuments.ts b/docusaurus-search-local/src/server/utils/scanDocuments.ts
@@ -1,7 +1,11 @@
 import fs from "fs";
 import path from "path";
 import util from "util";
-import { DocInfoWithFilePath, SearchDocument } from "../../shared/interfaces";
+import {
+  DocInfoWithFilePath,
+  SearchDocument,
+  ProcessedPluginOptions,
+} from "../../shared/interfaces";
 import { parse } from "./parse";
 import { debugVerbose } from "./debug";
 
@@ -13,7 +17,8 @@ const getNextDocId = () => {
 };
 
 export async function scanDocuments(
-  DocInfoWithFilePathList: DocInfoWithFilePath[]
+  DocInfoWithFilePathList: DocInfoWithFilePath[],
+  config: ProcessedPluginOptions
 ): Promise<SearchDocument[][]> {
   const titleDocuments: SearchDocument[] = [];
   const headingDocuments: SearchDocument[] = [];
@@ -30,7 +35,12 @@ export async function scanDocuments(
       );
 
       const html = await readFileAsync(filePath, { encoding: "utf8" });
-      const { pageTitle, sections, breadcrumb } = parse(html, type, url);
+      const { pageTitle, sections, breadcrumb } = parse(
+        html,
+        type,
+        url,
+        config
+      );
 
       const titleId = getNextDocId();
 

diff --git a/docusaurus-search-local/src/server/utils/validateOptions.spec.ts b/docusaurus-search-local/src/server/utils/validateOptions.spec.ts
@@ -52,6 +52,7 @@ describe("validateOptions", () => {
         explicitSearchResultPath: false,
         searchResultContextMaxLength: 50,
         ignoreFiles: [],
+        ignoreClasses: [],
         searchBarShortcut: true,
         searchBarShortcutHint: true,
         searchBarPosition: "auto",
@@ -78,6 +79,7 @@ describe("validateOptions", () => {
         explicitSearchResultPath: false,
         searchResultContextMaxLength: 50,
         ignoreFiles: "file1",
+        ignoreClasses: [],
         searchBarShortcut: true,
         searchBarShortcutHint: true,
         searchBarPosition: "auto",
@@ -104,6 +106,7 @@ describe("validateOptions", () => {
         explicitSearchResultPath: false,
         searchResultContextMaxLength: 50,
         ignoreFiles: [/__meta__$/, "file1"],
+        ignoreClasses: [],
         searchBarShortcut: true,
         searchBarShortcutHint: true,
         searchBarPosition: "auto",
@@ -130,6 +133,7 @@ describe("validateOptions", () => {
         explicitSearchResultPath: false,
         searchResultContextMaxLength: 50,
         ignoreFiles: [],
+        ignoreClasses: [],
         searchBarShortcut: true,
         searchBarShortcutHint: true,
         searchBarPosition: "auto",
@@ -167,6 +171,7 @@ describe("validateOptions", () => {
         explicitSearchResultPath: false,
         searchResultContextMaxLength: 30,
         ignoreFiles: [],
+        ignoreClasses: [],
         searchBarShortcut: false,
         searchBarShortcutHint: true,
         searchBarPosition: "auto",
@@ -198,6 +203,7 @@ describe("validateOptions", () => {
         explicitSearchResultPath: false,
         searchResultContextMaxLength: 50,
         ignoreFiles: [],
+        ignoreClasses: [],
         searchBarShortcut: true,
         searchBarShortcutHint: false,
         searchBarPosition: "auto",
@@ -233,6 +239,7 @@ describe("validateOptions", () => {
         explicitSearchResultPath: false,
         searchResultContextMaxLength: 50,
         ignoreFiles: [],
+        ignoreClasses: [],
         searchBarShortcut: true,
         searchBarShortcutHint: true,
         searchBarPosition: "left",
@@ -269,6 +276,7 @@ describe("validateOptions", () => {
         explicitSearchResultPath: false,
         searchResultContextMaxLength: 50,
         ignoreFiles: [],
+        ignoreClasses: [],
         searchBarShortcut: true,
         searchBarShortcutHint: true,
         searchBarPosition: "left",

diff --git a/docusaurus-search-local/src/server/utils/validateOptions.ts b/docusaurus-search-local/src/server/utils/validateOptions.ts
@@ -36,6 +36,7 @@ const schema = Joi.object<PluginOptions>({
   searchResultContextMaxLength: Joi.number().default(50),
   explicitSearchResultPath: Joi.boolean().default(false),
   ignoreFiles: isArrayOfStringsOrRegExpsOrStringOrRegExp.default([]),
+  ignoreClasses: isStringOrArrayOfStrings.default([]),
   searchBarShortcut: Joi.boolean().default(true),
   searchBarShortcutHint: Joi.boolean().default(true),
   searchBarPosition: Joi.string().default("auto"),

diff --git a/docusaurus-search-local/src/shared/interfaces.ts b/docusaurus-search-local/src/shared/interfaces.ts
@@ -143,6 +143,7 @@ export type ProcessedPluginOptions = Required<
     | "docsDir"
     | "blogDir"
     | "ignoreFiles"
+    | "ignoreClasses"
   >
 > & {
   docsRouteBasePath: string[];
@@ -151,6 +152,7 @@ export type ProcessedPluginOptions = Required<
   docsDir: string[];
   blogDir: string[];
   ignoreFiles: (string | RegExp)[];
+  ignoreClasses: string[];
 };
 
 export interface PostBuildData {