feat!: Rule Tester checks for missing placeholder data in the message

docs/src/use/migrate-to-9.0.0.md

@@ -553,6 +553,7 @@ In order to aid in the development of high-quality custom rules that are free fr
 1. **Suggestions must generate valid syntax.** In order for rule suggestions to be helpful, they need to be valid syntax. `RuleTester` now parses the output of suggestions using the same language options as the `code` value and throws an error if parsing fails.
 1. **Test cases must be unique.** Identical test cases can cause confusion and be hard to detect manually in a long test file. Duplicates are now automatically detected and can be safely removed.
 1. **`filename` and `only` must be of the expected type.** `RuleTester` now checks the type of `filename` and `only` properties of test objects. If specified, `filename` must be a string value. If specified, `only` must be a boolean value.
+1. **Messages cannot have unsubstituted placeholders.** The rule tester now checks if there are `{{ placeholder }}` still in the message as their values were not passed via `data` in the respective `context.report`.
 
 **To address:** Run your rule tests using `RuleTester` and fix any errors that occur. The changes you'll need to make to satisfy `RuleTester` are compatible with ESLint v8.x.
 

lib/linter/index.js

@@ -1,7 +1,7 @@
 "use strict";
 
 const { Linter } = require("./linter");
-const interpolate = require("./interpolate");
+const { interpolate } = require("./interpolate");
 const SourceCodeFixer = require("./source-code-fixer");
 
 module.exports = {

lib/linter/interpolate.js

@@ -9,20 +9,27 @@
 // Public Interface
 //------------------------------------------------------------------------------
 
-module.exports = (text, data) => {
-    if (!data) {
-        return text;
-    }
+module.exports = {
+    getPlaceholderMatcher() {
+        return /\{\{([^{}]+?)\}\}/gu;
+    },
+    interpolate(text, data) {
+        if (!data) {
+            return text;
+        }
 
-    // Substitution content for any {{ }} markers.
-    return text.replace(/\{\{([^{}]+?)\}\}/gu, (fullMatch, termWithWhitespace) => {
-        const term = termWithWhitespace.trim();
+        const matcher = module.exports.getPlaceholderMatcher();
 
-        if (term in data) {
-            return data[term];
-        }
+        // Substitution content for any {{ }} markers.
+        return text.replace(matcher, (fullMatch, termWithWhitespace) => {
+            const term = termWithWhitespace.trim();
 
-        // Preserve old behavior: If parameter name not provided, don't replace it.
-        return fullMatch;
-    });
+            if (term in data) {
+                return data[term];
+            }
+
+            // Preserve old behavior: If parameter name not provided, don't replace it.
+            return fullMatch;
+        });
+    }
 };

lib/linter/report-translator.js

@@ -11,7 +11,7 @@
 
 const assert = require("assert");
 const ruleFixer = require("./rule-fixer");
-const interpolate = require("./interpolate");
+const { interpolate } = require("./interpolate");
 
 //------------------------------------------------------------------------------
 // Typedefs

lib/rule-tester/rule-tester.js

@@ -17,7 +17,8 @@ const
     equal = require("fast-deep-equal"),
     Traverser = require("../shared/traverser"),
     { getRuleOptionsSchema } = require("../config/flat-config-helpers"),
-    { Linter, SourceCodeFixer, interpolate } = require("../linter"),
+    { Linter, SourceCodeFixer } = require("../linter"),
+    { interpolate, getPlaceholderMatcher } = require("../linter/interpolate"),
     stringify = require("json-stable-stringify-without-jsonify");
 
 const { FlatConfigArray } = require("../config/flat-config-array");
@@ -304,6 +305,39 @@ function throwForbiddenMethodError(methodName, prototype) {
     };
 }
 
+/**
+ * Extracts names of {{ placeholders }} from the reported message.
+ * @param   {string} message Reported message
+ * @returns {string[]} Array of placeholder names
+ */
+function getMessagePlaceholders(message) {
+    const matcher = getPlaceholderMatcher();
+
+    return Array.from(message.matchAll(matcher), ([, name]) => name.trim());
+}
+
+/**
+ * Returns the placeholders in the reported messages but
+ * only includes the placeholders available in the raw message and not in the provided data.
+ * @param {string} message The reported message
+ * @param {string} raw The raw message specified in the rule meta.messages
+ * @param {undefined|Record<unknown, unknown>} data The passed
+ * @returns {string[]} Missing placeholder names
+ */
+function getUnsubstitutedMessagePlaceholders(message, raw, data = {}) {
+    const unsubstituted = getMessagePlaceholders(message);
+
+    if (unsubstituted.length === 0) {
+        return [];
+    }
+
+    // Remove false positives by only counting placeholders in the raw message, which were not provided in the data matcher or added with a data property
+    const known = getMessagePlaceholders(raw);
+    const provided = Object.keys(data);
+
+    return unsubstituted.filter(name => known.includes(name) && !provided.includes(name));
+}
+
 const metaSchemaDescription = `
 \t- If the rule has options, set \`meta.schema\` to an array or non-empty object to enable options validation.
 \t- If the rule doesn't have options, omit \`meta.schema\` to enforce that no options can be passed to the rule.
@@ -997,6 +1031,18 @@ class RuleTester {
                                 error.messageId,
                                 `messageId '${message.messageId}' does not match expected messageId '${error.messageId}'.`
                             );
+
+                            const unsubstitutedPlaceholders = getUnsubstitutedMessagePlaceholders(
+                                message.message,
+                                rule.meta.messages[message.messageId],
+                                error.data
+                            );
+
+                            assert.ok(
+                                unsubstitutedPlaceholders.length === 0,
+                                `The reported message has ${unsubstitutedPlaceholders.length > 1 ? `unsubstituted placeholders: ${unsubstitutedPlaceholders.map(name => `'${name}'`).join(", ")}` : `an unsubstituted placeholder '${unsubstitutedPlaceholders[0]}'`}. Please provide them via the 'data' property in the context.report() call.`
+                            );
+
                             if (hasOwnProperty(error, "data")) {
 
                                 /*
@@ -1096,6 +1142,18 @@ class RuleTester {
                                                 expectedSuggestion.messageId,
                                                 `${suggestionPrefix} messageId should be '${expectedSuggestion.messageId}' but got '${actualSuggestion.messageId}' instead.`
                                             );
+
+                                            const missingPlaceholders = getUnsubstitutedMessagePlaceholders(
+                                                actualSuggestion.desc,
+                                                rule.meta.messages[expectedSuggestion.messageId],
+                                                expectedSuggestion.data
+                                            );
+
+                                            assert.ok(
+                                                missingPlaceholders.length === 0,
+                                                `The message of the suggestion has ${missingPlaceholders.length > 1 ? `unsubstituted placeholders: ${missingPlaceholders.map(name => `'${name}'`).join(", ")}` : `an unsubstituted placeholder '${missingPlaceholders[0]}'`}. Please provide them via the 'data' property for the suggestion in the context.report() call.`
+                                            );
+
                                             if (hasOwnProperty(expectedSuggestion, "data")) {
                                                 const unformattedMetaMessage = rule.meta.messages[expectedSuggestion.messageId];
                                                 const rehydratedDesc = interpolate(unformattedMetaMessage, expectedSuggestion.data);

tests/fixtures/testers/rule-tester/messageId.js

@@ -34,3 +34,89 @@ module.exports.withMessageOnly = {
         };
     }
 };
+
+module.exports.withMissingData = {
+    meta: {
+        messages: {
+            avoidFoo: "Avoid using variables named '{{ name }}'.",
+            unused: "An unused key"
+        }
+    },
+    create(context) {
+        return {
+            Identifier(node) {
+                if (node.name === "foo") {
+                    context.report({
+                        node,
+                        messageId: "avoidFoo",
+                    });
+                }
+            }
+        };
+    }
+};
+
+module.exports.withMultipleMissingDataProperties = {
+    meta: {
+        messages: {
+            avoidFoo: "Avoid using {{ type }} named '{{ name }}'.",
+            unused: "An unused key"
+        }
+    },
+    create(context) {
+        return {
+            Identifier(node) {
+                if (node.name === "foo") {
+                    context.report({
+                        node,
+                        messageId: "avoidFoo",
+                    });
+                }
+            }
+        };
+    }
+};
+
+module.exports.withPlaceholdersInData = {
+    meta: {
+        messages: {
+            avoidFoo: "Avoid using variables named '{{ name }}'.",
+            unused: "An unused key"
+        }
+    },
+    create(context) {
+        return {
+            Identifier(node) {
+                if (node.name === "foo") {
+                    context.report({
+                        node,
+                        messageId: "avoidFoo",
+                        data:      { name: '{{ placeholder }}' },
+                    });
+                }
+            }
+        };
+    }
+};
+
+module.exports.withSamePlaceholdersInData = {
+    meta: {
+        messages: {
+            avoidFoo: "Avoid using variables named '{{ name }}'.",
+            unused: "An unused key"
+        }
+    },
+    create(context) {
+        return {
+            Identifier(node) {
+                if (node.name === "foo") {
+                    context.report({
+                        node,
+                        messageId: "avoidFoo",
+                        data:      { name: '{{ name }}' },
+                    });
+                }
+            }
+        };
+    }
+};

tests/fixtures/testers/rule-tester/suggestions.js

@@ -198,3 +198,61 @@ module.exports.withFailingFixer = {
         };
     }
 };
+
+module.exports.withMissingPlaceholderData = {
+    meta: {
+        messages: {
+            avoidFoo: "Avoid using identifiers named '{{ name }}'.",
+            renameFoo: "Rename identifier 'foo' to '{{ newName }}'"
+        },
+        hasSuggestions: true
+    },
+    create(context) {
+        return {
+            Identifier(node) {
+                if (node.name === "foo") {
+                    context.report({
+                        node,
+                        messageId: "avoidFoo",
+                        data: {
+                            name: "foo"
+                        },
+                        suggest: [{
+                            messageId: "renameFoo",
+                            fix: fixer => fixer.replaceText(node, "bar")
+                        }]
+                    });
+                }
+            }
+        };
+    }
+};
+
+module.exports.withMultipleMissingPlaceholderDataProperties = {
+    meta: {
+        messages: {
+            avoidFoo: "Avoid using identifiers named '{{ name }}'.",
+            rename: "Rename identifier '{{ currentName }}' to '{{ newName }}'"
+        },
+        hasSuggestions: true
+    },
+    create(context) {
+        return {
+            Identifier(node) {
+                if (node.name === "foo") {
+                    context.report({
+                        node,
+                        messageId: "avoidFoo",
+                        data: {
+                            name: "foo"
+                        },
+                        suggest: [{
+                            messageId: "rename",
+                            fix: fixer => fixer.replaceText(node, "bar")
+                        }]
+                    });
+                }
+            }
+        };
+    }
+};

tests/lib/linter/interpolate.js

@@ -5,12 +5,40 @@
 //------------------------------------------------------------------------------
 
 const assert = require("chai").assert;
-const interpolate = require("../../../lib/linter/interpolate");
+const { getPlaceholderMatcher, interpolate } = require("../../../lib/linter/interpolate");
 
 //------------------------------------------------------------------------------
 // Tests
 //------------------------------------------------------------------------------
 
+describe("getPlaceholderMatcher", () => {
+    it("returns a global regular expression", () => {
+        const matcher = getPlaceholderMatcher();
+
+        assert.strictEqual(matcher.global, true);
+    });
+
+    it("matches text with placeholders", () => {
+        const matcher = getPlaceholderMatcher();
+
+        assert.match("{{ placeholder }}", matcher);
+    });
+
+    it("does not match text without placeholders", () => {
+        const matcher = getPlaceholderMatcher();
+
+        assert.notMatch("no placeholders in sight", matcher);
+    });
+
+    it("captures the text inside the placeholder", () => {
+        const matcher = getPlaceholderMatcher();
+        const text = "{{ placeholder }}";
+        const matches = Array.from(text.matchAll(matcher));
+
+        assert.deepStrictEqual(matches, [[text, " placeholder "]]);
+    });
+});
+
 describe("interpolate()", () => {
     it("passes through text without {{ }}", () => {
         const message = "This is a very important message!";