eslint/lib/rule-tester/rule-tester.js

   1 /**
   2  * @fileoverview Mocha test wrapper
   3  * @author Ilya Volodin
   4  */
   5 "use strict";
   6
   7 /* globals describe, it -- Mocha globals */
   8
   9 /*
  10  * This is a wrapper around mocha to allow for DRY unittests for eslint
  11  * Format:
  12  * RuleTester.run("{ruleName}", {
  13  *      valid: [
  14  *          "{code}",
  15  *          { code: "{code}", options: {options}, globals: {globals}, parser: "{parser}", settings: {settings} }
  16  *      ],
  17  *      invalid: [
  18  *          { code: "{code}", errors: {numErrors} },
  19  *          { code: "{code}", errors: ["{errorMessage}"] },
  20  *          { code: "{code}", options: {options}, globals: {globals}, parser: "{parser}", settings: {settings}, errors: [{ message: "{errorMessage}", type: "{errorNodeType}"}] }
  21  *      ]
  22  *  });
  23  *
  24  * Variables:
  25  * {code} - String that represents the code to be tested
  26  * {options} - Arguments that are passed to the configurable rules.
  27  * {globals} - An object representing a list of variables that are
  28  *             registered as globals
  29  * {parser} - String representing the parser to use
  30  * {settings} - An object representing global settings for all rules
  31  * {numErrors} - If failing case doesn't need to check error message,
  32  *               this integer will specify how many errors should be
  33  *               received
  34  * {errorMessage} - Message that is returned by the rule on failure
  35  * {errorNodeType} - AST node type that is returned by they rule as
  36  *                   a cause of the failure.
  37  */
  38
  39 //------------------------------------------------------------------------------
  40 // Requirements
  41 //------------------------------------------------------------------------------
  42
  43 const
  44     assert = require("assert"),
  45     path = require("path"),
  46     util = require("util"),
  47     merge = require("lodash.merge"),
  48     equal = require("fast-deep-equal"),
  49     Traverser = require("../../lib/shared/traverser"),
  50     { getRuleOptionsSchema, validate } = require("../shared/config-validator"),
  51     { Linter, SourceCodeFixer, interpolate } = require("../linter");
  52
  53 const ajv = require("../shared/ajv")({ strictDefaults: true });
  54
  55 const espreePath = require.resolve("espree");
  56 const parserSymbol = Symbol.for("eslint.RuleTester.parser");
  57
  58 const { SourceCode } = require("../source-code");
  59
  60 //------------------------------------------------------------------------------
  61 // Typedefs
  62 //------------------------------------------------------------------------------
  63
  64 /** @typedef {import("../shared/types").Parser} Parser */
  65
  66 /* eslint-disable jsdoc/valid-types -- https://github.com/jsdoc-type-pratt-parser/jsdoc-type-pratt-parser/issues/4#issuecomment-778805577 */
  67 /**
  68  * A test case that is expected to pass lint.
  69  * @typedef {Object} ValidTestCase
  70  * @property {string} [name] Name for the test case.
  71  * @property {string} code Code for the test case.
  72  * @property {any[]} [options] Options for the test case.
  73  * @property {{ [name: string]: any }} [settings] Settings for the test case.
  74  * @property {string} [filename] The fake filename for the test case. Useful for rules that make assertion about filenames.
  75  * @property {string} [parser] The absolute path for the parser.
  76  * @property {{ [name: string]: any }} [parserOptions] Options for the parser.
  77  * @property {{ [name: string]: "readonly" | "writable" | "off" }} [globals] The additional global variables.
  78  * @property {{ [name: string]: boolean }} [env] Environments for the test case.
  79  * @property {boolean} [only] Run only this test case or the subset of test cases with this property.
  80  */
  81
  82 /**
  83  * A test case that is expected to fail lint.
  84  * @typedef {Object} InvalidTestCase
  85  * @property {string} [name] Name for the test case.
  86  * @property {string} code Code for the test case.
  87  * @property {number | Array<TestCaseError | string | RegExp>} errors Expected errors.
  88  * @property {string | null} [output] The expected code after autofixes are applied. If set to `null`, the test runner will assert that no autofix is suggested.
  89  * @property {any[]} [options] Options for the test case.
  90  * @property {{ [name: string]: any }} [settings] Settings for the test case.
  91  * @property {string} [filename] The fake filename for the test case. Useful for rules that make assertion about filenames.
  92  * @property {string} [parser] The absolute path for the parser.
  93  * @property {{ [name: string]: any }} [parserOptions] Options for the parser.
  94  * @property {{ [name: string]: "readonly" | "writable" | "off" }} [globals] The additional global variables.
  95  * @property {{ [name: string]: boolean }} [env] Environments for the test case.
  96  * @property {boolean} [only] Run only this test case or the subset of test cases with this property.
  97  */
  98
  99 /**
 100  * A description of a reported error used in a rule tester test.
 101  * @typedef {Object} TestCaseError
 102  * @property {string | RegExp} [message] Message.
 103  * @property {string} [messageId] Message ID.
 104  * @property {string} [type] The type of the reported AST node.
 105  * @property {{ [name: string]: string }} [data] The data used to fill the message template.
 106  * @property {number} [line] The 1-based line number of the reported start location.
 107  * @property {number} [column] The 1-based column number of the reported start location.
 108  * @property {number} [endLine] The 1-based line number of the reported end location.
 109  * @property {number} [endColumn] The 1-based column number of the reported end location.
 110  */
 111 /* eslint-enable jsdoc/valid-types -- https://github.com/jsdoc-type-pratt-parser/jsdoc-type-pratt-parser/issues/4#issuecomment-778805577 */
 112
 113 //------------------------------------------------------------------------------
 114 // Private Members
 115 //------------------------------------------------------------------------------
 116
 117 /*
 118  * testerDefaultConfig must not be modified as it allows to reset the tester to
 119  * the initial default configuration
 120  */
 121 const testerDefaultConfig = { rules: {} };
 122 let defaultConfig = { rules: {} };
 123
 124 /*
 125  * List every parameters possible on a test case that are not related to eslint
 126  * configuration
 127  */
 128 const RuleTesterParameters = [
 129     "name",
 130     "code",
 131     "filename",
 132     "options",
 133     "errors",
 134     "output",
 135     "only"
 136 ];
 137
 138 /*
 139  * All allowed property names in error objects.
 140  */
 141 const errorObjectParameters = new Set([
 142     "message",
 143     "messageId",
 144     "data",
 145     "type",
 146     "line",
 147     "column",
 148     "endLine",
 149     "endColumn",
 150     "suggestions"
 151 ]);
 152 const friendlyErrorObjectParameterList = `[${[...errorObjectParameters].map(key => `'${key}'`).join(", ")}]`;
 153
 154 /*
 155  * All allowed property names in suggestion objects.
 156  */
 157 const suggestionObjectParameters = new Set([
 158     "desc",
 159     "messageId",
 160     "data",
 161     "output"
 162 ]);
 163 const friendlySuggestionObjectParameterList = `[${[...suggestionObjectParameters].map(key => `'${key}'`).join(", ")}]`;
 164
 165 const hasOwnProperty = Function.call.bind(Object.hasOwnProperty);
 166
 167 /**
 168  * Clones a given value deeply.
 169  * Note: This ignores `parent` property.
 170  * @param {any} x A value to clone.
 171  * @returns {any} A cloned value.
 172  */
 173 function cloneDeeplyExcludesParent(x) {
 174     if (typeof x === "object" && x !== null) {
 175         if (Array.isArray(x)) {
 176             return x.map(cloneDeeplyExcludesParent);
 177         }
 178
 179         const retv = {};
 180
 181         for (const key in x) {
 182             if (key !== "parent" && hasOwnProperty(x, key)) {
 183                 retv[key] = cloneDeeplyExcludesParent(x[key]);
 184             }
 185         }
 186
 187         return retv;
 188     }
 189
 190     return x;
 191 }
 192
 193 /**
 194  * Freezes a given value deeply.
 195  * @param {any} x A value to freeze.
 196  * @returns {void}
 197  */
 198 function freezeDeeply(x) {
 199     if (typeof x === "object" && x !== null) {
 200         if (Array.isArray(x)) {
 201             x.forEach(freezeDeeply);
 202         } else {
 203             for (const key in x) {
 204                 if (key !== "parent" && hasOwnProperty(x, key)) {
 205                     freezeDeeply(x[key]);
 206                 }
 207             }
 208         }
 209         Object.freeze(x);
 210     }
 211 }
 212
 213 /**
 214  * Replace control characters by `\u00xx` form.
 215  * @param {string} text The text to sanitize.
 216  * @returns {string} The sanitized text.
 217  */
 218 function sanitize(text) {
 219     if (typeof text !== "string") {
 220         return "";
 221     }
 222     return text.replace(
 223         /[\u0000-\u0009\u000b-\u001a]/gu, // eslint-disable-line no-control-regex -- Escaping controls
 224         c => `\\u${c.codePointAt(0).toString(16).padStart(4, "0")}`
 225     );
 226 }
 227
 228 /**
 229  * Define `start`/`end` properties as throwing error.
 230  * @param {string} objName Object name used for error messages.
 231  * @param {ASTNode} node The node to define.
 232  * @returns {void}
 233  */
 234 function defineStartEndAsError(objName, node) {
 235     Object.defineProperties(node, {
 236         start: {
 237             get() {
 238                 throw new Error(`Use ${objName}.range[0] instead of ${objName}.start`);
 239             },
 240             configurable: true,
 241             enumerable: false
 242         },
 243         end: {
 244             get() {
 245                 throw new Error(`Use ${objName}.range[1] instead of ${objName}.end`);
 246             },
 247             configurable: true,
 248             enumerable: false
 249         }
 250     });
 251 }
 252
 253
 254 /**
 255  * Define `start`/`end` properties of all nodes of the given AST as throwing error.
 256  * @param {ASTNode} ast The root node to errorize `start`/`end` properties.
 257  * @param {Object} [visitorKeys] Visitor keys to be used for traversing the given ast.
 258  * @returns {void}
 259  */
 260 function defineStartEndAsErrorInTree(ast, visitorKeys) {
 261     Traverser.traverse(ast, { visitorKeys, enter: defineStartEndAsError.bind(null, "node") });
 262     ast.tokens.forEach(defineStartEndAsError.bind(null, "token"));
 263     ast.comments.forEach(defineStartEndAsError.bind(null, "token"));
 264 }
 265
 266 /**
 267  * Wraps the given parser in order to intercept and modify return values from the `parse` and `parseForESLint` methods, for test purposes.
 268  * In particular, to modify ast nodes, tokens and comments to throw on access to their `start` and `end` properties.
 269  * @param {Parser} parser Parser object.
 270  * @returns {Parser} Wrapped parser object.
 271  */
 272 function wrapParser(parser) {
 273
 274     if (typeof parser.parseForESLint === "function") {
 275         return {
 276             [parserSymbol]: parser,
 277             parseForESLint(...args) {
 278                 const ret = parser.parseForESLint(...args);
 279
 280                 defineStartEndAsErrorInTree(ret.ast, ret.visitorKeys);
 281                 return ret;
 282             }
 283         };
 284     }
 285
 286     return {
 287         [parserSymbol]: parser,
 288         parse(...args) {
 289             const ast = parser.parse(...args);
 290
 291             defineStartEndAsErrorInTree(ast);
 292             return ast;
 293         }
 294     };
 295 }
 296
 297 /**
 298  * Function to replace `SourceCode.prototype.getComments`.
 299  * @returns {void}
 300  * @throws {Error} Deprecation message.
 301  */
 302 function getCommentsDeprecation() {
 303     throw new Error(
 304         "`SourceCode#getComments()` is deprecated and will be removed in a future major version. Use `getCommentsBefore()`, `getCommentsAfter()`, and `getCommentsInside()` instead."
 305     );
 306 }
 307
 308 /**
 309  * Emit a deprecation warning if function-style format is being used.
 310  * @param {string} ruleName Name of the rule.
 311  * @returns {void}
 312  */
 313 function emitLegacyRuleAPIWarning(ruleName) {
 314     if (!emitLegacyRuleAPIWarning[`warned-${ruleName}`]) {
 315         emitLegacyRuleAPIWarning[`warned-${ruleName}`] = true;
 316         process.emitWarning(
 317             `"${ruleName}" rule is using the deprecated function-style format and will stop working in ESLint v9. Please use object-style format: https://eslint.org/docs/developer-guide/working-with-rules`,
 318             "DeprecationWarning"
 319         );
 320     }
 321 }
 322
 323 /**
 324  * Emit a deprecation warning if rule has options but is missing the "meta.schema" property
 325  * @param {string} ruleName Name of the rule.
 326  * @returns {void}
 327  */
 328 function emitMissingSchemaWarning(ruleName) {
 329     if (!emitMissingSchemaWarning[`warned-${ruleName}`]) {
 330         emitMissingSchemaWarning[`warned-${ruleName}`] = true;
 331         process.emitWarning(
 332             `"${ruleName}" rule has options but is missing the "meta.schema" property and will stop working in ESLint v9. Please add a schema: https://eslint.org/docs/developer-guide/working-with-rules#options-schemas`,
 333             "DeprecationWarning"
 334         );
 335     }
 336 }
 337
 338 //------------------------------------------------------------------------------
 339 // Public Interface
 340 //------------------------------------------------------------------------------
 341
 342 // default separators for testing
 343 const DESCRIBE = Symbol("describe");
 344 const IT = Symbol("it");
 345 const IT_ONLY = Symbol("itOnly");
 346
 347 /**
 348  * This is `it` default handler if `it` don't exist.
 349  * @this {Mocha}
 350  * @param {string} text The description of the test case.
 351  * @param {Function} method The logic of the test case.
 352  * @throws {Error} Any error upon execution of `method`.
 353  * @returns {any} Returned value of `method`.
 354  */
 355 function itDefaultHandler(text, method) {
 356     try {
 357         return method.call(this);
 358     } catch (err) {
 359         if (err instanceof assert.AssertionError) {
 360             err.message += ` (${util.inspect(err.actual)} ${err.operator} ${util.inspect(err.expected)})`;
 361         }
 362         throw err;
 363     }
 364 }
 365
 366 /**
 367  * This is `describe` default handler if `describe` don't exist.
 368  * @this {Mocha}
 369  * @param {string} text The description of the test case.
 370  * @param {Function} method The logic of the test case.
 371  * @returns {any} Returned value of `method`.
 372  */
 373 function describeDefaultHandler(text, method) {
 374     return method.call(this);
 375 }
 376
 377 /**
 378  * Mocha test wrapper.
 379  */
 380 class RuleTester {
 381
 382     /**
 383      * Creates a new instance of RuleTester.
 384      * @param {Object} [testerConfig] Optional, extra configuration for the tester
 385      */
 386     constructor(testerConfig) {
 387
 388         /**
 389          * The configuration to use for this tester. Combination of the tester
 390          * configuration and the default configuration.
 391          * @type {Object}
 392          */
 393         this.testerConfig = merge(
 394             {},
 395             defaultConfig,
 396             testerConfig,
 397             { rules: { "rule-tester/validate-ast": "error" } }
 398         );
 399
 400         /**
 401          * Rule definitions to define before tests.
 402          * @type {Object}
 403          */
 404         this.rules = {};
 405         this.linter = new Linter();
 406     }
 407
 408     /**
 409      * Set the configuration to use for all future tests
 410      * @param {Object} config the configuration to use.
 411      * @throws {TypeError} If non-object config.
 412      * @returns {void}
 413      */
 414     static setDefaultConfig(config) {
 415         if (typeof config !== "object") {
 416             throw new TypeError("RuleTester.setDefaultConfig: config must be an object");
 417         }
 418         defaultConfig = config;
 419
 420         // Make sure the rules object exists since it is assumed to exist later
 421         defaultConfig.rules = defaultConfig.rules || {};
 422     }
 423
 424     /**
 425      * Get the current configuration used for all tests
 426      * @returns {Object} the current configuration
 427      */
 428     static getDefaultConfig() {
 429         return defaultConfig;
 430     }
 431
 432     /**
 433      * Reset the configuration to the initial configuration of the tester removing
 434      * any changes made until now.
 435      * @returns {void}
 436      */
 437     static resetDefaultConfig() {
 438         defaultConfig = merge({}, testerDefaultConfig);
 439     }
 440
 441
 442     /*
 443      * If people use `mocha test.js --watch` command, `describe` and `it` function
 444      * instances are different for each execution. So `describe` and `it` should get fresh instance
 445      * always.
 446      */
 447     static get describe() {
 448         return (
 449             this[DESCRIBE] ||
 450             (typeof describe === "function" ? describe : describeDefaultHandler)
 451         );
 452     }
 453
 454     static set describe(value) {
 455         this[DESCRIBE] = value;
 456     }
 457
 458     static get it() {
 459         return (
 460             this[IT] ||
 461             (typeof it === "function" ? it : itDefaultHandler)
 462         );
 463     }
 464
 465     static set it(value) {
 466         this[IT] = value;
 467     }
 468
 469     /**
 470      * Adds the `only` property to a test to run it in isolation.
 471      * @param {string | ValidTestCase | InvalidTestCase} item A single test to run by itself.
 472      * @returns {ValidTestCase | InvalidTestCase} The test with `only` set.
 473      */
 474     static only(item) {
 475         if (typeof item === "string") {
 476             return { code: item, only: true };
 477         }
 478
 479         return { ...item, only: true };
 480     }
 481
 482     static get itOnly() {
 483         if (typeof this[IT_ONLY] === "function") {
 484             return this[IT_ONLY];
 485         }
 486         if (typeof this[IT] === "function" && typeof this[IT].only === "function") {
 487             return Function.bind.call(this[IT].only, this[IT]);
 488         }
 489         if (typeof it === "function" && typeof it.only === "function") {
 490             return Function.bind.call(it.only, it);
 491         }
 492
 493         if (typeof this[DESCRIBE] === "function" || typeof this[IT] === "function") {
 494             throw new Error(
 495                 "Set `RuleTester.itOnly` to use `only` with a custom test framework.\n" +
 496                 "See https://eslint.org/docs/developer-guide/nodejs-api#customizing-ruletester for more."
 497             );
 498         }
 499         if (typeof it === "function") {
 500             throw new Error("The current test framework does not support exclusive tests with `only`.");
 501         }
 502         throw new Error("To use `only`, use RuleTester with a test framework that provides `it.only()` like Mocha.");
 503     }
 504
 505     static set itOnly(value) {
 506         this[IT_ONLY] = value;
 507     }
 508
 509     /**
 510      * Define a rule for one particular run of tests.
 511      * @param {string} name The name of the rule to define.
 512      * @param {Function} rule The rule definition.
 513      * @returns {void}
 514      */
 515     defineRule(name, rule) {
 516         this.rules[name] = rule;
 517     }
 518
 519     /**
 520      * Adds a new rule test to execute.
 521      * @param {string} ruleName The name of the rule to run.
 522      * @param {Function} rule The rule to test.
 523      * @param {{
 524      *   valid: (ValidTestCase | string)[],
 525      *   invalid: InvalidTestCase[]
 526      * }} test The collection of tests to run.
 527      * @throws {TypeError|Error} If non-object `test`, or if a required
 528      * scenario of the given type is missing.
 529      * @returns {void}
 530      */
 531     run(ruleName, rule, test) {
 532
 533         const testerConfig = this.testerConfig,
 534             requiredScenarios = ["valid", "invalid"],
 535             scenarioErrors = [],
 536             linter = this.linter;
 537
 538         if (!test || typeof test !== "object") {
 539             throw new TypeError(`Test Scenarios for rule ${ruleName} : Could not find test scenario object`);
 540         }
 541
 542         requiredScenarios.forEach(scenarioType => {
 543             if (!test[scenarioType]) {
 544                 scenarioErrors.push(`Could not find any ${scenarioType} test scenarios`);
 545             }
 546         });
 547
 548         if (scenarioErrors.length > 0) {
 549             throw new Error([
 550                 `Test Scenarios for rule ${ruleName} is invalid:`
 551             ].concat(scenarioErrors).join("\n"));
 552         }
 553
 554         if (typeof rule === "function") {
 555             emitLegacyRuleAPIWarning(ruleName);
 556         }
 557
 558         linter.defineRule(ruleName, Object.assign({}, rule, {
 559
 560             // Create a wrapper rule that freezes the `context` properties.
 561             create(context) {
 562                 freezeDeeply(context.options);
 563                 freezeDeeply(context.settings);
 564                 freezeDeeply(context.parserOptions);
 565
 566                 return (typeof rule === "function" ? rule : rule.create)(context);
 567             }
 568         }));
 569
 570         linter.defineRules(this.rules);
 571
 572         /**
 573          * Run the rule for the given item
 574          * @param {string|Object} item Item to run the rule against
 575          * @throws {Error} If an invalid schema.
 576          * @returns {Object} Eslint run result
 577          * @private
 578          */
 579         function runRuleForItem(item) {
 580             let config = merge({}, testerConfig),
 581                 code, filename, output, beforeAST, afterAST;
 582
 583             if (typeof item === "string") {
 584                 code = item;
 585             } else {
 586                 code = item.code;
 587
 588                 /*
 589                  * Assumes everything on the item is a config except for the
 590                  * parameters used by this tester
 591                  */
 592                 const itemConfig = { ...item };
 593
 594                 for (const parameter of RuleTesterParameters) {
 595                     delete itemConfig[parameter];
 596                 }
 597
 598                 /*
 599                  * Create the config object from the tester config and this item
 600                  * specific configurations.
 601                  */
 602                 config = merge(
 603                     config,
 604                     itemConfig
 605                 );
 606             }
 607
 608             if (item.filename) {
 609                 filename = item.filename;
 610             }
 611
 612             if (hasOwnProperty(item, "options")) {
 613                 assert(Array.isArray(item.options), "options must be an array");
 614                 if (
 615                     item.options.length > 0 &&
 616                     typeof rule === "object" &&
 617                     (
 618                         !rule.meta || (rule.meta && (typeof rule.meta.schema === "undefined" || rule.meta.schema === null))
 619                     )
 620                 ) {
 621                     emitMissingSchemaWarning(ruleName);
 622                 }
 623                 config.rules[ruleName] = [1].concat(item.options);
 624             } else {
 625                 config.rules[ruleName] = 1;
 626             }
 627
 628             const schema = getRuleOptionsSchema(rule);
 629
 630             /*
 631              * Setup AST getters.
 632              * The goal is to check whether or not AST was modified when
 633              * running the rule under test.
 634              */
 635             linter.defineRule("rule-tester/validate-ast", () => ({
 636                 Program(node) {
 637                     beforeAST = cloneDeeplyExcludesParent(node);
 638                 },
 639                 "Program:exit"(node) {
 640                     afterAST = node;
 641                 }
 642             }));
 643
 644             if (typeof config.parser === "string") {
 645                 assert(path.isAbsolute(config.parser), "Parsers provided as strings to RuleTester must be absolute paths");
 646             } else {
 647                 config.parser = espreePath;
 648             }
 649
 650             linter.defineParser(config.parser, wrapParser(require(config.parser)));
 651
 652             if (schema) {
 653                 ajv.validateSchema(schema);
 654
 655                 if (ajv.errors) {
 656                     const errors = ajv.errors.map(error => {
 657                         const field = error.dataPath[0] === "." ? error.dataPath.slice(1) : error.dataPath;
 658
 659                         return `\t${field}: ${error.message}`;
 660                     }).join("\n");
 661
 662                     throw new Error([`Schema for rule ${ruleName} is invalid:`, errors]);
 663                 }
 664
 665                 /*
 666                  * `ajv.validateSchema` checks for errors in the structure of the schema (by comparing the schema against a "meta-schema"),
 667                  * and it reports those errors individually. However, there are other types of schema errors that only occur when compiling
 668                  * the schema (e.g. using invalid defaults in a schema), and only one of these errors can be reported at a time. As a result,
 669                  * the schema is compiled here separately from checking for `validateSchema` errors.
 670                  */
 671                 try {
 672                     ajv.compile(schema);
 673                 } catch (err) {
 674                     throw new Error(`Schema for rule ${ruleName} is invalid: ${err.message}`);
 675                 }
 676             }
 677
 678             validate(config, "rule-tester", id => (id === ruleName ? rule : null));
 679
 680             // Verify the code.
 681             const { getComments } = SourceCode.prototype;
 682             let messages;
 683
 684             try {
 685                 SourceCode.prototype.getComments = getCommentsDeprecation;
 686                 messages = linter.verify(code, config, filename);
 687             } finally {
 688                 SourceCode.prototype.getComments = getComments;
 689             }
 690
 691             const fatalErrorMessage = messages.find(m => m.fatal);
 692
 693             assert(!fatalErrorMessage, `A fatal parsing error occurred: ${fatalErrorMessage && fatalErrorMessage.message}`);
 694
 695             // Verify if autofix makes a syntax error or not.
 696             if (messages.some(m => m.fix)) {
 697                 output = SourceCodeFixer.applyFixes(code, messages).output;
 698                 const errorMessageInFix = linter.verify(output, config, filename).find(m => m.fatal);
 699
 700                 assert(!errorMessageInFix, [
 701                     "A fatal parsing error occurred in autofix.",
 702                     `Error: ${errorMessageInFix && errorMessageInFix.message}`,
 703                     "Autofix output:",
 704                     output
 705                 ].join("\n"));
 706             } else {
 707                 output = code;
 708             }
 709
 710             return {
 711                 messages,
 712                 output,
 713                 beforeAST,
 714                 afterAST: cloneDeeplyExcludesParent(afterAST)
 715             };
 716         }
 717
 718         /**
 719          * Check if the AST was changed
 720          * @param {ASTNode} beforeAST AST node before running
 721          * @param {ASTNode} afterAST AST node after running
 722          * @returns {void}
 723          * @private
 724          */
 725         function assertASTDidntChange(beforeAST, afterAST) {
 726             if (!equal(beforeAST, afterAST)) {
 727                 assert.fail("Rule should not modify AST.");
 728             }
 729         }
 730
 731         /**
 732          * Check if the template is valid or not
 733          * all valid cases go through this
 734          * @param {string|Object} item Item to run the rule against
 735          * @returns {void}
 736          * @private
 737          */
 738         function testValidTemplate(item) {
 739             const code = typeof item === "object" ? item.code : item;
 740
 741             assert.ok(typeof code === "string", "Test case must specify a string value for 'code'");
 742             if (item.name) {
 743                 assert.ok(typeof item.name === "string", "Optional test case property 'name' must be a string");
 744             }
 745
 746             const result = runRuleForItem(item);
 747             const messages = result.messages;
 748
 749             assert.strictEqual(messages.length, 0, util.format("Should have no errors but had %d: %s",
 750                 messages.length,
 751                 util.inspect(messages)));
 752
 753             assertASTDidntChange(result.beforeAST, result.afterAST);
 754         }
 755
 756         /**
 757          * Asserts that the message matches its expected value. If the expected
 758          * value is a regular expression, it is checked against the actual
 759          * value.
 760          * @param {string} actual Actual value
 761          * @param {string|RegExp} expected Expected value
 762          * @returns {void}
 763          * @private
 764          */
 765         function assertMessageMatches(actual, expected) {
 766             if (expected instanceof RegExp) {
 767
 768                 // assert.js doesn't have a built-in RegExp match function
 769                 assert.ok(
 770                     expected.test(actual),
 771                     `Expected '${actual}' to match ${expected}`
 772                 );
 773             } else {
 774                 assert.strictEqual(actual, expected);
 775             }
 776         }
 777
 778         /**
 779          * Check if the template is invalid or not
 780          * all invalid cases go through this.
 781          * @param {string|Object} item Item to run the rule against
 782          * @returns {void}
 783          * @private
 784          */
 785         function testInvalidTemplate(item) {
 786             assert.ok(typeof item.code === "string", "Test case must specify a string value for 'code'");
 787             if (item.name) {
 788                 assert.ok(typeof item.name === "string", "Optional test case property 'name' must be a string");
 789             }
 790             assert.ok(item.errors || item.errors === 0,
 791                 `Did not specify errors for an invalid test of ${ruleName}`);
 792
 793             if (Array.isArray(item.errors) && item.errors.length === 0) {
 794                 assert.fail("Invalid cases must have at least one error");
 795             }
 796
 797             const ruleHasMetaMessages = hasOwnProperty(rule, "meta") && hasOwnProperty(rule.meta, "messages");
 798             const friendlyIDList = ruleHasMetaMessages ? `[${Object.keys(rule.meta.messages).map(key => `'${key}'`).join(", ")}]` : null;
 799
 800             const result = runRuleForItem(item);
 801             const messages = result.messages;
 802
 803             if (typeof item.errors === "number") {
 804
 805                 if (item.errors === 0) {
 806                     assert.fail("Invalid cases must have 'error' value greater than 0");
 807                 }
 808
 809                 assert.strictEqual(messages.length, item.errors, util.format("Should have %d error%s but had %d: %s",
 810                     item.errors,
 811                     item.errors === 1 ? "" : "s",
 812                     messages.length,
 813                     util.inspect(messages)));
 814             } else {
 815                 assert.strictEqual(
 816                     messages.length, item.errors.length, util.format(
 817                         "Should have %d error%s but had %d: %s",
 818                         item.errors.length,
 819                         item.errors.length === 1 ? "" : "s",
 820                         messages.length,
 821                         util.inspect(messages)
 822                     )
 823                 );
 824
 825                 const hasMessageOfThisRule = messages.some(m => m.ruleId === ruleName);
 826
 827                 for (let i = 0, l = item.errors.length; i < l; i++) {
 828                     const error = item.errors[i];
 829                     const message = messages[i];
 830
 831                     assert(hasMessageOfThisRule, "Error rule name should be the same as the name of the rule being tested");
 832
 833                     if (typeof error === "string" || error instanceof RegExp) {
 834
 835                         // Just an error message.
 836                         assertMessageMatches(message.message, error);
 837                     } else if (typeof error === "object" && error !== null) {
 838
 839                         /*
 840                          * Error object.
 841                          * This may have a message, messageId, data, node type, line, and/or
 842                          * column.
 843                          */
 844
 845                         Object.keys(error).forEach(propertyName => {
 846                             assert.ok(
 847                                 errorObjectParameters.has(propertyName),
 848                                 `Invalid error property name '${propertyName}'. Expected one of ${friendlyErrorObjectParameterList}.`
 849                             );
 850                         });
 851
 852                         if (hasOwnProperty(error, "message")) {
 853                             assert.ok(!hasOwnProperty(error, "messageId"), "Error should not specify both 'message' and a 'messageId'.");
 854                             assert.ok(!hasOwnProperty(error, "data"), "Error should not specify both 'data' and 'message'.");
 855                             assertMessageMatches(message.message, error.message);
 856                         } else if (hasOwnProperty(error, "messageId")) {
 857                             assert.ok(
 858                                 ruleHasMetaMessages,
 859                                 "Error can not use 'messageId' if rule under test doesn't define 'meta.messages'."
 860                             );
 861                             if (!hasOwnProperty(rule.meta.messages, error.messageId)) {
 862                                 assert(false, `Invalid messageId '${error.messageId}'. Expected one of ${friendlyIDList}.`);
 863                             }
 864                             assert.strictEqual(
 865                                 message.messageId,
 866                                 error.messageId,
 867                                 `messageId '${message.messageId}' does not match expected messageId '${error.messageId}'.`
 868                             );
 869                             if (hasOwnProperty(error, "data")) {
 870
 871                                 /*
 872                                  *  if data was provided, then directly compare the returned message to a synthetic
 873                                  *  interpolated message using the same message ID and data provided in the test.
 874                                  *  See https://github.com/eslint/eslint/issues/9890 for context.
 875                                  */
 876                                 const unformattedOriginalMessage = rule.meta.messages[error.messageId];
 877                                 const rehydratedMessage = interpolate(unformattedOriginalMessage, error.data);
 878
 879                                 assert.strictEqual(
 880                                     message.message,
 881                                     rehydratedMessage,
 882                                     `Hydrated message "${rehydratedMessage}" does not match "${message.message}"`
 883                                 );
 884                             }
 885                         }
 886
 887                         assert.ok(
 888                             hasOwnProperty(error, "data") ? hasOwnProperty(error, "messageId") : true,
 889                             "Error must specify 'messageId' if 'data' is used."
 890                         );
 891
 892                         if (error.type) {
 893                             assert.strictEqual(message.nodeType, error.type, `Error type should be ${error.type}, found ${message.nodeType}`);
 894                         }
 895
 896                         if (hasOwnProperty(error, "line")) {
 897                             assert.strictEqual(message.line, error.line, `Error line should be ${error.line}`);
 898                         }
 899
 900                         if (hasOwnProperty(error, "column")) {
 901                             assert.strictEqual(message.column, error.column, `Error column should be ${error.column}`);
 902                         }
 903
 904                         if (hasOwnProperty(error, "endLine")) {
 905                             assert.strictEqual(message.endLine, error.endLine, `Error endLine should be ${error.endLine}`);
 906                         }
 907
 908                         if (hasOwnProperty(error, "endColumn")) {
 909                             assert.strictEqual(message.endColumn, error.endColumn, `Error endColumn should be ${error.endColumn}`);
 910                         }
 911
 912                         if (hasOwnProperty(error, "suggestions")) {
 913
 914                             // Support asserting there are no suggestions
 915                             if (!error.suggestions || (Array.isArray(error.suggestions) && error.suggestions.length === 0)) {
 916                                 if (Array.isArray(message.suggestions) && message.suggestions.length > 0) {
 917                                     assert.fail(`Error should have no suggestions on error with message: "${message.message}"`);
 918                                 }
 919                             } else {
 920                                 assert.strictEqual(Array.isArray(message.suggestions), true, `Error should have an array of suggestions. Instead received "${message.suggestions}" on error with message: "${message.message}"`);
 921                                 assert.strictEqual(message.suggestions.length, error.suggestions.length, `Error should have ${error.suggestions.length} suggestions. Instead found ${message.suggestions.length} suggestions`);
 922
 923                                 error.suggestions.forEach((expectedSuggestion, index) => {
 924                                     assert.ok(
 925                                         typeof expectedSuggestion === "object" && expectedSuggestion !== null,
 926                                         "Test suggestion in 'suggestions' array must be an object."
 927                                     );
 928                                     Object.keys(expectedSuggestion).forEach(propertyName => {
 929                                         assert.ok(
 930                                             suggestionObjectParameters.has(propertyName),
 931                                             `Invalid suggestion property name '${propertyName}'. Expected one of ${friendlySuggestionObjectParameterList}.`
 932                                         );
 933                                     });
 934
 935                                     const actualSuggestion = message.suggestions[index];
 936                                     const suggestionPrefix = `Error Suggestion at index ${index} :`;
 937
 938                                     if (hasOwnProperty(expectedSuggestion, "desc")) {
 939                                         assert.ok(
 940                                             !hasOwnProperty(expectedSuggestion, "data"),
 941                                             `${suggestionPrefix} Test should not specify both 'desc' and 'data'.`
 942                                         );
 943                                         assert.strictEqual(
 944                                             actualSuggestion.desc,
 945                                             expectedSuggestion.desc,
 946                                             `${suggestionPrefix} desc should be "${expectedSuggestion.desc}" but got "${actualSuggestion.desc}" instead.`
 947                                         );
 948                                     }
 949
 950                                     if (hasOwnProperty(expectedSuggestion, "messageId")) {
 951                                         assert.ok(
 952                                             ruleHasMetaMessages,
 953                                             `${suggestionPrefix} Test can not use 'messageId' if rule under test doesn't define 'meta.messages'.`
 954                                         );
 955                                         assert.ok(
 956                                             hasOwnProperty(rule.meta.messages, expectedSuggestion.messageId),
 957                                             `${suggestionPrefix} Test has invalid messageId '${expectedSuggestion.messageId}', the rule under test allows only one of ${friendlyIDList}.`
 958                                         );
 959                                         assert.strictEqual(
 960                                             actualSuggestion.messageId,
 961                                             expectedSuggestion.messageId,
 962                                             `${suggestionPrefix} messageId should be '${expectedSuggestion.messageId}' but got '${actualSuggestion.messageId}' instead.`
 963                                         );
 964                                         if (hasOwnProperty(expectedSuggestion, "data")) {
 965                                             const unformattedMetaMessage = rule.meta.messages[expectedSuggestion.messageId];
 966                                             const rehydratedDesc = interpolate(unformattedMetaMessage, expectedSuggestion.data);
 967
 968                                             assert.strictEqual(
 969                                                 actualSuggestion.desc,
 970                                                 rehydratedDesc,
 971                                                 `${suggestionPrefix} Hydrated test desc "${rehydratedDesc}" does not match received desc "${actualSuggestion.desc}".`
 972                                             );
 973                                         }
 974                                     } else {
 975                                         assert.ok(
 976                                             !hasOwnProperty(expectedSuggestion, "data"),
 977                                             `${suggestionPrefix} Test must specify 'messageId' if 'data' is used.`
 978                                         );
 979                                     }
 980
 981                                     if (hasOwnProperty(expectedSuggestion, "output")) {
 982                                         const codeWithAppliedSuggestion = SourceCodeFixer.applyFixes(item.code, [actualSuggestion]).output;
 983
 984                                         assert.strictEqual(codeWithAppliedSuggestion, expectedSuggestion.output, `Expected the applied suggestion fix to match the test suggestion output for suggestion at index: ${index} on error with message: "${message.message}"`);
 985                                     }
 986                                 });
 987                             }
 988                         }
 989                     } else {
 990
 991                         // Message was an unexpected type
 992                         assert.fail(`Error should be a string, object, or RegExp, but found (${util.inspect(message)})`);
 993                     }
 994                 }
 995             }
 996
 997             if (hasOwnProperty(item, "output")) {
 998                 if (item.output === null) {
 999                     assert.strictEqual(
1000                         result.output,
1001                         item.code,
1002                         "Expected no autofixes to be suggested"
1003                     );
1004                 } else {
1005                     assert.strictEqual(result.output, item.output, "Output is incorrect.");
1006                 }
1007             } else {
1008                 assert.strictEqual(
1009                     result.output,
1010                     item.code,
1011                     "The rule fixed the code. Please add 'output' property."
1012                 );
1013             }
1014
1015             assertASTDidntChange(result.beforeAST, result.afterAST);
1016         }
1017
1018         /*
1019          * This creates a mocha test suite and pipes all supplied info through
1020          * one of the templates above.
1021          */
1022         this.constructor.describe(ruleName, () => {
1023             this.constructor.describe("valid", () => {
1024                 test.valid.forEach(valid => {
1025                     this.constructor[valid.only ? "itOnly" : "it"](
1026                         sanitize(typeof valid === "object" ? valid.name || valid.code : valid),
1027                         () => {
1028                             testValidTemplate(valid);
1029                         }
1030                     );
1031                 });
1032             });
1033
1034             this.constructor.describe("invalid", () => {
1035                 test.invalid.forEach(invalid => {
1036                     this.constructor[invalid.only ? "itOnly" : "it"](
1037                         sanitize(invalid.name || invalid.code),
1038                         () => {
1039                             testInvalidTemplate(invalid);
1040                         }
1041                     );
1042                 });
1043             });
1044         });
1045     }
1046 }
1047
1048 RuleTester[DESCRIBE] = RuleTester[IT] = RuleTester[IT_ONLY] = null;
1049
1050 module.exports = RuleTester;