eslint/tools/fuzzer-runner.js

   1 /**
   2  * @fileoverview An opinionated wrapper around eslint-fuzzer
   3  * @author Teddy Katz
   4  */
   5
   6 "use strict";
   7
   8 //------------------------------------------------------------------------------
   9 // Requirements
  10 //------------------------------------------------------------------------------
  11
  12 const ProgressBar = require("progress");
  13 const fuzz = require("./eslint-fuzzer");
  14 const eslint = require("..");
  15 const linter = new eslint.Linter();
  16
  17 //------------------------------------------------------------------------------
  18 // Helpers
  19 //------------------------------------------------------------------------------
  20
  21 // An estimate of how many times faster it is to do a crash-only fuzzer run versus an autofixing run.
  22 const ESTIMATED_CRASH_AUTOFIX_PERFORMANCE_RATIO = 4;
  23
  24 // The number of crash-only tests to run for each autofix test. Right now, this is mostly arbitrary.
  25 const CRASH_AUTOFIX_TEST_COUNT_RATIO = 3;
  26
  27 //------------------------------------------------------------------------------
  28 // Public API
  29 //------------------------------------------------------------------------------
  30
  31 /**
  32  * Runs the fuzzer and outputs a progress bar
  33  * @param {Object} [options] Options for the fuzzer
  34  * @param {number} [options.amount=300] A positive integer indicating how much testing to do. Larger values result in a higher
  35  * chance of finding bugs, but cause the testing to take longer (linear increase). With the default value, the fuzzer
  36  * takes about 15 seconds to run.
  37  * @param {boolean} [options.fuzzBrokenAutofixes=true] true if the fuzzer should look for invalid autofixes in addition to rule crashes
  38  * @returns {Object[]} A list of objects, where each object represents a problem detected by the fuzzer. The objects have the same
  39  * schema as objects returned from eslint-fuzzer.
  40  */
  41 function run({ amount = 300, fuzzBrokenAutofixes = true } = {}) {
  42     const crashTestCount = amount * CRASH_AUTOFIX_TEST_COUNT_RATIO;
  43     const autofixTestCount = fuzzBrokenAutofixes ? amount : 0;
  44
  45     /*
  46      * To keep the progress bar moving at a roughly constant speed, apply a different weight for finishing
  47      * a crash-only fuzzer run versus an autofix fuzzer run.
  48      */
  49     const progressBar = new ProgressBar(
  50         "Fuzzing rules [:bar] :percent, :elapseds elapsed, eta :etas, errors so far: :elapsedErrors",
  51         { width: 30, total: crashTestCount + autofixTestCount * ESTIMATED_CRASH_AUTOFIX_PERFORMANCE_RATIO }
  52     );
  53
  54     // Start displaying the progress bar.
  55     progressBar.tick(0, { elapsedErrors: 0 });
  56
  57     const crashTestResults = fuzz({
  58         linter,
  59         count: crashTestCount,
  60         checkAutofixes: false,
  61         progressCallback(elapsedErrors) {
  62             progressBar.tick(1, { elapsedErrors });
  63             progressBar.render();
  64         }
  65     });
  66
  67     const autofixTestResults = fuzz({
  68         linter,
  69         count: autofixTestCount,
  70         checkAutofixes: true,
  71         progressCallback(elapsedErrors) {
  72             progressBar.tick(ESTIMATED_CRASH_AUTOFIX_PERFORMANCE_RATIO, { elapsedErrors: crashTestResults.length + elapsedErrors });
  73             progressBar.render();
  74         }
  75     });
  76
  77     return crashTestResults.concat(autofixTestResults);
  78
  79 }
  80
  81 module.exports = { run };