eslint/lib/cli-engine/file-enumerator.js

   1 /**
   2  * @fileoverview `FileEnumerator` class.
   3  *
   4  * `FileEnumerator` class has two responsibilities:
   5  *
   6  * 1. Find target files by processing glob patterns.
   7  * 2. Tie each target file and appropriate configuration.
   8  *
   9  * It provides a method:
  10  *
  11  * - `iterateFiles(patterns)`
  12  *     Iterate files which are matched by given patterns together with the
  13  *     corresponded configuration. This is for `CLIEngine#executeOnFiles()`.
  14  *     While iterating files, it loads the configuration file of each directory
  15  *     before iterate files on the directory, so we can use the configuration
  16  *     files to determine target files.
  17  *
  18  * @example
  19  * const enumerator = new FileEnumerator();
  20  * const linter = new Linter();
  21  *
  22  * for (const { config, filePath } of enumerator.iterateFiles(["*.js"])) {
  23  *     const code = fs.readFileSync(filePath, "utf8");
  24  *     const messages = linter.verify(code, config, filePath);
  25  *
  26  *     console.log(messages);
  27  * }
  28  *
  29  * @author Toru Nagashima <https://github.com/mysticatea>
  30  */
  31 "use strict";
  32
  33 //------------------------------------------------------------------------------
  34 // Requirements
  35 //------------------------------------------------------------------------------
  36
  37 const fs = require("fs");
  38 const path = require("path");
  39 const getGlobParent = require("glob-parent");
  40 const isGlob = require("is-glob");
  41 const { escapeRegExp } = require("lodash");
  42 const { Minimatch } = require("minimatch");
  43
  44 const {
  45     Legacy: {
  46         IgnorePattern,
  47         CascadingConfigArrayFactory
  48     }
  49 } = require("@eslint/eslintrc");
  50 const debug = require("debug")("eslint:file-enumerator");
  51
  52 //------------------------------------------------------------------------------
  53 // Helpers
  54 //------------------------------------------------------------------------------
  55
  56 const minimatchOpts = { dot: true, matchBase: true };
  57 const dotfilesPattern = /(?:(?:^\.)|(?:[/\\]\.))[^/\\.].*/u;
  58 const NONE = 0;
  59 const IGNORED_SILENTLY = 1;
  60 const IGNORED = 2;
  61
  62 // For VSCode intellisense
  63 /** @typedef {ReturnType<CascadingConfigArrayFactory["getConfigArrayForFile"]>} ConfigArray */
  64
  65 /**
  66  * @typedef {Object} FileEnumeratorOptions
  67  * @property {CascadingConfigArrayFactory} [configArrayFactory] The factory for config arrays.
  68  * @property {string} [cwd] The base directory to start lookup.
  69  * @property {string[]} [extensions] The extensions to match files for directory patterns.
  70  * @property {boolean} [globInputPaths] Set to false to skip glob resolution of input file paths to lint (default: true). If false, each input file paths is assumed to be a non-glob path to an existing file.
  71  * @property {boolean} [ignore] The flag to check ignored files.
  72  * @property {string[]} [rulePaths] The value of `--rulesdir` option.
  73  */
  74
  75 /**
  76  * @typedef {Object} FileAndConfig
  77  * @property {string} filePath The path to a target file.
  78  * @property {ConfigArray} config The config entries of that file.
  79  * @property {boolean} ignored If `true` then this file should be ignored and warned because it was directly specified.
  80  */
  81
  82 /**
  83  * @typedef {Object} FileEntry
  84  * @property {string} filePath The path to a target file.
  85  * @property {ConfigArray} config The config entries of that file.
  86  * @property {NONE|IGNORED_SILENTLY|IGNORED} flag The flag.
  87  * - `NONE` means the file is a target file.
  88  * - `IGNORED_SILENTLY` means the file should be ignored silently.
  89  * - `IGNORED` means the file should be ignored and warned because it was directly specified.
  90  */
  91
  92 /**
  93  * @typedef {Object} FileEnumeratorInternalSlots
  94  * @property {CascadingConfigArrayFactory} configArrayFactory The factory for config arrays.
  95  * @property {string} cwd The base directory to start lookup.
  96  * @property {RegExp|null} extensionRegExp The RegExp to test if a string ends with specific file extensions.
  97  * @property {boolean} globInputPaths Set to false to skip glob resolution of input file paths to lint (default: true). If false, each input file paths is assumed to be a non-glob path to an existing file.
  98  * @property {boolean} ignoreFlag The flag to check ignored files.
  99  * @property {(filePath:string, dot:boolean) => boolean} defaultIgnores The default predicate function to ignore files.
 100  */
 101
 102 /** @type {WeakMap<FileEnumerator, FileEnumeratorInternalSlots>} */
 103 const internalSlotsMap = new WeakMap();
 104
 105 /**
 106  * Check if a string is a glob pattern or not.
 107  * @param {string} pattern A glob pattern.
 108  * @returns {boolean} `true` if the string is a glob pattern.
 109  */
 110 function isGlobPattern(pattern) {
 111     return isGlob(path.sep === "\\" ? pattern.replace(/\\/gu, "/") : pattern);
 112 }
 113
 114 /**
 115  * Get stats of a given path.
 116  * @param {string} filePath The path to target file.
 117  * @returns {fs.Stats|null} The stats.
 118  * @private
 119  */
 120 function statSafeSync(filePath) {
 121     try {
 122         return fs.statSync(filePath);
 123     } catch (error) {
 124         /* istanbul ignore next */
 125         if (error.code !== "ENOENT") {
 126             throw error;
 127         }
 128         return null;
 129     }
 130 }
 131
 132 /**
 133  * Get filenames in a given path to a directory.
 134  * @param {string} directoryPath The path to target directory.
 135  * @returns {import("fs").Dirent[]} The filenames.
 136  * @private
 137  */
 138 function readdirSafeSync(directoryPath) {
 139     try {
 140         return fs.readdirSync(directoryPath, { withFileTypes: true });
 141     } catch (error) {
 142         /* istanbul ignore next */
 143         if (error.code !== "ENOENT") {
 144             throw error;
 145         }
 146         return [];
 147     }
 148 }
 149
 150 /**
 151  * Create a `RegExp` object to detect extensions.
 152  * @param {string[] | null} extensions The extensions to create.
 153  * @returns {RegExp | null} The created `RegExp` object or null.
 154  */
 155 function createExtensionRegExp(extensions) {
 156     if (extensions) {
 157         const normalizedExts = extensions.map(ext => escapeRegExp(
 158             ext.startsWith(".")
 159                 ? ext.slice(1)
 160                 : ext
 161         ));
 162
 163         return new RegExp(
 164             `.\\.(?:${normalizedExts.join("|")})$`,
 165             "u"
 166         );
 167     }
 168     return null;
 169 }
 170
 171 /**
 172  * The error type when no files match a glob.
 173  */
 174 class NoFilesFoundError extends Error {
 175
 176     // eslint-disable-next-line jsdoc/require-description
 177     /**
 178      * @param {string} pattern The glob pattern which was not found.
 179      * @param {boolean} globDisabled If `true` then the pattern was a glob pattern, but glob was disabled.
 180      */
 181     constructor(pattern, globDisabled) {
 182         super(`No files matching '${pattern}' were found${globDisabled ? " (glob was disabled)" : ""}.`);
 183         this.messageTemplate = "file-not-found";
 184         this.messageData = { pattern, globDisabled };
 185     }
 186 }
 187
 188 /**
 189  * The error type when there are files matched by a glob, but all of them have been ignored.
 190  */
 191 class AllFilesIgnoredError extends Error {
 192
 193     // eslint-disable-next-line jsdoc/require-description
 194     /**
 195      * @param {string} pattern The glob pattern which was not found.
 196      */
 197     constructor(pattern) {
 198         super(`All files matched by '${pattern}' are ignored.`);
 199         this.messageTemplate = "all-files-ignored";
 200         this.messageData = { pattern };
 201     }
 202 }
 203
 204 /**
 205  * This class provides the functionality that enumerates every file which is
 206  * matched by given glob patterns and that configuration.
 207  */
 208 class FileEnumerator {
 209
 210     /**
 211      * Initialize this enumerator.
 212      * @param {FileEnumeratorOptions} options The options.
 213      */
 214     constructor({
 215         cwd = process.cwd(),
 216         configArrayFactory = new CascadingConfigArrayFactory({
 217             cwd,
 218             eslintRecommendedPath: path.resolve(__dirname, "../../conf/eslint-recommended.js"),
 219             eslintAllPath: path.resolve(__dirname, "../../conf/eslint-all.js")
 220         }),
 221         extensions = null,
 222         globInputPaths = true,
 223         errorOnUnmatchedPattern = true,
 224         ignore = true
 225     } = {}) {
 226         internalSlotsMap.set(this, {
 227             configArrayFactory,
 228             cwd,
 229             defaultIgnores: IgnorePattern.createDefaultIgnore(cwd),
 230             extensionRegExp: createExtensionRegExp(extensions),
 231             globInputPaths,
 232             errorOnUnmatchedPattern,
 233             ignoreFlag: ignore
 234         });
 235     }
 236
 237     /**
 238      * Check if a given file is target or not.
 239      * @param {string} filePath The path to a candidate file.
 240      * @param {ConfigArray} [providedConfig] Optional. The configuration for the file.
 241      * @returns {boolean} `true` if the file is a target.
 242      */
 243     isTargetPath(filePath, providedConfig) {
 244         const {
 245             configArrayFactory,
 246             extensionRegExp
 247         } = internalSlotsMap.get(this);
 248
 249         // If `--ext` option is present, use it.
 250         if (extensionRegExp) {
 251             return extensionRegExp.test(filePath);
 252         }
 253
 254         // `.js` file is target by default.
 255         if (filePath.endsWith(".js")) {
 256             return true;
 257         }
 258
 259         // use `overrides[].files` to check additional targets.
 260         const config =
 261             providedConfig ||
 262             configArrayFactory.getConfigArrayForFile(
 263                 filePath,
 264                 { ignoreNotFoundError: true }
 265             );
 266
 267         return config.isAdditionalTargetPath(filePath);
 268     }
 269
 270     /**
 271      * Iterate files which are matched by given glob patterns.
 272      * @param {string|string[]} patternOrPatterns The glob patterns to iterate files.
 273      * @returns {IterableIterator<FileAndConfig>} The found files.
 274      */
 275     *iterateFiles(patternOrPatterns) {
 276         const { globInputPaths, errorOnUnmatchedPattern } = internalSlotsMap.get(this);
 277         const patterns = Array.isArray(patternOrPatterns)
 278             ? patternOrPatterns
 279             : [patternOrPatterns];
 280
 281         debug("Start to iterate files: %o", patterns);
 282
 283         // The set of paths to remove duplicate.
 284         const set = new Set();
 285
 286         for (const pattern of patterns) {
 287             let foundRegardlessOfIgnored = false;
 288             let found = false;
 289
 290             // Skip empty string.
 291             if (!pattern) {
 292                 continue;
 293             }
 294
 295             // Iterate files of this pattern.
 296             for (const { config, filePath, flag } of this._iterateFiles(pattern)) {
 297                 foundRegardlessOfIgnored = true;
 298                 if (flag === IGNORED_SILENTLY) {
 299                     continue;
 300                 }
 301                 found = true;
 302
 303                 // Remove duplicate paths while yielding paths.
 304                 if (!set.has(filePath)) {
 305                     set.add(filePath);
 306                     yield {
 307                         config,
 308                         filePath,
 309                         ignored: flag === IGNORED
 310                     };
 311                 }
 312             }
 313
 314             // Raise an error if any files were not found.
 315             if (errorOnUnmatchedPattern) {
 316                 if (!foundRegardlessOfIgnored) {
 317                     throw new NoFilesFoundError(
 318                         pattern,
 319                         !globInputPaths && isGlob(pattern)
 320                     );
 321                 }
 322                 if (!found) {
 323                     throw new AllFilesIgnoredError(pattern);
 324                 }
 325             }
 326         }
 327
 328         debug(`Complete iterating files: ${JSON.stringify(patterns)}`);
 329     }
 330
 331     /**
 332      * Iterate files which are matched by a given glob pattern.
 333      * @param {string} pattern The glob pattern to iterate files.
 334      * @returns {IterableIterator<FileEntry>} The found files.
 335      */
 336     _iterateFiles(pattern) {
 337         const { cwd, globInputPaths } = internalSlotsMap.get(this);
 338         const absolutePath = path.resolve(cwd, pattern);
 339         const isDot = dotfilesPattern.test(pattern);
 340         const stat = statSafeSync(absolutePath);
 341
 342         if (stat && stat.isDirectory()) {
 343             return this._iterateFilesWithDirectory(absolutePath, isDot);
 344         }
 345         if (stat && stat.isFile()) {
 346             return this._iterateFilesWithFile(absolutePath);
 347         }
 348         if (globInputPaths && isGlobPattern(pattern)) {
 349             return this._iterateFilesWithGlob(absolutePath, isDot);
 350         }
 351
 352         return [];
 353     }
 354
 355     /**
 356      * Iterate a file which is matched by a given path.
 357      * @param {string} filePath The path to the target file.
 358      * @returns {IterableIterator<FileEntry>} The found files.
 359      * @private
 360      */
 361     _iterateFilesWithFile(filePath) {
 362         debug(`File: ${filePath}`);
 363
 364         const { configArrayFactory } = internalSlotsMap.get(this);
 365         const config = configArrayFactory.getConfigArrayForFile(filePath);
 366         const ignored = this._isIgnoredFile(filePath, { config, direct: true });
 367         const flag = ignored ? IGNORED : NONE;
 368
 369         return [{ config, filePath, flag }];
 370     }
 371
 372     /**
 373      * Iterate files in a given path.
 374      * @param {string} directoryPath The path to the target directory.
 375      * @param {boolean} dotfiles If `true` then it doesn't skip dot files by default.
 376      * @returns {IterableIterator<FileEntry>} The found files.
 377      * @private
 378      */
 379     _iterateFilesWithDirectory(directoryPath, dotfiles) {
 380         debug(`Directory: ${directoryPath}`);
 381
 382         return this._iterateFilesRecursive(
 383             directoryPath,
 384             { dotfiles, recursive: true, selector: null }
 385         );
 386     }
 387
 388     /**
 389      * Iterate files which are matched by a given glob pattern.
 390      * @param {string} pattern The glob pattern to iterate files.
 391      * @param {boolean} dotfiles If `true` then it doesn't skip dot files by default.
 392      * @returns {IterableIterator<FileEntry>} The found files.
 393      * @private
 394      */
 395     _iterateFilesWithGlob(pattern, dotfiles) {
 396         debug(`Glob: ${pattern}`);
 397
 398         const directoryPath = path.resolve(getGlobParent(pattern));
 399         const globPart = pattern.slice(directoryPath.length + 1);
 400
 401         /*
 402          * recursive if there are `**` or path separators in the glob part.
 403          * Otherwise, patterns such as `src/*.js`, it doesn't need recursive.
 404          */
 405         const recursive = /\*\*|\/|\\/u.test(globPart);
 406         const selector = new Minimatch(pattern, minimatchOpts);
 407
 408         debug(`recursive? ${recursive}`);
 409
 410         return this._iterateFilesRecursive(
 411             directoryPath,
 412             { dotfiles, recursive, selector }
 413         );
 414     }
 415
 416     /**
 417      * Iterate files in a given path.
 418      * @param {string} directoryPath The path to the target directory.
 419      * @param {Object} options The options to iterate files.
 420      * @param {boolean} [options.dotfiles] If `true` then it doesn't skip dot files by default.
 421      * @param {boolean} [options.recursive] If `true` then it dives into sub directories.
 422      * @param {InstanceType<Minimatch>} [options.selector] The matcher to choose files.
 423      * @returns {IterableIterator<FileEntry>} The found files.
 424      * @private
 425      */
 426     *_iterateFilesRecursive(directoryPath, options) {
 427         debug(`Enter the directory: ${directoryPath}`);
 428         const { configArrayFactory } = internalSlotsMap.get(this);
 429
 430         /** @type {ConfigArray|null} */
 431         let config = null;
 432
 433         // Enumerate the files of this directory.
 434         for (const entry of readdirSafeSync(directoryPath)) {
 435             const filePath = path.join(directoryPath, entry.name);
 436
 437             // Check if the file is matched.
 438             if (entry.isFile()) {
 439                 if (!config) {
 440                     config = configArrayFactory.getConfigArrayForFile(
 441                         filePath,
 442
 443                         /*
 444                          * We must ignore `ConfigurationNotFoundError` at this
 445                          * point because we don't know if target files exist in
 446                          * this directory.
 447                          */
 448                         { ignoreNotFoundError: true }
 449                     );
 450                 }
 451                 const matched = options.selector
 452
 453                     // Started with a glob pattern; choose by the pattern.
 454                     ? options.selector.match(filePath)
 455
 456                     // Started with a directory path; choose by file extensions.
 457                     : this.isTargetPath(filePath, config);
 458
 459                 if (matched) {
 460                     const ignored = this._isIgnoredFile(filePath, { ...options, config });
 461                     const flag = ignored ? IGNORED_SILENTLY : NONE;
 462
 463                     debug(`Yield: ${entry.name}${ignored ? " but ignored" : ""}`);
 464                     yield {
 465                         config: configArrayFactory.getConfigArrayForFile(filePath),
 466                         filePath,
 467                         flag
 468                     };
 469                 } else {
 470                     debug(`Didn't match: ${entry.name}`);
 471                 }
 472
 473             // Dive into the sub directory.
 474             } else if (options.recursive && entry.isDirectory()) {
 475                 if (!config) {
 476                     config = configArrayFactory.getConfigArrayForFile(
 477                         filePath,
 478                         { ignoreNotFoundError: true }
 479                     );
 480                 }
 481                 const ignored = this._isIgnoredFile(
 482                     filePath + path.sep,
 483                     { ...options, config }
 484                 );
 485
 486                 if (!ignored) {
 487                     yield* this._iterateFilesRecursive(filePath, options);
 488                 }
 489             }
 490         }
 491
 492         debug(`Leave the directory: ${directoryPath}`);
 493     }
 494
 495     /**
 496      * Check if a given file should be ignored.
 497      * @param {string} filePath The path to a file to check.
 498      * @param {Object} options Options
 499      * @param {ConfigArray} [options.config] The config for this file.
 500      * @param {boolean} [options.dotfiles] If `true` then this is not ignore dot files by default.
 501      * @param {boolean} [options.direct] If `true` then this is a direct specified file.
 502      * @returns {boolean} `true` if the file should be ignored.
 503      * @private
 504      */
 505     _isIgnoredFile(filePath, {
 506         config: providedConfig,
 507         dotfiles = false,
 508         direct = false
 509     }) {
 510         const {
 511             configArrayFactory,
 512             defaultIgnores,
 513             ignoreFlag
 514         } = internalSlotsMap.get(this);
 515
 516         if (ignoreFlag) {
 517             const config =
 518                 providedConfig ||
 519                 configArrayFactory.getConfigArrayForFile(
 520                     filePath,
 521                     { ignoreNotFoundError: true }
 522                 );
 523             const ignores =
 524                 config.extractConfig(filePath).ignores || defaultIgnores;
 525
 526             return ignores(filePath, dotfiles);
 527         }
 528
 529         return !direct && defaultIgnores(filePath, dotfiles);
 530     }
 531 }
 532
 533 //------------------------------------------------------------------------------
 534 // Public Interface
 535 //------------------------------------------------------------------------------
 536
 537 module.exports = { FileEnumerator };