eslint/lib/cli-engine/file-enumerator.js

   1 /**
   2  * @fileoverview `FileEnumerator` class.
   3  *
   4  * `FileEnumerator` class has two responsibilities:
   5  *
   6  * 1. Find target files by processing glob patterns.
   7  * 2. Tie each target file and appropriate configuration.
   8  *
   9  * It provides a method:
  10  *
  11  * - `iterateFiles(patterns)`
  12  *     Iterate files which are matched by given patterns together with the
  13  *     corresponded configuration. This is for `CLIEngine#executeOnFiles()`.
  14  *     While iterating files, it loads the configuration file of each directory
  15  *     before iterate files on the directory, so we can use the configuration
  16  *     files to determine target files.
  17  *
  18  * @example
  19  * const enumerator = new FileEnumerator();
  20  * const linter = new Linter();
  21  *
  22  * for (const { config, filePath } of enumerator.iterateFiles(["*.js"])) {
  23  *     const code = fs.readFileSync(filePath, "utf8");
  24  *     const messages = linter.verify(code, config, filePath);
  25  *
  26  *     console.log(messages);
  27  * }
  28  *
  29  * @author Toru Nagashima <https://github.com/mysticatea>
  30  */
  31 "use strict";
  32
  33 //------------------------------------------------------------------------------
  34 // Requirements
  35 //------------------------------------------------------------------------------
  36
  37 const fs = require("fs");
  38 const path = require("path");
  39 const getGlobParent = require("glob-parent");
  40 const isGlob = require("is-glob");
  41 const escapeRegExp = require("escape-string-regexp");
  42 const { Minimatch } = require("minimatch");
  43
  44 const {
  45     Legacy: {
  46         IgnorePattern,
  47         CascadingConfigArrayFactory
  48     }
  49 } = require("@eslint/eslintrc");
  50 const debug = require("debug")("eslint:file-enumerator");
  51
  52 //------------------------------------------------------------------------------
  53 // Helpers
  54 //------------------------------------------------------------------------------
  55
  56 const minimatchOpts = { dot: true, matchBase: true };
  57 const dotfilesPattern = /(?:(?:^\.)|(?:[/\\]\.))[^/\\.].*/u;
  58 const NONE = 0;
  59 const IGNORED_SILENTLY = 1;
  60 const IGNORED = 2;
  61
  62 // For VSCode intellisense
  63 /** @typedef {ReturnType<CascadingConfigArrayFactory.getConfigArrayForFile>} ConfigArray */
  64
  65 /**
  66  * @typedef {Object} FileEnumeratorOptions
  67  * @property {CascadingConfigArrayFactory} [configArrayFactory] The factory for config arrays.
  68  * @property {string} [cwd] The base directory to start lookup.
  69  * @property {string[]} [extensions] The extensions to match files for directory patterns.
  70  * @property {boolean} [globInputPaths] Set to false to skip glob resolution of input file paths to lint (default: true). If false, each input file paths is assumed to be a non-glob path to an existing file.
  71  * @property {boolean} [ignore] The flag to check ignored files.
  72  * @property {string[]} [rulePaths] The value of `--rulesdir` option.
  73  */
  74
  75 /**
  76  * @typedef {Object} FileAndConfig
  77  * @property {string} filePath The path to a target file.
  78  * @property {ConfigArray} config The config entries of that file.
  79  * @property {boolean} ignored If `true` then this file should be ignored and warned because it was directly specified.
  80  */
  81
  82 /**
  83  * @typedef {Object} FileEntry
  84  * @property {string} filePath The path to a target file.
  85  * @property {ConfigArray} config The config entries of that file.
  86  * @property {NONE|IGNORED_SILENTLY|IGNORED} flag The flag.
  87  * - `NONE` means the file is a target file.
  88  * - `IGNORED_SILENTLY` means the file should be ignored silently.
  89  * - `IGNORED` means the file should be ignored and warned because it was directly specified.
  90  */
  91
  92 /**
  93  * @typedef {Object} FileEnumeratorInternalSlots
  94  * @property {CascadingConfigArrayFactory} configArrayFactory The factory for config arrays.
  95  * @property {string} cwd The base directory to start lookup.
  96  * @property {RegExp|null} extensionRegExp The RegExp to test if a string ends with specific file extensions.
  97  * @property {boolean} globInputPaths Set to false to skip glob resolution of input file paths to lint (default: true). If false, each input file paths is assumed to be a non-glob path to an existing file.
  98  * @property {boolean} ignoreFlag The flag to check ignored files.
  99  * @property {(filePath:string, dot:boolean) => boolean} defaultIgnores The default predicate function to ignore files.
 100  */
 101
 102 /** @type {WeakMap<FileEnumerator, FileEnumeratorInternalSlots>} */
 103 const internalSlotsMap = new WeakMap();
 104
 105 /**
 106  * Check if a string is a glob pattern or not.
 107  * @param {string} pattern A glob pattern.
 108  * @returns {boolean} `true` if the string is a glob pattern.
 109  */
 110 function isGlobPattern(pattern) {
 111     return isGlob(path.sep === "\\" ? pattern.replace(/\\/gu, "/") : pattern);
 112 }
 113
 114 /**
 115  * Get stats of a given path.
 116  * @param {string} filePath The path to target file.
 117  * @throws {Error} As may be thrown by `fs.statSync`.
 118  * @returns {fs.Stats|null} The stats.
 119  * @private
 120  */
 121 function statSafeSync(filePath) {
 122     try {
 123         return fs.statSync(filePath);
 124     } catch (error) {
 125
 126         /* c8 ignore next */
 127         if (error.code !== "ENOENT") {
 128             throw error;
 129         }
 130         return null;
 131     }
 132 }
 133
 134 /**
 135  * Get filenames in a given path to a directory.
 136  * @param {string} directoryPath The path to target directory.
 137  * @throws {Error} As may be thrown by `fs.readdirSync`.
 138  * @returns {import("fs").Dirent[]} The filenames.
 139  * @private
 140  */
 141 function readdirSafeSync(directoryPath) {
 142     try {
 143         return fs.readdirSync(directoryPath, { withFileTypes: true });
 144     } catch (error) {
 145
 146         /* c8 ignore next */
 147         if (error.code !== "ENOENT") {
 148             throw error;
 149         }
 150         return [];
 151     }
 152 }
 153
 154 /**
 155  * Create a `RegExp` object to detect extensions.
 156  * @param {string[] | null} extensions The extensions to create.
 157  * @returns {RegExp | null} The created `RegExp` object or null.
 158  */
 159 function createExtensionRegExp(extensions) {
 160     if (extensions) {
 161         const normalizedExts = extensions.map(ext => escapeRegExp(
 162             ext.startsWith(".")
 163                 ? ext.slice(1)
 164                 : ext
 165         ));
 166
 167         return new RegExp(
 168             `.\\.(?:${normalizedExts.join("|")})$`,
 169             "u"
 170         );
 171     }
 172     return null;
 173 }
 174
 175 /**
 176  * The error type when no files match a glob.
 177  */
 178 class NoFilesFoundError extends Error {
 179
 180     /**
 181      * @param {string} pattern The glob pattern which was not found.
 182      * @param {boolean} globDisabled If `true` then the pattern was a glob pattern, but glob was disabled.
 183      */
 184     constructor(pattern, globDisabled) {
 185         super(`No files matching '${pattern}' were found${globDisabled ? " (glob was disabled)" : ""}.`);
 186         this.messageTemplate = "file-not-found";
 187         this.messageData = { pattern, globDisabled };
 188     }
 189 }
 190
 191 /**
 192  * The error type when there are files matched by a glob, but all of them have been ignored.
 193  */
 194 class AllFilesIgnoredError extends Error {
 195
 196     /**
 197      * @param {string} pattern The glob pattern which was not found.
 198      */
 199     constructor(pattern) {
 200         super(`All files matched by '${pattern}' are ignored.`);
 201         this.messageTemplate = "all-files-ignored";
 202         this.messageData = { pattern };
 203     }
 204 }
 205
 206 /**
 207  * This class provides the functionality that enumerates every file which is
 208  * matched by given glob patterns and that configuration.
 209  */
 210 class FileEnumerator {
 211
 212     /**
 213      * Initialize this enumerator.
 214      * @param {FileEnumeratorOptions} options The options.
 215      */
 216     constructor({
 217         cwd = process.cwd(),
 218         configArrayFactory = new CascadingConfigArrayFactory({
 219             cwd,
 220             getEslintRecommendedConfig: () => require("../../conf/eslint-recommended.js"),
 221             getEslintAllConfig: () => require("../../conf/eslint-all.js")
 222         }),
 223         extensions = null,
 224         globInputPaths = true,
 225         errorOnUnmatchedPattern = true,
 226         ignore = true
 227     } = {}) {
 228         internalSlotsMap.set(this, {
 229             configArrayFactory,
 230             cwd,
 231             defaultIgnores: IgnorePattern.createDefaultIgnore(cwd),
 232             extensionRegExp: createExtensionRegExp(extensions),
 233             globInputPaths,
 234             errorOnUnmatchedPattern,
 235             ignoreFlag: ignore
 236         });
 237     }
 238
 239     /**
 240      * Check if a given file is target or not.
 241      * @param {string} filePath The path to a candidate file.
 242      * @param {ConfigArray} [providedConfig] Optional. The configuration for the file.
 243      * @returns {boolean} `true` if the file is a target.
 244      */
 245     isTargetPath(filePath, providedConfig) {
 246         const {
 247             configArrayFactory,
 248             extensionRegExp
 249         } = internalSlotsMap.get(this);
 250
 251         // If `--ext` option is present, use it.
 252         if (extensionRegExp) {
 253             return extensionRegExp.test(filePath);
 254         }
 255
 256         // `.js` file is target by default.
 257         if (filePath.endsWith(".js")) {
 258             return true;
 259         }
 260
 261         // use `overrides[].files` to check additional targets.
 262         const config =
 263             providedConfig ||
 264             configArrayFactory.getConfigArrayForFile(
 265                 filePath,
 266                 { ignoreNotFoundError: true }
 267             );
 268
 269         return config.isAdditionalTargetPath(filePath);
 270     }
 271
 272     /**
 273      * Iterate files which are matched by given glob patterns.
 274      * @param {string|string[]} patternOrPatterns The glob patterns to iterate files.
 275      * @throws {NoFilesFoundError|AllFilesIgnoredError} On an unmatched pattern.
 276      * @returns {IterableIterator<FileAndConfig>} The found files.
 277      */
 278     *iterateFiles(patternOrPatterns) {
 279         const { globInputPaths, errorOnUnmatchedPattern } = internalSlotsMap.get(this);
 280         const patterns = Array.isArray(patternOrPatterns)
 281             ? patternOrPatterns
 282             : [patternOrPatterns];
 283
 284         debug("Start to iterate files: %o", patterns);
 285
 286         // The set of paths to remove duplicate.
 287         const set = new Set();
 288
 289         for (const pattern of patterns) {
 290             let foundRegardlessOfIgnored = false;
 291             let found = false;
 292
 293             // Skip empty string.
 294             if (!pattern) {
 295                 continue;
 296             }
 297
 298             // Iterate files of this pattern.
 299             for (const { config, filePath, flag } of this._iterateFiles(pattern)) {
 300                 foundRegardlessOfIgnored = true;
 301                 if (flag === IGNORED_SILENTLY) {
 302                     continue;
 303                 }
 304                 found = true;
 305
 306                 // Remove duplicate paths while yielding paths.
 307                 if (!set.has(filePath)) {
 308                     set.add(filePath);
 309                     yield {
 310                         config,
 311                         filePath,
 312                         ignored: flag === IGNORED
 313                     };
 314                 }
 315             }
 316
 317             // Raise an error if any files were not found.
 318             if (errorOnUnmatchedPattern) {
 319                 if (!foundRegardlessOfIgnored) {
 320                     throw new NoFilesFoundError(
 321                         pattern,
 322                         !globInputPaths && isGlob(pattern)
 323                     );
 324                 }
 325                 if (!found) {
 326                     throw new AllFilesIgnoredError(pattern);
 327                 }
 328             }
 329         }
 330
 331         debug(`Complete iterating files: ${JSON.stringify(patterns)}`);
 332     }
 333
 334     /**
 335      * Iterate files which are matched by a given glob pattern.
 336      * @param {string} pattern The glob pattern to iterate files.
 337      * @returns {IterableIterator<FileEntry>} The found files.
 338      */
 339     _iterateFiles(pattern) {
 340         const { cwd, globInputPaths } = internalSlotsMap.get(this);
 341         const absolutePath = path.resolve(cwd, pattern);
 342         const isDot = dotfilesPattern.test(pattern);
 343         const stat = statSafeSync(absolutePath);
 344
 345         if (stat && stat.isDirectory()) {
 346             return this._iterateFilesWithDirectory(absolutePath, isDot);
 347         }
 348         if (stat && stat.isFile()) {
 349             return this._iterateFilesWithFile(absolutePath);
 350         }
 351         if (globInputPaths && isGlobPattern(pattern)) {
 352             return this._iterateFilesWithGlob(absolutePath, isDot);
 353         }
 354
 355         return [];
 356     }
 357
 358     /**
 359      * Iterate a file which is matched by a given path.
 360      * @param {string} filePath The path to the target file.
 361      * @returns {IterableIterator<FileEntry>} The found files.
 362      * @private
 363      */
 364     _iterateFilesWithFile(filePath) {
 365         debug(`File: ${filePath}`);
 366
 367         const { configArrayFactory } = internalSlotsMap.get(this);
 368         const config = configArrayFactory.getConfigArrayForFile(filePath);
 369         const ignored = this._isIgnoredFile(filePath, { config, direct: true });
 370         const flag = ignored ? IGNORED : NONE;
 371
 372         return [{ config, filePath, flag }];
 373     }
 374
 375     /**
 376      * Iterate files in a given path.
 377      * @param {string} directoryPath The path to the target directory.
 378      * @param {boolean} dotfiles If `true` then it doesn't skip dot files by default.
 379      * @returns {IterableIterator<FileEntry>} The found files.
 380      * @private
 381      */
 382     _iterateFilesWithDirectory(directoryPath, dotfiles) {
 383         debug(`Directory: ${directoryPath}`);
 384
 385         return this._iterateFilesRecursive(
 386             directoryPath,
 387             { dotfiles, recursive: true, selector: null }
 388         );
 389     }
 390
 391     /**
 392      * Iterate files which are matched by a given glob pattern.
 393      * @param {string} pattern The glob pattern to iterate files.
 394      * @param {boolean} dotfiles If `true` then it doesn't skip dot files by default.
 395      * @returns {IterableIterator<FileEntry>} The found files.
 396      * @private
 397      */
 398     _iterateFilesWithGlob(pattern, dotfiles) {
 399         debug(`Glob: ${pattern}`);
 400
 401         const directoryPath = path.resolve(getGlobParent(pattern));
 402         const globPart = pattern.slice(directoryPath.length + 1);
 403
 404         /*
 405          * recursive if there are `**` or path separators in the glob part.
 406          * Otherwise, patterns such as `src/*.js`, it doesn't need recursive.
 407          */
 408         const recursive = /\*\*|\/|\\/u.test(globPart);
 409         const selector = new Minimatch(pattern, minimatchOpts);
 410
 411         debug(`recursive? ${recursive}`);
 412
 413         return this._iterateFilesRecursive(
 414             directoryPath,
 415             { dotfiles, recursive, selector }
 416         );
 417     }
 418
 419     /**
 420      * Iterate files in a given path.
 421      * @param {string} directoryPath The path to the target directory.
 422      * @param {Object} options The options to iterate files.
 423      * @param {boolean} [options.dotfiles] If `true` then it doesn't skip dot files by default.
 424      * @param {boolean} [options.recursive] If `true` then it dives into sub directories.
 425      * @param {InstanceType<Minimatch>} [options.selector] The matcher to choose files.
 426      * @returns {IterableIterator<FileEntry>} The found files.
 427      * @private
 428      */
 429     *_iterateFilesRecursive(directoryPath, options) {
 430         debug(`Enter the directory: ${directoryPath}`);
 431         const { configArrayFactory } = internalSlotsMap.get(this);
 432
 433         /** @type {ConfigArray|null} */
 434         let config = null;
 435
 436         // Enumerate the files of this directory.
 437         for (const entry of readdirSafeSync(directoryPath)) {
 438             const filePath = path.join(directoryPath, entry.name);
 439             const fileInfo = entry.isSymbolicLink() ? statSafeSync(filePath) : entry;
 440
 441             if (!fileInfo) {
 442                 continue;
 443             }
 444
 445             // Check if the file is matched.
 446             if (fileInfo.isFile()) {
 447                 if (!config) {
 448                     config = configArrayFactory.getConfigArrayForFile(
 449                         filePath,
 450
 451                         /*
 452                          * We must ignore `ConfigurationNotFoundError` at this
 453                          * point because we don't know if target files exist in
 454                          * this directory.
 455                          */
 456                         { ignoreNotFoundError: true }
 457                     );
 458                 }
 459                 const matched = options.selector
 460
 461                     // Started with a glob pattern; choose by the pattern.
 462                     ? options.selector.match(filePath)
 463
 464                     // Started with a directory path; choose by file extensions.
 465                     : this.isTargetPath(filePath, config);
 466
 467                 if (matched) {
 468                     const ignored = this._isIgnoredFile(filePath, { ...options, config });
 469                     const flag = ignored ? IGNORED_SILENTLY : NONE;
 470
 471                     debug(`Yield: ${entry.name}${ignored ? " but ignored" : ""}`);
 472                     yield {
 473                         config: configArrayFactory.getConfigArrayForFile(filePath),
 474                         filePath,
 475                         flag
 476                     };
 477                 } else {
 478                     debug(`Didn't match: ${entry.name}`);
 479                 }
 480
 481             // Dive into the sub directory.
 482             } else if (options.recursive && fileInfo.isDirectory()) {
 483                 if (!config) {
 484                     config = configArrayFactory.getConfigArrayForFile(
 485                         filePath,
 486                         { ignoreNotFoundError: true }
 487                     );
 488                 }
 489                 const ignored = this._isIgnoredFile(
 490                     filePath + path.sep,
 491                     { ...options, config }
 492                 );
 493
 494                 if (!ignored) {
 495                     yield* this._iterateFilesRecursive(filePath, options);
 496                 }
 497             }
 498         }
 499
 500         debug(`Leave the directory: ${directoryPath}`);
 501     }
 502
 503     /**
 504      * Check if a given file should be ignored.
 505      * @param {string} filePath The path to a file to check.
 506      * @param {Object} options Options
 507      * @param {ConfigArray} [options.config] The config for this file.
 508      * @param {boolean} [options.dotfiles] If `true` then this is not ignore dot files by default.
 509      * @param {boolean} [options.direct] If `true` then this is a direct specified file.
 510      * @returns {boolean} `true` if the file should be ignored.
 511      * @private
 512      */
 513     _isIgnoredFile(filePath, {
 514         config: providedConfig,
 515         dotfiles = false,
 516         direct = false
 517     }) {
 518         const {
 519             configArrayFactory,
 520             defaultIgnores,
 521             ignoreFlag
 522         } = internalSlotsMap.get(this);
 523
 524         if (ignoreFlag) {
 525             const config =
 526                 providedConfig ||
 527                 configArrayFactory.getConfigArrayForFile(
 528                     filePath,
 529                     { ignoreNotFoundError: true }
 530                 );
 531             const ignores =
 532                 config.extractConfig(filePath).ignores || defaultIgnores;
 533
 534             return ignores(filePath, dotfiles);
 535         }
 536
 537         return !direct && defaultIgnores(filePath, dotfiles);
 538     }
 539 }
 540
 541 //------------------------------------------------------------------------------
 542 // Public Interface
 543 //------------------------------------------------------------------------------
 544
 545 module.exports = { FileEnumerator };