eslint/tools/fetch-docs-links.js

   1 /**
   2  * @fileoverview Script to fetch link data.
   3  *
   4  * To fetch info about all files:
   5  *
   6  *      node tools/fetch-docs-links.js
   7  *
   8  * To fetch info for just selected files (for use with lint-staged):
   9  *
  10  *      node tools/fetch-docs-links.js docs/src/use/index.md
  11  *
  12  * @author Nicholas C. Zakas
  13  */
  14
  15 "use strict";
  16
  17 //-----------------------------------------------------------------------------
  18 // Requirements
  19 //-----------------------------------------------------------------------------
  20
  21 const matter = require("gray-matter");
  22 const metascraper = require("metascraper")([
  23     require("metascraper-image")(),
  24     require("metascraper-logo")(),
  25     require("metascraper-logo-favicon")(),
  26     require("metascraper-title")(),
  27     require("metascraper-description")()
  28 ]);
  29 const got = require("got");
  30 const path = require("path");
  31 const fs = require("fs").promises;
  32 const glob = require("fast-glob");
  33
  34 //-----------------------------------------------------------------------------
  35 // Data
  36 //-----------------------------------------------------------------------------
  37
  38 const BASE_DIR = path.resolve(__dirname, "../");
  39 const SRC_DIR = path.resolve(BASE_DIR, "docs/src");
  40 const DATA_DIR = path.resolve(SRC_DIR, "_data");
  41 const DATA_FILE_PATH = path.resolve(DATA_DIR, "further_reading_links.json");
  42
  43 // determine which files to check
  44 let filenames = process.argv.slice(2);
  45
  46 if (filenames.length === 0) {
  47     filenames = glob.sync("docs/src/rules/*.md", { cwd: BASE_DIR });
  48 }
  49
  50 filenames = filenames.map(filename => path.resolve(BASE_DIR, filename));
  51
  52 //-----------------------------------------------------------------------------
  53 // Helpers
  54 //-----------------------------------------------------------------------------
  55
  56 /**
  57  * Fetches metadata information for a given URL.
  58  * @param {string} url The URL to fetch data for.
  59  * @returns {Promise<object>} An object with metadata info.
  60  */
  61 async function fetchLinkMeta(url) {
  62     const { body: html, url: returnedURL } = await got(url);
  63     const metadata = await metascraper({ html, url: returnedURL });
  64     const domain = (new URL(returnedURL)).hostname;
  65
  66     return {
  67         domain,
  68         url,
  69         logo: metadata.logo,
  70         title: metadata.title,
  71         description: metadata.description
  72     };
  73 }
  74
  75
  76 //-----------------------------------------------------------------------------
  77 // Main
  78 //-----------------------------------------------------------------------------
  79
  80 (async () => {
  81
  82     // First read in the current data file
  83     const links = JSON.parse(await fs.readFile(DATA_FILE_PATH, "utf8"));
  84
  85     // check each file
  86     for (const filename of filenames) {
  87
  88         const text = await fs.readFile(filename, "utf8");
  89         const frontmatter = matter(text).data;
  90
  91         if (frontmatter.further_reading) {
  92
  93             for (const url of frontmatter.further_reading) {
  94                 if (!links[url]) {
  95                     try {
  96                         links[url] = await fetchLinkMeta(url);
  97                     } catch (ex) {
  98                         console.error("Error in ", filename);
  99                         console.error("Could not fetch data for", url);
 100                         console.error(ex.message);
 101                         console.error(ex.stack);
 102                         process.exit(1);
 103                     }
 104                 }
 105             }
 106
 107         }
 108     }
 109
 110     // Last write new data into the current data file
 111     await fs.writeFile(DATA_FILE_PATH, JSON.stringify(links, null, 4), "utf8");
 112 })();