check_whence.py

   1 #!/usr/bin/python3
   2
   3 import os, re, sys
   4 from io import open
   5
   6
   7 def list_whence():
   8     with open("WHENCE", encoding="utf-8") as whence:
   9         for line in whence:
  10             match = re.match(r'(?:RawFile|File|Source):\s*"(.*)"', line)
  11             if match:
  12                 yield match.group(1)
  13                 continue
  14             match = re.match(r"(?:RawFile|File|Source):\s*(\S*)", line)
  15             if match:
  16                 yield match.group(1)
  17                 continue
  18             match = re.match(
  19                 r"Licen[cs]e: (?:.*\bSee (.*) for details\.?|(\S*))\n", line
  20             )
  21             if match:
  22                 if match.group(1):
  23                     for name in re.split(r", | and ", match.group(1)):
  24                         yield name
  25                     continue
  26                 if match.group(2):
  27                     # Just one word - may or may not be a filename
  28                     if not re.search(
  29                         r"unknown|distributable", match.group(2), re.IGNORECASE
  30                     ):
  31                         yield match.group(2)
  32                         continue
  33
  34
  35 def list_whence_files():
  36     with open("WHENCE", encoding="utf-8") as whence:
  37         for line in whence:
  38             match = re.match(r"(?:RawFile|File):\s*(.*)", line)
  39             if match:
  40                 yield match.group(1).replace("\ ", " ").replace('"', "")
  41                 continue
  42
  43
  44 def list_links_list():
  45     with open("WHENCE", encoding="utf-8") as whence:
  46         for line in whence:
  47             match = re.match(r"Link:\s*(.*)", line)
  48             if match:
  49                 linkname, target = match.group(1).split("->")
  50
  51                 linkname = linkname.strip().replace("\ ", " ").replace('"', "")
  52                 target = target.strip().replace("\ ", " ").replace('"', "")
  53
  54                 # Link target is relative to the link
  55                 target = os.path.join(os.path.dirname(linkname), target)
  56                 target = os.path.normpath(target)
  57
  58                 yield (linkname, target)
  59                 continue
  60
  61
  62 def list_git():
  63     with os.popen("git ls-files") as git_files:
  64         for line in git_files:
  65             yield line.rstrip("\n")
  66
  67
  68 def main():
  69     ret = 0
  70     whence_list = list(list_whence())
  71     whence_files = list(list_whence_files())
  72     links_list = list(list_links_list())
  73     known_files = set(name for name in whence_list if not name.endswith("/")) | set(
  74         [
  75             ".gitignore",
  76             ".codespell.cfg",
  77             ".gitlab-ci.yml",
  78             ".pre-commit-config.yaml",
  79             "build_packages.py",
  80             "check_whence.py",
  81             "configure",
  82             "Makefile",
  83             "README.md",
  84             "copy-firmware.sh",
  85             "WHENCE",
  86             "Dockerfile",
  87             "contrib/templates/debian.changelog",
  88             "contrib/templates/debian.control",
  89             "contrib/templates/debian.copyright",
  90             "contrib/templates/rpm.spec",
  91             "contrib/process_linux_firmware.py",
  92         ]
  93     )
  94     known_prefixes = set(name for name in whence_list if name.endswith("/"))
  95     git_files = set(list_git())
  96
  97     for name in set(name for name in whence_files if name.endswith("/")):
  98         sys.stderr.write("E: %s listed in WHENCE as File, but is directory\n" % name)
  99         ret = 1
 100
 101     for name in set(fw for fw in whence_files if whence_files.count(fw) > 1):
 102         sys.stderr.write("E: %s listed in WHENCE twice\n" % name)
 103         ret = 1
 104
 105     for name in set(link for link in whence_files if os.path.islink(link)):
 106         sys.stderr.write("E: %s listed in WHENCE as File, but is a symlink\n" % name)
 107         ret = 1
 108
 109     for name in set(link[0] for link in links_list if os.path.islink(link[0])):
 110         sys.stderr.write("E: %s listed in WHENCE as Link, is in tree\n" % name)
 111         ret = 1
 112
 113     for name in sorted(list(known_files - git_files)):
 114         sys.stderr.write("E: %s listed in WHENCE does not exist\n" % name)
 115         ret = 1
 116
 117     # A link can point to another link, or to a file...
 118     valid_targets = set(link[0] for link in links_list) | git_files
 119
 120     # ... or to a directory
 121     for target in set(valid_targets):
 122         dirname = target
 123         while True:
 124             dirname = os.path.dirname(dirname)
 125             if dirname == "":
 126                 break
 127             valid_targets.add(dirname)
 128
 129     for name, target in sorted(links_list):
 130         if target not in valid_targets:
 131             sys.stderr.write(
 132                 "E: target %s of link %s in WHENCE" " does not exist\n" % (target, name)
 133             )
 134             ret = 1
 135
 136     for name in sorted(list(git_files - known_files)):
 137         # Ignore subdirectory changelogs and GPG detached signatures
 138         if name.endswith("/ChangeLog") or (
 139             name.endswith(".asc") and name[:-4] in known_files
 140         ):
 141             continue
 142
 143         # Ignore unknown files in known directories
 144         for prefix in known_prefixes:
 145             if name.startswith(prefix):
 146                 break
 147         else:
 148             sys.stderr.write("E: %s not listed in WHENCE\n" % name)
 149             ret = 1
 150     return ret
 151
 152
 153 if __name__ == "__main__":
 154     sys.exit(main())