python/xref2vtysh.py

   1 # SPDX-License-Identifier: GPL-2.0-or-later
   2 # FRR xref vtysh command extraction
   3 #
   4 # Copyright (C) 2022  David Lamparter for NetDEF, Inc.
   5
   6 """
   7 Generate vtysh_cmd.c from frr .xref file(s).
   8
   9 This can run either standalone or as part of xrelfo.  The latter saves a
  10 non-negligible amount of time (0.5s on average systems, more on e.g. slow ARMs)
  11 since serializing and deserializing JSON is a significant bottleneck in this.
  12 """
  13
  14 import sys
  15 import os
  16 import re
  17 import pathlib
  18 import argparse
  19 from collections import defaultdict
  20 import difflib
  21
  22 import json
  23
  24 try:
  25     import ujson as json  # type: ignore
  26 except ImportError:
  27     pass
  28
  29 frr_top_src = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
  30
  31 # vtysh needs to know which daemon(s) to send commands to.  For lib/, this is
  32 # not quite obvious...
  33
  34 daemon_flags = {
  35     "lib/agentx.c": "VTYSH_ISISD|VTYSH_RIPD|VTYSH_OSPFD|VTYSH_OSPF6D|VTYSH_BGPD|VTYSH_ZEBRA",
  36     "lib/filter.c": "VTYSH_ACL",
  37     "lib/filter_cli.c": "VTYSH_ACL",
  38     "lib/if.c": "VTYSH_INTERFACE",
  39     "lib/keychain.c": "VTYSH_RIPD|VTYSH_EIGRPD|VTYSH_OSPF6D",
  40     "lib/lib_vty.c": "VTYSH_ALL",
  41     "lib/log_vty.c": "VTYSH_ALL",
  42     "lib/nexthop_group.c": "VTYSH_NH_GROUP",
  43     "lib/resolver.c": "VTYSH_NHRPD|VTYSH_BGPD",
  44     "lib/routemap.c": "VTYSH_RMAP",
  45     "lib/routemap_cli.c": "VTYSH_RMAP",
  46     "lib/spf_backoff.c": "VTYSH_ISISD",
  47     "lib/thread.c": "VTYSH_ALL",
  48     "lib/vrf.c": "VTYSH_VRF",
  49     "lib/vty.c": "VTYSH_ALL",
  50 }
  51
  52 vtysh_cmd_head = """/* autogenerated file, DO NOT EDIT! */
  53 #include <zebra.h>
  54
  55 #include "command.h"
  56 #include "linklist.h"
  57
  58 #include "vtysh/vtysh.h"
  59 """
  60
  61 if sys.stderr.isatty():
  62     _fmt_red = "\033[31m"
  63     _fmt_green = "\033[32m"
  64     _fmt_clear = "\033[m"
  65 else:
  66     _fmt_red = _fmt_green = _fmt_clear = ""
  67
  68
  69 def c_escape(text: str) -> str:
  70     """
  71     Escape string for output into C source code.
  72
  73     Handles only what's needed here.  CLI strings and help text don't contain
  74     weird special characters.
  75     """
  76     return text.replace("\\", "\\\\").replace('"', '\\"').replace("\n", "\\n")
  77
  78
  79 class NodeDict(defaultdict):
  80     """
  81     CLI node ID (integer) -> dict of commands in that node.
  82     """
  83
  84     nodenames = {}  # Dict[int, str]
  85
  86     def __init__(self):
  87         super().__init__(dict)
  88
  89     def items_named(self):
  90         for k, v in self.items():
  91             yield self.nodename(k), v
  92
  93     @classmethod
  94     def nodename(cls, nodeid: int) -> str:
  95         return cls.nodenames.get(nodeid, str(nodeid))
  96
  97     @classmethod
  98     def load_nodenames(cls):
  99         with open(os.path.join(frr_top_src, "lib", "command.h"), "r") as fd:
 100             command_h = fd.read()
 101
 102         nodes = re.search(r"enum\s+node_type\s+\{(.*?)\}", command_h, re.S)
 103         if nodes is None:
 104             raise RuntimeError(
 105                 "regex failed to match on lib/command.h (to get CLI node names)"
 106             )
 107
 108         text = nodes.group(1)
 109         text = re.sub(r"/\*.*?\*/", "", text, flags=re.S)
 110         text = re.sub(r"//.*?$", "", text, flags=re.M)
 111         text = text.replace(",", " ")
 112         text = text.split()
 113
 114         for i, name in enumerate(text):
 115             cls.nodenames[i] = name
 116
 117
 118 class CommandEntry:
 119     """
 120     CLI command definition.
 121
 122     - one DEFUN creates at most one of these, even if the same command is
 123       installed in multiple CLI nodes (e.g. BGP address-family nodes)
 124     - for each CLI node, commands with the same CLI string are merged.  This
 125       is *almost* irrelevant - ospfd & ospf6d define some identical commands
 126       in the route-map node.  Those must be merged for things to work
 127       correctly.
 128     """
 129
 130     all_defs = []  # List[CommandEntry]
 131     warn_counter = 0
 132
 133     def __init__(self, origin, name, spec):
 134         self.origin = origin
 135         self.name = name
 136         self._spec = spec
 137         self._registered = False
 138
 139         self.cmd = spec["string"]
 140         self._cmd_normalized = self.normalize_cmd(self.cmd)
 141
 142         self.hidden = "hidden" in spec.get("attrs", [])
 143         self.daemons = self._get_daemons()
 144
 145         self.doclines = self._spec["doc"].splitlines(keepends=True)
 146         if not self.doclines[-1].endswith("\n"):
 147             self.warn_loc("docstring does not end with \\n")
 148
 149     def warn_loc(self, wtext, nodename=None):
 150         """
 151         Print warning with parseable (compiler style) location
 152
 153         Matching the way compilers emit file/lineno means editors/IDE can
 154         identify / jump to the error location.
 155         """
 156
 157         if nodename:
 158             prefix = ": [%s] %s:" % (nodename, self.name)
 159         else:
 160             prefix = ": %s:" % (self.name,)
 161
 162         for line in wtext.rstrip("\n").split("\n"):
 163             sys.stderr.write(
 164                 "%s:%d%s %s\n"
 165                 % (
 166                     self._spec["defun"]["file"],
 167                     self._spec["defun"]["line"],
 168                     prefix,
 169                     line,
 170                 )
 171             )
 172             prefix = "-    "
 173
 174         CommandEntry.warn_counter += 1
 175
 176     def _get_daemons(self):
 177         path = pathlib.Path(self.origin)
 178         if path.name == "vtysh":
 179             return {}
 180
 181         defun_file = os.path.relpath(self._spec["defun"]["file"], frr_top_src)
 182         defun_path = pathlib.Path(defun_file)
 183
 184         if defun_path.parts[0] != "lib":
 185             if "." not in path.name:
 186                 # daemons don't have dots in their filename
 187                 return {"VTYSH_" + path.name.upper()}
 188
 189             # loadable modules - use directory name to determine daemon
 190             return {"VTYSH_" + path.parts[-2].upper()}
 191
 192         if defun_file in daemon_flags:
 193             return {daemon_flags[defun_file]}
 194
 195         v6_cmd = "ipv6" in self.name
 196         if defun_file == "lib/plist.c":
 197             if v6_cmd:
 198                 return {
 199                     "VTYSH_RIPNGD|VTYSH_OSPF6D|VTYSH_BGPD|VTYSH_ZEBRA|VTYSH_PIM6D|VTYSH_BABELD|VTYSH_ISISD|VTYSH_FABRICD"
 200                 }
 201             else:
 202                 return {
 203                     "VTYSH_RIPD|VTYSH_OSPFD|VTYSH_BGPD|VTYSH_ZEBRA|VTYSH_PIMD|VTYSH_EIGRPD|VTYSH_BABELD|VTYSH_ISISD|VTYSH_FABRICD"
 204                 }
 205
 206         if defun_file == "lib/if_rmap.c":
 207             if v6_cmd:
 208                 return {"VTYSH_RIPNGD"}
 209             else:
 210                 return {"VTYSH_RIPD"}
 211
 212         return {}
 213
 214     def __repr__(self):
 215         return "<CommandEntry %s: %r>" % (self.name, self.cmd)
 216
 217     def register(self):
 218         """Track DEFUNs so each is only output once."""
 219         if not self._registered:
 220             self.all_defs.append(self)
 221             self._registered = True
 222         return self
 223
 224     def merge(self, other, nodename):
 225         if self._cmd_normalized != other._cmd_normalized:
 226             self.warn_loc(
 227                 "command definition mismatch, first definied as:\n%r" % (self.cmd,),
 228                 nodename=nodename,
 229             )
 230             other.warn_loc("later defined as:\n%r" % (other.cmd,), nodename=nodename)
 231
 232         if self._spec["doc"] != other._spec["doc"]:
 233             self.warn_loc(
 234                 "help string mismatch, first defined here (-)", nodename=nodename
 235             )
 236             other.warn_loc(
 237                 "later defined here (+)\nnote: both commands define %r in same node (%s)"
 238                 % (self.cmd, nodename),
 239                 nodename=nodename,
 240             )
 241
 242             d = difflib.Differ()
 243             for diffline in d.compare(self.doclines, other.doclines):
 244                 if diffline.startswith("  "):
 245                     continue
 246                 if diffline.startswith("+ "):
 247                     diffline = _fmt_green + diffline
 248                 elif diffline.startswith("- "):
 249                     diffline = _fmt_red + diffline
 250                 sys.stderr.write("\t" + diffline.rstrip("\n") + _fmt_clear + "\n")
 251
 252         if self.hidden != other.hidden:
 253             self.warn_loc(
 254                 "hidden flag mismatch, first %r here" % (self.hidden,),
 255                 nodename=nodename,
 256             )
 257             other.warn_loc(
 258                 "later %r here (+)\nnote: both commands define %r in same node (%s)"
 259                 % (other.hidden, self.cmd, nodename),
 260                 nodename=nodename,
 261             )
 262
 263         # ensure name is deterministic regardless of input DEFUN order
 264         self.name = min([self.name, other.name], key=lambda i: (len(i), i))
 265         self.daemons.update(other.daemons)
 266
 267     def get_def(self):
 268         doc = "\n".join(['\t"%s"' % c_escape(line) for line in self.doclines])
 269         defsh = "DEFSH_HIDDEN" if self.hidden else "DEFSH"
 270
 271         # make daemon list deterministic
 272         daemons = set()
 273         for daemon in self.daemons:
 274             daemons.update(daemon.split("|"))
 275         daemon_str = "|".join(sorted(daemons))
 276
 277         return """
 278 %s (%s, %s_vtysh,
 279 \t"%s",
 280 %s)
 281 """ % (
 282             defsh,
 283             daemon_str,
 284             self.name,
 285             c_escape(self.cmd),
 286             doc,
 287         )
 288
 289     # accept slightly different command definitions that result in the same command
 290     re_collapse_ws = re.compile(r"\s+")
 291     re_remove_varnames = re.compile(r"\$[a-z][a-z0-9_]*")
 292
 293     @classmethod
 294     def normalize_cmd(cls, cmd):
 295         cmd = cmd.strip()
 296         cmd = cls.re_collapse_ws.sub(" ", cmd)
 297         cmd = cls.re_remove_varnames.sub("", cmd)
 298         return cmd
 299
 300     @classmethod
 301     def process(cls, nodes, name, origin, spec):
 302         if "nosh" in spec.get("attrs", []):
 303             return
 304         if origin == "vtysh/vtysh":
 305             return
 306
 307         if origin == "isisd/fabricd":
 308             # dirty workaround :(
 309             name = "fabricd_" + name
 310
 311         entry = cls(origin, name, spec)
 312         if not entry.daemons:
 313             return
 314
 315         for nodedata in spec.get("nodes", []):
 316             node = nodes[nodedata["node"]]
 317             if entry._cmd_normalized not in node:
 318                 node[entry._cmd_normalized] = entry.register()
 319             else:
 320                 node[entry._cmd_normalized].merge(
 321                     entry, nodes.nodename(nodedata["node"])
 322                 )
 323
 324     @classmethod
 325     def load(cls, xref):
 326         nodes = NodeDict()
 327
 328         for cmd_name, origins in xref.get("cli", {}).items():
 329             for origin, spec in origins.items():
 330                 CommandEntry.process(nodes, cmd_name, origin, spec)
 331         return nodes
 332
 333     @classmethod
 334     def output_defs(cls, ofd):
 335         for entry in sorted(cls.all_defs, key=lambda i: i.name):
 336             ofd.write(entry.get_def())
 337
 338     @classmethod
 339     def output_install(cls, ofd, nodes):
 340         ofd.write("\nvoid vtysh_init_cmd(void)\n{\n")
 341
 342         for name, items in sorted(nodes.items_named()):
 343             for item in sorted(items.values(), key=lambda i: i.name):
 344                 ofd.write("\tinstall_element(%s, &%s_vtysh);\n" % (name, item.name))
 345
 346         ofd.write("}\n")
 347
 348     @classmethod
 349     def run(cls, xref, ofd):
 350         ofd.write(vtysh_cmd_head)
 351
 352         NodeDict.load_nodenames()
 353         nodes = cls.load(xref)
 354         cls.output_defs(ofd)
 355         cls.output_install(ofd, nodes)
 356
 357
 358 def main():
 359     argp = argparse.ArgumentParser(description="FRR xref to vtysh defs")
 360     argp.add_argument(
 361         "xreffile", metavar="XREFFILE", type=str, help=".xref file to read"
 362     )
 363     argp.add_argument("-Werror", action="store_const", const=True)
 364     args = argp.parse_args()
 365
 366     with open(args.xreffile, "r") as fd:
 367         data = json.load(fd)
 368
 369     CommandEntry.run(data, sys.stdout)
 370
 371     if args.Werror and CommandEntry.warn_counter:
 372         sys.exit(1)
 373
 374
 375 if __name__ == "__main__":
 376     main()