python/xref2vtysh.py

   1 # FRR xref vtysh command extraction
   2 #
   3 # Copyright (C) 2022  David Lamparter for NetDEF, Inc.
   4 #
   5 # This program is free software; you can redistribute it and/or modify it
   6 # under the terms of the GNU General Public License as published by the Free
   7 # Software Foundation; either version 2 of the License, or (at your option)
   8 # any later version.
   9 #
  10 # This program is distributed in the hope that it will be useful, but WITHOUT
  11 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12 # FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  13 # more details.
  14 #
  15 # You should have received a copy of the GNU General Public License along
  16 # with this program; see the file COPYING; if not, write to the Free Software
  17 # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
  18
  19 """
  20 Generate vtysh_cmd.c from frr .xref file(s).
  21
  22 This can run either standalone or as part of xrelfo.  The latter saves a
  23 non-negligible amount of time (0.5s on average systems, more on e.g. slow ARMs)
  24 since serializing and deserializing JSON is a significant bottleneck in this.
  25 """
  26
  27 import sys
  28 import os
  29 import re
  30 import pathlib
  31 import argparse
  32 from collections import defaultdict
  33 import difflib
  34
  35 import json
  36
  37 try:
  38     import ujson as json  # type: ignore
  39 except ImportError:
  40     pass
  41
  42 frr_top_src = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
  43
  44 # vtysh needs to know which daemon(s) to send commands to.  For lib/, this is
  45 # not quite obvious...
  46
  47 daemon_flags = {
  48     "lib/agentx.c": "VTYSH_ISISD|VTYSH_RIPD|VTYSH_OSPFD|VTYSH_OSPF6D|VTYSH_BGPD|VTYSH_ZEBRA",
  49     "lib/filter.c": "VTYSH_ACL",
  50     "lib/filter_cli.c": "VTYSH_ACL",
  51     "lib/if.c": "VTYSH_INTERFACE",
  52     "lib/keychain.c": "VTYSH_RIPD|VTYSH_EIGRPD|VTYSH_OSPF6D",
  53     "lib/lib_vty.c": "VTYSH_ALL",
  54     "lib/log_vty.c": "VTYSH_ALL",
  55     "lib/nexthop_group.c": "VTYSH_NH_GROUP",
  56     "lib/resolver.c": "VTYSH_NHRPD|VTYSH_BGPD",
  57     "lib/routemap.c": "VTYSH_RMAP",
  58     "lib/routemap_cli.c": "VTYSH_RMAP",
  59     "lib/spf_backoff.c": "VTYSH_ISISD",
  60     "lib/thread.c": "VTYSH_ALL",
  61     "lib/vrf.c": "VTYSH_VRF",
  62     "lib/vty.c": "VTYSH_ALL",
  63 }
  64
  65 vtysh_cmd_head = """/* autogenerated file, DO NOT EDIT! */
  66 #include <zebra.h>
  67
  68 #include "command.h"
  69 #include "linklist.h"
  70
  71 #include "vtysh/vtysh.h"
  72 """
  73
  74 if sys.stderr.isatty():
  75     _fmt_red = "\033[31m"
  76     _fmt_green = "\033[32m"
  77     _fmt_clear = "\033[m"
  78 else:
  79     _fmt_red = _fmt_green = _fmt_clear = ""
  80
  81
  82 def c_escape(text: str) -> str:
  83     """
  84     Escape string for output into C source code.
  85
  86     Handles only what's needed here.  CLI strings and help text don't contain
  87     weird special characters.
  88     """
  89     return text.replace("\\", "\\\\").replace('"', '\\"').replace("\n", "\\n")
  90
  91
  92 class NodeDict(defaultdict):
  93     """
  94     CLI node ID (integer) -> dict of commands in that node.
  95     """
  96
  97     nodenames = {}  # Dict[int, str]
  98
  99     def __init__(self):
 100         super().__init__(dict)
 101
 102     def items_named(self):
 103         for k, v in self.items():
 104             yield self.nodename(k), v
 105
 106     @classmethod
 107     def nodename(cls, nodeid: int) -> str:
 108         return cls.nodenames.get(nodeid, str(nodeid))
 109
 110     @classmethod
 111     def load_nodenames(cls):
 112         with open(os.path.join(frr_top_src, "lib", "command.h"), "r") as fd:
 113             command_h = fd.read()
 114
 115         nodes = re.search(r"enum\s+node_type\s+\{(.*?)\}", command_h, re.S)
 116         if nodes is None:
 117             raise RuntimeError(
 118                 "regex failed to match on lib/command.h (to get CLI node names)"
 119             )
 120
 121         text = nodes.group(1)
 122         text = re.sub(r"/\*.*?\*/", "", text, flags=re.S)
 123         text = re.sub(r"//.*?$", "", text, flags=re.M)
 124         text = text.replace(",", " ")
 125         text = text.split()
 126
 127         for i, name in enumerate(text):
 128             cls.nodenames[i] = name
 129
 130
 131 class CommandEntry:
 132     """
 133     CLI command definition.
 134
 135     - one DEFUN creates at most one of these, even if the same command is
 136       installed in multiple CLI nodes (e.g. BGP address-family nodes)
 137     - for each CLI node, commands with the same CLI string are merged.  This
 138       is *almost* irrelevant - ospfd & ospf6d define some identical commands
 139       in the route-map node.  Those must be merged for things to work
 140       correctly.
 141     """
 142
 143     all_defs = []  # List[CommandEntry]
 144     warn_counter = 0
 145
 146     def __init__(self, origin, name, spec):
 147         self.origin = origin
 148         self.name = name
 149         self._spec = spec
 150         self._registered = False
 151
 152         self.cmd = spec["string"]
 153         self._cmd_normalized = self.normalize_cmd(self.cmd)
 154
 155         self.hidden = "hidden" in spec.get("attrs", [])
 156         self.daemons = self._get_daemons()
 157
 158         self.doclines = self._spec["doc"].splitlines(keepends=True)
 159         if not self.doclines[-1].endswith("\n"):
 160             self.warn_loc("docstring does not end with \\n")
 161
 162     def warn_loc(self, wtext, nodename=None):
 163         """
 164         Print warning with parseable (compiler style) location
 165
 166         Matching the way compilers emit file/lineno means editors/IDE can
 167         identify / jump to the error location.
 168         """
 169
 170         if nodename:
 171             prefix = ": [%s] %s:" % (nodename, self.name)
 172         else:
 173             prefix = ": %s:" % (self.name,)
 174
 175         for line in wtext.rstrip("\n").split("\n"):
 176             sys.stderr.write(
 177                 "%s:%d%s %s\n"
 178                 % (
 179                     self._spec["defun"]["file"],
 180                     self._spec["defun"]["line"],
 181                     prefix,
 182                     line,
 183                 )
 184             )
 185             prefix = "-    "
 186
 187         CommandEntry.warn_counter += 1
 188
 189     def _get_daemons(self):
 190         path = pathlib.Path(self.origin)
 191         if path.name == "vtysh":
 192             return {}
 193
 194         defun_file = os.path.relpath(self._spec["defun"]["file"], frr_top_src)
 195         defun_path = pathlib.Path(defun_file)
 196
 197         if defun_path.parts[0] != "lib":
 198             if "." not in path.name:
 199                 # daemons don't have dots in their filename
 200                 return {"VTYSH_" + path.name.upper()}
 201
 202             # loadable modules - use directory name to determine daemon
 203             return {"VTYSH_" + path.parts[-2].upper()}
 204
 205         if defun_file in daemon_flags:
 206             return {daemon_flags[defun_file]}
 207
 208         v6_cmd = "ipv6" in self.name
 209         if defun_file == "lib/plist.c":
 210             if v6_cmd:
 211                 return {
 212                     "VTYSH_RIPNGD|VTYSH_OSPF6D|VTYSH_BGPD|VTYSH_ZEBRA|VTYSH_PIM6D|VTYSH_BABELD|VTYSH_ISISD|VTYSH_FABRICD"
 213                 }
 214             else:
 215                 return {
 216                     "VTYSH_RIPD|VTYSH_OSPFD|VTYSH_BGPD|VTYSH_ZEBRA|VTYSH_PIMD|VTYSH_EIGRPD|VTYSH_BABELD|VTYSH_ISISD|VTYSH_FABRICD"
 217                 }
 218
 219         if defun_file == "lib/if_rmap.c":
 220             if v6_cmd:
 221                 return {"VTYSH_RIPNGD"}
 222             else:
 223                 return {"VTYSH_RIPD"}
 224
 225         return {}
 226
 227     def __repr__(self):
 228         return "<CommandEntry %s: %r>" % (self.name, self.cmd)
 229
 230     def register(self):
 231         """Track DEFUNs so each is only output once."""
 232         if not self._registered:
 233             self.all_defs.append(self)
 234             self._registered = True
 235         return self
 236
 237     def merge(self, other, nodename):
 238         if self._cmd_normalized != other._cmd_normalized:
 239             self.warn_loc(
 240                 "command definition mismatch, first definied as:\n%r" % (self.cmd,),
 241                 nodename=nodename,
 242             )
 243             other.warn_loc("later defined as:\n%r" % (other.cmd,), nodename=nodename)
 244
 245         if self._spec["doc"] != other._spec["doc"]:
 246             self.warn_loc(
 247                 "help string mismatch, first defined here (-)", nodename=nodename
 248             )
 249             other.warn_loc(
 250                 "later defined here (+)\nnote: both commands define %r in same node (%s)"
 251                 % (self.cmd, nodename),
 252                 nodename=nodename,
 253             )
 254
 255             d = difflib.Differ()
 256             for diffline in d.compare(self.doclines, other.doclines):
 257                 if diffline.startswith("  "):
 258                     continue
 259                 if diffline.startswith("+ "):
 260                     diffline = _fmt_green + diffline
 261                 elif diffline.startswith("- "):
 262                     diffline = _fmt_red + diffline
 263                 sys.stderr.write("\t" + diffline.rstrip("\n") + _fmt_clear + "\n")
 264
 265         if self.hidden != other.hidden:
 266             self.warn_loc(
 267                 "hidden flag mismatch, first %r here" % (self.hidden,),
 268                 nodename=nodename,
 269             )
 270             other.warn_loc(
 271                 "later %r here (+)\nnote: both commands define %r in same node (%s)"
 272                 % (other.hidden, self.cmd, nodename),
 273                 nodename=nodename,
 274             )
 275
 276         # ensure name is deterministic regardless of input DEFUN order
 277         self.name = min([self.name, other.name], key=lambda i: (len(i), i))
 278         self.daemons.update(other.daemons)
 279
 280     def get_def(self):
 281         doc = "\n".join(['\t"%s"' % c_escape(line) for line in self.doclines])
 282         defsh = "DEFSH_HIDDEN" if self.hidden else "DEFSH"
 283
 284         # make daemon list deterministic
 285         daemons = set()
 286         for daemon in self.daemons:
 287             daemons.update(daemon.split("|"))
 288         daemon_str = "|".join(sorted(daemons))
 289
 290         return """
 291 %s (%s, %s_vtysh,
 292 \t"%s",
 293 %s)
 294 """ % (
 295             defsh,
 296             daemon_str,
 297             self.name,
 298             c_escape(self.cmd),
 299             doc,
 300         )
 301
 302     # accept slightly different command definitions that result in the same command
 303     re_collapse_ws = re.compile(r"\s+")
 304     re_remove_varnames = re.compile(r"\$[a-z][a-z0-9_]*")
 305
 306     @classmethod
 307     def normalize_cmd(cls, cmd):
 308         cmd = cmd.strip()
 309         cmd = cls.re_collapse_ws.sub(" ", cmd)
 310         cmd = cls.re_remove_varnames.sub("", cmd)
 311         return cmd
 312
 313     @classmethod
 314     def process(cls, nodes, name, origin, spec):
 315         if "nosh" in spec.get("attrs", []):
 316             return
 317         if origin == "vtysh/vtysh":
 318             return
 319
 320         if origin == "isisd/fabricd":
 321             # dirty workaround :(
 322             name = "fabricd_" + name
 323
 324         entry = cls(origin, name, spec)
 325         if not entry.daemons:
 326             return
 327
 328         for nodedata in spec.get("nodes", []):
 329             node = nodes[nodedata["node"]]
 330             if entry._cmd_normalized not in node:
 331                 node[entry._cmd_normalized] = entry.register()
 332             else:
 333                 node[entry._cmd_normalized].merge(
 334                     entry, nodes.nodename(nodedata["node"])
 335                 )
 336
 337     @classmethod
 338     def load(cls, xref):
 339         nodes = NodeDict()
 340
 341         for cmd_name, origins in xref.get("cli", {}).items():
 342             for origin, spec in origins.items():
 343                 CommandEntry.process(nodes, cmd_name, origin, spec)
 344         return nodes
 345
 346     @classmethod
 347     def output_defs(cls, ofd):
 348         for entry in sorted(cls.all_defs, key=lambda i: i.name):
 349             ofd.write(entry.get_def())
 350
 351     @classmethod
 352     def output_install(cls, ofd, nodes):
 353         ofd.write("\nvoid vtysh_init_cmd(void)\n{\n")
 354
 355         for name, items in sorted(nodes.items_named()):
 356             for item in sorted(items.values(), key=lambda i: i.name):
 357                 ofd.write("\tinstall_element(%s, &%s_vtysh);\n" % (name, item.name))
 358
 359         ofd.write("}\n")
 360
 361     @classmethod
 362     def run(cls, xref, ofd):
 363         ofd.write(vtysh_cmd_head)
 364
 365         NodeDict.load_nodenames()
 366         nodes = cls.load(xref)
 367         cls.output_defs(ofd)
 368         cls.output_install(ofd, nodes)
 369
 370
 371 def main():
 372     argp = argparse.ArgumentParser(description="FRR xref to vtysh defs")
 373     argp.add_argument(
 374         "xreffile", metavar="XREFFILE", type=str, help=".xref file to read"
 375     )
 376     argp.add_argument("-Werror", action="store_const", const=True)
 377     args = argp.parse_args()
 378
 379     with open(args.xreffile, "r") as fd:
 380         data = json.load(fd)
 381
 382     CommandEntry.run(data, sys.stdout)
 383
 384     if args.Werror and CommandEntry.warn_counter:
 385         sys.exit(1)
 386
 387
 388 if __name__ == "__main__":
 389     main()