lib/defun_lex.l

   1 /* SPDX-License-Identifier: GPL-2.0-or-later */
   2 /*
   3  * clippy (CLI preparator in python) C pseudo-lexer
   4  * Copyright (C) 2016-2017  David Lamparter for NetDEF, Inc.
   5  */
   6
   7 /* This is just enough of a lexer to make rough sense of a C source file.
   8  * It handles C preprocessor directives, strings, and looks for FRR-specific
   9  * idioms (aka DEFUN).
  10  *
  11  * There is some preliminary support for documentation comments for DEFUNs.
  12  * They would look like this (note the ~):  (replace \ by /)
  13  *
  14  * \*~  documentation for foobar_cmd
  15  *  *   parameter does xyz
  16  *  *\
  17  * DEFUN(foobar_cmd, ...)
  18  *
  19  * This is intended for user documentation / command reference.  Don't put
  20  * code documentation in it.
  21  */
  22
  23 %top{
  24 #ifdef HAVE_CONFIG_H
  25 #include "config.h"
  26 #endif
  27 }
  28 %{
  29 /* ignore harmless bugs in old versions of flex */
  30 #pragma GCC diagnostic ignored "-Wsign-compare"
  31 #pragma GCC diagnostic ignored "-Wunused-value"
  32
  33 #include "config.h"
  34 #include <Python.h>
  35 #include <string.h>
  36 #include <stdlib.h>
  37
  38 #include "command_graph.h"
  39 #include "clippy.h"
  40
  41 #define ID              258
  42 #define PREPROC         259
  43 #define OPERATOR        260
  44 #define STRING          261
  45 #define COMMENT         262
  46 #define SPECIAL         263
  47
  48 #define DEFUNNY         270
  49 #define INSTALL         271
  50 #define AUXILIARY       272
  51
  52 int comment_link;
  53 char string_end;
  54
  55 char *value;
  56 static const char *yyfilename;
  57
  58 static void extendbuf(char **what, const char *arg)
  59 {
  60         if (!*what)
  61                 *what = strdup(arg);
  62         else {
  63                 size_t vall = strlen(*what), argl = strlen(arg);
  64                 *what = realloc(*what, vall + argl + 1);
  65                 memcpy(*what + vall, arg, argl);
  66                 (*what)[vall + argl] = '\0';
  67         }
  68 }
  69 #define extend(x) extendbuf(&value, x)
  70
  71 #ifndef __clang_analyzer__
  72
  73 %}
  74
  75 ID              [A-Za-z0-9_]+
  76 OPERATOR        [!%&/\[\]{}=?:^|\*.;><~'\\+-]
  77 SPECIAL         [(),]
  78
  79 %pointer
  80 %option yylineno
  81 %option noyywrap
  82 %option noinput
  83 %option nounput
  84 %option outfile="lib/defun_lex.c"
  85 %option prefix="def_yy"
  86 %option 8bit
  87
  88 %s linestart
  89 %x comment
  90 %x linecomment
  91 %x preproc
  92 %x rstring
  93 %%
  94                                 BEGIN(linestart);
  95
  96 \n                              BEGIN(linestart);
  97
  98 <INITIAL,linestart,preproc>"/*" comment_link = YY_START; extend(yytext); BEGIN(comment);
  99 <comment>[^*\n]*                extend(yytext);
 100 <comment>"*"+[^*/\n]*           extend(yytext);
 101 <comment>\n                     extend(yytext);
 102 <comment>"*"+"/"                extend(yytext); BEGIN(comment_link); return COMMENT;
 103
 104 <INITIAL,linestart,preproc>"//" comment_link = YY_START; extend(yytext); BEGIN(linecomment);
 105 <linecomment>[^\n]*             extend(yytext);
 106 <linecomment>\n                 BEGIN((comment_link == INITIAL) ? linestart : comment_link); return COMMENT;
 107
 108 <linestart>#                    BEGIN(preproc);
 109 <preproc>\n                     BEGIN(INITIAL); return PREPROC;
 110 <preproc>[^\n\\]+               extend(yytext);
 111 <preproc>\\\n                   extend(yytext);
 112 <preproc>\\+[^\n]               extend(yytext);
 113
 114 [\"\']                          string_end = yytext[0]; extend(yytext); BEGIN(rstring);
 115 <rstring>[\"\']                 {
 116                                         extend(yytext);
 117                                         if (yytext[0] == string_end) {
 118                                                 BEGIN(INITIAL);
 119                                                 return STRING;
 120                                         }
 121                                 }
 122 <rstring>\\\n                   /* ignore */
 123 <rstring>\n                     {
 124                                         fprintf(stderr,
 125                                                 "%s:%d: string continues past the end of the line\n",
 126                                                 yyfilename, yylineno);
 127                                         free(value);
 128                                         value = NULL;
 129                                         BEGIN(INITIAL);
 130                                         return STRING;
 131                                 }
 132 <rstring>\\.                    extend(yytext);
 133 <rstring>[^\\\"\'\n]+           extend(yytext);
 134
 135 "DEFUN"                         value = strdup(yytext); return DEFUNNY;
 136 "DEFUN_NOSH"                    value = strdup(yytext); return DEFUNNY;
 137 "DEFUN_HIDDEN"                  value = strdup(yytext); return DEFUNNY;
 138 "DEFPY"                         value = strdup(yytext); return DEFUNNY;
 139 "DEFPY_NOSH"                    value = strdup(yytext); return DEFUNNY;
 140 "DEFPY_ATTR"                    value = strdup(yytext); return DEFUNNY;
 141 "DEFPY_HIDDEN"                  value = strdup(yytext); return DEFUNNY;
 142 "DEFPY_YANG"                    value = strdup(yytext); return DEFUNNY;
 143 "DEFPY_YANG_NOSH"               value = strdup(yytext); return DEFUNNY;
 144 "ALIAS"                         value = strdup(yytext); return DEFUNNY;
 145 "ALIAS_HIDDEN"                  value = strdup(yytext); return DEFUNNY;
 146 "install_element"               value = strdup(yytext); return INSTALL;
 147 "VTYSH_TARGETS"                 value = strdup(yytext); return AUXILIARY;
 148 "VTYSH_NODESWITCH"              value = strdup(yytext); return AUXILIARY;
 149
 150 [ \t\n]+                        /* ignore */
 151 \\                              /* ignore */
 152 {ID}                            BEGIN(INITIAL); value = strdup(yytext); return ID;
 153 {OPERATOR}                      BEGIN(INITIAL); value = strdup(yytext); return OPERATOR;
 154 {SPECIAL}                       BEGIN(INITIAL); value = strdup(yytext); return SPECIAL;
 155 .                               /* printf("-- '%s' in init\n", yytext); */ BEGIN(INITIAL); return yytext[0];
 156
 157 %%
 158
 159 #endif /* __clang_analyzer__ */
 160
 161 static int yylex_clr(char **retbuf)
 162 {
 163         int rv = def_yylex();
 164         *retbuf = value;
 165         value = NULL;
 166         return rv;
 167 }
 168
 169 static PyObject *get_args(const char *filename, int lineno)
 170 {
 171         PyObject *pyObj = PyList_New(0);
 172         PyObject *pyArg = NULL;
 173
 174         char *tval;
 175         int depth = 1;
 176         int token;
 177
 178         while ((token = yylex_clr(&tval)) != YY_NULL) {
 179                 if (token == SPECIAL && tval[0] == '(') {
 180                         free(tval);
 181                         break;
 182                 }
 183                 if (token == COMMENT) {
 184                         free(tval);
 185                         continue;
 186                 }
 187                 fprintf(stderr, "invalid input!\n");
 188                 exit(1);
 189         }
 190
 191         while ((token = yylex_clr(&tval)) != YY_NULL) {
 192                 if (token == COMMENT) {
 193                         free(tval);
 194                         continue;
 195                 }
 196                 if (token == PREPROC) {
 197                         free(tval);
 198                         Py_DECREF(pyObj);
 199                         return PyErr_Format(PyExc_ValueError,
 200                                         "%s:%d: cannot process CPP directive within argument list",
 201                                         filename, lineno);
 202                 }
 203                 if (token == SPECIAL) {
 204                         if (depth == 1 && (tval[0] == ',' || tval[0] == ')')) {
 205                                 if (pyArg)
 206                                         PyList_Append(pyObj, pyArg);
 207                                 pyArg = NULL;
 208                                 if (tval[0] == ')') {
 209                                         free(tval);
 210                                         break;
 211                                 }
 212                                 free(tval);
 213                                 continue;
 214                         }
 215                         if (tval[0] == '(')
 216                                 depth++;
 217                         if (tval[0] == ')')
 218                                 depth--;
 219                 }
 220                 if (!tval)
 221                         return PyErr_Format(PyExc_ValueError,
 222                                         "%s:%d: invalid token in DEFPY parameters",
 223                                         filename, lineno);
 224                 if (!pyArg)
 225                         pyArg = PyList_New(0);
 226                 PyList_Append(pyArg, PyUnicode_FromString(tval));
 227                 free(tval);
 228         }
 229         return pyObj;
 230 }
 231
 232 /* _clippy.parse() -- read a C file, returning a list of interesting bits.
 233  * note this ditches most of the actual C code. */
 234 PyObject *clippy_parse(PyObject *self, PyObject *args)
 235 {
 236         const char *filename;
 237         if (!PyArg_ParseTuple(args, "s", &filename))
 238                 return NULL;
 239
 240         FILE *fd = fopen(filename, "r");
 241         if (!fd)
 242                 return PyErr_SetFromErrnoWithFilename(PyExc_IOError, filename);
 243
 244         char *tval;
 245         int token;
 246         yyin = fd;
 247         value = NULL;
 248         yyfilename = filename;
 249
 250         PyObject *pyCont = PyDict_New();
 251         PyObject *pyObj = PyList_New(0);
 252         PyDict_SetItemString(pyCont, "filename", PyUnicode_FromString(filename));
 253         PyDict_SetItemString(pyCont, "data", pyObj);
 254
 255         while ((token = yylex_clr(&tval)) != YY_NULL) {
 256                 int lineno = yylineno;
 257                 PyObject *pyItem = NULL, *pyArgs;
 258                 switch (token) {
 259                 case DEFUNNY:
 260                 case INSTALL:
 261                 case AUXILIARY:
 262                         pyArgs = get_args(filename, lineno);
 263                         if (!pyArgs) {
 264                                 free(tval);
 265                                 Py_DECREF(pyCont);
 266                                 yyfilename = NULL;
 267                                 return NULL;
 268                         }
 269                         pyItem = PyDict_New();
 270                         PyDict_SetItemString(pyItem, "type", PyUnicode_FromString(tval));
 271                         PyDict_SetItemString(pyItem, "args", pyArgs);
 272                         break;
 273                 case COMMENT:
 274                         if (strncmp(tval, "//~", 3) && strncmp(tval, "/*~", 3))
 275                                 break;
 276                         pyItem = PyDict_New();
 277                         PyDict_SetItemString(pyItem, "type", PyUnicode_FromString("COMMENT"));
 278                         PyDict_SetItemString(pyItem, "line", PyUnicode_FromString(tval));
 279                         break;
 280                 case PREPROC:
 281                         pyItem = PyDict_New();
 282                         PyDict_SetItemString(pyItem, "type", PyUnicode_FromString("PREPROC"));
 283                         PyDict_SetItemString(pyItem, "line", PyUnicode_FromString(tval));
 284                         lineno--;
 285                         break;
 286                 }
 287                 if (pyItem) {
 288                         PyDict_SetItemString(pyItem, "lineno", PyLong_FromLong(lineno));
 289                         PyList_Append(pyObj, pyItem);
 290                 }
 291                 free(tval);
 292         }
 293         def_yylex_destroy();
 294         fclose(fd);
 295         yyfilename = NULL;
 296         return pyCont;
 297 }