lib/defun_lex.l

   1 /*
   2  * clippy (CLI preparator in python) C pseudo-lexer
   3  * Copyright (C) 2016-2017  David Lamparter for NetDEF, Inc.
   4  *
   5  * This program is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License as published by the Free
   7  * Software Foundation; either version 2 of the License, or (at your option)
   8  * any later version.
   9  *
  10  * This program is distributed in the hope that it will be useful, but WITHOUT
  11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  13  * more details.
  14  *
  15  * You should have received a copy of the GNU General Public License along
  16  * with this program; see the file COPYING; if not, write to the Free Software
  17  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
  18  */
  19
  20 /* This is just enough of a lexer to make rough sense of a C source file.
  21  * It handles C preprocessor directives, strings, and looks for FRR-specific
  22  * idioms (aka DEFUN).
  23  *
  24  * There is some preliminary support for documentation comments for DEFUNs.
  25  * They would look like this (note the ~):  (replace \ by /)
  26  *
  27  * \*~  documentation for foobar_cmd
  28  *  *   parameter does xyz
  29  *  *\
  30  * DEFUN(foobar_cmd, ...)
  31  *
  32  * This is intended for user documentation / command reference.  Don't put
  33  * code documentation in it.
  34  */
  35
  36 %top{
  37 #ifdef HAVE_CONFIG_H
  38 #include "config.h"
  39 #endif
  40 }
  41 %{
  42 /* ignore harmless bugs in old versions of flex */
  43 #pragma GCC diagnostic ignored "-Wsign-compare"
  44 #pragma GCC diagnostic ignored "-Wunused-value"
  45
  46 #include "config.h"
  47 #include <Python.h>
  48 #include <string.h>
  49 #include <stdlib.h>
  50
  51 #include "command_graph.h"
  52 #include "clippy.h"
  53
  54 #define ID              258
  55 #define PREPROC         259
  56 #define OPERATOR        260
  57 #define STRING          261
  58 #define COMMENT         262
  59 #define SPECIAL         263
  60
  61 #define DEFUNNY         270
  62 #define INSTALL         271
  63 #define AUXILIARY       272
  64
  65 int comment_link;
  66 char string_end;
  67
  68 char *value;
  69
  70 static void extendbuf(char **what, const char *arg)
  71 {
  72         if (!*what)
  73                 *what = strdup(arg);
  74         else {
  75                 size_t vall = strlen(*what), argl = strlen(arg);
  76                 *what = realloc(*what, vall + argl + 1);
  77                 memcpy(*what + vall, arg, argl);
  78                 (*what)[vall + argl] = '\0';
  79         }
  80 }
  81 #define extend(x) extendbuf(&value, x)
  82
  83 #ifndef __clang_analyzer__
  84
  85 %}
  86
  87 ID              [A-Za-z0-9_]+
  88 OPERATOR        [!%&/\[\]{}=?:^|\*.;><~'\\+-]
  89 SPECIAL         [(),]
  90
  91 %pointer
  92 %option yylineno
  93 %option noyywrap
  94 %option noinput
  95 %option nounput
  96 %option outfile="lib/defun_lex.c"
  97 %option prefix="def_yy"
  98 %option 8bit
  99
 100 %s linestart
 101 %x comment
 102 %x linecomment
 103 %x preproc
 104 %x rstring
 105 %%
 106                                 BEGIN(linestart);
 107
 108 \n                              BEGIN(linestart);
 109
 110 <INITIAL,linestart,preproc>"/*" comment_link = YY_START; extend(yytext); BEGIN(comment);
 111 <comment>[^*\n]*                extend(yytext);
 112 <comment>"*"+[^*/\n]*           extend(yytext);
 113 <comment>\n                     extend(yytext);
 114 <comment>"*"+"/"                extend(yytext); BEGIN(comment_link); return COMMENT;
 115
 116 <INITIAL,linestart,preproc>"//" comment_link = YY_START; extend(yytext); BEGIN(linecomment);
 117 <linecomment>[^\n]*             extend(yytext);
 118 <linecomment>\n                 BEGIN((comment_link == INITIAL) ? linestart : comment_link); return COMMENT;
 119
 120 <linestart>#                    BEGIN(preproc);
 121 <preproc>\n                     BEGIN(INITIAL); return PREPROC;
 122 <preproc>[^\n\\]+               extend(yytext);
 123 <preproc>\\\n                   extend(yytext);
 124 <preproc>\\+[^\n]               extend(yytext);
 125
 126 [\"\']                          string_end = yytext[0]; extend(yytext); BEGIN(rstring);
 127 <rstring>[\"\']                 {
 128                                         extend(yytext);
 129                                         if (yytext[0] == string_end) {
 130                                                 BEGIN(INITIAL);
 131                                                 return STRING;
 132                                         }
 133                                 }
 134 <rstring>\\\n                   /* ignore */
 135 <rstring>\\.                    extend(yytext);
 136 <rstring>[^\\\"\']+             extend(yytext);
 137
 138 "DEFUN"                         value = strdup(yytext); return DEFUNNY;
 139 "DEFUN_NOSH"                    value = strdup(yytext); return DEFUNNY;
 140 "DEFUN_HIDDEN"                  value = strdup(yytext); return DEFUNNY;
 141 "DEFPY"                         value = strdup(yytext); return DEFUNNY;
 142 "DEFPY_NOSH"                    value = strdup(yytext); return DEFUNNY;
 143 "DEFPY_ATTR"                    value = strdup(yytext); return DEFUNNY;
 144 "DEFPY_HIDDEN"                  value = strdup(yytext); return DEFUNNY;
 145 "DEFPY_YANG"                    value = strdup(yytext); return DEFUNNY;
 146 "DEFPY_YANG_NOSH"               value = strdup(yytext); return DEFUNNY;
 147 "ALIAS"                         value = strdup(yytext); return DEFUNNY;
 148 "ALIAS_HIDDEN"                  value = strdup(yytext); return DEFUNNY;
 149 "install_element"               value = strdup(yytext); return INSTALL;
 150 "VTYSH_TARGETS"                 value = strdup(yytext); return AUXILIARY;
 151 "VTYSH_NODESWITCH"              value = strdup(yytext); return AUXILIARY;
 152
 153 [ \t\n]+                        /* ignore */
 154 \\                              /* ignore */
 155 {ID}                            BEGIN(INITIAL); value = strdup(yytext); return ID;
 156 {OPERATOR}                      BEGIN(INITIAL); value = strdup(yytext); return OPERATOR;
 157 {SPECIAL}                       BEGIN(INITIAL); value = strdup(yytext); return SPECIAL;
 158 .                               /* printf("-- '%s' in init\n", yytext); */ BEGIN(INITIAL); return yytext[0];
 159
 160 %%
 161
 162 #endif /* __clang_analyzer__ */
 163
 164 static int yylex_clr(char **retbuf)
 165 {
 166         int rv = def_yylex();
 167         *retbuf = value;
 168         value = NULL;
 169         return rv;
 170 }
 171
 172 static PyObject *get_args(const char *filename, int lineno)
 173 {
 174         PyObject *pyObj = PyList_New(0);
 175         PyObject *pyArg = NULL;
 176
 177         char *tval;
 178         int depth = 1;
 179         int token;
 180
 181         while ((token = yylex_clr(&tval)) != YY_NULL) {
 182                 if (token == SPECIAL && tval[0] == '(') {
 183                         free(tval);
 184                         break;
 185                 }
 186                 if (token == COMMENT) {
 187                         free(tval);
 188                         continue;
 189                 }
 190                 fprintf(stderr, "invalid input!\n");
 191                 exit(1);
 192         }
 193
 194         while ((token = yylex_clr(&tval)) != YY_NULL) {
 195                 if (token == COMMENT) {
 196                         free(tval);
 197                         continue;
 198                 }
 199                 if (token == PREPROC) {
 200                         free(tval);
 201                         Py_DECREF(pyObj);
 202                         return PyErr_Format(PyExc_ValueError,
 203                                         "%s:%d: cannot process CPP directive within argument list",
 204                                         filename, lineno);
 205                 }
 206                 if (token == SPECIAL) {
 207                         if (depth == 1 && (tval[0] == ',' || tval[0] == ')')) {
 208                                 if (pyArg)
 209                                         PyList_Append(pyObj, pyArg);
 210                                 pyArg = NULL;
 211                                 if (tval[0] == ')') {
 212                                         free(tval);
 213                                         break;
 214                                 }
 215                                 free(tval);
 216                                 continue;
 217                         }
 218                         if (tval[0] == '(')
 219                                 depth++;
 220                         if (tval[0] == ')')
 221                                 depth--;
 222                 }
 223                 if (!pyArg)
 224                         pyArg = PyList_New(0);
 225                 PyList_Append(pyArg, PyUnicode_FromString(tval));
 226                 free(tval);
 227         }
 228         return pyObj;
 229 }
 230
 231 /* _clippy.parse() -- read a C file, returning a list of interesting bits.
 232  * note this ditches most of the actual C code. */
 233 PyObject *clippy_parse(PyObject *self, PyObject *args)
 234 {
 235         const char *filename;
 236         if (!PyArg_ParseTuple(args, "s", &filename))
 237                 return NULL;
 238
 239         FILE *fd = fopen(filename, "r");
 240         if (!fd)
 241                 return PyErr_SetFromErrnoWithFilename(PyExc_IOError, filename);
 242
 243         char *tval;
 244         int token;
 245         yyin = fd;
 246         value = NULL;
 247
 248         PyObject *pyCont = PyDict_New();
 249         PyObject *pyObj = PyList_New(0);
 250         PyDict_SetItemString(pyCont, "filename", PyUnicode_FromString(filename));
 251         PyDict_SetItemString(pyCont, "data", pyObj);
 252
 253         while ((token = yylex_clr(&tval)) != YY_NULL) {
 254                 int lineno = yylineno;
 255                 PyObject *pyItem = NULL, *pyArgs;
 256                 switch (token) {
 257                 case DEFUNNY:
 258                 case INSTALL:
 259                 case AUXILIARY:
 260                         pyArgs = get_args(filename, lineno);
 261                         if (!pyArgs) {
 262                                 free(tval);
 263                                 Py_DECREF(pyCont);
 264                                 return NULL;
 265                         }
 266                         pyItem = PyDict_New();
 267                         PyDict_SetItemString(pyItem, "type", PyUnicode_FromString(tval));
 268                         PyDict_SetItemString(pyItem, "args", pyArgs);
 269                         break;
 270                 case COMMENT:
 271                         if (strncmp(tval, "//~", 3) && strncmp(tval, "/*~", 3))
 272                                 break;
 273                         pyItem = PyDict_New();
 274                         PyDict_SetItemString(pyItem, "type", PyUnicode_FromString("COMMENT"));
 275                         PyDict_SetItemString(pyItem, "line", PyUnicode_FromString(tval));
 276                         break;
 277                 case PREPROC:
 278                         pyItem = PyDict_New();
 279                         PyDict_SetItemString(pyItem, "type", PyUnicode_FromString("PREPROC"));
 280                         PyDict_SetItemString(pyItem, "line", PyUnicode_FromString(tval));
 281                         lineno--;
 282                         break;
 283                 }
 284                 if (pyItem) {
 285                         PyDict_SetItemString(pyItem, "lineno", PyLong_FromLong(lineno));
 286                         PyList_Append(pyObj, pyItem);
 287                 }
 288                 free(tval);
 289         }
 290         def_yylex_destroy();
 291         fclose(fd);
 292         return pyCont;
 293 }