lib/defun_lex.l

   1 %{
   2 /*
   3  * clippy (CLI preparator in python) C pseudo-lexer
   4  * Copyright (C) 2016-2017  David Lamparter for NetDEF, Inc.
   5  *
   6  * This program is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU General Public License as published by the Free
   8  * Software Foundation; either version 2 of the License, or (at your option)
   9  * any later version.
  10  *
  11  * This program is distributed in the hope that it will be useful, but WITHOUT
  12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  14  * more details.
  15  *
  16  * You should have received a copy of the GNU General Public License along
  17  * with this program; see the file COPYING; if not, write to the Free Software
  18  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
  19  */
  20
  21 /* This is just enough of a lexer to make rough sense of a C source file.
  22  * It handles C preprocessor directives, strings, and looks for FRR-specific
  23  * idioms (aka DEFUN).
  24  *
  25  * There is some preliminary support for documentation comments for DEFUNs.
  26  * They would look like this (note the ~):  (replace \ by /)
  27  *
  28  * \*~  documentation for foobar_cmd
  29  *  *   parameter does xyz
  30  *  *\
  31  * DEFUN(foobar_cmd, ...)
  32  *
  33  * This is intended for user documentation / command reference.  Don't put
  34  * code documentation in it.
  35  */
  36
  37 /* ignore harmless bug in old versions of flex */
  38 #pragma GCC diagnostic ignored "-Wsign-compare"
  39
  40 #include "config.h"
  41 #include <Python.h>
  42 #include <string.h>
  43 #include <stdlib.h>
  44
  45 #include "command_graph.h"
  46 #include "clippy.h"
  47
  48 #define ID              258
  49 #define PREPROC         259
  50 #define OPERATOR        260
  51 #define STRING          261
  52 #define COMMENT         262
  53 #define SPECIAL         263
  54
  55 #define DEFUNNY         270
  56 #define INSTALL         271
  57 #define AUXILIARY       272
  58
  59 int comment_link;
  60 char string_end;
  61
  62 char *value;
  63
  64 static void extendbuf(char **what, const char *arg)
  65 {
  66         if (!*what)
  67                 *what = strdup(arg);
  68         else {
  69                 size_t vall = strlen(*what), argl = strlen(arg);
  70                 *what = realloc(*what, vall + argl + 1);
  71                 memcpy(*what + vall, arg, argl);
  72                 (*what)[vall + argl] = '\0';
  73         }
  74 }
  75 #define extend(x) extendbuf(&value, x)
  76
  77 %}
  78
  79 ID              [A-Za-z0-9_]+
  80 OPERATOR        [!%&/\[\]{}=?:^|\*.;><~'\\+-]
  81 SPECIAL         [(),]
  82
  83 %pointer
  84 %option yylineno
  85 %option noyywrap
  86 %option noinput
  87 %option nounput
  88 %option outfile="lib/defun_lex.c"
  89 %option prefix="def_yy"
  90 %option 8bit
  91
  92 %s linestart
  93 %x comment
  94 %x linecomment
  95 %x preproc
  96 %x rstring
  97 %%
  98                                 BEGIN(linestart);
  99
 100 \n                              BEGIN(linestart);
 101
 102 <INITIAL,linestart,preproc>"/*" comment_link = YY_START; extend(yytext); BEGIN(comment);
 103 <comment>[^*\n]*                extend(yytext);
 104 <comment>"*"+[^*/\n]*           extend(yytext);
 105 <comment>\n                     extend(yytext);
 106 <comment>"*"+"/"                extend(yytext); BEGIN(comment_link); return COMMENT;
 107
 108 <INITIAL,linestart,preproc>"//" comment_link = YY_START; extend(yytext); BEGIN(linecomment);
 109 <linecomment>[^\n]*             extend(yytext);
 110 <linecomment>\n                 BEGIN((comment_link == INITIAL) ? linestart : comment_link); return COMMENT;
 111
 112 <linestart>#                    BEGIN(preproc);
 113 <preproc>\n                     BEGIN(INITIAL); return PREPROC;
 114 <preproc>[^\n\\]+               extend(yytext);
 115 <preproc>\\\n                   extend(yytext);
 116 <preproc>\\+[^\n]               extend(yytext);
 117
 118 [\"\']                          string_end = yytext[0]; extend(yytext); BEGIN(rstring);
 119 <rstring>[\"\']                 {
 120                                         extend(yytext);
 121                                         if (yytext[0] == string_end) {
 122                                                 BEGIN(INITIAL);
 123                                                 return STRING;
 124                                         }
 125                                 }
 126 <rstring>\\\n                   /* ignore */
 127 <rstring>\\.                    extend(yytext);
 128 <rstring>[^\\\"\']+             extend(yytext);
 129
 130 "DEFUN"                         value = strdup(yytext); return DEFUNNY;
 131 "DEFUN_NOSH"                    value = strdup(yytext); return DEFUNNY;
 132 "DEFUN_HIDDEN"                  value = strdup(yytext); return DEFUNNY;
 133 "DEFPY"                         value = strdup(yytext); return DEFUNNY;
 134 "ALIAS"                         value = strdup(yytext); return DEFUNNY;
 135 "ALIAS_HIDDEN"                  value = strdup(yytext); return DEFUNNY;
 136 "install_element"               value = strdup(yytext); return INSTALL;
 137 "VTYSH_TARGETS"                 value = strdup(yytext); return AUXILIARY;
 138 "VTYSH_NODESWITCH"              value = strdup(yytext); return AUXILIARY;
 139
 140 [ \t\n]+                        /* ignore */
 141 \\                              /* ignore */
 142 {ID}                            BEGIN(INITIAL); value = strdup(yytext); return ID;
 143 {OPERATOR}                      BEGIN(INITIAL); value = strdup(yytext); return OPERATOR;
 144 {SPECIAL}                       BEGIN(INITIAL); value = strdup(yytext); return SPECIAL;
 145 .                               /* printf("-- '%s' in init\n", yytext); */ BEGIN(INITIAL); return yytext[0];
 146
 147 %%
 148
 149 static int yylex_clr(char **retbuf)
 150 {
 151         int rv = def_yylex();
 152         *retbuf = value;
 153         value = NULL;
 154         return rv;
 155 }
 156
 157 static PyObject *get_args(void)
 158 {
 159         PyObject *pyObj = PyList_New(0);
 160         PyObject *pyArg = NULL;
 161
 162         char *tval;
 163         int depth = 1;
 164         int token;
 165
 166         while ((token = yylex_clr(&tval)) != YY_NULL) {
 167                 if (token == SPECIAL && tval[0] == '(') {
 168                         free(tval);
 169                         break;
 170                 }
 171                 if (token == COMMENT) {
 172                         free(tval);
 173                         continue;
 174                 }
 175                 fprintf(stderr, "invalid input!\n");
 176                 exit(1);
 177         }
 178
 179         while ((token = yylex_clr(&tval)) != YY_NULL) {
 180                 if (token == COMMENT) {
 181                         free(tval);
 182                         continue;
 183                 }
 184                 if (token == SPECIAL) {
 185                         if (depth == 1 && (tval[0] == ',' || tval[0] == ')')) {
 186                                 if (pyArg)
 187                                         PyList_Append(pyObj, pyArg);
 188                                 pyArg = NULL;
 189                                 if (tval[0] == ')') {
 190                                         free(tval);
 191                                         break;
 192                                 }
 193                                 free(tval);
 194                                 continue;
 195                         }
 196                         if (tval[0] == '(')
 197                                 depth++;
 198                         if (tval[0] == ')')
 199                                 depth--;
 200                 }
 201                 if (!pyArg)
 202                         pyArg = PyList_New(0);
 203                 PyList_Append(pyArg, PyUnicode_FromString(tval));
 204                 free(tval);
 205         }
 206         return pyObj;
 207 }
 208
 209 /* _clippy.parse() -- read a C file, returning a list of interesting bits.
 210  * note this ditches most of the actual C code. */
 211 PyObject *clippy_parse(PyObject *self, PyObject *args)
 212 {
 213         const char *filename;
 214         if (!PyArg_ParseTuple(args, "s", &filename))
 215                 return NULL;
 216
 217         FILE *fd = fopen(filename, "r");
 218         if (!fd)
 219                 return PyErr_SetFromErrnoWithFilename(PyExc_IOError, filename);
 220
 221         char *tval;
 222         int token;
 223         yyin = fd;
 224         value = NULL;
 225
 226         PyObject *pyCont = PyDict_New();
 227         PyObject *pyObj = PyList_New(0);
 228         PyDict_SetItemString(pyCont, "filename", PyUnicode_FromString(filename));
 229         PyDict_SetItemString(pyCont, "data", pyObj);
 230
 231         while ((token = yylex_clr(&tval)) != YY_NULL) {
 232                 int lineno = yylineno;
 233                 PyObject *pyItem = NULL, *pyArgs;
 234                 switch (token) {
 235                 case DEFUNNY:
 236                 case INSTALL:
 237                 case AUXILIARY:
 238                         pyArgs = get_args();
 239                         pyItem = PyDict_New();
 240                         PyDict_SetItemString(pyItem, "type", PyUnicode_FromString(tval));
 241                         PyDict_SetItemString(pyItem, "args", pyArgs);
 242                         break;
 243                 case COMMENT:
 244                         if (strncmp(tval, "//~", 3) && strncmp(tval, "/*~", 3))
 245                                 break;
 246                         pyItem = PyDict_New();
 247                         PyDict_SetItemString(pyItem, "type", PyUnicode_FromString("COMMENT"));
 248                         PyDict_SetItemString(pyItem, "line", PyUnicode_FromString(tval));
 249                         break;
 250                 case PREPROC:
 251                         pyItem = PyDict_New();
 252                         PyDict_SetItemString(pyItem, "type", PyUnicode_FromString("PREPROC"));
 253                         PyDict_SetItemString(pyItem, "line", PyUnicode_FromString(tval));
 254                         break;
 255                 }
 256                 if (pyItem) {
 257                         PyDict_SetItemString(pyItem, "lineno", PyLong_FromLong(lineno));
 258                         PyList_Append(pyObj, pyItem);
 259                 }
 260                 free(tval);
 261         }
 262         def_yylex_destroy();
 263         fclose(fd);
 264         return pyCont;
 265 }