lib/defun_lex.l

   1 %{
   2 /*
   3  * clippy (CLI preparator in python) C pseudo-lexer
   4  * Copyright (C) 2016-2017  David Lamparter for NetDEF, Inc.
   5  *
   6  * This program is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU General Public License as published by the Free
   8  * Software Foundation; either version 2 of the License, or (at your option)
   9  * any later version.
  10  *
  11  * This program is distributed in the hope that it will be useful, but WITHOUT
  12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  14  * more details.
  15  *
  16  * You should have received a copy of the GNU General Public License along
  17  * with this program; see the file COPYING; if not, write to the Free Software
  18  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
  19  */
  20
  21 /* This is just enough of a lexer to make rough sense of a C source file.
  22  * It handles C preprocessor directives, strings, and looks for FRR-specific
  23  * idioms (aka DEFUN).
  24  *
  25  * There is some preliminary support for documentation comments for DEFUNs.
  26  * They would look like this (note the ~):  (replace \ by /)
  27  *
  28  * \*~  documentation for foobar_cmd
  29  *  *   parameter does xyz
  30  *  *\
  31  * DEFUN(foobar_cmd, ...)
  32  *
  33  * This is intended for user documentation / command reference.  Don't put
  34  * code documentation in it.
  35  */
  36
  37 /* ignore harmless bugs in old versions of flex */
  38 #pragma GCC diagnostic ignored "-Wsign-compare"
  39 #pragma GCC diagnostic ignored "-Wunused-value"
  40
  41 #include "config.h"
  42 #include <Python.h>
  43 #include <string.h>
  44 #include <stdlib.h>
  45
  46 #include "command_graph.h"
  47 #include "clippy.h"
  48
  49 #define ID              258
  50 #define PREPROC         259
  51 #define OPERATOR        260
  52 #define STRING          261
  53 #define COMMENT         262
  54 #define SPECIAL         263
  55
  56 #define DEFUNNY         270
  57 #define INSTALL         271
  58 #define AUXILIARY       272
  59
  60 int comment_link;
  61 char string_end;
  62
  63 char *value;
  64
  65 static void extendbuf(char **what, const char *arg)
  66 {
  67         if (!*what)
  68                 *what = strdup(arg);
  69         else {
  70                 size_t vall = strlen(*what), argl = strlen(arg);
  71                 *what = realloc(*what, vall + argl + 1);
  72                 memcpy(*what + vall, arg, argl);
  73                 (*what)[vall + argl] = '\0';
  74         }
  75 }
  76 #define extend(x) extendbuf(&value, x)
  77
  78 %}
  79
  80 ID              [A-Za-z0-9_]+
  81 OPERATOR        [!%&/\[\]{}=?:^|\*.;><~'\\+-]
  82 SPECIAL         [(),]
  83
  84 %pointer
  85 %option yylineno
  86 %option noyywrap
  87 %option noinput
  88 %option nounput
  89 %option outfile="lib/defun_lex.c"
  90 %option prefix="def_yy"
  91 %option 8bit
  92
  93 %s linestart
  94 %x comment
  95 %x linecomment
  96 %x preproc
  97 %x rstring
  98 %%
  99                                 BEGIN(linestart);
 100
 101 \n                              BEGIN(linestart);
 102
 103 <INITIAL,linestart,preproc>"/*" comment_link = YY_START; extend(yytext); BEGIN(comment);
 104 <comment>[^*\n]*                extend(yytext);
 105 <comment>"*"+[^*/\n]*           extend(yytext);
 106 <comment>\n                     extend(yytext);
 107 <comment>"*"+"/"                extend(yytext); BEGIN(comment_link); return COMMENT;
 108
 109 <INITIAL,linestart,preproc>"//" comment_link = YY_START; extend(yytext); BEGIN(linecomment);
 110 <linecomment>[^\n]*             extend(yytext);
 111 <linecomment>\n                 BEGIN((comment_link == INITIAL) ? linestart : comment_link); return COMMENT;
 112
 113 <linestart>#                    BEGIN(preproc);
 114 <preproc>\n                     BEGIN(INITIAL); return PREPROC;
 115 <preproc>[^\n\\]+               extend(yytext);
 116 <preproc>\\\n                   extend(yytext);
 117 <preproc>\\+[^\n]               extend(yytext);
 118
 119 [\"\']                          string_end = yytext[0]; extend(yytext); BEGIN(rstring);
 120 <rstring>[\"\']                 {
 121                                         extend(yytext);
 122                                         if (yytext[0] == string_end) {
 123                                                 BEGIN(INITIAL);
 124                                                 return STRING;
 125                                         }
 126                                 }
 127 <rstring>\\\n                   /* ignore */
 128 <rstring>\\.                    extend(yytext);
 129 <rstring>[^\\\"\']+             extend(yytext);
 130
 131 "DEFUN"                         value = strdup(yytext); return DEFUNNY;
 132 "DEFUN_NOSH"                    value = strdup(yytext); return DEFUNNY;
 133 "DEFUN_HIDDEN"                  value = strdup(yytext); return DEFUNNY;
 134 "DEFPY"                         value = strdup(yytext); return DEFUNNY;
 135 "ALIAS"                         value = strdup(yytext); return DEFUNNY;
 136 "ALIAS_HIDDEN"                  value = strdup(yytext); return DEFUNNY;
 137 "install_element"               value = strdup(yytext); return INSTALL;
 138 "VTYSH_TARGETS"                 value = strdup(yytext); return AUXILIARY;
 139 "VTYSH_NODESWITCH"              value = strdup(yytext); return AUXILIARY;
 140
 141 [ \t\n]+                        /* ignore */
 142 \\                              /* ignore */
 143 {ID}                            BEGIN(INITIAL); value = strdup(yytext); return ID;
 144 {OPERATOR}                      BEGIN(INITIAL); value = strdup(yytext); return OPERATOR;
 145 {SPECIAL}                       BEGIN(INITIAL); value = strdup(yytext); return SPECIAL;
 146 .                               /* printf("-- '%s' in init\n", yytext); */ BEGIN(INITIAL); return yytext[0];
 147
 148 %%
 149
 150 static int yylex_clr(char **retbuf)
 151 {
 152         int rv = def_yylex();
 153         *retbuf = value;
 154         value = NULL;
 155         return rv;
 156 }
 157
 158 static PyObject *get_args(void)
 159 {
 160         PyObject *pyObj = PyList_New(0);
 161         PyObject *pyArg = NULL;
 162
 163         char *tval;
 164         int depth = 1;
 165         int token;
 166
 167         while ((token = yylex_clr(&tval)) != YY_NULL) {
 168                 if (token == SPECIAL && tval[0] == '(') {
 169                         free(tval);
 170                         break;
 171                 }
 172                 if (token == COMMENT) {
 173                         free(tval);
 174                         continue;
 175                 }
 176                 fprintf(stderr, "invalid input!\n");
 177                 exit(1);
 178         }
 179
 180         while ((token = yylex_clr(&tval)) != YY_NULL) {
 181                 if (token == COMMENT) {
 182                         free(tval);
 183                         continue;
 184                 }
 185                 if (token == SPECIAL) {
 186                         if (depth == 1 && (tval[0] == ',' || tval[0] == ')')) {
 187                                 if (pyArg)
 188                                         PyList_Append(pyObj, pyArg);
 189                                 pyArg = NULL;
 190                                 if (tval[0] == ')') {
 191                                         free(tval);
 192                                         break;
 193                                 }
 194                                 free(tval);
 195                                 continue;
 196                         }
 197                         if (tval[0] == '(')
 198                                 depth++;
 199                         if (tval[0] == ')')
 200                                 depth--;
 201                 }
 202                 if (!pyArg)
 203                         pyArg = PyList_New(0);
 204                 PyList_Append(pyArg, PyUnicode_FromString(tval));
 205                 free(tval);
 206         }
 207         return pyObj;
 208 }
 209
 210 /* _clippy.parse() -- read a C file, returning a list of interesting bits.
 211  * note this ditches most of the actual C code. */
 212 PyObject *clippy_parse(PyObject *self, PyObject *args)
 213 {
 214         const char *filename;
 215         if (!PyArg_ParseTuple(args, "s", &filename))
 216                 return NULL;
 217
 218         FILE *fd = fopen(filename, "r");
 219         if (!fd)
 220                 return PyErr_SetFromErrnoWithFilename(PyExc_IOError, filename);
 221
 222         char *tval;
 223         int token;
 224         yyin = fd;
 225         value = NULL;
 226
 227         PyObject *pyCont = PyDict_New();
 228         PyObject *pyObj = PyList_New(0);
 229         PyDict_SetItemString(pyCont, "filename", PyUnicode_FromString(filename));
 230         PyDict_SetItemString(pyCont, "data", pyObj);
 231
 232         while ((token = yylex_clr(&tval)) != YY_NULL) {
 233                 int lineno = yylineno;
 234                 PyObject *pyItem = NULL, *pyArgs;
 235                 switch (token) {
 236                 case DEFUNNY:
 237                 case INSTALL:
 238                 case AUXILIARY:
 239                         pyArgs = get_args();
 240                         pyItem = PyDict_New();
 241                         PyDict_SetItemString(pyItem, "type", PyUnicode_FromString(tval));
 242                         PyDict_SetItemString(pyItem, "args", pyArgs);
 243                         break;
 244                 case COMMENT:
 245                         if (strncmp(tval, "//~", 3) && strncmp(tval, "/*~", 3))
 246                                 break;
 247                         pyItem = PyDict_New();
 248                         PyDict_SetItemString(pyItem, "type", PyUnicode_FromString("COMMENT"));
 249                         PyDict_SetItemString(pyItem, "line", PyUnicode_FromString(tval));
 250                         break;
 251                 case PREPROC:
 252                         pyItem = PyDict_New();
 253                         PyDict_SetItemString(pyItem, "type", PyUnicode_FromString("PREPROC"));
 254                         PyDict_SetItemString(pyItem, "line", PyUnicode_FromString(tval));
 255                         break;
 256                 }
 257                 if (pyItem) {
 258                         PyDict_SetItemString(pyItem, "lineno", PyLong_FromLong(lineno));
 259                         PyList_Append(pyObj, pyItem);
 260                 }
 261                 free(tval);
 262         }
 263         def_yylex_destroy();
 264         fclose(fd);
 265         return pyCont;
 266 }