lib/defun_lex.l

   1 /*
   2  * clippy (CLI preparator in python) C pseudo-lexer
   3  * Copyright (C) 2016-2017  David Lamparter for NetDEF, Inc.
   4  *
   5  * This program is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License as published by the Free
   7  * Software Foundation; either version 2 of the License, or (at your option)
   8  * any later version.
   9  *
  10  * This program is distributed in the hope that it will be useful, but WITHOUT
  11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  13  * more details.
  14  *
  15  * You should have received a copy of the GNU General Public License along
  16  * with this program; see the file COPYING; if not, write to the Free Software
  17  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
  18  */
  19
  20 /* This is just enough of a lexer to make rough sense of a C source file.
  21  * It handles C preprocessor directives, strings, and looks for FRR-specific
  22  * idioms (aka DEFUN).
  23  *
  24  * There is some preliminary support for documentation comments for DEFUNs.
  25  * They would look like this (note the ~):  (replace \ by /)
  26  *
  27  * \*~  documentation for foobar_cmd
  28  *  *   parameter does xyz
  29  *  *\
  30  * DEFUN(foobar_cmd, ...)
  31  *
  32  * This is intended for user documentation / command reference.  Don't put
  33  * code documentation in it.
  34  */
  35
  36 %top{
  37 #ifdef HAVE_CONFIG_H
  38 #include "config.h"
  39 #endif
  40 }
  41 %{
  42 /* ignore harmless bugs in old versions of flex */
  43 #pragma GCC diagnostic ignored "-Wsign-compare"
  44 #pragma GCC diagnostic ignored "-Wunused-value"
  45
  46 #include "config.h"
  47 #include <Python.h>
  48 #include <string.h>
  49 #include <stdlib.h>
  50
  51 #include "command_graph.h"
  52 #include "clippy.h"
  53
  54 #define ID              258
  55 #define PREPROC         259
  56 #define OPERATOR        260
  57 #define STRING          261
  58 #define COMMENT         262
  59 #define SPECIAL         263
  60
  61 #define DEFUNNY         270
  62 #define INSTALL         271
  63 #define AUXILIARY       272
  64
  65 int comment_link;
  66 char string_end;
  67
  68 char *value;
  69
  70 static void extendbuf(char **what, const char *arg)
  71 {
  72         if (!*what)
  73                 *what = strdup(arg);
  74         else {
  75                 size_t vall = strlen(*what), argl = strlen(arg);
  76                 *what = realloc(*what, vall + argl + 1);
  77                 memcpy(*what + vall, arg, argl);
  78                 (*what)[vall + argl] = '\0';
  79         }
  80 }
  81 #define extend(x) extendbuf(&value, x)
  82
  83 %}
  84
  85 ID              [A-Za-z0-9_]+
  86 OPERATOR        [!%&/\[\]{}=?:^|\*.;><~'\\+-]
  87 SPECIAL         [(),]
  88
  89 %pointer
  90 %option yylineno
  91 %option noyywrap
  92 %option noinput
  93 %option nounput
  94 %option outfile="lib/defun_lex.c"
  95 %option prefix="def_yy"
  96 %option 8bit
  97
  98 %s linestart
  99 %x comment
 100 %x linecomment
 101 %x preproc
 102 %x rstring
 103 %%
 104                                 BEGIN(linestart);
 105
 106 \n                              BEGIN(linestart);
 107
 108 <INITIAL,linestart,preproc>"/*" comment_link = YY_START; extend(yytext); BEGIN(comment);
 109 <comment>[^*\n]*                extend(yytext);
 110 <comment>"*"+[^*/\n]*           extend(yytext);
 111 <comment>\n                     extend(yytext);
 112 <comment>"*"+"/"                extend(yytext); BEGIN(comment_link); return COMMENT;
 113
 114 <INITIAL,linestart,preproc>"//" comment_link = YY_START; extend(yytext); BEGIN(linecomment);
 115 <linecomment>[^\n]*             extend(yytext);
 116 <linecomment>\n                 BEGIN((comment_link == INITIAL) ? linestart : comment_link); return COMMENT;
 117
 118 <linestart>#                    BEGIN(preproc);
 119 <preproc>\n                     BEGIN(INITIAL); return PREPROC;
 120 <preproc>[^\n\\]+               extend(yytext);
 121 <preproc>\\\n                   extend(yytext);
 122 <preproc>\\+[^\n]               extend(yytext);
 123
 124 [\"\']                          string_end = yytext[0]; extend(yytext); BEGIN(rstring);
 125 <rstring>[\"\']                 {
 126                                         extend(yytext);
 127                                         if (yytext[0] == string_end) {
 128                                                 BEGIN(INITIAL);
 129                                                 return STRING;
 130                                         }
 131                                 }
 132 <rstring>\\\n                   /* ignore */
 133 <rstring>\\.                    extend(yytext);
 134 <rstring>[^\\\"\']+             extend(yytext);
 135
 136 "DEFUN"                         value = strdup(yytext); return DEFUNNY;
 137 "DEFUN_NOSH"                    value = strdup(yytext); return DEFUNNY;
 138 "DEFUN_HIDDEN"                  value = strdup(yytext); return DEFUNNY;
 139 "DEFPY"                         value = strdup(yytext); return DEFUNNY;
 140 "DEFPY_NOSH"                    value = strdup(yytext); return DEFUNNY;
 141 "DEFPY_ATTR"                    value = strdup(yytext); return DEFUNNY;
 142 "DEFPY_HIDDEN"                  value = strdup(yytext); return DEFUNNY;
 143 "ALIAS"                         value = strdup(yytext); return DEFUNNY;
 144 "ALIAS_HIDDEN"                  value = strdup(yytext); return DEFUNNY;
 145 "install_element"               value = strdup(yytext); return INSTALL;
 146 "VTYSH_TARGETS"                 value = strdup(yytext); return AUXILIARY;
 147 "VTYSH_NODESWITCH"              value = strdup(yytext); return AUXILIARY;
 148
 149 [ \t\n]+                        /* ignore */
 150 \\                              /* ignore */
 151 {ID}                            BEGIN(INITIAL); value = strdup(yytext); return ID;
 152 {OPERATOR}                      BEGIN(INITIAL); value = strdup(yytext); return OPERATOR;
 153 {SPECIAL}                       BEGIN(INITIAL); value = strdup(yytext); return SPECIAL;
 154 .                               /* printf("-- '%s' in init\n", yytext); */ BEGIN(INITIAL); return yytext[0];
 155
 156 %%
 157
 158 static int yylex_clr(char **retbuf)
 159 {
 160         int rv = def_yylex();
 161         *retbuf = value;
 162         value = NULL;
 163         return rv;
 164 }
 165
 166 static PyObject *get_args(void)
 167 {
 168         PyObject *pyObj = PyList_New(0);
 169         PyObject *pyArg = NULL;
 170
 171         char *tval;
 172         int depth = 1;
 173         int token;
 174
 175         while ((token = yylex_clr(&tval)) != YY_NULL) {
 176                 if (token == SPECIAL && tval[0] == '(') {
 177                         free(tval);
 178                         break;
 179                 }
 180                 if (token == COMMENT) {
 181                         free(tval);
 182                         continue;
 183                 }
 184                 fprintf(stderr, "invalid input!\n");
 185                 exit(1);
 186         }
 187
 188         while ((token = yylex_clr(&tval)) != YY_NULL) {
 189                 if (token == COMMENT) {
 190                         free(tval);
 191                         continue;
 192                 }
 193                 if (token == SPECIAL) {
 194                         if (depth == 1 && (tval[0] == ',' || tval[0] == ')')) {
 195                                 if (pyArg)
 196                                         PyList_Append(pyObj, pyArg);
 197                                 pyArg = NULL;
 198                                 if (tval[0] == ')') {
 199                                         free(tval);
 200                                         break;
 201                                 }
 202                                 free(tval);
 203                                 continue;
 204                         }
 205                         if (tval[0] == '(')
 206                                 depth++;
 207                         if (tval[0] == ')')
 208                                 depth--;
 209                 }
 210                 if (!pyArg)
 211                         pyArg = PyList_New(0);
 212                 PyList_Append(pyArg, PyUnicode_FromString(tval));
 213                 free(tval);
 214         }
 215         return pyObj;
 216 }
 217
 218 /* _clippy.parse() -- read a C file, returning a list of interesting bits.
 219  * note this ditches most of the actual C code. */
 220 PyObject *clippy_parse(PyObject *self, PyObject *args)
 221 {
 222         const char *filename;
 223         if (!PyArg_ParseTuple(args, "s", &filename))
 224                 return NULL;
 225
 226         FILE *fd = fopen(filename, "r");
 227         if (!fd)
 228                 return PyErr_SetFromErrnoWithFilename(PyExc_IOError, filename);
 229
 230         char *tval;
 231         int token;
 232         yyin = fd;
 233         value = NULL;
 234
 235         PyObject *pyCont = PyDict_New();
 236         PyObject *pyObj = PyList_New(0);
 237         PyDict_SetItemString(pyCont, "filename", PyUnicode_FromString(filename));
 238         PyDict_SetItemString(pyCont, "data", pyObj);
 239
 240         while ((token = yylex_clr(&tval)) != YY_NULL) {
 241                 int lineno = yylineno;
 242                 PyObject *pyItem = NULL, *pyArgs;
 243                 switch (token) {
 244                 case DEFUNNY:
 245                 case INSTALL:
 246                 case AUXILIARY:
 247                         pyArgs = get_args();
 248                         pyItem = PyDict_New();
 249                         PyDict_SetItemString(pyItem, "type", PyUnicode_FromString(tval));
 250                         PyDict_SetItemString(pyItem, "args", pyArgs);
 251                         break;
 252                 case COMMENT:
 253                         if (strncmp(tval, "//~", 3) && strncmp(tval, "/*~", 3))
 254                                 break;
 255                         pyItem = PyDict_New();
 256                         PyDict_SetItemString(pyItem, "type", PyUnicode_FromString("COMMENT"));
 257                         PyDict_SetItemString(pyItem, "line", PyUnicode_FromString(tval));
 258                         break;
 259                 case PREPROC:
 260                         pyItem = PyDict_New();
 261                         PyDict_SetItemString(pyItem, "type", PyUnicode_FromString("PREPROC"));
 262                         PyDict_SetItemString(pyItem, "line", PyUnicode_FromString(tval));
 263                         break;
 264                 }
 265                 if (pyItem) {
 266                         PyDict_SetItemString(pyItem, "lineno", PyLong_FromLong(lineno));
 267                         PyList_Append(pyObj, pyItem);
 268                 }
 269                 free(tval);
 270         }
 271         def_yylex_destroy();
 272         fclose(fd);
 273         return pyCont;
 274 }