lib/defun_lex.l

   1 %{
   2 /*
   3  * clippy (CLI preparator in python) C pseudo-lexer
   4  * Copyright (C) 2016-2017  David Lamparter for NetDEF, Inc.
   5  *
   6  * This program is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU General Public License as published by the Free
   8  * Software Foundation; either version 2 of the License, or (at your option)
   9  * any later version.
  10  *
  11  * This program is distributed in the hope that it will be useful, but WITHOUT
  12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  14  * more details.
  15  *
  16  * You should have received a copy of the GNU General Public License along
  17  * with this program; see the file COPYING; if not, write to the Free Software
  18  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
  19  */
  20
  21 /* This is just enough of a lexer to make rough sense of a C source file.
  22  * It handles C preprocessor directives, strings, and looks for FRR-specific
  23  * idioms (aka DEFUN).
  24  *
  25  * There is some preliminary support for documentation comments for DEFUNs.
  26  * They would look like this (note the ~):  (replace \ by /)
  27  *
  28  * \*~  documentation for foobar_cmd
  29  *  *   parameter does xyz
  30  *  *\
  31  * DEFUN(foobar_cmd, ...)
  32  *
  33  * This is intended for user documentation / command reference.  Don't put
  34  * code documentation in it.
  35  */
  36
  37 /* ignore harmless bugs in old versions of flex */
  38 #pragma GCC diagnostic ignored "-Wsign-compare"
  39 #pragma GCC diagnostic ignored "-Wunused-value"
  40
  41 #include "config.h"
  42 #include <Python.h>
  43 #include <string.h>
  44 #include <stdlib.h>
  45
  46 #include "command_graph.h"
  47 #include "clippy.h"
  48
  49 #define ID              258
  50 #define PREPROC         259
  51 #define OPERATOR        260
  52 #define STRING          261
  53 #define COMMENT         262
  54 #define SPECIAL         263
  55
  56 #define DEFUNNY         270
  57 #define INSTALL         271
  58 #define AUXILIARY       272
  59
  60 int comment_link;
  61 char string_end;
  62
  63 char *value;
  64
  65 static void extendbuf(char **what, const char *arg)
  66 {
  67         if (!*what)
  68                 *what = strdup(arg);
  69         else {
  70                 size_t vall = strlen(*what), argl = strlen(arg);
  71                 *what = realloc(*what, vall + argl + 1);
  72                 memcpy(*what + vall, arg, argl);
  73                 (*what)[vall + argl] = '\0';
  74         }
  75 }
  76 #define extend(x) extendbuf(&value, x)
  77
  78 %}
  79
  80 ID              [A-Za-z0-9_]+
  81 OPERATOR        [!%&/\[\]{}=?:^|\*.;><~'\\+-]
  82 SPECIAL         [(),]
  83
  84 %pointer
  85 %option yylineno
  86 %option noyywrap
  87 %option noinput
  88 %option nounput
  89 %option outfile="lib/defun_lex.c"
  90 %option prefix="def_yy"
  91 %option 8bit
  92
  93 %s linestart
  94 %x comment
  95 %x linecomment
  96 %x preproc
  97 %x rstring
  98 %%
  99                                 BEGIN(linestart);
 100
 101 \n                              BEGIN(linestart);
 102
 103 <INITIAL,linestart,preproc>"/*" comment_link = YY_START; extend(yytext); BEGIN(comment);
 104 <comment>[^*\n]*                extend(yytext);
 105 <comment>"*"+[^*/\n]*           extend(yytext);
 106 <comment>\n                     extend(yytext);
 107 <comment>"*"+"/"                extend(yytext); BEGIN(comment_link); return COMMENT;
 108
 109 <INITIAL,linestart,preproc>"//" comment_link = YY_START; extend(yytext); BEGIN(linecomment);
 110 <linecomment>[^\n]*             extend(yytext);
 111 <linecomment>\n                 BEGIN((comment_link == INITIAL) ? linestart : comment_link); return COMMENT;
 112
 113 <linestart>#                    BEGIN(preproc);
 114 <preproc>\n                     BEGIN(INITIAL); return PREPROC;
 115 <preproc>[^\n\\]+               extend(yytext);
 116 <preproc>\\\n                   extend(yytext);
 117 <preproc>\\+[^\n]               extend(yytext);
 118
 119 [\"\']                          string_end = yytext[0]; extend(yytext); BEGIN(rstring);
 120 <rstring>[\"\']                 {
 121                                         extend(yytext);
 122                                         if (yytext[0] == string_end) {
 123                                                 BEGIN(INITIAL);
 124                                                 return STRING;
 125                                         }
 126                                 }
 127 <rstring>\\\n                   /* ignore */
 128 <rstring>\\.                    extend(yytext);
 129 <rstring>[^\\\"\']+             extend(yytext);
 130
 131 "DEFUN"                         value = strdup(yytext); return DEFUNNY;
 132 "DEFUN_NOSH"                    value = strdup(yytext); return DEFUNNY;
 133 "DEFUN_HIDDEN"                  value = strdup(yytext); return DEFUNNY;
 134 "DEFPY"                         value = strdup(yytext); return DEFUNNY;
 135 "DEFPY_ATTR"                    value = strdup(yytext); return DEFUNNY;
 136 "DEFPY_HIDDEN"                  value = strdup(yytext); return DEFUNNY;
 137 "ALIAS"                         value = strdup(yytext); return DEFUNNY;
 138 "ALIAS_HIDDEN"                  value = strdup(yytext); return DEFUNNY;
 139 "install_element"               value = strdup(yytext); return INSTALL;
 140 "VTYSH_TARGETS"                 value = strdup(yytext); return AUXILIARY;
 141 "VTYSH_NODESWITCH"              value = strdup(yytext); return AUXILIARY;
 142
 143 [ \t\n]+                        /* ignore */
 144 \\                              /* ignore */
 145 {ID}                            BEGIN(INITIAL); value = strdup(yytext); return ID;
 146 {OPERATOR}                      BEGIN(INITIAL); value = strdup(yytext); return OPERATOR;
 147 {SPECIAL}                       BEGIN(INITIAL); value = strdup(yytext); return SPECIAL;
 148 .                               /* printf("-- '%s' in init\n", yytext); */ BEGIN(INITIAL); return yytext[0];
 149
 150 %%
 151
 152 static int yylex_clr(char **retbuf)
 153 {
 154         int rv = def_yylex();
 155         *retbuf = value;
 156         value = NULL;
 157         return rv;
 158 }
 159
 160 static PyObject *get_args(void)
 161 {
 162         PyObject *pyObj = PyList_New(0);
 163         PyObject *pyArg = NULL;
 164
 165         char *tval;
 166         int depth = 1;
 167         int token;
 168
 169         while ((token = yylex_clr(&tval)) != YY_NULL) {
 170                 if (token == SPECIAL && tval[0] == '(') {
 171                         free(tval);
 172                         break;
 173                 }
 174                 if (token == COMMENT) {
 175                         free(tval);
 176                         continue;
 177                 }
 178                 fprintf(stderr, "invalid input!\n");
 179                 exit(1);
 180         }
 181
 182         while ((token = yylex_clr(&tval)) != YY_NULL) {
 183                 if (token == COMMENT) {
 184                         free(tval);
 185                         continue;
 186                 }
 187                 if (token == SPECIAL) {
 188                         if (depth == 1 && (tval[0] == ',' || tval[0] == ')')) {
 189                                 if (pyArg)
 190                                         PyList_Append(pyObj, pyArg);
 191                                 pyArg = NULL;
 192                                 if (tval[0] == ')') {
 193                                         free(tval);
 194                                         break;
 195                                 }
 196                                 free(tval);
 197                                 continue;
 198                         }
 199                         if (tval[0] == '(')
 200                                 depth++;
 201                         if (tval[0] == ')')
 202                                 depth--;
 203                 }
 204                 if (!pyArg)
 205                         pyArg = PyList_New(0);
 206                 PyList_Append(pyArg, PyUnicode_FromString(tval));
 207                 free(tval);
 208         }
 209         return pyObj;
 210 }
 211
 212 /* _clippy.parse() -- read a C file, returning a list of interesting bits.
 213  * note this ditches most of the actual C code. */
 214 PyObject *clippy_parse(PyObject *self, PyObject *args)
 215 {
 216         const char *filename;
 217         if (!PyArg_ParseTuple(args, "s", &filename))
 218                 return NULL;
 219
 220         FILE *fd = fopen(filename, "r");
 221         if (!fd)
 222                 return PyErr_SetFromErrnoWithFilename(PyExc_IOError, filename);
 223
 224         char *tval;
 225         int token;
 226         yyin = fd;
 227         value = NULL;
 228
 229         PyObject *pyCont = PyDict_New();
 230         PyObject *pyObj = PyList_New(0);
 231         PyDict_SetItemString(pyCont, "filename", PyUnicode_FromString(filename));
 232         PyDict_SetItemString(pyCont, "data", pyObj);
 233
 234         while ((token = yylex_clr(&tval)) != YY_NULL) {
 235                 int lineno = yylineno;
 236                 PyObject *pyItem = NULL, *pyArgs;
 237                 switch (token) {
 238                 case DEFUNNY:
 239                 case INSTALL:
 240                 case AUXILIARY:
 241                         pyArgs = get_args();
 242                         pyItem = PyDict_New();
 243                         PyDict_SetItemString(pyItem, "type", PyUnicode_FromString(tval));
 244                         PyDict_SetItemString(pyItem, "args", pyArgs);
 245                         break;
 246                 case COMMENT:
 247                         if (strncmp(tval, "//~", 3) && strncmp(tval, "/*~", 3))
 248                                 break;
 249                         pyItem = PyDict_New();
 250                         PyDict_SetItemString(pyItem, "type", PyUnicode_FromString("COMMENT"));
 251                         PyDict_SetItemString(pyItem, "line", PyUnicode_FromString(tval));
 252                         break;
 253                 case PREPROC:
 254                         pyItem = PyDict_New();
 255                         PyDict_SetItemString(pyItem, "type", PyUnicode_FromString("PREPROC"));
 256                         PyDict_SetItemString(pyItem, "line", PyUnicode_FromString(tval));
 257                         break;
 258                 }
 259                 if (pyItem) {
 260                         PyDict_SetItemString(pyItem, "lineno", PyLong_FromLong(lineno));
 261                         PyList_Append(pyObj, pyItem);
 262                 }
 263                 free(tval);
 264         }
 265         def_yylex_destroy();
 266         fclose(fd);
 267         return pyCont;
 268 }