[mirror_frr.git] / lib / defun_lex.l

%{
/*
 * clippy (CLI preparator in python) C pseudo-lexer
 * Copyright (C) 2016-2017  David Lamparter for NetDEF, Inc.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the Free
 * Software Foundation; either version 2 of the License, or (at your option)
 * any later version.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
 * more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; see the file COPYING; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
 */

/* This is just enough of a lexer to make rough sense of a C source file.
 * It handles C preprocessor directives, strings, and looks for FRR-specific
 * idioms (aka DEFUN).
 *
 * There is some preliminary support for documentation comments for DEFUNs.
 * They would look like this (note the ~):  (replace \ by /)
 *
 * \*~  documentation for foobar_cmd
 *  *   parameter does xyz
 *  *\
 * DEFUN(foobar_cmd, ...)
 *
 * This is intended for user documentation / command reference.  Don't put
 * code documentation in it.
 */

/* ignore harmless bug in old versions of flex */
#pragma GCC diagnostic ignored "-Wsign-compare"

#include "config.h"
#include <Python.h>
#include <string.h>
#include <stdlib.h>

#include "command_graph.h"
#include "clippy.h"

#define ID		258
#define PREPROC		259
#define OPERATOR	260
#define STRING		261
#define COMMENT		262
#define SPECIAL		263

#define DEFUNNY		270
#define INSTALL		271
#define AUXILIARY	272

int comment_link;
char string_end;

char *value;

static void extendbuf(char **what, const char *arg)
{
	if (!*what)
		*what = strdup(arg);
	else {
		size_t vall = strlen(*what), argl = strlen(arg);
		*what = realloc(*what, vall + argl + 1);
		memcpy(*what + vall, arg, argl);
		(*what)[vall + argl] = '\0';
	}
}
#define extend(x) extendbuf(&value, x)

%}

ID		[A-Za-z0-9_]+
OPERATOR	[!%&/\[\]{}=?:^|\*.;><~'\\+-]
SPECIAL		[(),]

%pointer
%option yylineno
%option noyywrap
%option noinput
%option nounput
%option outfile="lib/defun_lex.c"
%option prefix="def_yy"
%option 8bit

%s linestart
%x comment
%x linecomment
%x preproc
%x rstring
%%
				BEGIN(linestart);

\n				BEGIN(linestart);

<INITIAL,linestart,preproc>"/*"	comment_link = YY_START; extend(yytext); BEGIN(comment);
<comment>[^*\n]*		extend(yytext);
<comment>"*"+[^*/\n]*		extend(yytext);
<comment>\n			extend(yytext);
<comment>"*"+"/"		extend(yytext); BEGIN(comment_link); return COMMENT;

<INITIAL,linestart,preproc>"//"	comment_link = YY_START; extend(yytext); BEGIN(linecomment);
<linecomment>[^\n]*		extend(yytext);
<linecomment>\n			BEGIN((comment_link == INITIAL) ? linestart : comment_link); return COMMENT;

<linestart>#			BEGIN(preproc);
<preproc>\n			BEGIN(INITIAL); return PREPROC;
<preproc>[^\n\\]+		extend(yytext);
<preproc>\\\n			extend(yytext);
<preproc>\\+[^\n]		extend(yytext);

[\"\']				string_end = yytext[0]; extend(yytext); BEGIN(rstring);
<rstring>[\"\']			{
					extend(yytext);
					if (yytext[0] == string_end) {
						BEGIN(INITIAL);
						return STRING;
					}
				}
<rstring>\\\n			/* ignore */
<rstring>\\.			extend(yytext);
<rstring>[^\\\"\']+		extend(yytext);

"DEFUN"				value = strdup(yytext); return DEFUNNY;
"DEFUN_NOSH"			value = strdup(yytext); return DEFUNNY;
"DEFUN_HIDDEN"			value = strdup(yytext); return DEFUNNY;
"DEFPY"				value = strdup(yytext); return DEFUNNY;
"ALIAS"				value = strdup(yytext); return DEFUNNY;
"ALIAS_HIDDEN"			value = strdup(yytext); return DEFUNNY;
"install_element"		value = strdup(yytext); return INSTALL;
"VTYSH_TARGETS"			value = strdup(yytext); return AUXILIARY;
"VTYSH_NODESWITCH"		value = strdup(yytext); return AUXILIARY;

[ \t\n]+			/* ignore */
\\				/* ignore */
{ID}				BEGIN(INITIAL); value = strdup(yytext); return ID;
{OPERATOR}			BEGIN(INITIAL); value = strdup(yytext); return OPERATOR;
{SPECIAL}			BEGIN(INITIAL); value = strdup(yytext); return SPECIAL;
.				/* printf("-- '%s' in init\n", yytext); */ BEGIN(INITIAL); return yytext[0];

%%

static int yylex_clr(char **retbuf)
{
	int rv = def_yylex();
	*retbuf = value;
	value = NULL;
	return rv;
}

static PyObject *get_args(void)
{
	PyObject *pyObj = PyList_New(0);
	PyObject *pyArg = NULL;

	char *tval;
	int depth = 1;
	int token;

	while ((token = yylex_clr(&tval)) != YY_NULL) {
		if (token == SPECIAL && tval[0] == '(') {
			free(tval);
			break;
		}
		if (token == COMMENT) {
			free(tval);
			continue;
		}
		fprintf(stderr, "invalid input!\n");
		exit(1);
	}

	while ((token = yylex_clr(&tval)) != YY_NULL) {
		if (token == COMMENT) {
			free(tval);
			continue;
		}
		if (token == SPECIAL) {
			if (depth == 1 && (tval[0] == ',' || tval[0] == ')')) {
				if (pyArg)
					PyList_Append(pyObj, pyArg);
				pyArg = NULL;
				if (tval[0] == ')') {
					free(tval);
					break;
				}
				free(tval);
				continue;
			}
			if (tval[0] == '(')
				depth++;
			if (tval[0] == ')')
				depth--;
		}
		if (!pyArg)
			pyArg = PyList_New(0);
		PyList_Append(pyArg, PyUnicode_FromString(tval));
		free(tval);
	}
	return pyObj;
}

/* _clippy.parse() -- read a C file, returning a list of interesting bits.
 * note this ditches most of the actual C code. */
PyObject *clippy_parse(PyObject *self, PyObject *args)
{
	const char *filename;
	if (!PyArg_ParseTuple(args, "s", &filename))
		return NULL;
	
	FILE *fd = fopen(filename, "r");
	if (!fd)
		return PyErr_SetFromErrnoWithFilename(PyExc_IOError, filename);

	char *tval;
	int token;
	yyin = fd;
	value = NULL;

	PyObject *pyCont = PyDict_New();
	PyObject *pyObj = PyList_New(0);
	PyDict_SetItemString(pyCont, "filename", PyUnicode_FromString(filename));
	PyDict_SetItemString(pyCont, "data", pyObj);

	while ((token = yylex_clr(&tval)) != YY_NULL) {
                int lineno = yylineno;
		PyObject *pyItem = NULL, *pyArgs;
		switch (token) {
		case DEFUNNY:
		case INSTALL:
		case AUXILIARY:
			pyArgs = get_args();
			pyItem = PyDict_New();
			PyDict_SetItemString(pyItem, "type", PyUnicode_FromString(tval));
			PyDict_SetItemString(pyItem, "args", pyArgs);
			break;
		case COMMENT:
                        if (strncmp(tval, "//~", 3) && strncmp(tval, "/*~", 3))
                                break;
			pyItem = PyDict_New();
			PyDict_SetItemString(pyItem, "type", PyUnicode_FromString("COMMENT"));
			PyDict_SetItemString(pyItem, "line", PyUnicode_FromString(tval));
			break;
		case PREPROC:
			pyItem = PyDict_New();
			PyDict_SetItemString(pyItem, "type", PyUnicode_FromString("PREPROC"));
			PyDict_SetItemString(pyItem, "line", PyUnicode_FromString(tval));
			break;
		}
		if (pyItem) {
			PyDict_SetItemString(pyItem, "lineno", PyLong_FromLong(lineno));
			PyList_Append(pyObj, pyItem);
		}
		free(tval);
	}
	def_yylex_destroy();
	fclose(fd);
	return pyCont;
}
Commit	Line	Data
29ad6f68 DL	1	%{
	2	/*
	3	* clippy (CLI preparator in python) C pseudo-lexer
	4	* Copyright (C) 2016-2017 David Lamparter for NetDEF, Inc.
	5	*
	6	* This program is free software; you can redistribute it and/or modify it
	7	* under the terms of the GNU General Public License as published by the Free
	8	* Software Foundation; either version 2 of the License, or (at your option)
	9	* any later version.
	10	*
	11	* This program is distributed in the hope that it will be useful, but WITHOUT
	12	* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
	13	* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
	14	* more details.
	15	*
	16	* You should have received a copy of the GNU General Public License along
	17	* with this program; see the file COPYING; if not, write to the Free Software
	18	* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	19	*/
	20
	21	/* This is just enough of a lexer to make rough sense of a C source file.
	22	* It handles C preprocessor directives, strings, and looks for FRR-specific
	23	* idioms (aka DEFUN).
	24	*
	25	* There is some preliminary support for documentation comments for DEFUNs.
	26	* They would look like this (note the ~): (replace \ by /)
	27	*
	28	* \*~ documentation for foobar_cmd
	29	* * parameter does xyz
	30	* *\
	31	* DEFUN(foobar_cmd, ...)
	32	*
	33	* This is intended for user documentation / command reference. Don't put
	34	* code documentation in it.
	35	*/
	36
	37	/* ignore harmless bug in old versions of flex */
	38	#pragma GCC diagnostic ignored "-Wsign-compare"
	39
	40	#include "config.h"
	41	#include <Python.h>
	42	#include <string.h>
	43	#include <stdlib.h>
	44
	45	#include "command_graph.h"
	46	#include "clippy.h"
	47
	48	#define ID 258
	49	#define PREPROC 259
	50	#define OPERATOR 260
	51	#define STRING 261
	52	#define COMMENT 262
	53	#define SPECIAL 263
	54
	55	#define DEFUNNY 270
	56	#define INSTALL 271
	57	#define AUXILIARY 272
	58
	59	int comment_link;
	60	char string_end;
	61
	62	char *value;
	63
	64	static void extendbuf(char *what, const char arg)
65	{
66	if (!*what)
67	*what = strdup(arg);
68	else {
69	size_t vall = strlen(*what), argl = strlen(arg);
70	what = realloc(what, vall + argl + 1);
71	memcpy(*what + vall, arg, argl);
72	(*what)[vall + argl] = '\0';
73	}
74	}
75	#define extend(x) extendbuf(&value, x)
76
77	%}
78
79	ID [A-Za-z0-9_]+
80	OPERATOR [!%&/\[\]{}=?:^\|\*.;><~'\\+-]
81	SPECIAL [(),]
82
83	%pointer
84	%option yylineno
85	%option noyywrap
86	%option noinput
87	%option nounput
4a121f99	88	%option outfile="lib/defun_lex.c"
29ad6f68 DL	89	%option prefix="def_yy"
	90	%option 8bit
	91
	92	%s linestart
	93	%x comment
	94	%x linecomment
	95	%x preproc
	96	%x rstring
	97	%%
	98	BEGIN(linestart);
	99
	100	\n BEGIN(linestart);
	101
	102	<INITIAL,linestart,preproc>"/*" comment_link = YY_START; extend(yytext); BEGIN(comment);
	103	<comment>[^\n] extend(yytext);
	104	<comment>""+[^/\n]* extend(yytext);
	105	<comment>\n extend(yytext);
	106	<comment>"*"+"/" extend(yytext); BEGIN(comment_link); return COMMENT;
	107
	108	<INITIAL,linestart,preproc>"//" comment_link = YY_START; extend(yytext); BEGIN(linecomment);
	109	<linecomment>[^\n]* extend(yytext);
	110	<linecomment>\n BEGIN((comment_link == INITIAL) ? linestart : comment_link); return COMMENT;
	111
	112	<linestart># BEGIN(preproc);
	113	<preproc>\n BEGIN(INITIAL); return PREPROC;
	114	<preproc>[^\n\\]+ extend(yytext);
	115	<preproc>\\\n extend(yytext);
	116	<preproc>\\+[^\n] extend(yytext);
	117
	118	[\"\'] string_end = yytext[0]; extend(yytext); BEGIN(rstring);
	119	<rstring>[\"\'] {
	120	extend(yytext);
	121	if (yytext[0] == string_end) {
	122	BEGIN(INITIAL);
	123	return STRING;
	124	}
	125	}
	126	<rstring>\\\n /* ignore */
	127	<rstring>\\. extend(yytext);
	128	<rstring>[^\\\"\']+ extend(yytext);
	129
	130	"DEFUN" value = strdup(yytext); return DEFUNNY;
	131	"DEFUN_NOSH" value = strdup(yytext); return DEFUNNY;
	132	"DEFUN_HIDDEN" value = strdup(yytext); return DEFUNNY;
	133	"DEFPY" value = strdup(yytext); return DEFUNNY;
	134	"ALIAS" value = strdup(yytext); return DEFUNNY;
	135	"ALIAS_HIDDEN" value = strdup(yytext); return DEFUNNY;
	136	"install_element" value = strdup(yytext); return INSTALL;
	137	"VTYSH_TARGETS" value = strdup(yytext); return AUXILIARY;
	138	"VTYSH_NODESWITCH" value = strdup(yytext); return AUXILIARY;
	139
	140	[ \t\n]+ /* ignore */
	141	\\ /* ignore */
	142	{ID} BEGIN(INITIAL); value = strdup(yytext); return ID;
	143	{OPERATOR} BEGIN(INITIAL); value = strdup(yytext); return OPERATOR;
	144	{SPECIAL} BEGIN(INITIAL); value = strdup(yytext); return SPECIAL;
	145	. /* printf("-- '%s' in init\n", yytext); */ BEGIN(INITIAL); return yytext[0];
	146
	147	%%
	148
	149	static int yylex_clr(char **retbuf)
	150	{
	151	int rv = def_yylex();
	152	*retbuf = value;
153	value = NULL;
154	return rv;
155	}
156
157	static PyObject *get_args(void)
158	{
159	PyObject *pyObj = PyList_New(0);
160	PyObject *pyArg = NULL;
161
162	char *tval;
163	int depth = 1;
164	int token;
165
166	while ((token = yylex_clr(&tval)) != YY_NULL) {
167	if (token == SPECIAL && tval[0] == '(') {
168	free(tval);
169	break;
170	}
171	if (token == COMMENT) {
172	free(tval);
173	continue;
174	}
175	fprintf(stderr, "invalid input!\n");
176	exit(1);
177	}
178
179	while ((token = yylex_clr(&tval)) != YY_NULL) {
180	if (token == COMMENT) {
181	free(tval);
182	continue;
183	}
184	if (token == SPECIAL) {
185	if (depth == 1 && (tval[0] == ',' \|\| tval[0] == ')')) {
186	if (pyArg)
187	PyList_Append(pyObj, pyArg);
188	pyArg = NULL;
189	if (tval[0] == ')') {
190	free(tval);
191	break;
192	}
193	free(tval);
194	continue;
195	}
196	if (tval[0] == '(')
197	depth++;
198	if (tval[0] == ')')
199	depth--;
200	}
201	if (!pyArg)
202	pyArg = PyList_New(0);
203	PyList_Append(pyArg, PyUnicode_FromString(tval));
204	free(tval);
205	}
206	return pyObj;
207	}
208
209	/* _clippy.parse() -- read a C file, returning a list of interesting bits.
210	* note this ditches most of the actual C code. */
211	PyObject clippy_parse(PyObject self, PyObject *args)
212	{
213	const char *filename;
214	if (!PyArg_ParseTuple(args, "s", &filename))
215	return NULL;
216
217	FILE *fd = fopen(filename, "r");
218	if (!fd)
219	return PyErr_SetFromErrnoWithFilename(PyExc_IOError, filename);
220
221	char *tval;
222	int token;
223	yyin = fd;
224	value = NULL;
225
226	PyObject *pyCont = PyDict_New();
227	PyObject *pyObj = PyList_New(0);
228	PyDict_SetItemString(pyCont, "filename", PyUnicode_FromString(filename));
229	PyDict_SetItemString(pyCont, "data", pyObj);
230
231	while ((token = yylex_clr(&tval)) != YY_NULL) {
232	int lineno = yylineno;
233	PyObject pyItem = NULL, pyArgs;
234	switch (token) {
235	case DEFUNNY:
236	case INSTALL:
237	case AUXILIARY:
238	pyArgs = get_args();
239	pyItem = PyDict_New();
240	PyDict_SetItemString(pyItem, "type", PyUnicode_FromString(tval));
241	PyDict_SetItemString(pyItem, "args", pyArgs);
242	break;
243	case COMMENT:
244	if (strncmp(tval, "//~", 3) && strncmp(tval, "/*~", 3))
245	break;
246	pyItem = PyDict_New();
247	PyDict_SetItemString(pyItem, "type", PyUnicode_FromString("COMMENT"));
248	PyDict_SetItemString(pyItem, "line", PyUnicode_FromString(tval));
249	break;
250	case PREPROC:
251	pyItem = PyDict_New();
252	PyDict_SetItemString(pyItem, "type", PyUnicode_FromString("PREPROC"));
253	PyDict_SetItemString(pyItem, "line", PyUnicode_FromString(tval));
254	break;
255	}
256	if (pyItem) {
257	PyDict_SetItemString(pyItem, "lineno", PyLong_FromLong(lineno));
258	PyList_Append(pyObj, pyItem);
259	}
260	free(tval);
261	}
262	def_yylex_destroy();
263	fclose(fd);
264	return pyCont;
265	}