[mirror_frr.git] / lib / defun_lex.l

/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
 * clippy (CLI preparator in python) C pseudo-lexer
 * Copyright (C) 2016-2017  David Lamparter for NetDEF, Inc.
 */

/* This is just enough of a lexer to make rough sense of a C source file.
 * It handles C preprocessor directives, strings, and looks for FRR-specific
 * idioms (aka DEFUN).
 *
 * There is some preliminary support for documentation comments for DEFUNs.
 * They would look like this (note the ~):  (replace \ by /)
 *
 * \*~  documentation for foobar_cmd
 *  *   parameter does xyz
 *  *\
 * DEFUN(foobar_cmd, ...)
 *
 * This is intended for user documentation / command reference.  Don't put
 * code documentation in it.
 */

%top{
#ifdef HAVE_CONFIG_H
#include "config.h"
#endif
}
%{
/* ignore harmless bugs in old versions of flex */
#pragma GCC diagnostic ignored "-Wsign-compare"
#pragma GCC diagnostic ignored "-Wunused-value"

#include "config.h"
#include <Python.h>
#include <string.h>
#include <stdlib.h>

#include "command_graph.h"
#include "clippy.h"

#define ID		258
#define PREPROC		259
#define OPERATOR	260
#define STRING		261
#define COMMENT		262
#define SPECIAL		263

#define DEFUNNY		270
#define INSTALL		271
#define AUXILIARY	272

int comment_link;
char string_end;

char *value;
static const char *yyfilename;

static void extendbuf(char **what, const char *arg)
{
	if (!*what)
		*what = strdup(arg);
	else {
		size_t vall = strlen(*what), argl = strlen(arg);
		*what = realloc(*what, vall + argl + 1);
		memcpy(*what + vall, arg, argl);
		(*what)[vall + argl] = '\0';
	}
}
#define extend(x) extendbuf(&value, x)

#ifndef __clang_analyzer__

%}

ID		[A-Za-z0-9_]+
OPERATOR	[!%&/\[\]{}=?:^|\*.;><~'\\+-]
SPECIAL		[(),]

%pointer
%option yylineno
%option noyywrap
%option noinput
%option nounput
%option outfile="lib/defun_lex.c"
%option prefix="def_yy"
%option 8bit

%s linestart
%x comment
%x linecomment
%x preproc
%x rstring
%%
				BEGIN(linestart);

\n				BEGIN(linestart);

<INITIAL,linestart,preproc>"/*"	comment_link = YY_START; extend(yytext); BEGIN(comment);
<comment>[^*\n]*		extend(yytext);
<comment>"*"+[^*/\n]*		extend(yytext);
<comment>\n			extend(yytext);
<comment>"*"+"/"		extend(yytext); BEGIN(comment_link); return COMMENT;

<INITIAL,linestart,preproc>"//"	comment_link = YY_START; extend(yytext); BEGIN(linecomment);
<linecomment>[^\n]*		extend(yytext);
<linecomment>\n			BEGIN((comment_link == INITIAL) ? linestart : comment_link); return COMMENT;

<linestart>#			BEGIN(preproc);
<preproc>\n			BEGIN(INITIAL); return PREPROC;
<preproc>[^\n\\]+		extend(yytext);
<preproc>\\\n			extend(yytext);
<preproc>\\+[^\n]		extend(yytext);

[\"\']				string_end = yytext[0]; extend(yytext); BEGIN(rstring);
<rstring>[\"\']			{
					extend(yytext);
					if (yytext[0] == string_end) {
						BEGIN(INITIAL);
						return STRING;
					}
				}
<rstring>\\\n			/* ignore */
<rstring>\n			{
					fprintf(stderr,
						"%s:%d: string continues past the end of the line\n",
						yyfilename, yylineno);
					free(value);
					value = NULL;
					BEGIN(INITIAL);
					return STRING;
				}
<rstring>\\.			extend(yytext);
<rstring>[^\\\"\'\n]+		extend(yytext);

"DEFUN"				value = strdup(yytext); return DEFUNNY;
"DEFUN_NOSH"			value = strdup(yytext); return DEFUNNY;
"DEFUN_HIDDEN"			value = strdup(yytext); return DEFUNNY;
"DEFPY"				value = strdup(yytext); return DEFUNNY;
"DEFPY_NOSH"			value = strdup(yytext); return DEFUNNY;
"DEFPY_ATTR"			value = strdup(yytext); return DEFUNNY;
"DEFPY_HIDDEN"			value = strdup(yytext); return DEFUNNY;
"DEFPY_YANG"			value = strdup(yytext); return DEFUNNY;
"DEFPY_YANG_NOSH"		value = strdup(yytext); return DEFUNNY;
"ALIAS"				value = strdup(yytext); return DEFUNNY;
"ALIAS_HIDDEN"			value = strdup(yytext); return DEFUNNY;
"install_element"		value = strdup(yytext); return INSTALL;
"VTYSH_TARGETS"			value = strdup(yytext); return AUXILIARY;
"VTYSH_NODESWITCH"		value = strdup(yytext); return AUXILIARY;

[ \t\n]+			/* ignore */
\\				/* ignore */
{ID}				BEGIN(INITIAL); value = strdup(yytext); return ID;
{OPERATOR}			BEGIN(INITIAL); value = strdup(yytext); return OPERATOR;
{SPECIAL}			BEGIN(INITIAL); value = strdup(yytext); return SPECIAL;
.				/* printf("-- '%s' in init\n", yytext); */ BEGIN(INITIAL); return yytext[0];

%%

#endif /* __clang_analyzer__ */

static int yylex_clr(char **retbuf)
{
	int rv = def_yylex();
	*retbuf = value;
	value = NULL;
	return rv;
}

static PyObject *get_args(const char *filename, int lineno)
{
	PyObject *pyObj = PyList_New(0);
	PyObject *pyArg = NULL;

	char *tval;
	int depth = 1;
	int token;

	while ((token = yylex_clr(&tval)) != YY_NULL) {
		if (token == SPECIAL && tval[0] == '(') {
			free(tval);
			break;
		}
		if (token == COMMENT) {
			free(tval);
			continue;
		}
		fprintf(stderr, "invalid input!\n");
		exit(1);
	}

	while ((token = yylex_clr(&tval)) != YY_NULL) {
		if (token == COMMENT) {
			free(tval);
			continue;
		}
		if (token == PREPROC) {
			free(tval);
			Py_DECREF(pyObj);
			return PyErr_Format(PyExc_ValueError,
					"%s:%d: cannot process CPP directive within argument list",
					filename, lineno);
		}
		if (token == SPECIAL) {
			if (depth == 1 && (tval[0] == ',' || tval[0] == ')')) {
				if (pyArg)
					PyList_Append(pyObj, pyArg);
				pyArg = NULL;
				if (tval[0] == ')') {
					free(tval);
					break;
				}
				free(tval);
				continue;
			}
			if (tval[0] == '(')
				depth++;
			if (tval[0] == ')')
				depth--;
		}
		if (!tval)
			return PyErr_Format(PyExc_ValueError,
					"%s:%d: invalid token in DEFPY parameters",
					filename, lineno);
		if (!pyArg)
			pyArg = PyList_New(0);
		PyList_Append(pyArg, PyUnicode_FromString(tval));
		free(tval);
	}
	return pyObj;
}

/* _clippy.parse() -- read a C file, returning a list of interesting bits.
 * note this ditches most of the actual C code. */
PyObject *clippy_parse(PyObject *self, PyObject *args)
{
	const char *filename;
	if (!PyArg_ParseTuple(args, "s", &filename))
		return NULL;
	
	FILE *fd = fopen(filename, "r");
	if (!fd)
		return PyErr_SetFromErrnoWithFilename(PyExc_IOError, filename);

	char *tval;
	int token;
	yyin = fd;
	value = NULL;
	yyfilename = filename;

	PyObject *pyCont = PyDict_New();
	PyObject *pyObj = PyList_New(0);
	PyDict_SetItemString(pyCont, "filename", PyUnicode_FromString(filename));
	PyDict_SetItemString(pyCont, "data", pyObj);

	while ((token = yylex_clr(&tval)) != YY_NULL) {
                int lineno = yylineno;
		PyObject *pyItem = NULL, *pyArgs;
		switch (token) {
		case DEFUNNY:
		case INSTALL:
		case AUXILIARY:
			pyArgs = get_args(filename, lineno);
			if (!pyArgs) {
				free(tval);
				Py_DECREF(pyCont);
				yyfilename = NULL;
				return NULL;
			}
			pyItem = PyDict_New();
			PyDict_SetItemString(pyItem, "type", PyUnicode_FromString(tval));
			PyDict_SetItemString(pyItem, "args", pyArgs);
			break;
		case COMMENT:
                        if (strncmp(tval, "//~", 3) && strncmp(tval, "/*~", 3))
                                break;
			pyItem = PyDict_New();
			PyDict_SetItemString(pyItem, "type", PyUnicode_FromString("COMMENT"));
			PyDict_SetItemString(pyItem, "line", PyUnicode_FromString(tval));
			break;
		case PREPROC:
			pyItem = PyDict_New();
			PyDict_SetItemString(pyItem, "type", PyUnicode_FromString("PREPROC"));
			PyDict_SetItemString(pyItem, "line", PyUnicode_FromString(tval));
			lineno--;
			break;
		}
		if (pyItem) {
			PyDict_SetItemString(pyItem, "lineno", PyLong_FromLong(lineno));
			PyList_Append(pyObj, pyItem);
		}
		free(tval);
	}
	def_yylex_destroy();
	fclose(fd);
	yyfilename = NULL;
	return pyCont;
}
Commit	Line	Data
47a3a827	1	/* SPDX-License-Identifier: GPL-2.0-or-later */
29ad6f68 DL	2	/*
	3	* clippy (CLI preparator in python) C pseudo-lexer
	4	* Copyright (C) 2016-2017 David Lamparter for NetDEF, Inc.
29ad6f68 DL	5	*/
	6
	7	/* This is just enough of a lexer to make rough sense of a C source file.
	8	* It handles C preprocessor directives, strings, and looks for FRR-specific
	9	* idioms (aka DEFUN).
	10	*
	11	* There is some preliminary support for documentation comments for DEFUNs.
	12	* They would look like this (note the ~): (replace \ by /)
	13	*
	14	* \*~ documentation for foobar_cmd
	15	* * parameter does xyz
	16	* *\
	17	* DEFUN(foobar_cmd, ...)
	18	*
	19	* This is intended for user documentation / command reference. Don't put
	20	* code documentation in it.
	21	*/
	22
7b34167d DL	23	%top{
	24	#ifdef HAVE_CONFIG_H
	25	#include "config.h"
	26	#endif
	27	}
	28	%{
a37bd5e0	29	/* ignore harmless bugs in old versions of flex */
29ad6f68	30	#pragma GCC diagnostic ignored "-Wsign-compare"
a37bd5e0	31	#pragma GCC diagnostic ignored "-Wunused-value"
29ad6f68 DL	32
	33	#include "config.h"
	34	#include <Python.h>
	35	#include <string.h>
	36	#include <stdlib.h>
	37
	38	#include "command_graph.h"
	39	#include "clippy.h"
	40
	41	#define ID 258
	42	#define PREPROC 259
	43	#define OPERATOR 260
	44	#define STRING 261
	45	#define COMMENT 262
	46	#define SPECIAL 263
	47
	48	#define DEFUNNY 270
	49	#define INSTALL 271
	50	#define AUXILIARY 272
	51
	52	int comment_link;
	53	char string_end;
	54
	55	char *value;
be95afe1	56	static const char *yyfilename;
29ad6f68 DL	57
	58	static void extendbuf(char *what, const char arg)
	59	{
	60	if (!*what)
	61	*what = strdup(arg);
	62	else {
	63	size_t vall = strlen(*what), argl = strlen(arg);
	64	what = realloc(what, vall + argl + 1);
	65	memcpy(*what + vall, arg, argl);
	66	(*what)[vall + argl] = '\0';
	67	}
	68	}
	69	#define extend(x) extendbuf(&value, x)
	70
4da4b9d4 QY	71	#ifndef __clang_analyzer__
4da4b9d4 QY	72
29ad6f68 DL	73	%}
	74
	75	ID [A-Za-z0-9_]+
	76	OPERATOR [!%&/\[\]{}=?:^\|\*.;><~'\\+-]
	77	SPECIAL [(),]
	78
	79	%pointer
	80	%option yylineno
	81	%option noyywrap
	82	%option noinput
	83	%option nounput
4a121f99	84	%option outfile="lib/defun_lex.c"
29ad6f68 DL	85	%option prefix="def_yy"
	86	%option 8bit
	87
	88	%s linestart
	89	%x comment
	90	%x linecomment
	91	%x preproc
	92	%x rstring
	93	%%
	94	BEGIN(linestart);
	95
	96	\n BEGIN(linestart);
	97
	98	<INITIAL,linestart,preproc>"/*" comment_link = YY_START; extend(yytext); BEGIN(comment);
	99	<comment>[^\n] extend(yytext);
	100	<comment>""+[^/\n]* extend(yytext);
	101	<comment>\n extend(yytext);
	102	<comment>"*"+"/" extend(yytext); BEGIN(comment_link); return COMMENT;
	103
	104	<INITIAL,linestart,preproc>"//" comment_link = YY_START; extend(yytext); BEGIN(linecomment);
	105	<linecomment>[^\n]* extend(yytext);
	106	<linecomment>\n BEGIN((comment_link == INITIAL) ? linestart : comment_link); return COMMENT;
	107
	108	<linestart># BEGIN(preproc);
	109	<preproc>\n BEGIN(INITIAL); return PREPROC;
	110	<preproc>[^\n\\]+ extend(yytext);
	111	<preproc>\\\n extend(yytext);
	112	<preproc>\\+[^\n] extend(yytext);
	113
	114	[\"\'] string_end = yytext[0]; extend(yytext); BEGIN(rstring);
	115	<rstring>[\"\'] {
	116	extend(yytext);
	117	if (yytext[0] == string_end) {
	118	BEGIN(INITIAL);
	119	return STRING;
	120	}
	121	}
	122	<rstring>\\\n /* ignore */
be95afe1 DL	123	<rstring>\n {
	124	fprintf(stderr,
	125	"%s:%d: string continues past the end of the line\n",
	126	yyfilename, yylineno);
	127	free(value);
	128	value = NULL;
	129	BEGIN(INITIAL);
	130	return STRING;
	131	}
29ad6f68	132	<rstring>\\. extend(yytext);
be95afe1	133	<rstring>[^\\\"\'\n]+ extend(yytext);
29ad6f68 DL	134
	135	"DEFUN" value = strdup(yytext); return DEFUNNY;
	136	"DEFUN_NOSH" value = strdup(yytext); return DEFUNNY;
	137	"DEFUN_HIDDEN" value = strdup(yytext); return DEFUNNY;
	138	"DEFPY" value = strdup(yytext); return DEFUNNY;
26d19c83	139	"DEFPY_NOSH" value = strdup(yytext); return DEFUNNY;
e31f4dbe QY	140	"DEFPY_ATTR" value = strdup(yytext); return DEFUNNY;
e31f4dbe QY	141	"DEFPY_HIDDEN" value = strdup(yytext); return DEFUNNY;
ca77b518 RW	142	"DEFPY_YANG" value = strdup(yytext); return DEFUNNY;
ca77b518 RW	143	"DEFPY_YANG_NOSH" value = strdup(yytext); return DEFUNNY;
29ad6f68 DL	144	"ALIAS" value = strdup(yytext); return DEFUNNY;
	145	"ALIAS_HIDDEN" value = strdup(yytext); return DEFUNNY;
	146	"install_element" value = strdup(yytext); return INSTALL;
	147	"VTYSH_TARGETS" value = strdup(yytext); return AUXILIARY;
	148	"VTYSH_NODESWITCH" value = strdup(yytext); return AUXILIARY;
	149
	150	[ \t\n]+ /* ignore */
	151	\\ /* ignore */
	152	{ID} BEGIN(INITIAL); value = strdup(yytext); return ID;
	153	{OPERATOR} BEGIN(INITIAL); value = strdup(yytext); return OPERATOR;
	154	{SPECIAL} BEGIN(INITIAL); value = strdup(yytext); return SPECIAL;
	155	. /* printf("-- '%s' in init\n", yytext); */ BEGIN(INITIAL); return yytext[0];
	156
	157	%%
	158
4da4b9d4 QY	159	#endif /* __clang_analyzer__ */
4da4b9d4 QY	160
29ad6f68 DL	161	static int yylex_clr(char **retbuf)
	162	{
	163	int rv = def_yylex();
	164	*retbuf = value;
	165	value = NULL;
	166	return rv;
	167	}
	168
3779776a	169	static PyObject get_args(const char filename, int lineno)
29ad6f68 DL	170	{
	171	PyObject *pyObj = PyList_New(0);
	172	PyObject *pyArg = NULL;
	173
	174	char *tval;
	175	int depth = 1;
	176	int token;
	177
	178	while ((token = yylex_clr(&tval)) != YY_NULL) {
	179	if (token == SPECIAL && tval[0] == '(') {
	180	free(tval);
	181	break;
	182	}
	183	if (token == COMMENT) {
	184	free(tval);
	185	continue;
	186	}
	187	fprintf(stderr, "invalid input!\n");
	188	exit(1);
	189	}
	190
	191	while ((token = yylex_clr(&tval)) != YY_NULL) {
	192	if (token == COMMENT) {
	193	free(tval);
	194	continue;
	195	}
3779776a DL	196	if (token == PREPROC) {
	197	free(tval);
	198	Py_DECREF(pyObj);
	199	return PyErr_Format(PyExc_ValueError,
	200	"%s:%d: cannot process CPP directive within argument list",
	201	filename, lineno);
	202	}
29ad6f68 DL	203	if (token == SPECIAL) {
	204	if (depth == 1 && (tval[0] == ',' \|\| tval[0] == ')')) {
	205	if (pyArg)
	206	PyList_Append(pyObj, pyArg);
	207	pyArg = NULL;
	208	if (tval[0] == ')') {
	209	free(tval);
	210	break;
	211	}
	212	free(tval);
	213	continue;
	214	}
	215	if (tval[0] == '(')
	216	depth++;
	217	if (tval[0] == ')')
	218	depth--;
	219	}
9e92984f DL	220	if (!tval)
	221	return PyErr_Format(PyExc_ValueError,
	222	"%s:%d: invalid token in DEFPY parameters",
	223	filename, lineno);
29ad6f68 DL	224	if (!pyArg)
	225	pyArg = PyList_New(0);
	226	PyList_Append(pyArg, PyUnicode_FromString(tval));
	227	free(tval);
	228	}
	229	return pyObj;
	230	}
	231
	232	/* _clippy.parse() -- read a C file, returning a list of interesting bits.
	233	* note this ditches most of the actual C code. */
	234	PyObject clippy_parse(PyObject self, PyObject *args)
	235	{
	236	const char *filename;
	237	if (!PyArg_ParseTuple(args, "s", &filename))
	238	return NULL;
	239
	240	FILE *fd = fopen(filename, "r");
	241	if (!fd)
	242	return PyErr_SetFromErrnoWithFilename(PyExc_IOError, filename);
	243
	244	char *tval;
	245	int token;
	246	yyin = fd;
	247	value = NULL;
be95afe1	248	yyfilename = filename;
29ad6f68 DL	249
	250	PyObject *pyCont = PyDict_New();
	251	PyObject *pyObj = PyList_New(0);
	252	PyDict_SetItemString(pyCont, "filename", PyUnicode_FromString(filename));
	253	PyDict_SetItemString(pyCont, "data", pyObj);
	254
	255	while ((token = yylex_clr(&tval)) != YY_NULL) {
	256	int lineno = yylineno;
	257	PyObject pyItem = NULL, pyArgs;
	258	switch (token) {
	259	case DEFUNNY:
	260	case INSTALL:
	261	case AUXILIARY:
3779776a DL	262	pyArgs = get_args(filename, lineno);
	263	if (!pyArgs) {
	264	free(tval);
	265	Py_DECREF(pyCont);
be95afe1	266	yyfilename = NULL;
3779776a DL	267	return NULL;
3779776a DL	268	}
29ad6f68 DL	269	pyItem = PyDict_New();
	270	PyDict_SetItemString(pyItem, "type", PyUnicode_FromString(tval));
	271	PyDict_SetItemString(pyItem, "args", pyArgs);
	272	break;
	273	case COMMENT:
	274	if (strncmp(tval, "//~", 3) && strncmp(tval, "/*~", 3))
	275	break;
	276	pyItem = PyDict_New();
	277	PyDict_SetItemString(pyItem, "type", PyUnicode_FromString("COMMENT"));
	278	PyDict_SetItemString(pyItem, "line", PyUnicode_FromString(tval));
	279	break;
	280	case PREPROC:
	281	pyItem = PyDict_New();
	282	PyDict_SetItemString(pyItem, "type", PyUnicode_FromString("PREPROC"));
	283	PyDict_SetItemString(pyItem, "line", PyUnicode_FromString(tval));
3779776a	284	lineno--;
29ad6f68 DL	285	break;
	286	}
	287	if (pyItem) {
	288	PyDict_SetItemString(pyItem, "lineno", PyLong_FromLong(lineno));
	289	PyList_Append(pyObj, pyItem);
	290	}
	291	free(tval);
	292	}
	293	def_yylex_destroy();
	294	fclose(fd);
be95afe1	295	yyfilename = NULL;
29ad6f68 DL	296	return pyCont;
29ad6f68 DL	297	}