[mirror_edk2.git] / AppPkg / Applications / Python / Python-2.7.2 / Parser / parsetok.c

/** @file\r
  Parser-tokenizer link implementation.\r
\r
  Copyright (c) 2014, Intel Corporation. All rights reserved.<BR>\r
  This program and the accompanying materials are licensed and made available under\r
  the terms and conditions of the BSD License that accompanies this distribution.\r
  The full text of the license may be found at\r
  http://opensource.org/licenses/bsd-license.php.\r
\r
  THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,\r
  WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.\r
**/\r
#include "pgenheaders.h"\r
#include "tokenizer.h"\r
#include "node.h"\r
#include "grammar.h"\r
#include "parser.h"\r
#include "parsetok.h"\r
#include "errcode.h"\r
#include "graminit.h"\r
\r
int Py_TabcheckFlag;\r
\r
\r
/* Forward */\r
static node *parsetok(struct tok_state *, grammar *, int, perrdetail *, int *);\r
static void initerr(perrdetail *err_ret, const char* filename);\r
\r
/* Parse input coming from a string.  Return error code, print some errors. */\r
node *\r
PyParser_ParseString(const char *s, grammar *g, int start, perrdetail *err_ret)\r
{\r
    return PyParser_ParseStringFlagsFilename(s, NULL, g, start, err_ret, 0);\r
}\r
\r
node *\r
PyParser_ParseStringFlags(const char *s, grammar *g, int start,\r
                          perrdetail *err_ret, int flags)\r
{\r
    return PyParser_ParseStringFlagsFilename(s, NULL,\r
                                             g, start, err_ret, flags);\r
}\r
\r
node *\r
PyParser_ParseStringFlagsFilename(const char *s, const char *filename,\r
                          grammar *g, int start,\r
                          perrdetail *err_ret, int flags)\r
{\r
    int iflags = flags;\r
    return PyParser_ParseStringFlagsFilenameEx(s, filename, g, start,\r
                                               err_ret, &iflags);\r
}\r
\r
node *\r
PyParser_ParseStringFlagsFilenameEx(const char *s, const char *filename,\r
                          grammar *g, int start,\r
                          perrdetail *err_ret, int *flags)\r
{\r
    struct tok_state *tok;\r
\r
    initerr(err_ret, filename);\r
\r
    if ((tok = PyTokenizer_FromString(s, start == file_input)) == NULL) {\r
        err_ret->error = PyErr_Occurred() ? E_DECODE : E_NOMEM;\r
        return NULL;\r
    }\r
\r
    tok->filename = filename ? filename : "<string>";\r
    if (Py_TabcheckFlag || Py_VerboseFlag) {\r
        tok->altwarning = (tok->filename != NULL);\r
        if (Py_TabcheckFlag >= 2)\r
            tok->alterror++;\r
    }\r
\r
    return parsetok(tok, g, start, err_ret, flags);\r
}\r
\r
/* Parse input coming from a file.  Return error code, print some errors. */\r
\r
node *\r
PyParser_ParseFile(FILE *fp, const char *filename, grammar *g, int start,\r
                   char *ps1, char *ps2, perrdetail *err_ret)\r
{\r
    return PyParser_ParseFileFlags(fp, filename, g, start, ps1, ps2,\r
                                   err_ret, 0);\r
}\r
\r
node *\r
PyParser_ParseFileFlags(FILE *fp, const char *filename, grammar *g, int start,\r
                        char *ps1, char *ps2, perrdetail *err_ret, int flags)\r
{\r
    int iflags = flags;\r
    return PyParser_ParseFileFlagsEx(fp, filename, g, start, ps1, ps2, err_ret, &iflags);\r
}\r
\r
node *\r
PyParser_ParseFileFlagsEx(FILE *fp, const char *filename, grammar *g, int start,\r
                          char *ps1, char *ps2, perrdetail *err_ret, int *flags)\r
{\r
    struct tok_state *tok;\r
\r
    initerr(err_ret, filename);\r
\r
    if ((tok = PyTokenizer_FromFile(fp, ps1, ps2)) == NULL) {\r
        err_ret->error = E_NOMEM;\r
        return NULL;\r
    }\r
    tok->filename = filename;\r
    if (Py_TabcheckFlag || Py_VerboseFlag) {\r
        tok->altwarning = (filename != NULL);\r
        if (Py_TabcheckFlag >= 2)\r
            tok->alterror++;\r
    }\r
\r
    return parsetok(tok, g, start, err_ret, flags);\r
}\r
\r
#if 0\r
static char with_msg[] =\r
"%s:%d: Warning: 'with' will become a reserved keyword in Python 2.6\n";\r
\r
static char as_msg[] =\r
"%s:%d: Warning: 'as' will become a reserved keyword in Python 2.6\n";\r
\r
static void\r
warn(const char *msg, const char *filename, int lineno)\r
{\r
    if (filename == NULL)\r
        filename = "<string>";\r
    PySys_WriteStderr(msg, filename, lineno);\r
}\r
#endif\r
\r
/* Parse input coming from the given tokenizer structure.\r
   Return error code. */\r
\r
static node *\r
parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret,\r
         int *flags)\r
{\r
    parser_state *ps;\r
    node *n;\r
    int started = 0;\r
    //int handling_import = 0;\r
    //int handling_with = 0;\r
\r
    if ((ps = PyParser_New(g, start)) == NULL) {\r
        fprintf(stderr, "no mem for new parser\n");\r
        err_ret->error = E_NOMEM;\r
        PyTokenizer_Free(tok);\r
        return NULL;\r
    }\r
#ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD\r
    if (*flags & PyPARSE_PRINT_IS_FUNCTION) {\r
        ps->p_flags |= CO_FUTURE_PRINT_FUNCTION;\r
    }\r
    if (*flags & PyPARSE_UNICODE_LITERALS) {\r
        ps->p_flags |= CO_FUTURE_UNICODE_LITERALS;\r
    }\r
\r
#endif\r
\r
    for (;;) {\r
        char *a, *b;\r
        int type;\r
        size_t len;\r
        char *str;\r
        int col_offset;\r
\r
        type = PyTokenizer_Get(tok, &a, &b);\r
        if (type == ERRORTOKEN) {\r
            err_ret->error = tok->done;\r
            break;\r
        }\r
        if (type == ENDMARKER && started) {\r
            type = NEWLINE; /* Add an extra newline */\r
            //handling_with = 0;\r
            //handling_import = 0;\r
            started = 0;\r
            /* Add the right number of dedent tokens,\r
               except if a certain flag is given --\r
               codeop.py uses this. */\r
            if (tok->indent &&\r
                !(*flags & PyPARSE_DONT_IMPLY_DEDENT))\r
            {\r
                tok->pendin = -tok->indent;\r
                tok->indent = 0;\r
            }\r
        }\r
        else\r
            started = 1;\r
        len = b - a; /* XXX this may compute NULL - NULL */\r
        str = (char *) PyObject_MALLOC(len + 1);\r
        if (str == NULL) {\r
            fprintf(stderr, "no mem for next token\n");\r
            err_ret->error = E_NOMEM;\r
            break;\r
        }\r
        if (len > 0)\r
            strncpy(str, a, len);\r
        str[len] = '\0';\r
\r
#ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD\r
#endif\r
        if (a >= tok->line_start)\r
            col_offset = a - tok->line_start;\r
        else\r
            col_offset = -1;\r
\r
        if ((err_ret->error =\r
             PyParser_AddToken(ps, (int)type, str, tok->lineno, col_offset,\r
                               &(err_ret->expected))) != E_OK) {\r
            if (err_ret->error != E_DONE) {\r
                PyObject_FREE(str);\r
                err_ret->token = type;\r
            }\r
            break;\r
        }\r
    }\r
\r
    if (err_ret->error == E_DONE) {\r
        n = ps->p_tree;\r
        ps->p_tree = NULL;\r
    }\r
    else\r
        n = NULL;\r
\r
#ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD\r
    *flags = ps->p_flags;\r
#endif\r
    PyParser_Delete(ps);\r
\r
    if (n == NULL) {\r
        if (tok->lineno <= 1 && tok->done == E_EOF)\r
            err_ret->error = E_EOF;\r
        err_ret->lineno = tok->lineno;\r
        if (tok->buf != NULL) {\r
            char *text = NULL;\r
            size_t len;\r
            assert(tok->cur - tok->buf < INT_MAX);\r
            err_ret->offset = (int)(tok->cur - tok->buf);\r
            len = tok->inp - tok->buf;\r
#ifdef Py_USING_UNICODE\r
            text = PyTokenizer_RestoreEncoding(tok, len, &err_ret->offset);\r
\r
#endif\r
            if (text == NULL) {\r
                text = (char *) PyObject_MALLOC(len + 1);\r
                if (text != NULL) {\r
                    if (len > 0)\r
                        strncpy(text, tok->buf, len);\r
                    text[len] = '\0';\r
                }\r
            }\r
            err_ret->text = text;\r
        }\r
    } else if (tok->encoding != NULL) {\r
        /* 'nodes->n_str' uses PyObject_*, while 'tok->encoding' was\r
         * allocated using PyMem_\r
         */\r
        node* r = PyNode_New(encoding_decl);\r
        if (r)\r
            r->n_str = PyObject_MALLOC(strlen(tok->encoding)+1);\r
        if (!r || !r->n_str) {\r
            err_ret->error = E_NOMEM;\r
            if (r)\r
                PyObject_FREE(r);\r
            n = NULL;\r
            goto done;\r
        }\r
        strcpy(r->n_str, tok->encoding);\r
        PyMem_FREE(tok->encoding);\r
        tok->encoding = NULL;\r
        r->n_nchildren = 1;\r
        r->n_child = n;\r
        n = r;\r
    }\r
\r
done:\r
    PyTokenizer_Free(tok);\r
\r
    return n;\r
}\r
\r
static void\r
initerr(perrdetail *err_ret, const char *filename)\r
{\r
    err_ret->error = E_OK;\r
    err_ret->filename = filename;\r
    err_ret->lineno = 0;\r
    err_ret->offset = 0;\r
    err_ret->text = NULL;\r
    err_ret->token = -1;\r
    err_ret->expected = -1;\r
}\r
Commit	Line	Data
b81cc7d6 OM	1	/** @file\r
	2	Parser-tokenizer link implementation.\r
	3	\r
	4	Copyright (c) 2014, Intel Corporation. All rights reserved.<BR>\r
	5	This program and the accompanying materials are licensed and made available under\r
	6	the terms and conditions of the BSD License that accompanies this distribution.\r
	7	The full text of the license may be found at\r
	8	http://opensource.org/licenses/bsd-license.php.\r
	9	\r
	10	THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,\r
	11	WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.\r
	12	**/\r
4710c53d	13	#include "pgenheaders.h"\r
	14	#include "tokenizer.h"\r
	15	#include "node.h"\r
	16	#include "grammar.h"\r
	17	#include "parser.h"\r
	18	#include "parsetok.h"\r
	19	#include "errcode.h"\r
	20	#include "graminit.h"\r
	21	\r
	22	int Py_TabcheckFlag;\r
	23	\r
	24	\r
	25	/* Forward */\r
	26	static node parsetok(struct tok_state , grammar , int, perrdetail , int *);\r
	27	static void initerr(perrdetail err_ret, const char filename);\r
	28	\r
	29	/* Parse input coming from a string. Return error code, print some errors. */\r
	30	node *\r
	31	PyParser_ParseString(const char s, grammar g, int start, perrdetail *err_ret)\r
	32	{\r
	33	return PyParser_ParseStringFlagsFilename(s, NULL, g, start, err_ret, 0);\r
	34	}\r
	35	\r
	36	node *\r
	37	PyParser_ParseStringFlags(const char s, grammar g, int start,\r
	38	perrdetail *err_ret, int flags)\r
	39	{\r
	40	return PyParser_ParseStringFlagsFilename(s, NULL,\r
	41	g, start, err_ret, flags);\r
	42	}\r
	43	\r
	44	node *\r
	45	PyParser_ParseStringFlagsFilename(const char s, const char filename,\r
	46	grammar *g, int start,\r
	47	perrdetail *err_ret, int flags)\r
	48	{\r
	49	int iflags = flags;\r
	50	return PyParser_ParseStringFlagsFilenameEx(s, filename, g, start,\r
	51	err_ret, &iflags);\r
	52	}\r
	53	\r
	54	node *\r
	55	PyParser_ParseStringFlagsFilenameEx(const char s, const char filename,\r
	56	grammar *g, int start,\r
	57	perrdetail err_ret, int flags)\r
	58	{\r
	59	struct tok_state *tok;\r
	60	\r
	61	initerr(err_ret, filename);\r
	62	\r
	63	if ((tok = PyTokenizer_FromString(s, start == file_input)) == NULL) {\r
	64	err_ret->error = PyErr_Occurred() ? E_DECODE : E_NOMEM;\r
	65	return NULL;\r
	66	}\r
	67	\r
	68	tok->filename = filename ? filename : "<string>";\r
	69	if (Py_TabcheckFlag \|\| Py_VerboseFlag) {\r
	70	tok->altwarning = (tok->filename != NULL);\r
	71	if (Py_TabcheckFlag >= 2)\r
	72	tok->alterror++;\r
	73	}\r
	74	\r
	75	return parsetok(tok, g, start, err_ret, flags);\r
	76	}\r
77	\r
78	/* Parse input coming from a file. Return error code, print some errors. */\r
79	\r
80	node *\r
81	PyParser_ParseFile(FILE fp, const char filename, grammar *g, int start,\r
82	char ps1, char ps2, perrdetail *err_ret)\r
83	{\r
84	return PyParser_ParseFileFlags(fp, filename, g, start, ps1, ps2,\r
85	err_ret, 0);\r
86	}\r
87	\r
88	node *\r
89	PyParser_ParseFileFlags(FILE fp, const char filename, grammar *g, int start,\r
90	char ps1, char ps2, perrdetail *err_ret, int flags)\r
91	{\r
92	int iflags = flags;\r
93	return PyParser_ParseFileFlagsEx(fp, filename, g, start, ps1, ps2, err_ret, &iflags);\r
94	}\r
95	\r
96	node *\r
97	PyParser_ParseFileFlagsEx(FILE fp, const char filename, grammar *g, int start,\r
98	char ps1, char ps2, perrdetail err_ret, int flags)\r
99	{\r
100	struct tok_state *tok;\r
101	\r
102	initerr(err_ret, filename);\r
103	\r
104	if ((tok = PyTokenizer_FromFile(fp, ps1, ps2)) == NULL) {\r
105	err_ret->error = E_NOMEM;\r
106	return NULL;\r
107	}\r
108	tok->filename = filename;\r
109	if (Py_TabcheckFlag \|\| Py_VerboseFlag) {\r
110	tok->altwarning = (filename != NULL);\r
111	if (Py_TabcheckFlag >= 2)\r
112	tok->alterror++;\r
113	}\r
114	\r
115	return parsetok(tok, g, start, err_ret, flags);\r
116	}\r
117	\r
118	#if 0\r
119	static char with_msg[] =\r
120	"%s:%d: Warning: 'with' will become a reserved keyword in Python 2.6\n";\r
121	\r
122	static char as_msg[] =\r
123	"%s:%d: Warning: 'as' will become a reserved keyword in Python 2.6\n";\r
124	\r
125	static void\r
126	warn(const char msg, const char filename, int lineno)\r
127	{\r
128	if (filename == NULL)\r
129	filename = "<string>";\r
130	PySys_WriteStderr(msg, filename, lineno);\r
131	}\r
132	#endif\r
133	\r
134	/* Parse input coming from the given tokenizer structure.\r
135	Return error code. */\r
136	\r
137	static node *\r
138	parsetok(struct tok_state tok, grammar g, int start, perrdetail *err_ret,\r
139	int *flags)\r
140	{\r
141	parser_state *ps;\r
142	node *n;\r
b81cc7d6 OM	143	int started = 0;\r
	144	//int handling_import = 0;\r
	145	//int handling_with = 0;\r
4710c53d	146	\r
	147	if ((ps = PyParser_New(g, start)) == NULL) {\r
	148	fprintf(stderr, "no mem for new parser\n");\r
	149	err_ret->error = E_NOMEM;\r
	150	PyTokenizer_Free(tok);\r
	151	return NULL;\r
	152	}\r
	153	#ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD\r
	154	if (*flags & PyPARSE_PRINT_IS_FUNCTION) {\r
	155	ps->p_flags \|= CO_FUTURE_PRINT_FUNCTION;\r
	156	}\r
	157	if (*flags & PyPARSE_UNICODE_LITERALS) {\r
	158	ps->p_flags \|= CO_FUTURE_UNICODE_LITERALS;\r
	159	}\r
	160	\r
	161	#endif\r
	162	\r
	163	for (;;) {\r
	164	char a, b;\r
	165	int type;\r
	166	size_t len;\r
	167	char *str;\r
	168	int col_offset;\r
	169	\r
	170	type = PyTokenizer_Get(tok, &a, &b);\r
	171	if (type == ERRORTOKEN) {\r
	172	err_ret->error = tok->done;\r
	173	break;\r
	174	}\r
	175	if (type == ENDMARKER && started) {\r
	176	type = NEWLINE; /* Add an extra newline */\r
b81cc7d6 OM	177	//handling_with = 0;\r
b81cc7d6 OM	178	//handling_import = 0;\r
4710c53d	179	started = 0;\r
	180	/* Add the right number of dedent tokens,\r
	181	except if a certain flag is given --\r
	182	codeop.py uses this. */\r
	183	if (tok->indent &&\r
	184	!(*flags & PyPARSE_DONT_IMPLY_DEDENT))\r
	185	{\r
	186	tok->pendin = -tok->indent;\r
	187	tok->indent = 0;\r
	188	}\r
	189	}\r
	190	else\r
	191	started = 1;\r
	192	len = b - a; /* XXX this may compute NULL - NULL */\r
	193	str = (char *) PyObject_MALLOC(len + 1);\r
	194	if (str == NULL) {\r
	195	fprintf(stderr, "no mem for next token\n");\r
	196	err_ret->error = E_NOMEM;\r
	197	break;\r
	198	}\r
	199	if (len > 0)\r
	200	strncpy(str, a, len);\r
	201	str[len] = '\0';\r
	202	\r
	203	#ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD\r
	204	#endif\r
	205	if (a >= tok->line_start)\r
	206	col_offset = a - tok->line_start;\r
	207	else\r
	208	col_offset = -1;\r
	209	\r
	210	if ((err_ret->error =\r
	211	PyParser_AddToken(ps, (int)type, str, tok->lineno, col_offset,\r
	212	&(err_ret->expected))) != E_OK) {\r
	213	if (err_ret->error != E_DONE) {\r
	214	PyObject_FREE(str);\r
	215	err_ret->token = type;\r
	216	}\r
	217	break;\r
	218	}\r
	219	}\r
	220	\r
	221	if (err_ret->error == E_DONE) {\r
	222	n = ps->p_tree;\r
	223	ps->p_tree = NULL;\r
	224	}\r
	225	else\r
	226	n = NULL;\r
	227	\r
	228	#ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD\r
	229	*flags = ps->p_flags;\r
	230	#endif\r
	231	PyParser_Delete(ps);\r
	232	\r
	233	if (n == NULL) {\r
	234	if (tok->lineno <= 1 && tok->done == E_EOF)\r
	235	err_ret->error = E_EOF;\r
	236	err_ret->lineno = tok->lineno;\r
	237	if (tok->buf != NULL) {\r
	238	char *text = NULL;\r
	239	size_t len;\r
	240	assert(tok->cur - tok->buf < INT_MAX);\r
	241	err_ret->offset = (int)(tok->cur - tok->buf);\r
	242	len = tok->inp - tok->buf;\r
243	#ifdef Py_USING_UNICODE\r
244	text = PyTokenizer_RestoreEncoding(tok, len, &err_ret->offset);\r
245	\r
246	#endif\r
247	if (text == NULL) {\r
248	text = (char *) PyObject_MALLOC(len + 1);\r
249	if (text != NULL) {\r
250	if (len > 0)\r
251	strncpy(text, tok->buf, len);\r
252	text[len] = '\0';\r
253	}\r
254	}\r
255	err_ret->text = text;\r
256	}\r
257	} else if (tok->encoding != NULL) {\r
258	/* 'nodes->n_str' uses PyObject_*, while 'tok->encoding' was\r
259	* allocated using PyMem_\r
260	*/\r
261	node* r = PyNode_New(encoding_decl);\r
262	if (r)\r
263	r->n_str = PyObject_MALLOC(strlen(tok->encoding)+1);\r
264	if (!r \|\| !r->n_str) {\r
265	err_ret->error = E_NOMEM;\r
266	if (r)\r
267	PyObject_FREE(r);\r
268	n = NULL;\r
269	goto done;\r
270	}\r
271	strcpy(r->n_str, tok->encoding);\r
272	PyMem_FREE(tok->encoding);\r
273	tok->encoding = NULL;\r
274	r->n_nchildren = 1;\r
275	r->n_child = n;\r
276	n = r;\r
277	}\r
278	\r
279	done:\r
280	PyTokenizer_Free(tok);\r
281	\r
282	return n;\r
283	}\r
284	\r
285	static void\r
286	initerr(perrdetail err_ret, const char filename)\r
287	{\r
288	err_ret->error = E_OK;\r
289	err_ret->filename = filename;\r
290	err_ret->lineno = 0;\r
291	err_ret->offset = 0;\r
292	err_ret->text = NULL;\r
293	err_ret->token = -1;\r
294	err_ret->expected = -1;\r
295	}\r