-\r
-/* Tokenizer implementation */\r
-\r
-#include "Python.h"\r
-#include "pgenheaders.h"\r
-\r
-#include <ctype.h>\r
-#include <assert.h>\r
-\r
-#include "tokenizer.h"\r
-#include "errcode.h"\r
-\r
-#ifndef PGEN\r
-#include "unicodeobject.h"\r
-#include "stringobject.h"\r
-#include "fileobject.h"\r
-#include "codecs.h"\r
-#include "abstract.h"\r
-#include "pydebug.h"\r
-#endif /* PGEN */\r
-\r
-extern char *PyOS_Readline(FILE *, FILE *, char *);\r
-/* Return malloc'ed string including trailing \n;\r
- empty malloc'ed string for EOF;\r
- NULL if interrupted */\r
-\r
-/* Don't ever change this -- it would break the portability of Python code */\r
-#define TABSIZE 8\r
-\r
-/* Forward */\r
-static struct tok_state *tok_new(void);\r
-static int tok_nextc(struct tok_state *tok);\r
-static void tok_backup(struct tok_state *tok, int c);\r
-\r
-/* Token names */\r
-\r
-char *_PyParser_TokenNames[] = {\r
- "ENDMARKER",\r
- "NAME",\r
- "NUMBER",\r
- "STRING",\r
- "NEWLINE",\r
- "INDENT",\r
- "DEDENT",\r
- "LPAR",\r
- "RPAR",\r
- "LSQB",\r
- "RSQB",\r
- "COLON",\r
- "COMMA",\r
- "SEMI",\r
- "PLUS",\r
- "MINUS",\r
- "STAR",\r
- "SLASH",\r
- "VBAR",\r
- "AMPER",\r
- "LESS",\r
- "GREATER",\r
- "EQUAL",\r
- "DOT",\r
- "PERCENT",\r
- "BACKQUOTE",\r
- "LBRACE",\r
- "RBRACE",\r
- "EQEQUAL",\r
- "NOTEQUAL",\r
- "LESSEQUAL",\r
- "GREATEREQUAL",\r
- "TILDE",\r
- "CIRCUMFLEX",\r
- "LEFTSHIFT",\r
- "RIGHTSHIFT",\r
- "DOUBLESTAR",\r
- "PLUSEQUAL",\r
- "MINEQUAL",\r
- "STAREQUAL",\r
- "SLASHEQUAL",\r
- "PERCENTEQUAL",\r
- "AMPEREQUAL",\r
- "VBAREQUAL",\r
- "CIRCUMFLEXEQUAL",\r
- "LEFTSHIFTEQUAL",\r
- "RIGHTSHIFTEQUAL",\r
- "DOUBLESTAREQUAL",\r
- "DOUBLESLASH",\r
- "DOUBLESLASHEQUAL",\r
- "AT",\r
- /* This table must match the #defines in token.h! */\r
- "OP",\r
- "<ERRORTOKEN>",\r
- "<N_TOKENS>"\r
-};\r
-\r
-/* Create and initialize a new tok_state structure */\r
-\r
-static struct tok_state *\r
-tok_new(void)\r
-{\r
- struct tok_state *tok = (struct tok_state *)PyMem_MALLOC(\r
- sizeof(struct tok_state));\r
- if (tok == NULL)\r
- return NULL;\r
- tok->buf = tok->cur = tok->end = tok->inp = tok->start = NULL;\r
- tok->done = E_OK;\r
- tok->fp = NULL;\r
- tok->input = NULL;\r
- tok->tabsize = TABSIZE;\r
- tok->indent = 0;\r
- tok->indstack[0] = 0;\r
- tok->atbol = 1;\r
- tok->pendin = 0;\r
- tok->prompt = tok->nextprompt = NULL;\r
- tok->lineno = 0;\r
- tok->level = 0;\r
- tok->filename = NULL;\r
- tok->altwarning = 0;\r
- tok->alterror = 0;\r
- tok->alttabsize = 1;\r
- tok->altindstack[0] = 0;\r
- tok->decoding_state = 0;\r
- tok->decoding_erred = 0;\r
- tok->read_coding_spec = 0;\r
- tok->encoding = NULL;\r
- tok->cont_line = 0;\r
-#ifndef PGEN\r
- tok->decoding_readline = NULL;\r
- tok->decoding_buffer = NULL;\r
-#endif\r
- return tok;\r
-}\r
-\r
-static char *\r
-new_string(const char *s, Py_ssize_t len)\r
-{\r
- char* result = (char *)PyMem_MALLOC(len + 1);\r
- if (result != NULL) {\r
- memcpy(result, s, len);\r
- result[len] = '\0';\r
- }\r
- return result;\r
-}\r
-\r
-#ifdef PGEN\r
-\r
-static char *\r
-decoding_fgets(char *s, int size, struct tok_state *tok)\r
-{\r
- return fgets(s, size, tok->fp);\r
-}\r
-\r
-static int\r
-decoding_feof(struct tok_state *tok)\r
-{\r
- return feof(tok->fp);\r
-}\r
-\r
-static char *\r
-decode_str(const char *str, int exec_input, struct tok_state *tok)\r
-{\r
- return new_string(str, strlen(str));\r
-}\r
-\r
-#else /* PGEN */\r
-\r
-static char *\r
-error_ret(struct tok_state *tok) /* XXX */\r
-{\r
- tok->decoding_erred = 1;\r
- if (tok->fp != NULL && tok->buf != NULL) /* see PyTokenizer_Free */\r
- PyMem_FREE(tok->buf);\r
- tok->buf = NULL;\r
- return NULL; /* as if it were EOF */\r
-}\r
-\r
-\r
-static char *\r
-get_normal_name(char *s) /* for utf-8 and latin-1 */\r
-{\r
- char buf[13];\r
- int i;\r
- for (i = 0; i < 12; i++) {\r
- int c = s[i];\r
- if (c == '\0')\r
- break;\r
- else if (c == '_')\r
- buf[i] = '-';\r
- else\r
- buf[i] = tolower(c);\r
- }\r
- buf[i] = '\0';\r
- if (strcmp(buf, "utf-8") == 0 ||\r
- strncmp(buf, "utf-8-", 6) == 0)\r
- return "utf-8";\r
- else if (strcmp(buf, "latin-1") == 0 ||\r
- strcmp(buf, "iso-8859-1") == 0 ||\r
- strcmp(buf, "iso-latin-1") == 0 ||\r
- strncmp(buf, "latin-1-", 8) == 0 ||\r
- strncmp(buf, "iso-8859-1-", 11) == 0 ||\r
- strncmp(buf, "iso-latin-1-", 12) == 0)\r
- return "iso-8859-1";\r
- else\r
- return s;\r
-}\r
-\r
-/* Return the coding spec in S, or NULL if none is found. */\r
-\r
-static char *\r
-get_coding_spec(const char *s, Py_ssize_t size)\r
-{\r
- Py_ssize_t i;\r
- /* Coding spec must be in a comment, and that comment must be\r
- * the only statement on the source code line. */\r
- for (i = 0; i < size - 6; i++) {\r
- if (s[i] == '#')\r
- break;\r
- if (s[i] != ' ' && s[i] != '\t' && s[i] != '\014')\r
- return NULL;\r
- }\r
- for (; i < size - 6; i++) { /* XXX inefficient search */\r
- const char* t = s + i;\r
- if (strncmp(t, "coding", 6) == 0) {\r
- const char* begin = NULL;\r
- t += 6;\r
- if (t[0] != ':' && t[0] != '=')\r
- continue;\r
- do {\r
- t++;\r
- } while (t[0] == '\x20' || t[0] == '\t');\r
-\r
- begin = t;\r
- while (Py_ISALNUM(t[0]) ||\r
- t[0] == '-' || t[0] == '_' || t[0] == '.')\r
- t++;\r
-\r
- if (begin < t) {\r
- char* r = new_string(begin, t - begin);\r
- char* q = get_normal_name(r);\r
- if (r != q) {\r
- PyMem_FREE(r);\r
- r = new_string(q, strlen(q));\r
- }\r
- return r;\r
- }\r
- }\r
- }\r
- return NULL;\r
-}\r
-\r
-/* Check whether the line contains a coding spec. If it does,\r
- invoke the set_readline function for the new encoding.\r
- This function receives the tok_state and the new encoding.\r
- Return 1 on success, 0 on failure. */\r
-\r
-static int\r
-check_coding_spec(const char* line, Py_ssize_t size, struct tok_state *tok,\r
- int set_readline(struct tok_state *, const char *))\r
-{\r
- char * cs;\r
- int r = 1;\r
-\r
- if (tok->cont_line) {\r
- /* It's a continuation line, so it can't be a coding spec. */\r
- tok->read_coding_spec = 1;\r
- return 1;\r
- }\r
- cs = get_coding_spec(line, size);\r
- if (!cs) {\r
- Py_ssize_t i;\r
- for (i = 0; i < size; i++) {\r
- if (line[i] == '#' || line[i] == '\n' || line[i] == '\r')\r
- break;\r
- if (line[i] != ' ' && line[i] != '\t' && line[i] != '\014') {\r
- /* Stop checking coding spec after a line containing\r
- * anything except a comment. */\r
- tok->read_coding_spec = 1;\r
- break;\r
- }\r
- }\r
- } else {\r
- tok->read_coding_spec = 1;\r
- if (tok->encoding == NULL) {\r
- assert(tok->decoding_state == 1); /* raw */\r
- if (strcmp(cs, "utf-8") == 0 ||\r
- strcmp(cs, "iso-8859-1") == 0) {\r
- tok->encoding = cs;\r
- } else {\r
-#ifdef Py_USING_UNICODE\r
- r = set_readline(tok, cs);\r
- if (r) {\r
- tok->encoding = cs;\r
- tok->decoding_state = -1;\r
- }\r
- else {\r
- PyErr_Format(PyExc_SyntaxError,\r
- "encoding problem: %s", cs);\r
- PyMem_FREE(cs);\r
- }\r
-#else\r
- /* Without Unicode support, we cannot\r
- process the coding spec. Since there\r
- won't be any Unicode literals, that\r
- won't matter. */\r
- PyMem_FREE(cs);\r
-#endif\r
- }\r
- } else { /* then, compare cs with BOM */\r
- r = (strcmp(tok->encoding, cs) == 0);\r
- if (!r)\r
- PyErr_Format(PyExc_SyntaxError,\r
- "encoding problem: %s with BOM", cs);\r
- PyMem_FREE(cs);\r
- }\r
- }\r
- return r;\r
-}\r
-\r
-/* See whether the file starts with a BOM. If it does,\r
- invoke the set_readline function with the new encoding.\r
- Return 1 on success, 0 on failure. */\r
-\r
-static int\r
-check_bom(int get_char(struct tok_state *),\r
- void unget_char(int, struct tok_state *),\r
- int set_readline(struct tok_state *, const char *),\r
- struct tok_state *tok)\r
-{\r
- int ch1, ch2, ch3;\r
- ch1 = get_char(tok);\r
- tok->decoding_state = 1;\r
- if (ch1 == EOF) {\r
- return 1;\r
- } else if (ch1 == 0xEF) {\r
- ch2 = get_char(tok);\r
- if (ch2 != 0xBB) {\r
- unget_char(ch2, tok);\r
- unget_char(ch1, tok);\r
- return 1;\r
- }\r
- ch3 = get_char(tok);\r
- if (ch3 != 0xBF) {\r
- unget_char(ch3, tok);\r
- unget_char(ch2, tok);\r
- unget_char(ch1, tok);\r
- return 1;\r
- }\r
-#if 0\r
- /* Disable support for UTF-16 BOMs until a decision\r
- is made whether this needs to be supported. */\r
- } else if (ch1 == 0xFE) {\r
- ch2 = get_char(tok);\r
- if (ch2 != 0xFF) {\r
- unget_char(ch2, tok);\r
- unget_char(ch1, tok);\r
- return 1;\r
- }\r
- if (!set_readline(tok, "utf-16-be"))\r
- return 0;\r
- tok->decoding_state = -1;\r
- } else if (ch1 == 0xFF) {\r
- ch2 = get_char(tok);\r
- if (ch2 != 0xFE) {\r
- unget_char(ch2, tok);\r
- unget_char(ch1, tok);\r
- return 1;\r
- }\r
- if (!set_readline(tok, "utf-16-le"))\r
- return 0;\r
- tok->decoding_state = -1;\r
-#endif\r
- } else {\r
- unget_char(ch1, tok);\r
- return 1;\r
- }\r
- if (tok->encoding != NULL)\r
- PyMem_FREE(tok->encoding);\r
- tok->encoding = new_string("utf-8", 5); /* resulting is in utf-8 */\r
- return 1;\r
-}\r
-\r
-/* Read a line of text from TOK into S, using the stream in TOK.\r
- Return NULL on failure, else S.\r
-\r
- On entry, tok->decoding_buffer will be one of:\r
- 1) NULL: need to call tok->decoding_readline to get a new line\r
- 2) PyUnicodeObject *: decoding_feof has called tok->decoding_readline and\r
- stored the result in tok->decoding_buffer\r
- 3) PyStringObject *: previous call to fp_readl did not have enough room\r
- (in the s buffer) to copy entire contents of the line read\r
- by tok->decoding_readline. tok->decoding_buffer has the overflow.\r
- In this case, fp_readl is called in a loop (with an expanded buffer)\r
- until the buffer ends with a '\n' (or until the end of the file is\r
- reached): see tok_nextc and its calls to decoding_fgets.\r
-*/\r
-\r
-static char *\r
-fp_readl(char *s, int size, struct tok_state *tok)\r
-{\r
-#ifndef Py_USING_UNICODE\r
- /* In a non-Unicode built, this should never be called. */\r
- Py_FatalError("fp_readl should not be called in this build.");\r
- return NULL; /* Keep compiler happy (not reachable) */\r
-#else\r
- PyObject* utf8 = NULL;\r
- PyObject* buf = tok->decoding_buffer;\r
- char *str;\r
- Py_ssize_t utf8len;\r
-\r
- /* Ask for one less byte so we can terminate it */\r
- assert(size > 0);\r
- size--;\r
-\r
- if (buf == NULL) {\r
- buf = PyObject_CallObject(tok->decoding_readline, NULL);\r
- if (buf == NULL)\r
- return error_ret(tok);\r
- if (!PyUnicode_Check(buf)) {\r
- Py_DECREF(buf);\r
- PyErr_SetString(PyExc_SyntaxError,\r
- "codec did not return a unicode object");\r
- return error_ret(tok);\r
- }\r
- } else {\r
- tok->decoding_buffer = NULL;\r
- if (PyString_CheckExact(buf))\r
- utf8 = buf;\r
- }\r
- if (utf8 == NULL) {\r
- utf8 = PyUnicode_AsUTF8String(buf);\r
- Py_DECREF(buf);\r
- if (utf8 == NULL)\r
- return error_ret(tok);\r
- }\r
- str = PyString_AsString(utf8);\r
- utf8len = PyString_GET_SIZE(utf8);\r
- if (utf8len > size) {\r
- tok->decoding_buffer = PyString_FromStringAndSize(str+size, utf8len-size);\r
- if (tok->decoding_buffer == NULL) {\r
- Py_DECREF(utf8);\r
- return error_ret(tok);\r
- }\r
- utf8len = size;\r
- }\r
- memcpy(s, str, utf8len);\r
- s[utf8len] = '\0';\r
- Py_DECREF(utf8);\r
- if (utf8len == 0)\r
- return NULL; /* EOF */\r
- return s;\r
-#endif\r
-}\r
-\r
-/* Set the readline function for TOK to a StreamReader's\r
- readline function. The StreamReader is named ENC.\r
-\r
- This function is called from check_bom and check_coding_spec.\r
-\r
- ENC is usually identical to the future value of tok->encoding,\r
- except for the (currently unsupported) case of UTF-16.\r
-\r
- Return 1 on success, 0 on failure. */\r
-\r
-static int\r
-fp_setreadl(struct tok_state *tok, const char* enc)\r
-{\r
- PyObject *reader, *stream, *readline;\r
-\r
- /* XXX: constify filename argument. */\r
- stream = PyFile_FromFile(tok->fp, (char*)tok->filename, "rb", NULL);\r
- if (stream == NULL)\r
- return 0;\r
-\r
- reader = PyCodec_StreamReader(enc, stream, NULL);\r
- Py_DECREF(stream);\r
- if (reader == NULL)\r
- return 0;\r
-\r
- readline = PyObject_GetAttrString(reader, "readline");\r
- Py_DECREF(reader);\r
- if (readline == NULL)\r
- return 0;\r
-\r
- tok->decoding_readline = readline;\r
- return 1;\r
-}\r
-\r
-/* Fetch the next byte from TOK. */\r
-\r
-static int fp_getc(struct tok_state *tok) {\r
- return getc(tok->fp);\r
-}\r
-\r
-/* Unfetch the last byte back into TOK. */\r
-\r
-static void fp_ungetc(int c, struct tok_state *tok) {\r
- ungetc(c, tok->fp);\r
-}\r
-\r
-/* Read a line of input from TOK. Determine encoding\r
- if necessary. */\r
-\r
-static char *\r
-decoding_fgets(char *s, int size, struct tok_state *tok)\r
-{\r
- char *line = NULL;\r
- int badchar = 0;\r
- for (;;) {\r
- if (tok->decoding_state < 0) {\r
- /* We already have a codec associated with\r
- this input. */\r
- line = fp_readl(s, size, tok);\r
- break;\r
- } else if (tok->decoding_state > 0) {\r
- /* We want a 'raw' read. */\r
- line = Py_UniversalNewlineFgets(s, size,\r
- tok->fp, NULL);\r
- break;\r
- } else {\r
- /* We have not yet determined the encoding.\r
- If an encoding is found, use the file-pointer\r
- reader functions from now on. */\r
- if (!check_bom(fp_getc, fp_ungetc, fp_setreadl, tok))\r
- return error_ret(tok);\r
- assert(tok->decoding_state != 0);\r
- }\r
- }\r
- if (line != NULL && tok->lineno < 2 && !tok->read_coding_spec) {\r
- if (!check_coding_spec(line, strlen(line), tok, fp_setreadl)) {\r
- return error_ret(tok);\r
- }\r
- }\r
-#ifndef PGEN\r
- /* The default encoding is ASCII, so make sure we don't have any\r
- non-ASCII bytes in it. */\r
- if (line && !tok->encoding) {\r
- unsigned char *c;\r
- for (c = (unsigned char *)line; *c; c++)\r
- if (*c > 127) {\r
- badchar = *c;\r
- break;\r
- }\r
- }\r
- if (badchar) {\r
- char buf[500];\r
- /* Need to add 1 to the line number, since this line\r
- has not been counted, yet. */\r
- sprintf(buf,\r
- "Non-ASCII character '\\x%.2x' "\r
- "in file %.200s on line %i, "\r
- "but no encoding declared; "\r
- "see http://python.org/dev/peps/pep-0263/ for details",\r
- badchar, tok->filename, tok->lineno + 1);\r
- PyErr_SetString(PyExc_SyntaxError, buf);\r
- return error_ret(tok);\r
- }\r
-#endif\r
- return line;\r
-}\r
-\r
-static int\r
-decoding_feof(struct tok_state *tok)\r
-{\r
- if (tok->decoding_state >= 0) {\r
- return feof(tok->fp);\r
- } else {\r
- PyObject* buf = tok->decoding_buffer;\r
- if (buf == NULL) {\r
- buf = PyObject_CallObject(tok->decoding_readline, NULL);\r
- if (buf == NULL) {\r
- error_ret(tok);\r
- return 1;\r
- } else {\r
- tok->decoding_buffer = buf;\r
- }\r
- }\r
- return PyObject_Length(buf) == 0;\r
- }\r
-}\r
-\r
-/* Fetch a byte from TOK, using the string buffer. */\r
-\r
-static int\r
-buf_getc(struct tok_state *tok) {\r
- return Py_CHARMASK(*tok->str++);\r
-}\r
-\r
-/* Unfetch a byte from TOK, using the string buffer. */\r
-\r
-static void\r
-buf_ungetc(int c, struct tok_state *tok) {\r
- tok->str--;\r
- assert(Py_CHARMASK(*tok->str) == c); /* tok->cur may point to read-only segment */\r
-}\r
-\r
-/* Set the readline function for TOK to ENC. For the string-based\r
- tokenizer, this means to just record the encoding. */\r
-\r
-static int\r
-buf_setreadl(struct tok_state *tok, const char* enc) {\r
- tok->enc = enc;\r
- return 1;\r
-}\r
-\r
-/* Return a UTF-8 encoding Python string object from the\r
- C byte string STR, which is encoded with ENC. */\r
-\r
-#ifdef Py_USING_UNICODE\r
-static PyObject *\r
-translate_into_utf8(const char* str, const char* enc) {\r
- PyObject *utf8;\r
- PyObject* buf = PyUnicode_Decode(str, strlen(str), enc, NULL);\r
- if (buf == NULL)\r
- return NULL;\r
- utf8 = PyUnicode_AsUTF8String(buf);\r
- Py_DECREF(buf);\r
- return utf8;\r
-}\r
-#endif\r
-\r
-\r
-static char *\r
-translate_newlines(const char *s, int exec_input, struct tok_state *tok) {\r
- int skip_next_lf = 0, needed_length = strlen(s) + 2, final_length;\r
- char *buf, *current;\r
- char c = '\0';\r
- buf = PyMem_MALLOC(needed_length);\r
- if (buf == NULL) {\r
- tok->done = E_NOMEM;\r
- return NULL;\r
- }\r
- for (current = buf; *s; s++, current++) {\r
- c = *s;\r
- if (skip_next_lf) {\r
- skip_next_lf = 0;\r
- if (c == '\n') {\r
- c = *++s;\r
- if (!c)\r
- break;\r
- }\r
- }\r
- if (c == '\r') {\r
- skip_next_lf = 1;\r
- c = '\n';\r
- }\r
- *current = c;\r
- }\r
- /* If this is exec input, add a newline to the end of the string if\r
- there isn't one already. */\r
- if (exec_input && c != '\n') {\r
- *current = '\n';\r
- current++;\r
- }\r
- *current = '\0';\r
- final_length = current - buf + 1;\r
- if (final_length < needed_length && final_length)\r
- /* should never fail */\r
- buf = PyMem_REALLOC(buf, final_length);\r
- return buf;\r
-}\r
-\r
-/* Decode a byte string STR for use as the buffer of TOK.\r
- Look for encoding declarations inside STR, and record them\r
- inside TOK. */\r
-\r
-static const char *\r
-decode_str(const char *input, int single, struct tok_state *tok)\r
-{\r
- PyObject* utf8 = NULL;\r
- const char *str;\r
- const char *s;\r
- const char *newl[2] = {NULL, NULL};\r
- int lineno = 0;\r
- tok->input = str = translate_newlines(input, single, tok);\r
- if (str == NULL)\r
- return NULL;\r
- tok->enc = NULL;\r
- tok->str = str;\r
- if (!check_bom(buf_getc, buf_ungetc, buf_setreadl, tok))\r
- return error_ret(tok);\r
- str = tok->str; /* string after BOM if any */\r
- assert(str);\r
-#ifdef Py_USING_UNICODE\r
- if (tok->enc != NULL) {\r
- utf8 = translate_into_utf8(str, tok->enc);\r
- if (utf8 == NULL)\r
- return error_ret(tok);\r
- str = PyString_AsString(utf8);\r
- }\r
-#endif\r
- for (s = str;; s++) {\r
- if (*s == '\0') break;\r
- else if (*s == '\n') {\r
- assert(lineno < 2);\r
- newl[lineno] = s;\r
- lineno++;\r
- if (lineno == 2) break;\r
- }\r
- }\r
- tok->enc = NULL;\r
- /* need to check line 1 and 2 separately since check_coding_spec\r
- assumes a single line as input */\r
- if (newl[0]) {\r
- if (!check_coding_spec(str, newl[0] - str, tok, buf_setreadl))\r
- return error_ret(tok);\r
- if (tok->enc == NULL && !tok->read_coding_spec && newl[1]) {\r
- if (!check_coding_spec(newl[0]+1, newl[1] - newl[0],\r
- tok, buf_setreadl))\r
- return error_ret(tok);\r
- }\r
- }\r
-#ifdef Py_USING_UNICODE\r
- if (tok->enc != NULL) {\r
- assert(utf8 == NULL);\r
- utf8 = translate_into_utf8(str, tok->enc);\r
- if (utf8 == NULL)\r
- return error_ret(tok);\r
- str = PyString_AsString(utf8);\r
- }\r
-#endif\r
- assert(tok->decoding_buffer == NULL);\r
- tok->decoding_buffer = utf8; /* CAUTION */\r
- return str;\r
-}\r
-\r
-#endif /* PGEN */\r
-\r
-/* Set up tokenizer for string */\r
-\r
-struct tok_state *\r
-PyTokenizer_FromString(const char *str, int exec_input)\r
-{\r
- struct tok_state *tok = tok_new();\r
- if (tok == NULL)\r
- return NULL;\r
- str = (char *)decode_str(str, exec_input, tok);\r
- if (str == NULL) {\r
- PyTokenizer_Free(tok);\r
- return NULL;\r
- }\r
-\r
- /* XXX: constify members. */\r
- tok->buf = tok->cur = tok->end = tok->inp = (char*)str;\r
- return tok;\r
-}\r
-\r
-\r
-/* Set up tokenizer for file */\r
-\r
-struct tok_state *\r
-PyTokenizer_FromFile(FILE *fp, char *ps1, char *ps2)\r
-{\r
- struct tok_state *tok = tok_new();\r
- if (tok == NULL)\r
- return NULL;\r
- if ((tok->buf = (char *)PyMem_MALLOC(BUFSIZ)) == NULL) {\r
- PyTokenizer_Free(tok);\r
- return NULL;\r
- }\r
- tok->cur = tok->inp = tok->buf;\r
- tok->end = tok->buf + BUFSIZ;\r
- tok->fp = fp;\r
- tok->prompt = ps1;\r
- tok->nextprompt = ps2;\r
- return tok;\r
-}\r
-\r
-\r
-/* Free a tok_state structure */\r
-\r
-void\r
-PyTokenizer_Free(struct tok_state *tok)\r
-{\r
- if (tok->encoding != NULL)\r
- PyMem_FREE(tok->encoding);\r
-#ifndef PGEN\r
- Py_XDECREF(tok->decoding_readline);\r
- Py_XDECREF(tok->decoding_buffer);\r
-#endif\r
- if (tok->fp != NULL && tok->buf != NULL)\r
- PyMem_FREE(tok->buf);\r
- if (tok->input)\r
- PyMem_FREE((char *)tok->input);\r
- PyMem_FREE(tok);\r
-}\r
-\r
-#if !defined(PGEN) && defined(Py_USING_UNICODE)\r
-static int\r
-tok_stdin_decode(struct tok_state *tok, char **inp)\r
-{\r
- PyObject *enc, *sysstdin, *decoded, *utf8;\r
- const char *encoding;\r
- char *converted;\r
-\r
- if (PySys_GetFile((char *)"stdin", NULL) != stdin)\r
- return 0;\r
- sysstdin = PySys_GetObject("stdin");\r
- if (sysstdin == NULL || !PyFile_Check(sysstdin))\r
- return 0;\r
-\r
- enc = ((PyFileObject *)sysstdin)->f_encoding;\r
- if (enc == NULL || !PyString_Check(enc))\r
- return 0;\r
- Py_INCREF(enc);\r
-\r
- encoding = PyString_AsString(enc);\r
- decoded = PyUnicode_Decode(*inp, strlen(*inp), encoding, NULL);\r
- if (decoded == NULL)\r
- goto error_clear;\r
-\r
- utf8 = PyUnicode_AsEncodedString(decoded, "utf-8", NULL);\r
- Py_DECREF(decoded);\r
- if (utf8 == NULL)\r
- goto error_clear;\r
-\r
- assert(PyString_Check(utf8));\r
- converted = new_string(PyString_AS_STRING(utf8),\r
- PyString_GET_SIZE(utf8));\r
- Py_DECREF(utf8);\r
- if (converted == NULL)\r
- goto error_nomem;\r
-\r
- PyMem_FREE(*inp);\r
- *inp = converted;\r
- if (tok->encoding != NULL)\r
- PyMem_FREE(tok->encoding);\r
- tok->encoding = new_string(encoding, strlen(encoding));\r
- if (tok->encoding == NULL)\r
- goto error_nomem;\r
-\r
- Py_DECREF(enc);\r
- return 0;\r
-\r
-error_nomem:\r
- Py_DECREF(enc);\r
- tok->done = E_NOMEM;\r
- return -1;\r
-\r
-error_clear:\r
- Py_DECREF(enc);\r
- if (!PyErr_ExceptionMatches(PyExc_UnicodeDecodeError)) {\r
- tok->done = E_ERROR;\r
- return -1;\r
- }\r
- /* Fallback to iso-8859-1: for backward compatibility */\r
- PyErr_Clear();\r
- return 0;\r
-}\r
-#endif\r
-\r
-/* Get next char, updating state; error code goes into tok->done */\r
-\r
-static int\r
-tok_nextc(register struct tok_state *tok)\r
-{\r
- for (;;) {\r
- if (tok->cur != tok->inp) {\r
- return Py_CHARMASK(*tok->cur++); /* Fast path */\r
- }\r
- if (tok->done != E_OK)\r
- return EOF;\r
- if (tok->fp == NULL) {\r
- char *end = strchr(tok->inp, '\n');\r
- if (end != NULL)\r
- end++;\r
- else {\r
- end = strchr(tok->inp, '\0');\r
- if (end == tok->inp) {\r
- tok->done = E_EOF;\r
- return EOF;\r
- }\r
- }\r
- if (tok->start == NULL)\r
- tok->buf = tok->cur;\r
- tok->line_start = tok->cur;\r
- tok->lineno++;\r
- tok->inp = end;\r
- return Py_CHARMASK(*tok->cur++);\r
- }\r
- if (tok->prompt != NULL) {\r
- char *newtok = PyOS_Readline(stdin, stdout, tok->prompt);\r
- if (tok->nextprompt != NULL)\r
- tok->prompt = tok->nextprompt;\r
- if (newtok == NULL)\r
- tok->done = E_INTR;\r
- else if (*newtok == '\0') {\r
- PyMem_FREE(newtok);\r
- tok->done = E_EOF;\r
- }\r
-#if !defined(PGEN) && defined(Py_USING_UNICODE)\r
- else if (tok_stdin_decode(tok, &newtok) != 0)\r
- PyMem_FREE(newtok);\r
-#endif\r
- else if (tok->start != NULL) {\r
- size_t start = tok->start - tok->buf;\r
- size_t oldlen = tok->cur - tok->buf;\r
- size_t newlen = oldlen + strlen(newtok);\r
- char *buf = tok->buf;\r
- buf = (char *)PyMem_REALLOC(buf, newlen+1);\r
- tok->lineno++;\r
- if (buf == NULL) {\r
- PyMem_FREE(tok->buf);\r
- tok->buf = NULL;\r
- PyMem_FREE(newtok);\r
- tok->done = E_NOMEM;\r
- return EOF;\r
- }\r
- tok->buf = buf;\r
- tok->cur = tok->buf + oldlen;\r
- tok->line_start = tok->cur;\r
- strcpy(tok->buf + oldlen, newtok);\r
- PyMem_FREE(newtok);\r
- tok->inp = tok->buf + newlen;\r
- tok->end = tok->inp + 1;\r
- tok->start = tok->buf + start;\r
- }\r
- else {\r
- tok->lineno++;\r
- if (tok->buf != NULL)\r
- PyMem_FREE(tok->buf);\r
- tok->buf = newtok;\r
- tok->line_start = tok->buf;\r
- tok->cur = tok->buf;\r
- tok->line_start = tok->buf;\r
- tok->inp = strchr(tok->buf, '\0');\r
- tok->end = tok->inp + 1;\r
- }\r
- }\r
- else {\r
- int done = 0;\r
- Py_ssize_t cur = 0;\r
- char *pt;\r
- if (tok->start == NULL) {\r
- if (tok->buf == NULL) {\r
- tok->buf = (char *)\r
- PyMem_MALLOC(BUFSIZ);\r
- if (tok->buf == NULL) {\r
- tok->done = E_NOMEM;\r
- return EOF;\r
- }\r
- tok->end = tok->buf + BUFSIZ;\r
- }\r
- if (decoding_fgets(tok->buf, (int)(tok->end - tok->buf),\r
- tok) == NULL) {\r
- tok->done = E_EOF;\r
- done = 1;\r
- }\r
- else {\r
- tok->done = E_OK;\r
- tok->inp = strchr(tok->buf, '\0');\r
- done = tok->inp[-1] == '\n';\r
- }\r
- }\r
- else {\r
- cur = tok->cur - tok->buf;\r
- if (decoding_feof(tok)) {\r
- tok->done = E_EOF;\r
- done = 1;\r
- }\r
- else\r
- tok->done = E_OK;\r
- }\r
- tok->lineno++;\r
- /* Read until '\n' or EOF */\r
- while (!done) {\r
- Py_ssize_t curstart = tok->start == NULL ? -1 :\r
- tok->start - tok->buf;\r
- Py_ssize_t curvalid = tok->inp - tok->buf;\r
- Py_ssize_t newsize = curvalid + BUFSIZ;\r
- char *newbuf = tok->buf;\r
- newbuf = (char *)PyMem_REALLOC(newbuf,\r
- newsize);\r
- if (newbuf == NULL) {\r
- tok->done = E_NOMEM;\r
- tok->cur = tok->inp;\r
- return EOF;\r
- }\r
- tok->buf = newbuf;\r
- tok->inp = tok->buf + curvalid;\r
- tok->end = tok->buf + newsize;\r
- tok->start = curstart < 0 ? NULL :\r
- tok->buf + curstart;\r
- if (decoding_fgets(tok->inp,\r
- (int)(tok->end - tok->inp),\r
- tok) == NULL) {\r
- /* Break out early on decoding\r
- errors, as tok->buf will be NULL\r
- */\r
- if (tok->decoding_erred)\r
- return EOF;\r
- /* Last line does not end in \n,\r
- fake one */\r
- strcpy(tok->inp, "\n");\r
- }\r
- tok->inp = strchr(tok->inp, '\0');\r
- done = tok->inp[-1] == '\n';\r
- }\r
- if (tok->buf != NULL) {\r
- tok->cur = tok->buf + cur;\r
- tok->line_start = tok->cur;\r
- /* replace "\r\n" with "\n" */\r
- /* For Mac leave the \r, giving a syntax error */\r
- pt = tok->inp - 2;\r
- if (pt >= tok->buf && *pt == '\r') {\r
- *pt++ = '\n';\r
- *pt = '\0';\r
- tok->inp = pt;\r
- }\r
- }\r
- }\r
- if (tok->done != E_OK) {\r
- if (tok->prompt != NULL)\r
- PySys_WriteStderr("\n");\r
- tok->cur = tok->inp;\r
- return EOF;\r
- }\r
- }\r
- /*NOTREACHED*/\r
-}\r
-\r
-\r
-/* Back-up one character */\r
-\r
-static void\r
-tok_backup(register struct tok_state *tok, register int c)\r
-{\r
- if (c != EOF) {\r
- if (--tok->cur < tok->buf)\r
- Py_FatalError("tok_backup: beginning of buffer");\r
- if (*tok->cur != c)\r
- *tok->cur = c;\r
- }\r
-}\r
-\r
-\r
-/* Return the token corresponding to a single character */\r
-\r
-int\r
-PyToken_OneChar(int c)\r
-{\r
- switch (c) {\r
- case '(': return LPAR;\r
- case ')': return RPAR;\r
- case '[': return LSQB;\r
- case ']': return RSQB;\r
- case ':': return COLON;\r
- case ',': return COMMA;\r
- case ';': return SEMI;\r
- case '+': return PLUS;\r
- case '-': return MINUS;\r
- case '*': return STAR;\r
- case '/': return SLASH;\r
- case '|': return VBAR;\r
- case '&': return AMPER;\r
- case '<': return LESS;\r
- case '>': return GREATER;\r
- case '=': return EQUAL;\r
- case '.': return DOT;\r
- case '%': return PERCENT;\r
- case '`': return BACKQUOTE;\r
- case '{': return LBRACE;\r
- case '}': return RBRACE;\r
- case '^': return CIRCUMFLEX;\r
- case '~': return TILDE;\r
- case '@': return AT;\r
- default: return OP;\r
- }\r
-}\r
-\r
-\r
-int\r
-PyToken_TwoChars(int c1, int c2)\r
-{\r
- switch (c1) {\r
- case '=':\r
- switch (c2) {\r
- case '=': return EQEQUAL;\r
- }\r
- break;\r
- case '!':\r
- switch (c2) {\r
- case '=': return NOTEQUAL;\r
- }\r
- break;\r
- case '<':\r
- switch (c2) {\r
- case '>': return NOTEQUAL;\r
- case '=': return LESSEQUAL;\r
- case '<': return LEFTSHIFT;\r
- }\r
- break;\r
- case '>':\r
- switch (c2) {\r
- case '=': return GREATEREQUAL;\r
- case '>': return RIGHTSHIFT;\r
- }\r
- break;\r
- case '+':\r
- switch (c2) {\r
- case '=': return PLUSEQUAL;\r
- }\r
- break;\r
- case '-':\r
- switch (c2) {\r
- case '=': return MINEQUAL;\r
- }\r
- break;\r
- case '*':\r
- switch (c2) {\r
- case '*': return DOUBLESTAR;\r
- case '=': return STAREQUAL;\r
- }\r
- break;\r
- case '/':\r
- switch (c2) {\r
- case '/': return DOUBLESLASH;\r
- case '=': return SLASHEQUAL;\r
- }\r
- break;\r
- case '|':\r
- switch (c2) {\r
- case '=': return VBAREQUAL;\r
- }\r
- break;\r
- case '%':\r
- switch (c2) {\r
- case '=': return PERCENTEQUAL;\r
- }\r
- break;\r
- case '&':\r
- switch (c2) {\r
- case '=': return AMPEREQUAL;\r
- }\r
- break;\r
- case '^':\r
- switch (c2) {\r
- case '=': return CIRCUMFLEXEQUAL;\r
- }\r
- break;\r
- }\r
- return OP;\r
-}\r
-\r
-int\r
-PyToken_ThreeChars(int c1, int c2, int c3)\r
-{\r
- switch (c1) {\r
- case '<':\r
- switch (c2) {\r
- case '<':\r
- switch (c3) {\r
- case '=':\r
- return LEFTSHIFTEQUAL;\r
- }\r
- break;\r
- }\r
- break;\r
- case '>':\r
- switch (c2) {\r
- case '>':\r
- switch (c3) {\r
- case '=':\r
- return RIGHTSHIFTEQUAL;\r
- }\r
- break;\r
- }\r
- break;\r
- case '*':\r
- switch (c2) {\r
- case '*':\r
- switch (c3) {\r
- case '=':\r
- return DOUBLESTAREQUAL;\r
- }\r
- break;\r
- }\r
- break;\r
- case '/':\r
- switch (c2) {\r
- case '/':\r
- switch (c3) {\r
- case '=':\r
- return DOUBLESLASHEQUAL;\r
- }\r
- break;\r
- }\r
- break;\r
- }\r
- return OP;\r
-}\r
-\r
-static int\r
-indenterror(struct tok_state *tok)\r
-{\r
- if (tok->alterror) {\r
- tok->done = E_TABSPACE;\r
- tok->cur = tok->inp;\r
- return 1;\r
- }\r
- if (tok->altwarning) {\r
- PySys_WriteStderr("%s: inconsistent use of tabs and spaces "\r
- "in indentation\n", tok->filename);\r
- tok->altwarning = 0;\r
- }\r
- return 0;\r
-}\r
-\r
-/* Get next token, after space stripping etc. */\r
-\r
-static int\r
-tok_get(register struct tok_state *tok, char **p_start, char **p_end)\r
-{\r
- register int c;\r
- int blankline;\r
-\r
- *p_start = *p_end = NULL;\r
- nextline:\r
- tok->start = NULL;\r
- blankline = 0;\r
-\r
- /* Get indentation level */\r
- if (tok->atbol) {\r
- register int col = 0;\r
- register int altcol = 0;\r
- tok->atbol = 0;\r
- for (;;) {\r
- c = tok_nextc(tok);\r
- if (c == ' ')\r
- col++, altcol++;\r
- else if (c == '\t') {\r
- col = (col/tok->tabsize + 1) * tok->tabsize;\r
- altcol = (altcol/tok->alttabsize + 1)\r
- * tok->alttabsize;\r
- }\r
- else if (c == '\014') /* Control-L (formfeed) */\r
- col = altcol = 0; /* For Emacs users */\r
- else\r
- break;\r
- }\r
- tok_backup(tok, c);\r
- if (c == '#' || c == '\n') {\r
- /* Lines with only whitespace and/or comments\r
- shouldn't affect the indentation and are\r
- not passed to the parser as NEWLINE tokens,\r
- except *totally* empty lines in interactive\r
- mode, which signal the end of a command group. */\r
- if (col == 0 && c == '\n' && tok->prompt != NULL)\r
- blankline = 0; /* Let it through */\r
- else\r
- blankline = 1; /* Ignore completely */\r
- /* We can't jump back right here since we still\r
- may need to skip to the end of a comment */\r
- }\r
- if (!blankline && tok->level == 0) {\r
- if (col == tok->indstack[tok->indent]) {\r
- /* No change */\r
- if (altcol != tok->altindstack[tok->indent]) {\r
- if (indenterror(tok))\r
- return ERRORTOKEN;\r
- }\r
- }\r
- else if (col > tok->indstack[tok->indent]) {\r
- /* Indent -- always one */\r
- if (tok->indent+1 >= MAXINDENT) {\r
- tok->done = E_TOODEEP;\r
- tok->cur = tok->inp;\r
- return ERRORTOKEN;\r
- }\r
- if (altcol <= tok->altindstack[tok->indent]) {\r
- if (indenterror(tok))\r
- return ERRORTOKEN;\r
- }\r
- tok->pendin++;\r
- tok->indstack[++tok->indent] = col;\r
- tok->altindstack[tok->indent] = altcol;\r
- }\r
- else /* col < tok->indstack[tok->indent] */ {\r
- /* Dedent -- any number, must be consistent */\r
- while (tok->indent > 0 &&\r
- col < tok->indstack[tok->indent]) {\r
- tok->pendin--;\r
- tok->indent--;\r
- }\r
- if (col != tok->indstack[tok->indent]) {\r
- tok->done = E_DEDENT;\r
- tok->cur = tok->inp;\r
- return ERRORTOKEN;\r
- }\r
- if (altcol != tok->altindstack[tok->indent]) {\r
- if (indenterror(tok))\r
- return ERRORTOKEN;\r
- }\r
- }\r
- }\r
- }\r
-\r
- tok->start = tok->cur;\r
-\r
- /* Return pending indents/dedents */\r
- if (tok->pendin != 0) {\r
- if (tok->pendin < 0) {\r
- tok->pendin++;\r
- return DEDENT;\r
- }\r
- else {\r
- tok->pendin--;\r
- return INDENT;\r
- }\r
- }\r
-\r
- again:\r
- tok->start = NULL;\r
- /* Skip spaces */\r
- do {\r
- c = tok_nextc(tok);\r
- } while (c == ' ' || c == '\t' || c == '\014');\r
-\r
- /* Set start of current token */\r
- tok->start = tok->cur - 1;\r
-\r
- /* Skip comment, while looking for tab-setting magic */\r
- if (c == '#') {\r
- static char *tabforms[] = {\r
- "tab-width:", /* Emacs */\r
- ":tabstop=", /* vim, full form */\r
- ":ts=", /* vim, abbreviated form */\r
- "set tabsize=", /* will vi never die? */\r
- /* more templates can be added here to support other editors */\r
- };\r
- char cbuf[80];\r
- char *tp, **cp;\r
- tp = cbuf;\r
- do {\r
- *tp++ = c = tok_nextc(tok);\r
- } while (c != EOF && c != '\n' &&\r
- (size_t)(tp - cbuf + 1) < sizeof(cbuf));\r
- *tp = '\0';\r
- for (cp = tabforms;\r
- cp < tabforms + sizeof(tabforms)/sizeof(tabforms[0]);\r
- cp++) {\r
- if ((tp = strstr(cbuf, *cp))) {\r
- int newsize = atoi(tp + strlen(*cp));\r
-\r
- if (newsize >= 1 && newsize <= 40) {\r
- tok->tabsize = newsize;\r
- if (Py_VerboseFlag)\r
- PySys_WriteStderr(\r
- "Tab size set to %d\n",\r
- newsize);\r
- }\r
- }\r
- }\r
- while (c != EOF && c != '\n')\r
- c = tok_nextc(tok);\r
- }\r
-\r
- /* Check for EOF and errors now */\r
- if (c == EOF) {\r
- return tok->done == E_EOF ? ENDMARKER : ERRORTOKEN;\r
- }\r
-\r
- /* Identifier (most frequent token!) */\r
- if (Py_ISALPHA(c) || c == '_') {\r
- /* Process r"", u"" and ur"" */\r
- switch (c) {\r
- case 'b':\r
- case 'B':\r
- c = tok_nextc(tok);\r
- if (c == 'r' || c == 'R')\r
- c = tok_nextc(tok);\r
- if (c == '"' || c == '\'')\r
- goto letter_quote;\r
- break;\r
- case 'r':\r
- case 'R':\r
- c = tok_nextc(tok);\r
- if (c == '"' || c == '\'')\r
- goto letter_quote;\r
- break;\r
- case 'u':\r
- case 'U':\r
- c = tok_nextc(tok);\r
- if (c == 'r' || c == 'R')\r
- c = tok_nextc(tok);\r
- if (c == '"' || c == '\'')\r
- goto letter_quote;\r
- break;\r
- }\r
- while (c != EOF && (Py_ISALNUM(c) || c == '_')) {\r
- c = tok_nextc(tok);\r
- }\r
- tok_backup(tok, c);\r
- *p_start = tok->start;\r
- *p_end = tok->cur;\r
- return NAME;\r
- }\r
-\r
- /* Newline */\r
- if (c == '\n') {\r
- tok->atbol = 1;\r
- if (blankline || tok->level > 0)\r
- goto nextline;\r
- *p_start = tok->start;\r
- *p_end = tok->cur - 1; /* Leave '\n' out of the string */\r
- tok->cont_line = 0;\r
- return NEWLINE;\r
- }\r
-\r
- /* Period or number starting with period? */\r
- if (c == '.') {\r
- c = tok_nextc(tok);\r
- if (isdigit(c)) {\r
- goto fraction;\r
- }\r
- else {\r
- tok_backup(tok, c);\r
- *p_start = tok->start;\r
- *p_end = tok->cur;\r
- return DOT;\r
- }\r
- }\r
-\r
- /* Number */\r
- if (isdigit(c)) {\r
- if (c == '0') {\r
- /* Hex, octal or binary -- maybe. */\r
- c = tok_nextc(tok);\r
- if (c == '.')\r
- goto fraction;\r
-#ifndef WITHOUT_COMPLEX\r
- if (c == 'j' || c == 'J')\r
- goto imaginary;\r
-#endif\r
- if (c == 'x' || c == 'X') {\r
-\r
- /* Hex */\r
- c = tok_nextc(tok);\r
- if (!isxdigit(c)) {\r
- tok->done = E_TOKEN;\r
- tok_backup(tok, c);\r
- return ERRORTOKEN;\r
- }\r
- do {\r
- c = tok_nextc(tok);\r
- } while (isxdigit(c));\r
- }\r
- else if (c == 'o' || c == 'O') {\r
- /* Octal */\r
- c = tok_nextc(tok);\r
- if (c < '0' || c >= '8') {\r
- tok->done = E_TOKEN;\r
- tok_backup(tok, c);\r
- return ERRORTOKEN;\r
- }\r
- do {\r
- c = tok_nextc(tok);\r
- } while ('0' <= c && c < '8');\r
- }\r
- else if (c == 'b' || c == 'B') {\r
- /* Binary */\r
- c = tok_nextc(tok);\r
- if (c != '0' && c != '1') {\r
- tok->done = E_TOKEN;\r
- tok_backup(tok, c);\r
- return ERRORTOKEN;\r
- }\r
- do {\r
- c = tok_nextc(tok);\r
- } while (c == '0' || c == '1');\r
- }\r
- else {\r
- int found_decimal = 0;\r
- /* Octal; c is first char of it */\r
- /* There's no 'isoctdigit' macro, sigh */\r
- while ('0' <= c && c < '8') {\r
- c = tok_nextc(tok);\r
- }\r
- if (isdigit(c)) {\r
- found_decimal = 1;\r
- do {\r
- c = tok_nextc(tok);\r
- } while (isdigit(c));\r
- }\r
- if (c == '.')\r
- goto fraction;\r
- else if (c == 'e' || c == 'E')\r
- goto exponent;\r
-#ifndef WITHOUT_COMPLEX\r
- else if (c == 'j' || c == 'J')\r
- goto imaginary;\r
-#endif\r
- else if (found_decimal) {\r
- tok->done = E_TOKEN;\r
- tok_backup(tok, c);\r
- return ERRORTOKEN;\r
- }\r
- }\r
- if (c == 'l' || c == 'L')\r
- c = tok_nextc(tok);\r
- }\r
- else {\r
- /* Decimal */\r
- do {\r
- c = tok_nextc(tok);\r
- } while (isdigit(c));\r
- if (c == 'l' || c == 'L')\r
- c = tok_nextc(tok);\r
- else {\r
- /* Accept floating point numbers. */\r
- if (c == '.') {\r
- fraction:\r
- /* Fraction */\r
- do {\r
- c = tok_nextc(tok);\r
- } while (isdigit(c));\r
- }\r
- if (c == 'e' || c == 'E') {\r
- int e;\r
- exponent:\r
- e = c;\r
- /* Exponent part */\r
- c = tok_nextc(tok);\r
- if (c == '+' || c == '-') {\r
- c = tok_nextc(tok);\r
- if (!isdigit(c)) {\r
- tok->done = E_TOKEN;\r
- tok_backup(tok, c);\r
- return ERRORTOKEN;\r
- }\r
- } else if (!isdigit(c)) {\r
- tok_backup(tok, c);\r
- tok_backup(tok, e);\r
- *p_start = tok->start;\r
- *p_end = tok->cur;\r
- return NUMBER;\r
- }\r
- do {\r
- c = tok_nextc(tok);\r
- } while (isdigit(c));\r
- }\r
-#ifndef WITHOUT_COMPLEX\r
- if (c == 'j' || c == 'J')\r
- /* Imaginary part */\r
- imaginary:\r
- c = tok_nextc(tok);\r
-#endif\r
- }\r
- }\r
- tok_backup(tok, c);\r
- *p_start = tok->start;\r
- *p_end = tok->cur;\r
- return NUMBER;\r
- }\r
-\r
- letter_quote:\r
- /* String */\r
- if (c == '\'' || c == '"') {\r
- Py_ssize_t quote2 = tok->cur - tok->start + 1;\r
- int quote = c;\r
- int triple = 0;\r
- int tripcount = 0;\r
- for (;;) {\r
- c = tok_nextc(tok);\r
- if (c == '\n') {\r
- if (!triple) {\r
- tok->done = E_EOLS;\r
- tok_backup(tok, c);\r
- return ERRORTOKEN;\r
- }\r
- tripcount = 0;\r
- tok->cont_line = 1; /* multiline string. */\r
- }\r
- else if (c == EOF) {\r
- if (triple)\r
- tok->done = E_EOFS;\r
- else\r
- tok->done = E_EOLS;\r
- tok->cur = tok->inp;\r
- return ERRORTOKEN;\r
- }\r
- else if (c == quote) {\r
- tripcount++;\r
- if (tok->cur - tok->start == quote2) {\r
- c = tok_nextc(tok);\r
- if (c == quote) {\r
- triple = 1;\r
- tripcount = 0;\r
- continue;\r
- }\r
- tok_backup(tok, c);\r
- }\r
- if (!triple || tripcount == 3)\r
- break;\r
- }\r
- else if (c == '\\') {\r
- tripcount = 0;\r
- c = tok_nextc(tok);\r
- if (c == EOF) {\r
- tok->done = E_EOLS;\r
- tok->cur = tok->inp;\r
- return ERRORTOKEN;\r
- }\r
- }\r
- else\r
- tripcount = 0;\r
- }\r
- *p_start = tok->start;\r
- *p_end = tok->cur;\r
- return STRING;\r
- }\r
-\r
- /* Line continuation */\r
- if (c == '\\') {\r
- c = tok_nextc(tok);\r
- if (c != '\n') {\r
- tok->done = E_LINECONT;\r
- tok->cur = tok->inp;\r
- return ERRORTOKEN;\r
- }\r
- tok->cont_line = 1;\r
- goto again; /* Read next line */\r
- }\r
-\r
- /* Check for two-character token */\r
- {\r
- int c2 = tok_nextc(tok);\r
- int token = PyToken_TwoChars(c, c2);\r
-#ifndef PGEN\r
- if (Py_Py3kWarningFlag && token == NOTEQUAL && c == '<') {\r
- if (PyErr_WarnExplicit(PyExc_DeprecationWarning,\r
- "<> not supported in 3.x; use !=",\r
- tok->filename, tok->lineno,\r
- NULL, NULL)) {\r
- return ERRORTOKEN;\r
- }\r
- }\r
-#endif\r
- if (token != OP) {\r
- int c3 = tok_nextc(tok);\r
- int token3 = PyToken_ThreeChars(c, c2, c3);\r
- if (token3 != OP) {\r
- token = token3;\r
- } else {\r
- tok_backup(tok, c3);\r
- }\r
- *p_start = tok->start;\r
- *p_end = tok->cur;\r
- return token;\r
- }\r
- tok_backup(tok, c2);\r
- }\r
-\r
- /* Keep track of parentheses nesting level */\r
- switch (c) {\r
- case '(':\r
- case '[':\r
- case '{':\r
- tok->level++;\r
- break;\r
- case ')':\r
- case ']':\r
- case '}':\r
- tok->level--;\r
- break;\r
- }\r
-\r
- /* Punctuation character */\r
- *p_start = tok->start;\r
- *p_end = tok->cur;\r
- return PyToken_OneChar(c);\r
-}\r
-\r
-int\r
-PyTokenizer_Get(struct tok_state *tok, char **p_start, char **p_end)\r
-{\r
- int result = tok_get(tok, p_start, p_end);\r
- if (tok->decoding_erred) {\r
- result = ERRORTOKEN;\r
- tok->done = E_DECODE;\r
- }\r
- return result;\r
-}\r
-\r
-/* This function is only called from parsetok. However, it cannot live\r
- there, as it must be empty for PGEN, and we can check for PGEN only\r
- in this file. */\r
-\r
-#if defined(PGEN) || !defined(Py_USING_UNICODE)\r
-char*\r
-PyTokenizer_RestoreEncoding(struct tok_state* tok, int len, int* offset)\r
-{\r
- return NULL;\r
-}\r
-#else\r
-#ifdef Py_USING_UNICODE\r
-static PyObject *\r
-dec_utf8(const char *enc, const char *text, size_t len) {\r
- PyObject *ret = NULL;\r
- PyObject *unicode_text = PyUnicode_DecodeUTF8(text, len, "replace");\r
- if (unicode_text) {\r
- ret = PyUnicode_AsEncodedString(unicode_text, enc, "replace");\r
- Py_DECREF(unicode_text);\r
- }\r
- if (!ret) {\r
- PyErr_Clear();\r
- }\r
- return ret;\r
-}\r
-char *\r
-PyTokenizer_RestoreEncoding(struct tok_state* tok, int len, int *offset)\r
-{\r
- char *text = NULL;\r
- if (tok->encoding) {\r
- /* convert source to original encondig */\r
- PyObject *lineobj = dec_utf8(tok->encoding, tok->buf, len);\r
- if (lineobj != NULL) {\r
- int linelen = PyString_Size(lineobj);\r
- const char *line = PyString_AsString(lineobj);\r
- text = PyObject_MALLOC(linelen + 1);\r
- if (text != NULL && line != NULL) {\r
- if (linelen)\r
- strncpy(text, line, linelen);\r
- text[linelen] = '\0';\r
- }\r
- Py_DECREF(lineobj);\r
-\r
- /* adjust error offset */\r
- if (*offset > 1) {\r
- PyObject *offsetobj = dec_utf8(tok->encoding,\r
- tok->buf, *offset-1);\r
- if (offsetobj) {\r
- *offset = PyString_Size(offsetobj) + 1;\r
- Py_DECREF(offsetobj);\r
- }\r
- }\r
-\r
- }\r
- }\r
- return text;\r
-\r
-}\r
-#endif /* defined(Py_USING_UNICODE) */\r
-#endif\r
-\r
-\r
-#ifdef Py_DEBUG\r
-\r
-void\r
-tok_dump(int type, char *start, char *end)\r
-{\r
- printf("%s", _PyParser_TokenNames[type]);\r
- if (type == NAME || type == NUMBER || type == STRING || type == OP)\r
- printf("(%.*s)", (int)(end - start), start);\r
-}\r
-\r
-#endif\r
projects / mirror_edk2.git / blobdiff