projects / mirror_edk2.git / blame
| Commit | Line | Data |
|---|---|---|
| c8042e10 DM |
1 | #ifndef Py_TOKENIZER_H\r |
| 2 | #define Py_TOKENIZER_H\r | |
| 3 | #ifdef __cplusplus\r | |
| 4 | extern "C" {\r | |
| 5 | #endif\r | |
| 6 | \r | |
| 7 | #include "object.h"\r | |
| 8 | \r | |
| 9 | /* Tokenizer interface */\r | |
| 10 | \r | |
| 11 | #include "token.h" /* For token types */\r | |
| 12 | \r | |
| 13 | #define MAXINDENT 100 /* Max indentation level */\r | |
| 14 | \r | |
| 15 | /* Tokenizer state */\r | |
| 16 | struct tok_state {\r | |
| 17 | /* Input state; buf <= cur <= inp <= end */\r | |
| 18 | /* NB an entire line is held in the buffer */\r | |
| 19 | char *buf; /* Input buffer, or NULL; malloc'ed if fp != NULL */\r | |
| 20 | char *cur; /* Next character in buffer */\r | |
| 21 | char *inp; /* End of data in buffer */\r | |
| 22 | char *end; /* End of input buffer if buf != NULL */\r | |
| 23 | char *start; /* Start of current token if not NULL */\r | |
| 24 | int done; /* E_OK normally, E_EOF at EOF, otherwise error code */\r | |
| 25 | /* NB If done != E_OK, cur must be == inp!!! */\r | |
| 26 | FILE *fp; /* Rest of input; NULL if tokenizing a string */\r | |
| 27 | int tabsize; /* Tab spacing */\r | |
| 28 | int indent; /* Current indentation index */\r | |
| 29 | int indstack[MAXINDENT]; /* Stack of indents */\r | |
| 30 | int atbol; /* Nonzero if at begin of new line */\r | |
| 31 | int pendin; /* Pending indents (if > 0) or dedents (if < 0) */\r | |
| 32 | char *prompt, *nextprompt; /* For interactive prompting */\r | |
| 33 | int lineno; /* Current line number */\r | |
| 34 | int level; /* () [] {} Parentheses nesting level */\r | |
| 35 | /* Used to allow free continuations inside them */\r | |
| 36 | /* Stuff for checking on different tab sizes */\r | |
| 37 | const char *filename; /* For error messages */\r | |
| 38 | int altwarning; /* Issue warning if alternate tabs don't match */\r | |
| 39 | int alterror; /* Issue error if alternate tabs don't match */\r | |
| 40 | int alttabsize; /* Alternate tab spacing */\r | |
| 41 | int altindstack[MAXINDENT]; /* Stack of alternate indents */\r | |
| 42 | /* Stuff for PEP 0263 */\r | |
| 43 | int decoding_state; /* -1:decoding, 0:init, 1:raw */\r | |
| 44 | int decoding_erred; /* whether erred in decoding */\r | |
| 45 | int read_coding_spec; /* whether 'coding:...' has been read */\r | |
| 46 | char *encoding;\r | |
| 47 | int cont_line; /* whether we are in a continuation line. */\r | |
| 48 | const char* line_start; /* pointer to start of current line */\r | |
| 49 | #ifndef PGEN\r | |
| 50 | PyObject *decoding_readline; /* codecs.open(...).readline */\r | |
| 51 | PyObject *decoding_buffer;\r | |
| 52 | #endif\r | |
| 53 | const char* enc;\r | |
| 54 | const char* str;\r | |
| 55 | const char* input; /* Tokenizer's newline translated copy of the string. */\r | |
| 56 | };\r | |
| 57 | \r | |
| 58 | extern struct tok_state *PyTokenizer_FromString(const char *, int);\r | |
| 59 | extern struct tok_state *PyTokenizer_FromFile(FILE *, char *, char *);\r | |
| 60 | extern void PyTokenizer_Free(struct tok_state *);\r | |
| 61 | extern int PyTokenizer_Get(struct tok_state *, char **, char **);\r | |
| 62 | #if defined(PGEN) || defined(Py_USING_UNICODE)\r | |
| 63 | extern char * PyTokenizer_RestoreEncoding(struct tok_state* tok,\r | |
| 64 | int len, int *offset);\r | |
| 65 | #endif\r | |
| 66 | \r | |
| 67 | #ifdef __cplusplus\r | |
| 68 | }\r | |
| 69 | #endif\r | |
| 70 | #endif /* !Py_TOKENIZER_H */\r |