]>
Commit | Line | Data |
---|---|---|
c8042e10 DM |
1 | #ifndef Py_TOKENIZER_H\r |
2 | #define Py_TOKENIZER_H\r | |
3 | #ifdef __cplusplus\r | |
4 | extern "C" {\r | |
5 | #endif\r | |
6 | \r | |
7 | #include "object.h"\r | |
8 | \r | |
9 | /* Tokenizer interface */\r | |
10 | \r | |
11 | #include "token.h" /* For token types */\r | |
12 | \r | |
13 | #define MAXINDENT 100 /* Max indentation level */\r | |
14 | \r | |
15 | /* Tokenizer state */\r | |
16 | struct tok_state {\r | |
17 | /* Input state; buf <= cur <= inp <= end */\r | |
18 | /* NB an entire line is held in the buffer */\r | |
19 | char *buf; /* Input buffer, or NULL; malloc'ed if fp != NULL */\r | |
20 | char *cur; /* Next character in buffer */\r | |
21 | char *inp; /* End of data in buffer */\r | |
22 | char *end; /* End of input buffer if buf != NULL */\r | |
23 | char *start; /* Start of current token if not NULL */\r | |
24 | int done; /* E_OK normally, E_EOF at EOF, otherwise error code */\r | |
25 | /* NB If done != E_OK, cur must be == inp!!! */\r | |
26 | FILE *fp; /* Rest of input; NULL if tokenizing a string */\r | |
27 | int tabsize; /* Tab spacing */\r | |
28 | int indent; /* Current indentation index */\r | |
29 | int indstack[MAXINDENT]; /* Stack of indents */\r | |
30 | int atbol; /* Nonzero if at begin of new line */\r | |
31 | int pendin; /* Pending indents (if > 0) or dedents (if < 0) */\r | |
32 | char *prompt, *nextprompt; /* For interactive prompting */\r | |
33 | int lineno; /* Current line number */\r | |
34 | int level; /* () [] {} Parentheses nesting level */\r | |
35 | /* Used to allow free continuations inside them */\r | |
36 | /* Stuff for checking on different tab sizes */\r | |
37 | const char *filename; /* For error messages */\r | |
38 | int altwarning; /* Issue warning if alternate tabs don't match */\r | |
39 | int alterror; /* Issue error if alternate tabs don't match */\r | |
40 | int alttabsize; /* Alternate tab spacing */\r | |
41 | int altindstack[MAXINDENT]; /* Stack of alternate indents */\r | |
42 | /* Stuff for PEP 0263 */\r | |
43 | int decoding_state; /* -1:decoding, 0:init, 1:raw */\r | |
44 | int decoding_erred; /* whether erred in decoding */\r | |
45 | int read_coding_spec; /* whether 'coding:...' has been read */\r | |
46 | char *encoding;\r | |
47 | int cont_line; /* whether we are in a continuation line. */\r | |
48 | const char* line_start; /* pointer to start of current line */\r | |
49 | #ifndef PGEN\r | |
50 | PyObject *decoding_readline; /* codecs.open(...).readline */\r | |
51 | PyObject *decoding_buffer;\r | |
52 | #endif\r | |
53 | const char* enc;\r | |
54 | const char* str;\r | |
55 | const char* input; /* Tokenizer's newline translated copy of the string. */\r | |
56 | };\r | |
57 | \r | |
58 | extern struct tok_state *PyTokenizer_FromString(const char *, int);\r | |
59 | extern struct tok_state *PyTokenizer_FromFile(FILE *, char *, char *);\r | |
60 | extern void PyTokenizer_Free(struct tok_state *);\r | |
61 | extern int PyTokenizer_Get(struct tok_state *, char **, char **);\r | |
62 | #if defined(PGEN) || defined(Py_USING_UNICODE)\r | |
63 | extern char * PyTokenizer_RestoreEncoding(struct tok_state* tok,\r | |
64 | int len, int *offset);\r | |
65 | #endif\r | |
66 | \r | |
67 | #ifdef __cplusplus\r | |
68 | }\r | |
69 | #endif\r | |
70 | #endif /* !Py_TOKENIZER_H */\r |