--- /dev/null
+\r
+/* Token types */\r
+\r
+#ifndef Py_TOKEN_H\r
+#define Py_TOKEN_H\r
+#ifdef __cplusplus\r
+extern "C" {\r
+#endif\r
+\r
+#undef TILDE /* Prevent clash of our definition with system macro. Ex AIX, ioctl.h */\r
+\r
+#define ENDMARKER 0\r
+#define NAME 1\r
+#define NUMBER 2\r
+#define STRING 3\r
+#define NEWLINE 4\r
+#define INDENT 5\r
+#define DEDENT 6\r
+#define LPAR 7\r
+#define RPAR 8\r
+#define LSQB 9\r
+#define RSQB 10\r
+#define COLON 11\r
+#define COMMA 12\r
+#define SEMI 13\r
+#define PLUS 14\r
+#define MINUS 15\r
+#define STAR 16\r
+#define SLASH 17\r
+#define VBAR 18\r
+#define AMPER 19\r
+#define LESS 20\r
+#define GREATER 21\r
+#define EQUAL 22\r
+#define DOT 23\r
+#define PERCENT 24\r
+#define BACKQUOTE 25\r
+#define LBRACE 26\r
+#define RBRACE 27\r
+#define EQEQUAL 28\r
+#define NOTEQUAL 29\r
+#define LESSEQUAL 30\r
+#define GREATEREQUAL 31\r
+#define TILDE 32\r
+#define CIRCUMFLEX 33\r
+#define LEFTSHIFT 34\r
+#define RIGHTSHIFT 35\r
+#define DOUBLESTAR 36\r
+#define PLUSEQUAL 37\r
+#define MINEQUAL 38\r
+#define STAREQUAL 39\r
+#define SLASHEQUAL 40\r
+#define PERCENTEQUAL 41\r
+#define AMPEREQUAL 42\r
+#define VBAREQUAL 43\r
+#define CIRCUMFLEXEQUAL 44\r
+#define LEFTSHIFTEQUAL 45\r
+#define RIGHTSHIFTEQUAL 46\r
+#define DOUBLESTAREQUAL 47\r
+#define DOUBLESLASH 48\r
+#define DOUBLESLASHEQUAL 49\r
+#define AT 50 \r
+/* Don't forget to update the table _PyParser_TokenNames in tokenizer.c! */\r
+#define OP 51\r
+#define ERRORTOKEN 52\r
+#define N_TOKENS 53\r
+\r
+/* Special definitions for cooperation with parser */\r
+\r
+#define NT_OFFSET 256\r
+\r
+#define ISTERMINAL(x) ((x) < NT_OFFSET)\r
+#define ISNONTERMINAL(x) ((x) >= NT_OFFSET)\r
+#define ISEOF(x) ((x) == ENDMARKER)\r
+\r
+\r
+PyAPI_DATA(char *) _PyParser_TokenNames[]; /* Token names */\r
+PyAPI_FUNC(int) PyToken_OneChar(int);\r
+PyAPI_FUNC(int) PyToken_TwoChars(int, int);\r
+PyAPI_FUNC(int) PyToken_ThreeChars(int, int, int);\r
+\r
+#ifdef __cplusplus\r
+}\r
+#endif\r
+#endif /* !Py_TOKEN_H */\r