]>
Commit | Line | Data |
---|---|---|
3257aa99 DM |
1 | #\r |
2 | # Secret Labs' Regular Expression Engine\r | |
3 | #\r | |
4 | # re-compatible interface for the sre matching engine\r | |
5 | #\r | |
6 | # Copyright (c) 1998-2001 by Secret Labs AB. All rights reserved.\r | |
7 | #\r | |
8 | # This version of the SRE library can be redistributed under CNRI's\r | |
9 | # Python 1.6 license. For any other use, please contact Secret Labs\r | |
10 | # AB (info@pythonware.com).\r | |
11 | #\r | |
12 | # Portions of this engine have been developed in cooperation with\r | |
13 | # CNRI. Hewlett-Packard provided funding for 1.6 integration and\r | |
14 | # other compatibility work.\r | |
15 | #\r | |
16 | \r | |
17 | r"""Support for regular expressions (RE).\r | |
18 | \r | |
19 | This module provides regular expression matching operations similar to\r | |
20 | those found in Perl. It supports both 8-bit and Unicode strings; both\r | |
21 | the pattern and the strings being processed can contain null bytes and\r | |
22 | characters outside the US ASCII range.\r | |
23 | \r | |
24 | Regular expressions can contain both special and ordinary characters.\r | |
25 | Most ordinary characters, like "A", "a", or "0", are the simplest\r | |
26 | regular expressions; they simply match themselves. You can\r | |
27 | concatenate ordinary characters, so last matches the string 'last'.\r | |
28 | \r | |
29 | The special characters are:\r | |
30 | "." Matches any character except a newline.\r | |
31 | "^" Matches the start of the string.\r | |
32 | "$" Matches the end of the string or just before the newline at\r | |
33 | the end of the string.\r | |
34 | "*" Matches 0 or more (greedy) repetitions of the preceding RE.\r | |
35 | Greedy means that it will match as many repetitions as possible.\r | |
36 | "+" Matches 1 or more (greedy) repetitions of the preceding RE.\r | |
37 | "?" Matches 0 or 1 (greedy) of the preceding RE.\r | |
38 | *?,+?,?? Non-greedy versions of the previous three special characters.\r | |
39 | {m,n} Matches from m to n repetitions of the preceding RE.\r | |
40 | {m,n}? Non-greedy version of the above.\r | |
41 | "\\" Either escapes special characters or signals a special sequence.\r | |
42 | [] Indicates a set of characters.\r | |
43 | A "^" as the first character indicates a complementing set.\r | |
44 | "|" A|B, creates an RE that will match either A or B.\r | |
45 | (...) Matches the RE inside the parentheses.\r | |
46 | The contents can be retrieved or matched later in the string.\r | |
47 | (?iLmsux) Set the I, L, M, S, U, or X flag for the RE (see below).\r | |
48 | (?:...) Non-grouping version of regular parentheses.\r | |
49 | (?P<name>...) The substring matched by the group is accessible by name.\r | |
50 | (?P=name) Matches the text matched earlier by the group named name.\r | |
51 | (?#...) A comment; ignored.\r | |
52 | (?=...) Matches if ... matches next, but doesn't consume the string.\r | |
53 | (?!...) Matches if ... doesn't match next.\r | |
54 | (?<=...) Matches if preceded by ... (must be fixed length).\r | |
55 | (?<!...) Matches if not preceded by ... (must be fixed length).\r | |
56 | (?(id/name)yes|no) Matches yes pattern if the group with id/name matched,\r | |
57 | the (optional) no pattern otherwise.\r | |
58 | \r | |
59 | The special sequences consist of "\\" and a character from the list\r | |
60 | below. If the ordinary character is not on the list, then the\r | |
61 | resulting RE will match the second character.\r | |
62 | \number Matches the contents of the group of the same number.\r | |
63 | \A Matches only at the start of the string.\r | |
64 | \Z Matches only at the end of the string.\r | |
65 | \b Matches the empty string, but only at the start or end of a word.\r | |
66 | \B Matches the empty string, but not at the start or end of a word.\r | |
67 | \d Matches any decimal digit; equivalent to the set [0-9].\r | |
68 | \D Matches any non-digit character; equivalent to the set [^0-9].\r | |
69 | \s Matches any whitespace character; equivalent to [ \t\n\r\f\v].\r | |
70 | \S Matches any non-whitespace character; equiv. to [^ \t\n\r\f\v].\r | |
71 | \w Matches any alphanumeric character; equivalent to [a-zA-Z0-9_].\r | |
72 | With LOCALE, it will match the set [0-9_] plus characters defined\r | |
73 | as letters for the current locale.\r | |
74 | \W Matches the complement of \w.\r | |
75 | \\ Matches a literal backslash.\r | |
76 | \r | |
77 | This module exports the following functions:\r | |
78 | match Match a regular expression pattern to the beginning of a string.\r | |
79 | search Search a string for the presence of a pattern.\r | |
80 | sub Substitute occurrences of a pattern found in a string.\r | |
81 | subn Same as sub, but also return the number of substitutions made.\r | |
82 | split Split a string by the occurrences of a pattern.\r | |
83 | findall Find all occurrences of a pattern in a string.\r | |
84 | finditer Return an iterator yielding a match object for each match.\r | |
85 | compile Compile a pattern into a RegexObject.\r | |
86 | purge Clear the regular expression cache.\r | |
87 | escape Backslash all non-alphanumerics in a string.\r | |
88 | \r | |
89 | Some of the functions in this module takes flags as optional parameters:\r | |
90 | I IGNORECASE Perform case-insensitive matching.\r | |
91 | L LOCALE Make \w, \W, \b, \B, dependent on the current locale.\r | |
92 | M MULTILINE "^" matches the beginning of lines (after a newline)\r | |
93 | as well as the string.\r | |
94 | "$" matches the end of lines (before a newline) as well\r | |
95 | as the end of the string.\r | |
96 | S DOTALL "." matches any character at all, including the newline.\r | |
97 | X VERBOSE Ignore whitespace and comments for nicer looking RE's.\r | |
98 | U UNICODE Make \w, \W, \b, \B, dependent on the Unicode locale.\r | |
99 | \r | |
100 | This module also defines an exception 'error'.\r | |
101 | \r | |
102 | """\r | |
103 | \r | |
104 | import sys\r | |
105 | import sre_compile\r | |
106 | import sre_parse\r | |
107 | try:\r | |
108 | import _locale\r | |
109 | except ImportError:\r | |
110 | _locale = None\r | |
111 | \r | |
112 | # public symbols\r | |
113 | __all__ = [ "match", "search", "sub", "subn", "split", "findall",\r | |
114 | "compile", "purge", "template", "escape", "I", "L", "M", "S", "X",\r | |
115 | "U", "IGNORECASE", "LOCALE", "MULTILINE", "DOTALL", "VERBOSE",\r | |
116 | "UNICODE", "error" ]\r | |
117 | \r | |
118 | __version__ = "2.2.1"\r | |
119 | \r | |
120 | # flags\r | |
121 | I = IGNORECASE = sre_compile.SRE_FLAG_IGNORECASE # ignore case\r | |
122 | L = LOCALE = sre_compile.SRE_FLAG_LOCALE # assume current 8-bit locale\r | |
123 | U = UNICODE = sre_compile.SRE_FLAG_UNICODE # assume unicode locale\r | |
124 | M = MULTILINE = sre_compile.SRE_FLAG_MULTILINE # make anchors look for newline\r | |
125 | S = DOTALL = sre_compile.SRE_FLAG_DOTALL # make dot match newline\r | |
126 | X = VERBOSE = sre_compile.SRE_FLAG_VERBOSE # ignore whitespace and comments\r | |
127 | \r | |
128 | # sre extensions (experimental, don't rely on these)\r | |
129 | T = TEMPLATE = sre_compile.SRE_FLAG_TEMPLATE # disable backtracking\r | |
130 | DEBUG = sre_compile.SRE_FLAG_DEBUG # dump pattern after compilation\r | |
131 | \r | |
132 | # sre exception\r | |
133 | error = sre_compile.error\r | |
134 | \r | |
135 | # --------------------------------------------------------------------\r | |
136 | # public interface\r | |
137 | \r | |
138 | def match(pattern, string, flags=0):\r | |
139 | """Try to apply the pattern at the start of the string, returning\r | |
140 | a match object, or None if no match was found."""\r | |
141 | return _compile(pattern, flags).match(string)\r | |
142 | \r | |
143 | def search(pattern, string, flags=0):\r | |
144 | """Scan through string looking for a match to the pattern, returning\r | |
145 | a match object, or None if no match was found."""\r | |
146 | return _compile(pattern, flags).search(string)\r | |
147 | \r | |
148 | def sub(pattern, repl, string, count=0, flags=0):\r | |
149 | """Return the string obtained by replacing the leftmost\r | |
150 | non-overlapping occurrences of the pattern in string by the\r | |
151 | replacement repl. repl can be either a string or a callable;\r | |
152 | if a string, backslash escapes in it are processed. If it is\r | |
153 | a callable, it's passed the match object and must return\r | |
154 | a replacement string to be used."""\r | |
155 | return _compile(pattern, flags).sub(repl, string, count)\r | |
156 | \r | |
157 | def subn(pattern, repl, string, count=0, flags=0):\r | |
158 | """Return a 2-tuple containing (new_string, number).\r | |
159 | new_string is the string obtained by replacing the leftmost\r | |
160 | non-overlapping occurrences of the pattern in the source\r | |
161 | string by the replacement repl. number is the number of\r | |
162 | substitutions that were made. repl can be either a string or a\r | |
163 | callable; if a string, backslash escapes in it are processed.\r | |
164 | If it is a callable, it's passed the match object and must\r | |
165 | return a replacement string to be used."""\r | |
166 | return _compile(pattern, flags).subn(repl, string, count)\r | |
167 | \r | |
168 | def split(pattern, string, maxsplit=0, flags=0):\r | |
169 | """Split the source string by the occurrences of the pattern,\r | |
170 | returning a list containing the resulting substrings."""\r | |
171 | return _compile(pattern, flags).split(string, maxsplit)\r | |
172 | \r | |
173 | def findall(pattern, string, flags=0):\r | |
174 | """Return a list of all non-overlapping matches in the string.\r | |
175 | \r | |
176 | If one or more groups are present in the pattern, return a\r | |
177 | list of groups; this will be a list of tuples if the pattern\r | |
178 | has more than one group.\r | |
179 | \r | |
180 | Empty matches are included in the result."""\r | |
181 | return _compile(pattern, flags).findall(string)\r | |
182 | \r | |
183 | if sys.hexversion >= 0x02020000:\r | |
184 | __all__.append("finditer")\r | |
185 | def finditer(pattern, string, flags=0):\r | |
186 | """Return an iterator over all non-overlapping matches in the\r | |
187 | string. For each match, the iterator returns a match object.\r | |
188 | \r | |
189 | Empty matches are included in the result."""\r | |
190 | return _compile(pattern, flags).finditer(string)\r | |
191 | \r | |
192 | def compile(pattern, flags=0):\r | |
193 | "Compile a regular expression pattern, returning a pattern object."\r | |
194 | return _compile(pattern, flags)\r | |
195 | \r | |
196 | def purge():\r | |
197 | "Clear the regular expression cache"\r | |
198 | _cache.clear()\r | |
199 | _cache_repl.clear()\r | |
200 | \r | |
201 | def template(pattern, flags=0):\r | |
202 | "Compile a template pattern, returning a pattern object"\r | |
203 | return _compile(pattern, flags|T)\r | |
204 | \r | |
205 | _alphanum = frozenset(\r | |
206 | "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789")\r | |
207 | \r | |
208 | def escape(pattern):\r | |
209 | "Escape all non-alphanumeric characters in pattern."\r | |
210 | s = list(pattern)\r | |
211 | alphanum = _alphanum\r | |
212 | for i, c in enumerate(pattern):\r | |
213 | if c not in alphanum:\r | |
214 | if c == "\000":\r | |
215 | s[i] = "\\000"\r | |
216 | else:\r | |
217 | s[i] = "\\" + c\r | |
218 | return pattern[:0].join(s)\r | |
219 | \r | |
220 | # --------------------------------------------------------------------\r | |
221 | # internals\r | |
222 | \r | |
223 | _cache = {}\r | |
224 | _cache_repl = {}\r | |
225 | \r | |
226 | _pattern_type = type(sre_compile.compile("", 0))\r | |
227 | \r | |
228 | _MAXCACHE = 100\r | |
229 | \r | |
230 | def _compile(*key):\r | |
231 | # internal: compile pattern\r | |
232 | pattern, flags = key\r | |
233 | bypass_cache = flags & DEBUG\r | |
234 | if not bypass_cache:\r | |
235 | cachekey = (type(key[0]),) + key\r | |
236 | try:\r | |
237 | p, loc = _cache[cachekey]\r | |
238 | if loc is None or loc == _locale.setlocale(_locale.LC_CTYPE):\r | |
239 | return p\r | |
240 | except KeyError:\r | |
241 | pass\r | |
242 | if isinstance(pattern, _pattern_type):\r | |
243 | if flags:\r | |
244 | raise ValueError('Cannot process flags argument with a compiled pattern')\r | |
245 | return pattern\r | |
246 | if not sre_compile.isstring(pattern):\r | |
247 | raise TypeError, "first argument must be string or compiled pattern"\r | |
248 | try:\r | |
249 | p = sre_compile.compile(pattern, flags)\r | |
250 | except error, v:\r | |
251 | raise error, v # invalid expression\r | |
252 | if not bypass_cache:\r | |
253 | if len(_cache) >= _MAXCACHE:\r | |
254 | _cache.clear()\r | |
255 | if p.flags & LOCALE:\r | |
256 | if not _locale:\r | |
257 | return p\r | |
258 | loc = _locale.setlocale(_locale.LC_CTYPE)\r | |
259 | else:\r | |
260 | loc = None\r | |
261 | _cache[cachekey] = p, loc\r | |
262 | return p\r | |
263 | \r | |
264 | def _compile_repl(*key):\r | |
265 | # internal: compile replacement pattern\r | |
266 | p = _cache_repl.get(key)\r | |
267 | if p is not None:\r | |
268 | return p\r | |
269 | repl, pattern = key\r | |
270 | try:\r | |
271 | p = sre_parse.parse_template(repl, pattern)\r | |
272 | except error, v:\r | |
273 | raise error, v # invalid expression\r | |
274 | if len(_cache_repl) >= _MAXCACHE:\r | |
275 | _cache_repl.clear()\r | |
276 | _cache_repl[key] = p\r | |
277 | return p\r | |
278 | \r | |
279 | def _expand(pattern, match, template):\r | |
280 | # internal: match.expand implementation hook\r | |
281 | template = sre_parse.parse_template(template, pattern)\r | |
282 | return sre_parse.expand_template(template, match)\r | |
283 | \r | |
284 | def _subx(pattern, template):\r | |
285 | # internal: pattern.sub/subn implementation helper\r | |
286 | template = _compile_repl(template, pattern)\r | |
287 | if not template[0] and len(template[1]) == 1:\r | |
288 | # literal replacement\r | |
289 | return template[1][0]\r | |
290 | def filter(match, template=template):\r | |
291 | return sre_parse.expand_template(template, match)\r | |
292 | return filter\r | |
293 | \r | |
294 | # register myself for pickling\r | |
295 | \r | |
296 | import copy_reg\r | |
297 | \r | |
298 | def _pickle(p):\r | |
299 | return _compile, (p.pattern, p.flags)\r | |
300 | \r | |
301 | copy_reg.pickle(_pattern_type, _pickle, _compile)\r | |
302 | \r | |
303 | # --------------------------------------------------------------------\r | |
304 | # experimental stuff (see python-dev discussions for details)\r | |
305 | \r | |
306 | class Scanner:\r | |
307 | def __init__(self, lexicon, flags=0):\r | |
308 | from sre_constants import BRANCH, SUBPATTERN\r | |
309 | self.lexicon = lexicon\r | |
310 | # combine phrases into a compound pattern\r | |
311 | p = []\r | |
312 | s = sre_parse.Pattern()\r | |
313 | s.flags = flags\r | |
314 | for phrase, action in lexicon:\r | |
315 | p.append(sre_parse.SubPattern(s, [\r | |
316 | (SUBPATTERN, (len(p)+1, sre_parse.parse(phrase, flags))),\r | |
317 | ]))\r | |
318 | s.groups = len(p)+1\r | |
319 | p = sre_parse.SubPattern(s, [(BRANCH, (None, p))])\r | |
320 | self.scanner = sre_compile.compile(p)\r | |
321 | def scan(self, string):\r | |
322 | result = []\r | |
323 | append = result.append\r | |
324 | match = self.scanner.scanner(string).match\r | |
325 | i = 0\r | |
326 | while 1:\r | |
327 | m = match()\r | |
328 | if not m:\r | |
329 | break\r | |
330 | j = m.end()\r | |
331 | if i == j:\r | |
332 | break\r | |
333 | action = self.lexicon[m.lastindex-1][1]\r | |
334 | if hasattr(action, '__call__'):\r | |
335 | self.match = m\r | |
336 | action = action(self, m.group())\r | |
337 | if action is not None:\r | |
338 | append(action)\r | |
339 | i = j\r | |
340 | return result, string[i:]\r |