+++ /dev/null
-# Copyright (C) 2004-2006 Python Software Foundation\r
-# Authors: Baxter, Wouters and Warsaw\r
-# Contact: email-sig@python.org\r
-\r
-"""FeedParser - An email feed parser.\r
-\r
-The feed parser implements an interface for incrementally parsing an email\r
-message, line by line. This has advantages for certain applications, such as\r
-those reading email messages off a socket.\r
-\r
-FeedParser.feed() is the primary interface for pushing new data into the\r
-parser. It returns when there's nothing more it can do with the available\r
-data. When you have no more data to push into the parser, call .close().\r
-This completes the parsing and returns the root message object.\r
-\r
-The other advantage of this parser is that it will never throw a parsing\r
-exception. Instead, when it finds something unexpected, it adds a 'defect' to\r
-the current message. Defects are just instances that live on the message\r
-object's .defects attribute.\r
-"""\r
-\r
-__all__ = ['FeedParser']\r
-\r
-import re\r
-\r
-from email import errors\r
-from email import message\r
-\r
-NLCRE = re.compile('\r\n|\r|\n')\r
-NLCRE_bol = re.compile('(\r\n|\r|\n)')\r
-NLCRE_eol = re.compile('(\r\n|\r|\n)\Z')\r
-NLCRE_crack = re.compile('(\r\n|\r|\n)')\r
-# RFC 2822 $3.6.8 Optional fields. ftext is %d33-57 / %d59-126, Any character\r
-# except controls, SP, and ":".\r
-headerRE = re.compile(r'^(From |[\041-\071\073-\176]{1,}:|[\t ])')\r
-EMPTYSTRING = ''\r
-NL = '\n'\r
-\r
-NeedMoreData = object()\r
-\r
-\r
-\f\r
-class BufferedSubFile(object):\r
- """A file-ish object that can have new data loaded into it.\r
-\r
- You can also push and pop line-matching predicates onto a stack. When the\r
- current predicate matches the current line, a false EOF response\r
- (i.e. empty string) is returned instead. This lets the parser adhere to a\r
- simple abstraction -- it parses until EOF closes the current message.\r
- """\r
- def __init__(self):\r
- # The last partial line pushed into this object.\r
- self._partial = ''\r
- # The list of full, pushed lines, in reverse order\r
- self._lines = []\r
- # The stack of false-EOF checking predicates.\r
- self._eofstack = []\r
- # A flag indicating whether the file has been closed or not.\r
- self._closed = False\r
-\r
- def push_eof_matcher(self, pred):\r
- self._eofstack.append(pred)\r
-\r
- def pop_eof_matcher(self):\r
- return self._eofstack.pop()\r
-\r
- def close(self):\r
- # Don't forget any trailing partial line.\r
- self._lines.append(self._partial)\r
- self._partial = ''\r
- self._closed = True\r
-\r
- def readline(self):\r
- if not self._lines:\r
- if self._closed:\r
- return ''\r
- return NeedMoreData\r
- # Pop the line off the stack and see if it matches the current\r
- # false-EOF predicate.\r
- line = self._lines.pop()\r
- # RFC 2046, section 5.1.2 requires us to recognize outer level\r
- # boundaries at any level of inner nesting. Do this, but be sure it's\r
- # in the order of most to least nested.\r
- for ateof in self._eofstack[::-1]:\r
- if ateof(line):\r
- # We're at the false EOF. But push the last line back first.\r
- self._lines.append(line)\r
- return ''\r
- return line\r
-\r
- def unreadline(self, line):\r
- # Let the consumer push a line back into the buffer.\r
- assert line is not NeedMoreData\r
- self._lines.append(line)\r
-\r
- def push(self, data):\r
- """Push some new data into this object."""\r
- # Handle any previous leftovers\r
- data, self._partial = self._partial + data, ''\r
- # Crack into lines, but preserve the newlines on the end of each\r
- parts = NLCRE_crack.split(data)\r
- # The *ahem* interesting behaviour of re.split when supplied grouping\r
- # parentheses is that the last element of the resulting list is the\r
- # data after the final RE. In the case of a NL/CR terminated string,\r
- # this is the empty string.\r
- self._partial = parts.pop()\r
- #GAN 29Mar09 bugs 1555570, 1721862 Confusion at 8K boundary ending with \r:\r
- # is there a \n to follow later?\r
- if not self._partial and parts and parts[-1].endswith('\r'):\r
- self._partial = parts.pop(-2)+parts.pop()\r
- # parts is a list of strings, alternating between the line contents\r
- # and the eol character(s). Gather up a list of lines after\r
- # re-attaching the newlines.\r
- lines = []\r
- for i in range(len(parts) // 2):\r
- lines.append(parts[i*2] + parts[i*2+1])\r
- self.pushlines(lines)\r
-\r
- def pushlines(self, lines):\r
- # Reverse and insert at the front of the lines.\r
- self._lines[:0] = lines[::-1]\r
-\r
- def is_closed(self):\r
- return self._closed\r
-\r
- def __iter__(self):\r
- return self\r
-\r
- def next(self):\r
- line = self.readline()\r
- if line == '':\r
- raise StopIteration\r
- return line\r
-\r
-\r
-\f\r
-class FeedParser:\r
- """A feed-style parser of email."""\r
-\r
- def __init__(self, _factory=message.Message):\r
- """_factory is called with no arguments to create a new message obj"""\r
- self._factory = _factory\r
- self._input = BufferedSubFile()\r
- self._msgstack = []\r
- self._parse = self._parsegen().next\r
- self._cur = None\r
- self._last = None\r
- self._headersonly = False\r
-\r
- # Non-public interface for supporting Parser's headersonly flag\r
- def _set_headersonly(self):\r
- self._headersonly = True\r
-\r
- def feed(self, data):\r
- """Push more data into the parser."""\r
- self._input.push(data)\r
- self._call_parse()\r
-\r
- def _call_parse(self):\r
- try:\r
- self._parse()\r
- except StopIteration:\r
- pass\r
-\r
- def close(self):\r
- """Parse all remaining data and return the root message object."""\r
- self._input.close()\r
- self._call_parse()\r
- root = self._pop_message()\r
- assert not self._msgstack\r
- # Look for final set of defects\r
- if root.get_content_maintype() == 'multipart' \\r
- and not root.is_multipart():\r
- root.defects.append(errors.MultipartInvariantViolationDefect())\r
- return root\r
-\r
- def _new_message(self):\r
- msg = self._factory()\r
- if self._cur and self._cur.get_content_type() == 'multipart/digest':\r
- msg.set_default_type('message/rfc822')\r
- if self._msgstack:\r
- self._msgstack[-1].attach(msg)\r
- self._msgstack.append(msg)\r
- self._cur = msg\r
- self._last = msg\r
-\r
- def _pop_message(self):\r
- retval = self._msgstack.pop()\r
- if self._msgstack:\r
- self._cur = self._msgstack[-1]\r
- else:\r
- self._cur = None\r
- return retval\r
-\r
- def _parsegen(self):\r
- # Create a new message and start by parsing headers.\r
- self._new_message()\r
- headers = []\r
- # Collect the headers, searching for a line that doesn't match the RFC\r
- # 2822 header or continuation pattern (including an empty line).\r
- for line in self._input:\r
- if line is NeedMoreData:\r
- yield NeedMoreData\r
- continue\r
- if not headerRE.match(line):\r
- # If we saw the RFC defined header/body separator\r
- # (i.e. newline), just throw it away. Otherwise the line is\r
- # part of the body so push it back.\r
- if not NLCRE.match(line):\r
- self._input.unreadline(line)\r
- break\r
- headers.append(line)\r
- # Done with the headers, so parse them and figure out what we're\r
- # supposed to see in the body of the message.\r
- self._parse_headers(headers)\r
- # Headers-only parsing is a backwards compatibility hack, which was\r
- # necessary in the older parser, which could throw errors. All\r
- # remaining lines in the input are thrown into the message body.\r
- if self._headersonly:\r
- lines = []\r
- while True:\r
- line = self._input.readline()\r
- if line is NeedMoreData:\r
- yield NeedMoreData\r
- continue\r
- if line == '':\r
- break\r
- lines.append(line)\r
- self._cur.set_payload(EMPTYSTRING.join(lines))\r
- return\r
- if self._cur.get_content_type() == 'message/delivery-status':\r
- # message/delivery-status contains blocks of headers separated by\r
- # a blank line. We'll represent each header block as a separate\r
- # nested message object, but the processing is a bit different\r
- # than standard message/* types because there is no body for the\r
- # nested messages. A blank line separates the subparts.\r
- while True:\r
- self._input.push_eof_matcher(NLCRE.match)\r
- for retval in self._parsegen():\r
- if retval is NeedMoreData:\r
- yield NeedMoreData\r
- continue\r
- break\r
- msg = self._pop_message()\r
- # We need to pop the EOF matcher in order to tell if we're at\r
- # the end of the current file, not the end of the last block\r
- # of message headers.\r
- self._input.pop_eof_matcher()\r
- # The input stream must be sitting at the newline or at the\r
- # EOF. We want to see if we're at the end of this subpart, so\r
- # first consume the blank line, then test the next line to see\r
- # if we're at this subpart's EOF.\r
- while True:\r
- line = self._input.readline()\r
- if line is NeedMoreData:\r
- yield NeedMoreData\r
- continue\r
- break\r
- while True:\r
- line = self._input.readline()\r
- if line is NeedMoreData:\r
- yield NeedMoreData\r
- continue\r
- break\r
- if line == '':\r
- break\r
- # Not at EOF so this is a line we're going to need.\r
- self._input.unreadline(line)\r
- return\r
- if self._cur.get_content_maintype() == 'message':\r
- # The message claims to be a message/* type, then what follows is\r
- # another RFC 2822 message.\r
- for retval in self._parsegen():\r
- if retval is NeedMoreData:\r
- yield NeedMoreData\r
- continue\r
- break\r
- self._pop_message()\r
- return\r
- if self._cur.get_content_maintype() == 'multipart':\r
- boundary = self._cur.get_boundary()\r
- if boundary is None:\r
- # The message /claims/ to be a multipart but it has not\r
- # defined a boundary. That's a problem which we'll handle by\r
- # reading everything until the EOF and marking the message as\r
- # defective.\r
- self._cur.defects.append(errors.NoBoundaryInMultipartDefect())\r
- lines = []\r
- for line in self._input:\r
- if line is NeedMoreData:\r
- yield NeedMoreData\r
- continue\r
- lines.append(line)\r
- self._cur.set_payload(EMPTYSTRING.join(lines))\r
- return\r
- # Create a line match predicate which matches the inter-part\r
- # boundary as well as the end-of-multipart boundary. Don't push\r
- # this onto the input stream until we've scanned past the\r
- # preamble.\r
- separator = '--' + boundary\r
- boundaryre = re.compile(\r
- '(?P<sep>' + re.escape(separator) +\r
- r')(?P<end>--)?(?P<ws>[ \t]*)(?P<linesep>\r\n|\r|\n)?$')\r
- capturing_preamble = True\r
- preamble = []\r
- linesep = False\r
- while True:\r
- line = self._input.readline()\r
- if line is NeedMoreData:\r
- yield NeedMoreData\r
- continue\r
- if line == '':\r
- break\r
- mo = boundaryre.match(line)\r
- if mo:\r
- # If we're looking at the end boundary, we're done with\r
- # this multipart. If there was a newline at the end of\r
- # the closing boundary, then we need to initialize the\r
- # epilogue with the empty string (see below).\r
- if mo.group('end'):\r
- linesep = mo.group('linesep')\r
- break\r
- # We saw an inter-part boundary. Were we in the preamble?\r
- if capturing_preamble:\r
- if preamble:\r
- # According to RFC 2046, the last newline belongs\r
- # to the boundary.\r
- lastline = preamble[-1]\r
- eolmo = NLCRE_eol.search(lastline)\r
- if eolmo:\r
- preamble[-1] = lastline[:-len(eolmo.group(0))]\r
- self._cur.preamble = EMPTYSTRING.join(preamble)\r
- capturing_preamble = False\r
- self._input.unreadline(line)\r
- continue\r
- # We saw a boundary separating two parts. Consume any\r
- # multiple boundary lines that may be following. Our\r
- # interpretation of RFC 2046 BNF grammar does not produce\r
- # body parts within such double boundaries.\r
- while True:\r
- line = self._input.readline()\r
- if line is NeedMoreData:\r
- yield NeedMoreData\r
- continue\r
- mo = boundaryre.match(line)\r
- if not mo:\r
- self._input.unreadline(line)\r
- break\r
- # Recurse to parse this subpart; the input stream points\r
- # at the subpart's first line.\r
- self._input.push_eof_matcher(boundaryre.match)\r
- for retval in self._parsegen():\r
- if retval is NeedMoreData:\r
- yield NeedMoreData\r
- continue\r
- break\r
- # Because of RFC 2046, the newline preceding the boundary\r
- # separator actually belongs to the boundary, not the\r
- # previous subpart's payload (or epilogue if the previous\r
- # part is a multipart).\r
- if self._last.get_content_maintype() == 'multipart':\r
- epilogue = self._last.epilogue\r
- if epilogue == '':\r
- self._last.epilogue = None\r
- elif epilogue is not None:\r
- mo = NLCRE_eol.search(epilogue)\r
- if mo:\r
- end = len(mo.group(0))\r
- self._last.epilogue = epilogue[:-end]\r
- else:\r
- payload = self._last.get_payload()\r
- if isinstance(payload, basestring):\r
- mo = NLCRE_eol.search(payload)\r
- if mo:\r
- payload = payload[:-len(mo.group(0))]\r
- self._last.set_payload(payload)\r
- self._input.pop_eof_matcher()\r
- self._pop_message()\r
- # Set the multipart up for newline cleansing, which will\r
- # happen if we're in a nested multipart.\r
- self._last = self._cur\r
- else:\r
- # I think we must be in the preamble\r
- assert capturing_preamble\r
- preamble.append(line)\r
- # We've seen either the EOF or the end boundary. If we're still\r
- # capturing the preamble, we never saw the start boundary. Note\r
- # that as a defect and store the captured text as the payload.\r
- # Everything from here to the EOF is epilogue.\r
- if capturing_preamble:\r
- self._cur.defects.append(errors.StartBoundaryNotFoundDefect())\r
- self._cur.set_payload(EMPTYSTRING.join(preamble))\r
- epilogue = []\r
- for line in self._input:\r
- if line is NeedMoreData:\r
- yield NeedMoreData\r
- continue\r
- self._cur.epilogue = EMPTYSTRING.join(epilogue)\r
- return\r
- # If the end boundary ended in a newline, we'll need to make sure\r
- # the epilogue isn't None\r
- if linesep:\r
- epilogue = ['']\r
- else:\r
- epilogue = []\r
- for line in self._input:\r
- if line is NeedMoreData:\r
- yield NeedMoreData\r
- continue\r
- epilogue.append(line)\r
- # Any CRLF at the front of the epilogue is not technically part of\r
- # the epilogue. Also, watch out for an empty string epilogue,\r
- # which means a single newline.\r
- if epilogue:\r
- firstline = epilogue[0]\r
- bolmo = NLCRE_bol.match(firstline)\r
- if bolmo:\r
- epilogue[0] = firstline[len(bolmo.group(0)):]\r
- self._cur.epilogue = EMPTYSTRING.join(epilogue)\r
- return\r
- # Otherwise, it's some non-multipart type, so the entire rest of the\r
- # file contents becomes the payload.\r
- lines = []\r
- for line in self._input:\r
- if line is NeedMoreData:\r
- yield NeedMoreData\r
- continue\r
- lines.append(line)\r
- self._cur.set_payload(EMPTYSTRING.join(lines))\r
-\r
- def _parse_headers(self, lines):\r
- # Passed a list of lines that make up the headers for the current msg\r
- lastheader = ''\r
- lastvalue = []\r
- for lineno, line in enumerate(lines):\r
- # Check for continuation\r
- if line[0] in ' \t':\r
- if not lastheader:\r
- # The first line of the headers was a continuation. This\r
- # is illegal, so let's note the defect, store the illegal\r
- # line, and ignore it for purposes of headers.\r
- defect = errors.FirstHeaderLineIsContinuationDefect(line)\r
- self._cur.defects.append(defect)\r
- continue\r
- lastvalue.append(line)\r
- continue\r
- if lastheader:\r
- # XXX reconsider the joining of folded lines\r
- lhdr = EMPTYSTRING.join(lastvalue)[:-1].rstrip('\r\n')\r
- self._cur[lastheader] = lhdr\r
- lastheader, lastvalue = '', []\r
- # Check for envelope header, i.e. unix-from\r
- if line.startswith('From '):\r
- if lineno == 0:\r
- # Strip off the trailing newline\r
- mo = NLCRE_eol.search(line)\r
- if mo:\r
- line = line[:-len(mo.group(0))]\r
- self._cur.set_unixfrom(line)\r
- continue\r
- elif lineno == len(lines) - 1:\r
- # Something looking like a unix-from at the end - it's\r
- # probably the first line of the body, so push back the\r
- # line and stop.\r
- self._input.unreadline(line)\r
- return\r
- else:\r
- # Weirdly placed unix-from line. Note this as a defect\r
- # and ignore it.\r
- defect = errors.MisplacedEnvelopeHeaderDefect(line)\r
- self._cur.defects.append(defect)\r
- continue\r
- # Split the line on the colon separating field name from value.\r
- i = line.find(':')\r
- if i < 0:\r
- defect = errors.MalformedHeaderDefect(line)\r
- self._cur.defects.append(defect)\r
- continue\r
- lastheader = line[:i]\r
- lastvalue = [line[i+1:].lstrip()]\r
- # Done with all the lines, so handle the last header.\r
- if lastheader:\r
- # XXX reconsider the joining of folded lines\r
- self._cur[lastheader] = EMPTYSTRING.join(lastvalue).rstrip('\r\n')\r
projects / mirror_edk2.git / blobdiff