+++ /dev/null
-"""HTTP cookie handling for web clients.\r
-\r
-This module has (now fairly distant) origins in Gisle Aas' Perl module\r
-HTTP::Cookies, from the libwww-perl library.\r
-\r
-Docstrings, comments and debug strings in this code refer to the\r
-attributes of the HTTP cookie system as cookie-attributes, to distinguish\r
-them clearly from Python attributes.\r
-\r
-Class diagram (note that BSDDBCookieJar and the MSIE* classes are not\r
-distributed with the Python standard library, but are available from\r
-http://wwwsearch.sf.net/):\r
-\r
- CookieJar____\r
- / \ \\r
- FileCookieJar \ \\r
- / | \ \ \\r
- MozillaCookieJar | LWPCookieJar \ \\r
- | | \\r
- | ---MSIEBase | \\r
- | / | | \\r
- | / MSIEDBCookieJar BSDDBCookieJar\r
- |/\r
- MSIECookieJar\r
-\r
-"""\r
-\r
-__all__ = ['Cookie', 'CookieJar', 'CookiePolicy', 'DefaultCookiePolicy',\r
- 'FileCookieJar', 'LWPCookieJar', 'lwp_cookie_str', 'LoadError',\r
- 'MozillaCookieJar']\r
-\r
-import re, urlparse, copy, time, urllib\r
-try:\r
- import threading as _threading\r
-except ImportError:\r
- import dummy_threading as _threading\r
-import httplib # only for the default HTTP port\r
-from calendar import timegm\r
-\r
-debug = False # set to True to enable debugging via the logging module\r
-logger = None\r
-\r
-def _debug(*args):\r
- if not debug:\r
- return\r
- global logger\r
- if not logger:\r
- import logging\r
- logger = logging.getLogger("cookielib")\r
- return logger.debug(*args)\r
-\r
-\r
-DEFAULT_HTTP_PORT = str(httplib.HTTP_PORT)\r
-MISSING_FILENAME_TEXT = ("a filename was not supplied (nor was the CookieJar "\r
- "instance initialised with one)")\r
-\r
-def _warn_unhandled_exception():\r
- # There are a few catch-all except: statements in this module, for\r
- # catching input that's bad in unexpected ways. Warn if any\r
- # exceptions are caught there.\r
- import warnings, traceback, StringIO\r
- f = StringIO.StringIO()\r
- traceback.print_exc(None, f)\r
- msg = f.getvalue()\r
- warnings.warn("cookielib bug!\n%s" % msg, stacklevel=2)\r
-\r
-\r
-# Date/time conversion\r
-# -----------------------------------------------------------------------------\r
-\r
-EPOCH_YEAR = 1970\r
-def _timegm(tt):\r
- year, month, mday, hour, min, sec = tt[:6]\r
- if ((year >= EPOCH_YEAR) and (1 <= month <= 12) and (1 <= mday <= 31) and\r
- (0 <= hour <= 24) and (0 <= min <= 59) and (0 <= sec <= 61)):\r
- return timegm(tt)\r
- else:\r
- return None\r
-\r
-DAYS = ["Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"]\r
-MONTHS = ["Jan", "Feb", "Mar", "Apr", "May", "Jun",\r
- "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"]\r
-MONTHS_LOWER = []\r
-for month in MONTHS: MONTHS_LOWER.append(month.lower())\r
-\r
-def time2isoz(t=None):\r
- """Return a string representing time in seconds since epoch, t.\r
-\r
- If the function is called without an argument, it will use the current\r
- time.\r
-\r
- The format of the returned string is like "YYYY-MM-DD hh:mm:ssZ",\r
- representing Universal Time (UTC, aka GMT). An example of this format is:\r
-\r
- 1994-11-24 08:49:37Z\r
-\r
- """\r
- if t is None: t = time.time()\r
- year, mon, mday, hour, min, sec = time.gmtime(t)[:6]\r
- return "%04d-%02d-%02d %02d:%02d:%02dZ" % (\r
- year, mon, mday, hour, min, sec)\r
-\r
-def time2netscape(t=None):\r
- """Return a string representing time in seconds since epoch, t.\r
-\r
- If the function is called without an argument, it will use the current\r
- time.\r
-\r
- The format of the returned string is like this:\r
-\r
- Wed, DD-Mon-YYYY HH:MM:SS GMT\r
-\r
- """\r
- if t is None: t = time.time()\r
- year, mon, mday, hour, min, sec, wday = time.gmtime(t)[:7]\r
- return "%s %02d-%s-%04d %02d:%02d:%02d GMT" % (\r
- DAYS[wday], mday, MONTHS[mon-1], year, hour, min, sec)\r
-\r
-\r
-UTC_ZONES = {"GMT": None, "UTC": None, "UT": None, "Z": None}\r
-\r
-TIMEZONE_RE = re.compile(r"^([-+])?(\d\d?):?(\d\d)?$")\r
-def offset_from_tz_string(tz):\r
- offset = None\r
- if tz in UTC_ZONES:\r
- offset = 0\r
- else:\r
- m = TIMEZONE_RE.search(tz)\r
- if m:\r
- offset = 3600 * int(m.group(2))\r
- if m.group(3):\r
- offset = offset + 60 * int(m.group(3))\r
- if m.group(1) == '-':\r
- offset = -offset\r
- return offset\r
-\r
-def _str2time(day, mon, yr, hr, min, sec, tz):\r
- # translate month name to number\r
- # month numbers start with 1 (January)\r
- try:\r
- mon = MONTHS_LOWER.index(mon.lower())+1\r
- except ValueError:\r
- # maybe it's already a number\r
- try:\r
- imon = int(mon)\r
- except ValueError:\r
- return None\r
- if 1 <= imon <= 12:\r
- mon = imon\r
- else:\r
- return None\r
-\r
- # make sure clock elements are defined\r
- if hr is None: hr = 0\r
- if min is None: min = 0\r
- if sec is None: sec = 0\r
-\r
- yr = int(yr)\r
- day = int(day)\r
- hr = int(hr)\r
- min = int(min)\r
- sec = int(sec)\r
-\r
- if yr < 1000:\r
- # find "obvious" year\r
- cur_yr = time.localtime(time.time())[0]\r
- m = cur_yr % 100\r
- tmp = yr\r
- yr = yr + cur_yr - m\r
- m = m - tmp\r
- if abs(m) > 50:\r
- if m > 0: yr = yr + 100\r
- else: yr = yr - 100\r
-\r
- # convert UTC time tuple to seconds since epoch (not timezone-adjusted)\r
- t = _timegm((yr, mon, day, hr, min, sec, tz))\r
-\r
- if t is not None:\r
- # adjust time using timezone string, to get absolute time since epoch\r
- if tz is None:\r
- tz = "UTC"\r
- tz = tz.upper()\r
- offset = offset_from_tz_string(tz)\r
- if offset is None:\r
- return None\r
- t = t - offset\r
-\r
- return t\r
-\r
-STRICT_DATE_RE = re.compile(\r
- r"^[SMTWF][a-z][a-z], (\d\d) ([JFMASOND][a-z][a-z]) "\r
- "(\d\d\d\d) (\d\d):(\d\d):(\d\d) GMT$")\r
-WEEKDAY_RE = re.compile(\r
- r"^(?:Sun|Mon|Tue|Wed|Thu|Fri|Sat)[a-z]*,?\s*", re.I)\r
-LOOSE_HTTP_DATE_RE = re.compile(\r
- r"""^\r
- (\d\d?) # day\r
- (?:\s+|[-\/])\r
- (\w+) # month\r
- (?:\s+|[-\/])\r
- (\d+) # year\r
- (?:\r
- (?:\s+|:) # separator before clock\r
- (\d\d?):(\d\d) # hour:min\r
- (?::(\d\d))? # optional seconds\r
- )? # optional clock\r
- \s*\r
- ([-+]?\d{2,4}|(?![APap][Mm]\b)[A-Za-z]+)? # timezone\r
- \s*\r
- (?:\(\w+\))? # ASCII representation of timezone in parens.\r
- \s*$""", re.X)\r
-def http2time(text):\r
- """Returns time in seconds since epoch of time represented by a string.\r
-\r
- Return value is an integer.\r
-\r
- None is returned if the format of str is unrecognized, the time is outside\r
- the representable range, or the timezone string is not recognized. If the\r
- string contains no timezone, UTC is assumed.\r
-\r
- The timezone in the string may be numerical (like "-0800" or "+0100") or a\r
- string timezone (like "UTC", "GMT", "BST" or "EST"). Currently, only the\r
- timezone strings equivalent to UTC (zero offset) are known to the function.\r
-\r
- The function loosely parses the following formats:\r
-\r
- Wed, 09 Feb 1994 22:23:32 GMT -- HTTP format\r
- Tuesday, 08-Feb-94 14:15:29 GMT -- old rfc850 HTTP format\r
- Tuesday, 08-Feb-1994 14:15:29 GMT -- broken rfc850 HTTP format\r
- 09 Feb 1994 22:23:32 GMT -- HTTP format (no weekday)\r
- 08-Feb-94 14:15:29 GMT -- rfc850 format (no weekday)\r
- 08-Feb-1994 14:15:29 GMT -- broken rfc850 format (no weekday)\r
-\r
- The parser ignores leading and trailing whitespace. The time may be\r
- absent.\r
-\r
- If the year is given with only 2 digits, the function will select the\r
- century that makes the year closest to the current date.\r
-\r
- """\r
- # fast exit for strictly conforming string\r
- m = STRICT_DATE_RE.search(text)\r
- if m:\r
- g = m.groups()\r
- mon = MONTHS_LOWER.index(g[1].lower()) + 1\r
- tt = (int(g[2]), mon, int(g[0]),\r
- int(g[3]), int(g[4]), float(g[5]))\r
- return _timegm(tt)\r
-\r
- # No, we need some messy parsing...\r
-\r
- # clean up\r
- text = text.lstrip()\r
- text = WEEKDAY_RE.sub("", text, 1) # Useless weekday\r
-\r
- # tz is time zone specifier string\r
- day, mon, yr, hr, min, sec, tz = [None]*7\r
-\r
- # loose regexp parse\r
- m = LOOSE_HTTP_DATE_RE.search(text)\r
- if m is not None:\r
- day, mon, yr, hr, min, sec, tz = m.groups()\r
- else:\r
- return None # bad format\r
-\r
- return _str2time(day, mon, yr, hr, min, sec, tz)\r
-\r
-ISO_DATE_RE = re.compile(\r
- """^\r
- (\d{4}) # year\r
- [-\/]?\r
- (\d\d?) # numerical month\r
- [-\/]?\r
- (\d\d?) # day\r
- (?:\r
- (?:\s+|[-:Tt]) # separator before clock\r
- (\d\d?):?(\d\d) # hour:min\r
- (?::?(\d\d(?:\.\d*)?))? # optional seconds (and fractional)\r
- )? # optional clock\r
- \s*\r
- ([-+]?\d\d?:?(:?\d\d)?\r
- |Z|z)? # timezone (Z is "zero meridian", i.e. GMT)\r
- \s*$""", re.X)\r
-def iso2time(text):\r
- """\r
- As for http2time, but parses the ISO 8601 formats:\r
-\r
- 1994-02-03 14:15:29 -0100 -- ISO 8601 format\r
- 1994-02-03 14:15:29 -- zone is optional\r
- 1994-02-03 -- only date\r
- 1994-02-03T14:15:29 -- Use T as separator\r
- 19940203T141529Z -- ISO 8601 compact format\r
- 19940203 -- only date\r
-\r
- """\r
- # clean up\r
- text = text.lstrip()\r
-\r
- # tz is time zone specifier string\r
- day, mon, yr, hr, min, sec, tz = [None]*7\r
-\r
- # loose regexp parse\r
- m = ISO_DATE_RE.search(text)\r
- if m is not None:\r
- # XXX there's an extra bit of the timezone I'm ignoring here: is\r
- # this the right thing to do?\r
- yr, mon, day, hr, min, sec, tz, _ = m.groups()\r
- else:\r
- return None # bad format\r
-\r
- return _str2time(day, mon, yr, hr, min, sec, tz)\r
-\r
-\r
-# Header parsing\r
-# -----------------------------------------------------------------------------\r
-\r
-def unmatched(match):\r
- """Return unmatched part of re.Match object."""\r
- start, end = match.span(0)\r
- return match.string[:start]+match.string[end:]\r
-\r
-HEADER_TOKEN_RE = re.compile(r"^\s*([^=\s;,]+)")\r
-HEADER_QUOTED_VALUE_RE = re.compile(r"^\s*=\s*\"([^\"\\]*(?:\\.[^\"\\]*)*)\"")\r
-HEADER_VALUE_RE = re.compile(r"^\s*=\s*([^\s;,]*)")\r
-HEADER_ESCAPE_RE = re.compile(r"\\(.)")\r
-def split_header_words(header_values):\r
- r"""Parse header values into a list of lists containing key,value pairs.\r
-\r
- The function knows how to deal with ",", ";" and "=" as well as quoted\r
- values after "=". A list of space separated tokens are parsed as if they\r
- were separated by ";".\r
-\r
- If the header_values passed as argument contains multiple values, then they\r
- are treated as if they were a single value separated by comma ",".\r
-\r
- This means that this function is useful for parsing header fields that\r
- follow this syntax (BNF as from the HTTP/1.1 specification, but we relax\r
- the requirement for tokens).\r
-\r
- headers = #header\r
- header = (token | parameter) *( [";"] (token | parameter))\r
-\r
- token = 1*<any CHAR except CTLs or separators>\r
- separators = "(" | ")" | "<" | ">" | "@"\r
- | "," | ";" | ":" | "\" | <">\r
- | "/" | "[" | "]" | "?" | "="\r
- | "{" | "}" | SP | HT\r
-\r
- quoted-string = ( <"> *(qdtext | quoted-pair ) <"> )\r
- qdtext = <any TEXT except <">>\r
- quoted-pair = "\" CHAR\r
-\r
- parameter = attribute "=" value\r
- attribute = token\r
- value = token | quoted-string\r
-\r
- Each header is represented by a list of key/value pairs. The value for a\r
- simple token (not part of a parameter) is None. Syntactically incorrect\r
- headers will not necessarily be parsed as you would want.\r
-\r
- This is easier to describe with some examples:\r
-\r
- >>> split_header_words(['foo="bar"; port="80,81"; discard, bar=baz'])\r
- [[('foo', 'bar'), ('port', '80,81'), ('discard', None)], [('bar', 'baz')]]\r
- >>> split_header_words(['text/html; charset="iso-8859-1"'])\r
- [[('text/html', None), ('charset', 'iso-8859-1')]]\r
- >>> split_header_words([r'Basic realm="\"foo\bar\""'])\r
- [[('Basic', None), ('realm', '"foobar"')]]\r
-\r
- """\r
- assert not isinstance(header_values, basestring)\r
- result = []\r
- for text in header_values:\r
- orig_text = text\r
- pairs = []\r
- while text:\r
- m = HEADER_TOKEN_RE.search(text)\r
- if m:\r
- text = unmatched(m)\r
- name = m.group(1)\r
- m = HEADER_QUOTED_VALUE_RE.search(text)\r
- if m: # quoted value\r
- text = unmatched(m)\r
- value = m.group(1)\r
- value = HEADER_ESCAPE_RE.sub(r"\1", value)\r
- else:\r
- m = HEADER_VALUE_RE.search(text)\r
- if m: # unquoted value\r
- text = unmatched(m)\r
- value = m.group(1)\r
- value = value.rstrip()\r
- else:\r
- # no value, a lone token\r
- value = None\r
- pairs.append((name, value))\r
- elif text.lstrip().startswith(","):\r
- # concatenated headers, as per RFC 2616 section 4.2\r
- text = text.lstrip()[1:]\r
- if pairs: result.append(pairs)\r
- pairs = []\r
- else:\r
- # skip junk\r
- non_junk, nr_junk_chars = re.subn("^[=\s;]*", "", text)\r
- assert nr_junk_chars > 0, (\r
- "split_header_words bug: '%s', '%s', %s" %\r
- (orig_text, text, pairs))\r
- text = non_junk\r
- if pairs: result.append(pairs)\r
- return result\r
-\r
-HEADER_JOIN_ESCAPE_RE = re.compile(r"([\"\\])")\r
-def join_header_words(lists):\r
- """Do the inverse (almost) of the conversion done by split_header_words.\r
-\r
- Takes a list of lists of (key, value) pairs and produces a single header\r
- value. Attribute values are quoted if needed.\r
-\r
- >>> join_header_words([[("text/plain", None), ("charset", "iso-8859/1")]])\r
- 'text/plain; charset="iso-8859/1"'\r
- >>> join_header_words([[("text/plain", None)], [("charset", "iso-8859/1")]])\r
- 'text/plain, charset="iso-8859/1"'\r
-\r
- """\r
- headers = []\r
- for pairs in lists:\r
- attr = []\r
- for k, v in pairs:\r
- if v is not None:\r
- if not re.search(r"^\w+$", v):\r
- v = HEADER_JOIN_ESCAPE_RE.sub(r"\\\1", v) # escape " and \\r
- v = '"%s"' % v\r
- k = "%s=%s" % (k, v)\r
- attr.append(k)\r
- if attr: headers.append("; ".join(attr))\r
- return ", ".join(headers)\r
-\r
-def _strip_quotes(text):\r
- if text.startswith('"'):\r
- text = text[1:]\r
- if text.endswith('"'):\r
- text = text[:-1]\r
- return text\r
-\r
-def parse_ns_headers(ns_headers):\r
- """Ad-hoc parser for Netscape protocol cookie-attributes.\r
-\r
- The old Netscape cookie format for Set-Cookie can for instance contain\r
- an unquoted "," in the expires field, so we have to use this ad-hoc\r
- parser instead of split_header_words.\r
-\r
- XXX This may not make the best possible effort to parse all the crap\r
- that Netscape Cookie headers contain. Ronald Tschalar's HTTPClient\r
- parser is probably better, so could do worse than following that if\r
- this ever gives any trouble.\r
-\r
- Currently, this is also used for parsing RFC 2109 cookies.\r
-\r
- """\r
- known_attrs = ("expires", "domain", "path", "secure",\r
- # RFC 2109 attrs (may turn up in Netscape cookies, too)\r
- "version", "port", "max-age")\r
-\r
- result = []\r
- for ns_header in ns_headers:\r
- pairs = []\r
- version_set = False\r
- for ii, param in enumerate(re.split(r";\s*", ns_header)):\r
- param = param.rstrip()\r
- if param == "": continue\r
- if "=" not in param:\r
- k, v = param, None\r
- else:\r
- k, v = re.split(r"\s*=\s*", param, 1)\r
- k = k.lstrip()\r
- if ii != 0:\r
- lc = k.lower()\r
- if lc in known_attrs:\r
- k = lc\r
- if k == "version":\r
- # This is an RFC 2109 cookie.\r
- v = _strip_quotes(v)\r
- version_set = True\r
- if k == "expires":\r
- # convert expires date to seconds since epoch\r
- v = http2time(_strip_quotes(v)) # None if invalid\r
- pairs.append((k, v))\r
-\r
- if pairs:\r
- if not version_set:\r
- pairs.append(("version", "0"))\r
- result.append(pairs)\r
-\r
- return result\r
-\r
-\r
-IPV4_RE = re.compile(r"\.\d+$")\r
-def is_HDN(text):\r
- """Return True if text is a host domain name."""\r
- # XXX\r
- # This may well be wrong. Which RFC is HDN defined in, if any (for\r
- # the purposes of RFC 2965)?\r
- # For the current implementation, what about IPv6? Remember to look\r
- # at other uses of IPV4_RE also, if change this.\r
- if IPV4_RE.search(text):\r
- return False\r
- if text == "":\r
- return False\r
- if text[0] == "." or text[-1] == ".":\r
- return False\r
- return True\r
-\r
-def domain_match(A, B):\r
- """Return True if domain A domain-matches domain B, according to RFC 2965.\r
-\r
- A and B may be host domain names or IP addresses.\r
-\r
- RFC 2965, section 1:\r
-\r
- Host names can be specified either as an IP address or a HDN string.\r
- Sometimes we compare one host name with another. (Such comparisons SHALL\r
- be case-insensitive.) Host A's name domain-matches host B's if\r
-\r
- * their host name strings string-compare equal; or\r
-\r
- * A is a HDN string and has the form NB, where N is a non-empty\r
- name string, B has the form .B', and B' is a HDN string. (So,\r
- x.y.com domain-matches .Y.com but not Y.com.)\r
-\r
- Note that domain-match is not a commutative operation: a.b.c.com\r
- domain-matches .c.com, but not the reverse.\r
-\r
- """\r
- # Note that, if A or B are IP addresses, the only relevant part of the\r
- # definition of the domain-match algorithm is the direct string-compare.\r
- A = A.lower()\r
- B = B.lower()\r
- if A == B:\r
- return True\r
- if not is_HDN(A):\r
- return False\r
- i = A.rfind(B)\r
- if i == -1 or i == 0:\r
- # A does not have form NB, or N is the empty string\r
- return False\r
- if not B.startswith("."):\r
- return False\r
- if not is_HDN(B[1:]):\r
- return False\r
- return True\r
-\r
-def liberal_is_HDN(text):\r
- """Return True if text is a sort-of-like a host domain name.\r
-\r
- For accepting/blocking domains.\r
-\r
- """\r
- if IPV4_RE.search(text):\r
- return False\r
- return True\r
-\r
-def user_domain_match(A, B):\r
- """For blocking/accepting domains.\r
-\r
- A and B may be host domain names or IP addresses.\r
-\r
- """\r
- A = A.lower()\r
- B = B.lower()\r
- if not (liberal_is_HDN(A) and liberal_is_HDN(B)):\r
- if A == B:\r
- # equal IP addresses\r
- return True\r
- return False\r
- initial_dot = B.startswith(".")\r
- if initial_dot and A.endswith(B):\r
- return True\r
- if not initial_dot and A == B:\r
- return True\r
- return False\r
-\r
-cut_port_re = re.compile(r":\d+$")\r
-def request_host(request):\r
- """Return request-host, as defined by RFC 2965.\r
-\r
- Variation from RFC: returned value is lowercased, for convenient\r
- comparison.\r
-\r
- """\r
- url = request.get_full_url()\r
- host = urlparse.urlparse(url)[1]\r
- if host == "":\r
- host = request.get_header("Host", "")\r
-\r
- # remove port, if present\r
- host = cut_port_re.sub("", host, 1)\r
- return host.lower()\r
-\r
-def eff_request_host(request):\r
- """Return a tuple (request-host, effective request-host name).\r
-\r
- As defined by RFC 2965, except both are lowercased.\r
-\r
- """\r
- erhn = req_host = request_host(request)\r
- if req_host.find(".") == -1 and not IPV4_RE.search(req_host):\r
- erhn = req_host + ".local"\r
- return req_host, erhn\r
-\r
-def request_path(request):\r
- """Path component of request-URI, as defined by RFC 2965."""\r
- url = request.get_full_url()\r
- parts = urlparse.urlsplit(url)\r
- path = escape_path(parts.path)\r
- if not path.startswith("/"):\r
- # fix bad RFC 2396 absoluteURI\r
- path = "/" + path\r
- return path\r
-\r
-def request_port(request):\r
- host = request.get_host()\r
- i = host.find(':')\r
- if i >= 0:\r
- port = host[i+1:]\r
- try:\r
- int(port)\r
- except ValueError:\r
- _debug("nonnumeric port: '%s'", port)\r
- return None\r
- else:\r
- port = DEFAULT_HTTP_PORT\r
- return port\r
-\r
-# Characters in addition to A-Z, a-z, 0-9, '_', '.', and '-' that don't\r
-# need to be escaped to form a valid HTTP URL (RFCs 2396 and 1738).\r
-HTTP_PATH_SAFE = "%/;:@&=+$,!~*'()"\r
-ESCAPED_CHAR_RE = re.compile(r"%([0-9a-fA-F][0-9a-fA-F])")\r
-def uppercase_escaped_char(match):\r
- return "%%%s" % match.group(1).upper()\r
-def escape_path(path):\r
- """Escape any invalid characters in HTTP URL, and uppercase all escapes."""\r
- # There's no knowing what character encoding was used to create URLs\r
- # containing %-escapes, but since we have to pick one to escape invalid\r
- # path characters, we pick UTF-8, as recommended in the HTML 4.0\r
- # specification:\r
- # http://www.w3.org/TR/REC-html40/appendix/notes.html#h-B.2.1\r
- # And here, kind of: draft-fielding-uri-rfc2396bis-03\r
- # (And in draft IRI specification: draft-duerst-iri-05)\r
- # (And here, for new URI schemes: RFC 2718)\r
- if isinstance(path, unicode):\r
- path = path.encode("utf-8")\r
- path = urllib.quote(path, HTTP_PATH_SAFE)\r
- path = ESCAPED_CHAR_RE.sub(uppercase_escaped_char, path)\r
- return path\r
-\r
-def reach(h):\r
- """Return reach of host h, as defined by RFC 2965, section 1.\r
-\r
- The reach R of a host name H is defined as follows:\r
-\r
- * If\r
-\r
- - H is the host domain name of a host; and,\r
-\r
- - H has the form A.B; and\r
-\r
- - A has no embedded (that is, interior) dots; and\r
-\r
- - B has at least one embedded dot, or B is the string "local".\r
- then the reach of H is .B.\r
-\r
- * Otherwise, the reach of H is H.\r
-\r
- >>> reach("www.acme.com")\r
- '.acme.com'\r
- >>> reach("acme.com")\r
- 'acme.com'\r
- >>> reach("acme.local")\r
- '.local'\r
-\r
- """\r
- i = h.find(".")\r
- if i >= 0:\r
- #a = h[:i] # this line is only here to show what a is\r
- b = h[i+1:]\r
- i = b.find(".")\r
- if is_HDN(h) and (i >= 0 or b == "local"):\r
- return "."+b\r
- return h\r
-\r
-def is_third_party(request):\r
- """\r
-\r
- RFC 2965, section 3.3.6:\r
-\r
- An unverifiable transaction is to a third-party host if its request-\r
- host U does not domain-match the reach R of the request-host O in the\r
- origin transaction.\r
-\r
- """\r
- req_host = request_host(request)\r
- if not domain_match(req_host, reach(request.get_origin_req_host())):\r
- return True\r
- else:\r
- return False\r
-\r
-\r
-class Cookie:\r
- """HTTP Cookie.\r
-\r
- This class represents both Netscape and RFC 2965 cookies.\r
-\r
- This is deliberately a very simple class. It just holds attributes. It's\r
- possible to construct Cookie instances that don't comply with the cookie\r
- standards. CookieJar.make_cookies is the factory function for Cookie\r
- objects -- it deals with cookie parsing, supplying defaults, and\r
- normalising to the representation used in this class. CookiePolicy is\r
- responsible for checking them to see whether they should be accepted from\r
- and returned to the server.\r
-\r
- Note that the port may be present in the headers, but unspecified ("Port"\r
- rather than"Port=80", for example); if this is the case, port is None.\r
-\r
- """\r
-\r
- def __init__(self, version, name, value,\r
- port, port_specified,\r
- domain, domain_specified, domain_initial_dot,\r
- path, path_specified,\r
- secure,\r
- expires,\r
- discard,\r
- comment,\r
- comment_url,\r
- rest,\r
- rfc2109=False,\r
- ):\r
-\r
- if version is not None: version = int(version)\r
- if expires is not None: expires = int(expires)\r
- if port is None and port_specified is True:\r
- raise ValueError("if port is None, port_specified must be false")\r
-\r
- self.version = version\r
- self.name = name\r
- self.value = value\r
- self.port = port\r
- self.port_specified = port_specified\r
- # normalise case, as per RFC 2965 section 3.3.3\r
- self.domain = domain.lower()\r
- self.domain_specified = domain_specified\r
- # Sigh. We need to know whether the domain given in the\r
- # cookie-attribute had an initial dot, in order to follow RFC 2965\r
- # (as clarified in draft errata). Needed for the returned $Domain\r
- # value.\r
- self.domain_initial_dot = domain_initial_dot\r
- self.path = path\r
- self.path_specified = path_specified\r
- self.secure = secure\r
- self.expires = expires\r
- self.discard = discard\r
- self.comment = comment\r
- self.comment_url = comment_url\r
- self.rfc2109 = rfc2109\r
-\r
- self._rest = copy.copy(rest)\r
-\r
- def has_nonstandard_attr(self, name):\r
- return name in self._rest\r
- def get_nonstandard_attr(self, name, default=None):\r
- return self._rest.get(name, default)\r
- def set_nonstandard_attr(self, name, value):\r
- self._rest[name] = value\r
-\r
- def is_expired(self, now=None):\r
- if now is None: now = time.time()\r
- if (self.expires is not None) and (self.expires <= now):\r
- return True\r
- return False\r
-\r
- def __str__(self):\r
- if self.port is None: p = ""\r
- else: p = ":"+self.port\r
- limit = self.domain + p + self.path\r
- if self.value is not None:\r
- namevalue = "%s=%s" % (self.name, self.value)\r
- else:\r
- namevalue = self.name\r
- return "<Cookie %s for %s>" % (namevalue, limit)\r
-\r
- def __repr__(self):\r
- args = []\r
- for name in ("version", "name", "value",\r
- "port", "port_specified",\r
- "domain", "domain_specified", "domain_initial_dot",\r
- "path", "path_specified",\r
- "secure", "expires", "discard", "comment", "comment_url",\r
- ):\r
- attr = getattr(self, name)\r
- args.append("%s=%s" % (name, repr(attr)))\r
- args.append("rest=%s" % repr(self._rest))\r
- args.append("rfc2109=%s" % repr(self.rfc2109))\r
- return "Cookie(%s)" % ", ".join(args)\r
-\r
-\r
-class CookiePolicy:\r
- """Defines which cookies get accepted from and returned to server.\r
-\r
- May also modify cookies, though this is probably a bad idea.\r
-\r
- The subclass DefaultCookiePolicy defines the standard rules for Netscape\r
- and RFC 2965 cookies -- override that if you want a customised policy.\r
-\r
- """\r
- def set_ok(self, cookie, request):\r
- """Return true if (and only if) cookie should be accepted from server.\r
-\r
- Currently, pre-expired cookies never get this far -- the CookieJar\r
- class deletes such cookies itself.\r
-\r
- """\r
- raise NotImplementedError()\r
-\r
- def return_ok(self, cookie, request):\r
- """Return true if (and only if) cookie should be returned to server."""\r
- raise NotImplementedError()\r
-\r
- def domain_return_ok(self, domain, request):\r
- """Return false if cookies should not be returned, given cookie domain.\r
- """\r
- return True\r
-\r
- def path_return_ok(self, path, request):\r
- """Return false if cookies should not be returned, given cookie path.\r
- """\r
- return True\r
-\r
-\r
-class DefaultCookiePolicy(CookiePolicy):\r
- """Implements the standard rules for accepting and returning cookies."""\r
-\r
- DomainStrictNoDots = 1\r
- DomainStrictNonDomain = 2\r
- DomainRFC2965Match = 4\r
-\r
- DomainLiberal = 0\r
- DomainStrict = DomainStrictNoDots|DomainStrictNonDomain\r
-\r
- def __init__(self,\r
- blocked_domains=None, allowed_domains=None,\r
- netscape=True, rfc2965=False,\r
- rfc2109_as_netscape=None,\r
- hide_cookie2=False,\r
- strict_domain=False,\r
- strict_rfc2965_unverifiable=True,\r
- strict_ns_unverifiable=False,\r
- strict_ns_domain=DomainLiberal,\r
- strict_ns_set_initial_dollar=False,\r
- strict_ns_set_path=False,\r
- ):\r
- """Constructor arguments should be passed as keyword arguments only."""\r
- self.netscape = netscape\r
- self.rfc2965 = rfc2965\r
- self.rfc2109_as_netscape = rfc2109_as_netscape\r
- self.hide_cookie2 = hide_cookie2\r
- self.strict_domain = strict_domain\r
- self.strict_rfc2965_unverifiable = strict_rfc2965_unverifiable\r
- self.strict_ns_unverifiable = strict_ns_unverifiable\r
- self.strict_ns_domain = strict_ns_domain\r
- self.strict_ns_set_initial_dollar = strict_ns_set_initial_dollar\r
- self.strict_ns_set_path = strict_ns_set_path\r
-\r
- if blocked_domains is not None:\r
- self._blocked_domains = tuple(blocked_domains)\r
- else:\r
- self._blocked_domains = ()\r
-\r
- if allowed_domains is not None:\r
- allowed_domains = tuple(allowed_domains)\r
- self._allowed_domains = allowed_domains\r
-\r
- def blocked_domains(self):\r
- """Return the sequence of blocked domains (as a tuple)."""\r
- return self._blocked_domains\r
- def set_blocked_domains(self, blocked_domains):\r
- """Set the sequence of blocked domains."""\r
- self._blocked_domains = tuple(blocked_domains)\r
-\r
- def is_blocked(self, domain):\r
- for blocked_domain in self._blocked_domains:\r
- if user_domain_match(domain, blocked_domain):\r
- return True\r
- return False\r
-\r
- def allowed_domains(self):\r
- """Return None, or the sequence of allowed domains (as a tuple)."""\r
- return self._allowed_domains\r
- def set_allowed_domains(self, allowed_domains):\r
- """Set the sequence of allowed domains, or None."""\r
- if allowed_domains is not None:\r
- allowed_domains = tuple(allowed_domains)\r
- self._allowed_domains = allowed_domains\r
-\r
- def is_not_allowed(self, domain):\r
- if self._allowed_domains is None:\r
- return False\r
- for allowed_domain in self._allowed_domains:\r
- if user_domain_match(domain, allowed_domain):\r
- return False\r
- return True\r
-\r
- def set_ok(self, cookie, request):\r
- """\r
- If you override .set_ok(), be sure to call this method. If it returns\r
- false, so should your subclass (assuming your subclass wants to be more\r
- strict about which cookies to accept).\r
-\r
- """\r
- _debug(" - checking cookie %s=%s", cookie.name, cookie.value)\r
-\r
- assert cookie.name is not None\r
-\r
- for n in "version", "verifiability", "name", "path", "domain", "port":\r
- fn_name = "set_ok_"+n\r
- fn = getattr(self, fn_name)\r
- if not fn(cookie, request):\r
- return False\r
-\r
- return True\r
-\r
- def set_ok_version(self, cookie, request):\r
- if cookie.version is None:\r
- # Version is always set to 0 by parse_ns_headers if it's a Netscape\r
- # cookie, so this must be an invalid RFC 2965 cookie.\r
- _debug(" Set-Cookie2 without version attribute (%s=%s)",\r
- cookie.name, cookie.value)\r
- return False\r
- if cookie.version > 0 and not self.rfc2965:\r
- _debug(" RFC 2965 cookies are switched off")\r
- return False\r
- elif cookie.version == 0 and not self.netscape:\r
- _debug(" Netscape cookies are switched off")\r
- return False\r
- return True\r
-\r
- def set_ok_verifiability(self, cookie, request):\r
- if request.is_unverifiable() and is_third_party(request):\r
- if cookie.version > 0 and self.strict_rfc2965_unverifiable:\r
- _debug(" third-party RFC 2965 cookie during "\r
- "unverifiable transaction")\r
- return False\r
- elif cookie.version == 0 and self.strict_ns_unverifiable:\r
- _debug(" third-party Netscape cookie during "\r
- "unverifiable transaction")\r
- return False\r
- return True\r
-\r
- def set_ok_name(self, cookie, request):\r
- # Try and stop servers setting V0 cookies designed to hack other\r
- # servers that know both V0 and V1 protocols.\r
- if (cookie.version == 0 and self.strict_ns_set_initial_dollar and\r
- cookie.name.startswith("$")):\r
- _debug(" illegal name (starts with '$'): '%s'", cookie.name)\r
- return False\r
- return True\r
-\r
- def set_ok_path(self, cookie, request):\r
- if cookie.path_specified:\r
- req_path = request_path(request)\r
- if ((cookie.version > 0 or\r
- (cookie.version == 0 and self.strict_ns_set_path)) and\r
- not req_path.startswith(cookie.path)):\r
- _debug(" path attribute %s is not a prefix of request "\r
- "path %s", cookie.path, req_path)\r
- return False\r
- return True\r
-\r
- def set_ok_domain(self, cookie, request):\r
- if self.is_blocked(cookie.domain):\r
- _debug(" domain %s is in user block-list", cookie.domain)\r
- return False\r
- if self.is_not_allowed(cookie.domain):\r
- _debug(" domain %s is not in user allow-list", cookie.domain)\r
- return False\r
- if cookie.domain_specified:\r
- req_host, erhn = eff_request_host(request)\r
- domain = cookie.domain\r
- if self.strict_domain and (domain.count(".") >= 2):\r
- # XXX This should probably be compared with the Konqueror\r
- # (kcookiejar.cpp) and Mozilla implementations, but it's a\r
- # losing battle.\r
- i = domain.rfind(".")\r
- j = domain.rfind(".", 0, i)\r
- if j == 0: # domain like .foo.bar\r
- tld = domain[i+1:]\r
- sld = domain[j+1:i]\r
- if sld.lower() in ("co", "ac", "com", "edu", "org", "net",\r
- "gov", "mil", "int", "aero", "biz", "cat", "coop",\r
- "info", "jobs", "mobi", "museum", "name", "pro",\r
- "travel", "eu") and len(tld) == 2:\r
- # domain like .co.uk\r
- _debug(" country-code second level domain %s", domain)\r
- return False\r
- if domain.startswith("."):\r
- undotted_domain = domain[1:]\r
- else:\r
- undotted_domain = domain\r
- embedded_dots = (undotted_domain.find(".") >= 0)\r
- if not embedded_dots and domain != ".local":\r
- _debug(" non-local domain %s contains no embedded dot",\r
- domain)\r
- return False\r
- if cookie.version == 0:\r
- if (not erhn.endswith(domain) and\r
- (not erhn.startswith(".") and\r
- not ("."+erhn).endswith(domain))):\r
- _debug(" effective request-host %s (even with added "\r
- "initial dot) does not end end with %s",\r
- erhn, domain)\r
- return False\r
- if (cookie.version > 0 or\r
- (self.strict_ns_domain & self.DomainRFC2965Match)):\r
- if not domain_match(erhn, domain):\r
- _debug(" effective request-host %s does not domain-match "\r
- "%s", erhn, domain)\r
- return False\r
- if (cookie.version > 0 or\r
- (self.strict_ns_domain & self.DomainStrictNoDots)):\r
- host_prefix = req_host[:-len(domain)]\r
- if (host_prefix.find(".") >= 0 and\r
- not IPV4_RE.search(req_host)):\r
- _debug(" host prefix %s for domain %s contains a dot",\r
- host_prefix, domain)\r
- return False\r
- return True\r
-\r
- def set_ok_port(self, cookie, request):\r
- if cookie.port_specified:\r
- req_port = request_port(request)\r
- if req_port is None:\r
- req_port = "80"\r
- else:\r
- req_port = str(req_port)\r
- for p in cookie.port.split(","):\r
- try:\r
- int(p)\r
- except ValueError:\r
- _debug(" bad port %s (not numeric)", p)\r
- return False\r
- if p == req_port:\r
- break\r
- else:\r
- _debug(" request port (%s) not found in %s",\r
- req_port, cookie.port)\r
- return False\r
- return True\r
-\r
- def return_ok(self, cookie, request):\r
- """\r
- If you override .return_ok(), be sure to call this method. If it\r
- returns false, so should your subclass (assuming your subclass wants to\r
- be more strict about which cookies to return).\r
-\r
- """\r
- # Path has already been checked by .path_return_ok(), and domain\r
- # blocking done by .domain_return_ok().\r
- _debug(" - checking cookie %s=%s", cookie.name, cookie.value)\r
-\r
- for n in "version", "verifiability", "secure", "expires", "port", "domain":\r
- fn_name = "return_ok_"+n\r
- fn = getattr(self, fn_name)\r
- if not fn(cookie, request):\r
- return False\r
- return True\r
-\r
- def return_ok_version(self, cookie, request):\r
- if cookie.version > 0 and not self.rfc2965:\r
- _debug(" RFC 2965 cookies are switched off")\r
- return False\r
- elif cookie.version == 0 and not self.netscape:\r
- _debug(" Netscape cookies are switched off")\r
- return False\r
- return True\r
-\r
- def return_ok_verifiability(self, cookie, request):\r
- if request.is_unverifiable() and is_third_party(request):\r
- if cookie.version > 0 and self.strict_rfc2965_unverifiable:\r
- _debug(" third-party RFC 2965 cookie during unverifiable "\r
- "transaction")\r
- return False\r
- elif cookie.version == 0 and self.strict_ns_unverifiable:\r
- _debug(" third-party Netscape cookie during unverifiable "\r
- "transaction")\r
- return False\r
- return True\r
-\r
- def return_ok_secure(self, cookie, request):\r
- if cookie.secure and request.get_type() != "https":\r
- _debug(" secure cookie with non-secure request")\r
- return False\r
- return True\r
-\r
- def return_ok_expires(self, cookie, request):\r
- if cookie.is_expired(self._now):\r
- _debug(" cookie expired")\r
- return False\r
- return True\r
-\r
- def return_ok_port(self, cookie, request):\r
- if cookie.port:\r
- req_port = request_port(request)\r
- if req_port is None:\r
- req_port = "80"\r
- for p in cookie.port.split(","):\r
- if p == req_port:\r
- break\r
- else:\r
- _debug(" request port %s does not match cookie port %s",\r
- req_port, cookie.port)\r
- return False\r
- return True\r
-\r
- def return_ok_domain(self, cookie, request):\r
- req_host, erhn = eff_request_host(request)\r
- domain = cookie.domain\r
-\r
- # strict check of non-domain cookies: Mozilla does this, MSIE5 doesn't\r
- if (cookie.version == 0 and\r
- (self.strict_ns_domain & self.DomainStrictNonDomain) and\r
- not cookie.domain_specified and domain != erhn):\r
- _debug(" cookie with unspecified domain does not string-compare "\r
- "equal to request domain")\r
- return False\r
-\r
- if cookie.version > 0 and not domain_match(erhn, domain):\r
- _debug(" effective request-host name %s does not domain-match "\r
- "RFC 2965 cookie domain %s", erhn, domain)\r
- return False\r
- if cookie.version == 0 and not ("."+erhn).endswith(domain):\r
- _debug(" request-host %s does not match Netscape cookie domain "\r
- "%s", req_host, domain)\r
- return False\r
- return True\r
-\r
- def domain_return_ok(self, domain, request):\r
- # Liberal check of. This is here as an optimization to avoid\r
- # having to load lots of MSIE cookie files unless necessary.\r
- req_host, erhn = eff_request_host(request)\r
- if not req_host.startswith("."):\r
- req_host = "."+req_host\r
- if not erhn.startswith("."):\r
- erhn = "."+erhn\r
- if not (req_host.endswith(domain) or erhn.endswith(domain)):\r
- #_debug(" request domain %s does not match cookie domain %s",\r
- # req_host, domain)\r
- return False\r
-\r
- if self.is_blocked(domain):\r
- _debug(" domain %s is in user block-list", domain)\r
- return False\r
- if self.is_not_allowed(domain):\r
- _debug(" domain %s is not in user allow-list", domain)\r
- return False\r
-\r
- return True\r
-\r
- def path_return_ok(self, path, request):\r
- _debug("- checking cookie path=%s", path)\r
- req_path = request_path(request)\r
- if not req_path.startswith(path):\r
- _debug(" %s does not path-match %s", req_path, path)\r
- return False\r
- return True\r
-\r
-\r
-def vals_sorted_by_key(adict):\r
- keys = adict.keys()\r
- keys.sort()\r
- return map(adict.get, keys)\r
-\r
-def deepvalues(mapping):\r
- """Iterates over nested mapping, depth-first, in sorted order by key."""\r
- values = vals_sorted_by_key(mapping)\r
- for obj in values:\r
- mapping = False\r
- try:\r
- obj.items\r
- except AttributeError:\r
- pass\r
- else:\r
- mapping = True\r
- for subobj in deepvalues(obj):\r
- yield subobj\r
- if not mapping:\r
- yield obj\r
-\r
-\r
-# Used as second parameter to dict.get() method, to distinguish absent\r
-# dict key from one with a None value.\r
-class Absent: pass\r
-\r
-class CookieJar:\r
- """Collection of HTTP cookies.\r
-\r
- You may not need to know about this class: try\r
- urllib2.build_opener(HTTPCookieProcessor).open(url).\r
-\r
- """\r
-\r
- non_word_re = re.compile(r"\W")\r
- quote_re = re.compile(r"([\"\\])")\r
- strict_domain_re = re.compile(r"\.?[^.]*")\r
- domain_re = re.compile(r"[^.]*")\r
- dots_re = re.compile(r"^\.+")\r
-\r
- magic_re = r"^\#LWP-Cookies-(\d+\.\d+)"\r
-\r
- def __init__(self, policy=None):\r
- if policy is None:\r
- policy = DefaultCookiePolicy()\r
- self._policy = policy\r
-\r
- self._cookies_lock = _threading.RLock()\r
- self._cookies = {}\r
-\r
- def set_policy(self, policy):\r
- self._policy = policy\r
-\r
- def _cookies_for_domain(self, domain, request):\r
- cookies = []\r
- if not self._policy.domain_return_ok(domain, request):\r
- return []\r
- _debug("Checking %s for cookies to return", domain)\r
- cookies_by_path = self._cookies[domain]\r
- for path in cookies_by_path.keys():\r
- if not self._policy.path_return_ok(path, request):\r
- continue\r
- cookies_by_name = cookies_by_path[path]\r
- for cookie in cookies_by_name.values():\r
- if not self._policy.return_ok(cookie, request):\r
- _debug(" not returning cookie")\r
- continue\r
- _debug(" it's a match")\r
- cookies.append(cookie)\r
- return cookies\r
-\r
- def _cookies_for_request(self, request):\r
- """Return a list of cookies to be returned to server."""\r
- cookies = []\r
- for domain in self._cookies.keys():\r
- cookies.extend(self._cookies_for_domain(domain, request))\r
- return cookies\r
-\r
- def _cookie_attrs(self, cookies):\r
- """Return a list of cookie-attributes to be returned to server.\r
-\r
- like ['foo="bar"; $Path="/"', ...]\r
-\r
- The $Version attribute is also added when appropriate (currently only\r
- once per request).\r
-\r
- """\r
- # add cookies in order of most specific (ie. longest) path first\r
- cookies.sort(key=lambda arg: len(arg.path), reverse=True)\r
-\r
- version_set = False\r
-\r
- attrs = []\r
- for cookie in cookies:\r
- # set version of Cookie header\r
- # XXX\r
- # What should it be if multiple matching Set-Cookie headers have\r
- # different versions themselves?\r
- # Answer: there is no answer; was supposed to be settled by\r
- # RFC 2965 errata, but that may never appear...\r
- version = cookie.version\r
- if not version_set:\r
- version_set = True\r
- if version > 0:\r
- attrs.append("$Version=%s" % version)\r
-\r
- # quote cookie value if necessary\r
- # (not for Netscape protocol, which already has any quotes\r
- # intact, due to the poorly-specified Netscape Cookie: syntax)\r
- if ((cookie.value is not None) and\r
- self.non_word_re.search(cookie.value) and version > 0):\r
- value = self.quote_re.sub(r"\\\1", cookie.value)\r
- else:\r
- value = cookie.value\r
-\r
- # add cookie-attributes to be returned in Cookie header\r
- if cookie.value is None:\r
- attrs.append(cookie.name)\r
- else:\r
- attrs.append("%s=%s" % (cookie.name, value))\r
- if version > 0:\r
- if cookie.path_specified:\r
- attrs.append('$Path="%s"' % cookie.path)\r
- if cookie.domain.startswith("."):\r
- domain = cookie.domain\r
- if (not cookie.domain_initial_dot and\r
- domain.startswith(".")):\r
- domain = domain[1:]\r
- attrs.append('$Domain="%s"' % domain)\r
- if cookie.port is not None:\r
- p = "$Port"\r
- if cookie.port_specified:\r
- p = p + ('="%s"' % cookie.port)\r
- attrs.append(p)\r
-\r
- return attrs\r
-\r
- def add_cookie_header(self, request):\r
- """Add correct Cookie: header to request (urllib2.Request object).\r
-\r
- The Cookie2 header is also added unless policy.hide_cookie2 is true.\r
-\r
- """\r
- _debug("add_cookie_header")\r
- self._cookies_lock.acquire()\r
- try:\r
-\r
- self._policy._now = self._now = int(time.time())\r
-\r
- cookies = self._cookies_for_request(request)\r
-\r
- attrs = self._cookie_attrs(cookies)\r
- if attrs:\r
- if not request.has_header("Cookie"):\r
- request.add_unredirected_header(\r
- "Cookie", "; ".join(attrs))\r
-\r
- # if necessary, advertise that we know RFC 2965\r
- if (self._policy.rfc2965 and not self._policy.hide_cookie2 and\r
- not request.has_header("Cookie2")):\r
- for cookie in cookies:\r
- if cookie.version != 1:\r
- request.add_unredirected_header("Cookie2", '$Version="1"')\r
- break\r
-\r
- finally:\r
- self._cookies_lock.release()\r
-\r
- self.clear_expired_cookies()\r
-\r
- def _normalized_cookie_tuples(self, attrs_set):\r
- """Return list of tuples containing normalised cookie information.\r
-\r
- attrs_set is the list of lists of key,value pairs extracted from\r
- the Set-Cookie or Set-Cookie2 headers.\r
-\r
- Tuples are name, value, standard, rest, where name and value are the\r
- cookie name and value, standard is a dictionary containing the standard\r
- cookie-attributes (discard, secure, version, expires or max-age,\r
- domain, path and port) and rest is a dictionary containing the rest of\r
- the cookie-attributes.\r
-\r
- """\r
- cookie_tuples = []\r
-\r
- boolean_attrs = "discard", "secure"\r
- value_attrs = ("version",\r
- "expires", "max-age",\r
- "domain", "path", "port",\r
- "comment", "commenturl")\r
-\r
- for cookie_attrs in attrs_set:\r
- name, value = cookie_attrs[0]\r
-\r
- # Build dictionary of standard cookie-attributes (standard) and\r
- # dictionary of other cookie-attributes (rest).\r
-\r
- # Note: expiry time is normalised to seconds since epoch. V0\r
- # cookies should have the Expires cookie-attribute, and V1 cookies\r
- # should have Max-Age, but since V1 includes RFC 2109 cookies (and\r
- # since V0 cookies may be a mish-mash of Netscape and RFC 2109), we\r
- # accept either (but prefer Max-Age).\r
- max_age_set = False\r
-\r
- bad_cookie = False\r
-\r
- standard = {}\r
- rest = {}\r
- for k, v in cookie_attrs[1:]:\r
- lc = k.lower()\r
- # don't lose case distinction for unknown fields\r
- if lc in value_attrs or lc in boolean_attrs:\r
- k = lc\r
- if k in boolean_attrs and v is None:\r
- # boolean cookie-attribute is present, but has no value\r
- # (like "discard", rather than "port=80")\r
- v = True\r
- if k in standard:\r
- # only first value is significant\r
- continue\r
- if k == "domain":\r
- if v is None:\r
- _debug(" missing value for domain attribute")\r
- bad_cookie = True\r
- break\r
- # RFC 2965 section 3.3.3\r
- v = v.lower()\r
- if k == "expires":\r
- if max_age_set:\r
- # Prefer max-age to expires (like Mozilla)\r
- continue\r
- if v is None:\r
- _debug(" missing or invalid value for expires "\r
- "attribute: treating as session cookie")\r
- continue\r
- if k == "max-age":\r
- max_age_set = True\r
- try:\r
- v = int(v)\r
- except ValueError:\r
- _debug(" missing or invalid (non-numeric) value for "\r
- "max-age attribute")\r
- bad_cookie = True\r
- break\r
- # convert RFC 2965 Max-Age to seconds since epoch\r
- # XXX Strictly you're supposed to follow RFC 2616\r
- # age-calculation rules. Remember that zero Max-Age is a\r
- # is a request to discard (old and new) cookie, though.\r
- k = "expires"\r
- v = self._now + v\r
- if (k in value_attrs) or (k in boolean_attrs):\r
- if (v is None and\r
- k not in ("port", "comment", "commenturl")):\r
- _debug(" missing value for %s attribute" % k)\r
- bad_cookie = True\r
- break\r
- standard[k] = v\r
- else:\r
- rest[k] = v\r
-\r
- if bad_cookie:\r
- continue\r
-\r
- cookie_tuples.append((name, value, standard, rest))\r
-\r
- return cookie_tuples\r
-\r
- def _cookie_from_cookie_tuple(self, tup, request):\r
- # standard is dict of standard cookie-attributes, rest is dict of the\r
- # rest of them\r
- name, value, standard, rest = tup\r
-\r
- domain = standard.get("domain", Absent)\r
- path = standard.get("path", Absent)\r
- port = standard.get("port", Absent)\r
- expires = standard.get("expires", Absent)\r
-\r
- # set the easy defaults\r
- version = standard.get("version", None)\r
- if version is not None:\r
- try:\r
- version = int(version)\r
- except ValueError:\r
- return None # invalid version, ignore cookie\r
- secure = standard.get("secure", False)\r
- # (discard is also set if expires is Absent)\r
- discard = standard.get("discard", False)\r
- comment = standard.get("comment", None)\r
- comment_url = standard.get("commenturl", None)\r
-\r
- # set default path\r
- if path is not Absent and path != "":\r
- path_specified = True\r
- path = escape_path(path)\r
- else:\r
- path_specified = False\r
- path = request_path(request)\r
- i = path.rfind("/")\r
- if i != -1:\r
- if version == 0:\r
- # Netscape spec parts company from reality here\r
- path = path[:i]\r
- else:\r
- path = path[:i+1]\r
- if len(path) == 0: path = "/"\r
-\r
- # set default domain\r
- domain_specified = domain is not Absent\r
- # but first we have to remember whether it starts with a dot\r
- domain_initial_dot = False\r
- if domain_specified:\r
- domain_initial_dot = bool(domain.startswith("."))\r
- if domain is Absent:\r
- req_host, erhn = eff_request_host(request)\r
- domain = erhn\r
- elif not domain.startswith("."):\r
- domain = "."+domain\r
-\r
- # set default port\r
- port_specified = False\r
- if port is not Absent:\r
- if port is None:\r
- # Port attr present, but has no value: default to request port.\r
- # Cookie should then only be sent back on that port.\r
- port = request_port(request)\r
- else:\r
- port_specified = True\r
- port = re.sub(r"\s+", "", port)\r
- else:\r
- # No port attr present. Cookie can be sent back on any port.\r
- port = None\r
-\r
- # set default expires and discard\r
- if expires is Absent:\r
- expires = None\r
- discard = True\r
- elif expires <= self._now:\r
- # Expiry date in past is request to delete cookie. This can't be\r
- # in DefaultCookiePolicy, because can't delete cookies there.\r
- try:\r
- self.clear(domain, path, name)\r
- except KeyError:\r
- pass\r
- _debug("Expiring cookie, domain='%s', path='%s', name='%s'",\r
- domain, path, name)\r
- return None\r
-\r
- return Cookie(version,\r
- name, value,\r
- port, port_specified,\r
- domain, domain_specified, domain_initial_dot,\r
- path, path_specified,\r
- secure,\r
- expires,\r
- discard,\r
- comment,\r
- comment_url,\r
- rest)\r
-\r
- def _cookies_from_attrs_set(self, attrs_set, request):\r
- cookie_tuples = self._normalized_cookie_tuples(attrs_set)\r
-\r
- cookies = []\r
- for tup in cookie_tuples:\r
- cookie = self._cookie_from_cookie_tuple(tup, request)\r
- if cookie: cookies.append(cookie)\r
- return cookies\r
-\r
- def _process_rfc2109_cookies(self, cookies):\r
- rfc2109_as_ns = getattr(self._policy, 'rfc2109_as_netscape', None)\r
- if rfc2109_as_ns is None:\r
- rfc2109_as_ns = not self._policy.rfc2965\r
- for cookie in cookies:\r
- if cookie.version == 1:\r
- cookie.rfc2109 = True\r
- if rfc2109_as_ns:\r
- # treat 2109 cookies as Netscape cookies rather than\r
- # as RFC2965 cookies\r
- cookie.version = 0\r
-\r
- def make_cookies(self, response, request):\r
- """Return sequence of Cookie objects extracted from response object."""\r
- # get cookie-attributes for RFC 2965 and Netscape protocols\r
- headers = response.info()\r
- rfc2965_hdrs = headers.getheaders("Set-Cookie2")\r
- ns_hdrs = headers.getheaders("Set-Cookie")\r
-\r
- rfc2965 = self._policy.rfc2965\r
- netscape = self._policy.netscape\r
-\r
- if ((not rfc2965_hdrs and not ns_hdrs) or\r
- (not ns_hdrs and not rfc2965) or\r
- (not rfc2965_hdrs and not netscape) or\r
- (not netscape and not rfc2965)):\r
- return [] # no relevant cookie headers: quick exit\r
-\r
- try:\r
- cookies = self._cookies_from_attrs_set(\r
- split_header_words(rfc2965_hdrs), request)\r
- except Exception:\r
- _warn_unhandled_exception()\r
- cookies = []\r
-\r
- if ns_hdrs and netscape:\r
- try:\r
- # RFC 2109 and Netscape cookies\r
- ns_cookies = self._cookies_from_attrs_set(\r
- parse_ns_headers(ns_hdrs), request)\r
- except Exception:\r
- _warn_unhandled_exception()\r
- ns_cookies = []\r
- self._process_rfc2109_cookies(ns_cookies)\r
-\r
- # Look for Netscape cookies (from Set-Cookie headers) that match\r
- # corresponding RFC 2965 cookies (from Set-Cookie2 headers).\r
- # For each match, keep the RFC 2965 cookie and ignore the Netscape\r
- # cookie (RFC 2965 section 9.1). Actually, RFC 2109 cookies are\r
- # bundled in with the Netscape cookies for this purpose, which is\r
- # reasonable behaviour.\r
- if rfc2965:\r
- lookup = {}\r
- for cookie in cookies:\r
- lookup[(cookie.domain, cookie.path, cookie.name)] = None\r
-\r
- def no_matching_rfc2965(ns_cookie, lookup=lookup):\r
- key = ns_cookie.domain, ns_cookie.path, ns_cookie.name\r
- return key not in lookup\r
- ns_cookies = filter(no_matching_rfc2965, ns_cookies)\r
-\r
- if ns_cookies:\r
- cookies.extend(ns_cookies)\r
-\r
- return cookies\r
-\r
- def set_cookie_if_ok(self, cookie, request):\r
- """Set a cookie if policy says it's OK to do so."""\r
- self._cookies_lock.acquire()\r
- try:\r
- self._policy._now = self._now = int(time.time())\r
-\r
- if self._policy.set_ok(cookie, request):\r
- self.set_cookie(cookie)\r
-\r
-\r
- finally:\r
- self._cookies_lock.release()\r
-\r
- def set_cookie(self, cookie):\r
- """Set a cookie, without checking whether or not it should be set."""\r
- c = self._cookies\r
- self._cookies_lock.acquire()\r
- try:\r
- if cookie.domain not in c: c[cookie.domain] = {}\r
- c2 = c[cookie.domain]\r
- if cookie.path not in c2: c2[cookie.path] = {}\r
- c3 = c2[cookie.path]\r
- c3[cookie.name] = cookie\r
- finally:\r
- self._cookies_lock.release()\r
-\r
- def extract_cookies(self, response, request):\r
- """Extract cookies from response, where allowable given the request."""\r
- _debug("extract_cookies: %s", response.info())\r
- self._cookies_lock.acquire()\r
- try:\r
- self._policy._now = self._now = int(time.time())\r
-\r
- for cookie in self.make_cookies(response, request):\r
- if self._policy.set_ok(cookie, request):\r
- _debug(" setting cookie: %s", cookie)\r
- self.set_cookie(cookie)\r
- finally:\r
- self._cookies_lock.release()\r
-\r
- def clear(self, domain=None, path=None, name=None):\r
- """Clear some cookies.\r
-\r
- Invoking this method without arguments will clear all cookies. If\r
- given a single argument, only cookies belonging to that domain will be\r
- removed. If given two arguments, cookies belonging to the specified\r
- path within that domain are removed. If given three arguments, then\r
- the cookie with the specified name, path and domain is removed.\r
-\r
- Raises KeyError if no matching cookie exists.\r
-\r
- """\r
- if name is not None:\r
- if (domain is None) or (path is None):\r
- raise ValueError(\r
- "domain and path must be given to remove a cookie by name")\r
- del self._cookies[domain][path][name]\r
- elif path is not None:\r
- if domain is None:\r
- raise ValueError(\r
- "domain must be given to remove cookies by path")\r
- del self._cookies[domain][path]\r
- elif domain is not None:\r
- del self._cookies[domain]\r
- else:\r
- self._cookies = {}\r
-\r
- def clear_session_cookies(self):\r
- """Discard all session cookies.\r
-\r
- Note that the .save() method won't save session cookies anyway, unless\r
- you ask otherwise by passing a true ignore_discard argument.\r
-\r
- """\r
- self._cookies_lock.acquire()\r
- try:\r
- for cookie in self:\r
- if cookie.discard:\r
- self.clear(cookie.domain, cookie.path, cookie.name)\r
- finally:\r
- self._cookies_lock.release()\r
-\r
- def clear_expired_cookies(self):\r
- """Discard all expired cookies.\r
-\r
- You probably don't need to call this method: expired cookies are never\r
- sent back to the server (provided you're using DefaultCookiePolicy),\r
- this method is called by CookieJar itself every so often, and the\r
- .save() method won't save expired cookies anyway (unless you ask\r
- otherwise by passing a true ignore_expires argument).\r
-\r
- """\r
- self._cookies_lock.acquire()\r
- try:\r
- now = time.time()\r
- for cookie in self:\r
- if cookie.is_expired(now):\r
- self.clear(cookie.domain, cookie.path, cookie.name)\r
- finally:\r
- self._cookies_lock.release()\r
-\r
- def __iter__(self):\r
- return deepvalues(self._cookies)\r
-\r
- def __len__(self):\r
- """Return number of contained cookies."""\r
- i = 0\r
- for cookie in self: i = i + 1\r
- return i\r
-\r
- def __repr__(self):\r
- r = []\r
- for cookie in self: r.append(repr(cookie))\r
- return "<%s[%s]>" % (self.__class__, ", ".join(r))\r
-\r
- def __str__(self):\r
- r = []\r
- for cookie in self: r.append(str(cookie))\r
- return "<%s[%s]>" % (self.__class__, ", ".join(r))\r
-\r
-\r
-# derives from IOError for backwards-compatibility with Python 2.4.0\r
-class LoadError(IOError): pass\r
-\r
-class FileCookieJar(CookieJar):\r
- """CookieJar that can be loaded from and saved to a file."""\r
-\r
- def __init__(self, filename=None, delayload=False, policy=None):\r
- """\r
- Cookies are NOT loaded from the named file until either the .load() or\r
- .revert() method is called.\r
-\r
- """\r
- CookieJar.__init__(self, policy)\r
- if filename is not None:\r
- try:\r
- filename+""\r
- except:\r
- raise ValueError("filename must be string-like")\r
- self.filename = filename\r
- self.delayload = bool(delayload)\r
-\r
- def save(self, filename=None, ignore_discard=False, ignore_expires=False):\r
- """Save cookies to a file."""\r
- raise NotImplementedError()\r
-\r
- def load(self, filename=None, ignore_discard=False, ignore_expires=False):\r
- """Load cookies from a file."""\r
- if filename is None:\r
- if self.filename is not None: filename = self.filename\r
- else: raise ValueError(MISSING_FILENAME_TEXT)\r
-\r
- f = open(filename)\r
- try:\r
- self._really_load(f, filename, ignore_discard, ignore_expires)\r
- finally:\r
- f.close()\r
-\r
- def revert(self, filename=None,\r
- ignore_discard=False, ignore_expires=False):\r
- """Clear all cookies and reload cookies from a saved file.\r
-\r
- Raises LoadError (or IOError) if reversion is not successful; the\r
- object's state will not be altered if this happens.\r
-\r
- """\r
- if filename is None:\r
- if self.filename is not None: filename = self.filename\r
- else: raise ValueError(MISSING_FILENAME_TEXT)\r
-\r
- self._cookies_lock.acquire()\r
- try:\r
-\r
- old_state = copy.deepcopy(self._cookies)\r
- self._cookies = {}\r
- try:\r
- self.load(filename, ignore_discard, ignore_expires)\r
- except (LoadError, IOError):\r
- self._cookies = old_state\r
- raise\r
-\r
- finally:\r
- self._cookies_lock.release()\r
-\r
-from _LWPCookieJar import LWPCookieJar, lwp_cookie_str\r
-from _MozillaCookieJar import MozillaCookieJar\r
projects / mirror_edk2.git / blobdiff