AppPkg/Applications/Python/Python-2.7.2/Lib/email/charset.py

   1 # Copyright (C) 2001-2006 Python Software Foundation
   2 # Author: Ben Gertzfield, Barry Warsaw
   3 # Contact: email-sig@python.org
   4
   5 __all__ = [
   6     'Charset',
   7     'add_alias',
   8     'add_charset',
   9     'add_codec',
  10     ]
  11
  12 import codecs
  13 import email.base64mime
  14 import email.quoprimime
  15
  16 from email import errors
  17 from email.encoders import encode_7or8bit
  18
  19
  20 \f
  21 # Flags for types of header encodings
  22 QP          = 1 # Quoted-Printable
  23 BASE64      = 2 # Base64
  24 SHORTEST    = 3 # the shorter of QP and base64, but only for headers
  25
  26 # In "=?charset?q?hello_world?=", the =?, ?q?, and ?= add up to 7
  27 MISC_LEN = 7
  28
  29 DEFAULT_CHARSET = 'us-ascii'
  30
  31
  32 \f
  33 # Defaults
  34 CHARSETS = {
  35     # input        header enc  body enc output conv
  36     'iso-8859-1':  (QP,        QP,      None),
  37     'iso-8859-2':  (QP,        QP,      None),
  38     'iso-8859-3':  (QP,        QP,      None),
  39     'iso-8859-4':  (QP,        QP,      None),
  40     # iso-8859-5 is Cyrillic, and not especially used
  41     # iso-8859-6 is Arabic, also not particularly used
  42     # iso-8859-7 is Greek, QP will not make it readable
  43     # iso-8859-8 is Hebrew, QP will not make it readable
  44     'iso-8859-9':  (QP,        QP,      None),
  45     'iso-8859-10': (QP,        QP,      None),
  46     # iso-8859-11 is Thai, QP will not make it readable
  47     'iso-8859-13': (QP,        QP,      None),
  48     'iso-8859-14': (QP,        QP,      None),
  49     'iso-8859-15': (QP,        QP,      None),
  50     'iso-8859-16': (QP,        QP,      None),
  51     'windows-1252':(QP,        QP,      None),
  52     'viscii':      (QP,        QP,      None),
  53     'us-ascii':    (None,      None,    None),
  54     'big5':        (BASE64,    BASE64,  None),
  55     'gb2312':      (BASE64,    BASE64,  None),
  56     'euc-jp':      (BASE64,    None,    'iso-2022-jp'),
  57     'shift_jis':   (BASE64,    None,    'iso-2022-jp'),
  58     'iso-2022-jp': (BASE64,    None,    None),
  59     'koi8-r':      (BASE64,    BASE64,  None),
  60     'utf-8':       (SHORTEST,  BASE64, 'utf-8'),
  61     # We're making this one up to represent raw unencoded 8-bit
  62     '8bit':        (None,      BASE64, 'utf-8'),
  63     }
  64
  65 # Aliases for other commonly-used names for character sets.  Map
  66 # them to the real ones used in email.
  67 ALIASES = {
  68     'latin_1': 'iso-8859-1',
  69     'latin-1': 'iso-8859-1',
  70     'latin_2': 'iso-8859-2',
  71     'latin-2': 'iso-8859-2',
  72     'latin_3': 'iso-8859-3',
  73     'latin-3': 'iso-8859-3',
  74     'latin_4': 'iso-8859-4',
  75     'latin-4': 'iso-8859-4',
  76     'latin_5': 'iso-8859-9',
  77     'latin-5': 'iso-8859-9',
  78     'latin_6': 'iso-8859-10',
  79     'latin-6': 'iso-8859-10',
  80     'latin_7': 'iso-8859-13',
  81     'latin-7': 'iso-8859-13',
  82     'latin_8': 'iso-8859-14',
  83     'latin-8': 'iso-8859-14',
  84     'latin_9': 'iso-8859-15',
  85     'latin-9': 'iso-8859-15',
  86     'latin_10':'iso-8859-16',
  87     'latin-10':'iso-8859-16',
  88     'cp949':   'ks_c_5601-1987',
  89     'euc_jp':  'euc-jp',
  90     'euc_kr':  'euc-kr',
  91     'ascii':   'us-ascii',
  92     }
  93
  94
  95 # Map charsets to their Unicode codec strings.
  96 CODEC_MAP = {
  97     'gb2312':      'eucgb2312_cn',
  98     'big5':        'big5_tw',
  99     # Hack: We don't want *any* conversion for stuff marked us-ascii, as all
 100     # sorts of garbage might be sent to us in the guise of 7-bit us-ascii.
 101     # Let that stuff pass through without conversion to/from Unicode.
 102     'us-ascii':    None,
 103     }
 104
 105
 106 \f
 107 # Convenience functions for extending the above mappings
 108 def add_charset(charset, header_enc=None, body_enc=None, output_charset=None):
 109     """Add character set properties to the global registry.
 110
 111     charset is the input character set, and must be the canonical name of a
 112     character set.
 113
 114     Optional header_enc and body_enc is either Charset.QP for
 115     quoted-printable, Charset.BASE64 for base64 encoding, Charset.SHORTEST for
 116     the shortest of qp or base64 encoding, or None for no encoding.  SHORTEST
 117     is only valid for header_enc.  It describes how message headers and
 118     message bodies in the input charset are to be encoded.  Default is no
 119     encoding.
 120
 121     Optional output_charset is the character set that the output should be
 122     in.  Conversions will proceed from input charset, to Unicode, to the
 123     output charset when the method Charset.convert() is called.  The default
 124     is to output in the same character set as the input.
 125
 126     Both input_charset and output_charset must have Unicode codec entries in
 127     the module's charset-to-codec mapping; use add_codec(charset, codecname)
 128     to add codecs the module does not know about.  See the codecs module's
 129     documentation for more information.
 130     """
 131     if body_enc == SHORTEST:
 132         raise ValueError('SHORTEST not allowed for body_enc')
 133     CHARSETS[charset] = (header_enc, body_enc, output_charset)
 134
 135
 136 def add_alias(alias, canonical):
 137     """Add a character set alias.
 138
 139     alias is the alias name, e.g. latin-1
 140     canonical is the character set's canonical name, e.g. iso-8859-1
 141     """
 142     ALIASES[alias] = canonical
 143
 144
 145 def add_codec(charset, codecname):
 146     """Add a codec that map characters in the given charset to/from Unicode.
 147
 148     charset is the canonical name of a character set.  codecname is the name
 149     of a Python codec, as appropriate for the second argument to the unicode()
 150     built-in, or to the encode() method of a Unicode string.
 151     """
 152     CODEC_MAP[charset] = codecname
 153
 154
 155 \f
 156 class Charset:
 157     """Map character sets to their email properties.
 158
 159     This class provides information about the requirements imposed on email
 160     for a specific character set.  It also provides convenience routines for
 161     converting between character sets, given the availability of the
 162     applicable codecs.  Given a character set, it will do its best to provide
 163     information on how to use that character set in an email in an
 164     RFC-compliant way.
 165
 166     Certain character sets must be encoded with quoted-printable or base64
 167     when used in email headers or bodies.  Certain character sets must be
 168     converted outright, and are not allowed in email.  Instances of this
 169     module expose the following information about a character set:
 170
 171     input_charset: The initial character set specified.  Common aliases
 172                    are converted to their `official' email names (e.g. latin_1
 173                    is converted to iso-8859-1).  Defaults to 7-bit us-ascii.
 174
 175     header_encoding: If the character set must be encoded before it can be
 176                      used in an email header, this attribute will be set to
 177                      Charset.QP (for quoted-printable), Charset.BASE64 (for
 178                      base64 encoding), or Charset.SHORTEST for the shortest of
 179                      QP or BASE64 encoding.  Otherwise, it will be None.
 180
 181     body_encoding: Same as header_encoding, but describes the encoding for the
 182                    mail message's body, which indeed may be different than the
 183                    header encoding.  Charset.SHORTEST is not allowed for
 184                    body_encoding.
 185
 186     output_charset: Some character sets must be converted before the can be
 187                     used in email headers or bodies.  If the input_charset is
 188                     one of them, this attribute will contain the name of the
 189                     charset output will be converted to.  Otherwise, it will
 190                     be None.
 191
 192     input_codec: The name of the Python codec used to convert the
 193                  input_charset to Unicode.  If no conversion codec is
 194                  necessary, this attribute will be None.
 195
 196     output_codec: The name of the Python codec used to convert Unicode
 197                   to the output_charset.  If no conversion codec is necessary,
 198                   this attribute will have the same value as the input_codec.
 199     """
 200     def __init__(self, input_charset=DEFAULT_CHARSET):
 201         # RFC 2046, $4.1.2 says charsets are not case sensitive.  We coerce to
 202         # unicode because its .lower() is locale insensitive.  If the argument
 203         # is already a unicode, we leave it at that, but ensure that the
 204         # charset is ASCII, as the standard (RFC XXX) requires.
 205         try:
 206             if isinstance(input_charset, unicode):
 207                 input_charset.encode('ascii')
 208             else:
 209                 input_charset = unicode(input_charset, 'ascii')
 210         except UnicodeError:
 211             raise errors.CharsetError(input_charset)
 212         input_charset = input_charset.lower().encode('ascii')
 213         # Set the input charset after filtering through the aliases and/or codecs
 214         if not (input_charset in ALIASES or input_charset in CHARSETS):
 215             try:
 216                 input_charset = codecs.lookup(input_charset).name
 217             except LookupError:
 218                 pass
 219         self.input_charset = ALIASES.get(input_charset, input_charset)
 220         # We can try to guess which encoding and conversion to use by the
 221         # charset_map dictionary.  Try that first, but let the user override
 222         # it.
 223         henc, benc, conv = CHARSETS.get(self.input_charset,
 224                                         (SHORTEST, BASE64, None))
 225         if not conv:
 226             conv = self.input_charset
 227         # Set the attributes, allowing the arguments to override the default.
 228         self.header_encoding = henc
 229         self.body_encoding = benc
 230         self.output_charset = ALIASES.get(conv, conv)
 231         # Now set the codecs.  If one isn't defined for input_charset,
 232         # guess and try a Unicode codec with the same name as input_codec.
 233         self.input_codec = CODEC_MAP.get(self.input_charset,
 234                                          self.input_charset)
 235         self.output_codec = CODEC_MAP.get(self.output_charset,
 236                                           self.output_charset)
 237
 238     def __str__(self):
 239         return self.input_charset.lower()
 240
 241     __repr__ = __str__
 242
 243     def __eq__(self, other):
 244         return str(self) == str(other).lower()
 245
 246     def __ne__(self, other):
 247         return not self.__eq__(other)
 248
 249     def get_body_encoding(self):
 250         """Return the content-transfer-encoding used for body encoding.
 251
 252         This is either the string `quoted-printable' or `base64' depending on
 253         the encoding used, or it is a function in which case you should call
 254         the function with a single argument, the Message object being
 255         encoded.  The function should then set the Content-Transfer-Encoding
 256         header itself to whatever is appropriate.
 257
 258         Returns "quoted-printable" if self.body_encoding is QP.
 259         Returns "base64" if self.body_encoding is BASE64.
 260         Returns "7bit" otherwise.
 261         """
 262         assert self.body_encoding != SHORTEST
 263         if self.body_encoding == QP:
 264             return 'quoted-printable'
 265         elif self.body_encoding == BASE64:
 266             return 'base64'
 267         else:
 268             return encode_7or8bit
 269
 270     def convert(self, s):
 271         """Convert a string from the input_codec to the output_codec."""
 272         if self.input_codec != self.output_codec:
 273             return unicode(s, self.input_codec).encode(self.output_codec)
 274         else:
 275             return s
 276
 277     def to_splittable(self, s):
 278         """Convert a possibly multibyte string to a safely splittable format.
 279
 280         Uses the input_codec to try and convert the string to Unicode, so it
 281         can be safely split on character boundaries (even for multibyte
 282         characters).
 283
 284         Returns the string as-is if it isn't known how to convert it to
 285         Unicode with the input_charset.
 286
 287         Characters that could not be converted to Unicode will be replaced
 288         with the Unicode replacement character U+FFFD.
 289         """
 290         if isinstance(s, unicode) or self.input_codec is None:
 291             return s
 292         try:
 293             return unicode(s, self.input_codec, 'replace')
 294         except LookupError:
 295             # Input codec not installed on system, so return the original
 296             # string unchanged.
 297             return s
 298
 299     def from_splittable(self, ustr, to_output=True):
 300         """Convert a splittable string back into an encoded string.
 301
 302         Uses the proper codec to try and convert the string from Unicode back
 303         into an encoded format.  Return the string as-is if it is not Unicode,
 304         or if it could not be converted from Unicode.
 305
 306         Characters that could not be converted from Unicode will be replaced
 307         with an appropriate character (usually '?').
 308
 309         If to_output is True (the default), uses output_codec to convert to an
 310         encoded format.  If to_output is False, uses input_codec.
 311         """
 312         if to_output:
 313             codec = self.output_codec
 314         else:
 315             codec = self.input_codec
 316         if not isinstance(ustr, unicode) or codec is None:
 317             return ustr
 318         try:
 319             return ustr.encode(codec, 'replace')
 320         except LookupError:
 321             # Output codec not installed
 322             return ustr
 323
 324     def get_output_charset(self):
 325         """Return the output character set.
 326
 327         This is self.output_charset if that is not None, otherwise it is
 328         self.input_charset.
 329         """
 330         return self.output_charset or self.input_charset
 331
 332     def encoded_header_len(self, s):
 333         """Return the length of the encoded header string."""
 334         cset = self.get_output_charset()
 335         # The len(s) of a 7bit encoding is len(s)
 336         if self.header_encoding == BASE64:
 337             return email.base64mime.base64_len(s) + len(cset) + MISC_LEN
 338         elif self.header_encoding == QP:
 339             return email.quoprimime.header_quopri_len(s) + len(cset) + MISC_LEN
 340         elif self.header_encoding == SHORTEST:
 341             lenb64 = email.base64mime.base64_len(s)
 342             lenqp = email.quoprimime.header_quopri_len(s)
 343             return min(lenb64, lenqp) + len(cset) + MISC_LEN
 344         else:
 345             return len(s)
 346
 347     def header_encode(self, s, convert=False):
 348         """Header-encode a string, optionally converting it to output_charset.
 349
 350         If convert is True, the string will be converted from the input
 351         charset to the output charset automatically.  This is not useful for
 352         multibyte character sets, which have line length issues (multibyte
 353         characters must be split on a character, not a byte boundary); use the
 354         high-level Header class to deal with these issues.  convert defaults
 355         to False.
 356
 357         The type of encoding (base64 or quoted-printable) will be based on
 358         self.header_encoding.
 359         """
 360         cset = self.get_output_charset()
 361         if convert:
 362             s = self.convert(s)
 363         # 7bit/8bit encodings return the string unchanged (modulo conversions)
 364         if self.header_encoding == BASE64:
 365             return email.base64mime.header_encode(s, cset)
 366         elif self.header_encoding == QP:
 367             return email.quoprimime.header_encode(s, cset, maxlinelen=None)
 368         elif self.header_encoding == SHORTEST:
 369             lenb64 = email.base64mime.base64_len(s)
 370             lenqp = email.quoprimime.header_quopri_len(s)
 371             if lenb64 < lenqp:
 372                 return email.base64mime.header_encode(s, cset)
 373             else:
 374                 return email.quoprimime.header_encode(s, cset, maxlinelen=None)
 375         else:
 376             return s
 377
 378     def body_encode(self, s, convert=True):
 379         """Body-encode a string and convert it to output_charset.
 380
 381         If convert is True (the default), the string will be converted from
 382         the input charset to output charset automatically.  Unlike
 383         header_encode(), there are no issues with byte boundaries and
 384         multibyte charsets in email bodies, so this is usually pretty safe.
 385
 386         The type of encoding (base64 or quoted-printable) will be based on
 387         self.body_encoding.
 388         """
 389         if convert:
 390             s = self.convert(s)
 391         # 7bit/8bit encodings return the string unchanged (module conversions)
 392         if self.body_encoding is BASE64:
 393             return email.base64mime.body_encode(s)
 394         elif self.body_encoding is QP:
 395             return email.quoprimime.body_encode(s)
 396         else:
 397             return s