+++ /dev/null
-#\r
-# (re)generate unicode property and type databases\r
-#\r
-# this script converts a unicode 3.2 database file to\r
-# Modules/unicodedata_db.h, Modules/unicodename_db.h,\r
-# and Objects/unicodetype_db.h\r
-#\r
-# history:\r
-# 2000-09-24 fl created (based on bits and pieces from unidb)\r
-# 2000-09-25 fl merged tim's splitbin fixes, separate decomposition table\r
-# 2000-09-25 fl added character type table\r
-# 2000-09-26 fl added LINEBREAK, DECIMAL, and DIGIT flags/fields (2.0)\r
-# 2000-11-03 fl expand first/last ranges\r
-# 2001-01-19 fl added character name tables (2.1)\r
-# 2001-01-21 fl added decomp compression; dynamic phrasebook threshold\r
-# 2002-09-11 wd use string methods\r
-# 2002-10-18 mvl update to Unicode 3.2\r
-# 2002-10-22 mvl generate NFC tables\r
-# 2002-11-24 mvl expand all ranges, sort names version-independently\r
-# 2002-11-25 mvl add UNIDATA_VERSION\r
-# 2004-05-29 perky add east asian width information\r
-# 2006-03-10 mvl update to Unicode 4.1; add UCD 3.2 delta\r
-#\r
-# written by Fredrik Lundh (fredrik@pythonware.com)\r
-#\r
-\r
-import sys\r
-\r
-SCRIPT = sys.argv[0]\r
-VERSION = "2.6"\r
-\r
-# The Unicode Database\r
-UNIDATA_VERSION = "5.2.0"\r
-UNICODE_DATA = "UnicodeData%s.txt"\r
-COMPOSITION_EXCLUSIONS = "CompositionExclusions%s.txt"\r
-EASTASIAN_WIDTH = "EastAsianWidth%s.txt"\r
-UNIHAN = "Unihan%s.txt"\r
-DERIVEDNORMALIZATION_PROPS = "DerivedNormalizationProps%s.txt"\r
-LINE_BREAK = "LineBreak%s.txt"\r
-\r
-old_versions = ["3.2.0"]\r
-\r
-CATEGORY_NAMES = [ "Cn", "Lu", "Ll", "Lt", "Mn", "Mc", "Me", "Nd",\r
- "Nl", "No", "Zs", "Zl", "Zp", "Cc", "Cf", "Cs", "Co", "Cn", "Lm",\r
- "Lo", "Pc", "Pd", "Ps", "Pe", "Pi", "Pf", "Po", "Sm", "Sc", "Sk",\r
- "So" ]\r
-\r
-BIDIRECTIONAL_NAMES = [ "", "L", "LRE", "LRO", "R", "AL", "RLE", "RLO",\r
- "PDF", "EN", "ES", "ET", "AN", "CS", "NSM", "BN", "B", "S", "WS",\r
- "ON" ]\r
-\r
-EASTASIANWIDTH_NAMES = [ "F", "H", "W", "Na", "A", "N" ]\r
-\r
-MANDATORY_LINE_BREAKS = [ "BK", "CR", "LF", "NL" ]\r
-\r
-# note: should match definitions in Objects/unicodectype.c\r
-ALPHA_MASK = 0x01\r
-DECIMAL_MASK = 0x02\r
-DIGIT_MASK = 0x04\r
-LOWER_MASK = 0x08\r
-LINEBREAK_MASK = 0x10\r
-SPACE_MASK = 0x20\r
-TITLE_MASK = 0x40\r
-UPPER_MASK = 0x80\r
-NODELTA_MASK = 0x100\r
-NUMERIC_MASK = 0x200\r
-\r
-def maketables(trace=0):\r
-\r
- print "--- Reading", UNICODE_DATA % "", "..."\r
-\r
- version = ""\r
- unicode = UnicodeData(UNICODE_DATA % version,\r
- COMPOSITION_EXCLUSIONS % version,\r
- EASTASIAN_WIDTH % version,\r
- UNIHAN % version,\r
- DERIVEDNORMALIZATION_PROPS % version,\r
- LINE_BREAK % version)\r
-\r
- print len(filter(None, unicode.table)), "characters"\r
-\r
- for version in old_versions:\r
- print "--- Reading", UNICODE_DATA % ("-"+version), "..."\r
- old_unicode = UnicodeData(UNICODE_DATA % ("-"+version),\r
- COMPOSITION_EXCLUSIONS % ("-"+version),\r
- EASTASIAN_WIDTH % ("-"+version),\r
- UNIHAN % ("-"+version))\r
- print len(filter(None, old_unicode.table)), "characters"\r
- merge_old_version(version, unicode, old_unicode)\r
-\r
- makeunicodename(unicode, trace)\r
- makeunicodedata(unicode, trace)\r
- makeunicodetype(unicode, trace)\r
-\r
-# --------------------------------------------------------------------\r
-# unicode character properties\r
-\r
-def makeunicodedata(unicode, trace):\r
-\r
- dummy = (0, 0, 0, 0, 0, 0)\r
- table = [dummy]\r
- cache = {0: dummy}\r
- index = [0] * len(unicode.chars)\r
-\r
- FILE = "Modules/unicodedata_db.h"\r
-\r
- print "--- Preparing", FILE, "..."\r
-\r
- # 1) database properties\r
-\r
- for char in unicode.chars:\r
- record = unicode.table[char]\r
- if record:\r
- # extract database properties\r
- category = CATEGORY_NAMES.index(record[2])\r
- combining = int(record[3])\r
- bidirectional = BIDIRECTIONAL_NAMES.index(record[4])\r
- mirrored = record[9] == "Y"\r
- eastasianwidth = EASTASIANWIDTH_NAMES.index(record[15])\r
- normalizationquickcheck = record[17]\r
- item = (\r
- category, combining, bidirectional, mirrored, eastasianwidth,\r
- normalizationquickcheck\r
- )\r
- # add entry to index and item tables\r
- i = cache.get(item)\r
- if i is None:\r
- cache[item] = i = len(table)\r
- table.append(item)\r
- index[char] = i\r
-\r
- # 2) decomposition data\r
-\r
- decomp_data = [0]\r
- decomp_prefix = [""]\r
- decomp_index = [0] * len(unicode.chars)\r
- decomp_size = 0\r
-\r
- comp_pairs = []\r
- comp_first = [None] * len(unicode.chars)\r
- comp_last = [None] * len(unicode.chars)\r
-\r
- for char in unicode.chars:\r
- record = unicode.table[char]\r
- if record:\r
- if record[5]:\r
- decomp = record[5].split()\r
- if len(decomp) > 19:\r
- raise Exception, "character %x has a decomposition too large for nfd_nfkd" % char\r
- # prefix\r
- if decomp[0][0] == "<":\r
- prefix = decomp.pop(0)\r
- else:\r
- prefix = ""\r
- try:\r
- i = decomp_prefix.index(prefix)\r
- except ValueError:\r
- i = len(decomp_prefix)\r
- decomp_prefix.append(prefix)\r
- prefix = i\r
- assert prefix < 256\r
- # content\r
- decomp = [prefix + (len(decomp)<<8)] + [int(s, 16) for s in decomp]\r
- # Collect NFC pairs\r
- if not prefix and len(decomp) == 3 and \\r
- char not in unicode.exclusions and \\r
- unicode.table[decomp[1]][3] == "0":\r
- p, l, r = decomp\r
- comp_first[l] = 1\r
- comp_last[r] = 1\r
- comp_pairs.append((l,r,char))\r
- try:\r
- i = decomp_data.index(decomp)\r
- except ValueError:\r
- i = len(decomp_data)\r
- decomp_data.extend(decomp)\r
- decomp_size = decomp_size + len(decomp) * 2\r
- else:\r
- i = 0\r
- decomp_index[char] = i\r
-\r
- f = l = 0\r
- comp_first_ranges = []\r
- comp_last_ranges = []\r
- prev_f = prev_l = None\r
- for i in unicode.chars:\r
- if comp_first[i] is not None:\r
- comp_first[i] = f\r
- f += 1\r
- if prev_f is None:\r
- prev_f = (i,i)\r
- elif prev_f[1]+1 == i:\r
- prev_f = prev_f[0],i\r
- else:\r
- comp_first_ranges.append(prev_f)\r
- prev_f = (i,i)\r
- if comp_last[i] is not None:\r
- comp_last[i] = l\r
- l += 1\r
- if prev_l is None:\r
- prev_l = (i,i)\r
- elif prev_l[1]+1 == i:\r
- prev_l = prev_l[0],i\r
- else:\r
- comp_last_ranges.append(prev_l)\r
- prev_l = (i,i)\r
- comp_first_ranges.append(prev_f)\r
- comp_last_ranges.append(prev_l)\r
- total_first = f\r
- total_last = l\r
-\r
- comp_data = [0]*(total_first*total_last)\r
- for f,l,char in comp_pairs:\r
- f = comp_first[f]\r
- l = comp_last[l]\r
- comp_data[f*total_last+l] = char\r
-\r
- print len(table), "unique properties"\r
- print len(decomp_prefix), "unique decomposition prefixes"\r
- print len(decomp_data), "unique decomposition entries:",\r
- print decomp_size, "bytes"\r
- print total_first, "first characters in NFC"\r
- print total_last, "last characters in NFC"\r
- print len(comp_pairs), "NFC pairs"\r
-\r
- print "--- Writing", FILE, "..."\r
-\r
- fp = open(FILE, "w")\r
- print >>fp, "/* this file was generated by %s %s */" % (SCRIPT, VERSION)\r
- print >>fp\r
- print >>fp, '#define UNIDATA_VERSION "%s"' % UNIDATA_VERSION\r
- print >>fp, "/* a list of unique database records */"\r
- print >>fp, \\r
- "const _PyUnicode_DatabaseRecord _PyUnicode_Database_Records[] = {"\r
- for item in table:\r
- print >>fp, " {%d, %d, %d, %d, %d, %d}," % item\r
- print >>fp, "};"\r
- print >>fp\r
-\r
- print >>fp, "/* Reindexing of NFC first characters. */"\r
- print >>fp, "#define TOTAL_FIRST",total_first\r
- print >>fp, "#define TOTAL_LAST",total_last\r
- print >>fp, "struct reindex{int start;short count,index;};"\r
- print >>fp, "static struct reindex nfc_first[] = {"\r
- for start,end in comp_first_ranges:\r
- print >>fp," { %d, %d, %d}," % (start,end-start,comp_first[start])\r
- print >>fp," {0,0,0}"\r
- print >>fp,"};\n"\r
- print >>fp, "static struct reindex nfc_last[] = {"\r
- for start,end in comp_last_ranges:\r
- print >>fp," { %d, %d, %d}," % (start,end-start,comp_last[start])\r
- print >>fp," {0,0,0}"\r
- print >>fp,"};\n"\r
-\r
- # FIXME: <fl> the following tables could be made static, and\r
- # the support code moved into unicodedatabase.c\r
-\r
- print >>fp, "/* string literals */"\r
- print >>fp, "const char *_PyUnicode_CategoryNames[] = {"\r
- for name in CATEGORY_NAMES:\r
- print >>fp, " \"%s\"," % name\r
- print >>fp, " NULL"\r
- print >>fp, "};"\r
-\r
- print >>fp, "const char *_PyUnicode_BidirectionalNames[] = {"\r
- for name in BIDIRECTIONAL_NAMES:\r
- print >>fp, " \"%s\"," % name\r
- print >>fp, " NULL"\r
- print >>fp, "};"\r
-\r
- print >>fp, "const char *_PyUnicode_EastAsianWidthNames[] = {"\r
- for name in EASTASIANWIDTH_NAMES:\r
- print >>fp, " \"%s\"," % name\r
- print >>fp, " NULL"\r
- print >>fp, "};"\r
-\r
- print >>fp, "static const char *decomp_prefix[] = {"\r
- for name in decomp_prefix:\r
- print >>fp, " \"%s\"," % name\r
- print >>fp, " NULL"\r
- print >>fp, "};"\r
-\r
- # split record index table\r
- index1, index2, shift = splitbins(index, trace)\r
-\r
- print >>fp, "/* index tables for the database records */"\r
- print >>fp, "#define SHIFT", shift\r
- Array("index1", index1).dump(fp, trace)\r
- Array("index2", index2).dump(fp, trace)\r
-\r
- # split decomposition index table\r
- index1, index2, shift = splitbins(decomp_index, trace)\r
-\r
- print >>fp, "/* decomposition data */"\r
- Array("decomp_data", decomp_data).dump(fp, trace)\r
-\r
- print >>fp, "/* index tables for the decomposition data */"\r
- print >>fp, "#define DECOMP_SHIFT", shift\r
- Array("decomp_index1", index1).dump(fp, trace)\r
- Array("decomp_index2", index2).dump(fp, trace)\r
-\r
- index, index2, shift = splitbins(comp_data, trace)\r
- print >>fp, "/* NFC pairs */"\r
- print >>fp, "#define COMP_SHIFT", shift\r
- Array("comp_index", index).dump(fp, trace)\r
- Array("comp_data", index2).dump(fp, trace)\r
-\r
- # Generate delta tables for old versions\r
- for version, table, normalization in unicode.changed:\r
- cversion = version.replace(".","_")\r
- records = [table[0]]\r
- cache = {table[0]:0}\r
- index = [0] * len(table)\r
- for i, record in enumerate(table):\r
- try:\r
- index[i] = cache[record]\r
- except KeyError:\r
- index[i] = cache[record] = len(records)\r
- records.append(record)\r
- index1, index2, shift = splitbins(index, trace)\r
- print >>fp, "static const change_record change_records_%s[] = {" % cversion\r
- for record in records:\r
- print >>fp, "\t{ %s }," % ", ".join(map(str,record))\r
- print >>fp, "};"\r
- Array("changes_%s_index" % cversion, index1).dump(fp, trace)\r
- Array("changes_%s_data" % cversion, index2).dump(fp, trace)\r
- print >>fp, "static const change_record* get_change_%s(Py_UCS4 n)" % cversion\r
- print >>fp, "{"\r
- print >>fp, "\tint index;"\r
- print >>fp, "\tif (n >= 0x110000) index = 0;"\r
- print >>fp, "\telse {"\r
- print >>fp, "\t\tindex = changes_%s_index[n>>%d];" % (cversion, shift)\r
- print >>fp, "\t\tindex = changes_%s_data[(index<<%d)+(n & %d)];" % \\r
- (cversion, shift, ((1<<shift)-1))\r
- print >>fp, "\t}"\r
- print >>fp, "\treturn change_records_%s+index;" % cversion\r
- print >>fp, "}\n"\r
- print >>fp, "static Py_UCS4 normalization_%s(Py_UCS4 n)" % cversion\r
- print >>fp, "{"\r
- print >>fp, "\tswitch(n) {"\r
- for k, v in normalization:\r
- print >>fp, "\tcase %s: return 0x%s;" % (hex(k), v)\r
- print >>fp, "\tdefault: return 0;"\r
- print >>fp, "\t}\n}\n"\r
-\r
- fp.close()\r
-\r
-# --------------------------------------------------------------------\r
-# unicode character type tables\r
-\r
-def makeunicodetype(unicode, trace):\r
-\r
- FILE = "Objects/unicodetype_db.h"\r
-\r
- print "--- Preparing", FILE, "..."\r
-\r
- # extract unicode types\r
- dummy = (0, 0, 0, 0, 0, 0)\r
- table = [dummy]\r
- cache = {0: dummy}\r
- index = [0] * len(unicode.chars)\r
- numeric = {}\r
- spaces = []\r
- linebreaks = []\r
-\r
- for char in unicode.chars:\r
- record = unicode.table[char]\r
- if record:\r
- # extract database properties\r
- category = record[2]\r
- bidirectional = record[4]\r
- properties = record[16]\r
- flags = 0\r
- delta = True\r
- if category in ["Lm", "Lt", "Lu", "Ll", "Lo"]:\r
- flags |= ALPHA_MASK\r
- if category == "Ll":\r
- flags |= LOWER_MASK\r
- if 'Line_Break' in properties or bidirectional == "B":\r
- flags |= LINEBREAK_MASK\r
- linebreaks.append(char)\r
- if category == "Zs" or bidirectional in ("WS", "B", "S"):\r
- flags |= SPACE_MASK\r
- spaces.append(char)\r
- if category == "Lt":\r
- flags |= TITLE_MASK\r
- if category == "Lu":\r
- flags |= UPPER_MASK\r
- # use delta predictor for upper/lower/title if it fits\r
- if record[12]:\r
- upper = int(record[12], 16)\r
- else:\r
- upper = char\r
- if record[13]:\r
- lower = int(record[13], 16)\r
- else:\r
- lower = char\r
- if record[14]:\r
- title = int(record[14], 16)\r
- else:\r
- # UCD.html says that a missing title char means that\r
- # it defaults to the uppercase character, not to the\r
- # character itself. Apparently, in the current UCD (5.x)\r
- # this feature is never used\r
- title = upper\r
- upper_d = upper - char\r
- lower_d = lower - char\r
- title_d = title - char\r
- if -32768 <= upper_d <= 32767 and \\r
- -32768 <= lower_d <= 32767 and \\r
- -32768 <= title_d <= 32767:\r
- # use deltas\r
- upper = upper_d & 0xffff\r
- lower = lower_d & 0xffff\r
- title = title_d & 0xffff\r
- else:\r
- flags |= NODELTA_MASK\r
- # decimal digit, integer digit\r
- decimal = 0\r
- if record[6]:\r
- flags |= DECIMAL_MASK\r
- decimal = int(record[6])\r
- digit = 0\r
- if record[7]:\r
- flags |= DIGIT_MASK\r
- digit = int(record[7])\r
- if record[8]:\r
- flags |= NUMERIC_MASK\r
- numeric.setdefault(record[8], []).append(char)\r
- item = (\r
- upper, lower, title, decimal, digit, flags\r
- )\r
- # add entry to index and item tables\r
- i = cache.get(item)\r
- if i is None:\r
- cache[item] = i = len(table)\r
- table.append(item)\r
- index[char] = i\r
-\r
- print len(table), "unique character type entries"\r
- print sum(map(len, numeric.values())), "numeric code points"\r
- print len(spaces), "whitespace code points"\r
- print len(linebreaks), "linebreak code points"\r
-\r
- print "--- Writing", FILE, "..."\r
-\r
- fp = open(FILE, "w")\r
- print >>fp, "/* this file was generated by %s %s */" % (SCRIPT, VERSION)\r
- print >>fp\r
- print >>fp, "/* a list of unique character type descriptors */"\r
- print >>fp, "const _PyUnicode_TypeRecord _PyUnicode_TypeRecords[] = {"\r
- for item in table:\r
- print >>fp, " {%d, %d, %d, %d, %d, %d}," % item\r
- print >>fp, "};"\r
- print >>fp\r
-\r
- # split decomposition index table\r
- index1, index2, shift = splitbins(index, trace)\r
-\r
- print >>fp, "/* type indexes */"\r
- print >>fp, "#define SHIFT", shift\r
- Array("index1", index1).dump(fp, trace)\r
- Array("index2", index2).dump(fp, trace)\r
-\r
- # Generate code for _PyUnicode_ToNumeric()\r
- numeric_items = sorted(numeric.items())\r
- print >>fp, '/* Returns the numeric value as double for Unicode characters'\r
- print >>fp, ' * having this property, -1.0 otherwise.'\r
- print >>fp, ' */'\r
- print >>fp, 'double _PyUnicode_ToNumeric(Py_UNICODE ch)'\r
- print >>fp, '{'\r
- print >>fp, ' switch (ch) {'\r
- for value, codepoints in numeric_items:\r
- # Turn text into float literals\r
- parts = value.split('/')\r
- parts = [repr(float(part)) for part in parts]\r
- value = '/'.join(parts)\r
-\r
- haswide = False\r
- hasnonewide = False\r
- codepoints.sort()\r
- for codepoint in codepoints:\r
- if codepoint < 0x10000:\r
- hasnonewide = True\r
- if codepoint >= 0x10000 and not haswide:\r
- print >>fp, '#ifdef Py_UNICODE_WIDE'\r
- haswide = True\r
- print >>fp, ' case 0x%04X:' % (codepoint,)\r
- if haswide and hasnonewide:\r
- print >>fp, '#endif'\r
- print >>fp, ' return (double) %s;' % (value,)\r
- if haswide and not hasnonewide:\r
- print >>fp, '#endif'\r
- print >>fp,' }'\r
- print >>fp,' return -1.0;'\r
- print >>fp,'}'\r
- print >>fp\r
-\r
- # Generate code for _PyUnicode_IsWhitespace()\r
- print >>fp, "/* Returns 1 for Unicode characters having the bidirectional"\r
- print >>fp, " * type 'WS', 'B' or 'S' or the category 'Zs', 0 otherwise."\r
- print >>fp, " */"\r
- print >>fp, 'int _PyUnicode_IsWhitespace(register const Py_UNICODE ch)'\r
- print >>fp, '{'\r
- print >>fp, '#ifdef WANT_WCTYPE_FUNCTIONS'\r
- print >>fp, ' return iswspace(ch);'\r
- print >>fp, '#else'\r
- print >>fp, ' switch (ch) {'\r
-\r
- haswide = False\r
- hasnonewide = False\r
- for codepoint in sorted(spaces):\r
- if codepoint < 0x10000:\r
- hasnonewide = True\r
- if codepoint >= 0x10000 and not haswide:\r
- print >>fp, '#ifdef Py_UNICODE_WIDE'\r
- haswide = True\r
- print >>fp, ' case 0x%04X:' % (codepoint,)\r
- if haswide and hasnonewide:\r
- print >>fp, '#endif'\r
- print >>fp, ' return 1;'\r
- if haswide and not hasnonewide:\r
- print >>fp, '#endif'\r
-\r
- print >>fp,' }'\r
- print >>fp,' return 0;'\r
- print >>fp, '#endif'\r
- print >>fp,'}'\r
- print >>fp\r
-\r
- # Generate code for _PyUnicode_IsLinebreak()\r
- print >>fp, "/* Returns 1 for Unicode characters having the line break"\r
- print >>fp, " * property 'BK', 'CR', 'LF' or 'NL' or having bidirectional"\r
- print >>fp, " * type 'B', 0 otherwise."\r
- print >>fp, " */"\r
- print >>fp, 'int _PyUnicode_IsLinebreak(register const Py_UNICODE ch)'\r
- print >>fp, '{'\r
- print >>fp, ' switch (ch) {'\r
- haswide = False\r
- hasnonewide = False\r
- for codepoint in sorted(linebreaks):\r
- if codepoint < 0x10000:\r
- hasnonewide = True\r
- if codepoint >= 0x10000 and not haswide:\r
- print >>fp, '#ifdef Py_UNICODE_WIDE'\r
- haswide = True\r
- print >>fp, ' case 0x%04X:' % (codepoint,)\r
- if haswide and hasnonewide:\r
- print >>fp, '#endif'\r
- print >>fp, ' return 1;'\r
- if haswide and not hasnonewide:\r
- print >>fp, '#endif'\r
-\r
- print >>fp,' }'\r
- print >>fp,' return 0;'\r
- print >>fp,'}'\r
- print >>fp\r
-\r
- fp.close()\r
-\r
-# --------------------------------------------------------------------\r
-# unicode name database\r
-\r
-def makeunicodename(unicode, trace):\r
-\r
- FILE = "Modules/unicodename_db.h"\r
-\r
- print "--- Preparing", FILE, "..."\r
-\r
- # collect names\r
- names = [None] * len(unicode.chars)\r
-\r
- for char in unicode.chars:\r
- record = unicode.table[char]\r
- if record:\r
- name = record[1].strip()\r
- if name and name[0] != "<":\r
- names[char] = name + chr(0)\r
-\r
- print len(filter(lambda n: n is not None, names)), "distinct names"\r
-\r
- # collect unique words from names (note that we differ between\r
- # words inside a sentence, and words ending a sentence. the\r
- # latter includes the trailing null byte.\r
-\r
- words = {}\r
- n = b = 0\r
- for char in unicode.chars:\r
- name = names[char]\r
- if name:\r
- w = name.split()\r
- b = b + len(name)\r
- n = n + len(w)\r
- for w in w:\r
- l = words.get(w)\r
- if l:\r
- l.append(None)\r
- else:\r
- words[w] = [len(words)]\r
-\r
- print n, "words in text;", b, "bytes"\r
-\r
- wordlist = words.items()\r
-\r
- # sort on falling frequency, then by name\r
- def word_key(a):\r
- aword, alist = a\r
- return -len(alist), aword\r
- wordlist.sort(key=word_key)\r
-\r
- # figure out how many phrasebook escapes we need\r
- escapes = 0\r
- while escapes * 256 < len(wordlist):\r
- escapes = escapes + 1\r
- print escapes, "escapes"\r
-\r
- short = 256 - escapes\r
-\r
- assert short > 0\r
-\r
- print short, "short indexes in lexicon"\r
-\r
- # statistics\r
- n = 0\r
- for i in range(short):\r
- n = n + len(wordlist[i][1])\r
- print n, "short indexes in phrasebook"\r
-\r
- # pick the most commonly used words, and sort the rest on falling\r
- # length (to maximize overlap)\r
-\r
- wordlist, wordtail = wordlist[:short], wordlist[short:]\r
- wordtail.sort(key=lambda a: a[0], reverse=True)\r
- wordlist.extend(wordtail)\r
-\r
- # generate lexicon from words\r
-\r
- lexicon_offset = [0]\r
- lexicon = ""\r
- words = {}\r
-\r
- # build a lexicon string\r
- offset = 0\r
- for w, x in wordlist:\r
- # encoding: bit 7 indicates last character in word (chr(128)\r
- # indicates the last character in an entire string)\r
- ww = w[:-1] + chr(ord(w[-1])+128)\r
- # reuse string tails, when possible\r
- o = lexicon.find(ww)\r
- if o < 0:\r
- o = offset\r
- lexicon = lexicon + ww\r
- offset = offset + len(w)\r
- words[w] = len(lexicon_offset)\r
- lexicon_offset.append(o)\r
-\r
- lexicon = map(ord, lexicon)\r
-\r
- # generate phrasebook from names and lexicon\r
- phrasebook = [0]\r
- phrasebook_offset = [0] * len(unicode.chars)\r
- for char in unicode.chars:\r
- name = names[char]\r
- if name:\r
- w = name.split()\r
- phrasebook_offset[char] = len(phrasebook)\r
- for w in w:\r
- i = words[w]\r
- if i < short:\r
- phrasebook.append(i)\r
- else:\r
- # store as two bytes\r
- phrasebook.append((i>>8) + short)\r
- phrasebook.append(i&255)\r
-\r
- assert getsize(phrasebook) == 1\r
-\r
- #\r
- # unicode name hash table\r
-\r
- # extract names\r
- data = []\r
- for char in unicode.chars:\r
- record = unicode.table[char]\r
- if record:\r
- name = record[1].strip()\r
- if name and name[0] != "<":\r
- data.append((name, char))\r
-\r
- # the magic number 47 was chosen to minimize the number of\r
- # collisions on the current data set. if you like, change it\r
- # and see what happens...\r
-\r
- codehash = Hash("code", data, 47)\r
-\r
- print "--- Writing", FILE, "..."\r
-\r
- fp = open(FILE, "w")\r
- print >>fp, "/* this file was generated by %s %s */" % (SCRIPT, VERSION)\r
- print >>fp\r
- print >>fp, "#define NAME_MAXLEN", 256\r
- print >>fp\r
- print >>fp, "/* lexicon */"\r
- Array("lexicon", lexicon).dump(fp, trace)\r
- Array("lexicon_offset", lexicon_offset).dump(fp, trace)\r
-\r
- # split decomposition index table\r
- offset1, offset2, shift = splitbins(phrasebook_offset, trace)\r
-\r
- print >>fp, "/* code->name phrasebook */"\r
- print >>fp, "#define phrasebook_shift", shift\r
- print >>fp, "#define phrasebook_short", short\r
-\r
- Array("phrasebook", phrasebook).dump(fp, trace)\r
- Array("phrasebook_offset1", offset1).dump(fp, trace)\r
- Array("phrasebook_offset2", offset2).dump(fp, trace)\r
-\r
- print >>fp, "/* name->code dictionary */"\r
- codehash.dump(fp, trace)\r
-\r
- fp.close()\r
-\r
-\r
-def merge_old_version(version, new, old):\r
- # Changes to exclusion file not implemented yet\r
- if old.exclusions != new.exclusions:\r
- raise NotImplementedError, "exclusions differ"\r
-\r
- # In these change records, 0xFF means "no change"\r
- bidir_changes = [0xFF]*0x110000\r
- category_changes = [0xFF]*0x110000\r
- decimal_changes = [0xFF]*0x110000\r
- mirrored_changes = [0xFF]*0x110000\r
- # In numeric data, 0 means "no change",\r
- # -1 means "did not have a numeric value\r
- numeric_changes = [0] * 0x110000\r
- # normalization_changes is a list of key-value pairs\r
- normalization_changes = []\r
- for i in range(0x110000):\r
- if new.table[i] is None:\r
- # Characters unassigned in the new version ought to\r
- # be unassigned in the old one\r
- assert old.table[i] is None\r
- continue\r
- # check characters unassigned in the old version\r
- if old.table[i] is None:\r
- # category 0 is "unassigned"\r
- category_changes[i] = 0\r
- continue\r
- # check characters that differ\r
- if old.table[i] != new.table[i]:\r
- for k in range(len(old.table[i])):\r
- if old.table[i][k] != new.table[i][k]:\r
- value = old.table[i][k]\r
- if k == 2:\r
- #print "CATEGORY",hex(i), old.table[i][k], new.table[i][k]\r
- category_changes[i] = CATEGORY_NAMES.index(value)\r
- elif k == 4:\r
- #print "BIDIR",hex(i), old.table[i][k], new.table[i][k]\r
- bidir_changes[i] = BIDIRECTIONAL_NAMES.index(value)\r
- elif k == 5:\r
- #print "DECOMP",hex(i), old.table[i][k], new.table[i][k]\r
- # We assume that all normalization changes are in 1:1 mappings\r
- assert " " not in value\r
- normalization_changes.append((i, value))\r
- elif k == 6:\r
- #print "DECIMAL",hex(i), old.table[i][k], new.table[i][k]\r
- # we only support changes where the old value is a single digit\r
- assert value in "0123456789"\r
- decimal_changes[i] = int(value)\r
- elif k == 8:\r
- # print "NUMERIC",hex(i), `old.table[i][k]`, new.table[i][k]\r
- # Since 0 encodes "no change", the old value is better not 0\r
- if not value:\r
- numeric_changes[i] = -1\r
- else:\r
- numeric_changes[i] = float(value)\r
- assert numeric_changes[i] not in (0, -1)\r
- elif k == 9:\r
- if value == 'Y':\r
- mirrored_changes[i] = '1'\r
- else:\r
- mirrored_changes[i] = '0'\r
- elif k == 11:\r
- # change to ISO comment, ignore\r
- pass\r
- elif k == 12:\r
- # change to simple uppercase mapping; ignore\r
- pass\r
- elif k == 13:\r
- # change to simple lowercase mapping; ignore\r
- pass\r
- elif k == 14:\r
- # change to simple titlecase mapping; ignore\r
- pass\r
- elif k == 16:\r
- # change to properties; not yet\r
- pass\r
- else:\r
- class Difference(Exception):pass\r
- raise Difference, (hex(i), k, old.table[i], new.table[i])\r
- new.changed.append((version, zip(bidir_changes, category_changes,\r
- decimal_changes, mirrored_changes,\r
- numeric_changes),\r
- normalization_changes))\r
-\r
-\r
-# --------------------------------------------------------------------\r
-# the following support code is taken from the unidb utilities\r
-# Copyright (c) 1999-2000 by Secret Labs AB\r
-\r
-# load a unicode-data file from disk\r
-\r
-class UnicodeData:\r
- # Record structure:\r
- # [ID, name, category, combining, bidi, decomp, (6)\r
- # decimal, digit, numeric, bidi-mirrored, Unicode-1-name, (11)\r
- # ISO-comment, uppercase, lowercase, titlecase, ea-width, (16)\r
- # properties] (17)\r
-\r
- def __init__(self, filename, exclusions, eastasianwidth, unihan,\r
- derivednormalizationprops=None, linebreakprops=None,\r
- expand=1):\r
- self.changed = []\r
- file = open(filename)\r
- table = [None] * 0x110000\r
- while 1:\r
- s = file.readline()\r
- if not s:\r
- break\r
- s = s.strip().split(";")\r
- char = int(s[0], 16)\r
- table[char] = s\r
-\r
- # expand first-last ranges\r
- if expand:\r
- field = None\r
- for i in range(0, 0x110000):\r
- s = table[i]\r
- if s:\r
- if s[1][-6:] == "First>":\r
- s[1] = ""\r
- field = s\r
- elif s[1][-5:] == "Last>":\r
- s[1] = ""\r
- field = None\r
- elif field:\r
- f2 = field[:]\r
- f2[0] = "%X" % i\r
- table[i] = f2\r
-\r
- # public attributes\r
- self.filename = filename\r
- self.table = table\r
- self.chars = range(0x110000) # unicode 3.2\r
-\r
- file = open(exclusions)\r
- self.exclusions = {}\r
- for s in file:\r
- s = s.strip()\r
- if not s:\r
- continue\r
- if s[0] == '#':\r
- continue\r
- char = int(s.split()[0],16)\r
- self.exclusions[char] = 1\r
-\r
- widths = [None] * 0x110000\r
- for s in open(eastasianwidth):\r
- s = s.strip()\r
- if not s:\r
- continue\r
- if s[0] == '#':\r
- continue\r
- s = s.split()[0].split(';')\r
- if '..' in s[0]:\r
- first, last = [int(c, 16) for c in s[0].split('..')]\r
- chars = range(first, last+1)\r
- else:\r
- chars = [int(s[0], 16)]\r
- for char in chars:\r
- widths[char] = s[1]\r
- for i in range(0, 0x110000):\r
- if table[i] is not None:\r
- table[i].append(widths[i])\r
-\r
- for i in range(0, 0x110000):\r
- if table[i] is not None:\r
- table[i].append(set())\r
- if linebreakprops:\r
- for s in open(linebreakprops):\r
- s = s.partition('#')[0]\r
- s = [i.strip() for i in s.split(';')]\r
- if len(s) < 2 or s[1] not in MANDATORY_LINE_BREAKS:\r
- continue\r
- if '..' not in s[0]:\r
- first = last = int(s[0], 16)\r
- else:\r
- first, last = [int(c, 16) for c in s[0].split('..')]\r
- for char in range(first, last+1):\r
- table[char][-1].add('Line_Break')\r
-\r
- if derivednormalizationprops:\r
- quickchecks = [0] * 0x110000 # default is Yes\r
- qc_order = 'NFD_QC NFKD_QC NFC_QC NFKC_QC'.split()\r
- for s in open(derivednormalizationprops):\r
- if '#' in s:\r
- s = s[:s.index('#')]\r
- s = [i.strip() for i in s.split(';')]\r
- if len(s) < 2 or s[1] not in qc_order:\r
- continue\r
- quickcheck = 'MN'.index(s[2]) + 1 # Maybe or No\r
- quickcheck_shift = qc_order.index(s[1])*2\r
- quickcheck <<= quickcheck_shift\r
- if '..' not in s[0]:\r
- first = last = int(s[0], 16)\r
- else:\r
- first, last = [int(c, 16) for c in s[0].split('..')]\r
- for char in range(first, last+1):\r
- assert not (quickchecks[char]>>quickcheck_shift)&3\r
- quickchecks[char] |= quickcheck\r
- for i in range(0, 0x110000):\r
- if table[i] is not None:\r
- table[i].append(quickchecks[i])\r
-\r
- for line in open(unihan):\r
- if not line.startswith('U+'):\r
- continue\r
- code, tag, value = line.split(None, 3)[:3]\r
- if tag not in ('kAccountingNumeric', 'kPrimaryNumeric',\r
- 'kOtherNumeric'):\r
- continue\r
- value = value.strip().replace(',', '')\r
- i = int(code[2:], 16)\r
- # Patch the numeric field\r
- if table[i] is not None:\r
- table[i][8] = value\r
-\r
- def uselatin1(self):\r
- # restrict character range to ISO Latin 1\r
- self.chars = range(256)\r
-\r
-# hash table tools\r
-\r
-# this is a straight-forward reimplementation of Python's built-in\r
-# dictionary type, using a static data structure, and a custom string\r
-# hash algorithm.\r
-\r
-def myhash(s, magic):\r
- h = 0\r
- for c in map(ord, s.upper()):\r
- h = (h * magic) + c\r
- ix = h & 0xff000000L\r
- if ix:\r
- h = (h ^ ((ix>>24) & 0xff)) & 0x00ffffff\r
- return h\r
-\r
-SIZES = [\r
- (4,3), (8,3), (16,3), (32,5), (64,3), (128,3), (256,29), (512,17),\r
- (1024,9), (2048,5), (4096,83), (8192,27), (16384,43), (32768,3),\r
- (65536,45), (131072,9), (262144,39), (524288,39), (1048576,9),\r
- (2097152,5), (4194304,3), (8388608,33), (16777216,27)\r
-]\r
-\r
-class Hash:\r
- def __init__(self, name, data, magic):\r
- # turn a (key, value) list into a static hash table structure\r
-\r
- # determine table size\r
- for size, poly in SIZES:\r
- if size > len(data):\r
- poly = size + poly\r
- break\r
- else:\r
- raise AssertionError, "ran out of polynominals"\r
-\r
- print size, "slots in hash table"\r
-\r
- table = [None] * size\r
-\r
- mask = size-1\r
-\r
- n = 0\r
-\r
- hash = myhash\r
-\r
- # initialize hash table\r
- for key, value in data:\r
- h = hash(key, magic)\r
- i = (~h) & mask\r
- v = table[i]\r
- if v is None:\r
- table[i] = value\r
- continue\r
- incr = (h ^ (h >> 3)) & mask;\r
- if not incr:\r
- incr = mask\r
- while 1:\r
- n = n + 1\r
- i = (i + incr) & mask\r
- v = table[i]\r
- if v is None:\r
- table[i] = value\r
- break\r
- incr = incr << 1\r
- if incr > mask:\r
- incr = incr ^ poly\r
-\r
- print n, "collisions"\r
- self.collisions = n\r
-\r
- for i in range(len(table)):\r
- if table[i] is None:\r
- table[i] = 0\r
-\r
- self.data = Array(name + "_hash", table)\r
- self.magic = magic\r
- self.name = name\r
- self.size = size\r
- self.poly = poly\r
-\r
- def dump(self, file, trace):\r
- # write data to file, as a C array\r
- self.data.dump(file, trace)\r
- file.write("#define %s_magic %d\n" % (self.name, self.magic))\r
- file.write("#define %s_size %d\n" % (self.name, self.size))\r
- file.write("#define %s_poly %d\n" % (self.name, self.poly))\r
-\r
-# stuff to deal with arrays of unsigned integers\r
-\r
-class Array:\r
-\r
- def __init__(self, name, data):\r
- self.name = name\r
- self.data = data\r
-\r
- def dump(self, file, trace=0):\r
- # write data to file, as a C array\r
- size = getsize(self.data)\r
- if trace:\r
- print >>sys.stderr, self.name+":", size*len(self.data), "bytes"\r
- file.write("static ")\r
- if size == 1:\r
- file.write("unsigned char")\r
- elif size == 2:\r
- file.write("unsigned short")\r
- else:\r
- file.write("unsigned int")\r
- file.write(" " + self.name + "[] = {\n")\r
- if self.data:\r
- s = " "\r
- for item in self.data:\r
- i = str(item) + ", "\r
- if len(s) + len(i) > 78:\r
- file.write(s + "\n")\r
- s = " " + i\r
- else:\r
- s = s + i\r
- if s.strip():\r
- file.write(s + "\n")\r
- file.write("};\n\n")\r
-\r
-def getsize(data):\r
- # return smallest possible integer size for the given array\r
- maxdata = max(data)\r
- if maxdata < 256:\r
- return 1\r
- elif maxdata < 65536:\r
- return 2\r
- else:\r
- return 4\r
-\r
-def splitbins(t, trace=0):\r
- """t, trace=0 -> (t1, t2, shift). Split a table to save space.\r
-\r
- t is a sequence of ints. This function can be useful to save space if\r
- many of the ints are the same. t1 and t2 are lists of ints, and shift\r
- is an int, chosen to minimize the combined size of t1 and t2 (in C\r
- code), and where for each i in range(len(t)),\r
- t[i] == t2[(t1[i >> shift] << shift) + (i & mask)]\r
- where mask is a bitmask isolating the last "shift" bits.\r
-\r
- If optional arg trace is non-zero (default zero), progress info\r
- is printed to sys.stderr. The higher the value, the more info\r
- you'll get.\r
- """\r
-\r
- if trace:\r
- def dump(t1, t2, shift, bytes):\r
- print >>sys.stderr, "%d+%d bins at shift %d; %d bytes" % (\r
- len(t1), len(t2), shift, bytes)\r
- print >>sys.stderr, "Size of original table:", len(t)*getsize(t), \\r
- "bytes"\r
- n = len(t)-1 # last valid index\r
- maxshift = 0 # the most we can shift n and still have something left\r
- if n > 0:\r
- while n >> 1:\r
- n >>= 1\r
- maxshift += 1\r
- del n\r
- bytes = sys.maxint # smallest total size so far\r
- t = tuple(t) # so slices can be dict keys\r
- for shift in range(maxshift + 1):\r
- t1 = []\r
- t2 = []\r
- size = 2**shift\r
- bincache = {}\r
- for i in range(0, len(t), size):\r
- bin = t[i:i+size]\r
- index = bincache.get(bin)\r
- if index is None:\r
- index = len(t2)\r
- bincache[bin] = index\r
- t2.extend(bin)\r
- t1.append(index >> shift)\r
- # determine memory size\r
- b = len(t1)*getsize(t1) + len(t2)*getsize(t2)\r
- if trace > 1:\r
- dump(t1, t2, shift, b)\r
- if b < bytes:\r
- best = t1, t2, shift\r
- bytes = b\r
- t1, t2, shift = best\r
- if trace:\r
- print >>sys.stderr, "Best:",\r
- dump(t1, t2, shift, bytes)\r
- if __debug__:\r
- # exhaustively verify that the decomposition is correct\r
- mask = ~((~0) << shift) # i.e., low-bit mask of shift bits\r
- for i in xrange(len(t)):\r
- assert t[i] == t2[(t1[i >> shift] << shift) + (i & mask)]\r
- return best\r
-\r
-if __name__ == "__main__":\r
- maketables(1)\r
projects / mirror_edk2.git / blobdiff