vendor/unicode-script/scripts/unicode.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright 2011-2015 The Rust Project Developers. See the COPYRIGHT
   4 # file at the top-level directory of this distribution and at
   5 # http://rust-lang.org/COPYRIGHT.
   6 #
   7 # Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
   8 # http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
   9 # <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
  10 # option. This file may not be copied, modified, or distributed
  11 # except according to those terms.
  12
  13 # This script uses the following Unicode tables:
  14 # - PropertyValueAliases.txt
  15 # - ScriptExtensions.txt
  16 # - Scripts.txt
  17 #
  18 # Since this should not require frequent updates, we just store this
  19 # out-of-line and check the unicode.rs file into git.
  20
  21 import fileinput, re, os, sys
  22
  23 preamble = '''// Copyright 2012-2018 The Rust Project Developers. See the COPYRIGHT
  24 // file at the top-level directory of this distribution and at
  25 // http://rust-lang.org/COPYRIGHT.
  26 //
  27 // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
  28 // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
  29 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
  30 // option. This file may not be copied, modified, or distributed
  31 // except according to those terms.
  32
  33 // NOTE: The following code was generated by "scripts/unicode.py", do not edit directly
  34
  35 #![allow(missing_docs, non_upper_case_globals, non_snake_case)]
  36
  37 pub use tables_impl::*;
  38
  39 #[rustfmt::skip]
  40 mod tables_impl {
  41 use crate::ScriptExtension;
  42 '''
  43
  44 # Close `mod impl {`
  45 ending='''
  46 }
  47 '''
  48
  49 UNICODE_VERSION = (13, 0, 0)
  50
  51 UNICODE_VERSION_NUMBER = "%s.%s.%s" %UNICODE_VERSION
  52
  53 def escape_char(c):
  54     return "'\\u{%x}'" % c
  55
  56 def fetch(f):
  57     if not os.path.exists(os.path.basename(f)):
  58         if "emoji" in f:
  59             os.system("curl -O https://www.unicode.org/Public/emoji/%s.%s/%s"
  60                       % (UNICODE_VERSION[0], UNICODE_VERSION[1], f))
  61         else:
  62             os.system("curl -O http://www.unicode.org/Public/%s/ucd/%s"
  63                       % (UNICODE_VERSION_NUMBER, f))
  64
  65     if not os.path.exists(os.path.basename(f)):
  66         sys.stderr.write("cannot load %s" % f)
  67         exit(1)
  68
  69 def group_cats(cats):
  70     cats_out = {}
  71     for cat in cats:
  72         cats_out[cat] = group_cat(cats[cat])
  73     return cats_out
  74
  75 def aliases():
  76     """
  77     Fetch the shorthand aliases for each longhand Script name
  78     """
  79     fetch("PropertyValueAliases.txt")
  80     longforms = {}
  81     shortforms = {}
  82     re1 = re.compile(r"^ *sc *; *(\w+) *; *(\w+)")
  83     for line in fileinput.input(os.path.basename("PropertyValueAliases.txt")):
  84         m = re1.match(line)
  85         if m:
  86             l = m.group(2).strip()
  87             s = m.group(1).strip()
  88             assert(s not in longforms)
  89             assert(l not in shortforms)
  90             longforms[s] = l
  91             shortforms[l] = s
  92         else:
  93             continue
  94
  95     return (longforms, shortforms)
  96
  97 def format_table_content(f, content, indent):
  98     line = " "*indent
  99     first = True
 100     for chunk in content.split(","):
 101         if len(line) + len(chunk) < 98:
 102             if first:
 103                 line += chunk
 104             else:
 105                 line += ", " + chunk
 106             first = False
 107         else:
 108             f.write(line + ",\n")
 109             line = " "*indent + chunk
 110     f.write(line)
 111
 112 # Implementation from unicode-segmentation
 113 def load_properties(f, interestingprops):
 114     fetch(f)
 115     props = {}
 116     # Note: these regexes are different from those in unicode-segmentation,
 117     # becase we need to handle spaces here
 118     re1 = re.compile(r"^ *([0-9A-F]+) *; *([^#]+) *#")
 119     re2 = re.compile(r"^ *([0-9A-F]+)\.\.([0-9A-F]+) *; *([^#]+) *#")
 120
 121     for line in fileinput.input(os.path.basename(f)):
 122         prop = None
 123         d_lo = 0
 124         d_hi = 0
 125         m = re1.match(line)
 126         if m:
 127             d_lo = m.group(1)
 128             d_hi = m.group(1)
 129             prop = m.group(2).strip()
 130         else:
 131             m = re2.match(line)
 132             if m:
 133                 d_lo = m.group(1)
 134                 d_hi = m.group(2)
 135                 prop = m.group(3).strip()
 136             else:
 137                 continue
 138         if interestingprops and prop not in interestingprops:
 139             continue
 140         d_lo = int(d_lo, 16)
 141         d_hi = int(d_hi, 16)
 142         if prop not in props:
 143             props[prop] = []
 144         props[prop].append((d_lo, d_hi))
 145
 146     return props
 147
 148 # Implementation from unicode-segmentation
 149 def emit_table(f, name, t_data, t_type = "&'static [(char, char)]", is_pub=True,
 150         pfun=lambda x: "(%s,%s)" % (escape_char(x[0]), escape_char(x[1])), is_const=True):
 151     pub_string = "const"
 152     if not is_const:
 153         pub_string = "let"
 154     if is_pub:
 155         pub_string = "pub " + pub_string
 156     f.write("    %s %s: %s = &[\n" % (pub_string, name, t_type))
 157     data = ""
 158     first = True
 159     for dat in t_data:
 160         if not first:
 161             data += ","
 162         first = False
 163         data += pfun(dat)
 164     format_table_content(f, data, 8)
 165     f.write("\n    ];\n\n")
 166
 167 def emit_search(f):
 168     f.write("""
 169 pub fn bsearch_range_value_table<T: Copy>(c: char, r: &'static [(char, char, T)]) -> Option<T> {
 170     use core::cmp::Ordering::{Equal, Less, Greater};
 171     match r.binary_search_by(|&(lo, hi, _)| {
 172         if lo <= c && c <= hi { Equal }
 173         else if hi < c { Less }
 174         else { Greater }
 175     }) {
 176         Ok(idx) => {
 177             let (_, _, cat) = r[idx];
 178             Some(cat)
 179         }
 180         Err(_) => None
 181     }
 182 }
 183
 184 #[inline]
 185 pub fn get_script(c: char) -> Option<Script> {
 186     bsearch_range_value_table(c, SCRIPTS)
 187 }
 188
 189 #[inline]
 190 pub fn get_script_extension(c: char) -> Option<ScriptExtension> {
 191     bsearch_range_value_table(c, SCRIPT_EXTENSIONS)
 192 }
 193 """)
 194
 195 def emit_enums(f, script_list, extension_list, longforms):
 196     """
 197     Emit the Script and ScriptExtension enums as well as any related utility functions
 198     """
 199
 200     f.write("""
 201 #[derive(Clone, Copy, PartialEq, Eq, Debug, Hash)]
 202 #[non_exhaustive]
 203 #[allow(non_camel_case_types)]
 204 #[repr(u8)]
 205 /// A value of the `Script` property
 206 pub enum Script {
 207     /// Unknown script
 208     Unknown = 0xFF,
 209     /// Zyyy
 210     Common = 0xFE,
 211     /// Zinh,
 212     Inherited = 0xFD,
 213 """)
 214     for (i, script) in enumerate(script_list):
 215         f.write("    /// %s\n    %s = %s,\n" % (script, longforms[script], i))
 216     f.write("}\n")
 217     f.write("pub const NEXT_SCRIPT: u8 = %s;" % len(script_list))
 218     f.write("""
 219
 220 pub mod script_extensions {
 221     use crate::ScriptExtension;
 222     pub const COMMON: ScriptExtension = ScriptExtension::new_common();
 223     pub const INHERITED: ScriptExtension = ScriptExtension::new_inherited();
 224     pub const UNKNOWN: ScriptExtension = ScriptExtension::new_unknown();
 225 """)
 226     for (i, script) in enumerate(script_list):
 227         first = 0
 228         second = 0
 229         third = 0
 230         # need to replace L because `hex()` will spit out an L suffix for larger numbers
 231         if i < 64:
 232             first = hex(1 << i).replace("L", "")
 233         elif i < 128:
 234             second = hex(1 << (i - 64)).replace("L", "")
 235         else:
 236             third = hex(1 << (i - 128)).replace("L", "")
 237         f.write("    /// %s\n    pub const %s: ScriptExtension = ScriptExtension::new(%s, %s, %s);\n" %
 238                 (longforms[script], longforms[script].upper(), first, second, third))
 239         if script != longforms[script]:
 240             f.write("    /// %s\n    pub const %s: ScriptExtension = %s;\n" %
 241                     (longforms[script], script.upper(), longforms[script].upper()))
 242     for ext in extension_list:
 243         longform = ", ".join([longforms[s] for s in ext])
 244         name = "_".join([s.upper() for s in ext])
 245         expr = ext[0].upper()
 246         for e in ext[1:]:
 247             expr = "%s.union(%s)" % (expr, e.upper())
 248         f.write("    /// %s\n    pub const %s: ScriptExtension = %s;\n" % (longform, name, expr))
 249     f.write("""}
 250
 251 """)
 252
 253     # Generate implementation for the `Script`
 254     generate_script_impl(f)
 255
 256
 257 def generate_script_impl(f):
 258     """Generates an `impl Script { ... }` section with all the required functions"""
 259
 260     # Open `impl Script` section.
 261     f.write("""impl Script {
 262 """)
 263
 264     # Generate impl of `inner_full_name`.
 265     f.write("""
 266     #[inline]
 267     pub(crate) fn inner_full_name(self) -> &'static str {
 268         match self {
 269             Script::Unknown => "Unknown",
 270             Script::Common => "Common",
 271             Script::Inherited => "Inherited",
 272 """)
 273     for script in script_list:
 274         f.write("            Script::%s => \"%s\",\n" % (longforms[script], longforms[script]))
 275     f.write("""        }
 276     }
 277 """)
 278
 279     # Generate impl of `inner_from_full_name`.
 280     f.write("""
 281     #[inline]
 282     pub(crate) fn inner_from_full_name(input: &str) -> Option<Self> {
 283         match input {
 284             "Unknown" => Some(Script::Unknown),
 285             "Common" => Some(Script::Common),
 286             "Inherited" => Some(Script::Inherited),
 287 """)
 288     for script in script_list:
 289         f.write("            \"%s\" => Some(Script::%s),\n" % (longforms[script], longforms[script]))
 290     f.write("            _ => None,\n" )
 291     f.write("""        }
 292     }
 293 """)
 294
 295     # Generate impl of `inner_short_name`
 296     f.write("""
 297     #[inline]
 298     pub(crate) fn inner_short_name(self) -> &'static str {
 299         match self {
 300             Script::Unknown => "",
 301             Script::Common => "Zyyy",
 302             Script::Inherited => "Zinh",
 303 """)
 304     for script in script_list:
 305         f.write("            Script::%s => \"%s\",\n" % (longforms[script], script))
 306     f.write("""        }
 307     }
 308 """)
 309
 310     # Generate impl of `inner_from_short_name`
 311     f.write("""
 312     #[inline]
 313     pub(crate) fn inner_from_short_name(input: &str) -> Option<Self> {
 314         match input {
 315             "Zyyy" => Some(Script::Common),
 316             "Zinh" => Some(Script::Inherited),
 317 """)
 318     for script in script_list:
 319         f.write("            \"%s\" => Some(Script::%s),\n" % (script, longforms[script]))
 320     f.write("""        _ => None,\n""")
 321     f.write("""        }
 322     }
 323 """)
 324
 325     # Generate impl of `for_integer`
 326     f.write("""
 327     #[inline]
 328     pub(crate) fn for_integer(value: u8) -> Self {
 329         match value {
 330 """)
 331     for (i, script) in enumerate(script_list):
 332         f.write("            %s => Script::%s,\n" % (i, longforms[script]))
 333     f.write("""            _ => unreachable!(),
 334         }
 335     }
 336 """)
 337
 338     # Close `impl Script` section
 339     f.write("""
 340 }
 341 """)
 342
 343 def extension_name(ext):
 344     """Get the rust source for a given ScriptExtension"""
 345     return "script_extensions::%s" % "_".join([e.upper() for e in ext])
 346
 347
 348 if __name__ == "__main__":
 349     r = "tables.rs"
 350     if os.path.exists(r):
 351         os.remove(r)
 352     with open(r, "w") as rf:
 353         # write the file's preamble
 354         rf.write(preamble)
 355         rf.write("""
 356 /// The version of [Unicode](http://www.unicode.org/)
 357 /// that this version of unicode-script is based on.
 358 pub const UNICODE_VERSION: (u64, u64, u64) = (%s, %s, %s);
 359 """ % UNICODE_VERSION)
 360
 361
 362         (longforms, shortforms) = aliases()
 363
 364         scripts = load_properties("Scripts.txt", [])
 365
 366         script_table = []
 367         script_list = []
 368
 369         for script in scripts:
 370             if script not in ["Common", "Unknown", "Inherited"]:
 371                 script_list.append(shortforms[script])
 372             script_table.extend([(x, y, shortforms[script]) for (x, y) in scripts[script]])
 373         script_list.sort()
 374         script_table.sort(key=lambda w: w[0])
 375
 376
 377         extensions = load_properties("ScriptExtensions.txt", [])
 378         extension_table = []
 379         extension_list = []
 380
 381         for ext in extensions:
 382             split = ext.split(" ")
 383             split.sort()
 384             output_ext = [ext]
 385             if len(split) > 1:
 386                 extension_list.append(split)
 387                 output_ext = split
 388             extension_table.extend([(x, y, output_ext) for (x, y) in extensions[ext]])
 389         extension_table.sort(key=lambda w: w[0])
 390
 391
 392         emit_enums(rf, script_list, extension_list, longforms)
 393         emit_search(rf)
 394
 395         emit_table(rf, "SCRIPTS", script_table, t_type = "&'static [(char, char, Script)]",
 396                    is_pub=False , pfun=lambda x: "(%s,%s, Script::%s)" % (escape_char(x[0]), escape_char(x[1]), longforms[x[2]]))
 397         emit_table(rf, "SCRIPT_EXTENSIONS", extension_table, t_type = "&'static [(char, char, ScriptExtension)]",
 398                    is_pub=False , pfun=lambda x: "(%s,%s,%s)" % (escape_char(x[0]), escape_char(x[1]), extension_name(x[2])))
 399
 400         # emit_table(rf, "FOObar", properties)
 401
 402         rf.write(ending)