src/libcore/char.rs

   1 // Copyright 2012-2014 The Rust Project Developers. See the COPYRIGHT
   2 // file at the top-level directory of this distribution and at
   3 // http://rust-lang.org/COPYRIGHT.
   4 //
   5 // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
   6 // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
   7 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
   8 // option. This file may not be copied, modified, or distributed
   9 // except according to those terms.
  10
  11 //! Character manipulation.
  12 //!
  13 //! For more details, see ::rustc_unicode::char (a.k.a. std::char)
  14
  15 #![allow(non_snake_case)]
  16 #![stable(feature = "core_char", since = "1.2.0")]
  17
  18 use iter::Iterator;
  19 use mem::transmute;
  20 use option::Option::{None, Some};
  21 use option::Option;
  22 use slice::SliceExt;
  23
  24 // UTF-8 ranges and tags for encoding characters
  25 const TAG_CONT: u8    = 0b1000_0000;
  26 const TAG_TWO_B: u8   = 0b1100_0000;
  27 const TAG_THREE_B: u8 = 0b1110_0000;
  28 const TAG_FOUR_B: u8  = 0b1111_0000;
  29 const MAX_ONE_B: u32   =     0x80;
  30 const MAX_TWO_B: u32   =    0x800;
  31 const MAX_THREE_B: u32 =  0x10000;
  32
  33 /*
  34     Lu  Uppercase_Letter        an uppercase letter
  35     Ll  Lowercase_Letter        a lowercase letter
  36     Lt  Titlecase_Letter        a digraphic character, with first part uppercase
  37     Lm  Modifier_Letter         a modifier letter
  38     Lo  Other_Letter            other letters, including syllables and ideographs
  39     Mn  Nonspacing_Mark         a nonspacing combining mark (zero advance width)
  40     Mc  Spacing_Mark            a spacing combining mark (positive advance width)
  41     Me  Enclosing_Mark          an enclosing combining mark
  42     Nd  Decimal_Number          a decimal digit
  43     Nl  Letter_Number           a letterlike numeric character
  44     No  Other_Number            a numeric character of other type
  45     Pc  Connector_Punctuation   a connecting punctuation mark, like a tie
  46     Pd  Dash_Punctuation        a dash or hyphen punctuation mark
  47     Ps  Open_Punctuation        an opening punctuation mark (of a pair)
  48     Pe  Close_Punctuation       a closing punctuation mark (of a pair)
  49     Pi  Initial_Punctuation     an initial quotation mark
  50     Pf  Final_Punctuation       a final quotation mark
  51     Po  Other_Punctuation       a punctuation mark of other type
  52     Sm  Math_Symbol             a symbol of primarily mathematical use
  53     Sc  Currency_Symbol         a currency sign
  54     Sk  Modifier_Symbol         a non-letterlike modifier symbol
  55     So  Other_Symbol            a symbol of other type
  56     Zs  Space_Separator         a space character (of various non-zero widths)
  57     Zl  Line_Separator          U+2028 LINE SEPARATOR only
  58     Zp  Paragraph_Separator     U+2029 PARAGRAPH SEPARATOR only
  59     Cc  Control                 a C0 or C1 control code
  60     Cf  Format                  a format control character
  61     Cs  Surrogate               a surrogate code point
  62     Co  Private_Use             a private-use character
  63     Cn  Unassigned              a reserved unassigned code point or a noncharacter
  64 */
  65
  66 /// The highest valid code point a `char` can have.
  67 ///
  68 /// A [`char`] is a [Unicode Scalar Value], which means that it is a [Code
  69 /// Point], but only ones within a certain range. `MAX` is the highest valid
  70 /// code point that's a valid [Unicode Scalar Value].
  71 ///
  72 /// [`char`]: primitive.char.html
  73 /// [Unicode Scalar Value]: http://www.unicode.org/glossary/#unicode_scalar_value
  74 /// [Code Point]: http://www.unicode.org/glossary/#code_point
  75 #[stable(feature = "rust1", since = "1.0.0")]
  76 pub const MAX: char = '\u{10ffff}';
  77
  78 /// Converts a `u32` to a `char`.
  79 ///
  80 /// Note that all [`char`]s are valid [`u32`]s, and can be casted to one with
  81 /// [`as`]:
  82 ///
  83 /// ```
  84 /// let c = '💯';
  85 /// let i = c as u32;
  86 ///
  87 /// assert_eq!(128175, i);
  88 /// ```
  89 ///
  90 /// However, the reverse is not true: not all valid [`u32`]s are valid
  91 /// [`char`]s. `from_u32()` will return `None` if the input is not a valid value
  92 /// for a [`char`].
  93 ///
  94 /// [`char`]: primitive.char.html
  95 /// [`u32`]: primitive.u32.html
  96 /// [`as`]: ../book/casting-between-types.html#as
  97 ///
  98 /// For an unsafe version of this function which ignores these checks, see
  99 /// [`from_u32_unchecked()`].
 100 ///
 101 /// [`from_u32_unchecked()`]: fn.from_u32_unchecked.html
 102 ///
 103 /// # Examples
 104 ///
 105 /// Basic usage:
 106 ///
 107 /// ```
 108 /// use std::char;
 109 ///
 110 /// let c = char::from_u32(0x2764);
 111 ///
 112 /// assert_eq!(Some('❤'), c);
 113 /// ```
 114 ///
 115 /// Returning `None` when the input is not a valid [`char`]:
 116 ///
 117 /// ```
 118 /// use std::char;
 119 ///
 120 /// let c = char::from_u32(0x110000);
 121 ///
 122 /// assert_eq!(None, c);
 123 /// ```
 124 #[inline]
 125 #[stable(feature = "rust1", since = "1.0.0")]
 126 pub fn from_u32(i: u32) -> Option<char> {
 127     // catch out-of-bounds and surrogates
 128     if (i > MAX as u32) || (i >= 0xD800 && i <= 0xDFFF) {
 129         None
 130     } else {
 131         Some(unsafe { from_u32_unchecked(i) })
 132     }
 133 }
 134
 135 /// Converts a `u32` to a `char`, ignoring validity.
 136 ///
 137 /// Note that all [`char`]s are valid [`u32`]s, and can be casted to one with
 138 /// [`as`]:
 139 ///
 140 /// ```
 141 /// let c = '💯';
 142 /// let i = c as u32;
 143 ///
 144 /// assert_eq!(128175, i);
 145 /// ```
 146 ///
 147 /// However, the reverse is not true: not all valid [`u32`]s are valid
 148 /// [`char`]s. `from_u32_unchecked()` will ignore this, and blindly cast to
 149 /// [`char`], possibly creating an invalid one.
 150 ///
 151 /// [`char`]: primitive.char.html
 152 /// [`u32`]: primitive.u32.html
 153 /// [`as`]: ../book/casting-between-types.html#as
 154 ///
 155 /// # Safety
 156 ///
 157 /// This function is unsafe, as it may construct invalid `char` values.
 158 ///
 159 /// For a safe version of this function, see the [`from_u32()`] function.
 160 ///
 161 /// [`from_u32()`]: fn.from_u32.html
 162 ///
 163 /// # Examples
 164 ///
 165 /// Basic usage:
 166 ///
 167 /// ```
 168 /// use std::char;
 169 ///
 170 /// let c = unsafe { char::from_u32_unchecked(0x2764) };
 171 ///
 172 /// assert_eq!('❤', c);
 173 /// ```
 174 #[inline]
 175 #[stable(feature = "char_from_unchecked", since = "1.5.0")]
 176 pub unsafe fn from_u32_unchecked(i: u32) -> char {
 177     transmute(i)
 178 }
 179
 180 /// Converts a digit in the given radix to a `char`.
 181 ///
 182 /// A 'radix' here is sometimes also called a 'base'. A radix of two
 183 /// indicates a binary number, a radix of ten, decimal, and a radix of
 184 /// sixteen, hexicdecimal, to give some common values. Arbitrary
 185 /// radicum are supported.
 186 ///
 187 /// `from_digit()` will return `None` if the input is not a digit in
 188 /// the given radix.
 189 ///
 190 /// # Panics
 191 ///
 192 /// Panics if given a radix larger than 36.
 193 ///
 194 /// # Examples
 195 ///
 196 /// Basic usage:
 197 ///
 198 /// ```
 199 /// use std::char;
 200 ///
 201 /// let c = char::from_digit(4, 10);
 202 ///
 203 /// assert_eq!(Some('4'), c);
 204 ///
 205 /// // Decimal 11 is a single digit in base 16
 206 /// let c = char::from_digit(11, 16);
 207 ///
 208 /// assert_eq!(Some('b'), c);
 209 /// ```
 210 ///
 211 /// Returning `None` when the input is not a digit:
 212 ///
 213 /// ```
 214 /// use std::char;
 215 ///
 216 /// let c = char::from_digit(20, 10);
 217 ///
 218 /// assert_eq!(None, c);
 219 /// ```
 220 ///
 221 /// Passing a large radix, causing a panic:
 222 ///
 223 /// ```
 224 /// use std::thread;
 225 /// use std::char;
 226 ///
 227 /// let result = thread::spawn(|| {
 228 ///     // this panics
 229 ///     let c = char::from_digit(1, 37);
 230 /// }).join();
 231 ///
 232 /// assert!(result.is_err());
 233 /// ```
 234 #[inline]
 235 #[stable(feature = "rust1", since = "1.0.0")]
 236 pub fn from_digit(num: u32, radix: u32) -> Option<char> {
 237     if radix > 36 {
 238         panic!("from_digit: radix is too high (maximum 36)");
 239     }
 240     if num < radix {
 241         let num = num as u8;
 242         if num < 10 {
 243             Some((b'0' + num) as char)
 244         } else {
 245             Some((b'a' + num - 10) as char)
 246         }
 247     } else {
 248         None
 249     }
 250 }
 251
 252 // NB: the stabilization and documentation for this trait is in
 253 // unicode/char.rs, not here
 254 #[allow(missing_docs)] // docs in libunicode/u_char.rs
 255 #[doc(hidden)]
 256 #[unstable(feature = "core_char_ext",
 257            reason = "the stable interface is `impl char` in later crate",
 258            issue = "27701")]
 259 pub trait CharExt {
 260     #[stable(feature = "core", since = "1.6.0")]
 261     fn is_digit(self, radix: u32) -> bool;
 262     #[stable(feature = "core", since = "1.6.0")]
 263     fn to_digit(self, radix: u32) -> Option<u32>;
 264     #[stable(feature = "core", since = "1.6.0")]
 265     fn escape_unicode(self) -> EscapeUnicode;
 266     #[stable(feature = "core", since = "1.6.0")]
 267     fn escape_default(self) -> EscapeDefault;
 268     #[stable(feature = "core", since = "1.6.0")]
 269     fn len_utf8(self) -> usize;
 270     #[stable(feature = "core", since = "1.6.0")]
 271     fn len_utf16(self) -> usize;
 272     #[stable(feature = "core", since = "1.6.0")]
 273     fn encode_utf8(self, dst: &mut [u8]) -> Option<usize>;
 274     #[stable(feature = "core", since = "1.6.0")]
 275     fn encode_utf16(self, dst: &mut [u16]) -> Option<usize>;
 276 }
 277
 278 #[stable(feature = "core", since = "1.6.0")]
 279 impl CharExt for char {
 280     #[inline]
 281     fn is_digit(self, radix: u32) -> bool {
 282         self.to_digit(radix).is_some()
 283     }
 284
 285     #[inline]
 286     fn to_digit(self, radix: u32) -> Option<u32> {
 287         if radix > 36 {
 288             panic!("to_digit: radix is too high (maximum 36)");
 289         }
 290         let val = match self {
 291           '0' ... '9' => self as u32 - '0' as u32,
 292           'a' ... 'z' => self as u32 - 'a' as u32 + 10,
 293           'A' ... 'Z' => self as u32 - 'A' as u32 + 10,
 294           _ => return None,
 295         };
 296         if val < radix { Some(val) }
 297         else { None }
 298     }
 299
 300     #[inline]
 301     fn escape_unicode(self) -> EscapeUnicode {
 302         EscapeUnicode { c: self, state: EscapeUnicodeState::Backslash }
 303     }
 304
 305     #[inline]
 306     fn escape_default(self) -> EscapeDefault {
 307         let init_state = match self {
 308             '\t' => EscapeDefaultState::Backslash('t'),
 309             '\r' => EscapeDefaultState::Backslash('r'),
 310             '\n' => EscapeDefaultState::Backslash('n'),
 311             '\\' | '\'' | '"' => EscapeDefaultState::Backslash(self),
 312             '\x20' ... '\x7e' => EscapeDefaultState::Char(self),
 313             _ => EscapeDefaultState::Unicode(self.escape_unicode())
 314         };
 315         EscapeDefault { state: init_state }
 316     }
 317
 318     #[inline]
 319     fn len_utf8(self) -> usize {
 320         let code = self as u32;
 321         if code < MAX_ONE_B {
 322             1
 323         } else if code < MAX_TWO_B {
 324             2
 325         } else if code < MAX_THREE_B {
 326             3
 327         } else {
 328             4
 329         }
 330     }
 331
 332     #[inline]
 333     fn len_utf16(self) -> usize {
 334         let ch = self as u32;
 335         if (ch & 0xFFFF) == ch { 1 } else { 2 }
 336     }
 337
 338     #[inline]
 339     fn encode_utf8(self, dst: &mut [u8]) -> Option<usize> {
 340         encode_utf8_raw(self as u32, dst)
 341     }
 342
 343     #[inline]
 344     fn encode_utf16(self, dst: &mut [u16]) -> Option<usize> {
 345         encode_utf16_raw(self as u32, dst)
 346     }
 347 }
 348
 349 /// Encodes a raw u32 value as UTF-8 into the provided byte buffer,
 350 /// and then returns the number of bytes written.
 351 ///
 352 /// If the buffer is not large enough, nothing will be written into it
 353 /// and a `None` will be returned.
 354 #[inline]
 355 #[unstable(feature = "char_internals",
 356            reason = "this function should not be exposed publicly",
 357            issue = "0")]
 358 #[doc(hidden)]
 359 pub fn encode_utf8_raw(code: u32, dst: &mut [u8]) -> Option<usize> {
 360     // Marked #[inline] to allow llvm optimizing it away
 361     if code < MAX_ONE_B && !dst.is_empty() {
 362         dst[0] = code as u8;
 363         Some(1)
 364     } else if code < MAX_TWO_B && dst.len() >= 2 {
 365         dst[0] = (code >> 6 & 0x1F) as u8 | TAG_TWO_B;
 366         dst[1] = (code & 0x3F) as u8 | TAG_CONT;
 367         Some(2)
 368     } else if code < MAX_THREE_B && dst.len() >= 3  {
 369         dst[0] = (code >> 12 & 0x0F) as u8 | TAG_THREE_B;
 370         dst[1] = (code >>  6 & 0x3F) as u8 | TAG_CONT;
 371         dst[2] = (code & 0x3F) as u8 | TAG_CONT;
 372         Some(3)
 373     } else if dst.len() >= 4 {
 374         dst[0] = (code >> 18 & 0x07) as u8 | TAG_FOUR_B;
 375         dst[1] = (code >> 12 & 0x3F) as u8 | TAG_CONT;
 376         dst[2] = (code >>  6 & 0x3F) as u8 | TAG_CONT;
 377         dst[3] = (code & 0x3F) as u8 | TAG_CONT;
 378         Some(4)
 379     } else {
 380         None
 381     }
 382 }
 383
 384 /// Encodes a raw u32 value as UTF-16 into the provided `u16` buffer,
 385 /// and then returns the number of `u16`s written.
 386 ///
 387 /// If the buffer is not large enough, nothing will be written into it
 388 /// and a `None` will be returned.
 389 #[inline]
 390 #[unstable(feature = "char_internals",
 391            reason = "this function should not be exposed publicly",
 392            issue = "0")]
 393 #[doc(hidden)]
 394 pub fn encode_utf16_raw(mut ch: u32, dst: &mut [u16]) -> Option<usize> {
 395     // Marked #[inline] to allow llvm optimizing it away
 396     if (ch & 0xFFFF) == ch && !dst.is_empty() {
 397         // The BMP falls through (assuming non-surrogate, as it should)
 398         dst[0] = ch as u16;
 399         Some(1)
 400     } else if dst.len() >= 2 {
 401         // Supplementary planes break into surrogates.
 402         ch -= 0x1_0000;
 403         dst[0] = 0xD800 | ((ch >> 10) as u16);
 404         dst[1] = 0xDC00 | ((ch as u16) & 0x3FF);
 405         Some(2)
 406     } else {
 407         None
 408     }
 409 }
 410
 411 /// Returns an iterator that yields the hexadecimal Unicode escape of a
 412 /// character, as `char`s.
 413 ///
 414 /// This `struct` is created by the [`escape_unicode()`] method on [`char`]. See
 415 /// its documentation for more.
 416 ///
 417 /// [`escape_unicode()`]: primitive.char.html#method.escape_unicode
 418 /// [`char`]: primitive.char.html
 419 #[derive(Clone)]
 420 #[stable(feature = "rust1", since = "1.0.0")]
 421 pub struct EscapeUnicode {
 422     c: char,
 423     state: EscapeUnicodeState
 424 }
 425
 426 #[derive(Clone)]
 427 enum EscapeUnicodeState {
 428     Backslash,
 429     Type,
 430     LeftBrace,
 431     Value(usize),
 432     RightBrace,
 433     Done,
 434 }
 435
 436 #[stable(feature = "rust1", since = "1.0.0")]
 437 impl Iterator for EscapeUnicode {
 438     type Item = char;
 439
 440     fn next(&mut self) -> Option<char> {
 441         match self.state {
 442             EscapeUnicodeState::Backslash => {
 443                 self.state = EscapeUnicodeState::Type;
 444                 Some('\\')
 445             }
 446             EscapeUnicodeState::Type => {
 447                 self.state = EscapeUnicodeState::LeftBrace;
 448                 Some('u')
 449             }
 450             EscapeUnicodeState::LeftBrace => {
 451                 let mut n = 0;
 452                 while (self.c as u32) >> (4 * (n + 1)) != 0 {
 453                     n += 1;
 454                 }
 455                 self.state = EscapeUnicodeState::Value(n);
 456                 Some('{')
 457             }
 458             EscapeUnicodeState::Value(offset) => {
 459                 let c = from_digit(((self.c as u32) >> (offset * 4)) & 0xf, 16).unwrap();
 460                 if offset == 0 {
 461                     self.state = EscapeUnicodeState::RightBrace;
 462                 } else {
 463                     self.state = EscapeUnicodeState::Value(offset - 1);
 464                 }
 465                 Some(c)
 466             }
 467             EscapeUnicodeState::RightBrace => {
 468                 self.state = EscapeUnicodeState::Done;
 469                 Some('}')
 470             }
 471             EscapeUnicodeState::Done => None,
 472         }
 473     }
 474
 475     fn size_hint(&self) -> (usize, Option<usize>) {
 476         let mut n = 0;
 477         while (self.c as usize) >> (4 * (n + 1)) != 0 {
 478             n += 1;
 479         }
 480         let n = match self.state {
 481             EscapeUnicodeState::Backslash => n + 5,
 482             EscapeUnicodeState::Type => n + 4,
 483             EscapeUnicodeState::LeftBrace => n + 3,
 484             EscapeUnicodeState::Value(offset) => offset + 2,
 485             EscapeUnicodeState::RightBrace => 1,
 486             EscapeUnicodeState::Done => 0,
 487         };
 488         (n, Some(n))
 489     }
 490 }
 491
 492 /// An iterator that yields the literal escape code of a `char`.
 493 ///
 494 /// This `struct` is created by the [`escape_default()`] method on [`char`]. See
 495 /// its documentation for more.
 496 ///
 497 /// [`escape_default()`]: primitive.char.html#method.escape_default
 498 /// [`char`]: primitive.char.html
 499 #[derive(Clone)]
 500 #[stable(feature = "rust1", since = "1.0.0")]
 501 pub struct EscapeDefault {
 502     state: EscapeDefaultState
 503 }
 504
 505 #[derive(Clone)]
 506 enum EscapeDefaultState {
 507     Backslash(char),
 508     Char(char),
 509     Done,
 510     Unicode(EscapeUnicode),
 511 }
 512
 513 #[stable(feature = "rust1", since = "1.0.0")]
 514 impl Iterator for EscapeDefault {
 515     type Item = char;
 516
 517     fn next(&mut self) -> Option<char> {
 518         match self.state {
 519             EscapeDefaultState::Backslash(c) => {
 520                 self.state = EscapeDefaultState::Char(c);
 521                 Some('\\')
 522             }
 523             EscapeDefaultState::Char(c) => {
 524                 self.state = EscapeDefaultState::Done;
 525                 Some(c)
 526             }
 527             EscapeDefaultState::Done => None,
 528             EscapeDefaultState::Unicode(ref mut iter) => iter.next(),
 529         }
 530     }
 531
 532     fn size_hint(&self) -> (usize, Option<usize>) {
 533         match self.state {
 534             EscapeDefaultState::Char(_) => (1, Some(1)),
 535             EscapeDefaultState::Backslash(_) => (2, Some(2)),
 536             EscapeDefaultState::Unicode(ref iter) => iter.size_hint(),
 537             EscapeDefaultState::Done => (0, Some(0)),
 538         }
 539     }
 540 }