BaseTools/Scripts: Add ConvertUni.py script

This script uses python codecs to convert .uni string files between
the utf-16 and utf-8 formats.

The advantages of utf-8 data:
 * Generally smaller files
 * More commonly supported by editors
 * Not treated as binary data in patch files

The script was tested on MdePkg with both python 2.7 and python 3.4.
It was able to convert all MdePkg .uni files between utf-8 and utf-16
multiple times always producing the same files for each format.

v2:
 * Rename ConvertUtf16ToUtf8.py to ConvertUni.py
 * Also support utf-8 to utf-16 conversion (with --utf-16)

Contributed-under: TianoCore Contribution Agreement 1.0
Signed-off-by: Jordan Justen <jordan.l.justen@intel.com>
Reviewed-by: Jaben Carsey <jaben.carsey@intel.com>
Reviewed-by: Michael Kinney <michael.d.kinney@intel.com>
Reviewed-by: Yonghong Zhu <yonghong.zhu@intel.com>
git-svn-id: https://svn.code.sf.net/p/edk2/code/trunk/edk2@19247 6f19259b-4bc3-4df7-8a09-765794883524

diff --git a/BaseTools/Scripts/ConvertUni.py b/BaseTools/Scripts/ConvertUni.py
new file mode 100755 (executable)
index 0000000..2af55df
--- /dev/null
+++ b/BaseTools/Scripts/ConvertUni.py
@@ -0,0 +1,137 @@
+## @file\r
+#  Check a patch for various format issues\r
+#\r
+#  Copyright (c) 2015, Intel Corporation. All rights reserved.<BR>\r
+#\r
+#  This program and the accompanying materials are licensed and made\r
+#  available under the terms and conditions of the BSD License which\r
+#  accompanies this distribution. The full text of the license may be\r
+#  found at http://opensource.org/licenses/bsd-license.php\r
+#\r
+#  THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS"\r
+#  BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER\r
+#  EXPRESS OR IMPLIED.\r
+#\r
+\r
+from __future__ import print_function\r
+\r
+VersionNumber = '0.1'\r
+__copyright__ = "Copyright (c) 2015, Intel Corporation  All rights reserved."\r
+\r
+import argparse\r
+import codecs\r
+import os\r
+import sys\r
+\r
+try:\r
+    from io import StringIO\r
+except ImportError:\r
+    from StringIO import StringIO\r
+\r
+class ConvertOneArg:\r
+    """Converts utf-16 to utf-8 for one command line argument.\r
+\r
+       This could be a single file, or a directory.\r
+    """\r
+\r
+    def __init__(self, utf8, source):\r
+        self.utf8 = utf8\r
+        self.source = source\r
+\r
+        self.ok = True\r
+\r
+        if not os.path.exists(source):\r
+            self.ok = False\r
+        elif os.path.isdir(source):\r
+            for (root, dirs, files) in os.walk(source):\r
+                files = filter(lambda a: a.endswith('.uni'), files)\r
+                for filename in files:\r
+                    path = os.path.join(root, filename)\r
+                    self.ok &= self.convert_one_file(path)\r
+                    if not self.ok:\r
+                        break\r
+\r
+                if not self.ok:\r
+                    break\r
+        else:\r
+            self.ok &= self.convert_one_file(source)\r
+\r
+    def convert_one_file(self, source):\r
+        if self.utf8:\r
+            new_enc, old_enc = 'utf-8', 'utf-16'\r
+        else:\r
+            new_enc, old_enc = 'utf-16', 'utf-8'\r
+        #\r
+        # Read file\r
+        #\r
+        f = open(source, mode='rb')\r
+        file_content = f.read()\r
+        f.close()\r
+\r
+        #\r
+        # Detect UTF-16 Byte Order Mark at beginning of file.\r
+        #\r
+        bom = (file_content.startswith(codecs.BOM_UTF16_BE) or\r
+               file_content.startswith(codecs.BOM_UTF16_LE))\r
+        if bom != self.utf8:\r
+            print("%s: already %s" % (source, new_enc))\r
+            return True\r
+\r
+        #\r
+        # Decode old string data\r
+        #\r
+        str_content = file_content.decode(old_enc, 'ignore')\r
+\r
+        #\r
+        # Encode new string data\r
+        #\r
+        new_content = str_content.encode(new_enc, 'ignore')\r
+\r
+        #\r
+        # Write converted data back to file\r
+        #\r
+        f = open(source, mode='wb')\r
+        f.write(new_content)\r
+        f.close()\r
+\r
+        print(source + ": converted, size", len(file_content), '=>', len(new_content))\r
+        return True\r
+\r
+\r
+class ConvertUniApp:\r
+    """Converts .uni files between utf-16 and utf-8."""\r
+\r
+    def __init__(self):\r
+        self.parse_options()\r
+        sources = self.args.source\r
+\r
+        self.ok = True\r
+        for patch in sources:\r
+            self.process_one_arg(patch)\r
+\r
+        if self.ok:\r
+            self.retval = 0\r
+        else:\r
+            self.retval = -1\r
+\r
+    def process_one_arg(self, arg):\r
+        self.ok &= ConvertOneArg(self.utf8, arg).ok\r
+\r
+    def parse_options(self):\r
+        parser = argparse.ArgumentParser(description=__copyright__)\r
+        parser.add_argument('--version', action='version',\r
+                            version='%(prog)s ' + VersionNumber)\r
+        parser.add_argument('source', nargs='+',\r
+                            help='[uni file | directory]')\r
+        group = parser.add_mutually_exclusive_group()\r
+        group.add_argument("--utf-8",\r
+                           action="store_true",\r
+                           help="Convert from utf-16 to utf-8 [default]")\r
+        group.add_argument("--utf-16",\r
+                           action="store_true",\r
+                           help="Convert from utf-8 to utf-16")\r
+        self.args = parser.parse_args()\r
+        self.utf8 = not self.args.utf_16\r
+\r
+if __name__ == "__main__":\r
+    sys.exit(ConvertUniApp().retval)\r