BaseTools:Coding problems caused by special characters

BZ:https://bugzilla.tianocore.org/show_bug.cgi?id=1670

During BaseTools compiling under Chinese or Japanese
language Windows, python exception occurring.
UnicodeDecodeError: 'ascii' codec can't decode byte 0xbd
in position 3528: ordinal not in range(128)

Cc: Bob Feng <bob.c.feng@intel.com>
Cc: Liming Gao <liming.gao@intel.com>
Contributed-under: TianoCore Contribution Agreement 1.1
Signed-off-by: Zhiju.Fan <zhijux.fan@intel.com>
Reviewed-by: Bob Feng <bob.c.feng@intel.com>

diff --git a/BaseTools/Source/C/Makefiles/NmakeSubdirs.py b/BaseTools/Source/C/Makefiles/NmakeSubdirs.py
index 29bb5dfa7261cca41c2c46e38202e3ec493c551c..abcbea89a46c91a720282579b74624bd6561c9a8 100644 (file)
--- a/BaseTools/Source/C/Makefiles/NmakeSubdirs.py
+++ b/BaseTools/Source/C/Makefiles/NmakeSubdirs.py
@@ -44,7 +44,7 @@ def RunCommand(WorkDir=None, *Args, **kwargs):
     stdout, stderr = p.communicate()\r
     message = ""\r
     if stdout is not None:\r
-        message = stdout.decode() #for compatibility in python 2 and 3\r
+        message = stdout.decode(encoding='utf-8', errors='ignore') #for compatibility in python 2 and 3\r
 \r
     if p.returncode != 0:\r
         raise RuntimeError("Error while execute command \'{0}\' in direcotry {1}\n{2}".format(" ".join(Args), WorkDir, message))\r

diff --git a/BaseTools/Source/Python/Capsule/GenerateCapsule.py b/BaseTools/Source/Python/Capsule/GenerateCapsule.py
index 7b08918857d8c7a2c5f8952e4a5d3b66c3562413..5ce5daef685cd96a3c185e14215b76e173fd7778 100644 (file)
--- a/BaseTools/Source/Python/Capsule/GenerateCapsule.py
+++ b/BaseTools/Source/Python/Capsule/GenerateCapsule.py
@@ -94,7 +94,7 @@ def SignPayloadSignTool (Payload, ToolPath, PfxFile):
 \r
     if Process.returncode != 0:\r
         shutil.rmtree (TempDirectoryName)\r
-        print (Result[1].decode())\r
+        print (Result[1].decode(encoding='utf-8', errors='ignore'))\r
         raise ValueError ('GenerateCapsule: error: signtool failed.')\r
 \r
     #\r
@@ -132,12 +132,12 @@ def SignPayloadOpenSsl (Payload, ToolPath, SignerPrivateCertFile, OtherPublicCer
     try:\r
         Process = subprocess.Popen (Command, stdin = subprocess.PIPE, stdout = subprocess.PIPE, stderr = subprocess.PIPE, shell = True)\r
         Result = Process.communicate(input = Payload)\r
-        Signature = Result[0]\r
+        Signature = Result[0].decode(encoding='utf-8', errors='ignore')\r
     except:\r
         raise ValueError ('GenerateCapsule: error: can not run openssl.')\r
 \r
     if Process.returncode != 0:\r
-        print (Result[1].decode())\r
+        print (Result[1].decode(encoding='utf-8', errors='ignore'))\r
         raise ValueError ('GenerateCapsule: error: openssl failed.')\r
 \r
     return Signature\r
@@ -186,7 +186,7 @@ def VerifyPayloadOpenSsl (Payload, CertData, ToolPath, SignerPrivateCertFile, Ot
 \r
     if Process.returncode != 0:\r
         shutil.rmtree (TempDirectoryName)\r
-        print (Result[1].decode())\r
+        print (Result[1].decode(encoding='utf-8', errors='ignore'))\r
         raise ValueError ('GenerateCapsule: error: openssl failed.')\r
 \r
     shutil.rmtree (TempDirectoryName)\r

diff --git a/BaseTools/Source/Python/Common/Misc.py b/BaseTools/Source/Python/Common/Misc.py
index c7daf5417cdada9b6113467bfc4f3180dbf5536d..47f8e45222db3c8ad9cc06366620223149508172 100644 (file)
--- a/BaseTools/Source/Python/Common/Misc.py
+++ b/BaseTools/Source/Python/Common/Misc.py
@@ -1032,7 +1032,7 @@ def ParseFieldValue (Value):
             p.stderr.close()\r
         if err:\r
             raise BadExpression("DevicePath: %s" % str(err))\r
-        out = out.decode()\r
+        out = out.decode(encoding='utf-8', errors='ignore')\r
         Size = len(out.split())\r
         out = ','.join(out.split())\r
         return '{' + out + '}', Size\r

diff --git a/BaseTools/Source/Python/Pkcs7Sign/Pkcs7Sign.py b/BaseTools/Source/Python/Pkcs7Sign/Pkcs7Sign.py
index 003f052a9026634312f06c4846ac9c612a221672..706bf28e14ae218de6d038d4b6241fd3215f4061 100644 (file)
--- a/BaseTools/Source/Python/Pkcs7Sign/Pkcs7Sign.py
+++ b/BaseTools/Source/Python/Pkcs7Sign/Pkcs7Sign.py
@@ -122,7 +122,7 @@ if __name__ == '__main__':
   if Process.returncode != 0:\r
     print('ERROR: Open SSL command not available.  Please verify PATH or set OPENSSL_PATH')\r
     sys.exit(Process.returncode)\r
-  print(Version[0].decode())\r
+  print(Version[0].decode(encoding='utf-8', errors='ignore'))\r
 \r
   #\r
   # Read input file into a buffer and save input filename\r

diff --git a/BaseTools/Source/Python/Rsa2048Sha256Sign/Rsa2048Sha256GenerateKeys.py b/BaseTools/Source/Python/Rsa2048Sha256Sign/Rsa2048Sha256GenerateKeys.py
index c0b661d03c977124ca83fd16105ee3cc075f8353..3cc14e3bb9bff97e7c81f81194e0f200d19b7188 100644 (file)
--- a/BaseTools/Source/Python/Rsa2048Sha256Sign/Rsa2048Sha256GenerateKeys.py
+++ b/BaseTools/Source/Python/Rsa2048Sha256Sign/Rsa2048Sha256GenerateKeys.py
@@ -84,7 +84,7 @@ if __name__ == '__main__':
   if Process.returncode != 0:\r
     print('ERROR: Open SSL command not available.  Please verify PATH or set OPENSSL_PATH')\r
     sys.exit(Process.returncode)\r
-  print(Version[0].decode())\r
+  print(Version[0].decode(encoding='utf-8', errors='ignore'))\r
 \r
   args.PemFileName = []\r
 \r
@@ -125,7 +125,7 @@ if __name__ == '__main__':
     # Extract public key from private key into STDOUT\r
     #\r
     Process = subprocess.Popen('%s rsa -in %s -modulus -noout' % (OpenSslCommand, Item), stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)\r
-    PublicKeyHexString = Process.communicate()[0].split(b'=')[1].strip()\r
+    PublicKeyHexString = Process.communicate()[0].decode(encoding='utf-8', errors='ignore').split(b'=')[1].strip()\r
     if Process.returncode != 0:\r
       print('ERROR: Unable to extract public key from private key')\r
       sys.exit(Process.returncode)\r
@@ -138,7 +138,7 @@ if __name__ == '__main__':
     #\r
     Process = subprocess.Popen('%s dgst -sha256 -binary' % (OpenSslCommand), stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)\r
     Process.stdin.write (PublicKey)\r
-    PublicKeyHash = PublicKeyHash + Process.communicate()[0]\r
+    PublicKeyHash = PublicKeyHash + Process.communicate()[0].decode(encoding='utf-8', errors='ignore')\r
     if Process.returncode != 0:\r
       print('ERROR: Unable to extract SHA 256 hash of public key')\r
       sys.exit(Process.returncode)\r