BaseTools: Fix flexible PCD single quote and double quote bugs

1.The " and ' inside the string, must use escape character format
  (\", \')
2.'string' and L'string' format in --pcd, it must be double quoted
  first.

  Some examples that to match --pcd format and DSC format
  --pcd                             DSC format
  L"ABC"                            L"ABC"
  "AB\\\"C"                         "AB\"C"
  "AB\\\'C"                         "AB\'C"
  L"\'AB\\\"C\'"                    L'AB\"C'
  "\'AB\\\'C\'"                     'AB\'C'
  H"{0, L\"AB\\\"B\", \'ab\\\"c\'}" {0, L"AB\"B", 'ab\"c'}

Cc: Liming Gao <liming.gao@intel.com>
Cc: Yonghong Zhu <yonghong.zhu@intel.com>
Contributed-under: TianoCore Contribution Agreement 1.1
Signed-off-by: Yunhua Feng <yunhuax.feng@intel.com>
Reviewed-by: Yonghong Zhu <yonghong.zhu@intel.com>

diff --git a/BaseTools/Source/Python/AutoGen/GenMake.py b/BaseTools/Source/Python/AutoGen/GenMake.py
index afe6f2f99c47bee78e8bc76e426c4d5b98f551cd..4b924d21e099b0bee55554cd8a8194a95b12c3e6 100644 (file)
--- a/BaseTools/Source/Python/AutoGen/GenMake.py
+++ b/BaseTools/Source/Python/AutoGen/GenMake.py
@@ -1555,13 +1555,19 @@ class TopLevelMakefile(BuildFile):
             for index, option in enumerate(GlobalData.gCommand):\r
                 if "--pcd" == option and GlobalData.gCommand[index+1]:\r
                     pcdName, pcdValue = GlobalData.gCommand[index+1].split('=')\r
-                    if pcdValue.startswith('H'):\r
-                        pcdValue = 'H' + '"' + pcdValue[1:] + '"'\r
-                        ExtraOption += " --pcd " + pcdName + '=' + pcdValue\r
-                    elif pcdValue.startswith("L'"):\r
-                        ExtraOption += "--pcd " + pcdName + '=' + pcdValue\r
-                    elif pcdValue.startswith('L'):\r
-                        pcdValue = 'L' + '"' + pcdValue[1:] + '"'\r
+                    for Item in GlobalData.BuildOptionPcd:\r
+                        if '.'.join(Item[0:2]) == pcdName:\r
+                            pcdValue = Item[2]\r
+                            if pcdValue.startswith('L') or pcdValue.startswith('"'):\r
+                                pcdValue, Size = ParseFieldValue(pcdValue)\r
+                                NewVal = '{'\r
+                                for S in range(Size):\r
+                                    NewVal = NewVal + '0x%02X' % ((pcdValue >> S * 8) & 0xff)\r
+                                    NewVal += ','\r
+                                pcdValue =  NewVal[:-1] + '}'\r
+                            break\r
+                    if pcdValue.startswith('{'):\r
+                        pcdValue = 'H' + '"' + pcdValue + '"'\r
                         ExtraOption += " --pcd " + pcdName + '=' + pcdValue\r
                     else:\r
                         ExtraOption += " --pcd " + GlobalData.gCommand[index+1]\r

diff --git a/BaseTools/Source/Python/Common/Expression.py b/BaseTools/Source/Python/Common/Expression.py
index edb0a60de6007a367bdf80245fc061d665a3e469..f1516d5c7bb8c9bd39ffcc5dd1eabad2a2ff6d1d 100644 (file)
--- a/BaseTools/Source/Python/Common/Expression.py
+++ b/BaseTools/Source/Python/Common/Expression.py
@@ -45,15 +45,28 @@ ERR_IN_OPERAND          = 'Macro after IN operator can only be: $(FAMILY), $(ARC
 #  For example: abc"de\"f"ghi"jkl"mn will be: ['abc', '"de\"f"', 'ghi', '"jkl"', 'mn']\r
 #\r
 def SplitString(String):\r
-    # There might be escaped quote: "abc\"def\\\"ghi"\r
-    Str = String.replace('\\\\', '//').replace('\\\"', '\\\'')\r
+    # There might be escaped quote: "abc\"def\\\"ghi", 'abc\'def\\\'ghi'\r
+    Str = String\r
     RetList = []\r
-    InQuote = False\r
+    InSingleQuote = False\r
+    InDoubleQuote = False\r
     Item = ''\r
     for i, ch in enumerate(Str):\r
-        if ch == '"':\r
-            InQuote = not InQuote\r
-            if not InQuote:\r
+        if ch == '"' and not InSingleQuote:\r
+            if Str[i - 1] != '\\':\r
+                InDoubleQuote = not InDoubleQuote\r
+            if not InDoubleQuote:\r
+                Item += String[i]\r
+                RetList.append(Item)\r
+                Item = ''\r
+                continue\r
+            if Item:\r
+                RetList.append(Item)\r
+                Item = ''\r
+        elif ch == "'" and not InDoubleQuote:\r
+            if Str[i - 1] != '\\':\r
+                InSingleQuote = not InSingleQuote\r
+            if not InSingleQuote:\r
                 Item += String[i]\r
                 RetList.append(Item)\r
                 Item = ''\r
@@ -62,7 +75,7 @@ def SplitString(String):
                 RetList.append(Item)\r
                 Item = ''\r
         Item += String[i]\r
-    if InQuote:\r
+    if InSingleQuote or InDoubleQuote:\r
         raise BadExpression(ERR_STRING_TOKEN % Item)\r
     if Item:\r
         RetList.append(Item)\r
@@ -483,6 +496,8 @@ class ValueExpression(object):
             Flag = 0\r
             for Index in range(len(self._Token)):\r
                 if self._Token[Index] in ['"']:\r
+                    if self._Token[Index - 1] == '\\':\r
+                        continue\r
                     Flag += 1\r
             if Flag == 2 and self._Token.endswith('"'):\r
                 return True\r
@@ -490,6 +505,8 @@ class ValueExpression(object):
             Flag = 0\r
             for Index in range(len(self._Token)):\r
                 if self._Token[Index] in ["'"]:\r
+                    if self._Token[Index - 1] == '\\':\r
+                        continue\r
                     Flag += 1\r
             if Flag == 2 and self._Token.endswith("'"):\r
                 return True\r
@@ -537,16 +554,25 @@ class ValueExpression(object):
         self._Idx += 1\r
 \r
         # Replace escape \\\", \"\r
-        Expr = self._Expr[self._Idx:].replace('\\\\', '//').replace('\\\"', '\\\'')\r
-        for Ch in Expr:\r
-            self._Idx += 1\r
-            if Ch == '"' or Ch == "'":\r
-                break\r
-        self._Token = self._LiteralToken = self._Expr[Idx:self._Idx]\r
-        if self._Token.startswith('"') and not self._Token.endswith('"'):\r
-            raise BadExpression(ERR_STRING_TOKEN % self._Token)\r
-        if self._Token.startswith("'") and not self._Token.endswith("'"):\r
-            raise BadExpression(ERR_STRING_TOKEN % self._Token)\r
+        if self._Expr[Idx] == '"':\r
+            Expr = self._Expr[self._Idx:].replace('\\\\', '//').replace('\\\"', '\\\'')\r
+            for Ch in Expr:\r
+                self._Idx += 1\r
+                if Ch == '"':\r
+                    break\r
+            self._Token = self._LiteralToken = self._Expr[Idx:self._Idx]\r
+            if not self._Token.endswith('"'):\r
+                raise BadExpression(ERR_STRING_TOKEN % self._Token)\r
+        #Replace escape \\\', \'\r
+        elif self._Expr[Idx] == "'":\r
+            Expr = self._Expr[self._Idx:].replace('\\\\', '//').replace("\\\'", "\\\"")\r
+            for Ch in Expr:\r
+                self._Idx += 1\r
+                if Ch == "'":\r
+                    break\r
+            self._Token = self._LiteralToken = self._Expr[Idx:self._Idx]\r
+            if not self._Token.endswith("'"):\r
+                raise BadExpression(ERR_STRING_TOKEN % self._Token)\r
         self._Token = self._Token[1:-1]\r
         return self._Token\r
 \r

diff --git a/BaseTools/Source/Python/Common/Misc.py b/BaseTools/Source/Python/Common/Misc.py
index 1461d00669c20e54542f954d1bd336fe19a0a495..a7e7797d0499df0a70026c2f6674e83ae0a0054c 100644 (file)
--- a/BaseTools/Source/Python/Common/Misc.py
+++ b/BaseTools/Source/Python/Common/Misc.py
@@ -1443,21 +1443,26 @@ def ParseConsoleLog(Filename):
 \r
 def AnalyzePcdExpression(Setting):\r
     Setting = Setting.strip()\r
-    # There might be escaped quote in a string: \", \\\"\r
-    Data = Setting.replace('\\\\', '//').replace('\\\"', '\\\'')\r
+    # There might be escaped quote in a string: \", \\\" , \', \\\'\r
+    Data = Setting\r
     # There might be '|' in string and in ( ... | ... ), replace it with '-'\r
     NewStr = ''\r
-    InStr = False\r
+    InSingleQuoteStr = False\r
+    InDoubleQuoteStr = False\r
     Pair = 0\r
-    for ch in Data:\r
-        if ch == '"':\r
-            InStr = not InStr\r
-        elif ch == '(' and not InStr:\r
+    for Index, ch in enumerate(Data):\r
+        if ch == '"' and not InSingleQuoteStr:\r
+            if Data[Index - 1] != '\\':\r
+                InDoubleQuoteStr = not InDoubleQuoteStr\r
+        elif ch == "'" and not InDoubleQuoteStr:\r
+            if Data[Index - 1] != '\\':\r
+                InSingleQuoteStr = not InSingleQuoteStr\r
+        elif ch == '(' and not (InSingleQuoteStr or InDoubleQuoteStr):\r
             Pair += 1\r
-        elif ch == ')' and not InStr:\r
+        elif ch == ')' and not (InSingleQuoteStr or InDoubleQuoteStr):\r
             Pair -= 1\r
 \r
-        if (Pair > 0 or InStr) and ch == TAB_VALUE_SPLIT:\r
+        if (Pair > 0 or InSingleQuoteStr or InDoubleQuoteStr) and ch == TAB_VALUE_SPLIT:\r
             NewStr += '-'\r
         else:\r
             NewStr += ch\r
@@ -1549,7 +1554,7 @@ def ParseFieldValue (Value):
         return Value, 16\r
     if Value.startswith('L"') and Value.endswith('"'):\r
         # Unicode String\r
-        List = list(Value[2:-1])\r
+        List = list(eval(Value[1:]))  # translate escape character\r
         List.reverse()\r
         Value = 0\r
         for Char in List:\r
@@ -1557,7 +1562,7 @@ def ParseFieldValue (Value):
         return Value, (len(List) + 1) * 2\r
     if Value.startswith('"') and Value.endswith('"'):\r
         # ASCII String\r
-        List = list(Value[1:-1])\r
+        List = list(eval(Value))  # translate escape character\r
         List.reverse()\r
         Value = 0\r
         for Char in List:\r
@@ -1565,7 +1570,7 @@ def ParseFieldValue (Value):
         return Value, len(List) + 1\r
     if Value.startswith("L'") and Value.endswith("'"):\r
         # Unicode Character Constant\r
-        List = list(Value[2:-1])\r
+        List = list(eval(Value[1:])) # translate escape character\r
         if len(List) == 0:\r
             raise BadExpression('Length %s is %s' % (Value, len(List)))\r
         List.reverse()\r
@@ -1575,7 +1580,7 @@ def ParseFieldValue (Value):
         return Value, len(List) * 2\r
     if Value.startswith("'") and Value.endswith("'"):\r
         # Character constant\r
-        List = list(Value[1:-1])\r
+        List = list(eval(Value)) # translate escape character\r
         if len(List) == 0:\r
             raise BadExpression('Length %s is %s' % (Value, len(List)))\r
         List.reverse()\r

diff --git a/BaseTools/Source/Python/Common/String.py b/BaseTools/Source/Python/Common/String.py
index 4a8c03e88e28297e7c158fc27a420f0851e22d62..5e50beff5cc39539a0eab1684d0281b0533b9918 100644 (file)
--- a/BaseTools/Source/Python/Common/String.py
+++ b/BaseTools/Source/Python/Common/String.py
@@ -45,26 +45,32 @@ def GetSplitValueList(String, SplitTag=DataType.TAB_VALUE_SPLIT, MaxSplit= -1):
     ValueList = []\r
     Last = 0\r
     Escaped = False\r
-    InString = False\r
+    InSingleQuoteString = False\r
+    InDoubleQuoteString = False\r
     InParenthesis = 0\r
     for Index in range(0, len(String)):\r
         Char = String[Index]\r
 \r
         if not Escaped:\r
             # Found a splitter not in a string, split it\r
-            if not InString and InParenthesis == 0 and Char == SplitTag:\r
+            if (not InSingleQuoteString or not InDoubleQuoteString) and InParenthesis == 0 and Char == SplitTag:\r
                 ValueList.append(String[Last:Index].strip())\r
                 Last = Index + 1\r
                 if MaxSplit > 0 and len(ValueList) >= MaxSplit:\r
                     break\r
 \r
-            if Char == '\\' and InString:\r
+            if Char == '\\' and (InSingleQuoteString or InDoubleQuoteString):\r
                 Escaped = True\r
-            elif Char == '"':\r
-                if not InString:\r
-                    InString = True\r
+            elif Char == '"' and not InSingleQuoteString:\r
+                if not InDoubleQuoteString:\r
+                    InDoubleQuoteString = True\r
                 else:\r
-                    InString = False\r
+                    InDoubleQuoteString = False\r
+            elif Char == "'" and not InDoubleQuoteString:\r
+                if not InSingleQuoteString:\r
+                    InSingleQuoteString = True\r
+                else:\r
+                    InSingleQuoteString = False\r
             elif Char == '(':\r
                 InParenthesis = InParenthesis + 1\r
             elif Char == ')':\r
@@ -345,14 +351,17 @@ def CleanString(Line, CommentCharacter=DataType.TAB_COMMENT_SPLIT, AllowCppStyle
     #\r
     # remove comments, but we should escape comment character in string\r
     #\r
-    InString = False\r
+    InDoubleQuoteString = False\r
+    InSingleQuoteString = False\r
     CommentInString = False\r
     for Index in range(0, len(Line)):\r
-        if Line[Index] == '"':\r
-            InString = not InString\r
-        elif Line[Index] == CommentCharacter and InString :\r
+        if Line[Index] == '"' and not InSingleQuoteString:\r
+            InDoubleQuoteString = not InDoubleQuoteString\r
+        elif Line[Index] == "'" and not InDoubleQuoteString:\r
+            InSingleQuoteString = not InSingleQuoteString\r
+        elif Line[Index] == CommentCharacter and (InSingleQuoteString or InDoubleQuoteString):\r
             CommentInString = True\r
-        elif Line[Index] == CommentCharacter and not InString :\r
+        elif Line[Index] == CommentCharacter and not (InSingleQuoteString or InDoubleQuoteString):\r
             Line = Line[0: Index]\r
             break\r
 \r
@@ -402,15 +411,18 @@ def CleanString2(Line, CommentCharacter=DataType.TAB_COMMENT_SPLIT, AllowCppStyl
     #\r
     # separate comments and statements, but we should escape comment character in string\r
     #\r
-    InString = False\r
+    InDoubleQuoteString = False\r
+    InSingleQuoteString = False\r
     CommentInString = False\r
     Comment = ''\r
     for Index in range(0, len(Line)):\r
-        if Line[Index] == '"':\r
-            InString = not InString\r
-        elif Line[Index] == CommentCharacter and InString:\r
+        if Line[Index] == '"' and not InSingleQuoteString:\r
+            InDoubleQuoteString = not InDoubleQuoteString\r
+        elif Line[Index] == "'" and not InDoubleQuoteString:\r
+            InSingleQuoteString = not InSingleQuoteString\r
+        elif Line[Index] == CommentCharacter and (InDoubleQuoteString or InSingleQuoteString):\r
             CommentInString = True\r
-        elif Line[Index] == CommentCharacter and not InString:\r
+        elif Line[Index] == CommentCharacter and not (InDoubleQuoteString or InSingleQuoteString):\r
             Comment = Line[Index:].strip()\r
             Line = Line[0:Index].strip()\r
             break\r

diff --git a/BaseTools/Source/Python/Workspace/DscBuildData.py b/BaseTools/Source/Python/Workspace/DscBuildData.py
index f90da8e5017abf9aece9d62cf24b5d1f35ca5eb7..ea8d1847f757abbad6bf6f1f1bddfc5e973fdfbe 100644 (file)
--- a/BaseTools/Source/Python/Workspace/DscBuildData.py
+++ b/BaseTools/Source/Python/Workspace/DscBuildData.py
@@ -991,6 +991,8 @@ class DscBuildData(PlatformBuildClassObject):
                     NewValue = self.GetFieldValueFromComm(pcdvalue, TokenSpaceGuidCName, TokenCName, FieldName)\r
                     GlobalData.BuildOptionPcd[i] = (TokenSpaceGuidCName, TokenCName, FieldName,NewValue,("build command options",1))\r
                 else:\r
+                    # Replace \' to ', \\\' to \'\r
+                    pcdvalue = pcdvalue.replace("\\\\\\'", '\\\\\\"').replace('\\\'', '\'').replace('\\\\\\"', "\\'")\r
                     for key in self.DecPcds:\r
                         PcdItem = self.DecPcds[key]\r
                         if HasTokenSpace:\r
@@ -1002,7 +1004,7 @@ class DscBuildData(PlatformBuildClassObject):
                                     except BadExpression, Value:\r
                                         EdkLogger.error('Parser', FORMAT_INVALID, 'PCD [%s.%s] Value "%s",  %s' %\r
                                                         (TokenSpaceGuidCName, TokenCName, pcdvalue, Value))\r
-                                    if PcdDatumType == "VOID*":\r
+                                    if PcdDatumType not in [TAB_UINT8, TAB_UINT16, TAB_UINT32, TAB_UINT64, 'BOOLEAN']:\r
                                         pcdvalue = 'H' + pcdvalue\r
                                 elif pcdvalue.startswith("L'"):\r
                                     try:\r
@@ -1010,7 +1012,7 @@ class DscBuildData(PlatformBuildClassObject):
                                     except BadExpression, Value:\r
                                         EdkLogger.error('Parser', FORMAT_INVALID, 'PCD [%s.%s] Value "%s",  %s' %\r
                                                         (TokenSpaceGuidCName, TokenCName, pcdvalue, Value))\r
-                                    if pcdvalue.startswith('{'):\r
+                                    if PcdDatumType not in [TAB_UINT8, TAB_UINT16, TAB_UINT32, TAB_UINT64, 'BOOLEAN']:\r
                                         pcdvalue = 'H' + pcdvalue\r
                                 elif pcdvalue.startswith("'"):\r
                                     try:\r
@@ -1018,7 +1020,7 @@ class DscBuildData(PlatformBuildClassObject):
                                     except BadExpression, Value:\r
                                         EdkLogger.error('Parser', FORMAT_INVALID, 'PCD [%s.%s] Value "%s",  %s' %\r
                                                         (TokenSpaceGuidCName, TokenCName, pcdvalue, Value))\r
-                                    if pcdvalue.startswith('{'):\r
+                                    if PcdDatumType not in [TAB_UINT8, TAB_UINT16, TAB_UINT32, TAB_UINT64, 'BOOLEAN']:\r
                                         pcdvalue = 'H' + pcdvalue\r
                                 elif pcdvalue.startswith('L'):\r
                                     pcdvalue = 'L"' + pcdvalue[1:] + '"'\r
@@ -1031,8 +1033,12 @@ class DscBuildData(PlatformBuildClassObject):
                                     try:\r
                                         pcdvalue = ValueExpressionEx(pcdvalue, PcdDatumType, self._GuidDict)(True)\r
                                     except BadExpression, Value:\r
-                                        EdkLogger.error('Parser', FORMAT_INVALID, 'PCD [%s.%s] Value "%s",  %s' %\r
-                                                        (TokenSpaceGuidCName, TokenCName, pcdvalue, Value))\r
+                                        try:\r
+                                            pcdvalue = '"' + pcdvalue + '"'\r
+                                            pcdvalue = ValueExpressionEx(pcdvalue, PcdDatumType, self._GuidDict)(True)\r
+                                        except BadExpression, Value:\r
+                                            EdkLogger.error('Parser', FORMAT_INVALID, 'PCD [%s.%s] Value "%s",  %s' %\r
+                                                            (TokenSpaceGuidCName, TokenCName, pcdvalue, Value))\r
                                 NewValue = BuildOptionPcdValueFormat(TokenSpaceGuidCName, TokenCName, PcdDatumType, pcdvalue)\r
                                 FoundFlag = True\r
                         else:\r
@@ -1048,7 +1054,7 @@ class DscBuildData(PlatformBuildClassObject):
                                             except BadExpression, Value:\r
                                                 EdkLogger.error('Parser', FORMAT_INVALID, 'PCD [%s.%s] Value "%s", %s' %\r
                                                                 (TokenSpaceGuidCName, TokenCName, pcdvalue, Value))\r
-                                            if PcdDatumType == "VOID*":\r
+                                            if PcdDatumType not in [TAB_UINT8, TAB_UINT16, TAB_UINT32, TAB_UINT64,'BOOLEAN']:\r
                                                 pcdvalue = 'H' + pcdvalue\r
                                         elif pcdvalue.startswith("L'"):\r
                                             try:\r
@@ -1057,7 +1063,7 @@ class DscBuildData(PlatformBuildClassObject):
                                             except BadExpression, Value:\r
                                                 EdkLogger.error('Parser', FORMAT_INVALID, 'PCD [%s.%s] Value "%s",  %s' %\r
                                                                 (TokenSpaceGuidCName, TokenCName, pcdvalue, Value))\r
-                                            if pcdvalue.startswith('{'):\r
+                                            if PcdDatumType not in [TAB_UINT8, TAB_UINT16, TAB_UINT32, TAB_UINT64, 'BOOLEAN']:\r
                                                 pcdvalue = 'H' + pcdvalue\r
                                         elif pcdvalue.startswith("'"):\r
                                             try:\r
@@ -1066,7 +1072,7 @@ class DscBuildData(PlatformBuildClassObject):
                                             except BadExpression, Value:\r
                                                 EdkLogger.error('Parser', FORMAT_INVALID, 'PCD [%s.%s] Value "%s",  %s' %\r
                                                                 (TokenSpaceGuidCName, TokenCName, pcdvalue, Value))\r
-                                            if pcdvalue.startswith('{'):\r
+                                            if PcdDatumType not in [TAB_UINT8, TAB_UINT16, TAB_UINT32, TAB_UINT64, 'BOOLEAN']:\r
                                                 pcdvalue = 'H' + pcdvalue\r
                                         elif pcdvalue.startswith('L'):\r
                                             pcdvalue = 'L"' + pcdvalue[1:] + '"'\r
@@ -1080,9 +1086,12 @@ class DscBuildData(PlatformBuildClassObject):
                                             try:\r
                                                 pcdvalue = ValueExpressionEx(pcdvalue, PcdDatumType, self._GuidDict)(True)\r
                                             except BadExpression, Value:\r
-                                                EdkLogger.error('Parser', FORMAT_INVALID,\r
-                                                                'PCD [%s.%s] Value "%s",  %s' %\r
-                                                                (TokenSpaceGuidCName, TokenCName, pcdvalue, Value))\r
+                                                try:\r
+                                                    pcdvalue = '"' + pcdvalue + '"'\r
+                                                    pcdvalue = ValueExpressionEx(pcdvalue, PcdDatumType, self._GuidDict)(True)\r
+                                                except BadExpression, Value:\r
+                                                    EdkLogger.error('Parser', FORMAT_INVALID, 'PCD [%s.%s] Value "%s",  %s' %\r
+                                                                    (TokenSpaceGuidCName, TokenCName, pcdvalue, Value))\r
                                         NewValue = BuildOptionPcdValueFormat(TokenSpaceGuidCName, TokenCName, PcdDatumType, pcdvalue)\r
                                         FoundFlag = True\r
                                     else:\r