-# Copyright (c) 2007, Intel Corporation\r
-# All rights reserved. This program and the accompanying materials\r
+## @file\r
+# This file is used to parse a strings file and create or add to a string database \r
+# file.\r
+#\r
+# Copyright (c) 2007 - 2018, Intel Corporation. All rights reserved.<BR>\r
+# This program and the accompanying materials\r
# are licensed and made available under the terms and conditions of the BSD License\r
# which accompanies this distribution. The full text of the license may be found at\r
# http://opensource.org/licenses/bsd-license.php\r
# THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,\r
# WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.\r
\r
-#\r
-#This file is used to parse a strings file and create or add to a string database file.\r
-#\r
-\r
##\r
# Import Modules\r
#\r
import Common.EdkLogger as EdkLogger\r
from Common.BuildToolError import *\r
from UniClassObject import *\r
+from StringIO import StringIO\r
+from struct import pack, unpack\r
+from Common.LongFilePathSupport import OpenLongFilePath as open\r
\r
##\r
# Static definitions\r
COMMENT_NOT_REFERENCED = ' ' + COMMENT + NOT_REFERENCED\r
CHAR_ARRAY_DEFIN = 'unsigned char'\r
COMMON_FILE_NAME = 'Strings'\r
-OFFSET = 'offset'\r
-STRING = 'string'\r
-TO = 'to'\r
STRING_TOKEN = re.compile('STRING_TOKEN *\(([A-Z0-9_]+) *\)', re.MULTILINE | re.UNICODE)\r
\r
EFI_HII_ARRAY_SIZE_LENGTH = 4\r
# @retval: The formatted hex string\r
#\r
def DecToHexStr(Dec, Digit = 8):\r
- return eval("'0x%0" + str(Digit) + "X' % int(Dec)")\r
+ return '0x{0:0{1}X}'.format(Dec,Digit)\r
\r
## Convert a dec number to a hex list\r
#\r
# @retval: A list for formatted hex string\r
#\r
def DecToHexList(Dec, Digit = 8):\r
- Hex = eval("'%0" + str(Digit) + "X' % int(Dec)" )\r
- List = []\r
- for Bit in range(Digit - 2, -1, -2):\r
- List.append(HexHeader + Hex[Bit:Bit + 2])\r
- return List\r
+ Hex = '{0:0{1}X}'.format(Dec,Digit)\r
+ return ["0x" + Hex[Bit:Bit + 2] for Bit in range(Digit - 2, -1, -2)]\r
\r
## Convert a acsii string to a hex list\r
#\r
# @retval: A list for formatted hex string\r
#\r
def AscToHexList(Ascii):\r
- List = []\r
- for Item in Ascii:\r
- List.append('0x%2X' % ord(Item))\r
-\r
- return List\r
-\r
-## Create header of .h file\r
-#\r
-# Create a header of .h file\r
-#\r
-# @param BaseName: The basename of strings\r
-#\r
-# @retval Str: A string for .h file header\r
-#\r
-def CreateHFileHeader(BaseName):\r
- Str = ''\r
- for Item in H_C_FILE_HEADER:\r
- Str = WriteLine(Str, Item)\r
- Str = WriteLine(Str, '#ifndef _' + BaseName.upper() + '_STRINGS_DEFINE_H_')\r
- Str = WriteLine(Str, '#define _' + BaseName.upper() + '_STRINGS_DEFINE_H_')\r
- return Str\r
+ return ['0x{0:02X}'.format(ord(Item)) for Item in Ascii]\r
\r
## Create content of .h file\r
#\r
# Create content of .h file\r
#\r
-# @param BaseName: The basename of strings\r
-# @param UniObjectClass: A UniObjectClass instance\r
+# @param BaseName: The basename of strings\r
+# @param UniObjectClass A UniObjectClass instance\r
+# @param IsCompatibleMode Compatible mode\r
+# @param UniGenCFlag UniString is generated into AutoGen C file when it is set to True\r
#\r
# @retval Str: A string of .h file content\r
#\r
-def CreateHFileContent(BaseName, UniObjectClass):\r
+def CreateHFileContent(BaseName, UniObjectClass, IsCompatibleMode, UniGenCFlag):\r
Str = ''\r
ValueStartPtr = 60\r
Line = COMMENT_DEFINE_STR + ' ' + LANGUAGE_NAME_STRING_NAME + ' ' * (ValueStartPtr - len(DEFINE_STR + LANGUAGE_NAME_STRING_NAME)) + DecToHexStr(0, 4) + COMMENT_NOT_REFERENCED\r
Str = WriteLine(Str, Line)\r
Line = COMMENT_DEFINE_STR + ' ' + PRINTABLE_LANGUAGE_NAME_STRING_NAME + ' ' * (ValueStartPtr - len(DEFINE_STR + PRINTABLE_LANGUAGE_NAME_STRING_NAME)) + DecToHexStr(1, 4) + COMMENT_NOT_REFERENCED\r
Str = WriteLine(Str, Line)\r
+ UnusedStr = ''\r
+\r
+ #Group the referred/Unused STRING token together. \r
for Index in range(2, len(UniObjectClass.OrderedStringList[UniObjectClass.LanguageDef[0][0]])):\r
StringItem = UniObjectClass.OrderedStringList[UniObjectClass.LanguageDef[0][0]][Index]\r
Name = StringItem.StringName\r
Line = DEFINE_STR + ' ' + Name + ' ' + DecToHexStr(Token, 4)\r
else:\r
Line = DEFINE_STR + ' ' + Name + ' ' * (ValueStartPtr - len(DEFINE_STR + Name)) + DecToHexStr(Token, 4)\r
+ Str = WriteLine(Str, Line)\r
else:\r
if (ValueStartPtr - len(DEFINE_STR + Name)) <= 0:\r
Line = COMMENT_DEFINE_STR + ' ' + Name + ' ' + DecToHexStr(Token, 4) + COMMENT_NOT_REFERENCED\r
else:\r
Line = COMMENT_DEFINE_STR + ' ' + Name + ' ' * (ValueStartPtr - len(DEFINE_STR + Name)) + DecToHexStr(Token, 4) + COMMENT_NOT_REFERENCED\r
- Str = WriteLine(Str, Line)\r
+ UnusedStr = WriteLine(UnusedStr, Line)\r
+\r
+ Str = ''.join([Str, UnusedStr])\r
\r
- Str = WriteLine(Str, '')\r
- Str = WriteLine(Str, 'extern unsigned char ' + BaseName + 'Strings[];')\r
+ Str = WriteLine(Str, '')\r
+ if IsCompatibleMode or UniGenCFlag:\r
+ Str = WriteLine(Str, 'extern unsigned char ' + BaseName + 'Strings[];')\r
return Str\r
\r
## Create a complete .h file\r
#\r
# Create a complet .h file with file header and file content\r
#\r
-# @param BaseName: The basename of strings\r
-# @param UniObjectClass: A UniObjectClass instance\r
+# @param BaseName: The basename of strings\r
+# @param UniObjectClass A UniObjectClass instance\r
+# @param IsCompatibleMode Compatible mode\r
+# @param UniGenCFlag UniString is generated into AutoGen C file when it is set to True\r
#\r
# @retval Str: A string of complete .h file\r
#\r
-def CreateHFile(BaseName, UniObjectClass):\r
- HFile = WriteLine('', CreateHFileContent(BaseName, UniObjectClass))\r
+def CreateHFile(BaseName, UniObjectClass, IsCompatibleMode, UniGenCFlag):\r
+ HFile = WriteLine('', CreateHFileContent(BaseName, UniObjectClass, IsCompatibleMode, UniGenCFlag))\r
\r
return HFile\r
\r
-## Create header of .c file\r
-#\r
-# Create a header of .c file\r
+## Create a buffer to store all items in an array\r
#\r
-# @retval Str: A string for .c file header\r
+# @param BinBuffer Buffer to contain Binary data.\r
+# @param Array: The array need to be formatted\r
#\r
-def CreateCFileHeader():\r
- Str = ''\r
- for Item in H_C_FILE_HEADER:\r
- Str = WriteLine(Str, Item)\r
-\r
- return Str\r
+def CreateBinBuffer(BinBuffer, Array):\r
+ for Item in Array:\r
+ BinBuffer.write(pack("B", int(Item, 16)))\r
\r
## Create a formatted string all items in an array\r
#\r
Index = Index + 1\r
else:\r
ArrayItem = WriteLine(ArrayItem, Line)\r
- Line = ' ' + Item + ', '\r
+ Line = ' ' + Item + ', '\r
Index = 1\r
ArrayItem = Write(ArrayItem, Line.rstrip())\r
\r
\r
return Str\r
\r
+## GetFilteredLanguage\r
+#\r
+# apply get best language rules to the UNI language code list\r
+#\r
+# @param UniLanguageList: language code definition list in *.UNI file\r
+# @param LanguageFilterList: language code filter list of RFC4646 format in DSC file\r
+#\r
+# @retval UniLanguageListFiltered: the filtered language code\r
+#\r
+def GetFilteredLanguage(UniLanguageList, LanguageFilterList):\r
+ UniLanguageListFiltered = []\r
+ # if filter list is empty, then consider there is no filter\r
+ if LanguageFilterList == []:\r
+ UniLanguageListFiltered = UniLanguageList\r
+ return UniLanguageListFiltered\r
+ for Language in LanguageFilterList:\r
+ # first check for exact match\r
+ if Language in UniLanguageList:\r
+ if Language not in UniLanguageListFiltered:\r
+ UniLanguageListFiltered += [Language]\r
+ # find the first one with the same/equivalent primary tag\r
+ else:\r
+ if Language.find('-') != -1:\r
+ PrimaryTag = Language[0:Language.find('-')].lower()\r
+ else:\r
+ PrimaryTag = Language\r
+ \r
+ if len(PrimaryTag) == 3:\r
+ PrimaryTag = LangConvTable.get(PrimaryTag)\r
+ \r
+ for UniLanguage in UniLanguageList:\r
+ if UniLanguage.find('-') != -1:\r
+ UniLanguagePrimaryTag = UniLanguage[0:UniLanguage.find('-')].lower()\r
+ else:\r
+ UniLanguagePrimaryTag = UniLanguage\r
+ \r
+ if len(UniLanguagePrimaryTag) == 3:\r
+ UniLanguagePrimaryTag = LangConvTable.get(UniLanguagePrimaryTag)\r
+\r
+ if PrimaryTag == UniLanguagePrimaryTag:\r
+ if UniLanguage not in UniLanguageListFiltered:\r
+ UniLanguageListFiltered += [UniLanguage]\r
+ break\r
+ else:\r
+ # Here is rule 3 for "get best language"\r
+ # If tag is not listed in the Unicode file, the default ("en") tag should be used for that language\r
+ # for better processing, find the one that best suit for it.\r
+ DefaultTag = 'en'\r
+ if DefaultTag not in UniLanguageListFiltered:\r
+ # check whether language code with primary code equivalent with DefaultTag already in the list, if so, use that\r
+ for UniLanguage in UniLanguageList:\r
+ if UniLanguage.startswith('en-') or UniLanguage.startswith('eng-'):\r
+ if UniLanguage not in UniLanguageListFiltered:\r
+ UniLanguageListFiltered += [UniLanguage]\r
+ break\r
+ else:\r
+ UniLanguageListFiltered += [DefaultTag]\r
+ return UniLanguageListFiltered\r
+\r
\r
## Create content of .c file\r
#\r
# Create content of .c file\r
#\r
-# @param BaseName: The basename of strings\r
-# @param UniObjectClass: A UniObjectClass instance\r
+# @param BaseName: The basename of strings\r
+# @param UniObjectClass A UniObjectClass instance\r
+# @param IsCompatibleMode Compatible mode\r
+# @param UniBinBuffer UniBinBuffer to contain UniBinary data.\r
+# @param FilterInfo Platform language filter information \r
#\r
# @retval Str: A string of .c file content\r
#\r
-def CreateCFileContent(BaseName, UniObjectClass, IsCompatibleMode):\r
+def CreateCFileContent(BaseName, UniObjectClass, IsCompatibleMode, UniBinBuffer, FilterInfo):\r
#\r
# Init array length\r
#\r
Str = ''\r
Offset = 0\r
\r
+ EDK2Module = FilterInfo[0]\r
+ if EDK2Module:\r
+ LanguageFilterList = FilterInfo[1]\r
+ else:\r
+ # EDK module is using ISO639-2 format filter, convert to the RFC4646 format\r
+ LanguageFilterList = [LangConvTable.get(F.lower()) for F in FilterInfo[1]]\r
+ \r
+ UniLanguageList = []\r
+ for IndexI in range(len(UniObjectClass.LanguageDef)):\r
+ UniLanguageList += [UniObjectClass.LanguageDef[IndexI][0]]\r
+\r
+ UniLanguageListFiltered = GetFilteredLanguage(UniLanguageList, LanguageFilterList)\r
+ \r
+ \r
#\r
# Create lines for each language's strings\r
#\r
for IndexI in range(len(UniObjectClass.LanguageDef)):\r
Language = UniObjectClass.LanguageDef[IndexI][0]\r
- LangPrintName = UniObjectClass.LanguageDef[IndexI][1]\r
-\r
+ if Language not in UniLanguageListFiltered:\r
+ continue\r
+ \r
+ StringBuffer = StringIO()\r
StrStringValue = ''\r
ArrayLength = 0\r
- NumberOfUseOhterLangDef = 0\r
+ NumberOfUseOtherLangDef = 0\r
Index = 0\r
for IndexJ in range(1, len(UniObjectClass.OrderedStringList[UniObjectClass.LanguageDef[IndexI][0]])):\r
- Item = UniObjectClass.FindByToken(IndexJ, Language)\r
+ Item = UniObjectClass.OrderedStringListByToken[Language][IndexJ]\r
+\r
Name = Item.StringName\r
Value = Item.StringValueByteList\r
Referenced = Item.Referenced\r
Token = Item.Token\r
- Length = Item.Length\r
UseOtherLangDef = Item.UseOtherLangDef\r
\r
if UseOtherLangDef != '' and Referenced:\r
- NumberOfUseOhterLangDef = NumberOfUseOhterLangDef + 1\r
+ NumberOfUseOtherLangDef = NumberOfUseOtherLangDef + 1\r
Index = Index + 1\r
else:\r
- if NumberOfUseOhterLangDef > 0:\r
- StrStringValue = WriteLine(StrStringValue, CreateArrayItem([StringSkipType] + DecToHexList(NumberOfUseOhterLangDef, 4)))\r
- NumberOfUseOhterLangDef = 0\r
+ if NumberOfUseOtherLangDef > 0:\r
+ StrStringValue = WriteLine(StrStringValue, CreateArrayItem([StringSkipType] + DecToHexList(NumberOfUseOtherLangDef, 4)))\r
+ CreateBinBuffer (StringBuffer, ([StringSkipType] + DecToHexList(NumberOfUseOtherLangDef, 4)))\r
+ NumberOfUseOtherLangDef = 0\r
ArrayLength = ArrayLength + 3\r
if Referenced and Item.Token > 0:\r
Index = Index + 1\r
StrStringValue = WriteLine(StrStringValue, "// %s: %s:%s" % (DecToHexStr(Index, 4), Name, DecToHexStr(Token, 4)))\r
StrStringValue = Write(StrStringValue, CreateCFileStringValue(Value))\r
- Offset = Offset + Length\r
+ CreateBinBuffer (StringBuffer, [StringBlockType] + Value)\r
ArrayLength = ArrayLength + Item.Length + 1 # 1 is for the length of string type\r
\r
#\r
# Add an EFI_HII_SIBT_END at last\r
#\r
Str = WriteLine(Str, ' ' + EFI_HII_SIBT_END + ",")\r
+ \r
+ #\r
+ # Create binary UNI string\r
+ #\r
+ if UniBinBuffer:\r
+ CreateBinBuffer (UniBinBuffer, List)\r
+ UniBinBuffer.write (StringBuffer.getvalue())\r
+ UniBinBuffer.write (pack("B", int(EFI_HII_SIBT_END, 16)))\r
+ StringBuffer.close()\r
\r
#\r
# Create line for string variable name\r
# "unsigned char $(BaseName)Strings[] = {"\r
#\r
- AllStr = WriteLine('', CHAR_ARRAY_DEFIN + ' ' + BaseName + COMMON_FILE_NAME + '[] = {\n' )\r
+ AllStr = WriteLine('', CHAR_ARRAY_DEFIN + ' ' + BaseName + COMMON_FILE_NAME + '[] = {\n')\r
\r
- #\r
- # Create FRAMEWORK_EFI_HII_PACK_HEADER in compatible mode\r
- #\r
if IsCompatibleMode:\r
+ #\r
+ # Create FRAMEWORK_EFI_HII_PACK_HEADER in compatible mode\r
+ #\r
AllStr = WriteLine(AllStr, '// FRAMEWORK PACKAGE HEADER Length')\r
AllStr = WriteLine(AllStr, CreateArrayItem(DecToHexList(TotalLength + 2)) + '\n')\r
AllStr = WriteLine(AllStr, '// FRAMEWORK PACKAGE HEADER Type')\r
AllStr = WriteLine(AllStr, CreateArrayItem(DecToHexList(2, 4)) + '\n')\r
-\r
- #\r
- # Create whole array length in UEFI mode\r
- #\r
- if not IsCompatibleMode:\r
+ else:\r
+ #\r
+ # Create whole array length in UEFI mode\r
+ #\r
AllStr = WriteLine(AllStr, '// STRGATHER_OUTPUT_HEADER')\r
AllStr = WriteLine(AllStr, CreateArrayItem(DecToHexList(TotalLength)) + '\n')\r
\r
#\r
# Create a complete .c file\r
#\r
-# @param BaseName: The basename of strings\r
-# @param UniObjectClass: A UniObjectClass instance\r
+# @param BaseName: The basename of strings\r
+# @param UniObjectClass A UniObjectClass instance\r
+# @param IsCompatibleMode Compatible Mode\r
+# @param FilterInfo Platform language filter information \r
#\r
-# @retval CFile: A string of complete .c file\r
+# @retval CFile: A string of complete .c file\r
#\r
-def CreateCFile(BaseName, UniObjectClass, IsCompatibleMode):\r
+def CreateCFile(BaseName, UniObjectClass, IsCompatibleMode, FilterInfo):\r
CFile = ''\r
- #CFile = WriteLine(CFile, CreateCFileHeader())\r
- CFile = WriteLine(CFile, CreateCFileContent(BaseName, UniObjectClass, IsCompatibleMode))\r
+ CFile = WriteLine(CFile, CreateCFileContent(BaseName, UniObjectClass, IsCompatibleMode, None, FilterInfo))\r
CFile = WriteLine(CFile, CreateCFileEnd())\r
return CFile\r
\r
#\r
# @param UniObjectClass: Input UniObjectClass\r
# @param FileList: Search path list\r
+# @param IsCompatibleMode Compatible Mode\r
#\r
# @retval UniObjectClass: UniObjectClass after searched\r
#\r
-def SearchString(UniObjectClass, FileList):\r
+def SearchString(UniObjectClass, FileList, IsCompatibleMode):\r
if FileList == []:\r
return UniObjectClass\r
\r
if os.path.isfile(File):\r
Lines = open(File, 'r')\r
for Line in Lines:\r
- StringTokenList = STRING_TOKEN.findall(Line)\r
- for StrName in StringTokenList:\r
+ for StrName in STRING_TOKEN.findall(Line):\r
EdkLogger.debug(EdkLogger.DEBUG_5, "Found string identifier: " + StrName)\r
UniObjectClass.SetStringReferenced(StrName)\r
\r
# This function is used for UEFI2.1 spec\r
#\r
#\r
-def GetStringFiles(UniFilList, SourceFileList, IncludeList, SkipList, BaseName, IsCompatibleMode = False, ShellMode = False):\r
- Status = True\r
- ErrorMessage = ''\r
-\r
+def GetStringFiles(UniFilList, SourceFileList, IncludeList, IncludePathList, SkipList, BaseName, IsCompatibleMode = False, ShellMode = False, UniGenCFlag = True, UniGenBinBuffer = None, FilterInfo = [True, []]): \r
if len(UniFilList) > 0:\r
if ShellMode:\r
#\r
# support ISO 639-2 codes in .UNI files of EDK Shell\r
#\r
- Uni = UniFileClassObject(UniFilList, True)\r
+ Uni = UniFileClassObject(sorted (UniFilList), True, IncludePathList)\r
else:\r
- Uni = UniFileClassObject(UniFilList, IsCompatibleMode)\r
+ Uni = UniFileClassObject(sorted (UniFilList), IsCompatibleMode, IncludePathList)\r
else:\r
EdkLogger.error("UnicodeStringGather", AUTOGEN_ERROR, 'No unicode files given')\r
\r
FileList = GetFileList(SourceFileList, IncludeList, SkipList)\r
\r
- Uni = SearchString(Uni, FileList)\r
+ Uni = SearchString(Uni, sorted (FileList), IsCompatibleMode)\r
\r
- HFile = CreateHFile(BaseName, Uni)\r
- CFile = CreateCFile(BaseName, Uni, IsCompatibleMode)\r
+ HFile = CreateHFile(BaseName, Uni, IsCompatibleMode, UniGenCFlag)\r
+ CFile = None\r
+ if IsCompatibleMode or UniGenCFlag:\r
+ CFile = CreateCFile(BaseName, Uni, IsCompatibleMode, FilterInfo)\r
+ if UniGenBinBuffer:\r
+ CreateCFileContent(BaseName, Uni, IsCompatibleMode, UniGenBinBuffer, FilterInfo)\r
\r
return HFile, CFile\r
\r
# Write an item\r
#\r
def Write(Target, Item):\r
- return Target + Item\r
+ return ''.join([Target, Item])\r
\r
#\r
# Write an item with a break line\r
#\r
def WriteLine(Target, Item):\r
- return Target + Item + '\n'\r
+ return ''.join([Target, Item, '\n'])\r
\r
# This acts like the main() function for the script, unless it is 'import'ed into another\r
# script.\r
projects / mirror_edk2.git / blobdiff