BaseTools/Source/Python/AutoGen/StrGather.py

   1 # Copyright (c) 2007, Intel Corporation
   2 # All rights reserved. This program and the accompanying materials
   3 # are licensed and made available under the terms and conditions of the BSD License
   4 # which accompanies this distribution.  The full text of the license may be found at
   5 # http://opensource.org/licenses/bsd-license.php
   6 #
   7 # THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
   8 # WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
   9
  10 #
  11 #This file is used to parse a strings file and create or add to a string database file.
  12 #
  13
  14 ##
  15 # Import Modules
  16 #
  17 import re
  18 import Common.EdkLogger as EdkLogger
  19 from Common.BuildToolError import *
  20 from UniClassObject import *
  21
  22 ##
  23 # Static definitions
  24 #
  25 EFI_HII_SIBT_END = '0x00'
  26 EFI_HII_SIBT_STRING_SCSU = '0x10'
  27 EFI_HII_SIBT_STRING_SCSU_FONT = '0x11'
  28 EFI_HII_SIBT_STRINGS_SCSU = '0x12'
  29 EFI_HII_SIBT_STRINGS_SCSU_FONT = '0x13'
  30 EFI_HII_SIBT_STRING_UCS2 = '0x14'
  31 EFI_HII_SIBT_STRING_UCS2_FONT = '0x15'
  32 EFI_HII_SIBT_STRINGS_UCS2 = '0x16'
  33 EFI_HII_SIBT_STRINGS_UCS2_FONT = '0x17'
  34 EFI_HII_SIBT_DUPLICATE = '0x20'
  35 EFI_HII_SIBT_SKIP2 = '0x21'
  36 EFI_HII_SIBT_SKIP1 = '0x22'
  37 EFI_HII_SIBT_EXT1 = '0x30'
  38 EFI_HII_SIBT_EXT2 = '0x31'
  39 EFI_HII_SIBT_EXT4 = '0x32'
  40 EFI_HII_SIBT_FONT = '0x40'
  41
  42 EFI_HII_PACKAGE_STRINGS = '0x04'
  43 EFI_HII_PACKAGE_FORM = '0x02'
  44
  45 StringPackageType = EFI_HII_PACKAGE_STRINGS
  46 StringPackageForm = EFI_HII_PACKAGE_FORM
  47 StringBlockType = EFI_HII_SIBT_STRING_UCS2
  48 StringSkipType = EFI_HII_SIBT_SKIP2
  49
  50 HexHeader = '0x'
  51
  52 COMMENT = '// '
  53 DEFINE_STR = '#define'
  54 COMMENT_DEFINE_STR = COMMENT + DEFINE_STR
  55 NOT_REFERENCED = 'not referenced'
  56 COMMENT_NOT_REFERENCED = ' ' + COMMENT + NOT_REFERENCED
  57 CHAR_ARRAY_DEFIN = 'unsigned char'
  58 COMMON_FILE_NAME = 'Strings'
  59 OFFSET = 'offset'
  60 STRING = 'string'
  61 TO = 'to'
  62 STRING_TOKEN = re.compile('STRING_TOKEN *\(([A-Z0-9_]+) *\)', re.MULTILINE | re.UNICODE)
  63
  64 EFI_HII_ARRAY_SIZE_LENGTH = 4
  65 EFI_HII_PACKAGE_HEADER_LENGTH = 4
  66 EFI_HII_HDR_SIZE_LENGTH = 4
  67 EFI_HII_STRING_OFFSET_LENGTH = 4
  68 EFI_STRING_ID = 1
  69 EFI_STRING_ID_LENGTH = 2
  70 EFI_HII_LANGUAGE_WINDOW = 0
  71 EFI_HII_LANGUAGE_WINDOW_LENGTH = 2
  72 EFI_HII_LANGUAGE_WINDOW_NUMBER = 16
  73 EFI_HII_STRING_PACKAGE_HDR_LENGTH = EFI_HII_PACKAGE_HEADER_LENGTH + EFI_HII_HDR_SIZE_LENGTH + EFI_HII_STRING_OFFSET_LENGTH + EFI_HII_LANGUAGE_WINDOW_LENGTH * EFI_HII_LANGUAGE_WINDOW_NUMBER + EFI_STRING_ID_LENGTH
  74
  75 H_C_FILE_HEADER = ['//', \
  76                    '//  DO NOT EDIT -- auto-generated file', \
  77                    '//', \
  78                    '//  This file is generated by the StrGather utility', \
  79                    '//']
  80 LANGUAGE_NAME_STRING_NAME = '$LANGUAGE_NAME'
  81 PRINTABLE_LANGUAGE_NAME_STRING_NAME = '$PRINTABLE_LANGUAGE_NAME'
  82
  83 ## Convert a dec number to a hex string
  84 #
  85 # Convert a dec number to a formatted hex string in length digit
  86 # The digit is set to default 8
  87 # The hex string starts with "0x"
  88 # DecToHexStr(1000) is '0x000003E8'
  89 # DecToHexStr(1000, 6) is '0x0003E8'
  90 #
  91 # @param Dec:    The number in dec format
  92 # @param Digit:  The needed digit of hex string
  93 #
  94 # @retval:       The formatted hex string
  95 #
  96 def DecToHexStr(Dec, Digit = 8):
  97     return eval("'0x%0" + str(Digit) + "X' % int(Dec)")
  98
  99 ## Convert a dec number to a hex list
 100 #
 101 # Convert a dec number to a formatted hex list in size digit
 102 # The digit is set to default 8
 103 # DecToHexList(1000) is ['0xE8', '0x03', '0x00', '0x00']
 104 # DecToHexList(1000, 6) is ['0xE8', '0x03', '0x00']
 105 #
 106 # @param Dec:    The number in dec format
 107 # @param Digit:  The needed digit of hex list
 108 #
 109 # @retval:       A list for formatted hex string
 110 #
 111 def DecToHexList(Dec, Digit = 8):
 112     Hex = eval("'%0" + str(Digit) + "X' % int(Dec)" )
 113     List = []
 114     for Bit in range(Digit - 2, -1, -2):
 115         List.append(HexHeader + Hex[Bit:Bit + 2])
 116     return List
 117
 118 ## Convert a acsii string to a hex list
 119 #
 120 # Convert a acsii string to a formatted hex list
 121 # AscToHexList('en-US') is ['0x65', '0x6E', '0x2D', '0x55', '0x53']
 122 #
 123 # @param Ascii:  The acsii string
 124 #
 125 # @retval:       A list for formatted hex string
 126 #
 127 def AscToHexList(Ascii):
 128     List = []
 129     for Item in Ascii:
 130         List.append('0x%2X' % ord(Item))
 131
 132     return List
 133
 134 ## Create header of .h file
 135 #
 136 # Create a header of .h file
 137 #
 138 # @param BaseName: The basename of strings
 139 #
 140 # @retval Str:     A string for .h file header
 141 #
 142 def CreateHFileHeader(BaseName):
 143     Str = ''
 144     for Item in H_C_FILE_HEADER:
 145         Str = WriteLine(Str, Item)
 146     Str = WriteLine(Str, '#ifndef _' + BaseName.upper() + '_STRINGS_DEFINE_H_')
 147     Str = WriteLine(Str, '#define _' + BaseName.upper() + '_STRINGS_DEFINE_H_')
 148     return Str
 149
 150 ## Create content of .h file
 151 #
 152 # Create content of .h file
 153 #
 154 # @param BaseName:       The basename of strings
 155 # @param UniObjectClass: A UniObjectClass instance
 156 #
 157 # @retval Str:           A string of .h file content
 158 #
 159 def CreateHFileContent(BaseName, UniObjectClass):
 160     Str = ''
 161     ValueStartPtr = 60
 162     Line = COMMENT_DEFINE_STR + ' ' + LANGUAGE_NAME_STRING_NAME + ' ' * (ValueStartPtr - len(DEFINE_STR + LANGUAGE_NAME_STRING_NAME)) + DecToHexStr(0, 4) + COMMENT_NOT_REFERENCED
 163     Str = WriteLine(Str, Line)
 164     Line = COMMENT_DEFINE_STR + ' ' + PRINTABLE_LANGUAGE_NAME_STRING_NAME + ' ' * (ValueStartPtr - len(DEFINE_STR + PRINTABLE_LANGUAGE_NAME_STRING_NAME)) + DecToHexStr(1, 4) + COMMENT_NOT_REFERENCED
 165     Str = WriteLine(Str, Line)
 166     for Index in range(2, len(UniObjectClass.OrderedStringList[UniObjectClass.LanguageDef[0][0]])):
 167         StringItem = UniObjectClass.OrderedStringList[UniObjectClass.LanguageDef[0][0]][Index]
 168         Name = StringItem.StringName
 169         Token = StringItem.Token
 170         Referenced = StringItem.Referenced
 171         if Name != None:
 172             Line = ''
 173             if Referenced == True:
 174                 Line = DEFINE_STR + ' ' + Name + ' ' * (ValueStartPtr - len(DEFINE_STR + Name)) + DecToHexStr(Token, 4)
 175             else:
 176                 Line = COMMENT_DEFINE_STR + ' ' + Name + ' ' * (ValueStartPtr - len(DEFINE_STR + Name)) + DecToHexStr(Token, 4) + COMMENT_NOT_REFERENCED
 177             Str = WriteLine(Str, Line)
 178
 179     Str =  WriteLine(Str, '')
 180     Str = WriteLine(Str, 'extern unsigned char ' + BaseName + 'Strings[];')
 181     return Str
 182
 183 ## Create a complete .h file
 184 #
 185 # Create a complet .h file with file header and file content
 186 #
 187 # @param BaseName:       The basename of strings
 188 # @param UniObjectClass: A UniObjectClass instance
 189 #
 190 # @retval Str:           A string of complete .h file
 191 #
 192 def CreateHFile(BaseName, UniObjectClass):
 193     HFile = WriteLine('', CreateHFileContent(BaseName, UniObjectClass))
 194
 195     return HFile
 196
 197 ## Create header of .c file
 198 #
 199 # Create a header of .c file
 200 #
 201 # @retval Str:     A string for .c file header
 202 #
 203 def CreateCFileHeader():
 204     Str = ''
 205     for Item in H_C_FILE_HEADER:
 206         Str = WriteLine(Str, Item)
 207
 208     return Str
 209
 210 ## Create a formatted string all items in an array
 211 #
 212 # Use ',' to join each item in an array, and break an new line when reaching the width (default is 16)
 213 #
 214 # @param Array:      The array need to be formatted
 215 # @param Width:      The line length, the default value is set to 16
 216 #
 217 # @retval ArrayItem: A string for all formatted array items
 218 #
 219 def CreateArrayItem(Array, Width = 16):
 220     MaxLength = Width
 221     Index = 0
 222     Line = '  '
 223     ArrayItem = ''
 224
 225     for Item in Array:
 226         if Index < MaxLength:
 227             Line = Line + Item + ',  '
 228             Index = Index + 1
 229         else:
 230             ArrayItem = WriteLine(ArrayItem, Line)
 231             Line = '  ' + Item +  ',  '
 232             Index = 1
 233     ArrayItem = Write(ArrayItem, Line.rstrip())
 234
 235     return ArrayItem
 236
 237 ## CreateCFileStringValue
 238 #
 239 # Create a line with string value
 240 #
 241 # @param Value:  Value of the string
 242 #
 243 # @retval Str:   A formatted string with string value
 244 #
 245
 246 def CreateCFileStringValue(Value):
 247     Value = [StringBlockType] + Value
 248     Str = WriteLine('', CreateArrayItem(Value))
 249
 250     return Str
 251
 252
 253 ## Create content of .c file
 254 #
 255 # Create content of .c file
 256 #
 257 # @param BaseName:       The basename of strings
 258 # @param UniObjectClass: A UniObjectClass instance
 259 #
 260 # @retval Str:           A string of .c file content
 261 #
 262 def CreateCFileContent(BaseName, UniObjectClass, IsCompatibleMode):
 263     #
 264     # Init array length
 265     #
 266     TotalLength = EFI_HII_ARRAY_SIZE_LENGTH
 267     Str = ''
 268     Offset = 0
 269
 270     #
 271     # Create lines for each language's strings
 272     #
 273     for IndexI in range(len(UniObjectClass.LanguageDef)):
 274         Language = UniObjectClass.LanguageDef[IndexI][0]
 275         LangPrintName = UniObjectClass.LanguageDef[IndexI][1]
 276
 277         StrStringValue = ''
 278         ArrayLength = 0
 279         NumberOfUseOhterLangDef = 0
 280         Index = 0
 281         for IndexJ in range(1, len(UniObjectClass.OrderedStringList[UniObjectClass.LanguageDef[IndexI][0]])):
 282             Item = UniObjectClass.FindByToken(IndexJ, Language)
 283             Name = Item.StringName
 284             Value = Item.StringValueByteList
 285             Referenced = Item.Referenced
 286             Token = Item.Token
 287             Length = Item.Length
 288             UseOtherLangDef = Item.UseOtherLangDef
 289
 290             if UseOtherLangDef != '' and Referenced:
 291                 NumberOfUseOhterLangDef = NumberOfUseOhterLangDef + 1
 292                 Index = Index + 1
 293             else:
 294                 if NumberOfUseOhterLangDef > 0:
 295                     StrStringValue = WriteLine(StrStringValue, CreateArrayItem([StringSkipType] + DecToHexList(NumberOfUseOhterLangDef, 4)))
 296                     NumberOfUseOhterLangDef = 0
 297                     ArrayLength = ArrayLength + 3
 298                 if Referenced and Item.Token > 0:
 299                     Index = Index + 1
 300                     StrStringValue = WriteLine(StrStringValue, "// %s: %s:%s" % (DecToHexStr(Index, 4), Name, DecToHexStr(Token, 4)))
 301                     StrStringValue = Write(StrStringValue, CreateCFileStringValue(Value))
 302                     Offset = Offset + Length
 303                     ArrayLength = ArrayLength + Item.Length + 1 # 1 is for the length of string type
 304
 305         #
 306         # EFI_HII_PACKAGE_HEADER
 307         #
 308         Offset = EFI_HII_STRING_PACKAGE_HDR_LENGTH + len(Language) + 1
 309         ArrayLength = Offset + ArrayLength + 1
 310
 311         #
 312         # Create PACKAGE HEADER
 313         #
 314         Str = WriteLine(Str, '// PACKAGE HEADER\n')
 315         TotalLength = TotalLength + ArrayLength
 316
 317         List = DecToHexList(ArrayLength, 6) + \
 318                [StringPackageType] + \
 319                DecToHexList(Offset) + \
 320                DecToHexList(Offset) + \
 321                DecToHexList(EFI_HII_LANGUAGE_WINDOW, EFI_HII_LANGUAGE_WINDOW_LENGTH * 2) * EFI_HII_LANGUAGE_WINDOW_NUMBER + \
 322                DecToHexList(EFI_STRING_ID, 4) + \
 323                AscToHexList(Language) + \
 324                DecToHexList(0, 2)
 325         Str = WriteLine(Str, CreateArrayItem(List, 16) + '\n')
 326
 327         #
 328         # Create PACKAGE DATA
 329         #
 330         Str = WriteLine(Str, '// PACKAGE DATA\n')
 331         Str = Write(Str, StrStringValue)
 332
 333         #
 334         # Add an EFI_HII_SIBT_END at last
 335         #
 336         Str = WriteLine(Str, '  ' + EFI_HII_SIBT_END + ",")
 337
 338     #
 339     # Create line for string variable name
 340     # "unsigned char $(BaseName)Strings[] = {"
 341     #
 342     AllStr = WriteLine('', CHAR_ARRAY_DEFIN + ' ' + BaseName + COMMON_FILE_NAME + '[] = {\n' )
 343
 344     #
 345     # Create FRAMEWORK_EFI_HII_PACK_HEADER in compatible mode
 346     #
 347     if IsCompatibleMode:
 348         AllStr = WriteLine(AllStr, '// FRAMEWORK PACKAGE HEADER Length')
 349         AllStr = WriteLine(AllStr, CreateArrayItem(DecToHexList(TotalLength + 2)) + '\n')
 350         AllStr = WriteLine(AllStr, '// FRAMEWORK PACKAGE HEADER Type')
 351         AllStr = WriteLine(AllStr, CreateArrayItem(DecToHexList(2, 4)) + '\n')
 352
 353     #
 354     # Create whole array length in UEFI mode
 355     #
 356     if not IsCompatibleMode:
 357         AllStr = WriteLine(AllStr, '// STRGATHER_OUTPUT_HEADER')
 358         AllStr = WriteLine(AllStr, CreateArrayItem(DecToHexList(TotalLength)) + '\n')
 359
 360     #
 361     # Join package data
 362     #
 363     AllStr = Write(AllStr, Str)
 364
 365     return AllStr
 366
 367 ## Create end of .c file
 368 #
 369 # Create end of .c file
 370 #
 371 # @retval Str:           A string of .h file end
 372 #
 373 def CreateCFileEnd():
 374     Str = Write('', '};')
 375     return Str
 376
 377 ## Create a .c file
 378 #
 379 # Create a complete .c file
 380 #
 381 # @param BaseName:       The basename of strings
 382 # @param UniObjectClass: A UniObjectClass instance
 383 #
 384 # @retval CFile:         A string of complete .c file
 385 #
 386 def CreateCFile(BaseName, UniObjectClass, IsCompatibleMode):
 387     CFile = ''
 388     #CFile = WriteLine(CFile, CreateCFileHeader())
 389     CFile = WriteLine(CFile, CreateCFileContent(BaseName, UniObjectClass, IsCompatibleMode))
 390     CFile = WriteLine(CFile, CreateCFileEnd())
 391     return CFile
 392
 393 ## GetFileList
 394 #
 395 # Get a list for all files
 396 #
 397 # @param IncludeList:  A list of all path to be searched
 398 # @param SkipList:     A list of all types of file could be skipped
 399 #
 400 # @retval FileList:    A list of all files found
 401 #
 402 def GetFileList(SourceFileList, IncludeList, SkipList):
 403     if IncludeList == None:
 404         EdkLogger.error("UnicodeStringGather", AUTOGEN_ERROR, "Include path for unicode file is not defined")
 405
 406     FileList = []
 407     if SkipList == None:
 408         SkipList = []
 409
 410     for File in SourceFileList:
 411         for Dir in IncludeList:
 412             if not os.path.exists(Dir):
 413                 continue
 414             File = os.path.join(Dir, File.Path)
 415             #
 416             # Ignore Dir
 417             #
 418             if os.path.isfile(File) != True:
 419                 continue
 420             #
 421             # Ignore file listed in skip list
 422             #
 423             IsSkip = False
 424             for Skip in SkipList:
 425                 if os.path.splitext(File)[1].upper() == Skip.upper():
 426                     EdkLogger.verbose("Skipped %s for string token uses search" % File)
 427                     IsSkip = True
 428                     break
 429
 430             if not IsSkip:
 431                 FileList.append(File)
 432
 433             break
 434
 435     return FileList
 436
 437 ## SearchString
 438 #
 439 # Search whether all string defined in UniObjectClass are referenced
 440 # All string used should be set to Referenced
 441 #
 442 # @param UniObjectClass:  Input UniObjectClass
 443 # @param FileList:        Search path list
 444 #
 445 # @retval UniObjectClass: UniObjectClass after searched
 446 #
 447 def SearchString(UniObjectClass, FileList):
 448     if FileList == []:
 449         return UniObjectClass
 450
 451     for File in FileList:
 452         if os.path.isfile(File):
 453             Lines = open(File, 'r')
 454             for Line in Lines:
 455                 StringTokenList = STRING_TOKEN.findall(Line)
 456                 for StrName in StringTokenList:
 457                     EdkLogger.debug(EdkLogger.DEBUG_5, "Found string identifier: " + StrName)
 458                     UniObjectClass.SetStringReferenced(StrName)
 459
 460     UniObjectClass.ReToken()
 461
 462     return UniObjectClass
 463
 464 ## GetStringFiles
 465 #
 466 # This function is used for UEFI2.1 spec
 467 #
 468 #
 469 def GetStringFiles(UniFilList, SourceFileList, IncludeList, SkipList, BaseName, IsCompatibleMode = False, ShellMode = False):
 470     Status = True
 471     ErrorMessage = ''
 472
 473     if len(UniFilList) > 0:
 474         if ShellMode:
 475             #
 476             # support ISO 639-2 codes in .UNI files of EDK Shell
 477             #
 478             Uni = UniFileClassObject(UniFilList, True)
 479         else:
 480             Uni = UniFileClassObject(UniFilList, IsCompatibleMode)
 481     else:
 482         EdkLogger.error("UnicodeStringGather", AUTOGEN_ERROR, 'No unicode files given')
 483
 484     FileList = GetFileList(SourceFileList, IncludeList, SkipList)
 485
 486     Uni = SearchString(Uni, FileList)
 487
 488     HFile = CreateHFile(BaseName, Uni)
 489     CFile = CreateCFile(BaseName, Uni, IsCompatibleMode)
 490
 491     return HFile, CFile
 492
 493 #
 494 # Write an item
 495 #
 496 def Write(Target, Item):
 497     return Target + Item
 498
 499 #
 500 # Write an item with a break line
 501 #
 502 def WriteLine(Target, Item):
 503     return Target + Item + '\n'
 504
 505 # This acts like the main() function for the script, unless it is 'import'ed into another
 506 # script.
 507 if __name__ == '__main__':
 508     EdkLogger.info('start')
 509
 510     UniFileList = [
 511                    r'C:\\Edk\\Strings2.uni',
 512                    r'C:\\Edk\\Strings.uni'
 513     ]
 514
 515     SrcFileList = []
 516     for Root, Dirs, Files in os.walk('C:\\Edk'):
 517         for File in Files:
 518             SrcFileList.append(File)
 519
 520     IncludeList = [
 521                    r'C:\\Edk'
 522     ]
 523
 524     SkipList = ['.inf', '.uni']
 525     BaseName = 'DriverSample'
 526     (h, c) = GetStringFiles(UniFileList, SrcFileList, IncludeList, SkipList, BaseName, True)
 527     hfile = open('unistring.h', 'w')
 528     cfile = open('unistring.c', 'w')
 529     hfile.write(h)
 530     cfile.write(c)
 531
 532     EdkLogger.info('end')