BaseTools/Source/Python/AutoGen/StrGather.py

   1 # Copyright (c) 2007, Intel Corporation
   2 # All rights reserved. This program and the accompanying materials
   3 # are licensed and made available under the terms and conditions of the BSD License
   4 # which accompanies this distribution.  The full text of the license may be found at
   5 # http://opensource.org/licenses/bsd-license.php
   6 #
   7 # THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
   8 # WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
   9
  10 #
  11 #This file is used to parse a strings file and create or add to a string database file.
  12 #
  13
  14 ##
  15 # Import Modules
  16 #
  17 import re
  18 import Common.EdkLogger as EdkLogger
  19 from Common.BuildToolError import *
  20 from UniClassObject import *
  21
  22 ##
  23 # Static definitions
  24 #
  25 EFI_HII_SIBT_END = '0x00'
  26 EFI_HII_SIBT_STRING_SCSU = '0x10'
  27 EFI_HII_SIBT_STRING_SCSU_FONT = '0x11'
  28 EFI_HII_SIBT_STRINGS_SCSU = '0x12'
  29 EFI_HII_SIBT_STRINGS_SCSU_FONT = '0x13'
  30 EFI_HII_SIBT_STRING_UCS2 = '0x14'
  31 EFI_HII_SIBT_STRING_UCS2_FONT = '0x15'
  32 EFI_HII_SIBT_STRINGS_UCS2 = '0x16'
  33 EFI_HII_SIBT_STRINGS_UCS2_FONT = '0x17'
  34 EFI_HII_SIBT_DUPLICATE = '0x20'
  35 EFI_HII_SIBT_SKIP2 = '0x21'
  36 EFI_HII_SIBT_SKIP1 = '0x22'
  37 EFI_HII_SIBT_EXT1 = '0x30'
  38 EFI_HII_SIBT_EXT2 = '0x31'
  39 EFI_HII_SIBT_EXT4 = '0x32'
  40 EFI_HII_SIBT_FONT = '0x40'
  41
  42 EFI_HII_PACKAGE_STRINGS = '0x04'
  43 EFI_HII_PACKAGE_FORM = '0x02'
  44
  45 StringPackageType = EFI_HII_PACKAGE_STRINGS
  46 StringPackageForm = EFI_HII_PACKAGE_FORM
  47 StringBlockType = EFI_HII_SIBT_STRING_UCS2
  48 StringSkipType = EFI_HII_SIBT_SKIP2
  49
  50 HexHeader = '0x'
  51
  52 COMMENT = '// '
  53 DEFINE_STR = '#define'
  54 COMMENT_DEFINE_STR = COMMENT + DEFINE_STR
  55 NOT_REFERENCED = 'not referenced'
  56 COMMENT_NOT_REFERENCED = ' ' + COMMENT + NOT_REFERENCED
  57 CHAR_ARRAY_DEFIN = 'unsigned char'
  58 COMMON_FILE_NAME = 'Strings'
  59 OFFSET = 'offset'
  60 STRING = 'string'
  61 TO = 'to'
  62 STRING_TOKEN = re.compile('STRING_TOKEN *\(([A-Z0-9_]+) *\)', re.MULTILINE | re.UNICODE)
  63
  64 EFI_HII_ARRAY_SIZE_LENGTH = 4
  65 EFI_HII_PACKAGE_HEADER_LENGTH = 4
  66 EFI_HII_HDR_SIZE_LENGTH = 4
  67 EFI_HII_STRING_OFFSET_LENGTH = 4
  68 EFI_STRING_ID = 1
  69 EFI_STRING_ID_LENGTH = 2
  70 EFI_HII_LANGUAGE_WINDOW = 0
  71 EFI_HII_LANGUAGE_WINDOW_LENGTH = 2
  72 EFI_HII_LANGUAGE_WINDOW_NUMBER = 16
  73 EFI_HII_STRING_PACKAGE_HDR_LENGTH = EFI_HII_PACKAGE_HEADER_LENGTH + EFI_HII_HDR_SIZE_LENGTH + EFI_HII_STRING_OFFSET_LENGTH + EFI_HII_LANGUAGE_WINDOW_LENGTH * EFI_HII_LANGUAGE_WINDOW_NUMBER + EFI_STRING_ID_LENGTH
  74
  75 H_C_FILE_HEADER = ['//', \
  76                    '//  DO NOT EDIT -- auto-generated file', \
  77                    '//', \
  78                    '//  This file is generated by the StrGather utility', \
  79                    '//']
  80 LANGUAGE_NAME_STRING_NAME = '$LANGUAGE_NAME'
  81 PRINTABLE_LANGUAGE_NAME_STRING_NAME = '$PRINTABLE_LANGUAGE_NAME'
  82
  83 ## Convert a dec number to a hex string
  84 #
  85 # Convert a dec number to a formatted hex string in length digit
  86 # The digit is set to default 8
  87 # The hex string starts with "0x"
  88 # DecToHexStr(1000) is '0x000003E8'
  89 # DecToHexStr(1000, 6) is '0x0003E8'
  90 #
  91 # @param Dec:    The number in dec format
  92 # @param Digit:  The needed digit of hex string
  93 #
  94 # @retval:       The formatted hex string
  95 #
  96 def DecToHexStr(Dec, Digit = 8):
  97     return eval("'0x%0" + str(Digit) + "X' % int(Dec)")
  98
  99 ## Convert a dec number to a hex list
 100 #
 101 # Convert a dec number to a formatted hex list in size digit
 102 # The digit is set to default 8
 103 # DecToHexList(1000) is ['0xE8', '0x03', '0x00', '0x00']
 104 # DecToHexList(1000, 6) is ['0xE8', '0x03', '0x00']
 105 #
 106 # @param Dec:    The number in dec format
 107 # @param Digit:  The needed digit of hex list
 108 #
 109 # @retval:       A list for formatted hex string
 110 #
 111 def DecToHexList(Dec, Digit = 8):
 112     Hex = eval("'%0" + str(Digit) + "X' % int(Dec)" )
 113     List = []
 114     for Bit in range(Digit - 2, -1, -2):
 115         List.append(HexHeader + Hex[Bit:Bit + 2])
 116     return List
 117
 118 ## Convert a acsii string to a hex list
 119 #
 120 # Convert a acsii string to a formatted hex list
 121 # AscToHexList('en-US') is ['0x65', '0x6E', '0x2D', '0x55', '0x53']
 122 #
 123 # @param Ascii:  The acsii string
 124 #
 125 # @retval:       A list for formatted hex string
 126 #
 127 def AscToHexList(Ascii):
 128     List = []
 129     for Item in Ascii:
 130         List.append('0x%2X' % ord(Item))
 131
 132     return List
 133
 134 ## Create header of .h file
 135 #
 136 # Create a header of .h file
 137 #
 138 # @param BaseName: The basename of strings
 139 #
 140 # @retval Str:     A string for .h file header
 141 #
 142 def CreateHFileHeader(BaseName):
 143     Str = ''
 144     for Item in H_C_FILE_HEADER:
 145         Str = WriteLine(Str, Item)
 146     Str = WriteLine(Str, '#ifndef _' + BaseName.upper() + '_STRINGS_DEFINE_H_')
 147     Str = WriteLine(Str, '#define _' + BaseName.upper() + '_STRINGS_DEFINE_H_')
 148     return Str
 149
 150 ## Create content of .h file
 151 #
 152 # Create content of .h file
 153 #
 154 # @param BaseName:       The basename of strings
 155 # @param UniObjectClass: A UniObjectClass instance
 156 #
 157 # @retval Str:           A string of .h file content
 158 #
 159 def CreateHFileContent(BaseName, UniObjectClass):
 160     Str = ''
 161     ValueStartPtr = 60
 162     Line = COMMENT_DEFINE_STR + ' ' + LANGUAGE_NAME_STRING_NAME + ' ' * (ValueStartPtr - len(DEFINE_STR + LANGUAGE_NAME_STRING_NAME)) + DecToHexStr(0, 4) + COMMENT_NOT_REFERENCED
 163     Str = WriteLine(Str, Line)
 164     Line = COMMENT_DEFINE_STR + ' ' + PRINTABLE_LANGUAGE_NAME_STRING_NAME + ' ' * (ValueStartPtr - len(DEFINE_STR + PRINTABLE_LANGUAGE_NAME_STRING_NAME)) + DecToHexStr(1, 4) + COMMENT_NOT_REFERENCED
 165     Str = WriteLine(Str, Line)
 166     for Index in range(2, len(UniObjectClass.OrderedStringList[UniObjectClass.LanguageDef[0][0]])):
 167         StringItem = UniObjectClass.OrderedStringList[UniObjectClass.LanguageDef[0][0]][Index]
 168         Name = StringItem.StringName
 169         Token = StringItem.Token
 170         Referenced = StringItem.Referenced
 171         if Name != None:
 172             Line = ''
 173             if Referenced == True:
 174                 if (ValueStartPtr - len(DEFINE_STR + Name)) <= 0:
 175                     Line = DEFINE_STR + ' ' + Name + ' ' + DecToHexStr(Token, 4)
 176                 else:
 177                     Line = DEFINE_STR + ' ' + Name + ' ' * (ValueStartPtr - len(DEFINE_STR + Name)) + DecToHexStr(Token, 4)
 178             else:
 179                 if (ValueStartPtr - len(DEFINE_STR + Name)) <= 0:
 180                     Line = COMMENT_DEFINE_STR + ' ' + Name + ' ' + DecToHexStr(Token, 4) + COMMENT_NOT_REFERENCED
 181                 else:
 182                     Line = COMMENT_DEFINE_STR + ' ' + Name + ' ' * (ValueStartPtr - len(DEFINE_STR + Name)) + DecToHexStr(Token, 4) + COMMENT_NOT_REFERENCED
 183             Str = WriteLine(Str, Line)
 184
 185     Str =  WriteLine(Str, '')
 186     Str = WriteLine(Str, 'extern unsigned char ' + BaseName + 'Strings[];')
 187     return Str
 188
 189 ## Create a complete .h file
 190 #
 191 # Create a complet .h file with file header and file content
 192 #
 193 # @param BaseName:       The basename of strings
 194 # @param UniObjectClass: A UniObjectClass instance
 195 #
 196 # @retval Str:           A string of complete .h file
 197 #
 198 def CreateHFile(BaseName, UniObjectClass):
 199     HFile = WriteLine('', CreateHFileContent(BaseName, UniObjectClass))
 200
 201     return HFile
 202
 203 ## Create header of .c file
 204 #
 205 # Create a header of .c file
 206 #
 207 # @retval Str:     A string for .c file header
 208 #
 209 def CreateCFileHeader():
 210     Str = ''
 211     for Item in H_C_FILE_HEADER:
 212         Str = WriteLine(Str, Item)
 213
 214     return Str
 215
 216 ## Create a formatted string all items in an array
 217 #
 218 # Use ',' to join each item in an array, and break an new line when reaching the width (default is 16)
 219 #
 220 # @param Array:      The array need to be formatted
 221 # @param Width:      The line length, the default value is set to 16
 222 #
 223 # @retval ArrayItem: A string for all formatted array items
 224 #
 225 def CreateArrayItem(Array, Width = 16):
 226     MaxLength = Width
 227     Index = 0
 228     Line = '  '
 229     ArrayItem = ''
 230
 231     for Item in Array:
 232         if Index < MaxLength:
 233             Line = Line + Item + ',  '
 234             Index = Index + 1
 235         else:
 236             ArrayItem = WriteLine(ArrayItem, Line)
 237             Line = '  ' + Item +  ',  '
 238             Index = 1
 239     ArrayItem = Write(ArrayItem, Line.rstrip())
 240
 241     return ArrayItem
 242
 243 ## CreateCFileStringValue
 244 #
 245 # Create a line with string value
 246 #
 247 # @param Value:  Value of the string
 248 #
 249 # @retval Str:   A formatted string with string value
 250 #
 251
 252 def CreateCFileStringValue(Value):
 253     Value = [StringBlockType] + Value
 254     Str = WriteLine('', CreateArrayItem(Value))
 255
 256     return Str
 257
 258
 259 ## Create content of .c file
 260 #
 261 # Create content of .c file
 262 #
 263 # @param BaseName:       The basename of strings
 264 # @param UniObjectClass: A UniObjectClass instance
 265 #
 266 # @retval Str:           A string of .c file content
 267 #
 268 def CreateCFileContent(BaseName, UniObjectClass, IsCompatibleMode):
 269     #
 270     # Init array length
 271     #
 272     TotalLength = EFI_HII_ARRAY_SIZE_LENGTH
 273     Str = ''
 274     Offset = 0
 275
 276     #
 277     # Create lines for each language's strings
 278     #
 279     for IndexI in range(len(UniObjectClass.LanguageDef)):
 280         Language = UniObjectClass.LanguageDef[IndexI][0]
 281         LangPrintName = UniObjectClass.LanguageDef[IndexI][1]
 282
 283         StrStringValue = ''
 284         ArrayLength = 0
 285         NumberOfUseOhterLangDef = 0
 286         Index = 0
 287         for IndexJ in range(1, len(UniObjectClass.OrderedStringList[UniObjectClass.LanguageDef[IndexI][0]])):
 288             Item = UniObjectClass.FindByToken(IndexJ, Language)
 289             Name = Item.StringName
 290             Value = Item.StringValueByteList
 291             Referenced = Item.Referenced
 292             Token = Item.Token
 293             Length = Item.Length
 294             UseOtherLangDef = Item.UseOtherLangDef
 295
 296             if UseOtherLangDef != '' and Referenced:
 297                 NumberOfUseOhterLangDef = NumberOfUseOhterLangDef + 1
 298                 Index = Index + 1
 299             else:
 300                 if NumberOfUseOhterLangDef > 0:
 301                     StrStringValue = WriteLine(StrStringValue, CreateArrayItem([StringSkipType] + DecToHexList(NumberOfUseOhterLangDef, 4)))
 302                     NumberOfUseOhterLangDef = 0
 303                     ArrayLength = ArrayLength + 3
 304                 if Referenced and Item.Token > 0:
 305                     Index = Index + 1
 306                     StrStringValue = WriteLine(StrStringValue, "// %s: %s:%s" % (DecToHexStr(Index, 4), Name, DecToHexStr(Token, 4)))
 307                     StrStringValue = Write(StrStringValue, CreateCFileStringValue(Value))
 308                     Offset = Offset + Length
 309                     ArrayLength = ArrayLength + Item.Length + 1 # 1 is for the length of string type
 310
 311         #
 312         # EFI_HII_PACKAGE_HEADER
 313         #
 314         Offset = EFI_HII_STRING_PACKAGE_HDR_LENGTH + len(Language) + 1
 315         ArrayLength = Offset + ArrayLength + 1
 316
 317         #
 318         # Create PACKAGE HEADER
 319         #
 320         Str = WriteLine(Str, '// PACKAGE HEADER\n')
 321         TotalLength = TotalLength + ArrayLength
 322
 323         List = DecToHexList(ArrayLength, 6) + \
 324                [StringPackageType] + \
 325                DecToHexList(Offset) + \
 326                DecToHexList(Offset) + \
 327                DecToHexList(EFI_HII_LANGUAGE_WINDOW, EFI_HII_LANGUAGE_WINDOW_LENGTH * 2) * EFI_HII_LANGUAGE_WINDOW_NUMBER + \
 328                DecToHexList(EFI_STRING_ID, 4) + \
 329                AscToHexList(Language) + \
 330                DecToHexList(0, 2)
 331         Str = WriteLine(Str, CreateArrayItem(List, 16) + '\n')
 332
 333         #
 334         # Create PACKAGE DATA
 335         #
 336         Str = WriteLine(Str, '// PACKAGE DATA\n')
 337         Str = Write(Str, StrStringValue)
 338
 339         #
 340         # Add an EFI_HII_SIBT_END at last
 341         #
 342         Str = WriteLine(Str, '  ' + EFI_HII_SIBT_END + ",")
 343
 344     #
 345     # Create line for string variable name
 346     # "unsigned char $(BaseName)Strings[] = {"
 347     #
 348     AllStr = WriteLine('', CHAR_ARRAY_DEFIN + ' ' + BaseName + COMMON_FILE_NAME + '[] = {\n' )
 349
 350     #
 351     # Create FRAMEWORK_EFI_HII_PACK_HEADER in compatible mode
 352     #
 353     if IsCompatibleMode:
 354         AllStr = WriteLine(AllStr, '// FRAMEWORK PACKAGE HEADER Length')
 355         AllStr = WriteLine(AllStr, CreateArrayItem(DecToHexList(TotalLength + 2)) + '\n')
 356         AllStr = WriteLine(AllStr, '// FRAMEWORK PACKAGE HEADER Type')
 357         AllStr = WriteLine(AllStr, CreateArrayItem(DecToHexList(2, 4)) + '\n')
 358
 359     #
 360     # Create whole array length in UEFI mode
 361     #
 362     if not IsCompatibleMode:
 363         AllStr = WriteLine(AllStr, '// STRGATHER_OUTPUT_HEADER')
 364         AllStr = WriteLine(AllStr, CreateArrayItem(DecToHexList(TotalLength)) + '\n')
 365
 366     #
 367     # Join package data
 368     #
 369     AllStr = Write(AllStr, Str)
 370
 371     return AllStr
 372
 373 ## Create end of .c file
 374 #
 375 # Create end of .c file
 376 #
 377 # @retval Str:           A string of .h file end
 378 #
 379 def CreateCFileEnd():
 380     Str = Write('', '};')
 381     return Str
 382
 383 ## Create a .c file
 384 #
 385 # Create a complete .c file
 386 #
 387 # @param BaseName:       The basename of strings
 388 # @param UniObjectClass: A UniObjectClass instance
 389 #
 390 # @retval CFile:         A string of complete .c file
 391 #
 392 def CreateCFile(BaseName, UniObjectClass, IsCompatibleMode):
 393     CFile = ''
 394     #CFile = WriteLine(CFile, CreateCFileHeader())
 395     CFile = WriteLine(CFile, CreateCFileContent(BaseName, UniObjectClass, IsCompatibleMode))
 396     CFile = WriteLine(CFile, CreateCFileEnd())
 397     return CFile
 398
 399 ## GetFileList
 400 #
 401 # Get a list for all files
 402 #
 403 # @param IncludeList:  A list of all path to be searched
 404 # @param SkipList:     A list of all types of file could be skipped
 405 #
 406 # @retval FileList:    A list of all files found
 407 #
 408 def GetFileList(SourceFileList, IncludeList, SkipList):
 409     if IncludeList == None:
 410         EdkLogger.error("UnicodeStringGather", AUTOGEN_ERROR, "Include path for unicode file is not defined")
 411
 412     FileList = []
 413     if SkipList == None:
 414         SkipList = []
 415
 416     for File in SourceFileList:
 417         for Dir in IncludeList:
 418             if not os.path.exists(Dir):
 419                 continue
 420             File = os.path.join(Dir, File.Path)
 421             #
 422             # Ignore Dir
 423             #
 424             if os.path.isfile(File) != True:
 425                 continue
 426             #
 427             # Ignore file listed in skip list
 428             #
 429             IsSkip = False
 430             for Skip in SkipList:
 431                 if os.path.splitext(File)[1].upper() == Skip.upper():
 432                     EdkLogger.verbose("Skipped %s for string token uses search" % File)
 433                     IsSkip = True
 434                     break
 435
 436             if not IsSkip:
 437                 FileList.append(File)
 438
 439             break
 440
 441     return FileList
 442
 443 ## SearchString
 444 #
 445 # Search whether all string defined in UniObjectClass are referenced
 446 # All string used should be set to Referenced
 447 #
 448 # @param UniObjectClass:  Input UniObjectClass
 449 # @param FileList:        Search path list
 450 #
 451 # @retval UniObjectClass: UniObjectClass after searched
 452 #
 453 def SearchString(UniObjectClass, FileList):
 454     if FileList == []:
 455         return UniObjectClass
 456
 457     for File in FileList:
 458         if os.path.isfile(File):
 459             Lines = open(File, 'r')
 460             for Line in Lines:
 461                 StringTokenList = STRING_TOKEN.findall(Line)
 462                 for StrName in StringTokenList:
 463                     EdkLogger.debug(EdkLogger.DEBUG_5, "Found string identifier: " + StrName)
 464                     UniObjectClass.SetStringReferenced(StrName)
 465
 466     UniObjectClass.ReToken()
 467
 468     return UniObjectClass
 469
 470 ## GetStringFiles
 471 #
 472 # This function is used for UEFI2.1 spec
 473 #
 474 #
 475 def GetStringFiles(UniFilList, SourceFileList, IncludeList, SkipList, BaseName, IsCompatibleMode = False, ShellMode = False):
 476     Status = True
 477     ErrorMessage = ''
 478
 479     if len(UniFilList) > 0:
 480         if ShellMode:
 481             #
 482             # support ISO 639-2 codes in .UNI files of EDK Shell
 483             #
 484             Uni = UniFileClassObject(UniFilList, True)
 485         else:
 486             Uni = UniFileClassObject(UniFilList, IsCompatibleMode)
 487     else:
 488         EdkLogger.error("UnicodeStringGather", AUTOGEN_ERROR, 'No unicode files given')
 489
 490     FileList = GetFileList(SourceFileList, IncludeList, SkipList)
 491
 492     Uni = SearchString(Uni, FileList)
 493
 494     HFile = CreateHFile(BaseName, Uni)
 495     CFile = CreateCFile(BaseName, Uni, IsCompatibleMode)
 496
 497     return HFile, CFile
 498
 499 #
 500 # Write an item
 501 #
 502 def Write(Target, Item):
 503     return Target + Item
 504
 505 #
 506 # Write an item with a break line
 507 #
 508 def WriteLine(Target, Item):
 509     return Target + Item + '\n'
 510
 511 # This acts like the main() function for the script, unless it is 'import'ed into another
 512 # script.
 513 if __name__ == '__main__':
 514     EdkLogger.info('start')
 515
 516     UniFileList = [
 517                    r'C:\\Edk\\Strings2.uni',
 518                    r'C:\\Edk\\Strings.uni'
 519     ]
 520
 521     SrcFileList = []
 522     for Root, Dirs, Files in os.walk('C:\\Edk'):
 523         for File in Files:
 524             SrcFileList.append(File)
 525
 526     IncludeList = [
 527                    r'C:\\Edk'
 528     ]
 529
 530     SkipList = ['.inf', '.uni']
 531     BaseName = 'DriverSample'
 532     (h, c) = GetStringFiles(UniFileList, SrcFileList, IncludeList, SkipList, BaseName, True)
 533     hfile = open('unistring.h', 'w')
 534     cfile = open('unistring.c', 'w')
 535     hfile.write(h)
 536     cfile.write(c)
 537
 538     EdkLogger.info('end')