IntelFsp2Pkg/Tools/ConfigEditor/GenYamlCfg.py

   1 # @ GenYamlCfg.py
   2 #
   3 # Copyright (c) 2020 - 2021, Intel Corporation. All rights reserved.<BR>
   4 # SPDX-License-Identifier: BSD-2-Clause-Patent
   5 #
   6 #
   7
   8 import os
   9 import sys
  10 import re
  11 import marshal
  12 import string
  13 import operator as op
  14 import ast
  15 import tkinter.messagebox as messagebox
  16 import tkinter
  17
  18 from datetime import date
  19 from collections import OrderedDict
  20 from CommonUtility import value_to_bytearray, value_to_bytes, \
  21       bytes_to_value, get_bits_from_bytes, set_bits_to_bytes
  22
  23 # Generated file copyright header
  24 __copyright_tmp__ = """/** @file
  25
  26   Platform Configuration %s File.
  27
  28   Copyright (c) %4d, Intel Corporation. All rights reserved.<BR>
  29   SPDX-License-Identifier: BSD-2-Clause-Patent
  30
  31   This file is automatically generated. Please do NOT modify !!!
  32
  33 **/
  34 """
  35
  36
  37 def get_copyright_header(file_type, allow_modify=False):
  38     file_description = {
  39         'yaml': 'Boot Setting',
  40         'dlt': 'Delta',
  41         'inc': 'C Binary Blob',
  42         'h': 'C Struct Header'
  43     }
  44     if file_type in ['yaml', 'dlt']:
  45         comment_char = '#'
  46     else:
  47         comment_char = ''
  48     lines = __copyright_tmp__.split('\n')
  49     if allow_modify:
  50         lines = [line for line in lines if 'Please do NOT modify' not in line]
  51     copyright_hdr = '\n'.join('%s%s' % (comment_char, line)
  52                               for line in lines)[:-1] + '\n'
  53     return copyright_hdr % (file_description[file_type], date.today().year)
  54
  55
  56 def check_quote(text):
  57     if (text[0] == "'" and text[-1] == "'") or (text[0] == '"'
  58                                                 and text[-1] == '"'):
  59         return True
  60     return False
  61
  62
  63 def strip_quote(text):
  64     new_text = text.strip()
  65     if check_quote(new_text):
  66         return new_text[1:-1]
  67     return text
  68
  69
  70 def strip_delimiter(text, delim):
  71     new_text = text.strip()
  72     if new_text:
  73         if new_text[0] == delim[0] and new_text[-1] == delim[-1]:
  74             return new_text[1:-1]
  75     return text
  76
  77
  78 def bytes_to_bracket_str(bytes):
  79     return '{ %s }' % (', '.join('0x%02x' % i for i in bytes))
  80
  81
  82 def array_str_to_value(val_str):
  83     val_str = val_str.strip()
  84     val_str = strip_delimiter(val_str, '{}')
  85     val_str = strip_quote(val_str)
  86     value = 0
  87     for each in val_str.split(',')[::-1]:
  88         each = each.strip()
  89         value = (value << 8) | int(each, 0)
  90     return value
  91
  92
  93 def write_lines(lines, file):
  94     fo = open(file, "w")
  95     fo.write(''.join([x[0] for x in lines]))
  96     fo.close()
  97
  98
  99 def read_lines(file):
 100     if not os.path.exists(file):
 101         test_file = os.path.basename(file)
 102         if os.path.exists(test_file):
 103             file = test_file
 104     fi = open(file, 'r')
 105     lines = fi.readlines()
 106     fi.close()
 107     return lines
 108
 109
 110 def expand_file_value(path, value_str):
 111     result = bytearray()
 112     match = re.match("\\{\\s*FILE:(.+)\\}", value_str)
 113     if match:
 114         file_list = match.group(1).split(',')
 115         for file in file_list:
 116             file = file.strip()
 117             bin_path = os.path.join(path, file)
 118             result.extend(bytearray(open(bin_path, 'rb').read()))
 119             print('\n\n result ', result)
 120     return result
 121
 122
 123 class ExpressionEval(ast.NodeVisitor):
 124     operators = {
 125         ast.Add:    op.add,
 126         ast.Sub:    op.sub,
 127         ast.Mult:   op.mul,
 128         ast.Div:    op.floordiv,
 129         ast.Mod:    op.mod,
 130         ast.Eq:     op.eq,
 131         ast.NotEq:  op.ne,
 132         ast.Gt:     op.gt,
 133         ast.Lt:     op.lt,
 134         ast.GtE:    op.ge,
 135         ast.LtE:    op.le,
 136         ast.BitXor: op.xor,
 137         ast.BitAnd: op.and_,
 138         ast.BitOr:  op.or_,
 139         ast.Invert: op.invert,
 140         ast.USub:   op.neg
 141     }
 142
 143     def __init__(self):
 144         self._debug = False
 145         self._expression = ''
 146         self._namespace = {}
 147         self._get_variable = None
 148
 149     def eval(self, expr, vars={}):
 150         self._expression = expr
 151         if type(vars) is dict:
 152             self._namespace = vars
 153             self._get_variable = None
 154         else:
 155             self._namespace = {}
 156             self._get_variable = vars
 157         node = ast.parse(self._expression, mode='eval')
 158         result = self.visit(node.body)
 159         if self._debug:
 160             print('EVAL [ %s ] = %s' % (expr, str(result)))
 161         return result
 162
 163     def visit_Name(self, node):
 164         if self._get_variable is not None:
 165             return self._get_variable(node.id)
 166         else:
 167             return self._namespace[node.id]
 168
 169     def visit_Num(self, node):
 170         return node.n
 171
 172     def visit_NameConstant(self, node):
 173         return node.value
 174
 175     def visit_BoolOp(self, node):
 176         result = False
 177         if isinstance(node.op, ast.And):
 178             for value in node.values:
 179                 result = self.visit(value)
 180                 if not result:
 181                     break
 182         elif isinstance(node.op, ast.Or):
 183             for value in node.values:
 184                 result = self.visit(value)
 185                 if result:
 186                     break
 187         return True if result else False
 188
 189     def visit_UnaryOp(self, node):
 190         val = self.visit(node.operand)
 191         return ExpressionEval.operators[type(node.op)](val)
 192
 193     def visit_BinOp(self, node):
 194         lhs = self.visit(node.left)
 195         rhs = self.visit(node.right)
 196         return ExpressionEval.operators[type(node.op)](lhs, rhs)
 197
 198     def visit_Compare(self, node):
 199         right = self.visit(node.left)
 200         result = True
 201         for operation, comp in zip(node.ops, node.comparators):
 202             if not result:
 203                 break
 204             left = right
 205             right = self.visit(comp)
 206             result = ExpressionEval.operators[type(operation)](left, right)
 207         return result
 208
 209     def visit_Call(self, node):
 210         if node.func.id in ['ternary']:
 211             condition = self.visit(node.args[0])
 212             val_true = self.visit(node.args[1])
 213             val_false = self.visit(node.args[2])
 214             return val_true if condition else val_false
 215         elif node.func.id in ['offset', 'length']:
 216             if self._get_variable is not None:
 217                 return self._get_variable(node.args[0].s, node.func.id)
 218         else:
 219             raise ValueError("Unsupported function: " + repr(node))
 220
 221     def generic_visit(self, node):
 222         raise ValueError("malformed node or string: " + repr(node))
 223
 224
 225 class CFG_YAML():
 226     TEMPLATE = 'template'
 227     CONFIGS = 'configs'
 228     VARIABLE = 'variable'
 229
 230     def __init__(self):
 231         self.log_line = False
 232         self.allow_template = False
 233         self.cfg_tree = None
 234         self.tmp_tree = None
 235         self.var_dict = None
 236         self.def_dict = {}
 237         self.yaml_path = ''
 238         self.lines = []
 239         self.full_lines = []
 240         self.index = 0
 241         self.re_expand = re.compile(
 242             r'(.+:\s+|\s*\-\s*)!expand\s+\{\s*(\w+_TMPL)\s*:\s*\[(.+)]\s*\}')
 243         self.re_include = re.compile(r'(.+:\s+|\s*\-\s*)!include\s+(.+)')
 244
 245     @staticmethod
 246     def count_indent(line):
 247         return next((i for i, c in enumerate(line) if not c.isspace()),
 248                     len(line))
 249
 250     @staticmethod
 251     def substitue_args(text, arg_dict):
 252         for arg in arg_dict:
 253             text = text.replace('$' + arg, arg_dict[arg])
 254         return text
 255
 256     @staticmethod
 257     def dprint(*args):
 258         pass
 259
 260     def process_include(self, line, insert=True):
 261         match = self.re_include.match(line)
 262         if not match:
 263             raise Exception("Invalid !include format '%s' !" % line.strip())
 264
 265         prefix = match.group(1)
 266         include = match.group(2)
 267         if prefix.strip() == '-':
 268             prefix = ''
 269             adjust = 0
 270         else:
 271             adjust = 2
 272
 273         include = strip_quote(include)
 274         request = CFG_YAML.count_indent(line) + adjust
 275
 276         if self.log_line:
 277             # remove the include line itself
 278             del self.full_lines[-1]
 279
 280         inc_path = os.path.join(self.yaml_path, include)
 281         if not os.path.exists(inc_path):
 282             # try relative path to project root
 283             try_path = os.path.join(os.path.dirname(os.path.realpath(__file__)
 284                                                     ), "../..", include)
 285             if os.path.exists(try_path):
 286                 inc_path = try_path
 287             else:
 288                 raise Exception("ERROR: Cannot open file '%s'." % inc_path)
 289
 290         lines = read_lines(inc_path)
 291         current = 0
 292         same_line = False
 293         for idx, each in enumerate(lines):
 294             start = each.lstrip()
 295             if start == '' or start[0] == '#':
 296                 continue
 297
 298             if start[0] == '>':
 299                 # append the content directly at the same line
 300                 same_line = True
 301
 302             start = idx
 303             current = CFG_YAML.count_indent(each)
 304             break
 305
 306         lines = lines[start+1:] if same_line else lines[start:]
 307         leading = ''
 308         if same_line:
 309             request = len(prefix)
 310             leading = '>'
 311
 312         lines = [prefix + '%s\n' % leading] + [' ' * request +
 313                                                i[current:] for i in lines]
 314         if insert:
 315             self.lines = lines + self.lines
 316
 317         return lines
 318
 319     def process_expand(self, line):
 320         match = self.re_expand.match(line)
 321         if not match:
 322             raise Exception("Invalid !expand format '%s' !" % line.strip())
 323         lines = []
 324         prefix = match.group(1)
 325         temp_name = match.group(2)
 326         args = match.group(3)
 327
 328         if prefix.strip() == '-':
 329             indent = 0
 330         else:
 331             indent = 2
 332         lines = self.process_expand_template(temp_name, prefix, args, indent)
 333         self.lines = lines + self.lines
 334
 335     def process_expand_template(self, temp_name, prefix, args, indent=2):
 336         # expand text with arg substitution
 337         if temp_name not in self.tmp_tree:
 338             raise Exception("Could not find template '%s' !" % temp_name)
 339         parts = args.split(',')
 340         parts = [i.strip() for i in parts]
 341         num = len(parts)
 342         arg_dict = dict(zip(['(%d)' % (i + 1) for i in range(num)], parts))
 343         str_data = self.tmp_tree[temp_name]
 344         text = DefTemplate(str_data).safe_substitute(self.def_dict)
 345         text = CFG_YAML.substitue_args(text, arg_dict)
 346         target = CFG_YAML.count_indent(prefix) + indent
 347         current = CFG_YAML.count_indent(text)
 348         padding = target * ' '
 349         if indent == 0:
 350             leading = []
 351         else:
 352             leading = [prefix + '\n']
 353         text = leading + [(padding + i + '\n')[current:]
 354                           for i in text.splitlines()]
 355         return text
 356
 357     def load_file(self, yaml_file):
 358         self.index = 0
 359         self.lines = read_lines(yaml_file)
 360
 361     def peek_line(self):
 362         if len(self.lines) == 0:
 363             return None
 364         else:
 365             return self.lines[0]
 366
 367     def put_line(self, line):
 368         self.lines.insert(0, line)
 369         if self.log_line:
 370             del self.full_lines[-1]
 371
 372     def get_line(self):
 373         if len(self.lines) == 0:
 374             return None
 375         else:
 376             line = self.lines.pop(0)
 377             if self.log_line:
 378                 self.full_lines.append(line.rstrip())
 379             return line
 380
 381     def get_multiple_line(self, indent):
 382         text = ''
 383         newind = indent + 1
 384         while True:
 385             line = self.peek_line()
 386             if line is None:
 387                 break
 388             sline = line.strip()
 389             if sline != '':
 390                 newind = CFG_YAML.count_indent(line)
 391                 if newind <= indent:
 392                     break
 393             self.get_line()
 394             if sline != '':
 395                 text = text + line
 396         return text
 397
 398     def traverse_cfg_tree(self, handler):
 399         def _traverse_cfg_tree(root, level=0):
 400             # config structure
 401             for key in root:
 402                 if type(root[key]) is OrderedDict:
 403                     level += 1
 404                     handler(key, root[key], level)
 405                     _traverse_cfg_tree(root[key], level)
 406                     level -= 1
 407         _traverse_cfg_tree(self.cfg_tree)
 408
 409     def count(self):
 410         def _count(name, cfgs, level):
 411             num[0] += 1
 412         num = [0]
 413         self.traverse_cfg_tree(_count)
 414         return num[0]
 415
 416     def parse(self, parent_name='', curr=None, level=0):
 417         child = None
 418         last_indent = None
 419         key = ''
 420         temp_chk = {}
 421
 422         while True:
 423             line = self.get_line()
 424             if line is None:
 425                 break
 426
 427             curr_line = line.strip()
 428             if curr_line == '' or curr_line[0] == '#':
 429                 continue
 430
 431             indent = CFG_YAML.count_indent(line)
 432             if last_indent is None:
 433                 last_indent = indent
 434
 435             if indent != last_indent:
 436                 # outside of current block,  put the line back to queue
 437                 self.put_line(' ' * indent + curr_line)
 438
 439             if curr_line.endswith(': >'):
 440                 # multiline marker
 441                 old_count = len(self.full_lines)
 442                 line = self.get_multiple_line(indent)
 443                 if self.log_line and not self.allow_template \
 444                    and '!include ' in line:
 445                     # expand include in template
 446                     new_lines = []
 447                     lines = line.splitlines()
 448                     for idx, each in enumerate(lines):
 449                         if '!include ' in each:
 450                             new_line = ''.join(self.process_include(each,
 451                                                                     False))
 452                             new_lines.append(new_line)
 453                         else:
 454                             new_lines.append(each)
 455                     self.full_lines = self.full_lines[:old_count] + new_lines
 456                 curr_line = curr_line + line
 457
 458             if indent > last_indent:
 459                 # child nodes
 460                 if child is None:
 461                     raise Exception('Unexpected format at line: %s'
 462                                     % (curr_line))
 463
 464                 level += 1
 465                 self.parse(key, child, level)
 466                 level -= 1
 467                 line = self.peek_line()
 468                 if line is not None:
 469                     curr_line = line.strip()
 470                     indent = CFG_YAML.count_indent(line)
 471                     if indent >= last_indent:
 472                         # consume the line
 473                         self.get_line()
 474                 else:
 475                     # end of file
 476                     indent = -1
 477
 478             if curr is None:
 479                 curr = OrderedDict()
 480
 481             if indent < last_indent:
 482                 return curr
 483
 484             marker1 = curr_line[0]
 485             marker2 = curr_line[-1]
 486             start = 1 if marker1 == '-' else 0
 487             pos = curr_line.find(': ')
 488             if pos > 0:
 489                 child = None
 490                 key = curr_line[start:pos].strip()
 491                 if curr_line[pos + 2] == '>':
 492                     curr[key] = curr_line[pos + 3:]
 493                 else:
 494                     # XXXX: !include / !expand
 495                     if '!include ' in curr_line:
 496                         self.process_include(line)
 497                     elif '!expand ' in curr_line:
 498                         if self.allow_template and not self.log_line:
 499                             self.process_expand(line)
 500                     else:
 501                         value_str = curr_line[pos + 2:].strip()
 502                         curr[key] = value_str
 503                         if self.log_line and value_str[0] == '{':
 504                             # expand {FILE: xxxx} format in the log line
 505                             if value_str[1:].rstrip().startswith('FILE:'):
 506                                 value_bytes = expand_file_value(
 507                                     self.yaml_path, value_str)
 508                                 value_str = bytes_to_bracket_str(value_bytes)
 509                                 self.full_lines[-1] = line[
 510                                     :indent] + curr_line[:pos + 2] + value_str
 511
 512             elif marker2 == ':':
 513                 child = OrderedDict()
 514                 key = curr_line[start:-1].strip()
 515                 if key == '$ACTION':
 516                     # special virtual nodes, rename to ensure unique key
 517                     key = '$ACTION_%04X' % self.index
 518                     self.index += 1
 519                 if key in curr:
 520                     if key not in temp_chk:
 521                         # check for duplicated keys at same level
 522                         temp_chk[key] = 1
 523                     else:
 524                         raise Exception("Duplicated item '%s:%s' found !"
 525                                         % (parent_name, key))
 526
 527                 curr[key] = child
 528                 if self.var_dict is None and key == CFG_YAML.VARIABLE:
 529                     self.var_dict = child
 530                 if self.tmp_tree is None and key == CFG_YAML.TEMPLATE:
 531                     self.tmp_tree = child
 532                     if self.var_dict:
 533                         for each in self.var_dict:
 534                             txt = self.var_dict[each]
 535                             if type(txt) is str:
 536                                 self.def_dict['(%s)' % each] = txt
 537                 if self.tmp_tree and key == CFG_YAML.CONFIGS:
 538                     # apply template for the main configs
 539                     self.allow_template = True
 540             else:
 541                 child = None
 542                 # - !include cfg_opt.yaml
 543                 if '!include ' in curr_line:
 544                     self.process_include(line)
 545
 546         return curr
 547
 548     def load_yaml(self, opt_file):
 549         self.var_dict = None
 550         self.yaml_path = os.path.dirname(opt_file)
 551         self.load_file(opt_file)
 552         yaml_tree = self.parse()
 553         self.tmp_tree = yaml_tree[CFG_YAML.TEMPLATE]
 554         self.cfg_tree = yaml_tree[CFG_YAML.CONFIGS]
 555         return self.cfg_tree
 556
 557     def expand_yaml(self, opt_file):
 558         self.log_line = True
 559         self.load_yaml(opt_file)
 560         self.log_line = False
 561         text = '\n'.join(self.full_lines)
 562         self.full_lines = []
 563         return text
 564
 565
 566 class DefTemplate(string.Template):
 567     idpattern = '\\([_A-Z][_A-Z0-9]*\\)|[_A-Z][_A-Z0-9]*'
 568
 569
 570 class CGenYamlCfg:
 571     STRUCT = '$STRUCT'
 572     bits_width = {'b': 1, 'B': 8, 'W': 16, 'D': 32, 'Q': 64}
 573     builtin_option = {'$EN_DIS': [('0', 'Disable'), ('1', 'Enable')]}
 574     exclude_struct = ['FSP_UPD_HEADER', 'FSPT_ARCH_UPD',
 575                       'FSPM_ARCH_UPD', 'FSPS_ARCH_UPD',
 576                       'GPIO_GPP_*', 'GPIO_CFG_DATA',
 577                       'GpioConfPad*', 'GpioPinConfig',
 578                       'BOOT_OPTION*', 'PLATFORMID_CFG_DATA', '\\w+_Half[01]']
 579     include_tag = ['GPIO_CFG_DATA']
 580     keyword_set = set(['name', 'type', 'option', 'help', 'length',
 581                        'value', 'order', 'struct', 'condition'])
 582
 583     def __init__(self):
 584         self._mode = ''
 585         self._debug = False
 586         self._macro_dict = {}
 587         self.binseg_dict = {}
 588         self.initialize()
 589
 590     def initialize(self):
 591         self._old_bin = None
 592         self._cfg_tree = {}
 593         self._tmp_tree = {}
 594         self._cfg_list = []
 595         self._cfg_page = {'root': {'title': '', 'child': []}}
 596         self._cur_page = ''
 597         self._var_dict = {}
 598         self._def_dict = {}
 599         self._yaml_path = ''
 600
 601     @staticmethod
 602     def deep_convert_dict(layer):
 603         # convert OrderedDict to list + dict
 604         new_list = layer
 605         if isinstance(layer, OrderedDict):
 606             new_list = list(layer.items())
 607             for idx, pair in enumerate(new_list):
 608                 new_node = CGenYamlCfg.deep_convert_dict(pair[1])
 609                 new_list[idx] = dict({pair[0]: new_node})
 610         return new_list
 611
 612     @staticmethod
 613     def deep_convert_list(layer):
 614         if isinstance(layer, list):
 615             od = OrderedDict({})
 616             for each in layer:
 617                 if isinstance(each, dict):
 618                     key = next(iter(each))
 619                     od[key] = CGenYamlCfg.deep_convert_list(each[key])
 620             return od
 621         else:
 622             return layer
 623
 624     @staticmethod
 625     def expand_include_files(file_path, cur_dir=''):
 626         if cur_dir == '':
 627             cur_dir = os.path.dirname(file_path)
 628             file_path = os.path.basename(file_path)
 629
 630         input_file_path = os.path.join(cur_dir, file_path)
 631         file = open(input_file_path, "r")
 632         lines = file.readlines()
 633         file.close()
 634         new_lines = []
 635         for line_num, line in enumerate(lines):
 636             match = re.match("^!include\\s*(.+)?$", line.strip())
 637             if match:
 638                 inc_path = match.group(1)
 639                 tmp_path = os.path.join(cur_dir, inc_path)
 640                 org_path = tmp_path
 641                 if not os.path.exists(tmp_path):
 642                     cur_dir = os.path.join(os.path.dirname
 643                                            (os.path.realpath(__file__)
 644                                             ), "..", "..")
 645                 tmp_path = os.path.join(cur_dir, inc_path)
 646                 if not os.path.exists(tmp_path):
 647                     raise Exception("ERROR: Cannot open include\
 648                                     file '%s'." % org_path)
 649                 else:
 650                     new_lines.append(('# Included from file: %s\n' % inc_path,
 651                                       tmp_path, 0))
 652                     new_lines.append(('# %s\n' % ('=' * 80), tmp_path, 0))
 653                     new_lines.extend(CGenYamlCfg.expand_include_files
 654                                      (inc_path, cur_dir))
 655             else:
 656                 new_lines.append((line, input_file_path, line_num))
 657
 658         return new_lines
 659
 660     @staticmethod
 661     def format_struct_field_name(input, count=0):
 662         name = ''
 663         cap = True
 664         if '_' in input:
 665             input = input.lower()
 666         for each in input:
 667             if each == '_':
 668                 cap = True
 669                 continue
 670             elif cap:
 671                 each = each.upper()
 672                 cap = False
 673             name = name + each
 674
 675         if count > 1:
 676             name = '%s[%d]' % (name, count)
 677
 678         return name
 679
 680     def get_mode(self):
 681         return self._mode
 682
 683     def set_mode(self, mode):
 684         self._mode = mode
 685
 686     def get_last_error(self):
 687         return ''
 688
 689     def get_variable(self, var, attr='value'):
 690         if var in self._var_dict:
 691             var = self._var_dict[var]
 692             return var
 693
 694         item = self.locate_cfg_item(var, False)
 695         if item is None:
 696             raise ValueError("Cannot find variable '%s' !" % var)
 697
 698         if item:
 699             if 'indx' in item:
 700                 item = self.get_item_by_index(item['indx'])
 701             if attr == 'offset':
 702                 var = item['offset']
 703             elif attr == 'length':
 704                 var = item['length']
 705             elif attr == 'value':
 706                 var = self.get_cfg_item_value(item)
 707             else:
 708                 raise ValueError("Unsupported variable attribute '%s' !" %
 709                                  attr)
 710         return var
 711
 712     def eval(self, expr):
 713         def _handler(pattern):
 714             if pattern.group(1):
 715                 target = 1
 716             else:
 717                 target = 2
 718             result = self.get_variable(pattern.group(target))
 719             if result is None:
 720                 raise ValueError('Unknown variable $(%s) !' %
 721                                  pattern.group(target))
 722             return hex(result)
 723
 724         expr_eval = ExpressionEval()
 725         if '$' in expr:
 726             # replace known variable first
 727             expr = re.sub(r'\$\(([_a-zA-Z][\w\.]*)\)|\$([_a-zA-Z][\w\.]*)',
 728                           _handler, expr)
 729         return expr_eval.eval(expr, self.get_variable)
 730
 731     def parse_macros(self, macro_def_str):
 732         # ['-DABC=1', '-D', 'CFG_DEBUG=1', '-D', 'CFG_OUTDIR=Build']
 733         self._macro_dict = {}
 734         is_expression = False
 735         for macro in macro_def_str:
 736             if macro.startswith('-D'):
 737                 is_expression = True
 738                 if len(macro) > 2:
 739                     macro = macro[2:]
 740                 else:
 741                     continue
 742             if is_expression:
 743                 is_expression = False
 744                 match = re.match("(\\w+)=(.+)", macro)
 745                 if match:
 746                     self._macro_dict[match.group(1)] = match.group(2)
 747                 else:
 748                     match = re.match("(\\w+)", macro)
 749                     if match:
 750                         self._macro_dict[match.group(1)] = ''
 751         if len(self._macro_dict) == 0:
 752             error = 1
 753         else:
 754             error = 0
 755             if self._debug:
 756                 print("INFO : Macro dictionary:")
 757                 for each in self._macro_dict:
 758                     print("       $(%s) = [ %s ]"
 759                           % (each, self._macro_dict[each]))
 760         return error
 761
 762     def get_cfg_list(self, page_id=None):
 763         if page_id is None:
 764             # return full list
 765             return self._cfg_list
 766         else:
 767             # build a new list for items under a page ID
 768             cfgs = [i for i in self._cfg_list if i['cname'] and
 769                     (i['page'] == page_id)]
 770             return cfgs
 771
 772     def get_cfg_page(self):
 773         return self._cfg_page
 774
 775     def get_cfg_item_length(self, item):
 776         return item['length']
 777
 778     def get_cfg_item_value(self, item, array=False):
 779         value_str = item['value']
 780         length = item['length']
 781         return self.get_value(value_str, length, array)
 782
 783     def format_value_to_str(self, value, bit_length, old_value=''):
 784         # value is always int
 785         length = (bit_length + 7) // 8
 786         fmt = ''
 787         if old_value.startswith('0x'):
 788             fmt = '0x'
 789         elif old_value and (old_value[0] in ['"', "'", '{']):
 790             fmt = old_value[0]
 791         else:
 792             fmt = ''
 793
 794         bvalue = value_to_bytearray(value, length)
 795         if fmt in ['"', "'"]:
 796             svalue = bvalue.rstrip(b'\x00').decode()
 797             value_str = fmt + svalue + fmt
 798         elif fmt == "{":
 799             value_str = '{ ' + ', '.join(['0x%02x' % i for i in bvalue]) + ' }'
 800         elif fmt == '0x':
 801             hex_len = length * 2
 802             if len(old_value) == hex_len + 2:
 803                 fstr = '0x%%0%dx' % hex_len
 804             else:
 805                 fstr = '0x%x'
 806             value_str = fstr % value
 807         else:
 808             if length <= 2:
 809                 value_str = '%d' % value
 810             elif length <= 8:
 811                 value_str = '0x%x' % value
 812             else:
 813                 value_str = '{ ' + ', '.join(['0x%02x' % i for i in
 814                                               bvalue]) + ' }'
 815         return value_str
 816
 817     def reformat_value_str(self, value_str, bit_length, old_value=None):
 818         value = self.parse_value(value_str, bit_length, False)
 819         if old_value is None:
 820             old_value = value_str
 821         new_value = self.format_value_to_str(value, bit_length, old_value)
 822         return new_value
 823
 824     def get_value(self, value_str, bit_length, array=True):
 825         value_str = value_str.strip()
 826         if value_str[0] == "'" and value_str[-1] == "'" or \
 827            value_str[0] == '"' and value_str[-1] == '"':
 828             value_str = value_str[1:-1]
 829             bvalue = bytearray(value_str.encode())
 830             if len(bvalue) == 0:
 831                 bvalue = bytearray(b'\x00')
 832             if array:
 833                 return bvalue
 834             else:
 835                 return bytes_to_value(bvalue)
 836         else:
 837             if value_str[0] in '{':
 838                 value_str = value_str[1:-1].strip()
 839             value = 0
 840             for each in value_str.split(',')[::-1]:
 841                 each = each.strip()
 842                 value = (value << 8) | int(each, 0)
 843             if array:
 844                 length = (bit_length + 7) // 8
 845                 return value_to_bytearray(value, length)
 846             else:
 847                 return value
 848
 849     def parse_value(self, value_str, bit_length, array=True):
 850         length = (bit_length + 7) // 8
 851         if check_quote(value_str):
 852             value_str = bytes_to_bracket_str(value_str[1:-1].encode())
 853         elif (',' in value_str) and (value_str[0] != '{'):
 854             value_str = '{ %s }' % value_str
 855         if value_str[0] == '{':
 856             result = expand_file_value(self._yaml_path, value_str)
 857             if len(result) == 0:
 858                 bin_list = value_str[1:-1].split(',')
 859                 value = 0
 860                 bit_len = 0
 861                 unit_len = 1
 862                 for idx, element in enumerate(bin_list):
 863                     each = element.strip()
 864                     if len(each) == 0:
 865                         continue
 866
 867                     in_bit_field = False
 868                     if each[0] in "'" + '"':
 869                         each_value = bytearray(each[1:-1], 'utf-8')
 870                     elif ':' in each:
 871                         match = re.match("^(.+):(\\d+)([b|B|W|D|Q])$", each)
 872                         if match is None:
 873                             raise SystemExit("Exception: Invald value\
 874 list format '%s' !" % each)
 875                         if match.group(1) == '0' and match.group(2) == '0':
 876                             unit_len = CGenYamlCfg.bits_width[match.group(3)
 877                                                               ] // 8
 878                         cur_bit_len = int(match.group(2)
 879                                           ) * CGenYamlCfg.bits_width[
 880                                               match.group(3)]
 881                         value += ((self.eval(match.group(1)) & (
 882                             1 << cur_bit_len) - 1)) << bit_len
 883                         bit_len += cur_bit_len
 884                         each_value = bytearray()
 885                         if idx + 1 < len(bin_list):
 886                             in_bit_field = True
 887                     else:
 888                         try:
 889                             each_value = value_to_bytearray(
 890                                 self.eval(each.strip()), unit_len)
 891                         except Exception:
 892                             raise SystemExit("Exception: Value %d cannot \
 893 fit into %s bytes !" % (each, unit_len))
 894
 895                     if not in_bit_field:
 896                         if bit_len > 0:
 897                             if bit_len % 8 != 0:
 898                                 raise SystemExit("Exception: Invalid bit \
 899 field alignment '%s' !" % value_str)
 900                             result.extend(value_to_bytes(value, bit_len // 8))
 901                         value = 0
 902                         bit_len = 0
 903
 904                     result.extend(each_value)
 905
 906         elif check_quote(value_str):
 907             result = bytearray(value_str[1:-1], 'utf-8')  # Excluding quotes
 908         else:
 909             result = value_to_bytearray(self.eval(value_str), length)
 910
 911         if len(result) < length:
 912             result.extend(b'\x00' * (length - len(result)))
 913         elif len(result) > length:
 914             raise SystemExit("Exception: Value '%s' is too big to fit \
 915 into %d bytes !" % (value_str, length))
 916
 917         if array:
 918             return result
 919         else:
 920             return bytes_to_value(result)
 921
 922         return result
 923
 924     def get_cfg_item_options(self, item):
 925         tmp_list = []
 926         if item['type'] == "Combo":
 927             if item['option'] in CGenYamlCfg.builtin_option:
 928                 for op_val, op_str in CGenYamlCfg.builtin_option[item['option'
 929                                                                       ]]:
 930                     tmp_list.append((op_val, op_str))
 931             else:
 932                 opt_list = item['option'].split(',')
 933                 for option in opt_list:
 934                     option = option.strip()
 935                     try:
 936                         (op_val, op_str) = option.split(':')
 937                     except Exception:
 938                         raise SystemExit("Exception: Invalide \
 939 option format '%s' !" % option)
 940                     tmp_list.append((op_val, op_str))
 941         return tmp_list
 942
 943     def get_page_title(self, page_id, top=None):
 944         if top is None:
 945             top = self.get_cfg_page()['root']
 946         for node in top['child']:
 947             page_key = next(iter(node))
 948             if page_id == page_key:
 949                 return node[page_key]['title']
 950             else:
 951                 result = self.get_page_title(page_id, node[page_key])
 952                 if result is not None:
 953                     return result
 954         return None
 955
 956     def print_pages(self, top=None, level=0):
 957         if top is None:
 958             top = self.get_cfg_page()['root']
 959         for node in top['child']:
 960             page_id = next(iter(node))
 961             print('%s%s: %s' % ('  ' * level, page_id, node[page_id]['title']))
 962             level += 1
 963             self.print_pages(node[page_id], level)
 964             level -= 1
 965
 966     def get_item_by_index(self, index):
 967         return self._cfg_list[index]
 968
 969     def get_item_by_path(self, path):
 970         node = self.locate_cfg_item(path)
 971         if node:
 972             return self.get_item_by_index(node['indx'])
 973         else:
 974             return None
 975
 976     def locate_cfg_path(self, item):
 977         def _locate_cfg_path(root, level=0):
 978             # config structure
 979             if item is root:
 980                 return path
 981             for key in root:
 982                 if type(root[key]) is OrderedDict:
 983                     level += 1
 984                     path.append(key)
 985                     ret = _locate_cfg_path(root[key], level)
 986                     if ret:
 987                         return ret
 988                     path.pop()
 989             return None
 990         path = []
 991         return _locate_cfg_path(self._cfg_tree)
 992
 993     def locate_cfg_item(self, path, allow_exp=True):
 994         def _locate_cfg_item(root, path, level=0):
 995             if len(path) == level:
 996                 return root
 997             next_root = root.get(path[level], None)
 998             if next_root is None:
 999                 if allow_exp:
1000                     raise Exception('Not a valid CFG config option path: %s' %
1001                                     '.'.join(path[:level+1]))
1002                 else:
1003                     return None
1004             return _locate_cfg_item(next_root, path, level + 1)
1005
1006         path_nodes = path.split('.')
1007         return _locate_cfg_item(self._cfg_tree, path_nodes)
1008
1009     def traverse_cfg_tree(self, handler, top=None):
1010         def _traverse_cfg_tree(root, level=0):
1011             # config structure
1012             for key in root:
1013                 if type(root[key]) is OrderedDict:
1014                     level += 1
1015                     handler(key, root[key], level)
1016                     _traverse_cfg_tree(root[key], level)
1017                     level -= 1
1018
1019         if top is None:
1020             top = self._cfg_tree
1021         _traverse_cfg_tree(top)
1022
1023     def print_cfgs(self, root=None, short=True, print_level=256):
1024         def _print_cfgs(name, cfgs, level):
1025
1026             if 'indx' in cfgs:
1027                 act_cfg = self.get_item_by_index(cfgs['indx'])
1028             else:
1029                 offset = 0
1030                 length = 0
1031                 value = ''
1032                 if CGenYamlCfg.STRUCT in cfgs:
1033                     cfg = cfgs[CGenYamlCfg.STRUCT]
1034                     offset = int(cfg['offset'])
1035                     length = int(cfg['length'])
1036                     if 'value' in cfg:
1037                         value = cfg['value']
1038                 if length == 0:
1039                     return
1040                 act_cfg = dict({'value': value, 'offset': offset,
1041                                 'length': length})
1042             value = act_cfg['value']
1043             bit_len = act_cfg['length']
1044             offset = (act_cfg['offset'] + 7) // 8
1045             if value != '':
1046                 try:
1047                     value = self.reformat_value_str(act_cfg['value'],
1048                                                     act_cfg['length'])
1049
1050                 except Exception:
1051                     value = act_cfg['value']
1052             length = bit_len // 8
1053             bit_len = '(%db)' % bit_len if bit_len % 8 else '' * 4
1054             if level <= print_level:
1055                 if short and len(value) > 40:
1056                     value = '%s ... %s' % (value[:20], value[-20:])
1057                 print('%04X:%04X%-6s %s%s : %s' % (offset, length, bit_len,
1058                                                    '  ' * level, name, value))
1059
1060         self.traverse_cfg_tree(_print_cfgs)
1061
1062     def build_var_dict(self):
1063         def _build_var_dict(name, cfgs, level):
1064             if level <= 2:
1065                 if CGenYamlCfg.STRUCT in cfgs:
1066                     struct_info = cfgs[CGenYamlCfg.STRUCT]
1067                     self._var_dict['_LENGTH_%s_' % name] = struct_info[
1068                         'length'] // 8
1069                     self._var_dict['_OFFSET_%s_' % name] = struct_info[
1070                         'offset'] // 8
1071
1072         self._var_dict = {}
1073         self.traverse_cfg_tree(_build_var_dict)
1074         self._var_dict['_LENGTH_'] = self._cfg_tree[CGenYamlCfg.STRUCT][
1075             'length'] // 8
1076         return 0
1077
1078     def add_cfg_page(self, child, parent, title=''):
1079         def _add_cfg_page(cfg_page, child, parent):
1080             key = next(iter(cfg_page))
1081             if parent == key:
1082                 cfg_page[key]['child'].append({child: {'title': title,
1083                                                        'child': []}})
1084                 return True
1085             else:
1086                 result = False
1087                 for each in cfg_page[key]['child']:
1088                     if _add_cfg_page(each, child, parent):
1089                         result = True
1090                         break
1091                 return result
1092
1093         return _add_cfg_page(self._cfg_page, child, parent)
1094
1095     def set_cur_page(self, page_str):
1096         if not page_str:
1097             return
1098
1099         if ',' in page_str:
1100             page_list = page_str.split(',')
1101         else:
1102             page_list = [page_str]
1103         for page_str in page_list:
1104             parts = page_str.split(':')
1105             if len(parts) in [1, 3]:
1106                 page = parts[0].strip()
1107                 if len(parts) == 3:
1108                     # it is a new page definition, add it into tree
1109                     parent = parts[1] if parts[1] else 'root'
1110                     parent = parent.strip()
1111                     if parts[2][0] == '"' and parts[2][-1] == '"':
1112                         parts[2] = parts[2][1:-1]
1113
1114                     if not self.add_cfg_page(page, parent, parts[2]):
1115                         raise SystemExit("Error: Cannot find parent page \
1116 '%s'!" % parent)
1117             else:
1118                 raise SystemExit("Error: Invalid page format '%s' !"
1119                                  % page_str)
1120             self._cur_page = page
1121
1122     def extend_variable(self, line):
1123         # replace all variables
1124         if line == '':
1125             return line
1126         loop = 2
1127         while loop > 0:
1128             line_after = DefTemplate(line).safe_substitute(self._def_dict)
1129             if line == line_after:
1130                 break
1131             loop -= 1
1132             line = line_after
1133         return line_after
1134
1135     def reformat_number_per_type(self, itype, value):
1136         if check_quote(value) or value.startswith('{'):
1137             return value
1138         parts = itype.split(',')
1139         if len(parts) > 3 and parts[0] == 'EditNum':
1140             num_fmt = parts[1].strip()
1141         else:
1142             num_fmt = ''
1143         if num_fmt == 'HEX' and not value.startswith('0x'):
1144             value = '0x%X' % int(value, 10)
1145         elif num_fmt == 'DEC' and value.startswith('0x'):
1146             value = '%d' % int(value, 16)
1147         return value
1148
1149     def add_cfg_item(self, name, item, offset, path):
1150
1151         self.set_cur_page(item.get('page', ''))
1152
1153         if name[0] == '$':
1154             # skip all virtual node
1155             return 0
1156
1157         if not set(item).issubset(CGenYamlCfg.keyword_set):
1158             for each in list(item):
1159                 if each not in CGenYamlCfg.keyword_set:
1160                     raise Exception("Invalid attribute '%s' for '%s'!" %
1161                                     (each, '.'.join(path)))
1162
1163         length = item.get('length', 0)
1164         if type(length) is str:
1165             match = re.match("^(\\d+)([b|B|W|D|Q])([B|W|D|Q]?)\\s*$", length)
1166             if match:
1167                 unit_len = CGenYamlCfg.bits_width[match.group(2)]
1168                 length = int(match.group(1), 10) * unit_len
1169             else:
1170                 try:
1171                     length = int(length, 0) * 8
1172                 except Exception:
1173                     raise Exception("Invalid length field '%s' for '%s' !" %
1174                                     (length, '.'.join(path)))
1175
1176                 if offset % 8 > 0:
1177                     raise Exception("Invalid alignment for field '%s' for \
1178 '%s' !" % (name, '.'.join(path)))
1179         else:
1180             # define is length in bytes
1181             length = length * 8
1182
1183         if not name.isidentifier():
1184             raise Exception("Invalid config name '%s' for '%s' !" %
1185                             (name, '.'.join(path)))
1186
1187         itype = str(item.get('type', 'Reserved'))
1188         value = str(item.get('value', ''))
1189         if value:
1190             if not (check_quote(value) or value.startswith('{')):
1191                 if ',' in value:
1192                     value = '{ %s }' % value
1193                 else:
1194                     value = self.reformat_number_per_type(itype, value)
1195
1196         help = str(item.get('help', ''))
1197         if '\n' in help:
1198             help = ' '.join([i.strip() for i in help.splitlines()])
1199
1200         option = str(item.get('option', ''))
1201         if '\n' in option:
1202             option = ' '.join([i.strip() for i in option.splitlines()])
1203
1204         # extend variables for value and condition
1205         condition = str(item.get('condition', ''))
1206         if condition:
1207             condition = self.extend_variable(condition)
1208         value = self.extend_variable(value)
1209
1210         order = str(item.get('order', ''))
1211         if order:
1212             if '.' in order:
1213                 (major, minor) = order.split('.')
1214                 order = int(major, 16)
1215             else:
1216                 order = int(order, 16)
1217         else:
1218             order = offset
1219
1220         cfg_item = dict()
1221         cfg_item['length'] = length
1222         cfg_item['offset'] = offset
1223         cfg_item['value'] = value
1224         cfg_item['type'] = itype
1225         cfg_item['cname'] = str(name)
1226         cfg_item['name'] = str(item.get('name', ''))
1227         cfg_item['help'] = help
1228         cfg_item['option'] = option
1229         cfg_item['page'] = self._cur_page
1230         cfg_item['order'] = order
1231         cfg_item['path'] = '.'.join(path)
1232         cfg_item['condition'] = condition
1233         if 'struct' in item:
1234             cfg_item['struct'] = item['struct']
1235         self._cfg_list.append(cfg_item)
1236
1237         item['indx'] = len(self._cfg_list) - 1
1238
1239         # remove used info for reducing pkl size
1240         item.pop('option', None)
1241         item.pop('condition', None)
1242         item.pop('help', None)
1243         item.pop('name', None)
1244         item.pop('page', None)
1245
1246         return length
1247
1248     def build_cfg_list(self, cfg_name='', top=None, path=[],
1249                        info={'offset': 0}):
1250         if top is None:
1251             top = self._cfg_tree
1252             info.clear()
1253             info = {'offset': 0}
1254
1255         start = info['offset']
1256         is_leaf = True
1257         for key in top:
1258             path.append(key)
1259             if type(top[key]) is OrderedDict:
1260                 is_leaf = False
1261                 self.build_cfg_list(key, top[key], path, info)
1262             path.pop()
1263
1264         if is_leaf:
1265             length = self.add_cfg_item(cfg_name, top, info['offset'], path)
1266             info['offset'] += length
1267         elif cfg_name == '' or (cfg_name and cfg_name[0] != '$'):
1268             # check first element for struct
1269             first = next(iter(top))
1270             struct_str = CGenYamlCfg.STRUCT
1271             if first != struct_str:
1272                 struct_node = OrderedDict({})
1273                 top[struct_str] = struct_node
1274                 top.move_to_end(struct_str, False)
1275             else:
1276                 struct_node = top[struct_str]
1277             struct_node['offset'] = start
1278             struct_node['length'] = info['offset'] - start
1279             if struct_node['length'] % 8 != 0:
1280                 raise SystemExit("Error: Bits length not aligned for %s !" %
1281                                  str(path))
1282
1283     def get_field_value(self, top=None):
1284         def _get_field_value(name, cfgs, level):
1285             if 'indx' in cfgs:
1286                 act_cfg = self.get_item_by_index(cfgs['indx'])
1287                 if act_cfg['length'] == 0:
1288                     return
1289                 value = self.get_value(act_cfg['value'], act_cfg['length'],
1290                                        False)
1291                 set_bits_to_bytes(result, act_cfg['offset'] -
1292                                   struct_info['offset'], act_cfg['length'],
1293                                   value)
1294
1295         if top is None:
1296             top = self._cfg_tree
1297         struct_info = top[CGenYamlCfg.STRUCT]
1298         result = bytearray((struct_info['length'] + 7) // 8)
1299         self.traverse_cfg_tree(_get_field_value, top)
1300         return result
1301
1302     data_diff = ''
1303
1304     def find_data_difference(self, act_val, act_cfg):
1305         # checks for any difference between BSF and Binary file
1306         config_val = ''
1307         if act_val != act_cfg['value']:
1308
1309             if 'DEC' in act_cfg['type']:
1310                 bsf_val = '0x%x' % int(act_val)
1311                 if bsf_val != act_cfg['value']:
1312                     config_val = bsf_val
1313                 else:
1314                     config_val = ''
1315             else:
1316                 config_val = act_val
1317
1318             available_fv1 = 'none'
1319             available_fv2 = 'none'
1320
1321             if self.detect_fsp():
1322                 if len(self.available_fv) >= 1:
1323                     if len(self.available_fv) > 1:
1324                         available_fv1 = self.available_fv[1]
1325                         if self.available_fv[2]:
1326                             available_fv2 = self.available_fv[2]
1327             else:
1328                 available_fv1 = self.available_fv[1]
1329                 if act_cfg['length'] == 16:
1330                     config_val = int(config_val, 16)
1331                     config_val = '0x%x' % config_val
1332                     act_cfg['value'] = int(
1333                         act_cfg['value'], 16)
1334                     act_cfg['value'] = '0x%x' %  \
1335                         act_cfg['value']
1336
1337             if config_val:
1338                 string = ('.' + act_cfg['cname'])
1339                 if (act_cfg['path'].endswith(self.available_fv[0] + string)
1340                     or act_cfg['path'].endswith(available_fv1 + string)
1341                     or act_cfg['path'].endswith(available_fv2 + string)) \
1342                     and 'BsfSkip' not in act_cfg['cname'] \
1343                         and 'Reserved' not in act_cfg['name']:
1344                     if act_cfg['option'] != '':
1345                         if act_cfg['length'] == 8:
1346                             config_val = int(config_val, 16)
1347                             config_val = '0x%x' % config_val
1348                             act_cfg['value'] = int(
1349                                 act_cfg['value'], 16)
1350                             act_cfg['value'] = '0x%x' %  \
1351                                 act_cfg['value']
1352                         option = act_cfg['option']
1353
1354                         cfg_val = ''
1355                         bin_val = ''
1356                         for i in option.split(','):
1357                             if act_cfg['value'] in i:
1358                                 bin_val = i
1359                             elif config_val in i:
1360                                 cfg_val = i
1361                         if cfg_val != '' and bin_val != '':
1362                             self.data_diff += '\n\nBinary:        ' \
1363                                 + act_cfg['name'] \
1364                                 + ': ' + bin_val.replace(' ', '') \
1365                                 + '\nConfig file:   ' \
1366                                 + act_cfg['name'] + ': ' \
1367                                 + cfg_val.replace(' ', '') + '\n'
1368                     else:
1369                         self.data_diff += '\n\nBinary:        ' \
1370                             + act_cfg['name'] + ': ' + act_cfg['value'] \
1371                             + '\nConfig file:   ' + act_cfg['name'] \
1372                             + ': ' + config_val + '\n'
1373
1374     def set_field_value(self, top, value_bytes, force=False):
1375         def _set_field_value(name, cfgs, level):
1376             if 'indx' not in cfgs:
1377                 return
1378             act_cfg = self.get_item_by_index(cfgs['indx'])
1379             actual_offset = act_cfg['offset'] - struct_info['offset']
1380             if force or act_cfg['value'] == '':
1381                 value = get_bits_from_bytes(full_bytes,
1382                                             actual_offset,
1383                                             act_cfg['length'])
1384                 act_val = act_cfg['value']
1385                 if act_val == '':
1386                     act_val = '%d' % value
1387                 act_val = self.reformat_number_per_type(act_cfg
1388                                                         ['type'],
1389                                                         act_val)
1390                 act_cfg['value'] = self.format_value_to_str(
1391                     value, act_cfg['length'], act_val)
1392                 self.find_data_difference(act_val, act_cfg)
1393
1394         if 'indx' in top:
1395             # it is config option
1396             value = bytes_to_value(value_bytes)
1397             act_cfg = self.get_item_by_index(top['indx'])
1398             act_cfg['value'] = self.format_value_to_str(
1399                 value, act_cfg['length'], act_cfg['value'])
1400         else:
1401             # it is structure
1402             struct_info = top[CGenYamlCfg.STRUCT]
1403             length = struct_info['length'] // 8
1404             full_bytes = bytearray(value_bytes[:length])
1405             if len(full_bytes) < length:
1406                 full_bytes.extend(bytearray(length - len(value_bytes)))
1407             self.traverse_cfg_tree(_set_field_value, top)
1408
1409     def update_def_value(self):
1410         def _update_def_value(name, cfgs, level):
1411             if 'indx' in cfgs:
1412                 act_cfg = self.get_item_by_index(cfgs['indx'])
1413                 if act_cfg['value'] != '' and act_cfg['length'] > 0:
1414                     try:
1415                         act_cfg['value'] = self.reformat_value_str(
1416                             act_cfg['value'], act_cfg['length'])
1417                     except Exception:
1418                         raise Exception("Invalid value expression '%s' \
1419 for '%s' !" % (act_cfg['value'], act_cfg['path']))
1420             else:
1421                 if CGenYamlCfg.STRUCT in cfgs and 'value' in \
1422                    cfgs[CGenYamlCfg.STRUCT]:
1423                     curr = cfgs[CGenYamlCfg.STRUCT]
1424                     value_bytes = self.get_value(curr['value'],
1425                                                  curr['length'], True)
1426                     self.set_field_value(cfgs, value_bytes)
1427
1428         self.traverse_cfg_tree(_update_def_value, self._cfg_tree)
1429
1430     def evaluate_condition(self, item):
1431         expr = item['condition']
1432         result = self.parse_value(expr, 1, False)
1433         return result
1434
1435     def detect_fsp(self):
1436         cfg_segs = self.get_cfg_segment()
1437         if len(cfg_segs) == 3:
1438             fsp = True
1439             for idx, seg in enumerate(cfg_segs):
1440                 if not seg[0].endswith('UPD_%s' % 'TMS'[idx]):
1441                     fsp = False
1442                     break
1443         else:
1444             fsp = False
1445         if fsp:
1446             self.set_mode('FSP')
1447         return fsp
1448
1449     def get_cfg_segment(self):
1450         def _get_cfg_segment(name, cfgs, level):
1451             if 'indx' not in cfgs:
1452                 if name.startswith('$ACTION_'):
1453                     if 'find' in cfgs:
1454                         find[0] = cfgs['find']
1455             else:
1456                 if find[0]:
1457                     act_cfg = self.get_item_by_index(cfgs['indx'])
1458                     segments.append([find[0], act_cfg['offset'] // 8, 0])
1459                     find[0] = ''
1460                 return
1461
1462         find = ['']
1463         segments = []
1464         self.traverse_cfg_tree(_get_cfg_segment, self._cfg_tree)
1465         cfg_len = self._cfg_tree[CGenYamlCfg.STRUCT]['length'] // 8
1466         if len(segments) == 0:
1467             segments.append(['', 0, cfg_len])
1468
1469         segments.append(['', cfg_len, 0])
1470         cfg_segs = []
1471         for idx, each in enumerate(segments[:-1]):
1472             cfg_segs.append((each[0], each[1],
1473                              segments[idx+1][1] - each[1]))
1474
1475         return cfg_segs
1476
1477     def get_bin_segment(self, bin_data):
1478         cfg_segs = self.get_cfg_segment()
1479         bin_segs = []
1480         for seg in cfg_segs:
1481             key = seg[0].encode()
1482             if key == 0:
1483                 bin_segs.append([seg[0], 0, len(bin_data)])
1484                 break
1485             pos = bin_data.find(key)
1486             if pos >= 0:
1487                 # ensure no other match for the key
1488                 next_pos = bin_data.find(key, pos + len(seg[0]))
1489                 if next_pos >= 0:
1490                     if key == b'$SKLFSP$' or key == b'$BSWFSP$':
1491                         string = ('Warning: Multiple matches for %s in '
1492                                   'binary!\n\nA workaround applied to such '
1493                                   'FSP 1.x binary to use second'
1494                                   ' match instead of first match!' % key)
1495                         messagebox.showwarning('Warning!', string)
1496                         pos = next_pos
1497                     else:
1498                         print("Warning: Multiple matches for '%s' "
1499                               "in binary, the 1st instance will be used !"
1500                               % seg[0])
1501                 bin_segs.append([seg[0], pos, seg[2]])
1502                 self.binseg_dict[seg[0]] = pos
1503             else:
1504                 bin_segs.append([seg[0], -1, seg[2]])
1505                 self.binseg_dict[seg[0]] = -1
1506                 continue
1507
1508         return bin_segs
1509
1510     available_fv = []
1511     missing_fv = []
1512
1513     def extract_cfg_from_bin(self, bin_data):
1514         # get cfg bin length
1515         cfg_bins = bytearray()
1516         bin_segs = self.get_bin_segment(bin_data)
1517         Dummy_offset = 0
1518         for each in bin_segs:
1519             if each[1] != -1:
1520                 cfg_bins.extend(bin_data[each[1]:each[1] + each[2]])
1521                 self.available_fv.append(each[0])
1522             else:
1523                 self.missing_fv.append(each[0])
1524                 string = each[0] + ' is not availabe.'
1525                 messagebox.showinfo('', string)
1526                 cfg_bins.extend(bytearray(each[2]))
1527             Dummy_offset += each[2]
1528         return cfg_bins
1529
1530     def save_current_to_bin(self):
1531         cfg_bins = self.generate_binary_array()
1532         if self._old_bin is None:
1533             return cfg_bins
1534
1535         bin_data = bytearray(self._old_bin)
1536         bin_segs = self.get_bin_segment(self._old_bin)
1537         cfg_off = 0
1538         for each in bin_segs:
1539             length = each[2]
1540             if each[1] != -1:
1541                 bin_data[each[1]:each[1] + length] = cfg_bins[cfg_off:
1542                                                               cfg_off
1543                                                               + length]
1544                 cfg_off += length
1545             else:
1546                 cfg_off += length
1547
1548         print('Patched the loaded binary successfully !')
1549         return bin_data
1550
1551     def show_data_difference(self, data_diff):
1552         # Displays if any data difference detected in BSF and Binary file
1553         pop_up_text = 'There are differences in Config file and binary '\
1554             'data detected!\n'
1555         pop_up_text += data_diff
1556
1557         window = tkinter.Tk()
1558         window.title("Data Difference")
1559         window.resizable(1, 1)
1560         # Window Size
1561         window.geometry("800x400")
1562         frame = tkinter.Frame(window, height=800, width=700)
1563         frame.pack(side=tkinter.BOTTOM)
1564         # Vertical (y) Scroll Bar
1565         scroll = tkinter.Scrollbar(window)
1566         scroll.pack(side=tkinter.RIGHT, fill=tkinter.Y)
1567
1568         text = tkinter.Text(window, wrap=tkinter.NONE,
1569                             yscrollcommand=scroll.set,
1570                             width=700, height=400)
1571         text.insert(tkinter.INSERT, pop_up_text)
1572         text.pack()
1573         # Configure the scrollbars
1574         scroll.config(command=text.yview)
1575         exit_button = tkinter.Button(
1576             window, text="Close", command=window.destroy)
1577         exit_button.pack(in_=frame, side=tkinter.RIGHT, padx=20, pady=10)
1578
1579     def load_default_from_bin(self, bin_data):
1580         self._old_bin = bin_data
1581         cfg_bins = self.extract_cfg_from_bin(bin_data)
1582         self.set_field_value(self._cfg_tree, cfg_bins, True)
1583
1584         if self.data_diff:
1585             self.show_data_difference(self.data_diff)
1586         return cfg_bins
1587
1588     def generate_binary_array(self, path=''):
1589         if path == '':
1590             top = None
1591         else:
1592             top = self.locate_cfg_item(path)
1593             if not top:
1594                 raise Exception("Invalid configuration path '%s' !"
1595                                 % path)
1596
1597         return self.get_field_value(top)
1598
1599     def generate_binary(self, bin_file_name, path=''):
1600         bin_file = open(bin_file_name, "wb")
1601         bin_file.write(self.generate_binary_array(path))
1602         bin_file.close()
1603         return 0
1604
1605     def write_delta_file(self, out_file, platform_id, out_lines):
1606         dlt_fd = open(out_file, "w")
1607         dlt_fd.write("%s\n" % get_copyright_header('dlt', True))
1608         if platform_id is not None:
1609             dlt_fd.write('#\n')
1610             dlt_fd.write('# Delta configuration values for '
1611                          'platform ID 0x%04X\n'
1612                          % platform_id)
1613             dlt_fd.write('#\n\n')
1614         for line in out_lines:
1615             dlt_fd.write('%s\n' % line)
1616         dlt_fd.close()
1617
1618     def override_default_value(self, dlt_file):
1619         error = 0
1620         dlt_lines = CGenYamlCfg.expand_include_files(dlt_file)
1621
1622         platform_id = None
1623         for line, file_path, line_num in dlt_lines:
1624             line = line.strip()
1625             if not line or line.startswith('#'):
1626                 continue
1627             match = re.match("\\s*([\\w\\.]+)\\s*\\|\\s*(.+)", line)
1628             if not match:
1629                 raise Exception("Unrecognized line '%s' "
1630                                 "(File:'%s' Line:%d) !"
1631                                 % (line, file_path, line_num + 1))
1632
1633             path = match.group(1)
1634             value_str = match.group(2)
1635             top = self.locate_cfg_item(path)
1636             if not top:
1637                 raise Exception(
1638                     "Invalid configuration '%s' (File:'%s' Line:%d) !" %
1639                     (path, file_path, line_num + 1))
1640
1641             if 'indx' in top:
1642                 act_cfg = self.get_item_by_index(top['indx'])
1643                 bit_len = act_cfg['length']
1644             else:
1645                 struct_info = top[CGenYamlCfg.STRUCT]
1646                 bit_len = struct_info['length']
1647
1648             value_bytes = self.parse_value(value_str, bit_len)
1649             self.set_field_value(top, value_bytes, True)
1650
1651             if path == 'PLATFORMID_CFG_DATA.PlatformId':
1652                 platform_id = value_str
1653
1654         if platform_id is None:
1655             raise Exception(
1656                 "PLATFORMID_CFG_DATA.PlatformId is missing "
1657                 "in file '%s' !" %
1658                 (dlt_file))
1659
1660         return error
1661
1662     def generate_delta_file_from_bin(self, delta_file, old_data,
1663                                      new_data, full=False):
1664         new_data = self.load_default_from_bin(new_data)
1665         lines = []
1666         platform_id = None
1667         def_platform_id = 0
1668
1669         for item in self._cfg_list:
1670             if not full and (item['type'] in ['Reserved']):
1671                 continue
1672             old_val = get_bits_from_bytes(old_data, item['offset'],
1673                                           item['length'])
1674             new_val = get_bits_from_bytes(new_data, item['offset'],
1675                                           item['length'])
1676
1677             full_name = item['path']
1678             if 'PLATFORMID_CFG_DATA.PlatformId' == full_name:
1679                 def_platform_id = old_val
1680             if new_val != old_val or full:
1681                 val_str = self.reformat_value_str(item['value'],
1682                                                   item['length'])
1683                 text = '%-40s | %s' % (full_name, val_str)
1684                 lines.append(text)
1685
1686         if self.get_mode() != 'FSP':
1687             if platform_id is None or def_platform_id == platform_id:
1688                 platform_id = def_platform_id
1689                 print("WARNING: 'PlatformId' configuration is "
1690                       "same as default %d!" % platform_id)
1691
1692             lines.insert(0, '%-40s | %s\n\n' %
1693                          ('PLATFORMID_CFG_DATA.PlatformId',
1694                           '0x%04X' % platform_id))
1695         else:
1696             platform_id = None
1697
1698         self.write_delta_file(delta_file, platform_id, lines)
1699
1700         return 0
1701
1702     def generate_delta_file(self, delta_file, bin_file, bin_file2, full=False):
1703         fd = open(bin_file, 'rb')
1704         new_data = self.extract_cfg_from_bin(bytearray(fd.read()))
1705         fd.close()
1706
1707         if bin_file2 == '':
1708             old_data = self.generate_binary_array()
1709         else:
1710             old_data = new_data
1711             fd = open(bin_file2, 'rb')
1712             new_data = self.extract_cfg_from_bin(bytearray(fd.read()))
1713             fd.close()
1714
1715         return self.generate_delta_file_from_bin(delta_file,
1716                                                  old_data, new_data, full)
1717
1718     def prepare_marshal(self, is_save):
1719         if is_save:
1720             # Ordered dict is not marshallable, convert to list
1721             self._cfg_tree = CGenYamlCfg.deep_convert_dict(self._cfg_tree)
1722         else:
1723             # Revert it back
1724             self._cfg_tree = CGenYamlCfg.deep_convert_list(self._cfg_tree)
1725
1726     def generate_yml_file(self, in_file, out_file):
1727         cfg_yaml = CFG_YAML()
1728         text = cfg_yaml.expand_yaml(in_file)
1729         yml_fd = open(out_file, "w")
1730         yml_fd.write(text)
1731         yml_fd.close()
1732         return 0
1733
1734     def write_cfg_header_file(self, hdr_file_name, tag_mode,
1735                               tag_dict, struct_list):
1736         lines = []
1737         lines.append('\n\n')
1738         if self.get_mode() == 'FSP':
1739             lines.append('#include <FspUpd.h>\n')
1740
1741         tag_mode = tag_mode & 0x7F
1742         tag_list = sorted(list(tag_dict.items()), key=lambda x: x[1])
1743         for tagname, tagval in tag_list:
1744             if (tag_mode == 0 and tagval >= 0x100) or \
1745                (tag_mode == 1 and tagval < 0x100):
1746                 continue
1747             lines.append('#define    %-30s 0x%03X\n' % (
1748                 'CDATA_%s_TAG' % tagname[:-9], tagval))
1749         lines.append('\n\n')
1750
1751         name_dict = {}
1752         new_dict = {}
1753         for each in struct_list:
1754             if (tag_mode == 0 and each['tag'] >= 0x100) or \
1755                (tag_mode == 1 and each['tag'] < 0x100):
1756                 continue
1757             new_dict[each['name']] = (each['alias'], each['count'])
1758             if each['alias'] not in name_dict:
1759                 name_dict[each['alias']] = 1
1760                 lines.extend(self.create_struct(each['alias'],
1761                                                 each['node'], new_dict))
1762         lines.append('#pragma pack()\n\n')
1763
1764         self.write_header_file(lines, hdr_file_name)
1765
1766     def write_header_file(self, txt_body, file_name, type='h'):
1767         file_name_def = os.path.basename(file_name).replace('.', '_')
1768         file_name_def = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', file_name_def)
1769         file_name_def = re.sub('([a-z0-9])([A-Z])', r'\1_\2',
1770                                file_name_def).upper()
1771
1772         lines = []
1773         lines.append("%s\n" % get_copyright_header(type))
1774         lines.append("#ifndef __%s__\n" % file_name_def)
1775         lines.append("#define __%s__\n\n" % file_name_def)
1776         if type == 'h':
1777             lines.append("#pragma pack(1)\n\n")
1778         lines.extend(txt_body)
1779         if type == 'h':
1780             lines.append("#pragma pack()\n\n")
1781         lines.append("#endif\n")
1782
1783         # Don't rewrite if the contents are the same
1784         create = True
1785         if os.path.exists(file_name):
1786             hdr_file = open(file_name, "r")
1787             org_txt = hdr_file.read()
1788             hdr_file.close()
1789
1790             new_txt = ''.join(lines)
1791             if org_txt == new_txt:
1792                 create = False
1793
1794         if create:
1795             hdr_file = open(file_name, "w")
1796             hdr_file.write(''.join(lines))
1797             hdr_file.close()
1798
1799     def generate_data_inc_file(self, dat_inc_file_name, bin_file=None):
1800         # Put a prefix GUID before CFGDATA so that it can be located later on
1801         prefix = b'\xa7\xbd\x7f\x73\x20\x1e\x46\xd6\
1802 xbe\x8f\x64\x12\x05\x8d\x0a\xa8'
1803         if bin_file:
1804             fin = open(bin_file, 'rb')
1805             bin_dat = prefix + bytearray(fin.read())
1806             fin.close()
1807         else:
1808             bin_dat = prefix + self.generate_binary_array()
1809
1810         file_name = os.path.basename(dat_inc_file_name).upper()
1811         file_name = file_name.replace('.', '_')
1812
1813         txt_lines = []
1814
1815         txt_lines.append("UINT8  mConfigDataBlob[%d] = {\n" % len(bin_dat))
1816         count = 0
1817         line = ['  ']
1818         for each in bin_dat:
1819             line.append('0x%02X, ' % each)
1820             count = count + 1
1821             if (count & 0x0F) == 0:
1822                 line.append('\n')
1823                 txt_lines.append(''.join(line))
1824                 line = ['  ']
1825         if len(line) > 1:
1826             txt_lines.append(''.join(line) + '\n')
1827
1828         txt_lines.append("};\n\n")
1829         self.write_header_file(txt_lines, dat_inc_file_name, 'inc')
1830
1831         return 0
1832
1833     def get_struct_array_info(self, input):
1834         parts = input.split(':')
1835         if len(parts) > 1:
1836             var = parts[1]
1837             input = parts[0]
1838         else:
1839             var = ''
1840         array_str = input.split('[')
1841         name = array_str[0]
1842         if len(array_str) > 1:
1843             num_str = ''.join(c for c in array_str[-1] if c.isdigit())
1844             num_str = '1000' if len(num_str) == 0 else num_str
1845             array_num = int(num_str)
1846         else:
1847             array_num = 0
1848         return name, array_num, var
1849
1850     def process_multilines(self, string, max_char_length):
1851         multilines = ''
1852         string_length = len(string)
1853         current_string_start = 0
1854         string_offset = 0
1855         break_line_dict = []
1856         if len(string) <= max_char_length:
1857             while (string_offset < string_length):
1858                 if string_offset >= 1:
1859                     if string[string_offset - 1] == '\\' and string[
1860                        string_offset] == 'n':
1861                         break_line_dict.append(string_offset + 1)
1862                 string_offset += 1
1863             if break_line_dict != []:
1864                 for each in break_line_dict:
1865                     multilines += "  %s\n" % string[
1866                         current_string_start:each].lstrip()
1867                     current_string_start = each
1868                 if string_length - current_string_start > 0:
1869                     multilines += "  %s\n" % string[
1870                         current_string_start:].lstrip()
1871             else:
1872                 multilines = "  %s\n" % string
1873         else:
1874             new_line_start = 0
1875             new_line_count = 0
1876             found_space_char = False
1877             while (string_offset < string_length):
1878                 if string_offset >= 1:
1879                     if new_line_count >= max_char_length - 1:
1880                         if string[string_offset] == ' ' and \
1881                            string_length - string_offset > 10:
1882                             break_line_dict.append(new_line_start
1883                                                    + new_line_count)
1884                             new_line_start = new_line_start + new_line_count
1885                             new_line_count = 0
1886                             found_space_char = True
1887                         elif string_offset == string_length - 1 and \
1888                                 found_space_char is False:
1889                             break_line_dict.append(0)
1890                     if string[string_offset - 1] == '\\' and string[
1891                        string_offset] == 'n':
1892                         break_line_dict.append(string_offset + 1)
1893                         new_line_start = string_offset + 1
1894                         new_line_count = 0
1895                 string_offset += 1
1896                 new_line_count += 1
1897             if break_line_dict != []:
1898                 break_line_dict.sort()
1899                 for each in break_line_dict:
1900                     if each > 0:
1901                         multilines += "  %s\n" % string[
1902                             current_string_start:each].lstrip()
1903                     current_string_start = each
1904                 if string_length - current_string_start > 0:
1905                     multilines += "  %s\n" % \
1906                                   string[current_string_start:].lstrip()
1907         return multilines
1908
1909     def create_field(self, item, name, length, offset, struct,
1910                      bsf_name, help, option, bits_length=None):
1911         pos_name = 28
1912         name_line = ''
1913         # help_line = ''
1914         # option_line = ''
1915
1916         if length == 0 and name == 'dummy':
1917             return '\n'
1918
1919         if bits_length == 0:
1920             return '\n'
1921
1922         is_array = False
1923         if length in [1, 2, 4, 8]:
1924             type = "UINT%d" % (length * 8)
1925         else:
1926             is_array = True
1927             type = "UINT8"
1928
1929         if item and item['value'].startswith('{'):
1930             type = "UINT8"
1931             is_array = True
1932
1933         if struct != '':
1934             struct_base = struct.rstrip('*')
1935             name = '*' * (len(struct) - len(struct_base)) + name
1936             struct = struct_base
1937             type = struct
1938             if struct in ['UINT8', 'UINT16', 'UINT32', 'UINT64']:
1939                 is_array = True
1940                 unit = int(type[4:]) // 8
1941                 length = length / unit
1942             else:
1943                 is_array = False
1944
1945         if is_array:
1946             name = name + '[%d]' % length
1947
1948         if len(type) < pos_name:
1949             space1 = pos_name - len(type)
1950         else:
1951             space1 = 1
1952
1953         if bsf_name != '':
1954             name_line = " %s\n" % bsf_name
1955         else:
1956             name_line = "N/A\n"
1957
1958         # if help != '':
1959             # help_line = self.process_multilines(help, 80)
1960
1961         # if option != '':
1962             # option_line = self.process_multilines(option, 80)
1963
1964         if offset is None:
1965             offset_str = '????'
1966         else:
1967             offset_str = '0x%04X' % offset
1968
1969         if bits_length is None:
1970             bits_length = ''
1971         else:
1972             bits_length = ' : %d' % bits_length
1973
1974         # return "\n/** %s%s%s**/\n  %s%s%s%s;\n" % (name_line, help_line,
1975         # option_line, type, ' ' * space1, name, bits_length)
1976         return "\n  /* Offset %s: %s */\n  %s%s%s%s;\n" % (
1977             offset_str, name_line.strip(), type, ' ' * space1,
1978             name, bits_length)
1979
1980     def create_struct(self, cname, top, struct_dict):
1981         index = 0
1982         last = ''
1983         lines = []
1984         off_base = -1
1985
1986         if cname in struct_dict:
1987             if struct_dict[cname][2]:
1988                 return []
1989         lines.append('\ntypedef struct {\n')
1990         for field in top:
1991             if field[0] == '$':
1992                 continue
1993
1994             index += 1
1995
1996             t_item = top[field]
1997             if 'indx' not in t_item:
1998                 if CGenYamlCfg.STRUCT not in top[field]:
1999                     continue
2000
2001                 if struct_dict[field][1] == 0:
2002                     continue
2003
2004                 append = True
2005                 struct_info = top[field][CGenYamlCfg.STRUCT]
2006
2007                 if 'struct' in struct_info:
2008                     struct, array_num, var = self.get_struct_array_info(
2009                         struct_info['struct'])
2010                     if array_num > 0:
2011                         if last == struct:
2012                             append = False
2013                             last = struct
2014                         if var == '':
2015                             var = field
2016
2017                         field = CGenYamlCfg.format_struct_field_name(
2018                             var, struct_dict[field][1])
2019                 else:
2020                     struct = struct_dict[field][0]
2021                     field = CGenYamlCfg.format_struct_field_name(
2022                         field, struct_dict[field][1])
2023
2024                 if append:
2025                     offset = t_item['$STRUCT']['offset'] // 8
2026                     if off_base == -1:
2027                         off_base = offset
2028                     line = self.create_field(None, field, 0, 0, struct,
2029                                              '', '', '')
2030                     lines.append('  %s' % line)
2031                     last = struct
2032                 continue
2033
2034             item = self.get_item_by_index(t_item['indx'])
2035             if item['cname'] == 'CfgHeader' and index == 1 or \
2036                (item['cname'] == 'CondValue' and index == 2):
2037                 continue
2038
2039             bit_length = None
2040             length = (item['length'] + 7) // 8
2041             match = re.match("^(\\d+)([b|B|W|D|Q])([B|W|D|Q]?)",
2042                              t_item['length'])
2043             if match and match.group(2) == 'b':
2044                 bit_length = int(match.group(1))
2045                 if match.group(3) != '':
2046                     length = CGenYamlCfg.bits_width[match.group(3)] // 8
2047                 else:
2048                     length = 4
2049             offset = item['offset'] // 8
2050             if off_base == -1:
2051                 off_base = offset
2052             struct = item.get('struct', '')
2053             name = field
2054             prompt = item['name']
2055             help = item['help']
2056             option = item['option']
2057             line = self.create_field(item, name, length, offset, struct,
2058                                      prompt, help, option, bit_length)
2059             lines.append('  %s' % line)
2060             last = struct
2061
2062         lines.append('\n} %s;\n\n' % cname)
2063
2064         return lines
2065
2066     def write_fsp_sig_header_file(self, hdr_file_name):
2067         hdr_fd = open(hdr_file_name, 'w')
2068         hdr_fd.write("%s\n" % get_copyright_header('h'))
2069         hdr_fd.write("#ifndef __FSPUPD_H__\n"
2070                      "#define __FSPUPD_H__\n\n"
2071                      "#include <FspEas.h>\n\n"
2072                      "#pragma pack(1)\n\n")
2073         lines = []
2074         for fsp_comp in 'TMS':
2075             top = self.locate_cfg_item('FSP%s_UPD' % fsp_comp)
2076             if not top:
2077                 raise Exception('Could not find FSP UPD definition !')
2078             bins = self.get_field_value(top)
2079             lines.append("#define FSP%s_UPD_SIGNATURE"
2080                          "    0x%016X  /* '%s' */\n\n"
2081                          % (fsp_comp, bytes_to_value(bins[:8]),
2082                             bins[:8].decode()))
2083         hdr_fd.write(''.join(lines))
2084         hdr_fd.write("#pragma pack()\n\n"
2085                      "#endif\n")
2086         hdr_fd.close()
2087
2088     def create_header_file(self, hdr_file_name, com_hdr_file_name='', path=''):
2089
2090         def _build_header_struct(name, cfgs, level):
2091             if CGenYamlCfg.STRUCT in cfgs:
2092                 if 'CfgHeader' in cfgs:
2093                     # collect CFGDATA TAG IDs
2094                     cfghdr = self.get_item_by_index(cfgs['CfgHeader']['indx'])
2095                     tag_val = array_str_to_value(cfghdr['value']) >> 20
2096                     tag_dict[name] = tag_val
2097                     if level == 1:
2098                         tag_curr[0] = tag_val
2099                 struct_dict[name] = (level, tag_curr[0], cfgs)
2100         if path == 'FSP_SIG':
2101             self.write_fsp_sig_header_file(hdr_file_name)
2102             return
2103         tag_curr = [0]
2104         tag_dict = {}
2105         struct_dict = {}
2106
2107         if path == '':
2108             top = None
2109         else:
2110             top = self.locate_cfg_item(path)
2111             if not top:
2112                 raise Exception("Invalid configuration path '%s' !" % path)
2113             _build_header_struct(path, top, 0)
2114         self.traverse_cfg_tree(_build_header_struct, top)
2115
2116         if tag_curr[0] == 0:
2117             hdr_mode = 2
2118         else:
2119             hdr_mode = 1
2120
2121         if re.match('FSP[TMS]_UPD', path):
2122             hdr_mode |= 0x80
2123
2124         # filter out the items to be built for tags and structures
2125         struct_list = []
2126         for each in struct_dict:
2127             match = False
2128             for check in CGenYamlCfg.exclude_struct:
2129                 if re.match(check, each):
2130                     match = True
2131                     if each in tag_dict:
2132                         if each not in CGenYamlCfg.include_tag:
2133                             del tag_dict[each]
2134                     break
2135             if not match:
2136                 struct_list.append({'name': each, 'alias': '', 'count': 0,
2137                                     'level': struct_dict[each][0],
2138                                     'tag': struct_dict[each][1],
2139                                     'node': struct_dict[each][2]})
2140
2141         # sort by level so that the bottom level struct
2142         # will be build first to satisfy dependencies
2143         struct_list = sorted(struct_list, key=lambda x: x['level'],
2144                              reverse=True)
2145
2146         # Convert XXX_[0-9]+ to XXX as an array hint
2147         for each in struct_list:
2148             cfgs = each['node']
2149             if 'struct' in cfgs['$STRUCT']:
2150                 each['alias'], array_num, var = self.get_struct_array_info(
2151                     cfgs['$STRUCT']['struct'])
2152             else:
2153                 match = re.match('(\\w+)(_\\d+)', each['name'])
2154                 if match:
2155                     each['alias'] = match.group(1)
2156                 else:
2157                     each['alias'] = each['name']
2158
2159         # count items for array build
2160         for idx, each in enumerate(struct_list):
2161             if idx > 0:
2162                 last_struct = struct_list[idx-1]['node']['$STRUCT']
2163                 curr_struct = each['node']['$STRUCT']
2164                 if struct_list[idx-1]['alias'] == each['alias'] and \
2165                    curr_struct['length'] == last_struct['length'] and \
2166                    curr_struct['offset'] == last_struct['offset'] + \
2167                    last_struct['length']:
2168                     for idx2 in range(idx-1, -1, -1):
2169                         if struct_list[idx2]['count'] > 0:
2170                             struct_list[idx2]['count'] += 1
2171                             break
2172                     continue
2173             each['count'] = 1
2174
2175         # generate common header
2176         if com_hdr_file_name:
2177             self.write_cfg_header_file(com_hdr_file_name, 0, tag_dict,
2178                                        struct_list)
2179
2180         # generate platform header
2181         self.write_cfg_header_file(hdr_file_name, hdr_mode, tag_dict,
2182                                    struct_list)
2183
2184         return 0
2185
2186     def load_yaml(self, cfg_file):
2187         cfg_yaml = CFG_YAML()
2188         self.initialize()
2189         self._cfg_tree = cfg_yaml.load_yaml(cfg_file)
2190         self._def_dict = cfg_yaml.def_dict
2191         self._yaml_path = os.path.dirname(cfg_file)
2192         self.build_cfg_list()
2193         self.build_var_dict()
2194         self.update_def_value()
2195         return 0
2196
2197
2198 def usage():
2199     print('\n'.join([
2200           "GenYamlCfg Version 0.50",
2201           "Usage:",
2202           "    GenYamlCfg  GENINC  BinFile              IncOutFile "
2203           "  [-D Macros]",
2204
2205           "    GenYamlCfg  GENPKL  YamlFile             PklOutFile "
2206           "  [-D Macros]",
2207           "    GenYamlCfg  GENBIN  YamlFile[;DltFile]   BinOutFile "
2208           "  [-D Macros]",
2209           "    GenYamlCfg  GENDLT  YamlFile[;BinFile]   DltOutFile "
2210           "  [-D Macros]",
2211           "    GenYamlCfg  GENYML  YamlFile             YamlOutFile"
2212           "  [-D Macros]",
2213           "    GenYamlCfg  GENHDR  YamlFile             HdrOutFile "
2214           "  [-D Macros]"
2215           ]))
2216
2217
2218 def main():
2219     # Parse the options and args
2220     argc = len(sys.argv)
2221     if argc < 4:
2222         usage()
2223         return 1
2224
2225     gen_cfg_data = CGenYamlCfg()
2226     command = sys.argv[1].upper()
2227     out_file = sys.argv[3]
2228     if argc >= 5 and gen_cfg_data.parse_macros(sys.argv[4:]) != 0:
2229         raise Exception("ERROR: Macro parsing failed !")
2230
2231     file_list = sys.argv[2].split(';')
2232     if len(file_list) >= 2:
2233         yml_file = file_list[0]
2234         dlt_file = file_list[1]
2235     elif len(file_list) == 1:
2236         yml_file = file_list[0]
2237         dlt_file = ''
2238     else:
2239         raise Exception("ERROR: Invalid parameter '%s' !" % sys.argv[2])
2240     yml_scope = ''
2241     if '@' in yml_file:
2242         parts = yml_file.split('@')
2243         yml_file = parts[0]
2244         yml_scope = parts[1]
2245
2246     if command == "GENDLT" and yml_file.endswith('.dlt'):
2247         # It needs to expand an existing DLT file
2248         dlt_file = yml_file
2249         lines = gen_cfg_data.expand_include_files(dlt_file)
2250         write_lines(lines, out_file)
2251         return 0
2252
2253     if command == "GENYML":
2254         if not yml_file.lower().endswith('.yaml'):
2255             raise Exception('Only YAML file is supported !')
2256         gen_cfg_data.generate_yml_file(yml_file, out_file)
2257         return 0
2258
2259     bin_file = ''
2260     if (yml_file.lower().endswith('.bin')) and (command == "GENINC"):
2261         # It is binary file
2262         bin_file = yml_file
2263         yml_file = ''
2264
2265     if bin_file:
2266         gen_cfg_data.generate_data_inc_file(out_file, bin_file)
2267         return 0
2268
2269     cfg_bin_file = ''
2270     cfg_bin_file2 = ''
2271     if dlt_file:
2272         if command == "GENDLT":
2273             cfg_bin_file = dlt_file
2274             dlt_file = ''
2275             if len(file_list) >= 3:
2276                 cfg_bin_file2 = file_list[2]
2277
2278     if yml_file.lower().endswith('.pkl'):
2279         with open(yml_file, "rb") as pkl_file:
2280             gen_cfg_data.__dict__ = marshal.load(pkl_file)
2281         gen_cfg_data.prepare_marshal(False)
2282
2283         # Override macro definition again for Pickle file
2284         if argc >= 5:
2285             gen_cfg_data.parse_macros(sys.argv[4:])
2286     else:
2287         gen_cfg_data.load_yaml(yml_file)
2288         if command == 'GENPKL':
2289             gen_cfg_data.prepare_marshal(True)
2290             with open(out_file, "wb") as pkl_file:
2291                 marshal.dump(gen_cfg_data.__dict__, pkl_file)
2292             json_file = os.path.splitext(out_file)[0] + '.json'
2293             fo = open(json_file, 'w')
2294             path_list = []
2295             cfgs = {'_cfg_page': gen_cfg_data._cfg_page,
2296                     '_cfg_list': gen_cfg_data._cfg_list,
2297                     '_path_list': path_list}
2298             # optimize to reduce size
2299             path = None
2300             for each in cfgs['_cfg_list']:
2301                 new_path = each['path'][:-len(each['cname'])-1]
2302                 if path != new_path:
2303                     path = new_path
2304                     each['path'] = path
2305                     path_list.append(path)
2306                 else:
2307                     del each['path']
2308                 if each['order'] == each['offset']:
2309                     del each['order']
2310                 del each['offset']
2311
2312                 # value is just used to indicate display type
2313                 value = each['value']
2314                 if value.startswith('0x'):
2315                     hex_len = ((each['length'] + 7) // 8) * 2
2316                     if len(value) == hex_len:
2317                         value = 'x%d' % hex_len
2318                     else:
2319                         value = 'x'
2320                     each['value'] = value
2321                 elif value and value[0] in ['"', "'", '{']:
2322                     each['value'] = value[0]
2323                 else:
2324                     del each['value']
2325
2326             fo.write(repr(cfgs))
2327             fo.close()
2328             return 0
2329
2330     if dlt_file:
2331         gen_cfg_data.override_default_value(dlt_file)
2332
2333     gen_cfg_data.detect_fsp()
2334
2335     if command == "GENBIN":
2336         if len(file_list) == 3:
2337             old_data = gen_cfg_data.generate_binary_array()
2338             fi = open(file_list[2], 'rb')
2339             new_data = bytearray(fi.read())
2340             fi.close()
2341             if len(new_data) != len(old_data):
2342                 raise Exception("Binary file '%s' length does not match, \
2343 ignored !" % file_list[2])
2344             else:
2345                 gen_cfg_data.load_default_from_bin(new_data)
2346                 gen_cfg_data.override_default_value(dlt_file)
2347
2348         gen_cfg_data.generate_binary(out_file, yml_scope)
2349
2350     elif command == "GENDLT":
2351         full = True if 'FULL' in gen_cfg_data._macro_dict else False
2352         gen_cfg_data.generate_delta_file(out_file, cfg_bin_file,
2353                                          cfg_bin_file2, full)
2354
2355     elif command == "GENHDR":
2356         out_files = out_file.split(';')
2357         brd_out_file = out_files[0].strip()
2358         if len(out_files) > 1:
2359             com_out_file = out_files[1].strip()
2360         else:
2361             com_out_file = ''
2362         gen_cfg_data.create_header_file(brd_out_file, com_out_file, yml_scope)
2363
2364     elif command == "GENINC":
2365         gen_cfg_data.generate_data_inc_file(out_file)
2366
2367     elif command == "DEBUG":
2368         gen_cfg_data.print_cfgs()
2369
2370     else:
2371         raise Exception("Unsuported command '%s' !" % command)
2372
2373     return 0
2374
2375
2376 if __name__ == '__main__':
2377     sys.exit(main())