[mirror_edk2.git] / BaseTools / Source / Python / Ecc / MetaDataParser.py

## @file\r
# This file is used to define common parser functions for meta-data\r
#\r
# Copyright (c) 2008, Intel Corporation. All rights reserved.<BR>\r
# This program and the accompanying materials\r
# are licensed and made available under the terms and conditions of the BSD License\r
# which accompanies this distribution.  The full text of the license may be found at\r
# http://opensource.org/licenses/bsd-license.php\r
#\r
# THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,\r
# WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.\r
#\r
\r
import os\r
from CommonDataClass.DataClass import *\r
from EccToolError import *\r
import EccGlobalData\r
import re\r
## Get the inlcude path list for a source file\r
#\r
# 1. Find the source file belongs to which inf file\r
# 2. Find the inf's package\r
# 3. Return the include path list of the package\r
#\r
def GetIncludeListOfFile(WorkSpace, Filepath, Db):\r
    IncludeList = []\r
    Filepath = os.path.normpath(Filepath)\r
    SqlCommand = """\r
                select Value1, FullPath from Inf, File where Inf.Model = %s and Inf.BelongsToFile in(\r
                    select distinct B.BelongsToFile from File as A left join Inf as B\r
                        where A.ID = B.BelongsToFile and B.Model = %s and (A.Path || '%s' || B.Value1) = '%s')\r
                        and Inf.BelongsToFile = File.ID""" \\r
                % (MODEL_META_DATA_PACKAGE, MODEL_EFI_SOURCE_FILE, '\\', Filepath)\r
    RecordSet = Db.TblFile.Exec(SqlCommand)\r
    for Record in RecordSet:\r
        DecFullPath = os.path.normpath(os.path.join(WorkSpace, Record[0]))\r
        InfFullPath = os.path.normpath(os.path.join(WorkSpace, Record[1]))\r
        (DecPath, DecName) = os.path.split(DecFullPath)\r
        (InfPath, InfName) = os.path.split(InfFullPath)\r
        SqlCommand = """select Value1 from Dec where BelongsToFile =\r
                           (select ID from File where FullPath = '%s') and Model = %s""" \\r
                    % (DecFullPath, MODEL_EFI_INCLUDE)\r
        NewRecordSet = Db.TblDec.Exec(SqlCommand)\r
        if InfPath not in IncludeList:\r
            IncludeList.append(InfPath)\r
        for NewRecord in NewRecordSet:\r
            IncludePath = os.path.normpath(os.path.join(DecPath, NewRecord[0]))\r
            if IncludePath not in IncludeList:\r
                IncludeList.append(IncludePath)\r
\r
    return IncludeList\r
\r
## Get the file list\r
#\r
# Search table file and find all specific type files\r
#\r
def GetFileList(FileModel, Db):\r
    FileList = []\r
    SqlCommand = """select FullPath from File where Model = %s""" % str(FileModel)\r
    RecordSet = Db.TblFile.Exec(SqlCommand)\r
    for Record in RecordSet:\r
        FileList.append(Record[0])\r
\r
    return FileList\r
\r
## Get the table list\r
#\r
# Search table file and find all small tables\r
#\r
def GetTableList(FileModelList, Table, Db):\r
    TableList = []\r
    SqlCommand = """select ID from File where Model in %s""" % str(FileModelList)\r
    RecordSet = Db.TblFile.Exec(SqlCommand)\r
    for Record in RecordSet:\r
        TableName = Table + str(Record[0])\r
        TableList.append(TableName)\r
\r
    return TableList\r
\r
## ParseHeaderCommentSection\r
#\r
# Parse Header comment section lines, extract Abstract, Description, Copyright\r
# , License lines\r
#\r
# @param CommentList:   List of (Comment, LineNumber)\r
# @param FileName:      FileName of the comment\r
#\r
def ParseHeaderCommentSection(CommentList, FileName = None):\r
    \r
    Abstract = ''\r
    Description = ''\r
    Copyright = ''\r
    License = ''\r
    EndOfLine = "\n"\r
    STR_HEADER_COMMENT_START = "@file"\r
    \r
    #\r
    # used to indicate the state of processing header comment section of dec, \r
    # inf files\r
    #\r
    HEADER_COMMENT_NOT_STARTED = -1\r
    HEADER_COMMENT_STARTED     = 0\r
    HEADER_COMMENT_FILE        = 1\r
    HEADER_COMMENT_ABSTRACT    = 2\r
    HEADER_COMMENT_DESCRIPTION = 3\r
    HEADER_COMMENT_COPYRIGHT   = 4\r
    HEADER_COMMENT_LICENSE     = 5\r
    HEADER_COMMENT_END         = 6\r
    #\r
    # first find the last copyright line\r
    #\r
    Last = 0\r
    HeaderCommentStage = HEADER_COMMENT_NOT_STARTED\r
    for Index in xrange(len(CommentList)-1, 0, -1):\r
        Line = CommentList[Index][0]\r
        if _IsCopyrightLine(Line):\r
            Last = Index\r
            break\r
    \r
    for Item in CommentList:\r
        Line = Item[0]\r
        LineNo = Item[1]\r
        \r
        if not Line.startswith('#') and Line:\r
            SqlStatement = """ select ID from File where FullPath like '%s'""" % FileName\r
            ResultSet = EccGlobalData.gDb.TblFile.Exec(SqlStatement)\r
            for Result in ResultSet:\r
                Msg = 'Comment must start with #'\r
                EccGlobalData.gDb.TblReport.Insert(ERROR_DOXYGEN_CHECK_FILE_HEADER, Msg, "File", Result[0])\r
        Comment = CleanString2(Line)[1]\r
        Comment = Comment.strip()\r
        #\r
        # if there are blank lines between License or Description, keep them as they would be \r
        # indication of different block; or in the position that Abstract should be, also keep it\r
        # as it indicates that no abstract\r
        #\r
        if not Comment and HeaderCommentStage not in [HEADER_COMMENT_LICENSE, \\r
                                                      HEADER_COMMENT_DESCRIPTION, HEADER_COMMENT_ABSTRACT]:\r
            continue\r
        \r
        if HeaderCommentStage == HEADER_COMMENT_NOT_STARTED:\r
            if Comment.startswith(STR_HEADER_COMMENT_START):\r
                HeaderCommentStage = HEADER_COMMENT_ABSTRACT\r
            else:\r
                License += Comment + EndOfLine\r
        else:\r
            if HeaderCommentStage == HEADER_COMMENT_ABSTRACT:\r
                #\r
                # in case there is no abstract and description\r
                #\r
                if not Comment:\r
                    Abstract = ''\r
                    HeaderCommentStage = HEADER_COMMENT_DESCRIPTION\r
                elif _IsCopyrightLine(Comment):                    \r
                    Copyright += Comment + EndOfLine\r
                    HeaderCommentStage = HEADER_COMMENT_COPYRIGHT\r
                else:                    \r
                    Abstract += Comment + EndOfLine\r
                    HeaderCommentStage = HEADER_COMMENT_DESCRIPTION\r
            elif HeaderCommentStage == HEADER_COMMENT_DESCRIPTION:\r
                #\r
                # in case there is no description\r
                #                \r
                if _IsCopyrightLine(Comment):                    \r
                    Copyright += Comment + EndOfLine\r
                    HeaderCommentStage = HEADER_COMMENT_COPYRIGHT\r
                else:\r
                    Description += Comment + EndOfLine                \r
            elif HeaderCommentStage == HEADER_COMMENT_COPYRIGHT:\r
                if _IsCopyrightLine(Comment):                    \r
                    Copyright += Comment + EndOfLine\r
                else:\r
                    #\r
                    # Contents after copyright line are license, those non-copyright lines in between\r
                    # copyright line will be discarded \r
                    #\r
                    if LineNo > Last:\r
                        if License:\r
                            License += EndOfLine\r
                        License += Comment + EndOfLine\r
                        HeaderCommentStage = HEADER_COMMENT_LICENSE                \r
            else:\r
                if not Comment and not License:\r
                    continue\r
                License += Comment + EndOfLine\r
    \r
    if not Copyright:\r
        SqlStatement = """ select ID from File where FullPath like '%s'""" % FileName\r
        ResultSet = EccGlobalData.gDb.TblFile.Exec(SqlStatement)\r
        for Result in ResultSet:\r
            Msg = 'Header comment section must have copyright information'\r
            EccGlobalData.gDb.TblReport.Insert(ERROR_DOXYGEN_CHECK_FILE_HEADER, Msg, "File", Result[0])\r
\r
    if not License:\r
        SqlStatement = """ select ID from File where FullPath like '%s'""" % FileName\r
        ResultSet = EccGlobalData.gDb.TblFile.Exec(SqlStatement)\r
        for Result in ResultSet:\r
            Msg = 'Header comment section must have license information'\r
            EccGlobalData.gDb.TblReport.Insert(ERROR_DOXYGEN_CHECK_FILE_HEADER, Msg, "File", Result[0])\r
                     \r
    return Abstract.strip(), Description.strip(), Copyright.strip(), License.strip()\r
\r
## _IsCopyrightLine\r
# check whether current line is copyright line, the criteria is whether there is case insensitive keyword "Copyright" \r
# followed by zero or more white space characters followed by a "(" character \r
#\r
# @param LineContent:  the line need to be checked\r
# @return: True if current line is copyright line, False else\r
#\r
def _IsCopyrightLine (LineContent):\r
    LineContent = LineContent.upper()\r
    Result = False\r
    \r
    ReIsCopyrightRe = re.compile(r"""(^|\s)COPYRIGHT *\(""", re.DOTALL)\r
    if ReIsCopyrightRe.search(LineContent):\r
        Result = True\r
        \r
    return Result\r
\r
\r
## CleanString2\r
#\r
# Split comments in a string\r
# Remove spaces\r
#\r
# @param Line:              The string to be cleaned\r
# @param CommentCharacter:  Comment char, used to ignore comment content, \r
#                           default is DataType.TAB_COMMENT_SPLIT\r
#\r
def CleanString2(Line, CommentCharacter='#', AllowCppStyleComment=False):\r
    #\r
    # remove whitespace\r
    #\r
    Line = Line.strip()\r
    #\r
    # Replace EDK1's comment character\r
    #\r
    if AllowCppStyleComment:\r
        Line = Line.replace('//', CommentCharacter)\r
    #\r
    # separate comments and statements\r
    #\r
    LineParts = Line.split(CommentCharacter, 1)\r
    #\r
    # remove whitespace again\r
    #\r
    Line = LineParts[0].strip()\r
    if len(LineParts) > 1:\r
        Comment = LineParts[1].strip()\r
        #\r
        # Remove prefixed and trailing comment characters\r
        #\r
        Start = 0\r
        End = len(Comment)\r
        while Start < End and Comment.startswith(CommentCharacter, Start, End):\r
            Start += 1\r
        while End >= 0 and Comment.endswith(CommentCharacter, Start, End):\r
            End -= 1\r
        Comment = Comment[Start:End]\r
        Comment = Comment.strip()\r
    else:\r
        Comment = ''\r
\r
    return Line, Comment\r
Commit	Line	Data
30fdf114 LG	1	## @file\r
	2	# This file is used to define common parser functions for meta-data\r
	3	#\r
40d841f6 LG	4	# Copyright (c) 2008, Intel Corporation. All rights reserved.<BR>\r
40d841f6 LG	5	# This program and the accompanying materials\r
30fdf114 LG	6	# are licensed and made available under the terms and conditions of the BSD License\r
	7	# which accompanies this distribution. The full text of the license may be found at\r
	8	# http://opensource.org/licenses/bsd-license.php\r
	9	#\r
	10	# THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,\r
	11	# WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.\r
	12	#\r
	13	\r
	14	import os\r
	15	from CommonDataClass.DataClass import *\r
d0acc87a LG	16	from EccToolError import *\r
	17	import EccGlobalData\r
	18	import re\r
30fdf114 LG	19	## Get the inlcude path list for a source file\r
	20	#\r
	21	# 1. Find the source file belongs to which inf file\r
	22	# 2. Find the inf's package\r
	23	# 3. Return the include path list of the package\r
	24	#\r
	25	def GetIncludeListOfFile(WorkSpace, Filepath, Db):\r
	26	IncludeList = []\r
	27	Filepath = os.path.normpath(Filepath)\r
	28	SqlCommand = """\r
	29	select Value1, FullPath from Inf, File where Inf.Model = %s and Inf.BelongsToFile in(\r
e56468c0	30	select distinct B.BelongsToFile from File as A left join Inf as B\r
30fdf114 LG	31	where A.ID = B.BelongsToFile and B.Model = %s and (A.Path \|\| '%s' \|\| B.Value1) = '%s')\r
	32	and Inf.BelongsToFile = File.ID""" \\r
	33	% (MODEL_META_DATA_PACKAGE, MODEL_EFI_SOURCE_FILE, '\\', Filepath)\r
	34	RecordSet = Db.TblFile.Exec(SqlCommand)\r
	35	for Record in RecordSet:\r
	36	DecFullPath = os.path.normpath(os.path.join(WorkSpace, Record[0]))\r
	37	InfFullPath = os.path.normpath(os.path.join(WorkSpace, Record[1]))\r
	38	(DecPath, DecName) = os.path.split(DecFullPath)\r
	39	(InfPath, InfName) = os.path.split(InfFullPath)\r
e56468c0	40	SqlCommand = """select Value1 from Dec where BelongsToFile =\r
30fdf114 LG	41	(select ID from File where FullPath = '%s') and Model = %s""" \\r
	42	% (DecFullPath, MODEL_EFI_INCLUDE)\r
	43	NewRecordSet = Db.TblDec.Exec(SqlCommand)\r
	44	if InfPath not in IncludeList:\r
	45	IncludeList.append(InfPath)\r
	46	for NewRecord in NewRecordSet:\r
	47	IncludePath = os.path.normpath(os.path.join(DecPath, NewRecord[0]))\r
	48	if IncludePath not in IncludeList:\r
	49	IncludeList.append(IncludePath)\r
e56468c0	50	\r
30fdf114 LG	51	return IncludeList\r
30fdf114 LG	52	\r
e56468c0	53	## Get the file list\r
	54	#\r
	55	# Search table file and find all specific type files\r
	56	#\r
	57	def GetFileList(FileModel, Db):\r
	58	FileList = []\r
	59	SqlCommand = """select FullPath from File where Model = %s""" % str(FileModel)\r
	60	RecordSet = Db.TblFile.Exec(SqlCommand)\r
	61	for Record in RecordSet:\r
	62	FileList.append(Record[0])\r
	63	\r
	64	return FileList\r
	65	\r
30fdf114 LG	66	## Get the table list\r
	67	#\r
	68	# Search table file and find all small tables\r
	69	#\r
	70	def GetTableList(FileModelList, Table, Db):\r
	71	TableList = []\r
	72	SqlCommand = """select ID from File where Model in %s""" % str(FileModelList)\r
	73	RecordSet = Db.TblFile.Exec(SqlCommand)\r
	74	for Record in RecordSet:\r
	75	TableName = Table + str(Record[0])\r
	76	TableList.append(TableName)\r
e56468c0	77	\r
30fdf114 LG	78	return TableList\r
30fdf114 LG	79	\r
d0acc87a LG	80	## ParseHeaderCommentSection\r
	81	#\r
	82	# Parse Header comment section lines, extract Abstract, Description, Copyright\r
	83	# , License lines\r
	84	#\r
	85	# @param CommentList: List of (Comment, LineNumber)\r
	86	# @param FileName: FileName of the comment\r
	87	#\r
	88	def ParseHeaderCommentSection(CommentList, FileName = None):\r
	89	\r
	90	Abstract = ''\r
	91	Description = ''\r
	92	Copyright = ''\r
	93	License = ''\r
	94	EndOfLine = "\n"\r
	95	STR_HEADER_COMMENT_START = "@file"\r
	96	\r
	97	#\r
	98	# used to indicate the state of processing header comment section of dec, \r
	99	# inf files\r
	100	#\r
	101	HEADER_COMMENT_NOT_STARTED = -1\r
	102	HEADER_COMMENT_STARTED = 0\r
	103	HEADER_COMMENT_FILE = 1\r
	104	HEADER_COMMENT_ABSTRACT = 2\r
	105	HEADER_COMMENT_DESCRIPTION = 3\r
	106	HEADER_COMMENT_COPYRIGHT = 4\r
	107	HEADER_COMMENT_LICENSE = 5\r
	108	HEADER_COMMENT_END = 6\r
	109	#\r
	110	# first find the last copyright line\r
	111	#\r
	112	Last = 0\r
	113	HeaderCommentStage = HEADER_COMMENT_NOT_STARTED\r
	114	for Index in xrange(len(CommentList)-1, 0, -1):\r
	115	Line = CommentList[Index][0]\r
	116	if _IsCopyrightLine(Line):\r
	117	Last = Index\r
	118	break\r
	119	\r
	120	for Item in CommentList:\r
	121	Line = Item[0]\r
	122	LineNo = Item[1]\r
	123	\r
	124	if not Line.startswith('#') and Line:\r
	125	SqlStatement = """ select ID from File where FullPath like '%s'""" % FileName\r
	126	ResultSet = EccGlobalData.gDb.TblFile.Exec(SqlStatement)\r
	127	for Result in ResultSet:\r
	128	Msg = 'Comment must start with #'\r
	129	EccGlobalData.gDb.TblReport.Insert(ERROR_DOXYGEN_CHECK_FILE_HEADER, Msg, "File", Result[0])\r
	130	Comment = CleanString2(Line)[1]\r
	131	Comment = Comment.strip()\r
	132	#\r
	133	# if there are blank lines between License or Description, keep them as they would be \r
	134	# indication of different block; or in the position that Abstract should be, also keep it\r
	135	# as it indicates that no abstract\r
	136	#\r
	137	if not Comment and HeaderCommentStage not in [HEADER_COMMENT_LICENSE, \\r
	138	HEADER_COMMENT_DESCRIPTION, HEADER_COMMENT_ABSTRACT]:\r
	139	continue\r
	140	\r
	141	if HeaderCommentStage == HEADER_COMMENT_NOT_STARTED:\r
	142	if Comment.startswith(STR_HEADER_COMMENT_START):\r
	143	HeaderCommentStage = HEADER_COMMENT_ABSTRACT\r
144	else:\r
145	License += Comment + EndOfLine\r
146	else:\r
147	if HeaderCommentStage == HEADER_COMMENT_ABSTRACT:\r
148	#\r
149	# in case there is no abstract and description\r
150	#\r
151	if not Comment:\r
152	Abstract = ''\r
153	HeaderCommentStage = HEADER_COMMENT_DESCRIPTION\r
154	elif _IsCopyrightLine(Comment): \r
155	Copyright += Comment + EndOfLine\r
156	HeaderCommentStage = HEADER_COMMENT_COPYRIGHT\r
157	else: \r
158	Abstract += Comment + EndOfLine\r
159	HeaderCommentStage = HEADER_COMMENT_DESCRIPTION\r
160	elif HeaderCommentStage == HEADER_COMMENT_DESCRIPTION:\r
161	#\r
162	# in case there is no description\r
163	# \r
164	if _IsCopyrightLine(Comment): \r
165	Copyright += Comment + EndOfLine\r
166	HeaderCommentStage = HEADER_COMMENT_COPYRIGHT\r
167	else:\r
168	Description += Comment + EndOfLine \r
169	elif HeaderCommentStage == HEADER_COMMENT_COPYRIGHT:\r
170	if _IsCopyrightLine(Comment): \r
171	Copyright += Comment + EndOfLine\r
172	else:\r
173	#\r
174	# Contents after copyright line are license, those non-copyright lines in between\r
175	# copyright line will be discarded \r
176	#\r
177	if LineNo > Last:\r
178	if License:\r
179	License += EndOfLine\r
180	License += Comment + EndOfLine\r
181	HeaderCommentStage = HEADER_COMMENT_LICENSE \r
182	else:\r
183	if not Comment and not License:\r
184	continue\r
185	License += Comment + EndOfLine\r
186	\r
187	if not Copyright:\r
188	SqlStatement = """ select ID from File where FullPath like '%s'""" % FileName\r
189	ResultSet = EccGlobalData.gDb.TblFile.Exec(SqlStatement)\r
190	for Result in ResultSet:\r
191	Msg = 'Header comment section must have copyright information'\r
192	EccGlobalData.gDb.TblReport.Insert(ERROR_DOXYGEN_CHECK_FILE_HEADER, Msg, "File", Result[0])\r
193	\r
194	if not License:\r
195	SqlStatement = """ select ID from File where FullPath like '%s'""" % FileName\r
196	ResultSet = EccGlobalData.gDb.TblFile.Exec(SqlStatement)\r
197	for Result in ResultSet:\r
198	Msg = 'Header comment section must have license information'\r
199	EccGlobalData.gDb.TblReport.Insert(ERROR_DOXYGEN_CHECK_FILE_HEADER, Msg, "File", Result[0])\r
200	\r
201	return Abstract.strip(), Description.strip(), Copyright.strip(), License.strip()\r
202	\r
203	## _IsCopyrightLine\r
204	# check whether current line is copyright line, the criteria is whether there is case insensitive keyword "Copyright" \r
205	# followed by zero or more white space characters followed by a "(" character \r
206	#\r
207	# @param LineContent: the line need to be checked\r
208	# @return: True if current line is copyright line, False else\r
209	#\r
210	def _IsCopyrightLine (LineContent):\r
211	LineContent = LineContent.upper()\r
212	Result = False\r
213	\r
214	ReIsCopyrightRe = re.compile(r"""(^\|\s)COPYRIGHT *\(""", re.DOTALL)\r
215	if ReIsCopyrightRe.search(LineContent):\r
216	Result = True\r
217	\r
218	return Result\r
219	\r
220	\r
221	## CleanString2\r
222	#\r
223	# Split comments in a string\r
224	# Remove spaces\r
225	#\r
226	# @param Line: The string to be cleaned\r
227	# @param CommentCharacter: Comment char, used to ignore comment content, \r
228	# default is DataType.TAB_COMMENT_SPLIT\r
229	#\r
230	def CleanString2(Line, CommentCharacter='#', AllowCppStyleComment=False):\r
231	#\r
232	# remove whitespace\r
233	#\r
234	Line = Line.strip()\r
235	#\r
236	# Replace EDK1's comment character\r
237	#\r
238	if AllowCppStyleComment:\r
239	Line = Line.replace('//', CommentCharacter)\r
240	#\r
241	# separate comments and statements\r
242	#\r
243	LineParts = Line.split(CommentCharacter, 1)\r
244	#\r
245	# remove whitespace again\r
246	#\r
247	Line = LineParts[0].strip()\r
248	if len(LineParts) > 1:\r
249	Comment = LineParts[1].strip()\r
250	#\r
251	# Remove prefixed and trailing comment characters\r
252	#\r
253	Start = 0\r
254	End = len(Comment)\r
255	while Start < End and Comment.startswith(CommentCharacter, Start, End):\r
256	Start += 1\r
257	while End >= 0 and Comment.endswith(CommentCharacter, Start, End):\r
258	End -= 1\r
259	Comment = Comment[Start:End]\r
260	Comment = Comment.strip()\r
261	else:\r
262	Comment = ''\r
263	\r
264	return Line, Comment\r