[mirror_edk2.git] / .pytool / Plugin / SpellCheck / SpellCheck.py

# @file SpellCheck.py\r
#\r
# An edk2-pytool based plugin wrapper for cspell\r
#\r
# Copyright (c) Microsoft Corporation.\r
# SPDX-License-Identifier: BSD-2-Clause-Patent\r
##\r
import logging\r
import json\r
import yaml\r
from io import StringIO\r
import os\r
from edk2toolext.environment.plugintypes.ci_build_plugin import ICiBuildPlugin\r
from edk2toollib.utility_functions import RunCmd\r
from edk2toolext.environment.var_dict import VarDict\r
from edk2toollib.gitignore_parser import parse_gitignore_lines\r
from edk2toolext.environment import version_aggregator\r
\r
\r
class SpellCheck(ICiBuildPlugin):\r
    """\r
    A CiBuildPlugin that uses the cspell node module to scan the files\r
    from the package being tested for spelling errors.  The plugin contains\r
    the base cspell.json file then thru the configuration options other settings\r
    can be changed or extended.\r
\r
    Configuration options:\r
    "SpellCheck": {\r
        "AuditOnly": False,          # Don't fail the build if there are errors.  Just log them\r
        "IgnoreFiles": [],           # use gitignore syntax to ignore errors in matching files\r
        "ExtendWords": [],           # words to extend to the dictionary for this package\r
        "IgnoreStandardPaths": [],   # Standard Plugin defined paths that should be ignore\r
        "AdditionalIncludePaths": [] # Additional paths to spell check (wildcards supported)\r
    }\r
    """\r
\r
    #\r
    # A package can remove any of these using IgnoreStandardPaths\r
    #\r
    STANDARD_PLUGIN_DEFINED_PATHS = ("*.c", "*.h",\r
                                     "*.nasm", "*.asm", "*.masm", "*.s",\r
                                     "*.asl",\r
                                     "*.dsc", "*.dec", "*.fdf", "*.inf",\r
                                     "*.md", "*.txt"\r
                                     )\r
\r
    def GetTestName(self, packagename: str, environment: VarDict) -> tuple:\r
        """ Provide the testcase name and classname for use in reporting\r
\r
            Args:\r
              packagename: string containing name of package to build\r
              environment: The VarDict for the test to run in\r
            Returns:\r
                a tuple containing the testcase name and the classname\r
                (testcasename, classname)\r
                testclassname: a descriptive string for the testcase can include whitespace\r
                classname: should be patterned <packagename>.<plugin>.<optionally any unique condition>\r
        """\r
        return ("Spell check files in " + packagename, packagename + ".SpellCheck")\r
\r
    ##\r
    # External function of plugin.  This function is used to perform the task of the CiBuild Plugin\r
    #\r
    #   - package is the edk2 path to package.  This means workspace/packagepath relative.\r
    #   - edk2path object configured with workspace and packages path\r
    #   - PkgConfig Object (dict) for the pkg\r
    #   - EnvConfig Object\r
    #   - Plugin Manager Instance\r
    #   - Plugin Helper Obj Instance\r
    #   - Junit Logger\r
    #   - output_stream the StringIO output stream from this plugin via logging\r
\r
    def RunBuildPlugin(self, packagename, Edk2pathObj, pkgconfig, environment, PLM, PLMHelper, tc, output_stream=None):\r
        Errors = []\r
\r
        abs_pkg_path = Edk2pathObj.GetAbsolutePathOnThisSytemFromEdk2RelativePath(\r
            packagename)\r
\r
        if abs_pkg_path is None:\r
            tc.SetSkipped()\r
            tc.LogStdError("No package {0}".format(packagename))\r
            return -1\r
\r
        # check for node\r
        return_buffer = StringIO()\r
        ret = RunCmd("node", "--version", outstream=return_buffer)\r
        if (ret != 0):\r
            tc.SetSkipped()\r
            tc.LogStdError("NodeJs not installed. Test can't run")\r
            logging.warning("NodeJs not installed. Test can't run")\r
            return -1\r
        node_version = return_buffer.getvalue().strip()  # format vXX.XX.XX\r
        tc.LogStdOut(f"Node version: {node_version}")\r
        version_aggregator.GetVersionAggregator().ReportVersion(\r
            "NodeJs", node_version, version_aggregator.VersionTypes.INFO)\r
\r
        # Check for cspell\r
        return_buffer = StringIO()\r
        ret = RunCmd("cspell", "--version", outstream=return_buffer)\r
        if (ret != 0):\r
            tc.SetSkipped()\r
            tc.LogStdError("cspell not installed.  Test can't run")\r
            logging.warning("cspell not installed.  Test can't run")\r
            return -1\r
        cspell_version = return_buffer.getvalue().strip()  # format XX.XX.XX\r
        tc.LogStdOut(f"CSpell version: {cspell_version}")\r
        version_aggregator.GetVersionAggregator().ReportVersion(\r
            "CSpell", cspell_version, version_aggregator.VersionTypes.INFO)\r
\r
        # copy the default as a list\r
        package_relative_paths_to_spell_check = list(SpellCheck.STANDARD_PLUGIN_DEFINED_PATHS)\r
\r
        #\r
        # Allow the ci.yaml to remove any of the above standard paths\r
        #\r
        if("IgnoreStandardPaths" in pkgconfig):\r
            for a in pkgconfig["IgnoreStandardPaths"]:\r
                if(a in package_relative_paths_to_spell_check):\r
                    tc.LogStdOut(\r
                        f"ignoring standard path due to ci.yaml ignore: {a}")\r
                    package_relative_paths_to_spell_check.remove(a)\r
                else:\r
                    tc.LogStdOut(f"Invalid IgnoreStandardPaths value: {a}")\r
\r
        #\r
        # check for any additional include paths defined by package config\r
        #\r
        if("AdditionalIncludePaths" in pkgconfig):\r
            package_relative_paths_to_spell_check.extend(\r
                pkgconfig["AdditionalIncludePaths"])\r
\r
        #\r
        # Make the path string for cspell to check\r
        #\r
        relpath = os.path.relpath(abs_pkg_path)\r
        cpsell_paths = " ".join(\r
            # Double quote each path to defer expansion to cspell parameters\r
            [f'"{relpath}/**/{x}"' for x in package_relative_paths_to_spell_check])\r
\r
        # Make the config file\r
        config_file_path = os.path.join(\r
            Edk2pathObj.WorkspacePath, "Build", packagename, "cspell_actual_config.json")\r
        mydir = os.path.dirname(os.path.abspath(__file__))\r
        # load as yaml so it can have comments\r
        base = os.path.join(mydir, "cspell.base.yaml")\r
        with open(base, "r") as i:\r
            config = yaml.safe_load(i)\r
\r
        if("ExtendWords" in pkgconfig):\r
            config["words"].extend(pkgconfig["ExtendWords"])\r
        with open(config_file_path, "w") as o:\r
            json.dump(config, o)  # output as json so compat with cspell\r
\r
        All_Ignores = []\r
        # Parse the config for other ignores\r
        if "IgnoreFiles" in pkgconfig:\r
            All_Ignores.extend(pkgconfig["IgnoreFiles"])\r
\r
        # spell check all the files\r
        ignore = parse_gitignore_lines(All_Ignores, os.path.join(\r
            abs_pkg_path, "nofile.txt"), abs_pkg_path)\r
\r
        # result is a list of strings like this\r
        #  C:\src\sp-edk2\edk2\FmpDevicePkg\FmpDevicePkg.dec:53:9 - Unknown word (Capule)\r
        EasyFix = []\r
        results = self._check_spelling(cpsell_paths, config_file_path)\r
        for r in results:\r
            path, _, word = r.partition(" - Unknown word ")\r
            if len(word) == 0:\r
                # didn't find pattern\r
                continue\r
\r
            pathinfo = path.rsplit(":", 2)  # remove the line no info\r
            if(ignore(pathinfo[0])):  # check against ignore list\r
                tc.LogStdOut(f"ignoring error due to ci.yaml ignore: {r}")\r
                continue\r
\r
            # real error\r
            EasyFix.append(word.strip().strip("()"))\r
            Errors.append(r)\r
\r
        # Log all errors tc StdError\r
        for l in Errors:\r
            tc.LogStdError(l.strip())\r
\r
        # Helper - Log the syntax needed to add these words to dictionary\r
        if len(EasyFix) > 0:\r
            EasyFix = sorted(set(a.lower() for a in EasyFix))\r
            tc.LogStdOut("\n Easy fix:")\r
            OneString = "If these are not errors add this to your ci.yaml file.\n"\r
            OneString += '"SpellCheck": {\n  "ExtendWords": ['\r
            for a in EasyFix:\r
                tc.LogStdOut(f'\n"{a}",')\r
                OneString += f'\n    "{a}",'\r
            logging.info(OneString.rstrip(",") + '\n  ]\n}')\r
\r
        # add result to test case\r
        overall_status = len(Errors)\r
        if overall_status != 0:\r
            if "AuditOnly" in pkgconfig and pkgconfig["AuditOnly"]:\r
                # set as skipped if AuditOnly\r
                tc.SetSkipped()\r
                return -1\r
            else:\r
                tc.SetFailed("SpellCheck {0} Failed.  Errors {1}".format(\r
                    packagename, overall_status), "CHECK_FAILED")\r
        else:\r
            tc.SetSuccess()\r
        return overall_status\r
\r
    def _check_spelling(self, abs_file_to_check: str, abs_config_file_to_use: str) -> []:\r
        output = StringIO()\r
        ret = RunCmd(\r
            "cspell", f"--config {abs_config_file_to_use} {abs_file_to_check}", outstream=output)\r
        if ret == 0:\r
            return []\r
        else:\r
            return output.getvalue().strip().splitlines()\r
Commit	Line	Data
9da7846c SB	1	# @file SpellCheck.py\r
	2	#\r
	3	# An edk2-pytool based plugin wrapper for cspell\r
	4	#\r
	5	# Copyright (c) Microsoft Corporation.\r
	6	# SPDX-License-Identifier: BSD-2-Clause-Patent\r
	7	##\r
	8	import logging\r
	9	import json\r
	10	import yaml\r
	11	from io import StringIO\r
	12	import os\r
	13	from edk2toolext.environment.plugintypes.ci_build_plugin import ICiBuildPlugin\r
	14	from edk2toollib.utility_functions import RunCmd\r
	15	from edk2toolext.environment.var_dict import VarDict\r
	16	from edk2toollib.gitignore_parser import parse_gitignore_lines\r
	17	from edk2toolext.environment import version_aggregator\r
	18	\r
	19	\r
	20	class SpellCheck(ICiBuildPlugin):\r
	21	"""\r
	22	A CiBuildPlugin that uses the cspell node module to scan the files\r
	23	from the package being tested for spelling errors. The plugin contains\r
	24	the base cspell.json file then thru the configuration options other settings\r
	25	can be changed or extended.\r
	26	\r
	27	Configuration options:\r
	28	"SpellCheck": {\r
	29	"AuditOnly": False, # Don't fail the build if there are errors. Just log them\r
	30	"IgnoreFiles": [], # use gitignore syntax to ignore errors in matching files\r
	31	"ExtendWords": [], # words to extend to the dictionary for this package\r
	32	"IgnoreStandardPaths": [], # Standard Plugin defined paths that should be ignore\r
	33	"AdditionalIncludePaths": [] # Additional paths to spell check (wildcards supported)\r
	34	}\r
	35	"""\r
	36	\r
	37	#\r
	38	# A package can remove any of these using IgnoreStandardPaths\r
	39	#\r
288bd74a	40	STANDARD_PLUGIN_DEFINED_PATHS = (".c", ".h",\r
9da7846c SB	41	".nasm", ".asm", ".masm", ".s",\r
	42	"*.asl",\r
	43	".dsc", ".dec", ".fdf", ".inf",\r
	44	".md", ".txt"\r
288bd74a	45	)\r
9da7846c SB	46	\r
	47	def GetTestName(self, packagename: str, environment: VarDict) -> tuple:\r
	48	""" Provide the testcase name and classname for use in reporting\r
	49	\r
	50	Args:\r
	51	packagename: string containing name of package to build\r
	52	environment: The VarDict for the test to run in\r
	53	Returns:\r
	54	a tuple containing the testcase name and the classname\r
	55	(testcasename, classname)\r
	56	testclassname: a descriptive string for the testcase can include whitespace\r
	57	classname: should be patterned <packagename>.<plugin>.<optionally any unique condition>\r
	58	"""\r
	59	return ("Spell check files in " + packagename, packagename + ".SpellCheck")\r
	60	\r
	61	##\r
	62	# External function of plugin. This function is used to perform the task of the CiBuild Plugin\r
	63	#\r
	64	# - package is the edk2 path to package. This means workspace/packagepath relative.\r
	65	# - edk2path object configured with workspace and packages path\r
	66	# - PkgConfig Object (dict) for the pkg\r
	67	# - EnvConfig Object\r
	68	# - Plugin Manager Instance\r
	69	# - Plugin Helper Obj Instance\r
	70	# - Junit Logger\r
	71	# - output_stream the StringIO output stream from this plugin via logging\r
	72	\r
	73	def RunBuildPlugin(self, packagename, Edk2pathObj, pkgconfig, environment, PLM, PLMHelper, tc, output_stream=None):\r
	74	Errors = []\r
	75	\r
	76	abs_pkg_path = Edk2pathObj.GetAbsolutePathOnThisSytemFromEdk2RelativePath(\r
	77	packagename)\r
	78	\r
	79	if abs_pkg_path is None:\r
	80	tc.SetSkipped()\r
	81	tc.LogStdError("No package {0}".format(packagename))\r
	82	return -1\r
	83	\r
	84	# check for node\r
	85	return_buffer = StringIO()\r
	86	ret = RunCmd("node", "--version", outstream=return_buffer)\r
	87	if (ret != 0):\r
	88	tc.SetSkipped()\r
	89	tc.LogStdError("NodeJs not installed. Test can't run")\r
	90	logging.warning("NodeJs not installed. Test can't run")\r
	91	return -1\r
	92	node_version = return_buffer.getvalue().strip() # format vXX.XX.XX\r
	93	tc.LogStdOut(f"Node version: {node_version}")\r
	94	version_aggregator.GetVersionAggregator().ReportVersion(\r
	95	"NodeJs", node_version, version_aggregator.VersionTypes.INFO)\r
	96	\r
	97	# Check for cspell\r
	98	return_buffer = StringIO()\r
	99	ret = RunCmd("cspell", "--version", outstream=return_buffer)\r
	100	if (ret != 0):\r
	101	tc.SetSkipped()\r
	102	tc.LogStdError("cspell not installed. Test can't run")\r
	103	logging.warning("cspell not installed. Test can't run")\r
	104	return -1\r
	105	cspell_version = return_buffer.getvalue().strip() # format XX.XX.XX\r
	106	tc.LogStdOut(f"CSpell version: {cspell_version}")\r
	107	version_aggregator.GetVersionAggregator().ReportVersion(\r
	108	"CSpell", cspell_version, version_aggregator.VersionTypes.INFO)\r
	109	\r
288bd74a SB	110	# copy the default as a list\r
288bd74a SB	111	package_relative_paths_to_spell_check = list(SpellCheck.STANDARD_PLUGIN_DEFINED_PATHS)\r
9da7846c SB	112	\r
	113	#\r
	114	# Allow the ci.yaml to remove any of the above standard paths\r
	115	#\r
	116	if("IgnoreStandardPaths" in pkgconfig):\r
	117	for a in pkgconfig["IgnoreStandardPaths"]:\r
	118	if(a in package_relative_paths_to_spell_check):\r
	119	tc.LogStdOut(\r
	120	f"ignoring standard path due to ci.yaml ignore: {a}")\r
	121	package_relative_paths_to_spell_check.remove(a)\r
	122	else:\r
	123	tc.LogStdOut(f"Invalid IgnoreStandardPaths value: {a}")\r
	124	\r
	125	#\r
	126	# check for any additional include paths defined by package config\r
	127	#\r
	128	if("AdditionalIncludePaths" in pkgconfig):\r
	129	package_relative_paths_to_spell_check.extend(\r
	130	pkgconfig["AdditionalIncludePaths"])\r
	131	\r
	132	#\r
	133	# Make the path string for cspell to check\r
	134	#\r
	135	relpath = os.path.relpath(abs_pkg_path)\r
	136	cpsell_paths = " ".join(\r
f47c4676 SB	137	# Double quote each path to defer expansion to cspell parameters\r
f47c4676 SB	138	[f'"{relpath}/**/{x}"' for x in package_relative_paths_to_spell_check])\r
9da7846c SB	139	\r
	140	# Make the config file\r
	141	config_file_path = os.path.join(\r
	142	Edk2pathObj.WorkspacePath, "Build", packagename, "cspell_actual_config.json")\r
	143	mydir = os.path.dirname(os.path.abspath(__file__))\r
	144	# load as yaml so it can have comments\r
	145	base = os.path.join(mydir, "cspell.base.yaml")\r
	146	with open(base, "r") as i:\r
	147	config = yaml.safe_load(i)\r
	148	\r
	149	if("ExtendWords" in pkgconfig):\r
	150	config["words"].extend(pkgconfig["ExtendWords"])\r
	151	with open(config_file_path, "w") as o:\r
	152	json.dump(config, o) # output as json so compat with cspell\r
	153	\r
	154	All_Ignores = []\r
	155	# Parse the config for other ignores\r
	156	if "IgnoreFiles" in pkgconfig:\r
	157	All_Ignores.extend(pkgconfig["IgnoreFiles"])\r
	158	\r
	159	# spell check all the files\r
	160	ignore = parse_gitignore_lines(All_Ignores, os.path.join(\r
	161	abs_pkg_path, "nofile.txt"), abs_pkg_path)\r
	162	\r
	163	# result is a list of strings like this\r
	164	# C:\src\sp-edk2\edk2\FmpDevicePkg\FmpDevicePkg.dec:53:9 - Unknown word (Capule)\r
	165	EasyFix = []\r
	166	results = self._check_spelling(cpsell_paths, config_file_path)\r
	167	for r in results:\r
	168	path, _, word = r.partition(" - Unknown word ")\r
	169	if len(word) == 0:\r
	170	# didn't find pattern\r
	171	continue\r
	172	\r
	173	pathinfo = path.rsplit(":", 2) # remove the line no info\r
	174	if(ignore(pathinfo[0])): # check against ignore list\r
	175	tc.LogStdOut(f"ignoring error due to ci.yaml ignore: {r}")\r
	176	continue\r
	177	\r
	178	# real error\r
	179	EasyFix.append(word.strip().strip("()"))\r
	180	Errors.append(r)\r
	181	\r
	182	# Log all errors tc StdError\r
	183	for l in Errors:\r
	184	tc.LogStdError(l.strip())\r
	185	\r
	186	# Helper - Log the syntax needed to add these words to dictionary\r
	187	if len(EasyFix) > 0:\r
	188	EasyFix = sorted(set(a.lower() for a in EasyFix))\r
	189	tc.LogStdOut("\n Easy fix:")\r
	190	OneString = "If these are not errors add this to your ci.yaml file.\n"\r
	191	OneString += '"SpellCheck": {\n "ExtendWords": ['\r
	192	for a in EasyFix:\r
	193	tc.LogStdOut(f'\n"{a}",')\r
	194	OneString += f'\n "{a}",'\r
	195	logging.info(OneString.rstrip(",") + '\n ]\n}')\r
	196	\r
	197	# add result to test case\r
	198	overall_status = len(Errors)\r
	199	if overall_status != 0:\r
	200	if "AuditOnly" in pkgconfig and pkgconfig["AuditOnly"]:\r
	201	# set as skipped if AuditOnly\r
	202	tc.SetSkipped()\r
203	return -1\r
204	else:\r
205	tc.SetFailed("SpellCheck {0} Failed. Errors {1}".format(\r
206	packagename, overall_status), "CHECK_FAILED")\r
207	else:\r
208	tc.SetSuccess()\r
209	return overall_status\r
210	\r
211	def _check_spelling(self, abs_file_to_check: str, abs_config_file_to_use: str) -> []:\r
212	output = StringIO()\r
213	ret = RunCmd(\r
214	"cspell", f"--config {abs_config_file_to_use} {abs_file_to_check}", outstream=output)\r
215	if ret == 0:\r
216	return []\r
217	else:\r
218	return output.getvalue().strip().splitlines()\r