wehub-resource-sync
261 lines
8.0 KiB
Python
261 lines
8.0 KiB
Python
#!/usr/bin/env python3
|
|
# pylint: disable=protected-access, unused-variable, locally-disabled, len-as-condition
|
|
"""Lint helper to generate lint summary of source.
|
|
|
|
Copyright by Contributors.
|
|
|
|
Borrowed from dmlc-core/scripts/lint.py@939c052
|
|
"""
|
|
from __future__ import print_function
|
|
|
|
import argparse
|
|
import codecs
|
|
import os
|
|
import re
|
|
import sys
|
|
|
|
import cpplint
|
|
from cpplint import _cpplint_state
|
|
from pylint import epylint
|
|
|
|
CXX_SUFFIX = set(["cc", "c", "cpp", "h", "cu", "hpp", "cuh"])
|
|
PYTHON_SUFFIX = set(["py"])
|
|
|
|
|
|
def filepath_enumerate(paths):
|
|
"""Enumerate the file paths of all subfiles of the list of paths"""
|
|
out = []
|
|
for path in paths:
|
|
if os.path.isfile(path):
|
|
out.append(path)
|
|
else:
|
|
for root, dirs, files in os.walk(path):
|
|
for name in files:
|
|
out.append(os.path.normpath(os.path.join(root, name)))
|
|
return out
|
|
|
|
|
|
# pylint: disable=useless-object-inheritance
|
|
class LintHelper(object):
|
|
"""Class to help runing the lint and records summary"""
|
|
|
|
@staticmethod
|
|
def _print_summary_map(strm, result_map, ftype):
|
|
"""Print summary of certain result map."""
|
|
if len(result_map) == 0:
|
|
return 0
|
|
npass = sum(1 for x in result_map.values() if len(x) == 0)
|
|
strm.write(
|
|
f"====={npass}/{len(result_map)} {ftype} files passed check=====\n"
|
|
)
|
|
for fname, emap in result_map.items():
|
|
if len(emap) == 0:
|
|
continue
|
|
strm.write(
|
|
f"{fname}: {sum(emap.values())} Errors of {len(emap)} Categories map={str(emap)}\n"
|
|
)
|
|
return len(result_map) - npass
|
|
|
|
def __init__(self):
|
|
self.project_name = None
|
|
self.cpp_header_map = {}
|
|
self.cpp_src_map = {}
|
|
self.python_map = {}
|
|
pylint_disable = [
|
|
"superfluous-parens",
|
|
"too-many-instance-attributes",
|
|
"too-few-public-methods",
|
|
]
|
|
# setup pylint
|
|
self.pylint_opts = [
|
|
"--extension-pkg-whitelist=numpy",
|
|
"--disable=" + ",".join(pylint_disable),
|
|
]
|
|
|
|
self.pylint_cats = set(["error", "warning", "convention", "refactor"])
|
|
# setup cpp lint
|
|
cpplint_args = [
|
|
"--quiet",
|
|
"--extensions=" + (",".join(CXX_SUFFIX)),
|
|
".",
|
|
]
|
|
_ = cpplint.ParseArguments(cpplint_args)
|
|
cpplint._SetFilters(
|
|
",".join(
|
|
[
|
|
"-build/c++11",
|
|
"-build/namespaces",
|
|
"-build/include,",
|
|
"+build/include_what_you_use",
|
|
"+build/include_order",
|
|
]
|
|
)
|
|
)
|
|
cpplint._SetCountingStyle("toplevel")
|
|
cpplint._line_length = 80
|
|
|
|
def process_cpp(self, path, suffix):
|
|
"""Process a cpp file."""
|
|
_cpplint_state.ResetErrorCounts()
|
|
cpplint.ProcessFile(str(path), _cpplint_state.verbose_level)
|
|
_cpplint_state.PrintErrorCounts()
|
|
errors = _cpplint_state.errors_by_category.copy()
|
|
|
|
if suffix == "h":
|
|
self.cpp_header_map[str(path)] = errors
|
|
else:
|
|
self.cpp_src_map[str(path)] = errors
|
|
|
|
def process_python(self, path):
|
|
"""Process a python file."""
|
|
(pylint_stdout, pylint_stderr) = epylint.py_run(
|
|
" ".join([str(path)] + self.pylint_opts), return_std=True
|
|
)
|
|
emap = {}
|
|
err = pylint_stderr.read()
|
|
if len(err):
|
|
print(err)
|
|
for line in pylint_stdout:
|
|
sys.stderr.write(line)
|
|
key = line.split(":")[-1].split("(")[0].strip()
|
|
if key not in self.pylint_cats:
|
|
continue
|
|
if key not in emap:
|
|
emap[key] = 1
|
|
else:
|
|
emap[key] += 1
|
|
self.python_map[str(path)] = emap
|
|
|
|
def print_summary(self, strm):
|
|
"""Print summary of lint."""
|
|
nerr = 0
|
|
nerr += LintHelper._print_summary_map(
|
|
strm, self.cpp_header_map, "cpp-header"
|
|
)
|
|
nerr += LintHelper._print_summary_map(
|
|
strm, self.cpp_src_map, "cpp-source"
|
|
)
|
|
nerr += LintHelper._print_summary_map(strm, self.python_map, "python")
|
|
if nerr == 0:
|
|
strm.write("All passed!\n")
|
|
else:
|
|
strm.write(f"{nerr} files failed lint\n")
|
|
return nerr
|
|
|
|
|
|
# singleton helper for lint check
|
|
_HELPER = LintHelper()
|
|
|
|
|
|
def get_header_guard_dmlc(filename):
|
|
"""Get Header Guard Convention for DMLC Projects.
|
|
|
|
For headers in include, directly use the path
|
|
For headers in src, use project name plus path
|
|
|
|
Examples: with project-name = dmlc
|
|
include/dmlc/timer.h -> DMLC_TIMTER_H_
|
|
src/io/libsvm_parser.h -> DMLC_IO_LIBSVM_PARSER_H_
|
|
"""
|
|
fileinfo = cpplint.FileInfo(filename)
|
|
file_path_from_root = fileinfo.RepositoryName()
|
|
inc_list = ["include", "api", "wrapper", "contrib"]
|
|
if os.name == "nt":
|
|
inc_list.append("mshadow")
|
|
|
|
if (
|
|
file_path_from_root.find("src/") != -1
|
|
and _HELPER.project_name is not None
|
|
):
|
|
idx = file_path_from_root.find("src/")
|
|
file_path_from_root = (
|
|
_HELPER.project_name + file_path_from_root[idx + 3 :]
|
|
)
|
|
else:
|
|
idx = file_path_from_root.find("include/")
|
|
if idx != -1:
|
|
file_path_from_root = file_path_from_root[idx + 8 :]
|
|
for spath in inc_list:
|
|
prefix = spath + "/"
|
|
if file_path_from_root.startswith(prefix):
|
|
file_path_from_root = re.sub(
|
|
"^" + prefix, "", file_path_from_root
|
|
)
|
|
break
|
|
return re.sub(r"[-./\s]", "_", file_path_from_root).upper() + "_"
|
|
|
|
|
|
cpplint.GetHeaderGuardCPPVariable = get_header_guard_dmlc
|
|
|
|
|
|
def process(fname, allow_type):
|
|
"""Process a file."""
|
|
fname = str(fname)
|
|
arr = fname.rsplit(".", 1)
|
|
if fname.find("#") != -1 or arr[-1] not in allow_type:
|
|
return
|
|
if arr[-1] in CXX_SUFFIX:
|
|
_HELPER.process_cpp(fname, arr[-1])
|
|
if arr[-1] in PYTHON_SUFFIX:
|
|
_HELPER.process_python(fname)
|
|
|
|
|
|
def main():
|
|
"""Main entry function."""
|
|
parser = argparse.ArgumentParser(description="lint source codes")
|
|
parser.add_argument("project", help="project name")
|
|
parser.add_argument(
|
|
"filetype", choices=["python", "cpp", "all"], help="source code type"
|
|
)
|
|
parser.add_argument("path", nargs="+", help="path to traverse")
|
|
parser.add_argument(
|
|
"--exclude_path",
|
|
nargs="+",
|
|
default=[],
|
|
help="exclude this path, and all subfolders if path is a folder",
|
|
)
|
|
parser.add_argument(
|
|
"--quiet", action="store_true", help="run cpplint in quiet mode"
|
|
)
|
|
parser.add_argument("--pylint-rc", default=None, help="pylint rc file")
|
|
args = parser.parse_args()
|
|
|
|
_HELPER.project_name = args.project
|
|
if args.pylint_rc is not None:
|
|
_HELPER.pylint_opts = [
|
|
"--rcfile=" + args.pylint_rc,
|
|
]
|
|
file_type = args.filetype
|
|
allow_type = []
|
|
if file_type in ("python", "all"):
|
|
allow_type += PYTHON_SUFFIX
|
|
if file_type in ("cpp", "all"):
|
|
allow_type += CXX_SUFFIX
|
|
allow_type = set(allow_type)
|
|
if sys.version_info.major == 2 and os.name != "nt":
|
|
sys.stderr = codecs.StreamReaderWriter(
|
|
sys.stderr,
|
|
codecs.getreader("utf8"),
|
|
codecs.getwriter("utf8"),
|
|
"replace",
|
|
)
|
|
# get excluded files
|
|
excluded_paths = filepath_enumerate(args.exclude_path)
|
|
for path in args.path:
|
|
if os.path.isfile(path):
|
|
normpath = os.path.normpath(path)
|
|
if normpath not in excluded_paths:
|
|
process(path, allow_type)
|
|
else:
|
|
for root, dirs, files in os.walk(path):
|
|
for name in files:
|
|
file_path = os.path.normpath(os.path.join(root, name))
|
|
if file_path not in excluded_paths:
|
|
process(file_path, allow_type)
|
|
nerr = _HELPER.print_summary(sys.stderr)
|
|
sys.exit(nerr > 0)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|