#!/usr/bin/env python3 # pylint: disable=protected-access, unused-variable, locally-disabled, len-as-condition """Lint helper to generate lint summary of source. Copyright by Contributors. Borrowed from dmlc-core/scripts/lint.py@939c052 """ from __future__ import print_function import argparse import codecs import os import re import sys import cpplint from cpplint import _cpplint_state from pylint import epylint CXX_SUFFIX = set(["cc", "c", "cpp", "h", "cu", "hpp", "cuh"]) PYTHON_SUFFIX = set(["py"]) def filepath_enumerate(paths): """Enumerate the file paths of all subfiles of the list of paths""" out = [] for path in paths: if os.path.isfile(path): out.append(path) else: for root, dirs, files in os.walk(path): for name in files: out.append(os.path.normpath(os.path.join(root, name))) return out # pylint: disable=useless-object-inheritance class LintHelper(object): """Class to help runing the lint and records summary""" @staticmethod def _print_summary_map(strm, result_map, ftype): """Print summary of certain result map.""" if len(result_map) == 0: return 0 npass = sum(1 for x in result_map.values() if len(x) == 0) strm.write( f"====={npass}/{len(result_map)} {ftype} files passed check=====\n" ) for fname, emap in result_map.items(): if len(emap) == 0: continue strm.write( f"{fname}: {sum(emap.values())} Errors of {len(emap)} Categories map={str(emap)}\n" ) return len(result_map) - npass def __init__(self): self.project_name = None self.cpp_header_map = {} self.cpp_src_map = {} self.python_map = {} pylint_disable = [ "superfluous-parens", "too-many-instance-attributes", "too-few-public-methods", ] # setup pylint self.pylint_opts = [ "--extension-pkg-whitelist=numpy", "--disable=" + ",".join(pylint_disable), ] self.pylint_cats = set(["error", "warning", "convention", "refactor"]) # setup cpp lint cpplint_args = [ "--quiet", "--extensions=" + (",".join(CXX_SUFFIX)), ".", ] _ = cpplint.ParseArguments(cpplint_args) cpplint._SetFilters( ",".join( [ "-build/c++11", "-build/namespaces", "-build/include,", "+build/include_what_you_use", "+build/include_order", ] ) ) cpplint._SetCountingStyle("toplevel") cpplint._line_length = 80 def process_cpp(self, path, suffix): """Process a cpp file.""" _cpplint_state.ResetErrorCounts() cpplint.ProcessFile(str(path), _cpplint_state.verbose_level) _cpplint_state.PrintErrorCounts() errors = _cpplint_state.errors_by_category.copy() if suffix == "h": self.cpp_header_map[str(path)] = errors else: self.cpp_src_map[str(path)] = errors def process_python(self, path): """Process a python file.""" (pylint_stdout, pylint_stderr) = epylint.py_run( " ".join([str(path)] + self.pylint_opts), return_std=True ) emap = {} err = pylint_stderr.read() if len(err): print(err) for line in pylint_stdout: sys.stderr.write(line) key = line.split(":")[-1].split("(")[0].strip() if key not in self.pylint_cats: continue if key not in emap: emap[key] = 1 else: emap[key] += 1 self.python_map[str(path)] = emap def print_summary(self, strm): """Print summary of lint.""" nerr = 0 nerr += LintHelper._print_summary_map( strm, self.cpp_header_map, "cpp-header" ) nerr += LintHelper._print_summary_map( strm, self.cpp_src_map, "cpp-source" ) nerr += LintHelper._print_summary_map(strm, self.python_map, "python") if nerr == 0: strm.write("All passed!\n") else: strm.write(f"{nerr} files failed lint\n") return nerr # singleton helper for lint check _HELPER = LintHelper() def get_header_guard_dmlc(filename): """Get Header Guard Convention for DMLC Projects. For headers in include, directly use the path For headers in src, use project name plus path Examples: with project-name = dmlc include/dmlc/timer.h -> DMLC_TIMTER_H_ src/io/libsvm_parser.h -> DMLC_IO_LIBSVM_PARSER_H_ """ fileinfo = cpplint.FileInfo(filename) file_path_from_root = fileinfo.RepositoryName() inc_list = ["include", "api", "wrapper", "contrib"] if os.name == "nt": inc_list.append("mshadow") if ( file_path_from_root.find("src/") != -1 and _HELPER.project_name is not None ): idx = file_path_from_root.find("src/") file_path_from_root = ( _HELPER.project_name + file_path_from_root[idx + 3 :] ) else: idx = file_path_from_root.find("include/") if idx != -1: file_path_from_root = file_path_from_root[idx + 8 :] for spath in inc_list: prefix = spath + "/" if file_path_from_root.startswith(prefix): file_path_from_root = re.sub( "^" + prefix, "", file_path_from_root ) break return re.sub(r"[-./\s]", "_", file_path_from_root).upper() + "_" cpplint.GetHeaderGuardCPPVariable = get_header_guard_dmlc def process(fname, allow_type): """Process a file.""" fname = str(fname) arr = fname.rsplit(".", 1) if fname.find("#") != -1 or arr[-1] not in allow_type: return if arr[-1] in CXX_SUFFIX: _HELPER.process_cpp(fname, arr[-1]) if arr[-1] in PYTHON_SUFFIX: _HELPER.process_python(fname) def main(): """Main entry function.""" parser = argparse.ArgumentParser(description="lint source codes") parser.add_argument("project", help="project name") parser.add_argument( "filetype", choices=["python", "cpp", "all"], help="source code type" ) parser.add_argument("path", nargs="+", help="path to traverse") parser.add_argument( "--exclude_path", nargs="+", default=[], help="exclude this path, and all subfolders if path is a folder", ) parser.add_argument( "--quiet", action="store_true", help="run cpplint in quiet mode" ) parser.add_argument("--pylint-rc", default=None, help="pylint rc file") args = parser.parse_args() _HELPER.project_name = args.project if args.pylint_rc is not None: _HELPER.pylint_opts = [ "--rcfile=" + args.pylint_rc, ] file_type = args.filetype allow_type = [] if file_type in ("python", "all"): allow_type += PYTHON_SUFFIX if file_type in ("cpp", "all"): allow_type += CXX_SUFFIX allow_type = set(allow_type) if sys.version_info.major == 2 and os.name != "nt": sys.stderr = codecs.StreamReaderWriter( sys.stderr, codecs.getreader("utf8"), codecs.getwriter("utf8"), "replace", ) # get excluded files excluded_paths = filepath_enumerate(args.exclude_path) for path in args.path: if os.path.isfile(path): normpath = os.path.normpath(path) if normpath not in excluded_paths: process(path, allow_type) else: for root, dirs, files in os.walk(path): for name in files: file_path = os.path.normpath(os.path.join(root, name)) if file_path not in excluded_paths: process(file_path, allow_type) nerr = _HELPER.print_summary(sys.stderr) sys.exit(nerr > 0) if __name__ == "__main__": main()