lint.py 8.04 KB
Newer Older
1
2
3
4
5
6
7
8
9
#!/usr/bin/env python3
# pylint: disable=protected-access, unused-variable, locally-disabled, len-as-condition
"""Lint helper to generate lint summary of source.

Copyright by Contributors.

Borrowed from dmlc-core/scripts/lint.py@939c052
"""
from __future__ import print_function
10

11
12
13
import argparse
import codecs
import os
14
15
16
import re
import sys

17
18
19
20
import cpplint
from cpplint import _cpplint_state
from pylint import epylint

21
22
23
CXX_SUFFIX = set(["cc", "c", "cpp", "h", "cu", "hpp", "cuh"])
PYTHON_SUFFIX = set(["py"])

24
25
26
27
28
29
30
31
32
33
34
35
36

def filepath_enumerate(paths):
    """Enumerate the file paths of all subfiles of the list of paths"""
    out = []
    for path in paths:
        if os.path.isfile(path):
            out.append(path)
        else:
            for root, dirs, files in os.walk(path):
                for name in files:
                    out.append(os.path.normpath(os.path.join(root, name)))
    return out

37

38
39
40
41
42
43
44
45
46
47
# pylint: disable=useless-object-inheritance
class LintHelper(object):
    """Class to help runing the lint and records summary"""

    @staticmethod
    def _print_summary_map(strm, result_map, ftype):
        """Print summary of certain result map."""
        if len(result_map) == 0:
            return 0
        npass = sum(1 for x in result_map.values() if len(x) == 0)
48
49
50
        strm.write(
            f"====={npass}/{len(result_map)} {ftype} files passed check=====\n"
        )
51
52
53
54
        for fname, emap in result_map.items():
            if len(emap) == 0:
                continue
            strm.write(
55
56
                f"{fname}: {sum(emap.values())} Errors of {len(emap)} Categories map={str(emap)}\n"
            )
57
58
59
60
61
62
63
        return len(result_map) - npass

    def __init__(self):
        self.project_name = None
        self.cpp_header_map = {}
        self.cpp_src_map = {}
        self.python_map = {}
64
65
66
67
68
        pylint_disable = [
            "superfluous-parens",
            "too-many-instance-attributes",
            "too-few-public-methods",
        ]
69
        # setup pylint
70
71
72
73
        self.pylint_opts = [
            "--extension-pkg-whitelist=numpy",
            "--disable=" + ",".join(pylint_disable),
        ]
74

75
        self.pylint_cats = set(["error", "warning", "convention", "refactor"])
76
        # setup cpp lint
77
78
79
80
81
        cpplint_args = [
            "--quiet",
            "--extensions=" + (",".join(CXX_SUFFIX)),
            ".",
        ]
82
        _ = cpplint.ParseArguments(cpplint_args)
83
84
85
86
87
88
89
90
91
92
93
94
        cpplint._SetFilters(
            ",".join(
                [
                    "-build/c++11",
                    "-build/namespaces",
                    "-build/include,",
                    "+build/include_what_you_use",
                    "+build/include_order",
                ]
            )
        )
        cpplint._SetCountingStyle("toplevel")
95
        cpplint._line_length = 80
96
97
98
99
100
101
102
103

    def process_cpp(self, path, suffix):
        """Process a cpp file."""
        _cpplint_state.ResetErrorCounts()
        cpplint.ProcessFile(str(path), _cpplint_state.verbose_level)
        _cpplint_state.PrintErrorCounts()
        errors = _cpplint_state.errors_by_category.copy()

104
        if suffix == "h":
105
106
107
108
109
110
111
            self.cpp_header_map[str(path)] = errors
        else:
            self.cpp_src_map[str(path)] = errors

    def process_python(self, path):
        """Process a python file."""
        (pylint_stdout, pylint_stderr) = epylint.py_run(
112
113
            " ".join([str(path)] + self.pylint_opts), return_std=True
        )
114
115
116
117
118
119
        emap = {}
        err = pylint_stderr.read()
        if len(err):
            print(err)
        for line in pylint_stdout:
            sys.stderr.write(line)
120
            key = line.split(":")[-1].split("(")[0].strip()
121
122
123
124
125
126
127
128
129
130
131
            if key not in self.pylint_cats:
                continue
            if key not in emap:
                emap[key] = 1
            else:
                emap[key] += 1
        self.python_map[str(path)] = emap

    def print_summary(self, strm):
        """Print summary of lint."""
        nerr = 0
132
133
134
135
136
137
138
        nerr += LintHelper._print_summary_map(
            strm, self.cpp_header_map, "cpp-header"
        )
        nerr += LintHelper._print_summary_map(
            strm, self.cpp_src_map, "cpp-source"
        )
        nerr += LintHelper._print_summary_map(strm, self.python_map, "python")
139
        if nerr == 0:
140
            strm.write("All passed!\n")
141
        else:
142
            strm.write(f"{nerr} files failed lint\n")
143
144
        return nerr

145

146
147
148
# singleton helper for lint check
_HELPER = LintHelper()

149

150
151
152
153
154
155
156
157
158
159
160
161
def get_header_guard_dmlc(filename):
    """Get Header Guard Convention for DMLC Projects.

    For headers in include, directly use the path
    For headers in src, use project name plus path

    Examples: with project-name = dmlc
        include/dmlc/timer.h -> DMLC_TIMTER_H_
        src/io/libsvm_parser.h -> DMLC_IO_LIBSVM_PARSER_H_
    """
    fileinfo = cpplint.FileInfo(filename)
    file_path_from_root = fileinfo.RepositoryName()
162
163
    inc_list = ["include", "api", "wrapper", "contrib"]
    if os.name == "nt":
164
165
        inc_list.append("mshadow")

166
167
168
169
170
171
172
173
    if (
        file_path_from_root.find("src/") != -1
        and _HELPER.project_name is not None
    ):
        idx = file_path_from_root.find("src/")
        file_path_from_root = (
            _HELPER.project_name + file_path_from_root[idx + 3 :]
        )
174
175
176
    else:
        idx = file_path_from_root.find("include/")
        if idx != -1:
177
            file_path_from_root = file_path_from_root[idx + 8 :]
178
        for spath in inc_list:
179
            prefix = spath + "/"
180
            if file_path_from_root.startswith(prefix):
181
182
183
                file_path_from_root = re.sub(
                    "^" + prefix, "", file_path_from_root
                )
184
                break
185
186
    return re.sub(r"[-./\s]", "_", file_path_from_root).upper() + "_"

187
188
189

cpplint.GetHeaderGuardCPPVariable = get_header_guard_dmlc

190

191
192
193
def process(fname, allow_type):
    """Process a file."""
    fname = str(fname)
194
195
    arr = fname.rsplit(".", 1)
    if fname.find("#") != -1 or arr[-1] not in allow_type:
196
197
198
199
200
201
        return
    if arr[-1] in CXX_SUFFIX:
        _HELPER.process_cpp(fname, arr[-1])
    if arr[-1] in PYTHON_SUFFIX:
        _HELPER.process_python(fname)

202

203
204
205
def main():
    """Main entry function."""
    parser = argparse.ArgumentParser(description="lint source codes")
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
    parser.add_argument("project", help="project name")
    parser.add_argument(
        "filetype", choices=["python", "cpp", "all"], help="source code type"
    )
    parser.add_argument("path", nargs="+", help="path to traverse")
    parser.add_argument(
        "--exclude_path",
        nargs="+",
        default=[],
        help="exclude this path, and all subfolders if path is a folder",
    )
    parser.add_argument(
        "--quiet", action="store_true", help="run cpplint in quiet mode"
    )
    parser.add_argument("--pylint-rc", default=None, help="pylint rc file")
221
222
223
224
    args = parser.parse_args()

    _HELPER.project_name = args.project
    if args.pylint_rc is not None:
225
226
227
        _HELPER.pylint_opts = [
            "--rcfile=" + args.pylint_rc,
        ]
228
229
    file_type = args.filetype
    allow_type = []
230
    if file_type in ("python", "all"):
231
        allow_type += PYTHON_SUFFIX
232
    if file_type in ("cpp", "all"):
233
234
        allow_type += CXX_SUFFIX
    allow_type = set(allow_type)
235
236
237
238
239
240
241
    if sys.version_info.major == 2 and os.name != "nt":
        sys.stderr = codecs.StreamReaderWriter(
            sys.stderr,
            codecs.getreader("utf8"),
            codecs.getwriter("utf8"),
            "replace",
        )
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
    # get excluded files
    excluded_paths = filepath_enumerate(args.exclude_path)
    for path in args.path:
        if os.path.isfile(path):
            normpath = os.path.normpath(path)
            if normpath not in excluded_paths:
                process(path, allow_type)
        else:
            for root, dirs, files in os.walk(path):
                for name in files:
                    file_path = os.path.normpath(os.path.join(root, name))
                    if file_path not in excluded_paths:
                        process(file_path, allow_type)
    nerr = _HELPER.print_summary(sys.stderr)
    sys.exit(nerr > 0)

258
259

if __name__ == "__main__":
260
    main()