Skip to content

GitLab

Switch branch/tag
Find file Normal viewHistoryPermalink
language.py 969 Bytes
Newer Older
赵小蒙's avatar
Initial commit  
赵小蒙 committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36 
import pycld2 as cld2
import regex
import unicodedata


RE_BAD_CHARS = regex.compile(r"\p{Cc}|\p{Cs}")


def remove_bad_chars(text):
    return RE_BAD_CHARS.sub("", text)


def detect_lang(text: str) -> str:
    if len(text) == 0:
        return ""

    try:
        _, _, details = cld2.detect(text)
    except:
        # cld2 doesn't like control characters
        # https://github.com/mikemccand/chromium-compact-language-detector/issues/22#issuecomment-435904616
        html_no_ctrl_chars = ''.join([l for l in text if unicodedata.category(l)[0] not in ['C',]])
        _, _, details = cld2.detect(html_no_ctrl_chars)
    lang = ""
    try:
        lang = details[0][1].lower()
    except:
        lang = ""
    return lang


if __name__ == '__main__':
    print(detect_lang("This is a test."))
    print(detect_lang("<html>This is a test</html>"))
    print(detect_lang("这个是中文测试。"))
    print(detect_lang("<html>这个是中文测试。</html>"))