feat: add support for Korean and Latin configurations in OCR model processing

mineru/cli/gradio_app.py

@@ -133,15 +133,16 @@ latin_lang = [
 ]
 arabic_lang = ['ar', 'fa', 'ug', 'ur']
 cyrillic_lang = [
-        'ru', 'rs_cyrillic', 'be', 'bg', 'uk', 'mn', 'abq', 'ady', 'kbd', 'ava',  # noqa: E126
+        'rs_cyrillic', 'bg', 'mn', 'abq', 'ady', 'kbd', 'ava',  # noqa: E126
         'dar', 'inh', 'che', 'lbe', 'lez', 'tab'
 ]
+east_slavic_lang = ["ru", "be", "uk"]
 devanagari_lang = [
         'hi', 'mr', 'ne', 'bh', 'mai', 'ang', 'bho', 'mah', 'sck', 'new', 'gom',  # noqa: E126
         'sa', 'bgc'
 ]
 other_lang = ['ch', 'ch_lite', 'ch_server', 'en', 'korean', 'japan', 'chinese_cht', 'ta', 'te', 'ka']
-add_lang = ['latin', 'arabic', 'cyrillic', 'devanagari']
+add_lang = ['latin', 'arabic', 'east_slavic', 'cyrillic', 'devanagari']
 
 # all_lang = ['', 'auto']
 all_lang = []

mineru/model/ocr/paddleocr2pytorch/pytorch_paddle.py

@@ -26,9 +26,10 @@ latin_lang = [
 ]
 arabic_lang = ['ar', 'fa', 'ug', 'ur']
 cyrillic_lang = [
-        'ru', 'rs_cyrillic', 'be', 'bg', 'uk', 'mn', 'abq', 'ady', 'kbd', 'ava',  # noqa: E126
+        'rs_cyrillic', 'bg', 'mn', 'abq', 'ady', 'kbd', 'ava',  # noqa: E126
         'dar', 'inh', 'che', 'lbe', 'lez', 'tab'
 ]
+east_slavic_lang = ["ru", "be", "uk"]
 devanagari_lang = [
         'hi', 'mr', 'ne', 'bh', 'mai', 'ang', 'bho', 'mah', 'sck', 'new', 'gom',  # noqa: E126
         'sa', 'bgc'
@@ -69,6 +70,8 @@ class PytorchPaddleOCR(TextSystem):
             self.lang = 'cyrillic'
         elif self.lang in devanagari_lang:
             self.lang = 'devanagari'
+        elif self.lang in east_slavic_lang:
+            self.lang = 'east_slavic'
         else:
             pass
 

mineru/model/ocr/paddleocr2pytorch/pytorchocr/utils/resources/arch_config.yaml

@@ -490,3 +490,82 @@ devanagari_PP-OCRv3_rec_infer:
 #    out_channels: 169
     fc_decay: 0.00001
 
+korean_PP-OCRv5_rec_infer:
+  model_type: rec
+  algorithm: SVTR_HGNet
+  Transform:
+  Backbone:
+    name: PPLCNetV3
+    scale: 0.95
+  Head:
+    name: MultiHead
+    out_channels_list:
+      CTCLabelDecode: 11947
+    head_list:
+      - CTCHead:
+          Neck:
+            name: svtr
+            dims: 120
+            depth: 2
+            hidden_dims: 120
+            kernel_size: [ 1, 3 ]
+            use_guide: True
+          Head:
+            fc_decay: 0.00001
+      - NRTRHead:
+          nrtr_dim: 384
+          max_text_length: 25
+
+latin_PP-OCRv5_rec_infer:
+  model_type: rec
+  algorithm: SVTR_HGNet
+  Transform:
+  Backbone:
+    name: PPLCNetV3
+    scale: 0.95
+  Head:
+    name: MultiHead
+    out_channels_list:
+      CTCLabelDecode: 504
+    head_list:
+      - CTCHead:
+          Neck:
+            name: svtr
+            dims: 120
+            depth: 2
+            hidden_dims: 120
+            kernel_size: [ 1, 3 ]
+            use_guide: True
+          Head:
+            fc_decay: 0.00001
+      - NRTRHead:
+          nrtr_dim: 384
+          max_text_length: 25
+
+eslav_PP-OCRv5_rec_infer:
+  model_type: rec
+  algorithm: SVTR_HGNet
+  Transform:
+  Backbone:
+    name: PPLCNetV3
+    scale: 0.95
+  Head:
+    name: MultiHead
+    out_channels_list:
+      CTCLabelDecode: 519
+    head_list:
+      - CTCHead:
+          Neck:
+            name: svtr
+            dims: 120
+            depth: 2
+            hidden_dims: 120
+            kernel_size: [ 1, 3 ]
+            use_guide: True
+          Head:
+            fc_decay: 0.00001
+      - NRTRHead:
+          nrtr_dim: 384
+          max_text_length: 25
+
+

mineru/model/ocr/paddleocr2pytorch/pytorchocr/utils/resources/dict/ppocrv5_eslav_dict.txt

+!
+"
+#
+$
+%
+&
+'
+(
+)
+*
++
+,
+-
+.
+/
+0
+1
+2
+3
+4
+5
+6
+7
+8
+9
+:
+;
+<
+=
+>
+?
+A
+B
+C
+D
+E
+F
+G
+H
+I
+J
+K
+L
+M
+N
+O
+P
+Q
+R
+S
+T
+U
+V
+W
+X
+Y
+Z
+[
+]
+_
+`
+a
+b
+c
+d
+e
+f
+g
+h
+i
+j
+k
+l
+m
+n
+o
+p
+q
+r
+s
+t
+u
+v
+w
+x
+y
+z
+©
+‥
+{
+}
+\
+|
+@
+^
+~
+÷
+∕
+∙
+⋅
+·
+±
+∓
+∩
+∪
+□
+←
+↔
+⇒
+⇐
+⇔
+∀
+∃
+∄
+∴
+∵
+∝
+∞
+⊥
+∟
+∠
+∡
+∢
+′
+″
+∥
+⊾
+⊿
+∂
+∫
+∬
+∭
+∮
+∯
+∰
+∑
+∏
+√
+∛
+∜
+∱
+∲
+∳
+∶
+∷
+∼
+®
+℉
+Ω
+℧
+Å
+⌀
+ℏ
+⅀
+⍺
+⍵
+¢
+€
+£
+¥
+₿
+Ⅰ
+Ⅱ
+Ⅲ
+Ⅳ
+Ⅴ
+Ⅵ
+Ⅶ
+Ⅷ
+Ⅸ
+Ⅹ
+Ⅺ
+Ⅻ
+ⅰ
+ⅱ
+ⅲ
+ⅳ
+ⅴ
+ⅵ
+ⅶ
+ⅷ
+ⅸ
+ⅹ
+ⅺ
+ⅻ
+➀
+➁
+➂
+➃
+➄
+➅
+➆
+➇
+➈
+➉
+➊
+➋
+➌
+➍
+➎
+➏
+➐
+➑
+➒
+➓
+❶
+❷
+❸
+❹
+❺
+❻
+❼
+❽
+❾
+❿
+①
+②
+③
+④
+⑤
+⑥
+⑦
+⑧
+⑨
+⑩
+●
+▶
+𝑢
+︽
+–
+﹥
+𝜓
+•
+∋
+ƒ
+०
+⬆
+Ạ
+◀
+
+▫
+︾
+À
+Á
+Â
+Ã
+Ä
+Å
+Æ
+Ç
+È
+É
+Ê
+Ë
+Ì
+Í
+Î
+Ï
+Ð
+Ñ
+Ò
+Ó
+Ô
+Õ
+Ö
+Ø
+Ù
+Ú
+Û
+Ü
+Ý
+Þ
+à
+á
+â
+ã
+ä
+å
+æ
+ç
+è
+é
+ê
+ë
+ì
+í
+î
+ï
+ð
+ñ
+ò
+ó
+ô
+õ
+ö
+ø
+ù
+ú
+û
+ü
+ý
+þ
+ÿ
+¡
+¤
+¦
+§
+¨
+ª
+«
+¬
+¯
+°
+²
+³
+´
+µ
+¶
+¸
+¹
+º
+»
+¼
+½
+¾
+¿
+×
+‐
+‑
+‒
+—
+―
+‖
+‗
+‘
+’
+‚
+‛
+“
+”
+„
+‟
+†
+‡
+‣
+․
+…
+‧
+‰
+‴
+‵
+‶
+‷
+‸
+‹
+›
+※
+‼
+‽
+‾
+₤
+₡
+₹
+−
+∖
+∗
+≈
+≠
+≡
+≤
+≥
+⊂
+⊃
+↑
+→
+↓
+↕
+™
+Ω
+℮
+∆
+✓
+✗
+✘
+▪
+◼
+✔
+✕
+☑
+☒
+№
+₽
+₴
+Α
+α
+Β
+β
+Γ
+γ
+Δ
+δ
+Ε
+ε
+Ζ
+ζ
+Η
+η
+Θ
+θ
+Ι
+ι
+Κ
+κ
+Λ
+λ
+Μ
+μ
+Ν
+ν
+Ξ
+ξ
+Ο
+ο
+Π
+π
+Ρ
+ρ
+Σ
+σ
+ς
+Τ
+τ
+Υ
+υ
+Φ
+φ
+Χ
+χ
+Ψ
+ψ
+ω
+А
+Б
+В
+Г
+Ґ
+Д
+Е
+Ё
+Є
+Ж
+З
+И
+І
+Ї
+Й
+К
+Л
+М
+Н
+О
+П
+Р
+С
+Т
+У
+Ў
+Ф
+Х
+Ц
+Ч
+Ш
+Щ
+Ъ
+Ы
+Ь
+Э
+Ю
+Я
+а
+б
+в
+г
+ґ
+д
+е
+ё
+є
+ж
+з
+и
+і
+ї
+й
+к
+л
+м
+н
+о
+п
+р
+с
+т
+у
+ў
+ф
+х
+ц
+ч
+ш
+щ
+ъ
+ы
+ь
+э
+ю
+я

mineru/model/ocr/paddleocr2pytorch/pytorchocr/utils/resources/dict/ppocrv5_latin_dict.txt

+!
+"
+#
+$
+%
+&
+'
+(
+)
+*
++
+,
+-
+.
+/
+0
+1
+2
+3
+4
+5
+6
+7
+8
+9
+:
+;
+<
+=
+>
+?
+@
+A
+B
+C
+D
+E
+F
+G
+H
+I
+J
+K
+L
+M
+N
+O
+P
+Q
+R
+S
+T
+U
+V
+W
+X
+Y
+Z
+[
+\
+]
+^
+_
+`
+a
+b
+c
+d
+e
+f
+g
+h
+i
+j
+k
+l
+m
+n
+o
+p
+q
+r
+s
+t
+u
+v
+w
+x
+y
+z
+{
+|
+}
+~
+¡
+¢
+£
+¤
+¥
+¦
+§
+¨
+©
+ª
+«
+¬
+­
+®
+¯
+°
+±
+²
+³
+´
+µ
+¶
+·
+¸
+¹
+º
+»
+¼
+½
+¾
+¿
+À
+Á
+Â
+Ã
+Ä
+Å
+Æ
+Ç
+È
+É
+Ê
+Ë
+Ì
+Í
+Î
+Ï
+Ð
+Ñ
+Ò
+Ó
+Ô
+Õ
+Ö
+×
+Ø
+Ù
+Ú
+Û
+Ü
+Ý
+Þ
+ß
+à
+á
+â
+ã
+ä
+å
+æ
+ç
+è
+é
+ê
+ë
+ì
+í
+î
+ï
+ð
+ñ
+ò
+ó
+ô
+õ
+ö
+÷
+ø
+ù
+ú
+û
+ü
+ý
+þ
+ÿ
+Ą
+ą
+Ć
+ć
+Č
+č
+Ď
+ď
+Đ
+đ
+Ė
+ė
+Ę
+ę
+Ě
+ě
+Ğ
+ğ
+Į
+į
+İ
+ı
+Ĺ
+ĺ
+Ľ
+ľ
+Ł
+ł
+Ń
+ń
+Ň
+ň
+ō
+Ő
+ő
+Œ
+œ
+Ŕ
+ŕ
+Ř
+ř
+Ś
+ś
+Ş
+ş
+Š
+š
+Ť
+ť
+Ū
+ū
+Ů
+ů
+Ű
+ű
+Ų
+ų
+Ÿ
+Ź
+ź
+Ż
+ż
+Ž
+ž
+ƒ
+ʒ
+Ω
+α
+β
+γ
+δ
+ε
+ζ
+η
+θ
+ι
+κ
+λ
+μ
+ν
+ξ
+ο
+π
+ρ
+ς
+σ
+τ
+υ
+φ
+χ
+ψ
+ω
+з
+०
+Ṡ
+ẞ
+Ạ
+‐
+‑
+‒
+–
+—
+―
+‖
+‗
+‘
+’
+‚
+‛
+“
+”
+„
+‟
+†
+‡
+•
+‣
+․
+‥
+…
+‧
+‰
+′
+″
+‴
+‵
+‶
+‷
+‸
+‹
+›
+※
+‼
+‽
+‾
+⁄
+₂
+₃
+₡
+₤
+€
+₴
+₹
+₽
+₿
+℉
+ℏ
+№
+™
+Ω
+℧
+Å
+℮
+⅀
+Ⅰ
+Ⅱ
+Ⅲ
+Ⅳ
+Ⅴ
+Ⅵ
+Ⅶ
+Ⅷ
+Ⅸ
+Ⅹ
+Ⅺ
+Ⅻ
+ⅰ
+ⅱ
+ⅲ
+ⅳ
+ⅴ
+ⅵ
+ⅶ
+ⅷ
+ⅸ
+ⅹ
+ⅺ
+ⅻ
+←
+↑
+→
+↓
+↔
+↕
+⇐
+⇒
+⇔
+∀
+∂
+∃
+∄
+∅
+∆
+∋
+∏
+∑
+−
+∓
+∕
+∖
+∗
+∙
+√
+∛
+∜
+∝
+∞
+∟
+∠
+∡
+∢
+∥
+∧
+∨
+∩
+∪
+∫
+∬
+∭
+∮
+∯
+∰
+∱
+∲
+∳
+∴
+∵
+∶
+∷
+∼
+≈
+≠
+≡
+≤
+≥
+⊂
+⊃
+⊥
+⊾
+⊿
+⋅
+⌀
+⍵
+⍺
+①
+②
+③
+④
+⑤
+⑥
+⑦
+⑧
+⑨
+⑩
+─
+│
+└
+├
+■
+□
+▪
+▫
+▶
+◀
+●
+◼
+☑
+☒
+✓
+✔
+✕
+✗
+✘
+❶
+❷
+❸
+❹
+❺
+❻
+❼
+❽
+❾
+❿
+➀
+➁
+➂
+➃
+➄
+➅
+➆
+➇
+➈
+➉
+➊
+➋
+➌
+➍
+➎
+➏
+➐
+➑
+➒
+➓
+⬆
+、
+fi
+fl
+︽
+︾
+﹥
+�
+𝑢
+𝜓

mineru/model/ocr/paddleocr2pytorch/pytorchocr/utils/resources/models_config.yml

@@ -24,9 +24,9 @@ lang:
     rec: en_PP-OCRv4_rec_infer.pth
     dict: en_dict.txt
   korean:
-    det: Multilingual_PP-OCRv3_det_infer.pth
-    rec: korean_PP-OCRv3_rec_infer.pth
-    dict: korean_dict.txt
+    det: ch_PP-OCRv5_det_infer.pth
+    rec: korean_PP-OCRv5_rec_infer.pth
+    dict: ppocrv5_korean_dict.txt
   japan:
     det: ch_PP-OCRv5_det_infer.pth
     rec: ch_PP-OCRv5_rec_server_infer.pth
@@ -48,9 +48,9 @@ lang:
     rec: ka_PP-OCRv3_rec_infer.pth
     dict: ka_dict.txt
   latin:
-    det: en_PP-OCRv3_det_infer.pth
-    rec: latin_PP-OCRv3_rec_infer.pth
-    dict: latin_dict.txt
+    det: ch_PP-OCRv5_det_infer.pth
+    rec: latin_PP-OCRv5_rec_infer.pth
+    dict: ppocrv5_latin_dict.txt
   arabic:
     det: Multilingual_PP-OCRv3_det_infer.pth
     rec: arabic_PP-OCRv3_rec_infer.pth
@@ -63,3 +63,7 @@ lang:
     det: Multilingual_PP-OCRv3_det_infer.pth
     rec: devanagari_PP-OCRv3_rec_infer.pth
     dict: devanagari_dict.txt
+  east_slavic:
+    det: ch_PP-OCRv5_det_infer.pth
+    rec: eslav_PP-OCRv5_rec_infer.pth
+    dict: ppocrv5_eslav_dict.txt
\ No newline at end of file