Merge pull request #140 from dt-yy/master

add data

tests/test_cli/pdf_dev/annotations/cleaned/cleaned_ordinary_textbook_1d9a847603a5e37e379738316820850d.md

+# 数学新星问题征解 
+
+第十五期 (2016.06)
+
+主持: 牟晓生
+
+第一题. 设 $z_{1}, z_{2}, z_{3}$ 是单位复数. 证明存在单位复数 $z$ 使得:
+
+$$
+\frac{1}{\left|z-z_{1}\right|^{2}}+\frac{1}{\left|z-z_{2}\right|^{2}}+\frac{1}{\left|z-z_{3}\right|^{2}} \leq \frac{9}{4}
+$$
+
+(湖北武钢三中学生 王逸轩, 上海大学冷岗松 供题)
+
+第二题. 如图, $D$ 是正三角形 $A B C$ 的边 $B C$ 上一点, $B D>C D$. 记 $O_{1}, I_{1}$ 为 $\triangle A B D$ 的外心与内心, $O_{2}, I_{2}$ 为 $\triangle A C D$ 的外心与内心. 圆 $I_{1}$ 与圆 $I_{2}$ 除 $B C$外的另一条外公切线交 $A B, A C$ 于 $P, Q$. 设直线 $P I_{1}$与 $Q I_{2}$ 交于 $R$, 而直线 $O_{1} I_{1}$ 与 $O_{2} I_{2}$ 交于 $T$. 证明: $A T^{2}=A R^{2}+A D \cdot B C$.
+
+(广西钦州 卢圣 供题)
+
+
+
+第三题. 给定正整数 $m, n$, 考虑在 $m \times n$ 白棋盘上先将一些格染成黑色. 在之后的每一时刻, 若存在一个白格至少与两个黑格相邻, 则可将它也染成黑色. 求最初至少要染多少个黑色格才能在某一时刻染黑整个棋盘?
+
+(哈佛大学 牟晓生 供题)
+
+第四题. $A B C$ 是一个三角形, 而 $P, Q, R$ 分别是 $B C, C A, A B$ 上的点。证明 $\triangle P Q R$ 的周长不小于 $\triangle A Q R, \triangle B R P, \triangle C P Q$ 周长的最小值.
+
+(哈佛大学 牟晓生 供题)
+

tests/test_cli/test_bench.py

+"""
+bench
+"""
+import os
+import shutil
+import json
+from lib import calculate_score
+import pytest
+from conf import conf
+
+code_path = os.environ.get('GITHUB_WORKSPACE')
+pdf_dev_path = conf.conf["pdf_dev_path"]
+pdf_res_path = conf.conf["pdf_res_path"]
+last_simscore = 0
+last_editdistance = 0
+last_bleu = 0
+
+class TestBench():
+    """
+    test bench
+    """
+    def test_ci_ben(self):
+        """
+        ci benchmark
+        """
+        try:
+            fr = open(os.path.join(pdf_dev_path, "result.json"), "r", encoding="utf-8")
+            lines = fr.readlines()
+            last_line = lines[-1].strip()
+            last_score = json.loads(last_line)
+            last_simscore = last_score["average_sim_score"]
+            last_editdistance = last_score["average_edit_distance"]
+            last_bleu = last_score["average_bleu_score"]
+        except IOError:
+            print ("result.json not exist")
+        os.system(f"python tests/test_cli/lib/pre_clean.py --tool_name mineru --download_dir {pdf_dev_path}")
+        now_score = get_score()
+        print ("now_score:", now_score)
+        if not os.path.exists(os.path.join(pdf_dev_path, "ci")):
+            os.makedirs(os.path.join(pdf_dev_path, "ci"), exist_ok=True)
+        fw = open(os.path.join(pdf_dev_path, "ci", "result.json"), "w+", encoding="utf-8")
+        fw.write(json.dumps(now_score) + "\n")
+        now_simscore = now_score["average_sim_score"]
+        now_editdistance = now_score["average_edit_distance"]
+        now_bleu = now_score["average_bleu_score"]
+        assert last_simscore <= now_simscore
+        assert last_editdistance <= now_editdistance
+        assert last_bleu <= now_bleu
+
+
+def get_score():
+    """
+    get score
+    """
+    score = calculate_score.Scoring(os.path.join(pdf_dev_path, "result.json"))
+    score.calculate_similarity_total("mineru", pdf_dev_path)
+    res = score.summary_scores()
+    return res
+
+

tests/test_cli/test_cli.py

 import pytest
 import os
 from conf import conf
-import subprocess
+import os
+import json
+from magic_pdf.pipe.UNIPipe import UNIPipe
+from magic_pdf.rw.DiskReaderWriter import DiskReaderWriter
 from lib import common
-import logging
+
 pdf_res_path = conf.conf["pdf_res_path"]
 code_path = conf.conf["code_path"]
 pdf_dev_path = conf.conf["pdf_dev_path"]
 class TestCli:
-   
-    def test_pdf_specify_dir(self):
+    """
+    test cli
+    """
+    def test_pdf_sdk(self):
         """
-        输入pdf和指定目录的模型结果
+        pdf sdk 方式解析
         """
-        cmd = 'cd %s && export PYTHONPATH=. && find %s -type f -name "*.pdf" | xargs -I{} python magic_pdf/cli/magicpdf.py  pdf-command  --pdf {}' % (code_path, pdf_dev_path)
-        logging.info(cmd)
-        common.check_shell(cmd)
-        #common.count_folders_and_check_contents(pdf_res_path)      
-   
+        demo_names = list()
+        pdf_path = os.path.join(pdf_dev_path, "pdf")
+        for pdf_file in os.listdir(pdf_path):
+            if pdf_file.endswith('.pdf'):
+                demo_names.append(pdf_file.split('.')[0])
+        for demo_name in demo_names:
+            model_path = os.path.join(pdf_dev_path, f"{demo_name}_model.json")
+            pdf_path = os.path.join(pdf_dev_path, "pdf", f"{demo_name}.pdf")
+            pdf_bytes = open(pdf_path, "rb").read()
+            model_json = json.loads(open(model_path, "r", encoding="utf-8").read())
+            image_writer = DiskReaderWriter(pdf_dev_path)
+            image_dir = str(os.path.basename(pdf_dev_path))
+            jso_useful_key = {"_pdf_type": "", "model_list": model_json}
+            pipe = UNIPipe(pdf_bytes, jso_useful_key, image_writer)
+            pipe.pipe_classify()
+            pipe.pipe_parse()
+            md_content = pipe.pipe_mk_markdown(image_dir, drop_mode="none")
+            dir_path = os.path.join(pdf_dev_path, "mineru")
+            if not os.path.exists(dir_path):
+                os.makedirs(dir_path, exist_ok=True)
+            res_path = os.path.join(dir_path, f"{demo_name}.md")
+            with open(res_path, "w+", encoding="utf-8") as f:
+                f.write(md_content)
+            common.count_folders_and_check_contents(res_path)
+        
     # def test_pdf_specify_jsonl(self):
     #     """
     #     输入jsonl, 默认方式解析