Merge pull request #141 from dt-yy/master

add ci

tests/test_cli/lib/calculate_score.py

@@ -4,12 +4,12 @@ calculate_score
 import os
 import re
 import json
+from Levenshtein import distance
 from lib import scoring
 from nltk.translate.bleu_score import sentence_bleu, SmoothingFunction
 from nltk.tokenize import word_tokenize
 import nltk
 nltk.download('punkt')
-from Levenshtein import distance
 
 class Scoring:
     """

tests/test_cli/lib/pre_clean.py

@@ -118,9 +118,6 @@ def clean_data(prod_type, download_dir):
             with open(input_file, 'r', encoding='utf-8') as fr:
                 content = fr.read()
                 new_content = clean_markdown_images(content)
-                new_content = convert_html_table_to_md(new_content)
-                new_content = convert_latext_to_md(new_content)
-                new_content = convert_htmltale_to_md(new_content)
                 with open(output_file, 'w', encoding='utf-8') as fw:
                     fw.write(new_content)
 

tests/test_cli/lib/scoring.py

+"""
+Calculate simscore, refer to (https://github.com/VikParuchuri/marker?tab=readme-ov-file)
+"""
 import math
 
 from rapidfuzz import fuzz

tests/test_cli/pdf_dev/result.json

+{"average_sim_score":0.6505598645664856, "average_edit_distance":0.2514908429188901, "average_bleu_score": 0.5808819533975296}
\ No newline at end of file

tests/test_cli/test_bench.py

@@ -11,9 +11,6 @@ from conf import conf
 code_path = os.environ.get('GITHUB_WORKSPACE')
 pdf_dev_path = conf.conf["pdf_dev_path"]
 pdf_res_path = conf.conf["pdf_res_path"]
-last_simscore = 0
-last_editdistance = 0
-last_bleu = 0
 
 class TestBench():
     """
@@ -23,16 +20,13 @@ class TestBench():
         """
         ci benchmark
         """
-        try:
-            fr = open(os.path.join(pdf_dev_path, "result.json"), "r", encoding="utf-8")
-            lines = fr.readlines()
-            last_line = lines[-1].strip()
-            last_score = json.loads(last_line)
-            last_simscore = last_score["average_sim_score"]
-            last_editdistance = last_score["average_edit_distance"]
-            last_bleu = last_score["average_bleu_score"]
-        except IOError:
-            print ("result.json not exist")
+        fr = open(os.path.join(pdf_dev_path, "result.json"), "r", encoding="utf-8")
+        lines = fr.readlines()
+        last_line = lines[-1].strip()
+        last_score = json.loads(last_line)
+        last_simscore = last_score["average_sim_score"]
+        last_editdistance = last_score["average_edit_distance"]
+        last_bleu = last_score["average_bleu_score"]
         os.system(f"python tests/test_cli/lib/pre_clean.py --tool_name mineru --download_dir {pdf_dev_path}")
         now_score = get_score()
         print ("now_score:", now_score)