Skip to content

GitLab

Commit f99149b8 authored by 赵小蒙's avatar 赵小蒙
Browse files

重构目录结构

parent 59bc15e0
Show whitespace changes
Inline Side-by-side
Showing with 64 additions and 86 deletions
para/block_termination_processor.py → pdf_tools/para/block_termination_processor.py
View file @ f99149b8
import sys
from libs.commons import fitz
from termcolor import cprint
from para.commons import *
from pdf_tools.para.commons import *
if sys.version_info[0] >= 3:
......
para/commons.py → pdf_tools/para/commons.py
View file @ f99149b8
import sys
from libs.commons import fitz
from pdf_tools.libs.commons import fitz
from termcolor import cprint
......
para/denoise.py → pdf_tools/para/denoise.py
View file @ f99149b8
import sys
import math
from collections import defaultdict
from para.commons import *
from pdf_tools.para.commons import *
if sys.version_info[0] >= 3:
sys.stdout.reconfigure(encoding="utf-8") # type: ignore
......
para/draw.py → pdf_tools/para/draw.py
View file @ f99149b8
import sys
from pdf_tools.libs.commons import fitz
from libs.commons import fitz
from para.commons import *
from pdf_tools.para.commons import *
if sys.version_info[0] >= 3:
......
para/layout_match_processor.py → pdf_tools/para/layout_match_processor.py
View file @ f99149b8
import sys
import math
from para.commons import *
from pdf_tools.para.commons import *
if sys.version_info[0] >= 3:
......
para/para_pipeline.py → pdf_tools/para/para_pipeline.py
View file @ f99149b8
import os
import sys
import json
from para.commons import *
from para.raw_processor import RawBlockProcessor
from para.layout_match_processor import LayoutFilterProcessor
from para.stats import BlockStatisticsCalculator
from para.stats import DocStatisticsCalculator
from para.title_processor import TitleProcessor
from para.block_termination_processor import BlockTerminationProcessor
from para.block_continuation_processor import BlockContinuationProcessor
from para.draw import DrawAnnos
from para.exceptions import (
from pdf_tools.para.commons import *
from pdf_tools.para.raw_processor import RawBlockProcessor
from pdf_tools.para.layout_match_processor import LayoutFilterProcessor
from pdf_tools.para.stats import BlockStatisticsCalculator
from pdf_tools.para.stats import DocStatisticsCalculator
from pdf_tools.para.title_processor import TitleProcessor
from pdf_tools.para.block_termination_processor import BlockTerminationProcessor
from pdf_tools.para.block_continuation_processor import BlockContinuationProcessor
from pdf_tools.para.draw import DrawAnnos
from pdf_tools.para.exceptions import (
DenseSingleLineBlockException,
TitleDetectionException,
TitleLevelException,
......
para/raw_processor.py → pdf_tools/para/raw_processor.py
View file @ f99149b8
from para.commons import *
class RawBlockProcessor:
def __init__(self) -> None:
self.y_tolerance = 2
......@@ -186,7 +184,7 @@ class RawBlockProcessor:
The instance of the class.
----------
blocks : list
Input block is a list of raw blocks. Schema can refer to the value of key ""preproc_blocks", demo file is app/pdf_toolbox/test/preproc_2_parasplit_example.json.
Input block is a list of raw blocks. Schema can refer to the value of key ""preproc_blocks", demo file is app/pdf_toolbox/tests/preproc_2_parasplit_example.json.
Returns
-------
......
para/stats.py → pdf_tools/para/stats.py
View file @ f99149b8
import sys
from collections import Counter
import numpy as np
from para.commons import *
from pdf_tools.para.commons import *
if sys.version_info[0] >= 3:
......@@ -149,7 +148,7 @@ class BlockStatisticsCalculator:
The instance of the class.
----------
blocks : list
Input block is a list of raw blocks. Schema can refer to the value of key ""preproc_blocks", demo file is app/pdf_toolbox/test/preproc_2_parasplit_example.json
Input block is a list of raw blocks. Schema can refer to the value of key ""preproc_blocks", demo file is app/pdf_toolbox/tests/preproc_2_parasplit_example.json
Returns
-------
......
para/title_processor.py → pdf_tools/para/title_processor.py
View file @ f99149b8
import os
import sys
import re
import numpy as np
from libs.nlp_utils import NLPModels
from pdf_tools.libs.nlp_utils import NLPModels
from para.commons import *
from pdf_tools.para.commons import *
if sys.version_info[0] >= 3:
sys.stdout.reconfigure(encoding="utf-8") # type: ignore
......
pipeline/pdf_parse_by_model.py → pdf_tools/pipeline/pdf_parse_by_model.py
View file @ f99149b8
......@@ -2,28 +2,28 @@ import time
# from anyio import Path
from libs.commons import fitz, get_delta_time, get_img_s3_client
from pdf_tools.libs.commons import fitz, get_delta_time, get_img_s3_client
import json
import os
import math
from loguru import logger
from layout.bbox_sort import (
from pdf_tools.layout.bbox_sort import (
prepare_bboxes_for_layout_split,
)
from layout.layout_sort import LAYOUT_UNPROC, get_bboxes_layout, get_columns_cnt_of_layout, sort_text_block
from libs.drop_reason import DropReason
from libs.markdown_utils import escape_special_markdown_char
from libs.safe_filename import sanitize_filename
from libs.vis_utils import draw_bbox_on_page, draw_layout_bbox_on_page
from pre_proc.detect_images import parse_images
from pre_proc.detect_tables import parse_tables # 获取tables的bbox
from pre_proc.detect_equation import parse_equations # 获取equations的bbox
from pre_proc.detect_header import parse_headers # 获取headers的bbox
from pre_proc.detect_page_number import parse_pageNos # 获取pageNos的bbox
from pre_proc.detect_footnote import parse_footnotes_by_model, parse_footnotes_by_rule # 获取footnotes的bbox
from pre_proc.detect_footer_by_model import parse_footers # 获取footers的bbox
from post_proc.detect_para import (
from pdf_tools.layout.layout_sort import LAYOUT_UNPROC, get_bboxes_layout, get_columns_cnt_of_layout, sort_text_block
from pdf_tools.libs.drop_reason import DropReason
from pdf_tools.libs.markdown_utils import escape_special_markdown_char
from pdf_tools.libs.safe_filename import sanitize_filename
from pdf_tools.libs.vis_utils import draw_bbox_on_page, draw_layout_bbox_on_page
from pdf_tools.pre_proc.detect_images import parse_images
from pdf_tools.pre_proc.detect_tables import parse_tables # 获取tables的bbox
from pdf_tools.pre_proc.detect_equation import parse_equations # 获取equations的bbox
from pdf_tools.pre_proc.detect_header import parse_headers # 获取headers的bbox
from pdf_tools.pre_proc.detect_page_number import parse_pageNos # 获取pageNos的bbox
from pdf_tools.pre_proc.detect_footnote import parse_footnotes_by_model, parse_footnotes_by_rule # 获取footnotes的bbox
from pdf_tools.pre_proc.detect_footer_by_model import parse_footers # 获取footers的bbox
from pdf_tools.post_proc.detect_para import (
ParaProcessPipeline,
TitleDetectionException,
TitleLevelException,
......@@ -31,9 +31,9 @@ from post_proc.detect_para import (
ParaMergeException,
DenseSingleLineBlockException,
)
from pre_proc.main_text_font import get_main_text_font
from pre_proc.remove_colored_strip_bbox import remove_colored_strip_textblock
from pre_proc.remove_footer_header import remove_headder_footer_one_page
from pdf_tools.pre_proc.main_text_font import get_main_text_font
from pdf_tools.pre_proc.remove_colored_strip_bbox import remove_colored_strip_textblock
from pdf_tools.pre_proc.remove_footer_header import remove_headder_footer_one_page
'''
from para.para_pipeline import ParaProcessPipeline
......@@ -46,19 +46,19 @@ from para.exceptions import (
)
'''
from libs.commons import read_file, join_path
from libs.pdf_image_tools import save_images_by_bboxes
from post_proc.remove_footnote import merge_footnote_blocks, remove_footnote_blocks
from pre_proc.citationmarker_remove import remove_citation_marker
from pre_proc.equations_replace import combine_chars_to_pymudict, remove_chars_in_text_blocks, replace_equations_in_textblock
from pre_proc.pdf_pre_filter import pdf_filter
from pre_proc.detect_footer_header_by_statistics import drop_footer_header
from pre_proc.construct_paras import construct_page_component
from pre_proc.fix_image import combine_images, fix_image_vertical, fix_seperated_image, include_img_title
from post_proc.pdf_post_filter import pdf_post_filter
from pre_proc.remove_rotate_bbox import get_side_boundry, remove_rotate_side_textblock, remove_side_blank_block
from pre_proc.resolve_bbox_conflict import check_text_block_horizontal_overlap, resolve_bbox_overlap_conflict
from pre_proc.fix_table import fix_table_text_block, fix_tables, include_table_title
from pdf_tools.libs.commons import read_file, join_path
from pdf_tools.libs.pdf_image_tools import save_images_by_bboxes
from pdf_tools.post_proc.remove_footnote import merge_footnote_blocks, remove_footnote_blocks
from pdf_tools.pre_proc.citationmarker_remove import remove_citation_marker
from pdf_tools.pre_proc.equations_replace import combine_chars_to_pymudict, remove_chars_in_text_blocks, replace_equations_in_textblock
from pdf_tools.pre_proc.pdf_pre_filter import pdf_filter
from pdf_tools.pre_proc.detect_footer_header_by_statistics import drop_footer_header
from pdf_tools.pre_proc.construct_paras import construct_page_component
from pdf_tools.pre_proc.fix_image import combine_images, fix_image_vertical, fix_seperated_image, include_img_title
from pdf_tools.post_proc.pdf_post_filter import pdf_post_filter
from pdf_tools.pre_proc.remove_rotate_bbox import get_side_boundry, remove_rotate_side_textblock, remove_side_blank_block
from pdf_tools.pre_proc.resolve_bbox_conflict import check_text_block_horizontal_overlap, resolve_bbox_overlap_conflict
from pdf_tools.pre_proc.fix_table import fix_table_text_block, fix_tables, include_table_title
denseSingleLineBlockException_msg = DenseSingleLineBlockException().message
titleDetectionException_msg = TitleDetectionException().message
......@@ -108,7 +108,7 @@ def parse_pdf_by_model(
debug_mode=False,
):
pdf_bytes = read_file(s3_pdf_path, s3_pdf_profile)
save_tmp_path = os.path.join(os.path.dirname(__file__), "../..", "tmp", "unittest")
save_tmp_path = os.path.join(os.path.dirname(__file__), "../../..", "tmp", "unittest")
md_bookname_save_path = ""
book_name = sanitize_filename(book_name)
if debug_mode:
......
post_proc/detect_para.py → pdf_tools/post_proc/detect_para.py
View file @ f99149b8
......@@ -11,8 +11,8 @@ import numpy as np
from termcolor import cprint
from libs.commons import fitz
from libs.nlp_utils import NLPModels
from pdf_tools.libs.commons import fitz
from pdf_tools.libs.nlp_utils import NLPModels
if sys.version_info[0] >= 3:
......@@ -3404,7 +3404,7 @@ Params:
if __name__ == "__main__":
DEFAULT_PDF_PATH = (
"app/pdf_toolbox/test/assets/paper/paper.pdf" if os.name != "nt" else "app\\pdf_toolbox\\test\\assets\\paper\\paper.pdf"
"app/pdf_toolbox/tests/assets/paper/paper.pdf" if os.name != "nt" else "app\\pdf_toolbox\\tests\\assets\\paper\\paper.pdf"
)
input_pdf_path = sys.argv[1] if len(sys.argv) > 1 else DEFAULT_PDF_PATH
output_pdf_path = sys.argv[2] if len(sys.argv) > 2 else input_pdf_path.split(".")[0] + "_recogPara.pdf"
......
post_proc/pdf_post_filter.py → pdf_tools/post_proc/pdf_post_filter.py
View file @ f99149b8
from loguru import logger
from layout.layout_sort import get_columns_cnt_of_layout
from libs.drop_reason import DropReason
from pdf_tools.layout.layout_sort import get_columns_cnt_of_layout
from pdf_tools.libs.drop_reason import DropReason
def __is_pseudo_single_column(page_info) -> bool:
......
post_proc/remove_footnote.py → pdf_tools/post_proc/remove_footnote.py
View file @ f99149b8
from libs.boxbase import _is_in, _is_in_or_part_overlap
from pdf_tools.libs.boxbase import _is_in, _is_in_or_part_overlap
import collections # 统计库
......
pre_proc/citationmarker_remove.py → pdf_tools/pre_proc/citationmarker_remove.py
View file @ f99149b8
......@@ -3,8 +3,7 @@
https://aicarrier.feishu.cn/wiki/YLOPwo1PGiwFRdkwmyhcZmr0n3d
"""
import re
from loguru import logger
from libs.nlp_utils import NLPModels
from pdf_tools.libs.nlp_utils import NLPModels
__NLP_MODEL = NLPModels()
......
pre_proc/detect_equation.py → pdf_tools/pre_proc/detect_equation.py
View file @ f99149b8
import os
import collections # 统计库
import re
from libs.boxbase import _is_in # 正则
from libs.commons import fitz # pyMuPDF库
import json # json
from pathlib import Path
from pdf_tools.libs.boxbase import _is_in # 正则
from pdf_tools.libs.commons import fitz # pyMuPDF库
def __solve_contain_bboxs(all_bbox_list: list):
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment