Merge pull request #2625 from opendatalab/release-2.0.0

Release 2.0.0

mineru/cli/client.py

+# Copyright (c) Opendatalab. All rights reserved.
+import os
+import click
+from pathlib import Path
+import torch
+from loguru import logger
+
+from mineru.utils.config_reader import get_device
+from mineru.utils.model_utils import get_vram
+from ..version import __version__
+from .common import do_parse, read_fn, pdf_suffixes, image_suffixes
+
+
+@click.command()
+@click.version_option(__version__,
+                      '--version',
+                      '-v',
+                      help='display the version and exit')
+@click.option(
+    '-p',
+    '--path',
+    'input_path',
+    type=click.Path(exists=True),
+    required=True,
+    help='local filepath or directory. support pdf, png, jpg, jpeg files',
+)
+@click.option(
+    '-o',
+    '--output',
+    'output_dir',
+    type=click.Path(),
+    required=True,
+    help='output local directory',
+)
+@click.option(
+    '-m',
+    '--method',
+    'method',
+    type=click.Choice(['auto', 'txt', 'ocr']),
+    help="""the method for parsing pdf:
+    auto: Automatically determine the method based on the file type.
+    txt: Use text extraction method.
+    ocr: Use OCR method for image-based PDFs.
+    Without method specified, 'auto' will be used by default.
+    Adapted only for the case where the backend is set to "pipeline".""",
+    default='auto',
+)
+@click.option(
+    '-b',
+    '--backend',
+    'backend',
+    type=click.Choice(['pipeline', 'vlm-transformers', 'vlm-sglang-engine', 'vlm-sglang-client']),
+    help="""the backend for parsing pdf:
+    pipeline: More general.
+    vlm-transformers: More general.
+    vlm-sglang-engine: Faster(engine).
+    vlm-sglang-client: Faster(client).
+    without method specified, pipeline will be used by default.""",
+    default='pipeline',
+)
+@click.option(
+    '-l',
+    '--lang',
+    'lang',
+    type=click.Choice(['ch', 'ch_server', 'ch_lite', 'en', 'korean', 'japan', 'chinese_cht', 'ta', 'te', 'ka']),
+    help="""
+    Input the languages in the pdf (if known) to improve OCR accuracy.  Optional.
+    Without languages specified, 'ch' will be used by default.
+    Adapted only for the case where the backend is set to "pipeline".
+    """,
+    default='ch',
+)
+@click.option(
+    '-u',
+    '--url',
+    'server_url',
+    type=str,
+    help="""
+    When the backend is `sglang-client`, you need to specify the server_url, for example:`http://127.0.0.1:30000`
+    """,
+    default=None,
+)
+@click.option(
+    '-s',
+    '--start',
+    'start_page_id',
+    type=int,
+    help='The starting page for PDF parsing, beginning from 0.',
+    default=0,
+)
+@click.option(
+    '-e',
+    '--end',
+    'end_page_id',
+    type=int,
+    help='The ending page for PDF parsing, beginning from 0.',
+    default=None,
+)
+@click.option(
+    '-f',
+    '--formula',
+    'formula_enable',
+    type=bool,
+    help='Enable formula parsing. Default is True. Adapted only for the case where the backend is set to "pipeline".',
+    default=True,
+)
+@click.option(
+    '-t',
+    '--table',
+    'table_enable',
+    type=bool,
+    help='Enable table parsing. Default is True. Adapted only for the case where the backend is set to "pipeline".',
+    default=True,
+)
+@click.option(
+    '-d',
+    '--device',
+    'device_mode',
+    type=str,
+    help='Device mode for model inference, e.g., "cpu", "cuda", "cuda:0", "npu", "npu:0", "mps". Adapted only for the case where the backend is set to "pipeline". ',
+    default=None,
+)
+@click.option(
+    '--vram',
+    'virtual_vram',
+    type=int,
+    help='Upper limit of GPU memory occupied by a single process. Adapted only for the case where the backend is set to "pipeline". ',
+    default=None,
+)
+@click.option(
+    '--source',
+    'model_source',
+    type=click.Choice(['huggingface', 'modelscope', 'local']),
+    help="""
+    The source of the model repository. Default is 'huggingface'.
+    """,
+    default='huggingface',
+)
+
+
+def main(input_path, output_dir, method, backend, lang, server_url, start_page_id, end_page_id, formula_enable, table_enable, device_mode, virtual_vram, model_source):
+
+    if os.getenv('MINERU_FORMULA_ENABLE', None) is None:
+        os.environ['MINERU_FORMULA_ENABLE'] = str(formula_enable).lower()
+    if os.getenv('MINERU_TABLE_ENABLE', None) is None:
+        os.environ['MINERU_TABLE_ENABLE'] = str(table_enable).lower()
+    def get_device_mode() -> str:
+        if device_mode is not None:
+            return device_mode
+        else:
+            return get_device()
+    if os.getenv('MINERU_DEVICE_MODE', None) is None:
+        os.environ['MINERU_DEVICE_MODE'] = get_device_mode()
+
+    def get_virtual_vram_size() -> int:
+        if virtual_vram is not None:
+            return virtual_vram
+        if get_device_mode().startswith("cuda") or get_device_mode().startswith("npu"):
+            return round(get_vram(get_device_mode()))
+        return 1
+    if os.getenv('MINERU_VIRTUAL_VRAM_SIZE', None) is None:
+        os.environ['MINERU_VIRTUAL_VRAM_SIZE']= str(get_virtual_vram_size())
+
+    if os.getenv('MINERU_MODEL_SOURCE', None) is None:
+        os.environ['MINERU_MODEL_SOURCE'] = model_source
+
+    os.makedirs(output_dir, exist_ok=True)
+
+    def parse_doc(path_list: list[Path]):
+        try:
+            file_name_list = []
+            pdf_bytes_list = []
+            lang_list = []
+            for path in path_list:
+                file_name = str(Path(path).stem)
+                pdf_bytes = read_fn(path)
+                file_name_list.append(file_name)
+                pdf_bytes_list.append(pdf_bytes)
+                lang_list.append(lang)
+            do_parse(
+                output_dir=output_dir,
+                pdf_file_names=file_name_list,
+                pdf_bytes_list=pdf_bytes_list,
+                p_lang_list=lang_list,
+                backend=backend,
+                parse_method=method,
+                server_url=server_url,
+                start_page_id=start_page_id,
+                end_page_id=end_page_id
+            )
+        except Exception as e:
+            logger.exception(e)
+
+    if os.path.isdir(input_path):
+        doc_path_list = []
+        for doc_path in Path(input_path).glob('*'):
+            if doc_path.suffix in pdf_suffixes + image_suffixes:
+                doc_path_list.append(doc_path)
+        parse_doc(doc_path_list)
+    else:
+        parse_doc([Path(input_path)])
+
+if __name__ == '__main__':
+    main()

mineru/cli/common.py

+# Copyright (c) Opendatalab. All rights reserved.
+import io
+import json
+import os
+import copy
+from pathlib import Path
+
+import pypdfium2 as pdfium
+from loguru import logger
+
+from mineru.backend.pipeline.pipeline_middle_json_mkcontent import union_make as pipeline_union_make
+from mineru.backend.pipeline.model_json_to_middle_json import result_to_middle_json as pipeline_result_to_middle_json
+from mineru.backend.vlm.vlm_middle_json_mkcontent import union_make as vlm_union_make
+from mineru.backend.vlm.vlm_analyze import doc_analyze as vlm_doc_analyze
+from mineru.backend.pipeline.pipeline_analyze import doc_analyze as pipeline_doc_analyze
+from mineru.data.data_reader_writer import FileBasedDataWriter
+from mineru.utils.draw_bbox import draw_layout_bbox, draw_span_bbox
+from mineru.utils.enum_class import MakeMode
+from mineru.utils.pdf_image_tools import images_bytes_to_pdf_bytes
+
+pdf_suffixes = [".pdf"]
+image_suffixes = [".png", ".jpeg", ".jpg"]
+
+
+def read_fn(path):
+    if not isinstance(path, Path):
+        path = Path(path)
+    with open(str(path), "rb") as input_file:
+        file_bytes = input_file.read()
+        if path.suffix in image_suffixes:
+            return images_bytes_to_pdf_bytes(file_bytes)
+        elif path.suffix in pdf_suffixes:
+            return file_bytes
+        else:
+            raise Exception(f"Unknown file suffix: {path.suffix}")
+
+
+def prepare_env(output_dir, pdf_file_name, parse_method):
+    local_md_dir = str(os.path.join(output_dir, pdf_file_name, parse_method))
+    local_image_dir = os.path.join(str(local_md_dir), "images")
+    os.makedirs(local_image_dir, exist_ok=True)
+    os.makedirs(local_md_dir, exist_ok=True)
+    return local_image_dir, local_md_dir
+
+
+def convert_pdf_bytes_to_bytes_by_pypdfium2(pdf_bytes, start_page_id=0, end_page_id=None):
+
+    # 从字节数据加载PDF
+    pdf = pdfium.PdfDocument(pdf_bytes)
+
+    # 确定结束页
+    end_page_id = end_page_id if end_page_id is not None and end_page_id >= 0 else len(pdf) - 1
+    if end_page_id > len(pdf) - 1:
+        logger.warning("end_page_id is out of range, use pdf_docs length")
+        end_page_id = len(pdf) - 1
+
+    # 创建一个新的PDF文档
+    output_pdf = pdfium.PdfDocument.new()
+
+    # 选择要导入的页面索引
+    page_indices = list(range(start_page_id, end_page_id + 1))
+
+    # 从原PDF导入页面到新PDF
+    output_pdf.import_pages(pdf, page_indices)
+
+    # 将新PDF保存到内存缓冲区
+    output_buffer = io.BytesIO()
+    output_pdf.save(output_buffer)
+
+    # 获取字节数据
+    output_bytes = output_buffer.getvalue()
+
+    pdf.close()  # 关闭原PDF文档以释放资源
+    output_pdf.close()  # 关闭新PDF文档以释放资源
+
+    return output_bytes
+
+
+def do_parse(
+    output_dir,
+    pdf_file_names: list[str],
+    pdf_bytes_list: list[bytes],
+    p_lang_list: list[str],
+    backend="pipeline",
+    parse_method="auto",
+    p_formula_enable=True,
+    p_table_enable=True,
+    server_url=None,
+    f_draw_layout_bbox=True,
+    f_draw_span_bbox=True,
+    f_dump_md=True,
+    f_dump_middle_json=True,
+    f_dump_model_output=True,
+    f_dump_orig_pdf=True,
+    f_dump_content_list=True,
+    f_make_md_mode=MakeMode.MM_MD,
+    start_page_id=0,
+    end_page_id=None,
+):
+
+    if backend == "pipeline":
+        for idx, pdf_bytes in enumerate(pdf_bytes_list):
+            new_pdf_bytes = convert_pdf_bytes_to_bytes_by_pypdfium2(pdf_bytes, start_page_id, end_page_id)
+            pdf_bytes_list[idx] = new_pdf_bytes
+
+        infer_results, all_image_lists, all_pdf_docs, lang_list, ocr_enabled_list = pipeline_doc_analyze(pdf_bytes_list, p_lang_list, parse_method=parse_method, formula_enable=p_formula_enable,table_enable=p_table_enable)
+
+        for idx, model_list in enumerate(infer_results):
+            model_json = copy.deepcopy(model_list)
+            pdf_file_name = pdf_file_names[idx]
+            local_image_dir, local_md_dir = prepare_env(output_dir, pdf_file_name, parse_method)
+            image_writer, md_writer = FileBasedDataWriter(local_image_dir), FileBasedDataWriter(local_md_dir)
+
+            images_list = all_image_lists[idx]
+            pdf_doc = all_pdf_docs[idx]
+            _lang = lang_list[idx]
+            _ocr_enable = ocr_enabled_list[idx]
+            middle_json = pipeline_result_to_middle_json(model_list, images_list, pdf_doc, image_writer, _lang, _ocr_enable, p_formula_enable)
+
+            pdf_info = middle_json["pdf_info"]
+
+            pdf_bytes = pdf_bytes_list[idx]
+            if f_draw_layout_bbox:
+                draw_layout_bbox(pdf_info, pdf_bytes, local_md_dir, f"{pdf_file_name}_layout.pdf")
+
+            if f_draw_span_bbox:
+                draw_span_bbox(pdf_info, pdf_bytes, local_md_dir, f"{pdf_file_name}_span.pdf")
+
+            if f_dump_orig_pdf:
+                md_writer.write(
+                    f"{pdf_file_name}_origin.pdf",
+                    pdf_bytes,
+                )
+
+            if f_dump_md:
+                image_dir = str(os.path.basename(local_image_dir))
+                md_content_str = pipeline_union_make(pdf_info, f_make_md_mode, image_dir)
+                md_writer.write_string(
+                    f"{pdf_file_name}.md",
+                    md_content_str,
+                )
+
+            if f_dump_content_list:
+                image_dir = str(os.path.basename(local_image_dir))
+                content_list = pipeline_union_make(pdf_info, MakeMode.CONTENT_LIST, image_dir)
+                md_writer.write_string(
+                    f"{pdf_file_name}_content_list.json",
+                    json.dumps(content_list, ensure_ascii=False, indent=4),
+                )
+
+            if f_dump_middle_json:
+                md_writer.write_string(
+                    f"{pdf_file_name}_middle.json",
+                    json.dumps(middle_json, ensure_ascii=False, indent=4),
+                )
+
+            if f_dump_model_output:
+                md_writer.write_string(
+                    f"{pdf_file_name}_model.json",
+                    json.dumps(model_json, ensure_ascii=False, indent=4),
+                )
+
+            logger.info(f"local output dir is {local_md_dir}")
+    else:
+        if backend.startswith("vlm-"):
+            backend = backend[4:]
+
+        f_draw_span_bbox = False
+        parse_method = "vlm"
+        for idx, pdf_bytes in enumerate(pdf_bytes_list):
+            pdf_file_name = pdf_file_names[idx]
+            pdf_bytes = convert_pdf_bytes_to_bytes_by_pypdfium2(pdf_bytes, start_page_id, end_page_id)
+            local_image_dir, local_md_dir = prepare_env(output_dir, pdf_file_name, parse_method)
+            image_writer, md_writer = FileBasedDataWriter(local_image_dir), FileBasedDataWriter(local_md_dir)
+            middle_json, infer_result = vlm_doc_analyze(pdf_bytes, image_writer=image_writer, backend=backend, server_url=server_url)
+
+            pdf_info = middle_json["pdf_info"]
+
+            if f_draw_layout_bbox:
+                draw_layout_bbox(pdf_info, pdf_bytes, local_md_dir, f"{pdf_file_name}_layout.pdf")
+
+            if f_draw_span_bbox:
+                draw_span_bbox(pdf_info, pdf_bytes, local_md_dir, f"{pdf_file_name}_span.pdf")
+
+            if f_dump_orig_pdf:
+                md_writer.write(
+                    f"{pdf_file_name}_origin.pdf",
+                    pdf_bytes,
+                )
+
+            if f_dump_md:
+                image_dir = str(os.path.basename(local_image_dir))
+                md_content_str = vlm_union_make(pdf_info, f_make_md_mode, image_dir)
+                md_writer.write_string(
+                    f"{pdf_file_name}.md",
+                    md_content_str,
+                )
+
+            if f_dump_content_list:
+                image_dir = str(os.path.basename(local_image_dir))
+                content_list = vlm_union_make(pdf_info, MakeMode.CONTENT_LIST, image_dir)
+                md_writer.write_string(
+                    f"{pdf_file_name}_content_list.json",
+                    json.dumps(content_list, ensure_ascii=False, indent=4),
+                )
+
+            if f_dump_middle_json:
+                md_writer.write_string(
+                    f"{pdf_file_name}_middle.json",
+                    json.dumps(middle_json, ensure_ascii=False, indent=4),
+                )
+
+            if f_dump_model_output:
+                model_output = ("\n" + "-" * 50 + "\n").join(infer_result)
+                md_writer.write_string(
+                    f"{pdf_file_name}_model_output.txt",
+                    model_output,
+                )
+
+            logger.info(f"local output dir is {local_md_dir}")
+
+
+
+if __name__ == "__main__":
+    # pdf_path = "../../demo/pdfs/demo3.pdf"
+    pdf_path = "C:/Users/zhaoxiaomeng/Downloads/4546d0e2-ba60-40a5-a17e-b68555cec741.pdf"
+
+    try:
+       do_parse("./output", [Path(pdf_path).stem], [read_fn(Path(pdf_path))],["ch"],
+                end_page_id=10,
+                backend='vlm-huggingface'
+                # backend = 'pipeline'
+                )
+    except Exception as e:
+        logger.exception(e)

mineru/cli/models_download.py

+import json
+import os
+import sys
+import click
+import requests
+
+from mineru.utils.enum_class import ModelPath
+from mineru.utils.models_download_utils import auto_download_and_get_model_root_path
+
+
+def download_json(url):
+    """下载JSON文件"""
+    response = requests.get(url)
+    response.raise_for_status()
+    return response.json()
+
+
+def download_and_modify_json(url, local_filename, modifications):
+    """下载JSON并修改内容"""
+    if os.path.exists(local_filename):
+        data = json.load(open(local_filename))
+        config_version = data.get('config_version', '0.0.0')
+        if config_version < '1.3.0':
+            data = download_json(url)
+    else:
+        data = download_json(url)
+
+    # 修改内容
+    for key, value in modifications.items():
+        if key in data:
+            if isinstance(data[key], dict):
+                # 如果是字典，合并新值
+                data[key].update(value)
+            else:
+                # 否则直接替换
+                data[key] = value
+
+    # 保存修改后的内容
+    with open(local_filename, 'w', encoding='utf-8') as f:
+        json.dump(data, f, ensure_ascii=False, indent=4)
+
+
+def configure_model(model_dir, model_type):
+    """配置模型"""
+    # json_url = 'https://gcore.jsdelivr.net/gh/opendatalab/MinerU@master/mineru.template.json'
+    json_url = 'https://gcore.jsdelivr.net/gh/myhloli/Magic-PDF@dev/mineru.template.json'
+    config_file_name = 'mineru.json'
+    home_dir = os.path.expanduser('~')
+    config_file = os.path.join(home_dir, config_file_name)
+
+    json_mods = {
+        'models-dir': {
+            f'{model_type}': model_dir
+        }
+    }
+
+    download_and_modify_json(json_url, config_file, json_mods)
+    print(f'The configuration file has been successfully configured, the path is: {config_file}')
+
+
+@click.command()
+@click.option(
+    '-s',
+    '--source',
+    'model_source',
+    type=click.Choice(['huggingface', 'modelscope']),
+    help="""
+        The source of the model repository. 
+        """,
+    default=None,
+)
+@click.option(
+    '-m',
+    '--model_type',
+    'model_type',
+    type=click.Choice(['pipeline', 'vlm', 'all']),
+    help="""
+        The type of the model to download.
+        """,
+    default=None,
+)
+def download_models(model_source, model_type):
+    """Download MinerU model files.
+
+    Supports downloading pipeline or VLM models from ModelScope or HuggingFace.
+    """
+    # 如果未显式指定则交互式输入下载来源
+    if model_source is None:
+        model_source = click.prompt(
+            "Please select the model download source: ",
+            type=click.Choice(['huggingface', 'modelscope']),
+            default='huggingface'
+        )
+
+    if os.getenv('MINERU_MODEL_SOURCE', None) is None:
+        os.environ['MINERU_MODEL_SOURCE'] = model_source
+
+    # 如果未显式指定则交互式输入模型类型
+    if model_type is None:
+        model_type = click.prompt(
+            "Please select the model type to download: ",
+            type=click.Choice(['pipeline', 'vlm', 'all']),
+            default='all'
+        )
+
+    click.echo(f"Downloading {model_type} model from {os.getenv('MINERU_MODEL_SOURCE', None)}...")
+
+    def download_pipeline_models():
+        """下载Pipeline模型"""
+        model_paths = [
+            ModelPath.doclayout_yolo,
+            ModelPath.yolo_v8_mfd,
+            ModelPath.unimernet_small,
+            ModelPath.pytorch_paddle,
+            ModelPath.layout_reader,
+            ModelPath.slanet_plus
+        ]
+        download_finish_path = ""
+        for model_path in model_paths:
+            click.echo(f"Downloading model: {model_path}")
+            download_finish_path = auto_download_and_get_model_root_path(model_path, repo_mode='pipeline')
+        click.echo(f"Pipeline models downloaded successfully to: {download_finish_path}")
+        configure_model(download_finish_path, model_type)
+
+    def download_vlm_models():
+        """下载VLM模型"""
+        download_finish_path = auto_download_and_get_model_root_path("/", repo_mode='vlm')
+        click.echo(f"VLM models downloaded successfully to: {download_finish_path}")
+        configure_model(download_finish_path, model_type)
+
+    try:
+        if model_type == 'pipeline':
+            download_pipeline_models()
+        elif model_type == 'vlm':
+            download_vlm_models()
+        elif model_type == 'all':
+            download_pipeline_models()
+            download_vlm_models()
+        else:
+            click.echo(f"Unsupported model type: {model_type}", err=True)
+            sys.exit(1)
+
+    except Exception as e:
+        click.echo(f"Download failed: {str(e)}", err=True)
+        sys.exit(1)
+
+if __name__ == '__main__':
+    download_models()

mineru/cli/vlm_sglang_server.py

+from ..model.vlm_sglang_model.server import main
+
+if __name__ == "__main__":
+    main()

mineru/data/__init__.py

+# Copyright (c) Opendatalab. All rights reserved.

mineru/data/data_reader_writer/__init__.py

+from .base import DataReader, DataWriter
+from .dummy import DummyDataWriter
+from .filebase import FileBasedDataReader, FileBasedDataWriter
+from .multi_bucket_s3 import MultiBucketS3DataReader, MultiBucketS3DataWriter
+from .s3 import S3DataReader, S3DataWriter
+
+__all__ = [
+    "DataReader",
+    "DataWriter",
+    "FileBasedDataReader",
+    "FileBasedDataWriter",
+    "S3DataReader",
+    "S3DataWriter",
+    "MultiBucketS3DataReader",
+    "MultiBucketS3DataWriter",
+    "DummyDataWriter",
+]

mineru/data/data_reader_writer/dummy.py

+from .base import DataWriter
+
+
+class DummyDataWriter(DataWriter):
+    def write(self, path: str, data: bytes) -> None:
+        """Dummy write method that does nothing."""
+        pass
+
+    def write_string(self, path: str, data: str) -> None:
+        """Dummy write_string method that does nothing."""
+        pass

magic_pdf/data/data_reader_writer/filebase.py → mineru/data/data_reader_writer/filebase.py

 import os
 
-from magic_pdf.data.data_reader_writer.base import DataReader, DataWriter
+from .base import DataReader, DataWriter
 
 
 class FileBasedDataReader(DataReader):

magic_pdf/data/data_reader_writer/multi_bucket_s3.py → mineru/data/data_reader_writer/multi_bucket_s3.py

 
-from magic_pdf.config.exceptions import InvalidConfig, InvalidParams
-from magic_pdf.data.data_reader_writer.base import DataReader, DataWriter
-from magic_pdf.data.io.s3 import S3Reader, S3Writer
-from magic_pdf.data.schemas import S3Config
-from magic_pdf.libs.path_utils import (parse_s3_range_params, parse_s3path,
-                                       remove_non_official_s3_args)
+from ..utils.exceptions import InvalidConfig, InvalidParams
+from .base import DataReader, DataWriter
+from ..io.s3 import S3Reader, S3Writer
+from ..utils.schemas import S3Config
+from ..utils.path_utils import parse_s3_range_params, parse_s3path, remove_non_official_s3_args
 
 
 class MultiS3Mixin:

magic_pdf/data/data_reader_writer/s3.py → mineru/data/data_reader_writer/s3.py

-from magic_pdf.data.data_reader_writer.multi_bucket_s3 import (
-    MultiBucketS3DataReader, MultiBucketS3DataWriter)
-from magic_pdf.data.schemas import S3Config
+from .multi_bucket_s3 import MultiBucketS3DataReader, MultiBucketS3DataWriter
+from ..utils.schemas import S3Config
 
 
 class S3DataReader(MultiBucketS3DataReader):

mineru/data/io/__init__.py

+
+from .base import IOReader, IOWriter
+from .http import HttpReader, HttpWriter
+from .s3 import S3Reader, S3Writer
+
+__all__ = ['IOReader', 'IOWriter', 'HttpReader', 'HttpWriter', 'S3Reader', 'S3Writer']
\ No newline at end of file

magic_pdf/data/io/http.py → mineru/data/io/http.py

@@ -3,7 +3,7 @@ import io
 
 import requests
 
-from magic_pdf.data.io.base import IOReader, IOWriter
+from .base import IOReader, IOWriter
 
 
 class HttpReader(IOReader):

magic_pdf/data/io/s3.py → mineru/data/io/s3.py

 import boto3
 from botocore.config import Config
 
-from magic_pdf.data.io.base import IOReader, IOWriter
+from ..io.base import IOReader, IOWriter
 
 
 class S3Reader(IOReader):

mineru/data/utils/__init__.py

+# Copyright (c) Opendatalab. All rights reserved.

magic_pdf/config/exceptions.py → mineru/data/utils/exceptions.py

+# Copyright (c) Opendatalab. All rights reserved.
 
 class FileNotExisted(Exception):
 

magic_pdf/libs/path_utils.py → mineru/data/utils/path_utils.py

+# Copyright (c) Opendatalab. All rights reserved.
 
 
 def remove_non_official_s3_args(s3path):

magic_pdf/data/schemas.py → mineru/data/utils/schemas.py

+# Copyright (c) Opendatalab. All rights reserved.
 
 from pydantic import BaseModel, Field