Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
wangsen
MinerU
Commits
7e6926ff
Commit
7e6926ff
authored
Jul 09, 2025
by
myhloli
Browse files
feat: enhance heading level feature with conditional imports and error handling
parent
ca7a567e
Changes
2
Show whitespace changes
Inline
Side-by-side
Showing
2 changed files
with
37 additions
and
30 deletions
+37
-30
mineru/backend/vlm/token_to_middle_json.py
mineru/backend/vlm/token_to_middle_json.py
+35
-28
pyproject.toml
pyproject.toml
+2
-2
No files found.
mineru/backend/vlm/token_to_middle_json.py
View file @
7e6926ff
import
time
import
time
import
cv2
import
numpy
as
np
from
loguru
import
logger
from
loguru
import
logger
import
numpy
as
np
from
mineru.backend.pipeline.model_init
import
AtomModelSingleton
import
cv2
from
mineru.utils.config_reader
import
get_llm_aided_config
from
mineru.utils.config_reader
import
get_llm_aided_config
from
mineru.utils.cut_image
import
cut_image_and_table
from
mineru.utils.cut_image
import
cut_image_and_table
from
mineru.utils.enum_class
import
ContentType
from
mineru.utils.enum_class
import
ContentType
from
mineru.utils.hash_utils
import
str_md5
from
mineru.utils.hash_utils
import
str_md5
from
mineru.backend.vlm.vlm_magic_model
import
MagicModel
from
mineru.backend.vlm.vlm_magic_model
import
MagicModel
from
mineru.utils.llm_aided
import
llm_aided_title
from
mineru.utils.pdf_image_tools
import
get_crop_img
from
mineru.utils.pdf_image_tools
import
get_crop_img
from
mineru.version
import
__version__
from
mineru.version
import
__version__
heading_level_import_success
=
False
try
:
from
mineru.utils.llm_aided
import
llm_aided_title
from
mineru.backend.pipeline.model_init
import
AtomModelSingleton
heading_level_import_success
=
True
except
Exception
as
e
:
logger
.
warning
(
"The heading level feature cannot be used. If you need to use the heading level feature, "
"please execute `pip install mineru[pipeline]` to install the required packages."
)
def
token_to_page_info
(
token
,
image_dict
,
page
,
image_writer
,
page_index
)
->
dict
:
def
token_to_page_info
(
token
,
image_dict
,
page
,
image_writer
,
page_index
)
->
dict
:
"""将token转换为页面信息"""
"""将token转换为页面信息"""
...
@@ -37,6 +43,7 @@ def token_to_page_info(token, image_dict, page, image_writer, page_index) -> dic
...
@@ -37,6 +43,7 @@ def token_to_page_info(token, image_dict, page, image_writer, page_index) -> dic
title_aided_config
=
llm_aided_config
.
get
(
'title_aided'
,
None
)
title_aided_config
=
llm_aided_config
.
get
(
'title_aided'
,
None
)
if
title_aided_config
is
not
None
:
if
title_aided_config
is
not
None
:
if
title_aided_config
.
get
(
'enable'
,
False
):
if
title_aided_config
.
get
(
'enable'
,
False
):
if
heading_level_import_success
:
atom_model_manager
=
AtomModelSingleton
()
atom_model_manager
=
AtomModelSingleton
()
ocr_model
=
atom_model_manager
.
get_atom_model
(
ocr_model
=
atom_model_manager
.
get_atom_model
(
atom_model_name
=
'ocr'
,
atom_model_name
=
'ocr'
,
...
@@ -86,12 +93,12 @@ def result_to_middle_json(token_list, images_list, pdf_doc, image_writer):
...
@@ -86,12 +93,12 @@ def result_to_middle_json(token_list, images_list, pdf_doc, image_writer):
"""llm优化"""
"""llm优化"""
llm_aided_config
=
get_llm_aided_config
()
llm_aided_config
=
get_llm_aided_config
()
if
llm_aided_config
is
not
None
:
if
llm_aided_config
is
not
None
:
"""标题优化"""
"""标题优化"""
title_aided_config
=
llm_aided_config
.
get
(
'title_aided'
,
None
)
title_aided_config
=
llm_aided_config
.
get
(
'title_aided'
,
None
)
if
title_aided_config
is
not
None
:
if
title_aided_config
is
not
None
:
if
title_aided_config
.
get
(
'enable'
,
False
):
if
title_aided_config
.
get
(
'enable'
,
False
):
if
heading_level_import_success
:
llm_aided_title_start_time
=
time
.
time
()
llm_aided_title_start_time
=
time
.
time
()
llm_aided_title
(
middle_json
[
"pdf_info"
],
title_aided_config
)
llm_aided_title
(
middle_json
[
"pdf_info"
],
title_aided_config
)
logger
.
info
(
f
'llm aided title time:
{
round
(
time
.
time
()
-
llm_aided_title_start_time
,
2
)
}
'
)
logger
.
info
(
f
'llm aided title time:
{
round
(
time
.
time
()
-
llm_aided_title_start_time
,
2
)
}
'
)
...
...
pyproject.toml
View file @
7e6926ff
...
@@ -33,6 +33,8 @@ dependencies = [
...
@@ -33,6 +33,8 @@ dependencies = [
"modelscope>=1.26.0"
,
"modelscope>=1.26.0"
,
"huggingface-hub>=0.32.4"
,
"huggingface-hub>=0.32.4"
,
"json-repair>=0.46.2"
,
"json-repair>=0.46.2"
,
"opencv-python>=4.11.0.86"
,
"fast-langdetect>=0.2.3,<0.3.0"
,
]
]
[project.optional-dependencies]
[project.optional-dependencies]
...
@@ -60,7 +62,6 @@ pipeline = [
...
@@ -60,7 +62,6 @@ pipeline = [
"torch>=2.2.2,!=2.5.0,!=2.5.1,<3"
,
"torch>=2.2.2,!=2.5.0,!=2.5.1,<3"
,
"torchvision"
,
"torchvision"
,
"transformers>=4.49.0,!=4.51.0,<5.0.0"
,
"transformers>=4.49.0,!=4.51.0,<5.0.0"
,
"fast-langdetect>=0.2.3,<0.3.0"
,
]
]
api
=
[
api
=
[
"fastapi"
,
"fastapi"
,
...
@@ -97,7 +98,6 @@ pipeline_old_linux = [
...
@@ -97,7 +98,6 @@ pipeline_old_linux = [
"torch>=2.2.2,!=2.5.0,!=2.5.1,<3"
,
"torch>=2.2.2,!=2.5.0,!=2.5.1,<3"
,
"torchvision"
,
"torchvision"
,
"transformers>=4.49.0,!=4.51.0,<5.0.0"
,
"transformers>=4.49.0,!=4.51.0,<5.0.0"
,
"fast-langdetect>=0.2.3,<0.3.0"
,
]
]
[project.urls]
[project.urls]
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment