Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
wangsen
MinerU
Commits
99055af3
"vscode:/vscode.git/clone" did not exist on "3501a4bdf95c57ed102bce30836caae3466166ff"
Commit
99055af3
authored
Mar 21, 2024
by
赵小蒙
Browse files
英文文本拼接时,如果单个单词超过15个字符,则对该单词进行切分处理。行间公式/图片/表格独立占有一行
parent
1d5d7781
Changes
1
Show whitespace changes
Inline
Side-by-side
Showing
1 changed file
with
19 additions
and
5 deletions
+19
-5
magic_pdf/dict2md/ocr_mkcontent.py
magic_pdf/dict2md/ocr_mkcontent.py
+19
-5
No files found.
magic_pdf/dict2md/ocr_mkcontent.py
View file @
99055af3
from
magic_pdf.libs.commons
import
s3_image_save_path
,
join_path
from
magic_pdf.libs.commons
import
s3_image_save_path
,
join_path
from
magic_pdf.libs.markdown_utils
import
ocr_escape_special_markdown_char
from
magic_pdf.libs.markdown_utils
import
ocr_escape_special_markdown_char
from
magic_pdf.libs.ocr_content_type
import
ContentType
from
magic_pdf.libs.ocr_content_type
import
ContentType
import
wordninja
import
re
def
split_long_words
(
text
):
segments
=
text
.
split
(
' '
)
for
i
in
range
(
len
(
segments
)):
words
=
re
.
findall
(
r
'\w+|[^\w\s]'
,
segments
[
i
],
re
.
UNICODE
)
for
j
in
range
(
len
(
words
)):
if
len
(
words
[
j
])
>
15
:
words
[
j
]
=
' '
.
join
(
wordninja
.
split
(
words
[
j
]))
segments
[
i
]
=
''
.
join
(
words
)
return
' '
.
join
(
segments
)
def
ocr_mk_nlp_markdown
(
pdf_info_dict
:
dict
):
def
ocr_mk_nlp_markdown
(
pdf_info_dict
:
dict
):
...
@@ -67,17 +80,18 @@ def ocr_mk_mm_markdown_with_para(pdf_info_dict: dict):
...
@@ -67,17 +80,18 @@ def ocr_mk_mm_markdown_with_para(pdf_info_dict: dict):
for
span
in
line
[
'spans'
]:
for
span
in
line
[
'spans'
]:
span_type
=
span
.
get
(
'type'
)
span_type
=
span
.
get
(
'type'
)
if
span_type
==
ContentType
.
Text
:
if
span_type
==
ContentType
.
Text
:
content
=
span
[
'content'
]
content
=
split_long_words
(
span
[
'content'
])
# content = span['content']
elif
span_type
==
ContentType
.
InlineEquation
:
elif
span_type
==
ContentType
.
InlineEquation
:
content
=
f
"
$
{
span
[
'content'
]
}
$
"
content
=
f
"$
{
span
[
'content'
]
}
$"
elif
span_type
==
ContentType
.
InterlineEquation
:
elif
span_type
==
ContentType
.
InterlineEquation
:
content
=
f
"$$
\n
{
span
[
'content'
]
}
\n
$$
"
content
=
f
"
\n
$$
\n
{
span
[
'content'
]
}
\n
$$
\n
"
elif
span_type
in
[
ContentType
.
Image
,
ContentType
.
Table
]:
elif
span_type
in
[
ContentType
.
Image
,
ContentType
.
Table
]:
content
=
f
"
}
)"
content
=
f
"
\n

}
)
\n
"
para_text
+=
content
+
' '
para_text
+=
content
+
' '
markdown
.
append
(
para_text
.
strip
()
+
' '
)
markdown
.
append
(
para_text
.
strip
()
+
' '
)
return
'
\n
'
.
join
(
markdown
)
return
'
\n
\n
'
.
join
(
markdown
)
def
make_standard_format_with_para
(
pdf_info_dict
:
dict
):
def
make_standard_format_with_para
(
pdf_info_dict
:
dict
):
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment