Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
wangsen
MinerU
Commits
d637dab3
Commit
d637dab3
authored
Dec 27, 2024
by
icecraft
Browse files
fix: s3 path join method
parent
667b1a39
Changes
1
Show whitespace changes
Inline
Side-by-side
Showing
1 changed file
with
8 additions
and
6 deletions
+8
-6
magic_pdf/data/data_reader_writer/multi_bucket_s3.py
magic_pdf/data/data_reader_writer/multi_bucket_s3.py
+8
-6
No files found.
magic_pdf/data/data_reader_writer/multi_bucket_s3.py
View file @
d637dab3
import
os
from
magic_pdf.config.exceptions
import
InvalidConfig
,
InvalidParams
from
magic_pdf.config.exceptions
import
InvalidConfig
,
InvalidParams
from
magic_pdf.data.data_reader_writer.base
import
DataReader
,
DataWriter
from
magic_pdf.data.data_reader_writer.base
import
DataReader
,
DataWriter
from
magic_pdf.data.io.s3
import
S3Reader
,
S3Writer
from
magic_pdf.data.io.s3
import
S3Reader
,
S3Writer
...
@@ -23,9 +23,9 @@ class MultiS3Mixin:
...
@@ -23,9 +23,9 @@ class MultiS3Mixin:
if
len
(
default_prefix
)
==
0
:
if
len
(
default_prefix
)
==
0
:
raise
InvalidConfig
(
'default_prefix must be provided'
)
raise
InvalidConfig
(
'default_prefix must be provided'
)
arr
=
default_prefix
.
strip
(
"/"
).
split
(
"/"
)
arr
=
default_prefix
.
strip
(
'/'
).
split
(
'/'
)
self
.
default_bucket
=
arr
[
0
]
self
.
default_bucket
=
arr
[
0
]
self
.
default_prefix
=
"/"
.
join
(
arr
[
1
:])
self
.
default_prefix
=
'/'
.
join
(
arr
[
1
:])
found_default_bucket_config
=
False
found_default_bucket_config
=
False
for
conf
in
s3_configs
:
for
conf
in
s3_configs
:
...
@@ -103,7 +103,8 @@ class MultiBucketS3DataReader(DataReader, MultiS3Mixin):
...
@@ -103,7 +103,8 @@ class MultiBucketS3DataReader(DataReader, MultiS3Mixin):
s3_reader
=
self
.
__get_s3_client
(
bucket_name
)
s3_reader
=
self
.
__get_s3_client
(
bucket_name
)
else
:
else
:
s3_reader
=
self
.
__get_s3_client
(
self
.
default_bucket
)
s3_reader
=
self
.
__get_s3_client
(
self
.
default_bucket
)
path
=
os
.
path
.
join
(
self
.
default_prefix
,
path
)
if
self
.
default_prefix
:
path
=
self
.
default_prefix
+
'/'
+
path
return
s3_reader
.
read_at
(
path
,
offset
,
limit
)
return
s3_reader
.
read_at
(
path
,
offset
,
limit
)
...
@@ -139,5 +140,6 @@ class MultiBucketS3DataWriter(DataWriter, MultiS3Mixin):
...
@@ -139,5 +140,6 @@ class MultiBucketS3DataWriter(DataWriter, MultiS3Mixin):
s3_writer
=
self
.
__get_s3_client
(
bucket_name
)
s3_writer
=
self
.
__get_s3_client
(
bucket_name
)
else
:
else
:
s3_writer
=
self
.
__get_s3_client
(
self
.
default_bucket
)
s3_writer
=
self
.
__get_s3_client
(
self
.
default_bucket
)
path
=
os
.
path
.
join
(
self
.
default_prefix
,
path
)
if
self
.
default_prefix
:
path
=
self
.
default_prefix
+
'/'
+
path
return
s3_writer
.
write
(
path
,
data
)
return
s3_writer
.
write
(
path
,
data
)
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment