Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
OpenDAS
Megatron-LM
Commits
3a2d1e30
Commit
3a2d1e30
authored
Dec 05, 2021
by
zihanl
Browse files
update processing.py
parent
3c3447f1
Changes
1
Show whitespace changes
Inline
Side-by-side
Showing
1 changed file
with
4 additions
and
1 deletion
+4
-1
tasks/knwl_dialo/preprocessing.py
tasks/knwl_dialo/preprocessing.py
+4
-1
No files found.
tasks/knwl_dialo/preprocessing.py
View file @
3a2d1e30
...
@@ -30,7 +30,8 @@ def get_args():
...
@@ -30,7 +30,8 @@ def get_args():
parser
.
add_argument
(
"--model_file"
,
type
=
str
,
default
=
None
,
parser
.
add_argument
(
"--model_file"
,
type
=
str
,
default
=
None
,
help
=
"path of the model file"
)
help
=
"path of the model file"
)
parser
.
add_argument
(
"--data_type"
,
type
=
str
,
default
=
None
,
parser
.
add_argument
(
"--data_type"
,
type
=
str
,
default
=
None
,
help
=
"data types (wow_seen, wow_unseen, or woi)"
)
help
=
"data types, choose one out of three types:
\
wow_seen, wow_unseen, and woi"
)
parser
.
add_argument
(
"--seed"
,
type
=
int
,
default
=
1234
,
parser
.
add_argument
(
"--seed"
,
type
=
int
,
default
=
1234
,
help
=
"random seed"
)
help
=
"random seed"
)
...
@@ -229,6 +230,8 @@ def process_woi_dataset(raw_file, processed_file, knwl_ref_file, resp_ref_file):
...
@@ -229,6 +230,8 @@ def process_woi_dataset(raw_file, processed_file, knwl_ref_file, resp_ref_file):
def
get_database
(
test_datapath
,
train_datapath
,
data_type
):
def
get_database
(
test_datapath
,
train_datapath
,
data_type
):
"""Get the database by topics"""
"""Get the database by topics"""
assert
data_type
in
[
"wow_seen"
,
"wow_unseen"
,
"woi"
],
\
"Please input a correct data type!!"
# get test data topic list
# get test data topic list
print
(
"> reading test data from %s"
%
test_datapath
)
print
(
"> reading test data from %s"
%
test_datapath
)
test_topics
=
{}
test_topics
=
{}
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment