Skip to content

GitLab

Commit 23302cb2 authored by zihanl's avatar zihanl
Browse files

add comments

parent aaa7aa32
Show whitespace changes
Inline Side-by-side
Showing with 5 additions and 2 deletions
tasks/knwl_dialo/preprocessing.py
View file @ 23302cb2
...@@ -268,17 +268,20 @@ def get_database(test_datapath, train_datapath, data_type): ...@@ -268,17 +268,20 @@ def get_database(test_datapath, train_datapath, data_type):
turns = splits[1].split(" [SEP] ")[-3:] turns = splits[1].split(" [SEP] ")[-3:]
knowledge = splits[2] knowledge = splits[2]
response = splits[3] response = splits[3]
# filtering data samples # filtering data samples
if knowledge == "no_passages_used": if knowledge == "no_passages_used":
# when no knowledge is used
continue continue
if data_type != "wow_seen" and ("(" in knowledge or ")" in knowledge): if data_type != "wow_seen" and ("(" in knowledge or ")" in knowledge):
# when brackets existing in the knowledge
continue continue
if data_type != "wow_seen" and topic not in knowledge: if data_type != "wow_seen" and topic not in knowledge:
# when topic does not exist in the knowledge
continue continue
# get the instance # get the instance
last_turn = turns[-1] last_turn = turns[-1]
if data_type == "woi":
instance = "( " + last_turn + " ) " + topic + " => " + knowledge instance = "( " + last_turn + " ) " + topic + " => " + knowledge
# construct dialog example # construct dialog example
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment