Skip to content

GitLab

Commit 9c623a49 authored by zihanl's avatar zihanl
Browse files

update COMMANDs

parent 8c9c145d
Hide whitespace changes
Inline Side-by-side
Showing with 5 additions and 6 deletions
tasks/knwl_dialo/scripts/data_processing.sh
View file @ 9c623a49
......@@ -6,17 +6,16 @@
# WoI: https://parl.ai/projects/sea/
DIR=`pwd`
mkdir -p $DIR/tasks/knwl_dialo/data
# We provide the following script to process the raw data from Wizard of Wikipedia
python ${DIR}/tasks/knwl_dialo/preprocessing.py --func process_wow_dataset --input_file <PATH_OF_THE_INPUT_DATA> --output_file <PATH_OF_THE_OUTPUT_DATA>
python ${DIR}/tasks/knwl_dialo/preprocessing.py --func process_wow_dataset --raw_file <PATH_OF_THE_INPUT_DATA> --processed_file <PATH_OF_THE_OUTPUT_DATA> --knwl_ref_file <PATH_OF_THE_KNOWLEDGE_REFERENCE_OUTPUT_DATA> --resp_ref_file <PATH_OF_THE_RESPONSE_REFERENCE_OUTPUT_DATA>
# We provide the following script to process the raw data from Wizard of Internet
python ${DIR}/tasks/knwl_dialo/preprocessing.py --func process_woi_dataset --input_file <PATH_OF_THE_INPUT_DATA> --output_file <PATH_OF_THE_OUTPUT_DATA>
python ${DIR}/tasks/knwl_dialo/preprocessing.py --func process_woi_dataset --raw_file <PATH_OF_THE_INPUT_DATA> --processed_file <PATH_OF_THE_OUTPUT_DATA> --knwl_ref_file <PATH_OF_THE_KNOWLEDGE_REFERENCE_OUTPUT_DATA> --resp_ref_file <PATH_OF_THE_RESPONSE_REFERENCE_OUTPUT_DATA>
# Obtain the knowledge generation prompts
python ${DIR}/tasks/knwl_dialo/preprocessing.py --func get_knwl_gen_prompts --test_file <PATH_OF_THE_PROCESSED_TEST_DATA> --train_file <PATH_OF_THE_PROCESSED_TRAIN_DATA> --model_file <PATH_OF_THE_DPR_MODEL> --output_file <PATH_OF_THE_OUTPUT_FILE> --data_type <DATA_TYPE_OF_THE_INPUT_FILE>
python ${DIR}/tasks/knwl_dialo/preprocessing.py --func get_knwl_gen_prompts --test_file <PATH_OF_THE_PROCESSED_TEST_DATA> --train_file <PATH_OF_THE_PROCESSED_TRAIN_DATA> --model_file <PATH_OF_THE_DPR_MODEL> --processed_file <PATH_OF_THE_OUTPUT_FILE> --data_type <DATA_TYPE_OF_THE_INPUT_FILE>
# Obtain the response generation prompts
python ${DIR}/tasks/knwl_dialo/preprocessing.py --func get_resp_gen_prompts --train_file <PATH_OF_THE_PROCESSED_TRAIN_DATA> --output_file <PATH_OF_THE_OUTPUT_FILE>
python ${DIR}/tasks/knwl_dialo/preprocessing.py --func get_resp_gen_prompts --train_file <PATH_OF_THE_PROCESSED_TRAIN_DATA> --processed_file <PATH_OF_THE_OUTPUT_FILE>
tasks/knwl_dialo/scripts/prep_resp_gen.sh
View file @ 9c623a49
......@@ -3,4 +3,4 @@
# Preparing the input file for the response generation (second-stage prompting)
DIR=`pwd`
python ${DIR}/tasks/knwl_dialo/preprocessing.py --func prepare_input --test_file <PATH_OF_THE_PROCESSED_TEST_DATA> --knowledge_file <PATH_OF_THE_GENERATED_KNOWLEDGE_DATA> --output_file <PATH_OF_THE_OUTPUT_FILE>
python ${DIR}/tasks/knwl_dialo/preprocessing.py --func prepare_input --test_file <PATH_OF_THE_PROCESSED_TEST_DATA> --knowledge_gen_file <PATH_OF_THE_GENERATED_KNOWLEDGE_DATA> --processed_file <PATH_OF_THE_OUTPUT_FILE>
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment