v1.0

examples/MMPT/projects/task/test_crosstask_zs.yaml

+includes: projects/task/test.yaml
+dataset:
+  split: test
+  meta_processor: CrossTaskMetaProcessor
+  test_path: data/crosstask/crosstask_release/videos_val.csv
+  train_csv_path: data/crosstask/crosstask_release/videos.csv
+  val_path: data/crosstask/crosstask_release/videos_val.csv  # dummy
+  val_csv_path: data/crosstask/crosstask_release/videos_val.csv    
+  primary_path: data/crosstask/crosstask_release/tasks_primary.txt
+  related_path: data/crosstask/crosstask_release/tasks_related.txt
+  vfeat_dir: data/feat/feat_crosstask_s3d
+  annotation_path: data/crosstask/crosstask_release/annotations
+  n_train: 30
+  video_processor: CrossTaskVideoProcessor
+  text_processor: CrossTaskTextProcessor
+  aligner: CrossTaskAligner
+  num_iso_layer: 12
+  sliding_window: 16
+  sliding_window_size: 32
+model:
+  model_cls: MMFusionActionLocalization
+  mm_encoder_cls: MMBertForJoint
+eval:
+  save_path: runs/task/crosstask_zs/eval
+fairseq:
+  # read code and find what is the checkpoint arg.
+  dataset:
+    batch_size: 1
+  common_eval:
+    path: runs/task/checkpoint_best.pt  # load the best from how2 on ACL submission:  runs/task/checkpoint11.pt
+metric: CrossTaskMetric
+predictor: CrossTaskPredictor

examples/MMPT/projects/task/test_crosstask_zs_videoclip.yaml

+includes: projects/task/test_crosstask_zs.yaml
+model:
+  model_cls: MMFusionSeparateActionLocalization
+  mm_encoder_cls: 
+  video_encoder_cls: MMBertForEncoder
+  text_encoder_cls: BertModel  # dummy, not used.
+  num_hidden_video_layers: 6

examples/MMPT/projects/task/test_didemo_zs.yaml

+includes: projects/task/test.yaml
+dataset:
+  meta_processor: DiDeMoMetaProcessor
+  test_path: data/didemo/test_data.json
+  video_processor: VideoProcessor
+  vfeat_dir: data/feat/feat_didemo_s3d
+  text_processor: DiDeMoTextProcessor
+  aligner: DiDeMoAligner
+  num_iso_layer: 12
+model:
+  model_cls: MMFusionSeparate
+  mm_encoder_cls: 
+  video_encoder_cls: MMBertForEncoder
+  text_encoder_cls: BertModel
+  num_hidden_video_layers: 6
+eval:
+  save_path: runs/task/didemo_zs/eval
+fairseq:
+  # read code and find what is the checkpoint arg.
+  common_eval:
+    path: runs/task/checkpoint_best.pt
+metric: DiDeMoMetric
+predictor: DiDeMoPredictor

examples/MMPT/projects/task/test_vtt.yaml

+includes: projects/task/test.yaml
+dataset:
+  meta_processor: MSRVTTMetaProcessor
+  test_path: data/msrvtt/MSRVTT_JSFUSION_test.csv
+  video_processor: VideoProcessor
+  vfeat_dir: data/feat/feat_vtt_s3d
+  text_processor: MSRVTTTextProcessor
+  num_iso_layer: 12
+model:
+  model_cls: MMFusionJoint
+  mm_encoder_cls: MMBertForJoint
+eval:
+  save_path: runs/task/vtt/eval
+fairseq:
+  # read code and find what is the checkpoint arg.
+  common_eval:
+    path: runs/task/vtt/checkpoint_last.pt
+metric: RetrievalMetric
+predictor: RetrievalPredictor

examples/MMPT/projects/task/test_vtt_videoclip.yaml

+includes: projects/task/test_vtt.yaml
+model:
+  model_cls: MMFusionSeparate
+  mm_encoder_cls: 
+  video_encoder_cls: MMBertForEncoder
+  text_encoder_cls: BertModel
+  num_hidden_video_layers: 6
+

examples/MMPT/projects/task/test_vtt_zs.yaml

+includes: projects/task/test_vtt.yaml
+model:
+  model_cls: MMFusionSeparate
+  mm_encoder_cls: 
+  video_encoder_cls: MMBertForEncoder
+  text_encoder_cls: BertModel
+  num_hidden_video_layers: 6
+eval:
+  save_path: runs/task/vtt_zs/eval
+fairseq:
+  # read code and find what is the checkpoint arg.
+  common_eval:
+    path: runs/task/checkpoint_best.pt

examples/MMPT/projects/task/test_vttqa.yaml

+includes: projects/task/test.yaml
+dataset:
+  meta_processor: MSRVTTQAMetaProcessor
+  test_path: data/msrvtt-qa/MSR_MC_test.csv
+  video_processor: VideoProcessor
+  vfeat_dir: data/feat/feat_vtt_s3d
+  text_processor: MSRVTTQATextProcessor
+  aligner: MSRVTTQAAligner
+  num_iso_layer: 12
+model:
+  model_cls: MMFusionJoint
+  mm_encoder_cls: MMBertForJoint
+eval:
+  save_path: runs/task/vttqa/eval
+fairseq:
+  # read code and find what is the checkpoint arg.
+  common_eval:
+    path: runs/task/vttqa/checkpoint_last.pt
+metric: QAMetric
+predictor: QAPredictor

examples/MMPT/projects/task/test_vttqa_videoclip.yaml

+includes: projects/task/test_vttqa.yaml
+model:
+  model_cls: MMFusionSeparate
+  mm_encoder_cls: 
+  video_encoder_cls: MMBertForEncoder
+  text_encoder_cls: BertModel
+  num_hidden_video_layers: 6
+

examples/MMPT/projects/task/test_vttqa_zs.yaml

+includes: projects/task/test_vttqa.yaml
+model:
+  model_cls: MMFusionSeparate
+  mm_encoder_cls: 
+  video_encoder_cls: MMBertForEncoder
+  text_encoder_cls: BertModel
+  num_hidden_video_layers: 6
+eval:
+  save_path: runs/task/vttqa_zs/eval
+fairseq:
+  # read code and find what is the checkpoint arg.
+  common_eval:
+    path: runs/task/checkpoint_best.pt

examples/MMPT/projects/task/test_youcook.yaml

+includes: projects/task/test.yaml
+dataset:
+  meta_processor: YoucookMetaProcessor
+  test_path: data/youcook/youcook_val.pkl
+  trainval_annotation: data/youcook/youcookii_annotations_trainval.json
+  use_annotation_text: True
+  video_processor: YoucookVideoProcessor
+  vfeat_dir: data/feat/feat_youcook_s3d # /checkpoint/huxu/feat/youcook_vmz # /checkpoint/prarora/berniehuang/feat_youcook_vmz
+  text_processor: TextProcessor
+  aligner: DSAligner
+  num_iso_layer: 12
+model:
+  model_cls: MMFusionJoint
+  mm_encoder_cls: MMBertForJoint
+eval:
+  save_path: runs/task/youcook/eval
+fairseq:
+  # read code and find what is the checkpoint arg.
+  common_eval:
+    path: runs/task/youcook/checkpoint_last.pt
+metric: RetrievalMetric
+predictor: RetrievalPredictor

examples/MMPT/projects/task/test_youcook_videoclip.yaml

+includes: projects/task/test_youcook.yaml
+model:
+  model_cls: MMFusionSeparate
+  mm_encoder_cls: 
+  video_encoder_cls: MMBertForEncoder
+  text_encoder_cls: BertModel
+  num_hidden_video_layers: 6
+

examples/MMPT/projects/task/test_youcook_zs.yaml

+includes: projects/task/test_youcook.yaml
+model:
+  model_cls: MMFusionSeparate
+  mm_encoder_cls: 
+  video_encoder_cls: MMBertForEncoder
+  text_encoder_cls: BertModel
+  num_hidden_video_layers: 6
+eval:
+  save_path: runs/task/youcook_zs/eval
+fairseq:
+  # read code and find what is the checkpoint arg.
+  common_eval:
+    path: runs/task/checkpoint_best.pt

examples/MMPT/projects/task/test_youcookcap.yaml

+includes: projects/task/test.yaml
+dataset:
+  meta_processor: YoucookNLGMetaProcessor
+  test_path: data/youcook/val_list.txt
+  trainval_annotation: data/youcook/youcookii_annotations_trainval.json
+  video_processor: YoucookVideoProcessor
+  vfeat_dir: data/feat/feat_youcook_s3d
+  text_processor: NLGTextProcessor
+  aligner: DSNLGAligner
+model:
+  model_cls: MMFusionNLG
+  mm_encoder_cls: MMBertForNLG
+  max_decode_length: 24
+eval:
+  save_path: runs/task/youcookcap/eval
+fairseq:
+  # read code and find what is the checkpoint arg.
+  common_eval:
+    path: runs/task/youcookcap/checkpoint_best.pt
+metric: NLGMetric
+predictor: NLGPredictor
+gen_param:
+  num_beams: 5

examples/MMPT/projects/task/vtt.yaml

+includes: projects/task/ft.yaml
+dataset:
+  meta_processor: MSRVTTMetaProcessor
+  train_path: data/msrvtt/MSRVTT_train.csv
+  jsfusion_path: data/msrvtt/MSRVTT_JSFUSION_test.csv
+  full_test_path: data/msrvtt/MSRVTT_FULL_test.csv
+  dup: 20
+  val_path: data/msrvtt/MSRVTT_JSFUSION_test.csv
+  vfeat_dir: data/feat/feat_vtt_s3d
+  text_processor: MSRVTTTextProcessor
+  json_path: data/msrvtt/MSRVTT_data.json
+  aligner: DSAligner
+  num_iso_layer: 12
+model:
+  model_cls: MMFusionJoint
+  mm_encoder_cls: MMBertForJoint
+loss:
+  loss_cls: T2VContraLoss
+fairseq:
+  dataset:
+    batch_size: 256
+  optimization:
+    max_epoch: 10
+  checkpoint:
+    save_dir: runs/task/vtt

examples/MMPT/projects/task/vtt_videoclip.yaml

+includes: projects/task/vtt.yaml
+model:
+  model_cls: MMFusionSeparate
+  mm_encoder_cls: 
+  video_encoder_cls: MMBertForEncoder
+  text_encoder_cls: BertModel
+  num_hidden_video_layers: 6
+fairseq:
+  dataset:
+    batch_size: 224
+#   model_cls: MMFusionShare
+#   mm_encoder_cls: MMBertForEncoder

examples/MMPT/projects/task/vttqa.yaml

+includes: projects/task/ft.yaml
+dataset:
+  meta_processor: MSRVTTMetaProcessor
+  train_path: data/msrvtt/MSRVTT_train.csv
+  dup: 20
+  val_path: data/msrvtt/MSRVTT_JSFUSION_test.csv
+  vfeat_dir: data/feat/feat_vtt_s3d
+  text_processor: MSRVTTTextProcessor
+  json_path: data/msrvtt/MSRVTT_data.json
+  aligner: DSAligner
+  num_iso_layer: 12
+model:
+  model_cls: MMFusionJoint
+  mm_encoder_cls: MMBertForJoint
+loss:
+  loss_cls: V2TContraLoss
+fairseq:
+  dataset:
+    batch_size: 128
+  optimization:
+    max_epoch: 5
+  checkpoint:
+    save_dir: runs/task/vttqa

examples/MMPT/projects/task/vttqa_videoclip.yaml

+includes: projects/task/vttqa.yaml
+model:
+  model_cls: MMFusionSeparate
+  mm_encoder_cls: 
+  video_encoder_cls: MMBertForEncoder
+  text_encoder_cls: BertModel
+  num_hidden_video_layers: 6
+
+#   model_cls: MMFusionShare
+#   mm_encoder_cls: MMBertForEncoder

examples/MMPT/projects/task/youcook.yaml

+includes: projects/task/ft.yaml
+dataset:
+  meta_processor: YoucookMetaProcessor
+  train_path: data/youcook/youcook_train.pkl
+  val_path: data/youcook/youcook_val.pkl
+  trainval_annotation: data/youcook/youcookii_annotations_trainval.json
+  use_annotation_text: True
+  video_processor: YoucookVideoProcessor
+  vfeat_dir: data/feat/feat_youcook_s3d # /checkpoint/huxu/feat/youcook_vmz # /checkpoint/prarora/berniehuang/feat_youcook_vmz
+  text_processor: TextProcessor
+  aligner: DSAligner
+  num_iso_layer: 12
+model:
+  model_cls: MMFusionJoint
+  mm_encoder_cls: MMBertForJoint
+loss:
+  loss_cls: T2VContraLoss
+fairseq:
+  dataset:
+    batch_size: 128
+  optimization:
+    max_epoch: 10
+  checkpoint:
+    save_dir: runs/task/youcook
+ 

examples/MMPT/projects/task/youcook_videoclip.yaml

+includes: projects/task/youcook.yaml
+model:
+  model_cls: MMFusionSeparate
+  mm_encoder_cls: 
+  video_encoder_cls: MMBertForEncoder
+  text_encoder_cls: BertModel
+  num_hidden_video_layers: 6
+  # model_cls: MMFusionShare
+  # mm_encoder_cls: MMBertForEncoder

examples/MMPT/projects/task/youcookcap.yaml

+# finetuning for youcook captioning.
+includes: projects/task/ft.yaml
+dataset:
+  meta_processor: YoucookNLGMetaProcessor
+  train_path: data/youcook/train_list.txt
+  val_path: data/youcook/val_list.txt
+  trainval_annotation: data/youcook/youcookii_annotations_trainval.json
+  video_processor: YoucookVideoProcessor
+  vfeat_dir: data/feat/feat_youcook_s3d
+  text_processor: NLGTextProcessor
+  aligner: DSNLGAligner
+model:
+  model_cls: MMFusionNLG
+  mm_encoder_cls: MMBertForNLG
+loss:
+  loss_cls: NLGLoss
+fairseq:
+  dataset:
+    batch_size: 128
+  optimization:
+    max_epoch: 10
+  checkpoint:
+    save_dir: runs/task/youcookcap