chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,25 @@
|
||||
includes: projects/task/ft.yaml
|
||||
task_type: sweep_big
|
||||
dataset:
|
||||
meta_processor: COINActionSegmentationMetaProcessor
|
||||
train_path: data/coin/COIN.json
|
||||
val_path: data/coin/COIN.json
|
||||
vfeat_dir: data/feat/feat_coin_s3d
|
||||
video_processor: VideoProcessor
|
||||
text_processor: COINActionSegmentationTextProcessor
|
||||
aligner: COINActionSegmentationAligner
|
||||
num_iso_layer: 12
|
||||
sliding_window: 8
|
||||
sliding_window_size: 32
|
||||
model:
|
||||
model_cls: MMFusionActionSegmentation
|
||||
mm_encoder_cls: MMBertForTokenClassification
|
||||
loss:
|
||||
loss_cls: CrossEntropy
|
||||
fairseq:
|
||||
dataset:
|
||||
batch_size: 1
|
||||
optimization:
|
||||
max_epoch: 8
|
||||
checkpoint:
|
||||
save_dir: runs/task/coin
|
||||
@@ -0,0 +1,7 @@
|
||||
includes: projects/task/coin.yaml
|
||||
model:
|
||||
model_cls: MMFusionSeparateActionSegmentation
|
||||
mm_encoder_cls:
|
||||
video_encoder_cls: MMBertForTokenClassification
|
||||
text_encoder_cls: BertModel # dummy, not used.
|
||||
num_hidden_video_layers: 6
|
||||
@@ -0,0 +1,31 @@
|
||||
includes: projects/task/ft.yaml
|
||||
dataset:
|
||||
meta_processor: CrossTaskMetaProcessor
|
||||
train_path: data/crosstask/crosstask_release/videos.csv # dummy
|
||||
train_csv_path: data/crosstask/crosstask_release/videos.csv
|
||||
val_path: data/crosstask/crosstask_release/videos_val.csv # dummy
|
||||
val_csv_path: data/crosstask/crosstask_release/videos_val.csv
|
||||
primary_path: data/crosstask/crosstask_release/tasks_primary.txt
|
||||
related_path: data/crosstask/crosstask_release/tasks_related.txt
|
||||
vfeat_dir: data/feat/feat_crosstask_s3d
|
||||
annotation_path: data/crosstask/crosstask_release/annotations
|
||||
n_train: 30
|
||||
video_processor: CrossTaskVideoProcessor
|
||||
text_processor: CrossTaskTextProcessor
|
||||
aligner: CrossTaskAligner
|
||||
num_iso_layer: 12
|
||||
sliding_window: 16
|
||||
sliding_window_size: 32
|
||||
model:
|
||||
model_cls: MMFusionActionLocalization
|
||||
mm_encoder_cls: MMBertForJoint
|
||||
loss:
|
||||
loss_cls: BCE
|
||||
fairseq:
|
||||
dataset:
|
||||
batch_size: 1
|
||||
optimization:
|
||||
max_epoch: 5
|
||||
checkpoint:
|
||||
save_dir: runs/task/crosstask
|
||||
restore_file: runs/task/checkpoint11.pt # for VLM
|
||||
@@ -0,0 +1,10 @@
|
||||
includes: projects/task/crosstask.yaml
|
||||
model:
|
||||
model_cls: MMFusionSeparateActionLocalization
|
||||
mm_encoder_cls:
|
||||
video_encoder_cls: MMBertForEncoder
|
||||
text_encoder_cls: BertModel # dummy, not used.
|
||||
num_hidden_video_layers: 6
|
||||
fairseq:
|
||||
checkpoint:
|
||||
restore_file: runs/task/checkpoint_best.pt # overwrite the default of VLM.
|
||||
@@ -0,0 +1,20 @@
|
||||
# this yaml cannot be run alone. you must use `how2.yaml`, `vtt.yaml` etc for training.
|
||||
dataset:
|
||||
video_processor: VideoProcessor
|
||||
bert_name: bert-base-uncased
|
||||
fairseq:
|
||||
common:
|
||||
tensorboard_logdir: run
|
||||
log_interval: 1000
|
||||
dataset:
|
||||
num_workers: 4
|
||||
optimization:
|
||||
lr: [ 0.00005 ]
|
||||
clip_norm: 2.0
|
||||
optimizer: adam
|
||||
adam_betas: (0.9, 0.98)
|
||||
lr_scheduler: polynomial_decay
|
||||
total_num_update: 1000000 # backward compatible on fairseq 1.0.0a0+af0389f for reproducibility.
|
||||
warmup_updates: 1000
|
||||
weight_decay: 0.0
|
||||
ddp_backend: no_c10d
|
||||
@@ -0,0 +1,13 @@
|
||||
includes: projects/task/default.yaml
|
||||
# all derived config will be run by fairseq-train.
|
||||
task_type: sweep_small
|
||||
fairseq:
|
||||
optimization:
|
||||
warmup_updates: 122 # copied from roberta glue: https://github.com/pytorch/fairseq/blob/master/examples/roberta/README.glue.md
|
||||
checkpoint:
|
||||
# save_interval_updates: 512
|
||||
# borrowed from Roberta script.
|
||||
restore_file: runs/task/checkpoint_best.pt
|
||||
reset_optimizer: True
|
||||
reset_dataloader: True
|
||||
reset_meters: True
|
||||
@@ -0,0 +1,22 @@
|
||||
includes: projects/task/default.yaml
|
||||
task_type: sweep_big
|
||||
slurm_config: big
|
||||
dataset:
|
||||
meta_processor: ShardedHow2MetaProcessor
|
||||
train_path: data/how2/how2_s3d_train.lst
|
||||
val_path: data/how2/how2_s3d_val.lst
|
||||
video_processor: ShardedVideoProcessor
|
||||
vfeat_dir: data/feat/feat_how2_s3d_shard_small
|
||||
text_processor: ShardedTextProcessor
|
||||
tfeat_dir: data/feat/feat_how2_s3d_shard_small/raw_caption_dedup.bert-base-uncased.
|
||||
aligner: FixedLenAligner
|
||||
# disable direct running of this yaml
|
||||
eval:
|
||||
save_path: runs/task
|
||||
fairseq:
|
||||
checkpoint:
|
||||
save_dir: runs/task
|
||||
save_interval_updates: 1024
|
||||
keep_interval_updates: 2
|
||||
keep_last_epochs: 30
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
# this yaml cannot be run alone: implement a test_${dataset}.yaml
|
||||
slurm_config: big
|
||||
task_type: local_predict
|
||||
dataset:
|
||||
split: test
|
||||
video_processor: VideoProcessor
|
||||
aligner: DSAligner
|
||||
bert_name: bert-base-uncased
|
||||
fairseq:
|
||||
dataset:
|
||||
batch_size: 256
|
||||
valid_subset: test
|
||||
num_workers: 2
|
||||
@@ -0,0 +1,24 @@
|
||||
includes: projects/task/test.yaml
|
||||
dataset:
|
||||
split: test
|
||||
test_path: data/coin/COIN.json
|
||||
meta_processor: COINActionSegmentationMetaProcessor
|
||||
vfeat_dir: data/feat/feat_coin_s3d
|
||||
video_processor: VideoProcessor
|
||||
text_processor: COINActionSegmentationTextProcessor
|
||||
aligner: COINActionSegmentationAligner
|
||||
num_iso_layer: 12
|
||||
sliding_window: 16
|
||||
sliding_window_size: 32
|
||||
model:
|
||||
model_cls: MMFusionActionSegmentation
|
||||
mm_encoder_cls: MMBertForTokenClassification
|
||||
eval:
|
||||
save_path: runs/task/coin/eval
|
||||
fairseq:
|
||||
dataset:
|
||||
batch_size: 1
|
||||
common_eval:
|
||||
path: runs/task/coin/checkpoint_best.pt
|
||||
metric: COINActionSegmentationMetric
|
||||
predictor: COINPredictor
|
||||
@@ -0,0 +1,7 @@
|
||||
includes: projects/task/test_coin.yaml
|
||||
model:
|
||||
model_cls: MMFusionSeparateActionSegmentation
|
||||
mm_encoder_cls:
|
||||
video_encoder_cls: MMBertForTokenClassification
|
||||
text_encoder_cls: BertModel # dummy, not used.
|
||||
num_hidden_video_layers: 6
|
||||
@@ -0,0 +1,13 @@
|
||||
includes: projects/task/test_coin.yaml
|
||||
model:
|
||||
model_cls: MMFusionSeparate
|
||||
mm_encoder_cls:
|
||||
video_encoder_cls: MMBertForEncoder
|
||||
text_encoder_cls: BertModel
|
||||
num_hidden_video_layers: 6
|
||||
eval:
|
||||
save_path: runs/task/coin_zs/eval
|
||||
fairseq:
|
||||
common_eval:
|
||||
path: runs/task/checkpoint_best.pt
|
||||
predictor: COINZSPredictor
|
||||
@@ -0,0 +1,32 @@
|
||||
includes: projects/task/test.yaml
|
||||
dataset:
|
||||
split: test
|
||||
meta_processor: CrossTaskMetaProcessor
|
||||
test_path: data/crosstask/crosstask_release/videos_val.csv
|
||||
train_csv_path: data/crosstask/crosstask_release/videos.csv
|
||||
val_path: data/crosstask/crosstask_release/videos_val.csv # dummy
|
||||
val_csv_path: data/crosstask/crosstask_release/videos_val.csv
|
||||
primary_path: data/crosstask/crosstask_release/tasks_primary.txt
|
||||
related_path: data/crosstask/crosstask_release/tasks_related.txt
|
||||
vfeat_dir: data/feat/feat_crosstask_s3d
|
||||
annotation_path: data/crosstask/crosstask_release/annotations
|
||||
n_train: 30
|
||||
video_processor: CrossTaskVideoProcessor
|
||||
text_processor: CrossTaskTextProcessor
|
||||
aligner: CrossTaskAligner
|
||||
num_iso_layer: 12
|
||||
sliding_window: 16
|
||||
sliding_window_size: 32
|
||||
model:
|
||||
model_cls: MMFusionActionLocalization
|
||||
mm_encoder_cls: MMBertForJoint
|
||||
eval:
|
||||
save_path: runs/task/crosstask/eval
|
||||
fairseq:
|
||||
# read code and find what is the checkpoint arg.
|
||||
dataset:
|
||||
batch_size: 1
|
||||
common_eval:
|
||||
path: runs/task/crosstask/checkpoint_best.pt
|
||||
metric: CrossTaskMetric
|
||||
predictor: CrossTaskPredictor
|
||||
@@ -0,0 +1,7 @@
|
||||
includes: projects/task/test_crosstask.yaml
|
||||
model:
|
||||
model_cls: MMFusionSeparateActionLocalization
|
||||
mm_encoder_cls:
|
||||
video_encoder_cls: MMBertForEncoder
|
||||
text_encoder_cls: BertModel # dummy, not used.
|
||||
num_hidden_video_layers: 6
|
||||
@@ -0,0 +1,32 @@
|
||||
includes: projects/task/test.yaml
|
||||
dataset:
|
||||
split: test
|
||||
meta_processor: CrossTaskMetaProcessor
|
||||
test_path: data/crosstask/crosstask_release/videos_val.csv
|
||||
train_csv_path: data/crosstask/crosstask_release/videos.csv
|
||||
val_path: data/crosstask/crosstask_release/videos_val.csv # dummy
|
||||
val_csv_path: data/crosstask/crosstask_release/videos_val.csv
|
||||
primary_path: data/crosstask/crosstask_release/tasks_primary.txt
|
||||
related_path: data/crosstask/crosstask_release/tasks_related.txt
|
||||
vfeat_dir: data/feat/feat_crosstask_s3d
|
||||
annotation_path: data/crosstask/crosstask_release/annotations
|
||||
n_train: 30
|
||||
video_processor: CrossTaskVideoProcessor
|
||||
text_processor: CrossTaskTextProcessor
|
||||
aligner: CrossTaskAligner
|
||||
num_iso_layer: 12
|
||||
sliding_window: 16
|
||||
sliding_window_size: 32
|
||||
model:
|
||||
model_cls: MMFusionActionLocalization
|
||||
mm_encoder_cls: MMBertForJoint
|
||||
eval:
|
||||
save_path: runs/task/crosstask_zs/eval
|
||||
fairseq:
|
||||
# read code and find what is the checkpoint arg.
|
||||
dataset:
|
||||
batch_size: 1
|
||||
common_eval:
|
||||
path: runs/task/checkpoint_best.pt # load the best from how2 on ACL submission: runs/task/checkpoint11.pt
|
||||
metric: CrossTaskMetric
|
||||
predictor: CrossTaskPredictor
|
||||
@@ -0,0 +1,7 @@
|
||||
includes: projects/task/test_crosstask_zs.yaml
|
||||
model:
|
||||
model_cls: MMFusionSeparateActionLocalization
|
||||
mm_encoder_cls:
|
||||
video_encoder_cls: MMBertForEncoder
|
||||
text_encoder_cls: BertModel # dummy, not used.
|
||||
num_hidden_video_layers: 6
|
||||
@@ -0,0 +1,23 @@
|
||||
includes: projects/task/test.yaml
|
||||
dataset:
|
||||
meta_processor: DiDeMoMetaProcessor
|
||||
test_path: data/didemo/test_data.json
|
||||
video_processor: VideoProcessor
|
||||
vfeat_dir: data/feat/feat_didemo_s3d
|
||||
text_processor: DiDeMoTextProcessor
|
||||
aligner: DiDeMoAligner
|
||||
num_iso_layer: 12
|
||||
model:
|
||||
model_cls: MMFusionSeparate
|
||||
mm_encoder_cls:
|
||||
video_encoder_cls: MMBertForEncoder
|
||||
text_encoder_cls: BertModel
|
||||
num_hidden_video_layers: 6
|
||||
eval:
|
||||
save_path: runs/task/didemo_zs/eval
|
||||
fairseq:
|
||||
# read code and find what is the checkpoint arg.
|
||||
common_eval:
|
||||
path: runs/task/checkpoint_best.pt
|
||||
metric: DiDeMoMetric
|
||||
predictor: DiDeMoPredictor
|
||||
@@ -0,0 +1,19 @@
|
||||
includes: projects/task/test.yaml
|
||||
dataset:
|
||||
meta_processor: MSRVTTMetaProcessor
|
||||
test_path: data/msrvtt/MSRVTT_JSFUSION_test.csv
|
||||
video_processor: VideoProcessor
|
||||
vfeat_dir: data/feat/feat_vtt_s3d
|
||||
text_processor: MSRVTTTextProcessor
|
||||
num_iso_layer: 12
|
||||
model:
|
||||
model_cls: MMFusionJoint
|
||||
mm_encoder_cls: MMBertForJoint
|
||||
eval:
|
||||
save_path: runs/task/vtt/eval
|
||||
fairseq:
|
||||
# read code and find what is the checkpoint arg.
|
||||
common_eval:
|
||||
path: runs/task/vtt/checkpoint_last.pt
|
||||
metric: RetrievalMetric
|
||||
predictor: RetrievalPredictor
|
||||
@@ -0,0 +1,8 @@
|
||||
includes: projects/task/test_vtt.yaml
|
||||
model:
|
||||
model_cls: MMFusionSeparate
|
||||
mm_encoder_cls:
|
||||
video_encoder_cls: MMBertForEncoder
|
||||
text_encoder_cls: BertModel
|
||||
num_hidden_video_layers: 6
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
includes: projects/task/test_vtt.yaml
|
||||
model:
|
||||
model_cls: MMFusionSeparate
|
||||
mm_encoder_cls:
|
||||
video_encoder_cls: MMBertForEncoder
|
||||
text_encoder_cls: BertModel
|
||||
num_hidden_video_layers: 6
|
||||
eval:
|
||||
save_path: runs/task/vtt_zs/eval
|
||||
fairseq:
|
||||
# read code and find what is the checkpoint arg.
|
||||
common_eval:
|
||||
path: runs/task/checkpoint_best.pt
|
||||
@@ -0,0 +1,20 @@
|
||||
includes: projects/task/test.yaml
|
||||
dataset:
|
||||
meta_processor: MSRVTTQAMetaProcessor
|
||||
test_path: data/msrvtt-qa/MSR_MC_test.csv
|
||||
video_processor: VideoProcessor
|
||||
vfeat_dir: data/feat/feat_vtt_s3d
|
||||
text_processor: MSRVTTQATextProcessor
|
||||
aligner: MSRVTTQAAligner
|
||||
num_iso_layer: 12
|
||||
model:
|
||||
model_cls: MMFusionJoint
|
||||
mm_encoder_cls: MMBertForJoint
|
||||
eval:
|
||||
save_path: runs/task/vttqa/eval
|
||||
fairseq:
|
||||
# read code and find what is the checkpoint arg.
|
||||
common_eval:
|
||||
path: runs/task/vttqa/checkpoint_last.pt
|
||||
metric: QAMetric
|
||||
predictor: QAPredictor
|
||||
@@ -0,0 +1,8 @@
|
||||
includes: projects/task/test_vttqa.yaml
|
||||
model:
|
||||
model_cls: MMFusionSeparate
|
||||
mm_encoder_cls:
|
||||
video_encoder_cls: MMBertForEncoder
|
||||
text_encoder_cls: BertModel
|
||||
num_hidden_video_layers: 6
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
includes: projects/task/test_vttqa.yaml
|
||||
model:
|
||||
model_cls: MMFusionSeparate
|
||||
mm_encoder_cls:
|
||||
video_encoder_cls: MMBertForEncoder
|
||||
text_encoder_cls: BertModel
|
||||
num_hidden_video_layers: 6
|
||||
eval:
|
||||
save_path: runs/task/vttqa_zs/eval
|
||||
fairseq:
|
||||
# read code and find what is the checkpoint arg.
|
||||
common_eval:
|
||||
path: runs/task/checkpoint_best.pt
|
||||
@@ -0,0 +1,22 @@
|
||||
includes: projects/task/test.yaml
|
||||
dataset:
|
||||
meta_processor: YoucookMetaProcessor
|
||||
test_path: data/youcook/youcook_val.pkl
|
||||
trainval_annotation: data/youcook/youcookii_annotations_trainval.json
|
||||
use_annotation_text: True
|
||||
video_processor: YoucookVideoProcessor
|
||||
vfeat_dir: data/feat/feat_youcook_s3d # /checkpoint/huxu/feat/youcook_vmz # /checkpoint/prarora/berniehuang/feat_youcook_vmz
|
||||
text_processor: TextProcessor
|
||||
aligner: DSAligner
|
||||
num_iso_layer: 12
|
||||
model:
|
||||
model_cls: MMFusionJoint
|
||||
mm_encoder_cls: MMBertForJoint
|
||||
eval:
|
||||
save_path: runs/task/youcook/eval
|
||||
fairseq:
|
||||
# read code and find what is the checkpoint arg.
|
||||
common_eval:
|
||||
path: runs/task/youcook/checkpoint_last.pt
|
||||
metric: RetrievalMetric
|
||||
predictor: RetrievalPredictor
|
||||
@@ -0,0 +1,8 @@
|
||||
includes: projects/task/test_youcook.yaml
|
||||
model:
|
||||
model_cls: MMFusionSeparate
|
||||
mm_encoder_cls:
|
||||
video_encoder_cls: MMBertForEncoder
|
||||
text_encoder_cls: BertModel
|
||||
num_hidden_video_layers: 6
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
includes: projects/task/test_youcook.yaml
|
||||
model:
|
||||
model_cls: MMFusionSeparate
|
||||
mm_encoder_cls:
|
||||
video_encoder_cls: MMBertForEncoder
|
||||
text_encoder_cls: BertModel
|
||||
num_hidden_video_layers: 6
|
||||
eval:
|
||||
save_path: runs/task/youcook_zs/eval
|
||||
fairseq:
|
||||
# read code and find what is the checkpoint arg.
|
||||
common_eval:
|
||||
path: runs/task/checkpoint_best.pt
|
||||
@@ -0,0 +1,23 @@
|
||||
includes: projects/task/test.yaml
|
||||
dataset:
|
||||
meta_processor: YoucookNLGMetaProcessor
|
||||
test_path: data/youcook/val_list.txt
|
||||
trainval_annotation: data/youcook/youcookii_annotations_trainval.json
|
||||
video_processor: YoucookVideoProcessor
|
||||
vfeat_dir: data/feat/feat_youcook_s3d
|
||||
text_processor: NLGTextProcessor
|
||||
aligner: DSNLGAligner
|
||||
model:
|
||||
model_cls: MMFusionNLG
|
||||
mm_encoder_cls: MMBertForNLG
|
||||
max_decode_length: 24
|
||||
eval:
|
||||
save_path: runs/task/youcookcap/eval
|
||||
fairseq:
|
||||
# read code and find what is the checkpoint arg.
|
||||
common_eval:
|
||||
path: runs/task/youcookcap/checkpoint_best.pt
|
||||
metric: NLGMetric
|
||||
predictor: NLGPredictor
|
||||
gen_param:
|
||||
num_beams: 5
|
||||
@@ -0,0 +1,25 @@
|
||||
includes: projects/task/ft.yaml
|
||||
dataset:
|
||||
meta_processor: MSRVTTMetaProcessor
|
||||
train_path: data/msrvtt/MSRVTT_train.csv
|
||||
jsfusion_path: data/msrvtt/MSRVTT_JSFUSION_test.csv
|
||||
full_test_path: data/msrvtt/MSRVTT_FULL_test.csv
|
||||
dup: 20
|
||||
val_path: data/msrvtt/MSRVTT_JSFUSION_test.csv
|
||||
vfeat_dir: data/feat/feat_vtt_s3d
|
||||
text_processor: MSRVTTTextProcessor
|
||||
json_path: data/msrvtt/MSRVTT_data.json
|
||||
aligner: DSAligner
|
||||
num_iso_layer: 12
|
||||
model:
|
||||
model_cls: MMFusionJoint
|
||||
mm_encoder_cls: MMBertForJoint
|
||||
loss:
|
||||
loss_cls: T2VContraLoss
|
||||
fairseq:
|
||||
dataset:
|
||||
batch_size: 256
|
||||
optimization:
|
||||
max_epoch: 10
|
||||
checkpoint:
|
||||
save_dir: runs/task/vtt
|
||||
@@ -0,0 +1,12 @@
|
||||
includes: projects/task/vtt.yaml
|
||||
model:
|
||||
model_cls: MMFusionSeparate
|
||||
mm_encoder_cls:
|
||||
video_encoder_cls: MMBertForEncoder
|
||||
text_encoder_cls: BertModel
|
||||
num_hidden_video_layers: 6
|
||||
fairseq:
|
||||
dataset:
|
||||
batch_size: 224
|
||||
# model_cls: MMFusionShare
|
||||
# mm_encoder_cls: MMBertForEncoder
|
||||
@@ -0,0 +1,23 @@
|
||||
includes: projects/task/ft.yaml
|
||||
dataset:
|
||||
meta_processor: MSRVTTMetaProcessor
|
||||
train_path: data/msrvtt/MSRVTT_train.csv
|
||||
dup: 20
|
||||
val_path: data/msrvtt/MSRVTT_JSFUSION_test.csv
|
||||
vfeat_dir: data/feat/feat_vtt_s3d
|
||||
text_processor: MSRVTTTextProcessor
|
||||
json_path: data/msrvtt/MSRVTT_data.json
|
||||
aligner: DSAligner
|
||||
num_iso_layer: 12
|
||||
model:
|
||||
model_cls: MMFusionJoint
|
||||
mm_encoder_cls: MMBertForJoint
|
||||
loss:
|
||||
loss_cls: V2TContraLoss
|
||||
fairseq:
|
||||
dataset:
|
||||
batch_size: 128
|
||||
optimization:
|
||||
max_epoch: 5
|
||||
checkpoint:
|
||||
save_dir: runs/task/vttqa
|
||||
@@ -0,0 +1,10 @@
|
||||
includes: projects/task/vttqa.yaml
|
||||
model:
|
||||
model_cls: MMFusionSeparate
|
||||
mm_encoder_cls:
|
||||
video_encoder_cls: MMBertForEncoder
|
||||
text_encoder_cls: BertModel
|
||||
num_hidden_video_layers: 6
|
||||
|
||||
# model_cls: MMFusionShare
|
||||
# mm_encoder_cls: MMBertForEncoder
|
||||
@@ -0,0 +1,25 @@
|
||||
includes: projects/task/ft.yaml
|
||||
dataset:
|
||||
meta_processor: YoucookMetaProcessor
|
||||
train_path: data/youcook/youcook_train.pkl
|
||||
val_path: data/youcook/youcook_val.pkl
|
||||
trainval_annotation: data/youcook/youcookii_annotations_trainval.json
|
||||
use_annotation_text: True
|
||||
video_processor: YoucookVideoProcessor
|
||||
vfeat_dir: data/feat/feat_youcook_s3d # /checkpoint/huxu/feat/youcook_vmz # /checkpoint/prarora/berniehuang/feat_youcook_vmz
|
||||
text_processor: TextProcessor
|
||||
aligner: DSAligner
|
||||
num_iso_layer: 12
|
||||
model:
|
||||
model_cls: MMFusionJoint
|
||||
mm_encoder_cls: MMBertForJoint
|
||||
loss:
|
||||
loss_cls: T2VContraLoss
|
||||
fairseq:
|
||||
dataset:
|
||||
batch_size: 128
|
||||
optimization:
|
||||
max_epoch: 10
|
||||
checkpoint:
|
||||
save_dir: runs/task/youcook
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
includes: projects/task/youcook.yaml
|
||||
model:
|
||||
model_cls: MMFusionSeparate
|
||||
mm_encoder_cls:
|
||||
video_encoder_cls: MMBertForEncoder
|
||||
text_encoder_cls: BertModel
|
||||
num_hidden_video_layers: 6
|
||||
# model_cls: MMFusionShare
|
||||
# mm_encoder_cls: MMBertForEncoder
|
||||
@@ -0,0 +1,23 @@
|
||||
# finetuning for youcook captioning.
|
||||
includes: projects/task/ft.yaml
|
||||
dataset:
|
||||
meta_processor: YoucookNLGMetaProcessor
|
||||
train_path: data/youcook/train_list.txt
|
||||
val_path: data/youcook/val_list.txt
|
||||
trainval_annotation: data/youcook/youcookii_annotations_trainval.json
|
||||
video_processor: YoucookVideoProcessor
|
||||
vfeat_dir: data/feat/feat_youcook_s3d
|
||||
text_processor: NLGTextProcessor
|
||||
aligner: DSNLGAligner
|
||||
model:
|
||||
model_cls: MMFusionNLG
|
||||
mm_encoder_cls: MMBertForNLG
|
||||
loss:
|
||||
loss_cls: NLGLoss
|
||||
fairseq:
|
||||
dataset:
|
||||
batch_size: 128
|
||||
optimization:
|
||||
max_epoch: 10
|
||||
checkpoint:
|
||||
save_dir: runs/task/youcookcap
|
||||
Reference in New Issue
Block a user