chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,211 @@
|
||||
defaults:
|
||||
- hydra: default
|
||||
# - deepspeed@ds_cfg: train_hybrid_engine_zero3
|
||||
- deepspeed@ds_cfg: train_hybrid_engine_zero1
|
||||
- _self_ # see here for more details: https://hydra.cc/docs/tutorials/basic/your_first_app/defaults/#composition-order-of-primary-config
|
||||
|
||||
hydra:
|
||||
searchpath:
|
||||
- file://conf/
|
||||
|
||||
data_path_prefix: ""
|
||||
model_path_prefix: /mnt/fangkai_blob/share/models
|
||||
output_path_prefix: /mnt/fangkai_blob/reward_modeling/
|
||||
|
||||
train_file: "hf:codeparrot/apps"
|
||||
dev_file:
|
||||
test_file:
|
||||
|
||||
torch_dtype:
|
||||
_target_: general_util.training_utils.return_torch_dtype
|
||||
dtype: float16
|
||||
|
||||
tokenizer_init:
|
||||
_target_: general_util.tokenization_utils.init_tokenizer
|
||||
tokenizer_path: ${model_name_or_path}
|
||||
padding_side: left
|
||||
|
||||
device_map:
|
||||
_target_: models.utils.return_single_device_map
|
||||
|
||||
model:
|
||||
_target_: models.llama.LlamaForCausalLMDPO.from_pretrained_with_ref_model
|
||||
beta: 0.1
|
||||
gradient_checkpointing: True
|
||||
attn_implementation: "flash_attention_2"
|
||||
torch_dtype: ${torch_dtype}
|
||||
pad_token_id: 100001
|
||||
# device_map: ${device_map}
|
||||
ref_model:
|
||||
_target_: models.llama.LlamaForCausalLMDPO.from_pretrained
|
||||
pretrained_model_name_or_path: ${model_name_or_path}
|
||||
torch_dtype: ${torch_dtype}
|
||||
attn_implementation: "flash_attention_2"
|
||||
device_map: ${device_map}
|
||||
pad_token_id: 100001
|
||||
|
||||
|
||||
chat_prefix: "<｜begin▁of▁sentence｜>You are an AI programming assistant, utilizing the Deepseek Coder model, developed by Deepseek Company, and you only answer questions related to computer science. For politically sensitive questions, security and privacy issues, and other non-computer science questions, you will refuse to answer\n### Instruction:\n"
|
||||
chat_connect: "\n### Response:\n"
|
||||
chat_suffix: "\n<|EOT|>"
|
||||
prompt: "{question}\n\nPlease write a program to solve the above problem under the given time constraints and memory limits."
|
||||
|
||||
read_tensor:
|
||||
_target_: data.combine_dataset.MultiMappingDataset
|
||||
read_fn:
|
||||
_target_: data.apps.APPsReader
|
||||
split: train
|
||||
train_sub_split: train
|
||||
aligner:
|
||||
_target_: data.input_aligner.concat_aligner
|
||||
aligners:
|
||||
- _target_: data.input_aligner.field_extract_aligner
|
||||
input_index_field: problem_id
|
||||
extract_index_field: id
|
||||
extract_fields: [ "pos", "neg" ]
|
||||
extra_file: ${model_path_prefix}/deepseek-coder-7b-instruct-v1.5/apps/train.0shot.tem1.0.n10.v1.0.pseudo_test_case.exec.sc.dpo_v1.0.json
|
||||
- _target_: data.input_aligner.dpo_pair_aligner
|
||||
pos_field: pos
|
||||
neg_field: neg
|
||||
template:
|
||||
_target_: data.input_utils.recompose_template
|
||||
units:
|
||||
chat_prefix: ${chat_prefix}
|
||||
prompt: ${prompt}
|
||||
chat_connect: ${chat_connect}
|
||||
pos: "{pos}"
|
||||
neg: "{neg}"
|
||||
chat_suffix: ${chat_suffix}
|
||||
compositions:
|
||||
prompt: "{chat_prefix}{prompt}{chat_connect}"
|
||||
chosen: "{chat_prefix}{prompt}{chat_connect}{pos}{chat_suffix}"
|
||||
reject: "{chat_prefix}{prompt}{chat_connect}{neg}{chat_suffix}"
|
||||
index_field: problem_id
|
||||
kv_mapping:
|
||||
chosen: chosen
|
||||
reject: reject
|
||||
problem_id: index
|
||||
prompt: prompt
|
||||
|
||||
|
||||
dist_load_data_barrier: False
|
||||
|
||||
extended_vocab:
|
||||
|
||||
# Data collator
|
||||
collator:
|
||||
_target_: data.general_collator.DPOCollator
|
||||
tokenizer: ${tokenizer_init}
|
||||
max_seq_length: 3072
|
||||
|
||||
# Dataloader
|
||||
num_workers: 8
|
||||
prefetch_factor: 2
|
||||
|
||||
# Wiki path pretrain v8.2
|
||||
model_name_or_path: ${model_path_prefix}/deepseek-coder-7b-instruct-v1.5
|
||||
pretrain:
|
||||
resume:
|
||||
|
||||
dp_size:
|
||||
tp_size: 1
|
||||
pp_size: 1
|
||||
|
||||
|
||||
exp_name: deepseek-coder-v1.5-ins.7b.apps.code_gen.dpo.pseudo.A100.w8.v1.0.s${seed}
|
||||
exp_notes:
|
||||
output_dir: ${output_path_prefix}experiments/${exp_name} # Fix <pad token id>
|
||||
|
||||
|
||||
do_train: True
|
||||
evaluate_during_training: False
|
||||
|
||||
do_eval: False
|
||||
eval_sub_path: checkpoint-*
|
||||
|
||||
# Training hyper-parameters
|
||||
per_gpu_train_batch_size: 1
|
||||
per_gpu_eval_batch_size: 1
|
||||
#learning_rate: 1e-4
|
||||
learning_rate: 5e-6
|
||||
#learning_rate: 2e-5
|
||||
gradient_accumulation_steps: 16
|
||||
weight_decay: 0.1
|
||||
adam_epsilon: 1e-6
|
||||
adam_betas: "(0.9, 0.98)"
|
||||
#adam_betas: "(0.9, 0.999)"
|
||||
#max_grad_norm: 0.0
|
||||
total_dataset_len: -1
|
||||
max_grad_norm: 1.0
|
||||
num_train_epochs: 2
|
||||
max_steps: 500
|
||||
warmup_proportion: 0.1
|
||||
warmup_steps: 0
|
||||
|
||||
# Optimizer
|
||||
optimizer:
|
||||
use_nvlamb:
|
||||
bit_training:
|
||||
|
||||
logging_steps: 5
|
||||
save_ds_state: True
|
||||
save_steps: 100
|
||||
save_best: False
|
||||
eval_steps: 100
|
||||
ddp_eval: True
|
||||
no_cuda: False
|
||||
seed: 42
|
||||
local_rank: -1
|
||||
fp16: True
|
||||
fp16_opt_level: O1
|
||||
fp16_bfloat16: True
|
||||
|
||||
# Prediction config
|
||||
prediction_cfg:
|
||||
metric: "loss"
|
||||
measure: -1
|
||||
best_checkpoint:
|
||||
best_result:
|
||||
eval_forward_fn:
|
||||
_target_: general_util.evaluator.DefaultForwardFn
|
||||
post_process:
|
||||
_target_: post_processors.dpo.DPOEvalPostProcessor
|
||||
|
||||
|
||||
ds_cfg:
|
||||
train_micro_batch_size_per_gpu: ${per_gpu_train_batch_size}
|
||||
gradient_accumulation_steps: ${gradient_accumulation_steps}
|
||||
optimizer:
|
||||
type: AdamW
|
||||
params:
|
||||
lr: ${learning_rate}
|
||||
betas: [ 0.9, 0.95 ]
|
||||
weight_decay: ${weight_decay}
|
||||
# bf16:
|
||||
# enabled: False
|
||||
# fp16:
|
||||
# enabled: True
|
||||
# auto_cast: False
|
||||
# loss_scale: 0
|
||||
# initial_scale_power: 16
|
||||
# loss_scale_window: 1000
|
||||
# hysteresis: 2
|
||||
# consecutive_hysteresis: False
|
||||
# min_loss_scale: 1
|
||||
# zero_optimization:
|
||||
# offload_optimizer:
|
||||
# device: cpu
|
||||
# pin_memory: True
|
||||
|
||||
|
||||
summary_helper:
|
||||
_target_: general_util.tensorboard_helper.WandbWriter
|
||||
batch_index_or_keys:
|
||||
outputs_index_or_keys:
|
||||
|
||||
# Temporary variables
|
||||
n_gpu:
|
||||
device:
|
||||
train_batch_size:
|
||||
eval_batch_size:
|
||||
world_size:
|
||||
@@ -0,0 +1,213 @@
|
||||
defaults:
|
||||
- hydra: default
|
||||
- deepspeed@ds_cfg: train_hybrid_engine_zero3_optim_offload
|
||||
# - deepspeed@ds_cfg: train_hybrid_engine_zero1_optim_offload
|
||||
- _self_ # see here for more details: https://hydra.cc/docs/tutorials/basic/your_first_app/defaults/#composition-order-of-primary-config
|
||||
|
||||
hydra:
|
||||
searchpath:
|
||||
- file://conf/
|
||||
|
||||
data_path_prefix: ""
|
||||
model_path_prefix: ../pretrained-models/ # /mnt/fangkai_blob/share/models
|
||||
output_path_prefix: "" # /mnt/fangkai_blob/reward_modeling/
|
||||
|
||||
train_file: "hf:codeparrot/apps"
|
||||
dev_file:
|
||||
test_file:
|
||||
|
||||
torch_dtype:
|
||||
_target_: general_util.training_utils.return_torch_dtype
|
||||
dtype: float16
|
||||
|
||||
tokenizer_init:
|
||||
_target_: general_util.tokenization_utils.init_tokenizer
|
||||
tokenizer_path: ${model_name_or_path}
|
||||
padding_side: left
|
||||
|
||||
device_map:
|
||||
_target_: models.utils.return_single_device_map
|
||||
|
||||
model:
|
||||
_target_: models.llama.LlamaForCausalLMDPO.from_pretrained_with_ref_model
|
||||
beta: 0.1
|
||||
gradient_checkpointing: True
|
||||
attn_implementation: "eager"
|
||||
torch_dtype: ${torch_dtype}
|
||||
pad_token_id: 100001
|
||||
device_map: ${device_map}
|
||||
ref_model:
|
||||
_target_: models.ds_utils.init_ds_eval_engine
|
||||
model:
|
||||
_target_: models.llama.LlamaForCausalLMDPO.from_pretrained
|
||||
pretrained_model_name_or_path: ${model_name_or_path}
|
||||
torch_dtype: ${torch_dtype}
|
||||
attn_implementation: "eager"
|
||||
device_map: ${device_map}
|
||||
pad_token_id: 100001
|
||||
ds_cfg: ${ds_cfg}
|
||||
|
||||
|
||||
chat_prefix: "<｜begin▁of▁sentence｜>You are an AI programming assistant, utilizing the Deepseek Coder model, developed by Deepseek Company, and you only answer questions related to computer science. For politically sensitive questions, security and privacy issues, and other non-computer science questions, you will refuse to answer\n### Instruction:\n"
|
||||
chat_connect: "\n### Response:\n"
|
||||
chat_suffix: "\n<|EOT|>"
|
||||
prompt: "{question}\n\nPlease write a program to solve the above problem under the given time constraints and memory limits."
|
||||
|
||||
read_tensor:
|
||||
_target_: data.combine_dataset.MultiMappingDataset
|
||||
read_fn:
|
||||
_target_: data.apps.APPsReader
|
||||
split: train
|
||||
aligner:
|
||||
_target_: data.input_aligner.concat_aligner
|
||||
aligners:
|
||||
- _target_: data.input_aligner.field_extract_aligner
|
||||
input_index_field: problem_id
|
||||
extract_index_field: id
|
||||
extract_fields: [ "pos", "neg" ]
|
||||
extra_file: ${model_path_prefix}/deepseek-coder-7b-instruct-v1.5/apps/train.0shot.tem1.0.n10.dpo_v1.0.json
|
||||
- _target_: data.input_aligner.dpo_pair_aligner
|
||||
pos_field: pos
|
||||
neg_field: neg
|
||||
template:
|
||||
_target_: data.input_utils.recompose_template
|
||||
units:
|
||||
chat_prefix: ${chat_prefix}
|
||||
prompt: ${prompt}
|
||||
chat_connect: ${chat_connect}
|
||||
pos: "{pos}"
|
||||
neg: "{neg}"
|
||||
chat_suffix: ${chat_suffix}
|
||||
compositions:
|
||||
prompt: "{chat_prefix}{prompt}{chat_connect}"
|
||||
chosen: "{chat_prefix}{prompt}{chat_connect}{pos}{chat_suffix}"
|
||||
reject: "{chat_prefix}{prompt}{chat_connect}{neg}{chat_suffix}"
|
||||
index_field: problem_id
|
||||
kv_mapping:
|
||||
chosen: chosen
|
||||
reject: reject
|
||||
problem_id: index
|
||||
prompt: prompt
|
||||
|
||||
|
||||
dist_load_data_barrier: False
|
||||
|
||||
extended_vocab:
|
||||
|
||||
# Data collator
|
||||
collator:
|
||||
_target_: data.general_collator.DPOCollator
|
||||
tokenizer: ${tokenizer_init}
|
||||
max_seq_length: 2048
|
||||
|
||||
# Dataloader
|
||||
num_workers: 8
|
||||
prefetch_factor: 2
|
||||
|
||||
# Wiki path pretrain v8.2
|
||||
model_name_or_path: ${model_path_prefix}/deepseek-coder-7b-instruct-v1.5
|
||||
pretrain:
|
||||
resume:
|
||||
|
||||
dp_size:
|
||||
tp_size: 1
|
||||
pp_size: 1
|
||||
|
||||
|
||||
exp_name: deepseek-coder-v1.5-ins.7b.apps.code_gen.dpo.V100.w8.v1.0
|
||||
exp_notes:
|
||||
output_dir: ${output_path_prefix}experiments/${exp_name} # Fix <pad token id>
|
||||
|
||||
|
||||
do_train: True
|
||||
evaluate_during_training: False
|
||||
|
||||
do_eval: False
|
||||
eval_sub_path: checkpoint-*
|
||||
|
||||
# Training hyper-parameters
|
||||
per_gpu_train_batch_size: 1
|
||||
per_gpu_eval_batch_size: 1
|
||||
#learning_rate: 1e-4
|
||||
learning_rate: 5e-6
|
||||
#learning_rate: 2e-5
|
||||
gradient_accumulation_steps: 16
|
||||
weight_decay: 0.1
|
||||
adam_epsilon: 1e-6
|
||||
adam_betas: "(0.9, 0.98)"
|
||||
#adam_betas: "(0.9, 0.999)"
|
||||
#max_grad_norm: 0.0
|
||||
total_dataset_len: -1
|
||||
max_grad_norm: 1.0
|
||||
num_train_epochs: 1
|
||||
max_steps: 0
|
||||
warmup_proportion: 0.1
|
||||
warmup_steps: 0
|
||||
|
||||
# Optimizer
|
||||
optimizer:
|
||||
use_nvlamb:
|
||||
bit_training:
|
||||
|
||||
logging_steps: 5
|
||||
save_ds_state: True
|
||||
save_steps: 200
|
||||
save_best: False
|
||||
eval_steps: 200
|
||||
ddp_eval: True
|
||||
no_cuda: False
|
||||
seed: 42
|
||||
local_rank: -1
|
||||
fp16: True
|
||||
fp16_opt_level: O1
|
||||
fp16_bfloat16: True
|
||||
|
||||
# Prediction config
|
||||
prediction_cfg:
|
||||
metric: "loss"
|
||||
measure: -1
|
||||
best_checkpoint:
|
||||
best_result:
|
||||
eval_forward_fn:
|
||||
_target_: general_util.evaluator.DefaultForwardFn
|
||||
post_process:
|
||||
_target_: post_processors.dpo.DPOEvalPostProcessor
|
||||
|
||||
|
||||
ds_cfg:
|
||||
train_micro_batch_size_per_gpu: ${per_gpu_train_batch_size}
|
||||
gradient_accumulation_steps: ${gradient_accumulation_steps}
|
||||
optimizer:
|
||||
type: AdamW
|
||||
params:
|
||||
lr: ${learning_rate}
|
||||
betas: [ 0.9, 0.95 ]
|
||||
weight_decay: ${weight_decay}
|
||||
bf16:
|
||||
enabled: False
|
||||
fp16:
|
||||
enabled: True
|
||||
auto_cast: False
|
||||
loss_scale": 0
|
||||
initial_scale_power: 16
|
||||
loss_scale_window: 1000
|
||||
hysteresis: 2
|
||||
consecutive_hysteresis: False
|
||||
min_loss_scale: 1
|
||||
# zero_optimization:
|
||||
# offload_optimizer:
|
||||
# device: cpu
|
||||
# pin_memory: True
|
||||
|
||||
|
||||
summary_helper:
|
||||
_target_: general_util.tensorboard_helper.WandbWriter
|
||||
batch_index_or_keys:
|
||||
outputs_index_or_keys:
|
||||
|
||||
# Temporary variables
|
||||
n_gpu:
|
||||
device:
|
||||
train_batch_size:
|
||||
eval_batch_size:
|
||||
world_size:
|
||||
@@ -0,0 +1,210 @@
|
||||
defaults:
|
||||
- hydra: default
|
||||
# - deepspeed@ds_cfg: train_hybrid_engine_zero3
|
||||
- deepspeed@ds_cfg: train_hybrid_engine_zero2_optim_offload
|
||||
- _self_ # see here for more details: https://hydra.cc/docs/tutorials/basic/your_first_app/defaults/#composition-order-of-primary-config
|
||||
|
||||
hydra:
|
||||
searchpath:
|
||||
- file://conf/
|
||||
|
||||
data_path_prefix: ""
|
||||
model_path_prefix: ../pretrained-models/ # /mnt/fangkai_blob/share/models
|
||||
output_path_prefix: "" # /mnt/fangkai_blob/reward_modeling/
|
||||
|
||||
train_file: "hf:codeparrot/apps"
|
||||
dev_file:
|
||||
test_file:
|
||||
|
||||
torch_dtype:
|
||||
_target_: general_util.training_utils.return_torch_dtype
|
||||
dtype: float16
|
||||
|
||||
tokenizer_init:
|
||||
_target_: general_util.tokenization_utils.init_tokenizer
|
||||
tokenizer_path: ${model_name_or_path}
|
||||
padding_side: left
|
||||
|
||||
device_map:
|
||||
_target_: models.utils.return_single_device_map
|
||||
|
||||
model:
|
||||
_target_: models.llama_tp.LlamaForCausalLMDPO.from_pretrained_with_ref_model
|
||||
beta: 0.1
|
||||
gradient_checkpointing: True
|
||||
attn_implementation: "eager"
|
||||
torch_dtype: ${torch_dtype}
|
||||
pad_token_id: 100001
|
||||
# device_map: ${device_map}
|
||||
ref_model:
|
||||
_target_: models.llama_tp.LlamaForCausalLMDPO.from_pretrained
|
||||
pretrained_model_name_or_path: ${model_name_or_path}
|
||||
torch_dtype: ${torch_dtype}
|
||||
attn_implementation: "eager"
|
||||
# device_map: ${device_map}
|
||||
pad_token_id: 100001
|
||||
|
||||
|
||||
chat_prefix: "<｜begin▁of▁sentence｜>You are an AI programming assistant, utilizing the Deepseek Coder model, developed by Deepseek Company, and you only answer questions related to computer science. For politically sensitive questions, security and privacy issues, and other non-computer science questions, you will refuse to answer\n### Instruction:\n"
|
||||
chat_connect: "\n### Response:\n"
|
||||
chat_suffix: "\n<|EOT|>"
|
||||
prompt: "{question}\n\nPlease write a program to solve the above problem under the given time constraints and memory limits."
|
||||
|
||||
read_tensor:
|
||||
_target_: data.combine_dataset.MultiMappingDataset
|
||||
read_fn:
|
||||
_target_: data.apps.APPsReader
|
||||
split: train
|
||||
aligner:
|
||||
_target_: data.input_aligner.concat_aligner
|
||||
aligners:
|
||||
- _target_: data.input_aligner.field_extract_aligner
|
||||
input_index_field: problem_id
|
||||
extract_index_field: id
|
||||
extract_fields: [ "pos", "neg" ]
|
||||
extra_file: ${model_path_prefix}/deepseek-coder-7b-instruct-v1.5/apps/train.0shot.tem1.0.n10.dpo_v1.0.json
|
||||
- _target_: data.input_aligner.dpo_pair_aligner
|
||||
pos_field: pos
|
||||
neg_field: neg
|
||||
template:
|
||||
_target_: data.input_utils.recompose_template
|
||||
units:
|
||||
chat_prefix: ${chat_prefix}
|
||||
prompt: ${prompt}
|
||||
chat_connect: ${chat_connect}
|
||||
pos: "{pos}"
|
||||
neg: "{neg}"
|
||||
chat_suffix: ${chat_suffix}
|
||||
compositions:
|
||||
prompt: "{chat_prefix}{prompt}{chat_connect}"
|
||||
chosen: "{chat_prefix}{prompt}{chat_connect}{pos}{chat_suffix}"
|
||||
reject: "{chat_prefix}{prompt}{chat_connect}{neg}{chat_suffix}"
|
||||
index_field: problem_id
|
||||
kv_mapping:
|
||||
chosen: chosen
|
||||
reject: reject
|
||||
problem_id: index
|
||||
prompt: prompt
|
||||
|
||||
|
||||
dist_load_data_barrier: False
|
||||
|
||||
extended_vocab:
|
||||
|
||||
# Data collator
|
||||
collator:
|
||||
_target_: data.general_collator.DPOCollator
|
||||
tokenizer: ${tokenizer_init}
|
||||
max_seq_length: 2048
|
||||
|
||||
# Dataloader
|
||||
num_workers: 8
|
||||
prefetch_factor: 2
|
||||
|
||||
# Wiki path pretrain v8.2
|
||||
model_name_or_path: ${model_path_prefix}/deepseek-coder-7b-instruct-v1.5
|
||||
pretrain:
|
||||
resume:
|
||||
|
||||
dp_size:
|
||||
tp_size: 2
|
||||
pp_size: 1
|
||||
|
||||
|
||||
exp_name: deepseek-coder-v1.5-ins.7b.apps.code_gen.dpo.V100.tp2.dp4.v1.1.s${seed}
|
||||
exp_notes:
|
||||
output_dir: ${output_path_prefix}experiments/${exp_name} # Fix <pad token id>
|
||||
|
||||
|
||||
do_train: True
|
||||
evaluate_during_training: False
|
||||
|
||||
do_eval: False
|
||||
eval_sub_path: checkpoint-*
|
||||
|
||||
# Training hyper-parameters
|
||||
per_gpu_train_batch_size: 1
|
||||
per_gpu_eval_batch_size: 1
|
||||
#learning_rate: 1e-4
|
||||
learning_rate: 5e-6
|
||||
#learning_rate: 2e-5
|
||||
gradient_accumulation_steps: 32
|
||||
weight_decay: 0.1
|
||||
adam_epsilon: 1e-6
|
||||
adam_betas: "(0.9, 0.98)"
|
||||
#adam_betas: "(0.9, 0.999)"
|
||||
#max_grad_norm: 0.0
|
||||
total_dataset_len: -1
|
||||
max_grad_norm: 1.0
|
||||
num_train_epochs: 2
|
||||
max_steps: 500
|
||||
warmup_proportion: 0.1
|
||||
warmup_steps: 0
|
||||
|
||||
# Optimizer
|
||||
optimizer:
|
||||
use_nvlamb:
|
||||
bit_training:
|
||||
|
||||
logging_steps: 5
|
||||
save_ds_state: True
|
||||
save_steps: 100
|
||||
save_best: False
|
||||
eval_steps: 100
|
||||
ddp_eval: True
|
||||
no_cuda: False
|
||||
seed: 42
|
||||
local_rank: -1
|
||||
fp16: True
|
||||
fp16_opt_level: O1
|
||||
fp16_bfloat16: True
|
||||
|
||||
# Prediction config
|
||||
prediction_cfg:
|
||||
metric: "loss"
|
||||
measure: -1
|
||||
best_checkpoint:
|
||||
best_result:
|
||||
eval_forward_fn:
|
||||
_target_: general_util.evaluator.DefaultForwardFn
|
||||
post_process:
|
||||
_target_: post_processors.dpo.DPOEvalPostProcessor
|
||||
|
||||
|
||||
ds_cfg:
|
||||
train_micro_batch_size_per_gpu: ${per_gpu_train_batch_size}
|
||||
gradient_accumulation_steps: ${gradient_accumulation_steps}
|
||||
optimizer:
|
||||
type: AdamW
|
||||
params:
|
||||
lr: ${learning_rate}
|
||||
betas: [ 0.9, 0.95 ]
|
||||
weight_decay: ${weight_decay}
|
||||
bf16:
|
||||
enabled: False
|
||||
fp16:
|
||||
enabled: True
|
||||
auto_cast: False
|
||||
loss_scale": 0
|
||||
initial_scale_power: 16
|
||||
loss_scale_window: 1000
|
||||
hysteresis: 2
|
||||
consecutive_hysteresis: False
|
||||
min_loss_scale: 1
|
||||
# zero_optimization:
|
||||
# offload_optimizer:
|
||||
# device: cpu
|
||||
# pin_memory: True
|
||||
|
||||
|
||||
summary_helper:
|
||||
_target_: general_util.tensorboard_helper.WandbWriter
|
||||
batch_index_or_keys:
|
||||
outputs_index_or_keys:
|
||||
|
||||
# Temporary variables
|
||||
n_gpu:
|
||||
device:
|
||||
train_batch_size:
|
||||
eval_batch_size:
|
||||
world_size:
|
||||
@@ -0,0 +1,210 @@
|
||||
defaults:
|
||||
- hydra: default
|
||||
# - deepspeed@ds_cfg: train_hybrid_engine_zero3
|
||||
- deepspeed@ds_cfg: train_hybrid_engine_zero1_optim_offload
|
||||
- _self_ # see here for more details: https://hydra.cc/docs/tutorials/basic/your_first_app/defaults/#composition-order-of-primary-config
|
||||
|
||||
hydra:
|
||||
searchpath:
|
||||
- file://conf/
|
||||
|
||||
data_path_prefix: ""
|
||||
model_path_prefix: ../pretrained-models/ # /mnt/fangkai_blob/share/models
|
||||
output_path_prefix: "" # /mnt/fangkai_blob/reward_modeling/
|
||||
|
||||
train_file: "hf:codeparrot/apps"
|
||||
dev_file:
|
||||
test_file:
|
||||
|
||||
torch_dtype:
|
||||
_target_: general_util.training_utils.return_torch_dtype
|
||||
dtype: float16
|
||||
|
||||
tokenizer_init:
|
||||
_target_: general_util.tokenization_utils.init_tokenizer
|
||||
tokenizer_path: ${model_name_or_path}
|
||||
padding_side: left
|
||||
|
||||
device_map:
|
||||
_target_: models.utils.return_single_device_map
|
||||
|
||||
model:
|
||||
_target_: models.llama_tp.LlamaForCausalLMDPO.from_pretrained_with_ref_model
|
||||
beta: 0.1
|
||||
gradient_checkpointing: True
|
||||
attn_implementation: "eager"
|
||||
torch_dtype: ${torch_dtype}
|
||||
pad_token_id: 100001
|
||||
# device_map: ${device_map}
|
||||
ref_model:
|
||||
_target_: models.llama_tp.LlamaForCausalLMDPO.from_pretrained
|
||||
pretrained_model_name_or_path: ${model_name_or_path}
|
||||
torch_dtype: ${torch_dtype}
|
||||
attn_implementation: "eager"
|
||||
# device_map: ${device_map}
|
||||
pad_token_id: 100001
|
||||
|
||||
|
||||
chat_prefix: "<｜begin▁of▁sentence｜>You are an AI programming assistant, utilizing the Deepseek Coder model, developed by Deepseek Company, and you only answer questions related to computer science. For politically sensitive questions, security and privacy issues, and other non-computer science questions, you will refuse to answer\n### Instruction:\n"
|
||||
chat_connect: "\n### Response:\n"
|
||||
chat_suffix: "\n<|EOT|>"
|
||||
prompt: "{question}\n\nPlease write a program to solve the above problem under the given time constraints and memory limits."
|
||||
|
||||
read_tensor:
|
||||
_target_: data.combine_dataset.MultiMappingDataset
|
||||
read_fn:
|
||||
_target_: data.apps.APPsReader
|
||||
split: train
|
||||
aligner:
|
||||
_target_: data.input_aligner.concat_aligner
|
||||
aligners:
|
||||
- _target_: data.input_aligner.field_extract_aligner
|
||||
input_index_field: problem_id
|
||||
extract_index_field: id
|
||||
extract_fields: [ "pos", "neg" ]
|
||||
extra_file: ${model_path_prefix}/deepseek-coder-7b-instruct-v1.5/apps/train.0shot.tem1.0.n10.dpo_v1.0.json
|
||||
- _target_: data.input_aligner.dpo_pair_aligner
|
||||
pos_field: pos
|
||||
neg_field: neg
|
||||
template:
|
||||
_target_: data.input_utils.recompose_template
|
||||
units:
|
||||
chat_prefix: ${chat_prefix}
|
||||
prompt: ${prompt}
|
||||
chat_connect: ${chat_connect}
|
||||
pos: "{pos}"
|
||||
neg: "{neg}"
|
||||
chat_suffix: ${chat_suffix}
|
||||
compositions:
|
||||
prompt: "{chat_prefix}{prompt}{chat_connect}"
|
||||
chosen: "{chat_prefix}{prompt}{chat_connect}{pos}{chat_suffix}"
|
||||
reject: "{chat_prefix}{prompt}{chat_connect}{neg}{chat_suffix}"
|
||||
index_field: problem_id
|
||||
kv_mapping:
|
||||
chosen: chosen
|
||||
reject: reject
|
||||
problem_id: index
|
||||
prompt: prompt
|
||||
|
||||
|
||||
dist_load_data_barrier: False
|
||||
|
||||
extended_vocab:
|
||||
|
||||
# Data collator
|
||||
collator:
|
||||
_target_: data.general_collator.DPOCollator
|
||||
tokenizer: ${tokenizer_init}
|
||||
max_seq_length: 2048
|
||||
|
||||
# Dataloader
|
||||
num_workers: 8
|
||||
prefetch_factor: 2
|
||||
|
||||
# Wiki path pretrain v8.2
|
||||
model_name_or_path: ${model_path_prefix}/deepseek-coder-7b-instruct-v1.5
|
||||
pretrain:
|
||||
resume:
|
||||
|
||||
dp_size:
|
||||
tp_size: 4
|
||||
pp_size: 1
|
||||
|
||||
|
||||
exp_name: deepseek-coder-v1.5-ins.7b.apps.code_gen.dpo.V100.tp4.dp2.v1.1.s${seed}
|
||||
exp_notes:
|
||||
output_dir: ${output_path_prefix}experiments/${exp_name} # Fix <pad token id>
|
||||
|
||||
|
||||
do_train: True
|
||||
evaluate_during_training: False
|
||||
|
||||
do_eval: False
|
||||
eval_sub_path: checkpoint-*
|
||||
|
||||
# Training hyper-parameters
|
||||
per_gpu_train_batch_size: 1
|
||||
per_gpu_eval_batch_size: 1
|
||||
#learning_rate: 1e-4
|
||||
learning_rate: 5e-6
|
||||
#learning_rate: 2e-5
|
||||
gradient_accumulation_steps: 64
|
||||
weight_decay: 0.1
|
||||
adam_epsilon: 1e-6
|
||||
adam_betas: "(0.9, 0.98)"
|
||||
#adam_betas: "(0.9, 0.999)"
|
||||
#max_grad_norm: 0.0
|
||||
total_dataset_len: -1
|
||||
max_grad_norm: 1.0
|
||||
num_train_epochs: 2
|
||||
max_steps: 500
|
||||
warmup_proportion: 0.1
|
||||
warmup_steps: 0
|
||||
|
||||
# Optimizer
|
||||
optimizer:
|
||||
use_nvlamb:
|
||||
bit_training:
|
||||
|
||||
logging_steps: 5
|
||||
save_ds_state: True
|
||||
save_steps: 100
|
||||
save_best: False
|
||||
eval_steps: 100
|
||||
ddp_eval: True
|
||||
no_cuda: False
|
||||
seed: 42
|
||||
local_rank: -1
|
||||
fp16: True
|
||||
fp16_opt_level: O1
|
||||
fp16_bfloat16: True
|
||||
|
||||
# Prediction config
|
||||
prediction_cfg:
|
||||
metric: "loss"
|
||||
measure: -1
|
||||
best_checkpoint:
|
||||
best_result:
|
||||
eval_forward_fn:
|
||||
_target_: general_util.evaluator.DefaultForwardFn
|
||||
post_process:
|
||||
_target_: post_processors.dpo.DPOEvalPostProcessor
|
||||
|
||||
|
||||
ds_cfg:
|
||||
train_micro_batch_size_per_gpu: ${per_gpu_train_batch_size}
|
||||
gradient_accumulation_steps: ${gradient_accumulation_steps}
|
||||
optimizer:
|
||||
type: AdamW
|
||||
params:
|
||||
lr: ${learning_rate}
|
||||
betas: [ 0.9, 0.95 ]
|
||||
weight_decay: ${weight_decay}
|
||||
bf16:
|
||||
enabled: False
|
||||
fp16:
|
||||
enabled: True
|
||||
auto_cast: False
|
||||
loss_scale": 0
|
||||
initial_scale_power: 16
|
||||
loss_scale_window: 1000
|
||||
hysteresis: 2
|
||||
consecutive_hysteresis: False
|
||||
min_loss_scale: 1
|
||||
# zero_optimization:
|
||||
# offload_optimizer:
|
||||
# device: cpu
|
||||
# pin_memory: True
|
||||
|
||||
|
||||
summary_helper:
|
||||
_target_: general_util.tensorboard_helper.WandbWriter
|
||||
batch_index_or_keys:
|
||||
outputs_index_or_keys:
|
||||
|
||||
# Temporary variables
|
||||
n_gpu:
|
||||
device:
|
||||
train_batch_size:
|
||||
eval_batch_size:
|
||||
world_size:
|
||||
@@ -0,0 +1,213 @@
|
||||
defaults:
|
||||
- hydra: default
|
||||
- deepspeed@ds_cfg: train_hybrid_engine_zero3
|
||||
# - deepspeed@ds_cfg: train_hybrid_engine_zero1_optim_offload
|
||||
- _self_ # see here for more details: https://hydra.cc/docs/tutorials/basic/your_first_app/defaults/#composition-order-of-primary-config
|
||||
|
||||
hydra:
|
||||
searchpath:
|
||||
- file://conf/
|
||||
|
||||
data_path_prefix: ""
|
||||
model_path_prefix: ../pretrained-models/ # /mnt/fangkai_blob/share/models
|
||||
output_path_prefix: "" # /mnt/fangkai_blob/reward_modeling/
|
||||
|
||||
train_file: "hf:codeparrot/apps"
|
||||
dev_file:
|
||||
test_file:
|
||||
|
||||
torch_dtype:
|
||||
_target_: general_util.training_utils.return_torch_dtype
|
||||
dtype: float16
|
||||
|
||||
tokenizer_init:
|
||||
_target_: general_util.tokenization_utils.init_tokenizer
|
||||
tokenizer_path: ${model_name_or_path}
|
||||
padding_side: left
|
||||
|
||||
device_map:
|
||||
_target_: models.utils.return_single_device_map
|
||||
|
||||
model:
|
||||
_target_: models.llama.LlamaForCausalLMDPO.from_pretrained_with_ref_model
|
||||
beta: 0.1
|
||||
gradient_checkpointing: True
|
||||
attn_implementation: "eager"
|
||||
torch_dtype: ${torch_dtype}
|
||||
pad_token_id: 100001
|
||||
device_map: ${device_map}
|
||||
ref_model:
|
||||
_target_: models.ds_utils.init_ds_eval_engine
|
||||
model:
|
||||
_target_: models.llama.LlamaForCausalLMDPO.from_pretrained
|
||||
pretrained_model_name_or_path: ${model_name_or_path}
|
||||
torch_dtype: ${torch_dtype}
|
||||
attn_implementation: "eager"
|
||||
device_map: ${device_map}
|
||||
pad_token_id: 100001
|
||||
ds_cfg: ${ds_cfg}
|
||||
|
||||
|
||||
chat_prefix: "<｜begin▁of▁sentence｜>You are an AI programming assistant, utilizing the Deepseek Coder model, developed by Deepseek Company, and you only answer questions related to computer science. For politically sensitive questions, security and privacy issues, and other non-computer science questions, you will refuse to answer\n### Instruction:\n"
|
||||
chat_connect: "\n### Response:\n"
|
||||
chat_suffix: "\n<|EOT|>"
|
||||
prompt: "{question}\n\nPlease write a program to solve the above problem under the given time constraints and memory limits."
|
||||
|
||||
read_tensor:
|
||||
_target_: data.combine_dataset.MultiMappingDataset
|
||||
read_fn:
|
||||
_target_: data.apps.APPsReader
|
||||
split: train
|
||||
aligner:
|
||||
_target_: data.input_aligner.concat_aligner
|
||||
aligners:
|
||||
- _target_: data.input_aligner.field_extract_aligner
|
||||
input_index_field: problem_id
|
||||
extract_index_field: id
|
||||
extract_fields: [ "pos", "neg" ]
|
||||
extra_file: ${model_path_prefix}/deepseek-coder-7b-instruct-v1.5/apps/train.0shot.tem1.0.n10.dpo_v1.0.json
|
||||
- _target_: data.input_aligner.dpo_pair_aligner
|
||||
pos_field: pos
|
||||
neg_field: neg
|
||||
template:
|
||||
_target_: data.input_utils.recompose_template
|
||||
units:
|
||||
chat_prefix: ${chat_prefix}
|
||||
prompt: ${prompt}
|
||||
chat_connect: ${chat_connect}
|
||||
pos: "{pos}"
|
||||
neg: "{neg}"
|
||||
chat_suffix: ${chat_suffix}
|
||||
compositions:
|
||||
prompt: "{chat_prefix}{prompt}{chat_connect}"
|
||||
chosen: "{chat_prefix}{prompt}{chat_connect}{pos}{chat_suffix}"
|
||||
reject: "{chat_prefix}{prompt}{chat_connect}{neg}{chat_suffix}"
|
||||
index_field: problem_id
|
||||
kv_mapping:
|
||||
chosen: chosen
|
||||
reject: reject
|
||||
problem_id: index
|
||||
prompt: prompt
|
||||
|
||||
|
||||
dist_load_data_barrier: False
|
||||
|
||||
extended_vocab:
|
||||
|
||||
# Data collator
|
||||
collator:
|
||||
_target_: data.general_collator.DPOCollator
|
||||
tokenizer: ${tokenizer_init}
|
||||
max_seq_length: 2048
|
||||
|
||||
# Dataloader
|
||||
num_workers: 8
|
||||
prefetch_factor: 2
|
||||
|
||||
# Wiki path pretrain v8.2
|
||||
model_name_or_path: ${model_path_prefix}/deepseek-coder-7b-instruct-v1.5
|
||||
pretrain:
|
||||
resume:
|
||||
|
||||
dp_size:
|
||||
tp_size: 1
|
||||
pp_size: 1
|
||||
|
||||
|
||||
exp_name: deepseek-coder-v1.5-ins.7b.apps.code_gen.dpo.V100.w8.v1.1
|
||||
exp_notes:
|
||||
output_dir: ${output_path_prefix}experiments/${exp_name} # Fix <pad token id>
|
||||
|
||||
|
||||
do_train: True
|
||||
evaluate_during_training: False
|
||||
|
||||
do_eval: False
|
||||
eval_sub_path: checkpoint-*
|
||||
|
||||
# Training hyper-parameters
|
||||
per_gpu_train_batch_size: 1
|
||||
per_gpu_eval_batch_size: 1
|
||||
#learning_rate: 1e-4
|
||||
learning_rate: 5e-6
|
||||
#learning_rate: 2e-5
|
||||
gradient_accumulation_steps: 16
|
||||
weight_decay: 0.1
|
||||
adam_epsilon: 1e-6
|
||||
adam_betas: "(0.9, 0.98)"
|
||||
#adam_betas: "(0.9, 0.999)"
|
||||
#max_grad_norm: 0.0
|
||||
total_dataset_len: -1
|
||||
max_grad_norm: 1.0
|
||||
num_train_epochs: 2
|
||||
max_steps: 0
|
||||
warmup_proportion: 0.1
|
||||
warmup_steps: 0
|
||||
|
||||
# Optimizer
|
||||
optimizer:
|
||||
use_nvlamb:
|
||||
bit_training:
|
||||
|
||||
logging_steps: 5
|
||||
save_ds_state: True
|
||||
save_steps: 100
|
||||
save_best: False
|
||||
eval_steps: 100
|
||||
ddp_eval: True
|
||||
no_cuda: False
|
||||
seed: 42
|
||||
local_rank: -1
|
||||
fp16: True
|
||||
fp16_opt_level: O1
|
||||
fp16_bfloat16: True
|
||||
|
||||
# Prediction config
|
||||
prediction_cfg:
|
||||
metric: "loss"
|
||||
measure: -1
|
||||
best_checkpoint:
|
||||
best_result:
|
||||
eval_forward_fn:
|
||||
_target_: general_util.evaluator.DefaultForwardFn
|
||||
post_process:
|
||||
_target_: post_processors.dpo.DPOEvalPostProcessor
|
||||
|
||||
|
||||
ds_cfg:
|
||||
train_micro_batch_size_per_gpu: ${per_gpu_train_batch_size}
|
||||
gradient_accumulation_steps: ${gradient_accumulation_steps}
|
||||
# optimizer:
|
||||
# type: AdamW
|
||||
# params:
|
||||
# lr: ${learning_rate}
|
||||
# betas: [ 0.9, 0.95 ]
|
||||
# weight_decay: ${weight_decay}
|
||||
bf16:
|
||||
enabled: False
|
||||
fp16:
|
||||
enabled: True
|
||||
auto_cast: False
|
||||
loss_scale": 0
|
||||
initial_scale_power: 16
|
||||
loss_scale_window: 1000
|
||||
hysteresis: 2
|
||||
consecutive_hysteresis: False
|
||||
min_loss_scale: 1
|
||||
# zero_optimization:
|
||||
# offload_optimizer:
|
||||
# device: cpu
|
||||
# pin_memory: True
|
||||
|
||||
|
||||
summary_helper:
|
||||
_target_: general_util.tensorboard_helper.WandbWriter
|
||||
batch_index_or_keys:
|
||||
outputs_index_or_keys:
|
||||
|
||||
# Temporary variables
|
||||
n_gpu:
|
||||
device:
|
||||
train_batch_size:
|
||||
eval_batch_size:
|
||||
world_size:
|
||||
@@ -0,0 +1,210 @@
|
||||
defaults:
|
||||
- hydra: default
|
||||
# - deepspeed@ds_cfg: train_hybrid_engine_zero3
|
||||
- deepspeed@ds_cfg: train_hybrid_engine_zero1_optim_offload
|
||||
- _self_ # see here for more details: https://hydra.cc/docs/tutorials/basic/your_first_app/defaults/#composition-order-of-primary-config
|
||||
|
||||
hydra:
|
||||
searchpath:
|
||||
- file://conf/
|
||||
|
||||
data_path_prefix: ""
|
||||
model_path_prefix: ../pretrained-models/ # /mnt/fangkai_blob/share/models
|
||||
output_path_prefix: "" # /mnt/fangkai_blob/reward_modeling/
|
||||
|
||||
train_file: "hf:codeparrot/apps"
|
||||
dev_file:
|
||||
test_file:
|
||||
|
||||
torch_dtype:
|
||||
_target_: general_util.training_utils.return_torch_dtype
|
||||
dtype: float16
|
||||
|
||||
tokenizer_init:
|
||||
_target_: general_util.tokenization_utils.init_tokenizer
|
||||
tokenizer_path: ${model_name_or_path}
|
||||
padding_side: left
|
||||
|
||||
device_map:
|
||||
_target_: models.utils.return_single_device_map
|
||||
|
||||
model:
|
||||
_target_: models.llama_tp.LlamaForCausalLMDPO.from_pretrained_with_ref_model
|
||||
beta: 0.1
|
||||
gradient_checkpointing: True
|
||||
attn_implementation: "eager"
|
||||
torch_dtype: ${torch_dtype}
|
||||
pad_token_id: 100001
|
||||
# device_map: ${device_map}
|
||||
ref_model:
|
||||
_target_: models.llama_tp.LlamaForCausalLMDPO.from_pretrained
|
||||
pretrained_model_name_or_path: ${model_name_or_path}
|
||||
torch_dtype: ${torch_dtype}
|
||||
attn_implementation: "eager"
|
||||
# device_map: ${device_map}
|
||||
pad_token_id: 100001
|
||||
|
||||
|
||||
chat_prefix: "<｜begin▁of▁sentence｜>You are an AI programming assistant, utilizing the Deepseek Coder model, developed by Deepseek Company, and you only answer questions related to computer science. For politically sensitive questions, security and privacy issues, and other non-computer science questions, you will refuse to answer\n### Instruction:\n"
|
||||
chat_connect: "\n### Response:\n"
|
||||
chat_suffix: "\n<|EOT|>"
|
||||
prompt: "{question}\n\nPlease write a program to solve the above problem under the given time constraints and memory limits."
|
||||
|
||||
read_tensor:
|
||||
_target_: data.combine_dataset.MultiMappingDataset
|
||||
read_fn:
|
||||
_target_: data.apps.APPsReader
|
||||
split: train
|
||||
aligner:
|
||||
_target_: data.input_aligner.concat_aligner
|
||||
aligners:
|
||||
- _target_: data.input_aligner.field_extract_aligner
|
||||
input_index_field: problem_id
|
||||
extract_index_field: id
|
||||
extract_fields: [ "pos", "neg" ]
|
||||
extra_file: ${model_path_prefix}/deepseek-coder-7b-instruct-v1.5/apps/train.0shot.tem1.0.n10.v1.0.dpo_v1.1.json
|
||||
- _target_: data.input_aligner.dpo_pair_aligner
|
||||
pos_field: pos
|
||||
neg_field: neg
|
||||
template:
|
||||
_target_: data.input_utils.recompose_template
|
||||
units:
|
||||
chat_prefix: ${chat_prefix}
|
||||
prompt: ${prompt}
|
||||
chat_connect: ${chat_connect}
|
||||
pos: "{pos}"
|
||||
neg: "{neg}"
|
||||
chat_suffix: ${chat_suffix}
|
||||
compositions:
|
||||
prompt: "{chat_prefix}{prompt}{chat_connect}"
|
||||
chosen: "{chat_prefix}{prompt}{chat_connect}{pos}{chat_suffix}"
|
||||
reject: "{chat_prefix}{prompt}{chat_connect}{neg}{chat_suffix}"
|
||||
index_field: problem_id
|
||||
kv_mapping:
|
||||
chosen: chosen
|
||||
reject: reject
|
||||
problem_id: index
|
||||
prompt: prompt
|
||||
|
||||
|
||||
dist_load_data_barrier: False
|
||||
|
||||
extended_vocab:
|
||||
|
||||
# Data collator
|
||||
collator:
|
||||
_target_: data.general_collator.DPOCollator
|
||||
tokenizer: ${tokenizer_init}
|
||||
max_seq_length: 3072
|
||||
|
||||
# Dataloader
|
||||
num_workers: 8
|
||||
prefetch_factor: 2
|
||||
|
||||
# Wiki path pretrain v8.2
|
||||
model_name_or_path: ${model_path_prefix}/deepseek-coder-7b-instruct-v1.5
|
||||
pretrain:
|
||||
resume: latest
|
||||
|
||||
dp_size:
|
||||
tp_size: 4
|
||||
pp_size: 1
|
||||
|
||||
|
||||
exp_name: deepseek-coder-v1.5-ins.7b.apps.code_gen.dpo.V100.tp4.dp8.v1.2.s${seed}
|
||||
exp_notes:
|
||||
output_dir: ${output_path_prefix}experiments/${exp_name} # Fix <pad token id>
|
||||
|
||||
|
||||
do_train: True
|
||||
evaluate_during_training: False
|
||||
|
||||
do_eval: False
|
||||
eval_sub_path: checkpoint-*
|
||||
|
||||
# Training hyper-parameters
|
||||
per_gpu_train_batch_size: 1
|
||||
per_gpu_eval_batch_size: 1
|
||||
#learning_rate: 1e-4
|
||||
learning_rate: 5e-6
|
||||
#learning_rate: 2e-5
|
||||
gradient_accumulation_steps: 16
|
||||
weight_decay: 0.1
|
||||
adam_epsilon: 1e-6
|
||||
adam_betas: "(0.9, 0.98)"
|
||||
#adam_betas: "(0.9, 0.999)"
|
||||
#max_grad_norm: 0.0
|
||||
total_dataset_len: -1
|
||||
max_grad_norm: 1.0
|
||||
num_train_epochs: 2
|
||||
max_steps: 500
|
||||
warmup_proportion: 0.1
|
||||
warmup_steps: 0
|
||||
|
||||
# Optimizer
|
||||
optimizer:
|
||||
use_nvlamb:
|
||||
bit_training:
|
||||
|
||||
logging_steps: 5
|
||||
save_ds_state: True
|
||||
save_steps: 100
|
||||
save_best: False
|
||||
eval_steps: 100
|
||||
ddp_eval: True
|
||||
no_cuda: False
|
||||
seed: 42
|
||||
local_rank: -1
|
||||
fp16: True
|
||||
fp16_opt_level: O1
|
||||
fp16_bfloat16: False
|
||||
|
||||
# Prediction config
|
||||
prediction_cfg:
|
||||
metric: "loss"
|
||||
measure: -1
|
||||
best_checkpoint:
|
||||
best_result:
|
||||
eval_forward_fn:
|
||||
_target_: general_util.evaluator.DefaultForwardFn
|
||||
post_process:
|
||||
_target_: post_processors.dpo.DPOEvalPostProcessor
|
||||
|
||||
|
||||
ds_cfg:
|
||||
train_micro_batch_size_per_gpu: ${per_gpu_train_batch_size}
|
||||
gradient_accumulation_steps: ${gradient_accumulation_steps}
|
||||
optimizer:
|
||||
type: AdamW
|
||||
params:
|
||||
lr: ${learning_rate}
|
||||
betas: [ 0.9, 0.95 ]
|
||||
weight_decay: ${weight_decay}
|
||||
bf16:
|
||||
enabled: False
|
||||
fp16:
|
||||
enabled: True
|
||||
auto_cast: False
|
||||
loss_scale: 0
|
||||
initial_scale_power: 16
|
||||
loss_scale_window: 1000
|
||||
hysteresis: 2
|
||||
consecutive_hysteresis: False
|
||||
min_loss_scale: 1
|
||||
# zero_optimization:
|
||||
# offload_optimizer:
|
||||
# device: cpu
|
||||
# pin_memory: True
|
||||
|
||||
|
||||
summary_helper:
|
||||
_target_: general_util.tensorboard_helper.WandbWriter
|
||||
batch_index_or_keys:
|
||||
outputs_index_or_keys:
|
||||
|
||||
# Temporary variables
|
||||
n_gpu:
|
||||
device:
|
||||
train_batch_size:
|
||||
eval_batch_size:
|
||||
world_size:
|
||||
@@ -0,0 +1,211 @@
|
||||
defaults:
|
||||
- hydra: default
|
||||
# - deepspeed@ds_cfg: train_hybrid_engine_zero3
|
||||
- deepspeed@ds_cfg: train_hybrid_engine_zero1
|
||||
- _self_ # see here for more details: https://hydra.cc/docs/tutorials/basic/your_first_app/defaults/#composition-order-of-primary-config
|
||||
|
||||
hydra:
|
||||
searchpath:
|
||||
- file://conf/
|
||||
|
||||
data_path_prefix: ""
|
||||
model_path_prefix: /mnt/fangkai_blob/share/models
|
||||
output_path_prefix: /mnt/fangkai_blob/reward_modeling/
|
||||
|
||||
train_file: "hf:codeparrot/apps"
|
||||
dev_file:
|
||||
test_file:
|
||||
|
||||
torch_dtype:
|
||||
_target_: general_util.training_utils.return_torch_dtype
|
||||
dtype: float16
|
||||
|
||||
tokenizer_init:
|
||||
_target_: general_util.tokenization_utils.init_tokenizer
|
||||
tokenizer_path: ${model_name_or_path}
|
||||
padding_side: left
|
||||
|
||||
device_map:
|
||||
_target_: models.utils.return_single_device_map
|
||||
|
||||
model:
|
||||
_target_: models.llama.LlamaForCausalLMDPO.from_pretrained_with_ref_model
|
||||
beta: 0.1
|
||||
gradient_checkpointing: True
|
||||
attn_implementation: "flash_attention_2"
|
||||
torch_dtype: ${torch_dtype}
|
||||
pad_token_id: 100001
|
||||
# device_map: ${device_map}
|
||||
ref_model:
|
||||
_target_: models.llama.LlamaForCausalLMDPO.from_pretrained
|
||||
pretrained_model_name_or_path: ${model_name_or_path}
|
||||
torch_dtype: ${torch_dtype}
|
||||
attn_implementation: "flash_attention_2"
|
||||
device_map: ${device_map}
|
||||
pad_token_id: 100001
|
||||
|
||||
|
||||
chat_prefix: "<｜begin▁of▁sentence｜>You are an AI programming assistant, utilizing the Deepseek Coder model, developed by Deepseek Company, and you only answer questions related to computer science. For politically sensitive questions, security and privacy issues, and other non-computer science questions, you will refuse to answer\n### Instruction:\n"
|
||||
chat_connect: "\n### Response:\n"
|
||||
chat_suffix: "\n<|EOT|>"
|
||||
prompt: "{question}\n\nPlease write a program to solve the above problem under the given time constraints and memory limits."
|
||||
|
||||
read_tensor:
|
||||
_target_: data.combine_dataset.MultiMappingDataset
|
||||
read_fn:
|
||||
_target_: data.apps.APPsReader
|
||||
split: train
|
||||
train_sub_split: train
|
||||
aligner:
|
||||
_target_: data.input_aligner.concat_aligner
|
||||
aligners:
|
||||
- _target_: data.input_aligner.field_extract_aligner
|
||||
input_index_field: problem_id
|
||||
extract_index_field: id
|
||||
extract_fields: [ "pos", "neg" ]
|
||||
extra_file: ${model_path_prefix}/deepseek-coder-7b-instruct-v1.5/apps/train.0shot.tem1.0.n10.v1.0.dpo_v1.1.json
|
||||
- _target_: data.input_aligner.dpo_pair_aligner
|
||||
pos_field: pos
|
||||
neg_field: neg
|
||||
template:
|
||||
_target_: data.input_utils.recompose_template
|
||||
units:
|
||||
chat_prefix: ${chat_prefix}
|
||||
prompt: ${prompt}
|
||||
chat_connect: ${chat_connect}
|
||||
pos: "{pos}"
|
||||
neg: "{neg}"
|
||||
chat_suffix: ${chat_suffix}
|
||||
compositions:
|
||||
prompt: "{chat_prefix}{prompt}{chat_connect}"
|
||||
chosen: "{chat_prefix}{prompt}{chat_connect}{pos}{chat_suffix}"
|
||||
reject: "{chat_prefix}{prompt}{chat_connect}{neg}{chat_suffix}"
|
||||
index_field: problem_id
|
||||
kv_mapping:
|
||||
chosen: chosen
|
||||
reject: reject
|
||||
problem_id: index
|
||||
prompt: prompt
|
||||
|
||||
|
||||
dist_load_data_barrier: False
|
||||
|
||||
extended_vocab:
|
||||
|
||||
# Data collator
|
||||
collator:
|
||||
_target_: data.general_collator.DPOCollator
|
||||
tokenizer: ${tokenizer_init}
|
||||
max_seq_length: 3072
|
||||
|
||||
# Dataloader
|
||||
num_workers: 8
|
||||
prefetch_factor: 2
|
||||
|
||||
# Wiki path pretrain v8.2
|
||||
model_name_or_path: ${model_path_prefix}/deepseek-coder-7b-instruct-v1.5
|
||||
pretrain:
|
||||
resume:
|
||||
|
||||
dp_size:
|
||||
tp_size: 1
|
||||
pp_size: 1
|
||||
|
||||
|
||||
exp_name: deepseek-coder-v1.5-ins.7b.apps.code_gen.dpo.A100.w8.v1.3.s${seed}
|
||||
exp_notes:
|
||||
output_dir: ${output_path_prefix}experiments/${exp_name} # Fix <pad token id>
|
||||
|
||||
|
||||
do_train: True
|
||||
evaluate_during_training: False
|
||||
|
||||
do_eval: False
|
||||
eval_sub_path: checkpoint-*
|
||||
|
||||
# Training hyper-parameters
|
||||
per_gpu_train_batch_size: 1
|
||||
per_gpu_eval_batch_size: 1
|
||||
#learning_rate: 1e-4
|
||||
learning_rate: 5e-6
|
||||
#learning_rate: 2e-5
|
||||
gradient_accumulation_steps: 16
|
||||
weight_decay: 0.1
|
||||
adam_epsilon: 1e-6
|
||||
adam_betas: "(0.9, 0.98)"
|
||||
#adam_betas: "(0.9, 0.999)"
|
||||
#max_grad_norm: 0.0
|
||||
total_dataset_len: -1
|
||||
max_grad_norm: 1.0
|
||||
num_train_epochs: 2
|
||||
max_steps: 500
|
||||
warmup_proportion: 0.1
|
||||
warmup_steps: 0
|
||||
|
||||
# Optimizer
|
||||
optimizer:
|
||||
use_nvlamb:
|
||||
bit_training:
|
||||
|
||||
logging_steps: 5
|
||||
save_ds_state: True
|
||||
save_steps: 100
|
||||
save_best: False
|
||||
eval_steps: 100
|
||||
ddp_eval: True
|
||||
no_cuda: False
|
||||
seed: 42
|
||||
local_rank: -1
|
||||
fp16: True
|
||||
fp16_opt_level: O1
|
||||
fp16_bfloat16: True
|
||||
|
||||
# Prediction config
|
||||
prediction_cfg:
|
||||
metric: "loss"
|
||||
measure: -1
|
||||
best_checkpoint:
|
||||
best_result:
|
||||
eval_forward_fn:
|
||||
_target_: general_util.evaluator.DefaultForwardFn
|
||||
post_process:
|
||||
_target_: post_processors.dpo.DPOEvalPostProcessor
|
||||
|
||||
|
||||
ds_cfg:
|
||||
train_micro_batch_size_per_gpu: ${per_gpu_train_batch_size}
|
||||
gradient_accumulation_steps: ${gradient_accumulation_steps}
|
||||
optimizer:
|
||||
type: AdamW
|
||||
params:
|
||||
lr: ${learning_rate}
|
||||
betas: [ 0.9, 0.95 ]
|
||||
weight_decay: ${weight_decay}
|
||||
# bf16:
|
||||
# enabled: False
|
||||
# fp16:
|
||||
# enabled: True
|
||||
# auto_cast: False
|
||||
# loss_scale: 0
|
||||
# initial_scale_power: 16
|
||||
# loss_scale_window: 1000
|
||||
# hysteresis: 2
|
||||
# consecutive_hysteresis: False
|
||||
# min_loss_scale: 1
|
||||
# zero_optimization:
|
||||
# offload_optimizer:
|
||||
# device: cpu
|
||||
# pin_memory: True
|
||||
|
||||
|
||||
summary_helper:
|
||||
_target_: general_util.tensorboard_helper.WandbWriter
|
||||
batch_index_or_keys:
|
||||
outputs_index_or_keys:
|
||||
|
||||
# Temporary variables
|
||||
n_gpu:
|
||||
device:
|
||||
train_batch_size:
|
||||
eval_batch_size:
|
||||
world_size:
|
||||
@@ -0,0 +1,211 @@
|
||||
defaults:
|
||||
- hydra: default
|
||||
# - deepspeed@ds_cfg: train_hybrid_engine_zero3
|
||||
- deepspeed@ds_cfg: train_hybrid_engine_zero1_optim_offload
|
||||
- _self_ # see here for more details: https://hydra.cc/docs/tutorials/basic/your_first_app/defaults/#composition-order-of-primary-config
|
||||
|
||||
hydra:
|
||||
searchpath:
|
||||
- file://conf/
|
||||
|
||||
data_path_prefix: ""
|
||||
model_path_prefix: ../pretrained-models/ # /mnt/fangkai_blob/share/models
|
||||
output_path_prefix: "" # /mnt/fangkai_blob/reward_modeling/
|
||||
|
||||
train_file: "hf:codeparrot/apps"
|
||||
dev_file:
|
||||
test_file:
|
||||
|
||||
torch_dtype:
|
||||
_target_: general_util.training_utils.return_torch_dtype
|
||||
dtype: float16
|
||||
|
||||
tokenizer_init:
|
||||
_target_: general_util.tokenization_utils.init_tokenizer
|
||||
tokenizer_path: ${model_name_or_path}
|
||||
padding_side: left
|
||||
|
||||
device_map:
|
||||
_target_: models.utils.return_single_device_map
|
||||
|
||||
model:
|
||||
_target_: models.llama_tp.LlamaForCausalLMDPO.from_pretrained_with_ref_model
|
||||
beta: 0.1
|
||||
gradient_checkpointing: True
|
||||
attn_implementation: "eager"
|
||||
torch_dtype: ${torch_dtype}
|
||||
pad_token_id: 100001
|
||||
# device_map: ${device_map}
|
||||
ref_model:
|
||||
_target_: models.llama_tp.LlamaForCausalLMDPO.from_pretrained
|
||||
pretrained_model_name_or_path: ${model_name_or_path}
|
||||
torch_dtype: ${torch_dtype}
|
||||
attn_implementation: "eager"
|
||||
# device_map: ${device_map}
|
||||
pad_token_id: 100001
|
||||
|
||||
|
||||
chat_prefix: "<｜begin▁of▁sentence｜>You are an AI programming assistant, utilizing the Deepseek Coder model, developed by Deepseek Company, and you only answer questions related to computer science. For politically sensitive questions, security and privacy issues, and other non-computer science questions, you will refuse to answer\n### Instruction:\n"
|
||||
chat_connect: "\n### Response:\n"
|
||||
chat_suffix: "\n<|EOT|>"
|
||||
prompt: "{question}\n\nPlease write a program to solve the above problem under the given time constraints and memory limits."
|
||||
|
||||
read_tensor:
|
||||
_target_: data.combine_dataset.MultiMappingDataset
|
||||
read_fn:
|
||||
_target_: data.apps.APPsReader
|
||||
split: train
|
||||
train_sub_split: train
|
||||
aligner:
|
||||
_target_: data.input_aligner.concat_aligner
|
||||
aligners:
|
||||
- _target_: data.input_aligner.field_extract_aligner
|
||||
input_index_field: problem_id
|
||||
extract_index_field: id
|
||||
extract_fields: [ "pos", "neg" ]
|
||||
extra_file: ${model_path_prefix}/deepseek-coder-7b-instruct-v1.5/apps/train.0shot.tem1.0.n10.v1.0.dpo_v1.1.json
|
||||
- _target_: data.input_aligner.dpo_pair_aligner
|
||||
pos_field: pos
|
||||
neg_field: neg
|
||||
template:
|
||||
_target_: data.input_utils.recompose_template
|
||||
units:
|
||||
chat_prefix: ${chat_prefix}
|
||||
prompt: ${prompt}
|
||||
chat_connect: ${chat_connect}
|
||||
pos: "{pos}"
|
||||
neg: "{neg}"
|
||||
chat_suffix: ${chat_suffix}
|
||||
compositions:
|
||||
prompt: "{chat_prefix}{prompt}{chat_connect}"
|
||||
chosen: "{chat_prefix}{prompt}{chat_connect}{pos}{chat_suffix}"
|
||||
reject: "{chat_prefix}{prompt}{chat_connect}{neg}{chat_suffix}"
|
||||
index_field: problem_id
|
||||
kv_mapping:
|
||||
chosen: chosen
|
||||
reject: reject
|
||||
problem_id: index
|
||||
prompt: prompt
|
||||
|
||||
|
||||
dist_load_data_barrier: False
|
||||
|
||||
extended_vocab:
|
||||
|
||||
# Data collator
|
||||
collator:
|
||||
_target_: data.general_collator.DPOCollator
|
||||
tokenizer: ${tokenizer_init}
|
||||
max_seq_length: 3072
|
||||
|
||||
# Dataloader
|
||||
num_workers: 8
|
||||
prefetch_factor: 2
|
||||
|
||||
# Wiki path pretrain v8.2
|
||||
model_name_or_path: ${model_path_prefix}/deepseek-coder-7b-instruct-v1.5
|
||||
pretrain:
|
||||
resume: latest
|
||||
|
||||
dp_size:
|
||||
tp_size: 4
|
||||
pp_size: 1
|
||||
|
||||
|
||||
exp_name: deepseek-coder-v1.5-ins.7b.apps.code_gen.dpo.V100.tp4.dp8.v1.3.s${seed}
|
||||
exp_notes:
|
||||
output_dir: ${output_path_prefix}experiments/${exp_name} # Fix <pad token id>
|
||||
|
||||
|
||||
do_train: True
|
||||
evaluate_during_training: False
|
||||
|
||||
do_eval: False
|
||||
eval_sub_path: checkpoint-*
|
||||
|
||||
# Training hyper-parameters
|
||||
per_gpu_train_batch_size: 1
|
||||
per_gpu_eval_batch_size: 1
|
||||
#learning_rate: 1e-4
|
||||
learning_rate: 5e-6
|
||||
#learning_rate: 2e-5
|
||||
gradient_accumulation_steps: 16
|
||||
weight_decay: 0.1
|
||||
adam_epsilon: 1e-6
|
||||
adam_betas: "(0.9, 0.98)"
|
||||
#adam_betas: "(0.9, 0.999)"
|
||||
#max_grad_norm: 0.0
|
||||
total_dataset_len: -1
|
||||
max_grad_norm: 1.0
|
||||
num_train_epochs: 2
|
||||
max_steps: 500
|
||||
warmup_proportion: 0.1
|
||||
warmup_steps: 0
|
||||
|
||||
# Optimizer
|
||||
optimizer:
|
||||
use_nvlamb:
|
||||
bit_training:
|
||||
|
||||
logging_steps: 5
|
||||
save_ds_state: True
|
||||
save_steps: 100
|
||||
save_best: False
|
||||
eval_steps: 100
|
||||
ddp_eval: True
|
||||
no_cuda: False
|
||||
seed: 42
|
||||
local_rank: -1
|
||||
fp16: True
|
||||
fp16_opt_level: O1
|
||||
fp16_bfloat16: False
|
||||
|
||||
# Prediction config
|
||||
prediction_cfg:
|
||||
metric: "loss"
|
||||
measure: -1
|
||||
best_checkpoint:
|
||||
best_result:
|
||||
eval_forward_fn:
|
||||
_target_: general_util.evaluator.DefaultForwardFn
|
||||
post_process:
|
||||
_target_: post_processors.dpo.DPOEvalPostProcessor
|
||||
|
||||
|
||||
ds_cfg:
|
||||
train_micro_batch_size_per_gpu: ${per_gpu_train_batch_size}
|
||||
gradient_accumulation_steps: ${gradient_accumulation_steps}
|
||||
optimizer:
|
||||
type: AdamW
|
||||
params:
|
||||
lr: ${learning_rate}
|
||||
betas: [ 0.9, 0.95 ]
|
||||
weight_decay: ${weight_decay}
|
||||
bf16:
|
||||
enabled: False
|
||||
fp16:
|
||||
enabled: True
|
||||
auto_cast: False
|
||||
loss_scale: 0
|
||||
initial_scale_power: 16
|
||||
loss_scale_window: 1000
|
||||
hysteresis: 2
|
||||
consecutive_hysteresis: False
|
||||
min_loss_scale: 1
|
||||
# zero_optimization:
|
||||
# offload_optimizer:
|
||||
# device: cpu
|
||||
# pin_memory: True
|
||||
|
||||
|
||||
summary_helper:
|
||||
_target_: general_util.tensorboard_helper.WandbWriter
|
||||
batch_index_or_keys:
|
||||
outputs_index_or_keys:
|
||||
|
||||
# Temporary variables
|
||||
n_gpu:
|
||||
device:
|
||||
train_batch_size:
|
||||
eval_batch_size:
|
||||
world_size:
|
||||
@@ -0,0 +1,212 @@
|
||||
defaults:
|
||||
- hydra: default
|
||||
# - deepspeed@ds_cfg: train_hybrid_engine_zero3
|
||||
- deepspeed@ds_cfg: train_hybrid_engine_zero1_optim_offload
|
||||
- _self_ # see here for more details: https://hydra.cc/docs/tutorials/basic/your_first_app/defaults/#composition-order-of-primary-config
|
||||
|
||||
hydra:
|
||||
searchpath:
|
||||
- file://conf/
|
||||
|
||||
data_path_prefix: /mnt/fangkai_blob/share/
|
||||
model_path_prefix: /mnt/fangkai_blob/share/models
|
||||
output_path_prefix: /mnt/fangkai_blob/reward_modeling/
|
||||
|
||||
train_file: "hf:codeparrot/apps"
|
||||
dev_file:
|
||||
test_file:
|
||||
|
||||
torch_dtype:
|
||||
_target_: general_util.training_utils.return_torch_dtype
|
||||
dtype: bfloat16
|
||||
|
||||
tokenizer_init:
|
||||
_target_: general_util.tokenization_utils.init_tokenizer
|
||||
tokenizer_path: ${model_name_or_path}
|
||||
padding_side: left
|
||||
|
||||
device_map:
|
||||
_target_: models.utils.return_single_device_map
|
||||
|
||||
model:
|
||||
_target_: models.llama.LlamaForCausalLMDPO.from_pretrained_with_ref_model
|
||||
beta: 0.1
|
||||
gradient_checkpointing: True
|
||||
attn_implementation: "flash_attention_2"
|
||||
torch_dtype: ${torch_dtype}
|
||||
pad_token_id: 100001
|
||||
device_map: ${device_map}
|
||||
ref_model:
|
||||
_target_: models.llama.LlamaForCausalLMDPO.from_pretrained
|
||||
pretrained_model_name_or_path: ${model_name_or_path}
|
||||
torch_dtype: ${torch_dtype}
|
||||
attn_implementation: "flash_attention_2"
|
||||
device_map: ${device_map}
|
||||
pad_token_id: 100001
|
||||
|
||||
|
||||
chat_prefix: "<｜begin▁of▁sentence｜>You are an AI programming assistant, utilizing the Deepseek Coder model, developed by Deepseek Company, and you only answer questions related to computer science. For politically sensitive questions, security and privacy issues, and other non-computer science questions, you will refuse to answer\n### Instruction:\n"
|
||||
chat_connect: "\n### Response:\n"
|
||||
chat_suffix: "\n<|EOT|>"
|
||||
prompt: "{question}\n\nPlease write a program to solve the above problem under the given time constraints and memory limits."
|
||||
|
||||
read_tensor:
|
||||
_target_: data.combine_dataset.MultiMappingDataset
|
||||
read_fn:
|
||||
_target_: data.apps.APPsWithFunctionName
|
||||
split: train
|
||||
train_sub_split: train
|
||||
use_starter_code: True
|
||||
aligner:
|
||||
_target_: data.input_aligner.concat_aligner
|
||||
aligners:
|
||||
- _target_: data.input_aligner.field_extract_aligner
|
||||
input_index_field: problem_id
|
||||
extract_index_field: id
|
||||
extract_fields: [ "pos", "neg" ]
|
||||
extra_file: ${model_path_prefix}/deepseek-coder-7b-instruct-v1.5/apps/train.0shot.tem1.0.n10.v1.0.dpo_v1.1.json
|
||||
- _target_: data.input_aligner.dpo_pair_aligner
|
||||
pos_field: pos
|
||||
neg_field: neg
|
||||
template:
|
||||
_target_: data.input_utils.recompose_template
|
||||
units:
|
||||
chat_prefix: ${chat_prefix}
|
||||
prompt: ${prompt}
|
||||
chat_connect: ${chat_connect}
|
||||
pos: "{pos}"
|
||||
neg: "{neg}"
|
||||
chat_suffix: ${chat_suffix}
|
||||
compositions:
|
||||
prompt: "{chat_prefix}{prompt}{chat_connect}"
|
||||
chosen: "{chat_prefix}{prompt}{chat_connect}{pos}{chat_suffix}"
|
||||
reject: "{chat_prefix}{prompt}{chat_connect}{neg}{chat_suffix}"
|
||||
index_field: problem_id
|
||||
kv_mapping:
|
||||
chosen: chosen
|
||||
reject: reject
|
||||
problem_id: index
|
||||
prompt: prompt
|
||||
|
||||
|
||||
dist_load_data_barrier: False
|
||||
|
||||
extended_vocab:
|
||||
|
||||
# Data collator
|
||||
collator:
|
||||
_target_: data.general_collator.DPOCollator
|
||||
tokenizer: ${tokenizer_init}
|
||||
max_seq_length: 3072
|
||||
|
||||
# Dataloader
|
||||
num_workers: 8
|
||||
prefetch_factor: 2
|
||||
|
||||
# Wiki path pretrain v8.2
|
||||
model_name_or_path: ${model_path_prefix}/deepseek-coder-7b-instruct-v1.5
|
||||
pretrain:
|
||||
resume:
|
||||
|
||||
dp_size:
|
||||
tp_size: 1
|
||||
pp_size: 1
|
||||
|
||||
|
||||
exp_name: deepseek-coder-v1.5-ins.7b.apps.code_gen.dpo.A100.dp4.v1.4.s${seed}
|
||||
exp_notes:
|
||||
output_dir: ${output_path_prefix}experiments/${exp_name} # Fix <pad token id>
|
||||
|
||||
|
||||
do_train: True
|
||||
evaluate_during_training: False
|
||||
|
||||
do_eval: False
|
||||
eval_sub_path: checkpoint-*
|
||||
|
||||
# Training hyper-parameters
|
||||
per_gpu_train_batch_size: 1
|
||||
per_gpu_eval_batch_size: 1
|
||||
#learning_rate: 1e-4
|
||||
learning_rate: 5e-6
|
||||
#learning_rate: 2e-5
|
||||
gradient_accumulation_steps: 32
|
||||
weight_decay: 0.1
|
||||
adam_epsilon: 1e-6
|
||||
adam_betas: "(0.9, 0.98)"
|
||||
#adam_betas: "(0.9, 0.999)"
|
||||
#max_grad_norm: 0.0
|
||||
total_dataset_len: -1
|
||||
max_grad_norm: 1.0
|
||||
num_train_epochs: 2
|
||||
max_steps: 500
|
||||
warmup_proportion: 0.1
|
||||
warmup_steps: 0
|
||||
|
||||
# Optimizer
|
||||
optimizer:
|
||||
use_nvlamb:
|
||||
bit_training:
|
||||
|
||||
logging_steps: 5
|
||||
save_ds_state: True
|
||||
save_steps: 100
|
||||
save_best: False
|
||||
eval_steps: 100
|
||||
ddp_eval: True
|
||||
no_cuda: False
|
||||
seed: 42
|
||||
local_rank: -1
|
||||
fp16: True
|
||||
fp16_opt_level: O1
|
||||
fp16_bfloat16: True
|
||||
|
||||
# Prediction config
|
||||
prediction_cfg:
|
||||
metric: "loss"
|
||||
measure: -1
|
||||
best_checkpoint:
|
||||
best_result:
|
||||
eval_forward_fn:
|
||||
_target_: general_util.evaluator.DefaultForwardFn
|
||||
post_process:
|
||||
_target_: post_processors.dpo.DPOEvalPostProcessor
|
||||
|
||||
|
||||
ds_cfg:
|
||||
train_micro_batch_size_per_gpu: ${per_gpu_train_batch_size}
|
||||
gradient_accumulation_steps: ${gradient_accumulation_steps}
|
||||
optimizer:
|
||||
type: AdamW
|
||||
params:
|
||||
lr: ${learning_rate}
|
||||
betas: [ 0.9, 0.95 ]
|
||||
weight_decay: ${weight_decay}
|
||||
# bf16:
|
||||
# enabled: False
|
||||
# fp16:
|
||||
# enabled: True
|
||||
# auto_cast: False
|
||||
# loss_scale: 0
|
||||
# initial_scale_power: 16
|
||||
# loss_scale_window: 1000
|
||||
# hysteresis: 2
|
||||
# consecutive_hysteresis: False
|
||||
# min_loss_scale: 1
|
||||
# zero_optimization:
|
||||
# offload_optimizer:
|
||||
# device: cpu
|
||||
# pin_memory: True
|
||||
|
||||
|
||||
summary_helper:
|
||||
_target_: general_util.tensorboard_helper.WandbWriter
|
||||
batch_index_or_keys:
|
||||
outputs_index_or_keys:
|
||||
|
||||
# Temporary variables
|
||||
n_gpu:
|
||||
device:
|
||||
train_batch_size:
|
||||
eval_batch_size:
|
||||
world_size:
|
||||
@@ -0,0 +1,212 @@
|
||||
defaults:
|
||||
- hydra: default
|
||||
# - deepspeed@ds_cfg: train_hybrid_engine_zero3
|
||||
- deepspeed@ds_cfg: train_hybrid_engine_zero2_optim_offload
|
||||
- _self_ # see here for more details: https://hydra.cc/docs/tutorials/basic/your_first_app/defaults/#composition-order-of-primary-config
|
||||
|
||||
hydra:
|
||||
searchpath:
|
||||
- file://conf/
|
||||
|
||||
data_path_prefix: /mnt/fangkai_blob/share/
|
||||
model_path_prefix: /mnt/fangkai_blob/share/models
|
||||
output_path_prefix: /mnt/fangkai_blob/reward_modeling/
|
||||
|
||||
train_file: "hf:codeparrot/apps"
|
||||
dev_file:
|
||||
test_file:
|
||||
|
||||
torch_dtype:
|
||||
_target_: general_util.training_utils.return_torch_dtype
|
||||
dtype: float16
|
||||
|
||||
tokenizer_init:
|
||||
_target_: general_util.tokenization_utils.init_tokenizer
|
||||
tokenizer_path: ${model_name_or_path}
|
||||
padding_side: left
|
||||
|
||||
device_map:
|
||||
_target_: models.utils.return_single_device_map
|
||||
|
||||
model:
|
||||
_target_: models.llama_tp.LlamaForCausalLMDPO.from_pretrained_with_ref_model
|
||||
beta: 0.1
|
||||
gradient_checkpointing: True
|
||||
attn_implementation: "eager"
|
||||
torch_dtype: ${torch_dtype}
|
||||
pad_token_id: 100001
|
||||
# device_map: ${device_map}
|
||||
ref_model:
|
||||
_target_: models.llama_tp.LlamaForCausalLMDPO.from_pretrained
|
||||
pretrained_model_name_or_path: ${model_name_or_path}
|
||||
torch_dtype: ${torch_dtype}
|
||||
attn_implementation: "eager"
|
||||
# device_map: ${device_map}
|
||||
pad_token_id: 100001
|
||||
|
||||
|
||||
chat_prefix: "<｜begin▁of▁sentence｜>You are an AI programming assistant, utilizing the Deepseek Coder model, developed by Deepseek Company, and you only answer questions related to computer science. For politically sensitive questions, security and privacy issues, and other non-computer science questions, you will refuse to answer\n### Instruction:\n"
|
||||
chat_connect: "\n### Response:\n"
|
||||
chat_suffix: "\n<|EOT|>"
|
||||
prompt: "{question}\n\nPlease write a program to solve the above problem under the given time constraints and memory limits."
|
||||
|
||||
read_tensor:
|
||||
_target_: data.combine_dataset.MultiMappingDataset
|
||||
read_fn:
|
||||
_target_: data.apps.APPsWithFunctionName
|
||||
split: train
|
||||
train_sub_split: train
|
||||
use_starter_code: True
|
||||
aligner:
|
||||
_target_: data.input_aligner.concat_aligner
|
||||
aligners:
|
||||
- _target_: data.input_aligner.field_extract_aligner
|
||||
input_index_field: problem_id
|
||||
extract_index_field: id
|
||||
extract_fields: [ "pos", "neg" ]
|
||||
extra_file: ${model_path_prefix}/deepseek-coder-7b-instruct-v1.5/apps/train.0shot.tem1.0.n10.v1.0.dpo_v1.1.json
|
||||
- _target_: data.input_aligner.dpo_pair_aligner
|
||||
pos_field: pos
|
||||
neg_field: neg
|
||||
template:
|
||||
_target_: data.input_utils.recompose_template
|
||||
units:
|
||||
chat_prefix: ${chat_prefix}
|
||||
prompt: ${prompt}
|
||||
chat_connect: ${chat_connect}
|
||||
pos: "{pos}"
|
||||
neg: "{neg}"
|
||||
chat_suffix: ${chat_suffix}
|
||||
compositions:
|
||||
prompt: "{chat_prefix}{prompt}{chat_connect}"
|
||||
chosen: "{chat_prefix}{prompt}{chat_connect}{pos}{chat_suffix}"
|
||||
reject: "{chat_prefix}{prompt}{chat_connect}{neg}{chat_suffix}"
|
||||
index_field: problem_id
|
||||
kv_mapping:
|
||||
chosen: chosen
|
||||
reject: reject
|
||||
problem_id: index
|
||||
prompt: prompt
|
||||
|
||||
|
||||
dist_load_data_barrier: False
|
||||
|
||||
extended_vocab:
|
||||
|
||||
# Data collator
|
||||
collator:
|
||||
_target_: data.general_collator.DPOCollator
|
||||
tokenizer: ${tokenizer_init}
|
||||
max_seq_length: 3072
|
||||
|
||||
# Dataloader
|
||||
num_workers: 8
|
||||
prefetch_factor: 2
|
||||
|
||||
# Wiki path pretrain v8.2
|
||||
model_name_or_path: ${model_path_prefix}/deepseek-coder-7b-instruct-v1.5
|
||||
pretrain:
|
||||
resume:
|
||||
|
||||
dp_size:
|
||||
tp_size: 4
|
||||
pp_size: 1
|
||||
|
||||
|
||||
exp_name: deepseek-coder-v1.5-ins.7b.apps.code_gen.dpo.V100.tp4.dp4.v1.4.s${seed}
|
||||
exp_notes:
|
||||
output_dir: ${output_path_prefix}experiments/${exp_name} # Fix <pad token id>
|
||||
|
||||
|
||||
do_train: True
|
||||
evaluate_during_training: False
|
||||
|
||||
do_eval: False
|
||||
eval_sub_path: checkpoint-*
|
||||
|
||||
# Training hyper-parameters
|
||||
per_gpu_train_batch_size: 1
|
||||
per_gpu_eval_batch_size: 1
|
||||
#learning_rate: 1e-4
|
||||
learning_rate: 5e-6
|
||||
#learning_rate: 2e-5
|
||||
gradient_accumulation_steps: 32
|
||||
weight_decay: 0.1
|
||||
adam_epsilon: 1e-6
|
||||
adam_betas: "(0.9, 0.98)"
|
||||
#adam_betas: "(0.9, 0.999)"
|
||||
#max_grad_norm: 0.0
|
||||
total_dataset_len: -1
|
||||
max_grad_norm: 1.0
|
||||
num_train_epochs: 2
|
||||
max_steps: 500
|
||||
warmup_proportion: 0.1
|
||||
warmup_steps: 0
|
||||
|
||||
# Optimizer
|
||||
optimizer:
|
||||
use_nvlamb:
|
||||
bit_training:
|
||||
|
||||
logging_steps: 5
|
||||
save_ds_state: True
|
||||
save_steps: 100
|
||||
save_best: False
|
||||
eval_steps: 100
|
||||
ddp_eval: True
|
||||
no_cuda: False
|
||||
seed: 42
|
||||
local_rank: -1
|
||||
fp16: True
|
||||
fp16_opt_level: O1
|
||||
fp16_bfloat16: False
|
||||
|
||||
# Prediction config
|
||||
prediction_cfg:
|
||||
metric: "loss"
|
||||
measure: -1
|
||||
best_checkpoint:
|
||||
best_result:
|
||||
eval_forward_fn:
|
||||
_target_: general_util.evaluator.DefaultForwardFn
|
||||
post_process:
|
||||
_target_: post_processors.dpo.DPOEvalPostProcessor
|
||||
|
||||
|
||||
ds_cfg:
|
||||
train_micro_batch_size_per_gpu: ${per_gpu_train_batch_size}
|
||||
gradient_accumulation_steps: ${gradient_accumulation_steps}
|
||||
optimizer:
|
||||
type: AdamW
|
||||
params:
|
||||
lr: ${learning_rate}
|
||||
betas: [ 0.9, 0.95 ]
|
||||
weight_decay: ${weight_decay}
|
||||
bf16:
|
||||
enabled: False
|
||||
fp16:
|
||||
enabled: True
|
||||
auto_cast: False
|
||||
loss_scale: 0
|
||||
initial_scale_power: 16
|
||||
loss_scale_window: 1000
|
||||
hysteresis: 2
|
||||
consecutive_hysteresis: False
|
||||
min_loss_scale: 1
|
||||
# zero_optimization:
|
||||
# offload_optimizer:
|
||||
# device: cpu
|
||||
# pin_memory: True
|
||||
|
||||
|
||||
summary_helper:
|
||||
_target_: general_util.tensorboard_helper.WandbWriter
|
||||
batch_index_or_keys:
|
||||
outputs_index_or_keys:
|
||||
|
||||
# Temporary variables
|
||||
n_gpu:
|
||||
device:
|
||||
train_batch_size:
|
||||
eval_batch_size:
|
||||
world_size:
|
||||
@@ -0,0 +1,213 @@
|
||||
defaults:
|
||||
- hydra: default
|
||||
# - deepspeed@ds_cfg: train_hybrid_engine_zero3
|
||||
- deepspeed@ds_cfg: train_hybrid_engine_zero1_optim_offload
|
||||
- _self_ # see here for more details: https://hydra.cc/docs/tutorials/basic/your_first_app/defaults/#composition-order-of-primary-config
|
||||
|
||||
hydra:
|
||||
searchpath:
|
||||
- file://conf/
|
||||
|
||||
data_path_prefix: /mnt/fangkai_blob/share/
|
||||
model_path_prefix: /mnt/fangkai_blob/share/models
|
||||
output_path_prefix: /mnt/fangkai_blob/reward_modeling/
|
||||
|
||||
train_file: "hf:codeparrot/apps"
|
||||
dev_file:
|
||||
test_file:
|
||||
|
||||
torch_dtype:
|
||||
_target_: general_util.training_utils.return_torch_dtype
|
||||
dtype: float16
|
||||
|
||||
tokenizer_init:
|
||||
_target_: general_util.tokenization_utils.init_tokenizer
|
||||
tokenizer_path: ${model_name_or_path}
|
||||
padding_side: left
|
||||
|
||||
device_map:
|
||||
_target_: models.utils.return_single_device_map
|
||||
|
||||
model:
|
||||
_target_: models.llama_tp.LlamaForCausalLMDPO.from_pretrained_with_ref_model
|
||||
beta: 0.4
|
||||
gradient_checkpointing: True
|
||||
attn_implementation: "eager"
|
||||
torch_dtype: ${torch_dtype}
|
||||
pad_token_id: 100001
|
||||
# device_map: ${device_map}
|
||||
ref_model:
|
||||
_target_: models.llama_tp.LlamaForCausalLMDPO.from_pretrained
|
||||
pretrained_model_name_or_path: ${model_name_or_path}
|
||||
torch_dtype: ${torch_dtype}
|
||||
attn_implementation: "eager"
|
||||
# device_map: ${device_map}
|
||||
pad_token_id: 100001
|
||||
|
||||
|
||||
chat_prefix: "<｜begin▁of▁sentence｜>You are an AI programming assistant, utilizing the Deepseek Coder model, developed by Deepseek Company, and you only answer questions related to computer science. For politically sensitive questions, security and privacy issues, and other non-computer science questions, you will refuse to answer\n### Instruction:\n"
|
||||
chat_connect: "\n### Response:\n"
|
||||
chat_suffix: "\n<|EOT|>"
|
||||
prompt: "{question}\n\nPlease write a program to solve the above problem under the given time constraints and memory limits."
|
||||
|
||||
read_tensor:
|
||||
_target_: data.combine_dataset.MultiMappingDataset
|
||||
read_fn:
|
||||
_target_: data.apps.APPsWithFunctionName
|
||||
split: train
|
||||
train_sub_split: train
|
||||
use_starter_code: True
|
||||
aligner:
|
||||
_target_: data.input_aligner.concat_aligner
|
||||
aligners:
|
||||
- _target_: data.input_aligner.field_extract_aligner
|
||||
input_index_field: problem_id
|
||||
extract_index_field: id
|
||||
extract_fields: [ "pos", "neg" ]
|
||||
extra_file: ${model_path_prefix}/deepseek-coder-7b-instruct-v1.5/apps/train.0shot.tem1.0.n10.v2.0.pseudo_input_output.v1.0.clean.dpo_m2_low0.5.json
|
||||
- _target_: data.input_aligner.flat_aligner
|
||||
input_index_field: problem_id
|
||||
extract_field: [ "pos", "neg" ]
|
||||
mode: multi
|
||||
template:
|
||||
_target_: data.input_utils.recompose_template
|
||||
units:
|
||||
chat_prefix: ${chat_prefix}
|
||||
prompt: ${prompt}
|
||||
chat_connect: ${chat_connect}
|
||||
pos: "{pos}"
|
||||
neg: "{neg}"
|
||||
chat_suffix: ${chat_suffix}
|
||||
compositions:
|
||||
prompt: "{chat_prefix}{prompt}{chat_connect}"
|
||||
chosen: "{chat_prefix}{prompt}{chat_connect}{pos}{chat_suffix}"
|
||||
reject: "{chat_prefix}{prompt}{chat_connect}{neg}{chat_suffix}"
|
||||
index_field: problem_id
|
||||
kv_mapping:
|
||||
chosen: chosen
|
||||
reject: reject
|
||||
problem_id: index
|
||||
prompt: prompt
|
||||
|
||||
|
||||
dist_load_data_barrier: False
|
||||
|
||||
extended_vocab:
|
||||
|
||||
# Data collator
|
||||
collator:
|
||||
_target_: data.general_collator.DPOCollator
|
||||
tokenizer: ${tokenizer_init}
|
||||
max_seq_length: 3072
|
||||
|
||||
# Dataloader
|
||||
num_workers: 8
|
||||
prefetch_factor: 2
|
||||
|
||||
# Wiki path pretrain v8.2
|
||||
model_name_or_path: ${model_path_prefix}/deepseek-coder-7b-instruct-v1.5
|
||||
pretrain:
|
||||
resume: latest
|
||||
|
||||
dp_size:
|
||||
tp_size: 8
|
||||
pp_size: 1
|
||||
|
||||
|
||||
exp_name: deepseek-coder-v1.5-ins.7b.apps.code_gen.pseudo-test-10.sc.dpo.V100.tp8.dp4.v1.0.s${seed}
|
||||
exp_notes:
|
||||
output_dir: ${output_path_prefix}experiments/${exp_name} # Fix <pad token id>
|
||||
|
||||
|
||||
do_train: True
|
||||
evaluate_during_training: False
|
||||
|
||||
do_eval: False
|
||||
eval_sub_path: checkpoint-*
|
||||
|
||||
# Training hyper-parameters
|
||||
per_gpu_train_batch_size: 1
|
||||
per_gpu_eval_batch_size: 1
|
||||
#learning_rate: 1e-4
|
||||
learning_rate: 1e-6
|
||||
#learning_rate: 2e-5
|
||||
gradient_accumulation_steps: 32
|
||||
weight_decay: 0.1
|
||||
adam_epsilon: 1e-6
|
||||
adam_betas: "(0.9, 0.98)"
|
||||
#adam_betas: "(0.9, 0.999)"
|
||||
#max_grad_norm: 0.0
|
||||
total_dataset_len: -1
|
||||
max_grad_norm: 1.0
|
||||
num_train_epochs: 1
|
||||
max_steps: 0
|
||||
warmup_proportion: 0.1
|
||||
warmup_steps: 0
|
||||
|
||||
# Optimizer
|
||||
optimizer:
|
||||
use_nvlamb:
|
||||
bit_training:
|
||||
|
||||
logging_steps: 5
|
||||
save_ds_state: True
|
||||
save_steps: 100
|
||||
save_best: False
|
||||
eval_steps: 100
|
||||
ddp_eval: True
|
||||
no_cuda: False
|
||||
seed: 42
|
||||
local_rank: -1
|
||||
fp16: True
|
||||
fp16_opt_level: O1
|
||||
fp16_bfloat16: False
|
||||
|
||||
# Prediction config
|
||||
prediction_cfg:
|
||||
metric: "loss"
|
||||
measure: -1
|
||||
best_checkpoint:
|
||||
best_result:
|
||||
eval_forward_fn:
|
||||
_target_: general_util.evaluator.DefaultForwardFn
|
||||
post_process:
|
||||
_target_: post_processors.dpo.DPOEvalPostProcessor
|
||||
|
||||
|
||||
ds_cfg:
|
||||
train_micro_batch_size_per_gpu: ${per_gpu_train_batch_size}
|
||||
gradient_accumulation_steps: ${gradient_accumulation_steps}
|
||||
optimizer:
|
||||
type: AdamW
|
||||
params:
|
||||
lr: ${learning_rate}
|
||||
betas: [ 0.9, 0.95 ]
|
||||
weight_decay: ${weight_decay}
|
||||
bf16:
|
||||
enabled: False
|
||||
fp16:
|
||||
enabled: True
|
||||
auto_cast: False
|
||||
loss_scale: 0
|
||||
initial_scale_power: 16
|
||||
loss_scale_window: 1000
|
||||
hysteresis: 2
|
||||
consecutive_hysteresis: False
|
||||
min_loss_scale: 1
|
||||
# zero_optimization:
|
||||
# offload_optimizer:
|
||||
# device: cpu
|
||||
# pin_memory: True
|
||||
|
||||
|
||||
summary_helper:
|
||||
_target_: general_util.tensorboard_helper.WandbWriter
|
||||
batch_index_or_keys:
|
||||
outputs_index_or_keys:
|
||||
|
||||
# Temporary variables
|
||||
n_gpu:
|
||||
device:
|
||||
train_batch_size:
|
||||
eval_batch_size:
|
||||
world_size:
|
||||
@@ -0,0 +1,213 @@
|
||||
defaults:
|
||||
- hydra: default
|
||||
# - deepspeed@ds_cfg: train_hybrid_engine_zero3
|
||||
- deepspeed@ds_cfg: train_hybrid_engine_zero1
|
||||
- _self_ # see here for more details: https://hydra.cc/docs/tutorials/basic/your_first_app/defaults/#composition-order-of-primary-config
|
||||
|
||||
hydra:
|
||||
searchpath:
|
||||
- file://conf/
|
||||
|
||||
data_path_prefix: /mnt/fangkai_blob/share/
|
||||
model_path_prefix: /mnt/fangkai_blob/share/models
|
||||
output_path_prefix: /mnt/fangkai_blob/reward_modeling/
|
||||
|
||||
train_file: "hf:codeparrot/apps"
|
||||
dev_file:
|
||||
test_file:
|
||||
|
||||
torch_dtype:
|
||||
_target_: general_util.training_utils.return_torch_dtype
|
||||
dtype: float16
|
||||
|
||||
tokenizer_init:
|
||||
_target_: general_util.tokenization_utils.init_tokenizer
|
||||
tokenizer_path: ${model_name_or_path}
|
||||
padding_side: left
|
||||
|
||||
device_map:
|
||||
_target_: models.utils.return_single_device_map
|
||||
|
||||
model:
|
||||
_target_: models.llama.LlamaForCausalLMDPO.from_pretrained_with_ref_model
|
||||
beta: 0.4
|
||||
gradient_checkpointing: True
|
||||
attn_implementation: "flash_attention_2"
|
||||
torch_dtype: ${torch_dtype}
|
||||
pad_token_id: 100001
|
||||
device_map: ${device_map}
|
||||
ref_model:
|
||||
_target_: models.llama.LlamaForCausalLMDPO.from_pretrained
|
||||
pretrained_model_name_or_path: ${model_name_or_path}
|
||||
torch_dtype: ${torch_dtype}
|
||||
attn_implementation: "flash_attention_2"
|
||||
device_map: ${device_map}
|
||||
pad_token_id: 100001
|
||||
|
||||
|
||||
chat_prefix: "<｜begin▁of▁sentence｜>You are an AI programming assistant, utilizing the Deepseek Coder model, developed by Deepseek Company, and you only answer questions related to computer science. For politically sensitive questions, security and privacy issues, and other non-computer science questions, you will refuse to answer\n### Instruction:\n"
|
||||
chat_connect: "\n### Response:\n"
|
||||
chat_suffix: "\n<|EOT|>"
|
||||
prompt: "{question}\n\nPlease write a program to solve the above problem under the given time constraints and memory limits."
|
||||
|
||||
read_tensor:
|
||||
_target_: data.combine_dataset.MultiMappingDataset
|
||||
read_fn:
|
||||
_target_: data.apps.APPsWithFunctionName
|
||||
split: train
|
||||
train_sub_split: train
|
||||
use_starter_code: True
|
||||
aligner:
|
||||
_target_: data.input_aligner.concat_aligner
|
||||
aligners:
|
||||
- _target_: data.input_aligner.field_extract_aligner
|
||||
input_index_field: problem_id
|
||||
extract_index_field: id
|
||||
extract_fields: [ "pos", "neg" ]
|
||||
extra_file: ${model_path_prefix}/deepseek-coder-7b-instruct-v1.5/apps/train.0shot.tem1.0.n10.v2.0.pseudo_input_output.v1.0.clean.dpo_m3_low0.5.json
|
||||
- _target_: data.input_aligner.flat_aligner
|
||||
input_index_field: problem_id
|
||||
extract_field: [ "pos", "neg" ]
|
||||
mode: multi
|
||||
template:
|
||||
_target_: data.input_utils.recompose_template
|
||||
units:
|
||||
chat_prefix: ${chat_prefix}
|
||||
prompt: ${prompt}
|
||||
chat_connect: ${chat_connect}
|
||||
pos: "{pos}"
|
||||
neg: "{neg}"
|
||||
chat_suffix: ${chat_suffix}
|
||||
compositions:
|
||||
prompt: "{chat_prefix}{prompt}{chat_connect}"
|
||||
chosen: "{chat_prefix}{prompt}{chat_connect}{pos}{chat_suffix}"
|
||||
reject: "{chat_prefix}{prompt}{chat_connect}{neg}{chat_suffix}"
|
||||
index_field: problem_id
|
||||
kv_mapping:
|
||||
chosen: chosen
|
||||
reject: reject
|
||||
problem_id: index
|
||||
prompt: prompt
|
||||
|
||||
|
||||
dist_load_data_barrier: False
|
||||
|
||||
extended_vocab:
|
||||
|
||||
# Data collator
|
||||
collator:
|
||||
_target_: data.general_collator.DPOCollator
|
||||
tokenizer: ${tokenizer_init}
|
||||
max_seq_length: 3072
|
||||
|
||||
# Dataloader
|
||||
num_workers: 8
|
||||
prefetch_factor: 2
|
||||
|
||||
# Wiki path pretrain v8.2
|
||||
model_name_or_path: ${model_path_prefix}/deepseek-coder-7b-instruct-v1.5
|
||||
pretrain:
|
||||
resume:
|
||||
|
||||
dp_size:
|
||||
tp_size: 1
|
||||
pp_size: 1
|
||||
|
||||
|
||||
exp_name: deepseek-coder-v1.5-ins.7b.apps.code_gen.pseudo-test-10.sc.dpo.H100.dp8.v1.1.s${seed}
|
||||
exp_notes:
|
||||
output_dir: ${output_path_prefix}experiments/${exp_name} # Fix <pad token id>
|
||||
|
||||
|
||||
do_train: True
|
||||
evaluate_during_training: False
|
||||
|
||||
do_eval: False
|
||||
eval_sub_path: checkpoint-*
|
||||
|
||||
# Training hyper-parameters
|
||||
per_gpu_train_batch_size: 1
|
||||
per_gpu_eval_batch_size: 1
|
||||
#learning_rate: 1e-4
|
||||
learning_rate: 1e-6
|
||||
#learning_rate: 2e-5
|
||||
gradient_accumulation_steps: 16
|
||||
weight_decay: 0.1
|
||||
adam_epsilon: 1e-6
|
||||
adam_betas: "(0.9, 0.98)"
|
||||
#adam_betas: "(0.9, 0.999)"
|
||||
#max_grad_norm: 0.0
|
||||
total_dataset_len: -1
|
||||
max_grad_norm: 1.0
|
||||
num_train_epochs: 1
|
||||
max_steps: 0
|
||||
warmup_proportion: 0.1
|
||||
warmup_steps: 0
|
||||
|
||||
# Optimizer
|
||||
optimizer:
|
||||
use_nvlamb:
|
||||
bit_training:
|
||||
|
||||
logging_steps: 5
|
||||
save_ds_state: True
|
||||
save_steps: 100
|
||||
save_best: False
|
||||
eval_steps: 100
|
||||
ddp_eval: True
|
||||
no_cuda: False
|
||||
seed: 42
|
||||
local_rank: -1
|
||||
fp16: True
|
||||
fp16_opt_level: O1
|
||||
fp16_bfloat16: False
|
||||
|
||||
# Prediction config
|
||||
prediction_cfg:
|
||||
metric: "loss"
|
||||
measure: -1
|
||||
best_checkpoint:
|
||||
best_result:
|
||||
eval_forward_fn:
|
||||
_target_: general_util.evaluator.DefaultForwardFn
|
||||
post_process:
|
||||
_target_: post_processors.dpo.DPOEvalPostProcessor
|
||||
|
||||
|
||||
ds_cfg:
|
||||
train_micro_batch_size_per_gpu: ${per_gpu_train_batch_size}
|
||||
gradient_accumulation_steps: ${gradient_accumulation_steps}
|
||||
optimizer:
|
||||
type: AdamW
|
||||
params:
|
||||
lr: ${learning_rate}
|
||||
betas: [ 0.9, 0.95 ]
|
||||
weight_decay: ${weight_decay}
|
||||
bf16:
|
||||
enabled: False
|
||||
fp16:
|
||||
enabled: True
|
||||
auto_cast: False
|
||||
loss_scale: 0
|
||||
initial_scale_power: 16
|
||||
loss_scale_window: 1000
|
||||
hysteresis: 2
|
||||
consecutive_hysteresis: False
|
||||
min_loss_scale: 1
|
||||
# zero_optimization:
|
||||
# offload_optimizer:
|
||||
# device: cpu
|
||||
# pin_memory: True
|
||||
|
||||
|
||||
summary_helper:
|
||||
_target_: general_util.tensorboard_helper.WandbWriter
|
||||
batch_index_or_keys:
|
||||
outputs_index_or_keys:
|
||||
|
||||
# Temporary variables
|
||||
n_gpu:
|
||||
device:
|
||||
train_batch_size:
|
||||
eval_batch_size:
|
||||
world_size:
|
||||
@@ -0,0 +1,214 @@
|
||||
defaults:
|
||||
- hydra: default
|
||||
# - deepspeed@ds_cfg: train_hybrid_engine_zero3
|
||||
- deepspeed@ds_cfg: train_hybrid_engine_zero1_optim_offload
|
||||
- _self_ # see here for more details: https://hydra.cc/docs/tutorials/basic/your_first_app/defaults/#composition-order-of-primary-config
|
||||
|
||||
hydra:
|
||||
searchpath:
|
||||
- file://conf/
|
||||
|
||||
data_path_prefix: /mnt/fangkai_blob/share/
|
||||
model_path_prefix: /mnt/fangkai_blob/share/models
|
||||
output_path_prefix: /mnt/fangkai_blob/reward_modeling/
|
||||
|
||||
train_file: "hf:codeparrot/apps"
|
||||
dev_file:
|
||||
test_file:
|
||||
|
||||
torch_dtype:
|
||||
_target_: general_util.training_utils.return_torch_dtype
|
||||
dtype: float16
|
||||
|
||||
tokenizer_init:
|
||||
_target_: general_util.tokenization_utils.init_tokenizer
|
||||
tokenizer_path: ${model_name_or_path}
|
||||
padding_side: left
|
||||
|
||||
device_map:
|
||||
_target_: models.utils.return_single_device_map
|
||||
|
||||
model:
|
||||
_target_: models.llama_tp.LlamaForCausalLMDPO.from_pretrained_with_ref_model
|
||||
beta: 0.4
|
||||
gradient_checkpointing: True
|
||||
attn_implementation: "eager"
|
||||
torch_dtype: ${torch_dtype}
|
||||
pad_token_id: 100001
|
||||
# device_map: ${device_map}
|
||||
ref_model:
|
||||
_target_: models.llama_tp.LlamaForCausalLMDPO.from_pretrained
|
||||
pretrained_model_name_or_path: ${model_name_or_path}
|
||||
torch_dtype: ${torch_dtype}
|
||||
attn_implementation: "eager"
|
||||
# device_map: ${device_map}
|
||||
pad_token_id: 100001
|
||||
|
||||
|
||||
chat_prefix: "<｜begin▁of▁sentence｜>You are an AI programming assistant, utilizing the Deepseek Coder model, developed by Deepseek Company, and you only answer questions related to computer science. For politically sensitive questions, security and privacy issues, and other non-computer science questions, you will refuse to answer\n### Instruction:\n"
|
||||
chat_connect: "\n### Response:\n"
|
||||
chat_suffix: "\n<|EOT|>"
|
||||
prompt: "{question}\n\nPlease write a program to solve the above problem under the given time constraints and memory limits."
|
||||
|
||||
read_tensor:
|
||||
_target_: data.combine_dataset.MultiMappingDataset
|
||||
read_fn:
|
||||
_target_: data.apps.APPsWithFunctionName
|
||||
split: train
|
||||
train_sub_split: train
|
||||
use_starter_code: True
|
||||
aligner:
|
||||
_target_: data.input_aligner.concat_aligner
|
||||
aligners:
|
||||
- _target_: data.input_aligner.field_extract_aligner
|
||||
input_index_field: problem_id
|
||||
extract_index_field: id
|
||||
extract_fields: [ "pos", "neg" ]
|
||||
# extra_file: ${model_path_prefix}/deepseek-coder-7b-instruct-v1.5/apps/train.0shot.tem1.0.n10.v2.0.pseudo_input_output.v1.0.clean.dpo_m3_low0.5.json
|
||||
extra_file: ${model_path_prefix}/deepseek-coder-7b-instruct-v1.5/apps/train.0shot.tem1.0.n10.v2.0.pseudo_input_output.v1.0.clean.dpo_m3_low0.5.fix.json
|
||||
- _target_: data.input_aligner.flat_aligner
|
||||
input_index_field: problem_id
|
||||
extract_field: [ "pos", "neg" ]
|
||||
mode: multi
|
||||
template:
|
||||
_target_: data.input_utils.recompose_template
|
||||
units:
|
||||
chat_prefix: ${chat_prefix}
|
||||
prompt: ${prompt}
|
||||
chat_connect: ${chat_connect}
|
||||
pos: "{pos}"
|
||||
neg: "{neg}"
|
||||
chat_suffix: ${chat_suffix}
|
||||
compositions:
|
||||
prompt: "{chat_prefix}{prompt}{chat_connect}"
|
||||
chosen: "{chat_prefix}{prompt}{chat_connect}{pos}{chat_suffix}"
|
||||
reject: "{chat_prefix}{prompt}{chat_connect}{neg}{chat_suffix}"
|
||||
index_field: problem_id
|
||||
kv_mapping:
|
||||
chosen: chosen
|
||||
reject: reject
|
||||
problem_id: index
|
||||
prompt: prompt
|
||||
|
||||
|
||||
dist_load_data_barrier: False
|
||||
|
||||
extended_vocab:
|
||||
|
||||
# Data collator
|
||||
collator:
|
||||
_target_: data.general_collator.DPOCollator
|
||||
tokenizer: ${tokenizer_init}
|
||||
max_seq_length: 3072
|
||||
|
||||
# Dataloader
|
||||
num_workers: 8
|
||||
prefetch_factor: 2
|
||||
|
||||
# Wiki path pretrain v8.2
|
||||
model_name_or_path: ${model_path_prefix}/deepseek-coder-7b-instruct-v1.5
|
||||
pretrain:
|
||||
resume:
|
||||
|
||||
dp_size:
|
||||
tp_size: 8
|
||||
pp_size: 1
|
||||
|
||||
|
||||
exp_name: deepseek-coder-v1.5-ins.7b.apps.code_gen.pseudo-test-10.sc.dpo.V100.tp8.dp8.v1.1.s${seed}.fix-rerun
|
||||
exp_notes:
|
||||
output_dir: ${output_path_prefix}experiments/${exp_name} # Fix <pad token id>
|
||||
|
||||
|
||||
do_train: True
|
||||
evaluate_during_training: False
|
||||
|
||||
do_eval: False
|
||||
eval_sub_path: checkpoint-*
|
||||
|
||||
# Training hyper-parameters
|
||||
per_gpu_train_batch_size: 1
|
||||
per_gpu_eval_batch_size: 1
|
||||
#learning_rate: 1e-4
|
||||
learning_rate: 1e-6
|
||||
#learning_rate: 2e-5
|
||||
gradient_accumulation_steps: 16
|
||||
weight_decay: 0.1
|
||||
adam_epsilon: 1e-6
|
||||
adam_betas: "(0.9, 0.98)"
|
||||
#adam_betas: "(0.9, 0.999)"
|
||||
#max_grad_norm: 0.0
|
||||
total_dataset_len: -1
|
||||
max_grad_norm: 1.0
|
||||
num_train_epochs: 1
|
||||
max_steps: 0
|
||||
warmup_proportion: 0.1
|
||||
warmup_steps: 0
|
||||
|
||||
# Optimizer
|
||||
optimizer:
|
||||
use_nvlamb:
|
||||
bit_training:
|
||||
|
||||
logging_steps: 5
|
||||
save_ds_state: True
|
||||
save_steps: 100
|
||||
save_best: False
|
||||
eval_steps: 100
|
||||
ddp_eval: True
|
||||
no_cuda: False
|
||||
seed: 42
|
||||
local_rank: -1
|
||||
fp16: True
|
||||
fp16_opt_level: O1
|
||||
fp16_bfloat16: False
|
||||
|
||||
# Prediction config
|
||||
prediction_cfg:
|
||||
metric: "loss"
|
||||
measure: -1
|
||||
best_checkpoint:
|
||||
best_result:
|
||||
eval_forward_fn:
|
||||
_target_: general_util.evaluator.DefaultForwardFn
|
||||
post_process:
|
||||
_target_: post_processors.dpo.DPOEvalPostProcessor
|
||||
|
||||
|
||||
ds_cfg:
|
||||
train_micro_batch_size_per_gpu: ${per_gpu_train_batch_size}
|
||||
gradient_accumulation_steps: ${gradient_accumulation_steps}
|
||||
optimizer:
|
||||
type: AdamW
|
||||
params:
|
||||
lr: ${learning_rate}
|
||||
betas: [ 0.9, 0.95 ]
|
||||
weight_decay: ${weight_decay}
|
||||
bf16:
|
||||
enabled: False
|
||||
fp16:
|
||||
enabled: True
|
||||
auto_cast: False
|
||||
loss_scale: 0
|
||||
initial_scale_power: 16
|
||||
loss_scale_window: 1000
|
||||
hysteresis: 2
|
||||
consecutive_hysteresis: False
|
||||
min_loss_scale: 1
|
||||
# zero_optimization:
|
||||
# offload_optimizer:
|
||||
# device: cpu
|
||||
# pin_memory: True
|
||||
|
||||
|
||||
summary_helper:
|
||||
_target_: general_util.tensorboard_helper.WandbWriter
|
||||
batch_index_or_keys:
|
||||
outputs_index_or_keys:
|
||||
|
||||
# Temporary variables
|
||||
n_gpu:
|
||||
device:
|
||||
train_batch_size:
|
||||
eval_batch_size:
|
||||
world_size:
|
||||
@@ -0,0 +1,213 @@
|
||||
defaults:
|
||||
- hydra: default
|
||||
# - deepspeed@ds_cfg: train_hybrid_engine_zero3
|
||||
- deepspeed@ds_cfg: train_hybrid_engine_zero1
|
||||
- _self_ # see here for more details: https://hydra.cc/docs/tutorials/basic/your_first_app/defaults/#composition-order-of-primary-config
|
||||
|
||||
hydra:
|
||||
searchpath:
|
||||
- file://conf/
|
||||
|
||||
data_path_prefix: /mnt/fangkai_blob/share/
|
||||
model_path_prefix: /mnt/fangkai_blob/share/models
|
||||
output_path_prefix: /mnt/fangkai_blob/reward_modeling/
|
||||
|
||||
train_file: "hf:codeparrot/apps"
|
||||
dev_file:
|
||||
test_file:
|
||||
|
||||
torch_dtype:
|
||||
_target_: general_util.training_utils.return_torch_dtype
|
||||
dtype: float16
|
||||
|
||||
tokenizer_init:
|
||||
_target_: general_util.tokenization_utils.init_tokenizer
|
||||
tokenizer_path: ${model_name_or_path}
|
||||
padding_side: left
|
||||
|
||||
device_map:
|
||||
_target_: models.utils.return_single_device_map
|
||||
|
||||
model:
|
||||
_target_: models.llama.LlamaForCausalLMDPO.from_pretrained_with_ref_model
|
||||
beta: 0.4
|
||||
gradient_checkpointing: True
|
||||
attn_implementation: "flash_attention_2"
|
||||
torch_dtype: ${torch_dtype}
|
||||
pad_token_id: 100001
|
||||
device_map: ${device_map}
|
||||
ref_model:
|
||||
_target_: models.llama.LlamaForCausalLMDPO.from_pretrained
|
||||
pretrained_model_name_or_path: ${model_name_or_path}
|
||||
torch_dtype: ${torch_dtype}
|
||||
attn_implementation: "flash_attention_2"
|
||||
device_map: ${device_map}
|
||||
pad_token_id: 100001
|
||||
|
||||
|
||||
chat_prefix: "<｜begin▁of▁sentence｜>You are an AI programming assistant, utilizing the Deepseek Coder model, developed by Deepseek Company, and you only answer questions related to computer science. For politically sensitive questions, security and privacy issues, and other non-computer science questions, you will refuse to answer\n### Instruction:\n"
|
||||
chat_connect: "\n### Response:\n"
|
||||
chat_suffix: "\n<|EOT|>"
|
||||
prompt: "{question}\n\nPlease write a program to solve the above problem under the given time constraints and memory limits."
|
||||
|
||||
read_tensor:
|
||||
_target_: data.combine_dataset.MultiMappingDataset
|
||||
read_fn:
|
||||
_target_: data.apps.APPsWithFunctionName
|
||||
split: train
|
||||
train_sub_split: train
|
||||
use_starter_code: True
|
||||
aligner:
|
||||
_target_: data.input_aligner.concat_aligner
|
||||
aligners:
|
||||
- _target_: data.input_aligner.field_extract_aligner
|
||||
input_index_field: problem_id
|
||||
extract_index_field: id
|
||||
extract_fields: [ "pos", "neg" ]
|
||||
extra_file: ${model_path_prefix}/deepseek-coder-7b-instruct-v1.5/apps/train.0shot.tem1.0.n10.v2.0.pseudo_input_output.v1.0.clean.dpo_m6_low0.5.json
|
||||
- _target_: data.input_aligner.flat_aligner
|
||||
input_index_field: problem_id
|
||||
extract_field: [ "pos", "neg" ]
|
||||
mode: multi
|
||||
template:
|
||||
_target_: data.input_utils.recompose_template
|
||||
units:
|
||||
chat_prefix: ${chat_prefix}
|
||||
prompt: ${prompt}
|
||||
chat_connect: ${chat_connect}
|
||||
pos: "{pos}"
|
||||
neg: "{neg}"
|
||||
chat_suffix: ${chat_suffix}
|
||||
compositions:
|
||||
prompt: "{chat_prefix}{prompt}{chat_connect}"
|
||||
chosen: "{chat_prefix}{prompt}{chat_connect}{pos}{chat_suffix}"
|
||||
reject: "{chat_prefix}{prompt}{chat_connect}{neg}{chat_suffix}"
|
||||
index_field: problem_id
|
||||
kv_mapping:
|
||||
chosen: chosen
|
||||
reject: reject
|
||||
problem_id: index
|
||||
prompt: prompt
|
||||
|
||||
|
||||
dist_load_data_barrier: False
|
||||
|
||||
extended_vocab:
|
||||
|
||||
# Data collator
|
||||
collator:
|
||||
_target_: data.general_collator.DPOCollator
|
||||
tokenizer: ${tokenizer_init}
|
||||
max_seq_length: 3072
|
||||
|
||||
# Dataloader
|
||||
num_workers: 8
|
||||
prefetch_factor: 2
|
||||
|
||||
# Wiki path pretrain v8.2
|
||||
model_name_or_path: ${model_path_prefix}/deepseek-coder-7b-instruct-v1.5
|
||||
pretrain:
|
||||
resume:
|
||||
|
||||
dp_size:
|
||||
tp_size: 1
|
||||
pp_size: 1
|
||||
|
||||
|
||||
exp_name: deepseek-coder-v1.5-ins.7b.apps.code_gen.pseudo-test-10.sc.dpo.A100.dp8.v1.2.s${seed}
|
||||
exp_notes:
|
||||
output_dir: ${output_path_prefix}experiments/${exp_name} # Fix <pad token id>
|
||||
|
||||
|
||||
do_train: True
|
||||
evaluate_during_training: False
|
||||
|
||||
do_eval: False
|
||||
eval_sub_path: checkpoint-*
|
||||
|
||||
# Training hyper-parameters
|
||||
per_gpu_train_batch_size: 1
|
||||
per_gpu_eval_batch_size: 1
|
||||
#learning_rate: 1e-4
|
||||
learning_rate: 1e-6
|
||||
#learning_rate: 2e-5
|
||||
gradient_accumulation_steps: 16
|
||||
weight_decay: 0.1
|
||||
adam_epsilon: 1e-6
|
||||
adam_betas: "(0.9, 0.98)"
|
||||
#adam_betas: "(0.9, 0.999)"
|
||||
#max_grad_norm: 0.0
|
||||
total_dataset_len: -1
|
||||
max_grad_norm: 1.0
|
||||
num_train_epochs: 1
|
||||
max_steps: 0
|
||||
warmup_proportion: 0.1
|
||||
warmup_steps: 0
|
||||
|
||||
# Optimizer
|
||||
optimizer:
|
||||
use_nvlamb:
|
||||
bit_training:
|
||||
|
||||
logging_steps: 5
|
||||
save_ds_state: True
|
||||
save_steps: 100
|
||||
save_best: False
|
||||
eval_steps: 100
|
||||
ddp_eval: True
|
||||
no_cuda: False
|
||||
seed: 42
|
||||
local_rank: -1
|
||||
fp16: True
|
||||
fp16_opt_level: O1
|
||||
fp16_bfloat16: False
|
||||
|
||||
# Prediction config
|
||||
prediction_cfg:
|
||||
metric: "loss"
|
||||
measure: -1
|
||||
best_checkpoint:
|
||||
best_result:
|
||||
eval_forward_fn:
|
||||
_target_: general_util.evaluator.DefaultForwardFn
|
||||
post_process:
|
||||
_target_: post_processors.dpo.DPOEvalPostProcessor
|
||||
|
||||
|
||||
ds_cfg:
|
||||
train_micro_batch_size_per_gpu: ${per_gpu_train_batch_size}
|
||||
gradient_accumulation_steps: ${gradient_accumulation_steps}
|
||||
optimizer:
|
||||
type: AdamW
|
||||
params:
|
||||
lr: ${learning_rate}
|
||||
betas: [ 0.9, 0.95 ]
|
||||
weight_decay: ${weight_decay}
|
||||
bf16:
|
||||
enabled: False
|
||||
fp16:
|
||||
enabled: True
|
||||
auto_cast: False
|
||||
loss_scale: 0
|
||||
initial_scale_power: 16
|
||||
loss_scale_window: 1000
|
||||
hysteresis: 2
|
||||
consecutive_hysteresis: False
|
||||
min_loss_scale: 1
|
||||
# zero_optimization:
|
||||
# offload_optimizer:
|
||||
# device: cpu
|
||||
# pin_memory: True
|
||||
|
||||
|
||||
summary_helper:
|
||||
_target_: general_util.tensorboard_helper.WandbWriter
|
||||
batch_index_or_keys:
|
||||
outputs_index_or_keys:
|
||||
|
||||
# Temporary variables
|
||||
n_gpu:
|
||||
device:
|
||||
train_batch_size:
|
||||
eval_batch_size:
|
||||
world_size:
|
||||
@@ -0,0 +1,214 @@
|
||||
defaults:
|
||||
- hydra: default
|
||||
# - deepspeed@ds_cfg: train_hybrid_engine_zero3
|
||||
- deepspeed@ds_cfg: train_hybrid_engine_zero1_optim_offload
|
||||
- _self_ # see here for more details: https://hydra.cc/docs/tutorials/basic/your_first_app/defaults/#composition-order-of-primary-config
|
||||
|
||||
hydra:
|
||||
searchpath:
|
||||
- file://conf/
|
||||
|
||||
data_path_prefix: /mnt/fangkai_blob/share/
|
||||
model_path_prefix: /mnt/fangkai_blob/share/models
|
||||
output_path_prefix: /mnt/fangkai_blob/reward_modeling/
|
||||
|
||||
train_file: "hf:codeparrot/apps"
|
||||
dev_file:
|
||||
test_file:
|
||||
|
||||
torch_dtype:
|
||||
_target_: general_util.training_utils.return_torch_dtype
|
||||
dtype: float16
|
||||
|
||||
tokenizer_init:
|
||||
_target_: general_util.tokenization_utils.init_tokenizer
|
||||
tokenizer_path: ${model_name_or_path}
|
||||
padding_side: left
|
||||
|
||||
device_map:
|
||||
_target_: models.utils.return_single_device_map
|
||||
|
||||
model:
|
||||
_target_: models.llama_tp.LlamaForCausalLMDPO.from_pretrained_with_ref_model
|
||||
beta: 0.4
|
||||
gradient_checkpointing: True
|
||||
attn_implementation: "eager"
|
||||
torch_dtype: ${torch_dtype}
|
||||
pad_token_id: 100001
|
||||
# device_map: ${device_map}
|
||||
ref_model:
|
||||
_target_: models.llama_tp.LlamaForCausalLMDPO.from_pretrained
|
||||
pretrained_model_name_or_path: ${model_name_or_path}
|
||||
torch_dtype: ${torch_dtype}
|
||||
attn_implementation: "eager"
|
||||
# device_map: ${device_map}
|
||||
pad_token_id: 100001
|
||||
|
||||
|
||||
chat_prefix: "<｜begin▁of▁sentence｜>You are an AI programming assistant, utilizing the Deepseek Coder model, developed by Deepseek Company, and you only answer questions related to computer science. For politically sensitive questions, security and privacy issues, and other non-computer science questions, you will refuse to answer\n### Instruction:\n"
|
||||
chat_connect: "\n### Response:\n"
|
||||
chat_suffix: "\n<|EOT|>"
|
||||
prompt: "{question}\n\nPlease write a program to solve the above problem under the given time constraints and memory limits."
|
||||
|
||||
read_tensor:
|
||||
_target_: data.combine_dataset.MultiMappingDataset
|
||||
read_fn:
|
||||
_target_: data.apps.APPsWithFunctionName
|
||||
split: train
|
||||
train_sub_split: train
|
||||
use_starter_code: True
|
||||
aligner:
|
||||
_target_: data.input_aligner.concat_aligner
|
||||
aligners:
|
||||
- _target_: data.input_aligner.field_extract_aligner
|
||||
input_index_field: problem_id
|
||||
extract_index_field: id
|
||||
extract_fields: [ "pos", "neg" ]
|
||||
# extra_file: ${model_path_prefix}/deepseek-coder-7b-instruct-v1.5/apps/train.0shot.tem1.0.n10.v2.0.pseudo_input_output.v1.0.clean.dpo_m3_low0.5.json
|
||||
extra_file: ${model_path_prefix}/deepseek-coder-7b-instruct-v1.5/apps/train.0shot.tem1.0.n10.v2.0.pseudo_input_output.v1.0.clean.dpo_m6_low0.5.json
|
||||
- _target_: data.input_aligner.flat_aligner
|
||||
input_index_field: problem_id
|
||||
extract_field: [ "pos", "neg" ]
|
||||
mode: multi
|
||||
template:
|
||||
_target_: data.input_utils.recompose_template
|
||||
units:
|
||||
chat_prefix: ${chat_prefix}
|
||||
prompt: ${prompt}
|
||||
chat_connect: ${chat_connect}
|
||||
pos: "{pos}"
|
||||
neg: "{neg}"
|
||||
chat_suffix: ${chat_suffix}
|
||||
compositions:
|
||||
prompt: "{chat_prefix}{prompt}{chat_connect}"
|
||||
chosen: "{chat_prefix}{prompt}{chat_connect}{pos}{chat_suffix}"
|
||||
reject: "{chat_prefix}{prompt}{chat_connect}{neg}{chat_suffix}"
|
||||
index_field: problem_id
|
||||
kv_mapping:
|
||||
chosen: chosen
|
||||
reject: reject
|
||||
problem_id: index
|
||||
prompt: prompt
|
||||
|
||||
|
||||
dist_load_data_barrier: False
|
||||
|
||||
extended_vocab:
|
||||
|
||||
# Data collator
|
||||
collator:
|
||||
_target_: data.general_collator.DPOCollator
|
||||
tokenizer: ${tokenizer_init}
|
||||
max_seq_length: 3072
|
||||
|
||||
# Dataloader
|
||||
num_workers: 8
|
||||
prefetch_factor: 2
|
||||
|
||||
# Wiki path pretrain v8.2
|
||||
model_name_or_path: ${model_path_prefix}/deepseek-coder-7b-instruct-v1.5
|
||||
pretrain:
|
||||
resume:
|
||||
|
||||
dp_size:
|
||||
tp_size: 8
|
||||
pp_size: 1
|
||||
|
||||
|
||||
exp_name: deepseek-coder-v1.5-ins.7b.apps.code_gen.pseudo-test-10.sc.dpo.V100.tp8.dp8.v1.2.s${seed}
|
||||
exp_notes:
|
||||
output_dir: ${output_path_prefix}experiments/${exp_name} # Fix <pad token id>
|
||||
|
||||
|
||||
do_train: True
|
||||
evaluate_during_training: False
|
||||
|
||||
do_eval: False
|
||||
eval_sub_path: checkpoint-*
|
||||
|
||||
# Training hyper-parameters
|
||||
per_gpu_train_batch_size: 1
|
||||
per_gpu_eval_batch_size: 1
|
||||
#learning_rate: 1e-4
|
||||
learning_rate: 1e-6
|
||||
#learning_rate: 2e-5
|
||||
gradient_accumulation_steps: 16
|
||||
weight_decay: 0.1
|
||||
adam_epsilon: 1e-6
|
||||
adam_betas: "(0.9, 0.98)"
|
||||
#adam_betas: "(0.9, 0.999)"
|
||||
#max_grad_norm: 0.0
|
||||
total_dataset_len: -1
|
||||
max_grad_norm: 1.0
|
||||
num_train_epochs: 1
|
||||
max_steps: 0
|
||||
warmup_proportion: 0.1
|
||||
warmup_steps: 0
|
||||
|
||||
# Optimizer
|
||||
optimizer:
|
||||
use_nvlamb:
|
||||
bit_training:
|
||||
|
||||
logging_steps: 5
|
||||
save_ds_state: True
|
||||
save_steps: 100
|
||||
save_best: False
|
||||
eval_steps: 100
|
||||
ddp_eval: True
|
||||
no_cuda: False
|
||||
seed: 42
|
||||
local_rank: -1
|
||||
fp16: True
|
||||
fp16_opt_level: O1
|
||||
fp16_bfloat16: False
|
||||
|
||||
# Prediction config
|
||||
prediction_cfg:
|
||||
metric: "loss"
|
||||
measure: -1
|
||||
best_checkpoint:
|
||||
best_result:
|
||||
eval_forward_fn:
|
||||
_target_: general_util.evaluator.DefaultForwardFn
|
||||
post_process:
|
||||
_target_: post_processors.dpo.DPOEvalPostProcessor
|
||||
|
||||
|
||||
ds_cfg:
|
||||
train_micro_batch_size_per_gpu: ${per_gpu_train_batch_size}
|
||||
gradient_accumulation_steps: ${gradient_accumulation_steps}
|
||||
optimizer:
|
||||
type: AdamW
|
||||
params:
|
||||
lr: ${learning_rate}
|
||||
betas: [ 0.9, 0.95 ]
|
||||
weight_decay: ${weight_decay}
|
||||
bf16:
|
||||
enabled: False
|
||||
fp16:
|
||||
enabled: True
|
||||
auto_cast: False
|
||||
loss_scale: 0
|
||||
initial_scale_power: 16
|
||||
loss_scale_window: 1000
|
||||
hysteresis: 2
|
||||
consecutive_hysteresis: False
|
||||
min_loss_scale: 1
|
||||
# zero_optimization:
|
||||
# offload_optimizer:
|
||||
# device: cpu
|
||||
# pin_memory: True
|
||||
|
||||
|
||||
summary_helper:
|
||||
_target_: general_util.tensorboard_helper.WandbWriter
|
||||
batch_index_or_keys:
|
||||
outputs_index_or_keys:
|
||||
|
||||
# Temporary variables
|
||||
n_gpu:
|
||||
device:
|
||||
train_batch_size:
|
||||
eval_batch_size:
|
||||
world_size:
|
||||
@@ -0,0 +1,190 @@
|
||||
defaults:
|
||||
- hydra: default
|
||||
# - deepspeed@ds_cfg: train_hybrid_engine_zero3
|
||||
- deepspeed@ds_cfg: train_hybrid_engine_zero1_optim_offload
|
||||
- _self_ # see here for more details: https://hydra.cc/docs/tutorials/basic/your_first_app/defaults/#composition-order-of-primary-config
|
||||
|
||||
hydra:
|
||||
searchpath:
|
||||
- file://conf/
|
||||
|
||||
data_path_prefix: ""
|
||||
model_path_prefix: ../pretrained-models/ # /mnt/fangkai_blob/share/models
|
||||
output_path_prefix: "" # /mnt/fangkai_blob/reward_modeling/
|
||||
|
||||
train_file: "hf:codeparrot/apps"
|
||||
dev_file:
|
||||
test_file:
|
||||
|
||||
torch_dtype:
|
||||
_target_: general_util.training_utils.return_torch_dtype
|
||||
dtype: float16
|
||||
|
||||
tokenizer_init:
|
||||
_target_: general_util.tokenization_utils.init_tokenizer
|
||||
tokenizer_path: ${model_name_or_path}
|
||||
padding_side: left
|
||||
|
||||
model:
|
||||
_target_: models.llama.LlamaForCausalLM.from_pretrained
|
||||
gradient_checkpointing: True
|
||||
attn_implementation: "eager"
|
||||
torch_dtype: ${torch_dtype}
|
||||
pad_token_id: 100001
|
||||
# device_map:
|
||||
# _target_: models.utils.return_single_device_map
|
||||
|
||||
|
||||
chat_prefix: "<｜begin▁of▁sentence｜>You are an AI programming assistant, utilizing the Deepseek Coder model, developed by Deepseek Company, and you only answer questions related to computer science. For politically sensitive questions, security and privacy issues, and other non-computer science questions, you will refuse to answer\n### Instruction:\n"
|
||||
chat_connect: "\n### Response:\n"
|
||||
chat_suffix: "\n<|EOT|>"
|
||||
prompt: "{question}\n\nPlease write a program to solve the above problem under the given time constraints and memory limits."
|
||||
|
||||
read_tensor:
|
||||
_target_: data.combine_dataset.MultiMappingDataset
|
||||
read_fn:
|
||||
_target_: data.apps.APPsReader
|
||||
split: train
|
||||
aligner:
|
||||
_target_: data.input_aligner.flat_aligner
|
||||
input_index_field: problem_id
|
||||
extract_field: [ "solutions" ]
|
||||
mode: "multi"
|
||||
template:
|
||||
_target_: data.input_utils.recompose_template
|
||||
units:
|
||||
chat_prefix: ${chat_prefix}
|
||||
prompt: ${prompt}
|
||||
chat_connect: ${chat_connect}
|
||||
suffix: "{solutions}"
|
||||
compositions:
|
||||
prompt: "{chat_prefix}{prompt}{chat_connect}"
|
||||
chosen: "{chat_prefix}{prompt}{chat_connect}{suffix}"
|
||||
index_field: problem_id
|
||||
kv_mapping:
|
||||
chosen: chosen
|
||||
problem_id: index
|
||||
prompt: prompt
|
||||
|
||||
|
||||
dist_load_data_barrier: False
|
||||
|
||||
extended_vocab:
|
||||
|
||||
# Data collator
|
||||
collator:
|
||||
_target_: data.general_collator.DPODataSFTCollator
|
||||
tokenizer: ${tokenizer_init}
|
||||
max_seq_length: 4096
|
||||
|
||||
# Dataloader
|
||||
num_workers: 8
|
||||
prefetch_factor: 2
|
||||
|
||||
# Wiki path pretrain v8.2
|
||||
model_name_or_path: ${model_path_prefix}/deepseek-coder-7b-instruct-v1.5
|
||||
pretrain:
|
||||
resume:
|
||||
|
||||
dp_size:
|
||||
tp_size: 1
|
||||
pp_size: 1
|
||||
|
||||
|
||||
exp_name: deepseek-coder-v1.5-ins.7b.apps.code_gen.A100.w8.v1.0
|
||||
exp_notes:
|
||||
output_dir: ${output_path_prefix}experiments/${exp_name} # Fix <pad token id>
|
||||
|
||||
|
||||
do_train: True
|
||||
evaluate_during_training: True
|
||||
|
||||
do_eval: True
|
||||
eval_sub_path: checkpoint-*
|
||||
|
||||
# Training hyper-parameters
|
||||
per_gpu_train_batch_size: 2
|
||||
per_gpu_eval_batch_size: 2
|
||||
#learning_rate: 1e-4
|
||||
#learning_rate: 5e-6
|
||||
learning_rate: 2e-5
|
||||
gradient_accumulation_steps: 8
|
||||
weight_decay: 0.1
|
||||
adam_epsilon: 1e-6
|
||||
adam_betas: "(0.9, 0.98)"
|
||||
#adam_betas: "(0.9, 0.999)"
|
||||
#max_grad_norm: 0.0
|
||||
total_dataset_len: -1
|
||||
max_grad_norm: 1.0
|
||||
num_train_epochs: 1
|
||||
max_steps: 0
|
||||
warmup_proportion: 0.1
|
||||
warmup_steps: 0
|
||||
|
||||
# Optimizer
|
||||
optimizer:
|
||||
use_nvlamb:
|
||||
bit_training:
|
||||
|
||||
logging_steps: 5
|
||||
save_ds_state: True
|
||||
save_steps: 400
|
||||
save_best: False
|
||||
eval_steps: 400
|
||||
ddp_eval: True
|
||||
no_cuda: False
|
||||
seed: 42
|
||||
local_rank: -1
|
||||
fp16: True
|
||||
fp16_opt_level: O1
|
||||
fp16_bfloat16: True
|
||||
|
||||
# Prediction config
|
||||
prediction_cfg:
|
||||
metric: "acc"
|
||||
measure: -1
|
||||
best_checkpoint:
|
||||
best_result:
|
||||
eval_forward_fn:
|
||||
_target_: general_util.evaluator.DefaultForwardFn
|
||||
post_process:
|
||||
_target_: post_processors.dpo.SFTLossOnlyPostProcessor
|
||||
|
||||
|
||||
ds_cfg:
|
||||
train_micro_batch_size_per_gpu: ${per_gpu_train_batch_size}
|
||||
gradient_accumulation_steps: ${gradient_accumulation_steps}
|
||||
# optimizer:
|
||||
# type: AdamW
|
||||
# params:
|
||||
# lr: ${learning_rate}
|
||||
# betas: [ 0.9, 0.95 ]
|
||||
# weight_decay: ${weight_decay}
|
||||
# bf16:
|
||||
# enabled: False
|
||||
# fp16:
|
||||
# enabled: True
|
||||
# auto_cast: False
|
||||
# loss_scale": 0
|
||||
# initial_scale_power: 16
|
||||
# loss_scale_window: 1000
|
||||
# hysteresis: 2
|
||||
# consecutive_hysteresis: False
|
||||
# min_loss_scale: 1
|
||||
# zero_optimization:
|
||||
# offload_optimizer:
|
||||
# device: cpu
|
||||
# pin_memory: True
|
||||
|
||||
|
||||
summary_helper:
|
||||
_target_: general_util.tensorboard_helper.WandbWriter
|
||||
batch_index_or_keys:
|
||||
outputs_index_or_keys:
|
||||
|
||||
# Temporary variables
|
||||
n_gpu:
|
||||
device:
|
||||
train_batch_size:
|
||||
eval_batch_size:
|
||||
world_size:
|
||||
@@ -0,0 +1,190 @@
|
||||
defaults:
|
||||
- hydra: default
|
||||
# - deepspeed@ds_cfg: train_hybrid_engine_zero3
|
||||
- deepspeed@ds_cfg: train_hybrid_engine_zero2
|
||||
- _self_ # see here for more details: https://hydra.cc/docs/tutorials/basic/your_first_app/defaults/#composition-order-of-primary-config
|
||||
|
||||
hydra:
|
||||
searchpath:
|
||||
- file://conf/
|
||||
|
||||
data_path_prefix: ""
|
||||
model_path_prefix: ../pretrained-models/ # /mnt/fangkai_blob/share/models
|
||||
output_path_prefix: "" # /mnt/fangkai_blob/reward_modeling/
|
||||
|
||||
train_file: "hf:codeparrot/apps"
|
||||
dev_file:
|
||||
test_file:
|
||||
|
||||
torch_dtype:
|
||||
_target_: general_util.training_utils.return_torch_dtype
|
||||
dtype: float16
|
||||
|
||||
tokenizer_init:
|
||||
_target_: general_util.tokenization_utils.init_tokenizer
|
||||
tokenizer_path: ${model_name_or_path}
|
||||
padding_side: left
|
||||
|
||||
model:
|
||||
_target_: models.llama.LlamaForCausalLM.from_pretrained
|
||||
gradient_checkpointing: True
|
||||
attn_implementation: "eager"
|
||||
torch_dtype: ${torch_dtype}
|
||||
pad_token_id: 100001
|
||||
# device_map:
|
||||
# _target_: models.utils.return_single_device_map
|
||||
|
||||
|
||||
chat_prefix: "<｜begin▁of▁sentence｜>You are an AI programming assistant, utilizing the Deepseek Coder model, developed by Deepseek Company, and you only answer questions related to computer science. For politically sensitive questions, security and privacy issues, and other non-computer science questions, you will refuse to answer\n### Instruction:\n"
|
||||
chat_connect: "\n### Response:\n"
|
||||
chat_suffix: "\n<|EOT|>"
|
||||
prompt: "{question}\n\nPlease write a program to solve the above problem under the given time constraints and memory limits."
|
||||
|
||||
read_tensor:
|
||||
_target_: data.combine_dataset.MultiMappingDataset
|
||||
read_fn:
|
||||
_target_: data.apps.APPsReader
|
||||
split: train
|
||||
aligner:
|
||||
_target_: data.input_aligner.flat_aligner
|
||||
input_index_field: problem_id
|
||||
extract_field: [ "solutions" ]
|
||||
mode: "multi"
|
||||
template:
|
||||
_target_: data.input_utils.recompose_template
|
||||
units:
|
||||
chat_prefix: ${chat_prefix}
|
||||
prompt: ${prompt}
|
||||
chat_connect: ${chat_connect}
|
||||
suffix: "{solutions}"
|
||||
compositions:
|
||||
prompt: "{chat_prefix}{prompt}{chat_connect}"
|
||||
chosen: "{chat_prefix}{prompt}{chat_connect}{suffix}"
|
||||
index_field: problem_id
|
||||
kv_mapping:
|
||||
chosen: chosen
|
||||
problem_id: index
|
||||
prompt: prompt
|
||||
|
||||
|
||||
dist_load_data_barrier: False
|
||||
|
||||
extended_vocab:
|
||||
|
||||
# Data collator
|
||||
collator:
|
||||
_target_: data.general_collator.DPODataSFTCollator
|
||||
tokenizer: ${tokenizer_init}
|
||||
max_seq_length: 4096
|
||||
|
||||
# Dataloader
|
||||
num_workers: 8
|
||||
prefetch_factor: 2
|
||||
|
||||
# Wiki path pretrain v8.2
|
||||
model_name_or_path: ${model_path_prefix}/deepseek-coder-7b-instruct-v1.5
|
||||
pretrain:
|
||||
resume:
|
||||
|
||||
dp_size:
|
||||
tp_size: 1
|
||||
pp_size: 1
|
||||
|
||||
|
||||
exp_name: deepseek-coder-v1.5-ins.7b.apps.code_gen.V100.w8.v1.0
|
||||
exp_notes:
|
||||
output_dir: ${output_path_prefix}experiments/${exp_name} # Fix <pad token id>
|
||||
|
||||
|
||||
do_train: True
|
||||
evaluate_during_training: True
|
||||
|
||||
do_eval: True
|
||||
eval_sub_path: checkpoint-*
|
||||
|
||||
# Training hyper-parameters
|
||||
per_gpu_train_batch_size: 1
|
||||
per_gpu_eval_batch_size: 1
|
||||
#learning_rate: 1e-4
|
||||
#learning_rate: 5e-6
|
||||
learning_rate: 2e-5
|
||||
gradient_accumulation_steps: 16
|
||||
weight_decay: 0.1
|
||||
adam_epsilon: 1e-6
|
||||
adam_betas: "(0.9, 0.98)"
|
||||
#adam_betas: "(0.9, 0.999)"
|
||||
#max_grad_norm: 0.0
|
||||
total_dataset_len: -1
|
||||
max_grad_norm: 1.0
|
||||
num_train_epochs: 1
|
||||
max_steps: 0
|
||||
warmup_proportion: 0.1
|
||||
warmup_steps: 0
|
||||
|
||||
# Optimizer
|
||||
optimizer:
|
||||
use_nvlamb:
|
||||
bit_training:
|
||||
|
||||
logging_steps: 5
|
||||
save_ds_state: False
|
||||
save_steps: 400
|
||||
save_best: False
|
||||
eval_steps: 400
|
||||
ddp_eval: True
|
||||
no_cuda: False
|
||||
seed: 42
|
||||
local_rank: -1
|
||||
fp16: True
|
||||
fp16_opt_level: O1
|
||||
fp16_bfloat16: False
|
||||
|
||||
# Prediction config
|
||||
prediction_cfg:
|
||||
metric: "acc"
|
||||
measure: -1
|
||||
best_checkpoint:
|
||||
best_result:
|
||||
eval_forward_fn:
|
||||
_target_: general_util.evaluator.DefaultForwardFn
|
||||
post_process:
|
||||
_target_: post_processors.dpo.SFTLossOnlyPostProcessor
|
||||
|
||||
|
||||
ds_cfg:
|
||||
train_micro_batch_size_per_gpu: ${per_gpu_train_batch_size}
|
||||
gradient_accumulation_steps: ${gradient_accumulation_steps}
|
||||
optimizer:
|
||||
type: AdamW
|
||||
params:
|
||||
lr: ${learning_rate}
|
||||
betas: [ 0.9, 0.95 ]
|
||||
weight_decay: ${weight_decay}
|
||||
bf16:
|
||||
enabled: False
|
||||
fp16:
|
||||
enabled: True
|
||||
auto_cast: False
|
||||
loss_scale": 0
|
||||
initial_scale_power: 16
|
||||
loss_scale_window: 1000
|
||||
hysteresis: 2
|
||||
consecutive_hysteresis: False
|
||||
min_loss_scale: 1
|
||||
zero_optimization:
|
||||
offload_optimizer:
|
||||
device: cpu
|
||||
pin_memory: True
|
||||
|
||||
|
||||
summary_helper:
|
||||
_target_: general_util.tensorboard_helper.WandbWriter
|
||||
batch_index_or_keys:
|
||||
outputs_index_or_keys:
|
||||
|
||||
# Temporary variables
|
||||
n_gpu:
|
||||
device:
|
||||
train_batch_size:
|
||||
eval_batch_size:
|
||||
world_size:
|
||||
Reference in New Issue
Block a user