chore: import upstream snapshot with attribution

2026-07-13 13:39:21 +08:00
commit bc34f6df14
1149 changed files with 328099 additions and 0 deletions
@@ -0,0 +1,44 @@
+# the instruction and training config version
+version="llm-embedder"
+# the output folder
+output="llm-embedder"
+# the data root where you untar the data
+data_root="/data/llm-embedder"
+
+torchrun --nproc_per_node=8 run_dense.py --train_data \
+    llm-embedder:chat/msc/train.json \
+    llm-embedder:convsearch/qrecc/train.concat.json \
+    llm-embedder:lrlm/arxiv/train.json \
+    llm-embedder:lrlm/books3/train.json \
+    llm-embedder:lrlm/codeparrot/train.json \
+    llm-embedder:qa/msmarco/train.json \
+    llm-embedder:qa/nq/train.json \
+    llm-embedder:tool/toolbench/train.json \
+    llm-embedder:tool/toolbench/train.json \
+    llm-embedder:icl/icl/train.json \
+    --output_dir data/outputs/$output \
+    --save_steps 10000 \
+    --max_steps 10000 \
+    --logging_steps 100 \
+    --inbatch_same_dataset epoch \
+    --use_train_config \
+    --gradient_checkpointing \
+    --per_device_train_batch_size 100 \
+    --deepspeed data/deepspeed/stage0.json \
+    --version $version \
+    --learning_rate 5e-6 \
+    --data_root $data_root
+
+for model in "checkpoint-10000"
+do
+    torchrun --nproc_per_node 8 -m evaluation.eval_mmlu --query_encoder data/outputs/$output/$model/encoder --version $version --data_root $data_root
+    torchrun --nproc_per_node 8 -m evaluation.eval_popqa --query_encoder data/outputs/$output/$model/encoder --version $version --data_root $data_root
+    torchrun --nproc_per_node 8 -m evaluation.eval_msc --query_encoder data/outputs/$output/$model/encoder --version $version --data_root $data_root
+    torchrun --nproc_per_node 8 -m evaluation.eval_tool --query_encoder data/outputs/$output/$model/encoder --version $version --data_root $data_root
+    torchrun --nproc_per_node 8 -m evaluation.eval_lrlm --query_encoder data/outputs/$output/$model/encoder --eval_data llm-embedder:lrlm/books3/test.json --version $version --data_root $data_root
+    torchrun --nproc_per_node 8 -m evaluation.eval_lrlm --query_encoder data/outputs/$output/$model/encoder --eval_data llm-embedder:lrlm/arxiv/test.json --version $version --data_root $data_root
+    torchrun --nproc_per_node 8 -m evaluation.eval_lrlm --query_encoder data/outputs/$output/$model/encoder --eval_data llm-embedder:lrlm/codeparrot/test.json --version $version --data_root $data_root
+    torchrun --nproc_per_node 8 -m evaluation.eval_lrlm --query_encoder data/outputs/$output/$model/encoder --eval_data llm-embedder:lrlm/pg19/test.json --version $version --data_root $data_root
+    torchrun --nproc_per_node 8 -m evaluation.eval_icl --query_encoder data/outputs/$output/$model/encoder --version $version --data_root $data_root
+    torchrun --nproc_per_node 8 -m evaluation.eval_qrecc --query_encoder data/outputs/$output/$model/encoder --version $version --data_root $data_root
+done