chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
Executable
+85
@@ -0,0 +1,85 @@
|
||||
#!/bin/bash
|
||||
#SBATCH --job-name=llmc-multinode # job name
|
||||
#SBATCH --output=/home/ubuntu/llm.c/scripts/multi_node/%x_%j_%t.log # output file
|
||||
#SBATCH --error=/home/ubuntu/llm.c/scripts/multi_node/%x_%j_%t.err # error file
|
||||
#SBATCH --partition=llmc # Specify the GPU partition
|
||||
#SBATCH --ntasks=16 # total number of processes to launch on all nodes
|
||||
#SBATCH --nodes=2 # total number of nodes
|
||||
#SBATCH --ntasks-per-node=8 # assuming each node has 8 gpus
|
||||
#SBATCH --gres=gpu:8 # request 8 gpus from each node
|
||||
|
||||
# NOTE: change the above slurm arguments to match your system!
|
||||
# Run with `sbatch <path_to_this_script.sh>`
|
||||
|
||||
make train_gpt2cu USE_CUDNN=1 NO_USE_MPI=1
|
||||
|
||||
# NOTE: change the following to match your system
|
||||
binary_path="/home/ubuntu/llm.c/train_gpt2cu"
|
||||
out_dir="/ephemeral/data/fineweb/log_gpt2_124M_multi"
|
||||
train_data_path='/ephemeral/data/fineweb/bin_10B/fineweb_train_*.bin'
|
||||
val_data_path='/ephemeral/data/fineweb/bin_10B/fineweb_val_*.bin'
|
||||
sync_fs_path=$out_dir # needs to be a shared filesystem path that all nodes can access
|
||||
|
||||
# In case the file system is shared this is a no-op.
|
||||
# Otherwise, we need to copy the binary to all nodes.
|
||||
current_user=$USER
|
||||
hosts=$(scontrol show hostnames $SLURM_JOB_NODELIST) # get the hostnames of the allocated nodes
|
||||
current_host=$(hostname)
|
||||
for host in $hosts; do
|
||||
if [ $host == $current_host ]; then
|
||||
continue
|
||||
fi
|
||||
echo "copying $binary_path to $current_user@$host"
|
||||
scp -r $binary_path $current_user@$host:$binary_path
|
||||
done
|
||||
|
||||
# Use this for NCCL debugging if you run into issues
|
||||
# export NCCL_DEBUG=INFO
|
||||
# export NCCL_DEBUG_SUBSYS=ALL
|
||||
export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
|
||||
|
||||
# Optimization flags
|
||||
export NCCL_NET_GDR_LEVEL=2 # use GPUDirect RDMA - allows for direct memory access between GPUs across different nodes by bypassing the CPU
|
||||
export NCCL_IB_DISABLE=0 # use InfiniBand if available
|
||||
|
||||
# NOTE: change the following environment variables to match your system - or comment them out if you don't need them
|
||||
export NCCL_SOCKET_IFNAME=ens17
|
||||
export OMPI_MCA_btl_tcp_if_include=ens17
|
||||
export NCCL_P2P_LEVEL=PXB
|
||||
|
||||
if [ -z "$SLURM_JOB_ID" ]; then
|
||||
echo "Make sure you're running in a SLURM environment. Did you forget to run with sbatch? Aborting."
|
||||
exit 1
|
||||
else
|
||||
DATESTRING=`date "+%Y-%m-%dT%H:%M:%S"`
|
||||
echo "Running in a SLURM environment (job ID: $SLURM_JOB_ID, user: $current_user)"
|
||||
echo "Running on hosts: $(echo $(scontrol show hostname))"
|
||||
echo "$DATESTRING"
|
||||
fi
|
||||
|
||||
srun -l -u bash -c "
|
||||
$binary_path \
|
||||
-i '$train_data_path' \
|
||||
-j '$val_data_path' \
|
||||
-o $out_dir \
|
||||
-v 250 -s 20000 -g 144 \
|
||||
-h 1 \
|
||||
-b 64 -t 1024 \
|
||||
-d 2097152 \
|
||||
-r 0 \
|
||||
-z 1 \
|
||||
-c 0.1 \
|
||||
-l 0.0006 \
|
||||
-q 0.0 \
|
||||
-u 700 \
|
||||
-n 5000 \
|
||||
-y 1 \
|
||||
-e d12 \
|
||||
-pn \$SLURM_NTASKS \
|
||||
-pr \$SLURM_PROCID \
|
||||
-pg \$SLURM_NTASKS_PER_NODE \
|
||||
-pf $sync_fs_path \
|
||||
-pi "fs" \
|
||||
"
|
||||
|
||||
echo "$DATESTRING"
|
||||
Executable
+49
@@ -0,0 +1,49 @@
|
||||
|
||||
make train_gpt2cu USE_CUDNN=1
|
||||
|
||||
# NOTE: change the following to match your system
|
||||
binary_path="/home/ubuntu/llm.c/train_gpt2cu"
|
||||
out_dir="/ephemeral/data/fineweb/log_gpt2_124M_multi"
|
||||
train_data_path='/ephemeral/data/fineweb/bin_10B/fineweb_train_*.bin'
|
||||
val_data_path='/ephemeral/data/fineweb/bin_10B/fineweb_val_*.bin'
|
||||
# You can find these names either in `/etc/hosts`` file or in the terminal (user@host:~$).
|
||||
host1="h100-node-1-0" # master and worker node
|
||||
host2="h100-node-1-1" # worker node
|
||||
|
||||
# In case the file system is shared this is a no-op.
|
||||
# Otherwise, we need to copy the binary to all nodes.
|
||||
scp -r $binary_path $USER@$host2:$binary_path
|
||||
|
||||
# Use this for NCCL debugging if you run into issues
|
||||
# export NCCL_DEBUG=INFO
|
||||
# export NCCL_DEBUG_SUBSYS=ALL
|
||||
export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
|
||||
|
||||
# Optimization flags
|
||||
export NCCL_NET_GDR_LEVEL=2 # use GPUDirect RDMA - allows for direct memory access between GPUs across different nodes by bypassing the CPU
|
||||
export NCCL_IB_DISABLE=0 # use InfiniBand if available
|
||||
|
||||
# NOTE: change the following environment variables to match your system - or comment them out if you don't need them
|
||||
export NCCL_SOCKET_IFNAME=ens17
|
||||
export OMPI_MCA_btl_tcp_if_include=ens17
|
||||
export NCCL_P2P_LEVEL=PXB
|
||||
|
||||
mpirun -np 16 --host $host1:8,$host2:8 \
|
||||
$binary_path \
|
||||
-i "$train_data_path" \
|
||||
-j "$val_data_path" \
|
||||
-o $out_dir \
|
||||
-v 250 -s 20000 -g 144 \
|
||||
-h 1 \
|
||||
-b 64 -t 1024 \
|
||||
-d 2097152 \
|
||||
-r 0 \
|
||||
-z 1 \
|
||||
-c 0.1 \
|
||||
-l 0.0006 \
|
||||
-q 0.1 \
|
||||
-u 700 \
|
||||
-n 1000 \
|
||||
-y 0 \
|
||||
-e d12 \
|
||||
-pi "mpi" \
|
||||
Executable
+86
@@ -0,0 +1,86 @@
|
||||
#!/bin/bash
|
||||
#SBATCH --job-name=llmc-multinode # job name
|
||||
#SBATCH --output=/home/ubuntu/llm.c/scripts/multi_node/%x_%j_%t.log # output file
|
||||
#SBATCH --error=/home/ubuntu/llm.c/scripts/multi_node/%x_%j_%t.err # error file
|
||||
#SBATCH --partition=llmc # Specify the GPU partition
|
||||
#SBATCH --ntasks=16 # total number of processes to launch on all nodes
|
||||
#SBATCH --nodes=2 # total number of nodes
|
||||
#SBATCH --ntasks-per-node=8 # assuming each node has 8 gpus
|
||||
#SBATCH --gres=gpu:8 # request 8 gpus from each node
|
||||
|
||||
# NOTE: change the above slurm arguments to match your system!
|
||||
# Run with `sbatch <path_to_this_script.sh>`
|
||||
|
||||
make train_gpt2cu USE_CUDNN=1 NO_USE_MPI=1
|
||||
|
||||
# NOTE: change the following to match your system
|
||||
binary_path="/home/ubuntu/llm.c/train_gpt2cu"
|
||||
out_dir="/ephemeral/data/fineweb/log_gpt2_124M_multi"
|
||||
train_data_path='/ephemeral/data/fineweb/bin_10B/fineweb_train_*.bin'
|
||||
val_data_path='/ephemeral/data/fineweb/bin_10B/fineweb_val_*.bin'
|
||||
# NOTE: change the server_ip to the IP address of the machine that is running process zero
|
||||
server_ip="10.0.1.220"
|
||||
|
||||
# In case the file system is shared this is a no-op.
|
||||
# Otherwise, we need to copy the binary to all nodes.
|
||||
current_user=$USER
|
||||
hosts=$(scontrol show hostnames $SLURM_JOB_NODELIST) # get the hostnames of the allocated nodes
|
||||
current_host=$(hostname)
|
||||
for host in $hosts; do
|
||||
if [ $host == $current_host ]; then
|
||||
continue
|
||||
fi
|
||||
echo "copying $binary_path to $current_user@$host"
|
||||
scp -r $binary_path $current_user@$host:$binary_path
|
||||
done
|
||||
|
||||
# Use this for NCCL debugging if you run into issues
|
||||
# export NCCL_DEBUG=INFO
|
||||
# export NCCL_DEBUG_SUBSYS=ALL
|
||||
export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
|
||||
|
||||
# Optimization flags
|
||||
export NCCL_NET_GDR_LEVEL=2 # use GPUDirect RDMA - allows for direct memory access between GPUs across different nodes by bypassing the CPU
|
||||
export NCCL_IB_DISABLE=0 # use InfiniBand if available
|
||||
|
||||
# NOTE: change the following environment variables to match your system - or comment them out if you don't need them
|
||||
export NCCL_SOCKET_IFNAME=ens17
|
||||
export OMPI_MCA_btl_tcp_if_include=ens17
|
||||
export NCCL_P2P_LEVEL=PXB
|
||||
|
||||
if [ -z "$SLURM_JOB_ID" ]; then
|
||||
echo "Make sure you're running in a SLURM environment. Did you forget to run with sbatch? Aborting."
|
||||
exit 1
|
||||
else
|
||||
DATESTRING=`date "+%Y-%m-%dT%H:%M:%S"`
|
||||
echo "Running in a SLURM environment (job ID: $SLURM_JOB_ID, user: $current_user)"
|
||||
echo "Running on hosts: $(echo $(scontrol show hostname))"
|
||||
echo "$DATESTRING"
|
||||
fi
|
||||
|
||||
srun -l -u bash -c "
|
||||
$binary_path \
|
||||
-i '$train_data_path' \
|
||||
-j '$val_data_path' \
|
||||
-o $out_dir \
|
||||
-v 250 -s 20000 -g 144 \
|
||||
-h 1 \
|
||||
-b 64 -t 1024 \
|
||||
-d 2097152 \
|
||||
-r 0 \
|
||||
-z 1 \
|
||||
-c 0.1 \
|
||||
-l 0.0006 \
|
||||
-q 0.0 \
|
||||
-u 700 \
|
||||
-n 5000 \
|
||||
-y 1 \
|
||||
-e d12 \
|
||||
-pn \$SLURM_NTASKS \
|
||||
-pr \$SLURM_PROCID \
|
||||
-pg \$SLURM_NTASKS_PER_NODE \
|
||||
-ps $server_ip \
|
||||
-pi "tcp" \
|
||||
"
|
||||
|
||||
echo "$DATESTRING"
|
||||
Reference in New Issue
Block a user