wehub-resource-sync
94 lines
3.4 KiB
Bash
Executable File
94 lines
3.4 KiB
Bash
Executable File
#!/bin/bash
|
|
|
|
set -e
|
|
pushd $(cd $(dirname ${0})/..; pwd) > /dev/null
|
|
|
|
env_dir=./third_party/env/pytorch
|
|
src_dir=./third_party/source/pytorch
|
|
|
|
case "${OSTYPE}" in
|
|
msys*|cygwin*|mingw*) python="winpty python";;
|
|
*) python="python";;
|
|
esac
|
|
|
|
venv() {
|
|
[ -d "${env_dir}" ] || ${python} -m venv ${env_dir}
|
|
case "${OSTYPE}" in
|
|
msys*|cygwin*|mingw*) source ${env_dir}/Scripts/activate;;
|
|
*) source ${env_dir}/bin/activate;;
|
|
esac
|
|
${python} -m pip install --quiet --upgrade pip requests
|
|
}
|
|
|
|
clean() {
|
|
echo "pytorch clean"
|
|
rm -rf "${env_dir}"
|
|
rm -rf "${src_dir}"
|
|
rm -rf "./third_party/source/caffe2"
|
|
}
|
|
|
|
sync() {
|
|
echo "pytorch sync"
|
|
if [ ! -d "${src_dir}" ]; then
|
|
git clone --quiet --depth=1 --branch main --single-branch -c core.longpaths=true https://github.com/pytorch/pytorch.git "${src_dir}"
|
|
else
|
|
git -C "${src_dir}" fetch --quiet --depth=1 origin main
|
|
git -C "${src_dir}" reset --quiet --hard FETCH_HEAD
|
|
git -C "${src_dir}" gc --quiet --prune=all
|
|
fi
|
|
mkdir -p "./third_party/source/caffe2/proto"
|
|
curl --silent --show-error --location --output "./third_party/source/caffe2/proto/caffe2.proto" "https://github.com/pytorch/pytorch/raw/5e69e11d098a2cfccc8a59377c431e9c71cab9a8/caffe2/proto/caffe2.proto"
|
|
curl --silent --show-error --location --output "./third_party/source/caffe2/proto/torch.proto" "https://github.com/pytorch/pytorch/raw/5e69e11d098a2cfccc8a59377c431e9c71cab9a8/caffe2/proto/torch.proto"
|
|
}
|
|
|
|
install() {
|
|
echo "pytorch install"
|
|
venv
|
|
${python} -m pip install --quiet --upgrade wheel psutil
|
|
${python} -m pip install --quiet --upgrade torch torchvision torchao --pre --index-url https://download.pytorch.org/whl/nightly/cpu
|
|
# ${python} -m pip install --quiet --upgrade torch-neuron --index-url https://pip.repos.neuron.amazonaws.com
|
|
deactivate
|
|
}
|
|
|
|
|
|
schema() {
|
|
echo "pytorch schema"
|
|
[[ $(grep -U $'\x0D' ./source/caffe2-proto.js) ]] && crlf=1
|
|
node ./tools/protoc.js --binary --text --root caffe2 --out ./source/caffe2-proto.js ./third_party/source/caffe2/proto/caffe2.proto
|
|
if [[ -n ${crlf} ]]; then
|
|
unix2dos --quiet --newfile ./source/caffe2-proto.js ./source/caffe2-proto.js
|
|
fi
|
|
[[ $(grep -U $'\x0D' ./source/pytorch-proto.js) ]] && crlf=1
|
|
node ./tools/protoc.js --json --root pytorch --out ./source/pytorch-proto.js --path ./third_party/source ./third_party/source/caffe2/proto/torch.proto
|
|
if [[ -n ${crlf} ]]; then
|
|
unix2dos --quiet --newfile ./source/pytorch-proto.js ./source/pytorch-proto.js
|
|
fi
|
|
[[ $(grep -U $'\x0D' ./source/pytorch-schema.js) ]] && crlf=1
|
|
node ./tools/flatc.js --root torch --out ./source/pytorch-schema.js ./third_party/source/pytorch/torch/csrc/jit/serialization/mobile_bytecode.fbs
|
|
if [[ -n ${crlf} ]]; then
|
|
unix2dos --quiet --newfile ./source/pytorch-schema.js ./source/pytorch-schema.js
|
|
fi
|
|
}
|
|
|
|
metadata() {
|
|
echo "pytorch metadata"
|
|
[[ $(grep -U $'\x0D' ./source/pytorch-metadata.json) ]] && crlf=1
|
|
venv
|
|
${python} ./tools/pytorch_script.py
|
|
deactivate
|
|
if [[ -n ${crlf} ]]; then
|
|
unix2dos --quiet --newfile ./source/pytorch-metadata.json ./source/pytorch-metadata.json
|
|
fi
|
|
}
|
|
|
|
while [ "$#" != 0 ]; do
|
|
command="$1" && shift
|
|
case "${command}" in
|
|
"clean") clean;;
|
|
"install") install;;
|
|
"sync") sync;;
|
|
"schema") schema;;
|
|
"metadata") metadata;;
|
|
esac
|
|
done
|