wehub-resource-sync
a06f331eb8
CI / benchmark (push) Has been skipped
install-script / posix-syntax (push) Successful in 6m1s
CI / build-onnx (push) Failing after 6m43s
init-smoke / dry-run (push) Failing after 15m57s
security / govulncheck (push) Has been cancelled
security / trivy-fs (push) Has been cancelled
CI / test (1.26, ubuntu-latest) (push) Has been cancelled
Scorecard supply-chain security / Scorecard analysis (push) Has been cancelled
CI / test (1.26, macos-latest) (push) Has been cancelled
CI / build-windows (push) Has been cancelled
CI / lint (push) Has been cancelled
install-script / powershell-syntax (push) Has been cancelled
install-script / install (macos-14) (push) Has been cancelled
install-script / install (ubuntu-latest) (push) Has been cancelled
72 lines
2.0 KiB
Bash
Executable File
72 lines
2.0 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Prepare GloVe word vectors for the StaticProvider.
|
|
# Downloads GloVe 6B, extracts top 100k words (300d), converts to binary format.
|
|
#
|
|
# Usage: ./scripts/prepare_vectors.sh
|
|
#
|
|
# Output: internal/embedding/data/vectors.bin.gz (~3MB)
|
|
# Input: Downloads glove.6B.zip (~862MB) to /tmp/
|
|
|
|
set -euo pipefail
|
|
|
|
GLOVE_URL="https://nlp.stanford.edu/data/glove.6B.zip"
|
|
GLOVE_DIR="/tmp/glove"
|
|
OUTPUT_DIR="internal/embedding/data"
|
|
OUTPUT_FILE="${OUTPUT_DIR}/vectors.bin.gz"
|
|
MAX_WORDS=20000
|
|
DIMS=50
|
|
|
|
echo "=== GloVe Vector Preparation ==="
|
|
|
|
# Download if not cached.
|
|
if [ ! -f "${GLOVE_DIR}/glove.6B.${DIMS}d.txt" ]; then
|
|
echo "Downloading GloVe 6B..."
|
|
mkdir -p "${GLOVE_DIR}"
|
|
curl -L -o "${GLOVE_DIR}/glove.6B.zip" "${GLOVE_URL}"
|
|
echo "Extracting..."
|
|
unzip -o "${GLOVE_DIR}/glove.6B.zip" "glove.6B.${DIMS}d.txt" -d "${GLOVE_DIR}"
|
|
fi
|
|
|
|
echo "Converting to binary format (top ${MAX_WORDS} words, ${DIMS}d)..."
|
|
mkdir -p "${OUTPUT_DIR}"
|
|
|
|
python3 -c "
|
|
import struct, gzip, sys
|
|
|
|
input_file = '${GLOVE_DIR}/glove.6B.${DIMS}d.txt'
|
|
output_file = '${OUTPUT_FILE}'
|
|
max_words = ${MAX_WORDS}
|
|
dims = ${DIMS}
|
|
|
|
words = []
|
|
with open(input_file, 'r', encoding='utf-8') as f:
|
|
for i, line in enumerate(f):
|
|
if i >= max_words:
|
|
break
|
|
parts = line.strip().split(' ')
|
|
word = parts[0]
|
|
vec = [float(x) for x in parts[1:]]
|
|
if len(vec) != dims:
|
|
continue
|
|
words.append((word, vec))
|
|
|
|
print(f'Loaded {len(words)} words')
|
|
|
|
with gzip.open(output_file, 'wb') as f:
|
|
# Header: word count + dimensions
|
|
f.write(struct.pack('<II', len(words), dims))
|
|
for word, vec in words:
|
|
word_bytes = word.encode('utf-8')
|
|
f.write(struct.pack('<H', len(word_bytes)))
|
|
f.write(word_bytes)
|
|
f.write(struct.pack(f'<{dims}f', *vec))
|
|
|
|
import os
|
|
size_mb = os.path.getsize(output_file) / (1024 * 1024)
|
|
print(f'Written {output_file} ({size_mb:.1f} MB)')
|
|
"
|
|
|
|
echo "Done. Add the following to static_data.go:"
|
|
echo ' //go:embed data/vectors.bin.gz'
|
|
echo ' var vectorData []byte'
|