chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
Executable
+234
@@ -0,0 +1,234 @@
|
||||
#!/usr/bin/env bash
|
||||
# End-to-end smoke test for /storage/v1/s3.
|
||||
# Runs as a "cloud" test — only invoked by run-all-tests.sh when
|
||||
# AWS_S3_BUCKET and APP_KEY are already set.
|
||||
#
|
||||
# Requirements (from the test runner's perspective):
|
||||
# - Backend running at $TEST_API_BASE (default http://localhost:7130/api),
|
||||
# configured with a real S3 or S3-compatible backend.
|
||||
# - ACCESS_API_KEY set (project admin API key).
|
||||
# - aws CLI and jq installed.
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
|
||||
PROJECT_ROOT="$(dirname "$(dirname "$SCRIPT_DIR")")"
|
||||
|
||||
# Load .env like the sibling cloud test does.
|
||||
if [ -z "${AWS_S3_BUCKET:-}" ] && [ -f "$PROJECT_ROOT/.env" ]; then
|
||||
set -a
|
||||
source "$PROJECT_ROOT/.env"
|
||||
set +a
|
||||
fi
|
||||
|
||||
# TEST_API_BASE already includes /api (matches sibling cloud tests).
|
||||
if [ -n "${TEST_API_BASE:-}" ]; then
|
||||
API_BASE="$TEST_API_BASE"
|
||||
else
|
||||
API_BASE="http://localhost:7130/api"
|
||||
fi
|
||||
# Gateway base: strip trailing /api if present.
|
||||
GATEWAY_HOST="${API_BASE%/api}"
|
||||
GATEWAY_URL="$GATEWAY_HOST/storage/v1/s3"
|
||||
|
||||
API_KEY="${ACCESS_API_KEY:-}"
|
||||
|
||||
# Graceful skip if tooling or creds are missing — run-all-tests.sh treats
|
||||
# exit 0 as a pass, and CI should not fail for missing prerequisites.
|
||||
for tool in aws jq curl; do
|
||||
if ! command -v "$tool" > /dev/null 2>&1; then
|
||||
echo "SKIP: $tool not installed"
|
||||
exit 0
|
||||
fi
|
||||
done
|
||||
if [ -z "$API_KEY" ]; then
|
||||
echo "SKIP: ACCESS_API_KEY not set"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
BUCKET="${BUCKET:-s3gw-e2e-$$-$RANDOM}"
|
||||
|
||||
# Per-run temp directory so parallel runs don't collide on /tmp/s3gw-* paths,
|
||||
# and the EXIT trap always removes exactly the files this run created.
|
||||
WORK_DIR="$(mktemp -d -t s3gw-e2e.XXXXXX)"
|
||||
|
||||
log() { printf "\n\033[1;34m--> %s\033[0m\n" "$*"; }
|
||||
ok() { printf " \033[1;32mOK\033[0m %s\n" "$*"; }
|
||||
warn() { printf " \033[1;33mWARN\033[0m %s\n" "$*"; }
|
||||
|
||||
AK=""
|
||||
SK=""
|
||||
KID=""
|
||||
CREATED_BUCKET=0
|
||||
|
||||
cleanup() {
|
||||
local rc=$?
|
||||
set +e
|
||||
if [ -n "$AK" ] && [ $CREATED_BUCKET -eq 1 ]; then
|
||||
log "cleanup: removing bucket $BUCKET"
|
||||
AWS_ACCESS_KEY_ID="$AK" AWS_SECRET_ACCESS_KEY="$SK" AWS_DEFAULT_REGION="us-east-2" \
|
||||
aws --endpoint-url "$GATEWAY_URL" s3 rm "s3://$BUCKET/" --recursive >/dev/null 2>&1 || \
|
||||
warn "bucket contents cleanup failed"
|
||||
AWS_ACCESS_KEY_ID="$AK" AWS_SECRET_ACCESS_KEY="$SK" AWS_DEFAULT_REGION="us-east-2" \
|
||||
aws --endpoint-url "$GATEWAY_URL" s3 rb "s3://$BUCKET" >/dev/null 2>&1 || \
|
||||
warn "bucket removal failed"
|
||||
fi
|
||||
if [ -n "$KID" ]; then
|
||||
log "cleanup: revoking access key"
|
||||
curl -sS -X DELETE -H "x-api-key: $API_KEY" \
|
||||
"$API_BASE/storage/s3/access-keys/$KID" > /dev/null 2>&1 || \
|
||||
warn "access key revocation failed"
|
||||
fi
|
||||
rm -rf "$WORK_DIR"
|
||||
exit "$rc"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
log "Creating S3 access key"
|
||||
RESP=$(curl -sS -X POST -H "x-api-key: $API_KEY" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"description":"ci e2e"}' \
|
||||
"$API_BASE/storage/s3/access-keys")
|
||||
AK=$(echo "$RESP" | jq -r '.data.accessKeyId // .accessKeyId // empty')
|
||||
SK=$(echo "$RESP" | jq -r '.data.secretAccessKey // .secretAccessKey // empty')
|
||||
KID=$(echo "$RESP" | jq -r '.data.id // .id // empty')
|
||||
if [ -z "$AK" ] || [ -z "$SK" ] || [ -z "$KID" ]; then
|
||||
printf "failed to create access key, response was: %s\n" "$RESP" >&2
|
||||
exit 1
|
||||
fi
|
||||
ok "access key $AK"
|
||||
|
||||
export AWS_ACCESS_KEY_ID="$AK"
|
||||
export AWS_SECRET_ACCESS_KEY="$SK"
|
||||
export AWS_DEFAULT_REGION="us-east-2"
|
||||
|
||||
AWSCMD=(aws --endpoint-url "$GATEWAY_URL")
|
||||
|
||||
log "Listing buckets"
|
||||
"${AWSCMD[@]}" s3 ls
|
||||
|
||||
log "Creating bucket $BUCKET"
|
||||
"${AWSCMD[@]}" s3 mb "s3://$BUCKET"
|
||||
CREATED_BUCKET=1
|
||||
|
||||
log "Uploading a small file"
|
||||
echo "hello world" > "$WORK_DIR/small.txt"
|
||||
"${AWSCMD[@]}" s3 cp "$WORK_DIR/small.txt" "s3://$BUCKET/small.txt"
|
||||
ok "upload"
|
||||
|
||||
log "Downloading it back"
|
||||
"${AWSCMD[@]}" s3 cp "s3://$BUCKET/small.txt" "$WORK_DIR/small.out"
|
||||
diff "$WORK_DIR/small.txt" "$WORK_DIR/small.out"
|
||||
ok "round-trip identity"
|
||||
|
||||
log "Listing objects"
|
||||
"${AWSCMD[@]}" s3 ls "s3://$BUCKET/"
|
||||
|
||||
# Drive multipart explicitly via aws s3api so coverage doesn't depend on the
|
||||
# caller's aws-cli multipart_threshold config. Two 10 MiB parts + complete.
|
||||
log "Running multipart upload explicitly (2 parts via s3api)"
|
||||
dd if=/dev/urandom of="$WORK_DIR/big.bin" bs=1M count=20 status=none
|
||||
split -b $((10 * 1024 * 1024)) "$WORK_DIR/big.bin" "$WORK_DIR/part-"
|
||||
PART1="$WORK_DIR/part-aa"
|
||||
PART2="$WORK_DIR/part-ab"
|
||||
|
||||
UPLOAD_ID=$("${AWSCMD[@]}" s3api create-multipart-upload \
|
||||
--bucket "$BUCKET" --key big.bin | jq -r '.UploadId')
|
||||
[ -n "$UPLOAD_ID" ] || { echo "no UploadId returned"; exit 1; }
|
||||
ok "uploadId $UPLOAD_ID"
|
||||
|
||||
ETAG1=$("${AWSCMD[@]}" s3api upload-part \
|
||||
--bucket "$BUCKET" --key big.bin --part-number 1 --upload-id "$UPLOAD_ID" \
|
||||
--body "$PART1" | jq -r '.ETag')
|
||||
ETAG2=$("${AWSCMD[@]}" s3api upload-part \
|
||||
--bucket "$BUCKET" --key big.bin --part-number 2 --upload-id "$UPLOAD_ID" \
|
||||
--body "$PART2" | jq -r '.ETag')
|
||||
|
||||
"${AWSCMD[@]}" s3api complete-multipart-upload \
|
||||
--bucket "$BUCKET" --key big.bin --upload-id "$UPLOAD_ID" \
|
||||
--multipart-upload "$(jq -n --arg e1 "$ETAG1" --arg e2 "$ETAG2" \
|
||||
'{Parts: [{PartNumber: 1, ETag: $e1}, {PartNumber: 2, ETag: $e2}]}')" \
|
||||
> /dev/null
|
||||
|
||||
# AWS S3 multipart ETags are suffixed with "-<part count>" per the docs, so
|
||||
# this assertion verifies we went through the multipart code path regardless
|
||||
# of aws-cli config.
|
||||
HEAD_ETAG=$("${AWSCMD[@]}" s3api head-object --bucket "$BUCKET" --key big.bin | jq -r '.ETag')
|
||||
case "$HEAD_ETAG" in
|
||||
*-2\") ok "multipart upload (ETag=$HEAD_ETAG)" ;;
|
||||
*) printf "ETag %s is not multipart-style; expected *-2\n" "$HEAD_ETAG" >&2; exit 1 ;;
|
||||
esac
|
||||
|
||||
log "Downloading 20 MB file"
|
||||
"${AWSCMD[@]}" s3 cp "s3://$BUCKET/big.bin" "$WORK_DIR/big.out"
|
||||
diff "$WORK_DIR/big.bin" "$WORK_DIR/big.out"
|
||||
ok "multipart round-trip"
|
||||
|
||||
log "aws s3 sync roundtrip"
|
||||
mkdir -p "$WORK_DIR/dir/a" "$WORK_DIR/dir/b"
|
||||
echo A > "$WORK_DIR/dir/a/x.txt"
|
||||
echo B > "$WORK_DIR/dir/b/y.txt"
|
||||
"${AWSCMD[@]}" s3 sync "$WORK_DIR/dir" "s3://$BUCKET/dir/"
|
||||
"${AWSCMD[@]}" s3 sync "s3://$BUCKET/dir" "$WORK_DIR/dir-back"
|
||||
diff -r "$WORK_DIR/dir" "$WORK_DIR/dir-back"
|
||||
ok "sync"
|
||||
|
||||
# Modern aws-cli (≥2.30) turns on default integrity protections, which sends
|
||||
# PutObject / UploadPart bodies in aws-chunked framing with
|
||||
# x-amz-content-sha256: STREAMING-UNSIGNED-PAYLOAD-TRAILER. The 20 MiB `aws
|
||||
# s3 cp` upload below auto-splits into multipart parts, each one going
|
||||
# through the UploadPart path in that same format. If this test regresses
|
||||
# with "Body hash mismatch" or "Only the last chunk is allowed…", the
|
||||
# gateway has stopped handling that payload-hash variant.
|
||||
log "aws s3 cp multipart round-trip (exercises UNSIGNED-PAYLOAD-TRAILER UploadPart)"
|
||||
dd if=/dev/urandom of="$WORK_DIR/multi.bin" bs=1M count=20 status=none
|
||||
"${AWSCMD[@]}" s3 cp "$WORK_DIR/multi.bin" "s3://$BUCKET/multi.bin"
|
||||
"${AWSCMD[@]}" s3 cp "s3://$BUCKET/multi.bin" "$WORK_DIR/multi.out"
|
||||
diff "$WORK_DIR/multi.bin" "$WORK_DIR/multi.out"
|
||||
MULTI_ETAG=$("${AWSCMD[@]}" s3api head-object --bucket "$BUCKET" --key multi.bin | jq -r '.ETag')
|
||||
case "$MULTI_ETAG" in
|
||||
*-*\") ok "aws-cli multipart round-trip (ETag=$MULTI_ETAG)" ;;
|
||||
*) printf "ETag %s is not multipart-style; aws-cli didn't exercise UploadPart\n" "$MULTI_ETAG" >&2; exit 1 ;;
|
||||
esac
|
||||
|
||||
# Force STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER via the JS SDK. The
|
||||
# aws-cli path above covers UNSIGNED-PAYLOAD-TRAILER; the signed-trailer
|
||||
# variant is what clients pick when they enable default integrity *and*
|
||||
# keep per-chunk signing (the AWS SDK for JavaScript does this when you
|
||||
# set requestChecksumCalculation=WHEN_SUPPORTED and don't opt into
|
||||
# unsigned streaming).
|
||||
if command -v node >/dev/null 2>&1; then
|
||||
log "JS SDK v3 upload with signed-trailer (STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER)"
|
||||
NODE_SCRIPT="$WORK_DIR/signed-trailer.mjs"
|
||||
cat > "$NODE_SCRIPT" <<'JS'
|
||||
import { S3Client, PutObjectCommand, GetObjectCommand } from '@aws-sdk/client-s3';
|
||||
const [endpoint, bucket, ak, sk] = process.argv.slice(2);
|
||||
const client = new S3Client({
|
||||
endpoint,
|
||||
region: 'us-east-2',
|
||||
forcePathStyle: true,
|
||||
credentials: { accessKeyId: ak, secretAccessKey: sk },
|
||||
// WHEN_SUPPORTED triggers trailing checksums; request signing is on by
|
||||
// default, so the body flows as STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER.
|
||||
requestChecksumCalculation: 'WHEN_SUPPORTED',
|
||||
});
|
||||
const payload = Buffer.alloc(500_000, 0x41); // 500 KiB > 64 KiB → multi-chunk
|
||||
await client.send(new PutObjectCommand({
|
||||
Bucket: bucket, Key: 'signed-trailer.bin',
|
||||
Body: payload, ContentType: 'application/octet-stream',
|
||||
}));
|
||||
const got = await client.send(new GetObjectCommand({ Bucket: bucket, Key: 'signed-trailer.bin' }));
|
||||
const chunks = [];
|
||||
for await (const c of got.Body) chunks.push(c);
|
||||
const back = Buffer.concat(chunks);
|
||||
if (!back.equals(payload)) { console.error('round-trip mismatch'); process.exit(1); }
|
||||
console.log('OK ' + back.length + ' bytes round-tripped');
|
||||
JS
|
||||
(cd "$WORK_DIR" && npm init -y >/dev/null 2>&1 && npm install --silent --no-audit --no-fund @aws-sdk/client-s3 >/dev/null 2>&1)
|
||||
node "$NODE_SCRIPT" "$GATEWAY_URL" "$BUCKET" "$AK" "$SK"
|
||||
ok "JS SDK signed-trailer round-trip"
|
||||
else
|
||||
warn "node not installed — skipping signed-trailer test"
|
||||
fi
|
||||
|
||||
printf "\n\033[1;32mAll S3 gateway smoke checks passed.\033[0m\n"
|
||||
# cleanup() via trap handles access-key revocation and bucket teardown.
|
||||
Executable
+132
@@ -0,0 +1,132 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Test script for multi-tenant S3 storage
|
||||
# This script tests the storage API with S3 backend and app key folder structure
|
||||
|
||||
# Get the directory where this script is located
|
||||
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
|
||||
PROJECT_ROOT="$(dirname "$(dirname "$SCRIPT_DIR")")"
|
||||
|
||||
# Load .env file if AWS variables not already set
|
||||
if [ -z "$AWS_S3_BUCKET" ] && [ -f "$PROJECT_ROOT/.env" ]; then
|
||||
echo "Loading environment from .env file..."
|
||||
set -a # automatically export all variables
|
||||
source "$PROJECT_ROOT/.env"
|
||||
set +a # turn off automatic export
|
||||
fi
|
||||
|
||||
# Source test configuration
|
||||
source "$SCRIPT_DIR/../test-config.sh"
|
||||
|
||||
API_KEY="${ACCESS_API_KEY:-}"
|
||||
# Use TEST_API_BASE if set, otherwise default to localhost
|
||||
if [ -n "$TEST_API_BASE" ]; then
|
||||
# TEST_API_BASE already includes /api
|
||||
BASE_URL="$TEST_API_BASE"
|
||||
else
|
||||
BASE_URL="http://localhost:7130/api"
|
||||
fi
|
||||
|
||||
echo "=== Testing Multi-tenant S3 Storage ==="
|
||||
echo "Note: Ensure AWS_S3_BUCKET and APP_KEY are set in your .env file"
|
||||
echo "Base URL: $BASE_URL"
|
||||
echo "API Key: ${API_KEY:0:10}..." # Show first 10 chars only
|
||||
echo "AWS Bucket: ${AWS_S3_BUCKET:-not set}"
|
||||
echo "App Key: ${APP_KEY:-not set}"
|
||||
echo ""
|
||||
|
||||
# Create a test bucket
|
||||
echo "1. Creating test bucket..."
|
||||
curl -X POST "$BASE_URL/storage/buckets" \
|
||||
-H "Content-Type: application/json" \
|
||||
-H "x-api-key: $API_KEY" \
|
||||
-d '{"bucketName": "test-bucket", "isPublic": true}' | jq .
|
||||
|
||||
echo ""
|
||||
|
||||
# List buckets
|
||||
echo "2. Listing buckets..."
|
||||
curl "$BASE_URL/storage/buckets" \
|
||||
-H "x-api-key: $API_KEY" | jq .
|
||||
|
||||
echo ""
|
||||
|
||||
# Upload a file with specific key
|
||||
echo "3. Uploading file with specific key..."
|
||||
echo "Test content for S3" > test-file.txt
|
||||
curl -X PUT "$BASE_URL/storage/test-bucket/test-file.txt" \
|
||||
-H "x-api-key: $API_KEY" \
|
||||
-F "file=@test-file.txt" | jq .
|
||||
|
||||
echo ""
|
||||
|
||||
# Upload a file with auto-generated key
|
||||
echo "4. Uploading file with auto-generated key..."
|
||||
curl -X POST "$BASE_URL/storage/test-bucket" \
|
||||
-H "x-api-key: $API_KEY" \
|
||||
-F "file=@test-file.txt" | jq .
|
||||
|
||||
echo ""
|
||||
|
||||
# List objects in bucket
|
||||
echo "5. Listing objects in bucket..."
|
||||
curl "$BASE_URL/storage/test-bucket" \
|
||||
-H "x-api-key: $API_KEY" | jq .
|
||||
|
||||
echo ""
|
||||
|
||||
# Download a file (public bucket, no auth needed)
|
||||
echo "6. Downloading file from public bucket..."
|
||||
curl "$BASE_URL/storage/test-bucket/test-file.txt" \
|
||||
-o downloaded-file.txt
|
||||
echo "Downloaded content:"
|
||||
cat downloaded-file.txt
|
||||
echo ""
|
||||
|
||||
# Update bucket visibility to private
|
||||
echo "7. Making bucket private..."
|
||||
curl -X PATCH "$BASE_URL/storage/buckets/test-bucket" \
|
||||
-H "Content-Type: application/json" \
|
||||
-H "x-api-key: $API_KEY" \
|
||||
-d '{"isPublic": false}' | jq .
|
||||
|
||||
echo ""
|
||||
|
||||
# Try downloading from private bucket without auth (should fail)
|
||||
echo "8. Trying to download from private bucket without auth (should fail)..."
|
||||
curl -v "$BASE_URL/api/storage/test-bucket/test-file.txt" 2>&1 | grep "< HTTP"
|
||||
|
||||
echo ""
|
||||
|
||||
# Download from private bucket with auth
|
||||
echo "9. Downloading from private bucket with auth..."
|
||||
curl "$BASE_URL/storage/test-bucket/test-file.txt" \
|
||||
-H "x-api-key: $API_KEY" \
|
||||
-o downloaded-private.txt
|
||||
echo "Downloaded content:"
|
||||
cat downloaded-private.txt
|
||||
echo ""
|
||||
|
||||
# Delete a file
|
||||
echo "10. Deleting a file..."
|
||||
curl -X DELETE "$BASE_URL/storage/test-bucket/test-file.txt" \
|
||||
-H "x-api-key: $API_KEY" | jq .
|
||||
|
||||
echo ""
|
||||
|
||||
# Delete the bucket
|
||||
echo "11. Deleting the bucket..."
|
||||
curl -X DELETE "$BASE_URL/storage/test-bucket" \
|
||||
-H "x-api-key: $API_KEY" | jq .
|
||||
|
||||
echo ""
|
||||
|
||||
# Clean up
|
||||
rm -f test-file.txt downloaded-file.txt downloaded-private.txt
|
||||
|
||||
echo "=== S3 Multi-tenant Storage Test Complete ==="
|
||||
echo ""
|
||||
echo "When AWS_S3_BUCKET is set, files are stored in S3 with structure:"
|
||||
echo " s3://\${AWS_S3_BUCKET}/\${APP_KEY}/\${bucket}/\${key}"
|
||||
echo ""
|
||||
echo "Example: s3://my-bucket/app12345/test-bucket/test-file.txt"
|
||||
Reference in New Issue
Block a user