chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,194 @@
|
||||
#!/bin/bash
|
||||
# Bright Data - Dataset Wrapper
|
||||
# Provides easy access to all supported datasets
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
DATASET_TYPE="$1"
|
||||
shift
|
||||
|
||||
declare -A DATASETS=(
|
||||
["amazon_product"]="gd_l7q7dkf244hwjntr0"
|
||||
["amazon_product_reviews"]="gd_le8e811kzy4ggddlq"
|
||||
["amazon_product_search"]="gd_lwdb4vjm1ehb499uxs"
|
||||
["walmart_product"]="gd_l95fol7l1ru6rlo116"
|
||||
["walmart_seller"]="gd_m7ke48w81ocyu4hhz0"
|
||||
["ebay_product"]="gd_ltr9mjt81n0zzdk1fb"
|
||||
["homedepot_products"]="gd_lmusivh019i7g97q2n"
|
||||
["zara_products"]="gd_lct4vafw1tgx27d4o0"
|
||||
["etsy_products"]="gd_ltppk0jdv1jqz25mz"
|
||||
["bestbuy_products"]="gd_ltre1jqe1jfr7cccf"
|
||||
["linkedin_person_profile"]="gd_l1viktl72bvl7bjuj0"
|
||||
["linkedin_company_profile"]="gd_l1vikfnt1wgvvqz95w"
|
||||
["linkedin_job_listings"]="gd_lpfll7v5hcqtkxl6l"
|
||||
["linkedin_posts"]="gd_lyy3tktm25m4avu764"
|
||||
["linkedin_people_search"]="gd_m8d03he47z8nwb5xc"
|
||||
["crunchbase_company"]="gd_l1vijqt9jfj7olije"
|
||||
["zoominfo_company_profile"]="gd_m0ci4a4ivx3j5l6nx"
|
||||
["instagram_profiles"]="gd_l1vikfch901nx3by4"
|
||||
["instagram_posts"]="gd_lk5ns7kz21pck8jpis"
|
||||
["instagram_reels"]="gd_lyclm20il4r5helnj"
|
||||
["instagram_comments"]="gd_ltppn085pokosxh13"
|
||||
["facebook_posts"]="gd_lyclm1571iy3mv57zw"
|
||||
["facebook_marketplace_listings"]="gd_lvt9iwuh6fbcwmx1a"
|
||||
["facebook_company_reviews"]="gd_m0dtqpiu1mbcyc2g86"
|
||||
["facebook_events"]="gd_m14sd0to1jz48ppm51"
|
||||
["tiktok_profiles"]="gd_l1villgoiiidt09ci"
|
||||
["tiktok_posts"]="gd_lu702nij2f790tmv9h"
|
||||
["tiktok_shop"]="gd_m45m1u911dsa4274pi"
|
||||
["tiktok_comments"]="gd_lkf2st302ap89utw5k"
|
||||
["x_posts"]="gd_lwxkxvnf1cynvib9co"
|
||||
["youtube_profiles"]="gd_lk538t2k2p1k3oos71"
|
||||
["youtube_videos"]="gd_lk56epmy2i5g7lzu0k"
|
||||
["youtube_comments"]="gd_lk9q0ew71spt1mxywf"
|
||||
["reddit_posts"]="gd_lvz8ah06191smkebj4"
|
||||
["google_maps_reviews"]="gd_luzfs1dn2oa0teb81"
|
||||
["google_shopping"]="gd_ltppk50q18kdw67omz"
|
||||
["google_play_store"]="gd_lsk382l8xei8vzm4u"
|
||||
["apple_app_store"]="gd_lsk9ki3u2iishmwrui"
|
||||
["reuter_news"]="gd_lyptx9h74wtlvpnfu"
|
||||
["github_repository_file"]="gd_lyrexgxc24b3d4imjt"
|
||||
["yahoo_finance_business"]="gd_lmrpz3vxmz972ghd7"
|
||||
["zillow_properties_listing"]="gd_lfqkr8wm13ixtbd8f5"
|
||||
["booking_hotel_listings"]="gd_m5mbdl081229ln6t4a"
|
||||
)
|
||||
|
||||
if [ -z "$DATASET_TYPE" ]; then
|
||||
echo "Usage: $0 <dataset_type> <url> [additional_params...]" >&2
|
||||
echo "" >&2
|
||||
echo "Available datasets:" >&2
|
||||
echo "" >&2
|
||||
echo "E-COMMERCE:" >&2
|
||||
echo " amazon_product <url> - Amazon product data" >&2
|
||||
echo " amazon_product_reviews <url> - Amazon product reviews" >&2
|
||||
echo " amazon_product_search <keyword> <domain_url> - Amazon search" >&2
|
||||
echo " walmart_product <url> - Walmart product data" >&2
|
||||
echo " walmart_seller <url> - Walmart seller data" >&2
|
||||
echo " ebay_product <url> - eBay product data" >&2
|
||||
echo " homedepot_products <url> - Home Depot product data" >&2
|
||||
echo " zara_products <url> - Zara product data" >&2
|
||||
echo " etsy_products <url> - Etsy product data" >&2
|
||||
echo " bestbuy_products <url> - Best Buy product data" >&2
|
||||
echo "" >&2
|
||||
echo "PROFESSIONAL NETWORKS:" >&2
|
||||
echo " linkedin_person_profile <url> - LinkedIn person profile" >&2
|
||||
echo " linkedin_company_profile <url> - LinkedIn company profile" >&2
|
||||
echo " linkedin_job_listings <url> - LinkedIn job listings" >&2
|
||||
echo " linkedin_posts <url> - LinkedIn posts" >&2
|
||||
echo " linkedin_people_search <url> <first_name> <last_name> - LinkedIn people search" >&2
|
||||
echo " crunchbase_company <url> - Crunchbase company data" >&2
|
||||
echo " zoominfo_company_profile <url> - ZoomInfo company profile" >&2
|
||||
echo "" >&2
|
||||
echo "SOCIAL MEDIA - INSTAGRAM:" >&2
|
||||
echo " instagram_profiles <url> - Instagram profile data" >&2
|
||||
echo " instagram_posts <url> - Instagram post data" >&2
|
||||
echo " instagram_reels <url> - Instagram reel data" >&2
|
||||
echo " instagram_comments <url> - Instagram comments" >&2
|
||||
echo "" >&2
|
||||
echo "SOCIAL MEDIA - FACEBOOK:" >&2
|
||||
echo " facebook_posts <url> - Facebook post data" >&2
|
||||
echo " facebook_marketplace_listings <url> - Facebook marketplace" >&2
|
||||
echo " facebook_company_reviews <url> <num_reviews> - Company reviews" >&2
|
||||
echo " facebook_events <url> - Facebook events" >&2
|
||||
echo "" >&2
|
||||
echo "SOCIAL MEDIA - TIKTOK:" >&2
|
||||
echo " tiktok_profiles <url> - TikTok profile data" >&2
|
||||
echo " tiktok_posts <url> - TikTok post data" >&2
|
||||
echo " tiktok_shop <url> - TikTok shop product" >&2
|
||||
echo " tiktok_comments <url> - TikTok video comments" >&2
|
||||
echo "" >&2
|
||||
echo "SOCIAL MEDIA - OTHER:" >&2
|
||||
echo " x_posts <url> - X (Twitter) post data" >&2
|
||||
echo " youtube_profiles <url> - YouTube channel data" >&2
|
||||
echo " youtube_videos <url> - YouTube video data" >&2
|
||||
echo " youtube_comments <url> [num] - YouTube comments (default: 10)" >&2
|
||||
echo " reddit_posts <url> - Reddit post data" >&2
|
||||
echo "" >&2
|
||||
echo "GOOGLE SERVICES:" >&2
|
||||
echo " google_maps_reviews <url> [days] - Google Maps reviews (default: 3 days)" >&2
|
||||
echo " google_shopping <url> - Google Shopping product" >&2
|
||||
echo " google_play_store <url> - Google Play Store app" >&2
|
||||
echo "" >&2
|
||||
echo "OTHER:" >&2
|
||||
echo " apple_app_store <url> - Apple App Store app" >&2
|
||||
echo " reuter_news <url> - Reuters news article" >&2
|
||||
echo " github_repository_file <url> - GitHub repository file" >&2
|
||||
echo " yahoo_finance_business <url> - Yahoo Finance business" >&2
|
||||
echo " zillow_properties_listing <url> - Zillow property listing" >&2
|
||||
echo " booking_hotel_listings <url> - Booking.com hotel" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Get dataset ID
|
||||
DATASET_ID="${DATASETS[$DATASET_TYPE]}"
|
||||
|
||||
if [ -z "$DATASET_ID" ]; then
|
||||
echo "Error: Unknown dataset type '$DATASET_TYPE'" >&2
|
||||
echo "Run '$0' without arguments to see available datasets" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
case "$DATASET_TYPE" in
|
||||
amazon_product_search)
|
||||
KEYWORD="$1"
|
||||
DOMAIN_URL="$2"
|
||||
if [ -z "$KEYWORD" ] || [ -z "$DOMAIN_URL" ]; then
|
||||
echo "Usage: $0 amazon_product_search <keyword> <domain_url>" >&2
|
||||
exit 1
|
||||
fi
|
||||
INPUT_JSON=$(jq -n --arg keyword "$KEYWORD" --arg url "$DOMAIN_URL" \
|
||||
'{keyword: $keyword, url: $url, pages_to_search: "1"}')
|
||||
;;
|
||||
linkedin_people_search)
|
||||
URL="$1"
|
||||
FIRST_NAME="$2"
|
||||
LAST_NAME="$3"
|
||||
if [ -z "$URL" ] || [ -z "$FIRST_NAME" ] || [ -z "$LAST_NAME" ]; then
|
||||
echo "Usage: $0 linkedin_people_search <url> <first_name> <last_name>" >&2
|
||||
exit 1
|
||||
fi
|
||||
INPUT_JSON=$(jq -n --arg url "$URL" --arg first "$FIRST_NAME" --arg last "$LAST_NAME" \
|
||||
'{url: $url, first_name: $first, last_name: $last}')
|
||||
;;
|
||||
facebook_company_reviews)
|
||||
URL="$1"
|
||||
NUM_REVIEWS="${2:-10}"
|
||||
if [ -z "$URL" ]; then
|
||||
echo "Usage: $0 facebook_company_reviews <url> [num_reviews]" >&2
|
||||
exit 1
|
||||
fi
|
||||
INPUT_JSON=$(jq -n --arg url "$URL" --arg num "$NUM_REVIEWS" \
|
||||
'{url: $url, num_of_reviews: $num}')
|
||||
;;
|
||||
google_maps_reviews)
|
||||
URL="$1"
|
||||
DAYS_LIMIT="${2:-3}"
|
||||
if [ -z "$URL" ]; then
|
||||
echo "Usage: $0 google_maps_reviews <url> [days_limit]" >&2
|
||||
exit 1
|
||||
fi
|
||||
INPUT_JSON=$(jq -n --arg url "$URL" --arg days "$DAYS_LIMIT" \
|
||||
'{url: $url, days_limit: $days}')
|
||||
;;
|
||||
youtube_comments)
|
||||
URL="$1"
|
||||
NUM_COMMENTS="${2:-10}"
|
||||
if [ -z "$URL" ]; then
|
||||
echo "Usage: $0 youtube_comments <url> [num_comments]" >&2
|
||||
exit 1
|
||||
fi
|
||||
INPUT_JSON=$(jq -n --arg url "$URL" --arg num "$NUM_COMMENTS" \
|
||||
'{url: $url, num_of_comments: $num}')
|
||||
;;
|
||||
*)
|
||||
# Default: just URL input
|
||||
URL="$1"
|
||||
if [ -z "$URL" ]; then
|
||||
echo "Usage: $0 $DATASET_TYPE <url>" >&2
|
||||
exit 1
|
||||
fi
|
||||
INPUT_JSON=$(jq -n --arg url "$URL" '{url: $url}')
|
||||
;;
|
||||
esac
|
||||
|
||||
# Call the fetch script
|
||||
exec "$SCRIPT_DIR/fetch.sh" "$DATASET_ID" "$INPUT_JSON"
|
||||
@@ -0,0 +1,58 @@
|
||||
#!/bin/bash
|
||||
# Bright Data - Dataset Fetch with Polling
|
||||
# Triggers a dataset collection and polls until results are ready
|
||||
|
||||
DATASET_ID="$1"
|
||||
shift
|
||||
INPUT_JSON="$1"
|
||||
|
||||
if [ -z "$DATASET_ID" ] || [ -z "$INPUT_JSON" ]; then
|
||||
echo "Usage: $0 <dataset_id> '<json_input>'" >&2
|
||||
echo "Example: $0 gd_l1viktl72bvl7bjuj0 '{\"url\":\"https://linkedin.com/in/someone\"}'" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ -z "${BRIGHTDATA_API_KEY:-}" ]; then
|
||||
echo "Error: BRIGHTDATA_API_KEY is not set." >&2
|
||||
echo "Get a key from https://brightdata.com/cp" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
POLLING_TIMEOUT="${BRIGHTDATA_POLLING_TIMEOUT:-600}"
|
||||
|
||||
TRIGGER_RESPONSE=$(curl -s -X POST "https://api.brightdata.com/datasets/v3/trigger?dataset_id=${DATASET_ID}&include_errors=true" \
|
||||
-H "Authorization: Bearer $BRIGHTDATA_API_KEY" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "[$INPUT_JSON]")
|
||||
|
||||
SNAPSHOT_ID=$(echo "$TRIGGER_RESPONSE" | jq -r '.snapshot_id // empty')
|
||||
|
||||
if [ -z "$SNAPSHOT_ID" ]; then
|
||||
echo "Error: Failed to trigger dataset collection" >&2
|
||||
echo "$TRIGGER_RESPONSE" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Triggered collection with snapshot ID: $SNAPSHOT_ID" >&2
|
||||
|
||||
ATTEMPTS=0
|
||||
while [ $ATTEMPTS -lt $POLLING_TIMEOUT ]; do
|
||||
SNAPSHOT_RESPONSE=$(curl -s -X GET "https://api.brightdata.com/datasets/v3/snapshot/${SNAPSHOT_ID}?format=json" \
|
||||
-H "Authorization: Bearer $BRIGHTDATA_API_KEY")
|
||||
|
||||
STATUS=$(echo "$SNAPSHOT_RESPONSE" | jq -r '.status // empty')
|
||||
|
||||
if [ "$STATUS" = "running" ] || [ "$STATUS" = "building" ] || [ "$STATUS" = "starting" ]; then
|
||||
echo "Status: $STATUS - polling again (attempt $((ATTEMPTS + 1))/$POLLING_TIMEOUT)" >&2
|
||||
ATTEMPTS=$((ATTEMPTS + 1))
|
||||
sleep 1
|
||||
continue
|
||||
fi
|
||||
|
||||
echo "Data received after $((ATTEMPTS + 1)) attempts" >&2
|
||||
echo "$SNAPSHOT_RESPONSE" | jq '.'
|
||||
exit 0
|
||||
done
|
||||
|
||||
echo "Error: Timeout after $POLLING_TIMEOUT seconds waiting for data" >&2
|
||||
exit 1
|
||||
Reference in New Issue
Block a user