#!/bin/bash # Bright Data - Dataset Wrapper # Provides easy access to all supported datasets SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" DATASET_TYPE="$1" shift declare -A DATASETS=( ["amazon_product"]="gd_l7q7dkf244hwjntr0" ["amazon_product_reviews"]="gd_le8e811kzy4ggddlq" ["amazon_product_search"]="gd_lwdb4vjm1ehb499uxs" ["walmart_product"]="gd_l95fol7l1ru6rlo116" ["walmart_seller"]="gd_m7ke48w81ocyu4hhz0" ["ebay_product"]="gd_ltr9mjt81n0zzdk1fb" ["homedepot_products"]="gd_lmusivh019i7g97q2n" ["zara_products"]="gd_lct4vafw1tgx27d4o0" ["etsy_products"]="gd_ltppk0jdv1jqz25mz" ["bestbuy_products"]="gd_ltre1jqe1jfr7cccf" ["linkedin_person_profile"]="gd_l1viktl72bvl7bjuj0" ["linkedin_company_profile"]="gd_l1vikfnt1wgvvqz95w" ["linkedin_job_listings"]="gd_lpfll7v5hcqtkxl6l" ["linkedin_posts"]="gd_lyy3tktm25m4avu764" ["linkedin_people_search"]="gd_m8d03he47z8nwb5xc" ["crunchbase_company"]="gd_l1vijqt9jfj7olije" ["zoominfo_company_profile"]="gd_m0ci4a4ivx3j5l6nx" ["instagram_profiles"]="gd_l1vikfch901nx3by4" ["instagram_posts"]="gd_lk5ns7kz21pck8jpis" ["instagram_reels"]="gd_lyclm20il4r5helnj" ["instagram_comments"]="gd_ltppn085pokosxh13" ["facebook_posts"]="gd_lyclm1571iy3mv57zw" ["facebook_marketplace_listings"]="gd_lvt9iwuh6fbcwmx1a" ["facebook_company_reviews"]="gd_m0dtqpiu1mbcyc2g86" ["facebook_events"]="gd_m14sd0to1jz48ppm51" ["tiktok_profiles"]="gd_l1villgoiiidt09ci" ["tiktok_posts"]="gd_lu702nij2f790tmv9h" ["tiktok_shop"]="gd_m45m1u911dsa4274pi" ["tiktok_comments"]="gd_lkf2st302ap89utw5k" ["x_posts"]="gd_lwxkxvnf1cynvib9co" ["youtube_profiles"]="gd_lk538t2k2p1k3oos71" ["youtube_videos"]="gd_lk56epmy2i5g7lzu0k" ["youtube_comments"]="gd_lk9q0ew71spt1mxywf" ["reddit_posts"]="gd_lvz8ah06191smkebj4" ["google_maps_reviews"]="gd_luzfs1dn2oa0teb81" ["google_shopping"]="gd_ltppk50q18kdw67omz" ["google_play_store"]="gd_lsk382l8xei8vzm4u" ["apple_app_store"]="gd_lsk9ki3u2iishmwrui" ["reuter_news"]="gd_lyptx9h74wtlvpnfu" ["github_repository_file"]="gd_lyrexgxc24b3d4imjt" ["yahoo_finance_business"]="gd_lmrpz3vxmz972ghd7" ["zillow_properties_listing"]="gd_lfqkr8wm13ixtbd8f5" ["booking_hotel_listings"]="gd_m5mbdl081229ln6t4a" ) if [ -z "$DATASET_TYPE" ]; then echo "Usage: $0 [additional_params...]" >&2 echo "" >&2 echo "Available datasets:" >&2 echo "" >&2 echo "E-COMMERCE:" >&2 echo " amazon_product - Amazon product data" >&2 echo " amazon_product_reviews - Amazon product reviews" >&2 echo " amazon_product_search - Amazon search" >&2 echo " walmart_product - Walmart product data" >&2 echo " walmart_seller - Walmart seller data" >&2 echo " ebay_product - eBay product data" >&2 echo " homedepot_products - Home Depot product data" >&2 echo " zara_products - Zara product data" >&2 echo " etsy_products - Etsy product data" >&2 echo " bestbuy_products - Best Buy product data" >&2 echo "" >&2 echo "PROFESSIONAL NETWORKS:" >&2 echo " linkedin_person_profile - LinkedIn person profile" >&2 echo " linkedin_company_profile - LinkedIn company profile" >&2 echo " linkedin_job_listings - LinkedIn job listings" >&2 echo " linkedin_posts - LinkedIn posts" >&2 echo " linkedin_people_search - LinkedIn people search" >&2 echo " crunchbase_company - Crunchbase company data" >&2 echo " zoominfo_company_profile - ZoomInfo company profile" >&2 echo "" >&2 echo "SOCIAL MEDIA - INSTAGRAM:" >&2 echo " instagram_profiles - Instagram profile data" >&2 echo " instagram_posts - Instagram post data" >&2 echo " instagram_reels - Instagram reel data" >&2 echo " instagram_comments - Instagram comments" >&2 echo "" >&2 echo "SOCIAL MEDIA - FACEBOOK:" >&2 echo " facebook_posts - Facebook post data" >&2 echo " facebook_marketplace_listings - Facebook marketplace" >&2 echo " facebook_company_reviews - Company reviews" >&2 echo " facebook_events - Facebook events" >&2 echo "" >&2 echo "SOCIAL MEDIA - TIKTOK:" >&2 echo " tiktok_profiles - TikTok profile data" >&2 echo " tiktok_posts - TikTok post data" >&2 echo " tiktok_shop - TikTok shop product" >&2 echo " tiktok_comments - TikTok video comments" >&2 echo "" >&2 echo "SOCIAL MEDIA - OTHER:" >&2 echo " x_posts - X (Twitter) post data" >&2 echo " youtube_profiles - YouTube channel data" >&2 echo " youtube_videos - YouTube video data" >&2 echo " youtube_comments [num] - YouTube comments (default: 10)" >&2 echo " reddit_posts - Reddit post data" >&2 echo "" >&2 echo "GOOGLE SERVICES:" >&2 echo " google_maps_reviews [days] - Google Maps reviews (default: 3 days)" >&2 echo " google_shopping - Google Shopping product" >&2 echo " google_play_store - Google Play Store app" >&2 echo "" >&2 echo "OTHER:" >&2 echo " apple_app_store - Apple App Store app" >&2 echo " reuter_news - Reuters news article" >&2 echo " github_repository_file - GitHub repository file" >&2 echo " yahoo_finance_business - Yahoo Finance business" >&2 echo " zillow_properties_listing - Zillow property listing" >&2 echo " booking_hotel_listings - Booking.com hotel" >&2 exit 1 fi # Get dataset ID DATASET_ID="${DATASETS[$DATASET_TYPE]}" if [ -z "$DATASET_ID" ]; then echo "Error: Unknown dataset type '$DATASET_TYPE'" >&2 echo "Run '$0' without arguments to see available datasets" >&2 exit 1 fi case "$DATASET_TYPE" in amazon_product_search) KEYWORD="$1" DOMAIN_URL="$2" if [ -z "$KEYWORD" ] || [ -z "$DOMAIN_URL" ]; then echo "Usage: $0 amazon_product_search " >&2 exit 1 fi INPUT_JSON=$(jq -n --arg keyword "$KEYWORD" --arg url "$DOMAIN_URL" \ '{keyword: $keyword, url: $url, pages_to_search: "1"}') ;; linkedin_people_search) URL="$1" FIRST_NAME="$2" LAST_NAME="$3" if [ -z "$URL" ] || [ -z "$FIRST_NAME" ] || [ -z "$LAST_NAME" ]; then echo "Usage: $0 linkedin_people_search " >&2 exit 1 fi INPUT_JSON=$(jq -n --arg url "$URL" --arg first "$FIRST_NAME" --arg last "$LAST_NAME" \ '{url: $url, first_name: $first, last_name: $last}') ;; facebook_company_reviews) URL="$1" NUM_REVIEWS="${2:-10}" if [ -z "$URL" ]; then echo "Usage: $0 facebook_company_reviews [num_reviews]" >&2 exit 1 fi INPUT_JSON=$(jq -n --arg url "$URL" --arg num "$NUM_REVIEWS" \ '{url: $url, num_of_reviews: $num}') ;; google_maps_reviews) URL="$1" DAYS_LIMIT="${2:-3}" if [ -z "$URL" ]; then echo "Usage: $0 google_maps_reviews [days_limit]" >&2 exit 1 fi INPUT_JSON=$(jq -n --arg url "$URL" --arg days "$DAYS_LIMIT" \ '{url: $url, days_limit: $days}') ;; youtube_comments) URL="$1" NUM_COMMENTS="${2:-10}" if [ -z "$URL" ]; then echo "Usage: $0 youtube_comments [num_comments]" >&2 exit 1 fi INPUT_JSON=$(jq -n --arg url "$URL" --arg num "$NUM_COMMENTS" \ '{url: $url, num_of_comments: $num}') ;; *) # Default: just URL input URL="$1" if [ -z "$URL" ]; then echo "Usage: $0 $DATASET_TYPE " >&2 exit 1 fi INPUT_JSON=$(jq -n --arg url "$URL" '{url: $url}') ;; esac # Call the fetch script exec "$SCRIPT_DIR/fetch.sh" "$DATASET_ID" "$INPUT_JSON"