Data Feeds

#!/bin/bash
# Bright Data - Dataset Wrapper
# Provides easy access to all supported datasets

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
DATASET_TYPE="$1"
shift

declare -A DATASETS=(
    ["amazon_product"]="gd_l7q7dkf244hwjntr0"
    ["amazon_product_reviews"]="gd_le8e811kzy4ggddlq"
    ["amazon_product_search"]="gd_lwdb4vjm1ehb499uxs"
    ["walmart_product"]="gd_l95fol7l1ru6rlo116"
    ["walmart_seller"]="gd_m7ke48w81ocyu4hhz0"
    ["ebay_product"]="gd_ltr9mjt81n0zzdk1fb"
    ["homedepot_products"]="gd_lmusivh019i7g97q2n"
    ["zara_products"]="gd_lct4vafw1tgx27d4o0"
    ["etsy_products"]="gd_ltppk0jdv1jqz25mz"
    ["bestbuy_products"]="gd_ltre1jqe1jfr7cccf"
    ["linkedin_person_profile"]="gd_l1viktl72bvl7bjuj0"
    ["linkedin_company_profile"]="gd_l1vikfnt1wgvvqz95w"
    ["linkedin_job_listings"]="gd_lpfll7v5hcqtkxl6l"
    ["linkedin_posts"]="gd_lyy3tktm25m4avu764"
    ["linkedin_people_search"]="gd_m8d03he47z8nwb5xc"
    ["crunchbase_company"]="gd_l1vijqt9jfj7olije"
    ["zoominfo_company_profile"]="gd_m0ci4a4ivx3j5l6nx"
    ["instagram_profiles"]="gd_l1vikfch901nx3by4"
    ["instagram_posts"]="gd_lk5ns7kz21pck8jpis"
    ["instagram_reels"]="gd_lyclm20il4r5helnj"
    ["instagram_comments"]="gd_ltppn085pokosxh13"
    ["facebook_posts"]="gd_lyclm1571iy3mv57zw"
    ["facebook_marketplace_listings"]="gd_lvt9iwuh6fbcwmx1a"
    ["facebook_company_reviews"]="gd_m0dtqpiu1mbcyc2g86"
    ["facebook_events"]="gd_m14sd0to1jz48ppm51"
    ["tiktok_profiles"]="gd_l1villgoiiidt09ci"
    ["tiktok_posts"]="gd_lu702nij2f790tmv9h"
    ["tiktok_shop"]="gd_m45m1u911dsa4274pi"
    ["tiktok_comments"]="gd_lkf2st302ap89utw5k"
    ["x_posts"]="gd_lwxkxvnf1cynvib9co"
    ["youtube_profiles"]="gd_lk538t2k2p1k3oos71"
    ["youtube_videos"]="gd_lk56epmy2i5g7lzu0k"
    ["youtube_comments"]="gd_lk9q0ew71spt1mxywf"
    ["reddit_posts"]="gd_lvz8ah06191smkebj4"
    ["google_maps_reviews"]="gd_luzfs1dn2oa0teb81"
    ["google_shopping"]="gd_ltppk50q18kdw67omz"
    ["google_play_store"]="gd_lsk382l8xei8vzm4u"
    ["apple_app_store"]="gd_lsk9ki3u2iishmwrui"
    ["reuter_news"]="gd_lyptx9h74wtlvpnfu"
    ["github_repository_file"]="gd_lyrexgxc24b3d4imjt"
    ["yahoo_finance_business"]="gd_lmrpz3vxmz972ghd7"
    ["zillow_properties_listing"]="gd_lfqkr8wm13ixtbd8f5"
    ["booking_hotel_listings"]="gd_m5mbdl081229ln6t4a"
)

if [ -z "$DATASET_TYPE" ]; then
    echo "Usage: $0 <dataset_type> <url> [additional_params...]" >&2
    echo "" >&2
    echo "Available datasets:" >&2
    echo "" >&2
    echo "E-COMMERCE:" >&2
    echo "  amazon_product <url>              - Amazon product data" >&2
    echo "  amazon_product_reviews <url>      - Amazon product reviews" >&2
    echo "  amazon_product_search <keyword> <domain_url> - Amazon search" >&2
    echo "  walmart_product <url>             - Walmart product data" >&2
    echo "  walmart_seller <url>              - Walmart seller data" >&2
    echo "  ebay_product <url>                - eBay product data" >&2
    echo "  homedepot_products <url>          - Home Depot product data" >&2
    echo "  zara_products <url>               - Zara product data" >&2
    echo "  etsy_products <url>               - Etsy product data" >&2
    echo "  bestbuy_products <url>            - Best Buy product data" >&2
    echo "" >&2
    echo "PROFESSIONAL NETWORKS:" >&2
    echo "  linkedin_person_profile <url>     - LinkedIn person profile" >&2
    echo "  linkedin_company_profile <url>    - LinkedIn company profile" >&2
    echo "  linkedin_job_listings <url>       - LinkedIn job listings" >&2
    echo "  linkedin_posts <url>              - LinkedIn posts" >&2
    echo "  linkedin_people_search <url> <first_name> <last_name> - LinkedIn people search" >&2
    echo "  crunchbase_company <url>          - Crunchbase company data" >&2
    echo "  zoominfo_company_profile <url>    - ZoomInfo company profile" >&2
    echo "" >&2
    echo "SOCIAL MEDIA - INSTAGRAM:" >&2
    echo "  instagram_profiles <url>          - Instagram profile data" >&2
    echo "  instagram_posts <url>             - Instagram post data" >&2
    echo "  instagram_reels <url>             - Instagram reel data" >&2
    echo "  instagram_comments <url>          - Instagram comments" >&2
    echo "" >&2
    echo "SOCIAL MEDIA - FACEBOOK:" >&2
    echo "  facebook_posts <url>              - Facebook post data" >&2
    echo "  facebook_marketplace_listings <url> - Facebook marketplace" >&2
    echo "  facebook_company_reviews <url> <num_reviews> - Company reviews" >&2
    echo "  facebook_events <url>             - Facebook events" >&2
    echo "" >&2
    echo "SOCIAL MEDIA - TIKTOK:" >&2
    echo "  tiktok_profiles <url>             - TikTok profile data" >&2
    echo "  tiktok_posts <url>                - TikTok post data" >&2
    echo "  tiktok_shop <url>                 - TikTok shop product" >&2
    echo "  tiktok_comments <url>             - TikTok video comments" >&2
    echo "" >&2
    echo "SOCIAL MEDIA - OTHER:" >&2
    echo "  x_posts <url>                     - X (Twitter) post data" >&2
    echo "  youtube_profiles <url>            - YouTube channel data" >&2
    echo "  youtube_videos <url>              - YouTube video data" >&2
    echo "  youtube_comments <url> [num]      - YouTube comments (default: 10)" >&2
    echo "  reddit_posts <url>                - Reddit post data" >&2
    echo "" >&2
    echo "GOOGLE SERVICES:" >&2
    echo "  google_maps_reviews <url> [days]  - Google Maps reviews (default: 3 days)" >&2
    echo "  google_shopping <url>             - Google Shopping product" >&2
    echo "  google_play_store <url>           - Google Play Store app" >&2
    echo "" >&2
    echo "OTHER:" >&2
    echo "  apple_app_store <url>             - Apple App Store app" >&2
    echo "  reuter_news <url>                 - Reuters news article" >&2
    echo "  github_repository_file <url>      - GitHub repository file" >&2
    echo "  yahoo_finance_business <url>      - Yahoo Finance business" >&2
    echo "  zillow_properties_listing <url>   - Zillow property listing" >&2
    echo "  booking_hotel_listings <url>      - Booking.com hotel" >&2
    exit 1
fi

# Get dataset ID
DATASET_ID="${DATASETS[$DATASET_TYPE]}"

if [ -z "$DATASET_ID" ]; then
    echo "Error: Unknown dataset type '$DATASET_TYPE'" >&2
    echo "Run '$0' without arguments to see available datasets" >&2
    exit 1
fi

case "$DATASET_TYPE" in
    amazon_product_search)
        KEYWORD="$1"
        DOMAIN_URL="$2"
        if [ -z "$KEYWORD" ] || [ -z "$DOMAIN_URL" ]; then
            echo "Usage: $0 amazon_product_search <keyword> <domain_url>" >&2
            exit 1
        fi
        INPUT_JSON=$(jq -n --arg keyword "$KEYWORD" --arg url "$DOMAIN_URL" \
            '{keyword: $keyword, url: $url, pages_to_search: "1"}')
        ;;
    linkedin_people_search)
        URL="$1"
        FIRST_NAME="$2"
        LAST_NAME="$3"
        if [ -z "$URL" ] || [ -z "$FIRST_NAME" ] || [ -z "$LAST_NAME" ]; then
            echo "Usage: $0 linkedin_people_search <url> <first_name> <last_name>" >&2
            exit 1
        fi
        INPUT_JSON=$(jq -n --arg url "$URL" --arg first "$FIRST_NAME" --arg last "$LAST_NAME" \
            '{url: $url, first_name: $first, last_name: $last}')
        ;;
    facebook_company_reviews)
        URL="$1"
        NUM_REVIEWS="${2:-10}"
        if [ -z "$URL" ]; then
            echo "Usage: $0 facebook_company_reviews <url> [num_reviews]" >&2
            exit 1
        fi
        INPUT_JSON=$(jq -n --arg url "$URL" --arg num "$NUM_REVIEWS" \
            '{url: $url, num_of_reviews: $num}')
        ;;
    google_maps_reviews)
        URL="$1"
        DAYS_LIMIT="${2:-3}"
        if [ -z "$URL" ]; then
            echo "Usage: $0 google_maps_reviews <url> [days_limit]" >&2
            exit 1
        fi
        INPUT_JSON=$(jq -n --arg url "$URL" --arg days "$DAYS_LIMIT" \
            '{url: $url, days_limit: $days}')
        ;;
    youtube_comments)
        URL="$1"
        NUM_COMMENTS="${2:-10}"
        if [ -z "$URL" ]; then
            echo "Usage: $0 youtube_comments <url> [num_comments]" >&2
            exit 1
        fi
        INPUT_JSON=$(jq -n --arg url "$URL" --arg num "$NUM_COMMENTS" \
            '{url: $url, num_of_comments: $num}')
        ;;
    *)
        # Default: just URL input
        URL="$1"
        if [ -z "$URL" ]; then
            echo "Usage: $0 $DATASET_TYPE <url>" >&2
            exit 1
        fi
        INPUT_JSON=$(jq -n --arg url "$URL" '{url: $url}')
        ;;
esac

# Call the fetch script
exec "$SCRIPT_DIR/fetch.sh" "$DATASET_ID" "$INPUT_JSON"

#!/bin/bash
# Bright Data - Dataset Fetch with Polling
# Triggers a dataset collection and polls until results are ready

DATASET_ID="$1"
shift
INPUT_JSON="$1"

if [ -z "$DATASET_ID" ] || [ -z "$INPUT_JSON" ]; then
    echo "Usage: $0 <dataset_id> '<json_input>'" >&2
    echo "Example: $0 gd_l1viktl72bvl7bjuj0 '{\"url\":\"https://linkedin.com/in/someone\"}'" >&2
    exit 1
fi

if [ -z "${BRIGHTDATA_API_KEY:-}" ]; then
    echo "Error: BRIGHTDATA_API_KEY is not set." >&2
    echo "Get a key from https://brightdata.com/cp" >&2
    exit 1
fi

POLLING_TIMEOUT="${BRIGHTDATA_POLLING_TIMEOUT:-600}"

TRIGGER_RESPONSE=$(curl -s -X POST "https://api.brightdata.com/datasets/v3/trigger?dataset_id=${DATASET_ID}&include_errors=true" \
    -H "Authorization: Bearer $BRIGHTDATA_API_KEY" \
    -H "Content-Type: application/json" \
    -d "[$INPUT_JSON]")

SNAPSHOT_ID=$(echo "$TRIGGER_RESPONSE" | jq -r '.snapshot_id // empty')

if [ -z "$SNAPSHOT_ID" ]; then
    echo "Error: Failed to trigger dataset collection" >&2
    echo "$TRIGGER_RESPONSE" >&2
    exit 1
fi

echo "Triggered collection with snapshot ID: $SNAPSHOT_ID" >&2

ATTEMPTS=0
while [ $ATTEMPTS -lt $POLLING_TIMEOUT ]; do
    SNAPSHOT_RESPONSE=$(curl -s -X GET "https://api.brightdata.com/datasets/v3/snapshot/${SNAPSHOT_ID}?format=json" \
        -H "Authorization: Bearer $BRIGHTDATA_API_KEY")

    STATUS=$(echo "$SNAPSHOT_RESPONSE" | jq -r '.status // empty')

    if [ "$STATUS" = "running" ] || [ "$STATUS" = "building" ] || [ "$STATUS" = "starting" ]; then
        echo "Status: $STATUS - polling again (attempt $((ATTEMPTS + 1))/$POLLING_TIMEOUT)" >&2
        ATTEMPTS=$((ATTEMPTS + 1))
        sleep 1
        continue
    fi

    echo "Data received after $((ATTEMPTS + 1)) attempts" >&2
    echo "$SNAPSHOT_RESPONSE" | jq '.'
    exit 0
done

echo "Error: Timeout after $POLLING_TIMEOUT seconds waiting for data" >&2
exit 1