Yandex Search Api

# Config with secrets
config/config.json
config/service_account_key.json

# Cache files (generated at runtime)
cache/iam_token.json
cache/ops/
cache/results/

# macOS
.DS_Store

{
  "yandex_cloud_folder_id": "b1g...",
  "auth": {
    "service_account_key_file": "config/service_account_key.json",
    "openssl_bin": "openssl"
  },
  "search": {
    "mode": "sync",
    "region_id": 225,
    "search_type": "SEARCH_TYPE_RU",
    "family_mode": "FAMILY_MODE_MODERATE",
    "fix_typo_mode": "FIX_TYPO_MODE_ON",
    "results_per_page": 10,
    "pages": 1
  },
  "async": {
    "poll_interval_minutes": 10,
    "max_wait_minutes": 120,
    "batch_size": 10
  }
}

#!/bin/sh
# Common functions for Yandex Search API
# Zero external dependencies: python3 stdlib + openssl + curl

set -e

SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
SKILL_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
CONFIG_FILE="$SKILL_DIR/config/config.json"
CACHE_DIR="$SKILL_DIR/cache"
SEARCH_API_URL="https://searchapi.api.cloud.yandex.net"
IAM_API_URL="https://iam.api.cloud.yandex.net/iam/v1/tokens"
OPERATION_API_URL="https://operation.api.cloud.yandex.net/operations"

# --- Prerequisites check ---

check_python3() {
    if ! command -v python3 >/dev/null 2>&1; then
        echo "Error: python3 is required but not found." >&2
        echo "Install Python 3.7+ and ensure python3 is in PATH." >&2
        exit 1
    fi
}

check_openssl() {
    _ossl_bin="${1:-openssl}"
    if ! command -v "$_ossl_bin" >/dev/null 2>&1; then
        echo "Error: openssl not found at '$_ossl_bin'." >&2
        echo "Install OpenSSL 1.1.1+ or set auth.openssl_bin in config.json." >&2
        exit 1
    fi
    # Check version (need 1.1.1+ for PSS support)
    _ossl_ver="$("$_ossl_bin" version 2>/dev/null || true)"
    case "$_ossl_ver" in
        LibreSSL*)
            echo "Error: LibreSSL detected ($_ossl_ver). OpenSSL 1.1.1+ required for PS256." >&2
            echo "Install OpenSSL via: brew install openssl@3" >&2
            exit 1
            ;;
        "OpenSSL 0."*|"OpenSSL 1.0."*)
            echo "Error: OpenSSL version too old ($_ossl_ver). Need 1.1.1+." >&2
            exit 1
            ;;
    esac
}

check_curl() {
    if ! command -v curl >/dev/null 2>&1; then
        echo "Error: curl is required but not found." >&2
        exit 1
    fi
}

# Run all prerequisite checks
check_prerequisites() {
    check_python3
    check_curl
}

# --- Config loading (JSON via python3) ---

load_config() {
    if [ ! -f "$CONFIG_FILE" ]; then
        echo "Error: config.json not found at $CONFIG_FILE" >&2
        echo "Copy config.example.json to config.json and fill in your values." >&2
        echo "See config/README.md for instructions." >&2
        exit 1
    fi
}

# Read a value from config.json
# Usage: cfg_get "yandex_cloud_folder_id"
# Usage: cfg_get "auth.service_account_key_file"
# Usage: cfg_get "search.region_id" "225"  (with default)
cfg_get() {
    _key="$1"
    _default="${2:-}"
    _val=$(python3 -c "
import json, sys
with open('$CONFIG_FILE') as f:
    cfg = json.load(f)
keys = '$_key'.split('.')
v = cfg
for k in keys:
    if isinstance(v, dict) and k in v:
        v = v[k]
    else:
        v = None
        break
if v is None:
    d = '$_default'
    print(d if d else '')
else:
    print(v)
" 2>/dev/null)
    echo "$_val"
}

# --- Temp file management ---

# Create secure temp directory
# Usage: _tmpdir=$(make_secure_tmpdir)
make_secure_tmpdir() {
    _old_umask=$(umask)
    umask 077
    _td=$(mktemp -d "${TMPDIR:-/tmp}/ysa_XXXXXX")
    umask "$_old_umask"
    echo "$_td"
}

# --- JSON helpers (via python3) ---

# Extract field from JSON file
# Usage: json_file_get "file.json" "field.nested"
json_file_get() {
    _file="$1"
    _key="$2"
    python3 -c "
import json, sys
with open('$_file') as f:
    d = json.load(f)
keys = '$_key'.split('.')
v = d
for k in keys:
    if isinstance(v, dict) and k in v:
        v = v[k]
    else:
        print('')
        sys.exit(0)
print(v if v is not None else '')
" 2>/dev/null
}

# Extract field from JSON string on stdin
# Usage: echo '{"a":1}' | json_stdin_get "a"
json_stdin_get() {
    _key="$1"
    python3 -c "
import json, sys
d = json.load(sys.stdin)
keys = '$_key'.split('.')
v = d
for k in keys:
    if isinstance(v, dict) and k in v:
        v = v[k]
    else:
        print('')
        sys.exit(0)
print(v if v is not None else '')
"
}

# --- Base64 helpers (python3 stdlib, cross-platform) ---

# Base64 decode from stdin to stdout (binary)
b64_decode() {
    python3 -c "
import base64, sys
data = sys.stdin.read()
sys.stdout.buffer.write(base64.b64decode(data))
"
}

# Base64url encode from stdin to stdout (no padding)
b64url_encode() {
    python3 -c "
import base64, sys
data = sys.stdin.buffer.read()
print(base64.urlsafe_b64encode(data).rstrip(b'=').decode())
"
}

# --- HTTP helpers with retry ---

# HTTP request with retry (3 attempts, exponential backoff)
# Usage: http_request "POST" "url" "body_file_or_empty" "header1" "header2" ...
# body is passed via temp file to avoid shell injection
# Writes response to stdout, returns 0 on success, 1 on error
http_request() {
    _method="$1"
    _url="$2"
    _body="$3"
    shift 3

    _max_retries=3
    _attempt=0
    _backoff=2

    # Save headers to a persistent temp file BEFORE retry loop
    # so that set -- inside the loop doesn't lose them
    _hr_tmpdir=$(make_secure_tmpdir)
    _hr_headers_file="$_hr_tmpdir/headers_saved"
    : > "$_hr_headers_file"
    for _h in "$@"; do
        printf '%s\n' "$_h" >> "$_hr_headers_file"
    done

    while [ "$_attempt" -lt "$_max_retries" ]; do
        _attempt=$((_attempt + 1))

        _tmpdir_http=$(make_secure_tmpdir)
        _resp_file="$_tmpdir_http/response"
        _header_file="$_tmpdir_http/headers"
        _body_file="$_tmpdir_http/body"

        # Write body to temp file to avoid shell quoting issues
        if [ -n "$_body" ]; then
            printf '%s' "$_body" > "$_body_file"
        fi

        # Build curl args via set -- to avoid word splitting on spaces in headers
        set -- curl -s -w '%{http_code}' -o "$_resp_file" -D "$_header_file" -X "$_method"
        while IFS= read -r _hline; do
            set -- "$@" -H "$_hline"
        done < "$_hr_headers_file"
        if [ -n "$_body" ]; then
            set -- "$@" --data-binary "@$_body_file"
        fi
        set -- "$@" "$_url"

        _status=$("$@" 2>/dev/null) || _status="000"

        case "$_status" in
            2[0-9][0-9])
                cat "$_resp_file"
                rm -rf "$_tmpdir_http" "$_hr_tmpdir"
                return 0
                ;;
            401)
                cat "$_resp_file"
                rm -rf "$_tmpdir_http" "$_hr_tmpdir"
                return 1
                ;;
            403)
                echo "Error: 403 Forbidden. Check:" >&2
                echo "  - Role 'search-api.webSearch.user' assigned to SA" >&2
                echo "  - Correct folder_id in config.json" >&2
                cat "$_resp_file" >&2
                rm -rf "$_tmpdir_http" "$_hr_tmpdir"
                return 1
                ;;
            5[0-9][0-9]|000)
                if [ "$_attempt" -lt "$_max_retries" ]; then
                    echo "Request failed (status=$_status), retry in ${_backoff}s... ($_attempt/$_max_retries)" >&2
                    sleep "$_backoff"
                    _backoff=$((_backoff * 2))
                else
                    echo "Error: Request failed after $_max_retries attempts (last status=$_status)" >&2
                    cat "$_resp_file" >&2 2>/dev/null || true
                fi
                rm -rf "$_tmpdir_http"
                ;;
            *)
                echo "Error: HTTP $_status" >&2
                cat "$_resp_file" >&2 2>/dev/null || true
                rm -rf "$_tmpdir_http" "$_hr_tmpdir"
                return 1
                ;;
        esac
    done
    rm -rf "$_hr_tmpdir"
    return 1
}

# Authenticated request (adds IAM token and folder-id headers)
# Usage: auth_request "POST" "url" "body"
auth_request() {
    _ar_method="$1"
    _ar_url="$2"
    _ar_body="$3"

    _iam_token=$(get_cached_iam_token)
    if [ -z "$_iam_token" ]; then
        echo "Error: No valid IAM token. Run iam_token_get.sh first." >&2
        return 1
    fi

    _folder_id=$(cfg_get "yandex_cloud_folder_id")
    if [ -z "$_folder_id" ]; then
        echo "Error: yandex_cloud_folder_id not set in config.json" >&2
        return 1
    fi

    _result=$(http_request "$_ar_method" "$_ar_url" "$_ar_body" \
        "Authorization: Bearer $_iam_token" \
        "x-folder-id: $_folder_id" \
        "Content-Type: application/json") || {
        # On 401, auto-refresh token and retry once
        if echo "$_result" 2>/dev/null | python3 -c "import sys,json; d=json.load(sys.stdin); exit(0 if d.get('code')==16 else 1)" 2>/dev/null; then
            echo "Token expired, auto-refreshing..." >&2
            rm -f "$CACHE_DIR/iam_token.json"
            sh "$SCRIPT_DIR/iam_token_get.sh" >&2 || { echo "Error: Token refresh failed" >&2; return 1; }
            _iam_token=$(get_cached_iam_token)
            if [ -z "$_iam_token" ]; then
                echo "Error: Token refresh produced no token" >&2
                return 1
            fi
            # Retry with new token
            _result=$(http_request "$_ar_method" "$_ar_url" "$_ar_body" \
                "Authorization: Bearer $_iam_token" \
                "x-folder-id: $_folder_id" \
                "Content-Type: application/json") || return 1
        else
            return 1
        fi
    }
    echo "$_result"
}

# --- IAM Token cache ---

get_cached_iam_token() {
    _cache_file="$CACHE_DIR/iam_token.json"
    if [ ! -f "$_cache_file" ]; then
        return 0
    fi

    # Check expiry with 5-minute safety window
    _is_valid=$(python3 -c "
import json, time
with open('$_cache_file') as f:
    d = json.load(f)
exp = d.get('expires_at', 0)
now = time.time()
if exp - now > 300:
    print(d['iam_token'])
else:
    print('')
" 2>/dev/null)

    if [ -n "$_is_valid" ]; then
        echo "$_is_valid"
    else
        # Token expired, remove cache
        rm -f "$_cache_file"
    fi
}

save_iam_token() {
    _token="$1"
    _expires_at="$2"
    _cache_file="$CACHE_DIR/iam_token.json"

    # Atomic write with restricted permissions: tmp -> rename
    _old_umask=$(umask)
    umask 077
    _tmp_file="$CACHE_DIR/.iam_token_tmp_$$.json"
    # Pass token via environment to avoid shell injection
    _SAVE_TOKEN="$_token" python3 -c "
import json, os
d = {'iam_token': os.environ['_SAVE_TOKEN'], 'expires_at': $_expires_at}
with open('$_tmp_file', 'w') as f:
    json.dump(d, f)
"
    mv "$_tmp_file" "$_cache_file"
    umask "$_old_umask"
}

# --- XML parsing (python3 xml.etree.ElementTree) ---

# Parse Yandex Search API XML response to JSON
# Usage: parse_search_xml "input.xml" > "output.json"
parse_search_xml() {
    _xml_file="$1"
    python3 -c "
import xml.etree.ElementTree as ET
import json
import sys

def clean_hl(elem):
    parts = []
    if elem.text:
        parts.append(elem.text)
    for child in elem:
        if child.text:
            parts.append(child.text)
        if child.tail:
            parts.append(child.tail)
    return ''.join(parts).strip()

try:
    tree = ET.parse('$_xml_file')
    root = tree.getroot()

    results = []
    pos = 0

    for grouping in root.iter('grouping'):
        for group in grouping.iter('group'):
            for doc in group.iter('doc'):
                pos += 1
                url_elem = doc.find('url')
                title_elem = doc.find('title')
                snippet = ''
                for passages in doc.iter('passages'):
                    for passage in passages.iter('passage'):
                        snippet = clean_hl(passage)
                        if snippet:
                            break
                    if snippet:
                        break

                domain_elem = doc.find('domain')

                entry = {
                    'position': pos,
                    'url': url_elem.text if url_elem is not None else '',
                    'title': clean_hl(title_elem) if title_elem is not None else '',
                    'snippet': snippet[:300] if snippet else '',
                    'domain': domain_elem.text if domain_elem is not None else '',
                }
                results.append(entry)

    json.dump(results, sys.stdout, ensure_ascii=False, indent=2)
except ET.ParseError as e:
    print(json.dumps({'error': f'XML parse error: {e}', 'raw_saved': True}), file=sys.stdout)
    sys.exit(0)
except Exception as e:
    print(json.dumps({'error': str(e), 'raw_saved': True}), file=sys.stdout)
    sys.exit(0)
"
}

# --- Hash helper ---

file_hash() {
    _input="$1"
    echo "$_input" | python3 -c "
import hashlib, sys
print(hashlib.md5(sys.stdin.read().strip().encode()).hexdigest()[:12])
"
}

#!/bin/sh
# Get or refresh IAM token for Yandex Cloud Search API
# Uses JWT PS256 signed with Service Account key via openssl
# Zero external dependencies: python3 stdlib + openssl + curl

set -e

SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
. "$SCRIPT_DIR/common.sh"

check_prerequisites
load_config

# Check for cached valid token first
_cached=$(get_cached_iam_token)
if [ -n "$_cached" ]; then
    echo "IAM token is valid (cached)."
    exit 0
fi

echo "Generating new IAM token..." >&2

# Read SA key file path from config
SA_KEY_FILE=$(cfg_get "auth.service_account_key_file")
if [ -z "$SA_KEY_FILE" ]; then
    echo "Error: auth.service_account_key_file not set in config.json" >&2
    exit 1
fi

# Resolve relative path from skill dir
case "$SA_KEY_FILE" in
    /*) ;;
    *) SA_KEY_FILE="$SKILL_DIR/$SA_KEY_FILE" ;;
esac

if [ ! -f "$SA_KEY_FILE" ]; then
    echo "Error: Service account key file not found: $SA_KEY_FILE" >&2
    echo "Create it with: yc iam key create --service-account-name <name> --output <path>" >&2
    exit 1
fi

OPENSSL_BIN=$(cfg_get "auth.openssl_bin" "openssl")
check_openssl "$OPENSSL_BIN"

# Create secure temp directory for PEM and JWT files
SECURE_TMP=$(make_secure_tmpdir)
trap 'rm -rf "$SECURE_TMP"' EXIT INT TERM

# Extract SA credentials and create JWT via python3
python3 << PYEOF
import json, base64, time, os

sa_key_file = "$SA_KEY_FILE"
tmp_dir = "$SECURE_TMP"

with open(sa_key_file) as f:
    sa = json.load(f)

sa_id = sa['service_account_id']
key_id = sa['id']
private_key = sa['private_key']

# Write private key to temp PEM file
pem_path = os.path.join(tmp_dir, 'key.pem')
with open(pem_path, 'w') as f:
    f.write(private_key)

# Create JWT header
header = json.dumps({"typ": "JWT", "alg": "PS256", "kid": key_id}, separators=(',', ':'))
header_b64 = base64.urlsafe_b64encode(header.encode()).rstrip(b'=').decode()

# Create JWT payload
now = int(time.time())
payload = json.dumps({
    "iss": sa_id,
    "aud": "https://iam.api.cloud.yandex.net/iam/v1/tokens",
    "iat": now,
    "exp": now + 3600
}, separators=(',', ':'))
payload_b64 = base64.urlsafe_b64encode(payload.encode()).rstrip(b'=').decode()

# Write signing input
signing_input = f"{header_b64}.{payload_b64}"
signing_path = os.path.join(tmp_dir, 'signing_input.txt')
with open(signing_path, 'w') as f:
    f.write(signing_input)

# Write parts for shell to read
with open(os.path.join(tmp_dir, 'header_payload.txt'), 'w') as f:
    f.write(signing_input)
PYEOF

# Sign with openssl PS256
"$OPENSSL_BIN" dgst -sha256 \
    -sigopt rsa_padding_mode:pss \
    -sigopt rsa_pss_saltlen:-1 \
    -sign "$SECURE_TMP/key.pem" \
    -out "$SECURE_TMP/signature.bin" \
    "$SECURE_TMP/signing_input.txt"

# Base64url encode signature
SIGNATURE=$(cat "$SECURE_TMP/signature.bin" | b64url_encode)

# Assemble JWT
HEADER_PAYLOAD=$(cat "$SECURE_TMP/header_payload.txt")
JWT="${HEADER_PAYLOAD}.${SIGNATURE}"

# Exchange JWT for IAM token
IAM_RESPONSE=$(http_request "POST" "$IAM_API_URL" \
    "{\"jwt\":\"$JWT\"}" \
    "Content-Type: application/json")

if [ -z "$IAM_RESPONSE" ]; then
    echo "Error: Empty response from IAM API" >&2
    exit 1
fi

# Extract token and expiry
IAM_RESULT=$(echo "$IAM_RESPONSE" | python3 -c "
import json, sys
from datetime import datetime

d = json.load(sys.stdin)
token = d.get('iamToken', '')
expires_at_str = d.get('expiresAt', '')

if not token:
    print('ERROR: No iamToken in response', file=sys.stderr)
    sys.exit(1)

# Parse RFC3339 expiresAt to unix timestamp
if expires_at_str:
    ts = datetime.fromisoformat(expires_at_str.replace('Z', '+00:00')).timestamp()
    expires_at = int(ts)
else:
    # Fallback: assume 12 hours from now
    import time
    expires_at = int(time.time()) + 43200

print(f'{token}|{expires_at}')
")

IAM_TOKEN=$(echo "$IAM_RESULT" | cut -d'|' -f1)
EXPIRES_AT=$(echo "$IAM_RESULT" | cut -d'|' -f2)

if [ -z "$IAM_TOKEN" ]; then
    echo "Error: Failed to extract IAM token" >&2
    echo "Response: $IAM_RESPONSE" >&2
    exit 1
fi

# Save to cache (atomic)
save_iam_token "$IAM_TOKEN" "$EXPIRES_AT"

echo "IAM token generated and cached successfully."
echo "Expires at: $(python3 -c "from datetime import datetime; print(datetime.fromtimestamp($EXPIRES_AT).isoformat())")"

#!/bin/sh
# Show common Yandex Search API region IDs

echo "=== Common Region IDs (Yandex Search API) ==="
echo ""
echo "Countries:"
echo "  225   - Россия"
echo "  159   - Казахстан"
echo "  187   - Украина"
echo "  149   - Беларусь"
echo "  983   - Турция"
echo "  84    - США"
echo ""
echo "Federal Districts (Russia):"
echo "  3     - Центральный ФО"
echo "  17    - Северо-Западный ФО"
echo "  40    - Приволжский ФО"
echo "  52    - Уральский ФО"
echo "  59    - Сибирский ФО"
echo "  73    - Южный ФО"
echo "  26    - Дальневосточный ФО"
echo ""
echo "Major Cities:"
echo "  213   - Москва"
echo "  2     - Санкт-Петербург"
echo "  54    - Екатеринбург"
echo "  65    - Новосибирск"
echo "  43    - Казань"
echo "  35    - Краснодар"
echo "  47    - Нижний Новгород"
echo "  39    - Ростов-на-Дону"
echo "  51    - Самара"
echo "  172   - Уфа"
echo "  56    - Челябинск"
echo "  66    - Омск"
echo "  11    - Пермь"
echo "  14    - Воронеж"
echo ""
echo "Moscow Region:"
echo "  1     - Москва и область"
echo "  213   - Москва (город)"
echo "  10716 - Московская область"
echo ""
echo "Use these IDs with --region-id parameter in search scripts."
echo "Example: bash scripts/web_search_sync.sh --query \"test\" --region-id 213"

#!/bin/sh
# Search for region by name

SEARCH=""

while [ $# -gt 0 ]; do
    case $1 in
        --name|-n) SEARCH="$2"; shift 2 ;;
        *) echo "Unknown option: $1"; exit 1 ;;
    esac
done

if [ -z "$SEARCH" ]; then
    echo "Usage: search_region.sh --name \"city name\""
    echo ""
    echo "Examples:"
    echo "  bash scripts/search_region.sh --name \"Москва\""
    echo "  bash scripts/search_region.sh --name \"Казань\""
    exit 1
fi

echo "Searching for: $SEARCH"
echo ""

# Hardcoded common regions
REGIONS="
225|Россия
159|Казахстан
187|Украина
149|Беларусь
983|Турция
84|США
3|Центральный ФО
17|Северо-Западный ФО
40|Приволжский ФО
52|Уральский ФО
59|Сибирский ФО
73|Южный ФО
26|Дальневосточный ФО
1|Москва и область
213|Москва
10716|Московская область
2|Санкт-Петербург
10174|Ленинградская область
54|Екатеринбург
65|Новосибирск
43|Казань
35|Краснодар
47|Нижний Новгород
39|Ростов-на-Дону
51|Самара
172|Уфа
56|Челябинск
66|Омск
11|Пермь
14|Воронеж
38|Волгоград
37|Саратов
195|Тюмень
62|Красноярск
68|Томск
67|Барнаул
"

# Search (case-insensitive via python3 for portability)
python3 -c "
import sys
search = '$SEARCH'.lower()
regions = '''$REGIONS'''.strip().split('\n')
found = []
for line in regions:
    line = line.strip()
    if not line:
        continue
    if search in line.lower():
        parts = line.split('|', 1)
        if len(parts) == 2:
            found.append((parts[0].strip(), parts[1].strip()))

if not found:
    print('No regions found matching \"$SEARCH\"')
    print('')
    print('Try running regions_tree.sh to see all common regions')
else:
    print('Found:')
    print('')
    print('| ID | Name |')
    print('|----|------|')
    for rid, name in found:
        print(f'| {rid} | {name} |')
"

#!/bin/sh
# Asynchronous web search via Yandex Cloud Search API v2
# Creates batch of operations, polls for completion, resumes on restart
#
# Usage:
#   web_search_async.sh --file queries.txt [--region-id 225]
#   web_search_async.sh --resume  (continue polling pending operations)

set -e

SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
. "$SCRIPT_DIR/common.sh"

check_prerequisites
load_config

# Defaults from config
REGION_ID=$(cfg_get "search.region_id" "225")
SEARCH_TYPE=$(cfg_get "search.search_type" "SEARCH_TYPE_RU")
FAMILY_MODE=$(cfg_get "search.family_mode" "FAMILY_MODE_MODERATE")
FIX_TYPO=$(cfg_get "search.fix_typo_mode" "FIX_TYPO_MODE_ON")
RESULTS_PER_PAGE=$(cfg_get "search.results_per_page" "10")
POLL_INTERVAL=$(cfg_get "async.poll_interval_minutes" "10")
MAX_WAIT=$(cfg_get "async.max_wait_minutes" "120")
BATCH_SIZE=$(cfg_get "async.batch_size" "10")

QUERIES_FILE=""
RESUME_MODE=0

while [ $# -gt 0 ]; do
    case $1 in
        --file|-f) QUERIES_FILE="$2"; shift 2 ;;
        --region-id|-r) REGION_ID="$2"; shift 2 ;;
        --results|-n) RESULTS_PER_PAGE="$2"; shift 2 ;;
        --search-type) SEARCH_TYPE="$2"; shift 2 ;;
        --poll-interval) POLL_INTERVAL="$2"; shift 2 ;;
        --max-wait) MAX_WAIT="$2"; shift 2 ;;
        --resume) RESUME_MODE=1; shift ;;
        *) echo "Unknown option: $1"; exit 1 ;;
    esac
done

# Create directories
mkdir -p "$CACHE_DIR/ops"
mkdir -p "$CACHE_DIR/results"

# --- Helper: count pending operations ---
count_pending() {
    _pending=0
    for _op_file in "$CACHE_DIR/ops"/*.json; do
        [ -f "$_op_file" ] || continue
        _st=$(json_file_get "$_op_file" "status")
        if [ "$_st" = "pending" ] || [ "$_st" = "running" ]; then
            _pending=$((_pending + 1))
        fi
    done
    echo "$_pending"
}

# --- Helper: count completed operations ---
count_done() {
    _done=0
    for _op_file in "$CACHE_DIR/ops"/*.json; do
        [ -f "$_op_file" ] || continue
        _st=$(json_file_get "$_op_file" "status")
        if [ "$_st" = "done" ]; then
            _done=$((_done + 1))
        fi
    done
    echo "$_done"
}

# --- Helper: count total operations ---
count_total() {
    _total=0
    for _op_file in "$CACHE_DIR/ops"/*.json; do
        [ -f "$_op_file" ] || continue
        _total=$((_total + 1))
    done
    echo "$_total"
}

# --- Helper: submit a single async query ---
submit_query() {
    _aq_query="$1"
    _aq_hash=$(file_hash "$_aq_query")

    # Query passed via env to avoid shell injection
    _body=$(_YSA_QUERY="$_aq_query" python3 -c "
import json, os
body = {
    'query': {
        'searchType': '$SEARCH_TYPE',
        'queryText': os.environ['_YSA_QUERY'],
        'familyMode': '$FAMILY_MODE',
        'fixTypoMode': '$FIX_TYPO'
    },
    'sortSpec': {},
    'groupSpec': {
        'groupMode': 'GROUP_MODE_FLAT',
        'groupsOnPage': $RESULTS_PER_PAGE,
        'docsInGroup': 1
    },
    'maxPassages': 3,
    'region': '$REGION_ID',
    'l10n': 'LOCALIZATION_RU',
    'folderId': '$(cfg_get "yandex_cloud_folder_id")'
}
print(json.dumps(body, ensure_ascii=False))
")

    _response=$(auth_request "POST" "$SEARCH_API_URL/v2/web/searchAsync" "$_body") || {
        echo "Error: Failed to submit async query: $_aq_query" >&2
        return 1
    }

    _op_id=$(echo "$_response" | json_stdin_get "id")
    if [ -z "$_op_id" ]; then
        echo "Error: No operation ID in response for: $_aq_query" >&2
        return 1
    fi

    # Save operation state (query via env to avoid injection)
    _now=$(python3 -c "import time; print(int(time.time()))")
    _YSA_QUERY="$_aq_query" python3 -c "
import json, os
op = {
    'operation_id': '$_op_id',
    'query': os.environ['_YSA_QUERY'],
    'region_id': '$REGION_ID',
    'created_at': $_now,
    'last_checked_at': $_now,
    'status': 'pending',
    'error': None,
    'result_hash': '$_aq_hash'
}
with open('$CACHE_DIR/ops/${_op_id}.json', 'w') as f:
    json.dump(op, f, indent=2, ensure_ascii=False)
"

    echo "  Submitted: $_aq_query -> op=$_op_id" >&2
}

# --- Helper: check and process a single operation ---
check_operation() {
    _co_file="$1"
    _co_op_id=$(json_file_get "$_co_file" "operation_id")

    _poll_resp=$(auth_request "GET" "$OPERATION_API_URL/$_co_op_id" "") || {
        echo "  Warning: Failed to poll operation $_co_op_id" >&2
        return 1
    }

    _co_done=$(echo "$_poll_resp" | json_stdin_get "done")
    _now=$(python3 -c "import time; print(int(time.time()))")

    if [ "$_co_done" = "True" ] || [ "$_co_done" = "true" ]; then
        # Extract rawData from response
        _raw_b64=$(echo "$_poll_resp" | python3 -c "
import json, sys
d = json.load(sys.stdin)
resp = d.get('response', {})
print(resp.get('rawData', ''))
")
        _result_hash=$(json_file_get "$_co_file" "result_hash")
        _query=$(json_file_get "$_co_file" "query")

        if [ -n "$_raw_b64" ]; then
            # Decode and parse
            echo "$_raw_b64" | b64_decode > "$CACHE_DIR/results/${_result_hash}.raw"
            parse_search_xml "$CACHE_DIR/results/${_result_hash}.raw" > "$CACHE_DIR/results/${_result_hash}.json"

            # Update operation status
            python3 -c "
import json
with open('$_co_file') as f:
    op = json.load(f)
op['status'] = 'done'
op['last_checked_at'] = $_now
with open('$_co_file', 'w') as f:
    json.dump(op, f, indent=2, ensure_ascii=False)
"
            echo "  Done: $_query -> results/${_result_hash}.json" >&2
        else
            # Check for error in operation
            _error=$(echo "$_poll_resp" | python3 -c "
import json, sys
d = json.load(sys.stdin)
err = d.get('error', {})
print(err.get('message', 'Unknown error'))
" 2>/dev/null || echo "Unknown error")

            python3 -c "
import json
with open('$_co_file') as f:
    op = json.load(f)
op['status'] = 'error'
op['last_checked_at'] = $_now
op['error'] = '$_error'
with open('$_co_file', 'w') as f:
    json.dump(op, f, indent=2, ensure_ascii=False)
"
            echo "  Error: $_query -> $_error" >&2
        fi
    else
        # Still running, update last_checked_at
        python3 -c "
import json
with open('$_co_file') as f:
    op = json.load(f)
op['status'] = 'running'
op['last_checked_at'] = $_now
with open('$_co_file', 'w') as f:
    json.dump(op, f, indent=2, ensure_ascii=False)
"
    fi
}

# --- Main logic ---

# Check for pending operations (resume mode)
_existing_pending=$(count_pending)

if [ "$RESUME_MODE" -eq 1 ]; then
    if [ "$_existing_pending" -eq 0 ]; then
        echo "No pending operations to resume."
        echo "Done: $(count_done)/$(count_total)"
        exit 0
    fi
    echo "Resuming: $_existing_pending pending operations found."
elif [ -n "$QUERIES_FILE" ]; then
    # Check for existing pending (don't create duplicates)
    if [ "$_existing_pending" -gt 0 ]; then
        echo "WARNING: $_existing_pending pending operations already exist in cache/ops/" >&2
        echo "Use --resume to continue polling them, or delete cache/ops/ to start fresh." >&2
        echo "" >&2
        echo "Existing queries:" >&2
        for _ef in "$CACHE_DIR/ops"/*.json; do
            [ -f "$_ef" ] || continue
            _est=$(json_file_get "$_ef" "status")
            _eq=$(json_file_get "$_ef" "query")
            if [ "$_est" = "pending" ] || [ "$_est" = "running" ]; then
                echo "  [$_est] $_eq" >&2
            fi
        done
        exit 1
    fi

    if [ ! -f "$QUERIES_FILE" ]; then
        echo "Error: Queries file not found: $QUERIES_FILE" >&2
        exit 1
    fi

    # Ensure IAM token
    _token=$(get_cached_iam_token)
    if [ -z "$_token" ]; then
        echo "No valid IAM token. Generating..." >&2
        sh "$SCRIPT_DIR/iam_token_get.sh"
    fi

    echo "=== Async Search: Submitting queries ==="
    echo "NOTE: This may take minutes to hours. The script will poll every $POLL_INTERVAL minutes."
    echo "Max wait: $MAX_WAIT minutes. Use --resume to continue if interrupted."
    echo ""

    # Submit queries in batches
    _submitted=0
    while IFS= read -r _line || [ -n "$_line" ]; do
        _line=$(echo "$_line" | sed 's/^[[:space:]]*//' | sed 's/[[:space:]]*$//')
        if [ -n "$_line" ]; then
            submit_query "$_line" || true
            _submitted=$((_submitted + 1))

            # Batch pause to respect rate limits
            if [ $((_submitted % BATCH_SIZE)) -eq 0 ]; then
                echo "  Batch of $BATCH_SIZE submitted, pausing 2s..." >&2
                sleep 2
            fi
        fi
    done < "$QUERIES_FILE"

    echo ""
    echo "Submitted $_submitted queries. Starting polling..."
    echo ""
else
    echo "Usage: web_search_async.sh --file queries.txt [options]"
    echo "       web_search_async.sh --resume"
    echo ""
    echo "Options:"
    echo "  --file, -f         File with queries (one per line)"
    echo "  --region-id, -r    Region ID (default: $REGION_ID)"
    echo "  --results, -n      Results per page (default: $RESULTS_PER_PAGE)"
    echo "  --poll-interval    Poll interval in minutes (default: $POLL_INTERVAL)"
    echo "  --max-wait         Max wait in minutes (default: $MAX_WAIT)"
    echo "  --resume           Resume polling pending operations"
    exit 1
fi

# --- Polling loop ---

_poll_seconds=$((POLL_INTERVAL * 60))
_max_seconds=$((MAX_WAIT * 60))
_start_time=$(python3 -c "import time; print(int(time.time()))")
_poll_count=0

while true; do
    _pending=$(count_pending)
    _done_count=$(count_done)
    _total_count=$(count_total)

    if [ "$_pending" -eq 0 ]; then
        echo ""
        echo "=== All operations complete ==="
        echo "Done: $_done_count/$_total_count"
        echo ""

        # Print summary
        echo "Results:"
        for _rf in "$CACHE_DIR/ops"/*.json; do
            [ -f "$_rf" ] || continue
            _rq=$(json_file_get "$_rf" "query")
            _rs=$(json_file_get "$_rf" "status")
            _rh=$(json_file_get "$_rf" "result_hash")
            if [ "$_rs" = "done" ]; then
                echo "  OK: $_rq -> cache/results/${_rh}.json"
            else
                _re=$(json_file_get "$_rf" "error")
                echo "  FAIL: $_rq -> $_re"
            fi
        done
        exit 0
    fi

    # Check timeout
    _now=$(python3 -c "import time; print(int(time.time()))")
    _elapsed=$((_now - _start_time))
    if [ "$_elapsed" -ge "$_max_seconds" ]; then
        echo ""
        echo "=== Timeout: max_wait_minutes ($MAX_WAIT) exceeded ==="
        echo "Done: $_done_count/$_total_count | Pending: $_pending"
        echo ""
        echo "Pending queries:"
        for _pf in "$CACHE_DIR/ops"/*.json; do
            [ -f "$_pf" ] || continue
            _ps=$(json_file_get "$_pf" "status")
            _pq=$(json_file_get "$_pf" "query")
            if [ "$_ps" = "pending" ] || [ "$_ps" = "running" ]; then
                echo "  $_pq"
            fi
        done
        echo ""
        echo "To continue polling, run:"
        echo "  bash scripts/web_search_async.sh --resume"
        exit 0
    fi

    _poll_count=$((_poll_count + 1))
    _remaining_min=$(( (_max_seconds - _elapsed) / 60 ))
    echo "--- Poll #$_poll_count | Done: $_done_count/$_total_count | Pending: $_pending | Timeout in: ${_remaining_min}min ---"

    # Check each pending operation
    for _cf in "$CACHE_DIR/ops"/*.json; do
        [ -f "$_cf" ] || continue
        _cs=$(json_file_get "$_cf" "status")
        if [ "$_cs" = "pending" ] || [ "$_cs" = "running" ]; then
            check_operation "$_cf" || true
        fi
    done

    # Check if all done after this poll
    _pending_after=$(count_pending)
    if [ "$_pending_after" -eq 0 ]; then
        continue
    fi

    echo "Sleeping ${POLL_INTERVAL} minutes until next poll..." >&2
    sleep "$_poll_seconds"
done

#!/bin/sh
# Synchronous web search via Yandex Cloud Search API v2
# Usage: web_search_sync.sh --query "search text" [--region-id 225] [--results 10] [--page 0]

set -e

SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
. "$SCRIPT_DIR/common.sh"

check_prerequisites
load_config

# Defaults from config
QUERY=""
REGION_ID=$(cfg_get "search.region_id" "225")
SEARCH_TYPE=$(cfg_get "search.search_type" "SEARCH_TYPE_RU")
FAMILY_MODE=$(cfg_get "search.family_mode" "FAMILY_MODE_MODERATE")
FIX_TYPO=$(cfg_get "search.fix_typo_mode" "FIX_TYPO_MODE_ON")
RESULTS_PER_PAGE=$(cfg_get "search.results_per_page" "10")
PAGE=0
QUERIES_FILE=""

while [ $# -gt 0 ]; do
    case $1 in
        --query|-q) QUERY="$2"; shift 2 ;;
        --region-id|-r) REGION_ID="$2"; shift 2 ;;
        --results|-n) RESULTS_PER_PAGE="$2"; shift 2 ;;
        --page|-p) PAGE="$2"; shift 2 ;;
        --search-type) SEARCH_TYPE="$2"; shift 2 ;;
        --family-mode) FAMILY_MODE="$2"; shift 2 ;;
        --file|-f) QUERIES_FILE="$2"; shift 2 ;;
        *) echo "Unknown option: $1"; exit 1 ;;
    esac
done

if [ -z "$QUERY" ] && [ -z "$QUERIES_FILE" ]; then
    echo "Usage: web_search_sync.sh --query \"search text\" [options]"
    echo "       web_search_sync.sh --file queries.txt [options]"
    echo ""
    echo "Options:"
    echo "  --query, -q        Search query text"
    echo "  --file, -f         File with queries (one per line)"
    echo "  --region-id, -r    Region ID (default: $REGION_ID)"
    echo "  --results, -n      Results per page 1-100 (default: $RESULTS_PER_PAGE)"
    echo "  --page, -p         Page number 0+ (default: 0)"
    echo "  --search-type      SEARCH_TYPE_RU|SEARCH_TYPE_TR|SEARCH_TYPE_COM|SEARCH_TYPE_KK|SEARCH_TYPE_BE|SEARCH_TYPE_UZ"
    echo "  --family-mode      FAMILY_MODE_NONE|FAMILY_MODE_MODERATE|FAMILY_MODE_STRICT"
    echo ""
    echo "Examples:"
    echo "  bash scripts/web_search_sync.sh --query \"купить дымоход\" --region-id 213"
    echo "  bash scripts/web_search_sync.sh --file queries.txt --region-id 225"
    exit 1
fi

# Ensure IAM token is available
_token=$(get_cached_iam_token)
if [ -z "$_token" ]; then
    echo "No valid IAM token. Generating..." >&2
    sh "$SCRIPT_DIR/iam_token_get.sh"
fi

# Create results directory
mkdir -p "$CACHE_DIR/results"

# Function to search a single query
search_single() {
    _sq_query="$1"
    _sq_hash=$(file_hash "$_sq_query")

    echo "--- Searching: $_sq_query (hash: $_sq_hash) ---" >&2

    # Build request body (query passed via env to avoid shell injection)
    _body=$(_YSA_QUERY="$_sq_query" python3 -c "
import json, os
body = {
    'query': {
        'searchType': '$SEARCH_TYPE',
        'queryText': os.environ['_YSA_QUERY'],
        'familyMode': '$FAMILY_MODE',
        'fixTypoMode': '$FIX_TYPO',
        'page': $PAGE
    },
    'sortSpec': {},
    'groupSpec': {
        'groupMode': 'GROUP_MODE_FLAT',
        'groupsOnPage': $RESULTS_PER_PAGE,
        'docsInGroup': 1
    },
    'maxPassages': 3,
    'region': '$REGION_ID',
    'l10n': 'LOCALIZATION_RU',
    'folderId': '$(cfg_get "yandex_cloud_folder_id")'
}
print(json.dumps(body, ensure_ascii=False))
")

    # Make API call
    _response=$(auth_request "POST" "$SEARCH_API_URL/v2/web/search" "$_body") || {
        echo "Error: Search API call failed for query: $_sq_query" >&2
        return 1
    }

    # Extract rawData and decode
    _raw_b64=$(echo "$_response" | python3 -c "
import json, sys
d = json.load(sys.stdin)
print(d.get('rawData', ''))
")

    if [ -z "$_raw_b64" ]; then
        echo "Error: No rawData in response for query: $_sq_query" >&2
        echo "Response saved to: $CACHE_DIR/results/${_sq_hash}_error.json" >&2
        echo "$_response" > "$CACHE_DIR/results/${_sq_hash}_error.json"
        return 1
    fi

    # Decode base64 -> raw XML/HTML
    echo "$_raw_b64" | b64_decode > "$CACHE_DIR/results/${_sq_hash}.raw"

    # Parse XML to JSON
    parse_search_xml "$CACHE_DIR/results/${_sq_hash}.raw" > "$CACHE_DIR/results/${_sq_hash}.json"

    # Print summary
    echo ""
    echo "=== Results for: $_sq_query ==="
    echo "Region: $REGION_ID | Page: $PAGE"
    echo ""

    python3 -c "
import json
with open('$CACHE_DIR/results/${_sq_hash}.json') as f:
    results = json.load(f)

if isinstance(results, dict) and 'error' in results:
    print(f'  Parse error: {results[\"error\"]}')
    print(f'  Raw data saved to: $CACHE_DIR/results/${_sq_hash}.raw')
else:
    shown = min(len(results), 10)
    for r in results[:shown]:
        pos = r.get('position', '?')
        title = r.get('title', 'No title')[:80]
        url = r.get('url', '')
        snippet = r.get('snippet', '')[:120]
        print(f'  {pos}. {title}')
        print(f'     {url}')
        if snippet:
            print(f'     {snippet}')
        print()

    total = len(results)
    if total > shown:
        print(f'  ... and {total - shown} more results')
    print(f'  Total: {total} results')

print()
print(f'Files:')
print(f'  Raw: $CACHE_DIR/results/${_sq_hash}.raw')
print(f'  JSON: $CACHE_DIR/results/${_sq_hash}.json')
"
}

# Execute search(es)
if [ -n "$QUERIES_FILE" ]; then
    if [ ! -f "$QUERIES_FILE" ]; then
        echo "Error: Queries file not found: $QUERIES_FILE" >&2
        exit 1
    fi
    _total=0
    _ok=0
    while IFS= read -r _line || [ -n "$_line" ]; do
        _line=$(echo "$_line" | sed 's/^[[:space:]]*//' | sed 's/[[:space:]]*$//')
        if [ -n "$_line" ]; then
            _total=$((_total + 1))
            if search_single "$_line"; then
                _ok=$((_ok + 1))
            fi
        fi
    done < "$QUERIES_FILE"
    echo ""
    echo "=== Batch complete: $_ok/$_total queries processed ==="
else
    search_single "$QUERY"
fi