Yandex Webmaster

config/.env
cache/*
!cache/.gitkeep

# OAuth token for Yandex Webmaster API v4 (scopes: webmaster:hostinfo, webmaster:verify)
YANDEX_WEBMASTER_TOKEN="your_oauth_token_here"

# Session_id cookie из браузера (yandex.ru) — нужен для scripts/alice.sh
# (data SoV/Alice не имеет публичного API, парсится из window._initData).
# Берём в DevTools → Application → Cookies → yandex.ru → Session_id (httpOnly).
# Кавычки обязательны: значение содержит '|'.
SESSION_ID="paste_session_id_here"

#!/bin/sh
# Add a site to Yandex Webmaster
# Usage: add_site.sh --url <site_url>

set -e

SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
. "$SCRIPT_DIR/common.sh"
load_config

URL=""

while [ $# -gt 0 ]; do
    case "$1" in
        --url) URL="$2"; shift 2 ;;
        *)     shift ;;
    esac
done

if [ -z "$URL" ]; then
    echo "Error: --url <site_url> is required." >&2
    echo "Example: bash scripts/add_site.sh --url https://example.com" >&2
    exit 1
fi

ensure_user_id

TMPFILE="${WM_TMPDIR}/wm_add_site_$$.json"
trap 'rm -f "$TMPFILE"' EXIT

_escaped_url=$(json_escape "$URL")
webmaster_user_post "/hosts" "{\"host_url\":\"$_escaped_url\"}" > "$TMPFILE"

_host_id=$(json_extract_field_raw "$(cat "$TMPFILE")" "host_id")

echo "Site added successfully."
echo "Host ID: $_host_id"
echo ""
echo "Next step: verify ownership with:"
echo "  bash scripts/verify.sh --host-id \"$_host_id\" --action get --method DNS"

# Invalidate hosts cache
rm -f "$CACHE_DIR/hosts.tsv" "$CACHE_DIR/hosts.json"
echo "(hosts cache invalidated)"

#!/usr/bin/env python3
"""Yandex Webmaster — Alice (Share of Voice) efficiency extractor.

Не имеет публичного API: данные приходят в HTML через window._initData при SSR.
Авторизация — через cookie Session_id (длинноживущая, httpOnly).

Subcommands:
  fetch       — скачать страницу, извлечь alice.* из _initData, сохранить JSON
  sov         — Share-of-Voice: 12 недельных точек, TSV
  competitors — топ-10 сайтов в Alice (queries.GENERAL), TSV
  with-site   — запросы где наш сайт присутствует (hasOwnExamples), TSV
  without-site — запросы где наш сайт НЕ присутствует (noOwnExamples), TSV
  summary     — короткая сводка alertType + средний SoV + размеры списков

Все subcommand'ы кроме fetch читают из cache JSON; если кеша нет — делают fetch.
"""
import argparse
import json
import os
import re
import ssl
import sys
import urllib.error
import urllib.request

UA = (
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "
    "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36"
)

# Python.org installs ship without a CA bundle. Try common system locations
# before failing back to Python defaults (which may also be empty).
_CA_CANDIDATES = [
    os.environ.get("SSL_CERT_FILE", ""),
    "/etc/ssl/cert.pem",                          # macOS, FreeBSD
    "/etc/ssl/certs/ca-certificates.crt",         # Debian/Ubuntu
    "/etc/pki/tls/certs/ca-bundle.crt",           # RHEL/CentOS
    "/opt/homebrew/etc/ca-certificates/cert.pem", # Homebrew arm64
    "/usr/local/etc/ca-certificates/cert.pem",    # Homebrew x86_64
]


def _build_ssl_context() -> ssl.SSLContext:
    for path in _CA_CANDIDATES:
        if path and os.path.isfile(path):
            return ssl.create_default_context(cafile=path)
    return ssl.create_default_context()

ALICE_URL_TEMPLATE = (
    "https://webmaster.yandex.ru/site/{host_id}/efficiency/alice/"
    "?tab=GENERAL&tableType=GENERAL&onlyWithMySites=OFF"
)


def fail(msg, code=1):
    print(f"Error: {msg}", file=sys.stderr)
    sys.exit(code)


# ---------- HTML fetch ----------

def fetch_html(host_id: str, session_id: str) -> str:
    url = ALICE_URL_TEMPLATE.format(host_id=host_id)
    req = urllib.request.Request(
        url,
        headers={
            "User-Agent": UA,
            "Accept": "text/html,application/xhtml+xml",
            "Accept-Language": "ru,en;q=0.9",
            "Cookie": f"Session_id={session_id}",
        },
    )
    ctx = _build_ssl_context()
    try:
        with urllib.request.urlopen(req, timeout=30, context=ctx) as resp:
            charset = resp.headers.get_content_charset() or "utf-8"
            return resp.read().decode(charset, errors="replace")
    except urllib.error.HTTPError as e:
        fail(f"HTTP {e.code} from {url}: {e.reason}")
    except urllib.error.URLError as e:
        fail(f"network error: {e.reason}")


# ---------- _initData extraction ----------

def extract_init_data(html: str) -> dict:
    idx = html.find("window._initData")
    if idx < 0:
        fail("window._initData not found in HTML — session likely expired")
    eq = html.find("=", idx)
    start = html.find("{", eq)
    if start < 0:
        fail("opening brace of _initData not found")

    depth = 0
    in_str = False
    esc = False
    end = -1
    for i in range(start, len(html)):
        ch = html[i]
        if in_str:
            if esc:
                esc = False
            elif ch == "\\":
                esc = True
            elif ch == '"':
                in_str = False
        else:
            if ch == '"':
                in_str = True
            elif ch == "{":
                depth += 1
            elif ch == "}":
                depth -= 1
                if depth == 0:
                    end = i + 1
                    break
    if end < 0:
        fail("unbalanced braces in _initData")

    raw = html[start:end]
    try:
        return json.loads(raw)
    except json.JSONDecodeError as e:
        fail(f"failed to parse _initData JSON: {e}")


def assert_authed(init_data: dict) -> None:
    if not init_data.get("userIsAuth"):
        fail("userIsAuth=false — Session_id cookie expired or invalid")


# ---------- Cache I/O ----------

def cache_path(cache_dir: str, host_id: str) -> str:
    safe = re.sub(r"[^a-zA-Z0-9._-]", "_", host_id)
    d = os.path.join(cache_dir, f"host_{safe}", "alice")
    os.makedirs(d, exist_ok=True)
    return os.path.join(d, "init.json")


def load_or_fetch(args, force=False) -> dict:
    """Returns alice dict, using cache when fresh."""
    path = cache_path(args.cache_dir, args.host_id)
    if not force and os.path.exists(path):
        try:
            with open(path, "r", encoding="utf-8") as f:
                return json.load(f)
        except json.JSONDecodeError:
            pass

    if not args.session_id:
        fail("SESSION_ID is required for fetch (set in config/.env)")

    html = fetch_html(args.host_id, args.session_id)
    init = extract_init_data(html)
    assert_authed(init)

    alice = init.get("alice")
    if not alice:
        fail("init._initData.alice missing — page changed or wrong host")

    with open(path, "w", encoding="utf-8") as f:
        json.dump(alice, f, ensure_ascii=False)
    return alice


# ---------- Output helpers ----------

def write_tsv(rows, path: str):
    with open(path, "w", encoding="utf-8") as f:
        for row in rows:
            cleaned = ["" if c is None else str(c).replace("\t", " ").replace("\n", " ").replace("\r", " ")
                       for c in row]
            f.write("\t".join(cleaned) + "\n")


def emit_tsv(header, rows, out_path, head=20):
    write_tsv([header] + rows, out_path)
    print("\t".join(header))
    for r in rows[:head]:
        print("\t".join("" if c is None else str(c) for c in r))
    if len(rows) > head:
        print(f"... ({len(rows) - head} more rows, full data in: {out_path})")
    print(f"\nTotal: {len(rows)}")
    print(f"TSV: {out_path}")


# ---------- Subcommands ----------

def cmd_fetch(args):
    alice = load_or_fetch(args, force=True)
    path = cache_path(args.cache_dir, args.host_id)
    print(f"alice keys: {', '.join(alice.keys())}")
    print(f"sov points: {len(alice.get('sov', []))}")
    q = alice.get("queries", {}) or {}
    general = q.get("GENERAL") or []
    examples = q.get("EXAMPLES") or {}
    has = examples.get("hasOwnExamples") or [] if isinstance(examples, dict) else []
    no = examples.get("noOwnExamples") or [] if isinstance(examples, dict) else []
    print(f"competitors (GENERAL): {len(general)}")
    print(f"with-site (hasOwnExamples): {len(has)}")
    print(f"without-site (noOwnExamples): {len(no)}")
    print(f"alertType: {alice.get('alertType')}")
    print(f"cached: {path}")


def cmd_sov(args):
    alice = load_or_fetch(args)
    sov = alice.get("sov") or []
    rows = []
    for p in sov:
        share = p.get("sharePercent")
        rows.append([
            p.get("dateFrom", ""),
            p.get("dateTo", ""),
            f"{share:.4f}" if isinstance(share, (int, float)) else "",
            f"{share * 100:.2f}%" if isinstance(share, (int, float)) else "",
        ])
    out = os.path.join(os.path.dirname(cache_path(args.cache_dir, args.host_id)), "sov.tsv")
    emit_tsv(["date_from", "date_to", "share", "share_pct"], rows, out, head=20)


def cmd_competitors(args):
    alice = load_or_fetch(args)
    general = ((alice.get("queries") or {}).get("GENERAL")) or []
    rows = [[i + 1, item.get("url", "")] for i, item in enumerate(general)]
    out = os.path.join(os.path.dirname(cache_path(args.cache_dir, args.host_id)), "competitors.tsv")
    emit_tsv(["rank", "url"], rows, out, head=20)


def _flatten_examples(items):
    """Each item: {query, urls: [{url,title,host,favicon}]}.
    TSV: query, rank_in_query, host, url, title."""
    rows = []
    for item in items or []:
        q = item.get("query", "")
        urls = item.get("urls") or []
        for i, u in enumerate(urls, start=1):
            rows.append([
                q,
                i,
                u.get("host", ""),
                u.get("url", ""),
                u.get("title", ""),
            ])
    return rows


def cmd_with_site(args):
    alice = load_or_fetch(args)
    ex = ((alice.get("queries") or {}).get("EXAMPLES")) or {}
    items = ex.get("hasOwnExamples") or []
    rows = _flatten_examples(items)
    out = os.path.join(os.path.dirname(cache_path(args.cache_dir, args.host_id)), "with_site.tsv")
    emit_tsv(["query", "rank", "host", "url", "title"], rows, out, head=15)
    print(f"Unique queries: {len(items)}")


def cmd_without_site(args):
    alice = load_or_fetch(args)
    ex = ((alice.get("queries") or {}).get("EXAMPLES")) or {}
    items = ex.get("noOwnExamples") or []
    rows = _flatten_examples(items)
    out = os.path.join(os.path.dirname(cache_path(args.cache_dir, args.host_id)), "without_site.tsv")
    emit_tsv(["query", "rank", "host", "url", "title"], rows, out, head=15)
    print(f"Unique queries: {len(items)}")


def cmd_summary(args):
    alice = load_or_fetch(args)
    sov = alice.get("sov") or []
    shares = [p.get("sharePercent") for p in sov if isinstance(p.get("sharePercent"), (int, float))]
    avg = sum(shares) / len(shares) if shares else 0
    last = shares[-1] if shares else 0
    first = shares[0] if shares else 0
    q = alice.get("queries") or {}
    general = q.get("GENERAL") or []
    ex = q.get("EXAMPLES") or {}
    has = ex.get("hasOwnExamples") or []
    no = ex.get("noOwnExamples") or []
    print(f"alertType:        {alice.get('alertType')}")
    print(f"sov points:       {len(sov)}")
    if sov:
        print(f"sov range:        {sov[0].get('dateFrom')} → {sov[-1].get('dateTo')}")
        print(f"sov first:        {first * 100:.2f}%")
        print(f"sov last:         {last * 100:.2f}%")
        print(f"sov avg:          {avg * 100:.2f}%")
    print(f"competitors:      {len(general)}")
    print(f"with-site qs:     {len(has)}")
    print(f"without-site qs:  {len(no)}")


# ---------- CLI ----------

def build_parser():
    p = argparse.ArgumentParser(description="Yandex Webmaster Alice efficiency (SSR scraper)")
    p.add_argument("--host-id", required=True, help="Host id, e.g. https:metallik.ru:443")
    p.add_argument("--session-id", default=os.environ.get("SESSION_ID", ""),
                   help="Session_id cookie (default: $SESSION_ID)")
    p.add_argument("--cache-dir", required=True, help="Cache directory root")
    sub = p.add_subparsers(dest="cmd", required=True)
    sub.add_parser("fetch", help="force refresh + summary").set_defaults(func=cmd_fetch)
    sub.add_parser("sov", help="Share-of-Voice timeline").set_defaults(func=cmd_sov)
    sub.add_parser("competitors", help="Top sites in Alice").set_defaults(func=cmd_competitors)
    sub.add_parser("with-site", help="queries where own site appears").set_defaults(func=cmd_with_site)
    sub.add_parser("without-site", help="queries where own site is absent").set_defaults(func=cmd_without_site)
    sub.add_parser("summary", help="short summary").set_defaults(func=cmd_summary)
    return p


def main():
    args = build_parser().parse_args()
    args.func(args)


if __name__ == "__main__":
    main()

#!/bin/sh
# Yandex Webmaster — Alice (Share of Voice) efficiency
#
# Усложнение: данных по Alice нет в Webmaster API v4. Они приходят только в HTML
# через window._initData при SSR. Авторизация — через cookie Session_id.
# Cм. references/ALICE_EFFICIENCY.md.
#
# Usage:
#   alice.sh --host <domain> --action <action> [--no-cache]
#   alice.sh --host-id <id>  --action <action> [--no-cache]
#
# Actions:
#   summary       — короткая сводка (alertType, SoV, размеры списков). По умолчанию.
#   sov           — Share-of-Voice timeline (12 недель), TSV
#   competitors   — топ сайтов в Alice по теме, TSV
#   with-site     — запросы где наш сайт присутствует, TSV
#   without-site  — запросы где наш сайт отсутствует, TSV
#   fetch         — форс-обновить кеш и распечатать сводку

set -e

SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
. "$SCRIPT_DIR/common.sh"
load_config

if [ -z "$SESSION_ID" ]; then
    echo "Error: SESSION_ID not set in config/.env." >&2
    echo "Достаньте Session_id из браузера (DevTools → Application → Cookies → yandex.ru)." >&2
    echo "См. references/ALICE_EFFICIENCY.md." >&2
    exit 1
fi

parse_host_params "$@"

ACTION_DEFAULT="summary"
[ -z "$ACTION" ] && ACTION="$ACTION_DEFAULT"

# Resolve host (host search needs API call → user_id)
if [ -z "$HOST_ID" ] && [ -n "$HOST_SEARCH" ]; then
    ensure_user_id
fi
resolve_host
require_host

# Map action → python subcommand
case "$ACTION" in
    summary|sov|competitors|fetch) PY_CMD="$ACTION" ;;
    with-site)    PY_CMD="with-site" ;;
    without-site) PY_CMD="without-site" ;;
    *)
        echo "Error: unknown --action '$ACTION'" >&2
        echo "Valid: summary | sov | competitors | with-site | without-site | fetch" >&2
        exit 1
        ;;
esac

# Cache invalidation: --no-cache forces refresh
if [ -n "$NO_CACHE" ] && [ "$PY_CMD" != "fetch" ]; then
    _alice_cache="$CACHE_DIR/host_$(printf '%s' "$HOST_ID" | sed 's/[^a-zA-Z0-9._-]/_/g')/alice/init.json"
    rm -f "$_alice_cache"
fi

SESSION_ID="$SESSION_ID" exec python3 "$SCRIPT_DIR/alice_efficiency.py" \
    --host-id "$HOST_ID" \
    --cache-dir "$CACHE_DIR" \
    "$PY_CMD"

#!/bin/sh
# Export all indexed pages archive
# Usage: archive_export.sh --host <domain> --action start|status [--task-id ID]

set -e

SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
. "$SCRIPT_DIR/common.sh"
load_config

TASK_ID=""

_args=""
while [ $# -gt 0 ]; do
    case "$1" in
        --task-id) TASK_ID="$2"; shift 2 ;;
        *)         _args="$_args $1"; shift ;;
    esac
done
# shellcheck disable=SC2086
parse_host_params $_args
ensure_user_id
resolve_host
require_host

ACTION="${ACTION:-start}"
TMPFILE="${WM_TMPDIR}/wm_archive_$$.json"
trap 'rm -f "$TMPFILE"' EXIT

case "$ACTION" in
    start)
        webmaster_raw_post "/v4/user/${USER_ID}/hosts/${HOST_ID}/indexing/archive/" "{}" > "$TMPFILE"
        _tid=$(json_extract_field_raw "$(cat "$TMPFILE")" "task_id")
        echo "Archive export started."
        echo "Task ID: $_tid"
        echo "Check status: bash scripts/archive_export.sh --host ... --action status --task-id $_tid"
        echo "(Generation takes 10 seconds to 3 minutes)"
        ;;

    status)
        if [ -z "$TASK_ID" ]; then
            echo "Error: --task-id is required for --action status" >&2
            exit 1
        fi
        webmaster_raw_get "/v4/user/${USER_ID}/hosts/${HOST_ID}/indexing/archive/${TASK_ID}" > "$TMPFILE"
        _body=$(cat "$TMPFILE")
        _state=$(json_extract_field_raw "$_body" "state")
        _url=$(json_extract_field_raw "$_body" "download_url")

        echo "=== Archive Export ==="
        echo "Task ID: $TASK_ID"
        echo "State:   $_state"
        if [ -n "$_url" ]; then
            echo "Download: $_url"
            echo "(URL valid for 24 hours)"
        fi
        ;;

    *)
        echo "Error: unknown action '$ACTION'. Use: start, status" >&2
        exit 1
        ;;
esac

#!/bin/sh
# Common functions for Yandex Webmaster API skill
# POSIX sh compatible — no bashisms

set -e

SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
CONFIG_FILE="$SCRIPT_DIR/../config/.env"
CACHE_DIR="$SCRIPT_DIR/../cache"

WEBMASTER_API="https://api.webmaster.yandex.net"

WM_TMPDIR="${TMPDIR:-/tmp}"
mkdir -p "$WM_TMPDIR"

# --------------- Config ---------------

load_config() {
    if [ -f "$CONFIG_FILE" ]; then
        # shellcheck disable=SC1090
        . "$CONFIG_FILE"
    fi

    if [ -z "$YANDEX_WEBMASTER_TOKEN" ]; then
        echo "Error: YANDEX_WEBMASTER_TOKEN not found." >&2
        echo "Set in config/.env or environment. See config/README.md." >&2
        exit 1
    fi
}

# --------------- User ID ---------------

# ensure_user_id — fetches and caches user_id
ensure_user_id() {
    _eui_cache="$CACHE_DIR/user_id.txt"
    if [ -f "$_eui_cache" ] && [ -s "$_eui_cache" ]; then
        USER_ID=$(cat "$_eui_cache")
        return 0
    fi

    _eui_tmp="${WM_TMPDIR}/wm_user_$$.json"
    trap 'rm -f "$_eui_tmp"' EXIT

    webmaster_raw_get "/v4/user" > "$_eui_tmp"
    USER_ID=$(json_extract_number "$(cat "$_eui_tmp")" "user_id")

    if [ -z "$USER_ID" ]; then
        echo "Error: could not get user_id from API." >&2
        cat "$_eui_tmp" >&2
        rm -f "$_eui_tmp"
        exit 1
    fi

    mkdir -p "$CACHE_DIR"
    printf '%s' "$USER_ID" > "$_eui_cache"
    rm -f "$_eui_tmp"
}

# --------------- Host resolution ---------------

# resolve_host — sets HOST_ID from --host (domain search) or --host-id (direct)
# Requires USER_ID to be set (call ensure_user_id first)
resolve_host() {
    if [ -n "$HOST_ID" ]; then
        return 0
    fi

    if [ -n "$HOST_SEARCH" ]; then
        _rh_tsv="$CACHE_DIR/hosts.tsv"
        if [ ! -f "$_rh_tsv" ] || [ ! -s "$_rh_tsv" ]; then
            _refresh_hosts_cache
        fi

        _rh_match=$(grep -i "$HOST_SEARCH" "$_rh_tsv" | head -1)
        if [ -z "$_rh_match" ]; then
            echo "Error: no host matching '$HOST_SEARCH' in cache." >&2
            echo "Run: bash scripts/hosts.sh --no-cache" >&2
            exit 1
        fi
        HOST_ID=$(printf '%s' "$_rh_match" | cut -f1)
    fi

    if [ -z "$HOST_ID" ]; then
        echo "Error: --host <domain> or --host-id <id> is required." >&2
        exit 1
    fi
}

_refresh_hosts_cache() {
    _rhc_tmp="${WM_TMPDIR}/wm_hosts_$$.json"
    webmaster_raw_get "/v4/user/${USER_ID}/hosts" > "$_rhc_tmp"

    mkdir -p "$CACHE_DIR"
    cp "$_rhc_tmp" "$CACHE_DIR/hosts.json"

    # Generate TSV: host_id<TAB>url<TAB>verified<TAB>main_mirror
    {
        sed 's/},{/}\n{/g' "$_rhc_tmp" | while IFS= read -r _line || [ -n "$_line" ]; do
            _hid=$(json_extract_field_raw "$_line" "host_id")
            _url=$(json_extract_field_raw "$_line" "ascii_host_url")
            _ver=$(json_extract_field_raw "$_line" "verified")
            [ -z "$_hid" ] && continue
            printf '%s\t%s\t%s\n' "$_hid" "$_url" "$_ver"
        done
    } > "$CACHE_DIR/hosts.tsv"

    rm -f "$_rhc_tmp"
}

# --------------- API helpers ---------------

# webmaster_raw_get <full_path> [extra_curl_args...]
# Low-level GET with OAuth, 429 retry, error handling
webmaster_raw_get() {
    _wrg_path="$1"
    shift
    _wrg_url="${WEBMASTER_API}${_wrg_path}"
    _wrg_headers="${WM_TMPDIR}/wm_headers_$$.txt"

    _wrg_body=$(curl -s -G -D "$_wrg_headers" \
        -H "Authorization: OAuth $YANDEX_WEBMASTER_TOKEN" \
        "$@" \
        "$_wrg_url") || {
        rm -f "$_wrg_headers"
        echo "Error: curl failed for $_wrg_url" >&2
        return 1
    }

    _wrg_status=$(head -1 "$_wrg_headers" | grep -o '[0-9][0-9][0-9]' | head -1)

    # 429 retry
    if [ "$_wrg_status" = "429" ]; then
        _wrg_retry=$(grep -i 'Retry-After' "$_wrg_headers" | sed 's/[^0-9]//g' | head -1)
        rm -f "$_wrg_headers"
        if [ -z "${_WM_RETRY_DONE:-}" ] && [ -n "$_wrg_retry" ] && [ "$_wrg_retry" -le 60 ] 2>/dev/null; then
            _wrg_jitter=$(awk 'BEGIN{srand(); printf "%d", rand()*3}')
            _wrg_wait=$(( _wrg_retry + _wrg_jitter ))
            echo "Rate limited. Waiting ${_wrg_wait}s..." >&2
            sleep "$_wrg_wait"
            _WM_RETRY_DONE=1 webmaster_raw_get "$_wrg_path" "$@"
            return $?
        else
            echo "Error: Rate limit exceeded (429)." >&2
            return 1
        fi
    fi

    # HTTP errors
    if [ -n "$_wrg_status" ] && [ "$_wrg_status" -ge 400 ] 2>/dev/null; then
        rm -f "$_wrg_headers"
        _wrg_err_code=$(json_extract_field_raw "$_wrg_body" "error_code")
        _wrg_err_msg=$(json_extract_field_raw "$_wrg_body" "error_message")
        if [ -n "$_wrg_err_code" ]; then
            echo "Error: HTTP $_wrg_status — $_wrg_err_code: $_wrg_err_msg" >&2
        else
            echo "Error: HTTP $_wrg_status from $_wrg_url" >&2
            echo "$_wrg_body" >&2
        fi
        return 1
    fi

    rm -f "$_wrg_headers"
    printf '%s' "$_wrg_body"
}

# webmaster_raw_post <full_path> <json_body>
# Low-level POST with OAuth
webmaster_raw_post() {
    _wrp_path="$1"
    _wrp_body_data="$2"
    _wrp_url="${WEBMASTER_API}${_wrp_path}"
    _wrp_headers="${WM_TMPDIR}/wm_headers_$$.txt"

    _wrp_resp=$(curl -s -X POST -D "$_wrp_headers" \
        -H "Authorization: OAuth $YANDEX_WEBMASTER_TOKEN" \
        -H "Content-Type: application/json; charset=UTF-8" \
        -d "$_wrp_body_data" \
        "$_wrp_url") || {
        rm -f "$_wrp_headers"
        echo "Error: curl failed for $_wrp_url" >&2
        return 1
    }

    _wrp_status=$(head -1 "$_wrp_headers" | grep -o '[0-9][0-9][0-9]' | head -1)

    if [ "$_wrp_status" = "429" ]; then
        _wrp_retry=$(grep -i 'Retry-After' "$_wrp_headers" | sed 's/[^0-9]//g' | head -1)
        rm -f "$_wrp_headers"
        if [ -z "${_WM_RETRY_DONE:-}" ] && [ -n "$_wrp_retry" ] && [ "$_wrp_retry" -le 60 ] 2>/dev/null; then
            _wrp_jitter=$(awk 'BEGIN{srand(); printf "%d", rand()*3}')
            _wrp_wait=$(( _wrp_retry + _wrp_jitter ))
            echo "Rate limited. Waiting ${_wrp_wait}s..." >&2
            sleep "$_wrp_wait"
            _WM_RETRY_DONE=1 webmaster_raw_post "$_wrp_path" "$_wrp_body_data"
            return $?
        else
            echo "Error: Rate limit exceeded (429)." >&2
            return 1
        fi
    fi

    if [ -n "$_wrp_status" ] && [ "$_wrp_status" -ge 400 ] 2>/dev/null; then
        rm -f "$_wrp_headers"
        _wrp_err_code=$(json_extract_field_raw "$_wrp_resp" "error_code")
        _wrp_err_msg=$(json_extract_field_raw "$_wrp_resp" "error_message")
        if [ -n "$_wrp_err_code" ]; then
            echo "Error: HTTP $_wrp_status — $_wrp_err_code: $_wrp_err_msg" >&2
        else
            echo "Error: HTTP $_wrp_status from $_wrp_url" >&2
            echo "$_wrp_resp" >&2
        fi
        return 1
    fi

    rm -f "$_wrp_headers"
    printf '%s' "$_wrp_resp"
}

# webmaster_get <host_subpath> [extra_curl_args...]
# GET /v4/user/{USER_ID}/hosts/{HOST_ID}/<subpath>
webmaster_get() {
    _wg_sub="$1"
    shift
    webmaster_raw_get "/v4/user/${USER_ID}/hosts/${HOST_ID}${_wg_sub}" "$@"
}

# webmaster_post <host_subpath> <json_body>
# POST /v4/user/{USER_ID}/hosts/{HOST_ID}/<subpath>
webmaster_post() {
    _wp_sub="$1"
    _wp_body="$2"
    webmaster_raw_post "/v4/user/${USER_ID}/hosts/${HOST_ID}${_wp_sub}" "$_wp_body"
}

# webmaster_user_get [extra_curl_args...]
# GET /v4/user/{USER_ID}/hosts (user-level, no specific host)
webmaster_user_get() {
    _wug_sub="$1"
    shift 2>/dev/null || true
    webmaster_raw_get "/v4/user/${USER_ID}${_wug_sub}" "$@"
}

# webmaster_user_post <subpath> <json_body>
webmaster_user_post() {
    _wup_sub="$1"
    _wup_body="$2"
    webmaster_raw_post "/v4/user/${USER_ID}${_wup_sub}" "$_wup_body"
}

# --------------- Cache helpers ---------------

cache_key() {
    printf '%s' "$1" | cksum | awk '{print $1}'
}

cache_get() {
    if [ -f "$1" ] && [ -s "$1" ]; then
        cat "$1"
        return 0
    fi
    return 1
}

cache_put() {
    mkdir -p "$(dirname "$1")"
    cat > "$1"
}

# cache_get_ttl <file_path> <max_age_minutes>
# Returns 0 if file exists and is newer than max_age_minutes.
# Returns 1 (miss) if file is missing, empty, or older than TTL.
cache_get_ttl() {
    _cgt_file="$1"
    _cgt_ttl="${2:-1440}"
    if [ -f "$_cgt_file" ] && [ -s "$_cgt_file" ]; then
        # find returns the file if it's OLDER than ttl → stale
        _cgt_stale=$(find "$_cgt_file" -mmin +"$_cgt_ttl" 2>/dev/null)
        if [ -z "$_cgt_stale" ]; then
            return 0
        fi
        # Stale — delete and miss
        rm -f "$_cgt_file"
    fi
    return 1
}

cache_host_dir() {
    _chd_dir="$CACHE_DIR/host_$(printf '%s' "$HOST_ID" | sed 's/[^a-zA-Z0-9._-]/_/g')"
    mkdir -p "$_chd_dir"
    echo "$_chd_dir"
}

# --------------- JSON escape ---------------

# json_escape <string> — escapes special characters for JSON string values
json_escape() {
    printf '%s' "$1" | sed 's/\\/\\\\/g; s/"/\\"/g; s/	/\\t/g' | tr -d '\n\r'
}

# --------------- JSON helpers (no jq) ---------------

json_extract_field_raw() {
    printf '%s' "$1" | grep -o "\"$2\"[[:space:]]*:[[:space:]]*\"[^\"]*\"" | head -1 | sed 's/.*:[[:space:]]*"//;s/"$//'
}

json_extract_number() {
    printf '%s' "$1" | grep -o "\"$2\"[[:space:]]*:[[:space:]]*[0-9.e+-]*" | head -1 | sed 's/.*:[[:space:]]*//'
}

json_extract_bool() {
    printf '%s' "$1" | grep -o "\"$2\"[[:space:]]*:[[:space:]]*[a-z]*" | head -1 | sed 's/.*:[[:space:]]*//'
}

# json_extract_array_strings <json> <field> — extracts string array values, one per line
json_extract_array_strings() {
    printf '%s' "$1" | grep -o "\"$2\"[[:space:]]*:[[:space:]]*\[[^]]*\]" | head -1 | grep -o '"[^"]*"' | sed 's/"//g'
}

# --------------- Output helpers ---------------

print_tsv_head() {
    _pth_file="$1"
    _pth_n="${2:-30}"
    if [ -f "$_pth_file" ]; then
        head -n "$_pth_n" "$_pth_file"
        _pth_total=$(wc -l < "$_pth_file" | tr -d ' ')
        if [ "$_pth_total" -gt "$_pth_n" ]; then
            echo "... ($(( _pth_total - _pth_n )) more rows, full data in: $_pth_file)"
        fi
    fi
}

# --------------- Date helpers ---------------

# date_subtract_days <YYYY-MM-DD> <days>
# Returns date N days before the given date. POSIX-compatible.
date_subtract_days() {
    _dsd_base="$1"
    _dsd_days="$2"
    # Try macOS date first, then GNU date
    if date -v-1d +%Y-%m-%d >/dev/null 2>&1; then
        date -j -f "%Y-%m-%d" "$_dsd_base" -v-"${_dsd_days}d" +%Y-%m-%d 2>/dev/null && return 0
    fi
    if date -d "$_dsd_base - $_dsd_days days" +%Y-%m-%d 2>/dev/null; then
        return 0
    fi
    # Pure shell fallback: subtract days via simple month/day arithmetic
    # Good enough for 90-day ranges
    _dsd_y=$(echo "$_dsd_base" | cut -d- -f1)
    _dsd_m=$(echo "$_dsd_base" | cut -d- -f2 | sed 's/^0//')
    _dsd_d=$(echo "$_dsd_base" | cut -d- -f3 | sed 's/^0//')
    _dsd_remain="$_dsd_days"
    while [ "$_dsd_remain" -gt 0 ]; do
        if [ "$_dsd_d" -gt "$_dsd_remain" ]; then
            _dsd_d=$(( _dsd_d - _dsd_remain ))
            _dsd_remain=0
        else
            _dsd_remain=$(( _dsd_remain - _dsd_d ))
            _dsd_m=$(( _dsd_m - 1 ))
            if [ "$_dsd_m" -lt 1 ]; then
                _dsd_m=12
                _dsd_y=$(( _dsd_y - 1 ))
            fi
            # Days in the new month
            case "$_dsd_m" in
                1|3|5|7|8|10|12) _dsd_d=31 ;;
                4|6|9|11) _dsd_d=30 ;;
                2) if [ $(( _dsd_y % 4 )) -eq 0 ] && { [ $(( _dsd_y % 100 )) -ne 0 ] || [ $(( _dsd_y % 400 )) -eq 0 ]; }; then _dsd_d=29; else _dsd_d=28; fi ;;
            esac
        fi
    done
    printf '%04d-%02d-%02d\n' "$_dsd_y" "$_dsd_m" "$_dsd_d"
}

# default_date_from [reference_date]
# Returns date 90 days before reference_date (default: today)
default_date_from() {
    _ddf_ref="${1:-$(date +%Y-%m-%d)}"
    date_subtract_days "$_ddf_ref" 90
}

# apply_default_dates — sets DATE_FROM if empty (for history scripts)
# If --date-to is set but --date-from is not: DATE_FROM = DATE_TO - 90 days
# If neither is set: DATE_FROM = today - 90 days
apply_default_dates() {
    if [ -z "$DATE_FROM" ]; then
        if [ -n "$DATE_TO" ]; then
            DATE_FROM=$(default_date_from "$DATE_TO")
        else
            DATE_FROM=$(default_date_from)
        fi
    fi
}

# --------------- Common param parsing ---------------

# parse_host_params "$@"
# Sets: HOST_ID, HOST_SEARCH, ACTION, DATE_FROM, DATE_TO, LIMIT, OFFSET, NO_CACHE
parse_host_params() {
    HOST_ID=""
    HOST_SEARCH=""
    ACTION=""
    DATE_FROM=""
    DATE_TO=""
    LIMIT=""
    OFFSET=""
    NO_CACHE=""

    while [ $# -gt 0 ]; do
        case "$1" in
            --host-id)    HOST_ID="$2"; shift 2 ;;
            --host)       HOST_SEARCH="$2"; shift 2 ;;
            --action)     ACTION="$2"; shift 2 ;;
            --date-from)  DATE_FROM="$2"; shift 2 ;;
            --date-to)    DATE_TO="$2"; shift 2 ;;
            --limit)      LIMIT="$2"; shift 2 ;;
            --offset)     OFFSET="$2"; shift 2 ;;
            --no-cache)   NO_CACHE="1"; shift ;;
            *)            shift ;;
        esac
    done
}

# require_host — exits if host not resolved
require_host() {
    if [ -z "$HOST_ID" ]; then
        echo "Error: --host <domain> or --host-id <id> is required." >&2
        exit 1
    fi
}

# add_date_params — builds curl args for date_from/date_to
# Usage: eval "set -- $(add_date_params)"
add_date_params() {
    _adp=""
    if [ -n "$DATE_FROM" ]; then
        _adp="$_adp --data-urlencode \"date_from=${DATE_FROM}T00:00:00.000+0300\""
    fi
    if [ -n "$DATE_TO" ]; then
        _adp="$_adp --data-urlencode \"date_to=${DATE_TO}T00:00:00.000+0300\""
    fi
    echo "$_adp"
}

#!/bin/sh
# Site diagnostics — list all problems with severity and state
# Usage: diagnostics.sh --host <domain>
# NOT cached (always live data)

set -e

SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
. "$SCRIPT_DIR/common.sh"
load_config

parse_host_params "$@"
ensure_user_id
resolve_host
require_host

TMPFILE="${WM_TMPDIR}/wm_diag_$$.json"
trap 'rm -f "$TMPFILE"' EXIT

webmaster_get "/diagnostics" > "$TMPFILE"

echo "problem	severity	state	last_update"

# Parse each problem entry
tr -d '\n\r' < "$TMPFILE" | sed 's/"problems"[[:space:]]*:{//;s/}[[:space:]]*$//' | \
    grep -o '"[A-Z_]*"[[:space:]]*:{[^}]*}' | while IFS= read -r _entry; do
    _name=$(printf '%s' "$_entry" | grep -o '^"[^"]*"' | sed 's/"//g')
    _severity=$(json_extract_field_raw "$_entry" "severity")
    _state=$(json_extract_field_raw "$_entry" "state")
    _updated=$(json_extract_field_raw "$_entry" "last_state_update")
    _date=$(printf '%s' "$_updated" | cut -c1-10)
    printf '%s\t%s\t%s\t%s\n' "$_name" "$_severity" "$_state" "${_date:--}"
done

#!/bin/sh
# External links — samples and history
# Usage: external_links.sh --host <domain> --action samples|history
#        [--date-from] [--date-to] [--limit N] [--offset N]
# History defaults to last 90 days if --date-from is not specified.

set -e

SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
. "$SCRIPT_DIR/common.sh"
load_config

parse_host_params "$@"
ensure_user_id
resolve_host
require_host

ACTION="${ACTION:-samples}"
TMPFILE="${WM_TMPDIR}/wm_extlinks_$$.json"
trap 'rm -f "$TMPFILE"' EXIT

case "$ACTION" in
    samples)
        _curl_args=""
        if [ -n "$LIMIT" ]; then
            _curl_args="--data-urlencode limit=$LIMIT"
        fi
        if [ -n "$OFFSET" ]; then
            _curl_args="$_curl_args --data-urlencode offset=$OFFSET"
        fi

        # shellcheck disable=SC2086
        webmaster_get "/links/external/samples" $_curl_args > "$TMPFILE"

        _count=$(json_extract_number "$(cat "$TMPFILE")" "count")

        echo "source_url	destination_url	discovery_date"
        tr -d '\n\r' < "$TMPFILE" | sed 's/},{/}\n{/g' | while IFS= read -r _line || [ -n "$_line" ]; do
            _src=$(json_extract_field_raw "$_line" "source_url")
            [ -z "$_src" ] && continue
            _dst=$(json_extract_field_raw "$_line" "destination_url")
            _disc=$(json_extract_field_raw "$_line" "discovery_date")
            printf '%s\t%s\t%s\n' "$_src" "$_dst" "${_disc:--}"
        done | head -30

        echo ""
        echo "Total external links: ${_count:-?} (showing first 30)"
        ;;

    history)
        apply_default_dates
        _host_dir=$(cache_host_dir)
        mkdir -p "$_host_dir/links"
        _hash=$(cache_key "ext_links_history_${DATE_FROM}_${DATE_TO}")
        _out_file="$_host_dir/links/external_history_${_hash}.tsv"

        # TTL cache check (24h)
        if [ -z "$NO_CACHE" ] && cache_get_ttl "$_out_file" 1440; then
            print_tsv_head "$_out_file" 30
            echo ""
            echo "(cached: $_out_file)"
            exit 0
        fi

        _curl_args="--data-urlencode indicator=LINKS_TOTAL_COUNT"
        if [ -n "$DATE_FROM" ]; then
            _curl_args="$_curl_args --data-urlencode date_from=${DATE_FROM}T00:00:00.000+0300"
        fi
        if [ -n "$DATE_TO" ]; then
            _curl_args="$_curl_args --data-urlencode date_to=${DATE_TO}T00:00:00.000+0300"
        fi

        # shellcheck disable=SC2086
        webmaster_get "/links/external/history" $_curl_args > "$TMPFILE"

        {
            echo "date	total_links"
            tr -d '\n\r' < "$TMPFILE" | grep -o '"date":"[^"]*","value":"[^"]*"' | while IFS= read -r _match; do
                _date=$(printf '%s' "$_match" | sed 's/.*"date":"//;s/".*//' | cut -c1-10)
                _val=$(printf '%s' "$_match" | grep -o '"value":"[^"]*"' | sed 's/"value":"//;s/"$//')
                printf '%s\t%s\n' "$_date" "$_val"
            done
        } > "$_out_file"

        print_tsv_head "$_out_file" 30
        echo ""
        echo "Cached: $_out_file"
        ;;

    *)
        echo "Error: unknown action '$ACTION'. Use: samples, history" >&2
        exit 1
        ;;
esac

#!/bin/sh
# YML Feeds management — list, add, batch-add, change regions, check status
# Usage: feeds.sh --host <domain> --action list|add|batch-add|change|add-status
#        [--url URL] [--type REALTY|VACANCY|GOODS|DOCTORS|CARS|SERVICES|EDUCATION|ACTIVITY]
#        [--region-ids "225,213"] [--request-id ID]

set -e

SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
. "$SCRIPT_DIR/common.sh"
load_config

FEED_URL=""
FEED_TYPE=""
REGION_IDS=""
REQUEST_ID=""

_args=""
while [ $# -gt 0 ]; do
    case "$1" in
        --url)        FEED_URL="$2"; shift 2 ;;
        --type)       FEED_TYPE="$2"; shift 2 ;;
        --region-ids) REGION_IDS="$2"; shift 2 ;;
        --request-id) REQUEST_ID="$2"; shift 2 ;;
        *)            _args="$_args $1"; shift ;;
    esac
done
# shellcheck disable=SC2086
parse_host_params $_args
ensure_user_id
resolve_host
require_host

ACTION="${ACTION:-list}"
TMPFILE="${WM_TMPDIR}/wm_feeds_$$.json"
trap 'rm -f "$TMPFILE"' EXIT

case "$ACTION" in
    list)
        webmaster_get "/feeds/list" > "$TMPFILE"

        _list_tmp="${WM_TMPDIR}/wm_feeds_list_$$.tsv"
        {
            echo "url	type	regions"
            tr -d '\n\r' < "$TMPFILE" | sed 's/},{/}\n{/g' | while IFS= read -r _line || [ -n "$_line" ]; do
                _furl=$(json_extract_field_raw "$_line" "url")
                [ -z "$_furl" ] && continue
                _ftype=$(json_extract_field_raw "$_line" "type")
                _fregs=$(printf '%s' "$_line" | grep -o '"regionIds"[[:space:]]*:\[[^]]*\]' | head -1 | grep -o '[0-9]*' | tr '\n' ',' | sed 's/,$//')
                printf '%s\t%s\t%s\n' "$_furl" "${_ftype:--}" "${_fregs:--}"
            done
        } > "$_list_tmp"
        print_tsv_head "$_list_tmp" 30
        rm -f "$_list_tmp"
        ;;

    add)
        if [ -z "$FEED_URL" ] || [ -z "$FEED_TYPE" ]; then
            echo "Error: --url and --type are required for --action add" >&2
            exit 1
        fi

        _escaped_furl=$(json_escape "$FEED_URL")
        _json="{\"url\":\"$_escaped_furl\",\"type\":\"$FEED_TYPE\""
        if [ -n "$REGION_IDS" ]; then
            _json="$_json,\"regionIds\":[$REGION_IDS]"
        fi
        _json="$_json}"

        webmaster_post "/feeds/add/start" "$_json" > "$TMPFILE"
        _rid=$(json_extract_field_raw "$(cat "$TMPFILE")" "requestId")
        echo "Feed upload started."
        echo "Request ID: $_rid"
        echo "Check status: bash scripts/feeds.sh --host ... --action add-status --request-id $_rid"
        ;;

    batch-add)
        # Reads feed definitions from stdin (JSON array)
        # Example: echo '[{"url":"https://a.com/f.yml","type":"GOODS"},{"url":"https://b.com/f.yml","type":"REALTY","regionIds":[213]}]' | bash scripts/feeds.sh --host ... --action batch-add
        if [ -t 0 ]; then
            echo "Error: batch-add reads JSON array from stdin." >&2
            echo "Example: echo '[{\"url\":\"...\",\"type\":\"GOODS\"}]' | bash scripts/feeds.sh --host ... --action batch-add" >&2
            exit 1
        fi

        _batch_body=$(cat)
        webmaster_post "/feeds/batch/add" "$_batch_body" > "$TMPFILE"

        echo "url	status"
        tr -d '\n\r' < "$TMPFILE" | sed 's/},{/}\n{/g' | while IFS= read -r _line || [ -n "$_line" ]; do
            _burl=$(json_extract_field_raw "$_line" "url")
            [ -z "$_burl" ] && continue
            _bstatus=$(json_extract_field_raw "$_line" "status")
            printf '%s\t%s\n' "$_burl" "$_bstatus"
        done
        ;;

    change)
        if [ -z "$FEED_URL" ] || [ -z "$REGION_IDS" ]; then
            echo "Error: --url and --region-ids are required for --action change" >&2
            exit 1
        fi

        _escaped_furl=$(json_escape "$FEED_URL")
        webmaster_post "/feeds/change" "{\"url\":\"$_escaped_furl\",\"newRegionIds\":[$REGION_IDS]}" > "$TMPFILE"
        echo "Feed regions updated."
        ;;

    add-status)
        if [ -z "$REQUEST_ID" ]; then
            echo "Error: --request-id is required for --action add-status" >&2
            exit 1
        fi

        webmaster_get "/feeds/add/info" --data-urlencode "requestId=$REQUEST_ID" > "$TMPFILE"
        _status=$(json_extract_field_raw "$(cat "$TMPFILE")" "processStatus")
        echo "Upload status: $_status"
        ;;

    *)
        echo "Error: unknown action '$ACTION'. Use: list, add, change, add-status" >&2
        exit 1
        ;;
esac

#!/bin/sh
# Get site details and owners
# Usage: host_info.sh --host <domain> | --host-id <id>

set -e

SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
. "$SCRIPT_DIR/common.sh"
load_config

parse_host_params "$@"
ensure_user_id
resolve_host
require_host

TMPFILE="${WM_TMPDIR}/wm_host_info_$$.json"
trap 'rm -f "$TMPFILE" "${TMPFILE}_owners"' EXIT

# Get host info
webmaster_get "" > "$TMPFILE"

echo "=== Site Info ==="
_hi_url=$(json_extract_field_raw "$(cat "$TMPFILE")" "ascii_host_url")
_hi_ver=$(json_extract_bool "$(cat "$TMPFILE")" "verified")
_hi_status=$(json_extract_field_raw "$(cat "$TMPFILE")" "host_data_status")
_hi_display=$(json_extract_field_raw "$(cat "$TMPFILE")" "host_display_name")
_hi_mirror=$(json_extract_field_raw "$(cat "$TMPFILE")" "ascii_host_url")

echo "Host ID:      $HOST_ID"
echo "URL:          $_hi_url"
echo "Display name: ${_hi_display:--}"
echo "Verified:     $_hi_ver"
echo "Data status:  ${_hi_status:--}"

# Get owners
echo ""
echo "=== Owners ==="
webmaster_get "/owners" > "${TMPFILE}_owners"

echo "login	verification_type	verification_date"
tr -d '\n\r' < "${TMPFILE}_owners" | sed 's/},{/}\n{/g' | while IFS= read -r _line || [ -n "$_line" ]; do
    _login=$(json_extract_field_raw "$_line" "user_login")
    _vtype=$(json_extract_field_raw "$_line" "verification_type")
    _vdate=$(json_extract_field_raw "$_line" "verification_date")
    [ -z "$_login" ] && continue
    printf '%s\t%s\t%s\n' "$_login" "$_vtype" "${_vdate:--}"
done

#!/bin/sh
# List Yandex Webmaster sites with cache + TSV index
# Usage: hosts.sh [--no-cache] [--search <text>]

set -e

SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
. "$SCRIPT_DIR/common.sh"
load_config

SEARCH=""
NO_CACHE=""

while [ $# -gt 0 ]; do
    case "$1" in
        --no-cache) NO_CACHE="1"; shift ;;
        --search)   SEARCH="$2"; shift 2 ;;
        *)          shift ;;
    esac
done

ensure_user_id

CACHE_TSV="$CACHE_DIR/hosts.tsv"

# Try cache first
if [ -z "$NO_CACHE" ] && [ -f "$CACHE_TSV" ] && [ -s "$CACHE_TSV" ]; then
    if [ -n "$SEARCH" ]; then
        echo "host_id	url	verified"
        grep -i "$SEARCH" "$CACHE_TSV" || echo "(no matches for '$SEARCH')"
    else
        # Prepend header for print_tsv_head
        _tmp_display="${WM_TMPDIR}/wm_hosts_display_$$.tsv"
        { echo "host_id	url	verified"; cat "$CACHE_TSV"; } > "$_tmp_display"
        print_tsv_head "$_tmp_display" 30
        rm -f "$_tmp_display"
    fi
    echo ""
    echo "(cached: $CACHE_TSV)"
    exit 0
fi

# Fetch from API
echo "Fetching hosts from API..." >&2

TMPFILE="${WM_TMPDIR}/wm_hosts_list_$$.json"
trap 'rm -f "$TMPFILE"' EXIT

webmaster_user_get "/hosts" > "$TMPFILE"

# Save raw JSON
mkdir -p "$CACHE_DIR"
cp "$TMPFILE" "$CACHE_DIR/hosts.json"

# Generate TSV: host_id<TAB>url<TAB>verified
{
    tr -d '\n\r' < "$TMPFILE" | sed 's/},{/}\n{/g' | while IFS= read -r _line || [ -n "$_line" ]; do
        _hid=$(json_extract_field_raw "$_line" "host_id")
        _url=$(json_extract_field_raw "$_line" "ascii_host_url")
        _ver=$(json_extract_bool "$_line" "verified")
        [ -z "$_hid" ] && continue
        printf '%s\t%s\t%s\n' "$_hid" "$_url" "$_ver"
    done
} > "$CACHE_TSV"

# Output
echo "host_id	url	verified"
if [ -n "$SEARCH" ]; then
    grep -i "$SEARCH" "$CACHE_TSV" || echo "(no matches for '$SEARCH')"
else
    print_tsv_head "$CACHE_TSV" 30
fi
echo ""
echo "Total hosts: $(wc -l < "$CACHE_TSV" | tr -d ' ')"
echo "Cached: $CACHE_TSV"