Zettel Builder

__pycache__/
*.pyc
.DS_Store

MIT License

Copyright (c) 2026 Shuyi Wang

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

"""Shared helpers for zettel-builder scripts. Stdlib only."""
from __future__ import annotations

import json
import re
import os
from datetime import datetime, timezone
from pathlib import Path
from typing import Any

DEFAULT_ZETTEL_ROOT = Path(os.environ.get("ZETTEL_ROOT", Path.home() / "zettel"))
# v1.17.0 (2026-05-21): raw/ 现在直接在 zettel 仓内维护(hourly_pulse 拉取),
# 不再依赖 ~/wiki/published-articles/ 中转。
# 名字保留为 WIKI_ROOT 是历史兼容(scan_mechanical.py / backfill_source_timestamp.py 引用),
# 默认值改为 zettel_root,语义变为"raw 的父目录"。
DEFAULT_WIKI_ROOT = Path(os.environ.get("WIKI_ROOT", DEFAULT_ZETTEL_ROOT))


def now_iso() -> str:
    return datetime.now(timezone.utc).astimezone().isoformat(timespec="seconds")


def read_json(path: Path, default: Any = None) -> Any:
    if not path.exists():
        return default
    try:
        return json.loads(path.read_text(encoding="utf-8"))
    except json.JSONDecodeError:
        return default


def write_json(path: Path, data: Any) -> None:
    path.parent.mkdir(parents=True, exist_ok=True)
    tmp = path.with_suffix(path.suffix + ".tmp")
    tmp.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8")
    tmp.replace(path)


FRONTMATTER_RE = re.compile(r"^---\n(.*?)\n---\n(.*)$", re.DOTALL)


def parse_frontmatter(text: str) -> tuple[dict, str]:
    """Return (frontmatter_dict, body). Naive YAML — supports the subset we use.

    Avoids PyYAML dependency. Only the schema in card-format.md is supported.
    """
    m = FRONTMATTER_RE.match(text)
    if not m:
        return {}, text
    fm_raw, body = m.group(1), m.group(2)
    fm: dict[str, Any] = {}
    current_key = None
    current_list: list | None = None
    current_obj: dict | None = None
    obj_key: str | None = None

    for raw_line in fm_raw.splitlines():
        if not raw_line.strip():
            continue
        if not raw_line.startswith(" "):
            current_list = None
            current_obj = None
            obj_key = None
            if ":" not in raw_line:
                continue
            key, _, val = raw_line.partition(":")
            key = key.strip()
            val = val.strip()
            if val == "":
                # Defer container type: don't assign fm[key] yet. The first
                # indented child line decides — '- ' makes it list, 'k: v' makes it dict.
                if key in fm:
                    del fm[key]
                current_key = key
                current_list = None
                current_obj = None
                obj_key = key
            elif val.startswith("[") and val.endswith("]"):
                inner = val[1:-1].strip()
                fm[key] = [x.strip() for x in inner.split(",") if x.strip()] if inner else []
                current_key = None
            else:
                fm[key] = _coerce(val)
                current_key = None
        else:
            stripped = raw_line.strip()
            if stripped.startswith("- "):
                # List item: lazily make fm[current_key] a list
                if current_key:
                    if current_key not in fm or not isinstance(fm.get(current_key), list):
                        fm[current_key] = []
                item_body = stripped[2:].strip()
                if ":" in item_body:
                    new_item: dict[str, Any] = {}
                    k, _, v = item_body.partition(":")
                    new_item[k.strip()] = _coerce(v.strip())
                    if current_key:
                        fm[current_key].append(new_item)
                        current_obj = new_item
                else:
                    if current_key:
                        fm[current_key].append(_coerce(item_body))
            elif ":" in stripped and current_key:
                # Indented k:v line under a parent.
                # If we're inside a list (last '- ' seen), populate the last dict.
                # Else, this is a nested dict — lazily make fm[current_key] a dict.
                k, _, v = stripped.partition(":")
                if isinstance(fm.get(current_key), list) and current_obj is not None:
                    current_obj[k.strip()] = _coerce(v.strip())
                else:
                    if not isinstance(fm.get(current_key), dict):
                        fm[current_key] = {}
                    fm[current_key][k.strip()] = _coerce(v.strip())
                    current_obj = fm[current_key]
    return fm, body


def _coerce(val: str) -> Any:
    if val == "":
        return ""
    if val.lower() == "true":
        return True
    if val.lower() == "false":
        return False
    if val.lower() in ("null", "~"):
        return None
    if val.startswith('"') and val.endswith('"'):
        return val[1:-1]
    if val.startswith("'") and val.endswith("'"):
        return val[1:-1]
    if re.match(r"^-?\d+$", val):
        return int(val)
    if re.match(r"^-?\d+\.\d+$", val):
        return float(val)
    return val


def tokens_cn_en(text: str) -> list[str]:
    """Mixed Chinese/English tokenizer.

    - English: whitespace + lower
    - Chinese: 2-grams over consecutive CJK runs
    - Numbers kept as tokens
    """
    text = text.lower()
    out: list[str] = []
    # Strip markdown noise
    text = re.sub(r"[`*_#>\[\]()]", " ", text)
    for chunk in re.findall(r"[A-Za-z0-9]+|[一-鿿]+", text):
        if chunk[0].isascii():
            if len(chunk) >= 2:
                out.append(chunk)
        else:
            if len(chunk) == 1:
                continue  # skip 1-char CJK (too noisy)
            for i in range(len(chunk) - 1):
                out.append(chunk[i:i + 2])
    return out


def load_cards(cards_dir: Path) -> list[dict]:
    """Return list of card dicts: {id, path, frontmatter, body}."""
    cards = []
    if not cards_dir.exists():
        return cards
    for fp in sorted(cards_dir.glob("*.md")):
        try:
            text = fp.read_text(encoding="utf-8")
        except OSError:
            continue
        fm, body = parse_frontmatter(text)
        if not fm.get("id"):
            continue
        cards.append({"id": fm["id"], "path": str(fp), "fm": fm, "body": body})
    return cards


def log_line(log_path: Path, line: str) -> None:
    log_path.parent.mkdir(parents=True, exist_ok=True)
    with log_path.open("a", encoding="utf-8") as f:
        f.write(f"[{now_iso()}] {line}\n")

#!/usr/bin/env bash
# bootstrap_runner.sh — Persistent loop that drives Oven through the entire raw
# queue until empty, then auto-switches the system to steady state.
#
# Architecture:
# - Triggers `openclaw cron run zettel-bootstrap-hourly` and waits for the run
#   to land in ~/.openclaw/cron/runs/<id>.jsonl as finished/failed.
# - On success: backoff resets to 30s, next iteration triggers immediately.
# - On rate-limit/quota/timeout: exponential backoff (60 → 120 → 240 → 480 → 960s),
#   capped at 1920s (32 min). Backoff resets after one success.
# - When queue is empty for 2 consecutive checks (~5 min idle), runs cleanup:
#   disable bootstrap-hourly cron, enable event-driven-scan, flip mode.json to
#   steady, send Telegram completion notice, exit 0.
#
# Run as: systemctl --user start zettel-bootstrap-runner.service
# Or directly: bash zettel-bootstrap-runner.service
set -euo pipefail

ZETTEL_ROOT="${ZETTEL_ROOT:-$HOME/zettel}"
LOG="$ZETTEL_ROOT/.bootstrap.log"
exec > >(tee -a "$LOG") 2>&1

trigger_telegram() {
    local msg="$1"
    if [[ -f "$HOME/.config/secrets.zsh" ]]; then
        eval "$(grep '^export ' "$HOME/.config/secrets.zsh")"
        curl -s -X POST "https://api.telegram.org/bot${SIMPSON_BOT_TOKEN}/sendMessage" \
            -d "chat_id=${TELEGRAM_CHAT_ID}" -d "text=$msg" > /dev/null || true
    fi
}

resolve_job_id() {
    local name="$1"
    python3 -c "
import json
d = json.load(open('$HOME/.openclaw/cron/jobs.json'))
for j in d['jobs']:
    if j.get('name') == '$name':
        print(j['id']); break
" 2>/dev/null
}

BOOT_ID=$(resolve_job_id "zettel-bootstrap-hourly")
EVENT_ID=$(resolve_job_id "zettel-event-driven-scan")
if [[ -z "$BOOT_ID" ]]; then
    echo "FATAL: zettel-bootstrap-hourly job not found"
    trigger_telegram "❌ bootstrap_runner 启动失败:zettel-bootstrap-hourly cron job 找不到"
    exit 1
fi

echo "===== $(date '+%Y-%m-%d %H:%M:%S') bootstrap_runner start (boot=$BOOT_ID) ====="

backoff=30
consec_empty_checks=0
cards_at_start=$(ls "$ZETTEL_ROOT/cards"/*.md 2>/dev/null | wc -l)
trigger_telegram "🚀 bootstrap_runner 启动。当前卡数 $cards_at_start。开始消化全量库存。每跑完 100 张推一次进度。"
total_cards_added=0
last_telegram_milestone=0

while true; do
    queue_size=$(ls "$ZETTEL_ROOT/_queue"/*.json 2>/dev/null | wc -l)
    cards_now=$(ls "$ZETTEL_ROOT/cards"/*.md 2>/dev/null | wc -l)
    total_cards_added=$((cards_now - cards_at_start))

    echo "[$(date '+%H:%M:%S')] queue=$queue_size cards=$cards_now added_since_start=$total_cards_added backoff=$backoff"

    # Empty-queue exit condition: 3 consecutive empty checks = 2 × 150s waits = ~5 min idle
    # (Codex caught: >=2 only gave 1 wait = ~2.5 min, too eager)
    if [[ "$queue_size" -eq 0 ]]; then
        consec_empty_checks=$((consec_empty_checks + 1))
        if [[ "$consec_empty_checks" -ge 3 ]]; then
            echo "queue empty for ${consec_empty_checks} consecutive checks (~5 min); switching to steady state"

            # Cron flips first — if either fails, abort the switch and retry next loop
            # (Codex caught: don't flip mode.json + send Telegram if cron state didn't actually change)
            if ! openclaw cron disable "$BOOT_ID" 2>&1 | tail -1; then
                echo "ERROR: disable bootstrap-hourly failed; will retry next loop"
                consec_empty_checks=0
                sleep 60
                continue
            fi
            if [[ -n "$EVENT_ID" ]]; then
                if ! openclaw cron enable "$EVENT_ID" 2>&1 | tail -1; then
                    echo "ERROR: enable event-driven failed; re-enabling bootstrap and retrying"
                    openclaw cron enable "$BOOT_ID" 2>&1 | tail -1 || true
                    consec_empty_checks=0
                    sleep 60
                    continue
                fi
            fi

            # Now flip mode.json
            python3 -c "
import json, datetime
p = '$ZETTEL_ROOT/_state/mode.json'
d = json.load(open(p))
d['mode'] = 'steady'
d['switched_at'] = datetime.datetime.now(datetime.timezone.utc).astimezone().isoformat(timespec='seconds')
d['switched_by'] = 'bootstrap_runner: queue drained'
d['cards_at_switch'] = $cards_now
d['cards_added_during_bootstrap'] = $total_cards_added
json.dump(d, open(p,'w'), ensure_ascii=False, indent=2)
"
            # Commit + push (failures here are non-fatal — git can be retried)
            cd "$ZETTEL_ROOT"
            git add -A && git commit -m "chore: bootstrap complete, switched to steady" 2>&1 | tail -2 || true
            git pull --rebase origin main 2>&1 | tail -1 || true
            git push origin main 2>&1 | tail -1 || true
            trigger_telegram "✅ bootstrap 完成!网络已建立。本轮共生成 ${total_cards_added} 张新卡,总卡数 ${cards_now}。已切到 steady 态:bootstrap-hourly cron 已 disabled,event-driven 已 enabled。后续走 wiki-sync 触发的事件驱动模式。"
            echo "===== bootstrap_runner exit success ====="
            exit 0
        fi
        echo "queue empty (${consec_empty_checks}/3), sleeping 150s before recheck"
        sleep 150
        continue
    fi
    consec_empty_checks=0

    # Trigger one Oven run via openclaw cron run
    run_output=$(openclaw cron run "$BOOT_ID" 2>&1 || true)
    run_id=$(echo "$run_output" | python3 -c "import sys,json; d=json.load(sys.stdin); print(d.get('runId',''))" 2>/dev/null || true)
    if [[ -z "$run_id" ]]; then
        # Could be "already-running" or other transient
        reason=$(echo "$run_output" | python3 -c "import sys,json; d=json.load(sys.stdin); print(d.get('reason',''))" 2>/dev/null || true)
        if [[ "$reason" == "already-running" ]]; then
            echo "openclaw says already-running, waiting 60s and retry"
            sleep 60
            continue
        fi
        echo "ERROR: openclaw cron run failed: $run_output"
        echo "backoff ${backoff}s and retry"
        sleep "$backoff"
        backoff=$((backoff * 2))
        if [[ "$backoff" -gt 1920 ]]; then backoff=1920; fi
        continue
    fi
    echo "triggered run $run_id"

    # Wait for this specific run to finish (poll jsonl every 15s, timeout 30min)
    jsonl="$HOME/.openclaw/cron/runs/$BOOT_ID.jsonl"
    waited=0
    while [[ "$waited" -lt 1800 ]]; do
        sleep 15
        waited=$((waited + 15))
        if [[ -f "$jsonl" ]]; then
            last_line=$(tail -n 1 "$jsonl")
            this_run_id=$(echo "$last_line" | python3 -c "import sys,json; d=json.load(sys.stdin); print(d.get('runId',''))" 2>/dev/null || true)
            action=$(echo "$last_line" | python3 -c "import sys,json; d=json.load(sys.stdin); print(d.get('action',''))" 2>/dev/null || true)
            status=$(echo "$last_line" | python3 -c "import sys,json; d=json.load(sys.stdin); print(d.get('status',''))" 2>/dev/null || true)
            if [[ "$this_run_id" == "$run_id" ]] && [[ "$action" == "finished" ]]; then
                echo "run $run_id done: status=$status"
                if [[ "$status" == "ok" ]]; then
                    backoff=30
                else
                    error=$(echo "$last_line" | python3 -c "import sys,json; d=json.load(sys.stdin); print(d.get('error','')[:200])" 2>/dev/null || true)
                    echo "run failed: $error"
                    # Codex noted: OpenClaw's exact quota error string isn't documented.
                    # Conservative — any non-ok status triggers exponential backoff;
                    # known quota-like strings get longer back-off explicitly.
                    if echo "$error" | grep -qiE "quota|rate.?limit|429|too.?many|exceeded|throttle"; then
                        echo "quota/rate-like error, exponential backoff ${backoff}s"
                        sleep "$backoff"
                        backoff=$((backoff * 2))
                        if [[ "$backoff" -gt 1920 ]]; then backoff=1920; fi
                    else
                        echo "other error, baseline backoff 60s (backoff state preserved)"
                        sleep 60
                    fi
                fi
                break
            fi
        fi
    done
    if [[ "$waited" -ge 1800 ]]; then
        echo "WARN: run $run_id did not finish within 30 min, moving on"
    fi

    # Telegram milestone: every 100 cards added
    milestone=$((total_cards_added / 100))
    if [[ "$milestone" -gt "$last_telegram_milestone" ]] && [[ "$total_cards_added" -gt 0 ]]; then
        trigger_telegram "📊 bootstrap 进度:已新增 ${total_cards_added} 张卡,_queue 剩余 ${queue_size} 项。"
        last_telegram_milestone=$milestone
    fi
done

#!/usr/bin/env python3
"""Rebuild ~/zettel/_index/{tags,entities,term_inverted,links_graph}.json from cards/."""
from __future__ import annotations

import argparse
import sys
from pathlib import Path

sys.path.insert(0, str(Path(__file__).parent))
from _common import DEFAULT_ZETTEL_ROOT, load_cards, log_line, tokens_cn_en, write_json  # noqa: E402


def main() -> int:
    parser = argparse.ArgumentParser()
    parser.add_argument("--zettel-root", type=Path, default=DEFAULT_ZETTEL_ROOT)
    args = parser.parse_args()
    zroot: Path = args.zettel_root
    log_path = zroot / ".sync.log"

    log_line(log_path, "===== build_index start =====")

    cards = load_cards(zroot / "cards")
    tags_idx: dict[str, list[str]] = {}
    entities_idx: dict[str, list[str]] = {}
    term_inv: dict[str, list[str]] = {}
    links_graph: dict[str, dict] = {}

    for c in cards:
        cid = c["id"]
        fm = c["fm"]
        body = c["body"]

        for tag in fm.get("tags", []) or []:
            tags_idx.setdefault(str(tag), []).append(cid)
        for ent in fm.get("entities", []) or []:
            entities_idx.setdefault(str(ent), []).append(cid)

        # term inverted index (lite tf-idf input)
        seen_terms = set()
        for tok in tokens_cn_en(body):
            if tok in seen_terms:
                continue
            seen_terms.add(tok)
            term_inv.setdefault(tok, []).append(cid)

        # links graph
        node = links_graph.setdefault(cid, {"out": [], "in": []})
        for link in fm.get("links", []) or []:
            if isinstance(link, dict) and link.get("id"):
                target = str(link["id"])
                node["out"].append({"id": target, "reason": link.get("reason", "")})
                links_graph.setdefault(target, {"out": [], "in": []})["in"].append({"id": cid, "reason": link.get("reason", "")})

    idx_dir = zroot / "_index"
    write_json(idx_dir / "tags.json", tags_idx)
    write_json(idx_dir / "entities.json", entities_idx)
    write_json(idx_dir / "term_inverted.json", term_inv)
    write_json(idx_dir / "links_graph.json", links_graph)

    log_line(log_path, f"index built: cards={len(cards)} tags={len(tags_idx)} entities={len(entities_idx)} terms={len(term_inv)}")
    log_line(log_path, "===== build_index done =====")

    print(f"index: cards={len(cards)} tags={len(tags_idx)} entities={len(entities_idx)} terms={len(term_inv)}")
    return 0


if __name__ == "__main__":
    raise SystemExit(main())

#!/usr/bin/env python3
"""v1.15.0 experimental: multi-view cluster detection.

Four parallel candidate views:
1. link-cluster  — explicit [[]] dense subgraph (existing, relaxed thresholds)
2. tag-cluster   — cards sharing the same tag (≥ N members)
3. entity-cluster — cards sharing the same entity (≥ N members)
4. embedding-cluster — k-NN graph on BGE vectors, connected components (loose semantic neighborhood)

Outputs _state/multi_view_candidates.json for review without touching the
production pending_articles.json (run with --apply to also overwrite that).
"""
from __future__ import annotations

import argparse
import hashlib
import json
import sys
from collections import defaultdict
from pathlib import Path

sys.path.append(str(Path(__file__).parent))
from _common import DEFAULT_ZETTEL_ROOT, load_cards, now_iso, read_json, write_json  # noqa: E402


def _connected_components(adj: dict[str, set[str]], node_set: set[str] | None = None) -> list[set[str]]:
    seen = set()
    out = []
    pool = node_set if node_set is not None else set(adj.keys())
    for node in pool:
        if node in seen:
            continue
        comp = set()
        stack = [node]
        while stack:
            cur = stack.pop()
            if cur in seen:
                continue
            seen.add(cur)
            if cur not in pool:
                continue
            comp.add(cur)
            for nb in adj.get(cur, set()):
                if nb not in seen and nb in pool:
                    stack.append(nb)
        if comp:
            out.append(comp)
    return out


def _density(nodes: set[str], adj: dict[str, set[str]]) -> float:
    n = len(nodes)
    if n < 2:
        return 0.0
    edges = sum(len(adj.get(m, set()) & nodes) for m in nodes) // 2
    return edges / (n * (n - 1) / 2)


def _build_link_adj(cards: list[dict]) -> dict[str, set[str]]:
    perm_ids = {c["id"] for c in cards if c["fm"].get("status") == "permanent"}
    adj = {cid: set() for cid in perm_ids}
    for c in cards:
        if c["id"] not in perm_ids:
            continue
        for link in c["fm"].get("links", []) or []:
            tgt = link.get("id") if isinstance(link, dict) else None
            if tgt and tgt in perm_ids:
                adj[c["id"]].add(tgt)
                adj[tgt].add(c["id"])
    return adj


def _link_clusters(adj: dict[str, set[str]], min_size: int = 3, min_density: float = 0.3) -> list[dict]:
    out = []
    for comp in _connected_components(adj):
        if len(comp) < min_size:
            continue
        d = _density(comp, adj)
        if d >= min_density:
            out.append({"view": "link", "members": sorted(comp), "size": len(comp), "density": round(d, 3)})
    return out


def _tag_clusters(cards: list[dict], by_id: dict, min_size: int = 3,
                  max_size: int = 30) -> list[dict]:
    """Filter to focused topical tags (3-30 members). >30 = macro tag (too generic)."""
    tag_idx = defaultdict(set)
    for c in cards:
        if c["fm"].get("status") != "permanent":
            continue
        for t in c["fm"].get("tags", []) or []:
            tag_idx[str(t)].add(c["id"])
    out = []
    for tag, members in tag_idx.items():
        if not (min_size <= len(members) <= max_size):
            continue
        out.append({
            "view": "tag",
            "tag": tag,
            "members": sorted(members),
            "size": len(members),
        })
    return out


def _entity_clusters(cards: list[dict], by_id: dict, min_size: int = 3,
                     max_size: int = 30) -> list[dict]:
    ent_idx = defaultdict(set)
    for c in cards:
        if c["fm"].get("status") != "permanent":
            continue
        for e in c["fm"].get("entities", []) or []:
            ent_idx[str(e)].add(c["id"])
    out = []
    for ent, members in ent_idx.items():
        if not (min_size <= len(members) <= max_size):
            continue
        out.append({
            "view": "entity",
            "entity": ent,
            "members": sorted(members),
            "size": len(members),
        })
    return out


def _embedding_clusters(zroot: Path, by_id: dict,
                       k: int = 3, sim_threshold: float = 0.78,
                       min_size: int = 3, max_size: int = 30) -> list[dict]:
    try:
        from embed_lite import load_cache
    except ImportError:
        return []
    ids, vecs = load_cache(zroot)
    if not ids or vecs is None or vecs.shape[0] < min_size:
        return []
    import numpy as np
    # Filter to permanent cards only
    perm_mask = []
    for cid in ids:
        c = by_id.get(cid)
        perm_mask.append(c is not None and c["fm"].get("status") == "permanent")
    perm_mask = np.array(perm_mask)
    perm_ids = [ids[i] for i, m in enumerate(perm_mask) if m]
    perm_vecs = vecs[perm_mask]
    if perm_vecs.shape[0] < min_size:
        return []
    # k-NN graph: for each card, top-k cosine neighbors with sim ≥ threshold
    sim_mtx = perm_vecs @ perm_vecs.T  # cosine since BGE-normalized
    adj: dict[str, set[str]] = {cid: set() for cid in perm_ids}
    for i, cid in enumerate(perm_ids):
        # exclude self, sort by sim descending
        order = sim_mtx[i].argsort()[::-1]
        added = 0
        for j in order:
            if j == i:
                continue
            if sim_mtx[i][j] < sim_threshold:
                break
            adj[cid].add(perm_ids[j])
            adj[perm_ids[j]].add(cid)  # undirected
            added += 1
            if added >= k:
                break
    out = []
    for comp in _connected_components(adj):
        if not (min_size <= len(comp) <= max_size):
            continue
        d = _density(comp, adj)
        out.append({
            "view": "embedding",
            "members": sorted(comp),
            "size": len(comp),
            "density": round(d, 3),
        })
    return out


def _annotate(cluster: dict, by_id: dict) -> dict:
    members = cluster["members"]
    n = len(members)
    # core tags
    tag_counts = defaultdict(int)
    for m in members:
        for t in by_id[m]["fm"].get("tags", []) or []:
            tag_counts[str(t)] += 1
    core_tags = sorted([t for t, c in tag_counts.items() if c >= max(2, int(0.4 * n))])
    # voice ratio
    voice_count = sum(1 for m in members if by_id[m]["fm"].get("voice_passed") is True)
    voice_ratio = voice_count / n if n else 0
    cluster["core_tags"] = core_tags
    cluster["voice_ratio"] = round(voice_ratio, 3)
    cluster["cluster_id"] = hashlib.sha1((cluster["view"] + ":" + ",".join(members)).encode("utf-8")).hexdigest()[:12]
    # Sample member titles
    titles = []
    for m in members[:6]:
        body = by_id[m].get("body", "")
        for line in body.splitlines():
            if line.startswith("# "):
                titles.append({"id": m, "title": line[2:].strip()})
                break
    cluster["sample_titles"] = titles
    return cluster


def _merge_overlap(candidates: list[dict], jaccard_threshold: float = 0.6) -> list[dict]:
    """Merge clusters whose member sets overlap heavily (Jaccard >= threshold)."""
    # Sort by size descending; iterate, keep cluster if not strongly subsumed by any kept one.
    cands = sorted(candidates, key=lambda x: -x["size"])
    kept = []
    kept_sets = []
    for c in cands:
        m = set(c["members"])
        absorb = False
        for i, ks in enumerate(kept_sets):
            inter = len(m & ks)
            union = len(m | ks)
            jaccard = inter / union if union else 0
            if jaccard >= jaccard_threshold:
                # Absorbed: tag the kept cluster with all views it covers
                kept[i].setdefault("merged_views", set()).add(c["view"])
                if c["view"] == "tag":
                    kept[i].setdefault("merged_views", set()).add(f"tag:{c.get('tag','')}")
                elif c["view"] == "entity":
                    kept[i].setdefault("merged_views", set()).add(f"entity:{c.get('entity','')}")
                absorb = True
                break
        if not absorb:
            kept.append(c)
            kept_sets.append(m)
    # Convert merged_views set to sorted list for JSON
    for k in kept:
        if "merged_views" in k:
            k["merged_views"] = sorted(k["merged_views"])
    return kept


def main() -> int:
    parser = argparse.ArgumentParser()
    parser.add_argument("--zettel-root", type=Path, default=DEFAULT_ZETTEL_ROOT)
    parser.add_argument("--min-size", type=int, default=3)
    parser.add_argument("--min-link-density", type=float, default=0.3)
    parser.add_argument("--embed-k", type=int, default=3)
    parser.add_argument("--embed-sim", type=float, default=0.78)
    parser.add_argument("--max-size", type=int, default=30,
                        help="upper bound for candidate cluster size (skip macro tags)")
    parser.add_argument("--apply", action="store_true",
                        help="Also write to _state/pending_articles.json (overwriting prod)")
    parser.add_argument("--max-show", type=int, default=10)
    args = parser.parse_args()
    zroot = args.zettel_root

    cards = load_cards(zroot / "cards")
    by_id = {c["id"]: c for c in cards}
    adj = _build_link_adj(cards)

    link = _link_clusters(adj, min_size=args.min_size, min_density=args.min_link_density)
    tag = _tag_clusters(cards, by_id, min_size=args.min_size, max_size=args.max_size)
    ent = _entity_clusters(cards, by_id, min_size=args.min_size, max_size=args.max_size)
    emb = _embedding_clusters(zroot, by_id, k=args.embed_k,
                              sim_threshold=args.embed_sim,
                              min_size=args.min_size, max_size=args.max_size)

    print(f"per-view candidate counts:")
    print(f"  link:      {len(link)}")
    print(f"  tag:       {len(tag)}")
    print(f"  entity:    {len(ent)}")
    print(f"  embedding: {len(emb)}")

    all_cands = link + tag + ent + emb
    for c in all_cands:
        _annotate(c, by_id)

    merged = _merge_overlap(all_cands, jaccard_threshold=0.6)
    # Sort merged by size desc + density tiebreaker
    merged.sort(key=lambda x: (-x["size"], -x.get("density", 0)))

    print(f"\nafter overlap-merge (jaccard ≥ 0.6): {len(merged)} candidates")
    print(f"\ntop {args.max_show} merged candidates:")
    for i, c in enumerate(merged[:args.max_show]):
        view = c["view"]
        if view == "tag":
            label = f"tag:{c.get('tag','?')}"
        elif view == "entity":
            label = f"ent:{c.get('entity','?')}"
        else:
            label = view
        dens = f" density={c.get('density','-')}" if c.get("density") is not None else ""
        merged_in = f" [+{len(c.get('merged_views',[]))} other views]" if c.get('merged_views') else ""
        print(f"  {i+1}. [{label}] size={c['size']}{dens} voice={c.get('voice_ratio',0)} tags={c.get('core_tags',[])[:3]}{merged_in}")
        for t in c.get("sample_titles", [])[:3]:
            print(f"       - {t['title'][:60]}")

    out = {
        "generated_at": now_iso(),
        "summary": {
            "total_cards": len(cards),
            "link_candidates": len(link),
            "tag_candidates": len(tag),
            "entity_candidates": len(ent),
            "embedding_candidates": len(emb),
            "merged_candidates": len(merged),
        },
        "candidates": merged,
    }
    out_path = zroot / "_state" / "multi_view_candidates.json"
    write_json(out_path, out)
    print(f"\nwrote: {out_path}")
    if args.apply:
        prod = zroot / "_state" / "pending_articles.json"
        write_json(prod, merged)
        print(f"applied to: {prod}")
    return 0


if __name__ == "__main__":
    raise SystemExit(main())

#!/usr/bin/env python3
"""Compute top-K link candidates for a given card.

Primary: cosine similarity over BGE-small-zh-v1.5 embeddings (semantic).
Fallback: TF-IDF + tag/entity overlap (lexical) if embedder unavailable.

Final score (when embeddings available) blends semantic with structural signals
so cards that share tags/entities and link-neighbors rank higher than purely
text-similar ones.
"""
from __future__ import annotations

import argparse
import json
import sys
from pathlib import Path

sys.path.append(str(Path(__file__).parent))
from _common import DEFAULT_ZETTEL_ROOT, load_cards, read_json, tokens_cn_en  # noqa: E402


def jaccard(a: set, b: set) -> float:
    if not a and not b:
        return 0.0
    inter = len(a & b)
    union = len(a | b)
    return inter / union if union else 0.0


def _tfidf_cosine_lite(tokens_a: list[str], tokens_b: list[str], term_inv: dict, n_docs: int) -> float:
    import math

    def vec(tokens):
        tf: dict[str, int] = {}
        for t in tokens:
            tf[t] = tf.get(t, 0) + 1
        v = {}
        for t, c in tf.items():
            df = len(term_inv.get(t, [])) or 1
            idf = math.log((n_docs + 1) / df) + 1.0
            v[t] = c * idf
        return v

    va, vb = vec(tokens_a), vec(tokens_b)
    if not va or not vb:
        return 0.0
    common = set(va) & set(vb)
    dot = sum(va[t] * vb[t] for t in common)
    na = math.sqrt(sum(v * v for v in va.values()))
    nb = math.sqrt(sum(v * v for v in vb.values()))
    return dot / (na * nb) if na and nb else 0.0


def _structural_score(my_card: dict, other_card: dict, my_neighbors: set, link_graph: dict) -> tuple[float, dict]:
    my_tags = set(my_card["fm"].get("tags", []) or [])
    my_ents = set(my_card["fm"].get("entities", []) or [])
    o_tags = set(other_card["fm"].get("tags", []) or [])
    o_ents = set(other_card["fm"].get("entities", []) or [])

    t_score = jaccard(my_tags, o_tags)
    e_score = jaccard(my_ents, o_ents)

    o_nbrs = set()
    for entry in link_graph.get(other_card["id"], {}).get("out", []) or []:
        o_nbrs.add(entry["id"])
    for entry in link_graph.get(other_card["id"], {}).get("in", []) or []:
        o_nbrs.add(entry["id"])
    nbr_score = len(my_neighbors & o_nbrs) / max(1, len(my_neighbors | o_nbrs))

    return (
        0.5 * t_score + 0.3 * e_score + 0.2 * nbr_score,
        {"shared_tags": sorted(my_tags & o_tags), "shared_entities": sorted(my_ents & o_ents)},
    )


def _embedding_path(zroot: Path, card_id: str, cards: list[dict], by_id: dict, top_k: int, threshold: float) -> list[dict] | None:
    """Return candidates using embeddings, or None if unavailable."""
    try:
        from embed_lite import EmbedderUnavailable, refresh, top_k_similar
    except ImportError:
        return None

    try:
        refresh(zroot, verbose=False)
    except Exception as e:
        sys.stderr.write(f"embed refresh failed: {e}\n")
        return None

    sims = top_k_similar(card_id, k=max(top_k * 2, 20), zettel_root=zroot, threshold=threshold)
    if not sims:
        return None

    link_graph = read_json(zroot / "_index" / "links_graph.json", default={})
    my_card = by_id[card_id]
    my_nbrs = set()
    for entry in link_graph.get(card_id, {}).get("out", []) or []:
        my_nbrs.add(entry["id"])
    for entry in link_graph.get(card_id, {}).get("in", []) or []:
        my_nbrs.add(entry["id"])

    results = []
    for cid, sem in sims:
        if cid not in by_id:
            continue
        other = by_id[cid]
        struct, meta = _structural_score(my_card, other, my_nbrs, link_graph)
        # Semantic dominates; structural boosts ties and surfaces shared-context cards
        score = 0.7 * float(sem) + 0.3 * struct
        results.append({
            "id": cid,
            "score": round(score, 4),
            "semantic": round(float(sem), 4),
            "structural": round(struct, 4),
            "shared_tags": meta["shared_tags"],
            "shared_entities": meta["shared_entities"],
            "candidate_title_excerpt": (other["body"].splitlines()[0][:80] if other["body"] else ""),
            "source": "embedding",
        })

    results.sort(key=lambda x: -x["score"])
    return results[:top_k]


def _tfidf_path(zroot: Path, card_id: str, cards: list[dict], by_id: dict, top_k: int, threshold: float) -> list[dict]:
    """Fallback: TF-IDF + structural."""
    me = by_id[card_id]
    my_tokens = tokens_cn_en(me["body"])
    term_inv = read_json(zroot / "_index" / "term_inverted.json", default={})
    link_graph = read_json(zroot / "_index" / "links_graph.json", default={})
    my_nbrs = set()
    for entry in link_graph.get(card_id, {}).get("out", []) or []:
        my_nbrs.add(entry["id"])
    for entry in link_graph.get(card_id, {}).get("in", []) or []:
        my_nbrs.add(entry["id"])

    n_docs = len(cards)
    results = []
    for c in cards:
        if c["id"] == card_id:
            continue
        struct, meta = _structural_score(me, c, my_nbrs, link_graph)
        sem_lex = _tfidf_cosine_lite(my_tokens, tokens_cn_en(c["body"]), term_inv, n_docs)
        score = 0.5 * sem_lex + 0.5 * struct
        if score < threshold:
            continue
        results.append({
            "id": c["id"],
            "score": round(score, 4),
            "semantic": round(sem_lex, 4),
            "structural": round(struct, 4),
            "shared_tags": meta["shared_tags"],
            "shared_entities": meta["shared_entities"],
            "candidate_title_excerpt": (c["body"].splitlines()[0][:80] if c["body"] else ""),
            "source": "tfidf",
        })
    results.sort(key=lambda x: -x["score"])
    return results[:top_k]


def main() -> int:
    parser = argparse.ArgumentParser()
    parser.add_argument("--zettel-root", type=Path, default=DEFAULT_ZETTEL_ROOT)
    parser.add_argument("--card-id", required=True)
    parser.add_argument("--top-k", type=int, default=10)
    parser.add_argument("--threshold", type=float, default=0.05)
    parser.add_argument("--force-tfidf", action="store_true",
                        help="Skip embedding path; use TF-IDF only (for debugging or no-net machines)")
    args = parser.parse_args()
    zroot: Path = args.zettel_root

    cards = load_cards(zroot / "cards")
    by_id = {c["id"]: c for c in cards}
    if args.card_id not in by_id:
        for c in load_cards(zroot / "fleeting"):
            by_id[c["id"]] = c
    if args.card_id not in by_id:
        print(json.dumps({"error": f"card not found: {args.card_id}"}), file=sys.stderr)
        return 1

    results = None
    if not args.force_tfidf:
        results = _embedding_path(zroot, args.card_id, cards, by_id, args.top_k, args.threshold)

    if results is None:
        results = _tfidf_path(zroot, args.card_id, cards, by_id, args.top_k, args.threshold)

    print(json.dumps(results, ensure_ascii=False, indent=2))
    return 0


if __name__ == "__main__":
    raise SystemExit(main())

#!/usr/bin/env python3
"""Mode 5: command-line entry for 'propose' (命题作文).

User gives a seed (a case, an argument, an observation), and this script:
1. Embeds the seed text using fastembed
2. Finds top-K candidate cards by cosine similarity
3. Writes _state/proposal_<ts>.json with seed + candidates + metadata

The Oven (or any agent) then reads this JSON, opens each candidate card,
judges relevance + role (support/counter/case/context), drafts an outline,
and writes ~/zettel/_drafts/proposal_<ts>.md.

Usage:
    python3 propose_seed.py --seed "若数据集本身有偏,..."
    python3 propose_seed.py --seed-file /path/to/seed.txt
"""
from __future__ import annotations

import argparse
import json
import sys
from datetime import datetime, timezone
from pathlib import Path

sys.path.append(str(Path(__file__).parent))
from _common import DEFAULT_ZETTEL_ROOT, load_cards, now_iso  # noqa: E402


def main() -> int:
    parser = argparse.ArgumentParser()
    parser.add_argument("--zettel-root", type=Path, default=DEFAULT_ZETTEL_ROOT)
    parser.add_argument("--seed", default=None, help="Seed text (case / argument / question)")
    parser.add_argument("--seed-file", type=Path, default=None,
                        help="Read seed from a file instead of CLI")
    parser.add_argument("--top-k", type=int, default=30,
                        help="Number of candidate cards to surface for agent review")
    args = parser.parse_args()

    if args.seed_file:
        seed = args.seed_file.read_text(encoding="utf-8").strip()
    elif args.seed:
        seed = args.seed
    else:
        print("error: --seed or --seed-file required", file=sys.stderr)
        return 1

    if not seed:
        print("error: empty seed", file=sys.stderr)
        return 1

    try:
        from embed_lite import Embedder, load_cache
    except ImportError as e:
        print(f"error: embed_lite unavailable: {e}", file=sys.stderr)
        return 2

    # Embed the seed
    emb = Embedder.get()
    seed_vec = emb.embed([seed])[0]
    # Normalize (defensive — should already be normalized by Embedder.embed)
    import numpy as np
    n = float(np.linalg.norm(seed_vec))
    if n == 0:
        print("error: seed embedded to zero vector", file=sys.stderr)
        return 3
    seed_vec = seed_vec / n

    # Load card embeddings
    ids, vecs = load_cache(args.zettel_root)
    if not ids or vecs is None or vecs.shape[0] == 0:
        print("error: no card embeddings available (run build_index + embed_lite first)",
              file=sys.stderr)
        return 4

    sims = vecs @ seed_vec
    order = sims.argsort()[::-1][:args.top_k]

    cards = load_cards(args.zettel_root / "cards")
    by_id = {c["id"]: c for c in cards}

    candidates = []
    for i in order:
        cid = ids[i]
        card = by_id.get(cid)
        if not card:
            continue
        # Title from first H1
        title = ""
        for line in card["body"].splitlines():
            if line.startswith("# "):
                title = line[2:].strip()
                break
        # Judge excerpt
        judge_excerpt = ""
        ji = card["body"].find("## 我的判断")
        if ji >= 0:
            judge_excerpt = card["body"][ji:ji + 600]
        candidates.append({
            "id": cid,
            "title": title,
            "sim": round(float(sims[i]), 4),
            "tags": card["fm"].get("tags", []),
            "entities": card["fm"].get("entities", []),
            "status": card["fm"].get("status", ""),
            "path": card["path"],
            "judge_excerpt": judge_excerpt,
        })

    ts = datetime.now(timezone.utc).astimezone().strftime("%Y%m%dT%H%M")
    output = {
        "generated_at": now_iso(),
        "proposal_id": f"proposal_{ts}",
        "seed": seed,
        "n_candidates": len(candidates),
        "candidates": candidates,
    }
    out_path = args.zettel_root / "_state" / f"proposal_{ts}.json"
    out_path.parent.mkdir(parents=True, exist_ok=True)
    out_path.write_text(json.dumps(output, ensure_ascii=False, indent=2), encoding="utf-8")

    print(json.dumps({
        "ok": True,
        "proposal_id": output["proposal_id"],
        "output_path": str(out_path),
        "seed_length": len(seed),
        "n_candidates": len(candidates),
        "top_sim": candidates[0]["sim"] if candidates else 0,
    }, ensure_ascii=False))
    return 0


if __name__ == "__main__":
    raise SystemExit(main())

#!/usr/bin/env bash
# sync_raw_to_zettel.sh — Keep ~/zettel/raw/ in sync with upstream raw sources.
#
# Short-term: mirrors from wiki repo (where R2 processing is already centralized).
# Long-term: if wiki repo is decommissioned, replace step 2-3 with direct
# Dropbox + GetNote API sync (see ~/wiki/.../scripts/sync-raw.sh for reference).
#
# Idempotent: rsync --delete keeps target a clean mirror; symlink ensures
# images stay accessible locally.
set -euo pipefail

ZETTEL_ROOT="${ZETTEL_ROOT:-$HOME/zettel}"
WIKI_RAW="$HOME/wiki/published-articles/raw"
IMG_SRC="$HOME/knowledge/getnote-local/images"

mkdir -p "$ZETTEL_ROOT/raw/getnote" "$ZETTEL_ROOT/raw/articles"

# Mirror articles (text only, 4.3 MB)
if [[ -d "$WIKI_RAW/articles" ]]; then
    rsync -a --delete "$WIKI_RAW/articles/" "$ZETTEL_ROOT/raw/articles/"
fi

# Mirror getnote (text only with image refs, 21 MB)
if [[ -d "$WIKI_RAW/getnote" ]]; then
    rsync -a --delete "$WIKI_RAW/getnote/" "$ZETTEL_ROOT/raw/getnote/"
fi

# Symlink images (1.2 GB, not in git — points to local GetNote cache)
if [[ -d "$IMG_SRC" ]]; then
    if [[ -L "$ZETTEL_ROOT/raw/images" ]] || [[ ! -e "$ZETTEL_ROOT/raw/images" ]]; then
        ln -sfn "$IMG_SRC" "$ZETTEL_ROOT/raw/images"
    fi
fi

echo "sync_raw_to_zettel: $(ls "$ZETTEL_ROOT/raw/getnote" | wc -l) getnote, $(ls "$ZETTEL_ROOT/raw/articles" | wc -l) articles"