Happy App Audit

// Build the 4:5 infographic prompt from the already-generated Phase 1..5
// artifacts. This is the bridge between the audit pipeline and
// baoyu-imagine — the filled prompt is what the image model sees.

import { readFile, readdir } from "node:fs/promises";
import { basename, join } from "node:path";

interface Framework {
  name: string;
  path: string;
  size_bytes: number;
  archs?: string[];
}

interface Meta {
  bundle_id: string;
  version?: string;
  short_version?: string;
  sandboxed?: boolean | null;
  arbitrary_loads?: boolean | null;
  main_executable?: string | null;
  frameworks?: Framework[];
}

const PRIVACY_HOT = /applog|sscronet|ttnet|ime_net_sdk|parfait|sentry|bugly|umeng|firebase|sensorsdata|growingio|adjust|appsflyer|datadog|bytenn|sami|mars|mmkv/i;

const SDK_NOTES: Array<[RegExp, string]> = [
  [/onnxruntime/i, "设备端 ML 推理"],
  [/sscronet/i, "TTNet / Cronet 网络栈"],
  [/ime[_-]?net[_-]?sdk/i, "QUIC 云候选长链"],
  [/OimeEngine/i, "输入法引擎"],
  [/bytenn/i, "字节自研 NN 推理"],
  [/applogrs/i, "AppLog / TEA 埋点"],
  [/Parfait/i, "Parfait APM 采样"],
  [/ttnetdownloader/i, "TTNet 下载器"],
  [/sqlcipher/i, "加密 SQLite"],
  [/SwifterSwift/i, "Swift 工具库"],
  [/OimeCommon/i, "输入法公共库"],
  [/sentry/i, "Sentry 崩溃上报"],
  [/bugly/i, "Bugly 崩溃 / 埋点"],
  [/firebase/i, "Firebase 分析"],
  [/crashlytics/i, "Crashlytics"],
  [/mars/i, "mars 长链"],
  [/mmkv/i, "MMKV 存储"],
];

function sdkNote(name: string): string {
  for (const [re, note] of SDK_NOTES) if (re.test(name)) return note;
  return "—";
}

function formatSize(n: number): string {
  if (n < 1024) return `${n}B`;
  if (n < 1024 * 1024) return `${(n / 1024).toFixed(1)}K`;
  if (n < 1024 * 1024 * 1024) return `${(n / 1024 / 1024).toFixed(1)}M`;
  return `${(n / 1024 / 1024 / 1024).toFixed(2)}G`;
}

function padLeft(s: string, w: number): string {
  if (s.length >= w) return s;
  return " ".repeat(w - s.length) + s;
}

function padRight(s: string, w: number): string {
  if (s.length >= w) return s;
  return s + " ".repeat(w - s.length);
}

function redactLiteral(s: string): string {
  // 16+ consecutive digits → device id placeholder
  return s.replace(/\d{12,}/g, "〈redacted〉");
}

function buildSdkRows(meta: Meta): string {
  const top = (meta.frameworks || [])
    .slice()
    .sort((a, b) => b.size_bytes - a.size_bytes)
    .slice(0, 6);
  const sizeCol = Math.max(5, ...top.map((f) => formatSize(f.size_bytes).length));
  const nameCol = Math.max(12, ...top.map((f) => f.name.length));
  return top
    .map((f) => {
      const flag = PRIVACY_HOT.test(f.name) ? "   ← RED" : "";
      return `${padLeft(formatSize(f.size_bytes), sizeCol)}   ${padRight(
        f.name,
        nameCol,
      )}   ${sdkNote(f.name)}${flag}`;
    })
    .join("\n");
}

// Endpoints are ranked by reverse-lookup match (endpoints.md) first, then by
// raw URL / domain frequency in strings/*. Domains that are clearly static /
// CDN / documentation / code-table noise are suppressed.
const ENDPOINT_NOISE = /\.(png|svg|jpg|jpeg|gif|css|js|html|ico|json|pdf)(?:$|[/?#])/i;
const DOCS_HOSTS = /(^|\.)(github\.com|apple\.com|w3\.org|ietf\.org|example\.com|mozilla\.org|openssl\.org|rfc-editor\.org|yaml\.org|www\.google\.com)(\/|$)/i;
// Hosts with a doubled TLD (e.g. `foo.com...bar.com`) are glued string-table
// artifacts, not real endpoints. Also reject any host with an uppercase letter
// between lowercase (classic camelCase leak from a symbol table).
const GLUED_HOST = /(?:\.com|\.org|\.net|\.io|\.cn)[^.]*\.(?:com|org|net|io|cn)(?:$|\/)/i;

const ENDPOINT_CN_PURPOSE: Record<string, string> = {
  "endpoint.bytedance.ime": "每次按键云端候选",
  "endpoint.bytedance.sami": "语音实时推流",
  "endpoint.bytedance.applog": "埋点 / 行为日志",
  "endpoint.bytedance.ttnet": "TTNet 长链配置",
  "endpoint.bytedance.parfait": "APM 采样上报",
  "endpoint.bytedance.frontier": "长链接 frontier",
  "endpoint.bytedance.tnc": "TTNet 节点调度",
  "endpoint.bytedance.crash": "崩溃上报",
  "endpoint.tencent.mars": "mars 长链",
  "endpoint.sentry": "崩溃上报",
  "endpoint.firebase": "Firebase 分析",
  "endpoint.bugly": "Bugly 崩溃上报",
  "endpoint.umeng": "友盟统计",
  "endpoint.sensorsdata": "神策埋点",
  "endpoint.growingio": "GrowingIO 埋点",
};

function isKeystrokeLike(endpointLine: string): boolean {
  return /ime|keystroke|convert|input-method/i.test(endpointLine);
}

function synthProto(host: string, explicit?: string): string {
  if (explicit) return explicit.toUpperCase();
  if (/quic/i.test(host)) return "QUIC";
  if (/(^|[-.])ws([-.]|$)/i.test(host)) return "WSS";
  return "HTTPS";
}

function isValidHost(host: string): boolean {
  if (!host || !host.includes(".")) return false;
  if (host.length > 80) return false;
  if (!/^[a-z0-9.-]+$/.test(host)) return false;
  if (host.startsWith(".") || host.endsWith(".")) return false;
  if (host.includes("..")) return false;
  // Reject obvious glued-string artifacts
  if (GLUED_HOST.test(host)) return false;
  if (DOCS_HOSTS.test(host)) return false;
  return true;
}

function normalizeUrl(raw: string): { proto: string; host: string; path: string } | null {
  const m = raw.match(/^(https?|wss?|quic):\/\/([^/\s]+)(\/[^\s"'<>]*)?/i);
  if (!m) return null;
  const [, proto, host, path] = m;
  if (ENDPOINT_NOISE.test(raw)) return null;
  const lower = host.toLowerCase();
  if (!isValidHost(lower)) return null;
  return {
    proto: synthProto(lower, proto),
    host: lower,
    path: (path || "").replace(/[)>\]}]+$/, ""),
  };
}

// Parse a bare host (no scheme) — used for endpoint evidence from domains.md.
function normalizeHost(raw: string): { proto: string; host: string; path: string } | null {
  const cleaned = raw.trim().replace(/^`+|`+$/g, "");
  const lower = cleaned.toLowerCase();
  if (!isValidHost(lower)) return null;
  return { proto: synthProto(lower), host: lower, path: "" };
}

async function collectUrls(stringsDir: string): Promise<string[]> {
  const files = await readdir(stringsDir).catch(() => []);
  const urls: string[] = [];
  for (const f of files) {
    if (!f.endsWith(".urls.md")) continue;
    const text = await readFile(join(stringsDir, f), "utf8").catch(() => "");
    for (const line of text.split("\n")) {
      const m = line.match(/^- `(https?:\/\/[^`]+)`$/);
      if (m) urls.push(m[1]);
    }
  }
  return urls;
}

async function collectDomains(stringsDir: string): Promise<string[]> {
  const files = await readdir(stringsDir).catch(() => []);
  const hosts: string[] = [];
  for (const f of files) {
    if (!f.endsWith(".domains.md")) continue;
    const text = await readFile(join(stringsDir, f), "utf8").catch(() => "");
    for (const line of text.split("\n")) {
      const m = line.match(/^- `([^`]+)`$/);
      if (m) hosts.push(m[1]);
    }
  }
  return hosts;
}

async function parseConfirmedEndpointMatches(
  endpointsMdPath: string,
): Promise<Array<{ id: string; display: string }>> {
  const text = await readFile(endpointsMdPath, "utf8").catch(() => "");
  // rows look like: | endpoint.bytedance.ime | ime cloud convert | ... | **confirmed** |
  const rows: Array<{ id: string; display: string }> = [];
  for (const line of text.split("\n")) {
    const m = line.match(
      /^\|\s*(endpoint\.[\w.-]+)\s*\|\s*([^|]+?)\s*\|[^|]*\|[^|]*\|[^|]*\|\s*\*\*confirmed\*\*\s*\|/,
    );
    if (m) rows.push({ id: m[1].trim(), display: m[2].trim() });
  }
  return rows;
}

async function endpointEvidenceAll(
  endpointsMdPath: string,
  id: string,
): Promise<string[]> {
  const text = await readFile(endpointsMdPath, "utf8").catch(() => "");
  const sectionStart = text.indexOf(`### ${id}`);
  if (sectionStart < 0) return [];
  const after = text.slice(sectionStart);
  const end = after.indexOf("\n### ");
  const section = end < 0 ? after : after.slice(0, end);
  const out: string[] = [];
  for (const line of section.split("\n")) {
    const m = line.match(/^- `[^`]+`: `([^`]+)`$/);
    if (m) out.push(m[1]);
  }
  return out;
}

function purposeFor(
  id: string,
  englishDisplay: string,
): string {
  return ENDPOINT_CN_PURPOSE[id] || englishDisplay || "—";
}

export async function buildEndpointRows(
  endpointsMdPath: string,
  stringsDir: string,
  max = 6,
): Promise<string> {
  const confirmed = await parseConfirmedEndpointMatches(endpointsMdPath);
  const rows: Array<{
    star: boolean;
    proto: string;
    url: string;
    purpose: string;
  }> = [];
  const seenHosts = new Set<string>();

  for (const m of confirmed) {
    const evidence = await endpointEvidenceAll(endpointsMdPath, m.id);
    const purpose = purposeFor(m.id, m.display);
    for (const ev of evidence) {
      if (rows.length >= max) break;
      const norm = normalizeUrl(ev) || normalizeHost(ev);
      if (!norm) continue;
      const dedupKey = `${norm.host}${norm.path}`;
      if (seenHosts.has(dedupKey)) continue;
      seenHosts.add(dedupKey);
      const keystroke =
        isKeystrokeLike(m.id) || /\/convert\b|\/cloud\/|obric/i.test(norm.path);
      rows.push({
        star: keystroke,
        proto: norm.proto,
        url: norm.host + norm.path,
        purpose,
      });
    }
    if (rows.length >= max) break;
  }

  if (rows.length < max) {
    const urls = await collectUrls(stringsDir);
    for (const u of urls) {
      if (rows.length >= max) break;
      const norm = normalizeUrl(u);
      if (!norm) continue;
      if (seenHosts.has(norm.host)) continue;
      seenHosts.add(norm.host);
      const purpose = inferPurpose(norm.host);
      if (purpose === "—") continue;
      rows.push({
        star: false,
        proto: norm.proto,
        url: norm.host + norm.path.slice(0, 40),
        purpose,
      });
    }
  }

  if (rows.length < max) {
    const hosts = await collectDomains(stringsDir);
    for (const h of hosts) {
      if (rows.length >= max) break;
      const norm = normalizeHost(h);
      if (!norm) continue;
      if (seenHosts.has(norm.host)) continue;
      seenHosts.add(norm.host);
      const purpose = inferPurpose(norm.host);
      if (purpose === "—") continue;
      rows.push({ star: false, proto: norm.proto, url: norm.host, purpose });
    }
  }

  if (rows.length === 0) return "(no endpoints observed in static strings)";

  const protoCol = Math.max(5, ...rows.map((r) => r.proto.length));
  const urlCol = Math.max(24, ...rows.map((r) => r.url.length));
  return rows
    .map((r) => {
      const prefix = r.star ? "★ " : "  ";
      return `${prefix}${padRight(r.proto, protoCol)}   ${padRight(
        r.url,
        urlCol,
      )}   →  ${r.purpose}`;
    })
    .join("\n");
}

function inferPurpose(host: string): string {
  if (/log|applog|snssdk|zijieapi|klink/i.test(host)) return "埋点 / 日志上报";
  if (/crash|bugly|sentry|crashlytics/i.test(host)) return "崩溃上报";
  if (/frontier|sami|speech|audio/i.test(host)) return "语音 / 音频";
  if (/ime|convert|input/i.test(host)) return "输入法云端";
  if (/firebase|analytics|umeng|sensorsdata|growingio/i.test(host)) return "用户分析";
  if (/push|jpush|getui/i.test(host)) return "推送";
  if (/cdn|static/i.test(host)) return "静态资源";
  return "—";
}

// Local-data rows are picked by pattern priority: Parfait, alog, MMKV, sqlite,
// ttnet/server.json, other. Device ids in paths are redacted.
const LOCAL_PRIORITY: Array<[RegExp, string]> = [
  [/Parfait\/.*pftconfig/i, "APM 配置"],
  [/Parfait\/ready\//i, "APM 待上传队列"],
  [/alog\/(cache|log)\//i, "ALog 压缩日志"],
  [/ttnet\/server\.json/i, "TTNet 长轮询配置"],
  [/ttnet\/.*\.config|ttnet\/version/i, "TTNet 配置"],
  [/MMKV\//i, "MMKV 存储"],
  [/Recorder\/.*asrHistory/i, "ASR 录音历史"],
  [/Cache\.db(-wal|-shm)?$/i, "HTTP 缓存"],
  [/Crash\//i, "崩溃日志"],
  [/sqlcipher|\.db$/i, "SQLite"],
  [/\.plist$/i, "偏好"],
  [/Containers\/.*\/Data/i, "沙盒容器"],
];

interface LocalRow {
  size: string;
  kind: string;
  path: string;
  note: string;
  priority: number;
}

function classifyLocal(filePath: string): { kind: string; note: string; priority: number } {
  for (let i = 0; i < LOCAL_PRIORITY.length; i++) {
    const [re, note] = LOCAL_PRIORITY[i];
    if (re.test(filePath)) {
      if (/pftconfig/i.test(filePath)) return { kind: "cfg", note, priority: i };
      if (/alog/i.test(filePath)) return { kind: "log", note, priority: i };
      if (/\.json$/i.test(filePath)) return { kind: "json", note, priority: i };
      if (/\.db$/i.test(filePath)) return { kind: "sqlite", note, priority: i };
      if (/mmkv/i.test(filePath)) return { kind: "mmkv", note, priority: i };
      return { kind: "other", note, priority: i };
    }
  }
  return { kind: "other", note: "—", priority: 99 };
}

export async function buildLocalRows(
  localMdPath: string,
  homeDir: string,
  max = 5,
): Promise<string> {
  const text = await readFile(localMdPath, "utf8").catch(() => "");
  // Parse lines like: | 1.0K | cfg | `foo.pftconfig` |
  const rows: LocalRow[] = [];
  let currentDir = "";
  for (const line of text.split("\n")) {
    // Directory headings can contain spaces (e.g. "Application Support"), so
    // match to end-of-line, not \S.
    const dirMatch = line.match(/^##\s+(~\/.+|\/.+)\s*$/);
    if (dirMatch) {
      currentDir = dirMatch[1].trim();
      continue;
    }
    const fileMatch = line.match(/^\|\s*([\d.]+[BKMG])\s*\|\s*(\w+)\s*\|\s*`([^`]+)`\s*\|/);
    if (!fileMatch || !currentDir) continue;
    const [, size, , name] = fileMatch;
    const fullPath = `${currentDir}/${name}`;
    const { kind, note, priority } = classifyLocal(fullPath);
    const displayPath = shortenPath(fullPath);
    rows.push({ size, kind, path: redactLiteral(displayPath), note, priority });
  }

  // Collapse group = priority bucket + parent dir. Files in the *same* bucket
  // sharing a parent directory collapse into one row with `(×N)`. Keep the
  // larger size as the representative.
  const grouped = new Map<
    string,
    LocalRow & { count: number; totalBytes: number }
  >();
  for (const r of rows) {
    // Group by (note, top-2 notable segments) so that both casings of the app
    // dir (DoubaoIme vs doubaoime) and related subdirs (alog/cache + alog/log,
    // ttnet/server.json + ttnet/version) collapse into one representative row.
    const key = `${r.note}:${groupKey(r.path)}`;
    const bytes = parseSizeDesc(r.size);
    const prev = grouped.get(key);
    if (prev) {
      prev.count++;
      prev.totalBytes += bytes;
      if (bytes > parseSizeDesc(prev.size)) {
        prev.size = r.size;
        prev.path = r.path;
        prev.kind = r.kind;
      }
    } else {
      grouped.set(key, { ...r, count: 1, totalBytes: bytes });
    }
  }

  const collapsed = [...grouped.values()]
    .sort((a, b) => a.priority - b.priority || b.totalBytes - a.totalBytes)
    .slice(0, max)
    .map((r) => ({
      ...r,
      path: r.count > 1 ? `${pathTrunk(r.path)}/*  (×${r.count})` : r.path,
    }));

  if (collapsed.length === 0) return "(no on-disk surface observed)";

  const sizeCol = Math.max(4, ...collapsed.map((r) => r.size.length));
  const kindCol = Math.max(5, ...collapsed.map((r) => r.kind.length));
  const pathCol = Math.max(30, ...collapsed.map((r) => r.path.length));
  return collapsed
    .map(
      (r) =>
        `${padLeft(r.size, sizeCol)}   ${padRight(r.kind, kindCol)}   ${padRight(
          r.path,
          pathCol,
        )}   ${r.note}`,
    )
    .join("\n");
}

function shortenPath(p: string): string {
  // Shorten `/Users/xxx/Library/Application Support/App/Parfait/...` to
  // `~/…/App/Parfait/...` so one line fits in a card row.
  const m = p.match(/Application Support\/([^/]+)\/(.+)$/);
  if (m) return `~/…/${m[1]}/${m[2]}`;
  const m2 = p.match(/~\/Library\/(?:Application Support|Caches|Logs|Preferences)\/(.+)$/);
  if (m2) return `~/…/${m2[1]}`;
  return p;
}

function pathTrunk(p: string): string {
  // Strip the filename; keep dir. Used for collapsing repeated filenames.
  const i = p.lastIndexOf("/");
  return i < 0 ? p : p.slice(0, i);
}

function groupKey(p: string): string {
  // Canonical bucket: lowercase, strip `~/…/`, drop the filename, then keep
  // only the first 2 directory segments (so we collapse sibling files in the
  // same dir and related subdirs). Examples:
  //   `~/…/DoubaoIme/Log/alog/cache/foo.alog.cache` → `log/alog`
  //   `~/…/DoubaoIme/Log/alog/log/bar.alog.hot`    → `log/alog`
  //   `~/…/DoubaoIme/ttnet/tt_net_config.config`   → `ttnet`
  //   `~/…/DoubaoIme/ttnet/version`                → `ttnet`
  const stripped = p.replace(/^~\/…\//, "").toLowerCase();
  const parts = stripped.split("/");
  if (parts.length <= 1) return stripped;
  const dirs = parts.slice(0, -1); // drop filename
  const afterApp = dirs.slice(1);  // drop app-name segment
  return afterApp.slice(0, 2).join("/");
}

function parseSizeDesc(s: string): number {
  const m = s.match(/^([\d.]+)([BKMG])$/);
  if (!m) return 0;
  const n = parseFloat(m[1]);
  const unit = { B: 1, K: 1024, M: 1024 * 1024, G: 1024 * 1024 * 1024 }[m[2]] || 1;
  return n * unit;
}

export interface CardInputs {
  meta: Meta;
  endpointsMd: string;
  localMd: string;
  stringsDir: string;
  templatePath: string;
  homeDir: string;
}

export async function renderCardPrompt(i: CardInputs): Promise<string> {
  const tpl = await readFile(i.templatePath, "utf8");
  const sdkRows = buildSdkRows(i.meta);
  const endpointRows = await buildEndpointRows(i.endpointsMd, i.stringsDir);
  const localRows = await buildLocalRows(i.localMd, i.homeDir);

  const version = [i.meta.short_version, i.meta.version]
    .filter((x) => x && String(x).trim() !== "")
    .join(" ")
    .trim();

  const appDisplay = guessAppName(i.meta);

  return tpl
    .replaceAll("{{bundle_id}}", i.meta.bundle_id || "unknown")
    .replaceAll("{{version}}", version || "unknown")
    .replaceAll("{{sandboxed}}", String(i.meta.sandboxed ?? "unknown"))
    .replaceAll(
      "{{ats}}",
      i.meta.arbitrary_loads === true ? "arbitrary" : i.meta.arbitrary_loads === false ? "strict" : "unknown",
    )
    .replaceAll("{{app_display_name}}", appDisplay)
    .replaceAll("{{sdk_rows}}", sdkRows)
    .replaceAll("{{endpoint_rows}}", endpointRows)
    .replaceAll("{{local_rows}}", localRows);
}

function guessAppName(meta: Meta): string {
  const exe = meta.main_executable ? basename(meta.main_executable) : "";
  if (/doubao/i.test(exe)) return "豆包输入法";
  if (/wetype/i.test(exe)) return "微信输入法";
  if (/wechat|weixin/i.test(exe)) return "微信";
  if (/dingding|dingtalk/i.test(exe)) return "钉钉";
  return exe || meta.bundle_id || "未知 App";
}

// Whitelisted command runner. Anything not in ALLOWED is rejected.
// Rationale: skill is read-only by contract; this is the enforcement layer.

import { spawn } from "node:child_process";

const ALLOWED = new Set([
  "strings",
  "nm",
  "otool",
  "codesign",
  "plutil",
  "file",
  "find",
  "du",
  "sqlite3",
  "xxd",
  "stat",
  "head",
  "wc",
]);

export interface RunResult {
  ok: boolean;
  code: number;
  stdout: string;
  stderr: string;
  cmd: string;
}

export interface RunOpts {
  // Maximum bytes of stdout to capture. Anything beyond is truncated.
  maxBytes?: number;
  // Wall-clock timeout in ms.
  timeoutMs?: number;
  // If true, do not throw on non-zero exit.
  allowNonZero?: boolean;
}

export async function run(
  cmd: string,
  args: string[],
  opts: RunOpts = {},
): Promise<RunResult> {
  if (!ALLOWED.has(cmd)) {
    throw new Error(
      `[shell] command not in whitelist: ${cmd}. ` +
        `Allowed: ${[...ALLOWED].join(", ")}. ` +
        `If you genuinely need this, edit lib/shell.ts deliberately.`,
    );
  }

  const maxBytes = opts.maxBytes ?? 50 * 1024 * 1024;
  const timeoutMs = opts.timeoutMs ?? 60_000;

  return new Promise((resolve, reject) => {
    const child = spawn(cmd, args, { stdio: ["ignore", "pipe", "pipe"] });

    let outLen = 0;
    let errLen = 0;
    const outChunks: Buffer[] = [];
    const errChunks: Buffer[] = [];
    let truncated = false;

    const killTimer = setTimeout(() => {
      child.kill("SIGKILL");
    }, timeoutMs);

    child.stdout.on("data", (b: Buffer) => {
      if (outLen + b.length > maxBytes) {
        const room = Math.max(0, maxBytes - outLen);
        if (room > 0) outChunks.push(b.subarray(0, room));
        outLen = maxBytes;
        truncated = true;
        // do not kill yet — let it drain naturally; just stop accumulating.
      } else {
        outChunks.push(b);
        outLen += b.length;
      }
    });

    child.stderr.on("data", (b: Buffer) => {
      if (errLen + b.length <= 2 * 1024 * 1024) {
        errChunks.push(b);
        errLen += b.length;
      }
    });

    child.on("error", (e) => {
      clearTimeout(killTimer);
      reject(e);
    });

    child.on("close", (code) => {
      clearTimeout(killTimer);
      const stdout =
        Buffer.concat(outChunks).toString("utf8") +
        (truncated ? `\n[truncated at ${maxBytes} bytes]` : "");
      const stderr = Buffer.concat(errChunks).toString("utf8");
      const result: RunResult = {
        ok: code === 0,
        code: code ?? -1,
        stdout,
        stderr,
        cmd: `${cmd} ${args.join(" ")}`,
      };
      if (!result.ok && !opts.allowNonZero) {
        reject(
          new Error(
            `[shell] ${result.cmd} exited ${code}\nstderr: ${stderr.slice(0, 4000)}`,
          ),
        );
      } else {
        resolve(result);
      }
    });
  });
}

// Convenience: stream `strings -a -n 6 <file>` line by line into a callback.
// Used by classify_strings.ts to avoid loading 50MB into memory at once.
export async function streamStrings(
  filePath: string,
  onLine: (line: string) => void,
  opts: { minLen?: number; timeoutMs?: number } = {},
): Promise<void> {
  const minLen = opts.minLen ?? 6;
  return new Promise((resolve, reject) => {
    const child = spawn("strings", ["-a", "-n", String(minLen), filePath], {
      stdio: ["ignore", "pipe", "pipe"],
    });
    const killTimer = setTimeout(
      () => child.kill("SIGKILL"),
      opts.timeoutMs ?? 120_000,
    );
    let leftover = "";
    child.stdout.on("data", (b: Buffer) => {
      const text = leftover + b.toString("utf8");
      const lines = text.split("\n");
      leftover = lines.pop() ?? "";
      for (const line of lines) onLine(line);
    });
    child.on("error", (e) => {
      clearTimeout(killTimer);
      reject(e);
    });
    child.on("close", () => {
      clearTimeout(killTimer);
      if (leftover) onLine(leftover);
      resolve();
    });
  });
}

// Helpers for path safety used by Phase 0.
export function rejectSystemPath(p: string): string | null {
  const denied = [
    "/System/",
    "/usr/libexec/",
    "/private/var/db/com.apple.",
    "/usr/bin/",
    "/usr/sbin/",
  ];
  for (const prefix of denied) {
    if (p.startsWith(prefix) || p === prefix.slice(0, -1)) {
      return `path under ${prefix} is OS-owned, not a third-party telemetry target`;
    }
  }
  return null;
}

#!/usr/bin/env bun
// Phase 2: For each binary in meta.json, run `strings`, sort lines into buckets.
// Buckets: urls, domains, paths, sql, events, keys (counts only).

import { mkdir, readFile, writeFile } from "node:fs/promises";
import { join, basename } from "node:path";
import { streamStrings } from "../lib/shell.ts";

interface FrameworkEntry {
  name: string;
  path: string;
  size_bytes: number;
}

interface Meta {
  bundle_id: string;
  frameworks: FrameworkEntry[];
  main_executable: string | null;
}

interface Buckets {
  urls: Set<string>;
  domains: Set<string>;
  paths: Set<string>;
  sql: Set<string>;
  events: Set<string>;
  keysCount: number;
  keysPreview: Set<string>;
}

function emptyBuckets(): Buckets {
  return {
    urls: new Set(),
    domains: new Set(),
    paths: new Set(),
    sql: new Set(),
    events: new Set(),
    keysCount: 0,
    keysPreview: new Set(),
  };
}

const RE_URL_G = /https?:\/\/[^\s"'<>\x00-\x1f]{4,200}/gi;
// Real TLDs only — multi-part domains like log-klink.zijieapi.com are caught
// by the {1,4} subdomain repeat plus the actual TLD at the end.
const RE_DOMAIN_G = /\b([a-z0-9-]{2,}\.){1,5}(com|cn|net|org|io|app|dev|me|info|tv|co|us|jp|kr|ai|gg|xyz|cloud|qq|gov|edu|world|run|top|wang|site|live|tech)\b(:\d+)?/gi;
const RE_DOMAIN_STRICT = /^([a-z0-9-]+\.){1,5}[a-z]{2,}(:\d+)?$/i;
const RE_PATH = /(^|\s)(\/(Library|System|Applications|var|tmp|Users)\/|~\/)/;
const RE_SQL = /\b(CREATE\s+TABLE|INSERT\s+INTO|SELECT\s+.+FROM|UPDATE\s+\w+\s+SET|DROP\s+TABLE)\b/i;
const RE_EVENT_TOKEN = /\b([a-z][a-z0-9_]{8,80})\b/g;
const RE_KEY = /^[A-Za-z0-9+/=]{24,}$|^[A-Fa-f0-9]{24,}$/;
const RE_NOISE_PREFIX = /^(__|@_|_OBJC_|\.LFB|\.LBE|@OBJC_|libobjc|0x[0-9a-f]+|\([0-9])/;

const TLD_OK = /\.(com|cn|net|org|io|app|dev|me|info|tv|co|us|jp|kr|ai|gg|xyz|cloud|qq|gov|edu|world|run|top|wang|site|live|tech)(:\d+)?$/i;

function classify(line: string, b: Buckets) {
  const t = line.replace(/\x00/g, "").trim();
  if (t.length < 6) return;
  if (RE_NOISE_PREFIX.test(t)) return;

  // For ANY length line: multi-scan for URLs and domain-shaped tokens.
  // The key bug we are working around: Rust binaries cram many literals into
  // one continuous .rodata blob, so `strings` can return one 50KB "line" that
  // contains every endpoint, every event name, every error message.
  let foundSomething = false;

  // URLs (multi-match)
  const urlMatches = t.match(RE_URL_G);
  if (urlMatches) {
    for (const u of urlMatches) {
      b.urls.add(u);
      const hostMatch = u.match(/https?:\/\/([^/:]+)/i);
      if (hostMatch) b.domains.add(hostMatch[1].toLowerCase());
    }
    foundSomething = true;
  }

  // Domain-shaped tokens (multi-match) — only against a curated TLD set so
  // common version strings don't pollute.
  const domMatches = t.match(RE_DOMAIN_G);
  if (domMatches) {
    for (const d of domMatches) {
      const dn = d.toLowerCase();
      // Skip obvious file extensions
      if (/\.(swift|m|mm|cpp|h|hpp|js|ts|json|plist|xib|nib|po)$/.test(dn))
        continue;
      b.domains.add(dn);
    }
    foundSomething = true;
  }

  // Short-line specific buckets — only attempt full-line classification when
  // the line is short enough to be a single semantic token.
  if (t.length <= 300) {
    if (RE_SQL.test(t)) {
      b.sql.add(t.slice(0, 300));
      foundSomething = true;
    }
    if (RE_PATH.test(t)) {
      b.paths.add(t.slice(0, 300));
      foundSomething = true;
    }
    if (
      t.length < 100 &&
      RE_DOMAIN_STRICT.test(t) &&
      TLD_OK.test(t) &&
      !b.domains.has(t.toLowerCase())
    ) {
      b.domains.add(t.toLowerCase());
      foundSomething = true;
    }
    if (RE_KEY.test(t)) {
      b.keysCount++;
      if (b.keysPreview.size < 8) {
        b.keysPreview.add(`${t.slice(0, 12)}…(${t.length}ch)`);
      }
      foundSomething = true;
    }
  }

  // Event-name-shaped tokens (multi-match across the line). Only for lines
  // that aren't dominated by URLs (avoid grabbing fragments of paths).
  // Cap the harvest per line to avoid pathological blobs.
  let eventCount = 0;
  for (const m of t.matchAll(RE_EVENT_TOKEN)) {
    const tok = m[1];
    if (!tok.includes("_")) continue;
    if (tok.length < 9) continue;
    // Skip obvious noise tokens
    if (/^(get_|set_|src_|dst_|tmp_|str_|out_|err_|new_|old_|ret_|res_|val_)/.test(tok))
      continue;
    if (b.events.size > 5000) break; // hard cap per binary
    b.events.add(tok);
    if (++eventCount > 500) break;
  }

  void foundSomething;
}

function bucketToMd(title: string, items: Iterable<string>): string {
  const arr = [...items].sort();
  if (arr.length === 0) return `# ${title}\n\n_(empty)_\n`;
  const head = `# ${title}  (${arr.length})\n\n`;
  const body = arr.map((s) => `- \`${s.replace(/`/g, "ʹ")}\``).join("\n");
  return head + body + "\n";
}

function keysBucketMd(b: Buckets): string {
  const lines = [
    `# keys  (count=${b.keysCount})`,
    "",
    "Key-shaped strings are NOT stored verbatim. Below are first-12-char previews.",
    "",
  ];
  for (const p of b.keysPreview) lines.push(`- \`${p}\``);
  return lines.join("\n") + "\n";
}

async function processOne(
  binaryPath: string,
  outDir: string,
  maxBytes: number,
): Promise<{ name: string; counts: Record<string, number> } | null> {
  // Skip giant or missing binaries
  const name = basename(binaryPath);
  const buckets = emptyBuckets();

  let lineCount = 0;
  try {
    await streamStrings(
      binaryPath,
      (line) => {
        lineCount++;
        if (lineCount > 2_000_000) return; // hard cap
        classify(line, buckets);
      },
      { minLen: 6, timeoutMs: 180_000 },
    );
  } catch (e) {
    console.error(`skip ${name}: ${(e as Error).message}`);
    return null;
  }

  const safe = name.replace(/[^A-Za-z0-9._-]+/g, "_");
  await writeFile(join(outDir, `${safe}.urls.md`), bucketToMd("urls", buckets.urls));
  await writeFile(
    join(outDir, `${safe}.domains.md`),
    bucketToMd("domains", buckets.domains),
  );
  await writeFile(join(outDir, `${safe}.paths.md`), bucketToMd("paths", buckets.paths));
  await writeFile(join(outDir, `${safe}.sql.md`), bucketToMd("sql", buckets.sql));
  await writeFile(
    join(outDir, `${safe}.events.md`),
    bucketToMd("events", buckets.events),
  );
  await writeFile(join(outDir, `${safe}.keys.md`), keysBucketMd(buckets));

  return {
    name,
    counts: {
      urls: buckets.urls.size,
      domains: buckets.domains.size,
      paths: buckets.paths.size,
      sql: buckets.sql.size,
      events: buckets.events.size,
      keys: buckets.keysCount,
    },
  };
}

async function main() {
  const args = process.argv.slice(2);
  const metaPath = args.find((a) => !a.startsWith("--"));
  const outIdx = args.indexOf("--out");
  const out = outIdx >= 0 ? args[outIdx + 1] : "";
  if (!metaPath || !out) {
    console.error("usage: classify_strings.ts <meta.json> --out <strings-dir>");
    process.exit(2);
  }

  const meta: Meta = JSON.parse(await readFile(metaPath, "utf8"));
  await mkdir(out, { recursive: true });

  const targets: string[] = [];
  if (meta.main_executable) targets.push(meta.main_executable);
  for (const fw of meta.frameworks) {
    if (fw.size_bytes > 200 * 1024 * 1024) continue;
    targets.push(fw.path);
  }

  const summary: Array<{ name: string; counts: Record<string, number> }> = [];
  for (const t of targets) {
    const r = await processOne(t, out, 200 * 1024 * 1024);
    if (r) summary.push(r);
  }

  const idx = [
    "# strings index",
    "",
    "| binary | urls | domains | paths | sql | events | keys |",
    "|---|---:|---:|---:|---:|---:|---:|",
    ...summary.map(
      (s) =>
        `| ${s.name} | ${s.counts.urls} | ${s.counts.domains} | ${s.counts.paths} | ${s.counts.sql} | ${s.counts.events} | ${s.counts.keys} |`,
    ),
    "",
  ].join("\n");
  await writeFile(join(out, "INDEX.md"), idx);
  console.log(`classified ${summary.length} binaries -> ${out}`);
}

main().catch((e) => {
  console.error(e instanceof Error ? e.message : String(e));
  process.exit(1);
});

#!/usr/bin/env bun
// Phase 5: Inventory the on-disk surface of an app by bundle id.
// Lists files only. SQLite gets `.schema` (no row data). MMKV/JSON gets size only.

import { mkdir, readdir, stat, writeFile } from "node:fs/promises";
import { homedir } from "node:os";
import { join, dirname } from "node:path";
import { run } from "../lib/shell.ts";

interface FoundFile {
  path: string;
  size: number;
  kind: "sqlite" | "mmkv" | "json" | "plist" | "log" | "binary" | "other";
  schema?: string;
}

const HOME = homedir();

function bundlePrefix(bundleId: string): string {
  // group containers usually look like group.<vendor>.<...>
  const parts = bundleId.split(".");
  if (parts.length >= 2) return parts.slice(0, 2).join(".");
  return bundleId;
}

const ROOTS = (bundleId: string) => [
  join(HOME, "Library", "Application Support", bundleId),
  join(HOME, "Library", "Caches", bundleId),
  join(HOME, "Library", "Logs", bundleId),
  join(HOME, "Library", "Containers", bundleId, "Data"),
  join(HOME, "Library", "Preferences", `${bundleId}.plist`),
];

async function listGroupContainers(prefix: string): Promise<string[]> {
  const root = join(HOME, "Library", "Group Containers");
  try {
    const entries = await readdir(root);
    return entries
      .filter((e) => e.startsWith(`group.${prefix}`) || e.includes(prefix))
      .map((e) => join(root, e));
  } catch {
    return [];
  }
}

function classify(p: string): FoundFile["kind"] {
  if (/\.(sqlite|sqlite3|db)$/i.test(p)) return "sqlite";
  if (/\.(mmkv|crc)$/i.test(p)) return "mmkv";
  if (/\.json$/i.test(p)) return "json";
  if (/\.plist$/i.test(p)) return "plist";
  if (/\.(log|txt)$/i.test(p) || /\/Logs\//.test(p)) return "log";
  if (/\.(dylib|so|framework)$/i.test(p)) return "binary";
  return "other";
}

async function walk(
  root: string,
  out: FoundFile[],
  maxDepth = 6,
  depth = 0,
): Promise<void> {
  if (depth > maxDepth) return;
  let entries: string[];
  try {
    entries = await readdir(root);
  } catch {
    return;
  }
  for (const e of entries) {
    if (e.startsWith(".")) continue;
    const p = join(root, e);
    let s;
    try {
      s = await stat(p);
    } catch {
      continue;
    }
    if (s.isDirectory()) {
      await walk(p, out, maxDepth, depth + 1);
    } else if (s.isFile()) {
      out.push({ path: p, size: s.size, kind: classify(p) });
    }
  }
}

async function dumpSchema(sqlitePath: string): Promise<string> {
  try {
    const r = await run("sqlite3", [sqlitePath, ".schema"], {
      timeoutMs: 15_000,
      allowNonZero: true,
    });
    return r.stdout.trim() || "(no tables)";
  } catch (e) {
    return `(failed: ${(e as Error).message.slice(0, 120)})`;
  }
}

function fmtSize(n: number): string {
  if (n < 1024) return `${n}B`;
  if (n < 1024 * 1024) return `${(n / 1024).toFixed(1)}K`;
  if (n < 1024 * 1024 * 1024) return `${(n / 1024 / 1024).toFixed(1)}M`;
  return `${(n / 1024 / 1024 / 1024).toFixed(2)}G`;
}

async function main() {
  const args = process.argv.slice(2);
  const positional = args.filter((a) => !a.startsWith("--"));
  const bundle = positional[0];
  const aliases = positional.slice(1).filter(Boolean);
  const outIdx = args.indexOf("--out");
  const out = outIdx >= 0 ? args[outIdx + 1] : "";
  if (!bundle || !out) {
    console.error(
      "usage: inventory_data.ts <bundle.id> [<alias-name> ...] --out <local_data.md>",
    );
    process.exit(2);
  }

  // Apps don't always store data under bundle id — Input Methods commonly use
  // CFBundleExecutable as the directory name (e.g. ~/Library/Application
  // Support/DoubaoIme/ instead of .../com.bytedance.inputmethod.doubaoime/).
  // Accept aliases (typically the executable name + the last bundle segment)
  // to cover both layouts.
  const lastSeg = bundle.split(".").pop() || "";
  const allKeys = [bundle, ...aliases, lastSeg].filter(Boolean);
  const seen = new Set<string>();
  const rootsRaw: string[] = [];
  for (const k of allKeys) {
    for (const r of ROOTS(k)) rootsRaw.push(r);
  }
  const groupRoots = await listGroupContainers(bundlePrefix(bundle));
  for (const r of [...rootsRaw, ...groupRoots]) {
    if (!seen.has(r)) {
      seen.add(r);
    }
  }
  const roots = [...seen];

  const files: FoundFile[] = [];
  for (const r of roots) {
    let isFile = false;
    try {
      const s = await stat(r);
      if (s.isFile()) {
        files.push({ path: r, size: s.size, kind: classify(r) });
        isFile = true;
      }
    } catch {
      continue;
    }
    if (!isFile) await walk(r, files);
  }

  // Schema for SQLite — capped at 12 to avoid runaway
  let schemaCount = 0;
  for (const f of files) {
    if (f.kind === "sqlite" && schemaCount < 12) {
      f.schema = await dumpSchema(f.path);
      schemaCount++;
    }
  }

  // Group by parent dir
  const byDir = new Map<string, FoundFile[]>();
  for (const f of files) {
    const d = dirname(f.path);
    if (!byDir.has(d)) byDir.set(d, []);
    byDir.get(d)!.push(f);
  }
  const dirsSorted = [...byDir.keys()].sort();

  const lines: string[] = [
    `# local data — ${bundle}`,
    "",
    `Found ${files.length} files across ${dirsSorted.length} directories.`,
    "",
  ];

  if (files.length === 0) {
    lines.push("_(no on-disk surface found for this bundle id)_\n");
  } else {
    for (const d of dirsSorted) {
      lines.push(`## ${d.replace(HOME, "~")}`);
      lines.push("");
      lines.push("| size | kind | name |");
      lines.push("|---:|---|---|");
      for (const f of byDir.get(d)!.sort((a, b) => b.size - a.size)) {
        const name = f.path.slice(d.length + 1);
        lines.push(`| ${fmtSize(f.size)} | ${f.kind} | \`${name}\` |`);
      }
      lines.push("");
    }
    const sqlites = files.filter((f) => f.schema);
    if (sqlites.length) {
      lines.push("## SQLite schemas");
      lines.push("");
      for (const f of sqlites) {
        lines.push(`### \`${f.path.replace(HOME, "~")}\``);
        lines.push("");
        lines.push("```sql");
        lines.push(f.schema!.split("\n").slice(0, 80).join("\n"));
        lines.push("```");
        lines.push("");
      }
    }
  }

  await mkdir(dirname(out), { recursive: true });
  await writeFile(out, lines.join("\n"));
  console.log(`inventory: ${files.length} files -> ${out}`);
}

main().catch((e) => {
  console.error(e instanceof Error ? e.message : String(e));
  process.exit(1);
});