Joelclaw System Check

interface:
  icon_small: "./assets/small-logo.svg"
  icon_large: "./assets/large-logo.png"

<svg width="16" height="16" viewBox="0 0 512 512" xmlns="http://www.w3.org/2000/svg" role="img" aria-label="JoelClaw icon">
  <defs>
    <clipPath id="circle-clip">
      <circle cx="256" cy="256" r="248" />
    </clipPath>
  </defs>
  <circle cx="256" cy="256" r="248" fill="#0a0a0a" />
  <g clip-path="url(#circle-clip)">
    <path fill="#ff1493" d="M175.656 22.375l-48.47 82.094c-23.017 4.384-43.547 11.782-60.124 22.374-24.436 15.613-40.572 37.414-45.5 67.875-4.79 29.62 1.568 68.087 24.125 116.093 93.162 22.88 184.08-10.908 257.25-18.813 37.138-4.012 71.196-.898 96.344 22.97 22.33 21.19 36.21 56.808 41.908 113.436 29.246-35.682 44.538-69.065 49.343-99.594 5.543-35.207-2.526-66.97-20.31-95.593-8.52-13.708-19.368-26.618-32-38.626l14.217-33-41.218 10.625c-8.637-6.278-17.765-12.217-27.314-17.782l-7.03-59.782-38.157 37.406c-12.418-5.186-25.184-9.804-38.158-13.812l-8.375-71.28-57.625 56.5c-9.344-1.316-18.625-2.333-27.812-2.97l-31.094-78.125zM222 325.345c-39.146 7.525-82.183 14.312-127.156 11.686 47.403 113.454 207.056 224.082 260.125 87-101.18 33.84-95.303-49.595-132.97-98.686z" />
  </g>
</svg>

#!/bin/bash
# joelclaw system health check — outputs concise machine-friendly summary for agents
set -o pipefail

TOTAL=0
COUNT=0
ISSUES=()

REPO_DIR="${HOME}/Code/joelhooks/joelclaw"
SBUS_DIR="${REPO_DIR}/packages/system-bus"

check() {
  local name="$1" score="$2" detail="$3"
  TOTAL=$((TOTAL + score))
  COUNT=$((COUNT + 1))

  local icon="OK"
  [[ $score -le 5 ]] && icon="WARN"
  [[ $score -le 3 ]] && icon="FAIL"

  printf "%-5s %-28s %2d/10  %s\n" "$icon" "$name" "$score" "$detail"
  [[ $score -lt 7 ]] && ISSUES+=("$name ($score/10): $detail")
}

json_get() {
  local json="$1"
  local expr="$2"
  echo "$json" | jq -r "$expr" 2>/dev/null
}

echo "===================================================="
echo "  joelclaw system health -- $(date '+%Y-%m-%d %H:%M')"
echo "===================================================="
echo ""

# -- kubeconfig self-heal ----------------------------------------------
# Colima remaps k8s API (6443) and Talos API (50000) to random host ports
# on container restart. If kubectl can't reach the API, regenerate kubeconfig.
if ! kubectl cluster-info >/dev/null 2>&1; then
  if command -v talosctl >/dev/null 2>&1 && [[ -f ~/.talos/config ]]; then
    talosctl --talosconfig ~/.talos/config --nodes 127.0.0.1 kubeconfig --force >/dev/null 2>&1 || true
    # Find and use the new context
    NEW_CTX=$(kubectl config get-contexts -o name 2>/dev/null | grep 'joelclaw' | head -1)
    [[ -n "$NEW_CTX" ]] && kubectl config use-context "$NEW_CTX" >/dev/null 2>&1 || true
  fi
fi

# -- status snapshot ---------------------------------------------------
STATUS_JSON=$(joelclaw status --json 2>/dev/null || true)
STATUS_OK=$(json_get "$STATUS_JSON" '.ok // false')

# -- k8s cluster -------------------------------------------------------
if [[ "$STATUS_OK" == "true" ]]; then
  K8S_OK=$(json_get "$STATUS_JSON" '.result.k8s.ok // false')
  K8S_DETAIL=$(json_get "$STATUS_JSON" '.result.k8s.detail // "k8s detail unavailable"')
  if [[ "$K8S_OK" == "true" ]]; then
    check "k8s cluster" 10 "$K8S_DETAIL"
  else
    check "k8s cluster" 4 "$K8S_DETAIL"
  fi
else
  check "k8s cluster" 1 "joelclaw status unavailable"
fi

# -- pds health --------------------------------------------------------
# PDS Docker mapping: container:3000 → host:9627 (NodePort per ADR-0148)
PDS_PORT=9627
PDS_REPO="joel.pds.panda.tail7af24.ts.net"
PDS_VER=$(curl -sf "http://localhost:${PDS_PORT}/xrpc/_health" 2>/dev/null | jq -r '.version // empty' 2>/dev/null)
if [[ -n "$PDS_VER" ]]; then
  PDS_COLLS=$(curl -sf "http://localhost:${PDS_PORT}/xrpc/com.atproto.repo.describeRepo?repo=${PDS_REPO}" 2>/dev/null | jq -r '.collections | length // 0' 2>/dev/null)
  check "pds" 10 "v${PDS_VER}, ${PDS_COLLS} collections"
else
  PDS_POD=$(kubectl get pods -n joelclaw --no-headers 2>/dev/null | awk '/bluesky-pds/ && /Running/ {c++} END {print c+0}')
  if [[ "$PDS_POD" -gt 0 ]]; then
    check "pds" 4 "pod running but :${PDS_PORT} unreachable -- check Docker port mapping"
  else
    check "pds" 1 "pod not running -- run: kubectl rollout restart deployment/bluesky-pds -n joelclaw"
  fi
fi

# -- worker health -----------------------------------------------------
INNGEST_STATUS_JSON=$(joelclaw inngest status --json 2>/dev/null || true)
INNGEST_OK=$(json_get "$INNGEST_STATUS_JSON" '.ok // false')
if [[ "$INNGEST_OK" == "true" ]]; then
  WORKER_REACHABLE=$(json_get "$INNGEST_STATUS_JSON" '.result.worker.reachable // false')
  WORKER_FN_COUNT=$(json_get "$INNGEST_STATUS_JSON" '.result.worker.functionCount // 0')
  WORKER_ROLE=$(json_get "$INNGEST_STATUS_JSON" '.result.worker.role // "unknown"')
  if [[ "$WORKER_REACHABLE" == "true" && "$WORKER_FN_COUNT" -ge 55 ]]; then
    check "worker" 10 "${WORKER_FN_COUNT} functions, role=${WORKER_ROLE}"
  elif [[ "$WORKER_REACHABLE" == "true" ]]; then
    check "worker" 7 "reachable, ${WORKER_FN_COUNT} functions"
  else
    check "worker" 1 "worker not reachable"
  fi
else
  check "worker" 1 "joelclaw inngest status unavailable"
fi

# -- inngest server ----------------------------------------------------
if [[ "$STATUS_OK" == "true" ]]; then
  SERVER_OK=$(json_get "$STATUS_JSON" '.result.server.ok // false')
  if [[ "$SERVER_OK" == "true" ]]; then
    check "inngest server" 10 "responding on :8288"
  else
    SERVER_DETAIL=$(json_get "$STATUS_JSON" '.result.server.detail // "unreachable"')
    check "inngest server" 1 "$SERVER_DETAIL"
  fi
else
  check "inngest server" 1 "status check unavailable"
fi

# -- redis / gateway ---------------------------------------------------
GW_JSON=$(joelclaw gateway status --json 2>/dev/null || true)
GW_OK=$(json_get "$GW_JSON" '.ok // false')
if [[ "$GW_OK" == "true" ]]; then
  REDIS_STATE=$(json_get "$GW_JSON" '.result.redis // "error"')
  SESSION_COUNT=$(json_get "$GW_JSON" '.result.activeSessions | length // 0')
  PENDING_TOTAL=$(echo "$GW_JSON" | jq '[.result.activeSessions[]?.pending // 0] | add // 0' 2>/dev/null)
  if [[ "$REDIS_STATE" == "connected" ]]; then
    if [[ "$PENDING_TOTAL" -gt 25 ]]; then
      check "redis/gateway" 7 "redis=${REDIS_STATE}, sessions=${SESSION_COUNT}, pending=${PENDING_TOTAL}"
    else
      check "redis/gateway" 10 "redis=${REDIS_STATE}, sessions=${SESSION_COUNT}, pending=${PENDING_TOTAL}"
    fi
  else
    check "redis/gateway" 2 "redis=${REDIS_STATE}"
  fi
else
  check "redis/gateway" 1 "joelclaw gateway status unavailable"
fi

# -- typesense / otel --------------------------------------------------
TS_HEALTH=$(curl -sf http://localhost:8108/health 2>/dev/null | jq -r '.ok // false' 2>/dev/null)
OTEL_JSON=$(joelclaw otel stats --hours 1 --json 2>/dev/null || true)
OTEL_OK=$(json_get "$OTEL_JSON" '.ok // false')
if [[ "$TS_HEALTH" == "true" && "$OTEL_OK" == "true" ]]; then
  OTEL_TOTAL=$(json_get "$OTEL_JSON" '.result.total // 0')
  OTEL_ERRORS=$(json_get "$OTEL_JSON" '.result.errors // 0')
  OTEL_RATE=$(json_get "$OTEL_JSON" '.result.errorRate // 0')
  check "typesense/otel" 10 "typesense=ok, events=${OTEL_TOTAL}, errors=${OTEL_ERRORS}, rate=${OTEL_RATE}"
elif [[ "$TS_HEALTH" == "true" ]]; then
  check "typesense/otel" 6 "typesense=ok, otel query degraded"
else
  check "typesense/otel" 1 "typesense unavailable on :8108"
fi

# -- tests -------------------------------------------------------------
if [[ -d "$SBUS_DIR" ]]; then
  # Run test files in isolated Bun processes. Several legacy system-bus tests
  # monkey-patch globals or use `mock.module`, which leaks across files in a
  # single `bun test` process and creates false red health checks.
  TEST_PASS=0
  TEST_FAIL=0
  TEST_FILES=$(cd "$SBUS_DIR" && find src -name '*.test.ts' | sort)

  if [[ -n "$TEST_FILES" ]]; then
    while IFS= read -r TEST_FILE; do
      [[ -z "$TEST_FILE" ]] && continue
      TEST_STATUS=0
      TEST_OUTPUT=$(cd "$SBUS_DIR" && bun test "$TEST_FILE" 2>&1) || TEST_STATUS=$?
      FILE_PASS=$(echo "$TEST_OUTPUT" | grep -oE '[0-9]+ pass' | tail -1 | awk '{print $1}')
      FILE_FAIL=$(echo "$TEST_OUTPUT" | grep -oE '[0-9]+ fail' | tail -1 | awk '{print $1}')
      [[ -z "$FILE_PASS" ]] && FILE_PASS=0
      [[ -z "$FILE_FAIL" ]] && FILE_FAIL=0
      if [[ "$TEST_STATUS" -ne 0 && "$FILE_FAIL" == "0" ]]; then
        FILE_FAIL=1
      fi
      TEST_PASS=$((TEST_PASS + FILE_PASS))
      TEST_FAIL=$((TEST_FAIL + FILE_FAIL))
    done <<< "$TEST_FILES"
  fi

  if [[ "$TEST_FAIL" == "0" && "$TEST_PASS" -gt 0 ]]; then
    check "tests" 10 "${TEST_PASS} pass / ${TEST_FAIL} fail"
  elif [[ "$TEST_FAIL" -gt 0 ]]; then
    check "tests" 3 "${TEST_PASS} pass / ${TEST_FAIL} fail"
  else
    check "tests" 2 "tests did not report pass/fail counts"
  fi
else
  check "tests" 1 "missing ${SBUS_DIR}"
fi

# -- tsc ---------------------------------------------------------------
if [[ -d "$SBUS_DIR" ]]; then
  TSC_OUTPUT=$(cd "$SBUS_DIR" && bunx tsc --noEmit 2>&1)
  TSC_ERRORS=$(echo "$TSC_OUTPUT" | grep -c 'error TS')
  if [[ "$TSC_ERRORS" == "0" ]]; then
    check "tsc" 10 "clean"
  else
    check "tsc" 3 "${TSC_ERRORS} type errors"
  fi
else
  check "tsc" 1 "missing ${SBUS_DIR}"
fi

# -- repo sync ---------------------------------------------------------
if [[ -d "${REPO_DIR}/.git" ]]; then
  LOCAL_SHA=$(git -C "$REPO_DIR" rev-parse --short HEAD 2>/dev/null)
  git -C "$REPO_DIR" fetch origin main --quiet >/dev/null 2>&1 || true

  if git -C "$REPO_DIR" rev-parse --verify origin/main >/dev/null 2>&1; then
    COUNTS=$(git -C "$REPO_DIR" rev-list --left-right --count HEAD...origin/main 2>/dev/null)
    AHEAD=$(echo "$COUNTS" | awk '{print $1+0}')
    BEHIND=$(echo "$COUNTS" | awk '{print $2+0}')

    if [[ "$AHEAD" == "0" && "$BEHIND" == "0" ]]; then
      check "repo sync" 10 "HEAD=${LOCAL_SHA}, in sync with origin/main"
    elif [[ "$BEHIND" -gt 0 && "$AHEAD" == "0" ]]; then
      check "repo sync" 6 "HEAD=${LOCAL_SHA}, behind origin/main by ${BEHIND}"
    else
      check "repo sync" 7 "HEAD=${LOCAL_SHA}, ahead=${AHEAD}, behind=${BEHIND}"
    fi
  else
    check "repo sync" 5 "origin/main not available locally"
  fi
else
  check "repo sync" 1 "repo not found at ${REPO_DIR}"
fi

# -- memory pipeline ---------------------------------------------------
MEM_JSON=$(joelclaw inngest memory-health --hours 24 --json 2>/dev/null || true)
MEM_PARSE_OK=$(echo "$MEM_JSON" | jq -r 'if (.result | type) == "object" then "true" else "false" end' 2>/dev/null)
if [[ "$MEM_PARSE_OK" == "true" ]]; then
  MEM_HEALTH_OK=$(json_get "$MEM_JSON" '.result.ok // false')
  MEM_COUNT=$(json_get "$MEM_JSON" '.result.memory.count // 0')
  MEM_FAILED=$(json_get "$MEM_JSON" '.result.runs.failedMemoryRuns // 0')
  MEM_ACTIVE=$(json_get "$MEM_JSON" '.result.runs.activeMemoryRuns // 0')
  if [[ "$MEM_HEALTH_OK" == "true" ]]; then
    check "memory pipeline" 10 "count=${MEM_COUNT}, failed=${MEM_FAILED}, active=${MEM_ACTIVE}"
  else
    check "memory pipeline" 4 "degraded: count=${MEM_COUNT}, failed=${MEM_FAILED}, active=${MEM_ACTIVE}"
  fi
else
  check "memory pipeline" 2 "joelclaw inngest memory-health unavailable"
fi

# -- pi-tools ----------------------------------------------------------
PI_DEPS_OK=0
PI_TOOLS_NODE_MODULES="$HOME/.pi/agent/git/github.com/joelhooks/pi-tools/node_modules"
[[ -f "$PI_TOOLS_NODE_MODULES/@sinclair/typebox/package.json" ]] && PI_DEPS_OK=$((PI_DEPS_OK + 1))
if [[ -d "$PI_TOOLS_NODE_MODULES/@earendil-works/pi-coding-agent" || -d "$PI_TOOLS_NODE_MODULES/@mariozechner/pi-coding-agent" ]]; then
  PI_DEPS_OK=$((PI_DEPS_OK + 1))
fi
if [[ -d "$PI_TOOLS_NODE_MODULES/@earendil-works/pi-tui" || -d "$PI_TOOLS_NODE_MODULES/@mariozechner/pi-tui" ]]; then
  PI_DEPS_OK=$((PI_DEPS_OK + 1))
fi
if [[ "$PI_DEPS_OK" == "3" ]]; then
  check "pi-tools" 10 "all deps present"
else
  check "pi-tools" 3 "${PI_DEPS_OK}/3 deps present"
fi

# -- git config --------------------------------------------------------
GIT_NAME=$(git config --global user.name 2>/dev/null)
GIT_EMAIL=$(git config --global user.email 2>/dev/null)
if [[ -n "$GIT_NAME" && -n "$GIT_EMAIL" ]]; then
  check "git config" 10 "${GIT_NAME} <${GIT_EMAIL}>"
else
  check "git config" 2 "missing global user.name/user.email"
fi

# -- active loops ------------------------------------------------------
LOOP_JSON=$(joelclaw loop list --json 2>/dev/null || true)
LOOP_OK=$(json_get "$LOOP_JSON" '.ok // false')
if [[ "$LOOP_OK" == "true" ]]; then
  LOOP_COUNT=$(json_get "$LOOP_JSON" '.result.count // 0')
  check "active loops" 10 "${LOOP_COUNT} active"
else
  check "active loops" 4 "unable to query loop list"
fi

# -- disk --------------------------------------------------------------
DISK_FREE=$(df -h / | tail -1 | awk '{print $4}')
DISK_PCT=$(df -h / | tail -1 | awk '{gsub(/%/,"",$5); print $5}')
LOOP_TMP=$(du -sm /tmp/agent-loop/ 2>/dev/null | awk '{print $1}' || echo 0)
if [[ "$DISK_PCT" -lt 80 ]]; then
  SCORE=10
  [[ "$LOOP_TMP" -gt 2000 ]] && SCORE=8
  check "disk" "$SCORE" "${DISK_FREE} free (${DISK_PCT}% used), loop tmp=${LOOP_TMP}MB"
else
  check "disk" 4 "${DISK_FREE} free (${DISK_PCT}% used)"
fi

# -- gogcli ------------------------------------------------------------
GOG_KP=$(secrets lease gog_keyring_password --ttl 15m 2>/dev/null || echo "")
if [[ -n "$GOG_KP" ]]; then
  GOG_LIST=$(GOG_KEYRING_PASSWORD="$GOG_KP" gog auth list --check 2>&1)
  GOG_ACCT=$(echo "$GOG_LIST" | grep -c "true")
  if [[ "$GOG_ACCT" -gt 0 ]]; then
    check "gogcli" 10 "${GOG_ACCT} account(s) authed"
  else
    check "gogcli" 4 "auth configured, token check failed"
  fi
else
  GOG_LIST=$(gog auth list 2>&1)
  if echo "$GOG_LIST" | grep -q "@"; then
    check "gogcli" 5 "accounts detected but keyring password unavailable"
  else
    check "gogcli" 1 "not configured"
  fi
fi

# -- stale tests -------------------------------------------------------
# Only flag __tests__/ at the package root (loop artifact). Subdirs like
# src/inngest/functions/__tests__/ are legitimate test locations.
STALE_ROOT_TESTS=""
[[ -d "$SBUS_DIR/__tests__" ]] && STALE_ROOT_TESTS="__tests__/ exists at package root"
STALE_ACC=$(find "$SBUS_DIR/src" -name "*.acceptance.test.ts" 2>/dev/null | wc -l | tr -d ' ')
if [[ -z "$STALE_ROOT_TESTS" && "$STALE_ACC" == "0" ]]; then
  check "stale tests" 10 "clean"
else
  DETAIL=""
  [[ -n "$STALE_ROOT_TESTS" ]] && DETAIL="$STALE_ROOT_TESTS"
  [[ "$STALE_ACC" -gt 0 ]] && DETAIL="${DETAIL:+${DETAIL}, }${STALE_ACC} acceptance tests"
  check "stale tests" 4 "${DETAIL}"
fi

# -- summary -----------------------------------------------------------
echo ""
echo "===================================================="
if [[ "$COUNT" -gt 0 ]]; then
  PRECISE=$(awk "BEGIN { printf \"%.1f\", ${TOTAL}/${COUNT} }")
else
  PRECISE="0.0"
fi
printf "  OVERALL: %s/10  (%d checks)\n" "$PRECISE" "$COUNT"
echo "===================================================="

if [[ ${#ISSUES[@]} -gt 0 ]]; then
  echo ""
  echo "Issues:"
  for issue in "${ISSUES[@]}"; do
    echo "  - $issue"
  done
fi

echo ""
echo "Run: ~/Code/joelhooks/joelclaw/skills/joelclaw-system-check/scripts/health.sh"