Metric Validation Harness

__pycache__/
*.pyc

{
  "metric_cmd": "",
  "baseline_cmd": "",
  "corpus_dir": "",
  "labels_csv": "",
  "declared_min": "",
  "declared_max": "",
  "_setup_instructions": {
    "metric_cmd": "Command that computes YOUR candidate metric. It receives a file/dir path as its last argument and must print exactly ONE number to stdout (e.g. 'python3 /abs/path/mymetric.py'). Leave empty to use the bundled AST-node example metric. No spaces in the path or command.",
    "baseline_cmd": "Trivial baseline for the discriminant and predictive checks. Same contract as metric_cmd. Empty = bundled LOC counter.",
    "corpus_dir": "Directory of artifacts the property checks (determinism, invariance, monotonicity) iterate over. Empty = bundled fixtures.",
    "labels_csv": "CSV with header 'path[,outcome][,accepted]'. 'path' is relative to the CSV's own directory; 'outcome' is 0/1 for predictive validity; 'accepted' is an existing trusted measure for convergent validity. Empty = bundled corpus.csv.",
    "declared_min": "Optional lower bound the metric claims; the robustness check enforces it on edge inputs. Empty = unchecked.",
    "declared_max": "Optional upper bound the metric claims. Empty = unchecked."
  }
}

{
  "version": "0.1.0",
  "organization": "dot-skills",
  "technology": "Metric Validation Harness",
  "discipline": "composition",
  "type": "verification",
  "date": "May 2026",
  "abstract": "Read-only verification harness that empirically tests a candidate software metric against the properties the deterministic-metric-design skill says to prove. Point it at any metric (a command that takes a path and prints one number) plus a corpus, and it runs experiments: determinism across runs and hash seeds, invariance to cosmetic edits (which doubles as an anti-gaming probe), monotonicity under construct-increasing edits plus discrimination, robustness and range on edge inputs, near-linear tractability, and construct validity (Spearman convergent, discriminant vs LOC, predictive AUC, and lift over a baseline) using pure-stdlib statistics. Reports PASS/FAIL per property, each mapped to the design-skill category it checks. Ships an example metric, fixtures, and a self-test; safe to run unsupervised.",
  "references": [
    "https://psycnet.apa.org/doi/10.1037/h0046016",
    "https://doi.org/10.1016/j.patrec.2005.10.010",
    "https://doi.org/10.1109/32.6178",
    "https://arxiv.org/abs/1803.04585",
    "https://www.acm.org/publications/policies/artifact-review-and-badging-current"
  ]
}

#!/usr/bin/env bash
# check-determinism.sh — same input must yield the same number across runs and hash seeds.
# Maps to deterministic-metric-design: det-make-the-metric-a-pure-function,
# det-pin-iteration-and-tie-break-order.
set -euo pipefail

HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck source=lib/load-config.sh
source "$HERE/lib/load-config.sh"

pass=0
fail=0

while IFS= read -r f; do
  [[ -n "$f" ]] || continue
  name="$(basename "$f")"
  a="$(metric_of "$f")" || { echo "FAIL: $name — metric did not run"; fail=$((fail + 1)); continue; }
  b="$(metric_of "$f")" || { echo "FAIL: $name — metric did not run (2nd pass)"; fail=$((fail + 1)); continue; }
  c="$(PYTHONHASHSEED=0 metric_of "$f")" || c="ERR"
  d="$(PYTHONHASHSEED=1 metric_of "$f")" || d="ERR"
  if num_eq "$a" "$b" && num_eq "$a" "$c" && num_eq "$a" "$d"; then
    echo "PASS: $name — stable ($a) across repeats and hash seeds"
    pass=$((pass + 1))
  else
    echo "FAIL: $name — non-deterministic (repeats: $a,$b; seeds 0/1: $c,$d). Pin iteration/tie-break order; pass any time as input."
    fail=$((fail + 1))
  fi
done < <(list_fixtures)

echo "check-determinism: $pass passed, $fail failed"
[[ $fail -eq 0 ]]

#!/usr/bin/env bash
# check-invariance.sh — cosmetic edits (comments, blank lines, whitespace) must not change the score.
# This is invariance (prop-prove-invariance-under-irrelevant-transforms) AND anti-gaming
# (game-make-cheapest-improvement-the-right-one): if cosmetic edits move it, it is gameable.
set -euo pipefail

HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck source=lib/load-config.sh
source "$HERE/lib/load-config.sh"
TMP="$(mktemp -d)"
trap 'rm -rf "$TMP"' EXIT

pass=0
fail=0

while IFS= read -r f; do
  [[ -n "$f" ]] || continue
  name="$(basename "$f")"
  base="$(metric_of "$f")" || { echo "FAIL: $name — metric did not run"; fail=$((fail + 1)); continue; }
  variant="$TMP/$name"
  python3 "$HERE/lib/transforms.py" cosmetic "$f" "$variant"
  cos="$(metric_of "$variant")" || { echo "FAIL: $name — metric did not run on cosmetic variant"; fail=$((fail + 1)); continue; }
  if num_eq "$base" "$cos"; then
    echo "PASS: $name — invariant to cosmetic noise ($base)"
    pass=$((pass + 1))
  else
    echo "FAIL: $name — cosmetic noise moved the score ($base → $cos); it measures surface text and an optimizer can game it"
    fail=$((fail + 1))
  fi
done < <(list_fixtures)

echo "check-invariance: $pass passed, $fail failed"
[[ $fail -eq 0 ]]

#!/usr/bin/env bash
# check-monotonicity.sh — adding code (a construct-increasing edit) must not LOWER the score,
# and the metric must discriminate across distinct inputs (not saturate).
# Maps to prop-prove-monotonicity and prop-ensure-sensitivity-to-relevant-change.
set -euo pipefail

HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck source=lib/load-config.sh
source "$HERE/lib/load-config.sh"
TMP="$(mktemp -d)"
trap 'rm -rf "$TMP"' EXIT

pass=0
fail=0
values=()

while IFS= read -r f; do
  [[ -n "$f" ]] || continue
  name="$(basename "$f")"
  base="$(metric_of "$f")" || { echo "FAIL: $name — metric did not run"; fail=$((fail + 1)); continue; }
  values+=("$base")
  grown="$TMP/$name"
  python3 "$HERE/lib/transforms.py" grow "$f" "$grown" 50
  big="$(metric_of "$grown")" || { echo "FAIL: $name — metric did not run on grown variant"; fail=$((fail + 1)); continue; }
  if num_ge "$big" "$base"; then
    echo "PASS: $name — non-decreasing after adding code ($base → $big)"
    pass=$((pass + 1))
  else
    echo "FAIL: $name — score DROPPED after adding code ($base → $big); optimizing it could reward worse code"
    fail=$((fail + 1))
  fi
done < <(list_fixtures)

# Discrimination: distinct inputs must not all collapse to one value (saturation / no sensitivity).
distinct=0
if [[ ${#values[@]} -gt 0 ]]; then
  distinct="$(printf '%s\n' "${values[@]}" | sort -u | wc -l | tr -d ' ')"
fi
if [[ ${#values[@]} -ge 2 && $distinct -ge 2 ]]; then
  echo "PASS: discrimination — $distinct distinct values across ${#values[@]} fixtures (not saturated)"
  pass=$((pass + 1))
else
  echo "FAIL: discrimination — metric does not separate distinct inputs (saturated / no sensitivity)"
  fail=$((fail + 1))
fi

echo "check-monotonicity: $pass passed, $fail failed"
[[ $fail -eq 0 ]]

#!/usr/bin/env bash
# check-robustness.sh — edge inputs (empty, single statement) must give a finite, in-range number,
# never a crash or NaN. Maps to prop-prove-boundedness-and-handle-empty.
# Set declared_min / declared_max in config.json to enforce the metric's claimed range.
set -euo pipefail

HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck source=lib/load-config.sh
source "$HERE/lib/load-config.sh"
TMP="$(mktemp -d)"
trap 'rm -rf "$TMP"' EXIT

pass=0
fail=0

check_one() {  # check_one <label> <file>
  local label="$1" file="$2" val
  if ! val="$(metric_of "$file")"; then
    echo "FAIL: $label — metric crashed or returned a non-number on edge input"
    fail=$((fail + 1))
    return
  fi
  if [[ -n "$DECLARED_MIN" ]] && ! num_ge "$val" "$DECLARED_MIN"; then
    echo "FAIL: $label — value $val below declared min $DECLARED_MIN"
    fail=$((fail + 1))
    return
  fi
  if [[ -n "$DECLARED_MAX" ]] && ! num_ge "$DECLARED_MAX" "$val"; then
    echo "FAIL: $label — value $val above declared max $DECLARED_MAX"
    fail=$((fail + 1))
    return
  fi
  echo "PASS: $label — finite, in-range value ($val)"
  pass=$((pass + 1))
}

: > "$TMP/empty.py"                  # empty file
printf 'x = 0\n' > "$TMP/single.py" # single statement
check_one "empty input" "$TMP/empty.py"
check_one "single-statement input" "$TMP/single.py"

echo "check-robustness: $pass passed, $fail failed"
[[ $fail -eq 0 ]]

#!/usr/bin/env python3
"""check-tractability — the metric must scale near-linearly, not blow up (comp-keep-the-metric-tractable).

Generates inputs of increasing size, times the metric on each (best of 3), and fails if the
largest input exceeds a wall-clock budget or the least-squares log-log growth exponent is
super-quadratic. The wall-clock budget is the primary backstop; at small sizes the slope is
dampened by process-spawn overhead, so treat this as a blowup smoke test, not a microbenchmark
(profile your metric directly on production-sized inputs for real numbers).
"""
import math
import os
import shutil
import subprocess
import sys
import tempfile

HERE = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, os.path.join(HERE, "lib"))
import harness  # noqa: E402

SIZES = [100, 200, 400, 800, 1600]
BUDGET_SECONDS = float(os.environ.get("TRACTABILITY_BUDGET", "10.0"))
MAX_SLOPE = float(os.environ.get("TRACTABILITY_MAX_SLOPE", "2.2"))  # log-log exponent; <2 is sub-quadratic


def lstsq_slope(xs, ys):
    """Least-squares slope of ys vs xs (more robust than two-point endpoints)."""
    n = len(xs)
    mx, my = sum(xs) / n, sum(ys) / n
    den = sum((x - mx) ** 2 for x in xs)
    if den == 0:
        return 0.0
    return sum((x - mx) * (y - my) for x, y in zip(xs, ys)) / den


def main():
    cmd = harness.metric_cmd()
    tmp = tempfile.mkdtemp()
    try:
        subprocess.run(
            [sys.executable, os.path.join(HERE, "lib", "transforms.py"), "ramp", tmp, *map(str, SIZES)],
            check=True, capture_output=True,
        )
        times = []
        for n in SIZES:
            path = os.path.join(tmp, f"size_{n}.py")
            t = min(harness.timed_metric(cmd, path) for _ in range(3))
            times.append(max(t, 1e-6))
            print(f"  size {n:>4} funcs: {t * 1000:7.1f} ms")

        fails = 0
        if times[-1] > BUDGET_SECONDS:
            print(f"FAIL: largest input took {times[-1]:.2f}s (> {BUDGET_SECONDS}s budget)")
            fails += 1

        slope = lstsq_slope([math.log(n) for n in SIZES], [math.log(t) for t in times])
        if slope > MAX_SLOPE:
            print(f"FAIL: least-squares exponent ~{slope:.2f} (> {MAX_SLOPE}); the metric looks super-quadratic")
            fails += 1
        else:
            print(f"PASS: least-squares exponent ~{slope:.2f} (<= {MAX_SLOPE}); largest input within budget")

        print(f"check-tractability: {'0 failed' if fails == 0 else f'{fails} failed'}")
        return 1 if fails else 0
    finally:
        shutil.rmtree(tmp, ignore_errors=True)


if __name__ == "__main__":
    sys.exit(main())

#!/usr/bin/env python3
"""check-validity — empirical construct validity against a labeled corpus (valid-* rules).

Using whatever columns the corpus CSV provides (path, [accepted], [outcome]):
  convergent   Spearman(metric, accepted)        should be >= CONVERGENT_MIN
  discriminant |Spearman(metric, baseline LOC)|  FAIL only if > DISCRIMINANT_MAX (it's just size)
  predictive   AUC(metric, outcome)              should be >= PREDICTIVE_MIN
  lift         AUC(metric) - AUC(baseline)       should be >= 0 (beats the trivial baseline)

Thresholds are env-configurable; defaults are deliberately lenient — tighten them for your use
(the design skill argues for a strict discriminant bar, e.g. flag cyclomatic ~0.9 with LOC).
"""
import os
import sys

HERE = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, os.path.join(HERE, "lib"))
import harness  # noqa: E402

CONVERGENT_MIN = float(os.environ.get("CONVERGENT_MIN", "0.5"))
DISCRIMINANT_MAX = float(os.environ.get("DISCRIMINANT_MAX", "0.97"))
PREDICTIVE_MIN = float(os.environ.get("PREDICTIVE_MIN", "0.65"))


def main():
    labels = harness.setting("labels_csv", "LABELS_CSV", "{SKILL}/scripts/fixtures/corpus.csv")
    if not os.path.exists(labels):
        print(f"FAIL: labels CSV not found at {labels}. Set 'labels_csv' in config.json.")
        print("check-validity: 1 failed")
        return 1

    rows = list(harness.read_corpus(labels))
    if len(rows) < 3:
        print(f"FAIL: need >= 3 labeled rows for meaningful statistics, found {len(rows)}.")
        print("check-validity: 1 failed")
        return 1

    mcmd, bcmd = harness.metric_cmd(), harness.baseline_cmd()
    try:
        metric = [harness.run_metric(mcmd, r["_path"]) for r in rows]
        baseline = [harness.run_metric(bcmd, r["_path"]) for r in rows]
    except (RuntimeError, FileNotFoundError) as exc:
        print(f"FAIL: could not evaluate metric/baseline over the corpus — {exc}")
        print("check-validity: 1 failed")
        return 1

    fails = 0

    if rows[0].get("accepted"):
        accepted = [float(r["accepted"]) for r in rows]
        rho = harness.spearman(metric, accepted)
        if rho is not None and rho >= CONVERGENT_MIN:
            print(f"PASS: convergent — Spearman(metric, accepted) = {rho:.2f} (>= {CONVERGENT_MIN})")
        else:
            print(f"FAIL: convergent — Spearman(metric, accepted) = {rho} (< {CONVERGENT_MIN}); little agreement with the accepted measure")
            fails += 1
    else:
        print("SKIP: convergent — no 'accepted' column in corpus")

    rho_b = harness.spearman(metric, baseline)
    if rho_b is None:
        print("SKIP: discriminant — baseline has zero variance")
    elif abs(rho_b) <= DISCRIMINANT_MAX:
        print(f"PASS: discriminant — |Spearman(metric, LOC)| = {abs(rho_b):.2f} (<= {DISCRIMINANT_MAX}); adds signal beyond size")
    else:
        print(f"FAIL: discriminant — |Spearman(metric, LOC)| = {abs(rho_b):.2f} (> {DISCRIMINANT_MAX}); the metric is ~LOC relabeled")
        fails += 1

    if rows[0].get("outcome"):
        outcome = [int(float(r["outcome"])) for r in rows]
        a_m = harness.auc(metric, outcome)
        a_b = harness.auc(baseline, outcome)
        if a_m is None:
            print("SKIP: predictive — 'outcome' has only one class")
        else:
            if a_m >= PREDICTIVE_MIN:
                print(f"PASS: predictive — AUC(metric, outcome) = {a_m:.2f} (>= {PREDICTIVE_MIN})")
            else:
                print(f"FAIL: predictive — AUC(metric, outcome) = {a_m:.2f} (< {PREDICTIVE_MIN}); weak forecast of the outcome")
                fails += 1
            if a_b is not None:
                lift = a_m - a_b
                if lift >= 0:
                    print(f"PASS: beats baseline — AUC lift = {lift:+.2f} (metric {a_m:.2f} vs LOC {a_b:.2f})")
                else:
                    print(f"FAIL: beats baseline — metric AUC {a_m:.2f} < LOC AUC {a_b:.2f} ({lift:+.2f}); the cheaper baseline wins")
                    fails += 1
    else:
        print("SKIP: predictive — no 'outcome' column in corpus")

    print(f"check-validity: {'0 failed' if fails == 0 else f'{fails} failed'}")
    return 1 if fails else 0


if __name__ == "__main__":
    sys.exit(main())

#!/usr/bin/env python3
"""Example candidate metric: number of AST nodes in a Python file.

This is the `def-operationalize-behavior-and-size` "size" proxy from the
deterministic-metric-design skill: size measured on the parse tree, so it is
invariant to comments and whitespace (unlike LOC).

Adapter contract: take exactly one path argument, print ONE number to stdout.
"""
import ast
import sys


def ast_node_count(path: str) -> int:
    source = open(path, encoding="utf-8").read()
    return sum(1 for _ in ast.walk(ast.parse(source)))


if __name__ == "__main__":
    if len(sys.argv) != 2:
        print("usage: metric_ast_nodes.py <path>", file=sys.stderr)
        sys.exit(1)
    try:
        print(ast_node_count(sys.argv[1]))
    except SyntaxError:
        # Unparseable input → 0 is a defined, in-range value (see prop-prove-boundedness).
        print(0)

#!/usr/bin/env python3
"""Baseline metric: non-blank physical lines of code.

Used by the harness as the discriminant-validity baseline (`valid-discriminant-not-just-loc`).
It is deliberately gameable and NOT comment/whitespace-invariant — point the harness's
invariance check at this command and it will FAIL, which is the lesson.

Adapter contract: take one path argument, print ONE number to stdout.
"""
import sys


def loc(path: str) -> int:
    with open(path, encoding="utf-8") as f:
        return sum(1 for line in f if line.strip())


if __name__ == "__main__":
    if len(sys.argv) != 2:
        print("usage: metric_loc.py <path>", file=sys.stderr)
        sys.exit(1)
    print(loc(sys.argv[1]))

path,outcome,accepted
corpus/f6_tiny.py,0,1
corpus/f2_commented.py,0,2
corpus/f3_medium.py,1,5
corpus/f4_loops.py,1,7
corpus/f5_class.py,1,8
corpus/f1_dense.py,1,9

result = sum(a * b + c for a, b, c in zip(xs, ys, zs)) if xs and ys else (lambda n: n * 2)(0)

# Greeting helper for the onboarding flow.
# Kept intentionally tiny.
# Most of this file is explanation, not code.

def greet():
    # Return the canonical greeting.
    return "hi"

# End of module.

def add(a, b):
    return a + b


def subtract(a, b):
    return a - b

def signed_total(items):
    total = 0
    for item in items:
        if item > 0:
            total += item
        else:
            total -= item
    return total

class Counter:
    def __init__(self, start=0):
        self.value = start

    def increment(self, by=1):
        self.value += by
        return self.value

balance = 1

#!/usr/bin/env python3
"""Shared helpers for the Python-based harness checks.

Config resolution, metric invocation, and pure-stdlib statistics (Spearman, AUC) — no
numpy/scipy required, so the harness runs anywhere Python 3.8+ is present.
Setting precedence everywhere: environment variable > config.json > built-in default.
"""
import csv
import json
import os
import shlex
import subprocess
import time

SKILL_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))


def _config():
    path = os.environ.get("HARNESS_CONFIG") or os.path.join(SKILL_ROOT, "config.json")
    if os.path.exists(path):
        try:
            return json.load(open(path, encoding="utf-8"))
        except (json.JSONDecodeError, OSError):
            return {}
    return {}


def _resolve(value):
    return value.replace("{SKILL}", SKILL_ROOT) if isinstance(value, str) else value


def setting(name, env, default=""):
    if os.environ.get(env):
        return _resolve(os.environ[env])
    cfg = _config().get(name, "")
    return _resolve(cfg) if cfg else _resolve(default)


def metric_cmd():
    return setting("metric_cmd", "METRIC_CMD",
                   "python3 {SKILL}/scripts/examples/metric_ast_nodes.py")


def baseline_cmd():
    return setting("baseline_cmd", "BASELINE_CMD",
                   "python3 {SKILL}/scripts/examples/metric_loc.py")


def run_metric(cmd, path, timeout=120):
    """Invoke `cmd <path>`; return the single number printed. Raises on failure/non-numeric."""
    proc = subprocess.run(shlex.split(cmd) + [path],
                          capture_output=True, text=True, timeout=timeout)
    if proc.returncode != 0:
        raise RuntimeError(f"metric command failed on {path}: {proc.stderr.strip()}")
    out = proc.stdout.strip()
    try:
        return float(out)
    except ValueError:
        raise RuntimeError(
            f"metric did not print a number on {path} (got: {out!r}). "
            "The metric must print exactly ONE number to stdout."
        )


def timed_metric(cmd, path):
    start = time.perf_counter()
    run_metric(cmd, path)
    return time.perf_counter() - start


def read_corpus(labels_csv):
    base = os.path.dirname(os.path.abspath(labels_csv))
    with open(labels_csv, encoding="utf-8") as f:
        for row in csv.DictReader(f):
            row["_path"] = os.path.join(base, row["path"])
            yield row


def _rank(values):
    order = sorted(range(len(values)), key=lambda i: values[i])
    ranks = [0.0] * len(values)
    i = 0
    while i < len(values):
        j = i
        while j + 1 < len(values) and values[order[j + 1]] == values[order[i]]:
            j += 1
        avg = (i + j) / 2.0 + 1.0  # average (1-based) rank for ties
        for k in range(i, j + 1):
            ranks[order[k]] = avg
        i = j + 1
    return ranks


def spearman(x, y):
    """Spearman rank correlation. Returns None if undefined (n<2 or zero variance)."""
    if len(x) != len(y) or len(x) < 2:
        return None
    rx, ry = _rank(x), _rank(y)
    n = len(x)
    mx, my = sum(rx) / n, sum(ry) / n
    num = sum((a - mx) * (b - my) for a, b in zip(rx, ry))
    dx = sum((a - mx) ** 2 for a in rx) ** 0.5
    dy = sum((b - my) ** 2 for b in ry) ** 0.5
    if dx == 0 or dy == 0:
        return None
    return num / (dx * dy)


def auc(scores, labels):
    """Area under ROC (Mann–Whitney U). labels are 0/1. Returns None if a class is empty."""
    pos = [s for s, l in zip(scores, labels) if l == 1]
    neg = [s for s, l in zip(scores, labels) if l == 0]
    if not pos or not neg:
        return None
    wins = sum((1.0 if p > n else 0.5 if p == n else 0.0) for p in pos for n in neg)
    return wins / (len(pos) * len(neg))

# shellcheck shell=bash
# Sourced by the bash check scripts. Resolves SKILL_ROOT + config and defines helpers.
# Precedence for each setting: environment variable > config.json > built-in default.
# NOTE: paths in metric_cmd/skill root must not contain spaces (bash word-splits the command).

_LIB_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
SKILL_ROOT="$(cd "$_LIB_DIR/../.." && pwd)"
CONFIG_FILE="${HARNESS_CONFIG:-$SKILL_ROOT/config.json}"

_cfg() {  # _cfg <json-key>  → value or empty
  [[ -f "$CONFIG_FILE" ]] || { printf ''; return; }
  python3 -c 'import json,sys
try:
    d = json.load(open(sys.argv[1]))
except Exception:
    d = {}
print(d.get(sys.argv[2], ""))' "$CONFIG_FILE" "$1" 2>/dev/null || printf ''
}

_resolve() { printf '%s' "${1//\{SKILL\}/$SKILL_ROOT}"; }

_setting() {  # _setting <ENV_NAME> <json-key> <default>
  local env_name="$1" key="$2" def="$3" val
  val="${!env_name:-}"
  [[ -n "$val" ]] || val="$(_cfg "$key")"
  [[ -n "$val" ]] || val="$def"
  _resolve "$val"
}

METRIC_CMD="$(_setting METRIC_CMD   metric_cmd   'python3 {SKILL}/scripts/examples/metric_ast_nodes.py')"
BASELINE_CMD="$(_setting BASELINE_CMD baseline_cmd 'python3 {SKILL}/scripts/examples/metric_loc.py')"
CORPUS_DIR="$(_setting CORPUS_DIR   corpus_dir   '{SKILL}/scripts/fixtures/corpus')"
LABELS_CSV="$(_setting LABELS_CSV   labels_csv   '{SKILL}/scripts/fixtures/corpus.csv')"
DECLARED_MIN="$(_setting DECLARED_MIN declared_min '')"
DECLARED_MAX="$(_setting DECLARED_MAX declared_max '')"

case "$SKILL_ROOT" in
  *\ *) echo "warning: the skill path contains a space; the bash checks word-split \$METRIC_CMD and may fail. Move the skill to a space-free path or wrap your metric in a launcher script." >&2 ;;
esac

# metric_of <path> → prints the metric value; returns 1 with an actionable message on failure.
metric_of() {
  local p="$1" out
  out="$($METRIC_CMD "$p" 2>/dev/null)" || {
    echo "metric command failed on '$p'. Check 'metric_cmd' in config.json — it must run and exit 0." >&2
    return 1
  }
  out="$(printf '%s' "$out" | tr -d '[:space:]')"
  [[ "$out" =~ ^-?[0-9]+([.][0-9]+)?$ ]] || {
    echo "metric did not print one number on '$p' (got: '$out'). It must print exactly ONE number to stdout." >&2
    return 1
  }
  printf '%s' "$out"
}

num_eq() { [[ "$1" == "$2" ]]; }                              # exact string equality
num_ge() { awk -v a="$1" -v b="$2" 'BEGIN{exit !(a>=b)}'; }   # a >= b numerically
num_gt() { awk -v a="$1" -v b="$2" 'BEGIN{exit !(a>b)}'; }    # a >  b numerically

list_fixtures() { find "$CORPUS_DIR" -maxdepth 1 -name '*.py' -type f | sort; }

#!/usr/bin/env python3
"""Behavior-neutral and construct-changing source transforms used by the harness checks.

  cosmetic <src> <dst>      add comments, blank lines, and trailing whitespace — invisible to a
                            structure metric, visible to LOC (for invariance / anti-gaming).
  grow <src> <dst> [k=50]   append a function with k statements — strictly more construct
                            (for monotonicity).
  ramp <out_dir> n [n...]   write size_<n>.py with n functions each (for tractability timing).

All transforms are deterministic. The fixtures are Python; grow/ramp emit Python.
"""
import os
import sys


def cosmetic(src, dst):
    text = open(src, encoding="utf-8").read()
    noise = (
        "# cosmetic noise added by the harness\n"
        "# a structure metric must ignore this; LOC will not\n\n\n"
    )
    body = "\n".join(line + "   " for line in text.splitlines())  # trailing whitespace
    open(dst, "w", encoding="utf-8").write(noise + body + "\n")


def grow(src, dst, k=50):
    text = open(src, encoding="utf-8").read()
    extra = ["", "", "def _harness_grow():", "    x = 0"]
    extra += ["    x += 1" for _ in range(k)]
    open(dst, "w", encoding="utf-8").write(text.rstrip("\n") + "\n" + "\n".join(extra) + "\n")


def ramp(out_dir, sizes):
    os.makedirs(out_dir, exist_ok=True)
    paths = []
    for n in sizes:
        p = os.path.join(out_dir, f"size_{n}.py")
        with open(p, "w", encoding="utf-8") as f:
            for i in range(n):
                f.write(f"def f{i}(a, b):\n    return a + b + {i}\n\n")
        paths.append(p)
    return paths


def main(argv):
    if len(argv) < 2:
        print(__doc__, file=sys.stderr)
        return 1
    cmd = argv[1]
    if cmd == "cosmetic" and len(argv) == 4:
        cosmetic(argv[2], argv[3])
    elif cmd == "grow" and len(argv) in (4, 5):
        grow(argv[2], argv[3], int(argv[4]) if len(argv) == 5 else 50)
    elif cmd == "ramp" and len(argv) >= 4:
        for p in ramp(argv[2], [int(x) for x in argv[3:]]):
            print(p)
    else:
        print(__doc__, file=sys.stderr)
        return 1
    return 0


if __name__ == "__main__":
    sys.exit(main(sys.argv))

#!/usr/bin/env bash
# run-metric.sh <path> — invoke the configured candidate metric on <path> and print its number.
# Handy for checking your metric adapter before running the full harness.
set -euo pipefail

HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck source=lib/load-config.sh
source "$HERE/lib/load-config.sh"

if [[ $# -ne 1 ]]; then
  echo "Usage: $0 <path>" >&2
  exit 1
fi

value="$(metric_of "$1")" || exit 1
echo "$value"

#!/usr/bin/env bash
# selftest.sh — prove the harness works end to end against the bundled example metric + fixtures.
# Positive: the AST-node metric (comment/whitespace-invariant) passes every check.
# Negative: the LOC baseline FAILS invariance — proving the harness actually catches a bad metric.
set -euo pipefail  # the failing commands here are all in 'if' conditions, so -e is safe

HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
fail=0

echo "### 1. Positive run — the AST-node metric should pass every property check"
if bash "$HERE/verify.sh"; then
  echo "[selftest] positive run PASSED"
else
  echo "[selftest] positive run FAILED — the bundled example metric should pass"
  fail=1
fi
echo

echo "### 2. Negative run — the gameable LOC baseline should FAIL invariance"
loc_cmd="python3 $HERE/examples/metric_loc.py"
if METRIC_CMD="$loc_cmd" bash "$HERE/check-invariance.sh" >/dev/null 2>&1; then
  echo "[selftest] negative check FAILED — LOC unexpectedly passed invariance (harness not discriminating)"
  fail=1
else
  echo "[selftest] negative check PASSED — harness correctly flags LOC as non-invariant / gameable"
fi
echo

if [[ $fail -eq 0 ]]; then
  echo "SELFTEST: PASS"
  exit 0
fi
echo "SELFTEST: FAIL"
exit 1

#!/usr/bin/env bash
# verify.sh — run the full metric-validation harness and report PASS/FAIL per property.
# READ-ONLY: it computes and reports; it never modifies the metric, the corpus, or external state.
#
# Usage:
#   bash verify.sh                                  # uses config.json (or the bundled example metric)
#   METRIC_CMD="python3 path/to/mymetric.py" bash verify.sh
#
# Each property check maps to a category of the deterministic-metric-design skill.
set -euo pipefail  # checks are guarded with '|| rc=$?', so every check still runs to completion

HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck source=lib/load-config.sh
source "$HERE/lib/load-config.sh"

echo "== metric-validation-harness =="
echo "metric_cmd:   $METRIC_CMD"
echo "baseline_cmd: $BASELINE_CMD"
echo "corpus:       $LABELS_CSV"
echo

run_check() {  # run_check <label> <maps-to> <script>
  echo "── $1  ($2)"
  local rc=0
  if [[ "$3" == *.py ]]; then
    python3 "$HERE/$3" || rc=$?
  else
    bash "$HERE/$3" || rc=$?
  fi
  echo
  return $rc
}

total_fail=0
run_check "determinism"                   "det-"          "check-determinism.sh"  || total_fail=$((total_fail + 1))
run_check "invariance & anti-gaming"      "prop- / game-" "check-invariance.sh"   || total_fail=$((total_fail + 1))
run_check "monotonicity & discrimination" "prop-"         "check-monotonicity.sh" || total_fail=$((total_fail + 1))
run_check "robustness & range"            "prop-"         "check-robustness.sh"   || total_fail=$((total_fail + 1))
run_check "tractability"                  "comp-"         "check-tractability.py" || total_fail=$((total_fail + 1))
run_check "construct validity"            "valid-"        "check-validity.py"     || total_fail=$((total_fail + 1))

echo "════════════════════════════════════════════"
if [[ $total_fail -eq 0 ]]; then
  echo "RESULT: all property checks passed"
  exit 0
fi
echo "RESULT: $total_fail check group(s) failed — see the FAIL lines above"
exit 1