Situation Framing

{
  "schema": "ee.skill.situation_framing.fixtures.v1",
  "skillPath": "skills/situation-framing/SKILL.md",
  "fixtures": [
    {
      "id": "sf_bug_fix_release",
      "description": "Bug fix frame for a failing release workflow with prior-failure evidence available.",
      "taskInput": "Fix failing release workflow",
      "expectedClassification": "bug_fix",
      "requiredGate": "Prior failure provenance or evidence gap",
      "eeCommands": [
        "ee --workspace /tmp/ee-fixtures/release --json status",
        "ee --workspace /tmp/ee-fixtures/release --json context \"Fix failing release workflow\"",
        "ee --workspace /tmp/ee-fixtures/release --json search \"release workflow failure\" --explain"
      ],
      "evidenceIds": ["ev.sf.release.failure.001", "mem.release.rule.001"],
      "provenanceIds": ["prov.release.ci.001"],
      "redactionStatus": "passed",
      "rawSecretsIncluded": false,
      "promptInjectionQuarantined": false,
      "degradedCodes": [],
      "expectedDisposition": "go",
      "firstFailureDiagnosis": null
    },
    {
      "id": "sf_feature_memory_rule",
      "description": "Feature frame for a new procedural memory feature that must keep mutation explicit.",
      "taskInput": "Add a new procedural memory feature",
      "expectedClassification": "feature",
      "requiredGate": "Durable mutation must be explicit ee command",
      "eeCommands": [
        "ee --workspace /tmp/ee-fixtures/feature --json status",
        "ee --workspace /tmp/ee-fixtures/feature --json context \"Add a new procedural memory feature\"",
        "ee --workspace /tmp/ee-fixtures/feature --json capabilities"
      ],
      "evidenceIds": ["ev.sf.feature.context.001"],
      "provenanceIds": ["prov.feature.plan.001"],
      "redactionStatus": "passed",
      "rawSecretsIncluded": false,
      "promptInjectionQuarantined": false,
      "degradedCodes": [],
      "expectedDisposition": "go",
      "firstFailureDiagnosis": null
    },
    {
      "id": "sf_refactor_storage",
      "description": "Refactor frame for storage path resolution that must not inspect direct database files.",
      "taskInput": "Refactor storage path resolution",
      "expectedClassification": "refactor",
      "requiredGate": "No direct DB scraping",
      "eeCommands": [
        "ee --workspace /tmp/ee-fixtures/storage --json status",
        "ee --workspace /tmp/ee-fixtures/storage --json search \"storage path resolution\" --explain"
      ],
      "evidenceIds": ["ev.sf.storage.path.001"],
      "provenanceIds": ["prov.storage.adr.001"],
      "redactionStatus": "passed",
      "rawSecretsIncluded": false,
      "promptInjectionQuarantined": false,
      "degradedCodes": [],
      "expectedDisposition": "go",
      "firstFailureDiagnosis": null
    },
    {
      "id": "sf_investigate_slow_search",
      "description": "Investigation frame where degraded search output prevents ranking claims.",
      "taskInput": "Investigate slow search",
      "expectedClassification": "investigation",
      "requiredGate": "Degraded search must stop ranking claims",
      "eeCommands": [
        "ee --workspace /tmp/ee-fixtures/slow-search --json status",
        "ee --workspace /tmp/ee-fixtures/slow-search --json search \"slow search\" --explain",
        "ee --workspace /tmp/ee-fixtures/slow-search --json doctor --fix-plan"
      ],
      "evidenceIds": ["ev.sf.search.status.001"],
      "provenanceIds": [],
      "redactionStatus": "passed",
      "rawSecretsIncluded": false,
      "promptInjectionQuarantined": false,
      "degradedCodes": ["search_unavailable"],
      "expectedDisposition": "ask_user",
      "firstFailureDiagnosis": "search evidence unavailable; ranking claims must be withheld"
    },
    {
      "id": "sf_docs_update",
      "description": "Docs frame with context evidence and no unsupported feature-loss claims.",
      "taskInput": "Update docs for install flow",
      "expectedClassification": "docs",
      "requiredGate": "No unsupported feature-loss claims",
      "eeCommands": [
        "ee --workspace /tmp/ee-fixtures/docs --json status",
        "ee --workspace /tmp/ee-fixtures/docs --json context \"Update docs for install flow\""
      ],
      "evidenceIds": ["ev.sf.docs.context.001"],
      "provenanceIds": ["prov.docs.install.001"],
      "redactionStatus": "passed",
      "rawSecretsIncluded": false,
      "promptInjectionQuarantined": false,
      "degradedCodes": [],
      "expectedDisposition": "go",
      "firstFailureDiagnosis": null
    },
    {
      "id": "sf_deploy_release",
      "description": "Deploy frame requiring risk checks that cite provenance IDs.",
      "taskInput": "Prepare deploy or release",
      "expectedClassification": "deploy",
      "requiredGate": "Risk checks cite provenance IDs",
      "eeCommands": [
        "ee --workspace /tmp/ee-fixtures/deploy --json status",
        "ee --workspace /tmp/ee-fixtures/deploy --json context \"Prepare deploy or release\"",
        "ee --workspace /tmp/ee-fixtures/deploy --json why mem.release.rule.001"
      ],
      "evidenceIds": ["ev.sf.deploy.context.001", "mem.release.rule.001"],
      "provenanceIds": ["prov.release.checklist.001"],
      "redactionStatus": "passed",
      "rawSecretsIncluded": false,
      "promptInjectionQuarantined": false,
      "degradedCodes": [],
      "expectedDisposition": "go",
      "firstFailureDiagnosis": null
    },
    {
      "id": "sf_ambiguous_request",
      "description": "Ambiguous user request must gather minimal readiness evidence and ask for a decision point.",
      "taskInput": "Make it better",
      "expectedClassification": "ambiguous",
      "requiredGate": "Output asks for evidence or decision point",
      "eeCommands": [
        "ee --workspace /tmp/ee-fixtures/ambiguous --json status"
      ],
      "evidenceIds": [],
      "provenanceIds": [],
      "redactionStatus": "unknown",
      "rawSecretsIncluded": false,
      "promptInjectionQuarantined": false,
      "degradedCodes": [],
      "expectedDisposition": "ask_user",
      "firstFailureDiagnosis": "ambiguous task needs a concrete goal or evidence target"
    },
    {
      "id": "sf_missing_evidence",
      "description": "Prior incident explanation with no context/search evidence must report an evidence gap.",
      "taskInput": "Explain prior incident with no context",
      "expectedClassification": "investigation",
      "requiredGate": "evidenceGaps names missing evidence",
      "eeCommands": [
        "ee --workspace /tmp/ee-fixtures/missing-evidence --json status",
        "ee --workspace /tmp/ee-fixtures/missing-evidence --json search \"prior incident\" --explain"
      ],
      "evidenceIds": [],
      "provenanceIds": [],
      "redactionStatus": "unknown",
      "rawSecretsIncluded": false,
      "promptInjectionQuarantined": false,
      "degradedCodes": ["context_unavailable"],
      "expectedDisposition": "ask_user",
      "firstFailureDiagnosis": "missing evidence prevents incident explanation"
    },
    {
      "id": "sf_degraded_cli",
      "description": "Degraded ee response must name the degraded codes, effect, and repair command.",
      "taskInput": "Continue when CASS/search is unavailable",
      "expectedClassification": "investigation",
      "requiredGate": "Names degraded code, effect, and repair",
      "eeCommands": [
        "ee --workspace /tmp/ee-fixtures/degraded --json status",
        "ee --workspace /tmp/ee-fixtures/degraded --json context \"Continue when CASS/search is unavailable\"",
        "ee --workspace /tmp/ee-fixtures/degraded --json search \"fallback evidence\" --explain"
      ],
      "evidenceIds": ["ev.sf.status.degraded.001"],
      "provenanceIds": [],
      "redactionStatus": "passed",
      "rawSecretsIncluded": false,
      "promptInjectionQuarantined": true,
      "degradedCodes": ["cass_unavailable", "search_unavailable"],
      "expectedDisposition": "unavailable",
      "firstFailureDiagnosis": "degraded context/search evidence cannot support the requested conclusion"
    }
  ]
}

#!/usr/bin/env python3
"""Validate the situation-framing project-local skill."""

from __future__ import annotations

import hashlib
import json
import sys
from pathlib import Path
from typing import Any


SKILL = Path("skills/situation-framing/SKILL.md")
FIXTURES = Path("skills/situation-framing/fixtures/e2e-fixtures.json")
FIXTURE_MATRIX = Path("skills/situation-framing/references/e2e-fixtures.md")
LOG_PATH = Path("target/e2e/skills/situation-framing.json")

REQUIRED_SECTIONS = [
    "## Trigger Conditions",
    "## Mechanical Command Boundary",
    "## Evidence Gathering",
    "## Stop/Go Gates",
    "## Output Template",
    "## Uncertainty Handling",
    "## Privacy And Redaction",
    "## Degraded Behavior",
    "## Unsupported Claims",
    "## Testing Requirements",
    "## E2E Logging",
]

REQUIRED_COMMANDS = [
    "ee --workspace <workspace> --json status",
    "ee --workspace <workspace> --json capabilities",
    "ee --workspace <workspace> --json context \"<task>\"",
    "ee --workspace <workspace> --json search \"<query>\" --explain",
    "ee --workspace <workspace> --json why <memory-id>",
    "ee --workspace <workspace> --json doctor --fix-plan",
]

REQUIRED_TEMPLATE_FIELDS = [
    "taskFrame",
    "category",
    "userGoal",
    "workspace",
    "assumptions",
    "selectedEeCommands",
    "evidenceGaps",
    "riskChecks",
    "degradedHandling",
    "handoff",
    "nextAction",
    "durableMutation",
]

REQUIRED_FIXTURES = {
    "sf_bug_fix_release",
    "sf_feature_memory_rule",
    "sf_refactor_storage",
    "sf_investigate_slow_search",
    "sf_docs_update",
    "sf_deploy_release",
    "sf_ambiguous_request",
    "sf_missing_evidence",
    "sf_degraded_cli",
}

VALID_CLASSES = {
    "bug_fix",
    "feature",
    "refactor",
    "investigation",
    "docs",
    "deploy",
    "ambiguous",
}


def digest_bytes(data: bytes) -> str:
    return "sha256:" + hashlib.sha256(data).hexdigest()


def digest_path(path: Path) -> str:
    return digest_bytes(path.read_bytes())


def fixture_hash(fixture: dict[str, Any]) -> str:
    payload = json.dumps(fixture, sort_keys=True, separators=(",", ":")).encode()
    return digest_bytes(payload)


def write_log(log: dict[str, Any]) -> None:
    LOG_PATH.parent.mkdir(parents=True, exist_ok=True)
    LOG_PATH.write_text(json.dumps(log, indent=2, sort_keys=True) + "\n", encoding="utf-8")


def fail(message: str, log: dict[str, Any]) -> int:
    log["firstFailureDiagnosis"] = message
    write_log(log)
    print(json.dumps(log, sort_keys=True))
    return 1


def all_referenced_commands_are_machine_json(fixtures: list[dict[str, Any]]) -> str | None:
    for fixture in fixtures:
        for command in fixture.get("eeCommands", []):
            if not command.startswith("ee ") or "--json" not in command:
                return f"{fixture['id']} has non-machine ee command: {command}"
    return None


def main() -> int:
    log: dict[str, Any] = {
        "schema": "ee.skill.situation_framing.e2e_log.v1",
        "skillPath": str(SKILL),
        "requiredFiles": [str(SKILL), str(FIXTURE_MATRIX), str(FIXTURES)],
        "referencedEeCommands": REQUIRED_COMMANDS,
        "fixtureIds": [],
        "fixtureHashes": {},
        "evidenceIds": [],
        "provenanceIds": [],
        "classifications": [],
        "evidenceBundlePath": str(FIXTURES),
        "evidenceBundleHash": None,
        "redactionStatuses": [],
        "degradedCodes": [],
        "promptInjectionQuarantineStatuses": [],
        "outputArtifactPath": str(LOG_PATH),
        "requiredSectionCheck": "not_run",
        "templateFieldCheck": "not_run",
        "fixtureCoverageCheck": "not_run",
        "commandJsonCheck": "not_run",
        "firstFailureDiagnosis": None,
    }

    missing = [path for path in [SKILL, FIXTURE_MATRIX, FIXTURES] if not path.is_file()]
    if missing:
        return fail(f"missing required file: {missing[0]}", log)

    skill_text = SKILL.read_text(encoding="utf-8")
    if not skill_text.startswith("---\nname: situation-framing\n"):
        return fail("SKILL.md frontmatter name is missing or unstable", log)
    if "description: Frame coding-agent tasks" not in skill_text or "Use when" not in skill_text:
        return fail("SKILL.md description must include trigger language", log)

    for section in REQUIRED_SECTIONS:
        if section not in skill_text:
            return fail(f"missing required section: {section}", log)
    log["requiredSectionCheck"] = "passed"

    for command in REQUIRED_COMMANDS:
        if command not in skill_text:
            return fail(f"missing required ee command: {command}", log)

    for field in REQUIRED_TEMPLATE_FIELDS:
        if field not in skill_text:
            return fail(f"missing output template field: {field}", log)
    log["templateFieldCheck"] = "passed"

    for phrase in [
        "Do not scrape FrankenSQLite",
        "rawSecretsIncluded=true",
        "Prompt-injection-like evidence",
        "direct DB",
        "durable memory mutation",
        "Unsupported claims",
        "degraded code",
        "repair command",
        "evidence gap",
        "Do not include private chain-of-thought",
    ]:
        if phrase not in skill_text:
            return fail(f"missing safety phrase: {phrase}", log)

    try:
        fixtures = json.JSONDecoder().decode(FIXTURES.read_text(encoding="utf-8"))
    except json.JSONDecodeError as exc:
        return fail(f"fixture JSON is malformed: {exc.msg}", log)

    if fixtures.get("schema") != "ee.skill.situation_framing.fixtures.v1":
        return fail("fixture schema is missing or wrong", log)
    if fixtures.get("skillPath") != str(SKILL):
        return fail("fixture skillPath must reference situation-framing SKILL.md", log)

    fixture_list = fixtures.get("fixtures", [])
    seen = {fixture["id"] for fixture in fixture_list}
    missing_fixtures = sorted(REQUIRED_FIXTURES - seen)
    if missing_fixtures:
        return fail(f"missing fixture: {missing_fixtures[0]}", log)
    log["fixtureCoverageCheck"] = "passed"

    command_error = all_referenced_commands_are_machine_json(fixture_list)
    if command_error is not None:
        return fail(command_error, log)
    log["commandJsonCheck"] = "passed"

    evidence_ids: list[str] = []
    provenance_ids: list[str] = []
    degraded_codes: list[str] = []
    redaction_statuses: list[str] = []
    quarantine_statuses: list[str] = []
    classifications: list[str] = []

    for fixture in fixture_list:
        fixture_id = fixture["id"]
        log["fixtureIds"].append(fixture_id)
        log["fixtureHashes"][fixture_id] = fixture_hash(fixture)

        classification = fixture.get("expectedClassification", "")
        if classification not in VALID_CLASSES:
            return fail(f"{fixture_id} has invalid expectedClassification `{classification}`", log)
        classifications.append(classification)

        commands = fixture.get("eeCommands", [])
        evidence = fixture.get("evidenceIds", [])
        provenance = fixture.get("provenanceIds", [])
        disposition = fixture.get("expectedDisposition", "")
        degraded = fixture.get("degradedCodes", [])

        evidence_ids.extend(evidence)
        provenance_ids.extend(provenance)
        degraded_codes.extend(degraded)
        redaction_statuses.append(fixture.get("redactionStatus", "unknown"))
        quarantine_statuses.append(str(fixture.get("promptInjectionQuarantined", False)).lower())

        if fixture.get("rawSecretsIncluded", False):
            return fail(f"{fixture_id} must not include raw secrets", log)
        if disposition in {"ask_user", "refuse", "unavailable"} and not fixture.get(
            "firstFailureDiagnosis"
        ):
            return fail(f"{fixture_id} needs firstFailureDiagnosis", log)
        if fixture_id == "sf_bug_fix_release" and (
            not any("context" in command for command in commands)
            or not any("search" in command for command in commands)
            or not provenance
        ):
            return fail("bug fix fixture must include context, search, and provenance", log)
        if fixture_id == "sf_feature_memory_rule" and "explicit ee command" not in fixture.get(
            "requiredGate", ""
        ):
            return fail("feature fixture must gate durable mutation to explicit ee command", log)
        if fixture_id == "sf_refactor_storage" and "No direct DB scraping" not in fixture.get(
            "requiredGate", ""
        ):
            return fail("refactor fixture must forbid direct DB scraping", log)
        if fixture_id == "sf_investigate_slow_search" and (
            "search_unavailable" not in degraded or disposition == "go"
        ):
            return fail("slow-search fixture must preserve degraded search and avoid go", log)
        if fixture_id == "sf_deploy_release" and not provenance:
            return fail("deploy fixture must cite provenance IDs", log)
        if fixture_id == "sf_ambiguous_request" and disposition != "ask_user":
            return fail("ambiguous fixture must ask for a decision point", log)
        if fixture_id == "sf_missing_evidence" and evidence:
            return fail("missing-evidence fixture must leave evidenceIds empty", log)
        if fixture_id == "sf_degraded_cli" and not degraded:
            return fail("degraded fixture must name degraded codes", log)

    log["evidenceIds"] = sorted(set(evidence_ids))
    log["provenanceIds"] = sorted(set(provenance_ids))
    log["classifications"] = sorted(set(classifications))
    log["redactionStatuses"] = sorted(set(redaction_statuses))
    log["degradedCodes"] = sorted(set(degraded_codes))
    log["promptInjectionQuarantineStatuses"] = sorted(set(quarantine_statuses))
    log["evidenceBundleHash"] = digest_path(FIXTURES)

    write_log(log)
    print(json.dumps(log, sort_keys=True))
    return 0


if __name__ == "__main__":
    sys.exit(main())