Compliance Testing

{
  "$schema": "./config-schema.json",
  "_description": "Compliance Testing configuration. Auto-created on first run. Edit to customize.",
  "regulations": [],
  "scope": null,
  "options": {
    "dataClassification": null,
    "retentionPolicyDays": null,
    "auditLogRequired": true,
    "piiScanEnabled": true
  },
  "_setupPrompt": "If regulations is empty, ask: 'Which regulations apply to this project? (gdpr/ccpa/hipaa/soc2/pci-dss — comma-separated)'. If scope is null, ask: 'What is the compliance scope? (full-app/api-only/data-layer/specific-module)'."
}

# =============================================================================
# AQE Skill Evaluation Test Suite: Compliance Testing v1.0.0
# =============================================================================
#
# Comprehensive evaluation suite for the compliance-testing skill.
# Tests regulatory compliance detection across GDPR, HIPAA, SOC2, PCI-DSS, CCPA.
#
# Coverage:
# - Data privacy controls (GDPR/CCPA)
# - Healthcare data protection (HIPAA)
# - Payment security (PCI-DSS)
# - Security controls (SOC2)
# - Access control and audit logging
# - Multi-model consistency
#
# Schema: .claude/skills/.validation/schemas/skill-eval.schema.json
# Runner: scripts/run-skill-eval.ts
#
# =============================================================================

skill: compliance-testing
version: 1.0.0
description: >
  Comprehensive evaluation suite for the compliance-testing skill.
  Validates detection of compliance violations across major regulatory frameworks
  (GDPR, HIPAA, SOC2, PCI-DSS, CCPA), control assessment accuracy, risk scoring,
  and remediation quality. Integrates with ReasoningBank for pattern learning.

# =============================================================================
# Multi-Model Configuration
# =============================================================================

models_to_test:
  - claude-sonnet-4-6    # Primary (high accuracy expected)
  - claude-haiku-4-5     # Fast model (minimum quality floor)

# =============================================================================
# MCP Integration Configuration
# =============================================================================

mcp_integration:
  enabled: true
  namespace: skill-validation

  query_patterns: true
  track_outcomes: true
  store_patterns: true
  share_learning: true
  update_quality_gate: true

  target_agents:
    - qe-learning-coordinator
    - qe-queen-coordinator
    - qe-security-scanner
    - qe-security-auditor

# =============================================================================
# ReasoningBank Learning Configuration
# =============================================================================

learning:
  store_success_patterns: true
  store_failure_patterns: true
  pattern_ttl_days: 90
  min_confidence_to_store: 0.7
  cross_model_comparison: true

# =============================================================================
# Result Format Configuration
# =============================================================================

result_format:
  json_output: true
  markdown_report: true
  include_raw_output: false
  include_timing: true
  include_token_usage: true

# =============================================================================
# Environment Setup
# =============================================================================

setup:
  required_tools:
    - jq       # JSON processing

  environment_variables:
    COMPLIANCE_AUDIT_MODE: "comprehensive"
    GDPR_ENABLED: "true"
    HIPAA_ENABLED: "true"
    PCI_DSS_ENABLED: "true"
    SOC2_ENABLED: "true"
    CCPA_ENABLED: "true"

  fixtures:
    - name: gdpr_violation_app
      path: fixtures/gdpr-violation.js
      content: |
        // GDPR Violation: No consent tracking, no data subject rights
        const express = require('express');
        const app = express();

        // No consent management
        app.post('/newsletter/subscribe', (req, res) => {
          db.insert('subscribers', {
            email: req.body.email,
            // No consent timestamp, no IP, no opt-in record
          });
          res.send('Subscribed!');
        });

        // No right to erasure
        app.delete('/user/:id', (req, res) => {
          // Deletes user but not their data from other tables
          db.delete('users', { id: req.params.id });
          // Still has: orders, logs, analytics, backups
          res.send('Deleted');
        });

        // No data portability
        // Missing /user/:id/export endpoint

    - name: hipaa_violation_app
      path: fixtures/hipaa-violation.py
      content: |
        # HIPAA Violation: PHI exposed, no encryption, no audit logging
        from flask import Flask, request
        import sqlite3

        app = Flask(__name__)

        @app.route('/patient/<patient_id>')
        def get_patient(patient_id):
            # No access control check
            # No audit logging
            conn = sqlite3.connect('patients.db')
            cursor = conn.cursor()
            # PHI returned without encryption
            cursor.execute(f"SELECT ssn, medical_history, diagnosis FROM patients WHERE id = {patient_id}")
            return str(cursor.fetchone())  # Plain text response with PHI

        @app.route('/patient', methods=['POST'])
        def create_patient():
            # PHI stored without encryption
            data = request.json
            conn = sqlite3.connect('patients.db')
            cursor = conn.cursor()
            cursor.execute(f"""
                INSERT INTO patients (ssn, name, medical_history)
                VALUES ('{data['ssn']}', '{data['name']}', '{data['history']}')
            """)
            # No audit log of PHI access
            return 'Created'

    - name: pci_dss_violation_app
      path: fixtures/pci-violation.js
      content: |
        // PCI-DSS Violation: Card data stored, CVV logged
        const express = require('express');
        const app = express();

        app.post('/payment', (req, res) => {
          const { cardNumber, expiry, cvv, amount } = req.body;

          // Violation: Storing full card number
          db.insert('payments', {
            card_number: cardNumber,  // Should only store last 4
            expiry: expiry,           // Should not store
            cvv: cvv,                 // NEVER store CVV
            amount: amount
          });

          // Violation: Logging sensitive card data
          console.log(`Payment processed: ${cardNumber}, CVV: ${cvv}`);

          res.send('Payment processed');
        });

        // Violation: Exposing card data via API
        app.get('/payments/:id', (req, res) => {
          const payment = db.findOne('payments', { id: req.params.id });
          res.json(payment);  // Returns full card number
        });

# =============================================================================
# TEST CASES
# =============================================================================

test_cases:
  # ---------------------------------------------------------------------------
  # CATEGORY: GDPR Compliance (Data Privacy)
  # ---------------------------------------------------------------------------

  - id: tc001_gdpr_consent_violation
    description: "Detect missing consent management for data collection"
    category: gdpr
    priority: critical

    input:
      code: |
        app.post('/newsletter/subscribe', (req, res) => {
          db.insert('subscribers', {
            email: req.body.email,
            subscribed_at: new Date()
          });
          res.send('Subscribed!');
        });
      context:
        language: javascript
        framework: express
        regulation: GDPR

    expected_output:
      must_contain:
        - "consent"
        - "GDPR"
        - "Article 7"
        - "lawful basis"
      must_not_contain:
        - "compliant"
        - "no issues"
      severity_classification: high
      finding_count:
        min: 1

    validation:
      schema_check: true
      keyword_match_threshold: 0.8
      reasoning_quality_min: 0.7

  - id: tc002_gdpr_right_to_erasure
    description: "Detect incomplete implementation of right to erasure (Article 17)"
    category: gdpr
    priority: critical

    input:
      code: |
        app.delete('/user/:id', async (req, res) => {
          const userId = req.params.id;
          await db.delete('users', { id: userId });
          // Orders, logs, and analytics still contain user data
          res.send('User deleted');
        });
      context:
        language: javascript
        framework: express
        regulation: GDPR

    expected_output:
      must_contain:
        - "erasure"
        - "Article 17"
        - "right to be forgotten"
        - "incomplete"
        - "related data"
      must_match_regex:
        - "GDPR-Art17|Art\\.?\\s*17"
      severity_classification: high
      finding_count:
        min: 1

    validation:
      schema_check: true
      keyword_match_threshold: 0.7

  - id: tc003_gdpr_data_portability
    description: "Detect missing data portability implementation (Article 20)"
    category: gdpr
    priority: high

    input:
      code: |
        // User management API - No data export endpoint
        app.get('/user/:id', (req, res) => {
          const user = db.findOne('users', { id: req.params.id });
          res.json(user);
        });

        app.put('/user/:id', (req, res) => {
          db.update('users', { id: req.params.id }, req.body);
          res.send('Updated');
        });

        // Missing: GET /user/:id/export
      context:
        language: javascript
        framework: express
        regulation: GDPR

    expected_output:
      must_contain:
        - "portability"
        - "Article 20"
        - "export"
        - "machine-readable"
      severity_classification: medium

    validation:
      schema_check: true
      keyword_match_threshold: 0.7

  # ---------------------------------------------------------------------------
  # CATEGORY: HIPAA Compliance (Healthcare Data)
  # ---------------------------------------------------------------------------

  - id: tc004_hipaa_phi_encryption
    description: "Detect unencrypted PHI storage and transmission"
    category: hipaa
    priority: critical

    input:
      code: |
        @app.route('/patient', methods=['POST'])
        def create_patient():
            data = request.json
            conn = sqlite3.connect('patients.db')
            cursor = conn.cursor()
            cursor.execute(f"""
                INSERT INTO patients (ssn, name, diagnosis, medical_history)
                VALUES ('{data['ssn']}', '{data['name']}', '{data['diagnosis']}', '{data['history']}')
            """)
            conn.commit()
            return 'Patient created'
      context:
        language: python
        framework: flask
        regulation: HIPAA

    expected_output:
      must_contain:
        - "PHI"
        - "encryption"
        - "HIPAA"
        - "protected health information"
        - "encrypt at rest"
      must_match_regex:
        - "HIPAA-[0-9]+|164\\.312"
      severity_classification: critical
      finding_count:
        min: 1

    validation:
      schema_check: true
      keyword_match_threshold: 0.8
      reasoning_quality_min: 0.75

  - id: tc005_hipaa_audit_logging
    description: "Detect missing audit logging for PHI access"
    category: hipaa
    priority: critical

    input:
      code: |
        @app.route('/patient/<patient_id>')
        def get_patient(patient_id):
            conn = sqlite3.connect('patients.db')
            cursor = conn.cursor()
            cursor.execute(f"SELECT * FROM patients WHERE id = {patient_id}")
            return jsonify(cursor.fetchone())
      context:
        language: python
        framework: flask
        regulation: HIPAA

    expected_output:
      must_contain:
        - "audit"
        - "logging"
        - "HIPAA"
        - "access"
        - "who accessed"
      severity_classification: high

    validation:
      schema_check: true
      keyword_match_threshold: 0.7

  - id: tc006_hipaa_access_control
    description: "Detect missing access control for PHI"
    category: hipaa
    priority: critical

    input:
      code: |
        @app.route('/patient/<patient_id>/records')
        def get_patient_records(patient_id):
            # No authentication check
            # No authorization check (role-based access)
            # No minimum necessary principle
            conn = sqlite3.connect('patients.db')
            cursor = conn.cursor()
            cursor.execute("SELECT * FROM patient_records WHERE patient_id = ?", [patient_id])
            return jsonify(cursor.fetchall())  # Returns all fields, not minimum necessary
      context:
        language: python
        framework: flask
        regulation: HIPAA

    expected_output:
      must_contain:
        - "access control"
        - "authorization"
        - "minimum necessary"
        - "role-based"
      severity_classification: critical

    validation:
      schema_check: true
      keyword_match_threshold: 0.7

  # ---------------------------------------------------------------------------
  # CATEGORY: PCI-DSS Compliance (Payment Card Data)
  # ---------------------------------------------------------------------------

  - id: tc007_pci_card_storage
    description: "Detect prohibited storage of full card numbers"
    category: pci-dss
    priority: critical

    input:
      code: |
        app.post('/payment', (req, res) => {
          const { cardNumber, expiry, amount } = req.body;

          db.insert('payments', {
            card_number: cardNumber,  // Full card number stored
            expiry_date: expiry,
            amount: amount
          });

          res.send('Payment processed');
        });
      context:
        language: javascript
        framework: express
        regulation: PCI-DSS

    expected_output:
      must_contain:
        - "PCI"
        - "card number"
        - "storage"
        - "tokenize"
        - "last 4"
      must_match_regex:
        - "PCI-DSS|Requirement\\s*3"
      severity_classification: critical
      finding_count:
        min: 1

    validation:
      schema_check: true
      keyword_match_threshold: 0.8

  - id: tc008_pci_cvv_storage
    description: "Detect prohibited CVV/CVC storage"
    category: pci-dss
    priority: critical

    input:
      code: |
        app.post('/checkout', (req, res) => {
          const { cardNumber, cvv, expiry } = req.body;

          // Process payment
          const result = paymentGateway.charge({
            card: cardNumber,
            cvv: cvv,
            exp: expiry
          });

          // Store for later reference (VIOLATION)
          db.insert('transactions', {
            card_last4: cardNumber.slice(-4),
            cvv: cvv,  // NEVER store CVV
            transaction_id: result.id
          });

          res.json(result);
        });
      context:
        language: javascript
        framework: express
        regulation: PCI-DSS

    expected_output:
      must_contain:
        - "CVV"
        - "never store"
        - "PCI"
        - "Requirement 3.2"
      severity_classification: critical

    validation:
      schema_check: true
      keyword_match_threshold: 0.8

  - id: tc009_pci_logging_card_data
    description: "Detect card data in logs"
    category: pci-dss
    priority: critical

    input:
      code: |
        app.post('/payment', (req, res) => {
          const { cardNumber, amount } = req.body;

          // Log the transaction (VIOLATION)
          console.log(`Processing payment: card=${cardNumber}, amount=${amount}`);
          logger.info({ card: cardNumber, amount }, 'Payment request received');

          // Process payment
          const result = gateway.charge({ card: cardNumber, amount });

          res.json(result);
        });
      context:
        language: javascript
        framework: express
        regulation: PCI-DSS

    expected_output:
      must_contain:
        - "log"
        - "card"
        - "mask"
        - "PCI"
      severity_classification: high

    validation:
      schema_check: true
      keyword_match_threshold: 0.7

  # ---------------------------------------------------------------------------
  # CATEGORY: SOC2 Compliance (Security Controls)
  # ---------------------------------------------------------------------------

  - id: tc010_soc2_access_logging
    description: "Detect missing access logging for SOC2 CC6.1"
    category: soc2
    priority: high

    input:
      code: |
        app.get('/admin/users', requireAdmin, (req, res) => {
          // No access logging
          const users = db.findAll('users');
          res.json(users);
        });

        app.delete('/admin/users/:id', requireAdmin, (req, res) => {
          // No access logging for destructive operation
          db.delete('users', { id: req.params.id });
          res.send('Deleted');
        });
      context:
        language: javascript
        framework: express
        regulation: SOC2

    expected_output:
      must_contain:
        - "SOC2"
        - "CC6"
        - "logging"
        - "audit trail"
      severity_classification: high

    validation:
      schema_check: true
      keyword_match_threshold: 0.7

  - id: tc011_soc2_change_management
    description: "Detect missing change management controls for SOC2 CC8.1"
    category: soc2
    priority: medium

    input:
      code: |
        // Deployment script - no change management
        const deploy = async () => {
          // No approval workflow
          // No change tracking
          // No rollback capability

          await executeSQL('ALTER TABLE users ADD COLUMN admin BOOLEAN');
          await restartService('api');
          console.log('Deployed!');
        };

        deploy();
      context:
        language: javascript
        framework: nodejs
        regulation: SOC2

    expected_output:
      must_contain:
        - "SOC2"
        - "change management"
        - "CC8"
        - "approval"
        - "rollback"
      severity_classification: medium

    validation:
      schema_check: true
      keyword_match_threshold: 0.6

  # ---------------------------------------------------------------------------
  # CATEGORY: CCPA Compliance (California Consumer Privacy)
  # ---------------------------------------------------------------------------

  - id: tc012_ccpa_opt_out
    description: "Detect missing 'Do Not Sell' opt-out mechanism"
    category: ccpa
    priority: high

    input:
      code: |
        // Data sharing API - No opt-out mechanism
        app.post('/analytics/share', (req, res) => {
          const userData = db.findOne('users', { id: req.body.userId });

          // Share data with third parties without opt-out check
          thirdPartyAnalytics.send({
            email: userData.email,
            browsing_history: userData.history,
            purchases: userData.purchases
          });

          res.send('Data shared');
        });
      context:
        language: javascript
        framework: express
        regulation: CCPA

    expected_output:
      must_contain:
        - "CCPA"
        - "opt-out"
        - "do not sell"
        - "consumer"
      severity_classification: high

    validation:
      schema_check: true
      keyword_match_threshold: 0.7

  - id: tc013_ccpa_disclosure
    description: "Detect missing data collection disclosure"
    category: ccpa
    priority: high

    input:
      code: |
        // User signup - no disclosure of data practices
        app.post('/signup', (req, res) => {
          const user = db.insert('users', {
            email: req.body.email,
            name: req.body.name,
            ip_address: req.ip,
            device_info: req.headers['user-agent'],
            location: geoip.lookup(req.ip)
          });

          // Collect data without disclosing what's collected
          // No link to privacy policy
          // No categories of data disclosed

          res.json({ success: true });
        });
      context:
        language: javascript
        framework: express
        regulation: CCPA

    expected_output:
      must_contain:
        - "CCPA"
        - "disclosure"
        - "categories"
        - "privacy"
      severity_classification: medium

    validation:
      schema_check: true
      keyword_match_threshold: 0.7

  # ---------------------------------------------------------------------------
  # CATEGORY: Negative Tests (Compliant Code)
  # ---------------------------------------------------------------------------

  - id: tc014_compliant_gdpr_code
    description: "Verify compliant GDPR implementation is not flagged"
    category: negative
    priority: high

    input:
      code: |
        // GDPR-compliant user data handling
        app.post('/newsletter/subscribe', (req, res) => {
          // Verify consent
          if (!req.body.consent || !req.body.consent.marketing) {
            return res.status(400).json({ error: 'Consent required' });
          }

          db.insert('subscribers', {
            email: req.body.email,
            consent: {
              marketing: true,
              timestamp: new Date().toISOString(),
              ip_address: req.ip,
              version: 'consent-v2.1'
            }
          });

          res.json({ success: true, message: 'Subscribed with consent' });
        });

        // Right to erasure implementation
        app.delete('/user/:id/data', async (req, res) => {
          const userId = req.params.id;

          // Delete from all tables
          await Promise.all([
            db.delete('users', { id: userId }),
            db.delete('orders', { user_id: userId }),
            db.delete('analytics', { user_id: userId }),
            db.delete('preferences', { user_id: userId })
          ]);

          // Log the erasure (retain audit log)
          await db.insert('audit_log', {
            action: 'GDPR_ERASURE',
            user_id: userId,
            timestamp: new Date().toISOString()
          });

          res.json({ success: true, message: 'All data erased' });
        });

        // Data portability
        app.get('/user/:id/export', async (req, res) => {
          const userId = req.params.id;
          const userData = await collectAllUserData(userId);

          res.json({
            format: 'JSON',
            schema_version: '1.0',
            exported_at: new Date().toISOString(),
            data: userData
          });
        });
      context:
        language: javascript
        framework: express
        regulation: GDPR

    expected_output:
      must_contain:
        - "compliant"
        - "consent"
        - "erasure"
        - "portability"
      must_not_contain:
        - "critical"
        - "violation"
        - "missing"
      finding_count:
        max: 2  # Allow informational findings only

    validation:
      schema_check: true
      allow_partial: true

  - id: tc015_compliant_pci_code
    description: "Verify PCI-DSS compliant payment handling is not flagged"
    category: negative
    priority: high

    input:
      code: |
        // PCI-DSS compliant payment processing
        const processPayment = async (req, res) => {
          const { paymentToken, amount } = req.body;  // Only token, no raw card data

          // Log without sensitive data
          logger.info({
            amount,
            tokenId: paymentToken.slice(0, 8) + '***',
            timestamp: new Date().toISOString()
          }, 'Processing payment');

          // Process via tokenized gateway
          const result = await paymentGateway.charge({
            token: paymentToken,
            amount
          });

          // Store only safe reference
          await db.insert('transactions', {
            transaction_id: result.id,
            card_last4: result.card.last4,
            card_brand: result.card.brand,
            amount,
            status: result.status
          });

          res.json({
            success: true,
            transactionId: result.id
          });
        };
      context:
        language: javascript
        framework: express
        regulation: PCI-DSS

    expected_output:
      must_contain:
        - "token"
        - "compliant"
      must_not_contain:
        - "CVV"
        - "card number"
        - "violation"
        - "critical"
      finding_count:
        max: 1

    validation:
      schema_check: true
      allow_partial: true

  # ---------------------------------------------------------------------------
  # CATEGORY: Multi-Framework Tests
  # ---------------------------------------------------------------------------

  - id: tc016_multi_framework_violations
    description: "Detect violations across multiple compliance frameworks"
    category: multi-framework
    priority: high

    input:
      code: |
        // Healthcare payment app - violates HIPAA and PCI-DSS
        app.post('/patient/payment', (req, res) => {
          const { patientId, cardNumber, cvv, diagnosis } = req.body;

          // HIPAA violation: PHI without encryption/logging
          db.insert('patient_payments', {
            patient_id: patientId,
            diagnosis: diagnosis,  // PHI stored unencrypted
            // No audit log

            // PCI-DSS violation: Card data storage
            card_number: cardNumber,
            cvv: cvv  // Never store CVV
          });

          console.log(`Payment for patient ${patientId}, card ${cardNumber}`);

          res.send('Payment recorded');
        });
      context:
        language: javascript
        framework: express
        regulation: [HIPAA, PCI-DSS]

    expected_output:
      must_contain:
        - "HIPAA"
        - "PCI"
        - "PHI"
        - "CVV"
        - "encryption"
      finding_count:
        min: 3
        max: 8

    validation:
      schema_check: true
      keyword_match_threshold: 0.7

    timeout_ms: 45000

  - id: tc017_gdpr_ccpa_overlap
    description: "Detect privacy violations applicable to both GDPR and CCPA"
    category: multi-framework
    priority: high

    input:
      code: |
        // Privacy violations applicable to GDPR and CCPA
        app.post('/signup', (req, res) => {
          // No consent collection
          // No disclosure of data practices
          // No opt-out mechanism

          db.insert('users', {
            email: req.body.email,
            ip: req.ip,
            browser: req.headers['user-agent'],
            location: geoip.lookup(req.ip),
            referrer: req.headers.referer
          });

          // Share with third parties without consent
          analytics.track(req.body.email, 'signup');
          marketing.addLead(req.body.email);

          res.send('Signed up!');
        });
      context:
        language: javascript
        framework: express
        regulation: [GDPR, CCPA]

    expected_output:
      must_contain:
        - "consent"
        - "disclosure"
        - "third party"
      must_match_regex:
        - "GDPR|CCPA"
      finding_count:
        min: 2

    validation:
      schema_check: true

  # ---------------------------------------------------------------------------
  # CATEGORY: Edge Cases
  # ---------------------------------------------------------------------------

  - id: tc018_encrypted_but_logged
    description: "Detect sensitive data encrypted but logged in plain text"
    category: edge_cases
    priority: medium

    input:
      code: |
        app.post('/patient', async (req, res) => {
          const { ssn, diagnosis } = req.body;

          // Properly encrypted for storage
          const encryptedSSN = await encrypt(ssn);
          const encryptedDiagnosis = await encrypt(diagnosis);

          await db.insert('patients', {
            ssn: encryptedSSN,
            diagnosis: encryptedDiagnosis
          });

          // But logged in plain text (VIOLATION)
          console.log(`Created patient with SSN: ${ssn}, diagnosis: ${diagnosis}`);

          res.send('Patient created');
        });
      context:
        language: javascript
        framework: express
        regulation: HIPAA

    expected_output:
      must_contain:
        - "log"
        - "plain text"
        - "SSN"
        - "diagnosis"
      severity_classification: high

    validation:
      schema_check: true

  - id: tc019_partial_compliance
    description: "Detect partial compliance with some controls passing"
    category: edge_cases
    priority: medium

    input:
      code: |
        // Partial GDPR compliance
        app.post('/subscribe', (req, res) => {
          // PASS: Has consent
          if (!req.body.consent) {
            return res.status(400).json({ error: 'Consent required' });
          }

          db.insert('subscribers', {
            email: req.body.email,
            consent: true,
            // FAIL: Missing timestamp and IP
            subscribed_at: new Date()
          });

          res.send('Subscribed');
        });

        // PASS: Has data export
        app.get('/user/:id/export', (req, res) => {
          const data = collectUserData(req.params.id);
          res.json(data);
        });

        // FAIL: Incomplete erasure
        app.delete('/user/:id', (req, res) => {
          db.delete('users', { id: req.params.id });
          // Missing: orders, logs, analytics
          res.send('Deleted');
        });
      context:
        language: javascript
        framework: express
        regulation: GDPR

    expected_output:
      must_contain:
        - "partial"
        - "consent"
        - "erasure"
        - "incomplete"
      severity_classification: medium

    validation:
      schema_check: true
      allow_partial: true

  - id: tc020_typescript_compliance
    description: "Detect compliance issues in TypeScript code"
    category: language_support
    priority: medium

    input:
      code: |
        interface PatientRecord {
          id: string;
          ssn: string;  // PHI
          medicalHistory: string[];  // PHI
        }

        export const getPatient = async (
          patientId: string,
          requester: User  // Unused - no access control
        ): Promise<PatientRecord> => {
          // No audit logging
          // No encryption check
          const patient = await db.patients.findOne({ id: patientId });
          return patient as PatientRecord;
        };
      context:
        language: typescript
        framework: nodejs
        regulation: HIPAA

    expected_output:
      must_contain:
        - "HIPAA"
        - "PHI"
        - "access control"
        - "audit"

    validation:
      schema_check: true

# =============================================================================
# SUCCESS CRITERIA
# =============================================================================

success_criteria:
  # 90% of tests must pass overall
  pass_rate: 0.9

  # All critical tests must pass
  critical_pass_rate: 1.0

  # Average reasoning quality score
  avg_reasoning_quality: 0.75

  # Maximum suite execution time (5 minutes)
  max_execution_time_ms: 300000

  # Maximum variance between model results (15%)
  cross_model_variance: 0.15

# =============================================================================
# METADATA
# =============================================================================

metadata:
  author: "qe-security-auditor"
  created: "2026-02-02"
  last_updated: "2026-02-02"
  coverage_target: >
    Major compliance frameworks: GDPR (Articles 7, 17, 20), HIPAA (PHI protection,
    access control, audit logging), PCI-DSS (Requirements 3, 4, 10), SOC2 (CC6, CC8),
    CCPA (opt-out, disclosure). Covers JavaScript/TypeScript and Python applications.
  test_count: 20
  frameworks_covered:
    - GDPR
    - HIPAA
    - PCI-DSS
    - SOC2
    - CCPA

{
  "$schema": "https://json-schema.org/draft/2020-12/schema",
  "$id": "https://agentic-qe.dev/schemas/compliance-testing-output.json",
  "title": "Compliance Testing Skill Output Schema",
  "description": "Schema for compliance-testing skill output validation. Validates GDPR, HIPAA, SOC2, PCI-DSS, and CCPA compliance audit results.",
  "type": "object",
  "required": ["skillName", "version", "timestamp", "status", "trustTier", "output"],
  "properties": {
    "skillName": {
      "type": "string",
      "const": "compliance-testing",
      "description": "Must be 'compliance-testing'"
    },
    "version": {
      "type": "string",
      "pattern": "^\\d+\\.\\d+\\.\\d+(-[a-zA-Z0-9]+)?$",
      "description": "Semantic version of the skill"
    },
    "timestamp": {
      "type": "string",
      "format": "date-time",
      "description": "ISO 8601 timestamp of output generation"
    },
    "status": {
      "type": "string",
      "enum": ["success", "partial", "failed", "skipped"],
      "description": "Overall execution status"
    },
    "trustTier": {
      "type": "integer",
      "const": 3,
      "description": "Trust tier 3: has schema, validator, and eval suite"
    },
    "output": {
      "type": "object",
      "required": ["summary", "complianceFrameworks", "overallComplianceScore"],
      "properties": {
        "summary": {
          "type": "string",
          "minLength": 20,
          "maxLength": 3000,
          "description": "Human-readable summary of compliance audit findings"
        },
        "overallComplianceScore": {
          "$ref": "#/$defs/complianceScore",
          "description": "Overall compliance score across all frameworks"
        },
        "complianceFrameworks": {
          "type": "array",
          "items": {
            "$ref": "#/$defs/complianceFramework"
          },
          "minItems": 1,
          "maxItems": 10,
          "description": "List of compliance frameworks audited"
        },
        "controls": {
          "type": "array",
          "items": {
            "$ref": "#/$defs/complianceControl"
          },
          "maxItems": 500,
          "description": "Individual compliance controls evaluated"
        },
        "findings": {
          "type": "array",
          "items": {
            "$ref": "#/$defs/complianceFinding"
          },
          "maxItems": 200,
          "description": "Compliance violations and gaps identified"
        },
        "riskAssessment": {
          "$ref": "#/$defs/riskAssessment",
          "description": "Overall risk assessment from compliance gaps"
        },
        "recommendations": {
          "type": "array",
          "items": {
            "$ref": "#/$defs/recommendation"
          },
          "maxItems": 100,
          "description": "Actionable remediation recommendations"
        },
        "auditTrail": {
          "$ref": "#/$defs/auditTrail",
          "description": "Audit trail and evidence collection metadata"
        },
        "dataPrivacy": {
          "$ref": "#/$defs/dataPrivacyAssessment",
          "description": "Data privacy specific assessment (GDPR/CCPA)"
        },
        "artifacts": {
          "type": "array",
          "items": {
            "$ref": "#/$defs/artifact"
          },
          "maxItems": 50,
          "description": "Generated compliance reports and evidence"
        }
      }
    },
    "metadata": {
      "type": "object",
      "properties": {
        "executionTimeMs": {
          "type": "integer",
          "minimum": 0,
          "maximum": 3600000
        },
        "toolsUsed": {
          "type": "array",
          "items": {
            "type": "string"
          },
          "uniqueItems": true
        },
        "agentId": {
          "type": "string",
          "pattern": "^qe-[a-z][a-z0-9-]*$"
        },
        "modelUsed": {
          "type": "string"
        },
        "targetPath": {
          "type": "string"
        },
        "targetUrl": {
          "type": "string",
          "format": "uri"
        },
        "regulationsScoped": {
          "type": "array",
          "items": {
            "type": "string",
            "enum": ["GDPR", "HIPAA", "SOC2", "PCI-DSS", "CCPA", "ISO27001", "NIST", "FedRAMP"]
          },
          "description": "Regulations included in this compliance audit"
        },
        "auditDate": {
          "type": "string",
          "format": "date",
          "description": "Date of the compliance audit"
        },
        "auditScope": {
          "type": "string",
          "description": "Description of what was included in the audit"
        }
      }
    },
    "validation": {
      "type": "object",
      "properties": {
        "schemaValid": {
          "type": "boolean"
        },
        "contentValid": {
          "type": "boolean"
        },
        "confidence": {
          "type": "number",
          "minimum": 0,
          "maximum": 1
        },
        "warnings": {
          "type": "array",
          "items": {
            "type": "string"
          }
        },
        "errors": {
          "type": "array",
          "items": {
            "type": "string"
          }
        }
      }
    },
    "learning": {
      "type": "object",
      "properties": {
        "patternsDetected": {
          "type": "array",
          "items": {
            "type": "string"
          }
        },
        "reward": {
          "type": "number",
          "minimum": 0,
          "maximum": 1
        }
      }
    }
  },
  "$defs": {
    "complianceScore": {
      "type": "object",
      "required": ["value", "max", "percentage"],
      "properties": {
        "value": {
          "type": "number",
          "minimum": 0,
          "description": "Score value (controls passed)"
        },
        "max": {
          "type": "number",
          "minimum": 1,
          "description": "Maximum possible score (total controls)"
        },
        "percentage": {
          "type": "number",
          "minimum": 0,
          "maximum": 100,
          "description": "Compliance percentage"
        },
        "grade": {
          "type": "string",
          "pattern": "^[A-F][+-]?$",
          "description": "Letter grade (A, B, C, D, F)"
        },
        "status": {
          "type": "string",
          "enum": ["compliant", "partial", "non-compliant", "not-applicable"],
          "description": "Compliance status determination"
        },
        "trend": {
          "type": "string",
          "enum": ["improving", "stable", "declining", "unknown"],
          "description": "Trend compared to previous audits"
        }
      }
    },
    "complianceFramework": {
      "type": "object",
      "required": ["id", "name", "version", "score"],
      "properties": {
        "id": {
          "type": "string",
          "enum": ["GDPR", "HIPAA", "SOC2", "PCI-DSS", "CCPA", "ISO27001", "NIST", "FedRAMP"],
          "description": "Framework identifier"
        },
        "name": {
          "type": "string",
          "description": "Full framework name"
        },
        "version": {
          "type": "string",
          "description": "Framework version (e.g., 'v4.0' for PCI-DSS)"
        },
        "score": {
          "$ref": "#/$defs/complianceScore"
        },
        "controlCategories": {
          "type": "array",
          "items": {
            "$ref": "#/$defs/controlCategory"
          },
          "description": "Breakdown by control category"
        },
        "applicableRequirements": {
          "type": "integer",
          "minimum": 0,
          "description": "Number of requirements applicable to this system"
        },
        "exemptions": {
          "type": "array",
          "items": {
            "type": "string"
          },
          "description": "List of exempted requirements with justification"
        }
      }
    },
    "controlCategory": {
      "type": "object",
      "required": ["id", "name", "score"],
      "properties": {
        "id": {
          "type": "string",
          "description": "Category identifier (e.g., 'CC6' for SOC2, 'Art.17' for GDPR)"
        },
        "name": {
          "type": "string",
          "description": "Category name"
        },
        "score": {
          "$ref": "#/$defs/complianceScore"
        },
        "controlCount": {
          "type": "integer",
          "minimum": 0
        },
        "criticalGaps": {
          "type": "integer",
          "minimum": 0
        }
      }
    },
    "complianceControl": {
      "type": "object",
      "required": ["id", "requirement", "status"],
      "properties": {
        "id": {
          "type": "string",
          "pattern": "^[A-Z0-9]+-[A-Z0-9.-]+$",
          "description": "Control identifier (e.g., 'GDPR-Art17', 'PCI-3.4', 'SOC2-CC6.1')"
        },
        "framework": {
          "type": "string",
          "enum": ["GDPR", "HIPAA", "SOC2", "PCI-DSS", "CCPA", "ISO27001", "NIST", "FedRAMP"]
        },
        "requirement": {
          "type": "string",
          "minLength": 10,
          "maxLength": 1000,
          "description": "Full requirement text"
        },
        "status": {
          "type": "string",
          "enum": ["pass", "fail", "partial", "not-applicable", "not-tested"],
          "description": "Control compliance status"
        },
        "evidence": {
          "type": "string",
          "maxLength": 5000,
          "description": "Evidence supporting the status determination"
        },
        "evidenceType": {
          "type": "string",
          "enum": ["code", "config", "log", "document", "interview", "observation", "automated-test"],
          "description": "Type of evidence collected"
        },
        "testedBy": {
          "type": "string",
          "description": "Agent or tool that performed the test"
        },
        "testedAt": {
          "type": "string",
          "format": "date-time"
        },
        "location": {
          "$ref": "#/$defs/location"
        },
        "gaps": {
          "type": "array",
          "items": {
            "type": "string"
          },
          "description": "Specific gaps identified for this control"
        },
        "remediationRequired": {
          "type": "boolean",
          "description": "Whether remediation is required"
        },
        "remediationDeadline": {
          "type": "string",
          "format": "date",
          "description": "Deadline for remediation based on severity"
        }
      }
    },
    "complianceFinding": {
      "type": "object",
      "required": ["id", "title", "severity", "framework", "controlId"],
      "properties": {
        "id": {
          "type": "string",
          "pattern": "^COMP-\\d{3,6}$",
          "description": "Finding identifier (e.g., COMP-001)"
        },
        "title": {
          "type": "string",
          "minLength": 10,
          "maxLength": 200
        },
        "description": {
          "type": "string",
          "maxLength": 3000
        },
        "severity": {
          "type": "string",
          "enum": ["critical", "high", "medium", "low", "info"],
          "description": "Severity based on regulatory impact"
        },
        "framework": {
          "type": "string",
          "enum": ["GDPR", "HIPAA", "SOC2", "PCI-DSS", "CCPA", "ISO27001", "NIST", "FedRAMP"]
        },
        "controlId": {
          "type": "string",
          "description": "Related control identifier"
        },
        "category": {
          "type": "string",
          "enum": [
            "data-privacy",
            "access-control",
            "encryption",
            "audit-logging",
            "data-retention",
            "consent-management",
            "data-subject-rights",
            "incident-response",
            "vendor-management",
            "security-awareness",
            "physical-security",
            "change-management"
          ]
        },
        "evidence": {
          "type": "string",
          "maxLength": 5000
        },
        "location": {
          "$ref": "#/$defs/location"
        },
        "regulatoryImpact": {
          "type": "string",
          "maxLength": 1000,
          "description": "Potential regulatory consequences of non-compliance"
        },
        "penaltyRisk": {
          "type": "string",
          "maxLength": 500,
          "description": "Estimated penalty risk (e.g., 'Up to 4% global revenue')"
        },
        "remediation": {
          "type": "string",
          "maxLength": 2000
        },
        "remediationEffort": {
          "type": "string",
          "enum": ["trivial", "low", "medium", "high", "major"]
        },
        "dueDate": {
          "type": "string",
          "format": "date",
          "description": "Remediation due date"
        },
        "assignee": {
          "type": "string",
          "description": "Person/team responsible for remediation"
        }
      }
    },
    "riskAssessment": {
      "type": "object",
      "required": ["overallRiskLevel", "riskScore"],
      "properties": {
        "overallRiskLevel": {
          "type": "string",
          "enum": ["critical", "high", "medium", "low", "minimal"],
          "description": "Overall compliance risk level"
        },
        "riskScore": {
          "type": "number",
          "minimum": 0,
          "maximum": 100,
          "description": "Numeric risk score (0=no risk, 100=critical risk)"
        },
        "riskFactors": {
          "type": "array",
          "items": {
            "$ref": "#/$defs/riskFactor"
          },
          "description": "Individual risk factors contributing to overall risk"
        },
        "dataExposureRisk": {
          "type": "string",
          "enum": ["none", "low", "medium", "high", "critical"],
          "description": "Risk of sensitive data exposure"
        },
        "regulatoryActionRisk": {
          "type": "string",
          "enum": ["none", "low", "medium", "high", "critical"],
          "description": "Risk of regulatory enforcement action"
        },
        "reputationalRisk": {
          "type": "string",
          "enum": ["none", "low", "medium", "high", "critical"],
          "description": "Risk of reputational damage"
        },
        "financialImpact": {
          "type": "object",
          "properties": {
            "estimatedPenalty": {
              "type": "string",
              "description": "Estimated maximum penalty (e.g., '$10M or 4% revenue')"
            },
            "remediationCost": {
              "type": "string",
              "description": "Estimated remediation cost"
            }
          }
        }
      }
    },
    "riskFactor": {
      "type": "object",
      "required": ["factor", "impact", "likelihood"],
      "properties": {
        "factor": {
          "type": "string",
          "description": "Risk factor description"
        },
        "impact": {
          "type": "string",
          "enum": ["critical", "high", "medium", "low", "minimal"]
        },
        "likelihood": {
          "type": "string",
          "enum": ["certain", "likely", "possible", "unlikely", "rare"]
        },
        "mitigationStatus": {
          "type": "string",
          "enum": ["mitigated", "partial", "unmitigated"]
        }
      }
    },
    "auditTrail": {
      "type": "object",
      "properties": {
        "auditId": {
          "type": "string",
          "format": "uuid"
        },
        "auditDate": {
          "type": "string",
          "format": "date-time"
        },
        "auditor": {
          "type": "string",
          "description": "Agent or person conducting the audit"
        },
        "scope": {
          "type": "string",
          "description": "Audit scope description"
        },
        "methodology": {
          "type": "string",
          "enum": ["automated", "manual", "hybrid"],
          "description": "Audit methodology used"
        },
        "evidenceCount": {
          "type": "integer",
          "minimum": 0
        },
        "controlsTested": {
          "type": "integer",
          "minimum": 0
        },
        "findingsCount": {
          "type": "integer",
          "minimum": 0
        },
        "previousAuditId": {
          "type": "string",
          "format": "uuid",
          "description": "Reference to previous audit for trend analysis"
        }
      }
    },
    "dataPrivacyAssessment": {
      "type": "object",
      "description": "GDPR/CCPA specific data privacy assessment",
      "properties": {
        "piiDetected": {
          "type": "boolean",
          "description": "Whether PII was detected in scope"
        },
        "piiCategories": {
          "type": "array",
          "items": {
            "type": "string",
            "enum": [
              "name",
              "email",
              "phone",
              "address",
              "ssn",
              "financial",
              "health",
              "biometric",
              "genetic",
              "racial-ethnic",
              "political",
              "religious",
              "sexual-orientation",
              "criminal",
              "location",
              "ip-address",
              "device-id"
            ]
          }
        },
        "dataSubjectRights": {
          "type": "object",
          "properties": {
            "accessRight": {
              "$ref": "#/$defs/rightStatus"
            },
            "rectificationRight": {
              "$ref": "#/$defs/rightStatus"
            },
            "erasureRight": {
              "$ref": "#/$defs/rightStatus"
            },
            "portabilityRight": {
              "$ref": "#/$defs/rightStatus"
            },
            "restrictionRight": {
              "$ref": "#/$defs/rightStatus"
            },
            "objectionRight": {
              "$ref": "#/$defs/rightStatus"
            }
          }
        },
        "consentManagement": {
          "type": "object",
          "properties": {
            "consentTracked": {
              "type": "boolean"
            },
            "consentTimestamped": {
              "type": "boolean"
            },
            "consentWithdrawable": {
              "type": "boolean"
            },
            "consentGranular": {
              "type": "boolean"
            }
          }
        },
        "dataRetention": {
          "type": "object",
          "properties": {
            "retentionPolicyExists": {
              "type": "boolean"
            },
            "retentionEnforced": {
              "type": "boolean"
            },
            "retentionPeriods": {
              "type": "array",
              "items": {
                "type": "object",
                "properties": {
                  "dataCategory": {
                    "type": "string"
                  },
                  "retentionPeriod": {
                    "type": "string"
                  }
                }
              }
            }
          }
        },
        "encryptionStatus": {
          "type": "object",
          "properties": {
            "atRest": {
              "type": "boolean"
            },
            "inTransit": {
              "type": "boolean"
            },
            "algorithm": {
              "type": "string"
            }
          }
        },
        "crossBorderTransfer": {
          "type": "object",
          "properties": {
            "detected": {
              "type": "boolean"
            },
            "adequacyDecision": {
              "type": "boolean"
            },
            "transferMechanism": {
              "type": "string",
              "enum": ["adequacy", "scc", "bcr", "consent", "none"]
            }
          }
        }
      }
    },
    "rightStatus": {
      "type": "object",
      "properties": {
        "implemented": {
          "type": "boolean"
        },
        "tested": {
          "type": "boolean"
        },
        "responseTime": {
          "type": "string",
          "description": "Response time (e.g., '30 days')"
        },
        "issues": {
          "type": "array",
          "items": {
            "type": "string"
          }
        }
      }
    },
    "recommendation": {
      "type": "object",
      "required": ["id", "title", "priority"],
      "properties": {
        "id": {
          "type": "string",
          "pattern": "^REC-\\d{3,6}$"
        },
        "title": {
          "type": "string",
          "minLength": 5,
          "maxLength": 200
        },
        "description": {
          "type": "string",
          "maxLength": 2000
        },
        "priority": {
          "type": "string",
          "enum": ["critical", "high", "medium", "low"]
        },
        "effort": {
          "type": "string",
          "enum": ["trivial", "low", "medium", "high", "major"]
        },
        "impact": {
          "type": "integer",
          "minimum": 1,
          "maximum": 10
        },
        "relatedFindings": {
          "type": "array",
          "items": {
            "type": "string"
          }
        },
        "relatedControls": {
          "type": "array",
          "items": {
            "type": "string"
          }
        },
        "frameworks": {
          "type": "array",
          "items": {
            "type": "string",
            "enum": ["GDPR", "HIPAA", "SOC2", "PCI-DSS", "CCPA", "ISO27001", "NIST", "FedRAMP"]
          }
        },
        "codeExample": {
          "type": "string",
          "maxLength": 5000
        },
        "resources": {
          "type": "array",
          "items": {
            "type": "object",
            "required": ["title", "url"],
            "properties": {
              "title": {
                "type": "string"
              },
              "url": {
                "type": "string",
                "format": "uri"
              }
            }
          }
        }
      }
    },
    "artifact": {
      "type": "object",
      "required": ["type", "path"],
      "properties": {
        "type": {
          "type": "string",
          "enum": ["report", "evidence", "config", "log", "certificate", "policy", "matrix"]
        },
        "path": {
          "type": "string",
          "maxLength": 500
        },
        "format": {
          "type": "string",
          "enum": ["json", "html", "pdf", "md", "csv", "xlsx"]
        },
        "description": {
          "type": "string",
          "maxLength": 500
        },
        "sizeBytes": {
          "type": "integer",
          "minimum": 0
        }
      }
    },
    "location": {
      "type": "object",
      "properties": {
        "file": {
          "type": "string",
          "maxLength": 500
        },
        "line": {
          "type": "integer",
          "minimum": 1
        },
        "column": {
          "type": "integer",
          "minimum": 1
        },
        "url": {
          "type": "string",
          "format": "uri"
        },
        "component": {
          "type": "string",
          "description": "System component or service name"
        },
        "database": {
          "type": "string",
          "description": "Database name if applicable"
        },
        "table": {
          "type": "string",
          "description": "Table name if applicable"
        }
      }
    }
  }
}

{
  "skillName": "compliance-testing",
  "skillVersion": "1.0.0",
  "requiredTools": [
    "jq"
  ],
  "optionalTools": [
    "node",
    "python3",
    "ajv",
    "jsonschema"
  ],
  "schemaPath": "schemas/output.json",
  "requiredFields": [
    "skillName",
    "status",
    "output",
    "timestamp"
  ],
  "requiredNonEmptyFields": [
    "output.summary",
    "output.complianceFrameworks"
  ],
  "mustContainTerms": [],
  "mustNotContainTerms": [
    "TODO",
    "FIXME",
    "placeholder",
    "example.com"
  ],
  "enumValidations": {
    ".status": [
      "success",
      "partial",
      "failed",
      "skipped"
    ]
  }
}