┌─────────────────────────────────────────────────────────────────┐
│                    GOVERNANCE LAYERS                           │
├─────────────────────────────────────────────────────────────────┤
│ Layer 5: Business Governance                                     │
│    - Use case approval, risk assessment, value validation        │
├─────────────────────────────────────────────────────────────────┤
│ Layer 4: Policy Layer                                            │
│    - Rules, constraints, guardrails, compliance policies         │
├─────────────────────────────────────────────────────────────────┤
│ Layer 3: Control Plane                                           │
│    - Enforcement, monitoring, intervention capabilities          │
├─────────────────────────────────────────────────────────────────┤
│ Layer 2: Runtime Security                                        │
│    - Sandboxing, isolation, secure execution                     │
├─────────────────────────────────────────────────────────────────┤
│ Layer 1: Identity & Access                                     │
│    - Authentication, authorization, credential management        │
└─────────────────────────────────────────────────────────────────┘

# Zero-Trust Agent Configuration
agent_identity:
  authentication:
    method: mTLS_with_JWT
    token_lifetime: 15m
    refresh_window: 5m
  
  authorization:
    model: RBAC_with_ABAC
    dynamic_scopes: true
    context_aware: true
  
  credential_rotation:
    frequency: 24h
    automatic: true
    grace_period: 5m

network_security:
  segmentation:
    agent_vlans: isolated
    egress_filtering: strict
    east_west_inspection: enabled
  
  mTLS:
    mutual_verification: required
    cert_pinning: enabled
    revocation_check: ocsp_stapling

Global Policies (Organization-wide)
    ↓
Domain Policies (Finance, HR, Engineering)
    ↓
Workflow Policies (Specific agent capabilities)
    ↓
Runtime Policies (Execution-time constraints)

{
  "policy_id": "finance-agent-v1",
  "version": "1.2.0",
  "applies_to": ["agent:finance-*", "workflow:invoice-processing"],
  "rules": [
    {
      "name": "no_unapproved_transfers",
      "type": "action_block",
      "condition": "action.type == 'bank_transfer' && !action.approved_by",
      "effect": "DENY",
      "message": "Bank transfers require human approval"
    },
    {
      "name": "pii_protection",
      "type": "data_filter",
      "condition": "data.contains_pii && destination.external",
      "effect": "REDACT",
      "message": "PII redacted before external transmission"
    },
    {
      "name": "spend_limits",
      "type": "quota",
      "condition": "monthly_spend > 1000",
      "effect": "ALERT_AND_QUEUE",
      "message": "Monthly spend limit reached, awaiting approval"
    }
  ],
  "enforcement_mode": "ACTIVE",
  "audit_level": "FULL"
}

// Policy enforcement middleware
async function enforcePolicy(context, action) {
  const policies = await loadApplicablePolicies(context.agent);
  
  for (const policy of policies) {
    const result = await evaluatePolicy(policy, context, action);
    
    if (result.effect === 'DENY') {
      await logPolicyViolation(context, action, policy);
      throw new PolicyViolationError(result.message);
    }
    
    if (result.effect === 'ALERT') {
      await notifyComplianceTeam(context, action, policy);
    }
  }
  
  return true; // All policies passed
}

# Execution monitoring decorator
@policy_monitored(policy_id="financial-workflow")
async def process_invoice(invoice_data):
    # Check mid-execution policies
    await checkpoint_policy_check()
    
    # If policy violation detected, pause for review
    if policy_state.requires_review:
        await request_human_approval()
    
    continue_processing()

# Audit pipeline configuration
post_execution_audit:
  triggers:
    - workflow_complete
    - exception_raised
    - manual_review_requested
  
  analyzers:
    - pattern_detection
    - anomaly_scoring
    - compliance_validation
  
  actions:
    high_risk:
      - notify_security_team
      - create_incident_ticket
      - preserve_logs
    
    medium_risk:
      - add_to_review_queue
      - notify_manager
    
    low_risk:
      - log_only

service://agent/{domain}/{agent_name}/{instance_id}

Examples:
- service://agent/finance/invoice-processor/prod-001
- service://agent/support/ticket-classifier/prod-003
- service://agent/ops/system-monitor/prod-002

# Agent role definitions
roles:
  invoice_processor:
    permissions:
      - read:erp:invoices
      - write:erp:invoice_status
      - execute:workflow:approval_request
    constraints:
      max_daily_operations: 1000
      allowed_hours: ["08:00-18:00"]
      allowed_regions: ["us-east-1", "eu-west-1"]
  
  customer_support_agent:
    permissions:
      - read:crm:customer_data
      - write:crm:tickets
      - execute:tool:email_send
    constraints:
      requires_human_oversight: true
      pii_access: redacted

# ABAC policy evaluation
def evaluate_access(agent, resource, action, context):
    attributes = {
        'agent.clearance_level': agent.clearance,
        'agent.department': agent.department,
        'resource.classification': resource.classification,
        'resource.owner_department': resource.owner,
        'context.time_of_day': context.timestamp.hour,
        'context.risk_score': context.risk_score,
        'action.sensitivity': action.sensitivity
    }
    
    # Policy: Agents can access resources in their department
    # during business hours if risk score is low
    if (attributes['agent.department'] == attributes['resource.owner_department'] and
        9 <= attributes['context.time_of_day'] <= 17 and
        attributes['context.risk_score'] < 0.7):
        return AccessDecision.GRANT
    
    return AccessDecision.DENY

// Credential rotation service
interface CredentialRotation {
  // Generate new credentials
  async rotate(agentId: string): Promise<Credentials>;
  
  // Graceful transition
  async transition(agentId: string, gracePeriod: number): Promise<void>;
  
  // Emergency revocation
  async revoke(agentId: string, reason: string): Promise<void>;
}

// Implementation with automatic rotation
class ManagedCredentials implements CredentialRotation {
  async rotate(agentId: string): Promise<Credentials> {
    const newCreds = await this.vault.generate(agentId, {
      ttl: '15m',           // 15-minute lifespan
      renewable: true,      // Can be renewed while active
      max_ttl: '1h'         // Force rotation after 1 hour
    });
    
    // Update agent with new credentials
    await this.agentManager.updateCredentials(agentId, newCreds);
    
    // Schedule next rotation
    this.scheduler.schedule(`rotate-${agentId}`, '14m', () => {
      this.rotate(agentId);
    });
    
    return newCreds;
  }
}

# Kubernetes secret injection for agents
apiVersion: apps/v1
kind: Deployment
metadata:
  name: agent-processor
spec:
  template:
    spec:
      containers:
      - name: agent
        env:
        - name: API_KEY
          valueFrom:
            secretKeyRef:
              name: agent-credentials
              key: api-key
        volumeMounts:
        - name: certs
          mountPath: /etc/agent/certs
          readOnly: true
      volumes:
      - name: certs
        csi:
          driver: secrets-store.csi.k8s.io
          readOnly: true
          volumeAttributes:
            secretProviderClass: agent-mtls-certs

# Core agent metrics
AGENT_METRICS = {
    # Execution metrics
    'agent_execution_duration': Histogram(
        'agent_execution_duration_seconds',
        'Time spent executing agent workflow',
        ['agent_type', 'workflow_id']
    ),
    'agent_steps_completed': Counter(
        'agent_steps_completed_total',
        'Number of workflow steps completed',
        ['agent_type', 'status']
    ),
    
    # LLM metrics
    'llm_tokens_used': Counter(
        'llm_tokens_used_total',
        'Total tokens consumed by LLM calls',
        ['model', 'agent_type']
    ),
    'llm_latency': Histogram(
        'llm_latency_seconds',
        'LLM API response time',
        ['model', 'operation']
    ),
    
    # Tool usage metrics
    'tool_calls_total': Counter(
        'tool_calls_total',
        'Total tool invocations',
        ['tool_name', 'agent_type']
    ),
    'tool_error_rate': Gauge(
        'tool_error_rate',
        'Percentage of failed tool calls',
        ['tool_name']
    ),
    
    # Cost metrics
    'agent_cost_usd': Counter(
        'agent_cost_usd_total',
        'Total cost of agent execution',
        ['agent_type', 'model']
    )
}

# Business outcome metrics
BUSINESS_METRICS = {
    'tasks_completed': Counter(
        'business_tasks_completed_total',
        'Tasks successfully completed',
        ['task_type', 'priority']
    ),
    'time_saved_minutes': Counter(
        'business_time_saved_minutes',
        'Estimated time saved through automation',
        ['process_type']
    ),
    'human_escalations': Counter(
        'business_human_escalations_total',
        'Cases escalated to human',
        ['reason', 'urgency']
    ),
    'customer_satisfaction': Gauge(
        'business_csat_score',
        'Customer satisfaction rating',
        ['channel', 'agent_type']
    )
}

{
  "timestamp": "2026-06-28T09:45:32.123Z",
  "level": "INFO",
  "trace_id": "abc123-def456-ghi789",
  "span_id": "span123",
  "parent_span_id": "span000",
  "service": "agent-orchestrator",
  "agent": {
    "id": "finance-processor-001",
    "type": "invoice_automation",
    "version": "2.3.1"
  },
  "event": {
    "type": "tool_execution",
    "tool": "erp_api",
    "operation": "fetch_invoice",
    "status": "success",
    "duration_ms": 245
  },
  "context": {
    "workflow_id": "wf-invoice-12345",
    "user_id": "[email protected]",
    "tenant_id": "tenant-001"
  },
  "security": {
    "identity_verified": true,
    "permissions_checked": ["read:erp:invoices"],
    "data_classification": "internal"
  },
  "payload": {
    "invoice_id": "INV-2026-001",
    "amount": 5000.00,
    "currency": "USD"
  }
}

# Agent workflow tracing
from opentelemetry import trace
from opentelemetry.trace import Status, StatusCode

tracer = trace.get_tracer("agent.tracer")

class TracedAgent:
    async def execute_workflow(self, input_data):
        with tracer.start_as_current_span(
            name="agent_workflow",
            attributes={
                "agent.id": self.agent_id,
                "agent.type": self.agent_type,
                "workflow.input_size": len(input_data)
            }
        ) as workflow_span:
            try:
                # Step 1: Data validation
                with tracer.start_span("step_validation") as validation_span:
                    validated_data = await self.validate(input_data)
                    validation_span.set_attribute("validation.errors", 0)
                
                # Step 2: LLM reasoning
                with tracer.start_span("llm_reasoning") as llm_span:
                    plan = await self.llm.generate_plan(validated_data)
                    llm_span.set_attribute("llm.tokens_input", plan.tokens_input)
                    llm_span.set_attribute("llm.tokens_output", plan.tokens_output)
                    llm_span.set_attribute("llm.model", plan.model)
                
                # Step 3: Tool execution
                for idx, tool_call in enumerate(plan.tools):
                    with tracer.start_span(f"tool_{tool_call.name}") as tool_span:
                        tool_span.set_attribute("tool.name", tool_call.name)
                        tool_span.set_attribute("tool.args", str(tool_call.args))
                        
                        result = await self.execute_tool(tool_call)
                        
                        tool_span.set_attribute("tool.status", result.status)
                        tool_span.set_attribute("tool.duration_ms", result.duration)
                
                # Step 4: Response generation
                with tracer.start_span("generate_response") as response_span:
                    response = await self.generate_response(results)
                
                workflow_span.set_status(Status(StatusCode.OK))
                return response
                
            except Exception as e:
                workflow_span.set_status(
                    Status(StatusCode.ERROR, str(e))
                )
                workflow_span.record_exception(e)
                raise

# Propagate trace context through async boundaries
import contextvars

trace_context = contextvars.ContextVar('trace_context')

async def execute_subagent(parent_context, task):
    # Set trace context for subagent
    trace_context.set(parent_context)
    
    # Subagent automatically inherits parent's trace ID
    with tracer.start_span(
        name="subagent_execution",
        context=trace_context.get()
    ):
        return await subagent.execute(task)

# Cryptographically signed audit log
import hashlib
import json
from datetime import datetime

class ImmutableAuditLog:
    def __init__(self, storage_backend):
        self.storage = storage_backend
        self.previous_hash = self._get_last_hash()
    
    async def record_event(self, event: AuditEvent):
        # Create event with metadata
        audit_entry = {
            "timestamp": datetime.utcnow().isoformat(),
            "event_type": event.type,
            "event_data": event.data,
            "actor": event.actor,
            "resource": event.resource,
            "action": event.action,
            "result": event.result,
            "previous_hash": self.previous_hash,
            "sequence_number": await self._get_next_sequence()
        }
        
        # Calculate hash
        entry_json = json.dumps(audit_entry, sort_keys=True)
        entry_hash = hashlib.sha256(entry_json.encode()).hexdigest()
        audit_entry["hash"] = entry_hash
        
        # Sign with HSM if available
        if self.hsm_available:
            audit_entry["signature"] = await self._sign_with_hsm(entry_hash)
        
        # Store with write-once guarantee
        await self.storage.append(audit_entry)
        
        # Update chain
        self.previous_hash = entry_hash
        
        return entry_hash
    
    async def verify_integrity(self):
        """Verify the entire audit chain."""
        entries = await self.storage.read_all()
        
        for i, entry in enumerate(entries):
            # Verify hash chain
            if i > 0:
                expected_previous = entries[i-1]["hash"]
                if entry["previous_hash"] != expected_previous:
                    raise AuditIntegrityError(
                        f"Chain broken at entry {i}"
                    )
            
            # Verify entry hash
            entry_copy = {k: v for k, v in entry.items() if k != "hash"}
            expected_hash = hashlib.sha256(
                json.dumps(entry_copy, sort_keys=True).encode()
            ).hexdigest()
            
            if entry["hash"] != expected_hash:
                raise AuditIntegrityError(
                    f"Hash mismatch at entry {i}"
                )
        
        return True

# Real-time tamper detection
class TamperDetector:
    def __init__(self, audit_log, alert_service):
        self.audit = audit_log
        self.alerts = alert_service
        self.last_verified_hash = None
    
    async def continuous_verification(self):
        """Background task for continuous integrity checks."""
        while True:
            await asyncio.sleep(60)  # Check every minute
            
            try:
                current_hash = await self.audit.get_latest_hash()
                
                if self.last_verified_hash and \
                   current_hash != self.last_verified_hash:
                    # Check if this is legitimate new data
                    is_valid = await self.audit.verify_integrity()
                    
                    if not is_valid:
                        await self.alerts.send_critical(
                            "AUDIT_TAMPER_DETECTED",
                            "Audit log integrity check failed"
                        )
                
                self.last_verified_hash = current_hash
                
            except Exception as e:
                await self.alerts.send_critical(
                    "AUDIT_VERIFICATION_FAILED",
                    f"Audit verification error: {str(e)}"
                )

# Compliance event schema
events:
  AGENT_DEPLOYED:
    category: lifecycle
    retention: 7_years
    fields:
      - agent_id
      - agent_type
      - deployed_by
      - permissions_granted
      - approval_ticket
  
  POLICY_VIOLATION:
    category: security
    retention: 7_years
    fields:
      - policy_id
      - violation_type
      - agent_id
      - attempted_action
      - enforcement_result
      - remediation_action
  
  DATA_ACCESSED:
    category: data_governance
    retention: 3_years
    fields:
      - data_classification
      - access_purpose
      - agent_id
      - data_subject_id  # For GDPR
      - legal_basis
  
  HUMAN_APPROVAL_REQUESTED:
    category: workflow
    retention: 5_years
    fields:
      - request_type
      - requested_by
      - approver
      - approval_deadline
      - escalation_path
  
  LLM_INVOCATION:
    category: ai_operations
    retention: 1_year
    fields:
      - model_provider
      - model_version
      - tokens_input
      - tokens_output
      - cost_usd
      - data_sent_externally
      - pii_detected

# Compliance report generator
class ComplianceReporter:
    def __init__(self, audit_log, event_store):
        self.audit = audit_log
        self.events = event_store
    
    async def generate_gdpr_report(
        self,
        data_subject_id: str,
        start_date: datetime,
        end_date: datetime
    ) -> GDPRReport:
        """Generate GDPR Article 15 data subject access report."""
        
        # Query all events involving this data subject
        events = await self.events.query({
            "data_subject_id": data_subject_id,
            "timestamp": {
                "$gte": start_date,
                "$lte": end_date
            }
        })
        
        report = GDPRReport(
            data_subject_id=data_subject_id,
            report_period=(start_date, end_date),
            generated_at=datetime.utcnow()
        )
        
        # Categorize processing activities
        for event in events:
            if event.type == "DATA_ACCESSED":
                report.add_processing_activity(
                    purpose=event.access_purpose,
                    legal_basis=event.legal_basis,
                    data_categories=event.data_categories,
                    recipients=event.recipients,
                    retention_period=event.retention_period
                )
            
            elif event.type == "DATA_SHARED":
                report.add_third_party_disclosure(
                    recipient=event.recipient,
                    data_categories=event.data_categories,
                    legal_basis=event.legal_basis,
                    safeguards=event.safeguards
                )
        
        # Calculate automated decision-making instances
        automated_decisions = await self.events.count({
            "data_subject_id": data_subject_id,
            "type": "AUTOMATED_DECISION",
            "human_review": False
        })
        report.automated_decisions = automated_decisions
        
        return report
    
    async def generate_soc2_evidence(
        self,
        control_id: str,
        period: DateRange
    ) -> SOCTwoEvidence:
        """Generate SOC 2 Type II control evidence."""
        
        evidence = SOCTwoEvidence(control_id=control_id)
        
        # CC6.1 - Logical access security
        if control_id == "CC6.1":
            access_events = await self.events.query({
                "type": {"$in": ["ACCESS_GRANTED", "ACCESS_REVOKED"]},
                "timestamp": {"$gte": period.start, "$lte": period.end}
            })
            evidence.add_samples(access_events)
            
            # Add access review evidence
            reviews = await self.events.query({
                "type": "ACCESS_REVIEW_COMPLETED",
                "timestamp": {"$gte": period.start, "$lte": period.end}
            })
            evidence.add_samples(reviews)
        
        # CC7.1 - Security monitoring
        elif control_id == "CC7.1":
            monitoring_events = await self.events.query({
                "type": {"$in": ["THREAT_DETECTED", "ALERT_TRIGGERED"]},
                "timestamp": {"$gte": period.start, "$lte": period.end}
            })
            evidence.add_samples(monitoring_events)
        
        return evidence

# Alert severity classification
class AlertSeverity:
    CRITICAL = "critical"      # Immediate response required
    HIGH = "high"              # Response within 15 minutes
    MEDIUM = "medium"          # Response within 1 hour
    LOW = "low"                # Response within 24 hours
    INFO = "info"              # Log only, no action required

# Alert routing rules
ALERT_ROUTING = {
    "policy_violation": {
        "critical": ["security_team", "compliance_officer"],
        "high": ["security_team"]
    },
    "anomaly_detected": {
        "critical": ["ml_team", "ops_team"],
        "high": ["ops_team"]
    },
    "cost_threshold": {
        "critical": ["finance_team", "cto"],
        "high": ["finance_team"]
    },
    "system_error": {
        "critical": ["ops_team", "on_call"],
        "high": ["ops_team"]
    }
}

# Alert deduplication and grouping
class AlertManager:
    def __init__(self):
        self.recent_alerts = TTLCache(maxsize=1000, ttl=300)
        self.alert_groups = {}
    
    async def process_alert(self, alert: Alert):
        # Check for duplicates
        alert_key = self._generate_key(alert)
        if alert_key in self.recent_alerts:
            # Increment counter instead of new alert
            await self._update_existing_alert(alert_key, alert)
            return
        
        # Check for related alerts to group
        group_key = self._find_group_key(alert)
        if group_key:
            await self._add_to_group(group_key, alert)
            return
        
        # Rate limiting
        if await self._is_rate_limited(alert.type):
            await self._queue_alert(alert)
            return
        
        # Send new alert
        await self._dispatch_alert(alert)
        self.recent_alerts[alert_key] = alert
    
    def _generate_key(self, alert: Alert) -> str:
        """Generate unique key for deduplication."""
        return hashlib.md5(
            f"{alert.type}:{alert.agent_id}:{alert.error_code}".encode()
        ).hexdigest()
    
    async def _dispatch_alert(self, alert: Alert):
        # Determine severity
        severity = self._classify_severity(alert)
        
        # Get routing rules
        recipients = ALERT_ROUTING.get(alert.type, {}).get(severity, [])
        
        # Enrich with context
        enriched_alert = await self._enrich_alert(alert)
        
        # Send through appropriate channels
        for recipient in recipients:
            channel = self._get_channel(recipient)
            await channel.send(enriched_alert)

# Real-time dashboard data pipeline
class DashboardPipeline:
    def __init__(self, websocket_manager):
        self.websocket = websocket_manager
        self.metrics_buffer = []
    
    async def stream_metrics(self):
        """Stream real-time metrics to dashboard."""
        while True:
            metrics = await self._collect_current_metrics()
            
            dashboard_update = {
                "timestamp": datetime.utcnow().isoformat(),
                "agents": {
                    "active": metrics.active_agents,
                    "healthy": metrics.healthy_agents,
                    "degraded": metrics.degraded_agents,
                    "failed": metrics.failed_agents
                },
                "executions": {
                    "per_minute": metrics.executions_per_minute,
                    "success_rate": metrics.success_rate,
                    "avg_duration_ms": metrics.avg_duration
                },
                "costs": {
                    "hourly_usd": metrics.hourly_cost,
                    "projected_daily_usd": metrics.projected_daily_cost
                },
                "alerts": {
                    "critical": metrics.critical_alerts,
                    "open_incidents": metrics.open_incidents
                }
            }
            
            await self.websocket.broadcast("metrics", dashboard_update)
            await asyncio.sleep(5)  # Update every 5 seconds
    
    async def _collect_current_metrics(self):
        """Collect current system metrics."""
        return AgentMetrics(
            active_agents=await self._count_active_agents(),
            healthy_agents=await self._count_healthy_agents(),
            executions_per_minute=await self._calculate_execution_rate(),
            hourly_cost=await self._calculate_hourly_cost(),
            critical_alerts=await self._count_critical_alerts()
        )

# Token budget hierarchy
budgets:
  organization:
    daily_tokens: 10_000_000
    monthly_usd: 50_000
    alerts:
      - threshold: 80%
        notify: ["finance_team"]
      - threshold: 95%
        action: "throttle"
        notify: ["cto", "finance_team"]
  
  departments:
    engineering:
      daily_tokens: 4_000_000
      monthly_usd: 20_000
      
    support:
      daily_tokens: 3_000_000
      monthly_usd: 15_000
      
    marketing:
      daily_tokens: 2_000_000
      monthly_usd: 10_000
      
    operations:
      daily_tokens: 1_000_000
      monthly_usd: 5_000
  
  projects:
    invoice_automation:
      daily_tokens: 500_000
      model_tier: "standard"  # GPT-4o
      
    customer_chat:
      daily_tokens: 1_000_000
      model_tier: "fast"  # GPT-4o-mini
      
    code_review:
      daily_tokens: 200_000
      model_tier: "premium"  # GPT-5

# Token usage tracker
class TokenBudgetManager:
    def __init__(self, redis_client):
        self.redis = redis_client
        self.budgets = self._load_budgets()
    
    async def track_usage(
        self,
        agent_id: str,
        tokens_used: int,
        cost_usd: float,
        model: str
    ):
        """Track token usage against budgets."""
        
        timestamp = datetime.utcnow()
        
        # Update counters
        pipeline = self.redis.pipeline()
        
        # Daily counter
        daily_key = f"tokens:daily:{timestamp.strftime('%Y-%m-%d')}:{agent_id}"
        pipeline.incrby(daily_key, tokens_used)
        pipeline.expire(daily_key, 86400 * 2)  # Keep 2 days
        
        # Hourly counter for rate limiting
        hourly_key = f"tokens:hourly:{timestamp.strftime('%Y-%m-%d-%H')}:{agent_id}"
        pipeline.incrby(hourly_key, tokens_used)
        pipeline.expire(hourly_key, 7200)
        
        # Cost tracking
        cost_key = f"cost:daily:{timestamp.strftime('%Y-%m')}"
        pipeline.incrbyfloat(cost_key, cost_usd)
        
        await pipeline.execute()
        
        # Check budget thresholds
        await self._check_budgets(agent_id, tokens_used)
    
    async def _check_budgets(self, agent_id: str, tokens_used: int):
        """Check if usage exceeds budget thresholds."""
        
        agent_budget = self.budgets.get_agent_budget(agent_id)
        current_usage = await self._get_current_usage(agent_id)
        
        utilization = current_usage / agent_budget.daily_tokens
        
        if utilization >= 0.95:
            await self._enforce_throttle(agent_id)
            await self._alert_budget_exceeded(agent_id, utilization)
        elif utilization >= 0.80:
            await self._alert_budget_warning(agent_id, utilization)
    
    async def can_execute(
        self,
        agent_id: str,
        estimated_tokens: int
    ) -> BudgetCheckResult:
        """Check if execution is allowed under current budget."""
        
        agent_budget = self.budgets.get_agent_budget(agent_id)
        current_usage = await self._get_current_usage(agent_id)
        
        if current_usage + estimated_tokens > agent_budget.daily_tokens:
            return BudgetCheckResult(
                allowed=False,
                reason="Daily token budget exceeded",
                current_usage=current_usage,
                budget_limit=agent_budget.daily_tokens
            )
        
        return BudgetCheckResult(allowed=True)

# Dynamic model selection based on task complexity
class ModelRouter:
    def __init__(self):
        self.models = {
            "gpt-4o-mini": {"cost_per_1k": 0.0006, "capability": "basic"},
            "gpt-4o": {"cost_per_1k": 0.005, "capability": "advanced"},
            "gpt-5": {"cost_per_1k": 0.015, "capability": "expert"}
        }
    
    async def select_model(
        self,
        task: Task,
        required_quality: QualityLevel
    ) -> ModelSelection:
        """Select optimal model based on task and quality requirements."""
        
        # Analyze task complexity
        complexity = await self._analyze_complexity(task)
        
        # Get available budget
        budget = await self._get_remaining_budget(task.agent_id)
        
        # Quality-cost tradeoff
        if required_quality == QualityLevel.MINIMUM:
            return ModelSelection(
                model="gpt-4o-mini",
                reasoning="Minimum quality acceptable, maximize cost savings"
            )
        
        elif required_quality == QualityLevel.BALANCED:
            if complexity < 0.5 and budget.utilization < 0.7:
                return ModelSelection(
                    model="gpt-4o-mini",
                    reasoning="Low complexity task, budget available"
                )
            else:
                return ModelSelection(
                    model="gpt-4o",
                    reasoning="Higher complexity requires better model"
                )
        
        elif required_quality == QualityLevel.MAXIMUM:
            return ModelSelection(
                model="gpt-5",
                reasoning="Maximum quality required"
            )

# n8n security environment variables
environment:
  # Authentication
  N8N_BASIC_AUTH_ACTIVE: "true"
  N8N_BASIC_AUTH_USER: "${N8N_ADMIN_USER}"
  N8N_BASIC_AUTH_PASSWORD: "${N8N_ADMIN_PASSWORD}"
  
  # Security headers
  N8N_SECURITY_HEADERS: "true"
  N8N_SECURITY_ACCESS_CONTROL_ALLOW_ORIGIN: "https://tropical-media.work"
  
  # Execution settings
  N8N_EXECUTIONS_MODE: "queue"  # Separate worker processes
  N8N_EXECUTIONS_TIMEOUT: "300"  # 5 minute timeout
  N8N_EXECUTIONS_TIMEOUT_MAX: "7200"  # 2 hour max
  
  # Audit logging
  N8N_LOG_OUTPUT: "file"
  N8N_LOG_FILE_LOCATION: "/var/log/n8n/"
  N8N_LOG_FILE_COUNT: "30"
  
  # Credential security
  N8N_ENCRYPTION_KEY: "${N8N_ENCRYPTION_KEY}"
  
  # Workflow restrictions
  N8N_BLOCK_ENV_ACCESS_IN_NODE: "true"
  N8N_NODE_ALLOW_LIST: ".n8n/nodes/allowed.json"

// n8n Governance Check Node
const policyService = require('./services/policyService');
const auditService = require('./services/auditService');

// Pre-execution policy check
async function execute() {
    const workflowContext = {
        workflowId: $workflow.id,
        executionId: $execution.id,
        userId: $execution.userId,
        nodes: $workflow.nodes
    };
    
    // Check if workflow is allowed to run
    const policyCheck = await policyService.evaluateWorkflow(workflowContext);
    
    if (!policyCheck.allowed) {
        // Log policy violation
        await auditService.logEvent({
            type: 'WORKFLOW_BLOCKED',
            workflowId: workflowContext.workflowId,
            reason: policyCheck.reason,
            violatedPolicies: policyCheck.violatedPolicies
        });
        
        return [{ 
            json: { 
                error: 'Workflow execution blocked by policy',
                reason: policyCheck.reason 
            } 
        }];
    }
    
    // Log approved execution
    await auditService.logEvent({
        type: 'WORKFLOW_APPROVED',
        workflowId: workflowContext.workflowId,
        policiesChecked: policyCheck.checkedPolicies
    });
    
    return [{ json: { approved: true } }];
}

module.exports = { execute };

// n8n Custom Metrics Node
const prometheus = require('prom-client');

// Define metrics
const workflowDuration = new prometheus.Histogram({
    name: 'n8n_workflow_duration_seconds',
    help: 'Duration of workflow execution',
    labelNames: ['workflow_id', 'status'],
    buckets: [0.1, 0.5, 1, 2, 5, 10, 30, 60]
});

const nodeExecutions = new prometheus.Counter({
    name: 'n8n_node_executions_total',
    help: 'Total node executions',
    labelNames: ['workflow_id', 'node_type', 'status']
});

async function execute() {
    const startTime = Date.now();
    const workflowId = $workflow.id;
    
    try {
        // Track node execution
        nodeExecutions.inc({
            workflow_id: workflowId,
            node_type: 'custom_metrics',
            status: 'success'
        });
        
        // Calculate and record duration
        const duration = (Date.now() - startTime) / 1000;
        workflowDuration.observe(
            { workflow_id: workflowId, status: 'success' },
            duration
        );
        
        // Push to Prometheus
        await prometheus.pushAdd({
            jobName: 'n8n-workflows',
            groupings: { workflow_id: workflowId }
        }, 'http://prometheus-pushgateway:9091');
        
        return [{ json: { metrics_pushed: true } }];
        
    } catch (error) {
        nodeExecutions.inc({
            workflow_id: workflowId,
            node_type: 'custom_metrics',
            status: 'error'
        });
        throw error;
    }
}

module.exports = { execute };

# OpenClaw agent security configuration
agent_security:
  authentication:
    require_approval_for:
      - external_api_calls
      - database_writes
      - file_system_access
      - email_sending
    
    mcp_server_security:
      verify_signatures: true
      allowed_origins:
        - "https://mcp.tropical-media.work"
        - "https://internal-mcp.local"
      
      rate_limits:
        requests_per_minute: 60
        burst_allowance: 10
  
  execution_sandbox:
    mode: "restricted"
    network_access: "outbound_only"
    file_system: "read_only"
    environment_variables: "whitelisted"
    
  audit:
    log_all_tool_calls: true
    log_llm_prompts: true
    log_reasoning: true
    retention_days: 90

# OpenClaw governance integration
from openclaw import Agent, PolicyEnforcer

class GovernedOpenClawAgent:
    def __init__(self, agent_config):
        self.agent = Agent(agent_config)
        self.policy_enforcer = PolicyEnforcer()
        self.audit_logger = AuditLogger()
    
    async def execute_with_governance(self, task):
        """Execute agent task with full governance controls."""
        
        # Generate execution context
        context = ExecutionContext(
            agent_id=self.agent.id,
            user_id=task.user_id,
            session_id=task.session_id,
            timestamp=datetime.utcnow(),
            request_id=generate_uuid()
        )
        
        # Pre-execution policy check
        policy_result = await self.policy_enforcer.check(
            context=context,
            action=task.action,
            resources=task.resources
        )
        
        if not policy_result.allowed:
            await self.audit_logger.log_policy_violation(
                context=context,
                task=task,
                violation=policy_result
            )
            raise PolicyViolationError(policy_result.reason)
        
        # Execute with monitoring
        with self._create_span(context) as span:
            try:
                result = await self.agent.execute(task)
                
                # Post-execution audit
                await self.audit_logger.log_success(
                    context=context,
                    task=task,
                    result=result,
                    duration=span.duration
                )
                
                return result
                
            except Exception as e:
                await self.audit_logger.log_failure(
                    context=context,
                    task=task,
                    error=e
                )
                raise

# OpenClaw agent monitoring
class OpenClawMonitor:
    def __init__(self, metrics_collector):
        self.metrics = metrics_collector
    
    def instrument_agent(self, agent: Agent):
        """Add observability instrumentation to OpenClaw agent."""
        
        # Wrap tool execution
        original_execute_tool = agent.execute_tool
        
        async def monitored_execute_tool(tool_name, params):
            start_time = time.time()
            
            try:
                result = await original_execute_tool(tool_name, params)
                
                # Record success metrics
                self.metrics.record_tool_execution(
                    tool=tool_name,
                    status="success",
                    duration=time.time() - start_time
                )
                
                return result
                
            except Exception as e:
                # Record failure metrics
                self.metrics.record_tool_execution(
                    tool=tool_name,
                    status="error",
                    duration=time.time() - start_time,
                    error_type=type(e).__name__
                )
                raise
        
        agent.execute_tool = monitored_execute_tool
        
        # Wrap LLM calls
        original_llm_call = agent.llm.complete
        
        async def monitored_llm_call(prompt, **kwargs):
            start_time = time.time()
            
            result = await original_llm_call(prompt, **kwargs)
            
            # Record LLM metrics
            self.metrics.record_llm_invocation(
                model=kwargs.get('model', 'default'),
                tokens_input=result.usage.prompt_tokens,
                tokens_output=result.usage.completion_tokens,
                duration=time.time() - start_time,
                cost=result.usage.total_cost
            )
            
            return result
        
        agent.llm.complete = monitored_llm_call
        
        return agent

┌─────────────────────────────────────────────────────────────────┐
│                    GOVERNANCE DASHBOARD                        │
├─────────────────────────────────────────────────────────────────┤
│  ┌──────────────┐  ┌──────────────┐  ┌──────────────┐          │
│  │ Agent Health │  │ Compliance   │  │ Cost Tracking│          │
│  │ Status       │  │ Overview     │  │ & Budgets    │          │
│  └──────────────┘  └──────────────┘  └──────────────┘          │
├─────────────────────────────────────────────────────────────────┤
│  ┌─────────────────────────────────────────────────────────┐   │
│  │                 REAL-TIME ACTIVITY                     │   │
│  │  • Active workflows      • Current executions          │   │
│  │  • Tool usage heatmap    • LLM cost per hour           │   │
│  └─────────────────────────────────────────────────────────┘   │
├─────────────────────────────────────────────────────────────────┤
│  ┌───────────────────┐  ┌──────────────────────────────────┐   │
│  │   ALERTS          │  │   RECENT AUDIT EVENTS            │   │
│  │  • Critical: 2    │  │  • Policy violation - Finance    │   │
│  │  • High: 5        │  │  • Access granted - DevOps      │   │
│  │  • Open Incidents │  │  • Cost threshold exceeded       │   │
│  └───────────────────┘  └──────────────────────────────────┘   │
└─────────────────────────────────────────────────────────────────┘

# FastAPI governance dashboard backend
from fastapi import FastAPI, WebSocket
from fastapi.middleware.cors import CORSMiddleware

app = FastAPI(title="AI Governance Dashboard")

# CORS for dashboard access
app.add_middleware(
    CORSMiddleware,
    allow_origins=["https://governance.tropical-media.work"],
    allow_credentials=True,
    allow_methods=["*"],
    allow_headers=["*"],
)

# Dashboard data endpoints
@app.get("/api/dashboard/overview")
async def get_dashboard_overview():
    """Get high-level dashboard metrics."""
    return {
        "agents": {
            "total": await agent_registry.count(),
            "healthy": await health_checker.count_healthy(),
            "degraded": await health_checker.count_degraded(),
            "failed": await health_checker.count_failed()
        },
        "executions": {
            "last_hour": await execution_store.count_last_hour(),
            "success_rate": await execution_store.calculate_success_rate(),
            "avg_duration_ms": await execution_store.avg_duration()
        },
        "compliance": {
            "overall_score": await compliance_calculator.score(),
            "open_findings": await compliance_store.count_open_findings(),
            "last_audit": await audit_log.last_audit_date()
        },
        "costs": {
            "today_usd": await cost_tracker.today(),
            "mtd_usd": await cost_tracker.month_to_date(),
            "projected_month_usd": await cost_tracker.projected_month()
        }
    }

@app.get("/api/agents/{agent_id}/details")
async def get_agent_details(agent_id: str):
    """Get detailed information about a specific agent."""
    agent = await agent_registry.get(agent_id)
    
    return {
        "metadata": agent.metadata,
        "health": await health_checker.check(agent_id),
        "recent_executions": await execution_store.recent(agent_id, limit=20),
        "cost_breakdown": await cost_tracker.breakdown(agent_id),
        "compliance_status": await compliance_calculator.agent_score(agent_id),
        "policy_violations": await audit_log.violations(agent_id, days=30)
    }

@app.websocket("/ws/realtime")
async def realtime_websocket(websocket: WebSocket):
    """WebSocket for real-time dashboard updates."""
    await websocket.accept()
    
    try:
        while True:
            # Collect real-time metrics
            update = {
                "timestamp": datetime.utcnow().isoformat(),
                "active_executions": await execution_store.active_count(),
                "current_throughput": await metrics_collector.current_tps(),
                "alerts": await alert_manager.recent(5)
            }
            
            await websocket.send_json(update)
            await asyncio.sleep(5)
            
    except Exception:
        await websocket.close()

# Audit log query endpoint
@app.get("/api/audit/search")
async def search_audit_logs(
    start_date: datetime,
    end_date: datetime,
    event_type: Optional[str] = None,
    agent_id: Optional[str] = None,
    severity: Optional[str] = None,
    limit: int = 100
):
    """Search and filter audit logs."""
    query = AuditQuery(
        start_date=start_date,
        end_date=end_date,
        event_type=event_type,
        agent_id=agent_id,
        severity=severity,
        limit=limit
    )
    
    results = await audit_log.search(query)
    return {"results": results, "total": len(results)}

# SOC 2 Common Criteria implementation
soc2_controls:
  CC6.1:  # Logical Access Security
    implementations:
      - name: "Agent Identity Management"
        description: "All agents have unique service identities"
        evidence_sources:
          - agent_registry
          - identity_provider
        
      - name: "Role-Based Access Control"
        description: "RBAC enforced for all agent operations"
        evidence_sources:
          - policy_engine
          - access_control_logs
    
  CC6.2:  # Prior to Access
    implementations:
      - name: "Pre-Execution Policy Check"
        description: "All agent actions validated before execution"
        evidence_sources:
          - policy_enforcement_logs
          - approval_workflows
  
  CC7.1:  # Security Monitoring
    implementations:
      - name: "Real-Time Agent Monitoring"
        description: "Continuous monitoring of agent behavior"
        evidence_sources:
          - metrics_collector
          - alert_manager
      
      - name: "Anomaly Detection"
        description: "ML-based detection of unusual agent behavior"
        evidence_sources:
          - anomaly_detector
          - incident_response_logs
  
  CC7.2:  # Incident Detection
    implementations:
      - name: "Automated Alerting"
        description: "Real-time alerts for security events"
        evidence_sources:
          - alert_manager
          - notification_logs
  
  A1.2:  # Availability
    implementations:
      - name: "Agent Health Checks"
        description: "Continuous health monitoring and failover"
        evidence_sources:
          - health_checker
          - uptime_metrics

# Automated SOC 2 evidence collection
class SOC2EvidenceCollector:
    def __init__(self):
        self.evidence_store = EvidenceStore()
    
    async def collect_cc61_evidence(self, period: DateRange):
        """Collect evidence for CC6.1 - Logical Access."""
        
        evidence = {
            "control": "CC6.1",
            "period": period,
            "collected_at": datetime.utcnow(),
            "samples": []
        }
        
        # Sample 1: Agent identity configurations
        agent_configs = await self._sample_agent_configs(period, n=50)
        evidence["samples"].append({
            "type": "agent_identities",
            "description": "Service identity configurations for agents",
            "data": agent_configs,
            "count": len(agent_configs)
        })
        
        # Sample 2: Access control decisions
        access_decisions = await self._sample_access_decisions(period, n=100)
        evidence["samples"].append({
            "type": "access_control_logs",
            "description": "Authorization decisions logged",
            "data": access_decisions,
            "count": len(access_decisions)
        })
        
        # Sample 3: Credential rotation evidence
        credential_rotations = await self._get_credential_rotations(period)
        evidence["samples"].append({
            "type": "credential_rotations",
            "description": "Automated credential rotation events",
            "data": credential_rotations,
            "count": len(credential_rotations)
        })
        
        await self.evidence_store.store(evidence)
        return evidence

# GDPR data subject rights implementation
class GDPRAgentCompliance:
    def __init__(self, data_inventory, audit_log):
        self.data_inventory = data_inventory
        self.audit = audit_log
    
    async def handle_access_request(
        self,
        data_subject_id: str
    ) -> DataSubjectAccessReport:
        """Handle GDPR Article 15 access request."""
        
        # Identify all data involving this subject
        data_locations = await self.data_inventory.find_by_subject(
            data_subject_id
        )
        
        report = DataSubjectAccessReport(
            subject_id=data_subject_id,
            request_date=datetime.utcnow()
        )
        
        for location in data_locations:
            # Collect data from each system
            data = await self._collect_from_location(location, data_subject_id)
            report.add_data_source(location, data)
            
            # Log collection for audit
            await self.audit.log_event({
                "type": "GDPR_DATA_ACCESSED",
                "data_subject_id": data_subject_id,
                "system": location.system,
                "purpose": "access_request",
                "legal_basis": "GDPR_Article_15"
            })
        
        return report
    
    async def handle_deletion_request(
        self,
        data_subject_id: str
    ) -> DeletionConfirmation:
        """Handle GDPR Article 17 deletion request."""
        
        deletion_report = DeletionConfirmation(
            subject_id=data_subject_id,
            initiated_at=datetime.utcnow()
        )
        
        # Find all data locations
        locations = await self.data_inventory.find_by_subject(data_subject_id)
        
        for location in locations:
            try:
                # Attempt deletion
                result = await self._delete_from_location(
                    location,
                    data_subject_id
                )
                
                deletion_report.add_success(location, result)
                
            except Exception as e:
                # Log failure, may require manual review
                deletion_report.add_failure(location, str(e))
        
        # Propagate to any agent memory
        await self._purge_from_agent_memory(data_subject_id)
        
        # Log completion
        await self.audit.log_event({
            "type": "GDPR_DELETION_COMPLETED",
            "data_subject_id": data_subject_id,
            "systems_affected": len(locations),
            "timestamp": datetime.utcnow()
        })
        
        return deletion_report
    
    async def _purge_from_agent_memory(self, data_subject_id: str):
        """Remove data subject information from agent memory systems."""
        
        # Check all memory stores
        memory_stores = [
            "conversation_memory",
            "vector_store",
            "episodic_memory",
            "semantic_memory"
        ]
        
        for store in memory_stores:
            entries = await self._find_in_memory(store, data_subject_id)
            
            for entry in entries:
                await self._redact_memory_entry(store, entry)
                
                await self.audit.log_event({
                    "type": "MEMORY_REDACTED",
                    "data_subject_id": data_subject_id,
                    "memory_store": store,
                    "entry_id": entry.id
                })

# AI agent incident classification
class IncidentClassifier:
    SEVERITY_MATRIX = {
        # (impact, urgency) -> severity
        ("high", "immediate"): "CRITICAL",
        ("high", "high"): "HIGH",
        ("medium", "immediate"): "HIGH",
        ("high", "medium"): "MEDIUM",
        ("medium", "high"): "MEDIUM",
        ("low", "immediate"): "MEDIUM",
        ("medium", "medium"): "LOW",
        ("low", "high"): "LOW",
        ("low", "medium"): "INFO",
    }
    
    def classify(self, incident: AgentIncident) -> IncidentClassification:
        """Classify agent incident severity."""
        
        # Determine impact
        impact = self._assess_impact(incident)
        
        # Determine urgency
        urgency = self._assess_urgency(incident)
        
        # Get severity
        severity = self.SEVERITY_MATRIX.get(
            (impact, urgency),
            "LOW"
        )
        
        return IncidentClassification(
            severity=severity,
            impact=impact,
            urgency=urgency,
            recommended_response_time=self._response_time(severity)
        )
    
    def _assess_impact(self, incident: AgentIncident) -> str:
        """Assess business impact of incident."""
        
        # Check for data breach indicators
        if incident.type in ["DATA_EXFILTRATION", "UNAUTHORIZED_ACCESS"]:
            if incident.data_classification == "confidential":
                return "high"
        
        # Check for financial impact
        if incident.estimated_cost_usd and incident.estimated_cost_usd > 10000:
            return "high"
        
        # Check for operational impact
        if incident.affected_agents and len(incident.affected_agents) > 10:
            return "high"
        
        if incident.workflow_downtime_minutes and incident.workflow_downtime_minutes > 30:
            return "medium"
        
        return "low"

# Incident response playbooks
playbooks:
  policy_violation_detected:
    triggers:
      - event: POLICY_VIOLATION
        severity: [high, critical]
    
    steps:
      - action: isolate_agent
        description: "Immediately isolate the violating agent"
        timeout: 30s
      
      - action: preserve_logs
        description: "Capture and preserve all relevant logs"
        timeout: 60s
      
      - action: notify_security
        description: "Alert security team"
        channels: [slack, pagerduty]
      
      - action: create_incident_ticket
        description: "Create tracking ticket"
        system: jira
      
      - action: await_human_review
        description: "Pause for human investigation"
        condition: severity == "critical"
  
  anomalous_cost_spike:
    triggers:
      - event: COST_THRESHOLD_EXCEEDED
        threshold_percent: 200
    
    steps:
      - action: throttle_agent
        description: "Reduce agent execution rate"
        rate_limit: "10%"
      
      - action: analyze_spike
        description: "Identify cause of cost increase"
        timeout: 5m
      
      - action: notify_finance
        description: "Alert finance team"
        
      - action: conditional_escalate
        description: "Escalate if >$1000 over budget"
        condition: overage_usd > 1000
  
  agent_loop_detected:
    triggers:
      - event: INFINITE_LOOP_DETECTED
    
    steps:
      - action: terminate_execution
        description: "Kill the looping workflow"
        force: true
      
      - action: capture_state
        description: "Preserve execution state for debugging"
      
      - action: alert_engineering
        description: "Notify engineering team"
      
      - action: block_temporarily
        description: "Block agent until fixed"
        duration: "1h"

# Automated incident response
class IncidentResponder:
    def __init__(self):
        self.playbooks = self._load_playbooks()
        self.orchestrator = ResponseOrchestrator()
    
    async def handle_incident(self, incident: AgentIncident):
        """Automatically respond to agent incident."""
        
        # Classify incident
        classification = self.classifier.classify(incident)
        
        # Select appropriate playbook
        playbook = self._select_playbook(incident, classification)
        
        if not playbook:
            # No automated response, escalate to humans
            await self._escalate_to_human(incident, classification)
            return
        
        # Execute playbook
        execution_context = {
            "incident": incident,
            "classification": classification,
            "start_time": datetime.utcnow()
        }
        
        for step in playbook.steps:
            try:
                result = await self._execute_step(step, execution_context)
                
                if step.condition and not self._evaluate_condition(
                    step.condition,
                    execution_context
                ):
                    continue
                
                # Log step execution
                await self._log_step(incident, step, result)
                
            except Exception as e:
                # Step failed, escalate
                await self._escalate_step_failure(incident, step, e)
                break
        
        # Update incident status
        await self._update_incident_status(incident, "responded")
    
    async def _execute_step(
        self,
        step: PlaybookStep,
        context: Dict
    ) -> StepResult:
        """Execute a single response step."""
        
        action_handlers = {
            "isolate_agent": self._isolate_agent,
            "preserve_logs": self._preserve_logs,
            "notify_security": self._notify_security,
            "throttle_agent": self._throttle_agent,
            "terminate_execution": self._terminate_execution
        }
        
        handler = action_handlers.get(step.action)
        if not handler:
            raise ValueError(f"Unknown action: {step.action}")
        
        return await handler(context, step.parameters)

# Prompt injection detection and prevention
class PromptInjectionGuard:
    def __init__(self):
        self.classifier = self._load_injection_classifier()
        self.content_filter = ContentFilter()
    
    async def validate_input(self, user_input: str) -> ValidationResult:
        """Validate user input for prompt injection attempts."""
        
        # ML-based classification
        injection_score = await self.classifier.score(user_input)
        
        if injection_score > 0.8:
            return ValidationResult(
                valid=False,
                reason="High-confidence prompt injection detected",
                confidence=injection_score,
                action="BLOCK"
            )
        
        if injection_score > 0.5:
            return ValidationResult(
                valid=False,
                reason="Suspicious input pattern",
                confidence=injection_score,
                action="REQUIRE_REVIEW"
            )
        
        # Additional heuristic checks
        heuristics_result = self._check_heuristics(user_input)
        if heuristics_result.suspicious:
            return ValidationResult(
                valid=False,
                reason=heuristics_result.reason,
                action="REQUIRE_REVIEW"
            )
        
        return ValidationResult(valid=True)
    
    def _check_heuristics(self, text: str) -> HeuristicResult:
        """Apply heuristic checks for injection patterns."""
        
        patterns = [
            r"ignore (previous|above|earlier)",  # Instruction override
            r"system(?: prompt)?[:\s]*",  # System prompt access
            r"you are now.*?:",  # Role manipulation
            r"\{\{.*system.*\}\}",  # Template injection
            r"new instructions?:",  # Instruction injection
            r"disregard (all|previous).*constraint",  # Constraint bypass
        ]
        
        for pattern in patterns:
            if re.search(pattern, text, re.IGNORECASE):
                return HeuristicResult(
                    suspicious=True,
                    reason=f"Matched pattern: {pattern}"
                )
        
        return HeuristicResult(suspicious=False)

# Agent output sanitization
class OutputSanitizer:
    def __init__(self):
        self.pii_detector = PIIDetector()
        self.code_validator = CodeValidator()
    
    async def sanitize(
        self,
        agent_output: str,
        output_type: str,
        destination: str
    ) -> SanitizationResult:
        """Sanitize agent output before use.""""
        
        sanitized = agent_output
        actions_taken = []
        
        # Check for PII
        pii_findings = await self.pii_detector.scan(sanitized)
        if pii_findings:
            if destination == "external":
                # Redact PII for external destinations
                sanitized = await self._redact_pii(sanitized, pii_findings)
                actions_taken.append(f"Redacted {len(pii_findings)} PII instances")
            else:
                actions_taken.append(
                    f"Warning: {len(pii_findings)} PII instances detected"
                )
        
        # Validate code if present
        if output_type == "code":
            code_validation = await self.code_validator.validate(sanitized)
            if not code_validation.safe:
                return SanitizationResult(
                    safe=False,
                    reason=f"Unsafe code detected: {code_validation.issues}",
                    output=None
                )
        
        # Check for injection patterns in output
        if self._contains_injection_patterns(sanitized):
            return SanitizationResult(
                safe=False,
                reason="Output contains potential injection patterns",
                output=None
            )
        
        return SanitizationResult(
            safe=True,
            output=sanitized,
            actions_taken=actions_taken
        )

# Blue-green agent deployment
deployment:
  strategy: blue_green
  
  blue:
    version: "2.3.1"
    traffic_percent: 100
    agent_pool:
      replicas: 5
      resources:
        cpu: "2"
        memory: "4Gi"
    
  green:
    version: "2.4.0"
    traffic_percent: 0
    agent_pool:
      replicas: 5
      resources:
        cpu: "2"
        memory: "4Gi"
  
  promotion:
    health_checks:
      - metric: error_rate
        threshold: "< 0.01"
        duration: 5m
      - metric: latency_p99
        threshold: "< 500ms"
        duration: 5m
    
    canary_steps:
      - traffic: 5
        duration: 10m
      - traffic: 25
        duration: 15m
      - traffic: 50
        duration: 15m
      - traffic: 100
        duration: 0
  
  rollback:
    automatic_on:
      - error_rate > 0.05
      - latency_p99 > 2000ms
      - policy_violations > 0

# Agent deployment orchestrator
class AgentDeploymentOrchestrator:
    def __init__(self):
        self.k8s = KubernetesClient()
        self.metrics = MetricsCollector()
        self.governance = GovernanceValidator()
    
    async def deploy_agent(
        self,
        agent_config: AgentConfig,
        deployment_config: DeploymentConfig
    ) -> DeploymentResult:
        """Deploy agent with governance validation."""
        
        # Pre-deployment governance check
        governance_result = await self.governance.validate_deployment(
            agent_config
        )
        
        if not governance_result.approved:
            return DeploymentResult(
                success=False,
                reason=governance_result.reason,
                blocked_policies=governance_result.violated_policies
            )
        
        # Execute blue-green deployment
        if deployment_config.strategy == "blue_green":
            return await self._blue_green_deploy(agent_config, deployment_config)
        
        elif deployment_config.strategy == "canary":
            return await self._canary_deploy(agent_config, deployment_config)
        
        else:
            return await self._standard_deploy(agent_config, deployment_config)
    
    async def _blue_green_deploy(
        self,
        agent_config: AgentConfig,
        config: DeploymentConfig
    ) -> DeploymentResult:
        """Execute blue-green deployment."""
        
        # Deploy green version
        await self.k8s.deploy(
            name=f"{agent_config.name}-green",
            config=agent_config,
            replicas=config.green.replicas
        )
        
        # Run smoke tests
        smoke_result = await self._run_smoke_tests(
            f"{agent_config.name}-green"
        )
        
        if not smoke_result.success:
            await self.k8s.delete(f"{agent_config.name}-green")
            return DeploymentResult(
                success=False,
                reason=f"Smoke tests failed: {smoke_result.errors}"
            )
        
        # Gradual traffic shift
        for step in config.promotion.canary_steps:
            await self._shift_traffic(agent_config.name, step.traffic)
            
            # Monitor during step
            await asyncio.sleep(step.duration * 60)
            
            health = await self._check_health(agent_config.name)
            if not health.healthy:
                await self._rollback(agent_config.name)
                return DeploymentResult(
                    success=False,
                    reason=f"Health check failed at {step.traffic}% traffic"
                )
        
        # Promote green to blue
        await self._promote_green_to_blue(agent_config.name)
        
        return DeploymentResult(
            success=True,
            version_deployed=agent_config.version
        )

# Agent disaster recovery configuration
disaster_recovery:
  backup:
    frequency: hourly
    retention: 30d
    destinations:
      - s3://tropical-media-agent-backups/primary
      - s3://tropical-media-agent-backups-dr/secondary
    
    contents:
      - agent_configurations
      - workflow_definitions
      - execution_history
      - audit_logs
      - memory_snapshots
  
  rto: 15m  # Recovery Time Objective
  rpo: 5m   # Recovery Point Objective
  
  failover:
    regions:
      primary: us-east-1
      secondary: us-west-2
      tertiary: eu-west-1
    
    trigger_conditions:
      - primary_region_unavailable
      - error_rate > 0.5
      - latency_p99 > 5000ms
  
  testing:
    frequency: monthly
    scope: full_failover
    validation:
      - workflow_execution
      - data_integrity
      - performance_benchmark