┌─────────────────────────────────────────────────────────────────┐
│                    The Production Trap                            │
├─────────────────────────────────────────────────────────────────┤
│                                                                  │
│  Development                    Production                        │
│  ───────────                   ─────────                        │
│  Small datasets               │  Large datasets                   │
│  Fast APIs                  │  Rate-limited, slow APIs           │
│  Single user                │  Concurrent users                 │
│  Predictable inputs         │  Edge cases, malformed data        │
│  Fresh tokens               │  Expired credentials               │
│  Debug mode ON              │  Silent failures                   │
│  Manual testing             │  No automated validation          │
│  Local execution            │  Queue mode complexity             │
│                                                                  │
│  Result: 80% of workflows break within 30 days of deployment    │
│                                                                  │
└─────────────────────────────────────────────────────────────────┘

// Function node: Robust API Call with Retry
const axios = require('axios');

async function makeResilientRequest(config) {
  const maxRetries = config.retries || 3;
  const baseDelay = config.baseDelay || 1000;
  const maxDelay = config.maxDelay || 30000;
  
  for (let attempt = 1; attempt <= maxRetries; attempt++) {
    try {
      const response = await axios(config);
      return {
        success: true,
        data: response.data,
        status: response.status,
        attempt: attempt,
        duration: Date.now() - startTime
      };
    } catch (error) {
      const isRetryable = isRetryableError(error);
      const isLastAttempt = attempt === maxRetries;
      
      if (!isRetryable || isLastAttempt) {
        return {
          success: false,
          error: error.message,
          code: error.response?.status || error.code,
          attempt: attempt,
          retryable: isRetryable,
          timestamp: new Date().toISOString()
        };
      }
      
      // Exponential backoff with jitter
      const delay = Math.min(
        baseDelay * Math.pow(2, attempt - 1) + Math.random() * 1000,
        maxDelay
      );
      
      console.log(`Retry ${attempt}/${maxRetries} after ${delay}ms`);
      await new Promise(resolve => setTimeout(resolve, delay));
    }
  }
}

function isRetryableError(error) {
  const retryableCodes = [408, 429, 500, 502, 503, 504];
  const retryableNetworkErrors = ['ECONNRESET', 'ETIMEDOUT', 'ECONNREFUSED'];
  
  if (error.response) {
    return retryableCodes.includes(error.response.status);
  }
  
  return retryableNetworkErrors.includes(error.code);
}

// Usage in workflow
const result = await makeResilientRequest({
  method: 'POST',
  url: 'https://api.service.com/endpoint',
  data: $input.first().json.payload,
  headers: { 'Authorization': `Bearer ${$env.API_KEY}` },
  retries: 5,
  baseDelay: 2000
});

return [{ json: result }];

// Function node: Global Error Boundary
const { WorkflowInsights } = require('./workflow-insights');

async function handleWorkflowError(error, context) {
  const insights = new WorkflowInsights();
  
  // Capture error context
  const errorReport = {
    workflowId: $env.WORKFLOW_ID,
    executionId: $env.EXECUTION_ID,
    timestamp: new Date().toISOString(),
    node: $env.NODE_NAME,
    error: {
      message: error.message,
      stack: error.stack,
      code: error.code
    },
    context: {
      input: $input.first().json,
      env: Object.keys($env),
      runData: $run
    }
  };
  
  // Log to multiple destinations
  await Promise.all([
    // Primary error tracking
    insights.logError(errorReport),
    
    // Slack alert for critical errors
    notifySlack(errorReport),
    
    // PagerDuty for P0 issues
    error.severity === 'critical' && triggerPagerDuty(errorReport),
    
    // Store for analysis
    storeErrorForAnalysis(errorReport)
  ]);
  
  // Return graceful degradation
  return {
    success: false,
    error: 'Operation failed, team notified',
    referenceId: errorReport.executionId,
    fallbackData: getLastKnownGoodState(context)
  };
}

# docker-compose.yml - Error Workflow Configuration
version: '3.8'
services:
  n8n:
    image: n8nio/n8n:latest
    environment:
      - N8N_DEFAULT_BINARY_DATA_MODE=filesystem
      - EXECUTIONS_MODE=queue
      - N8N_ERROR_TRIGGER_TYPE=webhook
      - N8N_ERROR_TRIGGER_WEBHOOK_URL=https://hooks.service.com/error
      - N8N_ERROR_TRIGGER_LOG_LEVEL=error
    volumes:
      - ./error-workflows:/error-workflows

// circuit-breaker.js - Production-grade circuit breaker
class CircuitBreaker {
  constructor(options = {}) {
    this.failureThreshold = options.failureThreshold || 5;
    this.successThreshold = options.successThreshold || 2;
    this.timeout = options.timeout || 60000;
    this.halfOpenMaxCalls = options.halfOpenMaxCalls || 3;
    
    this.state = 'CLOSED'; // CLOSED, OPEN, HALF_OPEN
    this.failures = 0;
    this.successes = 0;
    this.lastFailureTime = null;
    this.halfOpenCalls = 0;
    
    this.metrics = {
      totalCalls: 0,
      successfulCalls: 0,
      failedCalls: 0,
      rejectedCalls: 0,
      stateTransitions: []
    };
  }

  async execute(operation, ...args) {
    this.metrics.totalCalls++;
    
    if (this.state === 'OPEN') {
      if (this.shouldAttemptReset()) {
        this.transitionTo('HALF_OPEN');
      } else {
        this.metrics.rejectedCalls++;
        throw new CircuitBreakerOpenError('Circuit breaker is OPEN');
      }
    }
    
    if (this.state === 'HALF_OPEN' && this.halfOpenCalls >= this.halfOpenMaxCalls) {
      this.metrics.rejectedCalls++;
      throw new CircuitBreakerOpenError('Circuit breaker is HALF_OPEN (max calls reached)');
    }
    
    if (this.state === 'HALF_OPEN') {
      this.halfOpenCalls++;
    }
    
    try {
      const result = await operation(...args);
      this.onSuccess();
      this.metrics.successfulCalls++;
      return result;
    } catch (error) {
      this.onFailure();
      this.metrics.failedCalls++;
      throw error;
    }
  }

  onSuccess() {
    this.failures = 0;
    
    if (this.state === 'HALF_OPEN') {
      this.successes++;
      if (this.successes >= this.successThreshold) {
        this.transitionTo('CLOSED');
      }
    }
  }

  onFailure() {
    this.failures++;
    this.lastFailureTime = Date.now();
    this.successes = 0;
    this.halfOpenCalls = 0;
    
    if (this.failures >= this.failureThreshold) {
      this.transitionTo('OPEN');
    }
  }

  shouldAttemptReset() {
    return Date.now() - this.lastFailureTime >= this.timeout;
  }

  transitionTo(newState) {
    const oldState = this.state;
    this.state = newState;
    this.failures = 0;
    this.successes = 0;
    this.halfOpenCalls = 0;
    
    this.metrics.stateTransitions.push({
      from: oldState,
      to: newState,
      timestamp: new Date().toISOString()
    });
    
    console.log(`Circuit breaker transitioned: ${oldState} -> ${newState}`);
  }

  getState() {
    return {
      state: this.state,
      failures: this.failures,
      successes: this.successes,
      lastFailureTime: this.lastFailureTime,
      metrics: this.metrics
    };
  }
}

class CircuitBreakerOpenError extends Error {
  constructor(message) {
    super(message);
    this.name = 'CircuitBreakerOpenError';
  }
}

// Usage in n8n
const circuitBreaker = new CircuitBreaker({
  failureThreshold: 5,
  successThreshold: 3,
  timeout: 120000 // 2 minutes
});

// Wrap external API calls
const result = await circuitBreaker.execute(
  async () => {
    return await callExternalAPI(data);
  }
);

module.exports = { CircuitBreaker, CircuitBreakerOpenError };

// fallback-strategies.js
class FallbackStrategies {
  
  // Strategy 1: Cached Response
  static async cachedFallback(cacheKey, primaryOperation, ttl = 3600) {
    const redis = getRedisClient();
    
    try {
      const result = await primaryOperation();
      await redis.setex(cacheKey, ttl, JSON.stringify(result));
      return { source: 'primary', data: result };
    } catch (error) {
      const cached = await redis.get(cacheKey);
      if (cached) {
        return { 
          source: 'cache', 
          data: JSON.parse(cached),
          stale: true 
        };
      }
      throw error;
    }
  }
  
  // Strategy 2: Degraded Mode
  static async degradedFallback(primaryOperation, degradedOperation) {
    try {
      const result = await primaryOperation();
      return { mode: 'full', data: result };
    } catch (error) {
      console.warn('Primary failed, switching to degraded mode:', error.message);
      const degraded = await degradedOperation();
      return { mode: 'degraded', data: degraded, originalError: error.message };
    }
  }
  
  // Strategy 3: Queue for Later
  static async queueFallback(operation, queueName) {
    const queue = getQueue(queueName);
    
    try {
      return await operation();
    } catch (error) {
      if (isRetryableError(error)) {
        await queue.add('delayed-task', operation, {
          delay: 60000,
          attempts: 5,
          backoff: {
            type: 'exponential',
            delay: 2000
          }
        });
        return { 
          status: 'queued', 
          message: 'Operation queued for retry',
          jobId: operation.id
        };
      }
      throw error;
    }
  }
  
  // Strategy 4: Mock/Synthetic Response
  static async mockFallback(primaryOperation, mockGenerator) {
    try {
      return await primaryOperation();
    } catch (error) {
      console.warn('Using mock data due to error:', error.message);
      const mock = await mockGenerator();
      return { 
        source: 'mock', 
        data: mock,
        warning: 'Using synthetic data'
      };
    }
  }
}

// Usage examples
// In n8n Function node:

// Example 1: Cached API calls
const userData = await FallbackStrategies.cachedFallback(
  `user:${userId}`,
  async () => await fetchUserFromAPI(userId),
  1800 // 30 min cache
);

// Example 2: Degraded search
const searchResults = await FallbackStrategies.degradedFallback(
  async () => await performAIEnhancedSearch(query),
  async () => await performBasicKeywordSearch(query)
);

// Example 3: Queue for later
const report = await FallbackStrategies.queueFallback(
  async () => await generateComplexReport(data),
  'report-generation'
);

module.exports = { FallbackStrategies };

┌─────────────────────────────────────────────────────────────────┐
│                    Testing Pyramid                                │
├─────────────────────────────────────────────────────────────────┤
│                                                                  │
│                    ┌──────────────┐                              │
│                    │   E2E Tests  │  5% - Full workflow runs      │
│                    │  (Cypress)   │                               │
│                    └──────┬───────┘                              │
│                           │                                      │
│              ┌────────────┼────────────┐                        │
│              │            │            │                         │
│         ┌────▼────┐ ┌────▼────┐ ┌────▼────┐                      │
│         │Integration│ │Integration│ │Integration│  25% - API     │
│         │  Tests   │ │  Tests   │ │  Tests   │       + nodes     │
│         └────┬────┘ └────┬────┘ └────┬────┘                      │
│              │            │            │                         │
│     ┌────────┴────────────┴────────────┴────────┐               │
│     │              Unit Tests (Jest)             │  70% - Logic   │
│     │        Function nodes, expressions          │               │
│     └─────────────────────────────────────────────┘               │
│                                                                  │
└─────────────────────────────────────────────────────────────────┘

// __tests__/data-transformer.test.js
const { DataTransformer } = require('../nodes/data-transformer');

describe('DataTransformer', () => {
  let transformer;
  
  beforeEach(() => {
    transformer = new DataTransformer();
  });
  
  describe('validateInput', () => {
    it('should accept valid email addresses', () => {
      const result = transformer.validateInput('[email protected]', 'email');
      expect(result.valid).toBe(true);
    });
    
    it('should reject invalid email addresses', () => {
      const result = transformer.validateInput('invalid-email', 'email');
      expect(result.valid).toBe(false);
      expect(result.error).toContain('Invalid email');
    });
    
    it('should handle null inputs gracefully', () => {
      const result = transformer.validateInput(null, 'email');
      expect(result.valid).toBe(false);
      expect(result.error).toContain('required');
    });
  });
  
  describe('transformOrderData', () => {
    it('should transform valid order data', () => {
      const input = {
        orderId: '12345',
        items: [{ sku: 'ABC', qty: 2, price: 29.99 }],
        customer: { id: 'C001', email: '[email protected]' }
      };
      
      const result = transformer.transformOrderData(input);
      
      expect(result.order_id).toBe('12345');
      expect(result.total_amount).toBe(59.98);
      expect(result.item_count).toBe(2);
      expect(result.customer_id).toBe('C001');
    });
    
    it('should calculate totals correctly with discounts', () => {
      const input = {
        orderId: '12345',
        items: [{ sku: 'ABC', qty: 2, price: 29.99 }],
        discount: { type: 'percentage', value: 10 }
      };
      
      const result = transformer.transformOrderData(input);
      
      expect(result.total_amount).toBeCloseTo(53.98, 2);
      expect(result.discount_applied).toBe(5.998);
    });
    
    it('should throw on missing required fields', () => {
      const input = { items: [] };
      
      expect(() => {
        transformer.transformOrderData(input);
      }).toThrow('orderId is required');
    });
  });
  
  describe('sanitizeData', () => {
    it('should remove sensitive fields', () => {
      const input = {
        name: 'John Doe',
        email: '[email protected]',
        ssn: '123-45-6789',
        creditCard: '4111111111111111'
      };
      
      const result = transformer.sanitizeData(input);
      
      expect(result.ssn).toBeUndefined();
      expect(result.creditCard).toBeUndefined();
      expect(result.name).toBe('John Doe');
    });
  });
});

// __tests__/integration/workflow-execution.test.js
const { WorkflowRunner } = require('../utils/workflow-runner');
const { MockWebhookServer } = require('../mocks/webhook-server');

describe('Order Processing Workflow', () => {
  let runner;
  let mockServer;
  
  beforeAll(async () => {
    mockServer = new MockWebhookServer();
    await mockServer.start(3001);
    
    runner = new WorkflowRunner({
      n8nUrl: process.env.N8N_TEST_URL || 'http://localhost:5678',
      apiKey: process.env.N8N_API_KEY
    });
  });
  
  afterAll(async () => {
    await mockServer.stop();
  });
  
  beforeEach(async () => {
    await mockServer.reset();
  });
  
  it('should process valid order and send confirmation', async () => {
    // Setup mocks
    mockServer.mockResponse('stripe', {
      status: 'succeeded',
      chargeId: 'ch_1234567890'
    });
    
    mockServer.mockResponse('sendgrid', { sent: true });
    
    // Execute workflow
    const result = await runner.execute('order-processing', {
      order: {
        id: 'ORD-001',
        amount: 99.99,
        customer: { email: '[email protected]' }
      },
      paymentToken: 'tok_visa'
    });
    
    // Assertions
    expect(result.status).toBe('success');
    expect(result.data.paymentStatus).toBe('succeeded');
    expect(result.data.emailSent).toBe(true);
    
    // Verify mock calls
    expect(mockServer.getCallCount('stripe')).toBe(1);
    expect(mockServer.getCallCount('sendgrid')).toBe(1);
  });
  
  it('should handle payment failure gracefully', async () => {
    mockServer.mockResponse('stripe', {
      error: { code: 'card_declined', message: 'Your card was declined' }
    }, 402);
    
    const result = await runner.execute('order-processing', {
      order: { id: 'ORD-002', amount: 99.99 },
      paymentToken: 'tok_declined'
    });
    
    expect(result.status).toBe('completed_with_errors');
    expect(result.error.code).toBe('payment_failed');
    expect(result.data.emailSent).toBe(true); // Should send failure email
  });
  
  it('should retry on network errors', async () => {
    mockServer.mockFailure('stripe', 'ECONNRESET');
    mockServer.mockFailure('stripe', 'ETIMEDOUT');
    mockServer.mockResponse('stripe', { status: 'succeeded' });
    
    const result = await runner.execute('order-processing', {
      order: { id: 'ORD-003', amount: 99.99 },
      paymentToken: 'tok_visa'
    });
    
    expect(result.status).toBe('success');
    expect(mockServer.getCallCount('stripe')).toBe(3);
  });
  
  it('should respect rate limits', async () => {
    mockServer.mockResponse('api', { data: 'ok' });
    
    const requests = Array(10).fill(null).map((_, i) => 
      runner.execute('rate-limited-workflow', { id: i })
    );
    
    await Promise.all(requests);
    
    const timestamps = mockServer.getTimestamps('api');
    const intervals = timestamps.slice(1).map((t, i) => t - timestamps[i]);
    
    // Should respect 100ms minimum interval
    intervals.forEach(interval => {
      expect(interval).toBeGreaterThanOrEqual(100);
    });
  });
});

// n8n-test-framework.js
class N8NTestFramework {
  constructor(config) {
    this.baseUrl = config.baseUrl;
    this.apiKey = config.apiKey;
    this.timeout = config.timeout || 30000;
  }

  async testWorkflow(workflowId, testCases) {
    const results = [];
    
    for (const testCase of testCases) {
      console.log(`Running test: ${testCase.name}`);
      
      const startTime = Date.now();
      
      try {
        // Trigger workflow execution
        const execution = await this.triggerWorkflow(
          workflowId, 
          testCase.input
        );
        
        // Wait for completion
        const result = await this.waitForExecution(
          execution.executionId,
          testCase.timeout || this.timeout
        );
        
        // Validate result
        const validation = await this.validateResult(
          result,
          testCase.expected
        );
        
        results.push({
          name: testCase.name,
          status: validation.success ? 'PASSED' : 'FAILED',
          duration: Date.now() - startTime,
          error: validation.error,
          output: result
        });
        
      } catch (error) {
        results.push({
          name: testCase.name,
          status: 'ERROR',
          duration: Date.now() - startTime,
          error: error.message
        });
      }
    }
    
    return this.generateReport(results);
  }

  async triggerWorkflow(workflowId, input) {
    const response = await fetch(
      `${this.baseUrl}/webhook/${workflowId}`,
      {
        method: 'POST',
        headers: {
          'Content-Type': 'application/json',
          'X-N8N-API-KEY': this.apiKey
        },
        body: JSON.stringify(input)
      }
    );
    
    if (!response.ok) {
      throw new Error(`Failed to trigger workflow: ${response.statusText}`);
    }
    
    return response.json();
  }

  async waitForExecution(executionId, timeout) {
    const startTime = Date.now();
    
    while (Date.now() - startTime < timeout) {
      const status = await this.getExecutionStatus(executionId);
      
      if (status.status === 'success' || status.status === 'error') {
        return status;
      }
      
      await new Promise(resolve => setTimeout(resolve, 1000));
    }
    
    throw new Error('Execution timeout');
  }

  async validateResult(result, expected) {
    const errors = [];
    
    // Check status
    if (expected.status && result.status !== expected.status) {
      errors.push(`Expected status ${expected.status}, got ${result.status}`);
    }
    
    // Check output fields
    if (expected.output) {
      for (const [key, value] of Object.entries(expected.output)) {
        const actual = this.getNestedValue(result.output, key);
        
        if (JSON.stringify(actual) !== JSON.stringify(value)) {
          errors.push(`Output.${key}: expected ${JSON.stringify(value)}, got ${JSON.stringify(actual)}`);
        }
      }
    }
    
    // Check with custom validator
    if (expected.validator) {
      const validation = expected.validator(result);
      if (!validation.valid) {
        errors.push(validation.error);
      }
    }
    
    return {
      success: errors.length === 0,
      error: errors.join('; ')
    };
  }

  generateReport(results) {
    const passed = results.filter(r => r.status === 'PASSED').length;
    const failed = results.filter(r => r.status === 'FAILED').length;
    const errors = results.filter(r => r.status === 'ERROR').length;
    
    return {
      summary: {
        total: results.length,
        passed,
        failed,
        errors,
        successRate: (passed / results.length * 100).toFixed(2) + '%',
        totalDuration: results.reduce((sum, r) => sum + r.duration, 0)
      },
      details: results
    };
  }
}

// Test case definitions
const testCases = [
  {
    name: 'Valid order processing',
    input: {
      orderId: 'ORD-001',
      amount: 100,
      customer: { email: '[email protected]' }
    },
    expected: {
      status: 'success',
      output: {
        'payment.status': 'completed',
        'email.sent': true
      }
    }
  },
  {
    name: 'Invalid email handling',
    input: {
      orderId: 'ORD-002',
      amount: 50,
      customer: { email: 'invalid-email' }
    },
    expected: {
      status: 'error',
      validator: (result) => ({
        valid: result.error && result.error.code === 'VALIDATION_ERROR',
        error: result.error?.message || 'No error message'
      })
    }
  },
  {
    name: 'Rate limit handling',
    input: { orderId: 'ORD-003' },
    expected: {
      validator: (result) => ({
        valid: result.retries >= 2,
        error: `Expected retries, got ${result.retries}`
      })
    }
  }
];

module.exports = { N8NTestFramework, testCases };

// metrics-collector.js
class MetricsCollector {
  constructor() {
    this.metrics = {
      executions: new Map(),
      errors: new Map(),
      latency: new Map(),
      custom: new Map()
    };
    
    this.flushInterval = setInterval(() => this.flush(), 60000);
  }

  recordExecution(workflowId, status, duration) {
    const key = `${workflowId}:${status}`;
    const current = this.metrics.executions.get(key) || 0;
    this.metrics.executions.set(key, current + 1);
    
    // Track latency percentiles
    if (!this.metrics.latency.has(workflowId)) {
      this.metrics.latency.set(workflowId, []);
    }
    this.metrics.latency.get(workflowId).push(duration);
  }

  recordError(workflowId, errorType, nodeName) {
    const key = `${workflowId}:${errorType}:${nodeName}`;
    const current = this.metrics.errors.get(key) || 0;
    this.metrics.errors.set(key, current + 1);
  }

  recordCustomMetric(name, value, labels = {}) {
    const key = JSON.stringify({ name, ...labels });
    if (!this.metrics.custom.has(key)) {
      this.metrics.custom.set(key, []);
    }
    this.metrics.custom.get(key).push({
      value,
      timestamp: Date.now()
    });
  }

  getPrometheusMetrics() {
    let output = '';
    
    // Execution metrics
    output += '# HELP n8n_workflow_executions_total Total workflow executions\n';
    output += '# TYPE n8n_workflow_executions_total counter\n';
    
    for (const [key, count] of this.metrics.executions) {
      const [workflowId, status] = key.split(':');
      output += `n8n_workflow_executions_total{workflow_id="${workflowId}",status="${status}"} ${count}\n`;
    }
    
    // Latency metrics
    output += '# HELP n8n_workflow_execution_duration_seconds Workflow execution duration\n';
    output += '# TYPE n8n_workflow_execution_duration_seconds histogram\n';
    
    for (const [workflowId, durations] of this.metrics.latency) {
      const buckets = [0.1, 0.5, 1, 2, 5, 10, 30];
      const sorted = durations.sort((a, b) => a - b);
      
      buckets.forEach(bucket => {
        const count = sorted.filter(d => d <= bucket).length;
        output += `n8n_workflow_execution_duration_seconds_bucket{workflow_id="${workflowId}",le="${bucket}"} ${count}\n`;
      });
      output += `n8n_workflow_execution_duration_seconds_count{workflow_id="${workflowId}"} ${sorted.length}\n`;
      output += `n8n_workflow_execution_duration_seconds_sum{workflow_id="${workflowId}"} ${sorted.reduce((a, b) => a + b, 0)}\n`;
    }
    
    // Error metrics
    output += '# HELP n8n_workflow_errors_total Total workflow errors\n';
    output += '# TYPE n8n_workflow_errors_total counter\n';
    
    for (const [key, count] of this.metrics.errors) {
      const [workflowId, errorType, nodeName] = key.split(':');
      output += `n8n_workflow_errors_total{workflow_id="${workflowId}",error_type="${errorType}",node="${nodeName}"} ${count}\n`;
    }
    
    return output;
  }

  async flush() {
    // Send metrics to Prometheus Pushgateway or other backend
    const metrics = this.getPrometheusMetrics();
    
    try {
      await fetch('http://prometheus-pushgateway:9091/metrics/job/n8n', {
        method: 'POST',
        body: metrics
      });
      
      // Clear counters after successful push
      this.clearCounters();
    } catch (error) {
      console.error('Failed to flush metrics:', error);
    }
  }

  clearCounters() {
    this.metrics.executions.clear();
    this.metrics.errors.clear();
    // Keep latency for histogram calculation
  }
}

// Usage in n8n workflows
const metrics = new MetricsCollector();

// At workflow start
const startTime = Date.now();

// At workflow end
metrics.recordExecution(
  $env.WORKFLOW_ID,
  $run.status,
  Date.now() - startTime
);

module.exports = { MetricsCollector };

// structured-logger.js
class StructuredLogger {
  constructor(config = {}) {
    this.service = config.service || 'n8n-workflow';
    this.environment = config.environment || 'production';
    this.version = config.version || '1.0.0';
    this.outputs = config.outputs || ['console'];
  }

  log(level, message, context = {}) {
    const logEntry = {
      timestamp: new Date().toISOString(),
      level,
      service: this.service,
      environment: this.environment,
      version: this.version,
      message,
      ...this.getExecutionContext(),
      ...context
    };

    // Output to configured destinations
    this.outputs.forEach(output => {
      switch (output) {
        case 'console':
          this.outputToConsole(logEntry);
          break;
        case 'loki':
          this.outputToLoki(logEntry);
          break;
        case 'elasticsearch':
          this.outputToElasticsearch(logEntry);
          break;
        case 'sentry':
          if (level === 'error') {
            this.outputToSentry(logEntry);
          }
          break;
      }
    });

    return logEntry;
  }

  getExecutionContext() {
    try {
      return {
        workflow_id: $env.WORKFLOW_ID,
        execution_id: $env.EXECUTION_ID,
        node: $env.NODE_NAME,
        run_mode: $env.EXECUTION_MODE
      };
    } catch (e) {
      return {};
    }
  }

  info(message, context) {
    return this.log('info', message, context);
  }

  warn(message, context) {
    return this.log('warn', message, context);
  }

  error(message, error, context = {}) {
    const errorContext = {
      error: {
        message: error.message,
        stack: error.stack,
        code: error.code,
        type: error.name
      },
      ...context
    };
    return this.log('error', message, errorContext);
  }

  debug(message, context) {
    if (this.environment !== 'production') {
      return this.log('debug', message, context);
    }
  }

  outputToConsole(entry) {
    const logLine = JSON.stringify(entry);
    console.log(logLine);
  }

  async outputToLoki(entry) {
    try {
      await fetch('http://loki:3100/loki/api/v1/push', {
        method: 'POST',
        headers: { 'Content-Type': 'application/json' },
        body: JSON.stringify({
          streams: [{
            stream: {
              service: this.service,
              level: entry.level
            },
            values: [[
              (Date.now() * 1000000).toString(),
              JSON.stringify(entry)
            ]]
          }]
        })
      });
    } catch (e) {
      console.error('Failed to send to Loki:', e);
    }
  }

  async outputToElasticsearch(entry) {
    try {
      await fetch('http://elasticsearch:9200/n8n-logs/_doc', {
        method: 'POST',
        headers: { 'Content-Type': 'application/json' },
        body: JSON.stringify(entry)
      });
    } catch (e) {
      console.error('Failed to send to Elasticsearch:', e);
    }
  }
}

// Usage
const logger = new StructuredLogger({
  service: 'order-processing',
  environment: 'production',
  outputs: ['console', 'loki', 'sentry']
});

logger.info('Processing order', {
  order_id: $input.first().json.orderId,
  customer_id: $input.first().json.customer.id
});

module.exports = { StructuredLogger };

// tracing.js
const { NodeSDK } = require('@opentelemetry/sdk-node');
const { JaegerExporter } = require('@opentelemetry/exporter-jaeger');
const { Resource } = require('@opentelemetry/resources');
const { SemanticResourceAttributes } = require('@opentelemetry/semantic-conventions');

class WorkflowTracer {
  constructor(serviceName) {
    this.serviceName = serviceName;
    this.traces = new Map();
    
    // Initialize OpenTelemetry
    this.sdk = new NodeSDK({
      traceExporter: new JaegerExporter({
        endpoint: process.env.JAEGER_ENDPOINT || 'http://jaeger:14268/api/traces'
      }),
      resource: new Resource({
        [SemanticResourceAttributes.SERVICE_NAME]: serviceName,
        [SemanticResourceAttributes.DEPLOYMENT_ENVIRONMENT]: process.env.ENVIRONMENT || 'production'
      })
    });
    
    this.sdk.start();
  }

  startSpan(name, attributes = {}) {
    const spanId = this.generateSpanId();
    const span = {
      id: spanId,
      name,
      startTime: Date.now(),
      attributes,
      events: [],
      status: 'ok'
    };
    
    this.traces.set(spanId, span);
    return spanId;
  }

  addEvent(spanId, name, attributes = {}) {
    const span = this.traces.get(spanId);
    if (span) {
      span.events.push({
        name,
        timestamp: Date.now(),
        attributes
      });
    }
  }

  setStatus(spanId, status, message) {
    const span = this.traces.get(spanId);
    if (span) {
      span.status = status;
      span.statusMessage = message;
    }
  }

  finishSpan(spanId) {
    const span = this.traces.get(spanId);
    if (span) {
      span.endTime = Date.now();
      span.duration = span.endTime - span.startTime;
      
      // Export to Jaeger
      this.exportSpan(span);
      
      // Clean up
      this.traces.delete(spanId);
      
      return span;
    }
    return null;
  }

  exportSpan(span) {
    // Convert to Jaeger format and send
    const jaegerSpan = {
      operationName: span.name,
      startTime: span.startTime * 1000, // Microseconds
      duration: span.duration * 1000,
      tags: Object.entries(span.attributes).map(([key, value]) => ({
        key,
        type: typeof value === 'number' ? 'int64' : 'string',
        value: String(value)
      })),
      logs: span.events.map(event => ({
        timestamp: event.timestamp * 1000,
        fields: Object.entries(event.attributes).map(([key, value]) => ({
          key,
          type: typeof value === 'number' ? 'int64' : 'string',
          value: String(value)
        }))
      })),
      spanID: span.id
    };
    
    // Send to Jaeger
    fetch('http://jaeger:14268/api/traces', {
      method: 'POST',
      headers: { 'Content-Type': 'application/json' },
      body: JSON.stringify({
        batches: [{
          serviceName: this.serviceName,
          spans: [jaegerSpan]
        }]
      })
    }).catch(err => console.error('Failed to export span:', err));
  }

  generateSpanId() {
    return Math.random().toString(36).substring(2, 15) + 
           Math.random().toString(36).substring(2, 15);
  }
}

// Usage in workflow
const tracer = new WorkflowTracer('order-processing');

const workflowSpan = tracer.startSpan('process_order', {
  'workflow.id': $env.WORKFLOW_ID,
  'execution.id': $env.EXECUTION_ID
});

try {
  tracer.addEvent(workflowSpan, 'validation_started');
  await validateOrder(order);
  
  tracer.addEvent(workflowSpan, 'payment_started');
  await processPayment(order);
  
  tracer.addEvent(workflowSpan, 'notification_sent');
  await sendNotification(order);
  
  tracer.setStatus(workflowSpan, 'ok');
} catch (error) {
  tracer.setStatus(workflowSpan, 'error', error.message);
  throw error;
} finally {
  tracer.finishSpan(workflowSpan);
}

module.exports = { WorkflowTracer };

// alerting-engine.js
class AlertingEngine {
  constructor(config) {
    this.rules = config.rules || [];
    this.channels = config.channels || {};
    this.alertHistory = new Map();
    this.cooldownPeriod = config.cooldownPeriod || 300000; // 5 minutes
  }

  async evaluateMetrics(metrics) {
    const alerts = [];
    
    for (const rule of this.rules) {
      const triggered = this.evaluateRule(rule, metrics);
      
      if (triggered) {
        const alertId = `${rule.name}:${metrics.workflow_id}`;
        
        // Check cooldown
        if (!this.isInCooldown(alertId)) {
          alerts.push({
            id: alertId,
            rule: rule.name,
            severity: rule.severity,
            message: this.formatAlertMessage(rule, metrics),
            timestamp: new Date().toISOString(),
            metrics
          });
          
          this.recordAlert(alertId);
        }
      }
    }
    
    // Send alerts
    await Promise.all(alerts.map(alert => this.sendAlert(alert)));
    
    return alerts;
  }

  evaluateRule(rule, metrics) {
    const value = this.getMetricValue(rule.metric, metrics);
    
    switch (rule.operator) {
      case '>':
        return value > rule.threshold;
      case '<':
        return value < rule.threshold;
      case '>=':
        return value >= rule.threshold;
      case '<=':
        return value <= rule.threshold;
      case '==':
        return value === rule.threshold;
      case 'contains':
        return String(value).includes(rule.threshold);
      default:
        return false;
    }
  }

  getMetricValue(metricPath, metrics) {
    return metricPath.split('.').reduce((obj, key) => obj?.[key], metrics);
  }

  formatAlertMessage(rule, metrics) {
    return rule.message
      .replace('{{value}}', this.getMetricValue(rule.metric, metrics))
      .replace('{{threshold}}', rule.threshold)
      .replace('{{workflow}}', metrics.workflow_id);
  }

  isInCooldown(alertId) {
    const lastAlert = this.alertHistory.get(alertId);
    if (!lastAlert) return false;
    
    return Date.now() - lastAlert < this.cooldownPeriod;
  }

  recordAlert(alertId) {
    this.alertHistory.set(alertId, Date.now());
  }

  async sendAlert(alert) {
    const channels = this.getChannelsForSeverity(alert.severity);
    
    await Promise.all(channels.map(channel => 
      this.sendToChannel(channel, alert)
    ));
  }

  getChannelsForSeverity(severity) {
    const mapping = {
      critical: ['pagerduty', 'slack', 'email'],
      high: ['slack', 'email'],
      medium: ['slack'],
      low: ['email']
    };
    
    return mapping[severity] || ['email'];
  }

  async sendToChannel(channel, alert) {
    switch (channel) {
      case 'slack':
        await this.sendSlackAlert(alert);
        break;
      case 'pagerduty':
        await this.sendPagerDutyAlert(alert);
        break;
      case 'email':
        await this.sendEmailAlert(alert);
        break;
    }
  }

  async sendSlackAlert(alert) {
    const color = {
      critical: '#FF0000',
      high: '#FFA500',
      medium: '#FFFF00',
      low: '#00FF00'
    }[alert.severity];
    
    await fetch(this.channels.slack.webhook, {
      method: 'POST',
      headers: { 'Content-Type': 'application/json' },
      body: JSON.stringify({
        attachments: [{
          color,
          title: `🚨 ${alert.severity.toUpperCase()}: ${alert.rule}`,
          text: alert.message,
          fields: [
            { title: 'Workflow', value: alert.metrics.workflow_id, short: true },
            { title: 'Time', value: alert.timestamp, short: true }
          ],
          footer: 'n8n Alerting',
          ts: Math.floor(Date.now() / 1000)
        }]
      })
    });
  }

  async sendPagerDutyAlert(alert) {
    await fetch('https://events.pagerduty.com/v2/enqueue', {
      method: 'POST',
      headers: { 'Content-Type': 'application/json' },
      body: JSON.stringify({
        routing_key: this.channels.pagerduty.key,
        event_action: 'trigger',
        dedup_key: alert.id,
        payload: {
          summary: alert.message,
          severity: alert.severity,
          source: 'n8n',
          custom_details: alert.metrics
        }
      })
    });
  }
}

// Alert rules configuration
const alertRules = [
  {
    name: 'high_error_rate',
    metric: 'error_rate',
    operator: '>',
    threshold: 0.05,
    severity: 'high',
    message: 'Error rate ({{value}}%) exceeds threshold ({{threshold}}%) for workflow {{workflow}}'
  },
  {
    name: 'workflow_execution_time',
    metric: 'execution_duration',
    operator: '>',
    threshold: 30000,
    severity: 'medium',
    message: 'Workflow {{workflow}} taking {{value}}ms (threshold: {{threshold}}ms)'
  },
  {
    name: 'api_rate_limit',
    metric: 'api.rate_limit_hits',
    operator: '>',
    threshold: 0,
    severity: 'medium',
    message: 'API rate limit hit {{value}} times'
  },
  {
    name: 'circuit_breaker_open',
    metric: 'circuit_breaker.state',
    operator: '==',
    threshold: 'OPEN',
    severity: 'critical',
    message: 'Circuit breaker OPEN for {{workflow}}'
  }
];

module.exports = { AlertingEngine, alertRules };