运行状况检查、断路器与重试逻辑

为什么自动故障检测很重要

ELB Health 检查

# Configure ALB target group health check
aws elbv2 modify-target-group \
  --target-group-arn arn:aws:elasticloadbalancing::123:targetgroup/my-tg/abc \
  --health-check-protocol HTTPS \
  --health-check-port 443 \
  --health-check-path /health \
  --health-check-interval-seconds 15 \
  --healthy-threshold-count 2 \
  --unhealthy-threshold-count 3 \
  --matcher HttpCode=200

Route 53 Health 检查

# Create endpoint health check
aws route53 create-health-check \
  --caller-reference ref-$(date +%s) \
  --health-check-config '{
    "Type": "HTTPS",
    "FullyQualifiedDomainName": "api.example.com",
    "Port": 443,
    "ResourcePath": "/health",
    "RequestInterval": 10,
    "FailureThreshold": 2,
    "EnableSNI": true
  }'

Auto Scaling Health 检查

# Configure ASG to use ELB health checks
aws autoscaling update-auto-scaling-group \
  --auto-scaling-group-name my-asg \
  --health-check-type ELB \
  --health-check-grace-period 300

# Grace period: time after launch before health checks start
# Prevents premature termination during startup

Circuit Breaker 模式

# Circuit breaker states
# CLOSED: All calls pass through
#   failureCount < threshold -> stay CLOSED
#   failureCount >= threshold -> open circuit

# OPEN: All calls fail immediately
#   After timeout -> enter HALF-OPEN

# HALF-OPEN: Allow limited test calls
#   Success -> return to CLOSED
#   Failure -> return to OPEN

# Example threshold: 5 failures in 10 seconds -> OPEN

用于 Circuit Breaking 的 AWS App Mesh

# App Mesh virtual node with circuit breaker
# (JSON configuration)
{
  'spec': {
    'listeners': [{
      'outlierDetection': {
        'consecutiveErrors': 5,
        'interval': {'unit': 'ms', 'value': 10000},
        'baseEjectionDuration': {'unit': 's', 'value': 30},
        'maxEjectionPercent': 50
      }
    }]
  }
}

重试逻辑与指数退避

# AWS SDK retries with exponential backoff automatically
# Default retry config for most AWS services:
# Max retries: 3-5 (varies by service)
# Base delay: 100ms
# Max delay: ~20 seconds

# Python boto3 custom retry configuration
import boto3
from botocore.config import Config

config = Config(
    retries={'max_attempts': 5, 'mode': 'adaptive'}
)
client = boto3.client('s3', config=config)

通过幂等性实现安全重试

# SQS message deduplication ID for FIFO queues
aws sqs send-message \
  --queue-url https://sqs.us-east-1.amazonaws.com/123/orders.fifo \
  --message-body '{"orderId":"ord-123","items":[...]}' \
  --message-group-id 'customer-456' \
  --message-deduplication-id 'ord-123-attempt-1'

# SQS deduplicates messages with same ID for 5 minutes

Timeout 配置

# Lambda: set execution timeout
aws lambda update-function-configuration \
  --function-name my-function \
  --timeout 30

# ALB: configure idle timeout
aws elbv2 modify-load-balancer-attributes \
  --load-balancer-arn <ALB-ARN> \
  --attributes Key=idle_timeout.timeout_seconds,Value=60

# API Gateway: integration timeout max 29000ms

用于失败处理的 Dead Letter 队列

# Configure DLQ on SQS queue
aws sqs set-queue-attributes \
  --queue-url https://sqs.us-east-1.amazonaws.com/123/main-queue \
  --attributes '{
    "RedrivePolicy": "{\"deadLetterTargetArn\":\"arn:aws:sqs:us-east-1:123:dlq\",\"maxReceiveCount\":3}"
  }'

# After 3 failed processing attempts, message goes to DLQ

使用 CloudWatch 观察 Failure

# CloudWatch alarm for unhealthy hosts
aws cloudwatch put-metric-alarm \
  --alarm-name 'ALB-UnhealthyHosts' \
  --alarm-description 'Alert when targets fail health checks' \
  --metric-name UnHealthyHostCount \
  --namespace AWS/ApplicationELB \
  --period 60 \
  --evaluation-periods 2 \
  --threshold 1 \
  --comparison-operator GreaterThanOrEqualToThreshold \
  --alarm-actions arn:aws:sns:us-east-1:123:ops-team

快速检查

课程回顾