Мониторинг и алерты: как узнать об атаке первым

version: '3.8'

services:
  prometheus:
    image: prom/prometheus:latest
    container_name: prometheus
    volumes:
      - ./prometheus.yml:/etc/prometheus/prometheus.yml
      - prometheus_data:/prometheus
    command:
      - '--config.file=/etc/prometheus/prometheus.yml'
      - '--storage.tsdb.retention.time=30d'
    ports:
      - "9090:9090"
    restart: unless-stopped

  grafana:
    image: grafana/grafana:latest
    container_name: grafana
    volumes:
      - grafana_data:/var/lib/grafana
    environment:
      - GF_SECURITY_ADMIN_PASSWORD=your_secure_password
      - GF_USERS_ALLOW_SIGN_UP=false
    ports:
      - "3000:3000"
    restart: unless-stopped

  node-exporter:
    image: prom/node-exporter:latest
    container_name: node-exporter
    volumes:
      - /proc:/host/proc:ro
      - /sys:/host/sys:ro
      - /:/rootfs:ro
    command:
      - '--path.procfs=/host/proc'
      - '--path.sysfs=/host/sys'
      - '--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)'
    ports:
      - "9100:9100"
    restart: unless-stopped

volumes:
  prometheus_data:
  grafana_data:

global:
  scrape_interval: 15s
  evaluation_interval: 15s

alerting:
  alertmanagers:
    - static_configs:
        - targets:
          - alertmanager:9093

rule_files:
  - /etc/prometheus/rules/*.yml

scrape_configs:
  # Сам Prometheus
  - job_name: 'prometheus'
    static_configs:
      - targets: ['localhost:9090']

  # Системные метрики серверов
  - job_name: 'node'
    static_configs:
      - targets: 
        - 'server1:9100'
        - 'server2:9100'

  # Nginx метрики
  - job_name: 'nginx'
    static_configs:
      - targets: ['nginx-exporter:9113']

  # Blackbox (внешние проверки)
  - job_name: 'blackbox-http'
    metrics_path: /probe
    params:
      module: [http_2xx]
    static_configs:
      - targets:
        - https://yoursite.com
        - https://yoursite.com/api/health
    relabel_configs:
      - source_labels: [__address__]
        target_label: __param_target
      - source_labels: [__param_target]
        target_label: instance
      - target_label: __address__
        replacement: blackbox:9115

groups:
  - name: ddos-detection
    interval: 10s  # Проверяем чаще для быстрого обнаружения
    rules:
      # Аномальный всплеск трафика
      - alert: TrafficSpike
        expr: rate(nginx_http_requests_total[1m]) > 10 * avg_over_time(rate(nginx_http_requests_total[1m])[1h:5m])
        for: 2m
        labels:
          severity: warning
        annotations:
          summary: "Всплеск трафика на {{ $labels.instance }}"
          description: "RPS в 10 раз выше среднего за последний час"

      # Много 5xx ошибок
      - alert: HighErrorRate
        expr: sum(rate(nginx_http_requests_total{status=~"5.."}[5m])) / sum(rate(nginx_http_requests_total[5m])) > 0.05
        for: 1m
        labels:
          severity: critical
        annotations:
          summary: "Высокий уровень 5xx ошибок"
          description: "Более 5% запросов возвращают 5xx. Текущее значение: {{ $value | humanizePercentage }}"

      # Высокая нагрузка на CPU
      - alert: HighCPUUsage
        expr: 100 - (avg by(instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 90
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "Высокая нагрузка CPU на {{ $labels.instance }}"
          description: "CPU usage > 90% в течение 5 минут"

      # Сервер недоступен извне
      - alert: SiteDown
        expr: probe_success{job="blackbox-http"} == 0
        for: 1m
        labels:
          severity: critical
        annotations:
          summary: "Сайт недоступен: {{ $labels.instance }}"
          description: "HTTP проверка не прошла в течение 1 минуты"

      # Медленные ответы
      - alert: SlowResponseTime
        expr: histogram_quantile(0.95, rate(nginx_http_request_duration_seconds_bucket[5m])) > 5
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Медленные ответы сервера"
          description: "95-й перцентиль времени ответа > 5 секунд"

global:
  resolve_timeout: 5m

route:
  receiver: 'telegram-notifications'
  group_by: ['alertname', 'severity']
  group_wait: 30s
  group_interval: 5m
  repeat_interval: 4h
  routes:
    - receiver: 'telegram-critical'
      match:
        severity: critical
      group_wait: 10s
      repeat_interval: 1h

receivers:
  - name: 'telegram-notifications'
    telegram_configs:
      - bot_token: 'YOUR_BOT_TOKEN'
        chat_id: YOUR_CHAT_ID
        message: |
           {{ .Status | toUpper }}
          {{ range .Alerts }}
          Alert: {{ .Labels.alertname }}
          Severity: {{ .Labels.severity }}
          {{ .Annotations.summary }}
          {{ .Annotations.description }}
          {{ end }}
        
  - name: 'telegram-critical'
    telegram_configs:
      - bot_token: 'YOUR_BOT_TOKEN'
        chat_id: YOUR_CHAT_ID
        message: |
           CRITICAL ALERT
          {{ range .Alerts }}
          {{ .Annotations.summary }}
          {{ .Annotations.description }}
          Started: {{ .StartsAt.Format "15:04:05" }}
          {{ end }}

# Чтобы создать Telegram бота:
# 1. Напишите @BotFather в Telegram
# 2. Создайте бота командой /newbot
# 3. Получите token
# 4. Добавьте бота в группу или получите chat_id через @userinfobot

app.get('/health', async (req, res) => {
  const health = {
    status: 'ok',
    timestamp: new Date().toISOString(),
    checks: {}
  };

  // Проверка базы данных
  try {
    await db.query('SELECT 1');
    health.checks.database = { status: 'ok' };
  } catch (err) {
    health.checks.database = { status: 'error', message: err.message };
    health.status = 'degraded';
  }

  // Проверка Redis
  try {
    await redis.ping();
    health.checks.redis = { status: 'ok' };
  } catch (err) {
    health.checks.redis = { status: 'error', message: err.message };
    health.status = 'degraded';
  }

  // Проверка внешнего API (если критичен)
  try {
    const response = await fetch('https://api.payment.com/health', { timeout: 5000 });
    health.checks.payment_api = { status: response.ok ? 'ok' : 'error' };
  } catch (err) {
    health.checks.payment_api = { status: 'error' };
    // Не меняем общий статус — внешний API не критичен
  }

  const statusCode = health.status === 'ok' ? 200 : 503;
  res.status(statusCode).json(health);
});

log_format detailed '$remote_addr - $remote_user [$time_local] '
                        '"$request" $status $body_bytes_sent '
                        '"$http_referer" "$http_user_agent" '
                        '$request_time $upstream_response_time '
                        '$http_x_forwarded_for $http_cf_connecting_ip '
                        '$http_cf_ipcountry $connection $connection_requests';

access_log /var/log/nginx/access.log detailed;

# Отдельный лог для ошибок с большей детализацией
error_log /var/log/nginx/error.log warn;

#!/bin/bash

# Настройки
ZONE_ID="your_cloudflare_zone_id"
API_TOKEN="your_cloudflare_api_token"
THRESHOLD_RPS=1000  # Порог RPS для включения защиты
LOG_FILE="/var/log/nginx/access.log"
STATE_FILE="/tmp/under_attack_enabled"

# Получить текущий RPS
CURRENT_RPS=$(tail -1000 "$LOG_FILE" | awk -v now="$(date +%s)" -v window=60 '
  BEGIN { count=0 }
  {
    # Парсим время из лога nginx
    gsub(/\[|\]/, "", $4)
    cmd = "date -d "" $4 "" +%s"
    cmd | getline log_time
    close(cmd)
    if (now - log_time < window) count++
  }
  END { print count/window }
')

echo "Current RPS: $CURRENT_RPS"

# Если RPS выше порога и защита не включена
if (( $(echo "$CURRENT_RPS > $THRESHOLD_RPS" | bc -l) )) && [ ! -f "$STATE_FILE" ]; then
  echo "Enabling Under Attack Mode..."
  curl -s -X PATCH "https://api.cloudflare.com/client/v4/zones/$ZONE_ID/settings/security_level" \
    -H "Authorization: Bearer $API_TOKEN" \
    -H "Content-Type: application/json" \
    --data '{"value":"under_attack"}'
  touch "$STATE_FILE"
  
  # Отправить уведомление
  curl -s -X POST "https://api.telegram.org/bot$TG_BOT_TOKEN/sendMessage" \
    -d "chat_id=$TG_CHAT_ID" \
    -d "text= Auto-enabled Under Attack Mode. RPS: $CURRENT_RPS"
fi

# Если RPS нормализовался и защита была включена автоматически
if (( $(echo "$CURRENT_RPS < $THRESHOLD_RPS / 2" | bc -l) )) && [ -f "$STATE_FILE" ]; then
  echo "Disabling Under Attack Mode..."
  curl -s -X PATCH "https://api.cloudflare.com/client/v4/zones/$ZONE_ID/settings/security_level" \
    -H "Authorization: Bearer $API_TOKEN" \
    -H "Content-Type: application/json" \
    --data '{"value":"medium"}'
  rm -f "$STATE_FILE"
  
  curl -s -X POST "https://api.telegram.org/bot$TG_BOT_TOKEN/sendMessage" \
    -d "chat_id=$TG_CHAT_ID" \
    -d "text= Auto-disabled Under Attack Mode. RPS normalized: $CURRENT_RPS"
fi