Devops Agent

# Ubuntu/Debian
sudo apt-get install -y apt-transport-https software-properties-common
wget -q -O - https://apt.grafana.com/gpg.key | sudo apt-key add -
echo "deb https://apt.grafana.com stable main" | sudo tee /etc/apt/sources.list.d/grafana.list
sudo apt-get update && sudo apt-get install -y grafana

sudo systemctl daemon-reload
sudo systemctl enable grafana-server
sudo systemctl start grafana-server

# 等待 Grafana 启动
sleep 5

# 添加 Prometheus 数据源
GRAFANA_ADMIN_PASS="${GRAFANA_ADMIN_PASS:-admin}"
curl -X POST -u "admin:${GRAFANA_ADMIN_PASS}" http://localhost:3000/api/datasources \
  -H "Content-Type: application/json" \
  -d '{
    "name": "Prometheus",
    "type": "prometheus",
    "url": "http://localhost:9090",
    "access": "proxy",
    "isDefault": true
  }'

# 导入 dashboard（使用 references/grafana-dashboards.md 中的 JSON）
curl -X POST -u "admin:${GRAFANA_ADMIN_PASS}" http://localhost:3000/api/dashboards/db \
  -H "Content-Type: application/json" \
  -d @dashboard.json

groups:
  - name: system_alerts
    rules:
      - alert: HighCPUUsage
        expr: 100 - (avg by(instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 80
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "CPU 使用率超过 80%"
          description: "实例 {{ $labels.instance }} CPU 使用率为 {{ $value }}%"

      - alert: HighMemoryUsage
        expr: (1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100 > 90
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "内存使用率超过 90%"

      - alert: DiskSpaceRunningOut
        expr: (1 - node_filesystem_avail_bytes{mountpoint="/"} / node_filesystem_size_bytes{mountpoint="/"}) * 100 > 85
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "磁盘使用率超过 85%"

      - alert: ServiceDown
        expr: up == 0
        for: 1m
        labels:
          severity: critical
        annotations:
          summary: "服务 {{ $labels.job }} 已停止"

receivers:
  - name: 'telegram'
    telegram_configs:
      - bot_token: '<BOT_TOKEN>'  # 提示用户提供，不硬编码
        chat_id: <CHAT_ID>
        message: '{{ template "telegram.default.message" . }}'

receivers:
  - name: 'slack'
    slack_configs:
      - api_url: '<WEBHOOK_URL>'  # 提示用户提供
        channel: '#alerts'
        title: '{{ .CommonAnnotations.summary }}'

receivers:
  - name: 'email'
    email_configs:
      - to: '<EMAIL>'  # 提示用户提供
        from: 'alertmanager@example.com'
        smarthost: 'smtp.example.com:587'
        auth_username: '<USERNAME>'
        auth_password: '<PASSWORD>'

# 监控配置报告

## 组件状态
- Prometheus: http://localhost:9090 ✅
- Grafana: http://localhost:3000 ✅
- node_exporter: http://localhost:9100 ✅
- <service>_exporter: http://localhost:<port> ✅

## 已配置告警规则
- CPU > 80% (warning, 5分钟持续)
- 内存 > 90% (critical, 5分钟持续)
- 磁盘 > 85% (warning, 10分钟持续)
- 服务 down (critical, 1分钟)

## 告警通知
- 渠道: <Telegram/Slack/Email>

## 访问信息
- Grafana 默认账号: admin / admin（首次登录请修改密码）

## 后续建议
- [ ] 修改 Grafana 默认密码
- [ ] 根据业务需求调整告警阈值
- [ ] 添加自定义 dashboard

/devops-agent backup <target> <destination> [--schedule CRON] [--encrypt] [--retain DAYS] [--dry-run]

# 数据库检测
psql --version 2>/dev/null        # PostgreSQL
mysql --version 2>/dev/null       # MySQL
mongodump --version 2>/dev/null   # MongoDB

# 如果 target 是路径，检查是否存在
[ -d "<target>" ] && echo "Directory backup mode"

#!/usr/bin/env bash
set -euo pipefail

# === 配置 ===
BACKUP_DIR="/var/backups/postgresql"
TIMESTAMP=$(date +%Y%m%d_%H%M%S)
BACKUP_FILE="${BACKUP_DIR}/pg_backup_${TIMESTAMP}.sql.gz"
RETAIN_DAYS=${RETAIN_DAYS:-30}
LOG_FILE="${HOME}/devops-agent.log"

# === 日志函数 ===
log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] [BACKUP] $*" | tee -a "$LOG_FILE"; }

# === 执行备份 ===
log "开始 PostgreSQL 备份..."
mkdir -p "$BACKUP_DIR"
pg_dumpall | gzip > "$BACKUP_FILE"

# === 加密（可选）===
# gpg --symmetric --cipher-algo AES256 --batch --passphrase-file /path/to/key "$BACKUP_FILE"

# === 校验完整性 ===
sha256sum "$BACKUP_FILE" > "${BACKUP_FILE}.sha256"
log "备份完成: $BACKUP_FILE ($(du -h "$BACKUP_FILE" | cut -f1))"

# === 上传到远程存储 ===
# aws s3 cp "$BACKUP_FILE" "s3://bucket/path/"
# 或: rsync -avz "$BACKUP_FILE" "user@remote:/path/"

# === 清理旧备份（保留策略）===
find "$BACKUP_DIR" -name "pg_backup_*.sql.gz" -mtime +${RETAIN_DAYS} -delete
log "已清理 ${RETAIN_DAYS} 天前的旧备份"

# === 通知 ===
# curl -s -X POST "https://api.telegram.org/bot<TOKEN>/sendMessage" \
#   -d "chat_id=<ID>&text=PostgreSQL 备份完成: $(du -h "$BACKUP_FILE" | cut -f1)"

mysqldump --all-databases --single-transaction --routines --triggers | gzip > "$BACKUP_FILE"

mongodump --archive="$BACKUP_FILE" --gzip

tar czf "$BACKUP_FILE" -C "$(dirname "$TARGET_DIR")" "$(basename "$TARGET_DIR")"

CRON_SCHEDULE="${SCHEDULE:-0 2 * * *}"
BACKUP_SCRIPT="/opt/scripts/backup_<target>.sh"

# 写入备份脚本
sudo cp generated_backup.sh "$BACKUP_SCRIPT"
sudo chmod +x "$BACKUP_SCRIPT"

# 添加 cron 任务（先检查是否已存在）
(crontab -l 2>/dev/null | grep -v "$BACKUP_SCRIPT"; echo "$CRON_SCHEDULE $BACKUP_SCRIPT") | crontab -

# 日备份：保留 7 天
# 周备份（周日）：保留 4 周
# 月备份（1号）：保留 12 个月

# 在备份脚本中实现轮转逻辑
DAY_OF_WEEK=$(date +%u)  # 1=Monday, 7=Sunday
DAY_OF_MONTH=$(date +%d)

if [ "$DAY_OF_MONTH" = "01" ]; then
    cp "$BACKUP_FILE" "${BACKUP_DIR}/monthly/"
elif [ "$DAY_OF_WEEK" = "7" ]; then
    cp "$BACKUP_FILE" "${BACKUP_DIR}/weekly/"
fi

# 清理策略
find "${BACKUP_DIR}/daily/" -mtime +7 -delete
find "${BACKUP_DIR}/weekly/" -mtime +28 -delete
find "${BACKUP_DIR}/monthly/" -mtime +365 -delete

# 验证备份文件
sha256sum -c "${BACKUP_FILE}.sha256"

# 数据库备份验证（可选 - 恢复到临时库测试）
# createdb test_restore && pg_restore -d test_restore "$BACKUP_FILE" && dropdb test_restore

# 备份配置报告

## 基本信息
- **备份目标**: <target>
- **备份目的地**: <destination>
- **备份计划**: <cron-schedule>
- **加密**: 是/否
- **保留策略**: <retain-days> 天

## 备份脚本
- **位置**: /opt/scripts/backup_<target>.sh
- **cron 条目**: <cron-line>

## 保留策略
- 日备份: 7 天
- 周备份: 4 周
- 月备份: 12 个月

## 验证
- **首次备份测试**: ✅ 成功
- **文件大小**: <size>
- **校验和**: <sha256>

## 恢复命令
- PostgreSQL: `gunzip -c <backup-file> | psql`
- MySQL: `gunzip -c <backup-file> | mysql`
- MongoDB: `mongorestore --gzip --archive=<backup-file>`
- 目录: `tar xzf <backup-file> -C <target-dir>`

/devops-agent diagnose <问题描述> [--deep] [--dry-run]

echo "=== 系统概况 ==="
uname -a                                    # 内核信息
uptime                                      # 运行时间和负载
free -h 2>/dev/null || vm_stat              # 内存使用
df -h                                       # 磁盘使用
cat /proc/loadavg 2>/dev/null               # 负载均衡
who                                         # 当前登录用户
last reboot | head -5                       # 最近重启记录

echo "=== 网络概况 ==="
ss -tlnp 2>/dev/null || netstat -tlnp       # 监听端口
ip addr 2>/dev/null || ifconfig             # 网络接口
cat /etc/resolv.conf                        # DNS 配置

# Nginx 状态
systemctl is-active nginx
nginx -t                                   # 配置语法检查
tail -50 /var/log/nginx/error.log          # 错误日志

# 端口监听
ss -tlnp | grep -E ':80|:443'

# 上游服务状态
# 解析 nginx 配置中的 upstream 端口，逐个检查
grep proxy_pass /etc/nginx/sites-enabled/* 2>/dev/null

# DNS 检查
dig <domain> +short
nslookup <domain>

# SSL 证书有效期
echo | openssl s_client -connect <domain>:443 2>/dev/null | openssl x509 -noout -dates

# 防火墙
sudo ufw status 2>/dev/null
sudo iptables -L -n 2>/dev/null

# CPU 进程排行
ps aux --sort=-%cpu | head -20

# IO wait
iostat -x 1 3 2>/dev/null

# 内存详情
vmstat 1 5

# 网络连接数
ss -s
ss -tnp | awk '{print $5}' | cut -d: -f1 | sort | uniq -c | sort -rn | head -10

# 数据库慢查询（PostgreSQL）
sudo -u postgres psql -c "SELECT pid, now() - pg_stat_activity.query_start AS duration, query FROM pg_stat_activity WHERE (now() - pg_stat_activity.query_start) > interval '5 seconds' ORDER BY duration DESC;" 2>/dev/null

# OOM killer 记录
dmesg | grep -i "oom\|killed process" | tail -10

# 磁盘使用概览
df -h

# 大文件定位（Top 20）
sudo find / -type f -size +100M -exec ls -lh {} \; 2>/dev/null | sort -k5 -rh | head -20

# 各目录占用
du -sh /var/log/ /var/lib/docker/ /tmp/ /home/ /opt/ 2>/dev/null

# 日志轮转检查
ls -la /etc/logrotate.d/
cat /var/lib/logrotate/status 2>/dev/null

# Docker 占用
docker system df 2>/dev/null

# 清理建议
echo "=== 清理建议 ==="
# apt 缓存
du -sh /var/cache/apt/archives/ 2>/dev/null
# journal 日志
journalctl --disk-usage 2>/dev/null
# Docker 未使用镜像
docker images --filter "dangling=true" -q 2>/dev/null | wc -l

# PostgreSQL 检查
pg_isready 2>/dev/null

# 连接数
sudo -u postgres psql -c "SELECT count(*) FROM pg_stat_activity;" 2>/dev/null
sudo -u postgres psql -c "SHOW max_connections;" 2>/dev/null

# 认证配置
sudo cat /etc/postgresql/*/main/pg_hba.conf 2>/dev/null | grep -v "^#" | grep -v "^$"

# 监听地址
sudo cat /etc/postgresql/*/main/postgresql.conf 2>/dev/null | grep listen_addresses

# 防火墙端口
ss -tlnp | grep 5432

# MySQL 检查
mysqladmin ping 2>/dev/null
mysqladmin status 2>/dev/null

# systemd 日志（最近 1 小时）
journalctl --since "1 hour ago" --priority=err --no-pager | tail -50

# Nginx 错误日志
tail -100 /var/log/nginx/error.log 2>/dev/null

# 应用日志（自动检测）
# 检查 /var/log/ 下的常见应用日志
ls -lt /var/log/*.log | head -10

# syslog / messages
tail -100 /var/log/syslog 2>/dev/null || tail -100 /var/log/messages 2>/dev/null

# dmesg（内核消息）
dmesg --time-format=iso | tail -50 2>/dev/null || dmesg | tail -50

# 故障诊断报告

## 问题描述
<用户描述>

## 诊断时间
<timestamp>

## 系统概况
- **OS**: <os-info>
- **运行时间**: <uptime>
- **负载**: <load-average>
- **内存使用**: <memory-usage>
- **磁盘使用**: <disk-usage>

## 发现的问题（按严重程度排序）

### 🔴 严重
1. <issue-description>
   - **证据**: <evidence>
   - **影响**: <impact>
   - **修复命令**: `<fix-command>`

### 🟡 警告
1. <issue-description>
   - **证据**: <evidence>
   - **修复建议**: <suggestion>

### 🔵 信息
1. <observation>

## 修复建议（按优先级排序）
1. [严重] <fix-1> — `<command>`
2. [警告] <fix-2> — `<command>`
3. [建议] <fix-3>

## 相关日志摘录