Log Aggregation

# Overall cluster status
curl -s localhost:9200/_cluster/health?pretty

# Node-level stats
curl -s localhost:9200/_nodes/stats?pretty

# Shard allocation explanation (when yellow/red)
curl -s localhost:9200/_cluster/allocation/explain?pretty

# Index-level health
curl -s localhost:9200/_cat/indices?v&health=yellow
curl -s localhost:9200/_cat/indices?v&health=red

# List indices with sizes
curl -s localhost:9200/_cat/indices?v&h=index,docs.count,store.size,pri,rep

# Create index with mapping
curl -X PUT localhost:9200/my-index -H 'Content-Type: application/json' -d '
{
  "mappings": {
    "properties": {
      "@timestamp": { "type": "date" },
      "message": { "type": "text" },
      "host": { "type": "keyword" },
      "level": { "type": "keyword" }
    }
  }
}'

# Delete index
curl -X DELETE localhost:9200/my-index

# Force merge (reduce segments)
curl -X POST "localhost:9200/my-index/_forcemerge?max_num_segments=1"

# /etc/logstash/pipeline/my-pipeline.conf

input {
  kafka {
    bootstrap_servers => "kafka:9092"
    topics => ["logs"]
    group_id => "logstash-consumer"
    consumer_threads => 4
  }
}

filter {
  grok {
    match => { "message" => "%{TIMESTAMP_ISO8601:timestamp} %{LOGLEVEL:level} %{GREEDYDATA:msg}" }
  }
  date {
    match => ["timestamp", "ISO8601"]
    target => "@timestamp"
  }
  mutate {
    remove_field => ["timestamp"]
  }
}

output {
  elasticsearch {
    hosts => ["elasticsearch:9200"]
    index => "logs-%{+YYYY.MM.dd}"
  }
}

# Check pipeline config syntax
/usr/share/logstash/bin/logstash --config.test_and_exit -f /etc/logstash/pipeline/my-pipeline.conf

# Watch logstash logs
journalctl -u logstash --no-pager -n 100 -f

# Check running pipelines
curl -s localhost:9600/_node/pipelines?pretty

# Logstash node stats
curl -s localhost:9600/_node/stats?pretty

# Common startup issues:
# - Config syntax errors (check with --config.test_and_exit)
# - JVM heap too low (check ES_JAVA_OPTS)
# - Pipeline worker failures (check logs for "exception")

# /etc/filebeat/filebeat.yml
filebeat.inputs:
  - type: log
    enabled: true
    paths:
      - /var/log/*.log
      - /var/log/syslog
    fields:
      env: production
      service: myapp
    fields_under_root: true

output.elasticsearch:
  hosts: ["elasticsearch:9200"]
  index: "filebeat-%{[agent.version]}-%{+yyyy.MM.dd}"

setup.ilm.enabled: false
setup.template.name: "filebeat"
setup.template.pattern: "filebeat-*"

# Test config
filebeat test config -c /etc/filebeat/filebeat.yml

# Test output connectivity
filebeat test output -c /etc/filebeat/filebeat.yml

# Check status
systemctl status filebeat

# View logs
journalctl -u filebeat --no-pager -n 100

# Run in foreground with debug
filebeat -e -d "*"

# Check Kibana status
curl -s localhost:5601/api/status

# Create index pattern via API
curl -X POST localhost:5601/api/saved_objects/index-pattern -H 'Content-Type: application/json' -H 'kbn-xsrf: true' -d '
{
  "attributes": {
    "title": "logs-*",
    "timeFieldName": "@timestamp"
  }
}'