Not performed correctly rollover for index.lifecycle.rollover_alias

24 views Asked by At

This is my problem I have configured filebit, logstash to create separate indexes for system and programs But I ran into a problem, the indexes are created, the log separation is fine, the life policy works, but I have a problem that logs are written to the first indexes, then new logs are created according to the life policy, but they are empty, then the first one is deleted and logs are written to it

# ============================== Filebeat inputs ===============================
filebeat.inputs:
- type: log
  enabled: true
  paths:
    - /var/log/*.log
  fields:
    log_type: system

  - type: log
  paths:
    - /var/lib/pgsql/13/data/log/*.log 
  fields:
    log_type: pgsql

- type: log
  paths:
     - /var/log/postgresql/*.log #Логи містять інформацію про події та стан PostgreSQL-сервера
  fields:
     log_type: postgresql
# ------------------------------ Logstash Output -------------------------------
output.logstash:
  # The Logstash hosts
    hosts: ["**********:****"]
# ================================= Processors =================================
processors:
  - add_host_metadata:
      when.not.contains.tags: forwarded
  - add_cloud_metadata: ~
  - add_docker_metadata: ~
  - add_kubernetes_metadata: ~

# ============================= Filebeat Logging metrics =======================
logging.metrics.enabled: true
logging.metrics.period: 30s

-------------
File Logstash
-------------
input {
  beats {
    port => 5044
  }
}

filter {
  if [fields][log_type] == "system" {
    grok {
      match => { "message" => "%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:syslog_message}" }
    }
    mutate {
      add_field => { '[@metadata][index]' => 'system_logs-%{+YYYY.MM.dd}-000001' }
    }
  } else if [fields][log_type] == "pgsql" {
    grok {
      match => { "message" => "%{TIMESTAMP_ISO8601:log_timestamp} %{WORD:log_level} %{GREEDYDATA:log_message}" }
    }
    mutate {
      add_field => { '[@metadata][index]' => 'pgsql_logs-%{+YYYY.MM.dd}-000001' }
    }
  } else if [fields][log_type] == "postgresql" {
    grok {
      match => { "message" => "%{TIMESTAMP_ISO8601:log_timestamp} %{WORD:log_level} %{GREEDYDATA:log_message}" }
    }
    mutate {
      add_field => { '[@metadata][index]' => 'postgresql_logs-%{+YYYY.MM.dd}-000001' }
    }

    mutate {
      add_field => { "[@metadata][index]" => "%{[@metadata][index]}-000001" }
    }
  }
}

output {
  stdout { codec => rubydebug }
  elasticsearch {
    hosts => ["https://***********:9000"]
    user => "*********"
    password => "***************"
    ssl => true
    cacert => '/etc*************'
    index => "%{[@metadata][index]}"
    manage_template => false
    ilm_enabled => true

    ilm_policy => "%{[@metadata][index]}-ilm"
    ilm_pattern => "000001"
  }
}



PUT _index_template/pgsql-template
{
  "index_patterns": ["pgsql-*"],
  "template": {
    "settings": {
      "number_of_shards": 1,
      "number_of_replicas": 1,
      "index.lifecycle.name": "pgsql-ilm",
      "index.lifecycle.rollover_alias": "pgsql_logs_alias"
    }
  }
}

PUT _index_template/system-template
{
  "index_patterns": ["system-*"],
  "template": {
    "settings": {
      "number_of_shards": 1,
      "number_of_replicas": 1,
      "index.lifecycle.name": "system-ilm",
      "index.lifecycle.rollover_alias": "system_logs_alias"
    }
  }
}
0

There are 0 answers