This is my problem I have configured filebit, logstash to create separate indexes for system and programs But I ran into a problem, the indexes are created, the log separation is fine, the life policy works, but I have a problem that logs are written to the first indexes, then new logs are created according to the life policy, but they are empty, then the first one is deleted and logs are written to it
# ============================== Filebeat inputs ===============================
filebeat.inputs:
- type: log
enabled: true
paths:
- /var/log/*.log
fields:
log_type: system
- type: log
paths:
- /var/lib/pgsql/13/data/log/*.log
fields:
log_type: pgsql
- type: log
paths:
- /var/log/postgresql/*.log #Логи містять інформацію про події та стан PostgreSQL-сервера
fields:
log_type: postgresql
# ------------------------------ Logstash Output -------------------------------
output.logstash:
# The Logstash hosts
hosts: ["**********:****"]
# ================================= Processors =================================
processors:
- add_host_metadata:
when.not.contains.tags: forwarded
- add_cloud_metadata: ~
- add_docker_metadata: ~
- add_kubernetes_metadata: ~
# ============================= Filebeat Logging metrics =======================
logging.metrics.enabled: true
logging.metrics.period: 30s
-------------
File Logstash
-------------
input {
beats {
port => 5044
}
}
filter {
if [fields][log_type] == "system" {
grok {
match => { "message" => "%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:syslog_message}" }
}
mutate {
add_field => { '[@metadata][index]' => 'system_logs-%{+YYYY.MM.dd}-000001' }
}
} else if [fields][log_type] == "pgsql" {
grok {
match => { "message" => "%{TIMESTAMP_ISO8601:log_timestamp} %{WORD:log_level} %{GREEDYDATA:log_message}" }
}
mutate {
add_field => { '[@metadata][index]' => 'pgsql_logs-%{+YYYY.MM.dd}-000001' }
}
} else if [fields][log_type] == "postgresql" {
grok {
match => { "message" => "%{TIMESTAMP_ISO8601:log_timestamp} %{WORD:log_level} %{GREEDYDATA:log_message}" }
}
mutate {
add_field => { '[@metadata][index]' => 'postgresql_logs-%{+YYYY.MM.dd}-000001' }
}
mutate {
add_field => { "[@metadata][index]" => "%{[@metadata][index]}-000001" }
}
}
}
output {
stdout { codec => rubydebug }
elasticsearch {
hosts => ["https://***********:9000"]
user => "*********"
password => "***************"
ssl => true
cacert => '/etc*************'
index => "%{[@metadata][index]}"
manage_template => false
ilm_enabled => true
ilm_policy => "%{[@metadata][index]}-ilm"
ilm_pattern => "000001"
}
}
PUT _index_template/pgsql-template
{
"index_patterns": ["pgsql-*"],
"template": {
"settings": {
"number_of_shards": 1,
"number_of_replicas": 1,
"index.lifecycle.name": "pgsql-ilm",
"index.lifecycle.rollover_alias": "pgsql_logs_alias"
}
}
}
PUT _index_template/system-template
{
"index_patterns": ["system-*"],
"template": {
"settings": {
"number_of_shards": 1,
"number_of_replicas": 1,
"index.lifecycle.name": "system-ilm",
"index.lifecycle.rollover_alias": "system_logs_alias"
}
}
}