阅读 148

单个logstash文件收集多个filebeat日志

一 背景说明

我现在安装了logstash,只配置了一个文件,想同时收集nginx和java的日志,nginx要显示ip,国家城市,状态码,等,java要显示日志的具体内容。

二  nginx filebeat设置

filebeat.inputs:
# 收集 nginx 日志
- type: log
  enabled: true
  paths:
    - /var/log/nginx/*.log
  tags: ["nginx_logs"]
 # 日志是json开启这个
  json.keys_under_root: true
  json.overwrite_keys: true
  json.add_error_key: true
  # 如果值为ture,那么fields存储在输出文档的顶级位置
  fields_under_root: true
  fields:
    app: easydong
    logtype: nginx_logs


filebeat.config.modules:
  path: ${path.config}/modules.d/*.yml
  reload.enabled: false
setup.template.settings:
  index.number_of_shards: 1
setup.kibana:
output.logstash:
  hosts: ["172.17.199.231:5044"]

processors:
  - add_host_metadata: ~
  - add_cloud_metadata: ~

三 java filebeat设置

filebeat.inputs:
- type: log
  enabled: true
  paths: 
    
     -  /xs/logs/app-front1/easydong-app/app.log 
  encoding: utf-8
  tail_files: true
  
  fields:
    app: easydong
    type: applog-V1-appfront1
  fields_under_root: true
  multiline:
     pattern: '^[0-2][0-9]:[0-5][0-9]:[0-5][0-9]'
     negate: true
     match: after
- type: log
  enabled: true
  paths:
     -  /xs/logs/app-front1/easydong-admin/admin.log
  encoding: utf-8
  tail_files: true
  fields:
    app: easydong
    type: adminlog-V1-appfront1
  fields_under_root: true
  multiline:
     pattern: '^[0-2][0-9]:[0-5][0-9]:[0-5][0-9]'
     negate: true
     match: after

- type: log
  enabled: true
  paths:
                   
     -  /xs/logs/app-front2/easydong-admin/admin.log
  encoding: utf-8
  tail_files: true
  fields:
    app: easydong
    type: adminlog-V1-appfront2
  fields_under_root: true
  multiline:
     pattern: '^[0-2][0-9]:[0-5][0-9]:[0-5][0-9]'
     negate: true
     match: after

- type: log
  enabled: true
  paths:
     -  /xs/logs/app-front2/easydong-app/app.log
  encoding: utf-8
  tail_files: true
  fields:
    app: easydong
    type: applog-V1-appfront2
  fields_under_root: true
  multiline:
     pattern: '^[0-2][0-9]:[0-5][0-9]:[0-5][0-9]'
     negate: true
     match: after

filebeat.config.modules:
  path: ${path.config}/modules.d/*.yml
  reload.enabled: false
setup.template.settings:
  index.number_of_shards: 1
setup.kibana:
output.logstash:
  hosts: ["39.96.179.187:5044"]

processors:
  - add_host_metadata: ~
  - add_cloud_metadata: ~

四 logstash设置

根据 if [logtype] == "nginx_logs 去判断是nginx日志还是java日志,

input {
 beats {
     port => 5044
     
  }



}


filter{
  if [type] == "adminlog-V1-appfront1"{

      grok {
         match => ["message", "%{SYSLOGBASE} %{GREEDYDATA:message}"]
         overwrite => ["message"]
     }
  }
  if [logtype] == "nginx_logs"{
       mutate {
          convert => [ "status","integer" ]
          convert => [ "size","integer" ]
          convert => [ "upstreatime","float" ]
          convert => ["[geoip][coordinates]", "float"]
          remove_field => "message"
        }

        date {
            match => [ "timestamp" ,"dd/MMM/YYYY:HH:mm:ss Z" ]
        }
        geoip {
          source => "client"  ##日志格式里的ip来源,这里是client这个字段(client":"$remote_addr")
          target => "geoip"
          database =>"/usr/share/logstash/GeoLite2-City.mmdb"   ##### 下载GeoIP库
          add_field => [ "[geoip][coordinates]", "%{[geoip][longitude]}" ]
          add_field => [ "[geoip][coordinates]", "%{[geoip][latitude]}"  ]
        }
       # mutate {
       #   remove_field => "timestamp"
       # }
    }

}



output {

    if [app] == "easydong" {
        if [type] == "applog-V1-appfront1" {
           elasticsearch {
              hosts => ["http://172.17.199.231:9200"]
              password => "111111"
              user => "elastic"
              index => "applog-v1-appfront1-%{+YYYY.MM.dd}"
           }
        }
        else if [type] == "adminlog-V1-appfront1" {
           elasticsearch {
              hosts => ["http://172.17.199.231:9200"]
              password => "111111"
              user => "elastic"
              index => "adminlog-v1-appfront1-%{+YYYY.MM.dd}"
           }
        }
        else if [type] == "applog-V1-appfront2" {
           elasticsearch {
              hosts => ["http://172.17.199.231:9200"]
              #password => "111111"
              #user => "elastic"
              index => "applog-v1-appfront2-%{+YYYY.MM.dd}"
           }
        }
        else if [type] == "adminlog-V1-appfront2" {
           elasticsearch {
              hosts => ["http://172.17.199.231:9200"]
              #password => "111111"
              #user => "elastic"
              index => "adminlog-v1-appfront2-%{+YYYY.MM.dd}"
           }
        }

        
        else if [type] == "applog-V2-appfront1" {
           elasticsearch {
              hosts => ["http://172.17.199.231:9200"]
              #password => "111111"
              #user => "elastic"
              index => "applog-v2-appfront1-%{+YYYY.MM.dd}"
           }  
        }  
        else if [type] == "adminlog-V2-appfront1" {
           elasticsearch {
              hosts => ["http://172.17.199.231:9200"]
              #password => "111111"
              #user => "elastic"
              index => "adminlog-v2-appfront1-%{+YYYY.MM.dd}"
           }  
        }  
        else if [type] == "applog-V2-appfront2" {
           elasticsearch {
              hosts => ["http://172.17.199.231:9200"]
              #password => "111111"
              #user => "elastic"
              index => "applog-v2-appfront2-%{+YYYY.MM.dd}"
           }  
        }  
        else if [type] == "adminlog-V2-appfront2" {
           elasticsearch {
              hosts => ["http://172.17.199.231:9200"]
              #password => "111111"
              #user => "elastic"
              index => "adminlog-v2-appfront2-%{+YYYY.MM.dd}"
           }  
        }  

        else if [type] == "indonesia-adminlog-1" {
           elasticsearch {
              hosts => ["http://172.17.199.231:9200"]
              #password => "111111"
              #user => "elastic"
              index => "indonesia-adminlog-1-%{+YYYY.MM.dd}"
           }
        }
 
    if [app] == "easydong" {
          if [logtype] == "nginx_logs"{
            elasticsearch {
            hosts => ["http://172.17.199.231:9200"]
            index => "logstash-sanwenqian-nginx-%{+YYYY-MM}"
          }
      }
  }
    
  }

  stdout { codec=> rubydebug }

最后重启logstash看能否正常启动,可以去/var/log/message查看是否有新的日志产生。


文章分类
后端
版权声明:本站是系统测试站点,无实际运营。本文内容由互联网用户自发贡献,该文观点仅代表作者本人。本站仅提供信息存储空间服务,不拥有所有权,不承担相关法律责任。如发现本站有涉嫌抄袭侵权/违法违规的内容, 请发送邮件至 XXXXXXo@163.com 举报,一经查实,本站将立刻删除。
相关推荐