docker-compose部署logstash、kafka

    1. version: '3.3'
    2. services:
    3.   pod-logstash:
    4.     image: logstash:7.13.4
    5.     volumes:
    6.     - /data/xtalpi/config/logstash-podlog.conf:/usr/share/logstash/config/logstash-podlog.conf
    7.     - /data/xtalpi/config/logstash.yml:/usr/share/logstash/config/logstash.yml
    8.     - /data:/data
    9.     - "/etc/localtime:/etc/localtime:ro"
    10.     - "/etc/timezone:/etc/timezone:ro"
    11.     restart: always
    12.     command: ["/usr/share/logstash/bin/logstash","-f","/usr/share/logstash/config/logstash-podlog.conf"]
    13.     # command: ["sleep","1000"]
    14.     user: root
    15.     links:
    16.       - kafka:kafka
    17.     depends_on:
    18.       - kafka
    19.     environment:
    20.       TZ: Asia/Shanghai
    22.   xpipline-logstash:
    23.     image: logstash:7.13.4
    24.     volumes:
    25.     - /data/xtalpi/config/logstash-podlog.conf.xpipline:/usr/share/logstash/config/logstash-x-podlog.conf
    26.     - /data/xtalpi/config/logstash.yml:/usr/share/logstash/config/logstash.yml
    27.     - /data:/data
    28.     - "/etc/localtime:/etc/localtime:ro"
    29.     - "/etc/timezone:/etc/timezone:ro"
    30.     restart: always
    31.     command: ["/usr/share/logstash/bin/logstash","-f","/usr/share/logstash/config/logstash-x-podlog.conf"]
    32.     # command: ["sleep","1000"]
    33.     user: root
    34.     links:
    35.       - kafka:kafka
    36.     depends_on:
    37.       - kafka
    38.       - pod-logstash
    39.     environment:
    40.       TZ: Asia/Shanghai
    42.   system-logstash:
    43.     image: logstash:7.13.4
    44.     volumes:
    45.     - /data/xtalpi/config/logstash-syslog.conf:/usr/share/logstash/config/logstash-syslog.conf
    46.     - /data/xtalpi/config/logstash.yml:/usr/share/logstash/config/logstash.yml
    47.     - /data:/data
    48.     - "/etc/localtime:/etc/localtime:ro"
    49.     - "/etc/timezone:/etc/timezone:ro"
    50.     restart: always
    51.     command: ["/usr/share/logstash/bin/logstash","-f","/usr/share/logstash/config/logstash-syslog.conf"]
    52.     user: root
    53.     depends_on:
    54.       - kafka
    55.     environment:
    56.       TZ: Asia/Shanghai
    58.   nginx:
    59.     image: nginx:1.21.1
    60.     volumes:
    61.     - /data/xtalpi/config/nginx.conf:/etc/nginx/nginx.conf
    62.     - /data/xtalpi/config/mime.types:/etc/nginx/mime.types
    63.     - /data/log/nginx:/data/nginx/logs
    64.     - /data:/data
    65.     - "/etc/localtime:/etc/localtime:ro"
    66.     - "/etc/timezone:/etc/timezone:ro"
    67.     restart: always
    68.     #command: ["sleep","1000000"]
    69.     links:
    70.       - getLog:getLog
    71.     ports:
    72.     - 8888:8888
    73.     environment:
    74.       TZ: Asia/Shanghai
    76.   getLog:
    77.     image: centos:centos7
    78.     volumes:
    79.     - /data/xtalpi/getLog:/data/xtalpi/getLog
    80.     - "/etc/localtime:/etc/localtime:ro"
    81.     - "/etc/timezone:/etc/timezone:ro"
    82.     - /data:/data
    83.     restart: always
    84.     working_dir: "/data/xtalpi/getLog/"
    85.     command: ["/data/xtalpi/getLog/getLog","-listen-addr","0.0.0.0:10000"]
    86.     ports:
    87.     - 10000:10000
    88.     environment:
    89.       TZ: Asia/Shanghai
    91.   zookeeper:
    92.     image: wurstmeister/zookeeper:3.4.6
    93.     volumes:
    94.     - "/etc/localtime:/etc/localtime:ro"
    95.     - "/etc/timezone:/etc/timezone:ro"
    96.     ports:
    97.       - "2181:2181"
    98.     environment:
    99.       TZ: Asia/Shanghai
    101.   kafka:
    102.     image: wurstmeister/kafka:2.12-2.5.0
    103.     volumes:
    104.     - /data/log/kafka:/kafka
    105.     - "/etc/localtime:/etc/localtime:ro"
    106.     - "/etc/timezone:/etc/timezone:ro"
    107.     user: root
    108.     ports:
    109.       - 9092:9092
    110.     links:
    111.       - zookeeper:zk
    112.     environment:
    113.       KAFKA_ADVERTISED_HOST_NAME: "10.41.16.11"
    114.       KAFKA_ADVERTISED_PORT: "9092"
    115.       KAFKA_ZOOKEEPER_CONNECT: "zk:2181"
    116.       KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://10.41.16.11:9092
    117.       TZ: Asia/Shanghai
    1. input {
    2.   file {
    3.     type => "xpipline-log"
    4.     path => [ "/data/podlog/*/production/xpipeline*.log" ]
    5.   }
    6. }
    7. filter {
    8.   grok {
    9.     match => {
    10.       "message" => " (?<fullTime>\S{10}:\S{8}) %{IPORHOST} \[%{IP:ip}\] - %{NUMBER:response_code:int} %{WORD:request_type} %{DATA:request_uri} %{NUMBER:response_time:int}us %{NUMBER:bytes:int}"
    11.     }
    12.     overwrite => ["message"]
    13.   }
    14.   #grok匹配失败日志记录直接删除
    15.   if "_grokparsefailure" in [tags] { drop {} }
    16.   #date匹配match => [ "字段", "时间格式" ]，target将匹配字段赋值给"@timestamp"
    17.   date {
    18.     match => [  "fullTime", "yyyy-MM-dd':'HH:mm:ss" ]
    19.     target => "@timestamp"
    20.   }
    21. }
    22. output {
    23.   elasticsearch {
    24.     hosts => ["10.42.0.5:9201"]
    25.     index => "newdrug-xpipline-%{+YYYY.MM}"
    26.   }
    27. }