配置清单

  • es.yaml ```yaml apiVersion: v1 kind: Service metadata:
    name: elasticsearch-logging
    namespace: kube-system
    labels:
    k8s-app: elasticsearch-logging
    kubernetes.io/cluster-service: “true”
    addonmanager.kubernetes.io/mode: Reconcile
    kubernetes.io/name: “Elasticsearch” spec:
    ports:
    • port: 9200
      protocol: TCP
      targetPort: db
      selector:
      k8s-app: elasticsearch-logging

RBAC authn and authz

apiVersion: v1 kind: ServiceAccount metadata: name: elasticsearch-logging namespace: kube-system labels: k8s-app: elasticsearch-logging kubernetes.io/cluster-service: “true”

  1. addonmanager.kubernetes.io/mode: Reconcile

kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: elasticsearch-logging labels: k8s-app: elasticsearch-logging kubernetes.io/cluster-service: “true” addonmanager.kubernetes.io/mode: Reconcile rules:

  • apiGroups:
    • “” resources:
    • “services”
    • “namespaces”
    • “endpoints” verbs:
    • “get”

kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: namespace: kube-system name: elasticsearch-logging labels: k8s-app: elasticsearch-logging kubernetes.io/cluster-service: “true” addonmanager.kubernetes.io/mode: Reconcile subjects:

  • kind: ServiceAccount name: elasticsearch-logging namespace: kube-system apiGroup: “” roleRef: kind: ClusterRole name: elasticsearch-logging apiGroup: “”

Elasticsearch deployment itself

apiVersion: apps/v1 kind: StatefulSet metadata: name: elasticsearch-logging namespace: kube-system labels: k8s-app: elasticsearch-logging version: v6.2.5 kubernetes.io/cluster-service: “true” addonmanager.kubernetes.io/mode: Reconcile spec: serviceName: elasticsearch-logging replicas: 1 selector: matchLabels: k8s-app: elasticsearch-logging version: v6.2.5 template: metadata: labels: k8s-app: elasticsearch-logging version: v6.2.5 kubernetes.io/cluster-service: “true” spec: serviceAccountName: elasticsearch-logging containers:

  1.   - image: harbor.querycap.com/rk-infra/elasticsearch:6.8.8-v0.4
  2.     name: elasticsearch-logging
  3.     resources:
  4.       # need more cpu upon initialization, therefore burstable class
  5.       limits:
  6.         cpu: 1000m
  7.       requests:
  8.         cpu: 100m
  9.     ports:
  10.     - containerPort: 9200
  11.       name: db
  12.       protocol: TCP
  13.     - containerPort: 9300
  14.       name: transport
  15.       protocol: TCP
  16.     volumeMounts:
  17.     - name: elasticsearch-logging
  18.       mountPath: /usr/share/elasticsearch/data/
  19.     - mountPath: /usr/share/elasticsearch/config/elastic-certificates.p12
  20.       name: keystore
  21.       readOnly: true
  22.       subPath: elastic-certificates.p12
  23.     env:
  24.     - name: "NAMESPACE"
  25.       valueFrom:
  26.         fieldRef:
  27.           fieldPath: metadata.namespace
  28.     - name: "xpack.security.enabled"
  29.       value: "true"
  30.     - name: "xpack.security.transport.ssl.enabled"
  31.       value: "true"
  32.     - name: xpack.security.transport.ssl.verification_mode
  33.       value: "certificate"
  34.     - name: xpack.security.transport.ssl.keystore.path
  35.       value: "/usr/share/elasticsearch/config/elastic-certificates.p12"
  36.     - name: xpack.security.transport.ssl.truststore.path
  37.       value: "/usr/share/elasticsearch/config/elastic-certificates.p12"
  38.     - name: "xpack.ml.enabled"
  39.       value: "false"
  40.     - name: "xpack.monitoring.collection.enabled"
  41.       value: "true"
  42.     - name: "xpack.license.self_generated.type"
  43.       value: "basic"
  44.   volumes:
  45.   - name: elasticsearch-logging
  46.     hostPath:
  47.       path: /data/es/
  48.   - name: keystore
  49.     secret:
  50.       secretName: es-keystore
  51.       defaultMode: 0444
  52.   nodeSelector:
  53.     es: data

tolerations:

- key: “dedicated”

operator: “Equal”

value: “es”

effect: “NoExecute”

  1.   # Elasticsearch requires vm.max_map_count to be at least 262144.
  2.   # If your OS already sets up this number to a higher value, feel free
  3.   # to remove this init container.
  4.   initContainers:
  5.   - name: elasticsearch-logging-init
  6.     image: alpine:3.6
  7.     command: ["/sbin/sysctl", "-w", "vm.max_map_count=262144"]
  8.     securityContext:
  9.       privileged: true
  10.   - name: elasticsearch-volume-init
  11.     image: alpine:3.6
  12.     command:
  13.       - chmod
  14.       - -R
  15.       - "777"
  16.       - /usr/share/elasticsearch/data/
  17.     volumeMounts:
  18.     - name: elasticsearch-logging
  19.       mountPath: /usr/share/elasticsearch/data/
  2. - es-secret.yaml
  3. ```yaml
  4. apiVersion: v1
  5. kind: Secret
  6. metadata:
  7.   name: es-keystore
  8.   namespace: kube-system
  9. data:
  10.   elastic-certificates.p12: MIINdwIBAzCCDTAGCSqGSIb3DQEHAaCCDSEEgg0dMIINGTCCBW0GCSqGSIb3DQEHAaCCBV4EggVaMIIFVjCCBVIGCyqGSIb3DQEMCgECoIIE+zCCBPcwKQYKKoZIhvcNAQwBAzAbBBQs3hsugdvGL7FWl+IJ6pxClg251AIDAMNQBIIEyF81+gPdcBuo0evFTyLXRtMD+YbLyJb7o3xzzS5BwZb+kvSCSw+lGkGybsL5iglxcBogmjTUbaSfjUbzCQ0xxisKVf1yR9DeP1e6Dm6lzF1lhOKYnxEtR813TXABy85ZEtczIDZkM5C+VKUoz/k4vuSLacaRaoEw4VKvXNo9mX+quzKajWKauZwDV9iP5Kf+4UZw6cQCPT4Xv2t7rab/l6e7peHL/r8hKDMmZAfRNgzjTMV1CrhIRHLPc9b5eOTUfk0uwIdjDhEE9ElB3dhbZFzbMItFPA2L2cCFi42eY1rQKATcoDpifF9NMjWJ8GtSnRmvc0irsJku8yoVZWDyghNgXAQT8JxGy9daV3WAMMbUIsqaV0r09WyH8kAikj/C+Cz9HwyctQs3vJc3fBMq5Dh7cxqaVctvVkxzcTPTv/OnCRg8dXto07xdsnTE37/qvrzw1cwyv5oZ2eBsem+490ulyWCIMrYN+owX1RScnPnI5RjOun2fRwN9G18LJrS6iJuSZVd4cQcWvMO/zmrsKga0KW3VFtJvTE+Pxad0wqN5h0Tx0DWhTJ8iqhMwPL+uHnhAQQUFvFPFSJEWwAYZgbGxYYRXkiZSML3KMWC57+2UYPKgSQHAVDc/RI+AJlMYfRw6QD174OZBG7PiBWJEiRtLqvH2dhcC9mGJEtOgrJyn/CDtSKgBnQm/beHG47BD4esX6A+RUgbvwsn4urYVMPy2qOlefAnSoru58bueQlt+j5CcsiQtaRlw3rzS+5V6U+VSFgVMZxsUzl8n3uV7uMfDSOx6abFGBMw8QNSMQhfaC9qqyU45QkyDeEvpyXY4ThDMV8Cw75g99+LpLfz1BHS2p/iG/6056Qmt7MC/3N6VVrvh5LCWQ6uFjGebqH9HKLDNHq31V7k50W0uZZdQLR0hFfnAX02zb4Rsen1vXvWH5kpu90tNe43IIzvFCD/dHM6L7tNzdNd3ke2bSmqsqp2bDAQew3Wpj3aWdI+HzjTmwONY7S5V88MQoQTeWXcjB8snSQkXYH4fh/vGfL+MpV+pSVtylIauDnMrj2JFOETGmQGHUuk9A1ofjzGdqcMIOGalzDrOb14eC5BVgfnKPOtR4bXar+UMmIIJwAaMQRqVO5D7mNhSPBwGPEW8SQQBbSfELaYHMi9q/Ii6B6D4knxt9eoWlDj7o4/M9omTuX53/HtWB1/4FbDinBNltr55dyazxPjRgzwAIr8mqUxxQoitWVys0oXos7yhizwdKn5/EaLJMo9ii9yIW058qvwpJ+juO36NCW1YK645X/fvu8+1UVC9VVHTxJSGei+RmHXJwRCVqSVG4ckHWPdlhKYPj3auMwkbWNqDOVipT13fwpTws2aGsxld4K6iWT9urbtxxhx8Hg0q35NMEILUODdjF3BbgsXZW40Hj3FN6ALvgFHolU7Homgna+Euvs/TJM62jrr02qZsRnZun5TvYRS9W5FVMU5Vjicfnp+ieCFM2Rl8iasd+h0t3DPBgv9m/xvYqpgD5aUyO+xyL0ikC8se0Nm+IxVKKV1ZATcLaXzZE3EkKu1Fk7T66ODEpkYmzkWzyP+XexqnDUEiQ35iR1yRyoKViArHgxmVx3u3TDo+Frt7c+wP2c/VkTFEMB8GCSqGSIb3DQEJFDESHhAAaQBuAHMAdABhAG4AYwBlMCEGCSqGSIb3DQEJFTEUBBJUaW1lIDE1ODkyNTQzODI2MjkwggekBgkqhkiG9w0BBwagggeVMIIHkQIBADCCB4oGCSqGSIb3DQEHATApBgoqhkiG9w0BDAEGMBsEFLzuguhJknOppoNt0cNAu/Yg9N7IAgMAw1CAggdQwSkHPuMWczUu39ME4ABWIYkp4Wk9MJ4HfM3ovhE+gskOjX6DG+o7H4sZ0okF0kcBoLAOo6Sgr/s9ZuVgYHTi5xtkGQaqbQUSMK03cpWUfue5f1ruRu93KVll6mMAz9z/s6/hG27hhanQUFDcZ4md9dFQHLv5ncBeMwrseeP2sUy/my1qYetlTIdskjfeNh2Dc2uZPTmDv6B/dj+nBHPd4B6zlqHB73JCvE8Qq6zWjjb6/d0geYMNKbdpbQjKbKxNgjzatand27Bh0Bpuk+OeqXF4+TrbwCyaiTnYYQPKbve/zsM7xVzvb5p9sdHmhOdI54+SuImIMuEQSikPmQk4MdPQaS27r7JAvsabdNm9jT1nzIMzINAj3b6LBTckbd94gP2ST2vizvq0maH2AtpZT5twX7AnfTnC2xuuKlZ1sJHsfl2Uw9JyvzNdMtZUaDO+0obesE1dBjH+nI5z75DzFLLbDl65HCJxL1qtvhSCXF676cQ2N4Mq2Sqs+nhzyVy+/NgRcfN/YtagWyYAzxdQmaW0lIQfdDcbYgW+1w+8RUht5imx5G/A0foKCcW1mGW2ocXzUkMxAqspHu3Tb3lazUpSS8BNWELtC+d3H8qx0qsPMenxe0sUxB/pUW5OFqcZCAC2hlUThtUD4E51joKJha0+DraiqmwaaUNFy2CcgqCd52WrwnpRV5gyS+Bk24d/4Dm3ezEZnMQxGiBwB55nHKWlgwAzTqnJWXmM/wYEkEVLsnN9bYBU2e8yM8Q9Qv6Vszg+8GqfIeXs4hjCAIJ8YWCDGtL/rM9+L0lsMfh73jo9Hj0UnvqLgIEaV6pqpk6O7NgdhiWEFDyIbbEWPEpFiLsGclpnELxZbxlWdDlcAM3dpU7hAMZ5AxSv6or7U7WkSfj/K8EzDjpQ66/klbsZiob/EEehdeBRv0ZFTR1uOLY+h5kJub0b/tflw36F0BA2wqxr6pCG8/zdKpET7CWu7TWKtX1Uw7/z16s3uJ6zYIyJJNXwB4c1bbdHiAjAfwh2SgVqGuRjQpN2/vKcwCRC/xWSa9ME/rAA6HhvtNgn0s5dTyKbbTKBkEvsWwfwEFAE3mlRO7t9waAmtWabNdJKSO2z70Qc3Irr+wJw+zDDD7+hfKcRxTFHZ65tlR75QgmWXRMMFMQW9nuKN9hY1k0XRv7qppD2jQRV19sR87Aq+5TbihzmbitYmeqBu87oi82HOu6/gReFGRgz1cGaS6y7+v00xoUiPU5Pc6k4bS0E2/1C/CUARN0K4ELCx+MqlWz01tDMnSzAOvDtB+EmgBIrCPCmVqwvoH2DS5hOoslYRx8LtQtQtb+DeFvxCRHCNzEi4+CwWC0GzWOW2Lz7feCyhwoDGkubRyESKbJ7VKWB4A5s5iyLp2Wc/ewFbBwaywpQ2AFezS8Ka2DiYPFLlf0shfycWjlQih6KYkxVHzd1i2E4mau0FWMNMirqRWkQ2rl5j7FoyNOqtfcy/K2N+FlzT6Y1Sjkw3imeqdxa+CWJAyC2W1dahDgnbR7JbLoKY5P8a++iJ6mbiHTRs8NcWtadcF0bbG1ycdcz/7jNzU7tDQODyfWH43Nf4GzcsPzv+WbDUmuUBuv8PrSeKIhMPc6XjmYV7Qgd8BEwwTpU3DTobvnO60HyoGBRJFsLfd7w0iXPUNqLWK4u1RSCyMjf1m+MfBHPunccqYaY4zgzj4cFkUMpAnpa6IziRDdUE0Vc3ePJSBAQ9WtA66sIIXm7lmwPzENBcUNcFLxX3H1PzWZzKuwBzTVsj7D+ECFL2jnjbq5HmvBKrC05YkT9bTonCWKYDwKwxheEZYSzexfJrHos98PZ28YiLA5F8ifQ/9Lc+T1prWuJ26hA8fUoicQHIaUS/rx3BuibnlfVroZUqsqC4zpYn3hKZDmO+jLrH3kxRTwHcS2Yo/qcseJy4OGseocvH5NDN5LnjHpEw+mqW7Qz7S9ZxmNxU9PwsYBq2Q39TKj0KjnGtIQjGYI4j+Ql6icC0Z3WbSCZseB4rEkG18LLpvdD/U0p6AwZ3MXPFYZOJn3k9TSs30Wbg4+bLUM9OT9LE8XLOFSEQCxgQyVAFpuQyfL3YnAThUoVJoWDr5DktNQ9pwtNhAo8xbQGdkAWUtKBkESRc4tekqSrxI0Zt0iD8VqluaTbt4RMJbtu2q+4vNLuJV6f3PeS0hzEsALwN3ZcNDHNRlVKhctDr0W2K1rVl+p+uGyDTvX7Yx6jEwghs0xoRhU3yU7f/FGhMhQZlZc/xLL/AJLXOAaTXIKO6PsySurWZrzHrboSk+65WmymdBOlEzNf+R/oriqeTNV/RoNVTglLyDdDmycYonbaexz03gHC/g40ypo7jECE3V130kApgnDvt6QRC/yd6NiNh1V4E8gVbm4J7k+gE++4tVE7KFGOSvWwPjkSbXe5PDejENuVX2v4UPuaMoUXAPUZSNNFao4bM4eS50U/g67pnl4PYbdtG/yrjCNnXCaRZRapDVD/MD4wITAJBgUrDgMCGgUABBR9QitEdRrOhirFpeeXYxRCiDXobgQUgdxm0RxqdfUMrM7V9NYJh5DmvWsCAwGGoA==
  1. ---
  2. # Source: filebeat/templates/filebeat-configmap.yaml
  3. ---
  4. apiVersion: v1
  5. kind: ConfigMap
  6. metadata:
  7.   name: filebeat-config
  8.   namespace: kube-system
  9.   labels:
  10.     k8s-app: filebeat
  11. data:
  12.   filebeat.yml: |-
  13.     filebeat.config:
  14.       max_procs: 1 
  15.       queue.mem.events: 16
  16.       queue.mem.flush.min_events: 8
  18.       inputs:
  19.         # Mounted `filebeat-inputs` configmap:
  20.         path: ${path.config}/inputs.d/*.yml
  21.         # Reload inputs configs as they change:
  22.         reload.enabled: true
  23.         enabled: true
  25.       modules:
  26.         path: ${path.config}/modules.d/*.yml
  27.         # Reload module configs as they change:
  28.         reload.enabled: true
  29.     #output.kafka:
  30.     #  hosts: ["kafka-log-01:9092", "kafka-log-02:9092", "kafka-log-03:9092"]
  31.     # topic: 'topic-test-log'
  32.     #  version: 2.0.0
  33.     output.logstash:
  34.        hosts: ["logstash:5044"]
  35.        enabled: true
  37. ---
  38. apiVersion: v1
  39. kind: ConfigMap
  40. metadata:
  41.   name: filebeat-inputs
  42.   namespace: kube-system
  43.   labels:
  44.     k8s-app: filebeat
  45. data:
  46.   kubernetes.yml: |-
  47.     - type: docker
  48.       #harvester_buffer_size: 4096000
  49.       close_inactive: 15m
  50.       scan_frequency: 30s
  51.       containers.ids:
  52.       - "*"
  53.       enabled: true
  54.       processors:
  55.         - add_kubernetes_metadata:
  56.             in_cluster: true
  68. ---
  69. # Source: filebeat/templates/filebeat-service-account.yaml
  70. apiVersion: v1
  71. kind: ServiceAccount
  72. metadata:
  73.   name: filebeat
  74.   namespace: kube-system
  75.   labels:
  76.     k8s-app: filebeat
  78. ---
  79. # Source: filebeat/templates/filebeat-role.yaml
  80. apiVersion: rbac.authorization.k8s.io/v1beta1
  81. kind: ClusterRole
  82. metadata:
  83.   name: filebeat
  84.   labels:
  85.     k8s-app: filebeat
  86. rules:
  87. - apiGroups: [""] # "" indicates the core API group
  88.   resources:
  89.   - namespaces
  90.   - pods
  91.   verbs:
  92.   - get
  93.   - watch
  94.   - list
  96. ---
  97. # Source: filebeat/templates/filebeat-role-binding.yaml
  98. apiVersion: rbac.authorization.k8s.io/v1beta1
  99. kind: ClusterRoleBinding
  100. metadata:
  101.   name: filebeat
  102. subjects:
  103. - kind: ServiceAccount
  104.   name: filebeat
  105.   namespace: kube-system
  106. roleRef:
  107.   kind: ClusterRole
  108.   name: filebeat
  109.   apiGroup: rbac.authorization.k8s.io
  111. ---
  112. # Source: filebeat/templates/filebeat-daemonset.yaml
  113. apiVersion: extensions/v1beta1
  114. kind: DaemonSet
  115. metadata:
  116.   name: filebeat
  117.   namespace: kube-system
  118.   labels:
  119.     k8s-app: filebeat
  120. spec:
  121.   template:
  122.     metadata:
  123.       labels:
  124.         k8s-app: filebeat
  125.     spec:
  126.       serviceAccountName: filebeat
  127.       terminationGracePeriodSeconds: 30
  128.       nodeSelector:
  129.         beta.kubernetes.io/filebeat-ds-ready: "true"
  130.       containers:
  131.       - name: filebeat
  132.         image: "harbor.querycap.com/rk-infra/filebeat:6.6.2"
  133.         args: [
  134.           "-c", "/etc/filebeat.yml",
  135.           "-e","-httpprof","0.0.0.0:6060"
  136.         ]
  137.         #ports:
  138.         #  - containerPort: 6060
  139.         #    hostPort: 6068
  140.         securityContext:
  141.           runAsUser: 0
  142.           # If using Red Hat OpenShift uncomment this:
  143.           #privileged: true
  144.         resources:
  145.           limits:
  146.             memory: 1000Mi
  147.             cpu: 1000m
  148.           requests:
  149.             memory: 100Mi
  150.             cpu: 100m          
  151.         volumeMounts:
  152.         - name: config
  153.           mountPath: /etc/filebeat.yml
  154.           readOnly: true
  155.           subPath: filebeat.yml
  156.         - name: inputs
  157.           mountPath: /usr/share/filebeat/inputs.d
  158.           readOnly: true          
  159.         - name: data
  160.           mountPath: /usr/share/filebeat/data
  161.         - name: varlibdockercontainers
  162.           mountPath: /var/lib/docker/containers
  163.           readOnly: true
  164.         - name: timezone
  165.           mountPath: /etc/localtime
  166.       imagePullSecrets:
  167.       - name: qingcloud-registry
  168.       volumes:
  169.       - name: config
  170.         configMap:
  171.           defaultMode: 0600
  172.           name: filebeat-config
  173.       - name: varlibdockercontainers
  174.         hostPath:
  175.           path: /data/var/lib/docker/containers
  176.       - name: inputs
  177.         configMap:
  178.           defaultMode: 0600
  179.           name: filebeat-inputs
  181.       # data folder stores a registry of read status for all files, so we don't send everything again on a Filebeat pod restart
  182.       - name: data
  183.         hostPath:
  184.           path: /data/filebeat-data
  185.           type: DirectoryOrCreate
  186.       - name: timezone
  187.         hostPath:
  188.           path: /etc/localtime
  189.       tolerations:
  190.       - effect: NoExecute
  191.         key: dedicated
  192.         operator: Equal
  193.         value: gpu
  194.       - effect: NoSchedule
  195.         operator: Exists
  • kibana.yaml ```yaml apiVersion: extensions/v1beta1 kind: Ingress metadata: name: kibana namespace: kube-system annotations: kubernetes.io/ingress.class: “nginx”
    spec: rules:
    • host: a.kibana.rockontrol.com http: paths:
      • backend: serviceName: kibana servicePort: 5601

apiVersion: v1 kind: Service metadata: name: kibana namespace: kube-system labels: k8s-app: kibana kubernetes.io/cluster-service: “true” addonmanager.kubernetes.io/mode: Reconcile kubernetes.io/name: “Kibana” spec: type: ClusterIP ports:

  • port: 5601 protocol: TCP targetPort: ui selector: k8s-app: kibana

apiVersion: apps/v1 kind: Deployment metadata: name: kibana namespace: kube-system labels: k8s-app: kibana kubernetes.io/cluster-service: “true” addonmanager.kubernetes.io/mode: Reconcile spec: replicas: 1 selector: matchLabels: k8s-app: kibana template: metadata: labels: k8s-app: kibana annotations: seccomp.security.alpha.kubernetes.io/pod: ‘docker/default’ spec: imagePullSecrets:

  1.   - name: qingcloud-registry
  2.   containers:
  3.   - name: kibana
  4.     image: harbor.querycap.com/rk-infra/kibana:6.8.8
  5.     resources:
  6.       # need more cpu upon initialization, therefore burstable class
  7.       limits:
  8.         cpu: 1000m
  9.       requests:
  10.         cpu: 100m
  11.     env:
  12.       - name: ELASTICSEARCH_URL
  13.         value: http://elasticsearch-logging:9200
  14.       - name: XPACK_SECURITY_ENABLED
  15.         value: "true"
  16.       - name: ELASTICSEARCH_PASSWORD
  17.         value: M50zLpCI0EeDcOh7
  18.       - name: ELASTICSEARCH_USERNAME
  19.         value: kibana
  20.       - name: XPACK_ML_ENABLED
  21.         value: "false"
  22.     ports:
  23.     - containerPort: 5601
  24.       name: ui
  25.       protocol: TCP
  2. - [**logstash.yaml**](https://git.querycap.com/infra/k8s-elk/-/blob/master/logstash.yaml)
  3. ```yaml
  4. apiVersion: v1
  5. kind: Service
  6. metadata:
  7.   name: logstash
  8.   namespace: kube-system
  9. spec:
  10.   ports:
  11.   - port: 5044
  12.     targetPort: beats
  13.   selector:
  14.     type: logstash
  15.   clusterIP: None
  16. ---
  17. apiVersion: extensions/v1beta1
  18. kind: Deployment
  19. metadata:
  20.   name: logstash
  21.   namespace: kube-system
  22. spec:
  23.   template:
  24.     metadata:
  25.       labels:
  26.         type: logstash
  27.     spec:
  28.       containers:
  29.       - image: harbor.querycap.com/rk-infra/logstash:v6.6.2
  30.         name: logstash
  31.         ports:
  32.         - containerPort: 5044
  33.           name: beats
  34.         command:
  35.         - logstash
  36.         - '-f'
  37.         - '/etc/logstash_c/logstash.conf'
  38.         env:
  39.         - name: "XPACK_MONITORING_ELASTICSEARCH_HOSTS"
  40.           value: "http://elasticsearch-logging:9200"      
  41.         volumeMounts:
  42.         - name: config-volume
  43.           mountPath: /etc/logstash_c/
  44.         - name: config-yml-volume
  45.           mountPath: /usr/share/logstash/config/
  46.         - name: timezone
  47.           mountPath: /etc/localtime
  48.         resources:
  49.           limits:
  50.             cpu: 1000m
  51.             memory: 2048Mi
  52.           # requests:
  53.           #   cpu: 512m
  54.           #   memory: 512Mi
  55.       imagePullSecrets:
  56.       - name: qingcloud-registry
  57.       volumes:
  58.       - name: config-volume
  59.         configMap:
  60.           name: logstash-conf
  61.           items:
  62.           - key: logstash.conf
  63.             path: logstash.conf
  64.       - name: timezone
  65.         hostPath:
  66.           path: /etc/localtime
  67.       - name: config-yml-volume
  68.         configMap:
  69.           name: logstash-yml
  70.           items:
  71.           - key: logstash.yml
  72.             path: logstash.yml
  74. ---
  75. apiVersion: v1
  76. kind: ConfigMap
  77. metadata:
  78.   name: logstash-conf
  79.   namespace: kube-system
  80.   labels:
  81.     type: logstash
  82. data:
  83.   logstash.conf: |-
  84.     input {
  85.       beats {
  86.       port => 5044
  87.       }
  88.      }
  90.      filter{
  91.      if [kubernetes][container][name] == "nginx-ingress-controller" {
  92.             json {
  93.        source => "message"
  94.        target => "ingress_log"
  95.      }
  96.      if [ingress_log][requesttime] {
  97.          mutate {
  98.         convert => ["[ingress_log][requesttime]", "float"]
  99.     }
  100.     }
  101.      if [ingress_log][upstremtime] {
  102.          mutate {
  103.         convert => ["[ingress_log][upstremtime]", "float"]
  104.     }
  106.     }
  107.      if [ingress_log][status] {
  108.          mutate {
  109.         convert => ["[ingress_log][status]", "float"]
  110.     }
  112.     }
  113.      if  [ingress_log][httphost] and [ingress_log][uri] {
  114.          mutate {
  115.         add_field => {"[ingress_log][entry]" => "%{[ingress_log][httphost]}%{[ingress_log][uri]}"}
  119.     }
  120.          mutate{
  121.          split => ["[ingress_log][entry]","/"]
  122.        }
  123.          if [ingress_log][entry][1] {
  124.          mutate{
  125.          add_field => {"[ingress_log][entrypoint]" => "%{[ingress_log][entry][0]}/%{[ingress_log][entry][1]}"}
  126.          remove_field => "[ingress_log][entry]"
  127.          }
  128.          }
  129.          else{
  130.           mutate{
  131.           add_field => {"[ingress_log][entrypoint]" => "%{[ingress_log][entry][0]}/"}
  132.           remove_field => "[ingress_log][entry]"
  133.          }
  134.          }
  139.     }
  140.     }
  142.      if [kubernetes][container][name] =~ /^srv*/ {
  143.        json {
  144.        source => "message"
  145.        target => "tmp"
  146.      }
  148.     if [kubernetes][namespace] == "kube-system" {
  150.        drop{}
  151.      }
  154.        if [tmp][level] {
  155.        mutate{
  156.        add_field => {"[applog][level]" => "%{[tmp][level]}"}
  158.      }
  159.        if [applog][level] == "debug"{
  160.        drop{}
  161.      }
  162.      }
  163.        if [tmp][msg]{
  164.       mutate{
  165.        add_field => {"[applog][msg]" => "%{[tmp][msg]}"}
  166.      }
  167.      }
  168.        if [tmp][func]{
  169.       mutate{
  170.        add_field => {"[applog][func]" => "%{[tmp][func]}"}
  171.      }
  172.      }
  173.       if [tmp][cost]{
  174.         if "ms" in [tmp][cost]{
  177.         mutate{
  178.           split => ["[tmp][cost]","m"]
  179.           add_field => {"[applog][cost]" => "%{[tmp][cost][0]}"} 
  180.           convert => ["[applog][cost]", "float"]
  181.         }
  182.         }
  183.         else{
  184.         mutate{
  185.         add_field => {"[applog][cost]" => "%{[tmp][cost]}"}
  186.       }
  187.       }
  188.      }
  189.       if [tmp][method]{
  190.       mutate{
  191.        add_field => {"[applog][method]" => "%{[tmp][method]}"}
  192.      }
  193.      }  
  194.       if [tmp][request_url]{
  195.       mutate{
  196.        add_field => {"[applog][request_url]" => "%{[tmp][request_url]}"}
  197.      }
  198.      }
  199.        if [tmp][meta._id]{
  200.        mutate{
  201.        add_field => {"[applog][traceId]" => "%{[tmp][meta._id]}"}
  202.      }
  203.      }
  206.        if [tmp][project] {
  207.        mutate{
  208.        add_field => {"[applog][project]" => "%{[tmp][project]}"}
  209.      }
  210.      }
  211.        if [tmp][time] {
  212.        mutate{
  213.        add_field => {"[applog][time]" => "%{[tmp][time]}"}
  214.      }
  215.      }
  216.        if [tmp][status] {
  217.        mutate{
  218.        add_field => {"[applog][status]" => "%{[tmp][status]}"}
  219.        convert => ["[applog][status]", "float"]
  220.      }
  221.      }            
  222.      }
  225.     mutate{
  226.       rename => ["kubernetes", "k8s"]
  227.       remove_field => "beat"
  228.       remove_field => "tmp"
  229.       remove_field => "[k8s][labels][app]"
  230.     }
  232.     }  
  235.     output{ 
  236.       elasticsearch {
  238.         hosts => ["http://elasticsearch-logging:9200"]
  240.         codec => json
  242.         index => "logstash-%{+YYYY.MM.dd}"
  244.         user => 'elastic'
  246.         password => 'P9IKmhwuYEEej52U'
  247.         }
  248.       }
  249. ---
  251. apiVersion: v1
  252. kind: ConfigMap
  253. metadata:
  254.   name: logstash-yml
  255.   namespace: kube-system
  256.   labels:
  257.     type: logstash
  258. data:
  259.   logstash.yml: |-
  260.     http.host: "0.0.0.0"
  261.     xpack.monitoring.enabled: true
  262.     xpack.monitoring.elasticsearch.username: logstash_system
  263.     xpack.monitoring.elasticsearch.password: hOsocygbtFEs1qty
  264.     xpack.monitoring.elasticsearch.url: http://elasticsearch-logging:9200
  • 还有一个elastic-certificates.p12，可以自己用二进制生成
    1. /usr/share/elasticsearch/bin/elasticsearch-certutil ca
    2. /usr/share/elasticsearch/bin/elasticsearch-certutil cert --ca elastic-certificates.p12
    3. #两条命令均一路回车即可，不需要给秘钥再添加密码。
    4. #证书创建完成之后，默认在es的数据目录，这里统一放到etc下：
    5. ls elastic-*
    6. elastic-certificates.p12  elastic-certificates-ca.p12

    部署步骤

    给部署es的节点加上标签

  1. kubectl label node <node> es=data

部署

  1. kubectl apply -f .

创建用户

  1. kubectl -n kube-system exec -it elasticsearch-logging-0 ./init-user.sh

这里的脚本是自己提前打包到镜像中的，没有使用官方镜像

  1. cat init-user.sh
  2. #!/usr/bin/expect
  3. set timeout -1
  4. spawn bin/elasticsearch-setup-passwords interactive
  5. expect "\[y/N\]"
  6. send  "y\r"
  7. expect "\[elastic\]"
  8. send "P9IKmhwuYEEej52U\r"
  9. expect "\[elastic\]"
  10. send "P9IKmhwuYEEej52U\r"
  11. expect "\[apm_system\]"
  12. send "1UmdUStBiKYqR0IJ\r"
  13. expect "\[apm_system\]"
  14. send "1UmdUStBiKYqR0IJ\r"
  15. expect "\[kibana\]"
  16. send "M50zLpCI0EeDcOh7\r"
  17. expect "\[kibana\]"
  18. send "M50zLpCI0EeDcOh7\r"
  19. expect "\[logstash_system\]"
  20. send "hOsocygbtFEs1qty\r"
  21. expect "\[logstash_system\]"
  22. send "hOsocygbtFEs1qty\r"
  23. expect "\[beats_system\]"
  24. send "xy28upiAPDNGfmgw\r"
  25. expect "\[beats_system\]"
  26. send "xy28upiAPDNGfmgw\r"
  27. expect "\[remote_monitoring_user\]"
  28. send "jooKJoYHlMiZwTXX\r"
  29. expect "\[remote_monitoring_user\]"
  30. send "jooKJoYHlMiZwTXX\r"

用户

  2. # 普通用户
  3. username: kibana
  4. password: M50zLpCI0EeDcOh7
  7. # 只读用户
  8. username: viewer
  9. password: WBQ3p4ZZ7WappBfEb

viewer 用的的权限配置

  1. 创建 viewer role 角色， 权限如下。

image.png

  1. 创建 viewer 账户， 授权 viewer 角色。