Heapster

在kubernets1.13之前，收集资源指标是Heapster，它是一个收集者，它从cAdvisor中收集数据并暂时存储，如果要永久存储，则要借助时序数据库，比如InfluxD，然后用Grafana展示。其架构如下：
image.png

部署

部署influxDB:

  1. apiVersion: apps/v1 
  2. kind: Deployment
  3. metadata:
  4.   name: monitoring-influxdb
  5.   namespace: kube-system
  6. spec:
  7.   replicas: 1
  8.   selector:
  9.     matchLabels:
  10.       task: monitoring
  11.       k8s-app: influxdb
  12.   template:
  13.     metadata:
  14.       labels:
  15.         task: monitoring
  16.         k8s-app: influxdb
  17.     spec:
  18.       containers:
  19.       - name: influxdb
  20.         image: k8s.gcr.io/heapster-influxdb-amd64:v1.5.2
  21.         volumeMounts:
  22.         - mountPath: /data
  23.           name: influxdb-storage
  24.       volumes:
  25.       - name: influxdb-storage
  26.         emptyDir: {}
  27. ---
  28. apiVersion: v1
  29. kind: Service
  30. metadata:
  31.   labels:
  32.     task: monitoring
  33.     # For use as a Cluster add-on (https://github.com/kubernetes/kubernetes/tree/master/cluster/addons)
  34.     # If you are NOT using this as an addon, you should comment out this line.
  35.     kubernetes.io/cluster-service: 'true'
  36.     kubernetes.io/name: monitoring-influxdb
  37.   name: monitoring-influxdb
  38.   namespace: kube-system
  39. spec:
  40.   ports:
  41.   - port: 8086
  42.     targetPort: 8086
  43.   selector:
  44.     k8s-app: influxdb

部署rbac：

  1. kind: ClusterRoleBinding
  2. apiVersion: rbac.authorization.k8s.io/v1beta1
  3. metadata:
  4.   name: heapster
  5. roleRef:
  6.   apiGroup: rbac.authorization.k8s.io
  7.   kind: ClusterRole
  8.   name: system:heapster
  9. subjects:
  10. - kind: ServiceAccount
  11.   name: heapster
  12.   namespace: kube-system

部署heapster：

  1. apiVersion: v1
  2. kind: ServiceAccount
  3. metadata:
  4.   name: heapster
  5.   namespace: kube-system
  6. ---
  7. apiVersion: apps/v1 
  8. kind: Deployment
  9. metadata:
  10.   name: heapster
  11.   namespace: kube-system
  12. spec:
  13.   replicas: 1
  14.   selector:
  15.     matchLabels:
  16.       task: monitoring
  17.       k8s-app: heapster
  18.   template:
  19.     metadata:
  20.       labels:
  21.         task: monitoring
  22.         k8s-app: heapster
  23.     spec:
  24.       serviceAccountName: heapster
  25.       containers:
  26.       - name: heapster
  27.         image: k8s.gcr.io/heapster-amd64:v1.5.4
  28.         imagePullPolicy: IfNotPresent
  29.         command:
  30.         - /heapster
  31.         - --source=kubernetes:https://kubernetes.default
  32.         - --sink=influxdb:http://monitoring-influxdb.kube-system.svc:8086
  33. ---
  34. apiVersion: v1
  35. kind: Service
  36. metadata:
  37.   labels:
  38.     task: monitoring
  39.     # For use as a Cluster add-on (https://github.com/kubernetes/kubernetes/tree/master/cluster/addons)
  40.     # If you are NOT using this as an addon, you should comment out this line.
  41.     kubernetes.io/cluster-service: 'true'
  42.     kubernetes.io/name: Heapster
  43.   name: heapster
  44.   namespace: kube-system
  45. spec:
  46.   ports:
  47.   - port: 80
  48.     targetPort: 8082
  49.   selector:
  50.     k8s-app: heapster

部署grafana：

  1. apiVersion: apps/v1 
  2. kind: Deployment
  3. metadata:
  4.   name: monitoring-grafana
  5.   namespace: kube-system
  6. spec:
  7.   replicas: 1
  8.   selector:
  9.     matchLabels:
  10.       task: monitoring
  11.       k8s-app: grafana
  12.   template:
  13.     metadata:
  14.       labels:
  15.         task: monitoring
  16.         k8s-app: grafana
  17.     spec:
  18.       containers:
  19.       - name: grafana
  20.         image: k8s.gcr.io/heapster-grafana-amd64:v5.0.4
  21.         ports:
  22.         - containerPort: 3000
  23.           protocol: TCP
  24.         volumeMounts:
  25.         - mountPath: /etc/ssl/certs
  26.           name: ca-certificates
  27.           readOnly: true
  28.         - mountPath: /var
  29.           name: grafana-storage
  30.         env:
  31.         - name: INFLUXDB_HOST
  32.           value: monitoring-influxdb
  33.         - name: GF_SERVER_HTTP_PORT
  34.           value: "3000"
  35.           # The following env variables are required to make Grafana accessible via
  36.           # the kubernetes api-server proxy. On production clusters, we recommend
  37.           # removing these env variables, setup auth for grafana, and expose the grafana
  38.           # service using a LoadBalancer or a public IP.
  39.         - name: GF_AUTH_BASIC_ENABLED
  40.           value: "false"
  41.         - name: GF_AUTH_ANONYMOUS_ENABLED
  42.           value: "true"
  43.         - name: GF_AUTH_ANONYMOUS_ORG_ROLE
  44.           value: Admin
  45.         - name: GF_SERVER_ROOT_URL
  46.           # If you're only using the API Server proxy, set this value instead:
  47.           # value: /api/v1/namespaces/kube-system/services/monitoring-grafana/proxy
  48.           value: /
  49.       volumes:
  50.       - name: ca-certificates
  51.         hostPath:
  52.           path: /etc/ssl/certs
  53.       - name: grafana-storage
  54.         emptyDir: {}
  55. ---
  56. apiVersion: v1
  57. kind: Service
  58. metadata:
  59.   labels:
  60.     # For use as a Cluster add-on (https://github.com/kubernetes/kubernetes/tree/master/cluster/addons)
  61.     # If you are NOT using this as an addon, you should comment out this line.
  62.     kubernetes.io/cluster-service: 'true'
  63.     kubernetes.io/name: monitoring-grafana
  64.   name: monitoring-grafana
  65.   namespace: kube-system
  66. spec:
  67.   # In a production setup, we recommend accessing Grafana through an external Loadbalancer
  68.   # or through a public IP.
  69.   # type: LoadBalancer
  70.   # You could also use NodePort to expose the service at a randomly-generated port
  71.   # type: NodePort
  72.   ports:
  73.   - port: 80
  74.     targetPort: 3000
  75.   type: nodePort
  76.   selector:
  77.     k8s-app: grafana

备注：生产中，volume不要使用empty，需要用自定义的PVC。
**

metrics-server

在Heapster被淘汰后，metrics-server独挑大梁了，其部署方式如下。

部署

github：https://github.com/kubernetes-sigs/metrics-server
稳定版：https://github.com/kubernetes/kubernetes/tree/master/cluster/addons/metrics-server

下载需要的YAML文件：

  1. for file in auth-delegator.yaml auth-reader.yaml metrics-apiservice.yaml metrics-server-deployment.yaml metrics-server-service.yaml resource-reader.yaml;do wget https://raw.githubusercontent.com/kubernetes/kubernetes/master/cluster/addons/metrics-server/${file}; done

然后由于网络原因，我们将镜像地址进行更改，metrics-server-deployment.yaml

  1.       containers:
  2.       - name: metrics-server
  3.         image: registry.cn-hangzhou.aliyuncs.com/rookieops/metrics-server-amd64:v0.3.6
  4.         imagePulllPolicy: IfNotPresent
  5.         command:
  6.         ......
  7.       - name: metrics-server-nanny
  8.         image: registry.cn-hangzhou.aliyuncs.com/rookieops/addon-resizer:1.8.6
  9.         imagePullPolicy: IfNotPresent
  10.         resources:
  11.                 ......

然后修改resource-reader.yaml

  1. ......
  2. rules:
  3. - apiGroups:
  4.   - ""
  5.   resources:
  6.   - pods
  7.   - nodes
  8.   - namespaces
  9.   - nodes/stats
  10. ......
  1. apiVersion: rbac.authorization.k8s.io/v1
  2. kind: ClusterRoleBinding
  3. metadata:
  4.   name: metrics-server:system:auth-delegator
  5.   labels:
  6.     kubernetes.io/cluster-service: "true"
  7.     addonmanager.kubernetes.io/mode: Reconcile
  8. roleRef:
  9.   apiGroup: rbac.authorization.k8s.io
  10.   kind: ClusterRole
  11.   name: system:auth-delegator
  12. subjects:
  13. - kind: ServiceAccount
  14.   name: metrics-server
  15.   namespace: kube-system
  16. ---
  17. apiVersion: rbac.authorization.k8s.io/v1
  18. kind: RoleBinding
  19. metadata:
  20.   name: metrics-server-auth-reader
  21.   namespace: kube-system
  22.   labels:
  23.     kubernetes.io/cluster-service: "true"
  24.     addonmanager.kubernetes.io/mode: Reconcile
  25. roleRef:
  26.   apiGroup: rbac.authorization.k8s.io
  27.   kind: Role
  28.   name: extension-apiserver-authentication-reader
  29. subjects:
  30. - kind: ServiceAccount
  31.   name: metrics-server
  32.   namespace: kube-system
  33. ---
  34. apiVersion: v1
  35. kind: ServiceAccount
  36. metadata:
  37.   name: metrics-server
  38.   namespace: kube-system
  39.   labels:
  40.     kubernetes.io/cluster-service: "true"
  41.     addonmanager.kubernetes.io/mode: Reconcile
  42. ---
  43. apiVersion: rbac.authorization.k8s.io/v1
  44. kind: ClusterRole
  45. metadata:
  46.   name: system:metrics-server
  47.   labels:
  48.     kubernetes.io/cluster-service: "true"
  49.     addonmanager.kubernetes.io/mode: Reconcile
  50. rules:
  51. - apiGroups:
  52.   - ""
  53.   resources:
  54.   - pods
  55.   - nodes
  56.   - nodes/stats
  57.   - namespaces
  58.   verbs:
  59.   - get
  60.   - list
  61.   - watch
  62. - apiGroups:
  63.   - "extensions"
  64.   resources:
  65.   - deployments
  66.   verbs:
  67.   - get
  68.   - list
  69.   - update
  70.   - watch
  71. ---
  72. apiVersion: rbac.authorization.k8s.io/v1
  73. kind: ClusterRoleBinding
  74. metadata:
  75.   name: system:metrics-server
  76.   labels:
  77.     kubernetes.io/cluster-service: "true"
  78.     addonmanager.kubernetes.io/mode: Reconcile
  79. roleRef:
  80.   apiGroup: rbac.authorization.k8s.io
  81.   kind: ClusterRole
  82.   name: system:metrics-server
  83. subjects:
  84. - kind: ServiceAccount
  85.   name: metrics-server
  86.   namespace: kube-system
  87. ---
  88. apiVersion: v1
  89. kind: ConfigMap
  90. metadata:
  91.   name: metrics-server-config
  92.   namespace: kube-system
  93.   labels:
  94.     kubernetes.io/cluster-service: "true"
  95.     addonmanager.kubernetes.io/mode: EnsureExists
  96. data:
  97.   NannyConfiguration: |-
  98.     apiVersion: nannyconfig/v1alpha1
  99.     kind: NannyConfiguration
  100. ---
  101. apiVersion: apps/v1
  102. kind: Deployment
  103. metadata:
  104.   name: metrics-server
  105.   namespace: kube-system
  106.   labels:
  107.     k8s-app: metrics-server
  108.     kubernetes.io/cluster-service: "true"
  109.     addonmanager.kubernetes.io/mode: Reconcile
  110.     version: v0.3.1
  111. spec:
  112.   selector:
  113.     matchLabels:
  114.       k8s-app: metrics-server
  115.       version: v0.3.1
  116.   template:
  117.     metadata:
  118.       name: metrics-server
  119.       labels:
  120.         k8s-app: metrics-server
  121.         version: v0.3.1
  122.       annotations:
  123.         scheduler.alpha.kubernetes.io/critical-pod: ''
  124.         seccomp.security.alpha.kubernetes.io/pod: 'docker/default'
  125.     spec:
  126.       priorityClassName: system-cluster-critical
  127.       serviceAccountName: metrics-server
  128.       containers:
  129.       - name: metrics-server
  130.         image: registry.cn-hangzhou.aliyuncs.com/rookieops/metrics-server-amd64:v0.3.6 
  131.         imagePullPolicy: IfNotPresent
  132.         command:
  133.         - /metrics-server
  134.         - --metric-resolution=30s
  135.         - --kubelet-preferred-address-types=InternalIP
  136.         - --kubelet-insecure-tls
  137.         ports:
  138.         - containerPort: 443
  139.           name: https
  140.           protocol: TCP
  141.       - name: metrics-server-nanny
  142.         image: registry.cn-hangzhou.aliyuncs.com/rookieops/addon-resizer:1.8.6
  143.         imagePullPolicy: IfNotPresent
  144.         resources:
  145.           limits:
  146.             cpu: 100m
  147.             memory: 300Mi
  148.           requests:
  149.             cpu: 5m
  150.             memory: 50Mi
  151.         env:
  152.           - name: MY_POD_NAME
  153.             valueFrom:
  154.               fieldRef:
  155.                 fieldPath: metadata.name
  156.           - name: MY_POD_NAMESPACE
  157.             valueFrom:
  158.               fieldRef:
  159.                 fieldPath: metadata.namespace
  160.         volumeMounts:
  161.         - name: metrics-server-config-volume
  162.           mountPath: /etc/config
  163.         command:
  164.           - /pod_nanny
  165.           - --config-dir=/etc/config
  166.           - --cpu=300m
  167.           - --extra-cpu=20m
  168.           - --memory=200Mi
  169.           - --extra-memory=10Mi
  170.           - --threshold=5
  171.           - --deployment=metrics-server
  172.           - --container=metrics-server
  173.           - --poll-period=300000
  174.           - --estimator=exponential
  175.           - --minClusterSize=2
  176.       volumes:
  177.         - name: metrics-server-config-volume
  178.           configMap:
  179.             name: metrics-server-config
  180.       tolerations:
  181.         - key: "CriticalAddonsOnly"
  182.           operator: "Exists"
  183.         - key: node-role.kubernetes.io/master
  184.           effect: NoSchedule
  185. ---
  186. apiVersion: v1
  187. kind: Service
  188. metadata:
  189.   name: metrics-server
  190.   namespace: kube-system
  191.   labels:
  192.     addonmanager.kubernetes.io/mode: Reconcile
  193.     kubernetes.io/cluster-service: "true"
  194.     kubernetes.io/name: "Metrics-server"
  195. spec:
  196.   selector:
  197.     k8s-app: metrics-server
  198.   ports:
  199.   - port: 443
  200.     protocol: TCP
  201.     targetPort: https
  202. ---
  203. apiVersion: apiregistration.k8s.io/v1beta1
  204. kind: APIService
  205. metadata:
  206.   name: v1beta1.metrics.k8s.io
  207.   labels:
  208.     kubernetes.io/cluster-service: "true"
  209.     addonmanager.kubernetes.io/mode: Reconcile
  210. spec:
  211.   service:
  212.     name: metrics-server
  213.     namespace: kube-system
  214.   group: metrics.k8s.io
  215.   version: v1beta1
  216.   insecureSkipTLSVerify: true
  217.   groupPriorityMinimum: 100
  218.   versionPriority: 100