一、需求

日志写入 kafka，再经由 lostash 写到 elasticsearch，最终由 kibana 展示。


k8s 练习 - 日志收集_k8s 日志收集


二、daemonset 方式收集

2.1 创建配置文件

收集每个节点上的系统日志和 pod 日志。

---
apiVersion: v1
data:
  logstash.conf: |
    input {
      file {
        #path => "/var/lib/docker/containers/*/*-json.log" #docker
        path => "/var/log/pods/*/*/*.log"
        start_position => "beginning"
        type => "jsonfile-daemonset-applog"
      }

      file {
        path => "/var/log/*.log"
        start_position => "beginning"
        type => "jsonfile-daemonset-syslog"
      }
    }

    output {
      if [type] == "jsonfile-daemonset-applog" {
        kafka {
          bootstrap_servers => "${KAFKA_SERVER}"
          topic_id => "${TOPIC_ID}"
          batch_size => 16384  #logstash每次向ES传输的数据量大小,单位为字节
          codec => "${CODEC}" 
       } }

      if [type] == "jsonfile-daemonset-syslog" {
        kafka {
          bootstrap_servers => "${KAFKA_SERVER}"
          topic_id => "${TOPIC_ID}"
          batch_size => 16384
          codec => "${CODEC}" #系统日志不是json格式
      }}
    }
  logstash.yml: |
    http.host: "0.0.0.0"
    #xpack.monitoring.elasticsearch.hosts: [ "http://elasticsearch:9200" ]
kind: ConfigMap
metadata:
  name: conf-logstash
  namespace: kube-system

2.2 创建 daemonset

---
apiVersion: apps/v1
kind: DaemonSet
metadata:
  labels:
    k8s-app: logstash-logging
    k8s.kuboard.cn/name: logstash-elasticsearch
  name: logstash-elasticsearch
  namespace: kube-system
spec:
  revisionHistoryLimit: 10
  selector:
    matchLabels:
      name: logstash-elasticsearch
  template:
    metadata:
      labels:
        name: logstash-elasticsearch
    spec:
      containers:
        - env:
            - name: KAFKA_SERVER
              value: '192.168.3.74:9092'
            - name: TOPIC_ID
              value: jsonfile-log-topic
            - name: CODEC
              value: json
          image: 'logstash:7.12.1'
          imagePullPolicy: IfNotPresent
          name: logstash-elasticsearch
          resources: {}
          terminationMessagePath: /dev/termination-log
          terminationMessagePolicy: File
          volumeMounts:
            - mountPath: /var/log
              name: varlog
            - mountPath: /var/log/pods
              name: varlibdockercontainers
            - mountPath: /usr/share/logstash/pipeline/logstash.conf
              name: logstash-config
              subPath: logstash.conf
            - mountPath: /usr/share/logstash/config/logstash.yml
              name: logstash-yaml
              subPath: logstash.yml
      dnsPolicy: ClusterFirst
      restartPolicy: Always
      schedulerName: default-scheduler
      securityContext:
        fsGroup: 0
        runAsGroup: 0
        runAsUser: 0
        seLinuxOptions: {}
      terminationGracePeriodSeconds: 30
      tolerations:
        - effect: NoSchedule
          key: node-role.kubernetes.io/master
          operator: Exists
      volumes:
        - hostPath:
            path: /var/log
            type: ''
          name: varlog
        - hostPath:
            path: /var/log/pods
            type: ''
          name: varlibdockercontainers
        - configMap:
            defaultMode: 420
            items:
              - key: logstash.conf
                path: logstash.conf
            name: conf-logstash
          name: logstash-config
        - configMap:
            defaultMode: 420
            items:
              - key: logstash.yml
                path: logstash.yml
            name: conf-logstash
          name: logstash-yaml
  updateStrategy:
    rollingUpdate:
      maxSurge: 0
      maxUnavailable: 1
    type: RollingUpdate

三、sidecar 方式收集

收集 tomcat 日志为例。

3.1 创建配置文件

---
apiVersion: v1
data:
  logstash.conf: |
    input {
      file {
        path => "/usr/local/tomcat/logs/catalina.*.log"
        start_position => "beginning"
        type => "tomcat-sidecar-catalina-log"
      }
      file {
        path => "/usr/local/tomcat/logs/localhost_access_log.*.txt"
        start_position => "beginning"
        type => "tomcat-sidecar-access-log"
      }
    }

    output {
      if [type] == "tomcat-sidecar-catalina-log" {
        kafka {
          bootstrap_servers => "${KAFKA_SERVER}"
          topic_id => "${TOPIC_ID}"
          batch_size => 16384  #logstash每次向ES传输的数据量大小,单位为字节
          codec => "${CODEC}" 
        }
      }

      if [type] == "tomcat-sidecar-access-log" {
        kafka {
          bootstrap_servers => "${KAFKA_SERVER}"
          topic_id => "${TOPIC_ID}"
          batch_size => 16384
          codec => "${CODEC}"
        }
      }
    }
  logstash.yml: |
    http.host: "0.0.0.0"
kind: ConfigMap
metadata:
  name: conf-tomcat-log
  namespace: default

3.2 创建 deployment & service

---
apiVersion: apps/v1
kind: Deployment
metadata:
  annotations: {}
  labels:
    app: tomcat
    k8s.kuboard.cn/name: tomcat
  name: tomcat
  namespace: default
spec:
  replicas: 3
  revisionHistoryLimit: 10
  selector:
    matchLabels:
      app: tomcat
  strategy:
    rollingUpdate:
      maxSurge: 25%
      maxUnavailable: 25%
    type: RollingUpdate
  template:
    metadata:
      labels:
        app: tomcat
    spec:
      containers:
        - env:
            - name: KAFKA_SERVER
              value: '192.168.3.74:9092'
            - name: TOPIC_ID
              value: tomcat-log
            - name: CODEC
              value: json
          image: 'myharbor.belkuy.top/base/logstash:7.12.1'
          imagePullPolicy: IfNotPresent
          name: sidecar-container
          resources: {}
          terminationMessagePath: /dev/termination-log
          terminationMessagePolicy: File
          volumeMounts:
            - mountPath: /usr/local/tomcat/logs
              name: tomcat-logs
            - mountPath: /usr/share/logstash/pipeline/logstash.conf
              name: logstash-config
              subPath: logstash.conf
            - mountPath: /usr/share/logstash/config/logstash.yml
              name: logstash-yaml
              subPath: logstash.yml
        - image: 'myharbor.belkuy.top/base/tomcat:9'
          imagePullPolicy: IfNotPresent
          name: tomcat-container
          ports:
            - containerPort: 8080
              name: http
              protocol: TCP
          resources: {}
          terminationMessagePath: /dev/termination-log
          terminationMessagePolicy: File
          volumeMounts:
            - mountPath: /usr/local/tomcat/logs
              name: tomcat-logs
      dnsPolicy: ClusterFirst
      restartPolicy: Always
      schedulerName: default-scheduler
      securityContext:
        fsGroup: 0
        runAsGroup: 0
        runAsUser: 0
        seLinuxOptions: {}
      terminationGracePeriodSeconds: 30
      volumes:
        - emptyDir: {}
          name: tomcat-logs
        - configMap:
            defaultMode: 420
            items:
              - key: logstash.conf
                path: logstash.conf
            name: conf-tomcat-log
          name: logstash-config
        - configMap:
            defaultMode: 420
            items:
              - key: logstash.yml
                path: logstash.yml
            name: conf-tomcat-log
          name: logstash-yaml

---
apiVersion: v1
kind: Service
metadata:
  labels:
    app: tomcat
  name: tomcat-svc
  namespace: default
spec:
  ports:
    - name: http
      nodePort: 30880
      port: 8080
      protocol: TCP
      targetPort: 8080
  selector:
    app: tomcat
  sessionAffinity: None
  type: NodePort

四、容器内置日志收集进程方式

以 tomcat 为例。

4.1 构建镜像

FROM tomcat:9

ADD run.sh /usr/local/tomcat/bin/run.sh

RUN curl https://artifacts.elastic.co/packages/7.x/apt/pool/main/f/filebeat/filebeat-7.12.1-amd64.deb -o filebeat-7.12.1-amd64.deb \
    && dpkg -i filebeat-7.12.1-amd64.deb \
    && mkdir -p /usr/local/tomcat/webapps/app \
    && echo 'test' > /usr/local/tomcat/webapps/app/index.html \
    && chmod +x /usr/local/tomcat/bin/run.sh

EXPOSE 8080 8443

CMD ["/usr/local/tomcat/bin/run.sh"]
#!/bin/bash
# run.sh
/usr/share/filebeat/bin/filebeat -e -c /etc/filebeat/filebeat.yml \
-path.home /usr/share/filebeat \
-path.config /etc/filebeat \
-path.data /var/lib/filebeat \
-path.logs /var/log/filebeat &
su - tomcat -c "/usr/local/tomcat/bin/catalina.sh start"
tail -f /var/log/lastlog

4.2 创建 filebeat 配置文件

---
apiVersion: v1
data:
  filebeat.yml: |
    filebeat.inputs:
    - type: log
      enabled: true
      paths:
        - /usr/local/tomcat/logs/catalina.*.log
      fields:
        type: tomcat-filebeat-catalina-log
    - type: log
      enabled: true
      paths:
        - /usr/local/tomcat/logs/localhost_access_log.*.txt 
      fields:
        type: tomcat-filebeat-access-log
    filebeat.config.modules:
      path: ${path.config}/modules.d/*.yml
      reload.enabled: false
    setup.template.settings:
      index.number_of_shards: 1
    # glibc >= 2.35 added a new rseq syscall that is not in our default list of allowed syscalls
    seccomp:
      default_action: allow 
      syscalls:
      - action: allow
        names:
        - rseq

    output.kafka:
      hosts: ["192.168.3.74:9092"]
      required_acks: 1
      topic: "tomcat-log"
      compression: gzip
      max_message_bytes: 1000000
kind: ConfigMap
metadata:
  name: conf-filebeat
  namespace: default

4.3 创建 deployment & service

---
apiVersion: apps/v1
kind: Deployment
metadata:
  labels:
    app: tomcat-app
  name: tomcat-app
  namespace: default
spec:
  replicas: 3
  revisionHistoryLimit: 10
  selector:
    matchLabels:
      app: tomcat-app
  strategy:
    rollingUpdate:
      maxSurge: 25%
      maxUnavailable: 25%
    type: RollingUpdate
  template:
    metadata:
      labels:
        app: tomcat-app
    spec:
      containers:
        - image: 'tomcat:9-app'
          imagePullPolicy: Always
          name: tomcat-app-filebeat
          ports:
            - containerPort: 8080
              name: http
              protocol: TCP
          resources: {}
          terminationMessagePath: /dev/termination-log
          terminationMessagePolicy: File
          volumeMounts:
            - mountPath: /etc/filebeat/filebeat.yml
              name: filebeat-yaml
              subPath: filebeat.yml
      dnsPolicy: ClusterFirst
      restartPolicy: Always
      schedulerName: default-scheduler
      securityContext: {}
      terminationGracePeriodSeconds: 30
      volumes:
        - configMap:
            defaultMode: 420
            items:
              - key: filebeat.yml
                path: filebeat.yml
            name: conf-filebeat
          name: filebeat-yaml

---
apiVersion: v1
kind: Service
metadata:
  annotations: {}
  labels:
    app: tomcat-app
  name: tomcat-app
  namespace: default
spec:
  ports:
    - name: http
      nodePort: 30088
      port: 8080
      protocol: TCP
      targetPort: 8080
  selector:
    app: tomcat-app
  sessionAffinity: None
  type: NodePort