在Kubernetes集群内收集ingress日志，我们是直接使用了阿里开源的log-pilot做为采集端，将日志经logstash过滤，在汇聚到elasticsearch进行分析。各组件配置示例如下：

一、log-pilot本质上也是一个日志的采集端，内部封装了filebeat和fluentd两种不同的采集工具，我们使用的是filebeat。

  • ingress-controller配置示例
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
# nginx-ingress-controller.yaml

apiVersion: extensions/v1beta1
kind: Deployment
metadata:
  name: nginx-ingress-controller 
  namespace: kube-system
  labels:
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx
spec:
  replicas: 1
  selector:
    matchLabels:
      app.kubernetes.io/name: ingress-nginx
      app.kubernetes.io/part-of: ingress-nginx
  template:
    metadata:
      labels:
        app.kubernetes.io/name: ingress-nginx
        app.kubernetes.io/part-of: ingress-nginx
      annotations:
        prometheus.io/port: "10254"
        prometheus.io/scrape: "true"
        enable-vts-status: "true"
    spec:
      hostNetwork: true
      dnsPolicy: ClusterFirstWithHostNet
      nodeSelector:
        node-role.kubernetes.io/system: ''
      serviceAccountName: nginx-ingress-serviceaccount 
      containers:
        - name: nginx-ingress-controller 
          image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.24.1
          imagePullPolicy: Always
          args:
            - /nginx-ingress-controller
            - --default-backend-service=$(POD_NAMESPACE)/default-http-backend 
            - --configmap=$(POD_NAMESPACE)/nginx-configuration 
            - --tcp-services-configmap=$(POD_NAMESPACE)/tcp-services 
            - --udp-services-configmap=$(POD_NAMESPACE)/udp-services 
            - --publish-service=$(POD_NAMESPACE)/ingress-nginx 
            - --annotations-prefix=nginx.ingress.kubernetes.io
          securityContext:
            capabilities:
              drop:
                - ALL
              add:
                - NET_BIND_SERVICE
            # www-data -> 33
            runAsUser: 33
          env:
            - name: POD_NAME
              valueFrom:
                fieldRef:
                  fieldPath: metadata.name
            - name: POD_NAMESPACE
              valueFrom:
                fieldRef:
                  fieldPath: metadata.namespace
            - name: aliyun_logs_ingress-nginx    ## 注意此处的环境变量对接log-pilot(原理请见参考一)
              value: "stdout"
          ports:
            - name: http
              containerPort: 80
            - name: https
              containerPort: 443
          resources:
            limits:
              cpu: 200m
              memory: 500Mi
            requests:
              cpu: 200m
              memory: 500Mi
          livenessProbe:
            failureThreshold: 3
            httpGet:
              path: /healthz
              port: 10254
              scheme: HTTP
            initialDelaySeconds: 30
            periodSeconds: 10
            successThreshold: 1
            timeoutSeconds: 5
          readinessProbe:
            failureThreshold: 3
            httpGet:
              path: /healthz
              port: 10254
              scheme: HTTP
            periodSeconds: 10
            successThreshold: 1
            timeoutSeconds: 1

  • log-pilot配置示例
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
# log-pilot-system.yaml

apiVersion: extensions/v1beta1
kind: DaemonSet
metadata:
  name: log-pilot-system
  labels:
    app: log-pilot-system
  namespace: kube-system
spec:
  updateStrategy:
    type: RollingUpdate
  template:
    metadata:
      labels:
        app: log-pilot-system
      annotations:
        scheduler.alpha.kubernetes.io/critical-pod: ''
    spec:
      hostNetwork: true
      containers:
      - name: log-pilot
        # 版本请参考https://github.com/AliyunContainerService/log-pilot/releases
        image: registry.cn-hangzhou.aliyuncs.com/acs/log-pilot:0.9.7-filebeat
        resources:
          limits:
            memory: 500Mi
          requests:
            cpu: 200m
            memory: 200Mi
        env:
          - name: "NODE_NAME"
            valueFrom:
              fieldRef:
                fieldPath: spec.nodeName
          # - name: "LOGGING_OUTPUT"       # 直接将日志打到es里面
          #   value: "elasticsearch"
          # - name: "ELASTICSEARCH_HOSTS"
          #   value: "192.168.7.75"
          # - name: "LOGSTASH_PORT"
          #   value: "9300"
          - name: "LOGGING_OUTPUT"         # 将日志打到logstash里面
            value: "logstash"
          - name: "LOGSTASH_HOST"
            value: "192.168.7.75"
          - name: "LOGSTASH_PORT"
            value: "5044"
        volumeMounts:
        - name: sock
          mountPath: /var/run/docker.sock
        - name: root
          mountPath: /host
          readOnly: true
        - name: varlib
          mountPath: /var/lib/filebeat
        - name: varlog
          mountPath: /var/log/filebeat
        - name: localtime
          mountPath: /etc/localtime
          readOnly: true
        livenessProbe:
          failureThreshold: 3
          exec:
            command:
            - /pilot/healthz
          initialDelaySeconds: 10
          periodSeconds: 10
          successThreshold: 1
          timeoutSeconds: 2
        securityContext:
          capabilities:
            add:
            - SYS_ADMIN
      terminationGracePeriodSeconds: 30
      # 是否允许部署到Master节点上
      tolerations:
      - key: node-role.kubernetes.io/master
        effect: NoSchedule
      affinity:
        nodeAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            nodeSelectorTerms:
            - matchExpressions:
              - key: node-role.kubernetes.io/worker
                operator: NotIn
                values:
                - ''
      volumes:
      - name: sock
        hostPath:
          path: /var/run/docker.sock
      - name: root
        hostPath:
          path: /
      - name: varlib
        hostPath:
          path: /var/lib/filebeat
          type: DirectoryOrCreate
      - name: varlog
        hostPath:
          path: /var/log/filebeat
          type: DirectoryOrCreate
      - name: localtime
        hostPath:
          path: /etc/localtime

二、ingress其实在kubernetes集群内部就是作为nginx来使用的，故logstash的过滤规则可以参考nginx来写。示例如下：

  • logstash配置示例：
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
# filebeat.yml

input {
  beats {
    port => 5044
  }
}

filter {
  grok {
    patterns_dir => "/opt/soft/logstash-6.2.4/config/patterns"
    match => {
      "message" => "%{INGRESSACCESS_GET}"
    }

    match => {
      "message" => "%{INGRESSACCESS_POST}"
    }
    remove_field => ["source", "IPV6", "IPV4", "QUOTEDSTRING", "upstream_response", "URIPARAM"]
  }

  if [request] == "/status.jsp" {
     drop{}
  }
  urldecode {
    all_fields => true   # 显示中文
  }
}

output {
  elasticsearch {                        #输出到elasticsearch中
    hosts => ["192.168.7.75:9200"]       #指定elasticsearch主机
    document_type => "log"               #设定Elasticsearch输出时的document的type字段，也可以用来给日志进行分类
    template_overwrite => true           #如果设置为true，模板名字一样的时候，新的模板会覆盖旧的模板
  }
}
  • patterns配置示例(此规则可覆盖大多数请求日志，post比较特殊，可根据自己需求修改)：
1
2
3
# patterns-ingress
INGRESSACCESS_GET %{IPORHOST:client_ip} - \[%{IPORHOST:x_forwarded_for}\] - %{DATA:client_identity} \[%{HTTPDATE:timestamp}\] \"(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|-)\" %{NUMBER:http_code} (?:%{NUMBER:bytes_sent;long}|-) (?:"(?:%{URI:referrer}|-)") %{QS:user_agent} %{NUMBER:request_length;long} %{NUMBER:response_time;double} %{DATA:upstream_proxy} %{NUMBER:upstream_bytes_sent;long} %{NUMBER:upstream_response_time} %{NUMBER:upstream_response} %{BASE16FLOAT:request_id}
INGRESSACCESS_POST %{IPORHOST:client_ip} - \[%{IPORHOST:x_forwarded_for}\] - %{DATA:client_identity} \[%{HTTPDATE:timestamp}\] \"(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|-)\" %{NUMBER:http_code} (?:%{NUMBER:bytes_sent;long}|-) "-" %{QS:user_agent} %{NUMBER:request_length;long} %{NUMBER:response_time;double} %{DATA:upstream_proxy} %{DATA:other}

参考：