在Kubernetes集群内收集ingress日志,我们是直接使用了阿里开源的log-pilot做为采集端,将日志经logstash过滤,在汇聚到elasticsearch进行分析。各组件配置示例如下:

一、log-pilot本质上也是一个日志的采集端,内部封装了filebeatfluentd两种不同的采集工具,我们使用的是filebeat

  • ingress-controller配置示例
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
# nginx-ingress-controller.yaml

apiVersion: extensions/v1beta1
kind: Deployment
metadata:
name: nginx-ingress-controller
namespace: kube-system
labels:
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
spec:
replicas: 1
selector:
matchLabels:
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
template:
metadata:
labels:
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
annotations:
prometheus.io/port: "10254"
prometheus.io/scrape: "true"
enable-vts-status: "true"
spec:
hostNetwork: true
dnsPolicy: ClusterFirstWithHostNet
nodeSelector:
node-role.kubernetes.io/system: ''
serviceAccountName: nginx-ingress-serviceaccount
containers:
- name: nginx-ingress-controller
image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.24.1
imagePullPolicy: Always
args:
- /nginx-ingress-controller
- --default-backend-service=$(POD_NAMESPACE)/default-http-backend
- --configmap=$(POD_NAMESPACE)/nginx-configuration
- --tcp-services-configmap=$(POD_NAMESPACE)/tcp-services
- --udp-services-configmap=$(POD_NAMESPACE)/udp-services
- --publish-service=$(POD_NAMESPACE)/ingress-nginx
- --annotations-prefix=nginx.ingress.kubernetes.io
securityContext:
capabilities:
drop:
- ALL
add:
- NET_BIND_SERVICE
# www-data -> 33
runAsUser: 33
env:
- name: POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: aliyun_logs_ingress-nginx ## 注意此处的环境变量对接log-pilot(原理请见参考一)
value: "stdout"
ports:
- name: http
containerPort: 80
- name: https
containerPort: 443
resources:
limits:
cpu: 200m
memory: 500Mi
requests:
cpu: 200m
memory: 500Mi
livenessProbe:
failureThreshold: 3
httpGet:
path: /healthz
port: 10254
scheme: HTTP
initialDelaySeconds: 30
periodSeconds: 10
successThreshold: 1
timeoutSeconds: 5
readinessProbe:
failureThreshold: 3
httpGet:
path: /healthz
port: 10254
scheme: HTTP
periodSeconds: 10
successThreshold: 1
timeoutSeconds: 1
  • log-pilot配置示例
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
# log-pilot-system.yaml

apiVersion: extensions/v1beta1
kind: DaemonSet
metadata:
name: log-pilot-system
labels:
app: log-pilot-system
namespace: kube-system
spec:
updateStrategy:
type: RollingUpdate
template:
metadata:
labels:
app: log-pilot-system
annotations:
scheduler.alpha.kubernetes.io/critical-pod: ''
spec:
hostNetwork: true
containers:
- name: log-pilot
# 版本请参考https://github.com/AliyunContainerService/log-pilot/releases
image: registry.cn-hangzhou.aliyuncs.com/acs/log-pilot:0.9.7-filebeat
resources:
limits:
memory: 500Mi
requests:
cpu: 200m
memory: 200Mi
env:
- name: "NODE_NAME"
valueFrom:
fieldRef:
fieldPath: spec.nodeName
# - name: "LOGGING_OUTPUT" # 直接将日志打到es里面
# value: "elasticsearch"
# - name: "ELASTICSEARCH_HOSTS"
# value: "192.168.7.75"
# - name: "LOGSTASH_PORT"
# value: "9300"
- name: "LOGGING_OUTPUT" # 将日志打到logstash里面
value: "logstash"
- name: "LOGSTASH_HOST"
value: "192.168.7.75"
- name: "LOGSTASH_PORT"
value: "5044"
volumeMounts:
- name: sock
mountPath: /var/run/docker.sock
- name: root
mountPath: /host
readOnly: true
- name: varlib
mountPath: /var/lib/filebeat
- name: varlog
mountPath: /var/log/filebeat
- name: localtime
mountPath: /etc/localtime
readOnly: true
livenessProbe:
failureThreshold: 3
exec:
command:
- /pilot/healthz
initialDelaySeconds: 10
periodSeconds: 10
successThreshold: 1
timeoutSeconds: 2
securityContext:
capabilities:
add:
- SYS_ADMIN
terminationGracePeriodSeconds: 30
# 是否允许部署到Master节点上
tolerations:
- key: node-role.kubernetes.io/master
effect: NoSchedule
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: node-role.kubernetes.io/worker
operator: NotIn
values:
- ''
volumes:
- name: sock
hostPath:
path: /var/run/docker.sock
- name: root
hostPath:
path: /
- name: varlib
hostPath:
path: /var/lib/filebeat
type: DirectoryOrCreate
- name: varlog
hostPath:
path: /var/log/filebeat
type: DirectoryOrCreate
- name: localtime
hostPath:
path: /etc/localtime

二、ingress其实在kubernetes集群内部就是作为nginx来使用的,故logstash的过滤规则可以参考nginx来写。示例如下:

  • logstash配置示例:
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
# filebeat.yml

input {
beats {
port => 5044
}
}

filter {
grok {
patterns_dir => "/opt/soft/logstash-6.2.4/config/patterns"
match => {
"message" => "%{INGRESSACCESS_GET}"
}

match => {
"message" => "%{INGRESSACCESS_POST}"
}
remove_field => ["source", "IPV6", "IPV4", "QUOTEDSTRING", "upstream_response", "URIPARAM"]
}

if [request] == "/status.jsp" {
drop{}
}
urldecode {
all_fields => true # 显示中文
}
}

output {
elasticsearch { #输出到elasticsearch中
hosts => ["192.168.7.75:9200"] #指定elasticsearch主机
document_type => "log" #设定Elasticsearch输出时的document的type字段,也可以用来给日志进行分类
template_overwrite => true #如果设置为true,模板名字一样的时候,新的模板会覆盖旧的模板
}
}
  • patterns配置示例(此规则可覆盖大多数请求日志,post比较特殊,可根据自己需求修改):
1
2
3
# patterns-ingress
INGRESSACCESS_GET %{IPORHOST:client_ip} - \[%{IPORHOST:x_forwarded_for}\] - %{DATA:client_identity} \[%{HTTPDATE:timestamp}\] \"(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|-)\" %{NUMBER:http_code} (?:%{NUMBER:bytes_sent;long}|-) (?:"(?:%{URI:referrer}|-)") %{QS:user_agent} %{NUMBER:request_length;long} %{NUMBER:response_time;double} %{DATA:upstream_proxy} %{NUMBER:upstream_bytes_sent;long} %{NUMBER:upstream_response_time} %{NUMBER:upstream_response} %{BASE16FLOAT:request_id}
INGRESSACCESS_POST %{IPORHOST:client_ip} - \[%{IPORHOST:x_forwarded_for}\] - %{DATA:client_identity} \[%{HTTPDATE:timestamp}\] \"(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|-)\" %{NUMBER:http_code} (?:%{NUMBER:bytes_sent;long}|-) "-" %{QS:user_agent} %{NUMBER:request_length;long} %{NUMBER:response_time;double} %{DATA:upstream_proxy} %{DATA:other}

参考: