Nginx Logs
yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: fluent-bit
namespace: monitor
selfLink: /api/v1/namespaces/monitor/configmaps/fluent-bit
uid: 8e6515f6-bd93-4a01-82f5-e272dc423fd1
resourceVersion: '568097977'
creationTimestamp: '2025-01-14T09:02:32Z'
labels:
app.kubernetes.io/instance: foobar-fluent-bit
app.kubernetes.io/managed-by: Helm
app.kubernetes.io/name: foobar-fluent-bit
app.kubernetes.io/version: 3.2.1
helm.sh/chart: foobar-fluent-bit-0.47.17
k8slens-edit-resource-version: v1
data:
custom_parsers.conf: >
[PARSER]
Name docker_no_time
Format json
Time_Keep Off
Time_Key time
Time_Format %Y-%m-%dT%H:%M:%S.%L
[PARSER]
Name container_log
Format regex
Regex ^(?<time>[^ ]+) (?<stream>stdout|stderr) (?<logtag>[^ ]*) (?<log>.*)$
Time_Key time
Time_Format %Y-%m-%dT%H:%M:%S.%L%z
# Example of nginx log
# 192.168.192.17 - - [07/Jan/2025:03:15:55 +0000] "GET / HTTP/1.1" 200 615
# "-" "kube-probe/1.26" "-" rt="0.000" uct="-" uht="-" urt="-"
#
# Define as follows
# log_format main '$remote_addr - $remote_user [$time_local] '
# '"$request" $status $body_bytes_sent "$http_referer" '
# '"$http_user_agent" "$http_x_forwarded_for" '
# 'rt="$request_time" uct="$upstream_connect_time" '
# 'uht="$upstream_header_time" '
# 'urt="$upstream_response_time"';
[PARSER]
Name nginx_access
Format regex
Regex ^(?<log_remote_addr>[^ ]*) - (?<log_remote_user>[^ ]*) \[(?<log_time_local>[^\]]*)\] "(?<log_request>[^"]*)" (?<log_status>[^ ]*) (?<log_body_bytes_sent>[^ ]*) "(?<log_http_referer>[^"]*)" "(?<log_http_user_agent>[^"]*)" "(?<log_http_x_forwarded_for>[^"]*)" rt="(?<log_request_time>[^"]*)" ua="(?<log_upstream_addr>[^"]*)" us="(?<log_upstream_status>[^"]*)" uct="(?<log_upstream_connect_time>[^"]*)" uht="(?<log_upstream_header_time>[^"]*)" urt="(?<log_upstream_response_time>[^"]*)" url="(?<log_upstream_response_length>[^"]*)"$
Time_Key log_time_local
Time_Format %d/%b/%Y:%H:%M:%S %z
[PARSER]
Name nginx_error
Format regex
#Regex ^(?<log_time>[0-9]{4}/[0-9]{2}/[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}) \[(?<log_level>[^\]]*)\] (?<log_pid>[0-9]*)#(?<log_tid>[0-9]*): \*(?<log_connection>[0-9]*) (?<log_error_msg>.*?), client: (?<log_remote_addr>[^,]*), server: (?<log_server>[^,]*), request: "(?<log_request>[^"]*)", host: "(?<log_host>[^"]*)(?:, referrer: "(?<log_http_referer>[^"]*)")?
Regex ^(?<log_time>[0-9]{4}/[0-9]{2}/[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}) \[(?<log_level>[^\]]*)\] (?<log_pid>[0-9]*)#(?<log_tid>[0-9]*): \*(?<log_connection>[0-9]*) (?<log_error_msg>.*?)(?:, client: (?<log_remote_addr>[^,]*))?(?:, server: (?<log_server>[^,]*))?(?:, request: "(?<log_request>[^"]*)")?(?:, upstream: "(?<log_upstream>[^"]*)")?(?:, host: "(?<log_host>[^"]*)")?(?:, referrer: "(?<log_http_referer>[^"]*)")?$
Time_Key log_time
Time_Format %Y/%m/%d %H:%M:%S
[PARSER]
Name request_parser
Format regex
Regex ^(?<log_request_method>[A-Z]+) (?<log_request_path>[^ ]*) (?<log_request_protocol>[^ ]*)$
[MULTILINE_PARSER]
name java-exception
type regex
flush_timeout 2000
rule "start_state" "\s*java\..*Exception:.*$" "cont"
rule "cont" "/\s+at.*/" "cont"
fluent-bit.conf: |
[SERVICE]
Daemon Off
Flush 1
Log_Level info
Parsers_File /fluent-bit/etc/parsers.conf
Parsers_File /fluent-bit/etc/conf/custom_parsers.conf
HTTP_Server On
HTTP_Listen 0.0.0.0
HTTP_Port 2020
Health_Check On
Mem_Buf_Limit 100MB
[INPUT]
Name tail
Path /var/log/containers/*.log
multiline.parser docker, cri, java-exception, go
Tag kube.*
Mem_Buf_Limit 5MB
Skip_Long_Lines On
#[INPUT]
# Name dummy
# Dummy {"message": "foobar"}
# Interval_Sec 5
#[INPUT]
# Name systemd
# Tag host.*
# Systemd_Filter _SYSTEMD_UNIT=kubelet.service
# Read_From_Tail On
[FILTER]
Name kubernetes
Match kube.*
Merge_Log On
Keep_Log Off
Labels Off
Annotations Off
K8S-Logging.Parser On
K8S-Logging.Exclude Off
[FILTER]
Name modify
Match kube.*
Rename message log
Rename msg log
[FILTER]
Name grep
Match kube.*
Exclude $log ^(GET /healthz|/ready|/health|/live|/ping)$
[FILTER]
Name nest
Match kube.*
Operation lift
Nested_under kubernetes
Add_prefix k8s_
[FILTER]
Name rewrite_tag
Match kube.*
Rule $k8s_pod_name foobar-reverproxy nginx false
Emitter_Name nginx
[FILTER]
Name parser
Match nginx
Key_Name log
Parser container_log
Reserve_Data True
[FILTER]
Name rewrite_tag
Match nginx
Rule $log ^\d{4}/\d{2}/\d{2} nginx.error false
# 如果不匹配上面的规则，重命名为 nginx.access
Rule $log .* nginx.access false
Emitter_Name nginx_log
[FILTER]
Name modify
Match nginx.access
Add log_type access
[FILTER]
Name modify
Match nginx.error
Add log_type error
[FILTER]
Name parser
Match nginx.access
Key_Name log
Parser nginx_access
Reserve_Data True
[FILTER]
Name parser
Match nginx.error
Key_Name log
Parser nginx_error
Reserve_Data True
[FILTER]
Name parser
Match nginx.*
Key_Name log_request
Parser request_parser
Preserve_Key True
Reserve_Data True
[FILTER]
Name grep
Match kube.*
Regex $k8s_namespace_name foobar|istio-system
#[FILTER]
# Name grep
# Match kube.*
# Regex $k8s_container_name foobar
#[FILTER]
# Name grep
# Match kube.*
# Exclude $k8s_container_name not-found
[OUTPUT]
Name kafka
Match kube.*
Brokers foobar-kafka.monitor.svc.cluster.local:9092
Topics otel_logs
Timestamp_Key timestamp
Format json
[OUTPUT]
Name kafka
Match nginx.*
Brokers foobar-kafka.monitor.svc.cluster.local:9092
Topics otel_logs_nginx
Timestamp_Key timestamp
Format json
#[OUTPUT]
# Name stdout
# Match kube.*
# Json_date_key timestamp
# Json_date_format iso8601
# Format json
#[OUTPUT]
# Name stdout
# Match nginx.*
# Json_date_key timestamp
# Json_date_format iso8601
# Format json
#[OUTPUT]
# Name kafka
# Match kube.*
# Brokers kafka.namespace.svc.cluster.local:9092
# Topics otel_logs
# Timestamp_Key timestamp
# Format json