32c2bfadfd
Metadata Agent Improvements Bump metadata agent version to 0.2-0.0.21-1. Expand the metadata agent's access to all API groups. Remove metadata agent config maps in favor of command line flags. Update the metadata agent's liveness probe to a new /healthz handler. Logging Agent Improvements Bump logging agent version to 0.2-1.5.33-1-k8s-1. Appropriately set log severity for k8s_container. Fix detect exceptions plugin to analyze message field instead of log field. Fix detect exceptions plugin to analyze streams based on local resource id. Disable the metadata agent for monitored resource construction in logging. Disable timestamp adjustment in logs to optimize performance. Reduce logging agent buffer chunk limit to 512k to optimize performance.
472 lines
18 KiB
YAML
472 lines
18 KiB
YAML
# This ConfigMap is used to ingest logs against new resources like
|
|
# "k8s_container" and "k8s_node" when $LOGGING_STACKDRIVER_RESOURCE_TYPES is set
|
|
# to "new".
|
|
# When $LOGGING_STACKDRIVER_RESOURCE_TYPES is set to "old", the ConfigMap in
|
|
# fluentd-gcp-configmap-old.yaml will be used for ingesting logs against old
|
|
# resources like "gke_container" and "gce_instance".
|
|
kind: ConfigMap
|
|
apiVersion: v1
|
|
data:
|
|
containers.input.conf: |-
|
|
# This configuration file for Fluentd is used
|
|
# to watch changes to Docker log files that live in the
|
|
# directory /var/lib/docker/containers/ and are symbolically
|
|
# linked to from the /var/log/containers directory using names that capture the
|
|
# pod name and container name. These logs are then submitted to
|
|
# Google Cloud Logging which assumes the installation of the cloud-logging plug-in.
|
|
#
|
|
# Example
|
|
# =======
|
|
# A line in the Docker log file might look like this JSON:
|
|
#
|
|
# {"log":"2014/09/25 21:15:03 Got request with path wombat\\n",
|
|
# "stream":"stderr",
|
|
# "time":"2014-09-25T21:15:03.499185026Z"}
|
|
#
|
|
# The original tag is derived from the log file's location.
|
|
# For example a Docker container's logs might be in the directory:
|
|
# /var/lib/docker/containers/997599971ee6366d4a5920d25b79286ad45ff37a74494f262e3bc98d909d0a7b
|
|
# and in the file:
|
|
# 997599971ee6366d4a5920d25b79286ad45ff37a74494f262e3bc98d909d0a7b-json.log
|
|
# where 997599971ee6... is the Docker ID of the running container.
|
|
# The Kubernetes kubelet makes a symbolic link to this file on the host
|
|
# machine in the /var/log/containers directory which includes the pod name,
|
|
# the namespace name and the Kubernetes container name:
|
|
# synthetic-logger-0.25lps-pod_default_synth-lgr-997599971ee6366d4a5920d25b79286ad45ff37a74494f262e3bc98d909d0a7b.log
|
|
# ->
|
|
# /var/lib/docker/containers/997599971ee6366d4a5920d25b79286ad45ff37a74494f262e3bc98d909d0a7b/997599971ee6366d4a5920d25b79286ad45ff37a74494f262e3bc98d909d0a7b-json.log
|
|
# The /var/log directory on the host is mapped to the /var/log directory in the container
|
|
# running this instance of Fluentd and we end up collecting the file:
|
|
# /var/log/containers/synthetic-logger-0.25lps-pod_default_synth-lgr-997599971ee6366d4a5920d25b79286ad45ff37a74494f262e3bc98d909d0a7b.log
|
|
# This results in the tag:
|
|
# var.log.containers.synthetic-logger-0.25lps-pod_default_synth-lgr-997599971ee6366d4a5920d25b79286ad45ff37a74494f262e3bc98d909d0a7b.log
|
|
# where 'synthetic-logger-0.25lps-pod' is the pod name, 'default' is the
|
|
# namespace name, 'synth-lgr' is the container name and '997599971ee6..' is
|
|
# the container ID.
|
|
# The record reformer is used to extract pod_name, namespace_name and
|
|
# container_name from the tag and set them in a local_resource_id in the
|
|
# format of:
|
|
# 'k8s_container.<NAMESPACE_NAME>.<POD_NAME>.<CONTAINER_NAME>'.
|
|
# The reformer also changes the tags to 'stderr' or 'stdout' based on the
|
|
# value of 'stream'.
|
|
# local_resource_id is later used by google_cloud plugin to determine the
|
|
# monitored resource to ingest logs against.
|
|
|
|
# Json Log Example:
|
|
# {"log":"[info:2016-02-16T16:04:05.930-08:00] Some log text here\n","stream":"stdout","time":"2016-02-17T00:04:05.931087621Z"}
|
|
# CRI Log Example:
|
|
# 2016-02-17T00:04:05.931087621Z stdout F [info:2016-02-16T16:04:05.930-08:00] Some log text here
|
|
<source>
|
|
@type tail
|
|
path /var/log/containers/*.log
|
|
pos_file /var/log/gcp-containers.log.pos
|
|
# Tags at this point are in the format of:
|
|
# reform.var.log.containers.<POD_NAME>_<NAMESPACE_NAME>_<CONTAINER_NAME>-<CONTAINER_ID>.log
|
|
tag reform.*
|
|
read_from_head true
|
|
format multi_format
|
|
<pattern>
|
|
format json
|
|
time_key time
|
|
time_format %Y-%m-%dT%H:%M:%S.%NZ
|
|
</pattern>
|
|
<pattern>
|
|
format /^(?<time>.+) (?<stream>stdout|stderr) [^ ]* (?<log>.*)$/
|
|
time_format %Y-%m-%dT%H:%M:%S.%N%:z
|
|
</pattern>
|
|
</source>
|
|
|
|
<filter reform.**>
|
|
@type parser
|
|
format /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<log>.*)/
|
|
reserve_data true
|
|
suppress_parse_error_log true
|
|
emit_invalid_record_to_error false
|
|
key_name log
|
|
</filter>
|
|
|
|
<match reform.**>
|
|
@type record_reformer
|
|
enable_ruby true
|
|
<record>
|
|
# Extract local_resource_id from tag for 'k8s_container' monitored
|
|
# resource. The format is:
|
|
# 'k8s_container.<namespace_name>.<pod_name>.<container_name>'.
|
|
"logging.googleapis.com/local_resource_id" ${"k8s_container.#{tag_suffix[4].rpartition('.')[0].split('_')[1]}.#{tag_suffix[4].rpartition('.')[0].split('_')[0]}.#{tag_suffix[4].rpartition('.')[0].split('_')[2].rpartition('-')[0]}"}
|
|
# Rename the field 'log' to a more generic field 'message'. This way the
|
|
# fluent-plugin-google-cloud knows to flatten the field as textPayload
|
|
# instead of jsonPayload after extracting 'time', 'severity' and
|
|
# 'stream' from the record.
|
|
message ${record['log']}
|
|
# If 'severity' is not set, assume stderr is ERROR and stdout is INFO.
|
|
severity ${record['severity'] || if record['stream'] == 'stderr' then 'ERROR' else 'INFO' end}
|
|
</record>
|
|
tag ${if record['stream'] == 'stderr' then 'raw.stderr' else 'raw.stdout' end}
|
|
remove_keys stream,log
|
|
</match>
|
|
|
|
# Detect exceptions in the log output and forward them as one log entry.
|
|
<match {raw.stderr,raw.stdout}>
|
|
@type detect_exceptions
|
|
|
|
remove_tag_prefix raw
|
|
message message
|
|
stream "logging.googleapis.com/local_resource_id"
|
|
multiline_flush_interval 5
|
|
max_bytes 500000
|
|
max_lines 1000
|
|
</match>
|
|
system.input.conf: |-
|
|
# Example:
|
|
# Dec 21 23:17:22 gke-foo-1-1-4b5cbd14-node-4eoj startupscript: Finished running startup script /var/run/google.startup.script
|
|
<source>
|
|
@type tail
|
|
format syslog
|
|
path /var/log/startupscript.log
|
|
pos_file /var/log/gcp-startupscript.log.pos
|
|
tag startupscript
|
|
</source>
|
|
|
|
# Examples:
|
|
# time="2016-02-04T06:51:03.053580605Z" level=info msg="GET /containers/json"
|
|
# time="2016-02-04T07:53:57.505612354Z" level=error msg="HTTP Error" err="No such image: -f" statusCode=404
|
|
# TODO(random-liu): Remove this after cri container runtime rolls out.
|
|
<source>
|
|
@type tail
|
|
format /^time="(?<time>[^)]*)" level=(?<severity>[^ ]*) msg="(?<message>[^"]*)"( err="(?<error>[^"]*)")?( statusCode=($<status_code>\d+))?/
|
|
path /var/log/docker.log
|
|
pos_file /var/log/gcp-docker.log.pos
|
|
tag docker
|
|
</source>
|
|
|
|
# Example:
|
|
# 2016/02/04 06:52:38 filePurge: successfully removed file /var/etcd/data/member/wal/00000000000006d0-00000000010a23d1.wal
|
|
<source>
|
|
@type tail
|
|
# Not parsing this, because it doesn't have anything particularly useful to
|
|
# parse out of it (like severities).
|
|
format none
|
|
path /var/log/etcd.log
|
|
pos_file /var/log/gcp-etcd.log.pos
|
|
tag etcd
|
|
</source>
|
|
|
|
# Multi-line parsing is required for all the kube logs because very large log
|
|
# statements, such as those that include entire object bodies, get split into
|
|
# multiple lines by glog.
|
|
|
|
# Example:
|
|
# I0204 07:32:30.020537 3368 server.go:1048] POST /stats/container/: (13.972191ms) 200 [[Go-http-client/1.1] 10.244.1.3:40537]
|
|
<source>
|
|
@type tail
|
|
format multiline
|
|
multiline_flush_interval 5s
|
|
format_firstline /^\w\d{4}/
|
|
format1 /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/
|
|
time_format %m%d %H:%M:%S.%N
|
|
path /var/log/kubelet.log
|
|
pos_file /var/log/gcp-kubelet.log.pos
|
|
tag kubelet
|
|
</source>
|
|
|
|
# Example:
|
|
# I1118 21:26:53.975789 6 proxier.go:1096] Port "nodePort for kube-system/default-http-backend:http" (:31429/tcp) was open before and is still needed
|
|
<source>
|
|
@type tail
|
|
format multiline
|
|
multiline_flush_interval 5s
|
|
format_firstline /^\w\d{4}/
|
|
format1 /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/
|
|
time_format %m%d %H:%M:%S.%N
|
|
path /var/log/kube-proxy.log
|
|
pos_file /var/log/gcp-kube-proxy.log.pos
|
|
tag kube-proxy
|
|
</source>
|
|
|
|
# Example:
|
|
# I0204 07:00:19.604280 5 handlers.go:131] GET /api/v1/nodes: (1.624207ms) 200 [[kube-controller-manager/v1.1.3 (linux/amd64) kubernetes/6a81b50] 127.0.0.1:38266]
|
|
<source>
|
|
@type tail
|
|
format multiline
|
|
multiline_flush_interval 5s
|
|
format_firstline /^\w\d{4}/
|
|
format1 /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/
|
|
time_format %m%d %H:%M:%S.%N
|
|
path /var/log/kube-apiserver.log
|
|
pos_file /var/log/gcp-kube-apiserver.log.pos
|
|
tag kube-apiserver
|
|
</source>
|
|
|
|
# Example:
|
|
# I0204 06:55:31.872680 5 servicecontroller.go:277] LB already exists and doesn't need update for service kube-system/kube-ui
|
|
<source>
|
|
@type tail
|
|
format multiline
|
|
multiline_flush_interval 5s
|
|
format_firstline /^\w\d{4}/
|
|
format1 /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/
|
|
time_format %m%d %H:%M:%S.%N
|
|
path /var/log/kube-controller-manager.log
|
|
pos_file /var/log/gcp-kube-controller-manager.log.pos
|
|
tag kube-controller-manager
|
|
</source>
|
|
|
|
# Example:
|
|
# W0204 06:49:18.239674 7 reflector.go:245] pkg/scheduler/factory/factory.go:193: watch of *api.Service ended with: 401: The event in requested index is outdated and cleared (the requested history has been cleared [2578313/2577886]) [2579312]
|
|
<source>
|
|
@type tail
|
|
format multiline
|
|
multiline_flush_interval 5s
|
|
format_firstline /^\w\d{4}/
|
|
format1 /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/
|
|
time_format %m%d %H:%M:%S.%N
|
|
path /var/log/kube-scheduler.log
|
|
pos_file /var/log/gcp-kube-scheduler.log.pos
|
|
tag kube-scheduler
|
|
</source>
|
|
|
|
# Example:
|
|
# I1104 10:36:20.242766 5 rescheduler.go:73] Running Rescheduler
|
|
<source>
|
|
@type tail
|
|
format multiline
|
|
multiline_flush_interval 5s
|
|
format_firstline /^\w\d{4}/
|
|
format1 /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/
|
|
time_format %m%d %H:%M:%S.%N
|
|
path /var/log/rescheduler.log
|
|
pos_file /var/log/gcp-rescheduler.log.pos
|
|
tag rescheduler
|
|
</source>
|
|
|
|
# Example:
|
|
# I0603 15:31:05.793605 6 cluster_manager.go:230] Reading config from path /etc/gce.conf
|
|
<source>
|
|
@type tail
|
|
format multiline
|
|
multiline_flush_interval 5s
|
|
format_firstline /^\w\d{4}/
|
|
format1 /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/
|
|
time_format %m%d %H:%M:%S.%N
|
|
path /var/log/glbc.log
|
|
pos_file /var/log/gcp-glbc.log.pos
|
|
tag glbc
|
|
</source>
|
|
|
|
# Example:
|
|
# I0603 15:31:05.793605 6 cluster_manager.go:230] Reading config from path /etc/gce.conf
|
|
<source>
|
|
@type tail
|
|
format multiline
|
|
multiline_flush_interval 5s
|
|
format_firstline /^\w\d{4}/
|
|
format1 /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/
|
|
time_format %m%d %H:%M:%S.%N
|
|
path /var/log/cluster-autoscaler.log
|
|
pos_file /var/log/gcp-cluster-autoscaler.log.pos
|
|
tag cluster-autoscaler
|
|
</source>
|
|
|
|
# Logs from systemd-journal for interesting services.
|
|
# TODO(random-liu): Keep this for compatibility, remove this after
|
|
# cri container runtime rolls out.
|
|
<source>
|
|
@type systemd
|
|
filters [{ "_SYSTEMD_UNIT": "docker.service" }]
|
|
pos_file /var/log/gcp-journald-docker.pos
|
|
read_from_head true
|
|
tag docker
|
|
</source>
|
|
|
|
<source>
|
|
@type systemd
|
|
filters [{ "_SYSTEMD_UNIT": "{{ container_runtime }}.service" }]
|
|
pos_file /var/log/gcp-journald-container-runtime.pos
|
|
read_from_head true
|
|
tag container-runtime
|
|
</source>
|
|
|
|
<source>
|
|
@type systemd
|
|
filters [{ "_SYSTEMD_UNIT": "kubelet.service" }]
|
|
pos_file /var/log/gcp-journald-kubelet.pos
|
|
read_from_head true
|
|
tag kubelet
|
|
</source>
|
|
|
|
<source>
|
|
@type systemd
|
|
filters [{ "_SYSTEMD_UNIT": "node-problem-detector.service" }]
|
|
pos_file /var/log/gcp-journald-node-problem-detector.pos
|
|
read_from_head true
|
|
tag node-problem-detector
|
|
</source>
|
|
|
|
# BEGIN_NODE_JOURNAL
|
|
# Whether to include node-journal or not is determined when starting the
|
|
# cluster. It is not changed when the cluster is already running.
|
|
<source>
|
|
@type systemd
|
|
pos_file /var/log/gcp-journald.pos
|
|
read_from_head true
|
|
tag node-journal
|
|
</source>
|
|
|
|
<filter node-journal>
|
|
@type grep
|
|
<exclude>
|
|
key _SYSTEMD_UNIT
|
|
pattern ^(docker|{{ container_runtime }}|kubelet|node-problem-detector)\.service$
|
|
</exclude>
|
|
</filter>
|
|
# END_NODE_JOURNAL
|
|
monitoring.conf: |-
|
|
# This source is used to acquire approximate process start timestamp,
|
|
# which purpose is explained before the corresponding output plugin.
|
|
<source>
|
|
@type exec
|
|
command /bin/sh -c 'date +%s'
|
|
tag process_start
|
|
time_format %Y-%m-%d %H:%M:%S
|
|
keys process_start_timestamp
|
|
</source>
|
|
|
|
# This filter is used to convert process start timestamp to integer
|
|
# value for correct ingestion in the prometheus output plugin.
|
|
<filter process_start>
|
|
@type record_transformer
|
|
enable_ruby true
|
|
auto_typecast true
|
|
<record>
|
|
process_start_timestamp ${record["process_start_timestamp"].to_i}
|
|
</record>
|
|
</filter>
|
|
output.conf: |-
|
|
# This match is placed before the all-matching output to provide metric
|
|
# exporter with a process start timestamp for correct exporting of
|
|
# cumulative metrics to Stackdriver.
|
|
<match process_start>
|
|
@type prometheus
|
|
|
|
<metric>
|
|
type gauge
|
|
name process_start_time_seconds
|
|
desc Timestamp of the process start in seconds
|
|
key process_start_timestamp
|
|
</metric>
|
|
</match>
|
|
|
|
# This filter allows to count the number of log entries read by fluentd
|
|
# before they are processed by the output plugin. This in turn allows to
|
|
# monitor the number of log entries that were read but never sent, e.g.
|
|
# because of liveness probe removing buffer.
|
|
<filter **>
|
|
@type prometheus
|
|
<metric>
|
|
type counter
|
|
name logging_entry_count
|
|
desc Total number of log entries generated by either application containers or system components
|
|
</metric>
|
|
</filter>
|
|
|
|
# This section is exclusive for k8s_container logs. Those come with
|
|
# 'stderr'/'stdout' tags.
|
|
# TODO(instrumentation): Reconsider this workaround later.
|
|
# Trim the entries which exceed slightly less than 100KB, to avoid
|
|
# dropping them. It is a necessity, because Stackdriver only supports
|
|
# entries that are up to 100KB in size.
|
|
<filter {stderr,stdout}>
|
|
@type record_transformer
|
|
enable_ruby true
|
|
<record>
|
|
message ${record['message'].length > 100000 ? "[Trimmed]#{record['message'][0..100000]}..." : record['message']}
|
|
</record>
|
|
</filter>
|
|
|
|
# Do not collect fluentd's own logs to avoid infinite loops.
|
|
<match fluent.**>
|
|
@type null
|
|
</match>
|
|
|
|
# This section is exclusive for k8s_container logs. These logs come with
|
|
# 'stderr'/'stdout' tags.
|
|
# We use a separate output stanza for 'k8s_node' logs with a smaller buffer
|
|
# because node logs are less important than user's container logs.
|
|
<match {stderr,stdout}>
|
|
@type google_cloud
|
|
|
|
# Try to detect JSON formatted log entries.
|
|
detect_json true
|
|
# Collect metrics in Prometheus registry about plugin activity.
|
|
enable_monitoring true
|
|
monitoring_type prometheus
|
|
# Allow log entries from multiple containers to be sent in the same request.
|
|
split_logs_by_tag false
|
|
# Set the buffer type to file to improve the reliability and reduce the memory consumption
|
|
buffer_type file
|
|
buffer_path /var/log/fluentd-buffers/kubernetes.containers.buffer
|
|
# Set queue_full action to block because we want to pause gracefully
|
|
# in case of the off-the-limits load instead of throwing an exception
|
|
buffer_queue_full_action block
|
|
# Set the chunk limit conservatively to avoid exceeding the recommended
|
|
# chunk size of 5MB per write request.
|
|
buffer_chunk_limit 512k
|
|
# Cap the combined memory usage of this buffer and the one below to
|
|
# 512KiB/chunk * (6 + 2) chunks = 4 MiB
|
|
buffer_queue_limit 6
|
|
# Never wait more than 5 seconds before flushing logs in the non-error case.
|
|
flush_interval 5s
|
|
# Never wait longer than 30 seconds between retries.
|
|
max_retry_wait 30
|
|
# Disable the limit on the number of retries (retry forever).
|
|
disable_retry_limit
|
|
# Use multiple threads for processing.
|
|
num_threads 2
|
|
use_grpc true
|
|
# Skip timestamp adjustment as this is in a controlled environment with
|
|
# known timestamp format. This helps with CPU usage.
|
|
adjust_invalid_timestamps false
|
|
</match>
|
|
|
|
# Attach local_resource_id for 'k8s_node' monitored resource.
|
|
<filter **>
|
|
@type record_transformer
|
|
enable_ruby true
|
|
<record>
|
|
"logging.googleapis.com/local_resource_id" ${"k8s_node.#{ENV['NODE_NAME']}"}
|
|
</record>
|
|
</filter>
|
|
|
|
# This section is exclusive for 'k8s_node' logs. These logs come with tags
|
|
# that are neither 'stderr' or 'stdout'.
|
|
# We use a separate output stanza for 'k8s_container' logs with a larger
|
|
# buffer because user's container logs are more important than node logs.
|
|
<match **>
|
|
@type google_cloud
|
|
|
|
detect_json true
|
|
enable_monitoring true
|
|
monitoring_type prometheus
|
|
# Allow entries from multiple system logs to be sent in the same request.
|
|
split_logs_by_tag false
|
|
detect_subservice false
|
|
buffer_type file
|
|
buffer_path /var/log/fluentd-buffers/kubernetes.system.buffer
|
|
buffer_queue_full_action block
|
|
buffer_chunk_limit 512k
|
|
buffer_queue_limit 2
|
|
flush_interval 5s
|
|
max_retry_wait 30
|
|
disable_retry_limit
|
|
num_threads 2
|
|
use_grpc true
|
|
# Skip timestamp adjustment as this is in a controlled environment with
|
|
# known timestamp format. This helps with CPU usage.
|
|
adjust_invalid_timestamps false
|
|
</match>
|
|
metadata:
|
|
name: fluentd-gcp-config-v1.2.5
|
|
namespace: kube-system
|
|
labels:
|
|
addonmanager.kubernetes.io/mode: Reconcile
|