8040f75594
changes to fluent-plugin-google-cloud to attach Kubernetes metadata to logs. Along with this, separate logs from containers in the cluster out from logs from the daemons running on the node by instantiating two instances of the output plugin, one which uses the new metadata (for containers) and one which doesn't (for things like docker and the kubelet).
157 lines
5.1 KiB
Plaintext
157 lines
5.1 KiB
Plaintext
# This configuration file for Fluentd / td-agent is used
|
|
# to watch changes to Docker log files that live in the
|
|
# directory /var/lib/docker/containers/ and are symbolically
|
|
# linked to from the /var/log directory using names that capture the
|
|
# pod name and container name. These logs are then submitted to
|
|
# Google Cloud Logging which assumes the installation of the cloud-logging plug-in.
|
|
#
|
|
# Example
|
|
# =======
|
|
# A line in the Docker log file might like like this JSON:
|
|
#
|
|
# {"log":"2014/09/25 21:15:03 Got request with path wombat\n",
|
|
# "stream":"stderr",
|
|
# "time":"2014-09-25T21:15:03.499185026Z"}
|
|
#
|
|
# Currently, the log information is ingested as plain text rather than JSON.
|
|
# TODO: When Cloud Logging supports it, ingest as JSON.
|
|
#
|
|
# The record reformer is used to write the tag to focus on the pod name
|
|
# and the Kubernetes container name. For example a Docker container's logs
|
|
# might be in the directory:
|
|
# /var/lib/docker/containers/997599971ee6366d4a5920d25b79286ad45ff37a74494f262e3bc98d909d0a7b
|
|
# and in the file:
|
|
# 997599971ee6366d4a5920d25b79286ad45ff37a74494f262e3bc98d909d0a7b-json.log
|
|
# where 997599971ee6... is the Docker ID of the running container.
|
|
# The Kubernetes kubelet makes a symbolic link to this file on the host machine
|
|
# in the /var/log/containers directory which includes the pod name and the Kubernetes
|
|
# container name:
|
|
# synthetic-logger-0.25lps-pod_default-synth-lgr-997599971ee6366d4a5920d25b79286ad45ff37a74494f262e3bc98d909d0a7b.log
|
|
# ->
|
|
# /var/lib/docker/containers/997599971ee6366d4a5920d25b79286ad45ff37a74494f262e3bc98d909d0a7b/997599971ee6366d4a5920d25b79286ad45ff37a74494f262e3bc98d909d0a7b-json.log
|
|
# The /var/log directory on the host is mapped to the /var/log directory in the container
|
|
# running this instance of Fluentd and we end up collecting the file:
|
|
# /var/log/containers/synthetic-logger-0.25lps-pod_default-synth-lgr-997599971ee6366d4a5920d25b79286ad45ff37a74494f262e3bc98d909d0a7b.log
|
|
# This results in the tag:
|
|
# var.log.containers.synthetic-logger-0.25lps-pod_default-synth-lgr-997599971ee6366d4a5920d25b79286ad45ff37a74494f262e3bc98d909d0a7b.log
|
|
# The record reformer is used is discard the var.log.containers prefix and
|
|
# the Docker container ID suffix and "kubernetes." is pre-pended giving the
|
|
# final tag which is ingested into Elasticsearch:
|
|
# kubernetes.synthetic-logger-0.25lps-pod_default-synth-lgr
|
|
# This makes it easier for users to search for logs by pod name or by
|
|
# the name of the Kubernetes container regardless of how many times the
|
|
# Kubernetes pod has been restarted (resulting in a several Docker container IDs).
|
|
|
|
<source>
|
|
type tail
|
|
format json
|
|
time_key time
|
|
path /var/log/containers/*.log
|
|
pos_file /var/log/gcp-containers.log.pos
|
|
time_format %Y-%m-%dT%H:%M:%S
|
|
tag reform.*
|
|
read_from_head true
|
|
</source>
|
|
|
|
<match reform.**>
|
|
type record_reformer
|
|
enable_ruby true
|
|
tag kubernetes.${tag_suffix[4].split('-')[0..-2].join('-')}
|
|
</match>
|
|
|
|
<source>
|
|
type tail
|
|
format none
|
|
path /var/log/salt/minion
|
|
pos_file /var/log/gcp-salt.pos
|
|
tag salt
|
|
</source>
|
|
|
|
<source>
|
|
type tail
|
|
format none
|
|
path /var/log/startupscript.log
|
|
pos_file /var/log/gcp-startupscript.log.pos
|
|
tag startupscript
|
|
</source>
|
|
|
|
<source>
|
|
type tail
|
|
format none
|
|
path /var/log/docker.log
|
|
pos_file /var/log/gcp-docker.log.pos
|
|
tag docker
|
|
</source>
|
|
|
|
<source>
|
|
type tail
|
|
format none
|
|
path /var/log/etcd.log
|
|
pos_file /var/log/gcp-etcd.log.pos
|
|
tag etcd
|
|
</source>
|
|
|
|
<source>
|
|
type tail
|
|
format none
|
|
path /var/log/kubelet.log
|
|
pos_file /var/log/gcp-kubelet.log.pos
|
|
tag kubelet
|
|
</source>
|
|
|
|
<source>
|
|
type tail
|
|
format none
|
|
path /var/log/kube-apiserver.log
|
|
pos_file /var/log/gcp-kube-apiserver.log.pos
|
|
tag kube-apiserver
|
|
</source>
|
|
|
|
<source>
|
|
type tail
|
|
format none
|
|
path /var/log/kube-controller-manager.log
|
|
pos_file /var/log/gcp-kube-controller-manager.log.pos
|
|
tag kube-controller-manager
|
|
</source>
|
|
|
|
<source>
|
|
type tail
|
|
format none
|
|
path /var/log/kube-scheduler.log
|
|
pos_file /var/log/gcp-kube-scheduler.log.pos
|
|
tag kube-scheduler
|
|
</source>
|
|
|
|
# We use 2 output stanzas - one to handle the container logs and one to handle
|
|
# the node daemon logs, the latter of which explicitly sends its logs to the
|
|
# compute.googleapis.com service rather than container.googleapis.com to keep
|
|
# them separate since most users don't care about the node logs.
|
|
<match kubernetes.**>
|
|
type google_cloud
|
|
# Set the chunk limit conservatively to avoid exceeding the GCL limit
|
|
# of 2MB per write request.
|
|
buffer_chunk_limit 512K
|
|
# Cap the combined memory usage of this buffer and the one below to
|
|
# 512KB/chunk * (96 + 32) chunks = 65 MB
|
|
buffer_queue_limit 96
|
|
# Never wait more than 5 seconds before flushing logs in the non-error case.
|
|
flush_interval 5s
|
|
# Never wait longer than 5 minutes between retries.
|
|
max_retry_wait 300
|
|
# Disable the limit on the number of retries (retry forever).
|
|
disable_retry_limit
|
|
</match>
|
|
|
|
# Keep a smaller buffer here since these logs are less important than the user's
|
|
# container logs.
|
|
<match **>
|
|
type google_cloud
|
|
detect_subservice false
|
|
buffer_chunk_limit 512K
|
|
buffer_queue_limit 32
|
|
flush_interval 5s
|
|
max_retry_wait 300
|
|
disable_retry_limit
|
|
</match>
|