Skip to content

Commit 1537df2

Browse files
joecoralljoecorall
committed
Copy fluent-bit.conf original
1 parent 0e5cfd2 commit 1537df2

1 file changed

Lines changed: 221 additions & 2 deletions

File tree

‎rootfs/home/cloud-compose/fluent-bit.conf‎

Lines changed: 221 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,225 @@
1+
#
2+
# Copyright 2022 Google LLC
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# https://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
#
16+
[SERVICE]
17+
# Flush
18+
# =====
19+
# set an interval of seconds before to flush records to a destination
20+
flush 1
21+
22+
# Daemon
23+
# ======
24+
# instruct Fluent Bit to run in foreground or background mode.
25+
daemon Off
26+
27+
# Log_Level
28+
# =========
29+
# Set the verbosity level of the service, values can be:
30+
#
31+
# - error
32+
# - warning
33+
# - info
34+
# - debug
35+
# - trace
36+
#
37+
# by default 'info' is set, that means it includes 'error' and 'warning'.
38+
log_level info
39+
40+
# Parsers File
41+
# ============
42+
# specify an optional 'Parsers' configuration file
43+
parsers_file parsers.conf
44+
45+
# Plugins File
46+
# ============
47+
# specify an optional 'Plugins' configuration file to load external plugins.
48+
#
49+
# plugins_file plugins.conf
50+
51+
# HTTP Server
52+
# ===========
53+
# Enable/Disable the built-in HTTP Server for metrics
54+
http_server Off
55+
# http_listen 0.0.0.0
56+
# http_port 2020
57+
58+
# Storage
59+
# =======
60+
# Fluent Bit can use memory and filesystem buffering based mechanisms
61+
#
62+
# - https://docs.fluentbit.io/manual/administration/buffering-and-storage
63+
#
64+
# storage metrics
65+
# ---------------
66+
# publish storage pipeline metrics in '/api/v1/storage'. The metrics are
67+
# exported only if the 'http_server' option is enabled.
68+
#
69+
storage.metrics on
70+
71+
# storage.path
72+
# ------------
73+
# absolute file system path to store filesystem data buffers (chunks).
74+
#
75+
# storage.path /tmp/storage
76+
77+
# storage.sync
78+
# ------------
79+
# configure the synchronization mode used to store the data into the
80+
# filesystem. It can take the values normal or full.
81+
#
82+
# storage.sync normal
83+
84+
# storage.checksum
85+
# ----------------
86+
# enable the data integrity check when writing and reading data from the
87+
# filesystem. The storage layer uses the CRC32 algorithm.
88+
#
89+
# storage.checksum off
90+
91+
# storage.backlog.mem_limit
92+
# -------------------------
93+
# if storage.path is set, Fluent Bit will look for data chunks that were
94+
# not delivered and are still in the storage layer, these are called
95+
# backlog data. This option configure a hint of maximum value of memory
96+
# to use when processing these records.
97+
#
98+
# storage.backlog.mem_limit 5M
99+
100+
# Collects docker.service logs.
1101
[INPUT]
2102
Name systemd
3103
Tag cos_docker
4-
Systemd_Filter _SYSTEMD_UNIT=cloud-compose.service
5-
DB /var/log/google-fluentbit/sync-conf.log.db
104+
Systemd_Filter _SYSTEMD_UNIT=docker.service
105+
DB /var/log/google-fluentbit/docker.log.db
6106
Read_From_Tail False
107+
108+
# Collects COS system services logs.
109+
[INPUT]
110+
Name systemd
111+
Tag cos_system
112+
Systemd_Filter _SYSTEMD_UNIT=docker-events-collector-fluent-bit.service
113+
Systemd_Filter _SYSTEMD_UNIT=konlet-startup.service
114+
Systemd_Filter _SYSTEMD_UNIT=kubelet.service
115+
Systemd_Filter _SYSTEMD_UNIT=crash-reporter.service
116+
Systemd_Filter _SYSTEMD_UNIT=crash-sender.service
117+
Systemd_Filter _SYSTEMD_UNIT=crash-boot-collect.service
118+
Systemd_Filter _SYSTEMD_UNIT=kdump-load-kernel.service
119+
Systemd_Filter _SYSTEMD_UNIT=kdump-save-dump.service
120+
Systemd_Filter _SYSTEMD_UNIT=cis-level1.service
121+
Systemd_Filter _SYSTEMD_UNIT=cis-level2.service
122+
Systemd_Filter _SYSTEMD_UNIT=cis-compliance-scanner.service
123+
Systemd_Filter _SYSTEMD_UNIT=systemd-networkd.service
124+
Systemd_Filter _SYSTEMD_UNIT=systemd-resolved.service
125+
Systemd_Filter _SYSTEMD_UNIT=systemd-logind.service
126+
Systemd_Filter _SYSTEMD_UNIT=chronyd.service
127+
Systemd_Filter _SYSTEMD_UNIT=device_policy_manager.service
128+
Systemd_Filter _SYSTEMD_UNIT=cloud-config.service
129+
Systemd_Filter _SYSTEMD_UNIT=cloud-init-local.service
130+
Systemd_Filter SYSLOG_IDENTIFIER=crash-sender.sh
131+
DB /var/log/google-fluentbit/system.log.db
132+
Read_From_Tail False
133+
134+
# Collects COS audit logs.
135+
[INPUT]
136+
Name systemd
137+
Tag cos_audit
138+
Systemd_Filter SYSLOG_IDENTIFIER=audit
139+
DB /var/log/google-fluentbit/audit.log.db
140+
Read_From_Tail False
141+
142+
# Collects all journal logs with priority >= warning
143+
# Change priority levels to make it more/less verbose.
144+
[INPUT]
145+
Name systemd
146+
Tag cos_journal_warning
147+
Systemd_Filter PRIORITY=0
148+
Systemd_Filter PRIORITY=1
149+
Systemd_Filter PRIORITY=2
150+
Systemd_Filter PRIORITY=3
151+
Systemd_Filter PRIORITY=4
152+
DB /var/log/google-fluentbit/journal.db
153+
Read_From_Tail False
154+
155+
# Docker container logs (when not running Kubernetes).
156+
# This will collect logs from all containers using json file logging driver.
157+
# To query logs for specific container, use below filter on GCP logging:
158+
# jsonPayload.container_id=CONTAINER_ID
159+
[INPUT]
160+
Name tail
161+
Tag cos_containers
162+
Path /var/lib/docker/containers/*/*.log
163+
Parser docker
164+
DB /var/log/google-fluentbit/containers.log.db
165+
Path_Key file_name
166+
Read_from_Head True
167+
Mem_Buf_Limit 5MB
168+
Skip_Long_Lines On
169+
170+
# 1) Add container_id field in container logs.
171+
[FILTER]
172+
Name parser
173+
Match cos_containers
174+
Key_Name file_name
175+
Parser container_filename_to_id
176+
Reserve_Data On
177+
178+
# 2) The following two filters save attrs[tag] as container_name field in
179+
# the record.
180+
[FILTER]
181+
Name nest
182+
Match cos_containers
183+
Operation lift
184+
Nested_under attrs
185+
Add_prefix attrs_
186+
187+
[FILTER]
188+
Name modify
189+
Match cos_containers
190+
Condition Key_exists attrs_tag
191+
Rename attrs_tag cos.googleapis.com/container_name
192+
193+
# If attrs[tag] doesn't exist, set container_name field as 'UNKNOWN'
194+
[FILTER]
195+
Name modify
196+
Match cos_containers
197+
Condition Key_does_not_exist attrs_tag
198+
Add cos.googleapis.com/container_name UNKNOWN
199+
200+
[FILTER]
201+
Name modify
202+
Match cos_containers
203+
Rename container_id cos.googleapis.com/container_id
204+
# 3) Rename field 'stream' to avoid collisions from container logs where
205+
# users may be also using 'stream' as a key
206+
Rename stream cos.googleapis.com/stream
207+
# 4) Rename field 'log' to a more generic field 'message'. This way Logs
208+
# Explorer UI will display the log message as summary of the log entry.
209+
Rename log message
210+
# Remove other fields under attrs.
211+
Remove_wildcard attrs_
212+
213+
# match fluentd jsonPayload structure
214+
[FILTER]
215+
Name nest
216+
Match cos_containers
217+
Operation lift
218+
Nested_under message
219+
220+
[OUTPUT]
221+
Name stackdriver
222+
Match *
223+
Resource gce_instance
224+
severity_key severity
225+
Cloud_Logging_Base_Url ${CLOUD_LOGGING_BASE_URL}

0 commit comments

Comments
 (0)