魏长东

weichangdong

fluentd配置文件

之前成功搞定了apache的flume,觉得功能很强大,但是发现fluentd也很强大。

下面是配置文件。

<match debug.**>
  type stdout
</match>
<source>
  type forward
</source>
<source>
  type http
  port 8888
</source>
<source>
  type debug_agent
  bind 127.0.0.1
  port 24230
</source>

<source>
  type tail
  format /^(?<remote>[^ ]*) (?<host>[^ ]*) (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)" "(?<xxx>[^\"]*)")?$/
  time_format %d/%b/%Y:%H:%M:%S %z
  path /data/logs/nginx/access.log
  pos_file /data/logs/nginx/access.log.pos
  tag s3.nginx.access
</source>

<match s3.*.*>
  type s3

  aws_key_id *****
  aws_sec_key *****
  s3_bucket web-log
  s3_region ap-southeast-1
  s3_object_key_format %{path}/%{time_slice}_s2.%{file_extension}
  path log
  buffer_path /var/log/td-agent/buffer/s3
  #flush_interval 1m

  time_slice_format %Y%m%d/%H
  time_slice_wait 10m
  utc

  format json
  include_time_key true
  include_tag_key false

  buffer_chunk_limit 2g
</match>