https://github.com/fluent/fluent-plugin-s3

    With fluentd v0.14 and fluent-plugin-s3 v1.0.0, use new buffer configuration to dynamic parameters.

    1. <match pattern>
    2.   @type s3
    4.   aws_key_id YOUR_AWS_KEY_ID
    5.   aws_sec_key YOUR_AWS_SECRET_KEY
    6.   s3_bucket YOUR_S3_BUCKET_NAME
    7.   s3_region ap-northeast-1
    9.   path logs/${tag}/%Y/%m/%d/
    10.   s3_object_key_format %{path}%{time_slice}_%{index}.%{file_extension}
    12.   # if you want to use ${tag} or %Y/%m/%d/ like syntax in path / s3_object_key_format,
    13.   # need to specify tag for ${tag} and time for %Y/%m/%d in <buffer> argument.
    14.   <buffer tag,time>
    15.     @type file
    16.     path /var/log/fluent/s3
    17.     timekey 3600 # 1 hour partition
    18.     timekey_wait 10m
    19.     timekey_use_utc true # use utc
    20.   </buffer>
    21.   <format>
    22.     @type json
    23.   </format>
    24. </match>

    https://devtron.ai/blog/pushing-k8s-cluster-logs-s3-bucket-fluentd/

    1. general.conf: |
    2.   # Prevent fluentd from handling records containing its own logs. Otherwise
    3.   # it can lead to an infinite loop, when error in sending one message generates
    4.   # another message which also fails to be sent and so on.
    5.   <match fluentd.**>
    6.     @type null
    7.   </match>
    8.   # Used for health checking
    9.   <source>
    10.     @type http
    11.     port 9880
    12.     bind 0.0.0.0
    13.   </source>
    14.   # Emits internal metrics to every minute, and also exposes them on port
    15.   # 24220. Useful for determining if an output plugin is retryring/erroring,
    16.   # or determining the buffer queue length.
    17.   <source>
    18.     @type monitor_agent
    19.     bind 0.0.0.0
    20.     port 24220
    21.     tag fluentd.monitor.metrics
    22.   </source>
    23. system.conf: |-
    24.   <system>
    25.     root_dir /tmp/fluentd-buffers/
    26.   </system>
    27. forward-input.conf: |
    28.   <source>
    29.     @type forward
    30.     port 24224
    31.     bind 0.0.0.0
    32.   </source>
    33. output.conf: |
    34.   <filter kube.**>
    35.      @type record_transformer
    36.      enable_ruby
    37.      <record>
    38.        kubernetes_tag ${"%s" % [record["kubernetes"]["labels"]["app"] || record["kubernetes"]["labels"]["k8s-app"] || record["kubernetes"]["labels"]["name"] || "unspecified-app-label"]}
    39.      </record>
    40.   </filter>
    41.   <match kube.**>
    42.      @type rewrite_tag_filter
    43.      <rule>
    44.        key     kubernetes_tag
    45.        pattern ^(.+)$
    46.        tag     $1
    47.      </rule>
    48.   </match>
    49.   <match **>
    50.      @type s3
    51.      s3_bucket <k8s-logs-bucket>
    52.      s3_region <ap-southeast-1>
    53.      s3_object_key_format "${tag}/%{time_slice}-events_%{index}.%{file_extension}"
    54.      time_slice_format %Y/%m/%d/%H
    55.      time_slice_wait 10m
    56.      path test-logs
    57.      # if you want to use ${tag} or %Y/%m/%d/ like syntax in path / s3_object_key_format,
    58.      # need to specify tag for ${tag} and time for %Y/%m/%d in <buffer> argument.
    59.      <buffer tag,time>
    60.        @type file
    61.        flush_mode interval
    62.        flush_interval 30s
    63.        path /var/log/fluent/s3
    64.        timekey 300 # 1 hour partition
    65.        timekey_wait 1m
    66.        timekey_use_utc true # use utc
    67.        chunk_limit_size 100m
    68.      </buffer>
    69.      <format>
    70.        @type json
    71.      </format>
    72.   </match>

    https://blog.csdn.net/hxpjava1/article/details/79447630

    https://docs.fluentd.org/buffer/file