Merge pull request #142 from kube-logging/onecermain

introduce v1.16-4.9 image with rdkafka2 support

.github/workflows/artifacts.yaml

@@ -20,6 +20,7 @@ jobs:
         fluentd:
           - v1.16
           - v1.16-4.8
+          - v1.16-4.9
         image-type:
           - base
           - filters

.github/workflows/ci.yaml

@@ -13,6 +13,7 @@ jobs:
         fluentd:
           - v1.16
           - v1.16-4.8
+          - v1.16-4.9
         image-type:
           - base
           - filters

v1.16-4.9/Dockerfile

@@ -0,0 +1,101 @@
+# TODO every time the base image is changed please make sure BASEIMAGEDEPS is updated (further down in this file)
+# https://gallery.ecr.aws/sumologic/kubernetes-fluentd
+FROM public.ecr.aws/sumologic/kubernetes-fluentd:1.16.5-sumo-0-alpine as sumo
+
+FROM ruby:3.2.4-alpine3.18 as base
+
+ARG BUILD_DEPS=" \
+      make gcc g++ libc-dev \
+      wget bzip2 zlib-dev git linux-headers \
+      automake autoconf libtool build-base \
+      ruby-dev libc6-compat geoip-dev \
+      snappy-dev gnupg bash \
+      "
+
+RUN addgroup -S -g 101 fluent && adduser -S -G fluent -u 100 fluent \
+    # for log storage (maybe shared with host)
+    && mkdir -p /fluentd/log \
+    # configuration/plugins path (default: copied from .)
+    && mkdir -p /fluentd/etc /fluentd/plugins \
+    && chown -R fluent /fluentd && chgrp -R fluent /fluentd \
+    && mkdir -p /buffers && chown -R fluent /buffers
+
+COPY --from=sumo --chown=fluent:fluent /usr/local/bundle /usr/local/bundle
+
+RUN apk add --no-cache \
+        ca-certificates \
+        ruby ruby-irb ruby-etc \
+        tini libmaxminddb geoip \
+        snappy \
+ && apk add --no-cache $BUILD_DEPS \
+ && apk add --no-cache "libssl3>=3.1.4-r1" \
+ && apk add --no-cache "libcrypto3>=3.1.4-r1" \
+ && echo 'gem: --no-document' >> /etc/gemrc \
+ # BASEIMAGEDEPS
+ # need to rebuild these gems to have the extensions installed
+ && fluent-gem install \
+         bigdecimal:1.4.4 \
+         oj:3.16.1 \
+         google-protobuf:3.21.12 \
+         nio4r:2.7.3 \
+         snappy:0.3.0 \
+ # The only required fluentd dependency is the label router
+ # kubeclient install is upgrading the one from the base image
+ && fluent-gem install specific_install -v 0.3.8 \
+ && fluent-gem specific_install -l https://github.com/ManageIQ/kubeclient.git --ref 054bff2c5e31a555004be2b3c4d32fb9dc5e6a0f \
+ && fluent-gem specific_install -l https://github.com/kube-logging/fluent-plugin-label-router.git --ref 6d9d34a7e3fd73a400a31fb35513781802e7b36d \
+ && find /usr/local/bundle/gems/ -newer /etc/gemrc -exec chown fluent:fluent {} \; \
+ && apk del $BUILD_DEPS \
+ && rm -rf /usr/local/bundle/cache/* && find /usr/local/bundle -name '*.o' -delete
+
+COPY fluent.conf /fluentd/etc/
+COPY entrypoint.sh /bin/
+COPY healthy.sh /bin/
+
+ENV FLUENTD_CONF="fluent.conf"
+ENV LD_PRELOAD=""
+EXPOSE 24224 5140
+
+USER fluent
+
+ENTRYPOINT ["tini",  "--", "/bin/entrypoint.sh"]
+CMD ["fluentd"]
+
+### Image with all the filters
+FROM base AS filters
+
+ADD ./filters/Gemfile /Gemfile.filters
+ADD ./filters/Gemfile.lock /Gemfile.filters.lock
+
+USER root
+
+ENV BUNDLE_WITHOUT="output,test,development"
+
+RUN apk add --no-cache $BUILD_DEPS \
+ && touch /etc/gemrc \
+ && fluent-gem install --file Gemfile.filters \
+ && find /usr/local/bundle/gems/ -newer /etc/gemrc -exec chown fluent:fluent {} \; \
+ && apk del $BUILD_DEPS \
+ && rm -rf /usr/local/bundle/cache/* && find /usr/local/bundle -name '*.o' -delete
+
+USER fluent
+
+### Image with all the filters and outputs
+FROM filters as full
+
+ADD ./outputs/Gemfile /Gemfile.outputs
+ADD ./outputs/Gemfile.lock /Gemfile.outputs.lock
+
+USER root
+
+RUN apk add --no-cache $BUILD_DEPS \
+ && touch /etc/gemrc \
+ # TODO get rid of this by submitting the patch upstream
+ && fluent-gem specific_install -l https://github.com/kube-logging/fluent-plugin-gcs.git --ref ad6a7cb0f166e2c1648954a928e1e9b8cdcbf93d \
+ && fluent-gem specific_install -l https://github.com/kube-logging/fluent-plugin-syslog_rfc5424.git --ref 4ab9f7df3757b0e31e4bc209acab05a518efdce3 \
+ && fluent-gem install --file /Gemfile.outputs \
+ && find /usr/local/bundle/gems/ -newer /etc/gemrc -exec chown fluent:fluent {} \; \
+ && apk del $BUILD_DEPS \
+ && rm -rf /usr/local/bundle/cache/* && find /usr/local/bundle -name '*.o' -delete
+
+USER fluent

v1.16-4.9/entrypoint.sh

@@ -0,0 +1,28 @@
+#!/bin/sh
+
+#source vars if file exists
+DEFAULT=/etc/default/fluentd
+
+if [ -r $DEFAULT ]; then
+    set -o allexport
+    . $DEFAULT
+    set +o allexport
+fi
+
+# If the user has supplied only arguments append them to `fluentd` command
+if [ "${1#-}" != "$1" ]; then
+    set -- fluentd "$@"
+fi
+
+# If user does not supply config file or plugins, use the default
+if [ "$1" = "fluentd" ]; then
+    if ! echo $@ | grep ' \-c' ; then
+       set -- "$@" -c /fluentd/etc/${FLUENTD_CONF}
+    fi
+
+    if ! echo $@ | grep ' \-p' ; then
+       set -- "$@" -p /fluentd/plugins
+    fi
+fi
+
+exec "$@"

v1.16-4.9/filters/Gemfile

@@ -0,0 +1,17 @@
+source 'https://rubygems.org'
+
+gem 'fluent-plugin-detect-exceptions', '0.0.15'
+gem 'fluent-plugin-prometheus', '2.1.0'
+gem 'fluent-plugin-dedot_filter', '1.0.0'
+gem 'fluent-plugin-geoip', '1.3.2'
+gem 'fluent-plugin-tag-normaliser', '0.1.3'
+gem 'fluent-plugin-concat', '2.5.0'
+gem 'fluent-plugin-parser-logfmt', '0.0.0'
+gem 'fluent-plugin-record-modifier', '2.2.0'
+gem 'fluent-plugin-kube-events-timestamp', '0.1.3'
+gem 'fluent-plugin-throttle', '0.0.5'
+gem 'fluent-plugin-rewrite-tag-filter', '2.4.0'
+gem 'fluent-plugin-multi-format-parser', '1.1.0'
+gem 'fluent-plugin-grok-parser', '2.6.2'
+gem 'fluent-plugin-mysqlslowquery', '0.0.9'
+gem 'fluent-plugin-ua-parser', '1.2.0'

v1.16-4.9/filters/Gemfile.lock

@@ -0,0 +1,101 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    concurrent-ruby (1.2.3)
+    cool.io (1.8.0)
+    dig_rb (1.0.1)
+    fluent-config-regexp-type (1.0.0)
+      fluentd (> 1.0.0, < 2)
+    fluent-plugin-concat (2.5.0)
+      fluentd (>= 0.14.0, < 2)
+    fluent-plugin-dedot_filter (1.0.0)
+      fluentd (>= 0.14.0, < 2)
+    fluent-plugin-detect-exceptions (0.0.15)
+      fluentd (>= 0.10)
+    fluent-plugin-geoip (1.3.2)
+      dig_rb
+      fluentd (>= 0.14.8, < 2)
+      geoip-c
+      geoip2_c
+    fluent-plugin-grok-parser (2.6.2)
+      fluentd (>= 0.14.6, < 2)
+    fluent-plugin-kube-events-timestamp (0.1.3)
+      fluentd (>= 0.14.0, < 2)
+    fluent-plugin-multi-format-parser (1.1.0)
+      fluentd (>= 0.14.0, < 2)
+    fluent-plugin-mysqlslowquery (0.0.9)
+      fluentd (>= 0.12.0, < 2)
+      myslog (~> 0.0)
+    fluent-plugin-parser-logfmt (0.0.0)
+      fluentd (>= 1, < 2)
+      logfmt (~> 0.0.8)
+    fluent-plugin-prometheus (2.1.0)
+      fluentd (>= 1.9.1, < 2)
+      prometheus-client (>= 2.1.0)
+    fluent-plugin-record-modifier (2.2.0)
+      fluentd (>= 1.1, < 2)
+    fluent-plugin-rewrite-tag-filter (2.4.0)
+      fluent-config-regexp-type
+      fluentd (>= 0.14.2, < 2)
+    fluent-plugin-tag-normaliser (0.1.3)
+      fluentd (>= 0.14.10, < 2)
+    fluent-plugin-throttle (0.0.5)
+      fluentd (~> 1.1)
+    fluent-plugin-ua-parser (1.2.0)
+      fluentd (>= 0.14, < 2)
+      lru_redux (>= 1.0.0)
+      user_agent_parser (>= 2.2.0)
+    fluentd (1.16.3)
+      bundler
+      cool.io (>= 1.4.5, < 2.0.0)
+      http_parser.rb (>= 0.5.1, < 0.9.0)
+      msgpack (>= 1.3.1, < 2.0.0)
+      serverengine (>= 2.3.2, < 3.0.0)
+      sigdump (~> 0.2.5)
+      strptime (>= 0.2.4, < 1.0.0)
+      tzinfo (>= 1.0, < 3.0)
+      tzinfo-data (~> 1.0)
+      webrick (~> 1.4)
+      yajl-ruby (~> 1.0)
+    geoip-c (0.9.1)
+    geoip2_c (0.3.4)
+    http_parser.rb (0.8.0)
+    logfmt (0.0.10)
+    lru_redux (1.1.0)
+    msgpack (1.7.2)
+    myslog (0.1.1)
+    prometheus-client (4.2.1)
+    serverengine (2.3.2)
+      sigdump (~> 0.2.2)
+    sigdump (0.2.5)
+    strptime (0.2.5)
+    tzinfo (2.0.6)
+      concurrent-ruby (~> 1.0)
+    tzinfo-data (1.2023.4)
+      tzinfo (>= 1.0.0)
+    user_agent_parser (2.16.0)
+    webrick (1.8.1)
+    yajl-ruby (1.4.3)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  fluent-plugin-concat (= 2.5.0)
+  fluent-plugin-dedot_filter (= 1.0.0)
+  fluent-plugin-detect-exceptions (= 0.0.15)
+  fluent-plugin-geoip (= 1.3.2)
+  fluent-plugin-grok-parser (= 2.6.2)
+  fluent-plugin-kube-events-timestamp (= 0.1.3)
+  fluent-plugin-multi-format-parser (= 1.1.0)
+  fluent-plugin-mysqlslowquery (= 0.0.9)
+  fluent-plugin-parser-logfmt (= 0.0.0)
+  fluent-plugin-prometheus (= 2.1.0)
+  fluent-plugin-record-modifier (= 2.2.0)
+  fluent-plugin-rewrite-tag-filter (= 2.4.0)
+  fluent-plugin-tag-normaliser (= 0.1.3)
+  fluent-plugin-throttle (= 0.0.5)
+  fluent-plugin-ua-parser (= 1.2.0)
+
+BUNDLED WITH
+   2.4.22

v1.16-4.9/fluent.conf

@@ -0,0 +1,8 @@
+# This is the root config file, which only includes components of the actual configuration
+
+# Do not collect fluentd's own logs to avoid infinite loops.
+<match fluent.**>
+  @type null
+</match>
+
+@include /fluentd/etc/conf.d/*.conf

v1.16-4.9/healthy.sh

@@ -0,0 +1,23 @@
+#!/bin/sh -x
+
+# Liveness probe is aimed to help in situations where fluentd
+# silently hangs for no apparent reasons until manual restart.
+# The idea of this probe is that if fluentd is not queueing or
+# flushing chunks for 5 minutes, something is not right. If
+# you want to change the fluentd configuration, reducing amount of
+# logs fluentd collects, consider changing the threshold or turning
+# liveness probe off completely.
+# soiurce https://github.com/kubernetes/kubernetes/blob/master/cluster/addons/fluentd-gcp/fluentd-gcp-ds.yaml#L58
+
+BUFFER_PATH=${BUFFER_PATH};
+LIVENESS_THRESHOLD_SECONDS=${LIVENESS_THRESHOLD_SECONDS:-300};
+
+if [ ! -e ${BUFFER_PATH} ];
+then
+  exit 1;
+fi;
+touch -d "@$(($(date +%s) - $LIVENESS_THRESHOLD_SECONDS))" /tmp/marker-liveness;
+if [ -z "$(find ${BUFFER_PATH} -type d -newer /tmp/marker-liveness -print -quit)" ];
+then
+  exit 1;
+fi;

v1.16-4.9/outputs/Gemfile

@@ -0,0 +1,25 @@
+source 'https://rubygems.org'
+
+gem 'fluent-plugin-aliyun-oss', '0.0.1'
+gem 'fluent-plugin-sumologic_output', '1.9.0'
+gem 'fluent-plugin-kafka', '0.19.2'
+gem 'fluent-plugin-grafana-loki', '1.2.20'
+gem 'fluent-plugin-kinesis', '3.5.0'
+gem 'fluent-plugin-splunk-hec', '1.3.3'
+gem 'fluent-plugin-elasticsearch', '5.4.3'
+gem 'fluent-plugin-newrelic', '1.2.3'
+gem 'fluent-plugin-cloudwatch-logs', '0.14.3'
+gem 'fluent-plugin-opensearch', '1.1.4'
+gem 'fluent-plugin-logzio', '0.2.2'
+gem 'fluent-plugin-datadog', '0.14.3'
+gem 'fluent-plugin-redis', '0.3.5'
+gem 'fluent-plugin-sqs', '3.0.0'
+gem 'fluent-plugin-mattermost', '0.2.2'
+gem 'fluent-plugin-remote-syslog', '1.1.0'
+gem 'fluent-plugin-webhdfs', '1.6.0'
+gem 'fluent-plugin-vmware-loginsight', '1.4.2'
+gem 'fluent-plugin-vmware-log-intelligence', '2.0.8'
+gem 'fluent-plugin-gelf-best', '1.3.3'
+gem 'rdkafka', '0.16.0'
+#gem 'fluent-plugin-aws-elasticsearch-service', '2.4.1'
+#gem 'fluent-plugin-logdna', '~> 0.4.0'