diff --git a/CHANGELOG.md b/CHANGELOG.md index c68a8f28f..732f25105 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ All notable changes to this project will be documented in this file. - stackable-base: Mitigate CVE-2023-37920 by removing e-Tugra root certificates ([#673]). - hdfs: Exclude unused jars and mitigate snappy-java CVEs by bumping dependency ([#682]). - druid: Build from source ([#684], [#696]). +- opa: Add log processing script to opa for decision logging ([#695]). ### Changed @@ -88,6 +89,7 @@ All notable changes to this project will be documented in this file. [#685]: https://github.com/stackabletech/docker-images/pull/685 [#688]: https://github.com/stackabletech/docker-images/pull/688 [#696]: https://github.com/stackabletech/docker-images/pull/696 +[#695]: https://github.com/stackabletech/docker-images/pull/695 ## [24.3.0] - 2024-03-20 diff --git a/opa/Dockerfile b/opa/Dockerfile index 769cb0cd4..c8515dfb7 100644 --- a/opa/Dockerfile +++ b/opa/Dockerfile @@ -109,6 +109,9 @@ LABEL name="Open Policy Agent" \ description="This image is deployed by the Stackable Operator for OPA." RUN microdnf update && \ + microdnf install \ + # Required for filtering logs + jq && \ microdnf clean all && \ rm -rf /var/cache/yum @@ -121,4 +124,8 @@ COPY --from=opa-builder /opa/opa /stackable/opa/opa COPY --from=opa-bundle-builder --chown=stackable:stackable /opa-bundle-builder/target/release/stackable-opa-bundle-builder /stackable/opa-bundle-builder COPY --from=multilog-builder --chown=stackable:stackable /daemontools/admin/daemontools/command/multilog /stackable/multilog -CMD ["./opa", "run", "-s"] +COPY --chown=stackable:stackable opa/stackable/bin /stackable/opa/bin + +ENV PATH="${PATH}:/stackable/opa:/stackable/opa/bin" + +CMD ["opa", "run", "-s"] diff --git a/opa/stackable/bin/process-logs b/opa/stackable/bin/process-logs new file mode 100755 index 000000000..75de8c6a5 --- /dev/null +++ b/opa/stackable/bin/process-logs @@ -0,0 +1,88 @@ +#!/usr/bin/env bash +# +# This file was created by the Stackable developers. +# +# Processes incoming log messages. Log messages are filtered by the set log levels +# and forwarded to the output destinations console and/or file. +# + +set -euo pipefail + +if [ -z "${CONSOLE_LEVEL}" ]; then + echo "ERROR: env variable CONSOLE_LEVEL cannot be empty" + exit 1 +fi + +if [ -z "${FILE_LEVEL}" ]; then + echo "ERROR: env variable FILE_LEVEL cannot be empty" + exit 1 +fi + +if [ -z "${DECISION_LEVEL}" ]; then + echo "ERROR: env variable DECISION_LEVEL cannot be empty" + exit 1 +fi + +if [ -z "${SERVER_LEVEL}" ]; then + echo "ERROR: env variable SERVER_LEVEL cannot be empty" + exit 1 +fi + +if [ -z "${OPA_ROLLING_LOG_FILE_SIZE_BYTES}" ]; then + echo "ERROR: env variable OPA_ROLLING_LOG_FILE_SIZE_BYTES cannot be empty" + exit 1 +fi + +if [ -z "${OPA_ROLLING_LOG_FILES}" ]; then + echo "ERROR: env variable OPA_ROLLING_LOG_FILES cannot be empty" + exit 1 +fi + +if [ -z "${STACKABLE_LOG_DIR}" ]; then + echo "ERROR: env variable STACKABLE_LOG_DIR cannot be empty" + exit 1 +fi + +if [ -z "${CONTAINER_NAME}" ]; then + echo "ERROR: env variable CONTAINER_NAME cannot be empty" + exit 1 +fi + +get_levels() { + case $1 in + fatal) + echo '["fatal"]' ;; + error) + echo '["error","fatal"]' ;; + warn) + echo '["warn","error","fatal"]' ;; + info) + echo '["info","warn","error","fatal"]' ;; + debug) + echo '["debug","info","warn","error","fatal"]' ;; + trace) + echo '["trace","debug","info","warn","error","fatal"]' ;; + *) + echo '[]' ;; + esac +} + +main() { + local DECISION_LEVELS + DECISION_LEVELS=$(get_levels "$DECISION_LEVEL") + local SERVER_LEVELS + SERVER_LEVELS=$(get_levels "$SERVER_LEVEL") + local CONSOLE_LEVELS + CONSOLE_LEVELS=$(get_levels "$CONSOLE_LEVEL") + local FILE_LEVELS + FILE_LEVELS=$(get_levels "$FILE_LEVEL") + + jq -R -c --unbuffered --arg decision_levels "$DECISION_LEVELS" --arg server_levels "$SERVER_LEVELS" \ + 'try (fromjson | if .decision_id then .logger = "decision" else .logger = "server" end | select(((.logger == "decision") and (.level | inside($decision_levels))) or + ((.logger == "server") and (.level | inside($server_levels))))) catch {"time":(now | todate),"level":"info","msg":"Could not process log message","error":true}' | + tee >(jq -c --unbuffered --arg file_levels "$FILE_LEVELS" 'select(.level | inside($file_levels))' \ + > >(/stackable/multilog s"$OPA_ROLLING_LOG_FILE_SIZE_BYTES" n"$OPA_ROLLING_LOG_FILES" "$STACKABLE_LOG_DIR"/"$CONTAINER_NAME")) | + jq -c --unbuffered --arg console_levels "$CONSOLE_LEVELS" 'select(.level | inside($console_levels))' +} + +main