Skip to content

Commit

Permalink
[System-Probe] remove code duplication in modules config pkgs (#30259)
Browse files Browse the repository at this point in the history
  • Loading branch information
@val06
val06 authored Oct 18, 2024
1 parent 16c16b2 commit ec63d4e
Show file tree
Hide file tree
Showing 3 changed files with 100 additions and 116 deletions.
48 changes: 21 additions & 27 deletions pkg/ebpf/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,6 @@
package ebpf

import (
"strings"

sysconfig "github.com/DataDog/datadog-agent/cmd/system-probe/config"
pkgconfigsetup "github.com/DataDog/datadog-agent/pkg/config/setup"
"github.com/DataDog/datadog-agent/pkg/util/kernel"
Expand Down Expand Up @@ -81,38 +79,34 @@ type Config struct {
BypassEnabled bool
}

func key(pieces ...string) string {
return strings.Join(pieces, ".")
}

// NewConfig creates a config with ebpf-related settings
func NewConfig() *Config {
cfg := pkgconfigsetup.SystemProbe()
sysconfig.Adjust(cfg)

c := &Config{
BPFDebug: cfg.GetBool(key(spNS, "bpf_debug")),
BPFDir: cfg.GetString(key(spNS, "bpf_dir")),
ExcludedBPFLinuxVersions: cfg.GetStringSlice(key(spNS, "excluded_linux_versions")),
EnableTracepoints: cfg.GetBool(key(spNS, "enable_tracepoints")),
BPFDebug: cfg.GetBool(sysconfig.FullKeyPath(spNS, "bpf_debug")),
BPFDir: cfg.GetString(sysconfig.FullKeyPath(spNS, "bpf_dir")),
ExcludedBPFLinuxVersions: cfg.GetStringSlice(sysconfig.FullKeyPath(spNS, "excluded_linux_versions")),
EnableTracepoints: cfg.GetBool(sysconfig.FullKeyPath(spNS, "enable_tracepoints")),
ProcRoot: kernel.ProcFSRoot(),
InternalTelemetryEnabled: cfg.GetBool(key(spNS, "telemetry_enabled")),

EnableCORE: cfg.GetBool(key(spNS, "enable_co_re")),
BTFPath: cfg.GetString(key(spNS, "btf_path")),

EnableRuntimeCompiler: cfg.GetBool(key(spNS, "enable_runtime_compiler")),
RuntimeCompilerOutputDir: cfg.GetString(key(spNS, "runtime_compiler_output_dir")),
EnableKernelHeaderDownload: cfg.GetBool(key(spNS, "enable_kernel_header_download")),
KernelHeadersDirs: cfg.GetStringSlice(key(spNS, "kernel_header_dirs")),
KernelHeadersDownloadDir: cfg.GetString(key(spNS, "kernel_header_download_dir")),
AptConfigDir: cfg.GetString(key(spNS, "apt_config_dir")),
YumReposDir: cfg.GetString(key(spNS, "yum_repos_dir")),
ZypperReposDir: cfg.GetString(key(spNS, "zypper_repos_dir")),
AllowPrecompiledFallback: cfg.GetBool(key(spNS, "allow_precompiled_fallback")),
AllowRuntimeCompiledFallback: cfg.GetBool(key(spNS, "allow_runtime_compiled_fallback")),

AttachKprobesWithKprobeEventsABI: cfg.GetBool(key(spNS, "attach_kprobes_with_kprobe_events_abi")),
InternalTelemetryEnabled: cfg.GetBool(sysconfig.FullKeyPath(spNS, "telemetry_enabled")),

EnableCORE: cfg.GetBool(sysconfig.FullKeyPath(spNS, "enable_co_re")),
BTFPath: cfg.GetString(sysconfig.FullKeyPath(spNS, "btf_path")),

EnableRuntimeCompiler: cfg.GetBool(sysconfig.FullKeyPath(spNS, "enable_runtime_compiler")),
RuntimeCompilerOutputDir: cfg.GetString(sysconfig.FullKeyPath(spNS, "runtime_compiler_output_dir")),
EnableKernelHeaderDownload: cfg.GetBool(sysconfig.FullKeyPath(spNS, "enable_kernel_header_download")),
KernelHeadersDirs: cfg.GetStringSlice(sysconfig.FullKeyPath(spNS, "kernel_header_dirs")),
KernelHeadersDownloadDir: cfg.GetString(sysconfig.FullKeyPath(spNS, "kernel_header_download_dir")),
AptConfigDir: cfg.GetString(sysconfig.FullKeyPath(spNS, "apt_config_dir")),
YumReposDir: cfg.GetString(sysconfig.FullKeyPath(spNS, "yum_repos_dir")),
ZypperReposDir: cfg.GetString(sysconfig.FullKeyPath(spNS, "zypper_repos_dir")),
AllowPrecompiledFallback: cfg.GetBool(sysconfig.FullKeyPath(spNS, "allow_precompiled_fallback")),
AllowRuntimeCompiledFallback: cfg.GetBool(sysconfig.FullKeyPath(spNS, "allow_runtime_compiled_fallback")),

AttachKprobesWithKprobeEventsABI: cfg.GetBool(sysconfig.FullKeyPath(spNS, "attach_kprobes_with_kprobe_events_abi")),
}

return c
Expand Down
15 changes: 5 additions & 10 deletions pkg/eventmonitor/config/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,7 @@
package config

import (
"strings"

sysconfig "github.com/DataDog/datadog-agent/cmd/system-probe/config"
pkgconfigsetup "github.com/DataDog/datadog-agent/pkg/config/setup"
"github.com/DataDog/datadog-agent/pkg/util/log"
)
Expand All @@ -34,21 +33,17 @@ type Config struct {
func NewConfig() *Config {
return &Config{
// event server
SocketPath: pkgconfigsetup.SystemProbe().GetString(join(evNS, "socket")),
EventServerBurst: pkgconfigsetup.SystemProbe().GetInt(join(evNS, "event_server.burst")),
SocketPath: pkgconfigsetup.SystemProbe().GetString(sysconfig.FullKeyPath(evNS, "socket")),
EventServerBurst: pkgconfigsetup.SystemProbe().GetInt(sysconfig.FullKeyPath(evNS, "event_server.burst")),

// consumers
ProcessConsumerEnabled: getBool("process.enabled"),
}
}

func join(pieces ...string) string {
return strings.Join(pieces, ".")
}

func getAllKeys(key string) (string, string) {
deprecatedKey := strings.Join([]string{rsNS, key}, ".")
newKey := strings.Join([]string{evNS, key}, ".")
deprecatedKey := sysconfig.FullKeyPath(rsNS, key)
newKey := sysconfig.FullKeyPath(evNS, key)
return deprecatedKey, newKey
}

Expand Down
153 changes: 74 additions & 79 deletions pkg/network/config/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@
package config

import (
"strings"
"time"

cebpf "github.com/cilium/ebpf"
Expand Down Expand Up @@ -292,10 +291,6 @@ type Config struct {
EnableUSMEventStream bool
}

func join(pieces ...string) string {
return strings.Join(pieces, ".")
}

// New creates a config for the network tracer
func New() *Config {
cfg := pkgconfigsetup.SystemProbe()
Expand All @@ -304,103 +299,103 @@ func New() *Config {
c := &Config{
Config: *ebpf.NewConfig(),

NPMEnabled: cfg.GetBool(join(netNS, "enabled")),
ServiceMonitoringEnabled: cfg.GetBool(join(smNS, "enabled")),
NPMEnabled: cfg.GetBool(sysconfig.FullKeyPath(netNS, "enabled")),
ServiceMonitoringEnabled: cfg.GetBool(sysconfig.FullKeyPath(smNS, "enabled")),

CollectTCPv4Conns: cfg.GetBool(join(netNS, "collect_tcp_v4")),
CollectTCPv6Conns: cfg.GetBool(join(netNS, "collect_tcp_v6")),
CollectTCPv4Conns: cfg.GetBool(sysconfig.FullKeyPath(netNS, "collect_tcp_v4")),
CollectTCPv6Conns: cfg.GetBool(sysconfig.FullKeyPath(netNS, "collect_tcp_v6")),
TCPConnTimeout: 2 * time.Minute,

CollectUDPv4Conns: cfg.GetBool(join(netNS, "collect_udp_v4")),
CollectUDPv6Conns: cfg.GetBool(join(netNS, "collect_udp_v6")),
CollectUDPv4Conns: cfg.GetBool(sysconfig.FullKeyPath(netNS, "collect_udp_v4")),
CollectUDPv6Conns: cfg.GetBool(sysconfig.FullKeyPath(netNS, "collect_udp_v6")),
UDPConnTimeout: defaultUDPTimeoutSeconds * time.Second,
UDPStreamTimeout: defaultUDPStreamTimeoutSeconds * time.Second,

OffsetGuessThreshold: uint64(cfg.GetInt64(join(spNS, "offset_guess_threshold"))),
ExcludedSourceConnections: cfg.GetStringMapStringSlice(join(spNS, "source_excludes")),
ExcludedDestinationConnections: cfg.GetStringMapStringSlice(join(spNS, "dest_excludes")),

TCPFailedConnectionsEnabled: cfg.GetBool(join(netNS, "enable_tcp_failed_connections")),
MaxTrackedConnections: uint32(cfg.GetInt64(join(spNS, "max_tracked_connections"))),
MaxClosedConnectionsBuffered: uint32(cfg.GetInt64(join(spNS, "max_closed_connections_buffered"))),
MaxFailedConnectionsBuffered: uint32(cfg.GetInt64(join(netNS, "max_failed_connections_buffered"))),
ClosedConnectionFlushThreshold: cfg.GetInt(join(spNS, "closed_connection_flush_threshold")),
ClosedChannelSize: cfg.GetInt(join(spNS, "closed_channel_size")),
MaxConnectionsStateBuffered: cfg.GetInt(join(spNS, "max_connection_state_buffered")),
OffsetGuessThreshold: uint64(cfg.GetInt64(sysconfig.FullKeyPath(spNS, "offset_guess_threshold"))),
ExcludedSourceConnections: cfg.GetStringMapStringSlice(sysconfig.FullKeyPath(spNS, "source_excludes")),
ExcludedDestinationConnections: cfg.GetStringMapStringSlice(sysconfig.FullKeyPath(spNS, "dest_excludes")),

TCPFailedConnectionsEnabled: cfg.GetBool(sysconfig.FullKeyPath(netNS, "enable_tcp_failed_connections")),
MaxTrackedConnections: uint32(cfg.GetInt64(sysconfig.FullKeyPath(spNS, "max_tracked_connections"))),
MaxClosedConnectionsBuffered: uint32(cfg.GetInt64(sysconfig.FullKeyPath(spNS, "max_closed_connections_buffered"))),
MaxFailedConnectionsBuffered: uint32(cfg.GetInt64(sysconfig.FullKeyPath(netNS, "max_failed_connections_buffered"))),
ClosedConnectionFlushThreshold: cfg.GetInt(sysconfig.FullKeyPath(spNS, "closed_connection_flush_threshold")),
ClosedChannelSize: cfg.GetInt(sysconfig.FullKeyPath(spNS, "closed_channel_size")),
MaxConnectionsStateBuffered: cfg.GetInt(sysconfig.FullKeyPath(spNS, "max_connection_state_buffered")),
ClientStateExpiry: 2 * time.Minute,

DNSInspection: !cfg.GetBool(join(spNS, "disable_dns_inspection")),
CollectDNSStats: cfg.GetBool(join(spNS, "collect_dns_stats")),
CollectLocalDNS: cfg.GetBool(join(spNS, "collect_local_dns")),
CollectDNSDomains: cfg.GetBool(join(spNS, "collect_dns_domains")),
MaxDNSStats: cfg.GetInt(join(spNS, "max_dns_stats")),
DNSInspection: !cfg.GetBool(sysconfig.FullKeyPath(spNS, "disable_dns_inspection")),
CollectDNSStats: cfg.GetBool(sysconfig.FullKeyPath(spNS, "collect_dns_stats")),
CollectLocalDNS: cfg.GetBool(sysconfig.FullKeyPath(spNS, "collect_local_dns")),
CollectDNSDomains: cfg.GetBool(sysconfig.FullKeyPath(spNS, "collect_dns_domains")),
MaxDNSStats: cfg.GetInt(sysconfig.FullKeyPath(spNS, "max_dns_stats")),
MaxDNSStatsBuffered: 75000,
DNSTimeout: time.Duration(cfg.GetInt(join(spNS, "dns_timeout_in_s"))) * time.Second,

ProtocolClassificationEnabled: cfg.GetBool(join(netNS, "enable_protocol_classification")),

NPMRingbuffersEnabled: cfg.GetBool(join(netNS, "enable_ringbuffers")),

EnableHTTPMonitoring: cfg.GetBool(join(smNS, "enable_http_monitoring")),
EnableHTTP2Monitoring: cfg.GetBool(join(smNS, "enable_http2_monitoring")),
EnableKafkaMonitoring: cfg.GetBool(join(smNS, "enable_kafka_monitoring")),
EnablePostgresMonitoring: cfg.GetBool(join(smNS, "enable_postgres_monitoring")),
EnableRedisMonitoring: cfg.GetBool(join(smNS, "enable_redis_monitoring")),
EnableNativeTLSMonitoring: cfg.GetBool(join(smNS, "tls", "native", "enabled")),
EnableIstioMonitoring: cfg.GetBool(join(smNS, "tls", "istio", "enabled")),
EnvoyPath: cfg.GetString(join(smNS, "tls", "istio", "envoy_path")),
EnableNodeJSMonitoring: cfg.GetBool(join(smNS, "tls", "nodejs", "enabled")),
MaxUSMConcurrentRequests: uint32(cfg.GetInt(join(smNS, "max_concurrent_requests"))),
MaxHTTPStatsBuffered: cfg.GetInt(join(smNS, "max_http_stats_buffered")),
MaxKafkaStatsBuffered: cfg.GetInt(join(smNS, "max_kafka_stats_buffered")),
MaxPostgresStatsBuffered: cfg.GetInt(join(smNS, "max_postgres_stats_buffered")),
MaxPostgresTelemetryBuffer: cfg.GetInt(join(smNS, "max_postgres_telemetry_buffer")),
MaxRedisStatsBuffered: cfg.GetInt(join(smNS, "max_redis_stats_buffered")),

MaxTrackedHTTPConnections: cfg.GetInt64(join(smNS, "max_tracked_http_connections")),
HTTPNotificationThreshold: cfg.GetInt64(join(smNS, "http_notification_threshold")),
HTTPMaxRequestFragment: cfg.GetInt64(join(smNS, "http_max_request_fragment")),

EnableConntrack: cfg.GetBool(join(spNS, "enable_conntrack")),
ConntrackMaxStateSize: cfg.GetInt(join(spNS, "conntrack_max_state_size")),
ConntrackRateLimit: cfg.GetInt(join(spNS, "conntrack_rate_limit")),
DNSTimeout: time.Duration(cfg.GetInt(sysconfig.FullKeyPath(spNS, "dns_timeout_in_s"))) * time.Second,

ProtocolClassificationEnabled: cfg.GetBool(sysconfig.FullKeyPath(netNS, "enable_protocol_classification")),

NPMRingbuffersEnabled: cfg.GetBool(sysconfig.FullKeyPath(netNS, "enable_ringbuffers")),

EnableHTTPMonitoring: cfg.GetBool(sysconfig.FullKeyPath(smNS, "enable_http_monitoring")),
EnableHTTP2Monitoring: cfg.GetBool(sysconfig.FullKeyPath(smNS, "enable_http2_monitoring")),
EnableKafkaMonitoring: cfg.GetBool(sysconfig.FullKeyPath(smNS, "enable_kafka_monitoring")),
EnablePostgresMonitoring: cfg.GetBool(sysconfig.FullKeyPath(smNS, "enable_postgres_monitoring")),
EnableRedisMonitoring: cfg.GetBool(sysconfig.FullKeyPath(smNS, "enable_redis_monitoring")),
EnableNativeTLSMonitoring: cfg.GetBool(sysconfig.FullKeyPath(smNS, "tls", "native", "enabled")),
EnableIstioMonitoring: cfg.GetBool(sysconfig.FullKeyPath(smNS, "tls", "istio", "enabled")),
EnvoyPath: cfg.GetString(sysconfig.FullKeyPath(smNS, "tls", "istio", "envoy_path")),
EnableNodeJSMonitoring: cfg.GetBool(sysconfig.FullKeyPath(smNS, "tls", "nodejs", "enabled")),
MaxUSMConcurrentRequests: uint32(cfg.GetInt(sysconfig.FullKeyPath(smNS, "max_concurrent_requests"))),
MaxHTTPStatsBuffered: cfg.GetInt(sysconfig.FullKeyPath(smNS, "max_http_stats_buffered")),
MaxKafkaStatsBuffered: cfg.GetInt(sysconfig.FullKeyPath(smNS, "max_kafka_stats_buffered")),
MaxPostgresStatsBuffered: cfg.GetInt(sysconfig.FullKeyPath(smNS, "max_postgres_stats_buffered")),
MaxPostgresTelemetryBuffer: cfg.GetInt(sysconfig.FullKeyPath(smNS, "max_postgres_telemetry_buffer")),
MaxRedisStatsBuffered: cfg.GetInt(sysconfig.FullKeyPath(smNS, "max_redis_stats_buffered")),

MaxTrackedHTTPConnections: cfg.GetInt64(sysconfig.FullKeyPath(smNS, "max_tracked_http_connections")),
HTTPNotificationThreshold: cfg.GetInt64(sysconfig.FullKeyPath(smNS, "http_notification_threshold")),
HTTPMaxRequestFragment: cfg.GetInt64(sysconfig.FullKeyPath(smNS, "http_max_request_fragment")),

EnableConntrack: cfg.GetBool(sysconfig.FullKeyPath(spNS, "enable_conntrack")),
ConntrackMaxStateSize: cfg.GetInt(sysconfig.FullKeyPath(spNS, "conntrack_max_state_size")),
ConntrackRateLimit: cfg.GetInt(sysconfig.FullKeyPath(spNS, "conntrack_rate_limit")),
ConntrackRateLimitInterval: 3 * time.Second,
EnableConntrackAllNamespaces: cfg.GetBool(join(spNS, "enable_conntrack_all_namespaces")),
IgnoreConntrackInitFailure: cfg.GetBool(join(netNS, "ignore_conntrack_init_failure")),
ConntrackInitTimeout: cfg.GetDuration(join(netNS, "conntrack_init_timeout")),
EnableEbpfConntracker: cfg.GetBool(join(netNS, "enable_ebpf_conntracker")),
EnableConntrackAllNamespaces: cfg.GetBool(sysconfig.FullKeyPath(spNS, "enable_conntrack_all_namespaces")),
IgnoreConntrackInitFailure: cfg.GetBool(sysconfig.FullKeyPath(netNS, "ignore_conntrack_init_failure")),
ConntrackInitTimeout: cfg.GetDuration(sysconfig.FullKeyPath(netNS, "conntrack_init_timeout")),
EnableEbpfConntracker: cfg.GetBool(sysconfig.FullKeyPath(netNS, "enable_ebpf_conntracker")),

EnableGatewayLookup: cfg.GetBool(join(netNS, "enable_gateway_lookup")),
EnableGatewayLookup: cfg.GetBool(sysconfig.FullKeyPath(netNS, "enable_gateway_lookup")),

EnableMonotonicCount: cfg.GetBool(join(spNS, "windows.enable_monotonic_count")),
EnableMonotonicCount: cfg.GetBool(sysconfig.FullKeyPath(spNS, "windows.enable_monotonic_count")),

RecordedQueryTypes: cfg.GetStringSlice(join(netNS, "dns_recorded_query_types")),
RecordedQueryTypes: cfg.GetStringSlice(sysconfig.FullKeyPath(netNS, "dns_recorded_query_types")),

EnableProcessEventMonitoring: cfg.GetBool(join(evNS, "network_process", "enabled")),
MaxProcessesTracked: cfg.GetInt(join(evNS, "network_process", "max_processes_tracked")),
EnableProcessEventMonitoring: cfg.GetBool(sysconfig.FullKeyPath(evNS, "network_process", "enabled")),
MaxProcessesTracked: cfg.GetInt(sysconfig.FullKeyPath(evNS, "network_process", "max_processes_tracked")),

EnableRootNetNs: cfg.GetBool(join(netNS, "enable_root_netns")),
EnableRootNetNs: cfg.GetBool(sysconfig.FullKeyPath(netNS, "enable_root_netns")),

HTTP2DynamicTableMapCleanerInterval: time.Duration(cfg.GetInt(join(smNS, "http2_dynamic_table_map_cleaner_interval_seconds"))) * time.Second,
HTTP2DynamicTableMapCleanerInterval: time.Duration(cfg.GetInt(sysconfig.FullKeyPath(smNS, "http2_dynamic_table_map_cleaner_interval_seconds"))) * time.Second,

HTTPMapCleanerInterval: time.Duration(cfg.GetInt(join(smNS, "http_map_cleaner_interval_in_s"))) * time.Second,
HTTPIdleConnectionTTL: time.Duration(cfg.GetInt(join(smNS, "http_idle_connection_ttl_in_s"))) * time.Second,
HTTPMapCleanerInterval: time.Duration(cfg.GetInt(sysconfig.FullKeyPath(smNS, "http_map_cleaner_interval_in_s"))) * time.Second,
HTTPIdleConnectionTTL: time.Duration(cfg.GetInt(sysconfig.FullKeyPath(smNS, "http_idle_connection_ttl_in_s"))) * time.Second,

EnableNPMConnectionRollup: cfg.GetBool(join(netNS, "enable_connection_rollup")),
EnableNPMConnectionRollup: cfg.GetBool(sysconfig.FullKeyPath(netNS, "enable_connection_rollup")),

EnableEbpfless: cfg.GetBool(join(netNS, "enable_ebpfless")),
EnableEbpfless: cfg.GetBool(sysconfig.FullKeyPath(netNS, "enable_ebpfless")),

// Service Monitoring
EnableGoTLSSupport: cfg.GetBool(join(smNS, "tls", "go", "enabled")),
GoTLSExcludeSelf: cfg.GetBool(join(smNS, "tls", "go", "exclude_self")),
EnableHTTPStatsByStatusCode: cfg.GetBool(join(smNS, "enable_http_stats_by_status_code")),
EnableUSMQuantization: cfg.GetBool(join(smNS, "enable_quantization")),
EnableUSMConnectionRollup: cfg.GetBool(join(smNS, "enable_connection_rollup")),
EnableUSMRingBuffers: cfg.GetBool(join(smNS, "enable_ring_buffers")),
EnableUSMEventStream: cfg.GetBool(join(smNS, "enable_event_stream")),
EnableGoTLSSupport: cfg.GetBool(sysconfig.FullKeyPath(smNS, "tls", "go", "enabled")),
GoTLSExcludeSelf: cfg.GetBool(sysconfig.FullKeyPath(smNS, "tls", "go", "exclude_self")),
EnableHTTPStatsByStatusCode: cfg.GetBool(sysconfig.FullKeyPath(smNS, "enable_http_stats_by_status_code")),
EnableUSMQuantization: cfg.GetBool(sysconfig.FullKeyPath(smNS, "enable_quantization")),
EnableUSMConnectionRollup: cfg.GetBool(sysconfig.FullKeyPath(smNS, "enable_connection_rollup")),
EnableUSMRingBuffers: cfg.GetBool(sysconfig.FullKeyPath(smNS, "enable_ring_buffers")),
EnableUSMEventStream: cfg.GetBool(sysconfig.FullKeyPath(smNS, "enable_event_stream")),
}

httpRRKey := join(smNS, "http_replace_rules")
httpRRKey := sysconfig.FullKeyPath(smNS, "http_replace_rules")
rr, err := parseReplaceRules(cfg, httpRRKey)
if err != nil {
log.Errorf("error parsing %q: %v", httpRRKey, err)
Expand Down

0 comments on commit ec63d4e

Please sign in to comment.