From 2f87a1d1c90e51679838b36609e07cd3acdc9bcf Mon Sep 17 00:00:00 2001 From: Xuan <112967240+xuan-cao-swi@users.noreply.github.com> Date: Fri, 23 Aug 2024 16:52:31 -0400 Subject: [PATCH 01/13] feat: initialize metrics exporter http gem (#1670) * feat: initialize metrics exporter http gem * remove version_rb_path for metrics exporter in release * lint * chore: Update metrics exporter README This removes content related to installing the gem before the metrics API and SDK gems were released. * chore: Update MetricsExporter class in example Now, there's an extra Metrics:: namespace * chore: Update class for MetricsExporter * revision * add force_flush --------- Co-authored-by: Kayla Reopelle --- .toys/.data/releases.yml | 4 + examples/metrics_sdk/metrics_collect_otlp.rb | 2 +- exporter/otlp-metrics/README.md | 46 +-- .../exporter/otlp/metrics/metrics_exporter.rb | 311 ++++++++++++++++++ .../exporter/otlp/metrics/util.rb | 141 ++++++++ .../exporter/otlp/{ => metrics}/version.rb | 6 +- .../exporter/otlp/metrics_exporter.rb | 309 ----------------- .../lib/opentelemetry/exporter/otlp/util.rb | 139 -------- .../opentelemetry/exporter/otlp_metrics.rb | 4 +- ...pentelemetry-exporter-otlp-metrics.gemspec | 10 +- .../exporter/otlp/metrics_exporter_test.rb | 96 +++--- 11 files changed, 523 insertions(+), 545 deletions(-) create mode 100644 exporter/otlp-metrics/lib/opentelemetry/exporter/otlp/metrics/metrics_exporter.rb create mode 100644 exporter/otlp-metrics/lib/opentelemetry/exporter/otlp/metrics/util.rb rename exporter/otlp-metrics/lib/opentelemetry/exporter/otlp/{ => metrics}/version.rb (62%) delete mode 100644 exporter/otlp-metrics/lib/opentelemetry/exporter/otlp/metrics_exporter.rb delete mode 100644 exporter/otlp-metrics/lib/opentelemetry/exporter/otlp/util.rb diff --git a/.toys/.data/releases.yml b/.toys/.data/releases.yml index 95c53559b2..a63f0be44f 100644 --- a/.toys/.data/releases.yml +++ b/.toys/.data/releases.yml @@ -63,6 +63,10 @@ gems: directory: exporter/otlp version_constant: [OpenTelemetry, Exporter, OTLP, VERSION] + - name: opentelemetry-exporter-otlp-metrics + directory: exporter/otlp-metrics + version_constant: [OpenTelemetry, Exporter, OTLP, Metrics, VERSION] + - name: opentelemetry-exporter-zipkin directory: exporter/zipkin version_constant: [OpenTelemetry, Exporter, Zipkin, VERSION] diff --git a/examples/metrics_sdk/metrics_collect_otlp.rb b/examples/metrics_sdk/metrics_collect_otlp.rb index d8b727a863..355c42d655 100644 --- a/examples/metrics_sdk/metrics_collect_otlp.rb +++ b/examples/metrics_sdk/metrics_collect_otlp.rb @@ -20,7 +20,7 @@ OpenTelemetry::SDK.configure -otlp_metric_exporter = OpenTelemetry::Exporter::OTLP::MetricsExporter.new +otlp_metric_exporter = OpenTelemetry::Exporter::OTLP::Metrics::MetricsExporter.new OpenTelemetry.meter_provider.add_metric_reader(otlp_metric_exporter) diff --git a/exporter/otlp-metrics/README.md b/exporter/otlp-metrics/README.md index 559cfe5285..3bbca793dd 100644 --- a/exporter/otlp-metrics/README.md +++ b/exporter/otlp-metrics/README.md @@ -18,41 +18,6 @@ Generally, *libraries* that produce telemetry data should avoid depending direct This gem supports the [v0.20.0 release][otel-proto-release] of OTLP. -## Prerequisite - -The exporter-oltp-metrics depends on two gems that have not been officially released: opentelemetry-metrics-sdk and opentelemetry-metrics-api. - -Within the .gemspec file, these gems are not listed as dependencies. However, for users who need utilize this metrics exporter, they must first install and load these two gems before they can use the exporter. - -To facilitate this, there are couple recommended approaches: - -#### 1. Download the source code - -1. Download the [opentelemetry-ruby](https://github.com/open-telemetry/opentelemetry-ruby). -2. Navigate to subfolder, then build the [metrics_sdk](https://github.com/open-telemetry/opentelemetry-ruby/tree/main/metrics_sdk) and [metrics_api](https://github.com/open-telemetry/opentelemetry-ruby/tree/main/metrics_api). -3. Execute `gem build *.gemspec`. -4. Lastly, install the built gem into the system. - -#### 2. Using `path:` option in Gemfile with downloaded source code - -git clone [opentelemetry-ruby](https://github.com/open-telemetry/opentelemetry-ruby) first, then use Gemfile - -```ruby -# Gemfile -source 'https://rubygems.org' -gem 'opentelemetry-metrics-api', path: "opentelemetry-ruby/metrics_api" -gem 'opentelemetry-metrics-sdk', path: "opentelemetry-ruby/metrics_sdk" -``` - -#### 3. Using `git:` option in Gemfile - -```ruby -# Gemfile -source 'https://rubygems.org' -gem 'opentelemetry-metrics-api', git: "https://github.com/open-telemetry/opentelemetry-ruby", glob: 'metrics_api/*.gemspec' -gem 'opentelemetry-metrics-sdk', git: "https://github.com/open-telemetry/opentelemetry-ruby", glob: 'metrics_sdk/*.gemspec' -``` - ## How do I get started? Install the gem using: @@ -60,13 +25,14 @@ Install the gem using: ```console gem install opentelemetry-sdk +gem install opentelemetry-metrics-sdk gem install opentelemetry-exporter-otlp-metrics ``` -Or, if you use [bundler][bundler-home], include `opentelemetry-sdk` in your `Gemfile`. +Or, if you use [bundler][bundler-home], include `opentelemetry-sdk`, `opentelemetry-metrics-sdk`, and `opentelemetry-exporter-otlp-metrics` in your `Gemfile`. -Then, configure the SDK to use the OTLP metrics exporter +Then, configure the SDK to use the OTLP metrics exporter ```ruby require 'opentelemetry/sdk' @@ -77,7 +43,7 @@ OpenTelemetry::SDK.configure # To start a trace you need to get a Tracer from the TracerProvider -otlp_metric_exporter = OpenTelemetry::Exporter::OTLP::MetricsExporter.new +otlp_metric_exporter = OpenTelemetry::Exporter::OTLP::Metrics::MetricsExporter.new OpenTelemetry.meter_provider.add_metric_reader(otlp_metric_exporter) @@ -112,7 +78,7 @@ The collector exporter can be configured explicitly in code, or via environment ## How can I get involved? -The `opentelemetry-exporter-otlp-metrics` gem source is [on github][repo-github], along with related gems including `opentelemetry-sdk`. +The `opentelemetry-exporter-otlp-metrics` gem source is [on github][repo-github], along with related gems including `opentelemetry-metrics-sdk`. The OpenTelemetry Ruby gems are maintained by the OpenTelemetry-Ruby special interest group (SIG). You can get involved by joining us in [GitHub Discussions][discussions-url] or attending our weekly meeting. See the [meeting calendar][community-meetings] for dates and times. For more information on this and other language SIGs, see the OpenTelemetry [community page][ruby-sig]. @@ -176,4 +142,4 @@ $> bundle exec rake test [protoc-install]: https://github.com/protocolbuffers/protobuf/releases/tag/v22.5 [ruby-downloads]: https://www.ruby-lang.org/en/downloads/ [otel-proto-github]: https://github.com/open-telemetry/opentelemetry-proto -[otel-proto-release]: https://github.com/open-telemetry/opentelemetry-proto/releases/tag/v0.20.0 \ No newline at end of file +[otel-proto-release]: https://github.com/open-telemetry/opentelemetry-proto/releases/tag/v0.20.0 diff --git a/exporter/otlp-metrics/lib/opentelemetry/exporter/otlp/metrics/metrics_exporter.rb b/exporter/otlp-metrics/lib/opentelemetry/exporter/otlp/metrics/metrics_exporter.rb new file mode 100644 index 0000000000..be21d5dde2 --- /dev/null +++ b/exporter/otlp-metrics/lib/opentelemetry/exporter/otlp/metrics/metrics_exporter.rb @@ -0,0 +1,311 @@ +# frozen_string_literal: true + +# Copyright The OpenTelemetry Authors +# +# SPDX-License-Identifier: Apache-2.0 + +require 'opentelemetry/common' +require 'opentelemetry/sdk' +require 'net/http' +require 'zlib' + +require 'google/rpc/status_pb' + +require 'opentelemetry/proto/common/v1/common_pb' +require 'opentelemetry/proto/resource/v1/resource_pb' +require 'opentelemetry/proto/metrics/v1/metrics_pb' +require 'opentelemetry/proto/collector/metrics/v1/metrics_service_pb' + +require 'opentelemetry/metrics' +require 'opentelemetry/sdk/metrics' + +require_relative './util' + +module OpenTelemetry + module Exporter + module OTLP + module Metrics + # An OpenTelemetry metrics exporter that sends metrics over HTTP as Protobuf encoded OTLP ExportMetricsServiceRequest. + class MetricsExporter < ::OpenTelemetry::SDK::Metrics::Export::MetricReader # rubocop:disable Metrics/ClassLength + include Util + + attr_reader :metric_snapshots + + SUCCESS = OpenTelemetry::SDK::Metrics::Export::SUCCESS + FAILURE = OpenTelemetry::SDK::Metrics::Export::FAILURE + private_constant(:SUCCESS, :FAILURE) + + def self.ssl_verify_mode + if ENV.key?('OTEL_RUBY_EXPORTER_OTLP_SSL_VERIFY_PEER') + OpenSSL::SSL::VERIFY_PEER + elsif ENV.key?('OTEL_RUBY_EXPORTER_OTLP_SSL_VERIFY_NONE') + OpenSSL::SSL::VERIFY_NONE + else + OpenSSL::SSL::VERIFY_PEER + end + end + + def initialize(endpoint: OpenTelemetry::Common::Utilities.config_opt('OTEL_EXPORTER_OTLP_METRICS_ENDPOINT', 'OTEL_EXPORTER_OTLP_ENDPOINT', default: 'http://localhost:4318/v1/metrics'), + certificate_file: OpenTelemetry::Common::Utilities.config_opt('OTEL_EXPORTER_OTLP_METRICS_CERTIFICATE', 'OTEL_EXPORTER_OTLP_CERTIFICATE'), + ssl_verify_mode: MetricsExporter.ssl_verify_mode, + headers: OpenTelemetry::Common::Utilities.config_opt('OTEL_EXPORTER_OTLP_METRICS_HEADERS', 'OTEL_EXPORTER_OTLP_HEADERS', default: {}), + compression: OpenTelemetry::Common::Utilities.config_opt('OTEL_EXPORTER_OTLP_METRICS_COMPRESSION', 'OTEL_EXPORTER_OTLP_COMPRESSION', default: 'gzip'), + timeout: OpenTelemetry::Common::Utilities.config_opt('OTEL_EXPORTER_OTLP_METRICS_TIMEOUT', 'OTEL_EXPORTER_OTLP_TIMEOUT', default: 10)) + raise ArgumentError, "invalid url for OTLP::MetricsExporter #{endpoint}" unless OpenTelemetry::Common::Utilities.valid_url?(endpoint) + raise ArgumentError, "unsupported compression key #{compression}" unless compression.nil? || %w[gzip none].include?(compression) + + # create the MetricStore object + super() + + @uri = if endpoint == ENV['OTEL_EXPORTER_OTLP_ENDPOINT'] + URI.join(endpoint, 'v1/metrics') + else + URI(endpoint) + end + + @http = http_connection(@uri, ssl_verify_mode, certificate_file) + + @path = @uri.path + @headers = prepare_headers(headers) + @timeout = timeout.to_f + @compression = compression + @mutex = Mutex.new + @shutdown = false + end + + # consolidate the metrics data into the form of MetricData + # + # return MetricData + def pull + export(collect) + end + + # metrics Array[MetricData] + def export(metrics, timeout: nil) + @mutex.synchronize do + send_bytes(encode(metrics), timeout: timeout) + end + end + + def send_bytes(bytes, timeout:) # rubocop:disable Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity + return FAILURE if bytes.nil? + + request = Net::HTTP::Post.new(@path) + + if @compression == 'gzip' + request.add_field('Content-Encoding', 'gzip') + body = Zlib.gzip(bytes) + else + body = bytes + end + + request.body = body + request.add_field('Content-Type', 'application/x-protobuf') + @headers.each { |key, value| request.add_field(key, value) } + + retry_count = 0 + timeout ||= @timeout + start_time = OpenTelemetry::Common::Utilities.timeout_timestamp + + around_request do + remaining_timeout = OpenTelemetry::Common::Utilities.maybe_timeout(timeout, start_time) + return FAILURE if remaining_timeout.zero? + + @http.open_timeout = remaining_timeout + @http.read_timeout = remaining_timeout + @http.write_timeout = remaining_timeout + @http.start unless @http.started? + response = measure_request_duration { @http.request(request) } + case response + when Net::HTTPOK + response.body # Read and discard body + SUCCESS + when Net::HTTPServiceUnavailable, Net::HTTPTooManyRequests + response.body # Read and discard body + redo if backoff?(retry_after: response['Retry-After'], retry_count: retry_count += 1, reason: response.code) + OpenTelemetry.logger.warn('Net::HTTPServiceUnavailable/Net::HTTPTooManyRequests in MetricsExporter#send_bytes') + FAILURE + when Net::HTTPRequestTimeOut, Net::HTTPGatewayTimeOut, Net::HTTPBadGateway + response.body # Read and discard body + redo if backoff?(retry_count: retry_count += 1, reason: response.code) + OpenTelemetry.logger.warn('Net::HTTPRequestTimeOut/Net::HTTPGatewayTimeOut/Net::HTTPBadGateway in MetricsExporter#send_bytes') + FAILURE + when Net::HTTPNotFound + OpenTelemetry.handle_error(message: "OTLP metrics_exporter received http.code=404 for uri: '#{@path}'") + FAILURE + when Net::HTTPBadRequest, Net::HTTPClientError, Net::HTTPServerError + log_status(response.body) + OpenTelemetry.logger.warn('Net::HTTPBadRequest/Net::HTTPClientError/Net::HTTPServerError in MetricsExporter#send_bytes') + FAILURE + when Net::HTTPRedirection + @http.finish + handle_redirect(response['location']) + redo if backoff?(retry_after: 0, retry_count: retry_count += 1, reason: response.code) + else + @http.finish + OpenTelemetry.logger.warn("Unexpected error in OTLP::MetricsExporter#send_bytes - #{response.message}") + FAILURE + end + rescue Net::OpenTimeout, Net::ReadTimeout + retry if backoff?(retry_count: retry_count += 1, reason: 'timeout') + OpenTelemetry.logger.warn('Net::OpenTimeout/Net::ReadTimeout in MetricsExporter#send_bytes') + return FAILURE + rescue OpenSSL::SSL::SSLError + retry if backoff?(retry_count: retry_count += 1, reason: 'openssl_error') + OpenTelemetry.logger.warn('OpenSSL::SSL::SSLError in MetricsExporter#send_bytes') + return FAILURE + rescue SocketError + retry if backoff?(retry_count: retry_count += 1, reason: 'socket_error') + OpenTelemetry.logger.warn('SocketError in MetricsExporter#send_bytes') + return FAILURE + rescue SystemCallError => e + retry if backoff?(retry_count: retry_count += 1, reason: e.class.name) + OpenTelemetry.logger.warn('SystemCallError in MetricsExporter#send_bytes') + return FAILURE + rescue EOFError + retry if backoff?(retry_count: retry_count += 1, reason: 'eof_error') + OpenTelemetry.logger.warn('EOFError in MetricsExporter#send_bytes') + return FAILURE + rescue Zlib::DataError + retry if backoff?(retry_count: retry_count += 1, reason: 'zlib_error') + OpenTelemetry.logger.warn('Zlib::DataError in MetricsExporter#send_bytes') + return FAILURE + rescue StandardError => e + OpenTelemetry.handle_error(exception: e, message: 'unexpected error in OTLP::MetricsExporter#send_bytes') + return FAILURE + end + ensure + # Reset timeouts to defaults for the next call. + @http.open_timeout = @timeout + @http.read_timeout = @timeout + @http.write_timeout = @timeout + end + + def encode(metrics_data) # rubocop:disable Metrics/MethodLength, Metrics/CyclomaticComplexity + Opentelemetry::Proto::Collector::Metrics::V1::ExportMetricsServiceRequest.encode( + Opentelemetry::Proto::Collector::Metrics::V1::ExportMetricsServiceRequest.new( + resource_metrics: metrics_data + .group_by(&:resource) + .map do |resource, scope_metrics| + Opentelemetry::Proto::Metrics::V1::ResourceMetrics.new( + resource: Opentelemetry::Proto::Resource::V1::Resource.new( + attributes: resource.attribute_enumerator.map { |key, value| as_otlp_key_value(key, value) } + ), + scope_metrics: scope_metrics + .group_by(&:instrumentation_scope) + .map do |instrumentation_scope, metrics| + Opentelemetry::Proto::Metrics::V1::ScopeMetrics.new( + scope: Opentelemetry::Proto::Common::V1::InstrumentationScope.new( + name: instrumentation_scope.name, + version: instrumentation_scope.version + ), + metrics: metrics.map { |sd| as_otlp_metrics(sd) } + ) + end + ) + end + ) + ) + rescue StandardError => e + OpenTelemetry.handle_error(exception: e, message: 'unexpected error in OTLP::MetricsExporter#encode') + nil + end + + # metrics_pb has following type of data: :gauge, :sum, :histogram, :exponential_histogram, :summary + # current metric sdk only implements instrument: :counter -> :sum, :histogram -> :histogram + # + # metrics [MetricData] + def as_otlp_metrics(metrics) # rubocop:disable Metrics/MethodLength + case metrics.instrument_kind + when :observable_gauge + Opentelemetry::Proto::Metrics::V1::Metric.new( + name: metrics.name, + description: metrics.description, + unit: metrics.unit, + gauge: Opentelemetry::Proto::Metrics::V1::Gauge.new( + aggregation_temporality: as_otlp_aggregation_temporality(metrics.aggregation_temporality), + data_points: metrics.data_points.map do |ndp| + number_data_point(ndp) + end + ) + ) + + when :counter, :up_down_counter + Opentelemetry::Proto::Metrics::V1::Metric.new( + name: metrics.name, + description: metrics.description, + unit: metrics.unit, + sum: Opentelemetry::Proto::Metrics::V1::Sum.new( + aggregation_temporality: as_otlp_aggregation_temporality(metrics.aggregation_temporality), + data_points: metrics.data_points.map do |ndp| + number_data_point(ndp) + end + ) + ) + + when :histogram + Opentelemetry::Proto::Metrics::V1::Metric.new( + name: metrics.name, + description: metrics.description, + unit: metrics.unit, + histogram: Opentelemetry::Proto::Metrics::V1::Histogram.new( + aggregation_temporality: as_otlp_aggregation_temporality(metrics.aggregation_temporality), + data_points: metrics.data_points.map do |hdp| + histogram_data_point(hdp) + end + ) + ) + end + end + + def as_otlp_aggregation_temporality(type) + case type + when :delta then Opentelemetry::Proto::Metrics::V1::AggregationTemporality::AGGREGATION_TEMPORALITY_DELTA + when :cumulative then Opentelemetry::Proto::Metrics::V1::AggregationTemporality::AGGREGATION_TEMPORALITY_CUMULATIVE + else Opentelemetry::Proto::Metrics::V1::AggregationTemporality::AGGREGATION_TEMPORALITY_UNSPECIFIED + end + end + + def histogram_data_point(hdp) + Opentelemetry::Proto::Metrics::V1::HistogramDataPoint.new( + attributes: hdp.attributes.map { |k, v| as_otlp_key_value(k, v) }, + start_time_unix_nano: hdp.start_time_unix_nano, + time_unix_nano: hdp.time_unix_nano, + count: hdp.count, + sum: hdp.sum, + bucket_counts: hdp.bucket_counts, + explicit_bounds: hdp.explicit_bounds, + exemplars: hdp.exemplars, + min: hdp.min, + max: hdp.max + ) + end + + def number_data_point(ndp) + Opentelemetry::Proto::Metrics::V1::NumberDataPoint.new( + attributes: ndp.attributes.map { |k, v| as_otlp_key_value(k, v) }, + as_int: ndp.value, + start_time_unix_nano: ndp.start_time_unix_nano, + time_unix_nano: ndp.time_unix_nano, + exemplars: ndp.exemplars # exemplars not implemented yet from metrics sdk + ) + end + + # may not need this + def reset + SUCCESS + end + + def force_flush(timeout: nil) + SUCCESS + end + + def shutdown(timeout: nil) + @shutdown = true + SUCCESS + end + end + end + end + end +end diff --git a/exporter/otlp-metrics/lib/opentelemetry/exporter/otlp/metrics/util.rb b/exporter/otlp-metrics/lib/opentelemetry/exporter/otlp/metrics/util.rb new file mode 100644 index 0000000000..568c9a0e48 --- /dev/null +++ b/exporter/otlp-metrics/lib/opentelemetry/exporter/otlp/metrics/util.rb @@ -0,0 +1,141 @@ +# frozen_string_literal: true + +# Copyright The OpenTelemetry Authors +# +# SPDX-License-Identifier: Apache-2.0 + +module OpenTelemetry + module Exporter + module OTLP + module Metrics + # Util module provide essential functionality for exporter + module Util # rubocop:disable Metrics/ModuleLength + KEEP_ALIVE_TIMEOUT = 30 + RETRY_COUNT = 5 + ERROR_MESSAGE_INVALID_HEADERS = 'headers must be a String with comma-separated URL Encoded UTF-8 k=v pairs or a Hash' + DEFAULT_USER_AGENT = "OTel-OTLP-MetricsExporter-Ruby/#{OpenTelemetry::Exporter::OTLP::Metrics::VERSION} Ruby/#{RUBY_VERSION} (#{RUBY_PLATFORM}; #{RUBY_ENGINE}/#{RUBY_ENGINE_VERSION})".freeze + + def http_connection(uri, ssl_verify_mode, certificate_file) + http = Net::HTTP.new(uri.host, uri.port) + http.use_ssl = uri.scheme == 'https' + http.verify_mode = ssl_verify_mode + http.ca_file = certificate_file unless certificate_file.nil? + http.keep_alive_timeout = KEEP_ALIVE_TIMEOUT + http + end + + def around_request + OpenTelemetry::Common::Utilities.untraced { yield } # rubocop:disable Style/ExplicitBlockArgument + end + + def as_otlp_key_value(key, value) + Opentelemetry::Proto::Common::V1::KeyValue.new(key: key, value: as_otlp_any_value(value)) + rescue Encoding::UndefinedConversionError => e + encoded_value = value.encode('UTF-8', invalid: :replace, undef: :replace, replace: '�') + OpenTelemetry.handle_error(exception: e, message: "encoding error for key #{key} and value #{encoded_value}") + Opentelemetry::Proto::Common::V1::KeyValue.new(key: key, value: as_otlp_any_value('Encoding Error')) + end + + def as_otlp_any_value(value) + result = Opentelemetry::Proto::Common::V1::AnyValue.new + case value + when String + result.string_value = value + when Integer + result.int_value = value + when Float + result.double_value = value + when true, false + result.bool_value = value + when Array + values = value.map { |element| as_otlp_any_value(element) } + result.array_value = Opentelemetry::Proto::Common::V1::ArrayValue.new(values: values) + end + result + end + + def prepare_headers(config_headers) + headers = case config_headers + when String then parse_headers(config_headers) + when Hash then config_headers.dup + else + raise ArgumentError, ERROR_MESSAGE_INVALID_HEADERS + end + + headers['User-Agent'] = "#{headers.fetch('User-Agent', '')} #{DEFAULT_USER_AGENT}".strip + + headers + end + + def measure_request_duration + start = Process.clock_gettime(Process::CLOCK_MONOTONIC) + begin + yield + ensure + stop = Process.clock_gettime(Process::CLOCK_MONOTONIC) + 1000.0 * (stop - start) + end + end + + def parse_headers(raw) + entries = raw.split(',') + raise ArgumentError, ERROR_MESSAGE_INVALID_HEADERS if entries.empty? + + entries.each_with_object({}) do |entry, headers| + k, v = entry.split('=', 2).map(&CGI.method(:unescape)) + begin + k = k.to_s.strip + v = v.to_s.strip + rescue Encoding::CompatibilityError + raise ArgumentError, ERROR_MESSAGE_INVALID_HEADERS + rescue ArgumentError => e + raise e, ERROR_MESSAGE_INVALID_HEADERS + end + raise ArgumentError, ERROR_MESSAGE_INVALID_HEADERS if k.empty? || v.empty? + + headers[k] = v + end + end + + def backoff?(retry_count:, reason:, retry_after: nil) # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity + return false if retry_count > RETRY_COUNT + + sleep_interval = nil + unless retry_after.nil? + sleep_interval = + begin + Integer(retry_after) + rescue ArgumentError + nil + end + sleep_interval ||= + begin + Time.httpdate(retry_after) - Time.now + rescue # rubocop:disable Style/RescueStandardError + nil + end + sleep_interval = nil unless sleep_interval&.positive? + end + sleep_interval ||= rand(2**retry_count) + + sleep(sleep_interval) + true + end + + def log_status(body) + status = Google::Rpc::Status.decode(body) + details = status.details.map do |detail| + klass_or_nil = ::Google::Protobuf::DescriptorPool.generated_pool.lookup(detail.type_name).msgclass + detail.unpack(klass_or_nil) if klass_or_nil + end.compact + OpenTelemetry.handle_error(message: "OTLP metrics_exporter received rpc.Status{message=#{status.message}, details=#{details}}") + rescue StandardError => e + OpenTelemetry.handle_error(exception: e, message: 'unexpected error decoding rpc.Status in OTLP::MetricsExporter#log_status') + end + + def handle_redirect(location); end + end + end + end + end +end diff --git a/exporter/otlp-metrics/lib/opentelemetry/exporter/otlp/version.rb b/exporter/otlp-metrics/lib/opentelemetry/exporter/otlp/metrics/version.rb similarity index 62% rename from exporter/otlp-metrics/lib/opentelemetry/exporter/otlp/version.rb rename to exporter/otlp-metrics/lib/opentelemetry/exporter/otlp/metrics/version.rb index a851733507..2d326a3580 100644 --- a/exporter/otlp-metrics/lib/opentelemetry/exporter/otlp/version.rb +++ b/exporter/otlp-metrics/lib/opentelemetry/exporter/otlp/metrics/version.rb @@ -7,8 +7,10 @@ module OpenTelemetry module Exporter module OTLP - ## Current OpenTelemetry OTLP exporter version - VERSION = '0.0.1' + module Metrics + ## Current OpenTelemetry OTLP exporter version + VERSION = '0.1.0' + end end end end diff --git a/exporter/otlp-metrics/lib/opentelemetry/exporter/otlp/metrics_exporter.rb b/exporter/otlp-metrics/lib/opentelemetry/exporter/otlp/metrics_exporter.rb deleted file mode 100644 index 36e70274ab..0000000000 --- a/exporter/otlp-metrics/lib/opentelemetry/exporter/otlp/metrics_exporter.rb +++ /dev/null @@ -1,309 +0,0 @@ -# frozen_string_literal: true - -# Copyright The OpenTelemetry Authors -# -# SPDX-License-Identifier: Apache-2.0 - -require 'opentelemetry/common' -require 'opentelemetry/sdk' -require 'net/http' -require 'csv' -require 'zlib' - -require 'google/rpc/status_pb' - -require 'opentelemetry/proto/common/v1/common_pb' -require 'opentelemetry/proto/resource/v1/resource_pb' -require 'opentelemetry/proto/metrics/v1/metrics_pb' -require 'opentelemetry/proto/collector/metrics/v1/metrics_service_pb' - -require 'opentelemetry/metrics' -require 'opentelemetry/sdk/metrics' - -require_relative './util' - -module OpenTelemetry - module Exporter - module OTLP - # An OpenTelemetry metrics exporter that sends metrics over HTTP as Protobuf encoded OTLP ExportMetricsServiceRequest. - class MetricsExporter < ::OpenTelemetry::SDK::Metrics::Export::MetricReader # rubocop:disable Metrics/ClassLength - include Util - - attr_reader :metric_snapshots - - SUCCESS = OpenTelemetry::SDK::Metrics::Export::SUCCESS - FAILURE = OpenTelemetry::SDK::Metrics::Export::FAILURE - private_constant(:SUCCESS, :FAILURE) - - WRITE_TIMEOUT_SUPPORTED = Gem::Version.new(RUBY_VERSION) >= Gem::Version.new('2.6') - private_constant(:WRITE_TIMEOUT_SUPPORTED) - - def self.ssl_verify_mode - if ENV.key?('OTEL_RUBY_EXPORTER_OTLP_SSL_VERIFY_PEER') - OpenSSL::SSL::VERIFY_PEER - elsif ENV.key?('OTEL_RUBY_EXPORTER_OTLP_SSL_VERIFY_NONE') - OpenSSL::SSL::VERIFY_NONE - else - OpenSSL::SSL::VERIFY_PEER - end - end - - def initialize(endpoint: OpenTelemetry::Common::Utilities.config_opt('OTEL_EXPORTER_OTLP_METRICS_ENDPOINT', 'OTEL_EXPORTER_OTLP_ENDPOINT', default: 'http://localhost:4318/v1/metrics'), - certificate_file: OpenTelemetry::Common::Utilities.config_opt('OTEL_EXPORTER_OTLP_METRICS_CERTIFICATE', 'OTEL_EXPORTER_OTLP_CERTIFICATE'), - ssl_verify_mode: MetricsExporter.ssl_verify_mode, - headers: OpenTelemetry::Common::Utilities.config_opt('OTEL_EXPORTER_OTLP_METRICS_HEADERS', 'OTEL_EXPORTER_OTLP_HEADERS', default: {}), - compression: OpenTelemetry::Common::Utilities.config_opt('OTEL_EXPORTER_OTLP_METRICS_COMPRESSION', 'OTEL_EXPORTER_OTLP_COMPRESSION', default: 'gzip'), - timeout: OpenTelemetry::Common::Utilities.config_opt('OTEL_EXPORTER_OTLP_METRICS_TIMEOUT', 'OTEL_EXPORTER_OTLP_TIMEOUT', default: 10)) - raise ArgumentError, "invalid url for OTLP::MetricsExporter #{endpoint}" unless OpenTelemetry::Common::Utilities.valid_url?(endpoint) - raise ArgumentError, "unsupported compression key #{compression}" unless compression.nil? || %w[gzip none].include?(compression) - - # create the MetricStore object - super() - - @uri = if endpoint == ENV['OTEL_EXPORTER_OTLP_ENDPOINT'] - URI.join(endpoint, 'v1/metrics') - else - URI(endpoint) - end - - @http = http_connection(@uri, ssl_verify_mode, certificate_file) - - @path = @uri.path - @headers = prepare_headers(headers) - @timeout = timeout.to_f - @compression = compression - @mutex = Mutex.new - @shutdown = false - end - - # consolidate the metrics data into the form of MetricData - # - # return MetricData - def pull - export(collect) - end - - # metrics Array[MetricData] - def export(metrics, timeout: nil) - @mutex.synchronize do - send_bytes(encode(metrics), timeout: timeout) - end - end - - def send_bytes(bytes, timeout:) # rubocop:disable Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity - return FAILURE if bytes.nil? - - request = Net::HTTP::Post.new(@path) - - if @compression == 'gzip' - request.add_field('Content-Encoding', 'gzip') - body = Zlib.gzip(bytes) - else - body = bytes - end - - request.body = body - request.add_field('Content-Type', 'application/x-protobuf') - @headers.each { |key, value| request.add_field(key, value) } - - retry_count = 0 - timeout ||= @timeout - start_time = OpenTelemetry::Common::Utilities.timeout_timestamp - - around_request do - remaining_timeout = OpenTelemetry::Common::Utilities.maybe_timeout(timeout, start_time) - return FAILURE if remaining_timeout.zero? - - @http.open_timeout = remaining_timeout - @http.read_timeout = remaining_timeout - @http.write_timeout = remaining_timeout if WRITE_TIMEOUT_SUPPORTED - @http.start unless @http.started? - response = measure_request_duration { @http.request(request) } - case response - when Net::HTTPOK - response.body # Read and discard body - SUCCESS - when Net::HTTPServiceUnavailable, Net::HTTPTooManyRequests - response.body # Read and discard body - redo if backoff?(retry_after: response['Retry-After'], retry_count: retry_count += 1, reason: response.code) - OpenTelemetry.logger.warn('Net::HTTPServiceUnavailable/Net::HTTPTooManyRequests in MetricsExporter#send_bytes') - FAILURE - when Net::HTTPRequestTimeOut, Net::HTTPGatewayTimeOut, Net::HTTPBadGateway - response.body # Read and discard body - redo if backoff?(retry_count: retry_count += 1, reason: response.code) - OpenTelemetry.logger.warn('Net::HTTPRequestTimeOut/Net::HTTPGatewayTimeOut/Net::HTTPBadGateway in MetricsExporter#send_bytes') - FAILURE - when Net::HTTPNotFound - OpenTelemetry.handle_error(message: "OTLP metrics_exporter received http.code=404 for uri: '#{@path}'") - FAILURE - when Net::HTTPBadRequest, Net::HTTPClientError, Net::HTTPServerError - log_status(response.body) - OpenTelemetry.logger.warn('Net::HTTPBadRequest/Net::HTTPClientError/Net::HTTPServerError in MetricsExporter#send_bytes') - FAILURE - when Net::HTTPRedirection - @http.finish - handle_redirect(response['location']) - redo if backoff?(retry_after: 0, retry_count: retry_count += 1, reason: response.code) - else - @http.finish - OpenTelemetry.logger.warn("Unexpected error in OTLP::MetricsExporter#send_bytes - #{response.message}") - FAILURE - end - rescue Net::OpenTimeout, Net::ReadTimeout - retry if backoff?(retry_count: retry_count += 1, reason: 'timeout') - OpenTelemetry.logger.warn('Net::OpenTimeout/Net::ReadTimeout in MetricsExporter#send_bytes') - return FAILURE - rescue OpenSSL::SSL::SSLError - retry if backoff?(retry_count: retry_count += 1, reason: 'openssl_error') - OpenTelemetry.logger.warn('OpenSSL::SSL::SSLError in MetricsExporter#send_bytes') - return FAILURE - rescue SocketError - retry if backoff?(retry_count: retry_count += 1, reason: 'socket_error') - OpenTelemetry.logger.warn('SocketError in MetricsExporter#send_bytes') - return FAILURE - rescue SystemCallError => e - retry if backoff?(retry_count: retry_count += 1, reason: e.class.name) - OpenTelemetry.logger.warn('SystemCallError in MetricsExporter#send_bytes') - return FAILURE - rescue EOFError - retry if backoff?(retry_count: retry_count += 1, reason: 'eof_error') - OpenTelemetry.logger.warn('EOFError in MetricsExporter#send_bytes') - return FAILURE - rescue Zlib::DataError - retry if backoff?(retry_count: retry_count += 1, reason: 'zlib_error') - OpenTelemetry.logger.warn('Zlib::DataError in MetricsExporter#send_bytes') - return FAILURE - rescue StandardError => e - OpenTelemetry.handle_error(exception: e, message: 'unexpected error in OTLP::MetricsExporter#send_bytes') - return FAILURE - end - ensure - # Reset timeouts to defaults for the next call. - @http.open_timeout = @timeout - @http.read_timeout = @timeout - @http.write_timeout = @timeout if WRITE_TIMEOUT_SUPPORTED - end - - def encode(metrics_data) # rubocop:disable Metrics/MethodLength, Metrics/CyclomaticComplexity - Opentelemetry::Proto::Collector::Metrics::V1::ExportMetricsServiceRequest.encode( - Opentelemetry::Proto::Collector::Metrics::V1::ExportMetricsServiceRequest.new( - resource_metrics: metrics_data - .group_by(&:resource) - .map do |resource, scope_metrics| - Opentelemetry::Proto::Metrics::V1::ResourceMetrics.new( - resource: Opentelemetry::Proto::Resource::V1::Resource.new( - attributes: resource.attribute_enumerator.map { |key, value| as_otlp_key_value(key, value) } - ), - scope_metrics: scope_metrics - .group_by(&:instrumentation_scope) - .map do |instrumentation_scope, metrics| - Opentelemetry::Proto::Metrics::V1::ScopeMetrics.new( - scope: Opentelemetry::Proto::Common::V1::InstrumentationScope.new( - name: instrumentation_scope.name, - version: instrumentation_scope.version - ), - metrics: metrics.map { |sd| as_otlp_metrics(sd) } - ) - end - ) - end - ) - ) - rescue StandardError => e - OpenTelemetry.handle_error(exception: e, message: 'unexpected error in OTLP::MetricsExporter#encode') - nil - end - - # metrics_pb has following type of data: :gauge, :sum, :histogram, :exponential_histogram, :summary - # current metric sdk only implements instrument: :counter -> :sum, :histogram -> :histogram - # - # metrics [MetricData] - def as_otlp_metrics(metrics) # rubocop:disable Metrics/MethodLength - case metrics.instrument_kind - when :observable_gauge - Opentelemetry::Proto::Metrics::V1::Metric.new( - name: metrics.name, - description: metrics.description, - unit: metrics.unit, - gauge: Opentelemetry::Proto::Metrics::V1::Gauge.new( - aggregation_temporality: as_otlp_aggregation_temporality(metrics.aggregation_temporality), - data_points: metrics.data_points.map do |ndp| - number_data_point(ndp) - end - ) - ) - - when :counter, :up_down_counter - Opentelemetry::Proto::Metrics::V1::Metric.new( - name: metrics.name, - description: metrics.description, - unit: metrics.unit, - sum: Opentelemetry::Proto::Metrics::V1::Sum.new( - aggregation_temporality: as_otlp_aggregation_temporality(metrics.aggregation_temporality), - data_points: metrics.data_points.map do |ndp| - number_data_point(ndp) - end - ) - ) - - when :histogram - Opentelemetry::Proto::Metrics::V1::Metric.new( - name: metrics.name, - description: metrics.description, - unit: metrics.unit, - histogram: Opentelemetry::Proto::Metrics::V1::Histogram.new( - aggregation_temporality: as_otlp_aggregation_temporality(metrics.aggregation_temporality), - data_points: metrics.data_points.map do |hdp| - histogram_data_point(hdp) - end - ) - ) - end - end - - def as_otlp_aggregation_temporality(type) - case type - when :delta then Opentelemetry::Proto::Metrics::V1::AggregationTemporality::AGGREGATION_TEMPORALITY_DELTA - when :cumulative then Opentelemetry::Proto::Metrics::V1::AggregationTemporality::AGGREGATION_TEMPORALITY_CUMULATIVE - else Opentelemetry::Proto::Metrics::V1::AggregationTemporality::AGGREGATION_TEMPORALITY_UNSPECIFIED - end - end - - def histogram_data_point(hdp) - Opentelemetry::Proto::Metrics::V1::HistogramDataPoint.new( - attributes: hdp.attributes.map { |k, v| as_otlp_key_value(k, v) }, - start_time_unix_nano: hdp.start_time_unix_nano, - time_unix_nano: hdp.time_unix_nano, - count: hdp.count, - sum: hdp.sum, - bucket_counts: hdp.bucket_counts, - explicit_bounds: hdp.explicit_bounds, - exemplars: hdp.exemplars, - min: hdp.min, - max: hdp.max - ) - end - - def number_data_point(ndp) - Opentelemetry::Proto::Metrics::V1::NumberDataPoint.new( - attributes: ndp.attributes.map { |k, v| as_otlp_key_value(k, v) }, - as_int: ndp.value, - start_time_unix_nano: ndp.start_time_unix_nano, - time_unix_nano: ndp.time_unix_nano, - exemplars: ndp.exemplars # exemplars not implemented yet from metrics sdk - ) - end - - # may not need this - def reset - SUCCESS - end - - def shutdown(timeout: nil) - @shutdown = true - SUCCESS - end - end - end - end -end diff --git a/exporter/otlp-metrics/lib/opentelemetry/exporter/otlp/util.rb b/exporter/otlp-metrics/lib/opentelemetry/exporter/otlp/util.rb deleted file mode 100644 index 9075893fea..0000000000 --- a/exporter/otlp-metrics/lib/opentelemetry/exporter/otlp/util.rb +++ /dev/null @@ -1,139 +0,0 @@ -# frozen_string_literal: true - -# Copyright The OpenTelemetry Authors -# -# SPDX-License-Identifier: Apache-2.0 - -module OpenTelemetry - module Exporter - module OTLP - # Util module provide essential functionality for exporter - module Util # rubocop:disable Metrics/ModuleLength - KEEP_ALIVE_TIMEOUT = 30 - RETRY_COUNT = 5 - ERROR_MESSAGE_INVALID_HEADERS = 'headers must be a String with comma-separated URL Encoded UTF-8 k=v pairs or a Hash' - DEFAULT_USER_AGENT = "OTel-OTLP-MetricsExporter-Ruby/#{OpenTelemetry::Exporter::OTLP::VERSION} Ruby/#{RUBY_VERSION} (#{RUBY_PLATFORM}; #{RUBY_ENGINE}/#{RUBY_ENGINE_VERSION})".freeze - - def http_connection(uri, ssl_verify_mode, certificate_file) - http = Net::HTTP.new(uri.host, uri.port) - http.use_ssl = uri.scheme == 'https' - http.verify_mode = ssl_verify_mode - http.ca_file = certificate_file unless certificate_file.nil? - http.keep_alive_timeout = KEEP_ALIVE_TIMEOUT - http - end - - def around_request - OpenTelemetry::Common::Utilities.untraced { yield } # rubocop:disable Style/ExplicitBlockArgument - end - - def as_otlp_key_value(key, value) - Opentelemetry::Proto::Common::V1::KeyValue.new(key: key, value: as_otlp_any_value(value)) - rescue Encoding::UndefinedConversionError => e - encoded_value = value.encode('UTF-8', invalid: :replace, undef: :replace, replace: '�') - OpenTelemetry.handle_error(exception: e, message: "encoding error for key #{key} and value #{encoded_value}") - Opentelemetry::Proto::Common::V1::KeyValue.new(key: key, value: as_otlp_any_value('Encoding Error')) - end - - def as_otlp_any_value(value) - result = Opentelemetry::Proto::Common::V1::AnyValue.new - case value - when String - result.string_value = value - when Integer - result.int_value = value - when Float - result.double_value = value - when true, false - result.bool_value = value - when Array - values = value.map { |element| as_otlp_any_value(element) } - result.array_value = Opentelemetry::Proto::Common::V1::ArrayValue.new(values: values) - end - result - end - - def prepare_headers(config_headers) - headers = case config_headers - when String then parse_headers(config_headers) - when Hash then config_headers.dup - else - raise ArgumentError, ERROR_MESSAGE_INVALID_HEADERS - end - - headers['User-Agent'] = "#{headers.fetch('User-Agent', '')} #{DEFAULT_USER_AGENT}".strip - - headers - end - - def measure_request_duration - start = Process.clock_gettime(Process::CLOCK_MONOTONIC) - begin - yield - ensure - stop = Process.clock_gettime(Process::CLOCK_MONOTONIC) - 1000.0 * (stop - start) - end - end - - def parse_headers(raw) - entries = raw.split(',') - raise ArgumentError, ERROR_MESSAGE_INVALID_HEADERS if entries.empty? - - entries.each_with_object({}) do |entry, headers| - k, v = entry.split('=', 2).map(&CGI.method(:unescape)) - begin - k = k.to_s.strip - v = v.to_s.strip - rescue Encoding::CompatibilityError - raise ArgumentError, ERROR_MESSAGE_INVALID_HEADERS - rescue ArgumentError => e - raise e, ERROR_MESSAGE_INVALID_HEADERS - end - raise ArgumentError, ERROR_MESSAGE_INVALID_HEADERS if k.empty? || v.empty? - - headers[k] = v - end - end - - def backoff?(retry_count:, reason:, retry_after: nil) # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity - return false if retry_count > RETRY_COUNT - - sleep_interval = nil - unless retry_after.nil? - sleep_interval = - begin - Integer(retry_after) - rescue ArgumentError - nil - end - sleep_interval ||= - begin - Time.httpdate(retry_after) - Time.now - rescue # rubocop:disable Style/RescueStandardError - nil - end - sleep_interval = nil unless sleep_interval&.positive? - end - sleep_interval ||= rand(2**retry_count) - - sleep(sleep_interval) - true - end - - def log_status(body) - status = Google::Rpc::Status.decode(body) - details = status.details.map do |detail| - klass_or_nil = ::Google::Protobuf::DescriptorPool.generated_pool.lookup(detail.type_name).msgclass - detail.unpack(klass_or_nil) if klass_or_nil - end.compact - OpenTelemetry.handle_error(message: "OTLP metrics_exporter received rpc.Status{message=#{status.message}, details=#{details}}") - rescue StandardError => e - OpenTelemetry.handle_error(exception: e, message: 'unexpected error decoding rpc.Status in OTLP::MetricsExporter#log_status') - end - - def handle_redirect(location); end - end - end - end -end diff --git a/exporter/otlp-metrics/lib/opentelemetry/exporter/otlp_metrics.rb b/exporter/otlp-metrics/lib/opentelemetry/exporter/otlp_metrics.rb index fe2d7d4c13..05f6b35972 100644 --- a/exporter/otlp-metrics/lib/opentelemetry/exporter/otlp_metrics.rb +++ b/exporter/otlp-metrics/lib/opentelemetry/exporter/otlp_metrics.rb @@ -4,8 +4,8 @@ # # SPDX-License-Identifier: Apache-2.0 -require 'opentelemetry/exporter/otlp/version' -require 'opentelemetry/exporter/otlp/metrics_exporter' +require 'opentelemetry/exporter/otlp/metrics/version' +require 'opentelemetry/exporter/otlp/metrics/metrics_exporter' # OpenTelemetry is an open source observability framework, providing a # general-purpose API, SDK, and related tools required for the instrumentation diff --git a/exporter/otlp-metrics/opentelemetry-exporter-otlp-metrics.gemspec b/exporter/otlp-metrics/opentelemetry-exporter-otlp-metrics.gemspec index 655be1d78c..cd4ee258e0 100644 --- a/exporter/otlp-metrics/opentelemetry-exporter-otlp-metrics.gemspec +++ b/exporter/otlp-metrics/opentelemetry-exporter-otlp-metrics.gemspec @@ -6,11 +6,11 @@ lib = File.expand_path('lib', __dir__) $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) -require 'opentelemetry/exporter/otlp/version' +require 'opentelemetry/exporter/otlp/metrics/version' Gem::Specification.new do |spec| spec.name = 'opentelemetry-exporter-otlp-metrics' - spec.version = OpenTelemetry::Exporter::OTLP::VERSION + spec.version = OpenTelemetry::Exporter::OTLP::Metrics::VERSION spec.authors = ['OpenTelemetry Authors'] spec.email = ['cncf-opentelemetry-contributors@lists.cncf.io'] @@ -29,6 +29,8 @@ Gem::Specification.new do |spec| spec.add_dependency 'google-protobuf', '>= 3.18', '< 5.0' spec.add_dependency 'opentelemetry-api', '~> 1.1' spec.add_dependency 'opentelemetry-common', '~> 0.20' + spec.add_dependency 'opentelemetry-metrics-api', '~> 0.1.0' + spec.add_dependency 'opentelemetry-metrics-sdk', '~> 0.1.0' spec.add_dependency 'opentelemetry-sdk', '~> 1.2' spec.add_dependency 'opentelemetry-semantic_conventions' @@ -46,9 +48,9 @@ Gem::Specification.new do |spec| spec.add_development_dependency 'yard-doctest', '~> 0.1.6' if spec.respond_to?(:metadata) - spec.metadata['changelog_uri'] = "https://open-telemetry.github.io/opentelemetry-ruby/opentelemetry-exporter-otlp-metrics/v#{OpenTelemetry::Exporter::OTLP::VERSION}/file.CHANGELOG.html" + spec.metadata['changelog_uri'] = "https://open-telemetry.github.io/opentelemetry-ruby/opentelemetry-exporter-otlp-metrics/v#{OpenTelemetry::Exporter::OTLP::Metrics::VERSION}/file.CHANGELOG.html" spec.metadata['source_code_uri'] = 'https://github.com/open-telemetry/opentelemetry-ruby/tree/main/exporter/otlp-metrics' spec.metadata['bug_tracker_uri'] = 'https://github.com/open-telemetry/opentelemetry-ruby/issues' - spec.metadata['documentation_uri'] = "https://open-telemetry.github.io/opentelemetry-ruby/opentelemetry-exporter-otlp-metrics/v#{OpenTelemetry::Exporter::OTLP::VERSION}" + spec.metadata['documentation_uri'] = "https://open-telemetry.github.io/opentelemetry-ruby/opentelemetry-exporter-otlp-metrics/v#{OpenTelemetry::Exporter::OTLP::Metrics::VERSION}" end end diff --git a/exporter/otlp-metrics/test/opentelemetry/exporter/otlp/metrics_exporter_test.rb b/exporter/otlp-metrics/test/opentelemetry/exporter/otlp/metrics_exporter_test.rb index f4d11883b6..5add4cbc55 100644 --- a/exporter/otlp-metrics/test/opentelemetry/exporter/otlp/metrics_exporter_test.rb +++ b/exporter/otlp-metrics/test/opentelemetry/exporter/otlp/metrics_exporter_test.rb @@ -8,15 +8,15 @@ require 'google/protobuf/wrappers_pb' require 'google/protobuf/well_known_types' -describe OpenTelemetry::Exporter::OTLP::MetricsExporter do +describe OpenTelemetry::Exporter::OTLP::Metrics::MetricsExporter do METRICS_SUCCESS = OpenTelemetry::SDK::Metrics::Export::SUCCESS METRICS_FAILURE = OpenTelemetry::SDK::Metrics::Export::FAILURE - METRICS_VERSION = OpenTelemetry::Exporter::OTLP::VERSION - METRICS_DEFAULT_USER_AGENT = OpenTelemetry::Exporter::OTLP::Util::DEFAULT_USER_AGENT + METRICS_VERSION = OpenTelemetry::Exporter::OTLP::Metrics::VERSION + METRICS_DEFAULT_USER_AGENT = OpenTelemetry::Exporter::OTLP::Metrics::Util::DEFAULT_USER_AGENT describe '#initialize' do it 'initializes with defaults' do - exp = OpenTelemetry::Exporter::OTLP::MetricsExporter.new + exp = OpenTelemetry::Exporter::OTLP::Metrics::MetricsExporter.new _(exp).wont_be_nil _(exp.instance_variable_get(:@headers)).must_equal('User-Agent' => METRICS_DEFAULT_USER_AGENT) _(exp.instance_variable_get(:@timeout)).must_equal 10.0 @@ -39,24 +39,24 @@ it 'refuses invalid endpoint' do assert_raises ArgumentError do - OpenTelemetry::Exporter::OTLP::MetricsExporter.new(endpoint: 'not a url') + OpenTelemetry::Exporter::OTLP::Metrics::MetricsExporter.new(endpoint: 'not a url') end end it 'uses endpoints path if provided' do - exp = OpenTelemetry::Exporter::OTLP::MetricsExporter.new(endpoint: 'https://localhost/custom/path') + exp = OpenTelemetry::Exporter::OTLP::Metrics::MetricsExporter.new(endpoint: 'https://localhost/custom/path') _(exp.instance_variable_get(:@path)).must_equal '/custom/path' end it 'only allows gzip compression or none' do assert_raises ArgumentError do - OpenTelemetry::Exporter::OTLP::MetricsExporter.new(compression: 'flate') + OpenTelemetry::Exporter::OTLP::Metrics::MetricsExporter.new(compression: 'flate') end - exp = OpenTelemetry::Exporter::OTLP::MetricsExporter.new(compression: nil) + exp = OpenTelemetry::Exporter::OTLP::Metrics::MetricsExporter.new(compression: nil) _(exp.instance_variable_get(:@compression)).must_be_nil %w[gzip none].each do |compression| - exp = OpenTelemetry::Exporter::OTLP::MetricsExporter.new(compression: compression) + exp = OpenTelemetry::Exporter::OTLP::Metrics::MetricsExporter.new(compression: compression) _(exp.instance_variable_get(:@compression)).must_equal(compression) end @@ -67,7 +67,7 @@ { envar: 'OTEL_EXPORTER_OTLP_METRICS_COMPRESSION', value: 'none' } ].each do |example| OpenTelemetry::TestHelpers.with_env(example[:envar] => example[:value]) do - exp = OpenTelemetry::Exporter::OTLP::MetricsExporter.new + exp = OpenTelemetry::Exporter::OTLP::Metrics::MetricsExporter.new _(exp.instance_variable_get(:@compression)).must_equal(example[:value]) end end @@ -80,7 +80,7 @@ 'OTEL_EXPORTER_OTLP_COMPRESSION' => 'gzip', 'OTEL_RUBY_EXPORTER_OTLP_SSL_VERIFY_NONE' => 'true', 'OTEL_EXPORTER_OTLP_TIMEOUT' => '11') do - OpenTelemetry::Exporter::OTLP::MetricsExporter.new + OpenTelemetry::Exporter::OTLP::Metrics::MetricsExporter.new end _(exp.instance_variable_get(:@headers)).must_equal('a' => 'b', 'c' => 'd', 'User-Agent' => METRICS_DEFAULT_USER_AGENT) _(exp.instance_variable_get(:@timeout)).must_equal 11.0 @@ -101,12 +101,12 @@ 'OTEL_EXPORTER_OTLP_COMPRESSION' => 'flate', 'OTEL_RUBY_EXPORTER_OTLP_SSL_VERIFY_PEER' => 'true', 'OTEL_EXPORTER_OTLP_TIMEOUT' => '11') do - OpenTelemetry::Exporter::OTLP::MetricsExporter.new(endpoint: 'http://localhost:4321', - certificate_file: '/baz', - headers: { 'x' => 'y' }, - compression: 'gzip', - ssl_verify_mode: OpenSSL::SSL::VERIFY_NONE, - timeout: 12) + OpenTelemetry::Exporter::OTLP::Metrics::MetricsExporter.new(endpoint: 'http://localhost:4321', + certificate_file: '/baz', + headers: { 'x' => 'y' }, + compression: 'gzip', + ssl_verify_mode: OpenSSL::SSL::VERIFY_NONE, + timeout: 12) end _(exp.instance_variable_get(:@headers)).must_equal('x' => 'y', 'User-Agent' => METRICS_DEFAULT_USER_AGENT) _(exp.instance_variable_get(:@timeout)).must_equal 12.0 @@ -124,7 +124,7 @@ exp = OpenTelemetry::TestHelpers.with_env( 'OTEL_EXPORTER_OTLP_ENDPOINT' => 'https://localhost:1234/' ) do - OpenTelemetry::Exporter::OTLP::MetricsExporter.new + OpenTelemetry::Exporter::OTLP::Metrics::MetricsExporter.new end _(exp.instance_variable_get(:@path)).must_equal '/v1/metrics' end @@ -133,20 +133,20 @@ exp = OpenTelemetry::TestHelpers.with_env( 'OTEL_EXPORTER_OTLP_ENDPOINT' => 'https://localhost:1234' ) do - OpenTelemetry::Exporter::OTLP::MetricsExporter.new + OpenTelemetry::Exporter::OTLP::Metrics::MetricsExporter.new end _(exp.instance_variable_get(:@path)).must_equal '/v1/metrics' end it 'restricts explicit headers to a String or Hash' do - exp = OpenTelemetry::Exporter::OTLP::MetricsExporter.new(headers: { 'token' => 'über' }) + exp = OpenTelemetry::Exporter::OTLP::Metrics::MetricsExporter.new(headers: { 'token' => 'über' }) _(exp.instance_variable_get(:@headers)).must_equal('token' => 'über', 'User-Agent' => METRICS_DEFAULT_USER_AGENT) - exp = OpenTelemetry::Exporter::OTLP::MetricsExporter.new(headers: 'token=%C3%BCber') + exp = OpenTelemetry::Exporter::OTLP::Metrics::MetricsExporter.new(headers: 'token=%C3%BCber') _(exp.instance_variable_get(:@headers)).must_equal('token' => 'über', 'User-Agent' => METRICS_DEFAULT_USER_AGENT) error = _ do - exp = OpenTelemetry::Exporter::OTLP::MetricsExporter.new(headers: Object.new) + exp = OpenTelemetry::Exporter::OTLP::Metrics::MetricsExporter.new(headers: Object.new) _(exp.instance_variable_get(:@headers)).must_equal('token' => 'über') end.must_raise(ArgumentError) _(error.message).must_match(/headers/i) @@ -154,7 +154,7 @@ it 'ignores later mutations of a headers Hash parameter' do a_hash_to_mutate_later = { 'token' => 'über' } - exp = OpenTelemetry::Exporter::OTLP::MetricsExporter.new(headers: a_hash_to_mutate_later) + exp = OpenTelemetry::Exporter::OTLP::Metrics::MetricsExporter.new(headers: a_hash_to_mutate_later) _(exp.instance_variable_get(:@headers)).must_equal('token' => 'über', 'User-Agent' => METRICS_DEFAULT_USER_AGENT) a_hash_to_mutate_later['token'] = 'unter' @@ -165,60 +165,60 @@ describe 'Headers Environment Variable' do it 'allows any number of the equal sign (=) characters in the value' do exp = OpenTelemetry::TestHelpers.with_env('OTEL_EXPORTER_OTLP_HEADERS' => 'a=b,c=d==,e=f') do - OpenTelemetry::Exporter::OTLP::MetricsExporter.new + OpenTelemetry::Exporter::OTLP::Metrics::MetricsExporter.new end _(exp.instance_variable_get(:@headers)).must_equal('a' => 'b', 'c' => 'd==', 'e' => 'f', 'User-Agent' => METRICS_DEFAULT_USER_AGENT) exp = OpenTelemetry::TestHelpers.with_env('OTEL_EXPORTER_OTLP_METRICS_HEADERS' => 'a=b,c=d==,e=f') do - OpenTelemetry::Exporter::OTLP::MetricsExporter.new + OpenTelemetry::Exporter::OTLP::Metrics::MetricsExporter.new end _(exp.instance_variable_get(:@headers)).must_equal('a' => 'b', 'c' => 'd==', 'e' => 'f', 'User-Agent' => METRICS_DEFAULT_USER_AGENT) end it 'trims any leading or trailing whitespaces in keys and values' do exp = OpenTelemetry::TestHelpers.with_env('OTEL_EXPORTER_OTLP_HEADERS' => 'a = b ,c=d , e=f') do - OpenTelemetry::Exporter::OTLP::MetricsExporter.new + OpenTelemetry::Exporter::OTLP::Metrics::MetricsExporter.new end _(exp.instance_variable_get(:@headers)).must_equal('a' => 'b', 'c' => 'd', 'e' => 'f', 'User-Agent' => METRICS_DEFAULT_USER_AGENT) exp = OpenTelemetry::TestHelpers.with_env('OTEL_EXPORTER_OTLP_METRICS_HEADERS' => 'a = b ,c=d , e=f') do - OpenTelemetry::Exporter::OTLP::MetricsExporter.new + OpenTelemetry::Exporter::OTLP::Metrics::MetricsExporter.new end _(exp.instance_variable_get(:@headers)).must_equal('a' => 'b', 'c' => 'd', 'e' => 'f', 'User-Agent' => METRICS_DEFAULT_USER_AGENT) end it 'decodes values as URL encoded UTF-8 strings' do exp = OpenTelemetry::TestHelpers.with_env('OTEL_EXPORTER_OTLP_HEADERS' => 'token=%C3%BCber') do - OpenTelemetry::Exporter::OTLP::MetricsExporter.new + OpenTelemetry::Exporter::OTLP::Metrics::MetricsExporter.new end _(exp.instance_variable_get(:@headers)).must_equal('token' => 'über', 'User-Agent' => METRICS_DEFAULT_USER_AGENT) exp = OpenTelemetry::TestHelpers.with_env('OTEL_EXPORTER_OTLP_HEADERS' => '%C3%BCber=token') do - OpenTelemetry::Exporter::OTLP::MetricsExporter.new + OpenTelemetry::Exporter::OTLP::Metrics::MetricsExporter.new end _(exp.instance_variable_get(:@headers)).must_equal('über' => 'token', 'User-Agent' => METRICS_DEFAULT_USER_AGENT) exp = OpenTelemetry::TestHelpers.with_env('OTEL_EXPORTER_OTLP_METRICS_HEADERS' => 'token=%C3%BCber') do - OpenTelemetry::Exporter::OTLP::MetricsExporter.new + OpenTelemetry::Exporter::OTLP::Metrics::MetricsExporter.new end _(exp.instance_variable_get(:@headers)).must_equal('token' => 'über', 'User-Agent' => METRICS_DEFAULT_USER_AGENT) exp = OpenTelemetry::TestHelpers.with_env('OTEL_EXPORTER_OTLP_METRICS_HEADERS' => '%C3%BCber=token') do - OpenTelemetry::Exporter::OTLP::MetricsExporter.new + OpenTelemetry::Exporter::OTLP::Metrics::MetricsExporter.new end _(exp.instance_variable_get(:@headers)).must_equal('über' => 'token', 'User-Agent' => METRICS_DEFAULT_USER_AGENT) end it 'appends the default user agent to one provided in config' do exp = OpenTelemetry::TestHelpers.with_env('OTEL_EXPORTER_OTLP_HEADERS' => 'User-Agent=%C3%BCber/3.2.1') do - OpenTelemetry::Exporter::OTLP::MetricsExporter.new + OpenTelemetry::Exporter::OTLP::Metrics::MetricsExporter.new end _(exp.instance_variable_get(:@headers)).must_equal('User-Agent' => "über/3.2.1 #{METRICS_DEFAULT_USER_AGENT}") end it 'prefers METRICS specific variable' do exp = OpenTelemetry::TestHelpers.with_env('OTEL_EXPORTER_OTLP_HEADERS' => 'a=b,c=d==,e=f', 'OTEL_EXPORTER_OTLP_METRICS_HEADERS' => 'token=%C3%BCber') do - OpenTelemetry::Exporter::OTLP::MetricsExporter.new + OpenTelemetry::Exporter::OTLP::Metrics::MetricsExporter.new end _(exp.instance_variable_get(:@headers)).must_equal('token' => 'über', 'User-Agent' => METRICS_DEFAULT_USER_AGENT) end @@ -226,14 +226,14 @@ it 'fails fast when header values are missing' do error = _ do OpenTelemetry::TestHelpers.with_env('OTEL_EXPORTER_OTLP_HEADERS' => 'a = ') do - OpenTelemetry::Exporter::OTLP::MetricsExporter.new + OpenTelemetry::Exporter::OTLP::Metrics::MetricsExporter.new end end.must_raise(ArgumentError) _(error.message).must_match(/headers/i) error = _ do OpenTelemetry::TestHelpers.with_env('OTEL_EXPORTER_OTLP_METRICS_HEADERS' => 'a = ') do - OpenTelemetry::Exporter::OTLP::MetricsExporter.new + OpenTelemetry::Exporter::OTLP::Metrics::MetricsExporter.new end end.must_raise(ArgumentError) _(error.message).must_match(/headers/i) @@ -242,14 +242,14 @@ it 'fails fast when header or values are not found' do error = _ do OpenTelemetry::TestHelpers.with_env('OTEL_EXPORTER_OTLP_HEADERS' => ',') do - OpenTelemetry::Exporter::OTLP::MetricsExporter.new + OpenTelemetry::Exporter::OTLP::Metrics::MetricsExporter.new end end.must_raise(ArgumentError) _(error.message).must_match(/headers/i) error = _ do OpenTelemetry::TestHelpers.with_env('OTEL_EXPORTER_OTLP_METRICS_HEADERS' => ',') do - OpenTelemetry::Exporter::OTLP::MetricsExporter.new + OpenTelemetry::Exporter::OTLP::Metrics::MetricsExporter.new end end.must_raise(ArgumentError) _(error.message).must_match(/headers/i) @@ -258,14 +258,14 @@ it 'fails fast when header values contain invalid escape characters' do error = _ do OpenTelemetry::TestHelpers.with_env('OTEL_EXPORTER_OTLP_HEADERS' => 'c=hi%F3') do - OpenTelemetry::Exporter::OTLP::MetricsExporter.new + OpenTelemetry::Exporter::OTLP::Metrics::MetricsExporter.new end end.must_raise(ArgumentError) _(error.message).must_match(/headers/i) error = _ do OpenTelemetry::TestHelpers.with_env('OTEL_EXPORTER_OTLP_METRICS_HEADERS' => 'c=hi%F3') do - OpenTelemetry::Exporter::OTLP::MetricsExporter.new + OpenTelemetry::Exporter::OTLP::Metrics::MetricsExporter.new end end.must_raise(ArgumentError) _(error.message).must_match(/headers/i) @@ -274,14 +274,14 @@ it 'fails fast when headers are invalid' do error = _ do OpenTelemetry::TestHelpers.with_env('OTEL_EXPORTER_OTLP_HEADERS' => 'this is not a header') do - OpenTelemetry::Exporter::OTLP::MetricsExporter.new + OpenTelemetry::Exporter::OTLP::Metrics::MetricsExporter.new end end.must_raise(ArgumentError) _(error.message).must_match(/headers/i) error = _ do OpenTelemetry::TestHelpers.with_env('OTEL_EXPORTER_OTLP_METRICS_HEADERS' => 'this is not a header') do - OpenTelemetry::Exporter::OTLP::MetricsExporter.new + OpenTelemetry::Exporter::OTLP::Metrics::MetricsExporter.new end end.must_raise(ArgumentError) _(error.message).must_match(/headers/i) @@ -292,7 +292,7 @@ describe 'ssl_verify_mode:' do it 'can be set to VERIFY_NONE by an envvar' do exp = OpenTelemetry::TestHelpers.with_env('OTEL_RUBY_EXPORTER_OTLP_SSL_VERIFY_NONE' => 'true') do - OpenTelemetry::Exporter::OTLP::MetricsExporter.new + OpenTelemetry::Exporter::OTLP::Metrics::MetricsExporter.new end http = exp.instance_variable_get(:@http) _(http.verify_mode).must_equal OpenSSL::SSL::VERIFY_NONE @@ -300,7 +300,7 @@ it 'can be set to VERIFY_PEER by an envvar' do exp = OpenTelemetry::TestHelpers.with_env('OTEL_RUBY_EXPORTER_OTLP_SSL_VERIFY_PEER' => 'true') do - OpenTelemetry::Exporter::OTLP::MetricsExporter.new + OpenTelemetry::Exporter::OTLP::Metrics::MetricsExporter.new end http = exp.instance_variable_get(:@http) _(http.verify_mode).must_equal OpenSSL::SSL::VERIFY_PEER @@ -309,7 +309,7 @@ it 'VERIFY_PEER will override VERIFY_NONE' do exp = OpenTelemetry::TestHelpers.with_env('OTEL_RUBY_EXPORTER_OTLP_SSL_VERIFY_NONE' => 'true', 'OTEL_RUBY_EXPORTER_OTLP_SSL_VERIFY_PEER' => 'true') do - OpenTelemetry::Exporter::OTLP::MetricsExporter.new + OpenTelemetry::Exporter::OTLP::Metrics::MetricsExporter.new end http = exp.instance_variable_get(:@http) _(http.verify_mode).must_equal OpenSSL::SSL::VERIFY_PEER @@ -317,14 +317,14 @@ end describe '#export' do - let(:exporter) { OpenTelemetry::Exporter::OTLP::MetricsExporter.new } + let(:exporter) { OpenTelemetry::Exporter::OTLP::Metrics::MetricsExporter.new } let(:meter_provider) { OpenTelemetry::SDK::Metrics::MeterProvider.new(resource: OpenTelemetry::SDK::Resources::Resource.telemetry_sdk) } it 'integrates with collector' do skip unless ENV['TRACING_INTEGRATION_TEST'] WebMock.disable_net_connect!(allow: 'localhost') metrics_data = create_metrics_data - exporter = OpenTelemetry::Exporter::OTLP::MetricsExporter.new(endpoint: 'http://localhost:4318', compression: 'gzip') + exporter = OpenTelemetry::Exporter::OTLP::Metrics::MetricsExporter.new(endpoint: 'http://localhost:4318', compression: 'gzip') result = exporter.export([metrics_data]) _(result).must_equal(METRICS_SUCCESS) end @@ -404,7 +404,7 @@ end it 'returns METRICS_FAILURE when encryption to receiver endpoint fails' do - exporter = OpenTelemetry::Exporter::OTLP::MetricsExporter.new(endpoint: 'https://localhost:4318/v1/metrics') + exporter = OpenTelemetry::Exporter::OTLP::Metrics::MetricsExporter.new(endpoint: 'https://localhost:4318/v1/metrics') stub_request(:post, 'https://localhost:4318/v1/metrics').to_raise(OpenSSL::SSL::SSLError.new('enigma wedged')) metrics_data = create_metrics_data exporter.stub(:backoff?, ->(**_) { false }) do @@ -506,7 +506,7 @@ end it 'compresses with gzip if enabled' do - exporter = OpenTelemetry::Exporter::OTLP::MetricsExporter.new(compression: 'gzip') + exporter = OpenTelemetry::Exporter::OTLP::Metrics::MetricsExporter.new(compression: 'gzip') stub_post = stub_request(:post, 'http://localhost:4318/v1/metrics').to_return do |request| Opentelemetry::Proto::Collector::Metrics::V1::ExportMetricsServiceRequest.decode(Zlib.gunzip(request.body)) { status: 200 } From de318dae6ebc2215fdb4efb3826ebdc711ccbaba Mon Sep 17 00:00:00 2001 From: Kayla Reopelle <87386821+kaylareopelle@users.noreply.github.com> Date: Fri, 23 Aug 2024 15:35:57 -0700 Subject: [PATCH 02/13] feat: Add log record processors (#1682) * feat: Add log record exporter interface * chore: Move ExportError within Export module * feat: Add simple and batch log record processors * test: Adjust for JRuby * Replace let variables with specific declarations in each test * Add more records for buffer-full tests --------- Co-authored-by: Matthew Wear --- logs_sdk/lib/opentelemetry/sdk/logs/export.rb | 2 + .../logs/export/batch_log_record_processor.rb | 219 +++++++ .../export/simple_log_record_processor.rb | 88 +++ .../export/batch_log_record_processor_test.rb | 535 ++++++++++++++++++ .../simple_log_record_processor_test.rb | 134 +++++ 5 files changed, 978 insertions(+) create mode 100644 logs_sdk/lib/opentelemetry/sdk/logs/export/batch_log_record_processor.rb create mode 100644 logs_sdk/lib/opentelemetry/sdk/logs/export/simple_log_record_processor.rb create mode 100644 logs_sdk/test/opentelemetry/sdk/logs/export/batch_log_record_processor_test.rb create mode 100644 logs_sdk/test/opentelemetry/sdk/logs/export/simple_log_record_processor_test.rb diff --git a/logs_sdk/lib/opentelemetry/sdk/logs/export.rb b/logs_sdk/lib/opentelemetry/sdk/logs/export.rb index 5cb94455a9..2565dbf85f 100644 --- a/logs_sdk/lib/opentelemetry/sdk/logs/export.rb +++ b/logs_sdk/lib/opentelemetry/sdk/logs/export.rb @@ -25,3 +25,5 @@ module Export end require_relative 'export/log_record_exporter' +require_relative 'export/simple_log_record_processor' +require_relative 'export/batch_log_record_processor' diff --git a/logs_sdk/lib/opentelemetry/sdk/logs/export/batch_log_record_processor.rb b/logs_sdk/lib/opentelemetry/sdk/logs/export/batch_log_record_processor.rb new file mode 100644 index 0000000000..65f99a1512 --- /dev/null +++ b/logs_sdk/lib/opentelemetry/sdk/logs/export/batch_log_record_processor.rb @@ -0,0 +1,219 @@ +# frozen_string_literal: true + +# Copyright The OpenTelemetry Authors +# +# SPDX-License-Identifier: Apache-2.0 + +module OpenTelemetry + module SDK + module Logs + module Export + # WARNING - The spec has some differences from the LogRecord version of this processor + # Implementation of the duck type LogRecordProcessor that batches + # log records exported by the SDK then pushes them to the exporter + # pipeline. + # + # Typically, the BatchLogRecordProcessor will be more suitable for + # production environments than the SimpleLogRecordProcessor. + class BatchLogRecordProcessor < LogRecordProcessor # rubocop:disable Metrics/ClassLength + # Returns a new instance of the {BatchLogRecordProcessor}. + # + # @param [LogRecordExporter] exporter The (duck type) LogRecordExporter to where the + # recorded LogRecords are pushed after batching. + # @param [Numeric] exporter_timeout The maximum allowed time to export data. + # Defaults to the value of the OTEL_BLRP_EXPORT_TIMEOUT + # environment variable, if set, or 30,000 (30 seconds). + # @param [Numeric] schedule_delay the delay interval between two consecutive exports. + # Defaults to the value of the OTEL_BLRP_SCHEDULE_DELAY environment + # variable, if set, or 1,000 (1 second). + # @param [Integer] max_queue_size the maximum queue size in log records. + # Defaults to the value of the OTEL_BLRP_MAX_QUEUE_SIZE environment + # variable, if set, or 2048. + # @param [Integer] max_export_batch_size the maximum batch size in log records. + # Defaults to the value of the OTEL_BLRP_MAX_EXPORT_BATCH_SIZE environment + # variable, if set, or 512. + # + # @return a new instance of the {BatchLogRecordProcessor}. + def initialize(exporter, + exporter_timeout: Float(ENV.fetch('OTEL_BLRP_EXPORT_TIMEOUT', 30_000)), + schedule_delay: Float(ENV.fetch('OTEL_BLRP_SCHEDULE_DELAY', 1000)), + max_queue_size: Integer(ENV.fetch('OTEL_BLRP_MAX_QUEUE_SIZE', 2048)), + max_export_batch_size: Integer(ENV.fetch('OTEL_BLRP_MAX_EXPORT_BATCH_SIZE', 512)), + start_thread_on_boot: String(ENV['OTEL_RUBY_BLRP_START_THREAD_ON_BOOT']) !~ /false/i) + + unless max_export_batch_size <= max_queue_size + raise ArgumentError, + 'max_export_batch_size much be less than or equal to max_queue_size' + end + + unless Common::Utilities.valid_exporter?(exporter) + raise ArgumentError, + "exporter #{exporter.inspect} does not appear to be a valid exporter" + end + + @exporter = exporter + @exporter_timeout_seconds = exporter_timeout / 1000.0 + @mutex = Mutex.new + @export_mutex = Mutex.new + @condition = ConditionVariable.new + @keep_running = true + @stopped = false + @delay_seconds = schedule_delay / 1000.0 + @max_queue_size = max_queue_size + @batch_size = max_export_batch_size + @log_records = [] + @pid = nil + @thread = nil + reset_on_fork(restart_thread: start_thread_on_boot) + end + + # Adds a log record to the batch. Thread-safe; may block on lock. + def on_emit(log_record, _context) + return if @stopped + + lock do + reset_on_fork + n = log_records.size + 1 - max_queue_size + if n.positive? + log_records.shift(n) + report_dropped_log_records(n, reason: 'buffer-full') + end + log_records << log_record + @condition.signal if log_records.size > batch_size + end + end + + # Export all emitted log records that have not yet been exported to + # the configured `Exporter`. + # + # This method should only be called in cases where it is absolutely + # necessary, such as when using some FaaS providers that may suspend + # the process after an invocation, but before the `Processor` exports + # the completed log records. + # + # @param [optional Numeric] timeout An optional timeout in seconds. + # @return [Integer] SUCCESS if no error occurred, FAILURE if a + # non-specific failure occurred, TIMEOUT if a timeout occurred. + def force_flush(timeout: nil) + start_time = OpenTelemetry::Common::Utilities.timeout_timestamp + + snapshot = lock do + reset_on_fork if @keep_running + log_records.shift(log_records.size) + end + + until snapshot.empty? + remaining_timeout = OpenTelemetry::Common::Utilities.maybe_timeout(timeout, start_time) + return TIMEOUT if remaining_timeout&.zero? + + batch = snapshot.shift(batch_size).map!(&:to_log_record_data) + result_code = export_batch(batch, timeout: remaining_timeout) + return result_code unless result_code == SUCCESS + end + + @exporter.force_flush(timeout: OpenTelemetry::Common::Utilities.maybe_timeout(timeout, start_time)) + ensure + # Unshift the remaining log records if we timed out. We drop excess + # log records from the snapshot because they're older than any + # records in the buffer. + lock do + n = log_records.size + snapshot.size - max_queue_size + + if n.positive? + snapshot.shift(n) + report_dropped_log_records(n, reason: 'buffer-full') + end + + log_records.unshift(*snapshot) unless snapshot.empty? + @condition.signal if log_records.size > max_queue_size / 2 + end + end + + # Shuts the consumer thread down and flushes the current accumulated + # buffer will block until the thread is finished. + # + # @param [optional Numeric] timeout An optional timeout in seconds. + # @return [Integer] SUCCESS if no error occurred, FAILURE if a + # non-specific failure occurred, TIMEOUT if a timeout occurred. + def shutdown(timeout: nil) + return if @stopped + + start_time = OpenTelemetry::Common::Utilities.timeout_timestamp + thread = lock do + @keep_running = false + @stopped = true + @condition.signal + @thread + end + + thread&.join(timeout) + force_flush(timeout: OpenTelemetry::Common::Utilities.maybe_timeout(timeout, start_time)) + dropped_log_records = lock { log_records.size } + report_dropped_log_records(dropped_log_records, reason: 'terminating') if dropped_log_records.positive? + + @exporter.shutdown(timeout: OpenTelemetry::Common::Utilities.maybe_timeout(timeout, start_time)) + end + + private + + attr_reader :log_records, :max_queue_size, :batch_size + + def work + loop do + batch = lock do + @condition.wait(@mutex, @delay_seconds) if log_records.size < batch_size && @keep_running + @condition.wait(@mutex, @delay_seconds) while log_records.empty? && @keep_running + return unless @keep_running + + fetch_batch + end + + export_batch(batch) + end + end + + def reset_on_fork(restart_thread: true) + pid = Process.pid + return if @pid == pid + + @pid = pid + log_records.clear + @thread = restart_thread ? Thread.new { work } : nil + rescue ThreadError => e + OpenTelemetry.handle_error(exception: e, message: 'unexpected error in BatchLogRecordProcessor#reset_on_fork') + end + + def export_batch(batch, timeout: @exporter_timeout_seconds) + result_code = @export_mutex.synchronize { @exporter.export(batch, timeout: timeout) } + report_result(result_code, batch) + result_code + rescue StandardError => e + report_result(FAILURE, batch) + OpenTelemetry.handle_error(exception: e, message: 'unexpected error in BatchLogRecordProcessor#export_batch') + end + + def report_result(result_code, batch) + if result_code == SUCCESS + OpenTelemetry.logger.debug("Successfully exported #{batch.size} log records") + else + OpenTelemetry.handle_error(exception: ExportError.new("Unable to export #{batch.size} log records")) + OpenTelemetry.logger.error("Result code: #{result_code}") + end + end + + def report_dropped_log_records(count, reason:) + OpenTelemetry.logger.warn("#{count} log record(s) dropped. Reason: #{reason}") + end + + def fetch_batch + log_records.shift(@batch_size).map!(&:to_log_record_data) + end + + def lock(&block) + @mutex.synchronize(&block) + end + end + end + end + end +end diff --git a/logs_sdk/lib/opentelemetry/sdk/logs/export/simple_log_record_processor.rb b/logs_sdk/lib/opentelemetry/sdk/logs/export/simple_log_record_processor.rb new file mode 100644 index 0000000000..9cf4fcbbd8 --- /dev/null +++ b/logs_sdk/lib/opentelemetry/sdk/logs/export/simple_log_record_processor.rb @@ -0,0 +1,88 @@ +# frozen_string_literal: true + +# Copyright The OpenTelemetry Authors +# +# SPDX-License-Identifier: Apache-2.0 + +module OpenTelemetry + module SDK + module Logs + module Export + # An implementation of {LogRecordProcessor} that converts the LogRecord + # into a ReadableLogRecord and passes it to the configured exporter + # on emit. + # + # Typically, the SimpleLogRecordProcessor will be most suitable for use + # in testing; it should be used with caution in production. It may be + # appropriate for production use in scenarios where creating multiple + # threads is not desirable as well as scenarios where different custom + # attributes should be added to individual log records based on code + # scopes. + class SimpleLogRecordProcessor < OpenTelemetry::SDK::Logs::LogRecordProcessor + # Returns a new {SimpleLogRecordProcessor} that converts log records + # to {ReadableLogRecords} and forwards them to the given + # log_record_exporter. + # + # @param log_record_exporter the LogRecordExporter to push the + # recorded log records. + # @return [SimpleLogRecordProcessor] + # @raise ArgumentError if the log_record_exporter is invalid or nil. + def initialize(log_record_exporter) + raise ArgumentError, "exporter #{log_record_exporter.inspect} does not appear to be a valid exporter" unless Common::Utilities.valid_exporter?(log_record_exporter) + + @log_record_exporter = log_record_exporter + @stopped = false + end + + # Called when a LogRecord is emitted. + # + # This method is called synchronously on the execution thread. It + # should not throw or block the execution thread. It may not be called + # after shutdown. + # + # @param [LogRecord] log_record The emitted {LogRecord} + # @param [Context] _context The current {Context} + def on_emit(log_record, _context) + return if @stopped + + @log_record_exporter&.export([log_record.to_log_record_data]) + rescue => e # rubocop:disable Style/RescueStandardError + OpenTelemetry.handle_error(exception: e, message: 'Unexpected error in Logger#on_emit') + end + + # Export all log records to the configured `Exporter` that have not + # yet been exported, then call {Exporter#force_flush}. + # + # This method should only be called in cases where it is absolutely + # necessary, such as when using some FaaS providers that may suspend + # the process after an invocation, but before the `Processor` exports + # the completed log records. + # + # @param [optional Numeric] timeout An optional timeout in seconds. + # @return [Integer] SUCCESS if no error occurred, FAILURE if a + # non-specific failure occurred, TIMEOUT if a timeout occurred. + # TODO: Should a rescue/handle error be added here for non-specific failures? + def force_flush(timeout: nil) + return if @stopped + + @log_record_exporter&.force_flush(timeout: timeout) || SUCCESS + end + + # Called when {LoggerProvider#shutdown} is called. + # + # @param [optional Numeric] timeout An optional timeout in seconds. + # @return [Integer] SUCCESS if no error occurred, FAILURE if a + # non-specific failure occurred, TIMEOUT if a timeout occurred. + # TODO: Should a rescue/handle error be added here for non-specific failures? + def shutdown(timeout: nil) + return if @stopped + + @log_record_exporter&.shutdown(timeout: timeout) || SUCCESS + ensure + @stopped = true + end + end + end + end + end +end diff --git a/logs_sdk/test/opentelemetry/sdk/logs/export/batch_log_record_processor_test.rb b/logs_sdk/test/opentelemetry/sdk/logs/export/batch_log_record_processor_test.rb new file mode 100644 index 0000000000..08aee76e85 --- /dev/null +++ b/logs_sdk/test/opentelemetry/sdk/logs/export/batch_log_record_processor_test.rb @@ -0,0 +1,535 @@ +# frozen_string_literal: true + +# Copyright The OpenTelemetry Authors +# +# SPDX-License-Identifier: Apache-2.0 + +require 'test_helper' + +# rubocop:disable Lint/ConstantDefinitionInBlock, Style/Documentation +describe OpenTelemetry::SDK::Logs::Export::BatchLogRecordProcessor do + BatchLogRecordProcessor = OpenTelemetry::SDK::Logs::Export::BatchLogRecordProcessor + SUCCESS = OpenTelemetry::SDK::Logs::Export::SUCCESS + FAILURE = OpenTelemetry::SDK::Logs::Export::FAILURE + TIMEOUT = OpenTelemetry::SDK::Logs::Export::TIMEOUT + + class TestExporter + def initialize(status_codes: nil) + @status_codes = status_codes || [] + @batches = [] + @failed_batches = [] + end + + attr_reader :batches, :failed_batches + + def export(batch, timeout: nil) + # If status codes are empty, return success for less verbose testing + s = @status_codes.shift + if s.nil? || s == SUCCESS + @batches << batch + SUCCESS + else + @failed_batches << batch + s + end + end + + def shutdown(timeout: nil); end + + def force_flush(timeout: nil); end + end + + class NotAnExporter + end + + class RaisingExporter + def export(batch, timeout: nil) + raise 'boom!' + end + + def shutdown(timeout: nil); end + + def force_flush(timeout: nil); end + end + + class TestLogRecord + def initialize(body = nil) + @body = body + end + + attr_reader :body + + def to_log_record_data + self + end + end + + let(:mock_context) { Minitest::Mock.new } + + describe 'initialization' do + it 'raises if max batch size is greater than max queue size' do + assert_raises ArgumentError do + BatchLogRecordProcessor.new(TestExporter.new, max_queue_size: 6, max_export_batch_size: 999) + end + end + + it 'raises if OTEL_BLRP_EXPORT_TIMEOUT env var is not numeric' do + assert_raises ArgumentError do + OpenTelemetry::TestHelpers.with_env('OTEL_BLRP_EXPORT_TIMEOUT' => 'foo') do + BatchLogRecordProcessor.new(TestExporter.new) + end + end + end + + it 'raises if exporter is nil' do + _(-> { BatchLogRecordProcessor.new(nil) }).must_raise(ArgumentError) + end + + it 'raises if exporter is not an exporter' do + _(-> { BatchLogRecordProcessor.new(NotAnExporter.new) }).must_raise(ArgumentError) + end + + it 'sets parameters from the environment' do + processor = OpenTelemetry::TestHelpers.with_env('OTEL_BLRP_EXPORT_TIMEOUT' => '4', + 'OTEL_BLRP_SCHEDULE_DELAY' => '3', + 'OTEL_BLRP_MAX_QUEUE_SIZE' => '2', + 'OTEL_BLRP_MAX_EXPORT_BATCH_SIZE' => '1') do + BatchLogRecordProcessor.new(TestExporter.new) + end + _(processor.instance_variable_get(:@exporter_timeout_seconds)).must_equal 0.004 + _(processor.instance_variable_get(:@delay_seconds)).must_equal 0.003 + _(processor.instance_variable_get(:@max_queue_size)).must_equal 2 + _(processor.instance_variable_get(:@batch_size)).must_equal 1 + end + + it 'prefers explicit parameters rather than the environment' do + processor = OpenTelemetry::TestHelpers.with_env('OTEL_BLRP_EXPORT_TIMEOUT' => '4', + 'OTEL_BLRP_SCHEDULE_DELAY' => '3', + 'OTEL_BLRP_MAX_QUEUE_SIZE' => '2', + 'OTEL_BLRP_MAX_EXPORT_BATCH_SIZE' => '1') do + BatchLogRecordProcessor.new(TestExporter.new, + exporter_timeout: 10, + schedule_delay: 9, + max_queue_size: 8, + max_export_batch_size: 7) + end + _(processor.instance_variable_get(:@exporter_timeout_seconds)).must_equal 0.01 + _(processor.instance_variable_get(:@delay_seconds)).must_equal 0.009 + _(processor.instance_variable_get(:@max_queue_size)).must_equal 8 + _(processor.instance_variable_get(:@batch_size)).must_equal 7 + end + + it 'sets defaults for parameters not in the environment' do + processor = BatchLogRecordProcessor.new(TestExporter.new) + _(processor.instance_variable_get(:@exporter_timeout_seconds)).must_equal 30.0 + _(processor.instance_variable_get(:@delay_seconds)).must_equal 1.0 + _(processor.instance_variable_get(:@max_queue_size)).must_equal 2048 + _(processor.instance_variable_get(:@batch_size)).must_equal 512 + end + + it 'spawns a thread on boot by default' do + mock = Minitest::Mock.new + mock.expect(:call, nil) + + Thread.stub(:new, mock) do + BatchLogRecordProcessor.new(TestExporter.new) + end + + mock.verify + end + + it 'spawns a thread on boot if OTEL_RUBY_BLRP_START_THREAD_ON_BOOT is true' do + mock = Minitest::Mock.new + mock.expect(:call, nil) + + Thread.stub(:new, mock) do + OpenTelemetry::TestHelpers.with_env('OTEL_RUBY_BLRP_START_THREAD_ON_BOOT' => 'true') do + BatchLogRecordProcessor.new(TestExporter.new) + end + end + + mock.verify + end + + it 'does not spawn a thread on boot if OTEL_RUBY_BLRP_START_THREAD_ON_BOOT is false' do + mock = Minitest::Mock.new + mock.expect(:call, nil) { assert false } + + Thread.stub(:new, mock) do + OpenTelemetry::TestHelpers.with_env('OTEL_RUBY_BLRP_START_THREAD_ON_BOOT' => 'false') do + BatchLogRecordProcessor.new(TestExporter.new) + end + end + end + + it 'prefers explicit start_thread_on_boot parameter rather than the environment' do + mock = Minitest::Mock.new + mock.expect(:call, nil) { assert false } + + Thread.stub(:new, mock) do + OpenTelemetry::TestHelpers.with_env('OTEL_RUBY_BLRP_START_THREAD_ON_BOOT' => 'true') do + BatchLogRecordProcessor.new(TestExporter.new, + start_thread_on_boot: false) + end + end + end + end + + describe '#on_emit' do + it 'adds the log record to the batch' do + processor = BatchLogRecordProcessor.new(TestExporter.new) + log_record = TestLogRecord.new + + processor.on_emit(log_record, mock_context) + + assert_includes(processor.instance_variable_get(:@log_records), log_record) + end + + it 'removes the older log records from the batch if full' do + processor = BatchLogRecordProcessor.new(TestExporter.new, max_queue_size: 1, max_export_batch_size: 1) + + older_log_record = TestLogRecord.new + newer_log_record = TestLogRecord.new + newest_log_record = TestLogRecord.new + + processor.on_emit(older_log_record, mock_context) + processor.on_emit(newer_log_record, mock_context) + processor.on_emit(newest_log_record, mock_context) + + records = processor.instance_variable_get(:@log_records) + + assert_includes(records, newest_log_record) + refute_includes(records, newer_log_record) + refute_includes(records, older_log_record) + end + + it 'logs a warning if a log record was emitted after the buffer is full' do + mock_otel_logger = Minitest::Mock.new + mock_otel_logger.expect(:warn, nil, ['1 log record(s) dropped. Reason: buffer-full']) + + OpenTelemetry.stub(:logger, mock_otel_logger) do + processor = BatchLogRecordProcessor.new(TestExporter.new, max_queue_size: 1, max_export_batch_size: 1) + + log_record = TestLogRecord.new + log_record2 = TestLogRecord.new + + processor.on_emit(log_record, mock_context) + processor.on_emit(log_record2, mock_context) + end + + mock_otel_logger.verify + end + + it 'does not emit a log record if stopped' do + processor = BatchLogRecordProcessor.new(TestExporter.new) + + processor.instance_variable_set(:@stopped, true) + processor.on_emit(TestLogRecord.new, mock_context) + + assert_empty(processor.instance_variable_get(:@log_records)) + end + end + + describe '#force_flush' do + it 'reenqueues excess log_records on timeout' do + exporter = TestExporter.new + processor = BatchLogRecordProcessor.new(exporter) + + processor.on_emit(TestLogRecord.new, mock_context) + result = processor.force_flush(timeout: 0) + + _(result).must_equal(TIMEOUT) + + _(exporter.failed_batches.size).must_equal(0) + _(exporter.batches.size).must_equal(0) + + _(processor.instance_variable_get(:@log_records).size).must_equal(1) + end + + it 'exports the log record data and calls #force_flush on the exporter' do + mock_exporter = Minitest::Mock.new + processor = BatchLogRecordProcessor.new(TestExporter.new) + processor.instance_variable_set(:@exporter, mock_exporter) + log_record = TestLogRecord.new + log_record_data_mock = Minitest::Mock.new + + log_record.stub(:to_log_record_data, log_record_data_mock) do + processor.on_emit(log_record, mock_context) + mock_exporter.expect(:export, 0, [[log_record_data_mock]], timeout: nil) + mock_exporter.expect(:force_flush, nil, timeout: nil) + processor.force_flush + mock_exporter.verify + end + end + + it 'returns failure code if export_batch fails' do + processor = BatchLogRecordProcessor.new(TestExporter.new) + + processor.stub(:export_batch, OpenTelemetry::SDK::Logs::Export::FAILURE) do + processor.on_emit(TestLogRecord.new, mock_context) + assert_equal(OpenTelemetry::SDK::Logs::Export::FAILURE, processor.force_flush) + end + end + + it 'reports dropped logs if timeout occurs with full buffer' do + mock_otel_logger = Minitest::Mock.new + mock_otel_logger.expect(:warn, nil, [/buffer-full/]) + + OpenTelemetry.stub(:logger, mock_otel_logger) do + OpenTelemetry::Common::Utilities.stub(:maybe_timeout, 0) do + processor = BatchLogRecordProcessor.new(TestExporter.new, max_queue_size: 1, max_export_batch_size: 1) + processor.instance_variable_set(:@log_records, [TestLogRecord.new, TestLogRecord.new, TestLogRecord.new]) + processor.force_flush + end + end + + mock_otel_logger.verify + end + end + + describe '#shutdown' do + it 'does not allow subsequent calls to emit after shutdown' do + processor = BatchLogRecordProcessor.new(TestExporter.new) + + processor.shutdown + processor.on_emit(TestLogRecord.new, mock_context) + + assert_empty(processor.instance_variable_get(:@log_records)) + end + + it 'does not send shutdown to exporter if already shutdown' do + exporter = TestExporter.new + processor = BatchLogRecordProcessor.new(exporter) + + processor.instance_variable_set(:@stopped, true) + + exporter.stub(:shutdown, ->(_) { raise 'whoops!' }) do + processor.shutdown + end + end + + it 'sets @stopped to true' do + processor = BatchLogRecordProcessor.new(TestExporter.new) + + refute(processor.instance_variable_get(:@stopped)) + + processor.shutdown + + assert(processor.instance_variable_get(:@stopped)) + end + + it 'respects the timeout' do + exporter = TestExporter.new + processor = BatchLogRecordProcessor.new(exporter) + + processor.on_emit(TestLogRecord.new, mock_context) + processor.shutdown(timeout: 0) + + _(exporter.failed_batches.size).must_equal(0) + _(exporter.batches.size).must_equal(0) + + _(processor.instance_variable_get(:@log_records).size).must_equal(1) + end + + it 'works if the thread is not running' do + processor = BatchLogRecordProcessor.new(TestExporter.new, start_thread_on_boot: false) + processor.shutdown(timeout: 0) + end + + it 'returns a SUCCESS status if no error' do + test_exporter = TestExporter.new + test_exporter.instance_eval do + def shutdown(timeout: nil) + SUCCESS + end + end + + processor = BatchLogRecordProcessor.new(test_exporter) + processor.on_emit(TestLogRecord.new, mock_context) + result = processor.shutdown(timeout: 0) + + _(result).must_equal(SUCCESS) + end + + it 'returns a FAILURE status if a non-specific export error occurs' do + test_exporter = TestExporter.new + test_exporter.instance_eval do + def shutdown(timeout: nil) + FAILURE + end + end + + processor = BatchLogRecordProcessor.new(test_exporter) + processor.on_emit(TestLogRecord.new, mock_context) + result = processor.shutdown(timeout: 0) + + _(result).must_equal(FAILURE) + end + + it 'returns a TIMEOUT status if a timeout export error occurs' do + test_exporter = TestExporter.new + test_exporter.instance_eval do + def shutdown(timeout: nil) + TIMEOUT + end + end + + processor = BatchLogRecordProcessor.new(test_exporter) + processor.on_emit(TestLogRecord.new, mock_context) + result = processor.shutdown(timeout: 0) + + _(result).must_equal(TIMEOUT) + end + end + + describe 'lifecycle' do + it 'should stop and start correctly' do + processor = BatchLogRecordProcessor.new(TestExporter.new) + processor.shutdown + end + + it 'should flush everything on shutdown' do + exporter = TestExporter.new + processor = BatchLogRecordProcessor.new(exporter) + log_record = TestLogRecord.new + + processor.on_emit(log_record, mock_context) + processor.shutdown + + _(exporter.batches).must_equal [[log_record]] + end + end + + describe 'batching' do + it 'should batch up to but not over the max_batch' do + exporter = TestExporter.new + processor = BatchLogRecordProcessor.new(exporter, max_queue_size: 6, max_export_batch_size: 3) + + log_records = [TestLogRecord.new, TestLogRecord.new, TestLogRecord.new, TestLogRecord.new] + log_records.each { |log_record| processor.on_emit(log_record, mock_context) } + processor.shutdown + + _(exporter.batches[0].size).must_equal(3) + end + end + + describe 'export retry' do + it 'should not retry on FAILURE exports' do + exporter = TestExporter.new(status_codes: [FAILURE, SUCCESS]) + processor = BatchLogRecordProcessor.new(exporter, + schedule_delay: 999, + max_queue_size: 6, + max_export_batch_size: 3) + log_records = [TestLogRecord.new, TestLogRecord.new, TestLogRecord.new, TestLogRecord.new] + log_records.each { |log_record| processor.on_emit(log_record, mock_context) } + + # Ensure that our work thread has time to loop + sleep(1) + processor.shutdown + + _(exporter.batches.size).must_equal(1) + _(exporter.batches[0].size).must_equal(1) + + _(exporter.failed_batches.size).must_equal(1) + _(exporter.failed_batches[0].size).must_equal(3) + end + end + + describe 'stress test' do + it 'does not blow up with a lot of things' do + exporter = TestExporter.new + processor = BatchLogRecordProcessor.new(exporter) + + producers = 10.times.map do |i| + Thread.new do + x = i * 10 + 10.times do |j| + processor.on_emit(TestLogRecord.new(x + j), mock_context) + end + sleep(rand(0.01)) + end + end + producers.each(&:join) + processor.shutdown + + out = exporter.batches.flatten.map(&:body).sort + + expected = 100.times.map { |i| i } + + _(out).must_equal(expected) + end + end + + describe 'faulty exporter' do + let(:exporter) { RaisingExporter.new } + let(:processor) { BatchLogRecordProcessor.new(exporter) } + + it 'reports export failures' do + mock_logger = Minitest::Mock.new + mock_logger.expect(:error, nil, [/Unable to export/]) + mock_logger.expect(:error, nil, [/Result code: 1/]) + mock_logger.expect(:error, nil, [/unexpected error in .*\#export_batch/]) + + OpenTelemetry.stub(:logger, mock_logger) do + log_records = [TestLogRecord.new, TestLogRecord.new, TestLogRecord.new, TestLogRecord.new] + log_records.each { |log_record| processor.on_emit(log_record, mock_context) } + processor.shutdown + end + + mock_logger.verify + end + end + + describe 'fork safety test' do + let(:exporter) { TestExporter.new } + let(:processor) do + BatchLogRecordProcessor.new(exporter, + max_queue_size: 10, + max_export_batch_size: 3) + end + + it 'when ThreadError is raised it handles it gracefully' do + parent_pid = processor.instance_variable_get(:@pid) + parent_work_thread_id = processor.instance_variable_get(:@thread).object_id + Process.stub(:pid, parent_pid + rand(1..10)) do + Thread.stub(:new, -> { raise ThreadError }) do + processor.on_emit(TestLogRecord.new, mock_context) + end + + current_pid = processor.instance_variable_get(:@pid) + current_work_thread_id = processor.instance_variable_get(:@thread).object_id + _(parent_pid).wont_equal current_pid + _(parent_work_thread_id).must_equal current_work_thread_id + end + end + + describe 'when a process fork occurs' do + it 'creates new work thread when emit is called' do + parent_pid = processor.instance_variable_get(:@pid) + parent_work_thread_id = processor.instance_variable_get(:@thread).object_id + Process.stub(:pid, parent_pid + rand(1..10)) do + # Emit a new log record on the forked process and export it. + processor.on_emit(TestLogRecord.new, mock_context) + current_pid = processor.instance_variable_get(:@pid) + current_work_thread_id = processor.instance_variable_get(:@thread).object_id + _(parent_pid).wont_equal current_pid + _(parent_work_thread_id).wont_equal current_work_thread_id + end + end + + it 'creates new work thread when force_flush' do + parent_pid = processor.instance_variable_get(:@pid) + parent_work_thread_id = processor.instance_variable_get(:@thread).object_id + Process.stub(:pid, parent_pid + rand(1..10)) do + # Force flush on the forked process. + processor.force_flush + current_pid = processor.instance_variable_get(:@pid) + current_work_thread_id = processor.instance_variable_get(:@thread).object_id + _(parent_pid).wont_equal current_pid + _(parent_work_thread_id).wont_equal current_work_thread_id + end + end + end + end +end +# rubocop:enable Lint/ConstantDefinitionInBlock, Style/Documentation diff --git a/logs_sdk/test/opentelemetry/sdk/logs/export/simple_log_record_processor_test.rb b/logs_sdk/test/opentelemetry/sdk/logs/export/simple_log_record_processor_test.rb new file mode 100644 index 0000000000..b7a1fa0927 --- /dev/null +++ b/logs_sdk/test/opentelemetry/sdk/logs/export/simple_log_record_processor_test.rb @@ -0,0 +1,134 @@ +# frozen_string_literal: true + +# Copyright The OpenTelemetry Authors +# +# SPDX-License-Identifier: Apache-2.0 + +require 'test_helper' + +describe OpenTelemetry::SDK::Logs::Export::SimpleLogRecordProcessor do + let(:exporter) { OpenTelemetry::SDK::Logs::Export::LogRecordExporter.new } + let(:processor) { OpenTelemetry::SDK::Logs::Export::SimpleLogRecordProcessor.new(exporter) } + let(:log_record) { OpenTelemetry::SDK::Logs::LogRecord.new } + let(:mock_context) { Minitest::Mock.new } + + describe '#initialize' do + it 'raises an error when exporter is invalid' do + OpenTelemetry::Common::Utilities.stub(:valid_exporter?, false) do + assert_raises(ArgumentError) { OpenTelemetry::SDK::Logs::Export::SimpleLogRecordProcessor.new(exporter) } + end + end + end + + describe '#on_emit' do + it 'exports the log records' do + mock_exporter = Minitest::Mock.new + processor.instance_variable_set(:@log_record_exporter, mock_exporter) + mock_log_record_data = Minitest::Mock.new + + log_record.stub(:to_log_record_data, mock_log_record_data) do + OpenTelemetry::Common::Utilities.stub(:valid_exporter?, true) do + mock_exporter.expect(:export, OpenTelemetry::SDK::Logs::Export::SUCCESS, [[mock_log_record_data]]) + processor.on_emit(log_record, mock_context) + mock_exporter.verify + end + end + end + + it 'does not export if stopped' do + processor.instance_variable_set(:@stopped, true) + # raise if export is invoked + exporter.stub(:export, ->(_) { raise 'whoops!' }) do + processor.on_emit(log_record, mock_context) + end + end + + it 'does not export if log_record is nil' do + # raise if export is invoked + exporter.stub(:export, ->(_) { raise 'whoops!' }) do + processor.on_emit(nil, mock_context) + end + end + + it 'does not raise if exporter is nil' do + processor.instance_variable_set(:@log_record_exporter, nil) + processor.on_emit(log_record, mock_context) + end + + it 'catches and logs exporter errors' do + error_message = 'uh oh' + logger_mock = Minitest::Mock.new + logger_mock.expect(:error, nil, [/#{error_message}/]) + # raise if exporter's emit call is invoked + OpenTelemetry.stub(:logger, logger_mock) do + exporter.stub(:export, ->(_) { raise error_message }) do + processor.on_emit(log_record, mock_context) + end + end + + logger_mock.verify + end + end + + describe '#force_flush' do + it 'does not attempt to flush if stopped' do + processor.instance_variable_set(:@stopped, true) + # raise if export is invoked + exporter.stub(:force_flush, ->(_) { raise 'whoops!' }) do + processor.force_flush + end + end + + it 'returns success when the exporter cannot be found' do + processor.instance_variable_set(:@log_record_exporter, nil) + assert_equal(OpenTelemetry::SDK::Logs::Export::SUCCESS, processor.force_flush) + end + + it 'calls #force_flush on the exporter' do + exporter = Minitest::Mock.new + processor.instance_variable_set(:@log_record_exporter, exporter) + exporter.expect(:force_flush, nil, timeout: nil) + processor.force_flush + exporter.verify + end + end + + describe '#shutdown' do + it 'does not attempt to shutdown if stopped' do + processor.instance_variable_set(:@stopped, true) + # raise if export is invoked + exporter.stub(:shutdown, ->(_) { raise 'whoops!' }) do + processor.shutdown + end + end + + describe 'when exporter is nil' do + it 'returns success' do + processor.instance_variable_set(:@log_record_exporter, nil) + assert_equal(OpenTelemetry::SDK::Logs::Export::SUCCESS, processor.shutdown) + end + + it 'sets stopped to true' do + processor.instance_variable_set(:@log_record_exporter, nil) + processor.shutdown + assert(processor.instance_variable_get(:@stopped)) + end + end + + it 'calls shutdown on the exporter' do + exporter = Minitest::Mock.new + processor.instance_variable_set(:@log_record_exporter, exporter) + exporter.expect(:shutdown, nil, timeout: nil) + processor.shutdown + exporter.verify + end + + it 'sets stopped to true after calling shutdown on the exporter' do + exporter = Minitest::Mock.new + processor.instance_variable_set(:@log_record_exporter, exporter) + exporter.expect(:shutdown, nil, timeout: nil) + processor.shutdown + assert(processor.instance_variable_get(:@stopped)) + end + end +end From 2f0f5917194e5362be6126317bf75284a87c0ccc Mon Sep 17 00:00:00 2001 From: Kayla Reopelle <87386821+kaylareopelle@users.noreply.github.com> Date: Fri, 23 Aug 2024 15:53:40 -0700 Subject: [PATCH 03/13] chore: Update ruby/setup-ruby GHA (#1675) * chore: Update ruby/setup-ruby GHA * chore: Bump bundler version to 2.5.17 2.5.11 has a known truffleruby problem --------- Co-authored-by: Matthew Wear --- .github/actions/test_gem/action.yml | 8 ++++---- .github/workflows/release-hook-on-closed.yml | 2 +- .github/workflows/release-hook-on-push.yml | 2 +- .github/workflows/release-perform.yml | 2 +- .github/workflows/release-request.yml | 2 +- .github/workflows/release-retry.yml | 2 +- 6 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/actions/test_gem/action.yml b/.github/actions/test_gem/action.yml index aa6486e360..009270fd81 100644 --- a/.github/actions/test_gem/action.yml +++ b/.github/actions/test_gem/action.yml @@ -58,21 +58,21 @@ runs: # ...but not for appraisals, sadly. - name: Install Ruby ${{ inputs.ruby }} with dependencies if: "${{ steps.setup.outputs.appraisals == 'false' }}" - uses: ruby/setup-ruby@v1.179.0 + uses: ruby/setup-ruby@v1.190.0 with: ruby-version: "${{ inputs.ruby }}" working-directory: "${{ steps.setup.outputs.gem_dir }}" - bundler: "2.5.10" + bundler: "2.5.17" bundler-cache: true cache-version: "v1-${{ steps.setup.outputs.cache_key }}" # If we're using appraisals, do it all manually. - name: Install Ruby ${{ inputs.ruby }} without dependencies if: "${{ steps.setup.outputs.appraisals == 'true' }}" - uses: ruby/setup-ruby@v1.179.0 + uses: ruby/setup-ruby@v1.190.0 with: ruby-version: "${{ inputs.ruby }}" - bundler: "2.5.10" + bundler: "2.5.17" working-directory: "${{ steps.setup.outputs.gem_dir }}" - name: Install dependencies and generate appraisals if: "${{ steps.setup.outputs.appraisals == 'true' }}" diff --git a/.github/workflows/release-hook-on-closed.yml b/.github/workflows/release-hook-on-closed.yml index 8e15e2124b..9b125d3f3f 100644 --- a/.github/workflows/release-hook-on-closed.yml +++ b/.github/workflows/release-hook-on-closed.yml @@ -12,7 +12,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Install Ruby ${{ env.ruby_version }} - uses: ruby/setup-ruby@v1.179.0 + uses: ruby/setup-ruby@v1.190.0 with: ruby-version: ${{ env.ruby_version }} - name: Checkout repo diff --git a/.github/workflows/release-hook-on-push.yml b/.github/workflows/release-hook-on-push.yml index 00cf10864c..5afad6cdc3 100644 --- a/.github/workflows/release-hook-on-push.yml +++ b/.github/workflows/release-hook-on-push.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Install Ruby ${{ env.ruby_version }} - uses: ruby/setup-ruby@v1.179.0 + uses: ruby/setup-ruby@v1.190.0 with: ruby-version: ${{ env.ruby_version }} - name: Checkout repo diff --git a/.github/workflows/release-perform.yml b/.github/workflows/release-perform.yml index 49aa216584..a4ac5d06b5 100644 --- a/.github/workflows/release-perform.yml +++ b/.github/workflows/release-perform.yml @@ -22,7 +22,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Install Ruby ${{ env.ruby_version }} - uses: ruby/setup-ruby@v1.179.0 + uses: ruby/setup-ruby@v1.190.0 with: ruby-version: ${{ env.ruby_version }} - name: Checkout repo diff --git a/.github/workflows/release-request.yml b/.github/workflows/release-request.yml index cd10d07fba..1b7d21f60f 100644 --- a/.github/workflows/release-request.yml +++ b/.github/workflows/release-request.yml @@ -16,7 +16,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Install Ruby ${{ env.ruby_version }} - uses: ruby/setup-ruby@v1.179.0 + uses: ruby/setup-ruby@v1.190.0 with: ruby-version: ${{ env.ruby_version }} - name: Checkout repo diff --git a/.github/workflows/release-retry.yml b/.github/workflows/release-retry.yml index dc2f940691..1e67aed366 100644 --- a/.github/workflows/release-retry.yml +++ b/.github/workflows/release-retry.yml @@ -19,7 +19,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Install Ruby ${{ env.ruby_version }} - uses: ruby/setup-ruby@v1.179.0 + uses: ruby/setup-ruby@v1.190.0 with: ruby-version: ${{ env.ruby_version }} - name: Checkout repo From bf3ce24cc4fbf133e424b0bc09914a716c0aee4a Mon Sep 17 00:00:00 2001 From: Kayla Reopelle <87386821+kaylareopelle@users.noreply.github.com> Date: Fri, 23 Aug 2024 16:30:13 -0700 Subject: [PATCH 04/13] chore: Update rubocop to ~> 1.65 (#1676) * chore: Update rubocop to ~> 1.65 * chore: Update add_metric_reader to use each_value Rubocop Style/HashEachMethods update --------- Co-authored-by: Matthew Wear --- api/opentelemetry-api.gemspec | 2 +- common/opentelemetry-common.gemspec | 2 +- exporter/jaeger/opentelemetry-exporter-jaeger.gemspec | 2 +- exporter/otlp-common/opentelemetry-exporter-otlp-common.gemspec | 2 +- exporter/otlp-grpc/opentelemetry-exporter-otlp-grpc.gemspec | 2 +- exporter/otlp-http/opentelemetry-exporter-otlp-http.gemspec | 2 +- .../otlp-metrics/opentelemetry-exporter-otlp-metrics.gemspec | 2 +- exporter/otlp/opentelemetry-exporter-otlp.gemspec | 2 +- exporter/zipkin/opentelemetry-exporter-zipkin.gemspec | 2 +- logs_api/opentelemetry-logs-api.gemspec | 2 +- logs_sdk/opentelemetry-logs-sdk.gemspec | 2 +- metrics_api/opentelemetry-metrics-api.gemspec | 2 +- metrics_sdk/lib/opentelemetry/sdk/metrics/meter.rb | 2 +- metrics_sdk/opentelemetry-metrics-sdk.gemspec | 2 +- propagator/b3/opentelemetry-propagator-b3.gemspec | 2 +- propagator/jaeger/opentelemetry-propagator-jaeger.gemspec | 2 +- registry/opentelemetry-registry.gemspec | 2 +- sdk/opentelemetry-sdk.gemspec | 2 +- sdk_experimental/opentelemetry-sdk-experimental.gemspec | 2 +- semantic_conventions/opentelemetry-semantic_conventions.gemspec | 2 +- test_helpers/opentelemetry-test-helpers.gemspec | 2 +- 21 files changed, 21 insertions(+), 21 deletions(-) diff --git a/api/opentelemetry-api.gemspec b/api/opentelemetry-api.gemspec index 12e2a2558e..f18bfeec02 100644 --- a/api/opentelemetry-api.gemspec +++ b/api/opentelemetry-api.gemspec @@ -31,7 +31,7 @@ Gem::Specification.new do |spec| spec.add_development_dependency 'minitest', '~> 5.0' spec.add_development_dependency 'opentelemetry-test-helpers' spec.add_development_dependency 'rake', '~> 12.0' - spec.add_development_dependency 'rubocop', '~> 1.30' + spec.add_development_dependency 'rubocop', '~> 1.65' spec.add_development_dependency 'simplecov', '~> 0.17' spec.add_development_dependency 'yard', '~> 0.9' spec.add_development_dependency 'yard-doctest', '~> 0.1.6' diff --git a/common/opentelemetry-common.gemspec b/common/opentelemetry-common.gemspec index 164e87e01d..fbc2837d16 100644 --- a/common/opentelemetry-common.gemspec +++ b/common/opentelemetry-common.gemspec @@ -31,7 +31,7 @@ Gem::Specification.new do |spec| spec.add_development_dependency 'minitest', '~> 5.0' spec.add_development_dependency 'opentelemetry-test-helpers' spec.add_development_dependency 'rake', '~> 12.0' - spec.add_development_dependency 'rubocop', '~> 1.3' + spec.add_development_dependency 'rubocop', '~> 1.65' spec.add_development_dependency 'simplecov', '~> 0.17' spec.add_development_dependency 'yard', '~> 0.9' spec.add_development_dependency 'yard-doctest', '~> 0.1.6' diff --git a/exporter/jaeger/opentelemetry-exporter-jaeger.gemspec b/exporter/jaeger/opentelemetry-exporter-jaeger.gemspec index 61656ebb59..6b043a91f4 100644 --- a/exporter/jaeger/opentelemetry-exporter-jaeger.gemspec +++ b/exporter/jaeger/opentelemetry-exporter-jaeger.gemspec @@ -38,7 +38,7 @@ Gem::Specification.new do |spec| spec.add_development_dependency 'opentelemetry-test-helpers' spec.add_development_dependency 'rake', '~> 12.0' spec.add_development_dependency 'rspec-mocks' - spec.add_development_dependency 'rubocop', '~> 1.3' + spec.add_development_dependency 'rubocop', '~> 1.65' spec.add_development_dependency 'simplecov', '~> 0.17' spec.add_development_dependency 'webmock', '~> 3.7.6' spec.add_development_dependency 'yard', '~> 0.9' diff --git a/exporter/otlp-common/opentelemetry-exporter-otlp-common.gemspec b/exporter/otlp-common/opentelemetry-exporter-otlp-common.gemspec index dda7839b93..23ac0dd50a 100644 --- a/exporter/otlp-common/opentelemetry-exporter-otlp-common.gemspec +++ b/exporter/otlp-common/opentelemetry-exporter-otlp-common.gemspec @@ -37,7 +37,7 @@ Gem::Specification.new do |spec| spec.add_development_dependency 'pry' spec.add_development_dependency 'pry-byebug' unless RUBY_ENGINE == 'jruby' spec.add_development_dependency 'rake', '~> 12.0' - spec.add_development_dependency 'rubocop', '~> 1.51.0' + spec.add_development_dependency 'rubocop', '~> 1.65' spec.add_development_dependency 'simplecov', '~> 0.17' spec.add_development_dependency 'webmock', '~> 3.7.6' spec.add_development_dependency 'yard', '~> 0.9' diff --git a/exporter/otlp-grpc/opentelemetry-exporter-otlp-grpc.gemspec b/exporter/otlp-grpc/opentelemetry-exporter-otlp-grpc.gemspec index beb9aa7f73..0a1c789cf4 100644 --- a/exporter/otlp-grpc/opentelemetry-exporter-otlp-grpc.gemspec +++ b/exporter/otlp-grpc/opentelemetry-exporter-otlp-grpc.gemspec @@ -37,7 +37,7 @@ Gem::Specification.new do |spec| spec.add_development_dependency 'opentelemetry-test-helpers' spec.add_development_dependency 'pry-byebug' unless RUBY_ENGINE == 'jruby' spec.add_development_dependency 'rake', '~> 12.0' - spec.add_development_dependency 'rubocop', '~> 1.51.0' + spec.add_development_dependency 'rubocop', '~> 1.65' spec.add_development_dependency 'simplecov', '~> 0.17' spec.add_development_dependency 'webmock', '~> 3.7.6' spec.add_development_dependency 'yard', '~> 0.9' diff --git a/exporter/otlp-http/opentelemetry-exporter-otlp-http.gemspec b/exporter/otlp-http/opentelemetry-exporter-otlp-http.gemspec index 26e7cb013a..7d12b22337 100644 --- a/exporter/otlp-http/opentelemetry-exporter-otlp-http.gemspec +++ b/exporter/otlp-http/opentelemetry-exporter-otlp-http.gemspec @@ -36,7 +36,7 @@ Gem::Specification.new do |spec| spec.add_development_dependency 'opentelemetry-test-helpers' spec.add_development_dependency 'pry-byebug' unless RUBY_ENGINE == 'jruby' spec.add_development_dependency 'rake', '~> 12.0' - spec.add_development_dependency 'rubocop', '~> 1.51.0' + spec.add_development_dependency 'rubocop', '~> 1.65' spec.add_development_dependency 'simplecov', '~> 0.17' spec.add_development_dependency 'webmock', '~> 3.7.6' spec.add_development_dependency 'yard', '~> 0.9' diff --git a/exporter/otlp-metrics/opentelemetry-exporter-otlp-metrics.gemspec b/exporter/otlp-metrics/opentelemetry-exporter-otlp-metrics.gemspec index cd4ee258e0..571b3b203e 100644 --- a/exporter/otlp-metrics/opentelemetry-exporter-otlp-metrics.gemspec +++ b/exporter/otlp-metrics/opentelemetry-exporter-otlp-metrics.gemspec @@ -41,7 +41,7 @@ Gem::Specification.new do |spec| spec.add_development_dependency 'opentelemetry-test-helpers' spec.add_development_dependency 'pry-byebug' unless RUBY_ENGINE == 'jruby' spec.add_development_dependency 'rake', '~> 12.0' - spec.add_development_dependency 'rubocop', '~> 1.3' + spec.add_development_dependency 'rubocop', '~> 1.65' spec.add_development_dependency 'simplecov', '~> 0.17' spec.add_development_dependency 'webmock', '~> 3.7.6' spec.add_development_dependency 'yard', '~> 0.9' diff --git a/exporter/otlp/opentelemetry-exporter-otlp.gemspec b/exporter/otlp/opentelemetry-exporter-otlp.gemspec index 3b79b3a0bd..0371621bbd 100644 --- a/exporter/otlp/opentelemetry-exporter-otlp.gemspec +++ b/exporter/otlp/opentelemetry-exporter-otlp.gemspec @@ -39,7 +39,7 @@ Gem::Specification.new do |spec| spec.add_development_dependency 'opentelemetry-test-helpers' spec.add_development_dependency 'pry-byebug' unless RUBY_ENGINE == 'jruby' spec.add_development_dependency 'rake', '~> 12.0' - spec.add_development_dependency 'rubocop', '~> 1.3' + spec.add_development_dependency 'rubocop', '~> 1.65' spec.add_development_dependency 'simplecov', '~> 0.17' spec.add_development_dependency 'webmock', '~> 3.7.6' spec.add_development_dependency 'yard', '~> 0.9' diff --git a/exporter/zipkin/opentelemetry-exporter-zipkin.gemspec b/exporter/zipkin/opentelemetry-exporter-zipkin.gemspec index bbad42c52c..40d86cde57 100644 --- a/exporter/zipkin/opentelemetry-exporter-zipkin.gemspec +++ b/exporter/zipkin/opentelemetry-exporter-zipkin.gemspec @@ -37,7 +37,7 @@ Gem::Specification.new do |spec| spec.add_development_dependency 'minitest', '~> 5.0' spec.add_development_dependency 'opentelemetry-test-helpers' spec.add_development_dependency 'rake', '~> 12.0' - spec.add_development_dependency 'rubocop', '~> 1.51.0' + spec.add_development_dependency 'rubocop', '~> 1.65' spec.add_development_dependency 'simplecov', '~> 0.17' spec.add_development_dependency 'webmock', '~> 3.7.6' spec.add_development_dependency 'yard', '~> 0.9' diff --git a/logs_api/opentelemetry-logs-api.gemspec b/logs_api/opentelemetry-logs-api.gemspec index 36b1729119..0426ff8759 100644 --- a/logs_api/opentelemetry-logs-api.gemspec +++ b/logs_api/opentelemetry-logs-api.gemspec @@ -29,7 +29,7 @@ Gem::Specification.new do |spec| spec.add_development_dependency 'bundler', '>= 1.17' spec.add_development_dependency 'minitest', '~> 5.19' spec.add_development_dependency 'rake', '~> 12.0' - spec.add_development_dependency 'rubocop', '~> 1.55' + spec.add_development_dependency 'rubocop', '~> 1.65' spec.add_development_dependency 'simplecov', '~> 0.22' spec.add_development_dependency 'yard', '~> 0.9' spec.add_development_dependency 'yard-doctest', '~> 0.1' diff --git a/logs_sdk/opentelemetry-logs-sdk.gemspec b/logs_sdk/opentelemetry-logs-sdk.gemspec index 29910cb296..766cf6be5d 100644 --- a/logs_sdk/opentelemetry-logs-sdk.gemspec +++ b/logs_sdk/opentelemetry-logs-sdk.gemspec @@ -32,7 +32,7 @@ Gem::Specification.new do |spec| spec.add_development_dependency 'minitest', '~> 5.19' spec.add_development_dependency 'opentelemetry-test-helpers', '~> 0.4' spec.add_development_dependency 'rake', '~> 13.0' - spec.add_development_dependency 'rubocop', '~> 1.56' + spec.add_development_dependency 'rubocop', '~> 1.65' spec.add_development_dependency 'simplecov', '~> 0.22' spec.add_development_dependency 'yard', '~> 0.9' spec.add_development_dependency 'yard-doctest', '~> 0.1.17' diff --git a/metrics_api/opentelemetry-metrics-api.gemspec b/metrics_api/opentelemetry-metrics-api.gemspec index e08b9bc3c9..ea541bc429 100644 --- a/metrics_api/opentelemetry-metrics-api.gemspec +++ b/metrics_api/opentelemetry-metrics-api.gemspec @@ -32,7 +32,7 @@ Gem::Specification.new do |spec| spec.add_development_dependency 'minitest', '~> 5.0' spec.add_development_dependency 'opentelemetry-test-helpers', '~> 0.3.0' spec.add_development_dependency 'rake', '~> 12.0' - spec.add_development_dependency 'rubocop', '~> 1.51.0' + spec.add_development_dependency 'rubocop', '~> 1.65' spec.add_development_dependency 'simplecov', '~> 0.17' spec.add_development_dependency 'yard', '~> 0.9' spec.add_development_dependency 'yard-doctest', '~> 0.1.6' diff --git a/metrics_sdk/lib/opentelemetry/sdk/metrics/meter.rb b/metrics_sdk/lib/opentelemetry/sdk/metrics/meter.rb index faf8e9adb1..8db3ea992c 100644 --- a/metrics_sdk/lib/opentelemetry/sdk/metrics/meter.rb +++ b/metrics_sdk/lib/opentelemetry/sdk/metrics/meter.rb @@ -28,7 +28,7 @@ def initialize(name, version, meter_provider) # @api private def add_metric_reader(metric_reader) - @instrument_registry.each do |_n, instrument| + @instrument_registry.each_value do |instrument| instrument.register_with_new_metric_store(metric_reader.metric_store) end end diff --git a/metrics_sdk/opentelemetry-metrics-sdk.gemspec b/metrics_sdk/opentelemetry-metrics-sdk.gemspec index c82bdf5977..f7459963e2 100644 --- a/metrics_sdk/opentelemetry-metrics-sdk.gemspec +++ b/metrics_sdk/opentelemetry-metrics-sdk.gemspec @@ -35,7 +35,7 @@ Gem::Specification.new do |spec| spec.add_development_dependency 'minitest', '~> 5.0' spec.add_development_dependency 'opentelemetry-test-helpers' spec.add_development_dependency 'rake', '~> 12.0' - spec.add_development_dependency 'rubocop', '~> 1.51.0' + spec.add_development_dependency 'rubocop', '~> 1.65' spec.add_development_dependency 'simplecov', '~> 0.17' spec.add_development_dependency 'yard', '~> 0.9' spec.add_development_dependency 'yard-doctest', '~> 0.1.6' diff --git a/propagator/b3/opentelemetry-propagator-b3.gemspec b/propagator/b3/opentelemetry-propagator-b3.gemspec index 46137babb4..3a75351735 100644 --- a/propagator/b3/opentelemetry-propagator-b3.gemspec +++ b/propagator/b3/opentelemetry-propagator-b3.gemspec @@ -31,7 +31,7 @@ Gem::Specification.new do |spec| spec.add_development_dependency 'bundler', '>= 1.17' spec.add_development_dependency 'minitest', '~> 5.0' spec.add_development_dependency 'rake', '~> 12.0' - spec.add_development_dependency 'rubocop', '~> 1.51.0' + spec.add_development_dependency 'rubocop', '~> 1.65' spec.add_development_dependency 'simplecov', '~> 0.17.1' spec.add_development_dependency 'yard', '~> 0.9' spec.add_development_dependency 'yard-doctest', '~> 0.1.6' diff --git a/propagator/jaeger/opentelemetry-propagator-jaeger.gemspec b/propagator/jaeger/opentelemetry-propagator-jaeger.gemspec index aada4f9e8d..4016ede97d 100644 --- a/propagator/jaeger/opentelemetry-propagator-jaeger.gemspec +++ b/propagator/jaeger/opentelemetry-propagator-jaeger.gemspec @@ -32,7 +32,7 @@ Gem::Specification.new do |spec| spec.add_development_dependency 'minitest', '~> 5.0' spec.add_development_dependency 'opentelemetry-sdk', '~> 1.2' spec.add_development_dependency 'rake', '~> 12.0' - spec.add_development_dependency 'rubocop', '~> 1.51.0' + spec.add_development_dependency 'rubocop', '~> 1.65' spec.add_development_dependency 'simplecov', '~> 0.17.1' spec.add_development_dependency 'yard', '~> 0.9' spec.add_development_dependency 'yard-doctest', '~> 0.1.6' diff --git a/registry/opentelemetry-registry.gemspec b/registry/opentelemetry-registry.gemspec index 510d12ceb9..9b9371e736 100644 --- a/registry/opentelemetry-registry.gemspec +++ b/registry/opentelemetry-registry.gemspec @@ -32,7 +32,7 @@ Gem::Specification.new do |spec| spec.add_development_dependency 'minitest', '~> 5.0' spec.add_development_dependency 'rake', '~> 12.3.3' spec.add_development_dependency 'rspec-mocks' - spec.add_development_dependency 'rubocop', '~> 1.3' + spec.add_development_dependency 'rubocop', '~> 1.65' spec.add_development_dependency 'simplecov', '~> 0.17.1' spec.add_development_dependency 'yard', '~> 0.9' spec.add_development_dependency 'yard-doctest', '~> 0.1.6' diff --git a/sdk/opentelemetry-sdk.gemspec b/sdk/opentelemetry-sdk.gemspec index de7edca8a4..decb49270f 100644 --- a/sdk/opentelemetry-sdk.gemspec +++ b/sdk/opentelemetry-sdk.gemspec @@ -43,7 +43,7 @@ Gem::Specification.new do |spec| spec.add_development_dependency 'opentelemetry-test-helpers' spec.add_development_dependency 'pry' spec.add_development_dependency 'rake', '~> 12.0' - spec.add_development_dependency 'rubocop', '~> 1.51.0' + spec.add_development_dependency 'rubocop', '~> 1.65' spec.add_development_dependency 'simplecov', '~> 0.17' spec.add_development_dependency 'yard', '~> 0.9' spec.add_development_dependency 'yard-doctest', '~> 0.1.6' diff --git a/sdk_experimental/opentelemetry-sdk-experimental.gemspec b/sdk_experimental/opentelemetry-sdk-experimental.gemspec index a9fa3dd5f4..647d71ee2e 100644 --- a/sdk_experimental/opentelemetry-sdk-experimental.gemspec +++ b/sdk_experimental/opentelemetry-sdk-experimental.gemspec @@ -34,7 +34,7 @@ Gem::Specification.new do |spec| spec.add_development_dependency 'minitest', '~> 5.0' spec.add_development_dependency 'opentelemetry-test-helpers' spec.add_development_dependency 'rake', '~> 12.0' - spec.add_development_dependency 'rubocop', '~> 1.51.0' + spec.add_development_dependency 'rubocop', '~> 1.65' spec.add_development_dependency 'simplecov', '~> 0.17' spec.add_development_dependency 'yard', '~> 0.9' spec.add_development_dependency 'yard-doctest', '~> 0.1.6' diff --git a/semantic_conventions/opentelemetry-semantic_conventions.gemspec b/semantic_conventions/opentelemetry-semantic_conventions.gemspec index e09b108c86..2eae221991 100644 --- a/semantic_conventions/opentelemetry-semantic_conventions.gemspec +++ b/semantic_conventions/opentelemetry-semantic_conventions.gemspec @@ -30,7 +30,7 @@ Gem::Specification.new do |spec| spec.add_development_dependency 'bundler', '>= 1.17' spec.add_development_dependency 'minitest', '~> 5.0' spec.add_development_dependency 'rake', '~> 12.0' - spec.add_development_dependency 'rubocop', '~> 1.3' + spec.add_development_dependency 'rubocop', '~> 1.65' spec.add_development_dependency 'simplecov', '~> 0.17' spec.add_development_dependency 'yard', '~> 0.9' spec.add_development_dependency 'yard-doctest', '~> 0.1.6' diff --git a/test_helpers/opentelemetry-test-helpers.gemspec b/test_helpers/opentelemetry-test-helpers.gemspec index 5aff33cada..faab90233b 100644 --- a/test_helpers/opentelemetry-test-helpers.gemspec +++ b/test_helpers/opentelemetry-test-helpers.gemspec @@ -31,7 +31,7 @@ Gem::Specification.new do |spec| spec.add_development_dependency 'pry' spec.add_development_dependency 'pry-byebug' unless RUBY_ENGINE == 'jruby' spec.add_development_dependency 'rake', '~> 12.0' - spec.add_development_dependency 'rubocop', '~> 1.3' + spec.add_development_dependency 'rubocop', '~> 1.65' spec.add_development_dependency 'simplecov', '~> 0.17' spec.add_development_dependency 'yard', '~> 0.9' spec.add_development_dependency 'yard-doctest', '~> 0.1.6' From 1fef415c071e293c90ed3eb027e9a3be55cfe979 Mon Sep 17 00:00:00 2001 From: Kayla Reopelle <87386821+kaylareopelle@users.noreply.github.com> Date: Fri, 23 Aug 2024 16:45:08 -0700 Subject: [PATCH 05/13] ci: Add weekly release request (#1677) opentelemetry-ruby-contrib uses this workflow to open a PR to release any gems with changes just before the SIG meeting. This may help keep core releases up to date with the repository. Co-authored-by: Matthew Wear --- .github/workflows/release-request-weekly.yml | 28 ++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 .github/workflows/release-request-weekly.yml diff --git a/.github/workflows/release-request-weekly.yml b/.github/workflows/release-request-weekly.yml new file mode 100644 index 0000000000..20a80a562d --- /dev/null +++ b/.github/workflows/release-request-weekly.yml @@ -0,0 +1,28 @@ +name: Open release request - Weekly + +on: + schedule: + - cron: "0 15 * * 2" + +jobs: + release-request: + if: ${{ github.repository == 'open-telemetry/opentelemetry-ruby' }} + env: + ruby_version: "3.0" + runs-on: ubuntu-latest + steps: + - name: Install Ruby ${{ env.ruby_version }} + uses: ruby/setup-ruby@v1.190.0 + with: + ruby-version: ${{ env.ruby_version }} + - name: Checkout repo + uses: actions/checkout@v4 + - name: Install Toys + run: "gem install --no-document toys -v 0.15.5" + - name: Open release pull request + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + toys release request --yes --verbose \ + "--release-ref=${{ github.ref }}" \ + < /dev/null From 21717b5d4d676a6ffd23a0f6c333d19f5dfd9656 Mon Sep 17 00:00:00 2001 From: Kayla Reopelle <87386821+kaylareopelle@users.noreply.github.com> Date: Fri, 23 Aug 2024 16:58:00 -0700 Subject: [PATCH 06/13] ci: Cancel workflows in progress on new push to PR (#1679) This ensures only one instance of the workflow is running per PR by canceling any previous runs of the workflow. Co-authored-by: Matthew Wear --- .github/workflows/ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8acd5270ca..1eadcc02c4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,6 +9,10 @@ on: branches: - main +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number }} # Ensure that only one instance of this workflow is running per Pull Request + cancel-in-progress: true # Cancel any previous runs of this workflow + jobs: base: strategy: From a217e9aec8ce9df2baa92ecff224bf67098268cf Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 28 Aug 2024 11:52:18 -0700 Subject: [PATCH 07/13] release: Release 4 gems (#1692) * opentelemetry-api 1.4.0 (was 1.3.0) * opentelemetry-exporter-otlp 0.29.0 (was 0.28.1) * opentelemetry-exporter-otlp-metrics 0.1.0 (initial release) * opentelemetry-metrics-sdk 0.2.0 (was 0.1.0) Co-authored-by: Daniel Azuma --- api/CHANGELOG.md | 4 ++++ api/lib/opentelemetry/version.rb | 2 +- exporter/otlp-metrics/CHANGELOG.md | 4 ++++ exporter/otlp/CHANGELOG.md | 4 ++++ exporter/otlp/lib/opentelemetry/exporter/otlp/version.rb | 2 +- metrics_sdk/CHANGELOG.md | 4 ++++ metrics_sdk/lib/opentelemetry/sdk/metrics/version.rb | 2 +- 7 files changed, 19 insertions(+), 3 deletions(-) diff --git a/api/CHANGELOG.md b/api/CHANGELOG.md index 35c3e87ad9..1cb9acd401 100644 --- a/api/CHANGELOG.md +++ b/api/CHANGELOG.md @@ -1,5 +1,9 @@ # Release History: opentelemetry-api +### v1.4.0 / 2024-08-27 + +* ADDED: Include backtrace first line for better debug info + ### v1.3.0 / 2024-07-24 * ADDED: Add add_link to span api/sdk diff --git a/api/lib/opentelemetry/version.rb b/api/lib/opentelemetry/version.rb index 009daf16ba..86295b3a96 100644 --- a/api/lib/opentelemetry/version.rb +++ b/api/lib/opentelemetry/version.rb @@ -6,5 +6,5 @@ module OpenTelemetry ## Current OpenTelemetry version - VERSION = '1.3.0' + VERSION = '1.4.0' end diff --git a/exporter/otlp-metrics/CHANGELOG.md b/exporter/otlp-metrics/CHANGELOG.md index 94a2c19729..b42812d565 100644 --- a/exporter/otlp-metrics/CHANGELOG.md +++ b/exporter/otlp-metrics/CHANGELOG.md @@ -1 +1,5 @@ # Release History: opentelemetry-exporter-otlp-metrics + +### v0.1.0 / 2024-08-27 + +Initial release. diff --git a/exporter/otlp/CHANGELOG.md b/exporter/otlp/CHANGELOG.md index 88ee03a7dc..bd86c03135 100644 --- a/exporter/otlp/CHANGELOG.md +++ b/exporter/otlp/CHANGELOG.md @@ -1,5 +1,9 @@ # Release History: opentelemetry-exporter-otlp +### v0.29.0 / 2024-08-27 + +* ADDED: Add support for mutual TLS. + ### v0.28.1 / 2024-07-24 * ADDED: Improve SSL error logging. diff --git a/exporter/otlp/lib/opentelemetry/exporter/otlp/version.rb b/exporter/otlp/lib/opentelemetry/exporter/otlp/version.rb index b12e7e7de2..fd37180664 100644 --- a/exporter/otlp/lib/opentelemetry/exporter/otlp/version.rb +++ b/exporter/otlp/lib/opentelemetry/exporter/otlp/version.rb @@ -8,7 +8,7 @@ module OpenTelemetry module Exporter module OTLP ## Current OpenTelemetry OTLP exporter version - VERSION = '0.28.1' + VERSION = '0.29.0' end end end diff --git a/metrics_sdk/CHANGELOG.md b/metrics_sdk/CHANGELOG.md index a1385c5c6b..1328411e29 100644 --- a/metrics_sdk/CHANGELOG.md +++ b/metrics_sdk/CHANGELOG.md @@ -1,5 +1,9 @@ # Release History: opentelemetry-metrics-sdk +### v0.2.0 / 2024-08-27 + +* ADDED: Add basic periodic exporting metric_reader + ### v0.1.0 / 2024-07-31 Initial release. diff --git a/metrics_sdk/lib/opentelemetry/sdk/metrics/version.rb b/metrics_sdk/lib/opentelemetry/sdk/metrics/version.rb index 41bd724c7a..4016895fb7 100644 --- a/metrics_sdk/lib/opentelemetry/sdk/metrics/version.rb +++ b/metrics_sdk/lib/opentelemetry/sdk/metrics/version.rb @@ -8,7 +8,7 @@ module OpenTelemetry module SDK module Metrics # Current OpenTelemetry metrics sdk version - VERSION = '0.1.0' + VERSION = '0.2.0' end end end From 3f96eba2f431d58d13aed723e06b1cd861cd1245 Mon Sep 17 00:00:00 2001 From: Kayla Reopelle <87386821+kaylareopelle@users.noreply.github.com> Date: Wed, 28 Aug 2024 13:05:30 -0700 Subject: [PATCH 08/13] chore: Loosen metrics gems versions (#1698) The OTLP metrics exporter will fail to install for tests because the latest metrics SDK is newer than the previous version constraint. The rest of the gems are installed based on minor versions. Now metrics_sdk/_api are in sync. --- .../otlp-metrics/opentelemetry-exporter-otlp-metrics.gemspec | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/exporter/otlp-metrics/opentelemetry-exporter-otlp-metrics.gemspec b/exporter/otlp-metrics/opentelemetry-exporter-otlp-metrics.gemspec index 571b3b203e..0729f3e377 100644 --- a/exporter/otlp-metrics/opentelemetry-exporter-otlp-metrics.gemspec +++ b/exporter/otlp-metrics/opentelemetry-exporter-otlp-metrics.gemspec @@ -29,8 +29,8 @@ Gem::Specification.new do |spec| spec.add_dependency 'google-protobuf', '>= 3.18', '< 5.0' spec.add_dependency 'opentelemetry-api', '~> 1.1' spec.add_dependency 'opentelemetry-common', '~> 0.20' - spec.add_dependency 'opentelemetry-metrics-api', '~> 0.1.0' - spec.add_dependency 'opentelemetry-metrics-sdk', '~> 0.1.0' + spec.add_dependency 'opentelemetry-metrics-api', '~> 0.1' + spec.add_dependency 'opentelemetry-metrics-sdk', '~> 0.2' spec.add_dependency 'opentelemetry-sdk', '~> 1.2' spec.add_dependency 'opentelemetry-semantic_conventions' From aad3cf7f508795b0ef2c6fb31bab59a94898c714 Mon Sep 17 00:00:00 2001 From: Kayla Reopelle <87386821+kaylareopelle@users.noreply.github.com> Date: Wed, 28 Aug 2024 14:57:34 -0700 Subject: [PATCH 09/13] feat: Add console and in-memory log exporters (#1687) The ConsoleLogRecordExporter will output logs to the console. The InMemoryLogRecordExporter is useful for testing OpenTelemetry integration. Co-authored-by: Matthew Wear --- logs_sdk/lib/opentelemetry/sdk/logs/export.rb | 2 + .../export/console_log_record_exporter.rb | 39 +++++++ .../export/in_memory_log_record_exporter.rb | 104 ++++++++++++++++++ .../console_log_record_exporter_test.rb | 56 ++++++++++ .../in_memory_log_record_exporter_test.rb | 93 ++++++++++++++++ 5 files changed, 294 insertions(+) create mode 100644 logs_sdk/lib/opentelemetry/sdk/logs/export/console_log_record_exporter.rb create mode 100644 logs_sdk/lib/opentelemetry/sdk/logs/export/in_memory_log_record_exporter.rb create mode 100644 logs_sdk/test/opentelemetry/sdk/logs/export/console_log_record_exporter_test.rb create mode 100644 logs_sdk/test/opentelemetry/sdk/logs/export/in_memory_log_record_exporter_test.rb diff --git a/logs_sdk/lib/opentelemetry/sdk/logs/export.rb b/logs_sdk/lib/opentelemetry/sdk/logs/export.rb index 2565dbf85f..414670e86c 100644 --- a/logs_sdk/lib/opentelemetry/sdk/logs/export.rb +++ b/logs_sdk/lib/opentelemetry/sdk/logs/export.rb @@ -24,6 +24,8 @@ module Export end end +require_relative 'export/console_log_record_exporter' +require_relative 'export/in_memory_log_record_exporter' require_relative 'export/log_record_exporter' require_relative 'export/simple_log_record_processor' require_relative 'export/batch_log_record_processor' diff --git a/logs_sdk/lib/opentelemetry/sdk/logs/export/console_log_record_exporter.rb b/logs_sdk/lib/opentelemetry/sdk/logs/export/console_log_record_exporter.rb new file mode 100644 index 0000000000..9f1af854d1 --- /dev/null +++ b/logs_sdk/lib/opentelemetry/sdk/logs/export/console_log_record_exporter.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +# Copyright The OpenTelemetry Authors +# +# SPDX-License-Identifier: Apache-2.0 + +module OpenTelemetry + module SDK + module Logs + module Export + # Outputs {LogRecordData} to the console. + # + # Potentially useful for exploratory purposes. + class ConsoleLogRecordExporter + def initialize + @stopped = false + end + + def export(log_records, timeout: nil) + return FAILURE if @stopped + + Array(log_records).each { |s| pp s } + + SUCCESS + end + + def force_flush(timeout: nil) + SUCCESS + end + + def shutdown(timeout: nil) + @stopped = true + SUCCESS + end + end + end + end + end +end diff --git a/logs_sdk/lib/opentelemetry/sdk/logs/export/in_memory_log_record_exporter.rb b/logs_sdk/lib/opentelemetry/sdk/logs/export/in_memory_log_record_exporter.rb new file mode 100644 index 0000000000..10c181e122 --- /dev/null +++ b/logs_sdk/lib/opentelemetry/sdk/logs/export/in_memory_log_record_exporter.rb @@ -0,0 +1,104 @@ +# frozen_string_literal: true + +# Copyright The OpenTelemetry Authors +# +# SPDX-License-Identifier: Apache-2.0 + +module OpenTelemetry + module SDK + module Logs + module Export + # A LogRecordExporter implementation that can be used to test OpenTelemetry integration. + # + # @example Usage in a test suite: + # class MyClassTest + # def setup + # @logger_provider = LoggerProvider.new + # @exporter = InMemoryLogRecordExporter.new + # @logger_provider.add_log_record_processor(SimpleLogRecordProcessor.new(@exporter)) + # end + # + # def test_emitted_log_records + # log_record = OpenTelemetry::SDK::Logs::LogRecord.new(body: 'log') + # @logger_provider.logger.on_emit(log_record, context) + # + # log_records = @exporter.emitted_log_records + + # refute_nil(log_records) + # assert_equal(1, log_records.size) + # assert_equal(log_records[0].body, 'log') + # end + # end + class InMemoryLogRecordExporter + # Returns a new instance of the {InMemoryLogRecordExporter}. + # + # @return a new instance of the {InMemoryLogRecordExporter}. + def initialize + @emitted_log_records = [] + @stopped = false + @mutex = Mutex.new + end + + # Returns a frozen array of the emitted {LogRecordData}s, represented by + # {io.opentelemetry.proto.trace.v1.LogRecord}. + # + # @return [Array] a frozen array of the emitted {LogRecordData}s. + def emitted_log_records + @mutex.synchronize do + @emitted_log_records.clone.freeze + end + end + + # Clears the internal collection of emitted {LogRecord}s. + # + # Does not reset the state of this exporter if already shutdown. + def reset + @mutex.synchronize do + @emitted_log_records.clear + end + end + + # Called to export {LogRecordData}s. + # + # @param [Enumerable] log_record_datas the list of {LogRecordData}s to be + # exported. + # @param [optional Numeric] timeout An optional timeout in seconds. + # @return [Integer] the result of the export, SUCCESS or + # FAILURE + def export(log_record_datas, timeout: nil) + @mutex.synchronize do + return FAILURE if @stopped + + @emitted_log_records.concat(log_record_datas.to_a) + end + SUCCESS + end + + # Called when {LoggerProvider#force_flush} is called, if this exporter is + # registered to a {LoggerProvider} object. + # + # @param [optional Numeric] timeout An optional timeout in seconds. + # @return [Integer] SUCCESS if no error occurred, FAILURE if a + # non-specific failure occurred, TIMEOUT if a timeout occurred. + def force_flush(timeout: nil) + SUCCESS + end + + # Called when {LoggerProvider#shutdown} is called, if this exporter is + # registered to a {LoggerProvider} object. + # + # @param [optional Numeric] timeout An optional timeout in seconds. + # @return [Integer] SUCCESS if no error occurred, FAILURE if a + # non-specific failure occurred, TIMEOUT if a timeout occurred. + def shutdown(timeout: nil) + @mutex.synchronize do + @emitted_log_records.clear + @stopped = true + end + SUCCESS + end + end + end + end + end +end diff --git a/logs_sdk/test/opentelemetry/sdk/logs/export/console_log_record_exporter_test.rb b/logs_sdk/test/opentelemetry/sdk/logs/export/console_log_record_exporter_test.rb new file mode 100644 index 0000000000..b549bc944a --- /dev/null +++ b/logs_sdk/test/opentelemetry/sdk/logs/export/console_log_record_exporter_test.rb @@ -0,0 +1,56 @@ +# frozen_string_literal: true + +# Copyright The OpenTelemetry Authors +# +# SPDX-License-Identifier: Apache-2.0 + +require 'test_helper' + +describe OpenTelemetry::SDK::Logs::Export::ConsoleLogRecordExporter do + export = OpenTelemetry::SDK::Logs::Export + + let(:captured_stdout) { StringIO.new } + let(:log_record_data1) { Logs::LogRecordData.new } + let(:log_record_data2) { Logs::LogRecordData.new } + let(:log_records) { [log_record_data1, log_record_data2] } + let(:exporter) { export::ConsoleLogRecordExporter.new } + + before do + @original_stdout = $stdout + $stdout = captured_stdout + end + + after do + $stdout = @original_stdout + end + + it 'accepts an Array of LogRecordData as arg to #export and succeeds' do + assert_equal(export::SUCCESS, exporter.export(log_records)) + end + + it 'accepts an Enumerable of LogRecordData as arg to #export and succeeds' do + enumerable = Struct.new(:log_record0, :log_record1).new(log_records[0], log_records[1]) + + assert_equal(export::SUCCESS, exporter.export(enumerable)) + end + + it 'outputs to console (stdout)' do + exporter.export(log_records) + + assert_match(/# Date: Wed, 28 Aug 2024 18:19:24 -0400 Subject: [PATCH 10/13] fix: remove WRITE_TIMEOUT_SUPPORTED (#1673) Co-authored-by: Matthew Wear --- .../lib/opentelemetry/exporter/otlp/http/trace_exporter.rb | 7 +++---- exporter/otlp/lib/opentelemetry/exporter/otlp/exporter.rb | 7 +++---- .../zipkin/lib/opentelemetry/exporter/zipkin/exporter.rb | 5 ++--- 3 files changed, 8 insertions(+), 11 deletions(-) diff --git a/exporter/otlp-http/lib/opentelemetry/exporter/otlp/http/trace_exporter.rb b/exporter/otlp-http/lib/opentelemetry/exporter/otlp/http/trace_exporter.rb index e71c1a9457..f6f0ecd98e 100644 --- a/exporter/otlp-http/lib/opentelemetry/exporter/otlp/http/trace_exporter.rb +++ b/exporter/otlp-http/lib/opentelemetry/exporter/otlp/http/trace_exporter.rb @@ -25,8 +25,7 @@ class TraceExporter # rubocop:disable Metrics/ClassLength # Default timeouts in seconds. KEEP_ALIVE_TIMEOUT = 30 RETRY_COUNT = 5 - WRITE_TIMEOUT_SUPPORTED = Gem::Version.new(RUBY_VERSION) >= Gem::Version.new('2.6') - private_constant(:KEEP_ALIVE_TIMEOUT, :RETRY_COUNT, :WRITE_TIMEOUT_SUPPORTED) + private_constant(:KEEP_ALIVE_TIMEOUT, :RETRY_COUNT) ERROR_MESSAGE_INVALID_HEADERS = 'headers must be a String with comma-separated URL Encoded UTF-8 k=v pairs or a Hash' private_constant(:ERROR_MESSAGE_INVALID_HEADERS) @@ -153,7 +152,7 @@ def send_bytes(bytes, timeout:) # rubocop:disable Metrics/MethodLength @http.open_timeout = remaining_timeout @http.read_timeout = remaining_timeout - @http.write_timeout = remaining_timeout if WRITE_TIMEOUT_SUPPORTED + @http.write_timeout = remaining_timeout @http.start unless @http.started? response = measure_request_duration { @http.request(request) } @@ -209,7 +208,7 @@ def send_bytes(bytes, timeout:) # rubocop:disable Metrics/MethodLength # Reset timeouts to defaults for the next call. @http.open_timeout = @timeout @http.read_timeout = @timeout - @http.write_timeout = @timeout if WRITE_TIMEOUT_SUPPORTED + @http.write_timeout = @timeout end def handle_redirect(location) diff --git a/exporter/otlp/lib/opentelemetry/exporter/otlp/exporter.rb b/exporter/otlp/lib/opentelemetry/exporter/otlp/exporter.rb index 6083692ffc..35754f1e91 100644 --- a/exporter/otlp/lib/opentelemetry/exporter/otlp/exporter.rb +++ b/exporter/otlp/lib/opentelemetry/exporter/otlp/exporter.rb @@ -28,8 +28,7 @@ class Exporter # rubocop:disable Metrics/ClassLength # Default timeouts in seconds. KEEP_ALIVE_TIMEOUT = 30 RETRY_COUNT = 5 - WRITE_TIMEOUT_SUPPORTED = Gem::Version.new(RUBY_VERSION) >= Gem::Version.new('2.6') - private_constant(:KEEP_ALIVE_TIMEOUT, :RETRY_COUNT, :WRITE_TIMEOUT_SUPPORTED) + private_constant(:KEEP_ALIVE_TIMEOUT, :RETRY_COUNT) ERROR_MESSAGE_INVALID_HEADERS = 'headers must be a String with comma-separated URL Encoded UTF-8 k=v pairs or a Hash' private_constant(:ERROR_MESSAGE_INVALID_HEADERS) @@ -153,7 +152,7 @@ def send_bytes(bytes, timeout:) # rubocop:disable Metrics/CyclomaticComplexity, @http.open_timeout = remaining_timeout @http.read_timeout = remaining_timeout - @http.write_timeout = remaining_timeout if WRITE_TIMEOUT_SUPPORTED + @http.write_timeout = remaining_timeout @http.start unless @http.started? response = measure_request_duration { @http.request(request) } @@ -213,7 +212,7 @@ def send_bytes(bytes, timeout:) # rubocop:disable Metrics/CyclomaticComplexity, # Reset timeouts to defaults for the next call. @http.open_timeout = @timeout @http.read_timeout = @timeout - @http.write_timeout = @timeout if WRITE_TIMEOUT_SUPPORTED + @http.write_timeout = @timeout end def handle_redirect(location) diff --git a/exporter/zipkin/lib/opentelemetry/exporter/zipkin/exporter.rb b/exporter/zipkin/lib/opentelemetry/exporter/zipkin/exporter.rb index 43efbee2cb..779df0022d 100644 --- a/exporter/zipkin/lib/opentelemetry/exporter/zipkin/exporter.rb +++ b/exporter/zipkin/lib/opentelemetry/exporter/zipkin/exporter.rb @@ -24,8 +24,7 @@ class Exporter # rubocop:disable Metrics/ClassLength # Default timeouts in seconds. KEEP_ALIVE_TIMEOUT = 30 RETRY_COUNT = 5 - WRITE_TIMEOUT_SUPPORTED = Gem::Version.new(RUBY_VERSION) >= Gem::Version.new('2.6') - private_constant(:KEEP_ALIVE_TIMEOUT, :RETRY_COUNT, :WRITE_TIMEOUT_SUPPORTED) + private_constant(:KEEP_ALIVE_TIMEOUT, :RETRY_COUNT) def initialize(endpoint: OpenTelemetry::Common::Utilities.config_opt('OTEL_EXPORTER_ZIPKIN_ENDPOINT', default: 'http://localhost:9411/api/v2/spans'), headers: OpenTelemetry::Common::Utilities.config_opt('OTEL_EXPORTER_ZIPKIN_TRACES_HEADERS', 'OTEL_EXPORTER_ZIPKIN_HEADERS'), @@ -130,7 +129,7 @@ def send_spans(zipkin_spans, timeout: nil) # rubocop:disable Metrics/MethodLengt @http.open_timeout = remaining_timeout @http.read_timeout = remaining_timeout - @http.write_timeout = remaining_timeout if WRITE_TIMEOUT_SUPPORTED + @http.write_timeout = remaining_timeout @http.start unless @http.started? response = measure_request_duration { @http.request(request) } From 75a48853b3525231a06aa17b0f76a247f4782060 Mon Sep 17 00:00:00 2001 From: Xuan <112967240+xuan-cao-swi@users.noreply.github.com> Date: Fri, 30 Aug 2024 18:23:21 -0400 Subject: [PATCH 11/13] Rubocop on metrics (#1683) * fix: use empty hash as default value for metric instrument attributes * fix: update otlp metrics rubocop yml * lint --------- Co-authored-by: Matthew Wear --- exporter/otlp-metrics/.rubocop.yml | 29 +- .../exporter/otlp/metrics/metrics_exporter.rb | 8 +- .../exporter/otlp/metrics/util.rb | 2 +- .../exporter/otlp/metrics_exporter.rb | 309 ++++++++++++++++++ .../lib/opentelemetry/exporter/otlp/util.rb | 139 ++++++++ .../metrics/instrument/counter.rb | 2 +- .../metrics/instrument/histogram.rb | 2 +- .../metrics/instrument/up_down_counter.rb | 2 +- .../sdk/metrics/instrument/histogram.rb | 2 +- .../sdk/metrics/instrument/up_down_counter.rb | 2 +- 10 files changed, 467 insertions(+), 30 deletions(-) create mode 100644 exporter/otlp-metrics/lib/opentelemetry/exporter/otlp/metrics_exporter.rb create mode 100644 exporter/otlp-metrics/lib/opentelemetry/exporter/otlp/util.rb diff --git a/exporter/otlp-metrics/.rubocop.yml b/exporter/otlp-metrics/.rubocop.yml index 684add8d91..52c76f71be 100644 --- a/exporter/otlp-metrics/.rubocop.yml +++ b/exporter/otlp-metrics/.rubocop.yml @@ -1,32 +1,21 @@ +inherit_from: ../../contrib/rubocop.yml + AllCops: - TargetRubyVersion: "3.0" - NewCops: disable - SuggestExtensions: false Exclude: - "lib/opentelemetry/proto/**/*" - "vendor/**/*" -Bundler/OrderedGems: - Exclude: - - gemfiles/**/* -Lint/UnusedMethodArgument: - Enabled: false -Lint/MissingSuper: - Enabled: false -Lint/ConstantDefinitionInBlock: - Exclude: - - "test/**/*" -Style/StringConcatenation: - Exclude: - - "test/**/*" -Metrics/AbcSize: +Metrics/CyclomaticComplexity: Enabled: false -Layout/LineLength: +Metrics/PerceivedComplexity: Enabled: false Metrics/MethodLength: - Max: 20 -Metrics/ParameterLists: Enabled: false +Metrics/ClassLength: + Enabled: false +Bundler/OrderedGems: + Exclude: + - gemfiles/**/* Style/FrozenStringLiteralComment: Exclude: - gemfiles/**/* diff --git a/exporter/otlp-metrics/lib/opentelemetry/exporter/otlp/metrics/metrics_exporter.rb b/exporter/otlp-metrics/lib/opentelemetry/exporter/otlp/metrics/metrics_exporter.rb index be21d5dde2..a1b0bf415f 100644 --- a/exporter/otlp-metrics/lib/opentelemetry/exporter/otlp/metrics/metrics_exporter.rb +++ b/exporter/otlp-metrics/lib/opentelemetry/exporter/otlp/metrics/metrics_exporter.rb @@ -26,7 +26,7 @@ module Exporter module OTLP module Metrics # An OpenTelemetry metrics exporter that sends metrics over HTTP as Protobuf encoded OTLP ExportMetricsServiceRequest. - class MetricsExporter < ::OpenTelemetry::SDK::Metrics::Export::MetricReader # rubocop:disable Metrics/ClassLength + class MetricsExporter < ::OpenTelemetry::SDK::Metrics::Export::MetricReader include Util attr_reader :metric_snapshots @@ -87,7 +87,7 @@ def export(metrics, timeout: nil) end end - def send_bytes(bytes, timeout:) # rubocop:disable Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity + def send_bytes(bytes, timeout:) return FAILURE if bytes.nil? request = Net::HTTP::Post.new(@path) @@ -181,7 +181,7 @@ def send_bytes(bytes, timeout:) # rubocop:disable Metrics/CyclomaticComplexity, @http.write_timeout = @timeout end - def encode(metrics_data) # rubocop:disable Metrics/MethodLength, Metrics/CyclomaticComplexity + def encode(metrics_data) Opentelemetry::Proto::Collector::Metrics::V1::ExportMetricsServiceRequest.encode( Opentelemetry::Proto::Collector::Metrics::V1::ExportMetricsServiceRequest.new( resource_metrics: metrics_data @@ -215,7 +215,7 @@ def encode(metrics_data) # rubocop:disable Metrics/MethodLength, Metrics/Cycloma # current metric sdk only implements instrument: :counter -> :sum, :histogram -> :histogram # # metrics [MetricData] - def as_otlp_metrics(metrics) # rubocop:disable Metrics/MethodLength + def as_otlp_metrics(metrics) case metrics.instrument_kind when :observable_gauge Opentelemetry::Proto::Metrics::V1::Metric.new( diff --git a/exporter/otlp-metrics/lib/opentelemetry/exporter/otlp/metrics/util.rb b/exporter/otlp-metrics/lib/opentelemetry/exporter/otlp/metrics/util.rb index 568c9a0e48..264b1e19bf 100644 --- a/exporter/otlp-metrics/lib/opentelemetry/exporter/otlp/metrics/util.rb +++ b/exporter/otlp-metrics/lib/opentelemetry/exporter/otlp/metrics/util.rb @@ -97,7 +97,7 @@ def parse_headers(raw) end end - def backoff?(retry_count:, reason:, retry_after: nil) # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity + def backoff?(retry_count:, reason:, retry_after: nil) return false if retry_count > RETRY_COUNT sleep_interval = nil diff --git a/exporter/otlp-metrics/lib/opentelemetry/exporter/otlp/metrics_exporter.rb b/exporter/otlp-metrics/lib/opentelemetry/exporter/otlp/metrics_exporter.rb new file mode 100644 index 0000000000..3d86f9e979 --- /dev/null +++ b/exporter/otlp-metrics/lib/opentelemetry/exporter/otlp/metrics_exporter.rb @@ -0,0 +1,309 @@ +# frozen_string_literal: true + +# Copyright The OpenTelemetry Authors +# +# SPDX-License-Identifier: Apache-2.0 + +require 'opentelemetry/common' +require 'opentelemetry/sdk' +require 'net/http' +require 'csv' +require 'zlib' + +require 'google/rpc/status_pb' + +require 'opentelemetry/proto/common/v1/common_pb' +require 'opentelemetry/proto/resource/v1/resource_pb' +require 'opentelemetry/proto/metrics/v1/metrics_pb' +require 'opentelemetry/proto/collector/metrics/v1/metrics_service_pb' + +require 'opentelemetry/metrics' +require 'opentelemetry/sdk/metrics' + +require_relative './util' + +module OpenTelemetry + module Exporter + module OTLP + # An OpenTelemetry metrics exporter that sends metrics over HTTP as Protobuf encoded OTLP ExportMetricsServiceRequest. + class MetricsExporter < ::OpenTelemetry::SDK::Metrics::Export::MetricReader + include Util + + attr_reader :metric_snapshots + + SUCCESS = OpenTelemetry::SDK::Metrics::Export::SUCCESS + FAILURE = OpenTelemetry::SDK::Metrics::Export::FAILURE + private_constant(:SUCCESS, :FAILURE) + + WRITE_TIMEOUT_SUPPORTED = Gem::Version.new(RUBY_VERSION) >= Gem::Version.new('2.6') + private_constant(:WRITE_TIMEOUT_SUPPORTED) + + def self.ssl_verify_mode + if ENV.key?('OTEL_RUBY_EXPORTER_OTLP_SSL_VERIFY_PEER') + OpenSSL::SSL::VERIFY_PEER + elsif ENV.key?('OTEL_RUBY_EXPORTER_OTLP_SSL_VERIFY_NONE') + OpenSSL::SSL::VERIFY_NONE + else + OpenSSL::SSL::VERIFY_PEER + end + end + + def initialize(endpoint: OpenTelemetry::Common::Utilities.config_opt('OTEL_EXPORTER_OTLP_METRICS_ENDPOINT', 'OTEL_EXPORTER_OTLP_ENDPOINT', default: 'http://localhost:4318/v1/metrics'), + certificate_file: OpenTelemetry::Common::Utilities.config_opt('OTEL_EXPORTER_OTLP_METRICS_CERTIFICATE', 'OTEL_EXPORTER_OTLP_CERTIFICATE'), + ssl_verify_mode: MetricsExporter.ssl_verify_mode, + headers: OpenTelemetry::Common::Utilities.config_opt('OTEL_EXPORTER_OTLP_METRICS_HEADERS', 'OTEL_EXPORTER_OTLP_HEADERS', default: {}), + compression: OpenTelemetry::Common::Utilities.config_opt('OTEL_EXPORTER_OTLP_METRICS_COMPRESSION', 'OTEL_EXPORTER_OTLP_COMPRESSION', default: 'gzip'), + timeout: OpenTelemetry::Common::Utilities.config_opt('OTEL_EXPORTER_OTLP_METRICS_TIMEOUT', 'OTEL_EXPORTER_OTLP_TIMEOUT', default: 10)) + raise ArgumentError, "invalid url for OTLP::MetricsExporter #{endpoint}" unless OpenTelemetry::Common::Utilities.valid_url?(endpoint) + raise ArgumentError, "unsupported compression key #{compression}" unless compression.nil? || %w[gzip none].include?(compression) + + # create the MetricStore object + super() + + @uri = if endpoint == ENV['OTEL_EXPORTER_OTLP_ENDPOINT'] + URI.join(endpoint, 'v1/metrics') + else + URI(endpoint) + end + + @http = http_connection(@uri, ssl_verify_mode, certificate_file) + + @path = @uri.path + @headers = prepare_headers(headers) + @timeout = timeout.to_f + @compression = compression + @mutex = Mutex.new + @shutdown = false + end + + # consolidate the metrics data into the form of MetricData + # + # return MetricData + def pull + export(collect) + end + + # metrics Array[MetricData] + def export(metrics, timeout: nil) + @mutex.synchronize do + send_bytes(encode(metrics), timeout: timeout) + end + end + + def send_bytes(bytes, timeout:) + return FAILURE if bytes.nil? + + request = Net::HTTP::Post.new(@path) + + if @compression == 'gzip' + request.add_field('Content-Encoding', 'gzip') + body = Zlib.gzip(bytes) + else + body = bytes + end + + request.body = body + request.add_field('Content-Type', 'application/x-protobuf') + @headers.each { |key, value| request.add_field(key, value) } + + retry_count = 0 + timeout ||= @timeout + start_time = OpenTelemetry::Common::Utilities.timeout_timestamp + + around_request do + remaining_timeout = OpenTelemetry::Common::Utilities.maybe_timeout(timeout, start_time) + return FAILURE if remaining_timeout.zero? + + @http.open_timeout = remaining_timeout + @http.read_timeout = remaining_timeout + @http.write_timeout = remaining_timeout if WRITE_TIMEOUT_SUPPORTED + @http.start unless @http.started? + response = measure_request_duration { @http.request(request) } + case response + when Net::HTTPOK + response.body # Read and discard body + SUCCESS + when Net::HTTPServiceUnavailable, Net::HTTPTooManyRequests + response.body # Read and discard body + redo if backoff?(retry_after: response['Retry-After'], retry_count: retry_count += 1, reason: response.code) + OpenTelemetry.logger.warn('Net::HTTPServiceUnavailable/Net::HTTPTooManyRequests in MetricsExporter#send_bytes') + FAILURE + when Net::HTTPRequestTimeOut, Net::HTTPGatewayTimeOut, Net::HTTPBadGateway + response.body # Read and discard body + redo if backoff?(retry_count: retry_count += 1, reason: response.code) + OpenTelemetry.logger.warn('Net::HTTPRequestTimeOut/Net::HTTPGatewayTimeOut/Net::HTTPBadGateway in MetricsExporter#send_bytes') + FAILURE + when Net::HTTPNotFound + OpenTelemetry.handle_error(message: "OTLP metrics_exporter received http.code=404 for uri: '#{@path}'") + FAILURE + when Net::HTTPBadRequest, Net::HTTPClientError, Net::HTTPServerError + log_status(response.body) + OpenTelemetry.logger.warn('Net::HTTPBadRequest/Net::HTTPClientError/Net::HTTPServerError in MetricsExporter#send_bytes') + FAILURE + when Net::HTTPRedirection + @http.finish + handle_redirect(response['location']) + redo if backoff?(retry_after: 0, retry_count: retry_count += 1, reason: response.code) + else + @http.finish + OpenTelemetry.logger.warn("Unexpected error in OTLP::MetricsExporter#send_bytes - #{response.message}") + FAILURE + end + rescue Net::OpenTimeout, Net::ReadTimeout + retry if backoff?(retry_count: retry_count += 1, reason: 'timeout') + OpenTelemetry.logger.warn('Net::OpenTimeout/Net::ReadTimeout in MetricsExporter#send_bytes') + return FAILURE + rescue OpenSSL::SSL::SSLError + retry if backoff?(retry_count: retry_count += 1, reason: 'openssl_error') + OpenTelemetry.logger.warn('OpenSSL::SSL::SSLError in MetricsExporter#send_bytes') + return FAILURE + rescue SocketError + retry if backoff?(retry_count: retry_count += 1, reason: 'socket_error') + OpenTelemetry.logger.warn('SocketError in MetricsExporter#send_bytes') + return FAILURE + rescue SystemCallError => e + retry if backoff?(retry_count: retry_count += 1, reason: e.class.name) + OpenTelemetry.logger.warn('SystemCallError in MetricsExporter#send_bytes') + return FAILURE + rescue EOFError + retry if backoff?(retry_count: retry_count += 1, reason: 'eof_error') + OpenTelemetry.logger.warn('EOFError in MetricsExporter#send_bytes') + return FAILURE + rescue Zlib::DataError + retry if backoff?(retry_count: retry_count += 1, reason: 'zlib_error') + OpenTelemetry.logger.warn('Zlib::DataError in MetricsExporter#send_bytes') + return FAILURE + rescue StandardError => e + OpenTelemetry.handle_error(exception: e, message: 'unexpected error in OTLP::MetricsExporter#send_bytes') + return FAILURE + end + ensure + # Reset timeouts to defaults for the next call. + @http.open_timeout = @timeout + @http.read_timeout = @timeout + @http.write_timeout = @timeout if WRITE_TIMEOUT_SUPPORTED + end + + def encode(metrics_data) + Opentelemetry::Proto::Collector::Metrics::V1::ExportMetricsServiceRequest.encode( + Opentelemetry::Proto::Collector::Metrics::V1::ExportMetricsServiceRequest.new( + resource_metrics: metrics_data + .group_by(&:resource) + .map do |resource, scope_metrics| + Opentelemetry::Proto::Metrics::V1::ResourceMetrics.new( + resource: Opentelemetry::Proto::Resource::V1::Resource.new( + attributes: resource.attribute_enumerator.map { |key, value| as_otlp_key_value(key, value) } + ), + scope_metrics: scope_metrics + .group_by(&:instrumentation_scope) + .map do |instrumentation_scope, metrics| + Opentelemetry::Proto::Metrics::V1::ScopeMetrics.new( + scope: Opentelemetry::Proto::Common::V1::InstrumentationScope.new( + name: instrumentation_scope.name, + version: instrumentation_scope.version + ), + metrics: metrics.map { |sd| as_otlp_metrics(sd) } + ) + end + ) + end + ) + ) + rescue StandardError => e + OpenTelemetry.handle_error(exception: e, message: 'unexpected error in OTLP::MetricsExporter#encode') + nil + end + + # metrics_pb has following type of data: :gauge, :sum, :histogram, :exponential_histogram, :summary + # current metric sdk only implements instrument: :counter -> :sum, :histogram -> :histogram + # + # metrics [MetricData] + def as_otlp_metrics(metrics) + case metrics.instrument_kind + when :observable_gauge + Opentelemetry::Proto::Metrics::V1::Metric.new( + name: metrics.name, + description: metrics.description, + unit: metrics.unit, + gauge: Opentelemetry::Proto::Metrics::V1::Gauge.new( + aggregation_temporality: as_otlp_aggregation_temporality(metrics.aggregation_temporality), + data_points: metrics.data_points.map do |ndp| + number_data_point(ndp) + end + ) + ) + + when :counter, :up_down_counter + Opentelemetry::Proto::Metrics::V1::Metric.new( + name: metrics.name, + description: metrics.description, + unit: metrics.unit, + sum: Opentelemetry::Proto::Metrics::V1::Sum.new( + aggregation_temporality: as_otlp_aggregation_temporality(metrics.aggregation_temporality), + data_points: metrics.data_points.map do |ndp| + number_data_point(ndp) + end + ) + ) + + when :histogram + Opentelemetry::Proto::Metrics::V1::Metric.new( + name: metrics.name, + description: metrics.description, + unit: metrics.unit, + histogram: Opentelemetry::Proto::Metrics::V1::Histogram.new( + aggregation_temporality: as_otlp_aggregation_temporality(metrics.aggregation_temporality), + data_points: metrics.data_points.map do |hdp| + histogram_data_point(hdp) + end + ) + ) + end + end + + def as_otlp_aggregation_temporality(type) + case type + when :delta then Opentelemetry::Proto::Metrics::V1::AggregationTemporality::AGGREGATION_TEMPORALITY_DELTA + when :cumulative then Opentelemetry::Proto::Metrics::V1::AggregationTemporality::AGGREGATION_TEMPORALITY_CUMULATIVE + else Opentelemetry::Proto::Metrics::V1::AggregationTemporality::AGGREGATION_TEMPORALITY_UNSPECIFIED + end + end + + def histogram_data_point(hdp) + Opentelemetry::Proto::Metrics::V1::HistogramDataPoint.new( + attributes: hdp.attributes.map { |k, v| as_otlp_key_value(k, v) }, + start_time_unix_nano: hdp.start_time_unix_nano, + time_unix_nano: hdp.time_unix_nano, + count: hdp.count, + sum: hdp.sum, + bucket_counts: hdp.bucket_counts, + explicit_bounds: hdp.explicit_bounds, + exemplars: hdp.exemplars, + min: hdp.min, + max: hdp.max + ) + end + + def number_data_point(ndp) + Opentelemetry::Proto::Metrics::V1::NumberDataPoint.new( + attributes: ndp.attributes.map { |k, v| as_otlp_key_value(k, v) }, + as_int: ndp.value, + start_time_unix_nano: ndp.start_time_unix_nano, + time_unix_nano: ndp.time_unix_nano, + exemplars: ndp.exemplars # exemplars not implemented yet from metrics sdk + ) + end + + # may not need this + def reset + SUCCESS + end + + def shutdown(timeout: nil) + @shutdown = true + SUCCESS + end + end + end + end +end diff --git a/exporter/otlp-metrics/lib/opentelemetry/exporter/otlp/util.rb b/exporter/otlp-metrics/lib/opentelemetry/exporter/otlp/util.rb new file mode 100644 index 0000000000..85070a6f43 --- /dev/null +++ b/exporter/otlp-metrics/lib/opentelemetry/exporter/otlp/util.rb @@ -0,0 +1,139 @@ +# frozen_string_literal: true + +# Copyright The OpenTelemetry Authors +# +# SPDX-License-Identifier: Apache-2.0 + +module OpenTelemetry + module Exporter + module OTLP + # Util module provide essential functionality for exporter + module Util # rubocop:disable Metrics/ModuleLength + KEEP_ALIVE_TIMEOUT = 30 + RETRY_COUNT = 5 + ERROR_MESSAGE_INVALID_HEADERS = 'headers must be a String with comma-separated URL Encoded UTF-8 k=v pairs or a Hash' + DEFAULT_USER_AGENT = "OTel-OTLP-MetricsExporter-Ruby/#{OpenTelemetry::Exporter::OTLP::VERSION} Ruby/#{RUBY_VERSION} (#{RUBY_PLATFORM}; #{RUBY_ENGINE}/#{RUBY_ENGINE_VERSION})".freeze + + def http_connection(uri, ssl_verify_mode, certificate_file) + http = Net::HTTP.new(uri.host, uri.port) + http.use_ssl = uri.scheme == 'https' + http.verify_mode = ssl_verify_mode + http.ca_file = certificate_file unless certificate_file.nil? + http.keep_alive_timeout = KEEP_ALIVE_TIMEOUT + http + end + + def around_request + OpenTelemetry::Common::Utilities.untraced { yield } # rubocop:disable Style/ExplicitBlockArgument + end + + def as_otlp_key_value(key, value) + Opentelemetry::Proto::Common::V1::KeyValue.new(key: key, value: as_otlp_any_value(value)) + rescue Encoding::UndefinedConversionError => e + encoded_value = value.encode('UTF-8', invalid: :replace, undef: :replace, replace: '�') + OpenTelemetry.handle_error(exception: e, message: "encoding error for key #{key} and value #{encoded_value}") + Opentelemetry::Proto::Common::V1::KeyValue.new(key: key, value: as_otlp_any_value('Encoding Error')) + end + + def as_otlp_any_value(value) + result = Opentelemetry::Proto::Common::V1::AnyValue.new + case value + when String + result.string_value = value + when Integer + result.int_value = value + when Float + result.double_value = value + when true, false + result.bool_value = value + when Array + values = value.map { |element| as_otlp_any_value(element) } + result.array_value = Opentelemetry::Proto::Common::V1::ArrayValue.new(values: values) + end + result + end + + def prepare_headers(config_headers) + headers = case config_headers + when String then parse_headers(config_headers) + when Hash then config_headers.dup + else + raise ArgumentError, ERROR_MESSAGE_INVALID_HEADERS + end + + headers['User-Agent'] = "#{headers.fetch('User-Agent', '')} #{DEFAULT_USER_AGENT}".strip + + headers + end + + def measure_request_duration + start = Process.clock_gettime(Process::CLOCK_MONOTONIC) + begin + yield + ensure + stop = Process.clock_gettime(Process::CLOCK_MONOTONIC) + 1000.0 * (stop - start) + end + end + + def parse_headers(raw) + entries = raw.split(',') + raise ArgumentError, ERROR_MESSAGE_INVALID_HEADERS if entries.empty? + + entries.each_with_object({}) do |entry, headers| + k, v = entry.split('=', 2).map(&CGI.method(:unescape)) + begin + k = k.to_s.strip + v = v.to_s.strip + rescue Encoding::CompatibilityError + raise ArgumentError, ERROR_MESSAGE_INVALID_HEADERS + rescue ArgumentError => e + raise e, ERROR_MESSAGE_INVALID_HEADERS + end + raise ArgumentError, ERROR_MESSAGE_INVALID_HEADERS if k.empty? || v.empty? + + headers[k] = v + end + end + + def backoff?(retry_count:, reason:, retry_after: nil) + return false if retry_count > RETRY_COUNT + + sleep_interval = nil + unless retry_after.nil? + sleep_interval = + begin + Integer(retry_after) + rescue ArgumentError + nil + end + sleep_interval ||= + begin + Time.httpdate(retry_after) - Time.now + rescue # rubocop:disable Style/RescueStandardError + nil + end + sleep_interval = nil unless sleep_interval&.positive? + end + sleep_interval ||= rand(2**retry_count) + + sleep(sleep_interval) + true + end + + def log_status(body) + status = Google::Rpc::Status.decode(body) + details = status.details.map do |detail| + klass_or_nil = ::Google::Protobuf::DescriptorPool.generated_pool.lookup(detail.type_name).msgclass + detail.unpack(klass_or_nil) if klass_or_nil + end.compact + OpenTelemetry.handle_error(message: "OTLP metrics_exporter received rpc.Status{message=#{status.message}, details=#{details}}") + rescue StandardError => e + OpenTelemetry.handle_error(exception: e, message: 'unexpected error decoding rpc.Status in OTLP::MetricsExporter#log_status') + end + + def handle_redirect(location); end + end + end + end +end diff --git a/metrics_api/lib/opentelemetry/metrics/instrument/counter.rb b/metrics_api/lib/opentelemetry/metrics/instrument/counter.rb index 870364ec89..f9e7225f0a 100644 --- a/metrics_api/lib/opentelemetry/metrics/instrument/counter.rb +++ b/metrics_api/lib/opentelemetry/metrics/instrument/counter.rb @@ -16,7 +16,7 @@ class Counter # Values must be non-nil and (array of) string, boolean or numeric type. # Array values must not contain nil elements and all elements must be of # the same basic type (string, numeric, boolean). - def add(increment, attributes: nil); end + def add(increment, attributes: {}); end end end end diff --git a/metrics_api/lib/opentelemetry/metrics/instrument/histogram.rb b/metrics_api/lib/opentelemetry/metrics/instrument/histogram.rb index 1de448baab..3672a08817 100644 --- a/metrics_api/lib/opentelemetry/metrics/instrument/histogram.rb +++ b/metrics_api/lib/opentelemetry/metrics/instrument/histogram.rb @@ -16,7 +16,7 @@ class Histogram # Values must be non-nil and (array of) string, boolean or numeric type. # Array values must not contain nil elements and all elements must be of # the same basic type (string, numeric, boolean). - def record(amount, attributes: nil); end + def record(amount, attributes: {}); end end end end diff --git a/metrics_api/lib/opentelemetry/metrics/instrument/up_down_counter.rb b/metrics_api/lib/opentelemetry/metrics/instrument/up_down_counter.rb index f9793d1508..b9a943db91 100644 --- a/metrics_api/lib/opentelemetry/metrics/instrument/up_down_counter.rb +++ b/metrics_api/lib/opentelemetry/metrics/instrument/up_down_counter.rb @@ -16,7 +16,7 @@ class UpDownCounter # Values must be non-nil and (array of) string, boolean or numeric type. # Array values must not contain nil elements and all elements must be of # the same basic type (string, numeric, boolean). - def add(amount, attributes: nil); end + def add(amount, attributes: {}); end end end end diff --git a/metrics_sdk/lib/opentelemetry/sdk/metrics/instrument/histogram.rb b/metrics_sdk/lib/opentelemetry/sdk/metrics/instrument/histogram.rb index 9cdcf60d80..5c8e00f157 100644 --- a/metrics_sdk/lib/opentelemetry/sdk/metrics/instrument/histogram.rb +++ b/metrics_sdk/lib/opentelemetry/sdk/metrics/instrument/histogram.rb @@ -24,7 +24,7 @@ def instrument_kind # Values must be non-nil and (array of) string, boolean or numeric type. # Array values must not contain nil elements and all elements must be of # the same basic type (string, numeric, boolean). - def record(amount, attributes: nil) + def record(amount, attributes: {}) update(amount, attributes) nil rescue StandardError => e diff --git a/metrics_sdk/lib/opentelemetry/sdk/metrics/instrument/up_down_counter.rb b/metrics_sdk/lib/opentelemetry/sdk/metrics/instrument/up_down_counter.rb index cf2dc0d8b4..bba734bb0c 100644 --- a/metrics_sdk/lib/opentelemetry/sdk/metrics/instrument/up_down_counter.rb +++ b/metrics_sdk/lib/opentelemetry/sdk/metrics/instrument/up_down_counter.rb @@ -24,7 +24,7 @@ def instrument_kind # Values must be non-nil and (array of) string, boolean or numeric type. # Array values must not contain nil elements and all elements must be of # the same basic type (string, numeric, boolean). - def add(amount, attributes: nil) + def add(amount, attributes: {}) update(amount, attributes) nil rescue StandardError => e From 0f5cbfc6f607bb877e2cc463041ecfc57494df9d Mon Sep 17 00:00:00 2001 From: Kayla Reopelle <87386821+kaylareopelle@users.noreply.github.com> Date: Wed, 4 Sep 2024 11:40:22 -0700 Subject: [PATCH 12/13] fix: Remove Metrics OTLP exporter `Util#measure_request_duration` and duplicate files (#1717) * fix: Remove measure_request_duration The OTLP metrics exporter does not use the metrics reporter present in the traces OTLP exporter. The measure_request_duration method is used in the traces exporter to provide a value for the metrics reporter. Since the metrics OTLP exporter does not use this metrics reporter, the method is not needed. Also, with the current code, the duration was being calculated in a void context. The value of the response variable has been and should be the result of @http.request(request). * fix: Remove old files outside nesting structure There were some duplicate files at the path exporter/otlp/* that were no longer used, now that the files are nested under exporter/otlp/metrics/* The test file location was updated to reflect the new location. * fix: Remove measure_request_duration from Util * style: Rubocop * style: Rubocop --- .../exporter/otlp/metrics/metrics_exporter.rb | 2 +- .../exporter/otlp/metrics/util.rb | 12 +- .../exporter/otlp/metrics_exporter.rb | 309 ------------------ .../lib/opentelemetry/exporter/otlp/util.rb | 139 -------- .../{ => metrics}/metrics_exporter_test.rb | 0 5 files changed, 2 insertions(+), 460 deletions(-) delete mode 100644 exporter/otlp-metrics/lib/opentelemetry/exporter/otlp/metrics_exporter.rb delete mode 100644 exporter/otlp-metrics/lib/opentelemetry/exporter/otlp/util.rb rename exporter/otlp-metrics/test/opentelemetry/exporter/otlp/{ => metrics}/metrics_exporter_test.rb (100%) diff --git a/exporter/otlp-metrics/lib/opentelemetry/exporter/otlp/metrics/metrics_exporter.rb b/exporter/otlp-metrics/lib/opentelemetry/exporter/otlp/metrics/metrics_exporter.rb index a1b0bf415f..44302e0e92 100644 --- a/exporter/otlp-metrics/lib/opentelemetry/exporter/otlp/metrics/metrics_exporter.rb +++ b/exporter/otlp-metrics/lib/opentelemetry/exporter/otlp/metrics/metrics_exporter.rb @@ -115,7 +115,7 @@ def send_bytes(bytes, timeout:) @http.read_timeout = remaining_timeout @http.write_timeout = remaining_timeout @http.start unless @http.started? - response = measure_request_duration { @http.request(request) } + response = @http.request(request) case response when Net::HTTPOK response.body # Read and discard body diff --git a/exporter/otlp-metrics/lib/opentelemetry/exporter/otlp/metrics/util.rb b/exporter/otlp-metrics/lib/opentelemetry/exporter/otlp/metrics/util.rb index 264b1e19bf..37f1896da0 100644 --- a/exporter/otlp-metrics/lib/opentelemetry/exporter/otlp/metrics/util.rb +++ b/exporter/otlp-metrics/lib/opentelemetry/exporter/otlp/metrics/util.rb @@ -9,7 +9,7 @@ module Exporter module OTLP module Metrics # Util module provide essential functionality for exporter - module Util # rubocop:disable Metrics/ModuleLength + module Util KEEP_ALIVE_TIMEOUT = 30 RETRY_COUNT = 5 ERROR_MESSAGE_INVALID_HEADERS = 'headers must be a String with comma-separated URL Encoded UTF-8 k=v pairs or a Hash' @@ -67,16 +67,6 @@ def prepare_headers(config_headers) headers end - def measure_request_duration - start = Process.clock_gettime(Process::CLOCK_MONOTONIC) - begin - yield - ensure - stop = Process.clock_gettime(Process::CLOCK_MONOTONIC) - 1000.0 * (stop - start) - end - end - def parse_headers(raw) entries = raw.split(',') raise ArgumentError, ERROR_MESSAGE_INVALID_HEADERS if entries.empty? diff --git a/exporter/otlp-metrics/lib/opentelemetry/exporter/otlp/metrics_exporter.rb b/exporter/otlp-metrics/lib/opentelemetry/exporter/otlp/metrics_exporter.rb deleted file mode 100644 index 3d86f9e979..0000000000 --- a/exporter/otlp-metrics/lib/opentelemetry/exporter/otlp/metrics_exporter.rb +++ /dev/null @@ -1,309 +0,0 @@ -# frozen_string_literal: true - -# Copyright The OpenTelemetry Authors -# -# SPDX-License-Identifier: Apache-2.0 - -require 'opentelemetry/common' -require 'opentelemetry/sdk' -require 'net/http' -require 'csv' -require 'zlib' - -require 'google/rpc/status_pb' - -require 'opentelemetry/proto/common/v1/common_pb' -require 'opentelemetry/proto/resource/v1/resource_pb' -require 'opentelemetry/proto/metrics/v1/metrics_pb' -require 'opentelemetry/proto/collector/metrics/v1/metrics_service_pb' - -require 'opentelemetry/metrics' -require 'opentelemetry/sdk/metrics' - -require_relative './util' - -module OpenTelemetry - module Exporter - module OTLP - # An OpenTelemetry metrics exporter that sends metrics over HTTP as Protobuf encoded OTLP ExportMetricsServiceRequest. - class MetricsExporter < ::OpenTelemetry::SDK::Metrics::Export::MetricReader - include Util - - attr_reader :metric_snapshots - - SUCCESS = OpenTelemetry::SDK::Metrics::Export::SUCCESS - FAILURE = OpenTelemetry::SDK::Metrics::Export::FAILURE - private_constant(:SUCCESS, :FAILURE) - - WRITE_TIMEOUT_SUPPORTED = Gem::Version.new(RUBY_VERSION) >= Gem::Version.new('2.6') - private_constant(:WRITE_TIMEOUT_SUPPORTED) - - def self.ssl_verify_mode - if ENV.key?('OTEL_RUBY_EXPORTER_OTLP_SSL_VERIFY_PEER') - OpenSSL::SSL::VERIFY_PEER - elsif ENV.key?('OTEL_RUBY_EXPORTER_OTLP_SSL_VERIFY_NONE') - OpenSSL::SSL::VERIFY_NONE - else - OpenSSL::SSL::VERIFY_PEER - end - end - - def initialize(endpoint: OpenTelemetry::Common::Utilities.config_opt('OTEL_EXPORTER_OTLP_METRICS_ENDPOINT', 'OTEL_EXPORTER_OTLP_ENDPOINT', default: 'http://localhost:4318/v1/metrics'), - certificate_file: OpenTelemetry::Common::Utilities.config_opt('OTEL_EXPORTER_OTLP_METRICS_CERTIFICATE', 'OTEL_EXPORTER_OTLP_CERTIFICATE'), - ssl_verify_mode: MetricsExporter.ssl_verify_mode, - headers: OpenTelemetry::Common::Utilities.config_opt('OTEL_EXPORTER_OTLP_METRICS_HEADERS', 'OTEL_EXPORTER_OTLP_HEADERS', default: {}), - compression: OpenTelemetry::Common::Utilities.config_opt('OTEL_EXPORTER_OTLP_METRICS_COMPRESSION', 'OTEL_EXPORTER_OTLP_COMPRESSION', default: 'gzip'), - timeout: OpenTelemetry::Common::Utilities.config_opt('OTEL_EXPORTER_OTLP_METRICS_TIMEOUT', 'OTEL_EXPORTER_OTLP_TIMEOUT', default: 10)) - raise ArgumentError, "invalid url for OTLP::MetricsExporter #{endpoint}" unless OpenTelemetry::Common::Utilities.valid_url?(endpoint) - raise ArgumentError, "unsupported compression key #{compression}" unless compression.nil? || %w[gzip none].include?(compression) - - # create the MetricStore object - super() - - @uri = if endpoint == ENV['OTEL_EXPORTER_OTLP_ENDPOINT'] - URI.join(endpoint, 'v1/metrics') - else - URI(endpoint) - end - - @http = http_connection(@uri, ssl_verify_mode, certificate_file) - - @path = @uri.path - @headers = prepare_headers(headers) - @timeout = timeout.to_f - @compression = compression - @mutex = Mutex.new - @shutdown = false - end - - # consolidate the metrics data into the form of MetricData - # - # return MetricData - def pull - export(collect) - end - - # metrics Array[MetricData] - def export(metrics, timeout: nil) - @mutex.synchronize do - send_bytes(encode(metrics), timeout: timeout) - end - end - - def send_bytes(bytes, timeout:) - return FAILURE if bytes.nil? - - request = Net::HTTP::Post.new(@path) - - if @compression == 'gzip' - request.add_field('Content-Encoding', 'gzip') - body = Zlib.gzip(bytes) - else - body = bytes - end - - request.body = body - request.add_field('Content-Type', 'application/x-protobuf') - @headers.each { |key, value| request.add_field(key, value) } - - retry_count = 0 - timeout ||= @timeout - start_time = OpenTelemetry::Common::Utilities.timeout_timestamp - - around_request do - remaining_timeout = OpenTelemetry::Common::Utilities.maybe_timeout(timeout, start_time) - return FAILURE if remaining_timeout.zero? - - @http.open_timeout = remaining_timeout - @http.read_timeout = remaining_timeout - @http.write_timeout = remaining_timeout if WRITE_TIMEOUT_SUPPORTED - @http.start unless @http.started? - response = measure_request_duration { @http.request(request) } - case response - when Net::HTTPOK - response.body # Read and discard body - SUCCESS - when Net::HTTPServiceUnavailable, Net::HTTPTooManyRequests - response.body # Read and discard body - redo if backoff?(retry_after: response['Retry-After'], retry_count: retry_count += 1, reason: response.code) - OpenTelemetry.logger.warn('Net::HTTPServiceUnavailable/Net::HTTPTooManyRequests in MetricsExporter#send_bytes') - FAILURE - when Net::HTTPRequestTimeOut, Net::HTTPGatewayTimeOut, Net::HTTPBadGateway - response.body # Read and discard body - redo if backoff?(retry_count: retry_count += 1, reason: response.code) - OpenTelemetry.logger.warn('Net::HTTPRequestTimeOut/Net::HTTPGatewayTimeOut/Net::HTTPBadGateway in MetricsExporter#send_bytes') - FAILURE - when Net::HTTPNotFound - OpenTelemetry.handle_error(message: "OTLP metrics_exporter received http.code=404 for uri: '#{@path}'") - FAILURE - when Net::HTTPBadRequest, Net::HTTPClientError, Net::HTTPServerError - log_status(response.body) - OpenTelemetry.logger.warn('Net::HTTPBadRequest/Net::HTTPClientError/Net::HTTPServerError in MetricsExporter#send_bytes') - FAILURE - when Net::HTTPRedirection - @http.finish - handle_redirect(response['location']) - redo if backoff?(retry_after: 0, retry_count: retry_count += 1, reason: response.code) - else - @http.finish - OpenTelemetry.logger.warn("Unexpected error in OTLP::MetricsExporter#send_bytes - #{response.message}") - FAILURE - end - rescue Net::OpenTimeout, Net::ReadTimeout - retry if backoff?(retry_count: retry_count += 1, reason: 'timeout') - OpenTelemetry.logger.warn('Net::OpenTimeout/Net::ReadTimeout in MetricsExporter#send_bytes') - return FAILURE - rescue OpenSSL::SSL::SSLError - retry if backoff?(retry_count: retry_count += 1, reason: 'openssl_error') - OpenTelemetry.logger.warn('OpenSSL::SSL::SSLError in MetricsExporter#send_bytes') - return FAILURE - rescue SocketError - retry if backoff?(retry_count: retry_count += 1, reason: 'socket_error') - OpenTelemetry.logger.warn('SocketError in MetricsExporter#send_bytes') - return FAILURE - rescue SystemCallError => e - retry if backoff?(retry_count: retry_count += 1, reason: e.class.name) - OpenTelemetry.logger.warn('SystemCallError in MetricsExporter#send_bytes') - return FAILURE - rescue EOFError - retry if backoff?(retry_count: retry_count += 1, reason: 'eof_error') - OpenTelemetry.logger.warn('EOFError in MetricsExporter#send_bytes') - return FAILURE - rescue Zlib::DataError - retry if backoff?(retry_count: retry_count += 1, reason: 'zlib_error') - OpenTelemetry.logger.warn('Zlib::DataError in MetricsExporter#send_bytes') - return FAILURE - rescue StandardError => e - OpenTelemetry.handle_error(exception: e, message: 'unexpected error in OTLP::MetricsExporter#send_bytes') - return FAILURE - end - ensure - # Reset timeouts to defaults for the next call. - @http.open_timeout = @timeout - @http.read_timeout = @timeout - @http.write_timeout = @timeout if WRITE_TIMEOUT_SUPPORTED - end - - def encode(metrics_data) - Opentelemetry::Proto::Collector::Metrics::V1::ExportMetricsServiceRequest.encode( - Opentelemetry::Proto::Collector::Metrics::V1::ExportMetricsServiceRequest.new( - resource_metrics: metrics_data - .group_by(&:resource) - .map do |resource, scope_metrics| - Opentelemetry::Proto::Metrics::V1::ResourceMetrics.new( - resource: Opentelemetry::Proto::Resource::V1::Resource.new( - attributes: resource.attribute_enumerator.map { |key, value| as_otlp_key_value(key, value) } - ), - scope_metrics: scope_metrics - .group_by(&:instrumentation_scope) - .map do |instrumentation_scope, metrics| - Opentelemetry::Proto::Metrics::V1::ScopeMetrics.new( - scope: Opentelemetry::Proto::Common::V1::InstrumentationScope.new( - name: instrumentation_scope.name, - version: instrumentation_scope.version - ), - metrics: metrics.map { |sd| as_otlp_metrics(sd) } - ) - end - ) - end - ) - ) - rescue StandardError => e - OpenTelemetry.handle_error(exception: e, message: 'unexpected error in OTLP::MetricsExporter#encode') - nil - end - - # metrics_pb has following type of data: :gauge, :sum, :histogram, :exponential_histogram, :summary - # current metric sdk only implements instrument: :counter -> :sum, :histogram -> :histogram - # - # metrics [MetricData] - def as_otlp_metrics(metrics) - case metrics.instrument_kind - when :observable_gauge - Opentelemetry::Proto::Metrics::V1::Metric.new( - name: metrics.name, - description: metrics.description, - unit: metrics.unit, - gauge: Opentelemetry::Proto::Metrics::V1::Gauge.new( - aggregation_temporality: as_otlp_aggregation_temporality(metrics.aggregation_temporality), - data_points: metrics.data_points.map do |ndp| - number_data_point(ndp) - end - ) - ) - - when :counter, :up_down_counter - Opentelemetry::Proto::Metrics::V1::Metric.new( - name: metrics.name, - description: metrics.description, - unit: metrics.unit, - sum: Opentelemetry::Proto::Metrics::V1::Sum.new( - aggregation_temporality: as_otlp_aggregation_temporality(metrics.aggregation_temporality), - data_points: metrics.data_points.map do |ndp| - number_data_point(ndp) - end - ) - ) - - when :histogram - Opentelemetry::Proto::Metrics::V1::Metric.new( - name: metrics.name, - description: metrics.description, - unit: metrics.unit, - histogram: Opentelemetry::Proto::Metrics::V1::Histogram.new( - aggregation_temporality: as_otlp_aggregation_temporality(metrics.aggregation_temporality), - data_points: metrics.data_points.map do |hdp| - histogram_data_point(hdp) - end - ) - ) - end - end - - def as_otlp_aggregation_temporality(type) - case type - when :delta then Opentelemetry::Proto::Metrics::V1::AggregationTemporality::AGGREGATION_TEMPORALITY_DELTA - when :cumulative then Opentelemetry::Proto::Metrics::V1::AggregationTemporality::AGGREGATION_TEMPORALITY_CUMULATIVE - else Opentelemetry::Proto::Metrics::V1::AggregationTemporality::AGGREGATION_TEMPORALITY_UNSPECIFIED - end - end - - def histogram_data_point(hdp) - Opentelemetry::Proto::Metrics::V1::HistogramDataPoint.new( - attributes: hdp.attributes.map { |k, v| as_otlp_key_value(k, v) }, - start_time_unix_nano: hdp.start_time_unix_nano, - time_unix_nano: hdp.time_unix_nano, - count: hdp.count, - sum: hdp.sum, - bucket_counts: hdp.bucket_counts, - explicit_bounds: hdp.explicit_bounds, - exemplars: hdp.exemplars, - min: hdp.min, - max: hdp.max - ) - end - - def number_data_point(ndp) - Opentelemetry::Proto::Metrics::V1::NumberDataPoint.new( - attributes: ndp.attributes.map { |k, v| as_otlp_key_value(k, v) }, - as_int: ndp.value, - start_time_unix_nano: ndp.start_time_unix_nano, - time_unix_nano: ndp.time_unix_nano, - exemplars: ndp.exemplars # exemplars not implemented yet from metrics sdk - ) - end - - # may not need this - def reset - SUCCESS - end - - def shutdown(timeout: nil) - @shutdown = true - SUCCESS - end - end - end - end -end diff --git a/exporter/otlp-metrics/lib/opentelemetry/exporter/otlp/util.rb b/exporter/otlp-metrics/lib/opentelemetry/exporter/otlp/util.rb deleted file mode 100644 index 85070a6f43..0000000000 --- a/exporter/otlp-metrics/lib/opentelemetry/exporter/otlp/util.rb +++ /dev/null @@ -1,139 +0,0 @@ -# frozen_string_literal: true - -# Copyright The OpenTelemetry Authors -# -# SPDX-License-Identifier: Apache-2.0 - -module OpenTelemetry - module Exporter - module OTLP - # Util module provide essential functionality for exporter - module Util # rubocop:disable Metrics/ModuleLength - KEEP_ALIVE_TIMEOUT = 30 - RETRY_COUNT = 5 - ERROR_MESSAGE_INVALID_HEADERS = 'headers must be a String with comma-separated URL Encoded UTF-8 k=v pairs or a Hash' - DEFAULT_USER_AGENT = "OTel-OTLP-MetricsExporter-Ruby/#{OpenTelemetry::Exporter::OTLP::VERSION} Ruby/#{RUBY_VERSION} (#{RUBY_PLATFORM}; #{RUBY_ENGINE}/#{RUBY_ENGINE_VERSION})".freeze - - def http_connection(uri, ssl_verify_mode, certificate_file) - http = Net::HTTP.new(uri.host, uri.port) - http.use_ssl = uri.scheme == 'https' - http.verify_mode = ssl_verify_mode - http.ca_file = certificate_file unless certificate_file.nil? - http.keep_alive_timeout = KEEP_ALIVE_TIMEOUT - http - end - - def around_request - OpenTelemetry::Common::Utilities.untraced { yield } # rubocop:disable Style/ExplicitBlockArgument - end - - def as_otlp_key_value(key, value) - Opentelemetry::Proto::Common::V1::KeyValue.new(key: key, value: as_otlp_any_value(value)) - rescue Encoding::UndefinedConversionError => e - encoded_value = value.encode('UTF-8', invalid: :replace, undef: :replace, replace: '�') - OpenTelemetry.handle_error(exception: e, message: "encoding error for key #{key} and value #{encoded_value}") - Opentelemetry::Proto::Common::V1::KeyValue.new(key: key, value: as_otlp_any_value('Encoding Error')) - end - - def as_otlp_any_value(value) - result = Opentelemetry::Proto::Common::V1::AnyValue.new - case value - when String - result.string_value = value - when Integer - result.int_value = value - when Float - result.double_value = value - when true, false - result.bool_value = value - when Array - values = value.map { |element| as_otlp_any_value(element) } - result.array_value = Opentelemetry::Proto::Common::V1::ArrayValue.new(values: values) - end - result - end - - def prepare_headers(config_headers) - headers = case config_headers - when String then parse_headers(config_headers) - when Hash then config_headers.dup - else - raise ArgumentError, ERROR_MESSAGE_INVALID_HEADERS - end - - headers['User-Agent'] = "#{headers.fetch('User-Agent', '')} #{DEFAULT_USER_AGENT}".strip - - headers - end - - def measure_request_duration - start = Process.clock_gettime(Process::CLOCK_MONOTONIC) - begin - yield - ensure - stop = Process.clock_gettime(Process::CLOCK_MONOTONIC) - 1000.0 * (stop - start) - end - end - - def parse_headers(raw) - entries = raw.split(',') - raise ArgumentError, ERROR_MESSAGE_INVALID_HEADERS if entries.empty? - - entries.each_with_object({}) do |entry, headers| - k, v = entry.split('=', 2).map(&CGI.method(:unescape)) - begin - k = k.to_s.strip - v = v.to_s.strip - rescue Encoding::CompatibilityError - raise ArgumentError, ERROR_MESSAGE_INVALID_HEADERS - rescue ArgumentError => e - raise e, ERROR_MESSAGE_INVALID_HEADERS - end - raise ArgumentError, ERROR_MESSAGE_INVALID_HEADERS if k.empty? || v.empty? - - headers[k] = v - end - end - - def backoff?(retry_count:, reason:, retry_after: nil) - return false if retry_count > RETRY_COUNT - - sleep_interval = nil - unless retry_after.nil? - sleep_interval = - begin - Integer(retry_after) - rescue ArgumentError - nil - end - sleep_interval ||= - begin - Time.httpdate(retry_after) - Time.now - rescue # rubocop:disable Style/RescueStandardError - nil - end - sleep_interval = nil unless sleep_interval&.positive? - end - sleep_interval ||= rand(2**retry_count) - - sleep(sleep_interval) - true - end - - def log_status(body) - status = Google::Rpc::Status.decode(body) - details = status.details.map do |detail| - klass_or_nil = ::Google::Protobuf::DescriptorPool.generated_pool.lookup(detail.type_name).msgclass - detail.unpack(klass_or_nil) if klass_or_nil - end.compact - OpenTelemetry.handle_error(message: "OTLP metrics_exporter received rpc.Status{message=#{status.message}, details=#{details}}") - rescue StandardError => e - OpenTelemetry.handle_error(exception: e, message: 'unexpected error decoding rpc.Status in OTLP::MetricsExporter#log_status') - end - - def handle_redirect(location); end - end - end - end -end diff --git a/exporter/otlp-metrics/test/opentelemetry/exporter/otlp/metrics_exporter_test.rb b/exporter/otlp-metrics/test/opentelemetry/exporter/otlp/metrics/metrics_exporter_test.rb similarity index 100% rename from exporter/otlp-metrics/test/opentelemetry/exporter/otlp/metrics_exporter_test.rb rename to exporter/otlp-metrics/test/opentelemetry/exporter/otlp/metrics/metrics_exporter_test.rb From 3ff81bdbc3112a69c41275f68153619b98e32209 Mon Sep 17 00:00:00 2001 From: Kayla Reopelle <87386821+kaylareopelle@users.noreply.github.com> Date: Wed, 4 Sep 2024 11:51:58 -0700 Subject: [PATCH 13/13] test: Misc logs fixes (#1697) * test: Add LogRecordProcessor tests * test: Update observed_timestamp LogRecord test Tests for timestamp were updated to use Time objects, but observed_timestamp was not. Now, they both evaluate Time. They are turned into floats when converted to LogRecordData. * test: Exclude test directory from SimpleCov * style: Misc docs/spacing updates * test: Skip flaky JRuby test with link to fix issue --------- Co-authored-by: Matthew Wear --- .../test/opentelemetry/logs/logger_test.rb | 2 +- logs_api/test/test_helper.rb | 7 +++- .../opentelemetry/sdk/logs/log_record_data.rb | 2 +- .../sdk/logs/log_record_processor.rb | 4 +-- logs_sdk/lib/opentelemetry/sdk/logs/logger.rb | 2 ++ .../export/batch_log_record_processor_test.rb | 4 +++ .../sdk/logs/log_record_processor_test.rb | 33 +++++++++++++++++++ .../opentelemetry/sdk/logs/log_record_test.rb | 10 ++---- logs_sdk/test/test_helper.rb | 7 +++- 9 files changed, 58 insertions(+), 13 deletions(-) create mode 100644 logs_sdk/test/opentelemetry/sdk/logs/log_record_processor_test.rb diff --git a/logs_api/test/opentelemetry/logs/logger_test.rb b/logs_api/test/opentelemetry/logs/logger_test.rb index 9fd35a651c..faf549f3d6 100644 --- a/logs_api/test/opentelemetry/logs/logger_test.rb +++ b/logs_api/test/opentelemetry/logs/logger_test.rb @@ -9,7 +9,7 @@ describe OpenTelemetry::Logs::Logger do let(:logger) { OpenTelemetry::Logs::Logger.new } - describe '#emit' do + describe '#on_emit' do it 'returns nil, as it is a no-op method' do assert_nil(logger.on_emit) end diff --git a/logs_api/test/test_helper.rb b/logs_api/test/test_helper.rb index 151c9213ec..a141bff9aa 100644 --- a/logs_api/test/test_helper.rb +++ b/logs_api/test/test_helper.rb @@ -5,7 +5,12 @@ # SPDX-License-Identifier: Apache-2.0 require 'simplecov' -SimpleCov.start { enable_coverage :branch } + +SimpleCov.start do + enable_coverage :branch + add_filter '/test/' +end + SimpleCov.minimum_coverage 85 require 'opentelemetry-logs-api' diff --git a/logs_sdk/lib/opentelemetry/sdk/logs/log_record_data.rb b/logs_sdk/lib/opentelemetry/sdk/logs/log_record_data.rb index 17d8b263fc..5a7e456d5c 100644 --- a/logs_sdk/lib/opentelemetry/sdk/logs/log_record_data.rb +++ b/logs_sdk/lib/opentelemetry/sdk/logs/log_record_data.rb @@ -13,7 +13,7 @@ module Logs :severity_text, # optional String :severity_number, # optional Integer :body, # optional String, Numeric, Boolean, Array, Hash{String => String, Numeric, Boolean, Array} - :attributes, # optional Hash{String => String, Numeric, Boolean, Array} + :attributes, # optional Hash{String => String, Numeric, Boolean, Array} :trace_id, # optional String (16-byte binary) :span_id, # optional String (8-byte binary) :trace_flags, # optional Integer (8-bit byte of bit flags) diff --git a/logs_sdk/lib/opentelemetry/sdk/logs/log_record_processor.rb b/logs_sdk/lib/opentelemetry/sdk/logs/log_record_processor.rb index 6b51b16915..6cca12ea14 100644 --- a/logs_sdk/lib/opentelemetry/sdk/logs/log_record_processor.rb +++ b/logs_sdk/lib/opentelemetry/sdk/logs/log_record_processor.rb @@ -26,7 +26,7 @@ def on_emit(log_record, context); end # the process after an invocation, but before the `Processor` exports # the completed spans. # - # @param [Numeric] timeout An optional timeout in seconds. + # @param [optional Numeric] timeout An optional timeout in seconds. # @return [Integer] Export::SUCCESS if no error occurred, Export::FAILURE if # a non-specific failure occurred, Export::TIMEOUT if a timeout occurred. def force_flush(timeout: nil) @@ -35,7 +35,7 @@ def force_flush(timeout: nil) # Called when {LoggerProvider#shutdown} is called. # - # @param [Numeric] timeout An optional timeout in seconds. + # @param [optional Numeric] timeout An optional timeout in seconds. # @return [Integer] Export::SUCCESS if no error occurred, Export::FAILURE if # a non-specific failure occurred, Export::TIMEOUT if a timeout occurred. def shutdown(timeout: nil) diff --git a/logs_sdk/lib/opentelemetry/sdk/logs/logger.rb b/logs_sdk/lib/opentelemetry/sdk/logs/logger.rb index 51f096c143..231999a33c 100644 --- a/logs_sdk/lib/opentelemetry/sdk/logs/logger.rb +++ b/logs_sdk/lib/opentelemetry/sdk/logs/logger.rb @@ -34,6 +34,8 @@ def initialize(name, version, logger_provider) # @param [optional OpenTelemetry::Trace::SpanContext] span_context The # OpenTelemetry::Trace::SpanContext to associate with the # {LogRecord}. + # @param [optional String] severity_text Original string representation of + # the severity as it is known at the source. Also known as log level. # @param severity_number [optional Integer] Numerical value of the # severity. Smaller numerical values correspond to less severe events # (such as debug events), larger numerical values correspond to more diff --git a/logs_sdk/test/opentelemetry/sdk/logs/export/batch_log_record_processor_test.rb b/logs_sdk/test/opentelemetry/sdk/logs/export/batch_log_record_processor_test.rb index 08aee76e85..64e81477a3 100644 --- a/logs_sdk/test/opentelemetry/sdk/logs/export/batch_log_record_processor_test.rb +++ b/logs_sdk/test/opentelemetry/sdk/logs/export/batch_log_record_processor_test.rb @@ -204,6 +204,10 @@ def to_log_record_data end it 'logs a warning if a log record was emitted after the buffer is full' do + # This will be fixed as part of Issue #1701 + # https://github.com/open-telemetry/opentelemetry-ruby/issues/1701 + skip if RUBY_ENGINE == 'jruby' + mock_otel_logger = Minitest::Mock.new mock_otel_logger.expect(:warn, nil, ['1 log record(s) dropped. Reason: buffer-full']) diff --git a/logs_sdk/test/opentelemetry/sdk/logs/log_record_processor_test.rb b/logs_sdk/test/opentelemetry/sdk/logs/log_record_processor_test.rb new file mode 100644 index 0000000000..9986b1915f --- /dev/null +++ b/logs_sdk/test/opentelemetry/sdk/logs/log_record_processor_test.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +# Copyright The OpenTelemetry Authors +# +# SPDX-License-Identifier: Apache-2.0 + +require 'test_helper' + +describe OpenTelemetry::SDK::Logs::LogRecordProcessor do + let(:processor) { OpenTelemetry::SDK::Logs::LogRecordProcessor.new } + let(:log_record) { nil } + let(:context) { nil } + + it 'implements #on_emit' do + processor.on_emit(log_record, context) + end + + it 'implements #force_flush' do + processor.force_flush + end + + it 'returns a success code when #force_flush is called' do + assert(OpenTelemetry::SDK::Logs::Export::SUCCESS, processor.force_flush) + end + + it 'implements #shutdown' do + processor.shutdown + end + + it 'returns a success code when #shutdown is called' do + assert(OpenTelemetry::SDK::Logs::Export::SUCCESS, processor.shutdown) + end +end diff --git a/logs_sdk/test/opentelemetry/sdk/logs/log_record_test.rb b/logs_sdk/test/opentelemetry/sdk/logs/log_record_test.rb index 7ba19048fe..ca38350e1f 100644 --- a/logs_sdk/test/opentelemetry/sdk/logs/log_record_test.rb +++ b/logs_sdk/test/opentelemetry/sdk/logs/log_record_test.rb @@ -15,7 +15,8 @@ describe '#initialize' do describe 'observed_timestamp' do describe 'when observed_timestamp is present' do - let(:observed_timestamp) { '1692661486.2841358' } + let(:current_time) { Time.now } + let(:observed_timestamp) { current_time + 1 } let(:args) { { observed_timestamp: observed_timestamp } } it 'is equal to observed_timestamp' do @@ -26,13 +27,8 @@ refute_equal(log_record.timestamp, log_record.observed_timestamp) end - # Process.clock_gettime is used to set the current time - # That method returns a Float. Since the stubbed value of - # observed_timestamp is a String, we can know the the - # observed_timestamp was not set to the value of Process.clock_gettime - # by making sure its value is not a Float. it 'is not equal to the current time' do - refute_instance_of(Float, log_record.observed_timestamp) + refute_equal(current_time, log_record.observed_timestamp) end end diff --git a/logs_sdk/test/test_helper.rb b/logs_sdk/test/test_helper.rb index 59d743987f..7f11ee9508 100644 --- a/logs_sdk/test/test_helper.rb +++ b/logs_sdk/test/test_helper.rb @@ -5,7 +5,12 @@ # SPDX-License-Identifier: Apache-2.0 require 'simplecov' -SimpleCov.start { enable_coverage :branch } + +SimpleCov.start do + enable_coverage :branch + add_filter '/test/' +end + SimpleCov.minimum_coverage 85 require 'opentelemetry-logs-api'