From d3378f2597974a3e63954891d7c2e8bbb3d5799c Mon Sep 17 00:00:00 2001 From: xuan-cao-swi Date: Thu, 26 Sep 2024 16:16:07 -0400 Subject: [PATCH 1/2] feat: add basic exponential histogram without merge --- .../opentelemetry/sdk/metrics/aggregation.rb | 2 + .../exponential_bucket_histogram.rb | 229 +++++++++++ .../exponential_histogram/buckets.rb | 113 ++++++ .../exponential_histogram/exponent_mapping.rb | 46 +++ .../exponential_histogram/ieee_754.rb | 46 +++ .../log2e_scale_factor.rb | 28 ++ .../logarithm_mapping.rb | 52 +++ .../exponential_histogram_data_point.rb | 32 ++ .../exponential_bucket_histogram_test.rb | 140 +++++++ .../aggregation/histogram_mapping_test.rb | 362 ++++++++++++++++++ 10 files changed, 1050 insertions(+) create mode 100644 metrics_sdk/lib/opentelemetry/sdk/metrics/aggregation/exponential_bucket_histogram.rb create mode 100644 metrics_sdk/lib/opentelemetry/sdk/metrics/aggregation/exponential_histogram/buckets.rb create mode 100644 metrics_sdk/lib/opentelemetry/sdk/metrics/aggregation/exponential_histogram/exponent_mapping.rb create mode 100644 metrics_sdk/lib/opentelemetry/sdk/metrics/aggregation/exponential_histogram/ieee_754.rb create mode 100644 metrics_sdk/lib/opentelemetry/sdk/metrics/aggregation/exponential_histogram/log2e_scale_factor.rb create mode 100644 metrics_sdk/lib/opentelemetry/sdk/metrics/aggregation/exponential_histogram/logarithm_mapping.rb create mode 100644 metrics_sdk/lib/opentelemetry/sdk/metrics/aggregation/exponential_histogram_data_point.rb create mode 100644 metrics_sdk/test/opentelemetry/sdk/metrics/aggregation/exponential_bucket_histogram_test.rb create mode 100644 metrics_sdk/test/opentelemetry/sdk/metrics/aggregation/histogram_mapping_test.rb diff --git a/metrics_sdk/lib/opentelemetry/sdk/metrics/aggregation.rb b/metrics_sdk/lib/opentelemetry/sdk/metrics/aggregation.rb index f36aef0152..a40ea8fdaa 100644 --- a/metrics_sdk/lib/opentelemetry/sdk/metrics/aggregation.rb +++ b/metrics_sdk/lib/opentelemetry/sdk/metrics/aggregation.rb @@ -19,3 +19,5 @@ module Aggregation require 'opentelemetry/sdk/metrics/aggregation/histogram_data_point' require 'opentelemetry/sdk/metrics/aggregation/explicit_bucket_histogram' require 'opentelemetry/sdk/metrics/aggregation/sum' +require 'opentelemetry/sdk/metrics/aggregation/exponential_histogram_data_point' +require 'opentelemetry/sdk/metrics/aggregation/exponential_bucket_histogram' diff --git a/metrics_sdk/lib/opentelemetry/sdk/metrics/aggregation/exponential_bucket_histogram.rb b/metrics_sdk/lib/opentelemetry/sdk/metrics/aggregation/exponential_bucket_histogram.rb new file mode 100644 index 0000000000..adad620b2c --- /dev/null +++ b/metrics_sdk/lib/opentelemetry/sdk/metrics/aggregation/exponential_bucket_histogram.rb @@ -0,0 +1,229 @@ +# frozen_string_literal: true + +# Copyright The OpenTelemetry Authors +# +# SPDX-License-Identifier: Apache-2.0 + +require_relative 'exponential_histogram/buckets' +require_relative 'exponential_histogram/log2e_scale_factor' +require_relative 'exponential_histogram/ieee_754' +require_relative 'exponential_histogram/logarithm_mapping' +require_relative 'exponential_histogram/exponent_mapping' + + +module OpenTelemetry + module SDK + module Metrics + module Aggregation + # Contains the implementation of the ExponentialBucketHistogram aggregation + # https://opentelemetry.io/docs/specs/otel/metrics/data-model/#exponentialhistogram + class ExponentialBucketHistogram + attr_reader :aggregation_temporality + + # relate to min max scale: https://opentelemetry.io/docs/specs/otel/metrics/sdk/#support-a-minimum-and-maximum-scale + MAX_SCALE = 20 + MIN_SCALE = -10 + MAX_SIZE = 160 + + # The default boundaries is calculated based on default max_size and max_scale value + def initialize( + aggregation_temporality: ENV.fetch('OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE', :delta), # TODO: aggregation_temporality should be renamed to collect_aggregation_temporality for clear definition + max_size: MAX_SIZE, + max_scale: MAX_SCALE, + record_min_max: true, + zero_threshold: 0 + ) + @data_points = {} + @aggregation_temporality = aggregation_temporality + @record_min_max = record_min_max + @min = Float::INFINITY + @max = -Float::INFINITY + @sum = 0 + @count = 0 + @zero_threshold = zero_threshold + @zero_count = 0 + @size = validate_size(max_size) + @scale = validate_scale(max_scale) + + @mapping = new_mapping(@scale) + end + + def collect(start_time, end_time) + if @aggregation_temporality == :delta + # Set timestamps and 'move' data point values to result. + hdps = @data_points.values.map! do |hdp| + hdp.start_time_unix_nano = start_time + hdp.time_unix_nano = end_time + hdp + end + @data_points.clear + hdps + else + # Assume merge is done in collector side at this point + @data_points.values.map! do |hdp| + hdp.start_time_unix_nano ||= start_time # Start time of a data point is from the first observation. + hdp.time_unix_nano = end_time + hdp = hdp.dup + hdp.positive = hdp.positive.dup + hdp.negative = hdp.negative.dup + hdp + end + end + end + + # ruby seems only record local min and max + def update(amount, attributes) + # fetch or initialize the ExponentialHistogramDataPoint + hdp = @data_points.fetch(attributes) do + + if @record_min_max + min = Float::INFINITY + max = -Float::INFINITY + end + + @data_points[attributes] = ExponentialHistogramDataPoint.new( + attributes, + nil, # :start_time_unix_nano + 0, # :time_unix_nano + 0, # :count + 0, # :sum + @scale, # :scale + @zero_count, # :zero_count + ExponentialHistogram::Buckets.new, # :positive + ExponentialHistogram::Buckets.new, # :negative + 0, # :flags + nil, # :exemplars + min, # :min + max, # :max + @zero_threshold, # :zero_threshold) + ) + end + + # Start to populate the data point (esp. the buckets) + if @record_min_max + hdp.max = amount if amount > hdp.max + hdp.min = amount if amount < hdp.min + end + + hdp.sum += amount + hdp.count += 1 + + if amount.abs <= @zero_threshold + hdp.zero_count += 1 + hdp.scale = 0 if hdp.count == hdp.zero_count # if always getting zero, then there is no point to keep doing the update + return + end + + # rescale, map to index, update the buckets here + buckets = amount.positive? ? hdp.positive : hdp.negative + amount = -amount if amount.negative? + + bucket_index = @mapping.map_to_index(amount) + + is_rescaling_needed = false + low = high = 0 + + if buckets.counts.empty? + buckets.index_start = bucket_index + buckets.index_end = bucket_index + buckets.index_base = bucket_index + + elsif bucket_index < buckets.index_start && (buckets.index_end - bucket_index) >= @size + is_rescaling_needed = true + low = bucket_index + high = buckets.index_end + + elsif bucket_index > buckets.index_end && (bucket_index - buckets.index_start) >= @size + is_rescaling_needed = true + low = buckets.index_start + high = bucket_index + end + + if is_rescaling_needed + scale_change = get_scale_change(low, high) + downscale(scale_change, hdp.positive, hdp.negative) + new_scale = @mapping.scale - scale_change + hdp.scale = new_scale + @mapping = new_mapping(new_scale) + bucket_index = @mapping.map_to_index(amount) + + OpenTelemetry.logger.debug "Rescaled with new scale #{new_scale} from #{low} and #{high}; bucket_index is updated to #{bucket_index}" + end + + # adjust buckets based on the bucket_index + if bucket_index < buckets.index_start + span = buckets.index_end - bucket_index + + if span >= buckets.counts.size + OpenTelemetry.logger.debug "buckets need to grow to #{span + 1} from #{buckets.counts.size} (max bucket size #{@size})" + buckets.grow(span + 1, @size) + end + + buckets.index_start = bucket_index + elsif bucket_index > buckets.index_end + span = bucket_index - buckets.index_start + + if span >= buckets.counts.size + OpenTelemetry.logger.debug "buckets need to grow to #{span + 1} from #{buckets.counts.size} (max bucket size #{@size})" + buckets.grow(span + 1, @size) + end + + buckets.index_end = bucket_index + end + + bucket_index = bucket_index - buckets.index_base + bucket_index += buckets.counts.size if bucket_index.negative? + + buckets.increment_bucket(bucket_index) + nil + end + + private + + def new_mapping(scale) + scale <= 0 ? ExponentialHistogram::ExponentMapping.new(scale) : ExponentialHistogram::LogarithmMapping.new(scale) + end + + def empty_counts + @boundaries ? Array.new(@boundaries.size + 1, 0) : nil + end + + def get_scale_change(low, high) + # puts "get_scale_change: low: #{low}, high: #{high}, @size: #{@size}" + # python code also produce 18 with 0,1048575, the high is little bit off + # just checked, the mapping is also ok, produce the 1048575 + change = 0 + while high - low >= @size + high >>= 1 + low >>= 1 + change += 1 + end + change + end + + def downscale(change, positive, negative) + return if change == 0 + raise "Invalid change of scale" if change.negative? + + positive.downscale(change) + negative.downscale(change) + end + + def validate_scale(scale) + return scale unless scale > MAX_SCALE || scale < MIN_SCALE + + OpenTelemetry.logger.warn "Scale #{scale} is invalid, using default max scale #{MAX_SCALE}" + MAX_SCALE + end + + def validate_size(size) + return size unless size > MAX_SIZE || size < 0 + + OpenTelemetry.logger.warn "Size #{size} is invalid, using default max size #{MAX_SIZE}" + MAX_SIZE + end + end + end + end + end +end diff --git a/metrics_sdk/lib/opentelemetry/sdk/metrics/aggregation/exponential_histogram/buckets.rb b/metrics_sdk/lib/opentelemetry/sdk/metrics/aggregation/exponential_histogram/buckets.rb new file mode 100644 index 0000000000..9516125852 --- /dev/null +++ b/metrics_sdk/lib/opentelemetry/sdk/metrics/aggregation/exponential_histogram/buckets.rb @@ -0,0 +1,113 @@ +# frozen_string_literal: true + +# Copyright The OpenTelemetry Authors +# +# SPDX-License-Identifier: Apache-2.0 + +module OpenTelemetry + module SDK + module Metrics + module Aggregation + module ExponentialHistogram + class Buckets + attr_accessor :index_start, :index_end, :index_base + attr_reader :counts + + def initialize + @counts = [0] + @index_base = 0 + @index_start = 0 + @index_end = 0 + end + + def grow(needed, max_size) + size = @counts.size + bias = @index_base - @index_start + old_positive_limit = size - bias + + new_size = [2**Math.log2(needed).ceil, max_size].min + + new_positive_limit = new_size - bias + + tmp = Array.new(new_size, 0) + tmp[new_positive_limit..-1] = @counts[old_positive_limit..-1] + tmp[0...old_positive_limit] = @counts[0...old_positive_limit] + @counts = tmp + end + + def offset + @index_start + end + + def get_offset_counts + bias = @index_base - @index_start + @counts[-bias..-1] + @counts[0...-bias] + end + alias_method :counts, :get_offset_counts + + def length + return 0 if @counts.empty? + return 0 if @index_end == @index_start && self[0] == 0 + + @index_end - @index_start + 1 + end + + def get_bucket(key) + bias = @index_base - @index_start + + key += @counts.size if key < bias + key -= bias + + @counts[key] + end + + def downscale(amount) + bias = @index_base - @index_start + + if bias != 0 + @index_base = @index_start + @counts.reverse! + @counts = @counts[0...bias].reverse + @counts[bias..-1].reverse + end + + size = 1 + @index_end - @index_start + each = 1 << amount + inpos = 0 + outpos = 0 + pos = @index_start + + while pos <= @index_end + mod = pos % each + mod += each if mod < 0 + + inds = mod + + while inds < each && inpos < size + if outpos != inpos + @counts[outpos] += @counts[inpos] + @counts[inpos] = 0 + end + + inpos += 1 + pos += 1 + inds += 1 + end + + outpos += 1 + end + + @index_start >>= amount + @index_end >>= amount + @index_base = @index_start + end + + def increment_bucket(bucket_index, increment = 1) + @counts[bucket_index] += increment + end + end + end + end + end + end +end + diff --git a/metrics_sdk/lib/opentelemetry/sdk/metrics/aggregation/exponential_histogram/exponent_mapping.rb b/metrics_sdk/lib/opentelemetry/sdk/metrics/aggregation/exponential_histogram/exponent_mapping.rb new file mode 100644 index 0000000000..e15d130284 --- /dev/null +++ b/metrics_sdk/lib/opentelemetry/sdk/metrics/aggregation/exponential_histogram/exponent_mapping.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +# Copyright The OpenTelemetry Authors +# +# SPDX-License-Identifier: Apache-2.0 + +module OpenTelemetry + module SDK + module Metrics + module Aggregation + module ExponentialHistogram + class ExponentMapping + attr_reader :scale + + def initialize(scale) + @scale = scale + @min_normal_lower_boundary_index = calculate_min_normal_lower_boundary_index(scale) + @max_normal_lower_boundary_index = IEEE754::MAX_NORMAL_EXPONENT >> -@scale + end + + def map_to_index(value) + + return @min_normal_lower_boundary_index if value < IEEE754::MIN_NORMAL_VALUE + + exponent = IEEE754.get_ieee_754_exponent(value) + correction = (IEEE754.get_ieee_754_mantissa(value) - 1) >> IEEE754::MANTISSA_WIDTH + (exponent + correction) >> -@scale + end + + def get_lower_boundary(inds) + raise StandardError, 'mapping underflow' if inds < @min_normal_lower_boundary_index || inds > @max_normal_lower_boundary_index + + Math.ldexp(1, inds << -@scale) + end + + def calculate_min_normal_lower_boundary_index(scale) + inds = IEEE754::MIN_NORMAL_EXPONENT >> -scale + inds -= 1 if -scale < 2 + inds + end + end + end + end + end + end +end diff --git a/metrics_sdk/lib/opentelemetry/sdk/metrics/aggregation/exponential_histogram/ieee_754.rb b/metrics_sdk/lib/opentelemetry/sdk/metrics/aggregation/exponential_histogram/ieee_754.rb new file mode 100644 index 0000000000..ff6686d077 --- /dev/null +++ b/metrics_sdk/lib/opentelemetry/sdk/metrics/aggregation/exponential_histogram/ieee_754.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +# Copyright The OpenTelemetry Authors +# +# SPDX-License-Identifier: Apache-2.0 + +require 'fiddle/import' + +module OpenTelemetry + module SDK + module Metrics + module Aggregation + module ExponentialHistogram + module IEEE754 + extend Fiddle::Importer + dlload Fiddle::Handle::DEFAULT + + MANTISSA_WIDTH = 52 + EXPONENT_WIDTH = 11 + + MANTISSA_MASK = (1 << MANTISSA_WIDTH) - 1 + EXPONENT_BIAS = (2 ** (EXPONENT_WIDTH - 1)) - 1 + EXPONENT_MASK = ((1 << EXPONENT_WIDTH) - 1) << MANTISSA_WIDTH + SIGN_MASK = 1 << (EXPONENT_WIDTH + MANTISSA_WIDTH) + + MIN_NORMAL_EXPONENT = -EXPONENT_BIAS + 1 + MAX_NORMAL_EXPONENT = EXPONENT_BIAS + + MIN_NORMAL_VALUE = Float::MIN + MAX_NORMAL_VALUE = Float::MAX + + def self.get_ieee_754_exponent(value) + bits = [value].pack('d').unpack1('Q') + ((bits & EXPONENT_MASK) >> MANTISSA_WIDTH) - EXPONENT_BIAS + end + + def self.get_ieee_754_mantissa(value) + bits = [value].pack('d').unpack1('Q') + bits & MANTISSA_MASK + end + end + end + end + end + end +end diff --git a/metrics_sdk/lib/opentelemetry/sdk/metrics/aggregation/exponential_histogram/log2e_scale_factor.rb b/metrics_sdk/lib/opentelemetry/sdk/metrics/aggregation/exponential_histogram/log2e_scale_factor.rb new file mode 100644 index 0000000000..cd7245eb5b --- /dev/null +++ b/metrics_sdk/lib/opentelemetry/sdk/metrics/aggregation/exponential_histogram/log2e_scale_factor.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +# Copyright The OpenTelemetry Authors +# +# SPDX-License-Identifier: Apache-2.0 + +module OpenTelemetry + module SDK + module Metrics + module Aggregation + module ExponentialHistogram + class Log2eScaleFactor + MAX_SCALE = 20 + + LOG2E_SCALE_BUCKETS = (0..MAX_SCALE).map do |scale| + log2e = 1 / Math.log(2) + Math.ldexp(log2e, scale) + end + + def self.log2e_scale_buckets + LOG2E_SCALE_BUCKETS + end + end + end + end + end + end +end \ No newline at end of file diff --git a/metrics_sdk/lib/opentelemetry/sdk/metrics/aggregation/exponential_histogram/logarithm_mapping.rb b/metrics_sdk/lib/opentelemetry/sdk/metrics/aggregation/exponential_histogram/logarithm_mapping.rb new file mode 100644 index 0000000000..14598b0c6e --- /dev/null +++ b/metrics_sdk/lib/opentelemetry/sdk/metrics/aggregation/exponential_histogram/logarithm_mapping.rb @@ -0,0 +1,52 @@ +# frozen_string_literal: true + +# Copyright The OpenTelemetry Authors +# +# SPDX-License-Identifier: Apache-2.0 + +module OpenTelemetry + module SDK + module Metrics + module Aggregation + module ExponentialHistogram + class LogarithmMapping + attr_reader :scale + + def initialize(scale) + @scale = scale + @scale_factor = Log2eScaleFactor::LOG2E_SCALE_BUCKETS[scale] # scale_factor is used for mapping the index + @min_normal_lower_boundary_index = IEEE754::MIN_NORMAL_EXPONENT << @scale + @max_normal_lower_boundary_index = ((IEEE754::MAX_NORMAL_EXPONENT + 1) << @scale) - 1 + end + + def map_to_index(value) + return @min_normal_lower_boundary_index - 1 if value <= IEEE754::MIN_NORMAL_VALUE + + if IEEE754.get_ieee_754_mantissa(value) == 0 + exponent = IEEE754.get_ieee_754_exponent(value) + return (exponent << @scale) - 1 + end + + [(Math.log(value) * @scale_factor).floor, @max_normal_lower_boundary_index].min + end + + def get_lower_boundary(inds) + if inds >= @max_normal_lower_boundary_index + return 2 * Math.exp((inds - (1 << @scale)) / @scale_factor) if inds == @max_normal_lower_boundary_index + raise StandardError, 'mapping overflow' + end + + if inds <= @min_normal_lower_boundary_index + return IEEE754::MIN_NORMAL_VALUE if inds == @min_normal_lower_boundary_index + return Math.exp((inds + (1 << @scale)) / @scale_factor) / 2 if inds == @min_normal_lower_boundary_index - 1 + raise StandardError, 'mapping underflow' + end + + Math.exp(inds / @scale_factor) + end + end + end + end + end + end +end diff --git a/metrics_sdk/lib/opentelemetry/sdk/metrics/aggregation/exponential_histogram_data_point.rb b/metrics_sdk/lib/opentelemetry/sdk/metrics/aggregation/exponential_histogram_data_point.rb new file mode 100644 index 0000000000..7ac46b45c3 --- /dev/null +++ b/metrics_sdk/lib/opentelemetry/sdk/metrics/aggregation/exponential_histogram_data_point.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +# Copyright The OpenTelemetry Authors +# +# SPDX-License-Identifier: Apache-2.0 + +module OpenTelemetry + module SDK + module Metrics + module Aggregation + # TODO: Deal with this later + # rubocop:disable Lint/StructNewOverride + + ExponentialHistogramDataPoint = Struct.new(:attributes, # optional Hash{String => String, Numeric, Boolean, Array} + :start_time_unix_nano, # Integer nanoseconds since Epoch + :time_unix_nano, # Integer nanoseconds since Epoch + :count, # Integer count is the number of values in the population. Must be non-negative + :sum, # Integer sum of the values in the population. If count is zero then this field then this field must be zero + :scale, # Integer scale factor + :zero_count, # Integer special bucket that count of observations that fall into the zero bucket + :positive, # Buckets representing the positive range of the histogram. + :negative, # Buckets representing the negative range of the histogram. + :flags, # Integer flags associated with the data point. + :exemplars, # optional List of exemplars collected from measurements that were used to form the data point + :min, # optional Float min is the minimum value over (start_time, end_time]. + :max, # optional Float max is the maximum value over (start_time, end_time]. + :zero_threshold) # optional Float the threshold for the zero bucket + # rubocop:enable Lint/StructNewOverride + end + end + end +end diff --git a/metrics_sdk/test/opentelemetry/sdk/metrics/aggregation/exponential_bucket_histogram_test.rb b/metrics_sdk/test/opentelemetry/sdk/metrics/aggregation/exponential_bucket_histogram_test.rb new file mode 100644 index 0000000000..25723c2233 --- /dev/null +++ b/metrics_sdk/test/opentelemetry/sdk/metrics/aggregation/exponential_bucket_histogram_test.rb @@ -0,0 +1,140 @@ +# frozen_string_literal: true + +# Copyright The OpenTelemetry Authors +# +# SPDX-License-Identifier: Apache-2.0 + +require 'test_helper' + +describe OpenTelemetry::SDK::Metrics::Aggregation::ExponentialBucketHistogram do + let(:expbh) do + OpenTelemetry::SDK::Metrics::Aggregation::ExponentialBucketHistogram.new( + aggregation_temporality: aggregation_temporality, + record_min_max: record_min_max, + max_size: max_size, + max_scale: max_scale, + zero_threshold: 0 + ) + end + let(:record_min_max) { true } + let(:max_size) { 20 } + let(:max_scale) { 5 } + let(:aggregation_temporality) { :delta } + # Time in nano + let(:start_time) { (Time.now.to_r * 1_000_000_000).to_i } + let(:end_time) { ((Time.now + 60).to_r * 1_000_000_000).to_i } + + describe '#collect' do + it 'returns all the data points' do + expbh.update(1.03, {}) + expbh.update(1.23, {}) + expbh.update(0, {}) + + expbh.update(1.45, {'foo' => 'bar'}) + expbh.update(1.67, {'foo' => 'bar'}) + + exphdps = expbh.collect(start_time, end_time) + + _(exphdps.size).must_equal(2) + _(exphdps[0].attributes).must_equal({}) + _(exphdps[0].count).must_equal(3) + _(exphdps[0].sum).must_equal(2.26) + _(exphdps[0].min).must_equal(0) + _(exphdps[0].max).must_equal(1.23) + _(exphdps[0].scale).must_equal(5) + _(exphdps[0].zero_count).must_equal(1) + _(exphdps[0].positive.counts).must_equal([0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0]) + _(exphdps[0].negative.counts).must_equal([0]) + _(exphdps[0].zero_threshold).must_equal(0) + + _(exphdps[1].attributes).must_equal('foo' => 'bar') + _(exphdps[1].count).must_equal(2) + _(exphdps[1].sum).must_equal(3.12) + _(exphdps[1].min).must_equal(1.45) + _(exphdps[1].max).must_equal(1.67) + _(exphdps[1].scale).must_equal(4) + _(exphdps[1].zero_count).must_equal(0) + _(exphdps[1].positive.counts).must_equal([0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0]) + _(exphdps[1].negative.counts).must_equal([0]) + _(exphdps[1].zero_threshold).must_equal(0) + end + + it 'rescale_with_alternating_growth_0' do + # Tests insertion of [2, 4, 1]. The index of 2 (i.e., 0) becomes + # `indexBase`, the 4 goes to its right and the 1 goes in the last + # position of the backing array. With 3 binary orders of magnitude + # and MaxSize=4, this must finish with scale=0; with minimum value 1 + # this must finish with offset=-1 (all scales). + + # The corresponding Go test is TestAlternatingGrowth1 where: + # agg := NewFloat64(NewConfig(WithMaxSize(4))) + # agg is an instance of github.com/lightstep/otel-launcher-go/lightstep/sdk/metric/aggregator/histogram/structure.Histogram[float64] + expbh = OpenTelemetry::SDK::Metrics::Aggregation::ExponentialBucketHistogram.new( + aggregation_temporality: aggregation_temporality, + record_min_max: record_min_max, + max_size: 4, + max_scale: 20, # use default value of max scale; should downscale to 0 + zero_threshold: 0 + ) + + expbh.update(2, {}) + expbh.update(4, {}) + expbh.update(1, {}) + + exphdps = expbh.collect(start_time, end_time) + + _(exphdps.size).must_equal(1) + _(exphdps[0].attributes).must_equal({}) + _(exphdps[0].count).must_equal(3) + _(exphdps[0].sum).must_equal(7) + _(exphdps[0].min).must_equal(1) + _(exphdps[0].max).must_equal(4) + _(exphdps[0].scale).must_equal(0) + _(exphdps[0].zero_count).must_equal(0) + _(exphdps[0].positive.offset).must_equal(-1) + _(exphdps[0].positive.counts).must_equal([1, 1, 1, 0]) + _(exphdps[0].negative.counts).must_equal([0]) + _(exphdps[0].zero_threshold).must_equal(0) + end + + it 'rescale_with_alternating_growth_1' do + # Tests insertion of [2, 2, 4, 1, 8, 0.5]. The test proceeds as + # above but then downscales once further to scale=-1, thus index -1 + # holds range [0.25, 1.0), index 0 holds range [1.0, 4), index 1 + # holds range [4, 16). + expbh = OpenTelemetry::SDK::Metrics::Aggregation::ExponentialBucketHistogram.new( + aggregation_temporality: aggregation_temporality, + record_min_max: record_min_max, + max_size: 4, + max_scale: 20, # use default value of max scale; should downscale to 0 + zero_threshold: 0 + ) + + expbh.update(2, {}) + expbh.update(2, {}) + expbh.update(2, {}) + expbh.update(1, {}) + expbh.update(8, {}) + expbh.update(0.5, {}) + + exphdps = expbh.collect(start_time, end_time) + + _(exphdps.size).must_equal(1) + _(exphdps[0].attributes).must_equal({}) + _(exphdps[0].count).must_equal(6) + _(exphdps[0].sum).must_equal(15.5) + _(exphdps[0].min).must_equal(0.5) + _(exphdps[0].max).must_equal(8) + _(exphdps[0].scale).must_equal(-1) + _(exphdps[0].zero_count).must_equal(0) + _(exphdps[0].positive.offset).must_equal(-1) + _(exphdps[0].positive.counts).must_equal([2, 3, 1, 0]) + _(exphdps[0].negative.counts).must_equal([0]) + _(exphdps[0].zero_threshold).must_equal(0) + end + + it 'test_merge' do + # TODO + end + end +end diff --git a/metrics_sdk/test/opentelemetry/sdk/metrics/aggregation/histogram_mapping_test.rb b/metrics_sdk/test/opentelemetry/sdk/metrics/aggregation/histogram_mapping_test.rb new file mode 100644 index 0000000000..9d7c699794 --- /dev/null +++ b/metrics_sdk/test/opentelemetry/sdk/metrics/aggregation/histogram_mapping_test.rb @@ -0,0 +1,362 @@ +# frozen_string_literal: true + +# Copyright The OpenTelemetry Authors +# +# SPDX-License-Identifier: Apache-2.0 + +require 'test_helper' + +def left_boundary(scale, inds) + while scale > 0 && inds < -1022 + inds /= 2.to_f + scale -= 1 + end + + result = 2.0**inds + + scale.times { result = Math.sqrt(result) } + result +end + +def right_boundary(scale, index) + result = 2**index + + scale.abs.times do + result *= result + end + + result +end + +describe OpenTelemetry::SDK::Metrics::Aggregation::ExponentialBucketHistogram do + + MAX_NORMAL_EXPONENT = OpenTelemetry::SDK::Metrics::Aggregation::ExponentialHistogram::IEEE754::MAX_NORMAL_EXPONENT + MIN_NORMAL_EXPONENT = OpenTelemetry::SDK::Metrics::Aggregation::ExponentialHistogram::IEEE754::MIN_NORMAL_EXPONENT + MAX_NORMAL_VALUE = OpenTelemetry::SDK::Metrics::Aggregation::ExponentialHistogram::IEEE754::MAX_NORMAL_VALUE + MIN_NORMAL_VALUE = OpenTelemetry::SDK::Metrics::Aggregation::ExponentialHistogram::IEEE754::MIN_NORMAL_VALUE + + describe 'logarithm_mapping' do + it 'test_logarithm_mapping_scale_one' do + logarithm_mapping = OpenTelemetry::SDK::Metrics::Aggregation::ExponentialHistogram::LogarithmMapping.new(1) + _(logarithm_mapping.scale).must_equal(1) + _(logarithm_mapping.map_to_index(15)).must_equal(7) + _(logarithm_mapping.map_to_index(9)).must_equal(6) + _(logarithm_mapping.map_to_index(7)).must_equal(5) + _(logarithm_mapping.map_to_index(5)).must_equal(4) + _(logarithm_mapping.map_to_index(3)).must_equal(3) + _(logarithm_mapping.map_to_index(2.5)).must_equal(2) + _(logarithm_mapping.map_to_index(1.5)).must_equal(1) + _(logarithm_mapping.map_to_index(1.2)).must_equal(0) + # This one is actually an exact test + _(logarithm_mapping.map_to_index(1)).must_equal(-1) + _(logarithm_mapping.map_to_index(0.75)).must_equal(-1) + _(logarithm_mapping.map_to_index(0.55)).must_equal(-2) + _(logarithm_mapping.map_to_index(0.45)).must_equal(-3) + end + + it 'test_logarithm_boundary' do + + [1, 2, 3, 4, 10, 15].each do |scale| + logarithm_mapping = OpenTelemetry::SDK::Metrics::Aggregation::ExponentialHistogram::LogarithmMapping.new(scale) + + [-100, -10, -1, 0, 1, 10, 100].each do |inds| + lower_boundary = logarithm_mapping.get_lower_boundary(inds) + mapped_index = logarithm_mapping.map_to_index(lower_boundary) + + _(mapped_index).must_be :<=, inds + _(mapped_index).must_be :>=, inds - 1 + + left_boundary_value = left_boundary(scale, inds) + _(lower_boundary).must_be_within_epsilon left_boundary_value, 1e-9 + end + end + end + + it 'test_logarithm_index_max' do + (1..20).each do |scale| + logarithm_mapping = OpenTelemetry::SDK::Metrics::Aggregation::ExponentialHistogram::LogarithmMapping.new(scale) + + inds = logarithm_mapping.map_to_index(OpenTelemetry::SDK::Metrics::Aggregation::ExponentialHistogram::IEEE754::MAX_NORMAL_VALUE) + max_index = ((OpenTelemetry::SDK::Metrics::Aggregation::ExponentialHistogram::IEEE754::MAX_NORMAL_EXPONENT + 1) << scale) - 1 + + _(inds).must_equal(max_index) + + boundary = logarithm_mapping.get_lower_boundary(inds) + base = logarithm_mapping.get_lower_boundary(1) + + _(boundary).must_be :<, OpenTelemetry::SDK::Metrics::Aggregation::ExponentialHistogram::IEEE754::MAX_NORMAL_VALUE + + _((OpenTelemetry::SDK::Metrics::Aggregation::ExponentialHistogram::IEEE754::MAX_NORMAL_VALUE - boundary) / boundary ).must_be_within_epsilon base - 1, 1e-6 + + error = assert_raises(StandardError) do + logarithm_mapping.get_lower_boundary(inds + 1) + end + assert_equal("mapping overflow", error.message) + + error = assert_raises(StandardError) do + logarithm_mapping.get_lower_boundary(inds + 2) + end + assert_equal("mapping overflow", error.message) + end + end + + it 'test_logarithm_index_min' do + (1..20).each do |scale| + logarithm_mapping = OpenTelemetry::SDK::Metrics::Aggregation::ExponentialHistogram::LogarithmMapping.new(scale) + + min_index = logarithm_mapping.map_to_index(MIN_NORMAL_VALUE) + correct_min_index = (MIN_NORMAL_EXPONENT << scale) - 1 + + _(min_index).must_equal(correct_min_index) + + correct_mapped = left_boundary(scale, correct_min_index) + _(correct_mapped).must_be :<, MIN_NORMAL_VALUE + + correct_mapped_upper = left_boundary(scale, correct_min_index + 1) + _(correct_mapped_upper).must_equal(MIN_NORMAL_VALUE) + + mapped = logarithm_mapping.get_lower_boundary(min_index + 1) + _(mapped).must_be_within_epsilon MIN_NORMAL_VALUE, 1e-6 + + _(logarithm_mapping.map_to_index(MIN_NORMAL_VALUE / 2)).must_equal(correct_min_index) + _(logarithm_mapping.map_to_index(MIN_NORMAL_VALUE / 3)).must_equal(correct_min_index) + _(logarithm_mapping.map_to_index(MIN_NORMAL_VALUE / 100)).must_equal(correct_min_index) + _(logarithm_mapping.map_to_index(2**-1050)).must_equal(correct_min_index) + _(logarithm_mapping.map_to_index(2**-1073)).must_equal(correct_min_index) + _(logarithm_mapping.map_to_index(1.1 * 2**-1073)).must_equal(correct_min_index) + _(logarithm_mapping.map_to_index(2**-1074)).must_equal(correct_min_index) + + mapped_lower = logarithm_mapping.get_lower_boundary(min_index) + _(correct_mapped).must_be_within_epsilon mapped_lower, 1e-6 + + error = assert_raises(StandardError) do + logarithm_mapping.get_lower_boundary(min_index - 1) + end + assert_equal('mapping underflow', error.message) + end + end + + end + + describe 'exponent_mapping' do + let(:exponent_mapping_min_scale) { -10 } + + it 'test_exponent_mapping_zero' do + + exponent_mapping = OpenTelemetry::SDK::Metrics::Aggregation::ExponentialHistogram::ExponentMapping.new(0) + + # This is the equivalent to 1.1 in hexadecimal + hex_1_1 = 1 + (1.0 / 16) + + # Testing with values near +inf + _(exponent_mapping.map_to_index(MAX_NORMAL_VALUE)).must_equal(MAX_NORMAL_EXPONENT) + _(exponent_mapping.map_to_index(MAX_NORMAL_VALUE)).must_equal(1023) + _(exponent_mapping.map_to_index(2**1023)).must_equal(1022) + _(exponent_mapping.map_to_index(2**1022)).must_equal(1021) + _(exponent_mapping.map_to_index(hex_1_1 * (2**1023))).must_equal(1023) + _(exponent_mapping.map_to_index(hex_1_1 * (2**1022))).must_equal(1022) + + # Testing with values near 1 + _(exponent_mapping.map_to_index(4)).must_equal(1) + _(exponent_mapping.map_to_index(3)).must_equal(1) + _(exponent_mapping.map_to_index(2)).must_equal(0) + _(exponent_mapping.map_to_index(1)).must_equal(-1) + _(exponent_mapping.map_to_index(0.75)).must_equal(-1) + _(exponent_mapping.map_to_index(0.51)).must_equal(-1) + _(exponent_mapping.map_to_index(0.5)).must_equal(-2) + _(exponent_mapping.map_to_index(0.26)).must_equal(-2) + _(exponent_mapping.map_to_index(0.25)).must_equal(-3) + _(exponent_mapping.map_to_index(0.126)).must_equal(-3) + _(exponent_mapping.map_to_index(0.125)).must_equal(-4) + + # Testing with values near 0 + _(exponent_mapping.map_to_index(2**-1022)).must_equal(-1023) + _(exponent_mapping.map_to_index(hex_1_1 * (2**-1022))).must_equal(-1022) + _(exponent_mapping.map_to_index(2**-1021)).must_equal(-1022) + _(exponent_mapping.map_to_index(hex_1_1 * (2**-1021))).must_equal(-1021) + _(exponent_mapping.map_to_index(2**-1022)).must_equal(MIN_NORMAL_EXPONENT - 1) + _(exponent_mapping.map_to_index(2**-1021)).must_equal(MIN_NORMAL_EXPONENT) + + # The smallest subnormal value is 2 ** -1074 = 5e-324. + # This value is also the result of: + # s = 1 + # while s / 2: + # s = s / 2 + # s == 5e-324 + _(exponent_mapping.map_to_index(2**-1074)).must_equal(MIN_NORMAL_EXPONENT - 1) + end + + it 'test_exponent_mapping_negative_one' do + exponent_mapping = OpenTelemetry::SDK::Metrics::Aggregation::ExponentialHistogram::ExponentMapping.new(-1) + _(exponent_mapping.map_to_index(17)).must_equal(2) + _(exponent_mapping.map_to_index(16)).must_equal(1) + _(exponent_mapping.map_to_index(15)).must_equal(1) + _(exponent_mapping.map_to_index(9)).must_equal(1) + _(exponent_mapping.map_to_index(8)).must_equal(1) + _(exponent_mapping.map_to_index(5)).must_equal(1) + _(exponent_mapping.map_to_index(4)).must_equal(0) + _(exponent_mapping.map_to_index(3)).must_equal(0) + _(exponent_mapping.map_to_index(2)).must_equal(0) + _(exponent_mapping.map_to_index(1.5)).must_equal(0) + _(exponent_mapping.map_to_index(1)).must_equal(-1) + _(exponent_mapping.map_to_index(0.75)).must_equal(-1) + _(exponent_mapping.map_to_index(0.5)).must_equal(-1) + _(exponent_mapping.map_to_index(0.25)).must_equal(-2) + _(exponent_mapping.map_to_index(0.20)).must_equal(-2) + _(exponent_mapping.map_to_index(0.13)).must_equal(-2) + _(exponent_mapping.map_to_index(0.125)).must_equal(-2) + _(exponent_mapping.map_to_index(0.10)).must_equal(-2) + _(exponent_mapping.map_to_index(0.0625)).must_equal(-3) + _(exponent_mapping.map_to_index(0.06)).must_equal(-3) + end + + it 'test_exponent_mapping_negative_four' do + exponent_mapping = OpenTelemetry::SDK::Metrics::Aggregation::ExponentialHistogram::ExponentMapping.new(-4) + + _(exponent_mapping.map_to_index(0x1.to_f)).must_equal(-1) + _(exponent_mapping.map_to_index(0x10.to_f)).must_equal(0) + _(exponent_mapping.map_to_index(0x100.to_f)).must_equal(0) + _(exponent_mapping.map_to_index(0x1000.to_f)).must_equal(0) + _(exponent_mapping.map_to_index(0x10000.to_f)).must_equal(0) # base == 2 ** 16 + _(exponent_mapping.map_to_index(0x100000.to_f)).must_equal(1) + _(exponent_mapping.map_to_index(0x1000000.to_f)).must_equal(1) + _(exponent_mapping.map_to_index(0x10000000.to_f)).must_equal(1) + _(exponent_mapping.map_to_index(0x100000000.to_f)).must_equal(1) # base == 2 ** 32 + + _(exponent_mapping.map_to_index(0x1000000000.to_f)).must_equal(2) + _(exponent_mapping.map_to_index(0x10000000000.to_f)).must_equal(2) + _(exponent_mapping.map_to_index(0x100000000000.to_f)).must_equal(2) + _(exponent_mapping.map_to_index(0x1000000000000.to_f)).must_equal(2) # base == 2 ** 48 + + _(exponent_mapping.map_to_index(0x10000000000000.to_f)).must_equal(3) + _(exponent_mapping.map_to_index(0x100000000000000.to_f)).must_equal(3) + _(exponent_mapping.map_to_index(0x1000000000000000.to_f)).must_equal(3) + _(exponent_mapping.map_to_index(0x10000000000000000.to_f)).must_equal(3) # base == 2 ** 64 + + _(exponent_mapping.map_to_index(0x100000000000000000.to_f)).must_equal(4) + _(exponent_mapping.map_to_index(0x1000000000000000000.to_f)).must_equal(4) + _(exponent_mapping.map_to_index(0x10000000000000000000.to_f)).must_equal(4) + _(exponent_mapping.map_to_index(0x100000000000000000000.to_f)).must_equal(4) # base == 2 ** 80 + _(exponent_mapping.map_to_index(0x1000000000000000000000.to_f)).must_equal(5) + + _(exponent_mapping.map_to_index(1 / 0x1.to_f)).must_equal(-1) + _(exponent_mapping.map_to_index(1 / 0x10.to_f)).must_equal(-1) + _(exponent_mapping.map_to_index(1 / 0x100.to_f)).must_equal(-1) + _(exponent_mapping.map_to_index(1 / 0x1000.to_f)).must_equal(-1) + _(exponent_mapping.map_to_index(1 / 0x10000.to_f)).must_equal(-2) # base == 2 ** -16 + _(exponent_mapping.map_to_index(1 / 0x100000.to_f)).must_equal(-2) + _(exponent_mapping.map_to_index(1 / 0x1000000.to_f)).must_equal(-2) + _(exponent_mapping.map_to_index(1 / 0x10000000.to_f)).must_equal(-2) + _(exponent_mapping.map_to_index(1 / 0x100000000.to_f)).must_equal(-3) # base == 2 ** -32 + _(exponent_mapping.map_to_index(1 / 0x1000000000.to_f)).must_equal(-3) + _(exponent_mapping.map_to_index(1 / 0x10000000000.to_f)).must_equal(-3) + _(exponent_mapping.map_to_index(1 / 0x100000000000.to_f)).must_equal(-3) + _(exponent_mapping.map_to_index(1 / 0x1000000000000.to_f)).must_equal(-4) # base == 2 ** -48 + _(exponent_mapping.map_to_index(1 / 0x10000000000000.to_f)).must_equal(-4) + _(exponent_mapping.map_to_index(1 / 0x100000000000000.to_f)).must_equal(-4) + _(exponent_mapping.map_to_index(1 / 0x1000000000000000.to_f)).must_equal(-4) + _(exponent_mapping.map_to_index(1 / 0x10000000000000000.to_f)).must_equal(-5) # base == 2 ** -64 + _(exponent_mapping.map_to_index(1 / 0x100000000000000000.to_f)).must_equal(-5) + + _(exponent_mapping.map_to_index(Float::MAX)).must_equal(63) + _(exponent_mapping.map_to_index(2**1023)).must_equal(63) + _(exponent_mapping.map_to_index(2**1019)).must_equal(63) + _(exponent_mapping.map_to_index(2**1009)).must_equal(63) + _(exponent_mapping.map_to_index(2**1008)).must_equal(62) + _(exponent_mapping.map_to_index(2**1007)).must_equal(62) + _(exponent_mapping.map_to_index(2**1000)).must_equal(62) + _(exponent_mapping.map_to_index(2**993)).must_equal(62) + _(exponent_mapping.map_to_index(2**992)).must_equal(61) + _(exponent_mapping.map_to_index(2**991)).must_equal(61) + + _(exponent_mapping.map_to_index(2**-1074)).must_equal(-64) + _(exponent_mapping.map_to_index(2**-1073)).must_equal(-64) + _(exponent_mapping.map_to_index(2**-1072)).must_equal(-64) + _(exponent_mapping.map_to_index(2**-1057)).must_equal(-64) + _(exponent_mapping.map_to_index(2**-1056)).must_equal(-64) + _(exponent_mapping.map_to_index(2**-1041)).must_equal(-64) + _(exponent_mapping.map_to_index(2**-1040)).must_equal(-64) + _(exponent_mapping.map_to_index(2**-1025)).must_equal(-64) + _(exponent_mapping.map_to_index(2**-1024)).must_equal(-64) + _(exponent_mapping.map_to_index(2**-1023)).must_equal(-64) + _(exponent_mapping.map_to_index(2**-1022)).must_equal(-64) + _(exponent_mapping.map_to_index(2**-1009)).must_equal(-64) + _(exponent_mapping.map_to_index(2**-1008)).must_equal(-64) + _(exponent_mapping.map_to_index(2**-1007)).must_equal(-63) + _(exponent_mapping.map_to_index(2**-993)).must_equal(-63) + _(exponent_mapping.map_to_index(2**-992)).must_equal(-63) + _(exponent_mapping.map_to_index(2**-991)).must_equal(-62) + _(exponent_mapping.map_to_index(2**-977)).must_equal(-62) + _(exponent_mapping.map_to_index(2**-976)).must_equal(-62) + _(exponent_mapping.map_to_index(2**-975)).must_equal(-61) + end + + it 'test_exponent_mapping_min_scale_negative_10' do + exponent_mapping = OpenTelemetry::SDK::Metrics::Aggregation::ExponentialHistogram::ExponentMapping.new(exponent_mapping_min_scale) + _(exponent_mapping.map_to_index(1.000001)).must_equal(0) + _(exponent_mapping.map_to_index(1)).must_equal(-1) + _(exponent_mapping.map_to_index(Float::MAX)).must_equal(0) + _(exponent_mapping.map_to_index(Float::MIN)).must_equal(-1) + end + + it 'test_exponent_index_max' do + (-10...0).each do |scale| + exponent_mapping = OpenTelemetry::SDK::Metrics::Aggregation::ExponentialHistogram::ExponentMapping.new(scale) + + inds = exponent_mapping.map_to_index(MAX_NORMAL_VALUE) + max_index = ((MAX_NORMAL_EXPONENT + 1) >> -scale) - 1 + + _(inds).must_equal(max_index) + + boundary = exponent_mapping.get_lower_boundary(inds) + _(boundary).must_equal(right_boundary(scale, max_index)) + + error = assert_raises(StandardError) do + exponent_mapping.get_lower_boundary(inds + 1) + end + assert_equal('mapping underflow', error.message) + end + + end + + it 'test_exponent_index_min' do + (-10..0).each do |scale| + exponent_mapping = OpenTelemetry::SDK::Metrics::Aggregation::ExponentialHistogram::ExponentMapping.new(scale) + + min_index = exponent_mapping.map_to_index(MIN_NORMAL_VALUE) + boundary = exponent_mapping.get_lower_boundary(min_index) + + correct_min_index = MIN_NORMAL_EXPONENT >> -scale + + correct_min_index -= 1 if MIN_NORMAL_EXPONENT % (1 << -scale) == 0 + + # We do not check for correct_min_index to be greater than the + # smallest integer because the smallest integer in Ruby is -Float::INFINITY. + + _(correct_min_index).must_equal(min_index) + + correct_boundary = right_boundary(scale, correct_min_index) + + _(correct_boundary).must_equal(boundary) + _(right_boundary(scale, correct_min_index + 1)).must_be :>, boundary + + _(correct_min_index).must_equal(exponent_mapping.map_to_index(MIN_NORMAL_VALUE / 2)) + _(correct_min_index).must_equal(exponent_mapping.map_to_index(MIN_NORMAL_VALUE / 3)) + _(correct_min_index).must_equal(exponent_mapping.map_to_index(MIN_NORMAL_VALUE / 100)) + _(correct_min_index).must_equal(exponent_mapping.map_to_index(2**-1050)) + _(correct_min_index).must_equal(exponent_mapping.map_to_index(2**-1073)) + _(correct_min_index).must_equal(exponent_mapping.map_to_index(1.1 * (2**-1073))) + _(correct_min_index).must_equal(exponent_mapping.map_to_index(2**-1074)) + + error = assert_raises(StandardError) do + exponent_mapping.get_lower_boundary(min_index - 1) + end + assert_equal('mapping underflow', error.message) + + _(exponent_mapping.map_to_index(Float::MIN.next_float)).must_equal(MIN_NORMAL_EXPONENT >> -scale) + end + end + + end + +end From 3eb733b588db895c9ec62d0edc1b19778f85e0b7 Mon Sep 17 00:00:00 2001 From: xuan-cao-swi Date: Fri, 27 Sep 2024 14:27:56 -0400 Subject: [PATCH 2/2] lint --- .../exponential_bucket_histogram.rb | 15 +-- .../exponential_histogram/buckets.rb | 13 +- .../exponential_histogram/exponent_mapping.rb | 2 +- .../exponential_histogram/ieee_754.rb | 3 +- .../log2e_scale_factor.rb | 3 +- .../logarithm_mapping.rb | 5 +- .../exponential_histogram_data_point.rb | 29 ++-- .../exponential_bucket_histogram_test.rb | 126 +++++++++++++++++- .../aggregation/histogram_mapping_test.rb | 27 ++-- 9 files changed, 167 insertions(+), 56 deletions(-) diff --git a/metrics_sdk/lib/opentelemetry/sdk/metrics/aggregation/exponential_bucket_histogram.rb b/metrics_sdk/lib/opentelemetry/sdk/metrics/aggregation/exponential_bucket_histogram.rb index adad620b2c..73b3e8eae6 100644 --- a/metrics_sdk/lib/opentelemetry/sdk/metrics/aggregation/exponential_bucket_histogram.rb +++ b/metrics_sdk/lib/opentelemetry/sdk/metrics/aggregation/exponential_bucket_histogram.rb @@ -10,14 +10,13 @@ require_relative 'exponential_histogram/logarithm_mapping' require_relative 'exponential_histogram/exponent_mapping' - module OpenTelemetry module SDK module Metrics module Aggregation # Contains the implementation of the ExponentialBucketHistogram aggregation # https://opentelemetry.io/docs/specs/otel/metrics/data-model/#exponentialhistogram - class ExponentialBucketHistogram + class ExponentialBucketHistogram # rubocop:disable Metrics/ClassLength attr_reader :aggregation_temporality # relate to min max scale: https://opentelemetry.io/docs/specs/otel/metrics/sdk/#support-a-minimum-and-maximum-scale @@ -71,11 +70,10 @@ def collect(start_time, end_time) end end - # ruby seems only record local min and max + # rubocop:disable Metrics/MethodLength, Metrics/CyclomaticComplexity def update(amount, attributes) # fetch or initialize the ExponentialHistogramDataPoint hdp = @data_points.fetch(attributes) do - if @record_min_max min = Float::INFINITY max = -Float::INFINITY @@ -95,7 +93,7 @@ def update(amount, attributes) nil, # :exemplars min, # :min max, # :max - @zero_threshold, # :zero_threshold) + @zero_threshold # :zero_threshold) ) end @@ -123,7 +121,7 @@ def update(amount, attributes) is_rescaling_needed = false low = high = 0 - if buckets.counts.empty? + if buckets.counts == [0] # special case of empty buckets.index_start = bucket_index buckets.index_end = bucket_index buckets.index_base = bucket_index @@ -171,12 +169,13 @@ def update(amount, attributes) buckets.index_end = bucket_index end - bucket_index = bucket_index - buckets.index_base + bucket_index -= buckets.index_base bucket_index += buckets.counts.size if bucket_index.negative? buckets.increment_bucket(bucket_index) nil end + # rubocop:enable Metrics/MethodLength, Metrics/CyclomaticComplexity private @@ -203,7 +202,7 @@ def get_scale_change(low, high) def downscale(change, positive, negative) return if change == 0 - raise "Invalid change of scale" if change.negative? + raise 'Invalid change of scale' if change.negative? positive.downscale(change) negative.downscale(change) diff --git a/metrics_sdk/lib/opentelemetry/sdk/metrics/aggregation/exponential_histogram/buckets.rb b/metrics_sdk/lib/opentelemetry/sdk/metrics/aggregation/exponential_histogram/buckets.rb index 9516125852..9da8939007 100644 --- a/metrics_sdk/lib/opentelemetry/sdk/metrics/aggregation/exponential_histogram/buckets.rb +++ b/metrics_sdk/lib/opentelemetry/sdk/metrics/aggregation/exponential_histogram/buckets.rb @@ -9,9 +9,9 @@ module SDK module Metrics module Aggregation module ExponentialHistogram + # Buckets is the fundamental building block of exponential histogram that store bucket/boundary value class Buckets attr_accessor :index_start, :index_end, :index_base - attr_reader :counts def initialize @counts = [0] @@ -30,7 +30,7 @@ def grow(needed, max_size) new_positive_limit = new_size - bias tmp = Array.new(new_size, 0) - tmp[new_positive_limit..-1] = @counts[old_positive_limit..-1] + tmp[new_positive_limit..-1] = @counts[old_positive_limit..] tmp[0...old_positive_limit] = @counts[0...old_positive_limit] @counts = tmp end @@ -39,11 +39,11 @@ def offset @index_start end - def get_offset_counts + def offset_counts bias = @index_base - @index_start - @counts[-bias..-1] + @counts[0...-bias] + @counts[-bias..] + @counts[0...-bias] end - alias_method :counts, :get_offset_counts + alias counts offset_counts def length return 0 if @counts.empty? @@ -67,7 +67,7 @@ def downscale(amount) if bias != 0 @index_base = @index_start @counts.reverse! - @counts = @counts[0...bias].reverse + @counts[bias..-1].reverse + @counts = @counts[0...bias].reverse + @counts[bias..].reverse end size = 1 + @index_end - @index_start @@ -110,4 +110,3 @@ def increment_bucket(bucket_index, increment = 1) end end end - diff --git a/metrics_sdk/lib/opentelemetry/sdk/metrics/aggregation/exponential_histogram/exponent_mapping.rb b/metrics_sdk/lib/opentelemetry/sdk/metrics/aggregation/exponential_histogram/exponent_mapping.rb index e15d130284..d207012f4e 100644 --- a/metrics_sdk/lib/opentelemetry/sdk/metrics/aggregation/exponential_histogram/exponent_mapping.rb +++ b/metrics_sdk/lib/opentelemetry/sdk/metrics/aggregation/exponential_histogram/exponent_mapping.rb @@ -9,6 +9,7 @@ module SDK module Metrics module Aggregation module ExponentialHistogram + # LogarithmMapping for mapping when scale < 0 class ExponentMapping attr_reader :scale @@ -19,7 +20,6 @@ def initialize(scale) end def map_to_index(value) - return @min_normal_lower_boundary_index if value < IEEE754::MIN_NORMAL_VALUE exponent = IEEE754.get_ieee_754_exponent(value) diff --git a/metrics_sdk/lib/opentelemetry/sdk/metrics/aggregation/exponential_histogram/ieee_754.rb b/metrics_sdk/lib/opentelemetry/sdk/metrics/aggregation/exponential_histogram/ieee_754.rb index ff6686d077..45aa009460 100644 --- a/metrics_sdk/lib/opentelemetry/sdk/metrics/aggregation/exponential_histogram/ieee_754.rb +++ b/metrics_sdk/lib/opentelemetry/sdk/metrics/aggregation/exponential_histogram/ieee_754.rb @@ -11,6 +11,7 @@ module SDK module Metrics module Aggregation module ExponentialHistogram + # IEEE754 standard for floating-point calculation module IEEE754 extend Fiddle::Importer dlload Fiddle::Handle::DEFAULT @@ -19,7 +20,7 @@ module IEEE754 EXPONENT_WIDTH = 11 MANTISSA_MASK = (1 << MANTISSA_WIDTH) - 1 - EXPONENT_BIAS = (2 ** (EXPONENT_WIDTH - 1)) - 1 + EXPONENT_BIAS = (2**(EXPONENT_WIDTH - 1)) - 1 EXPONENT_MASK = ((1 << EXPONENT_WIDTH) - 1) << MANTISSA_WIDTH SIGN_MASK = 1 << (EXPONENT_WIDTH + MANTISSA_WIDTH) diff --git a/metrics_sdk/lib/opentelemetry/sdk/metrics/aggregation/exponential_histogram/log2e_scale_factor.rb b/metrics_sdk/lib/opentelemetry/sdk/metrics/aggregation/exponential_histogram/log2e_scale_factor.rb index cd7245eb5b..be15092f11 100644 --- a/metrics_sdk/lib/opentelemetry/sdk/metrics/aggregation/exponential_histogram/log2e_scale_factor.rb +++ b/metrics_sdk/lib/opentelemetry/sdk/metrics/aggregation/exponential_histogram/log2e_scale_factor.rb @@ -9,6 +9,7 @@ module SDK module Metrics module Aggregation module ExponentialHistogram + # Log2eScaleFactor is precomputed scale factor value class Log2eScaleFactor MAX_SCALE = 20 @@ -25,4 +26,4 @@ def self.log2e_scale_buckets end end end -end \ No newline at end of file +end diff --git a/metrics_sdk/lib/opentelemetry/sdk/metrics/aggregation/exponential_histogram/logarithm_mapping.rb b/metrics_sdk/lib/opentelemetry/sdk/metrics/aggregation/exponential_histogram/logarithm_mapping.rb index 14598b0c6e..967bbe8c6c 100644 --- a/metrics_sdk/lib/opentelemetry/sdk/metrics/aggregation/exponential_histogram/logarithm_mapping.rb +++ b/metrics_sdk/lib/opentelemetry/sdk/metrics/aggregation/exponential_histogram/logarithm_mapping.rb @@ -9,12 +9,13 @@ module SDK module Metrics module Aggregation module ExponentialHistogram + # LogarithmMapping for mapping when scale > 0 class LogarithmMapping attr_reader :scale def initialize(scale) @scale = scale - @scale_factor = Log2eScaleFactor::LOG2E_SCALE_BUCKETS[scale] # scale_factor is used for mapping the index + @scale_factor = Log2eScaleFactor::LOG2E_SCALE_BUCKETS[scale] # scale_factor is used for mapping the index @min_normal_lower_boundary_index = IEEE754::MIN_NORMAL_EXPONENT << @scale @max_normal_lower_boundary_index = ((IEEE754::MAX_NORMAL_EXPONENT + 1) << @scale) - 1 end @@ -33,12 +34,14 @@ def map_to_index(value) def get_lower_boundary(inds) if inds >= @max_normal_lower_boundary_index return 2 * Math.exp((inds - (1 << @scale)) / @scale_factor) if inds == @max_normal_lower_boundary_index + raise StandardError, 'mapping overflow' end if inds <= @min_normal_lower_boundary_index return IEEE754::MIN_NORMAL_VALUE if inds == @min_normal_lower_boundary_index return Math.exp((inds + (1 << @scale)) / @scale_factor) / 2 if inds == @min_normal_lower_boundary_index - 1 + raise StandardError, 'mapping underflow' end diff --git a/metrics_sdk/lib/opentelemetry/sdk/metrics/aggregation/exponential_histogram_data_point.rb b/metrics_sdk/lib/opentelemetry/sdk/metrics/aggregation/exponential_histogram_data_point.rb index 7ac46b45c3..50491343c2 100644 --- a/metrics_sdk/lib/opentelemetry/sdk/metrics/aggregation/exponential_histogram_data_point.rb +++ b/metrics_sdk/lib/opentelemetry/sdk/metrics/aggregation/exponential_histogram_data_point.rb @@ -8,23 +8,22 @@ module OpenTelemetry module SDK module Metrics module Aggregation - # TODO: Deal with this later # rubocop:disable Lint/StructNewOverride - ExponentialHistogramDataPoint = Struct.new(:attributes, # optional Hash{String => String, Numeric, Boolean, Array} - :start_time_unix_nano, # Integer nanoseconds since Epoch - :time_unix_nano, # Integer nanoseconds since Epoch - :count, # Integer count is the number of values in the population. Must be non-negative - :sum, # Integer sum of the values in the population. If count is zero then this field then this field must be zero - :scale, # Integer scale factor - :zero_count, # Integer special bucket that count of observations that fall into the zero bucket - :positive, # Buckets representing the positive range of the histogram. - :negative, # Buckets representing the negative range of the histogram. - :flags, # Integer flags associated with the data point. - :exemplars, # optional List of exemplars collected from measurements that were used to form the data point - :min, # optional Float min is the minimum value over (start_time, end_time]. - :max, # optional Float max is the maximum value over (start_time, end_time]. - :zero_threshold) # optional Float the threshold for the zero bucket + ExponentialHistogramDataPoint = Struct.new(:attributes, # optional Hash{String => String, Numeric, Boolean, Array} + :start_time_unix_nano, # Integer nanoseconds since Epoch + :time_unix_nano, # Integer nanoseconds since Epoch + :count, # Integer count is the number of values in the population. Must be non-negative + :sum, # Integer sum of the values in the population. If count is zero then this field then this field must be zero + :scale, # Integer scale factor + :zero_count, # Integer special bucket that count of observations that fall into the zero bucket + :positive, # Buckets representing the positive range of the histogram. + :negative, # Buckets representing the negative range of the histogram. + :flags, # Integer flags associated with the data point. + :exemplars, # optional List of exemplars collected from measurements that were used to form the data point + :min, # optional Float min is the minimum value over (start_time, end_time]. + :max, # optional Float max is the maximum value over (start_time, end_time]. + :zero_threshold) # optional Float the threshold for the zero bucket # rubocop:enable Lint/StructNewOverride end end diff --git a/metrics_sdk/test/opentelemetry/sdk/metrics/aggregation/exponential_bucket_histogram_test.rb b/metrics_sdk/test/opentelemetry/sdk/metrics/aggregation/exponential_bucket_histogram_test.rb index 25723c2233..746ff11884 100644 --- a/metrics_sdk/test/opentelemetry/sdk/metrics/aggregation/exponential_bucket_histogram_test.rb +++ b/metrics_sdk/test/opentelemetry/sdk/metrics/aggregation/exponential_bucket_histogram_test.rb @@ -30,8 +30,8 @@ expbh.update(1.23, {}) expbh.update(0, {}) - expbh.update(1.45, {'foo' => 'bar'}) - expbh.update(1.67, {'foo' => 'bar'}) + expbh.update(1.45, { 'foo' => 'bar' }) + expbh.update(1.67, { 'foo' => 'bar' }) exphdps = expbh.collect(start_time, end_time) @@ -43,7 +43,7 @@ _(exphdps[0].max).must_equal(1.23) _(exphdps[0].scale).must_equal(5) _(exphdps[0].zero_count).must_equal(1) - _(exphdps[0].positive.counts).must_equal([0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0]) + _(exphdps[0].positive.counts).must_equal([1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0]) _(exphdps[0].negative.counts).must_equal([0]) _(exphdps[0].zero_threshold).must_equal(0) @@ -52,9 +52,9 @@ _(exphdps[1].sum).must_equal(3.12) _(exphdps[1].min).must_equal(1.45) _(exphdps[1].max).must_equal(1.67) - _(exphdps[1].scale).must_equal(4) + _(exphdps[1].scale).must_equal(5) _(exphdps[1].zero_count).must_equal(0) - _(exphdps[1].positive.counts).must_equal([0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0]) + _(exphdps[1].positive.counts).must_equal([1, 0, 0, 0, 0, 0, 1, 0]) _(exphdps[1].negative.counts).must_equal([0]) _(exphdps[1].zero_threshold).must_equal(0) end @@ -133,6 +133,122 @@ _(exphdps[0].zero_threshold).must_equal(0) end + it 'test_permutations' do + test_cases = [ + [ + [0.5, 1.0, 2.0], + { + scale: -1, + offset: -1, + len: 2, + at_zero: 2, + at_one: 1 + } + ], + [ + [1.0, 2.0, 4.0], + { + scale: -1, + offset: -1, + len: 2, + at_zero: 1, + at_one: 2 + } + ], + [ + [0.25, 0.5, 1.0], + { + scale: -1, + offset: -2, + len: 2, + at_zero: 1, + at_one: 2 + } + ] + ] + + test_cases.each do |test_values, expected| + test_values.permutation.each do |permutation| + expbh = OpenTelemetry::SDK::Metrics::Aggregation::ExponentialBucketHistogram.new( + aggregation_temporality: aggregation_temporality, + record_min_max: record_min_max, + max_size: 2, + max_scale: 20, # use default value of max scale; should downscale to 0 + zero_threshold: 0 + ) + + permutation.each do |value| + expbh.update(value, {}) + end + + exphdps = expbh.collect(start_time, end_time) + + assert_equal expected[:scale], exphdps[0].scale + assert_equal expected[:offset], exphdps[0].positive.offset + assert_equal expected[:len], exphdps[0].positive.length + assert_equal expected[:at_zero], exphdps[0].positive.counts[0] + assert_equal expected[:at_one], exphdps[0].positive.counts[1] + end + end + end + + it 'test_full_range' do + expbh = OpenTelemetry::SDK::Metrics::Aggregation::ExponentialBucketHistogram.new( + aggregation_temporality: aggregation_temporality, + record_min_max: record_min_max, + max_size: 2, + max_scale: 20, # use default value of max scale; should downscale to 0 + zero_threshold: 0 + ) + + expbh.update(Float::MAX, {}) + expbh.update(1, {}) + expbh.update(2**-1074, {}) + + exphdps = expbh.collect(start_time, end_time) + + assert_equal Float::MAX, exphdps[0].sum + assert_equal 3, exphdps[0].count + assert_equal(-10, exphdps[0].scale) + + assert_equal 2, exphdps[0].positive.length + assert_equal(-1, exphdps[0].positive.offset) + assert_operator exphdps[0].positive.counts[0], :<=, 2 + assert_operator exphdps[0].positive.counts[1], :<=, 1 + end + + it 'test_aggregator_min_max' do + expbh = OpenTelemetry::SDK::Metrics::Aggregation::ExponentialBucketHistogram.new( + aggregation_temporality: aggregation_temporality, + record_min_max: record_min_max, + zero_threshold: 0 + ) + + [1, 3, 5, 7, 9].each do |value| + expbh.update(value, {}) + end + + exphdps = expbh.collect(start_time, end_time) + + assert_equal 1, exphdps[0].min + assert_equal 9, exphdps[0].max + + expbh = OpenTelemetry::SDK::Metrics::Aggregation::ExponentialBucketHistogram.new( + aggregation_temporality: aggregation_temporality, + record_min_max: record_min_max, + zero_threshold: 0 + ) + + [-1, -3, -5, -7, -9].each do |value| + expbh.update(value, {}) + end + + exphdps = expbh.collect(start_time, end_time) + + assert_equal(-9, exphdps[0].min) + assert_equal(-1, exphdps[0].max) + end + it 'test_merge' do # TODO end diff --git a/metrics_sdk/test/opentelemetry/sdk/metrics/aggregation/histogram_mapping_test.rb b/metrics_sdk/test/opentelemetry/sdk/metrics/aggregation/histogram_mapping_test.rb index 9d7c699794..3caaa31f1e 100644 --- a/metrics_sdk/test/opentelemetry/sdk/metrics/aggregation/histogram_mapping_test.rb +++ b/metrics_sdk/test/opentelemetry/sdk/metrics/aggregation/histogram_mapping_test.rb @@ -29,7 +29,6 @@ def right_boundary(scale, index) end describe OpenTelemetry::SDK::Metrics::Aggregation::ExponentialBucketHistogram do - MAX_NORMAL_EXPONENT = OpenTelemetry::SDK::Metrics::Aggregation::ExponentialHistogram::IEEE754::MAX_NORMAL_EXPONENT MIN_NORMAL_EXPONENT = OpenTelemetry::SDK::Metrics::Aggregation::ExponentialHistogram::IEEE754::MIN_NORMAL_EXPONENT MAX_NORMAL_VALUE = OpenTelemetry::SDK::Metrics::Aggregation::ExponentialHistogram::IEEE754::MAX_NORMAL_VALUE @@ -55,7 +54,6 @@ def right_boundary(scale, index) end it 'test_logarithm_boundary' do - [1, 2, 3, 4, 10, 15].each do |scale| logarithm_mapping = OpenTelemetry::SDK::Metrics::Aggregation::ExponentialHistogram::LogarithmMapping.new(scale) @@ -86,17 +84,17 @@ def right_boundary(scale, index) _(boundary).must_be :<, OpenTelemetry::SDK::Metrics::Aggregation::ExponentialHistogram::IEEE754::MAX_NORMAL_VALUE - _((OpenTelemetry::SDK::Metrics::Aggregation::ExponentialHistogram::IEEE754::MAX_NORMAL_VALUE - boundary) / boundary ).must_be_within_epsilon base - 1, 1e-6 + _((OpenTelemetry::SDK::Metrics::Aggregation::ExponentialHistogram::IEEE754::MAX_NORMAL_VALUE - boundary) / boundary).must_be_within_epsilon base - 1, 1e-6 error = assert_raises(StandardError) do logarithm_mapping.get_lower_boundary(inds + 1) end - assert_equal("mapping overflow", error.message) + assert_equal('mapping overflow', error.message) error = assert_raises(StandardError) do logarithm_mapping.get_lower_boundary(inds + 2) end - assert_equal("mapping overflow", error.message) + assert_equal('mapping overflow', error.message) end end @@ -135,26 +133,24 @@ def right_boundary(scale, index) assert_equal('mapping underflow', error.message) end end - end describe 'exponent_mapping' do let(:exponent_mapping_min_scale) { -10 } - - it 'test_exponent_mapping_zero' do + it 'test_exponent_mapping_zero' do exponent_mapping = OpenTelemetry::SDK::Metrics::Aggregation::ExponentialHistogram::ExponentMapping.new(0) # This is the equivalent to 1.1 in hexadecimal - hex_1_1 = 1 + (1.0 / 16) + hex_one_one = 1 + (1.0 / 16) # Testing with values near +inf _(exponent_mapping.map_to_index(MAX_NORMAL_VALUE)).must_equal(MAX_NORMAL_EXPONENT) _(exponent_mapping.map_to_index(MAX_NORMAL_VALUE)).must_equal(1023) _(exponent_mapping.map_to_index(2**1023)).must_equal(1022) _(exponent_mapping.map_to_index(2**1022)).must_equal(1021) - _(exponent_mapping.map_to_index(hex_1_1 * (2**1023))).must_equal(1023) - _(exponent_mapping.map_to_index(hex_1_1 * (2**1022))).must_equal(1022) + _(exponent_mapping.map_to_index(hex_one_one * (2**1023))).must_equal(1023) + _(exponent_mapping.map_to_index(hex_one_one * (2**1022))).must_equal(1022) # Testing with values near 1 _(exponent_mapping.map_to_index(4)).must_equal(1) @@ -171,9 +167,9 @@ def right_boundary(scale, index) # Testing with values near 0 _(exponent_mapping.map_to_index(2**-1022)).must_equal(-1023) - _(exponent_mapping.map_to_index(hex_1_1 * (2**-1022))).must_equal(-1022) + _(exponent_mapping.map_to_index(hex_one_one * (2**-1022))).must_equal(-1022) _(exponent_mapping.map_to_index(2**-1021)).must_equal(-1022) - _(exponent_mapping.map_to_index(hex_1_1 * (2**-1021))).must_equal(-1021) + _(exponent_mapping.map_to_index(hex_one_one * (2**-1021))).must_equal(-1021) _(exponent_mapping.map_to_index(2**-1022)).must_equal(MIN_NORMAL_EXPONENT - 1) _(exponent_mapping.map_to_index(2**-1021)).must_equal(MIN_NORMAL_EXPONENT) @@ -211,7 +207,7 @@ def right_boundary(scale, index) end it 'test_exponent_mapping_negative_four' do - exponent_mapping = OpenTelemetry::SDK::Metrics::Aggregation::ExponentialHistogram::ExponentMapping.new(-4) + exponent_mapping = OpenTelemetry::SDK::Metrics::Aggregation::ExponentialHistogram::ExponentMapping.new(-4) _(exponent_mapping.map_to_index(0x1.to_f)).must_equal(-1) _(exponent_mapping.map_to_index(0x10.to_f)).must_equal(0) @@ -316,7 +312,6 @@ def right_boundary(scale, index) end assert_equal('mapping underflow', error.message) end - end it 'test_exponent_index_min' do @@ -356,7 +351,5 @@ def right_boundary(scale, index) _(exponent_mapping.map_to_index(Float::MIN.next_float)).must_equal(MIN_NORMAL_EXPONENT >> -scale) end end - end - end