From 8690e7a0ea810ac84b4678e770131d5e7e42b6a7 Mon Sep 17 00:00:00 2001 From: Stephen Toub Date: Thu, 17 Oct 2024 16:55:17 -0400 Subject: [PATCH] Update UseOpenTelemetry for latest genai spec updates (#5532) * Update UseOpenTelemetry for latest genai spec updates - Events are now expected to be emitted as body fields, and the newly-recommended way to achieve that is via ILogger. So UseOpenTelemetry now takes an optional logger that it uses for emitting such data. - I restructured the implementation to reduce duplication. - Added logging of response format and seed. - Added ChatOptions.TopK, as it's one of the parameters considered special by the spec. - Updated the Azure.AI.Inference provider name to match the convention and what the library itself uses - Updated the OpenAI client to use openai regardless of the kind of the actual client being used, per spec and recommendation * Address PR feedback --- .../ChatCompletion/ChatOptions.cs | 3 + .../AzureAIInferenceChatClient.cs | 11 +- .../OllamaChatClient.cs | 6 +- .../OpenAIChatClient.cs | 6 +- .../OpenAIEmbeddingGenerator.cs | 8 +- .../ChatCompletion/OpenTelemetryChatClient.cs | 494 +++++++++++------- ...penTelemetryChatClientBuilderExtensions.cs | 15 +- .../OpenTelemetryEmbeddingGenerator.cs | 97 ++-- ...etryEmbeddingGeneratorBuilderExtensions.cs | 17 +- .../Microsoft.Extensions.AI.csproj | 1 + .../OpenTelemetryConsts.cs | 42 +- .../AzureAIInferenceChatClientTests.cs | 2 +- .../ChatClientIntegrationTests.cs | 16 +- .../EmbeddingGeneratorIntegrationTests.cs | 2 +- .../OpenAIChatClientTests.cs | 29 +- .../OpenAIEmbeddingGeneratorTests.cs | 29 +- .../OpenTelemetryChatClientTests.cs | 221 ++++---- .../Microsoft.Extensions.AI.Tests.csproj | 1 + 18 files changed, 536 insertions(+), 464 deletions(-) diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/ChatCompletion/ChatOptions.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/ChatCompletion/ChatOptions.cs index 4f02815580e..b3b60c62bad 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/ChatCompletion/ChatOptions.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/ChatCompletion/ChatOptions.cs @@ -18,6 +18,9 @@ public class ChatOptions /// Gets or sets the "nucleus sampling" factor (or "top p") for generating chat responses. public float? TopP { get; set; } + /// Gets or sets a count indicating how many of the most probable tokens the model should consider when generating the next part of the text. + public int? TopK { get; set; } + /// Gets or sets the frequency penalty for generating chat responses. public float? FrequencyPenalty { get; set; } diff --git a/src/Libraries/Microsoft.Extensions.AI.AzureAIInference/AzureAIInferenceChatClient.cs b/src/Libraries/Microsoft.Extensions.AI.AzureAIInference/AzureAIInferenceChatClient.cs index c422e622065..c3313c0c85b 100644 --- a/src/Libraries/Microsoft.Extensions.AI.AzureAIInference/AzureAIInferenceChatClient.cs +++ b/src/Libraries/Microsoft.Extensions.AI.AzureAIInference/AzureAIInferenceChatClient.cs @@ -48,7 +48,7 @@ public AzureAIInferenceChatClient(ChatCompletionsClient chatCompletionsClient, s var providerUrl = typeof(ChatCompletionsClient).GetField("_endpoint", BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Instance) ?.GetValue(chatCompletionsClient) as Uri; - Metadata = new("AzureAIInference", providerUrl, modelId); + Metadata = new("az.ai.inference", providerUrl, modelId); } /// Gets or sets to use for any serialization activities related to tool call arguments and results. @@ -296,13 +296,19 @@ private ChatCompletionsOptions ToAzureAIOptions(IList chatContents, } } + // These properties are strongly-typed on ChatOptions but not on ChatCompletionsOptions. + if (options.TopK is int topK) + { + result.AdditionalProperties["top_k"] = BinaryData.FromObjectAsJson(topK, JsonContext.Default.Options); + } + if (options.AdditionalProperties is { } props) { foreach (var prop in props) { switch (prop.Key) { - // These properties are strongly-typed on the ChatCompletionsOptions class. + // These properties are strongly-typed on the ChatCompletionsOptions class but not on the ChatOptions class. case nameof(result.Seed) when prop.Value is long seed: result.Seed = seed; break; @@ -498,5 +504,6 @@ private static FunctionCallContent ParseCallContentFromJsonString(string json, s [JsonSerializable(typeof(AzureAIChatToolJson))] [JsonSerializable(typeof(IDictionary))] [JsonSerializable(typeof(JsonElement))] + [JsonSerializable(typeof(int))] private sealed partial class JsonContext : JsonSerializerContext; } diff --git a/src/Libraries/Microsoft.Extensions.AI.Ollama/OllamaChatClient.cs b/src/Libraries/Microsoft.Extensions.AI.Ollama/OllamaChatClient.cs index dac6f915d83..22ff6db6dab 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Ollama/OllamaChatClient.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Ollama/OllamaChatClient.cs @@ -259,7 +259,6 @@ private OllamaChatRequest ToOllamaChatRequest(IList chatMessages, C TransferMetadataValue(nameof(OllamaRequestOptions.repeat_penalty), (options, value) => options.repeat_penalty = value); TransferMetadataValue(nameof(OllamaRequestOptions.seed), (options, value) => options.seed = value); TransferMetadataValue(nameof(OllamaRequestOptions.tfs_z), (options, value) => options.tfs_z = value); - TransferMetadataValue(nameof(OllamaRequestOptions.top_k), (options, value) => options.top_k = value); TransferMetadataValue(nameof(OllamaRequestOptions.typical_p), (options, value) => options.typical_p = value); TransferMetadataValue(nameof(OllamaRequestOptions.use_mmap), (options, value) => options.use_mmap = value); TransferMetadataValue(nameof(OllamaRequestOptions.use_mlock), (options, value) => options.use_mlock = value); @@ -294,6 +293,11 @@ private OllamaChatRequest ToOllamaChatRequest(IList chatMessages, C { (request.Options ??= new()).top_p = topP; } + + if (options.TopK is int topK) + { + (request.Options ??= new()).top_k = topK; + } } return request; diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIChatClient.cs b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIChatClient.cs index 647c5aaf6ca..dbe415ad818 100644 --- a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIChatClient.cs +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIChatClient.cs @@ -50,11 +50,10 @@ public OpenAIChatClient(OpenAIClient openAIClient, string modelId) // The endpoint isn't currently exposed, so use reflection to get at it, temporarily. Once packages // implement the abstractions directly rather than providing adapters on top of the public APIs, // the package can provide such implementations separate from what's exposed in the public API. - string providerName = openAIClient.GetType().Name.StartsWith("Azure", StringComparison.Ordinal) ? "azureopenai" : "openai"; Uri providerUrl = typeof(OpenAIClient).GetField("_endpoint", BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Instance) ?.GetValue(openAIClient) as Uri ?? _defaultOpenAIEndpoint; - Metadata = new(providerName, providerUrl, modelId); + Metadata = new("openai", providerUrl, modelId); } /// Initializes a new instance of the class for the specified . @@ -69,13 +68,12 @@ public OpenAIChatClient(ChatClient chatClient) // The endpoint and model aren't currently exposed, so use reflection to get at them, temporarily. Once packages // implement the abstractions directly rather than providing adapters on top of the public APIs, // the package can provide such implementations separate from what's exposed in the public API. - string providerName = chatClient.GetType().Name.StartsWith("Azure", StringComparison.Ordinal) ? "azureopenai" : "openai"; Uri providerUrl = typeof(ChatClient).GetField("_endpoint", BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Instance) ?.GetValue(chatClient) as Uri ?? _defaultOpenAIEndpoint; string? model = typeof(ChatClient).GetField("_model", BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Instance) ?.GetValue(chatClient) as string; - Metadata = new(providerName, providerUrl, model); + Metadata = new("openai", providerUrl, model); } /// Gets or sets to use for any serialization activities related to tool call arguments and results. diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIEmbeddingGenerator.cs b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIEmbeddingGenerator.cs index 084e235df47..27bf001b3ff 100644 --- a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIEmbeddingGenerator.cs +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIEmbeddingGenerator.cs @@ -52,12 +52,11 @@ public OpenAIEmbeddingGenerator( // The endpoint isn't currently exposed, so use reflection to get at it, temporarily. Once packages // implement the abstractions directly rather than providing adapters on top of the public APIs, // the package can provide such implementations separate from what's exposed in the public API. - string providerName = openAIClient.GetType().Name.StartsWith("Azure", StringComparison.Ordinal) ? "azureopenai" : "openai"; string providerUrl = (typeof(OpenAIClient).GetField("_endpoint", BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Instance) ?.GetValue(openAIClient) as Uri)?.ToString() ?? DefaultOpenAIEndpoint; - Metadata = CreateMetadata(dimensions, providerName, providerUrl, modelId); + Metadata = CreateMetadata("openai", providerUrl, modelId, dimensions); } /// Initializes a new instance of the class. @@ -78,7 +77,6 @@ public OpenAIEmbeddingGenerator(EmbeddingClient embeddingClient, int? dimensions // The endpoint and model aren't currently exposed, so use reflection to get at them, temporarily. Once packages // implement the abstractions directly rather than providing adapters on top of the public APIs, // the package can provide such implementations separate from what's exposed in the public API. - string providerName = embeddingClient.GetType().Name.StartsWith("Azure", StringComparison.Ordinal) ? "azureopenai" : "openai"; string providerUrl = (typeof(EmbeddingClient).GetField("_endpoint", BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Instance) ?.GetValue(embeddingClient) as Uri)?.ToString() ?? DefaultOpenAIEndpoint; @@ -86,11 +84,11 @@ public OpenAIEmbeddingGenerator(EmbeddingClient embeddingClient, int? dimensions FieldInfo? modelField = typeof(EmbeddingClient).GetField("_model", BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Instance); string? model = modelField?.GetValue(embeddingClient) as string; - Metadata = CreateMetadata(dimensions, providerName, providerUrl, model); + Metadata = CreateMetadata("openai", providerUrl, model, dimensions); } /// Creates the for this instance. - private static EmbeddingGeneratorMetadata CreateMetadata(int? dimensions, string providerName, string providerUrl, string? model) => + private static EmbeddingGeneratorMetadata CreateMetadata(string providerName, string providerUrl, string? model, int? dimensions) => new(providerName, Uri.TryCreate(providerUrl, UriKind.Absolute, out Uri? providerUri) ? providerUri : null, model, dimensions); /// diff --git a/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/OpenTelemetryChatClient.cs b/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/OpenTelemetryChatClient.cs index a544e746ae2..46e26bea181 100644 --- a/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/OpenTelemetryChatClient.cs +++ b/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/OpenTelemetryChatClient.cs @@ -4,13 +4,17 @@ using System; using System.Collections.Generic; using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; using System.Diagnostics.Metrics; using System.Linq; using System.Runtime.CompilerServices; -using System.Text; using System.Text.Json; +using System.Text.Json.Nodes; +using System.Text.Json.Serialization; using System.Threading; using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Shared.Diagnostics; namespace Microsoft.Extensions.AI; @@ -20,34 +24,40 @@ namespace Microsoft.Extensions.AI; /// The draft specification this follows is available at https://opentelemetry.io/docs/specs/semconv/gen-ai/. /// The specification is still experimental and subject to change; as such, the telemetry output by this client is also subject to change. /// -public sealed class OpenTelemetryChatClient : DelegatingChatClient +public sealed partial class OpenTelemetryChatClient : DelegatingChatClient { + private const LogLevel EventLogLevel = LogLevel.Information; + private readonly ActivitySource _activitySource; private readonly Meter _meter; + private readonly ILogger _logger; private readonly Histogram _tokenUsageHistogram; private readonly Histogram _operationDurationHistogram; private readonly string? _modelId; - private readonly string? _modelProvider; - private readonly string? _endpointAddress; - private readonly int _endpointPort; + private readonly string? _system; + private readonly string? _serverAddress; + private readonly int _serverPort; private JsonSerializerOptions _jsonSerializerOptions; /// Initializes a new instance of the class. /// The underlying . + /// The to use for emitting events. /// An optional source name that will be used on the telemetry data. - public OpenTelemetryChatClient(IChatClient innerClient, string? sourceName = null) + public OpenTelemetryChatClient(IChatClient innerClient, ILogger? logger = null, string? sourceName = null) : base(innerClient) { Debug.Assert(innerClient is not null, "Should have been validated by the base ctor"); + _logger = logger ?? NullLogger.Instance; + ChatClientMetadata metadata = innerClient!.Metadata; _modelId = metadata.ModelId; - _modelProvider = metadata.ProviderName; - _endpointAddress = metadata.ProviderUri?.GetLeftPart(UriPartial.Path); - _endpointPort = metadata.ProviderUri?.Port ?? 0; + _system = metadata.ProviderName; + _serverAddress = metadata.ProviderUri?.GetLeftPart(UriPartial.Path); + _serverPort = metadata.ProviderUri?.Port ?? 0; string name = string.IsNullOrEmpty(sourceName) ? OpenTelemetryConsts.DefaultSourceName : sourceName!; _activitySource = new(name); @@ -88,27 +98,32 @@ protected override void Dispose(bool disposing) } /// - /// Gets or sets a value indicating whether potentially sensitive information (e.g. prompts) should be included in telemetry. + /// Gets or sets a value indicating whether potentially sensitive information should be included in telemetry. /// /// - /// The value is by default, meaning that telemetry will include metadata such as token counts but not the raw text of prompts or completions. + /// The value is by default, meaning that telemetry will include metadata such as token counts but not raw inputs + /// and outputs such as message content, function call arguments, and function call results. /// public bool EnableSensitiveData { get; set; } /// public override async Task CompleteAsync(IList chatMessages, ChatOptions? options = null, CancellationToken cancellationToken = default) { + _ = Throw.IfNull(chatMessages); _jsonSerializerOptions.MakeReadOnly(); - using Activity? activity = StartActivity(chatMessages, options); + using Activity? activity = CreateAndConfigureActivity(options); Stopwatch? stopwatch = _operationDurationHistogram.Enabled ? Stopwatch.StartNew() : null; string? requestModelId = options?.ModelId ?? _modelId; - ChatCompletion? response = null; + LogChatMessages(chatMessages); + + ChatCompletion? completion = null; Exception? error = null; try { - response = await base.CompleteAsync(chatMessages, options, cancellationToken).ConfigureAwait(false); + completion = await base.CompleteAsync(chatMessages, options, cancellationToken).ConfigureAwait(false); + return completion; } catch (Exception ex) { @@ -117,35 +132,37 @@ public override async Task CompleteAsync(IList chat } finally { - SetCompletionResponse(activity, requestModelId, response, error, stopwatch); + TraceCompletion(activity, requestModelId, completion, error, stopwatch); } - - return response; } /// public override async IAsyncEnumerable CompleteStreamingAsync( IList chatMessages, ChatOptions? options = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) { + _ = Throw.IfNull(chatMessages); _jsonSerializerOptions.MakeReadOnly(); - using Activity? activity = StartActivity(chatMessages, options); + using Activity? activity = CreateAndConfigureActivity(options); Stopwatch? stopwatch = _operationDurationHistogram.Enabled ? Stopwatch.StartNew() : null; string? requestModelId = options?.ModelId ?? _modelId; - IAsyncEnumerable response; + LogChatMessages(chatMessages); + + IAsyncEnumerable updates; try { - response = base.CompleteStreamingAsync(chatMessages, options, cancellationToken); + updates = base.CompleteStreamingAsync(chatMessages, options, cancellationToken); } catch (Exception ex) { - SetCompletionResponse(activity, requestModelId, null, ex, stopwatch); + TraceCompletion(activity, requestModelId, completion: null, ex, stopwatch); throw; } - var responseEnumerator = response.ConfigureAwait(false).GetAsyncEnumerator(); - List? streamedContents = activity is not null ? [] : null; + var responseEnumerator = updates.ConfigureAwait(false).GetAsyncEnumerator(); + List trackedUpdates = []; + Exception? error = null; try { while (true) @@ -162,167 +179,154 @@ public override async IAsyncEnumerable CompleteSt } catch (Exception ex) { - SetCompletionResponse(activity, requestModelId, null, ex, stopwatch); + error = ex; throw; } - streamedContents?.Add(update); + trackedUpdates.Add(update); yield return update; } } finally { - if (activity is not null) - { - UsageContent? usageContent = streamedContents?.SelectMany(c => c.Contents).OfType().LastOrDefault(); - SetCompletionResponse( - activity, - stopwatch, - requestModelId, - OrganizeStreamingContent(streamedContents), - streamedContents?.SelectMany(c => c.Contents).OfType(), - usage: usageContent?.Details); - } + TraceCompletion(activity, requestModelId, ComposeStreamingUpdatesIntoChatCompletion(trackedUpdates), error, stopwatch); await responseEnumerator.DisposeAsync(); } } - /// Gets a value indicating whether diagnostics are enabled. - private bool Enabled => _activitySource.HasListeners(); - - /// Convert chat history to a string aligned with the OpenAI format. - private static string ToOpenAIFormat(IEnumerable messages, JsonSerializerOptions serializerOptions) + /// Creates a from a collection of instances. + /// + /// This only propagates information that's later used by the telemetry. If additional information from the + /// is needed, this implementation should be updated to include it. + /// + private static ChatCompletion ComposeStreamingUpdatesIntoChatCompletion( + List updates) { - var sb = new StringBuilder().Append('['); - - string messageSeparator = string.Empty; - foreach (var message in messages) + // Group updates by choice index. + Dictionary> choices = []; + foreach (var update in updates) { - _ = sb.Append(messageSeparator); - messageSeparator = ", \n"; - - string text = string.Concat(message.Contents.OfType().Select(c => c.Text)); - _ = sb.Append("{\"role\": \"").Append(message.Role).Append("\", \"content\": ").Append(JsonSerializer.Serialize(text, serializerOptions.GetTypeInfo(typeof(string)))); - - if (message.Contents.OfType().Any()) + if (!choices.TryGetValue(update.ChoiceIndex, out var choiceContents)) { - _ = sb.Append(", \"tool_calls\": ").Append('['); - - string messageItemSeparator = string.Empty; - foreach (var functionCall in message.Contents.OfType()) - { - _ = sb.Append(messageItemSeparator); - messageItemSeparator = ", \n"; - - _ = sb.Append("{\"id\": \"").Append(functionCall.CallId) - .Append("\", \"function\": {\"arguments\": ").Append(JsonSerializer.Serialize(functionCall.Arguments, serializerOptions.GetTypeInfo(typeof(IDictionary)))) - .Append(", \"name\": \"").Append(functionCall.Name) - .Append("\"}, \"type\": \"function\"}"); - } - - _ = sb.Append(']'); + choices[update.ChoiceIndex] = choiceContents = []; } - _ = sb.Append('}'); - } - - _ = sb.Append(']'); - return sb.ToString(); - } - - /// Organize streaming content by choice index. - private static Dictionary> OrganizeStreamingContent(IEnumerable? contents) - { - Dictionary> choices = []; - if (contents is null) - { - return choices; + choiceContents.Add(update); } - foreach (var content in contents) + // Add a ChatMessage for each choice. + string? id = null; + ChatFinishReason? finishReason = null; + string? modelId = null; + List messages = new(choices.Count); + foreach (var choice in choices.OrderBy(c => c.Key)) { - if (!choices.TryGetValue(content.ChoiceIndex, out var choiceContents)) + ChatRole? role = null; + List items = []; + foreach (var update in choice.Value) { - choices[content.ChoiceIndex] = choiceContents = []; + id ??= update.CompletionId; + finishReason ??= update.FinishReason; + role ??= update.Role; + items.AddRange(update.Contents); + modelId ??= update.Contents.FirstOrDefault(c => c.ModelId is not null)?.ModelId; } - choiceContents.Add(content); + messages.Add(new ChatMessage(role ?? ChatRole.Assistant, items)); } - return choices; + return new(messages) + { + CompletionId = id, + FinishReason = finishReason, + ModelId = modelId, + Usage = updates.SelectMany(c => c.Contents).OfType().LastOrDefault()?.Details, + }; } /// Creates an activity for a chat completion request, or returns null if not enabled. - private Activity? StartActivity(IList chatMessages, ChatOptions? options) + private Activity? CreateAndConfigureActivity(ChatOptions? options) { Activity? activity = null; - if (Enabled) + if (_activitySource.HasListeners()) { string? modelId = options?.ModelId ?? _modelId; activity = _activitySource.StartActivity( - $"chat.completions {modelId}", - ActivityKind.Client, - default(ActivityContext), - [ - new(OpenTelemetryConsts.GenAI.Operation.Name, "chat"), - new(OpenTelemetryConsts.GenAI.Request.Model, modelId), - new(OpenTelemetryConsts.GenAI.System, _modelProvider), - ]); + $"{OpenTelemetryConsts.GenAI.Chat} {modelId}", + ActivityKind.Client); if (activity is not null) { - if (_endpointAddress is not null) + _ = activity + .AddTag(OpenTelemetryConsts.GenAI.Operation.Name, OpenTelemetryConsts.GenAI.Chat) + .AddTag(OpenTelemetryConsts.GenAI.Request.Model, modelId) + .AddTag(OpenTelemetryConsts.GenAI.SystemName, _system); + + if (_serverAddress is not null) { _ = activity - .SetTag(OpenTelemetryConsts.Server.Address, _endpointAddress) - .SetTag(OpenTelemetryConsts.Server.Port, _endpointPort); + .AddTag(OpenTelemetryConsts.Server.Address, _serverAddress) + .AddTag(OpenTelemetryConsts.Server.Port, _serverPort); } if (options is not null) { if (options.FrequencyPenalty is float frequencyPenalty) { - _ = activity.SetTag(OpenTelemetryConsts.GenAI.Request.FrequencyPenalty, frequencyPenalty); + _ = activity.AddTag(OpenTelemetryConsts.GenAI.Request.FrequencyPenalty, frequencyPenalty); } if (options.MaxOutputTokens is int maxTokens) { - _ = activity.SetTag(OpenTelemetryConsts.GenAI.Request.MaxTokens, maxTokens); + _ = activity.AddTag(OpenTelemetryConsts.GenAI.Request.MaxTokens, maxTokens); } if (options.PresencePenalty is float presencePenalty) { - _ = activity.SetTag(OpenTelemetryConsts.GenAI.Request.PresencePenalty, presencePenalty); + _ = activity.AddTag(OpenTelemetryConsts.GenAI.Request.PresencePenalty, presencePenalty); } if (options.StopSequences is IList stopSequences) { - _ = activity.SetTag(OpenTelemetryConsts.GenAI.Request.StopSequences, $"[{string.Join(", ", stopSequences.Select(s => $"\"{s}\""))}]"); + _ = activity.AddTag(OpenTelemetryConsts.GenAI.Request.StopSequences, $"[{string.Join(", ", stopSequences.Select(s => $"\"{s}\""))}]"); } if (options.Temperature is float temperature) { - _ = activity.SetTag(OpenTelemetryConsts.GenAI.Request.Temperature, temperature); + _ = activity.AddTag(OpenTelemetryConsts.GenAI.Request.Temperature, temperature); } - if (options.AdditionalProperties?.TryGetValue("top_k", out double topK) is true) + if (options.TopK is int topK) { - _ = activity.SetTag(OpenTelemetryConsts.GenAI.Request.TopK, topK); + _ = activity.AddTag(OpenTelemetryConsts.GenAI.Request.TopK, topK); } if (options.TopP is float top_p) { - _ = activity.SetTag(OpenTelemetryConsts.GenAI.Request.TopP, top_p); + _ = activity.AddTag(OpenTelemetryConsts.GenAI.Request.TopP, top_p); } - } - if (EnableSensitiveData) - { - _ = activity.AddEvent(new ActivityEvent( - OpenTelemetryConsts.GenAI.Content.Prompt, - tags: new ActivityTagsCollection([new(OpenTelemetryConsts.GenAI.Prompt, ToOpenAIFormat(chatMessages, _jsonSerializerOptions))]))); + if (_system is not null) + { + if (options.ResponseFormat is not null) + { + string responseFormat = options.ResponseFormat switch + { + ChatResponseFormatText => "text", + ChatResponseFormatJson { Schema: null } => "json_schema", + ChatResponseFormatJson => "json_object", + _ => "_OTHER", + }; + _ = activity.AddTag(OpenTelemetryConsts.GenAI.Request.PerProvider(_system, "response_format"), responseFormat); + } + + if (options.AdditionalProperties?.TryGetValue("seed", out long seed) is true) + { + _ = activity.AddTag(OpenTelemetryConsts.GenAI.Request.PerProvider(_system, "seed"), seed); + } + } } } } @@ -331,23 +335,18 @@ private static Dictionary> OrganizeStre } /// Adds chat completion information to the activity. - private void SetCompletionResponse( + private void TraceCompletion( Activity? activity, string? requestModelId, - ChatCompletion? completions, + ChatCompletion? completion, Exception? error, Stopwatch? stopwatch) { - if (!Enabled) - { - return; - } - if (_operationDurationHistogram.Enabled && stopwatch is not null) { TagList tags = default; - AddMetricTags(ref tags, requestModelId, completions); + AddMetricTags(ref tags, requestModelId, completion); if (error is not null) { tags.Add(OpenTelemetryConsts.Error.Type, error.GetType().FullName); @@ -356,13 +355,13 @@ private void SetCompletionResponse( _operationDurationHistogram.Record(stopwatch.Elapsed.TotalSeconds, tags); } - if (_tokenUsageHistogram.Enabled && completions?.Usage is { } usage) + if (_tokenUsageHistogram.Enabled && completion?.Usage is { } usage) { if (usage.InputTokenCount is int inputTokens) { TagList tags = default; tags.Add(OpenTelemetryConsts.GenAI.Token.Type, "input"); - AddMetricTags(ref tags, requestModelId, completions); + AddMetricTags(ref tags, requestModelId, completion); _tokenUsageHistogram.Record(inputTokens); } @@ -370,139 +369,230 @@ private void SetCompletionResponse( { TagList tags = default; tags.Add(OpenTelemetryConsts.GenAI.Token.Type, "output"); - AddMetricTags(ref tags, requestModelId, completions); + AddMetricTags(ref tags, requestModelId, completion); _tokenUsageHistogram.Record(outputTokens); } } - if (activity is null) - { - return; - } - if (error is not null) { - _ = activity - .SetTag(OpenTelemetryConsts.Error.Type, error.GetType().FullName) + _ = activity? + .AddTag(OpenTelemetryConsts.Error.Type, error.GetType().FullName) .SetStatus(ActivityStatusCode.Error, error.Message); - return; } - if (completions is not null) + if (completion is not null) { - if (completions.FinishReason is ChatFinishReason finishReason) + LogChatCompletion(completion); + + if (activity is not null) { + if (completion.FinishReason is ChatFinishReason finishReason) + { #pragma warning disable CA1308 // Normalize strings to uppercase - _ = activity.SetTag(OpenTelemetryConsts.GenAI.Response.FinishReasons, $"[\"{finishReason.Value.ToLowerInvariant()}\"]"); + _ = activity.AddTag(OpenTelemetryConsts.GenAI.Response.FinishReasons, $"[\"{finishReason.Value.ToLowerInvariant()}\"]"); #pragma warning restore CA1308 - } + } - if (!string.IsNullOrWhiteSpace(completions.CompletionId)) - { - _ = activity.SetTag(OpenTelemetryConsts.GenAI.Response.Id, completions.CompletionId); - } + if (!string.IsNullOrWhiteSpace(completion.CompletionId)) + { + _ = activity.AddTag(OpenTelemetryConsts.GenAI.Response.Id, completion.CompletionId); + } - if (completions.ModelId is not null) - { - _ = activity.SetTag(OpenTelemetryConsts.GenAI.Response.Model, completions.ModelId); + if (completion.ModelId is not null) + { + _ = activity.AddTag(OpenTelemetryConsts.GenAI.Response.Model, completion.ModelId); + } + + if (completion.Usage?.InputTokenCount is int inputTokens) + { + _ = activity.AddTag(OpenTelemetryConsts.GenAI.Response.InputTokens, inputTokens); + } + + if (completion.Usage?.OutputTokenCount is int outputTokens) + { + _ = activity.AddTag(OpenTelemetryConsts.GenAI.Response.OutputTokens, outputTokens); + } } + } + + void AddMetricTags(ref TagList tags, string? requestModelId, ChatCompletion? completions) + { + tags.Add(OpenTelemetryConsts.GenAI.Operation.Name, OpenTelemetryConsts.GenAI.Chat); - if (completions.Usage?.InputTokenCount is int inputTokens) + if (requestModelId is not null) { - _ = activity.SetTag(OpenTelemetryConsts.GenAI.Response.InputTokens, inputTokens); + tags.Add(OpenTelemetryConsts.GenAI.Request.Model, requestModelId); } - if (completions.Usage?.OutputTokenCount is int outputTokens) + tags.Add(OpenTelemetryConsts.GenAI.SystemName, _system); + + if (_serverAddress is string endpointAddress) { - _ = activity.SetTag(OpenTelemetryConsts.GenAI.Response.OutputTokens, outputTokens); + tags.Add(OpenTelemetryConsts.Server.Address, endpointAddress); + tags.Add(OpenTelemetryConsts.Server.Port, _serverPort); } - if (EnableSensitiveData) + if (completions?.ModelId is string responseModel) { - _ = activity.AddEvent(new ActivityEvent( - OpenTelemetryConsts.GenAI.Content.Completion, - tags: new ActivityTagsCollection([new(OpenTelemetryConsts.GenAI.Completion, ToOpenAIFormat(completions.Choices, _jsonSerializerOptions))]))); + tags.Add(OpenTelemetryConsts.GenAI.Response.Model, responseModel); } } } - /// Adds streaming chat completion information to the activity. - private void SetCompletionResponse( - Activity? activity, - Stopwatch? stopwatch, - string? requestModelId, - Dictionary> choices, - IEnumerable? toolCalls, - UsageDetails? usage) + private void LogChatMessages(IEnumerable messages) { - if (activity is null || !Enabled || choices.Count == 0) + if (!_logger.IsEnabled(EventLogLevel)) { return; } - string? id = null; - ChatFinishReason? finishReason = null; - string? modelId = null; - List messages = new(choices.Count); - - foreach (var choice in choices) + foreach (ChatMessage message in messages) { - ChatRole? role = null; - List items = []; - foreach (var update in choice.Value) + if (message.Role == ChatRole.Assistant) { - id ??= update.CompletionId; - role ??= update.Role; - finishReason ??= update.FinishReason; - foreach (AIContent content in update.Contents) + Log(new(1, OpenTelemetryConsts.GenAI.Assistant.Message), + JsonSerializer.Serialize(CreateAssistantEvent(message), OtelContext.Default.AssistantEvent)); + } + else if (message.Role == ChatRole.Tool) + { + foreach (FunctionResultContent frc in message.Contents.OfType()) { - items.Add(content); - modelId ??= content.ModelId; + Log(new(1, OpenTelemetryConsts.GenAI.Tool.Message), + JsonSerializer.Serialize(new() + { + Id = frc.CallId, + Content = EnableSensitiveData && frc.Result is object result ? + JsonSerializer.SerializeToNode(result, _jsonSerializerOptions.GetTypeInfo(result.GetType())) : + null, + }, OtelContext.Default.ToolEvent)); } } + else + { + Log(new(1, message.Role == ChatRole.System ? OpenTelemetryConsts.GenAI.System.Message : OpenTelemetryConsts.GenAI.User.Message), + JsonSerializer.Serialize(new() + { + Role = message.Role != ChatRole.System && message.Role != ChatRole.User && !string.IsNullOrWhiteSpace(message.Role.Value) ? message.Role.Value : null, + Content = GetMessageContent(message), + }, OtelContext.Default.SystemOrUserEvent)); + } + } + } - messages.Add(new ChatMessage(role ?? ChatRole.Assistant, items)); + private void LogChatCompletion(ChatCompletion completion) + { + if (!_logger.IsEnabled(EventLogLevel)) + { + return; } - if (toolCalls is not null && messages.FirstOrDefault()?.Contents is { } c) + EventId id = new(1, OpenTelemetryConsts.GenAI.Choice); + int choiceCount = completion.Choices.Count; + for (int choiceIndex = 0; choiceIndex < choiceCount; choiceIndex++) { - foreach (var functionCall in toolCalls) + Log(id, JsonSerializer.Serialize(new() { - c.Add(functionCall); - } + FinishReason = completion.FinishReason?.Value ?? "error", + Index = choiceIndex, + Message = CreateAssistantEvent(completion.Choices[choiceIndex]), + }, OtelContext.Default.ChoiceEvent)); } + } - ChatCompletion completion = new(messages) - { - CompletionId = id, - FinishReason = finishReason, - ModelId = modelId, - Usage = usage, - }; + private void Log(EventId id, [StringSyntax(StringSyntaxAttribute.Json)] string eventBodyJson) + { + // This is not the idiomatic way to log, but it's necessary for now in order to structure + // the data in a way that the OpenTelemetry collector can work with it. The event body + // can be very large and should not be logged as an attribute. - SetCompletionResponse(activity, requestModelId, completion, error: null, stopwatch); + KeyValuePair[] tags = + [ + new(OpenTelemetryConsts.Event.Name, id.Name), + new(OpenTelemetryConsts.GenAI.SystemName, _system), + ]; + + _logger.Log(EventLogLevel, id, tags, null, (_, __) => eventBodyJson); } - private void AddMetricTags(ref TagList tags, string? requestModelId, ChatCompletion? completions) + private AssistantEvent CreateAssistantEvent(ChatMessage message) { - tags.Add(OpenTelemetryConsts.GenAI.Operation.Name, "chat"); + var toolCalls = message.Contents.OfType().Select(fc => new ToolCall + { + Id = fc.CallId, + Function = new() + { + Name = fc.Name, + Arguments = EnableSensitiveData ? + JsonSerializer.SerializeToNode(fc.Arguments, _jsonSerializerOptions.GetTypeInfo(typeof(IDictionary))) : + null, + }, + }).ToArray(); + + return new() + { + Content = GetMessageContent(message), + ToolCalls = toolCalls.Length > 0 ? toolCalls : null, + }; + } - if (requestModelId is not null) + private string? GetMessageContent(ChatMessage message) + { + if (EnableSensitiveData) { - tags.Add(OpenTelemetryConsts.GenAI.Request.Model, requestModelId); + string content = string.Concat(message.Contents.OfType().Select(c => c.Text)); + if (content.Length > 0) + { + return content; + } } - tags.Add(OpenTelemetryConsts.GenAI.System, _modelProvider); + return null; + } - if (_endpointAddress is string endpointAddress) - { - tags.Add(OpenTelemetryConsts.Server.Address, endpointAddress); - tags.Add(OpenTelemetryConsts.Server.Port, _endpointPort); - } + private sealed class SystemOrUserEvent + { + public string? Role { get; set; } + public string? Content { get; set; } + } - if (completions?.ModelId is string responseModel) - { - tags.Add(OpenTelemetryConsts.GenAI.Response.Model, responseModel); - } + private sealed class AssistantEvent + { + public string? Content { get; set; } + public ToolCall[]? ToolCalls { get; set; } } + + private sealed class ToolEvent + { + public string? Id { get; set; } + public JsonNode? Content { get; set; } + } + + private sealed class ChoiceEvent + { + public string? FinishReason { get; set; } + public int Index { get; set; } + public AssistantEvent? Message { get; set; } + } + + private sealed class ToolCall + { + public string? Id { get; set; } + public string? Type { get; set; } = "function"; + public ToolCallFunction? Function { get; set; } + } + + private sealed class ToolCallFunction + { + public string? Name { get; set; } + public JsonNode? Arguments { get; set; } + } + + [JsonSourceGenerationOptions(PropertyNamingPolicy = JsonKnownNamingPolicy.SnakeCaseLower, DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull)] + [JsonSerializable(typeof(SystemOrUserEvent))] + [JsonSerializable(typeof(AssistantEvent))] + [JsonSerializable(typeof(ToolEvent))] + [JsonSerializable(typeof(ChoiceEvent))] + [JsonSerializable(typeof(object))] + private sealed partial class OtelContext : JsonSerializerContext; } diff --git a/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/OpenTelemetryChatClientBuilderExtensions.cs b/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/OpenTelemetryChatClientBuilderExtensions.cs index bf1ff4e9f0d..6e04e16f507 100644 --- a/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/OpenTelemetryChatClientBuilderExtensions.cs +++ b/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/OpenTelemetryChatClientBuilderExtensions.cs @@ -2,6 +2,8 @@ // The .NET Foundation licenses this file to you under the MIT license. using System; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; using Microsoft.Shared.Diagnostics; namespace Microsoft.Extensions.AI; @@ -17,15 +19,22 @@ public static class OpenTelemetryChatClientBuilderExtensions /// The specification is still experimental and subject to change; as such, the telemetry output by this client is also subject to change. /// /// The . + /// An optional to use to create a logger for logging events. /// An optional source name that will be used on the telemetry data. /// An optional callback that can be used to configure the instance. /// The . public static ChatClientBuilder UseOpenTelemetry( - this ChatClientBuilder builder, string? sourceName = null, Action? configure = null) => - Throw.IfNull(builder).Use(innerClient => + this ChatClientBuilder builder, + ILoggerFactory? loggerFactory = null, + string? sourceName = null, + Action? configure = null) => + Throw.IfNull(builder).Use((services, innerClient) => { - var chatClient = new OpenTelemetryChatClient(innerClient, sourceName); + loggerFactory ??= services.GetService(); + + var chatClient = new OpenTelemetryChatClient(innerClient, loggerFactory?.CreateLogger(typeof(OpenTelemetryChatClient)), sourceName); configure?.Invoke(chatClient); + return chatClient; }); } diff --git a/src/Libraries/Microsoft.Extensions.AI/Embeddings/OpenTelemetryEmbeddingGenerator.cs b/src/Libraries/Microsoft.Extensions.AI/Embeddings/OpenTelemetryEmbeddingGenerator.cs index 8105cc64bdf..c085aaef350 100644 --- a/src/Libraries/Microsoft.Extensions.AI/Embeddings/OpenTelemetryEmbeddingGenerator.cs +++ b/src/Libraries/Microsoft.Extensions.AI/Embeddings/OpenTelemetryEmbeddingGenerator.cs @@ -8,6 +8,7 @@ using System.Linq; using System.Threading; using System.Threading.Tasks; +using Microsoft.Extensions.Logging; using Microsoft.Shared.Diagnostics; namespace Microsoft.Extensions.AI; @@ -38,8 +39,11 @@ public sealed class OpenTelemetryEmbeddingGenerator : Delega /// Initializes a new instance of the class. /// /// The underlying , which is the next stage of the pipeline. + /// The to use for emitting events. /// An optional source name that will be used on the telemetry data. - public OpenTelemetryEmbeddingGenerator(IEmbeddingGenerator innerGenerator, string? sourceName = null) +#pragma warning disable IDE0060 // Remove unused parameter; it exists for future use and consistency with OpenTelemetryChatClient + public OpenTelemetryEmbeddingGenerator(IEmbeddingGenerator innerGenerator, ILogger? logger = null, string? sourceName = null) +#pragma warning restore IDE0060 : base(innerGenerator) { Debug.Assert(innerGenerator is not null, "Should have been validated by the base ctor."); @@ -68,27 +72,12 @@ public OpenTelemetryEmbeddingGenerator(IEmbeddingGenerator i advice: new() { HistogramBucketBoundaries = OpenTelemetryConsts.GenAI.Client.OperationDuration.ExplicitBucketBoundaries }); } - /// - protected override void Dispose(bool disposing) - { - if (disposing) - { - _activitySource.Dispose(); - _meter.Dispose(); - } - - base.Dispose(disposing); - } - - /// Gets a value indicating whether diagnostics are enabled. - private bool Enabled => _activitySource.HasListeners(); - /// public override async Task> GenerateAsync(IEnumerable values, EmbeddingGenerationOptions? options = null, CancellationToken cancellationToken = default) { _ = Throw.IfNull(values); - using Activity? activity = StartActivity(); + using Activity? activity = CreateAndConfigureActivity(); Stopwatch? stopwatch = _operationDurationHistogram.Enabled ? Stopwatch.StartNew() : null; GeneratedEmbeddings? response = null; @@ -104,26 +93,38 @@ public override async Task> GenerateAsync(IEnume } finally { - SetCompletionResponse(activity, response, error, stopwatch); + TraceCompletion(activity, response, error, stopwatch); } return response; } + /// + protected override void Dispose(bool disposing) + { + if (disposing) + { + _activitySource.Dispose(); + _meter.Dispose(); + } + + base.Dispose(disposing); + } + /// Creates an activity for an embedding generation request, or returns null if not enabled. - private Activity? StartActivity() + private Activity? CreateAndConfigureActivity() { Activity? activity = null; - if (Enabled) + if (_activitySource.HasListeners()) { activity = _activitySource.StartActivity( - $"embedding {_modelId}", + $"{OpenTelemetryConsts.GenAI.Embed} {_modelId}", ActivityKind.Client, default(ActivityContext), [ - new(OpenTelemetryConsts.GenAI.Operation.Name, "embedding"), + new(OpenTelemetryConsts.GenAI.Operation.Name, OpenTelemetryConsts.GenAI.Embed), new(OpenTelemetryConsts.GenAI.Request.Model, _modelId), - new(OpenTelemetryConsts.GenAI.System, _modelProvider), + new(OpenTelemetryConsts.GenAI.SystemName, _modelProvider), ]); if (activity is not null) @@ -131,13 +132,13 @@ public override async Task> GenerateAsync(IEnume if (_endpointAddress is not null) { _ = activity - .SetTag(OpenTelemetryConsts.Server.Address, _endpointAddress) - .SetTag(OpenTelemetryConsts.Server.Port, _endpointPort); + .AddTag(OpenTelemetryConsts.Server.Address, _endpointAddress) + .AddTag(OpenTelemetryConsts.Server.Port, _endpointPort); } if (_dimensions is int dimensions) { - _ = activity.SetTag(OpenTelemetryConsts.GenAI.Request.EmbeddingDimensions, dimensions); + _ = activity.AddTag(OpenTelemetryConsts.GenAI.Request.EmbeddingDimensions, dimensions); } } } @@ -146,17 +147,12 @@ public override async Task> GenerateAsync(IEnume } /// Adds embedding generation response information to the activity. - private void SetCompletionResponse( + private void TraceCompletion( Activity? activity, GeneratedEmbeddings? embeddings, Exception? error, Stopwatch? stopwatch) { - if (!Enabled) - { - return; - } - int? inputTokens = null; string? responseModelId = null; if (embeddings is not null) @@ -189,40 +185,37 @@ private void SetCompletionResponse( _tokenUsageHistogram.Record(inputTokens.Value); } - if (activity is null) + if (activity is not null) { - return; - } - - if (error is not null) - { - _ = activity - .SetTag(OpenTelemetryConsts.Error.Type, error.GetType().FullName) - .SetStatus(ActivityStatusCode.Error, error.Message); - return; - } + if (error is not null) + { + _ = activity + .AddTag(OpenTelemetryConsts.Error.Type, error.GetType().FullName) + .SetStatus(ActivityStatusCode.Error, error.Message); + } - if (inputTokens.HasValue) - { - _ = activity.SetTag(OpenTelemetryConsts.GenAI.Response.InputTokens, inputTokens); - } + if (inputTokens.HasValue) + { + _ = activity.AddTag(OpenTelemetryConsts.GenAI.Response.InputTokens, inputTokens); + } - if (responseModelId is not null) - { - _ = activity.SetTag(OpenTelemetryConsts.GenAI.Response.Model, responseModelId); + if (responseModelId is not null) + { + _ = activity.AddTag(OpenTelemetryConsts.GenAI.Response.Model, responseModelId); + } } } private void AddMetricTags(ref TagList tags, string? responseModelId) { - tags.Add(OpenTelemetryConsts.GenAI.Operation.Name, "embedding"); + tags.Add(OpenTelemetryConsts.GenAI.Operation.Name, OpenTelemetryConsts.GenAI.Embed); if (_modelId is string requestModel) { tags.Add(OpenTelemetryConsts.GenAI.Request.Model, requestModel); } - tags.Add(OpenTelemetryConsts.GenAI.System, _modelProvider); + tags.Add(OpenTelemetryConsts.GenAI.SystemName, _modelProvider); if (_endpointAddress is string endpointAddress) { diff --git a/src/Libraries/Microsoft.Extensions.AI/Embeddings/OpenTelemetryEmbeddingGeneratorBuilderExtensions.cs b/src/Libraries/Microsoft.Extensions.AI/Embeddings/OpenTelemetryEmbeddingGeneratorBuilderExtensions.cs index ba60847ef93..bffb9087abf 100644 --- a/src/Libraries/Microsoft.Extensions.AI/Embeddings/OpenTelemetryEmbeddingGeneratorBuilderExtensions.cs +++ b/src/Libraries/Microsoft.Extensions.AI/Embeddings/OpenTelemetryEmbeddingGeneratorBuilderExtensions.cs @@ -2,6 +2,8 @@ // The .NET Foundation licenses this file to you under the MIT license. using System; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; using Microsoft.Shared.Diagnostics; namespace Microsoft.Extensions.AI; @@ -19,15 +21,24 @@ public static class OpenTelemetryEmbeddingGeneratorBuilderExtensions /// The type of input used to produce embeddings. /// The type of embedding generated. /// The . + /// An optional to use to create a logger for logging events. /// An optional source name that will be used on the telemetry data. /// An optional callback that can be used to configure the instance. /// The . public static EmbeddingGeneratorBuilder UseOpenTelemetry( - this EmbeddingGeneratorBuilder builder, string? sourceName = null, Action>? configure = null) + this EmbeddingGeneratorBuilder builder, + ILoggerFactory? loggerFactory = null, + string? sourceName = null, + Action>? configure = null) where TEmbedding : Embedding => - Throw.IfNull(builder).Use(innerGenerator => + Throw.IfNull(builder).Use((services, innerGenerator) => { - var generator = new OpenTelemetryEmbeddingGenerator(innerGenerator, sourceName); + loggerFactory ??= services.GetService(); + + var generator = new OpenTelemetryEmbeddingGenerator( + innerGenerator, + loggerFactory?.CreateLogger(typeof(OpenTelemetryEmbeddingGenerator)), + sourceName); configure?.Invoke(generator); return generator; }); diff --git a/src/Libraries/Microsoft.Extensions.AI/Microsoft.Extensions.AI.csproj b/src/Libraries/Microsoft.Extensions.AI/Microsoft.Extensions.AI.csproj index f360e7d6c43..2d695c88fcb 100644 --- a/src/Libraries/Microsoft.Extensions.AI/Microsoft.Extensions.AI.csproj +++ b/src/Libraries/Microsoft.Extensions.AI/Microsoft.Extensions.AI.csproj @@ -20,6 +20,7 @@ true + true false diff --git a/src/Libraries/Microsoft.Extensions.AI/OpenTelemetryConsts.cs b/src/Libraries/Microsoft.Extensions.AI/OpenTelemetryConsts.cs index 31e61101a13..27a543705ba 100644 --- a/src/Libraries/Microsoft.Extensions.AI/OpenTelemetryConsts.cs +++ b/src/Libraries/Microsoft.Extensions.AI/OpenTelemetryConsts.cs @@ -3,7 +3,6 @@ namespace Microsoft.Extensions.AI; -#pragma warning disable S3218 // Inner class members should not shadow outer class "static" or type members #pragma warning disable CA1716 // Identifiers should not match keywords #pragma warning disable S4041 // Type names should not match namespaces @@ -15,6 +14,11 @@ internal static class OpenTelemetryConsts public const string SecondsUnit = "s"; public const string TokensUnit = "token"; + public static class Event + { + public const string Name = "event.name"; + } + public static class Error { public const string Type = "error.type"; @@ -22,9 +26,16 @@ public static class Error public static class GenAI { - public const string Completion = "gen_ai.completion"; - public const string Prompt = "gen_ai.prompt"; - public const string System = "gen_ai.system"; + public const string Choice = "gen_ai.choice"; + public const string SystemName = "gen_ai.system"; + + public const string Chat = "chat"; + public const string Embed = "embed"; + + public static class Assistant + { + public const string Message = "gen_ai.assistant.message"; + } public static class Client { @@ -43,12 +54,6 @@ public static class TokenUsage } } - public static class Content - { - public const string Completion = "gen_ai.content.completion"; - public const string Prompt = "gen_ai.content.prompt"; - } - public static class Operation { public const string Name = "gen_ai.operation.name"; @@ -65,6 +70,8 @@ public static class Request public const string Temperature = "gen_ai.request.temperature"; public const string TopK = "gen_ai.request.top_k"; public const string TopP = "gen_ai.request.top_p"; + + public static string PerProvider(string providerName, string parameterName) => $"gen_ai.{providerName}.request.{parameterName}"; } public static class Response @@ -76,10 +83,25 @@ public static class Response public const string OutputTokens = "gen_ai.response.output_tokens"; } + public static class System + { + public const string Message = "gen_ai.system.message"; + } + public static class Token { public const string Type = "gen_ai.token.type"; } + + public static class Tool + { + public const string Message = "gen_ai.tool.message"; + } + + public static class User + { + public const string Message = "gen_ai.user.message"; + } } public static class Server diff --git a/test/Libraries/Microsoft.Extensions.AI.AzureAIInference.Tests/AzureAIInferenceChatClientTests.cs b/test/Libraries/Microsoft.Extensions.AI.AzureAIInference.Tests/AzureAIInferenceChatClientTests.cs index fd4bd11a96f..be628c13d0d 100644 --- a/test/Libraries/Microsoft.Extensions.AI.AzureAIInference.Tests/AzureAIInferenceChatClientTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.AzureAIInference.Tests/AzureAIInferenceChatClientTests.cs @@ -47,7 +47,7 @@ public void AsChatClient_ProducesExpectedMetadata() ChatCompletionsClient client = new(endpoint, new AzureKeyCredential("key")); IChatClient chatClient = client.AsChatClient(model); - Assert.Equal("AzureAIInference", chatClient.Metadata.ProviderName); + Assert.Equal("az.ai.inference", chatClient.Metadata.ProviderName); Assert.Equal(endpoint, chatClient.Metadata.ProviderUri); Assert.Equal(model, chatClient.Metadata.ModelId); } diff --git a/test/Libraries/Microsoft.Extensions.AI.Integration.Tests/ChatClientIntegrationTests.cs b/test/Libraries/Microsoft.Extensions.AI.Integration.Tests/ChatClientIntegrationTests.cs index 09784e86d16..634e4a19f9e 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Integration.Tests/ChatClientIntegrationTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Integration.Tests/ChatClientIntegrationTests.cs @@ -545,13 +545,13 @@ public virtual async Task OpenTelemetry_CanEmitTracesAndMetrics() .Build(); var chatClient = new ChatClientBuilder() - .UseOpenTelemetry(sourceName, instance => { instance.EnableSensitiveData = true; }) + .UseOpenTelemetry(sourceName: sourceName) .Use(CreateChatClient()!); var response = await chatClient.CompleteAsync([new(ChatRole.User, "What's the biggest animal?")]); var activity = Assert.Single(activities); - Assert.StartsWith("chat.completions", activity.DisplayName); + Assert.StartsWith("chat", activity.DisplayName); Assert.StartsWith("http", (string)activity.GetTagItem("server.address")!); Assert.Equal(chatClient.Metadata.ProviderUri?.Port, (int)activity.GetTagItem("server.port")!); Assert.NotNull(activity.Id); @@ -559,18 +559,6 @@ public virtual async Task OpenTelemetry_CanEmitTracesAndMetrics() Assert.NotEqual(0, (int)activity.GetTagItem("gen_ai.response.input_tokens")!); Assert.NotEqual(0, (int)activity.GetTagItem("gen_ai.response.output_tokens")!); - Assert.Collection(activity.Events, - evt => - { - Assert.Equal("gen_ai.content.prompt", evt.Name); - Assert.Equal("""[{"role": "user", "content": "What\u0027s the biggest animal?"}]""", evt.Tags.FirstOrDefault(t => t.Key == "gen_ai.prompt").Value); - }, - evt => - { - Assert.Equal("gen_ai.content.completion", evt.Name); - Assert.Contains("whale", (string)evt.Tags.FirstOrDefault(t => t.Key == "gen_ai.completion").Value!); - }); - Assert.True(activity.Duration.TotalMilliseconds > 0); } diff --git a/test/Libraries/Microsoft.Extensions.AI.Integration.Tests/EmbeddingGeneratorIntegrationTests.cs b/test/Libraries/Microsoft.Extensions.AI.Integration.Tests/EmbeddingGeneratorIntegrationTests.cs index 252427836e8..29502f926c6 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Integration.Tests/EmbeddingGeneratorIntegrationTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Integration.Tests/EmbeddingGeneratorIntegrationTests.cs @@ -111,7 +111,7 @@ public virtual async Task OpenTelemetry_CanEmitTracesAndMetrics() .Build(); var embeddingGenerator = new EmbeddingGeneratorBuilder>() - .UseOpenTelemetry(sourceName) + .UseOpenTelemetry(sourceName: sourceName) .Use(CreateEmbeddingGenerator()!); _ = await embeddingGenerator.GenerateAsync("Hello, world!"); diff --git a/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIChatClientTests.cs b/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIChatClientTests.cs index 947deb2674d..adc245c58e8 100644 --- a/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIChatClientTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIChatClientTests.cs @@ -45,13 +45,17 @@ public void AsChatClient_InvalidArgs_Throws() Assert.Throws("modelId", () => client.AsChatClient(" ")); } - [Fact] - public void AsChatClient_OpenAIClient_ProducesExpectedMetadata() + [Theory] + [InlineData(false)] + [InlineData(true)] + public void AsChatClient_OpenAIClient_ProducesExpectedMetadata(bool useAzureOpenAI) { Uri endpoint = new("http://localhost/some/endpoint"); string model = "amazingModel"; - OpenAIClient client = new(new ApiKeyCredential("key"), new OpenAIClientOptions { Endpoint = endpoint }); + var client = useAzureOpenAI ? + new AzureOpenAIClient(endpoint, new ApiKeyCredential("key")) : + new OpenAIClient(new ApiKeyCredential("key"), new OpenAIClientOptions { Endpoint = endpoint }); IChatClient chatClient = client.AsChatClient(model); Assert.Equal("openai", chatClient.Metadata.ProviderName); @@ -64,25 +68,6 @@ public void AsChatClient_OpenAIClient_ProducesExpectedMetadata() Assert.Equal(model, chatClient.Metadata.ModelId); } - [Fact] - public void AsChatClient_AzureOpenAIClient_ProducesExpectedMetadata() - { - Uri endpoint = new("http://localhost/some/endpoint"); - string model = "amazingModel"; - - AzureOpenAIClient client = new(endpoint, new ApiKeyCredential("key")); - - IChatClient chatClient = client.AsChatClient(model); - Assert.Equal("azureopenai", chatClient.Metadata.ProviderName); - Assert.Equal(endpoint, chatClient.Metadata.ProviderUri); - Assert.Equal(model, chatClient.Metadata.ModelId); - - chatClient = client.GetChatClient(model).AsChatClient(); - Assert.Equal("azureopenai", chatClient.Metadata.ProviderName); - Assert.Equal(endpoint, chatClient.Metadata.ProviderUri); - Assert.Equal(model, chatClient.Metadata.ModelId); - } - [Fact] public void GetService_OpenAIClient_SuccessfullyReturnsUnderlyingClient() { diff --git a/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIEmbeddingGeneratorTests.cs b/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIEmbeddingGeneratorTests.cs index d08cf295a4b..50b64fc9196 100644 --- a/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIEmbeddingGeneratorTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIEmbeddingGeneratorTests.cs @@ -42,13 +42,17 @@ public void AsEmbeddingGenerator_InvalidArgs_Throws() Assert.Throws("modelId", () => client.AsEmbeddingGenerator(" ")); } - [Fact] - public void AsEmbeddingGenerator_OpenAIClient_ProducesExpectedMetadata() + [Theory] + [InlineData(false)] + [InlineData(true)] + public void AsEmbeddingGenerator_OpenAIClient_ProducesExpectedMetadata(bool useAzureOpenAI) { Uri endpoint = new("http://localhost/some/endpoint"); string model = "amazingModel"; - OpenAIClient client = new(new ApiKeyCredential("key"), new OpenAIClientOptions { Endpoint = endpoint }); + var client = useAzureOpenAI ? + new AzureOpenAIClient(endpoint, new ApiKeyCredential("key")) : + new OpenAIClient(new ApiKeyCredential("key"), new OpenAIClientOptions { Endpoint = endpoint }); IEmbeddingGenerator> embeddingGenerator = client.AsEmbeddingGenerator(model); Assert.Equal("openai", embeddingGenerator.Metadata.ProviderName); @@ -61,25 +65,6 @@ public void AsEmbeddingGenerator_OpenAIClient_ProducesExpectedMetadata() Assert.Equal(model, embeddingGenerator.Metadata.ModelId); } - [Fact] - public void AsEmbeddingGenerator_AzureOpenAIClient_ProducesExpectedMetadata() - { - Uri endpoint = new("http://localhost/some/endpoint"); - string model = "amazingModel"; - - AzureOpenAIClient client = new(endpoint, new ApiKeyCredential("key")); - - IEmbeddingGenerator> embeddingGenerator = client.AsEmbeddingGenerator(model); - Assert.Equal("azureopenai", embeddingGenerator.Metadata.ProviderName); - Assert.Equal(endpoint, embeddingGenerator.Metadata.ProviderUri); - Assert.Equal(model, embeddingGenerator.Metadata.ModelId); - - embeddingGenerator = client.GetEmbeddingClient(model).AsEmbeddingGenerator(); - Assert.Equal("azureopenai", embeddingGenerator.Metadata.ProviderName); - Assert.Equal(endpoint, embeddingGenerator.Metadata.ProviderUri); - Assert.Equal(model, embeddingGenerator.Metadata.ModelId); - } - [Fact] public void GetService_OpenAIClient_SuccessfullyReturnsUnderlyingClient() { diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/ChatCompletion/OpenTelemetryChatClientTests.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/ChatCompletion/OpenTelemetryChatClientTests.cs index d0056b21b91..2ad428fad76 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Tests/ChatCompletion/OpenTelemetryChatClientTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/ChatCompletion/OpenTelemetryChatClientTests.cs @@ -4,10 +4,11 @@ using System; using System.Collections.Generic; using System.Diagnostics; -using System.Linq; using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Testing; using OpenTelemetry.Trace; using Xunit; @@ -15,8 +16,12 @@ namespace Microsoft.Extensions.AI; public class OpenTelemetryChatClientTests { - [Fact] - public async Task ExpectedInformationLogged_NonStreaming_Async() + [Theory] + [InlineData(false, false)] + [InlineData(false, true)] + [InlineData(true, false)] + [InlineData(true, true)] + public async Task ExpectedInformationLogged_Async(bool enableSensitiveData, bool streaming) { var sourceName = Guid.NewGuid().ToString(); var activities = new List(); @@ -25,13 +30,16 @@ public async Task ExpectedInformationLogged_NonStreaming_Async() .AddInMemoryExporter(activities) .Build(); + var collector = new FakeLogCollector(); + using ILoggerFactory loggerFactory = LoggerFactory.Create(b => b.AddProvider(new FakeLoggerProvider(collector))); + using var innerClient = new TestChatClient { Metadata = new("testservice", new Uri("http://localhost:12345/something"), "amazingmodel"), CompleteAsyncCallback = async (messages, options, cancellationToken) => { await Task.Yield(); - return new ChatCompletion([new ChatMessage(ChatRole.Assistant, "blue whale")]) + return new ChatCompletion([new ChatMessage(ChatRole.Assistant, "The blue whale, I think.")]) { CompletionId = "id123", FinishReason = ChatFinishReason.Stop, @@ -42,99 +50,31 @@ public async Task ExpectedInformationLogged_NonStreaming_Async() TotalTokenCount = 42, }, }; - } - }; - - var chatClient = new ChatClientBuilder() - .UseOpenTelemetry(sourceName, instance => - { - instance.EnableSensitiveData = true; - instance.JsonSerializerOptions = TestJsonSerializerContext.Default.Options; - }) - .Use(innerClient); - - await chatClient.CompleteAsync( - [new(ChatRole.User, "What's the biggest animal?")], - new ChatOptions - { - FrequencyPenalty = 3.0f, - MaxOutputTokens = 123, - ModelId = "replacementmodel", - TopP = 4.0f, - PresencePenalty = 5.0f, - ResponseFormat = ChatResponseFormat.Json, - Temperature = 6.0f, - StopSequences = ["hello", "world"], - AdditionalProperties = new() { ["top_k"] = 7.0f }, - }); - - var activity = Assert.Single(activities); - - Assert.NotNull(activity.Id); - Assert.NotEmpty(activity.Id); - - Assert.Equal("http://localhost:12345/something", activity.GetTagItem("server.address")); - Assert.Equal(12345, (int)activity.GetTagItem("server.port")!); - - Assert.Equal("chat.completions replacementmodel", activity.DisplayName); - Assert.Equal("testservice", activity.GetTagItem("gen_ai.system")); - - Assert.Equal("replacementmodel", activity.GetTagItem("gen_ai.request.model")); - Assert.Equal(3.0f, activity.GetTagItem("gen_ai.request.frequency_penalty")); - Assert.Equal(4.0f, activity.GetTagItem("gen_ai.request.top_p")); - Assert.Equal(5.0f, activity.GetTagItem("gen_ai.request.presence_penalty")); - Assert.Equal(6.0f, activity.GetTagItem("gen_ai.request.temperature")); - Assert.Equal(7.0, activity.GetTagItem("gen_ai.request.top_k")); - Assert.Equal(123, activity.GetTagItem("gen_ai.request.max_tokens")); - Assert.Equal("""["hello", "world"]""", activity.GetTagItem("gen_ai.request.stop_sequences")); - - Assert.Equal("id123", activity.GetTagItem("gen_ai.response.id")); - Assert.Equal("""["stop"]""", activity.GetTagItem("gen_ai.response.finish_reasons")); - Assert.Equal(10, activity.GetTagItem("gen_ai.response.input_tokens")); - Assert.Equal(20, activity.GetTagItem("gen_ai.response.output_tokens")); - - Assert.Collection(activity.Events, - evt => - { - Assert.Equal("gen_ai.content.prompt", evt.Name); - Assert.Equal("""[{"role": "user", "content": "What\u0027s the biggest animal?"}]""", evt.Tags.FirstOrDefault(t => t.Key == "gen_ai.prompt").Value); }, - evt => - { - Assert.Equal("gen_ai.content.completion", evt.Name); - Assert.Contains("whale", (string)evt.Tags.FirstOrDefault(t => t.Key == "gen_ai.completion").Value!); - }); - - Assert.True(activity.Duration.TotalMilliseconds > 0); - } - - [Fact] - public async Task ExpectedInformationLogged_Streaming_Async() - { - var sourceName = Guid.NewGuid().ToString(); - var activities = new List(); - using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder() - .AddSource(sourceName) - .AddInMemoryExporter(activities) - .Build(); + CompleteStreamingAsyncCallback = CallbackAsync, + }; async static IAsyncEnumerable CallbackAsync( IList messages, ChatOptions? options, [EnumeratorCancellation] CancellationToken cancellationToken) { await Task.Yield(); - yield return new StreamingChatCompletionUpdate + + foreach (string text in new[] { "The ", "blue ", "whale,", " ", "", "I", " think." }) { - Role = ChatRole.Assistant, - Text = "blue ", - CompletionId = "id123", - }; - await Task.Yield(); + await Task.Yield(); + yield return new StreamingChatCompletionUpdate + { + Role = ChatRole.Assistant, + Text = text, + CompletionId = "id123", + }; + } + yield return new StreamingChatCompletionUpdate { - Role = ChatRole.Assistant, - Text = "whale", FinishReason = ChatFinishReason.Stop, }; + yield return new StreamingChatCompletionUpdate { Contents = [new UsageContent(new() @@ -146,36 +86,47 @@ async static IAsyncEnumerable CallbackAsync( }; } - using var innerClient = new TestChatClient - { - Metadata = new("testservice", new Uri("http://localhost:12345/something"), "amazingmodel"), - CompleteStreamingAsyncCallback = CallbackAsync, - }; - var chatClient = new ChatClientBuilder() - .UseOpenTelemetry(sourceName, instance => + .UseOpenTelemetry(loggerFactory, sourceName, configure: instance => { - instance.EnableSensitiveData = true; + instance.EnableSensitiveData = enableSensitiveData; instance.JsonSerializerOptions = TestJsonSerializerContext.Default.Options; }) .Use(innerClient); - await foreach (var update in chatClient.CompleteStreamingAsync( - [new(ChatRole.User, "What's the biggest animal?")], - new ChatOptions + List chatMessages = + [ + new(ChatRole.System, "You are a close friend."), + new(ChatRole.User, "Hey!"), + new(ChatRole.Assistant, [new FunctionCallContent("12345", "GetPersonName")]), + new(ChatRole.Tool, [new FunctionResultContent("12345", "GetPersonName", "John")]), + new(ChatRole.Assistant, "Hey John, what's up?"), + new(ChatRole.User, "What's the biggest animal?") + ]; + + var options = new ChatOptions + { + FrequencyPenalty = 3.0f, + MaxOutputTokens = 123, + ModelId = "replacementmodel", + TopP = 4.0f, + TopK = 7, + PresencePenalty = 5.0f, + ResponseFormat = ChatResponseFormat.Json, + Temperature = 6.0f, + StopSequences = ["hello", "world"], + }; + + if (streaming) + { + await foreach (var update in chatClient.CompleteStreamingAsync(chatMessages, options)) { - FrequencyPenalty = 3.0f, - MaxOutputTokens = 123, - ModelId = "replacementmodel", - TopP = 4.0f, - PresencePenalty = 5.0f, - ResponseFormat = ChatResponseFormat.Json, - Temperature = 6.0f, - StopSequences = ["hello", "world"], - AdditionalProperties = new() { ["top_k"] = 7.0 }, - })) + await Task.Yield(); + } + } + else { - // Drain the stream. + await chatClient.CompleteAsync(chatMessages, options); } var activity = Assert.Single(activities); @@ -186,7 +137,7 @@ async static IAsyncEnumerable CallbackAsync( Assert.Equal("http://localhost:12345/something", activity.GetTagItem("server.address")); Assert.Equal(12345, (int)activity.GetTagItem("server.port")!); - Assert.Equal("chat.completions replacementmodel", activity.DisplayName); + Assert.Equal("chat replacementmodel", activity.DisplayName); Assert.Equal("testservice", activity.GetTagItem("gen_ai.system")); Assert.Equal("replacementmodel", activity.GetTagItem("gen_ai.request.model")); @@ -194,7 +145,7 @@ async static IAsyncEnumerable CallbackAsync( Assert.Equal(4.0f, activity.GetTagItem("gen_ai.request.top_p")); Assert.Equal(5.0f, activity.GetTagItem("gen_ai.request.presence_penalty")); Assert.Equal(6.0f, activity.GetTagItem("gen_ai.request.temperature")); - Assert.Equal(7.0, activity.GetTagItem("gen_ai.request.top_k")); + Assert.Equal(7, activity.GetTagItem("gen_ai.request.top_k")); Assert.Equal(123, activity.GetTagItem("gen_ai.request.max_tokens")); Assert.Equal("""["hello", "world"]""", activity.GetTagItem("gen_ai.request.stop_sequences")); @@ -203,18 +154,44 @@ async static IAsyncEnumerable CallbackAsync( Assert.Equal(10, activity.GetTagItem("gen_ai.response.input_tokens")); Assert.Equal(20, activity.GetTagItem("gen_ai.response.output_tokens")); - Assert.Collection(activity.Events, - evt => - { - Assert.Equal("gen_ai.content.prompt", evt.Name); - Assert.Equal("""[{"role": "user", "content": "What\u0027s the biggest animal?"}]""", evt.Tags.FirstOrDefault(t => t.Key == "gen_ai.prompt").Value); - }, - evt => - { - Assert.Equal("gen_ai.content.completion", evt.Name); - Assert.Contains("whale", (string)evt.Tags.FirstOrDefault(t => t.Key == "gen_ai.completion").Value!); - }); - Assert.True(activity.Duration.TotalMilliseconds > 0); + + var logs = collector.GetSnapshot(); + if (enableSensitiveData) + { + Assert.Collection(logs, + log => Assert.Equal("""{"content":"You are a close friend."}""", log.Message), + log => Assert.Equal("""{"content":"Hey!"}""", log.Message), + log => Assert.Equal("""{"tool_calls":[{"id":"12345","type":"function","function":{"name":"GetPersonName"}}]}""", log.Message), + log => Assert.Equal("""{"id":"12345","content":"John"}""", log.Message), + log => Assert.Equal("""{"content":"Hey John, what\u0027s up?"}""", log.Message), + log => Assert.Equal("""{"content":"What\u0027s the biggest animal?"}""", log.Message), + log => Assert.Equal("""{"finish_reason":"stop","index":0,"message":{"content":"The blue whale, I think."}}""", log.Message)); + } + else + { + Assert.Collection(logs, + log => Assert.Equal("""{}""", log.Message), + log => Assert.Equal("""{}""", log.Message), + log => Assert.Equal("""{"tool_calls":[{"id":"12345","type":"function","function":{"name":"GetPersonName"}}]}""", log.Message), + log => Assert.Equal("""{"id":"12345"}""", log.Message), + log => Assert.Equal("""{}""", log.Message), + log => Assert.Equal("""{}""", log.Message), + log => Assert.Equal("""{"finish_reason":"stop","index":0,"message":{}}""", log.Message)); + } + + Assert.Collection(logs, + log => Assert.Equal(new KeyValuePair("event.name", "gen_ai.system.message"), ((IList>)log.State!)[0]), + log => Assert.Equal(new KeyValuePair("event.name", "gen_ai.user.message"), ((IList>)log.State!)[0]), + log => Assert.Equal(new KeyValuePair("event.name", "gen_ai.assistant.message"), ((IList>)log.State!)[0]), + log => Assert.Equal(new KeyValuePair("event.name", "gen_ai.tool.message"), ((IList>)log.State!)[0]), + log => Assert.Equal(new KeyValuePair("event.name", "gen_ai.assistant.message"), ((IList>)log.State!)[0]), + log => Assert.Equal(new KeyValuePair("event.name", "gen_ai.user.message"), ((IList>)log.State!)[0]), + log => Assert.Equal(new KeyValuePair("event.name", "gen_ai.choice"), ((IList>)log.State!)[0])); + + Assert.All(logs, log => + { + Assert.Equal(new KeyValuePair("gen_ai.system", "testservice"), ((IList>)log.State!)[1]); + }); } } diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/Microsoft.Extensions.AI.Tests.csproj b/test/Libraries/Microsoft.Extensions.AI.Tests/Microsoft.Extensions.AI.Tests.csproj index b3d5e8048f5..8675bdcf2f4 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Tests/Microsoft.Extensions.AI.Tests.csproj +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/Microsoft.Extensions.AI.Tests.csproj @@ -26,6 +26,7 @@ +