diff --git a/.gitignore b/.gitignore index 2359c60d..1a305311 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,4 @@ _repo.*/ .openpublishing.buildcore.ps1 .DS_Store +/.vs diff --git a/.openpublishing.publish.config.json b/.openpublishing.publish.config.json index c37fa624..25e35886 100644 --- a/.openpublishing.publish.config.json +++ b/.openpublishing.publish.config.json @@ -17,10 +17,10 @@ } ], "notification_subscribers": [ - "mabolan@microsoft.com" + "sopand@microsoft.com" ], "sync_notification_subscribers": [ - "mabolan@microsoft.com" + "sopand@microsoft.com" ], "branches_to_filter": [], "git_repository_url_open_to_public_contributors": "https://github.com/MicrosoftDocs/semantic-kernel-docs", diff --git a/.openpublishing.redirection.json b/.openpublishing.redirection.json index 7922ddfe..10934792 100644 --- a/.openpublishing.redirection.json +++ b/.openpublishing.redirection.json @@ -517,7 +517,7 @@ }, { "source_path": "semantic-kernel/agents/index.md", - "redirect_url": "/semantic-kernel/concepts/agents.md", + "redirect_url": "/semantic-kernel/concepts/agents", "redirect_document_id": false }, { @@ -714,6 +714,86 @@ "source_path": "semantic-kernel/when-to-use-ai/schillace-laws.md", "redirect_url": "/semantic-kernel/overview/index", "redirect_document_id": false + }, + { + "source_path": "semantic-kernel/concepts/ai-services/chat-completion/function-calling.md", + "redirect_url": "/semantic-kernel/concepts/ai-services/chat-completion/function-calling/index", + "redirect_document_id": false + }, + { + "source_path": "semantic-kernel/enterprise-readiness/observability/index.md", + "redirect_url": "/semantic-kernel/concepts/enterprise-readiness/observability/index", + "redirect_document_id": false + }, + { + "source_path": "semantic-kernel/enterprise-readiness/observability/telemetry-advanced.md", + "redirect_url": "/semantic-kernel/concepts/enterprise-readiness/observability/telemetry-advanced", + "redirect_document_id": false + }, + { + "source_path": "semantic-kernel/enterprise-readiness/observability/telemetry-with-app-insights.md", + "redirect_url": "/semantic-kernel/concepts/enterprise-readiness/observability/telemetry-with-app-insights", + "redirect_document_id": false + }, + { + "source_path": "semantic-kernel/enterprise-readiness/observability/telemetry-with-aspire-dashboard.md", + "redirect_url": "/semantic-kernel/concepts/enterprise-readiness/observability/telemetry-with-aspire-dashboard", + "redirect_document_id": false + }, + { + "source_path": "semantic-kernel/enterprise-readiness/observability/telemetry-with-console.md", + "redirect_url": "/semantic-kernel/concepts/enterprise-readiness/observability/telemetry-with-console", + "redirect_document_id": false + }, + { + "source_path": "semantic-kernel/how-to/vector-store-data-ingestion.md", + "redirect_url": "/semantic-kernel/concepts/vector-store-connectors/how-to/vector-store-data-ingestion", + "redirect_document_id": false + }, + { + "source_path": "semantic-kernel/concepts/vector-store-connectors/vector-store-data-ingestion.md", + "redirect_url": "/semantic-kernel/concepts/vector-store-connectors/how-to/vector-store-data-ingestion", + "redirect_document_id": false + }, + { + "source_path": "semantic-kernel/integrations/index.md", + "redirect_url": "/semantic-kernel/concepts/ai-services/integrations", + "redirect_document_id": false + }, + { + "source_path": "semantic-kernel/how-to/vector-store-custom-mapper.md", + "redirect_url": "/semantic-kernel/concepts/vector-store-connectors/how-to/vector-store-custom-mapper", + "redirect_document_id": false + }, + { + "source_path": "semantic-kernel/concepts/vector-store-connectors/vector-store-custom-mapper.md", + "redirect_url": "/semantic-kernel/concepts/vector-store-connectors/how-to/vector-store-custom-mapper", + "redirect_document_id": false + }, + { + "source_path": "semantic-kernel/get-started/hackathon-materials.md", + "redirect_url": "/semantic-kernel/support/hackathon-materials", + "redirect_document_id": false + }, + { + "source_path": "semantic-kernel/get-started/contributing.md", + "redirect_url": "/semantic-kernel/support/contributing", + "redirect_document_id": false + }, + { + "source_path": "semantic-kernel/frameworks/agent/agent-framework.md", + "redirect_url": "semantic-kernel/frameworks/agent", + "redirect_document_id": false + }, + { + "source_path": "semantic-kernel/Frameworks/agent/agents.md", + "redirect_url": "semantic-kernel/frameworks/agent", + "redirect_document_id": false + }, + { + "source_path": "semantic-kernel/Frameworks/agent/personas.md", + "redirect_url": "semantic-kernel/frameworks/agent", + "redirect_document_id": false } ] -} \ No newline at end of file +} diff --git a/.vscode/settings.json b/.vscode/settings.json index c76aec85..fed833e3 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -2,5 +2,6 @@ "python.terminal.activateEnvironment": true, "cSpell.words": [ "SKEXP" - ] + ], + "FSharp.suggestGitignore": false } \ No newline at end of file diff --git a/semantic-kernel-pr.sln b/semantic-kernel-pr.sln deleted file mode 100644 index 32a89ebc..00000000 --- a/semantic-kernel-pr.sln +++ /dev/null @@ -1,137 +0,0 @@ - -Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio Version 17 -VisualStudioVersion = 17.5.002.0 -MinimumVisualStudioVersion = 10.0.40219.1 -Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "samples", "samples", "{D5020C1C-C5E1-44C3-9285-7687C5C21552}" -EndProject -Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "dotnet", "dotnet", "{50F4184D-4621-405B-AF6A-0F3E93F198A7}" -EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "05-Nested-Functions-In-Prompts", "samples\dotnet\05-Nested-Functions-In-Prompts\05-Nested-Functions-In-Prompts.csproj", "{D3F29788-75FE-410E-A858-9D26963E0DB4}" -EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "07-Serializing-Prompts", "samples\dotnet\07-Serializing-Prompts\07-Serializing-Prompts.csproj", "{7A4EF630-44F2-4D00-96E4-2B0C98152FB8}" -EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "01-Kernel-Intro", "samples\dotnet\01-Kernel-Intro\01-Kernel-Intro.csproj", "{27ABE625-D6D3-46B9-A45E-968AC7A4C3A1}" -EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "08-Native-Functions-with-Context", "samples\dotnet\08-Native-Functions-with-Context\08-Native-Functions-with-Context.csproj", "{5F1BD319-7244-4E4F-8C0C-43AD335F4D5F}" -EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "11-Planner", "samples\dotnet\11-Planner\11-Planner.csproj", "{1C37EADA-45BE-491F-B0BC-131CB3FE58CE}" -EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "02-Adding-AI-Services", "samples\dotnet\02-Adding-AI-Services\02-Adding-AI-Services.csproj", "{0A4AF252-B9BC-4D0C-9AD4-8DA11F022D56}" -EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "10-Chaining-Functions", "samples\dotnet\10-Chaining-Functions\10-Chaining-Functions.csproj", "{65AC95D1-65EC-46A2-AC90-4A45BD11B88B}" -EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "03-Intro-to-Prompts", "samples\dotnet\03-Intro-to-Prompts\03-Intro-to-Prompts.csproj", "{A866A455-2A72-4BB6-9572-0424B73DCD99}" -EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "15-Using-Context-for-Chat", "samples\dotnet\15-Using-Context-for-Chat\15-Using-Context-for-Chat.csproj", "{0EAABCC5-FD3E-42C6-8BF6-BE267393547C}" -EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "08-Creating-Functions-For-AI", "samples\dotnet\08-Creating-Functions-For-AI\08-Creating-Functions-For-AI.csproj", "{88F66583-45B2-4B10-8D02-35C9808C643F}" -EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "09-Calling-Nested-Functions-in-Native-Functions", "samples\dotnet\09-Calling-Nested-Functions-in-Native-Functions\09-Calling-Nested-Functions-in-Native-Functions.csproj", "{2A531F4E-244C-4D9A-BC8B-009B0F77840C}" -EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "04-Templatizing-Prompts", "samples\dotnet\04-Templatizing-Prompts\04-Templatizing-Prompts.csproj", "{CE17D957-04F6-4813-8CB1-33ECF2D0CAD7}" -EndProject -Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "14-Create-ChatGPT-Plugin", "14-Create-ChatGPT-Plugin", "{0460D6D8-48F3-4FC6-A2C5-FE498154DA43}" -EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "14-Create-ChatGPT-Plugin", "samples\dotnet\14-Create-ChatGPT-Plugin\Solution\14-Create-ChatGPT-Plugin.csproj", "{07AFA7EA-545A-490A-A4C0-AC6866DF9FDB}" -EndProject -Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "MathPlugin", "MathPlugin", "{6564BDFD-8199-4FD7-9514-D749C45D8340}" -EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "sk-chatgpt-azure-function", "samples\dotnet\14-Create-ChatGPT-Plugin\MathPlugin\azure-function\sk-chatgpt-azure-function.csproj", "{25A50962-4D5A-4465-BBD1-EA87C016F6D3}" -EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "semantic-functions-generator", "samples\dotnet\14-Create-ChatGPT-Plugin\MathPlugin\semantic-functions-generator\semantic-functions-generator.csproj", "{8268AB69-32F0-4B29-B585-7BCC89A5FCBD}" -EndProject -Global - GlobalSection(SolutionConfigurationPlatforms) = preSolution - Debug|Any CPU = Debug|Any CPU - Release|Any CPU = Release|Any CPU - EndGlobalSection - GlobalSection(ProjectConfigurationPlatforms) = postSolution - {D3F29788-75FE-410E-A858-9D26963E0DB4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {D3F29788-75FE-410E-A858-9D26963E0DB4}.Debug|Any CPU.Build.0 = Debug|Any CPU - {D3F29788-75FE-410E-A858-9D26963E0DB4}.Release|Any CPU.ActiveCfg = Release|Any CPU - {D3F29788-75FE-410E-A858-9D26963E0DB4}.Release|Any CPU.Build.0 = Release|Any CPU - {7A4EF630-44F2-4D00-96E4-2B0C98152FB8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {7A4EF630-44F2-4D00-96E4-2B0C98152FB8}.Debug|Any CPU.Build.0 = Debug|Any CPU - {7A4EF630-44F2-4D00-96E4-2B0C98152FB8}.Release|Any CPU.ActiveCfg = Release|Any CPU - {7A4EF630-44F2-4D00-96E4-2B0C98152FB8}.Release|Any CPU.Build.0 = Release|Any CPU - {27ABE625-D6D3-46B9-A45E-968AC7A4C3A1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {27ABE625-D6D3-46B9-A45E-968AC7A4C3A1}.Debug|Any CPU.Build.0 = Debug|Any CPU - {27ABE625-D6D3-46B9-A45E-968AC7A4C3A1}.Release|Any CPU.ActiveCfg = Release|Any CPU - {27ABE625-D6D3-46B9-A45E-968AC7A4C3A1}.Release|Any CPU.Build.0 = Release|Any CPU - {5F1BD319-7244-4E4F-8C0C-43AD335F4D5F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {5F1BD319-7244-4E4F-8C0C-43AD335F4D5F}.Debug|Any CPU.Build.0 = Debug|Any CPU - {5F1BD319-7244-4E4F-8C0C-43AD335F4D5F}.Release|Any CPU.ActiveCfg = Release|Any CPU - {5F1BD319-7244-4E4F-8C0C-43AD335F4D5F}.Release|Any CPU.Build.0 = Release|Any CPU - {1C37EADA-45BE-491F-B0BC-131CB3FE58CE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {1C37EADA-45BE-491F-B0BC-131CB3FE58CE}.Debug|Any CPU.Build.0 = Debug|Any CPU - {1C37EADA-45BE-491F-B0BC-131CB3FE58CE}.Release|Any CPU.ActiveCfg = Release|Any CPU - {1C37EADA-45BE-491F-B0BC-131CB3FE58CE}.Release|Any CPU.Build.0 = Release|Any CPU - {0A4AF252-B9BC-4D0C-9AD4-8DA11F022D56}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {0A4AF252-B9BC-4D0C-9AD4-8DA11F022D56}.Debug|Any CPU.Build.0 = Debug|Any CPU - {0A4AF252-B9BC-4D0C-9AD4-8DA11F022D56}.Release|Any CPU.ActiveCfg = Release|Any CPU - {0A4AF252-B9BC-4D0C-9AD4-8DA11F022D56}.Release|Any CPU.Build.0 = Release|Any CPU - {65AC95D1-65EC-46A2-AC90-4A45BD11B88B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {65AC95D1-65EC-46A2-AC90-4A45BD11B88B}.Debug|Any CPU.Build.0 = Debug|Any CPU - {65AC95D1-65EC-46A2-AC90-4A45BD11B88B}.Release|Any CPU.ActiveCfg = Release|Any CPU - {65AC95D1-65EC-46A2-AC90-4A45BD11B88B}.Release|Any CPU.Build.0 = Release|Any CPU - {A866A455-2A72-4BB6-9572-0424B73DCD99}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {A866A455-2A72-4BB6-9572-0424B73DCD99}.Debug|Any CPU.Build.0 = Debug|Any CPU - {A866A455-2A72-4BB6-9572-0424B73DCD99}.Release|Any CPU.ActiveCfg = Release|Any CPU - {A866A455-2A72-4BB6-9572-0424B73DCD99}.Release|Any CPU.Build.0 = Release|Any CPU - {0EAABCC5-FD3E-42C6-8BF6-BE267393547C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {0EAABCC5-FD3E-42C6-8BF6-BE267393547C}.Debug|Any CPU.Build.0 = Debug|Any CPU - {0EAABCC5-FD3E-42C6-8BF6-BE267393547C}.Release|Any CPU.ActiveCfg = Release|Any CPU - {0EAABCC5-FD3E-42C6-8BF6-BE267393547C}.Release|Any CPU.Build.0 = Release|Any CPU - {88F66583-45B2-4B10-8D02-35C9808C643F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {88F66583-45B2-4B10-8D02-35C9808C643F}.Debug|Any CPU.Build.0 = Debug|Any CPU - {88F66583-45B2-4B10-8D02-35C9808C643F}.Release|Any CPU.ActiveCfg = Release|Any CPU - {88F66583-45B2-4B10-8D02-35C9808C643F}.Release|Any CPU.Build.0 = Release|Any CPU - {2A531F4E-244C-4D9A-BC8B-009B0F77840C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {2A531F4E-244C-4D9A-BC8B-009B0F77840C}.Debug|Any CPU.Build.0 = Debug|Any CPU - {2A531F4E-244C-4D9A-BC8B-009B0F77840C}.Release|Any CPU.ActiveCfg = Release|Any CPU - {2A531F4E-244C-4D9A-BC8B-009B0F77840C}.Release|Any CPU.Build.0 = Release|Any CPU - {CE17D957-04F6-4813-8CB1-33ECF2D0CAD7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {CE17D957-04F6-4813-8CB1-33ECF2D0CAD7}.Debug|Any CPU.Build.0 = Debug|Any CPU - {CE17D957-04F6-4813-8CB1-33ECF2D0CAD7}.Release|Any CPU.ActiveCfg = Release|Any CPU - {CE17D957-04F6-4813-8CB1-33ECF2D0CAD7}.Release|Any CPU.Build.0 = Release|Any CPU - {07AFA7EA-545A-490A-A4C0-AC6866DF9FDB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {07AFA7EA-545A-490A-A4C0-AC6866DF9FDB}.Debug|Any CPU.Build.0 = Debug|Any CPU - {07AFA7EA-545A-490A-A4C0-AC6866DF9FDB}.Release|Any CPU.ActiveCfg = Release|Any CPU - {07AFA7EA-545A-490A-A4C0-AC6866DF9FDB}.Release|Any CPU.Build.0 = Release|Any CPU - {25A50962-4D5A-4465-BBD1-EA87C016F6D3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {25A50962-4D5A-4465-BBD1-EA87C016F6D3}.Debug|Any CPU.Build.0 = Debug|Any CPU - {25A50962-4D5A-4465-BBD1-EA87C016F6D3}.Release|Any CPU.ActiveCfg = Release|Any CPU - {25A50962-4D5A-4465-BBD1-EA87C016F6D3}.Release|Any CPU.Build.0 = Release|Any CPU - {8268AB69-32F0-4B29-B585-7BCC89A5FCBD}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {8268AB69-32F0-4B29-B585-7BCC89A5FCBD}.Debug|Any CPU.Build.0 = Debug|Any CPU - {8268AB69-32F0-4B29-B585-7BCC89A5FCBD}.Release|Any CPU.ActiveCfg = Release|Any CPU - {8268AB69-32F0-4B29-B585-7BCC89A5FCBD}.Release|Any CPU.Build.0 = Release|Any CPU - EndGlobalSection - GlobalSection(SolutionProperties) = preSolution - HideSolutionNode = FALSE - EndGlobalSection - GlobalSection(NestedProjects) = preSolution - {50F4184D-4621-405B-AF6A-0F3E93F198A7} = {D5020C1C-C5E1-44C3-9285-7687C5C21552} - {D3F29788-75FE-410E-A858-9D26963E0DB4} = {50F4184D-4621-405B-AF6A-0F3E93F198A7} - {7A4EF630-44F2-4D00-96E4-2B0C98152FB8} = {50F4184D-4621-405B-AF6A-0F3E93F198A7} - {27ABE625-D6D3-46B9-A45E-968AC7A4C3A1} = {50F4184D-4621-405B-AF6A-0F3E93F198A7} - {5F1BD319-7244-4E4F-8C0C-43AD335F4D5F} = {50F4184D-4621-405B-AF6A-0F3E93F198A7} - {1C37EADA-45BE-491F-B0BC-131CB3FE58CE} = {50F4184D-4621-405B-AF6A-0F3E93F198A7} - {0A4AF252-B9BC-4D0C-9AD4-8DA11F022D56} = {50F4184D-4621-405B-AF6A-0F3E93F198A7} - {65AC95D1-65EC-46A2-AC90-4A45BD11B88B} = {50F4184D-4621-405B-AF6A-0F3E93F198A7} - {A866A455-2A72-4BB6-9572-0424B73DCD99} = {50F4184D-4621-405B-AF6A-0F3E93F198A7} - {0EAABCC5-FD3E-42C6-8BF6-BE267393547C} = {50F4184D-4621-405B-AF6A-0F3E93F198A7} - {88F66583-45B2-4B10-8D02-35C9808C643F} = {50F4184D-4621-405B-AF6A-0F3E93F198A7} - {2A531F4E-244C-4D9A-BC8B-009B0F77840C} = {50F4184D-4621-405B-AF6A-0F3E93F198A7} - {CE17D957-04F6-4813-8CB1-33ECF2D0CAD7} = {50F4184D-4621-405B-AF6A-0F3E93F198A7} - {0460D6D8-48F3-4FC6-A2C5-FE498154DA43} = {50F4184D-4621-405B-AF6A-0F3E93F198A7} - {07AFA7EA-545A-490A-A4C0-AC6866DF9FDB} = {0460D6D8-48F3-4FC6-A2C5-FE498154DA43} - {6564BDFD-8199-4FD7-9514-D749C45D8340} = {0460D6D8-48F3-4FC6-A2C5-FE498154DA43} - {25A50962-4D5A-4465-BBD1-EA87C016F6D3} = {6564BDFD-8199-4FD7-9514-D749C45D8340} - {8268AB69-32F0-4B29-B585-7BCC89A5FCBD} = {6564BDFD-8199-4FD7-9514-D749C45D8340} - EndGlobalSection - GlobalSection(ExtensibilityGlobals) = postSolution - SolutionGuid = {DDC042ED-DBAA-4B63-BD21-275D8D27EA11} - EndGlobalSection -EndGlobal diff --git a/semantic-kernel/Frameworks/TOC.yml b/semantic-kernel/Frameworks/TOC.yml new file mode 100644 index 00000000..49505e8c --- /dev/null +++ b/semantic-kernel/Frameworks/TOC.yml @@ -0,0 +1,4 @@ +- name: Agent Framework + href: agent/TOC.yml +- name: Process Framework + href: process/TOC.yml diff --git a/semantic-kernel/Frameworks/agent/TOC.yml b/semantic-kernel/Frameworks/agent/TOC.yml new file mode 100644 index 00000000..8c88375f --- /dev/null +++ b/semantic-kernel/Frameworks/agent/TOC.yml @@ -0,0 +1,18 @@ +- name: Overview + href: index.md +- name: Architecture + href: agent-architecture.md +- name: Chat Completion Agent + href: chat-completion-agent.md +- name: Open AI Assistant Agent + href: assistant-agent.md +- name: Agent Collaboration + href: agent-chat.md +- name: Create an Agent from a Template + href: agent-templates.md +- name: Configuring Agents with Plugins + href: agent-functions.md +- name: Streaming Agent Responses + href: agent-streaming.md +- name: How-To with Agents + href: examples/TOC.yml diff --git a/semantic-kernel/Frameworks/agent/agent-architecture.md b/semantic-kernel/Frameworks/agent/agent-architecture.md new file mode 100644 index 00000000..73dba076 --- /dev/null +++ b/semantic-kernel/Frameworks/agent/agent-architecture.md @@ -0,0 +1,280 @@ +--- +title: Semantic Kernel Agent Architecture (Experimental) +description: An overview of the architecture of the Semantic Kernel Agent Framework and how it aligns with core Semantic Kernel features. +zone_pivot_groups: programming-languages +author: crickman +ms.topic: tutorial +ms.author: crickman +ms.date: 09/13/2024 +ms.service: semantic-kernel +--- +# An Overview of the Agent Architecture (Experimental) + +> [!WARNING] +> The _Semantic Kernel Agent Framework_ is experimental, still in development and is subject to change. + +This article covers key concepts in the architecture of the Agent Framework, including foundational principles, design objectives, and strategic goals. + + +## Goals + +The _Agent Framework_ was developed with the following key priorities in mind: + +- The _Semantic Kernel_ framework serves as the core foundation for implementing agent functionalities. +- Multiple agents can collaborate within a single conversation, while integrating human input. +- An agent can engage in and manage multiple concurrent conversations simultaneously. +- Different types of agents can participate in the same conversation, each contributing their unique capabilities. + + +## Agent + +The abstract _Agent_ class serves as the core abstraction for all types of agents, providing a foundational structure that can be extended to create more specialized agents. One key subclass is _Kernel Agent_, which establishes a direct association with a [_Kernel_](../../concepts/kernel.md) object. This relationship forms the basis for more specific agent implementations, such as the [_Chat Completion Agent_](./chat-completion-agent.md) and the [_Open AI Assistant Agent_](./assistant-agent.md), both of which leverage the Kernel's capabilities to execute their respective functions. + +::: zone pivot="programming-language-csharp" + +- [`Agent`](/dotnet/api/microsoft.semantickernel.agents.agent) +- [`KernelAgent`](/dotnet/api/microsoft.semantickernel.agents.kernelagent) + +::: zone-end + +::: zone pivot="programming-language-python" + +- [`agent`](/python/api/semantic-kernel/semantic_kernel.agents.agent) + +::: zone-end + +::: zone pivot="programming-language-java" + +> Agents are currently unavailable in Java. + +::: zone-end + +Agents can either be invoked directly to perform tasks or orchestrated within an [_Agent Chat_](./agent-chat.md), where multiple agents may collaborate or interact dynamically with user inputs. This flexible structure allows agents to adapt to various conversational or task-driven scenarios, providing developers with robust tools for building intelligent, multi-agent systems. + +#### Deep Dive: + +- [`ChatCompletionAgent`](./chat-completion-agent.md) +- [`OpenAIAssistantAgent`](./assistant-agent.md) + + + + + +## Agent Chat + +The [_Agent Chat_](./agent-chat.md) class serves as the foundational component that enables agents of any type to engage in a specific conversation. This class provides the essential capabilities for managing agent interactions within a chat environment. Building on this, the [_Agent Group Chat_](./agent-chat.md#creating-an-agent-group-chat) class extends these capabilities by offering a stategy-based container, which allows multiple agents to collaborate across numerous interactions within the same conversation. + +This structure facilitates more complex, multi-agent scenarios where different agents can work together, share information, and dynamically respond to evolving conversations, making it an ideal solution for advanced use cases such as customer support, multi-faceted task management, or collaborative problem-solving environments. + +#### Deep Dive: +- [`AgentChat`](./agent-chat.md) + + +## Agent Channel + +The _Agent Channel_ class enables agents of various types to participate in an [_Agent Chat_](./agent-chat.md). This functionality is completely hidden from users of the _Agent Framework_ and only needs to be considered by developers creating a custom [_Agent_](#agent). + +::: zone pivot="programming-language-csharp" + +- [`AgentChannel`](/dotnet/api/microsoft.semantickernel.agents.agentchannel) + +::: zone-end + +::: zone pivot="programming-language-python" + +- [`agent_channel](/python/api/semantic-kernel/semantic_kernel.agents.channels.agent_channel) + +::: zone-end + +::: zone pivot="programming-language-java" + +> Agents are currently unavailable in Java. + +::: zone-end + +## Agent Alignment with _Semantic Kernel_ Features + +The _Agent Framework_ is built on the foundational concepts and features that many developers have come to know within the _Semantic Kernel_ ecosystem. These core principles serve as the building blocks for the Agent Framework’s design. By leveraging the familiar structure and capabilities of the _Semantic Kernel_, the Agent Framework extends its functionality to enable more advanced, autonomous agent behaviors, while maintaining consistency with the broader _Semantic Kernel_ architecture. This ensures a smooth transition for developers, allowing them to apply their existing knowledge to create intelligent, adaptable agents within the framework. + + +### The _Kernel_ + +At the heart of the _Semantic Kernel_ ecosystem is the [_Kernel_](../../concepts/kernel.md), which serves as the core object that drives AI operations and interactions. To create any agent within this framework, a _Kernel instance_ is required as it provides the foundational context and capabilities for the agent’s functionality. The _Kernel_ acts as the engine for processing instructions, managing state, and invoking the necessary AI services that power the agent's behavior. + +The [_Chat Completion Agent_](./chat-completion-agent.md) and [_Open AI Assistant Agent_](./assistant-agent.md) articles provide specific details on how to create each type of agent. + These resources offer step-by-step instructions and highlight the key configurations needed to tailor the agents to different conversational or task-based applications, demonstrating how the Kernel enables dynamic and intelligent agent behaviors across diverse use cases. + +#### Related API's: + +::: zone pivot="programming-language-csharp" + +- [`IKernelBuilder`](/dotnet/api/microsoft.semantickernel) +- [`Kernel`](/dotnet/api/microsoft.semantickernel.kernel) +- [`KernelBuilderExtensions`](/dotnet/api/microsoft.semantickernel.kernelbuilderextensions) +- [`KernelExtensions`](/dotnet/api/microsoft.semantickernel.kernelextensions) + +::: zone-end + +::: zone pivot="programming-language-python" + +- [`kernel`](/python/api/semantic-kernel/semantic_kernel.kernel.kernel) + +::: zone-end + +::: zone pivot="programming-language-java" + +> Agents are currently unavailable in Java. + +::: zone-end + + +### [Plugins and Function Calling](./agent-functions.md) + +Plugins are a fundamental aspect of the _Semantic Kernel_, enabling developers to integrate custom functionalities and extend the capabilities of an AI application. These plugins offer a flexible way to incorporate specialized features or business-specific logic into the core AI workflows. Additionally, agent capabilities within the framework can be significantly enhanced by utilizing [Plugins](../../concepts/plugins/index.md) and leveraging [Function Calling](../../concepts/ai-services/chat-completion/function-calling/index.md). This allows agents to dynamically interact with external services or execute complex tasks, further expanding the scope and versatility of the AI system within diverse applications. + +#### Example: + +- [How-To: _Chat Completion Agent_](./examples/example-chat-agent.md) + +#### Related API's: + +::: zone pivot="programming-language-csharp" + +- [`KernelFunctionFactory`](/dotnet/api/microsoft.semantickernel.kernelfunctionfactory) +- [`KernelFunction`](/dotnet/api/microsoft.semantickernel.kernelfunction) +- [`KernelPluginFactory`](/dotnet/api/microsoft.semantickernel.kernelpluginfactory) +- [`KernelPlugin`](/dotnet/api/microsoft.semantickernel.kernelplugin) +- [`Kernel.Plugins`](/dotnet/api/microsoft.semantickernel.kernel.plugins) + +::: zone-end + +::: zone pivot="programming-language-python" + +- [`kernel_function`](/python/api/semantic-kernel/semantic_kernel.functions.kernel_function) +- [`kernel_function_extension`](/python/api/semantic-kernel/semantic_kernel.functions.kernel_function_extension) +- [`kernel_plugin`](/python/api/semantic-kernel/semantic_kernel.functions.kernel_plugin) + +::: zone-end + +::: zone pivot="programming-language-java" + +> Agents are currently unavailable in Java. + +::: zone-end + + +### [Agent Messages](../../concepts/ai-services/chat-completion/chat-history.md) + +Agent messaging, including both input and response, is built upon the core content types of the _Semantic Kernel_, providing a unified structure for communication. This design choice simplifies the process of transitioning from traditional chat-completion patterns to more advanced agent-driven patterns in your application development. By leveraging familiar _Semantic Kernel_ content types, developers can seamlessly integrate agent capabilities into their applications without needing to overhaul existing systems. This streamlining ensures that as you evolve from basic conversational AI to more autonomous, task-oriented agents, the underlying framework remains consistent, making development faster and more efficient. + +> Note: The [_Open AI Assistant Agent_`_](./assistant-agent.md) introduced content types specific to its usage for _File References_ and _Content Annotation_: + +#### Related API's: + +::: zone pivot="programming-language-csharp" + +- [`ChatHistory`](/dotnet/api/microsoft.semantickernel.chatcompletion.chathistory) +- [`ChatMessageContent`](/dotnet/api/microsoft.semantickernel.chatmessagecontent) +- [`KernelContent`](/dotnet/api/microsoft.semantickernel.kernelcontent) +- [`StreamingKernelContent`](/dotnet/api/microsoft.semantickernel.streamingkernelcontent) +- [`FileReferenceContent`](/dotnet/api/microsoft.semantickernel.filereferencecontent) +- [`AnnotationContent`](/dotnet/api/microsoft.semantickernel.agents.openai.annotationcontent) + + +::: zone-end + +::: zone pivot="programming-language-python" + +- [`chat_history`](/python/api/semantic-kernel/semantic_kernel.contents.chat_history) +- [`chat_message_content`](/python/api/semantic-kernel/semantic_kernel.contents.chat_message_content) +- [`kernel_content`](/python/api/semantic-kernel/semantic_kernel.contents.kernel_content) +- [`streaming_chat_message_content`](/python/api/semantic-kernel/semantic_kernel.contents.streaming_chat_message_content) +- [`file_reference_content`](/python/api/semantic-kernel/semantic_kernel.contents.file_reference_content) +- [`annotation_content`](/python/api/semantic-kernel/semantic_kernel.contents.annotation_content) + +::: zone-end + +::: zone pivot="programming-language-java" + +> Agents are currently unavailable in Java. + +::: zone-end + + +### [Templating](./agent-templates.md) + +An agent's role is primarily shaped by the instructions it receives, which dictate its behavior and actions. Similar to invoking a _Kernel_ [prompt](../../concepts/prompts/index.md), an agent's instructions can include templated parameters—both values and functions—that are dynamically substituted during execution. This enables flexible, context-aware responses, allowing the agent to adjust its output based on real-time input. + +Additionally, an agent can be configured directly using a _Prompt Template Configuration_, providing developers with a structured and reusable way to define its behavior. This approach offers a powerful tool for standardizing and customizing agent instructions, ensuring consistency across various use cases while still maintaining dynamic adaptability. + +#### Example: + +- [How-To: _Chat Completion Agent_](./examples/example-chat-agent.md) + +#### Related API's: + +::: zone pivot="programming-language-csharp" + +- [`PromptTemplateConfig`](/dotnet/api/microsoft.semantickernel.prompttemplateconfig) +- [`KernelFunctionYaml.FromPromptYaml`](/dotnet/api/microsoft.semantickernel.kernelfunctionyaml.frompromptyaml#microsoft-semantickernel-kernelfunctionyaml-frompromptyaml(system-string-microsoft-semantickernel-iprompttemplatefactory-microsoft-extensions-logging-iloggerfactory)) +- [`IPromptTemplateFactory`](/dotnet/api/microsoft.semantickernel.iprompttemplatefactory) +- [`KernelPromptTemplateFactory`](/dotnet/api/microsoft.semantickernel.kernelprompttemplatefactory) +- [_Handlebars_](/dotnet/api/microsoft.semantickernel.prompttemplates.handlebars) +- [_Prompty_](/dotnet/api/microsoft.semantickernel.prompty) +- [_Liquid_](/dotnet/api/microsoft.semantickernel.prompttemplates.liquid) + +::: zone-end + +::: zone pivot="programming-language-python" + +- [`prompt_template_config`](/python/api/semantic-kernel/semantic_kernel.prompt_template.prompt_template_config) +- [`kernel_prompt_template`](/python/api/semantic-kernel/semantic_kernel.prompt_template.kernel_prompt_template) +- [`jinja2_prompt_template`](/python/api/semantic-kernel/semantic_kernel.prompt_template.jinja2_prompt_template) +- [`handlebars_prompt_teplate`](/python/api/semantic-kernel/semantic_kernel.prompt_template.handlebars_prompt_template) + +::: zone-end + +::: zone pivot="programming-language-java" + +> Agents are currently unavailable in Java. + +::: zone-end + + +### [Chat Completion](./chat-completion-agent.md) + +The [_Chat Completion Agent_](./chat-completion-agent.md) is designed around any _Semantic Kernel_ [AI service](../../concepts/ai-services/chat-completion/index.md), offering a flexible and convenient persona encapsulation that can be seamlessly integrated into a wide range of applications. This agent allows developers to easily bring conversational AI capabilities into their systems without having to deal with complex implementation details. It mirrors the features and patterns found in the underlying [AI service](../../concepts/ai-services/chat-completion/index.md), ensuring that all functionalities—such as natural language processing, dialogue management, and contextual understanding—are fully supported within the [_Chat Completion Agent_](./chat-completion-agent.md), making it a powerful tool for building conversational interfaces. + +#### Related API's: + +::: zone pivot="programming-language-csharp" + +- [`IChatCompletionService`](/dotnet/api/microsoft.semantickernel.chatcompletion.ichatcompletionservice) +- [`Microsoft.SemanticKernel.Connectors.AzureOpenAI`](/dotnet/api/microsoft.semantickernel.connectors.azureopenai) +- [`Microsoft.SemanticKernel.Connectors.OpenAI`](/dotnet/api/microsoft.semantickernel.connectors.openai) +- [`Microsoft.SemanticKernel.Connectors.Google`](/dotnet/api/microsoft.semantickernel.connectors.google) +- [`Microsoft.SemanticKernel.Connectors.HuggingFace`](/dotnet/api/microsoft.semantickernel.connectors.huggingface) +- [`Microsoft.SemanticKernel.Connectors.MistralAI`](/dotnet/api/microsoft.semantickernel.connectors.mistralai) +- [`Microsoft.SemanticKernel.Connectors.Onnx`](/dotnet/api/microsoft.semantickernel.connectors.onnx) + +::: zone-end + +::: zone pivot="programming-language-python" + +- [`azure_chat_completion`](/python/api/semantic-kernel/semantic_kernel.connectors.ai.open_ai.services.azure_chat_completion) +- [`open_ai_chat_completion`](/python/api/semantic-kernel/semantic_kernel.connectors.ai.open_ai.services.open_ai_chat_completion) + +::: zone-end + +::: zone pivot="programming-language-java" + +> Agents are currently unavailable in Java. + +::: zone-end + + +> [!div class="nextstepaction"] +> [Exploring Chat Completion Agent](./chat-completion-agent.md) + diff --git a/semantic-kernel/Frameworks/agent/agent-chat.md b/semantic-kernel/Frameworks/agent/agent-chat.md new file mode 100644 index 00000000..3c504d53 --- /dev/null +++ b/semantic-kernel/Frameworks/agent/agent-chat.md @@ -0,0 +1,678 @@ +--- +title: Exploring Agent Collaboration in Agent Chat (Experimental) +description: An exploration of defining and managing agent collaboration via Agent Chat. +zone_pivot_groups: programming-languages +author: crickman +ms.topic: tutorial +ms.author: crickman +ms.date: 09/13/2024 +ms.service: semantic-kernel +--- +# Exploring Agent Collaboration in _Agent Chat_ (Experimental) + +> [!WARNING] +> The _Semantic Kernel Agent Framework_ is experimental, still in development and is subject to change. + +Detailed API documentation related to this discussion is available at: + +::: zone pivot="programming-language-csharp" +- [`AgentChat`](/dotnet/api/microsoft.semantickernel.agents.agentchat) +- [`AgentGroupChat`](/dotnet/api/microsoft.semantickernel.agents.agentgroupchat) +- [`Microsoft.SemanticKernel.Agents.Chat`](/dotnet/api/microsoft.semantickernel.agents.chat) + +::: zone-end + +::: zone pivot="programming-language-python" + +- [`agent_chat`](/python/api/semantic-kernel/semantic_kernel.agents.group_chat.agent_chat) +- [`agent_group_chat`](/python/api/semantic-kernel/semantic_kernel.agents.group_chat.agent_group_chat) + +::: zone-end + +::: zone pivot="programming-language-java" + +> Agents are currently unavailable in Java. + +::: zone-end + + +## What is _Agent Chat_? + +_Agent Chat_ provides a framework that enables interaction between multiple agents, even if they are of different types. This makes it possible for a [_Chat Completion Agent_](./chat-completion-agent.md) and an [_Open AI Assistant Agent_](./assistant-agent.md) to work together within the same conversation. _Agent Chat_ also defines entry points for initiating collaboration between agents, whether through multiple responses or a single agent response. + +As an abstract class, _Agent Chat_ can be subclassed to support custom scenarios. + +One such subclass, _Agent Group Chat_, offers a concrete implementation of _Agent Chat_, using a strategy-based approach to manage conversation dynamics. + + +## Creating an _Agent Group Chat_ + +To create an _Agent Group Chat_, you may either specify the participating agents or create an empty chat and subsequently add agent participants. Configuring the _Chat-Settings_ and _Strategies_ is also performed during _Agent Group Chat_ initialization. These settings define how the conversation dynamics will function within the group. + +> Note: The default _Chat-Settings_ result in a conversation that is limited to a single response. See [_Agent Chat_ Behavior](#defining-agent-group-chat-behavior) for details on configuring _Chat-Settings. + +#### Creating _Agent Group Chat_ with _Agents_: + +::: zone pivot="programming-language-csharp" +```csharp +// Define agents +ChatCompletionAgent agent1 = ...; +OpenAIAssistantAgent agent2 = ...; + +// Create chat with participating agents. +AgentGroupChat chat = new(agent1, agent2); +``` +::: zone-end + +::: zone pivot="programming-language-python" +```python +# Define agents +agent1 = ChatCompletionAgent(...) +agent2 = OpenAIAssistantAgent(...) + +# Create chat with participating agents +chat = AgentGroupChat(agents=[agent1, agent2]) +``` +::: zone-end + +::: zone pivot="programming-language-java" + +> Agents are currently unavailable in Java. + +::: zone-end + +#### Adding _Agents_ to a _Agent Group Chat_: + +::: zone pivot="programming-language-csharp" +```csharp +// Define agents +ChatCompletionAgent agent1 = ...; +OpenAIAssistantAgent agent2 = ...; + +// Create an empty chat. +AgentGroupChat chat = new(); + +// Add agents to an existing chat. +chat.AddAgent(agent1); +chat.AddAgent(agent2); +``` +::: zone-end + +::: zone pivot="programming-language-python" +```python +# Define agents +agent1 = ChatCompletionAgent(...) +agent2 = OpenAIAssistantAgent(...) + +# Create an empty chat +chat = AgentGroupChat() + +# Add agents to an existing chat +chat.add_agent(agent=agent1) +chat.add_agent(agent=agent2) +``` +::: zone-end + +::: zone pivot="programming-language-java" + +> Agents are currently unavailable in Java. + +::: zone-end + + +## Using _Agent Group Chat_ + +_Agent Chat_ supports two modes of operation: _Single-Turn_ and _Multi-Turn_. In _single-turn_, a specific agent is designated to provide a response. In _multi-turn_, all agents in the conversation take turns responding until a termination criterion is met. In both modes, agents can collaborate by responding to one another to achieve a defined goal. + +### Providing Input + +Adding an input message to an _Agent Chat_ follows the same pattern as whit a _Chat History_ object. + +::: zone pivot="programming-language-csharp" +```csharp +AgentGroupChat chat = new(); + +chat.AddChatMessage(new ChatMessageContent(AuthorRole.User, "")); +``` +::: zone-end + +::: zone pivot="programming-language-python" +```python +chat = AgentGroupChat() + +await chat.add_chat_message(ChatMessageContent(role=AuthorRole.USER, content="")) +``` +::: zone-end + +::: zone pivot="programming-language-java" + +> Agents are currently unavailable in Java. + +::: zone-end + +### Single-Turn Agent Invocation + +In a multi-turn invocation, the system must decide which agent responds next and when the conversation should end. In contrast, a single-turn invocation simply returns a response from the specified agent, allowing the caller to directly manage agent participation. + +After an agent participates in the _Agent Chat_ through a single-turn invocation, it is added to the set of _agents_ eligible for multi-turn invocation. + +::: zone pivot="programming-language-csharp" +```csharp +// Define an agent +ChatCompletionAgent agent = ...; + +// Create an empty chat. +AgentGroupChat chat = new(); + +// Invoke an agent for its response +ChatMessageContent[] messages = await chat.InvokeAsync(agent).ToArrayAsync(); +``` +::: zone-end + +::: zone pivot="programming-language-python" +```python +# Define an agent +agent = ChatCompletionAgent(...) + +# Create an empty chat +chat = AgentGroupChat() + +# Invoke an agent for its response(s) +async for message in chat.invoke(agent) + # process message response(s) +``` +::: zone-end + +::: zone pivot="programming-language-java" + +> Agents are currently unavailable in Java. + +::: zone-end + +### Multi-Turn Agent Invocation + +While agent collaboration requires that a system must be in place that not only determines which agent should respond during each turn but also assesses when the conversation has achieved its intended goal, initiating multi-turn collaboration remains straightforward. + +Agent responses are returned asynchronously as they are generated, allowing the conversation to unfold in real-time. + +> Note: In following sections, [Agent Selection](#agent-selection) and [Chat Termination](#chat-termination), will delve into the _Execution Settings_ in detail. The default _Execution Settings_ employs sequential or round-robin selection and limits agent participation to a single turn. + +::: zone pivot="programming-language-csharp" + +.NET _Execution Settings_ API: [`AgentGroupChatSettings`](/dotnet/api/microsoft.semantickernel.agents.chat.agentgroupchatsettings) + +```csharp +// Define agents +ChatCompletionAgent agent1 = ...; +OpenAIAssistantAgent agent2 = ...; + +// Create chat with participating agents. +AgentGroupChat chat = + new(agent1, agent2) + { + // Override default execution settings + ExecutionSettings = + { + TerminationStrategy = { MaximumIterations = 10 } + } + }; + +// Invoke agents +await foreach (ChatMessageContent response in chat.InvokeAsync()) +{ + // Process agent response(s)... +} +``` +::: zone-end + +::: zone pivot="programming-language-python" +```python +# Define agents +agent1 = ChatCompletionAgent(...) +agent2 = OpenAIAssistantAgent(...) + +# Create chat with participating agents +chat = AgentGroupChat( + agents=[agent1, agent2], + termination_strategy=DefaultTerminationStrategy(maximum_iterations=10), +) + +async for response in chat.invoke(): + # process agent response(s) +``` +::: zone-end + +::: zone pivot="programming-language-java" + +> Agents are currently unavailable in Java. + +::: zone-end + +### Accessing Chat History + +The _Agent Chat_ conversation history is always accessible, even though messages are delivered through the invocation pattern. This ensures that past exchanges remain available throughout the conversation. + +> Note: The most recent message is provided first (descending order: newest to oldest). + +::: zone pivot="programming-language-csharp" +```csharp +// Define and use a chat +AgentGroupChat chat = ...; + +// Access history for a previously utilized AgentGroupChat +ChatMessageContent[] history = await chat.GetChatMessagesAsync().ToArrayAsync(); +``` +::: zone-end + +::: zone pivot="programming-language-python" +```python +# Define a group chat +chat = AgentGroupChat(...) + +# Access history for a previously utilized AgentGroupChat +history = await chat.get_chat_messages() +``` +::: zone-end + +::: zone pivot="programming-language-java" + +> Agents are currently unavailable in Java. + +::: zone-end + +Since different agent types or configurations may maintain their own version of the conversation history, agent specific history is also available by specifing an agent. (For example: [_Open AI Assistant_](./assistant-agent.md) versus [_Chat Completion Agent_](./chat-completion-agent.md).) + +::: zone pivot="programming-language-csharp" +```csharp +// Agents to participate in chat +ChatCompletionAgent agent1 = ...; +OpenAIAssistantAgent agent2 = ...; + +// Define a group chat +AgentGroupChat chat = ...; + +// Access history for a previously utilized AgentGroupChat +ChatMessageContent[] history1 = await chat.GetChatMessagesAsync(agent1).ToArrayAsync(); +ChatMessageContent[] history2 = await chat.GetChatMessagesAsync(agent2).ToArrayAsync(); +``` +::: zone-end + +::: zone pivot="programming-language-python" +```python +# Agents to participate in a chat +agent1 = ChatCompletionAgent(...) +agent2 = OpenAIAssistantAgent(...) + +# Define a group chat +chat = AgentGroupChat(...) + +# Access history for a previously utilized AgentGroupChat +history1 = await chat.get_chat_messages(agent=agent1) +history2 = await chat.get_chat_messages(agent=agent2) +``` +::: zone-end + +::: zone pivot="programming-language-java" + +> Agents are currently unavailable in Java. + +::: zone-end + + +## Defining _Agent Group Chat_ Behavior + +Collaboration among agents to solve complex tasks is a core agentic pattern. To use this pattern effectively, a system must be in place that not only determines which agent should respond during each turn but also assesses when the conversation has achieved its intended goal. This requires managing agent selection and establishing clear criteria for conversation termination, ensuring seamless cooperation between agents toward a solution. Both of these aspects are governed by the _Execution Settings_ property. + +The following sections, [Agent Selection](#agent-selection) and [Chat Termination](#chat-termination), will delve into these considerations in detail. + +### Agent Selection + +In multi-turn invocation, agent selection is guided by a _Selection Strategy_. This strategy is defined by a base class that can be extended to implement custom behaviors tailored to specific needs. For convenience, two predefined concrete _Selection Strategies_ are also available, offering ready-to-use approaches for handling agent selection during conversations. + +If known, an initial agent may be specified to always take the first turn. A history reducer may also be employed to limit token usage when using a strategy based on a _Kernel Function_. + +::: zone pivot="programming-language-csharp" + +.NET Selection Strategy API: +- [`SelectionStrategy`](/dotnet/api/microsoft.semantickernel.agents.chat.selectionstrategy) +- [`SequentialSelectionStrategy`](/dotnet/api/microsoft.semantickernel.agents.chat.sequentialselectionstrategy) +- [`KernelFunctionSelectionStrategy`](/dotnet/api/microsoft.semantickernel.agents.chat.kernelfunctionselectionstrategy) +- [`Microsoft.SemanticKernel.Agents.History`](/dotnet/api/microsoft.semantickernel.agents.history) + +```csharp +// Define the agent names for use in the function template +const string WriterName = "Writer"; +const string ReviewerName = "Reviewer"; + +// Initialize a Kernel with a chat-completion service +Kernel kernel = ...; + +// Create the agents +ChatCompletionAgent writerAgent = + new() + { + Name = WriterName, + Instructions = "", + Kernel = kernel + }; + +ChatCompletionAgent reviewerAgent = + new() + { + Name = ReviewerName, + Instructions = "", + Kernel = kernel + }; + +// Define a kernel function for the selection strategy +KernelFunction selectionFunction = + AgentGroupChat.CreatePromptFunctionForStrategy( + $$$""" + Determine which participant takes the next turn in a conversation based on the the most recent participant. + State only the name of the participant to take the next turn. + No participant should take more than one turn in a row. + + Choose only from these participants: + - {{{ReviewerName}}} + - {{{WriterName}}} + + Always follow these rules when selecting the next participant: + - After {{{WriterName}}}, it is {{{ReviewerName}}}'s turn. + - After {{{ReviewerName}}}, it is {{{WriterName}}}'s turn. + + History: + {{$history}} + """, + safeParameterNames: "history"); + +// Define the selection strategy +KernelFunctionSelectionStrategy selectionStrategy = + new(selectionFunction, kernel) + { + // Always start with the writer agent. + InitialAgent = writerAgent, + // Parse the function response. + ResultParser = (result) => result.GetValue() ?? WriterName, + // The prompt variable name for the history argument. + HistoryVariableName = "history", + // Save tokens by not including the entire history in the prompt + HistoryReducer = new ChatHistoryTruncationReducer(3), + }; + +// Create a chat using the defined selection strategy. +AgentGroupChat chat = + new(writerAgent, reviewerAgent) + { + ExecutionSettings = new() { SelectionStrategy = selectionStrategy } + }; +``` +::: zone-end + +::: zone pivot="programming-language-python" +```python +REVIEWER_NAME = "Reviewer" +WRITER_NAME = "Writer" + +agent_reviewer = ChatCompletionAgent( + service_id=REVIEWER_NAME, + kernel=kernel, + name=REVIEWER_NAME, + instructions="", +) + +agent_writer = ChatCompletionAgent( + service_id=WRITER_NAME, + kernel=kernel, + name=WRITER_NAME, + instructions="", +) + +selection_function = KernelFunctionFromPrompt( + function_name="selection", + prompt=f""" + Determine which participant takes the next turn in a conversation based on the the most recent participant. + State only the name of the participant to take the next turn. + No participant should take more than one turn in a row. + + Choose only from these participants: + - {REVIEWER_NAME} + - {WRITER_NAME} + + Always follow these rules when selecting the next participant: + - After user input, it is {WRITER_NAME}'s turn. + - After {WRITER_NAME} replies, it is {REVIEWER_NAME}'s turn. + - After {REVIEWER_NAME} provides feedback, it is {WRITER_NAME}'s turn. + + History: + {{{{$history}}}} + """, +) + +chat = AgentGroupChat( + agents=[agent_writer, agent_reviewer], + selection_strategy=KernelFunctionSelectionStrategy( + function=selection_function, + kernel=_create_kernel_with_chat_completion("selection"), + result_parser=lambda result: str(result.value[0]) if result.value is not None else COPYWRITER_NAME, + agent_variable_name="agents", + history_variable_name="history", + ), +) +``` +::: zone-end + +::: zone pivot="programming-language-java" + +> Agents are currently unavailable in Java. + +::: zone-end + + +### Chat Termination + +In _multi-turn_ invocation, the _Termination Strategy_ dictates when the final turn takes place. This strategy ensures the conversation ends at the appropriate point. + +This strategy is defined by a base class that can be extended to implement custom behaviors tailored to specific needs. For convenience, serveral predefined concrete _Selection Strategies_ are also available, offering ready-to-use approaches for defining termination criteria for an _Agent Chat_ conversations. + +::: zone pivot="programming-language-csharp" + +.NET Selection Strategy API: +- [`TerminationStrategy`](/dotnet/api/microsoft.semantickernel.agents.chat.terminationstrategy) +- [`RegexTerminationStrategy`](/dotnet/api/microsoft.semantickernel.agents.chat.regexterminationstrategy) +- [`KernelFunctionSelectionStrategy`](/dotnet/api/microsoft.semantickernel.agents.chat.kernelfunctionselectionstrategy) +- [`KernelFunctionTerminationStrategy`](/dotnet/api/microsoft.semantickernel.agents.chat.kernelfunctionterminationstrategy) +- [`AggregatorTerminationStrategy`](/dotnet/api/microsoft.semantickernel.agents.chat.aggregatorterminationstrategy) +- [`Microsoft.SemanticKernel.Agents.History`](/dotnet/api/microsoft.semantickernel.agents.history) + +```csharp +// Initialize a Kernel with a chat-completion service +Kernel kernel = ...; + +// Create the agents +ChatCompletionAgent writerAgent = + new() + { + Name = "Writer", + Instructions = "", + Kernel = kernel + }; + +ChatCompletionAgent reviewerAgent = + new() + { + Name = "Reviewer", + Instructions = "", + Kernel = kernel + }; + +// Define a kernel function for the selection strategy +KernelFunction terminationFunction = + AgentGroupChat.CreatePromptFunctionForStrategy( + $$$""" + Determine if the reviewer has approved. If so, respond with a single word: yes + + History: + {{$history}} + """, + safeParameterNames: "history"); + +// Define the termination strategy +KernelFunctionTerminationStrategy terminationStrategy = + new(selectionFunction, kernel) + { + // Only the reviewer may give approval. + Agents = [reviewerAgent], + // Parse the function response. + ResultParser = (result) => + result.GetValue()?.Contains("yes", StringComparison.OrdinalIgnoreCase) ?? false, + // The prompt variable name for the history argument. + HistoryVariableName = "history", + // Save tokens by not including the entire history in the prompt + HistoryReducer = new ChatHistoryTruncationReducer(1), + // Limit total number of turns no matter what + MaximumIterations = 10, +}; + +// Create a chat using the defined termination strategy. +AgentGroupChat chat = + new(writerAgent, reviewerAgent) + { + ExecutionSettings = new() { TerminationStrategy = terminationStrategy } + }; + +``` +::: zone-end + +::: zone pivot="programming-language-python" +```python +REVIEWER_NAME = "Reviewer" +WRITER_NAME = "Writer" + +agent_reviewer = ChatCompletionAgent( + service_id=REVIEWER_NAME, + kernel=kernel, + name=REVIEWER_NAME, + instructions="", +) + +agent_writer = ChatCompletionAgent( + service_id=WRITER_NAME, + kernel=kernel, + name=WRITER_NAME, + instructions="", +) + +termination_function = KernelFunctionFromPrompt( + function_name="termination", + prompt=""" + Determine if the copy has been approved. If so, respond with a single word: yes + + History: + {{$history}} + """, +) + +chat = AgentGroupChat( + agents=[agent_writer, agent_reviewer], + termination_strategy=KernelFunctionTerminationStrategy( + agents=[agent_reviewer], + function=termination_function, + kernel=_create_kernel_with_chat_completion("termination"), + result_parser=lambda result: str(result.value[0]).lower() == "yes", + history_variable_name="history", + maximum_iterations=10, + ), +) +``` +::: zone-end + +::: zone pivot="programming-language-java" + +> Agents are currently unavailable in Java. + +::: zone-end + + +### Resetting Chat Completion State + +Regardless of whether _Agent Group Chat_ is invoked using the single-turn or multi-turn approach, the state of the _Agent Group Chat_ is updated to indicate it is _completed_ once the termination criteria is met. This ensures that the system recognizes when a conversation has fully concluded. To continue using an _Agent Group Chat_ instance after it has reached the _Completed_ state, this state must be reset to allow further interactions. Without resetting, additional interactions or agent responses will not be possible. + +In the case of a multi-turn invocation that reaches the maximum turn limit, the system will cease agent invocation but will not mark the instance as _completed_. This allows for the possibility of extending the conversation without needing to reset the _Completion_ state. + +::: zone pivot="programming-language-csharp" +```csharp +// Define an use chat +AgentGroupChat chat = ...; + +// Evaluate if completion is met and reset. +if (chat.IsComplete) +{ + // Opt to take action on the chat result... + + // Reset completion state to continue use + chat.IsComplete = false; +} +``` +::: zone-end + +::: zone pivot="programming-language-python" +```python +# Define a group chat +chat = AgentGroupChat() + +# Evaluate if completion is met and reset +if chat.is_complete: + # Reset completion state to continue use + chat.is_complete = False +``` +::: zone-end + +::: zone pivot="programming-language-java" + +> Agents are currently unavailable in Java. + +::: zone-end + + +### Clear Full Conversation State + +When done using an _Agent Chat_ where an [_Open AI Assistant_](./assistant-agent.md) participated, it may be necessary to delete the remote _thread_ associated with the _assistant_. _Agent Chat_ supports resetting or clearing the entire conversation state, which includes deleting any remote _thread_ definition. This ensures that no residual conversation data remains linked to the assistant once the chat concludes. + +A full reset does not remove the _agents_ that had joined the _Agent Chat_ and leaves the _Agent Chat_ in a state where it can be reused. This allows for the continuation of interactions with the same agents without needing to reinitialize them, making future conversations more efficient. + +::: zone pivot="programming-language-csharp" +```csharp +// Define an use chat +AgentGroupChat chat = ...; + +// Clear the all conversation state +await chat.ResetAsync(); +``` +::: zone-end + +::: zone pivot="programming-language-python" +```python +# Define a group chat +chat = AgentGroupChat() + +# Clear the conversation state +await chat.reset() +``` +::: zone-end + +::: zone pivot="programming-language-java" + +> Agents are currently unavailable in Java. + +::: zone-end + + +## How-To + +For an end-to-end example for using _Agent Group Chat_ for _Agent_ collaboration, see: + +- [How to Coordinate Agent Collaboration using _Agent Group Chat_](./examples/example-agent-collaboration.md) + + +> [!div class="nextstepaction"] +> [Create an Agent from a Template](./agent-templates.md) diff --git a/semantic-kernel/Frameworks/agent/agent-functions.md b/semantic-kernel/Frameworks/agent/agent-functions.md new file mode 100644 index 00000000..e6ad9905 --- /dev/null +++ b/semantic-kernel/Frameworks/agent/agent-functions.md @@ -0,0 +1,213 @@ +--- +title: Configuring Agents with Semantic Kernel Plugins. (Experimental) +description: Describes how to use Semantic Kernal plugins and function calling with agents. +zone_pivot_groups: programming-languages +author: crickman +ms.topic: tutorial +ms.author: crickman +ms.date: 09/13/2024 +ms.service: semantic-kernel +--- +# Configuring Agents with Semantic Kernel Plugins (Experimental) + +> [!WARNING] +> The _Semantic Kernel Agent Framework_ is experimental, still in development and is subject to change. + +## Functions and Plugins in Semantic Kernel + +Function calling is a powerful tool that allows developers to add custom functionalities and expand the capabilities of AI applications. The _Semantic Kernel_ [Plugin](../../concepts/plugins/index.md) architecture offers a flexible framework to support [Function Calling](../../concepts/ai-services/chat-completion/function-calling/index.md). For an _Agent_, integrating [Plugins](../../concepts/plugins/index.md) and [Function Calling](../../concepts/ai-services/chat-completion/function-calling/index.md) is built on this foundational _Semantic Kernel_ feature. + +Once configured, an agent will choose when and how to call an available function, as it would in any usage outside of the _Agent Framework_. + +::: zone pivot="programming-language-csharp" + +- [`KernelFunctionFactory`](/dotnet/api/microsoft.semantickernel.kernelfunctionfactory) +- [`KernelFunction`](/dotnet/api/microsoft.semantickernel.kernelfunction) +- [`KernelPluginFactory`](/dotnet/api/microsoft.semantickernel.kernelpluginfactory) +- [`KernelPlugin`](/dotnet/api/microsoft.semantickernel.kernelplugin) +- [`Kernel.Plugins`](/dotnet/api/microsoft.semantickernel.kernel.plugins) + +::: zone-end + +::: zone pivot="programming-language-python" + +- [`kernel_function`](/python/api/semantic-kernel/semantic_kernel.functions.kernel_function) +- [`kernel_function_extension`](/python/api/semantic-kernel/semantic_kernel.functions.kernel_function_extension) +- [`kernel_plugin`](/python/api/semantic-kernel/semantic_kernel.functions.kernel_plugin) + +::: zone-end + +::: zone pivot="programming-language-java" + +> Agents are currently unavailable in Java. + +::: zone-end + + +## Adding Plugins to an Agent + +Any [Plugin](../../concepts/plugins/index.md) available to an _Agent_ is managed within its respective _Kernel_ instance. This setup enables each _Agent_ to access distinct functionalities based on its specific role. + +[Plugins](../../concepts/plugins/index.md) can be added to the _Kernel_ either before or after the _Agent_ is created. The process of initializing [Plugins](../../concepts/plugins/index.md) follows the same patterns used for any _Semantic Kernel_ implementation, allowing for consistency and ease of use in managing AI capabilities. + +> Note: For a [_Chat Completion Agent_](./chat-completion-agent.md), the function calling mode must be explicitly enabled. [_Open AI Assistant_](./assistant-agent.md) agent is always based on automatic function calling. + +::: zone pivot="programming-language-csharp" +```csharp +// Factory method to product an agent with a specific role. +// Could be incorporated into DI initialization. +ChatCompletionAgent CreateSpecificAgent(Kernel kernel, string credentials) +{ + // Clone kernel instance to allow for agent specific plug-in definition + Kernel agentKernel = kernel.Clone(); + + // Initialize plug-in from type + agentKernel.CreatePluginFromType(); + + // Initialize plug-in from object + agentKernel.CreatePluginFromObject(new StatefulPlugin(credentials)); + + // Create the agent + return + new ChatCompletionAgent() + { + Name = "", + Instructions = "", + Kernel = agentKernel, + Arguments = new KernelArguments( + new OpenAIPromptExecutionSettings() + { + FunctionChoiceBehavior = FunctionChoiceBehavior.Auto() + }) + }; +} +``` +::: zone-end + +::: zone pivot="programming-language-python" +```python +# Create the instance of the Kernel +kernel = Kernel() + +# Define the service ID +service_id = "" + +# Add the chat completion service to the Kernel +kernel.add_service(AzureChatCompletion(service_id=service_id)) + +# Get the AI Service settings for the specified service_id +settings = kernel.get_prompt_execution_settings_from_service_id(service_id=service_id) + +# Configure the function choice behavior to auto invoke kernel functions +settings.function_choice_behavior = FunctionChoiceBehavior.Auto() + +# Add the Plugin to the Kernel +kernel.add_plugin(SamplePlugin(), plugin_name="") + +# Create the agent +agent = ChatCompletionAgent( + service_id=service_id, + kernel=kernel, + name=, + instructions=, + execution_settings=settings, +) +``` +::: zone-end + +::: zone pivot="programming-language-java" + +> Agents are currently unavailable in Java. + +::: zone-end + + +## Adding Functions to an Agent + +A [Plugin](../../concepts/plugins/index.md) is the most common approach for configuring [Function Calling](../../concepts/ai-services/chat-completion/function-calling/index.md). However, individual functions can also be supplied independently including _prompt functions_. + +::: zone pivot="programming-language-csharp" +```csharp +// Factory method to product an agent with a specific role. +// Could be incorporated into DI initialization. +ChatCompletionAgent CreateSpecificAgent(Kernel kernel) +{ + // Clone kernel instance to allow for agent specific plug-in definition + Kernel agentKernel = kernel.Clone(); + + // Initialize plug-in from a static function + agentKernel.CreateFunctionFromMethod(StatelessPlugin.AStaticMethod); + + // Initialize plug-in from a prompt + agentKernel.CreateFunctionFromPrompt(""); + + // Create the agent + return + new ChatCompletionAgent() + { + Name = "", + Instructions = "", + Kernel = agentKernel, + Arguments = new KernelArguments( + new OpenAIPromptExecutionSettings() + { + FunctionChoiceBehavior = FunctionChoiceBehavior.Auto() + }) + }; +} +``` +::: zone-end + +::: zone pivot="programming-language-python" +```python +# Create the instance of the Kernel +kernel = Kernel() + +# Define the service ID +service_id = "" + +# Add the chat completion service to the Kernel +kernel.add_service(AzureChatCompletion(service_id=service_id)) + +# Get the AI Service settings for the specified service_id +settings = kernel.get_prompt_execution_settings_from_service_id(service_id=service_id) + +# Configure the function choice behavior to auto invoke kernel functions +settings.function_choice_behavior = FunctionChoiceBehavior.Auto() + +# Add the Plugin to the Kernel +kernel.add_plugin(SamplePlugin(), plugin_name="") + +# Create the agent +agent = ChatCompletionAgent( + service_id=service_id, + kernel=kernel, + name=, + instructions=, + execution_settings=settings, +) +``` +::: zone-end + +::: zone pivot="programming-language-java" + +> Agents are currently unavailable in Java. + +::: zone-end + + +## Limitations for Agent Function Calling + +When directly invoking a[_Chat Completion Agent_](./chat-completion-agent.md), all _Function Choice Behaviors_ are supported. However, when using an [_Open AI Assistant_](./assistant-agent.md) or [_Agent Chat_](./agent-chat.md), only _Automatic_ [Function Calling](../../concepts/ai-services/chat-completion/function-calling/index.md) is currently available. + + +## How-To + +For an end-to-end example for using function calling, see: + +- [How-To: _Chat Completion Agent_](./examples/example-chat-agent.md) + + +> [!div class="nextstepaction"] +> [How to Stream Agent Responses](./agent-streaming.md) + diff --git a/semantic-kernel/Frameworks/agent/agent-streaming.md b/semantic-kernel/Frameworks/agent/agent-streaming.md new file mode 100644 index 00000000..b652fa73 --- /dev/null +++ b/semantic-kernel/Frameworks/agent/agent-streaming.md @@ -0,0 +1,220 @@ +--- +title: How to Stream Agent Responses. (Experimental) +description: Describes how to utilize streamed responses for agents and agent chat. +zone_pivot_groups: programming-languages +author: crickman +ms.topic: tutorial +ms.author: crickman +ms.date: 09/13/2024 +ms.service: semantic-kernel +--- +# How to Stream Agent Responses. (Experimental) + +> [!WARNING] +> The _Semantic Kernel Agent Framework_ is experimental, still in development and is subject to change. + +## What is a Streamed Response? + +A streamed response delivers the message content in small, incremental chunks. This approach enhances the user experience by allowing them to view and engage with the message as it unfolds, rather than waiting for the entire response to load. Users can begin processing information immediately, improving the sense of responsiveness and interactivity. As a result, it minimizes delays and keeps users more engaged throughout the communication process. + +#### Streaming References: + +- [Open AI Streaming Guide](https://platform.openai.com/docs/api-reference/streaming) +- [Open AI Chat Completion Streaming](https://platform.openai.com/docs/api-reference/chat/create#chat-create-stream) +- [Open AI Assistant Streaming](https://platform.openai.com/docs/api-reference/assistants-streaming) +- [Azure OpenAI Service REST API](/azure/ai-services/openai/reference) + + +## Streaming in Semantic Kernel + +[AI Services](../../concepts/ai-services/index.md) that support streaming in Semantic Kernel use different content types compared to those used for fully-formed messages. These content types are specifically designed to handle the incremental nature of streaming data. The same content types are also utilized within the Agent Framework for similar purposes. This ensures consistency and efficiency across both systems when dealing with streaming information. + +::: zone pivot="programming-language-csharp" +- [`StreamingChatMessageContent`](/dotnet/api/microsoft.semantickernel.streamingchatmessagecontent) +- [`StreamingTextContent`](/dotnet/api/microsoft.semantickernel.streamingtextcontent) +- [`StreamingFileReferenceContent`](/dotnet/api/microsoft.semantickernel.streamingfilereferencecontent) +- [`StreamingAnnotationContent`](/dotnet/api/microsoft.semantickernel.agents.openai.streamingannotationcontent) + +::: zone-end + +::: zone pivot="programming-language-python" +- [`streaming_chat_message_content`](/python/api/semantic-kernel/semantic_kernel.contents.streaming_chat_message_content) +- [`streaming_text_content`](/python/api/semantic-kernel/semantic_kernel.contents.streaming_text_content) +- [`streaming_file_reference_content`](/python/api/semantic-kernel/semantic_kernel.contents.streaming_file_reference_content) +- [`streaming_annotation_content`](/python/api/semantic-kernel/semantic_kernel.contents.streaming_annotation_content) + +::: zone-end + +::: zone pivot="programming-language-java" + +> Agents are currently unavailable in Java. + +::: zone-end + + +## Streaming Agent Invocation + +The _Agent Framework_ supports _streamed_ responses when using [_Agent Chat_](./agent-chat.md) or when directly invoking a [_Chat Completion Agent_](./chat-completion-agent.md) or [_Open AI Assistant Agent_](./assistant-agent.md). In either mode, the framework delivers responses asynchronously as they are streamed. Alongside the streamed response, a consistent, non-streamed history is maintained to track the conversation. This ensures both real-time interaction and a reliable record of the conversation's flow. + +### Streamed response from _Chat Completion Agent_ + +When invoking a streamed response from a [_Chat Completion Agent_](./chat-completion-agent.md), the _Chat History_ is updated after the full response is received. Although the response is streamed incrementally, the history records only the complete message. This ensures that the _Chat History_ reflects fully formed responses for consistency. + +::: zone pivot="programming-language-csharp" +```csharp +// Define agent +ChatCompletionAgent agent = ...; + +// Create a ChatHistory object to maintain the conversation state. +ChatHistory chat = []; + +// Add a user message to the conversation +chat.Add(new ChatMessageContent(AuthorRole.User, "")); + +// Generate the streamed agent response(s) +await foreach (StreamingChatMessageContent response in agent.InvokeStreamingAsync(chat)) +{ + // Process streamed response(s)... +} +``` +::: zone-end + +::: zone pivot="programming-language-python" +```python +# Define agent +agent = ChatCompletionAgent(...) + +# Create a ChatHistory object to maintain the conversation state. +chat = ChatHistory() + +# Add a user message to the conversation +chat.add_message(ChatMessageContent(AuthorRole.USER, "")) + +# Generate the streamed agent response(s) +async for response in agent.invoke_stream(chat) +{ + # Process streamed response(s)... +} +``` +::: zone-end + +::: zone pivot="programming-language-java" + +> Agents are currently unavailable in Java. + +::: zone-end + +### Streamed response from _Open AI Assistant Agent_ + +When invoking a streamed response from an [_Open AI Assistant Agent_](./assistant-agent.md), an optional _Chat History_ can be provided to capture the complete messages for further analysis if needed. Since the assistant maintains the conversation state as a remote thread, capturing these messages is not always necessary. The decision to store and analyze the full response depends on the specific requirements of the interaction. + +::: zone pivot="programming-language-csharp" +```csharp +// Define agent +OpenAIAssistantAgent agent = ...; + +// Create a thread for the agent conversation. +string threadId = await agent.CreateThreadAsync(); + +// Add a user message to the conversation +chat.Add(threadId, new ChatMessageContent(AuthorRole.User, "")); + +// Generate the streamed agent response(s) +await foreach (StreamingChatMessageContent response in agent.InvokeStreamingAsync(threadId)) +{ + // Process streamed response(s)... +} + +// Delete the thread when it is no longer needed +await agent.DeleteThreadAsync(threadId); +``` +::: zone-end + +::: zone pivot="programming-language-python" +```python +# Define agent +agent = OpenAIAssistantAgent(...) + +# Create a thread for the agent conversation. +thread_id = await agent.create_thread() + +# Add user message to the conversation +await agent.add_chat_message(ChatMessageContent(role=AuthorRole.USER, content="")) + +# Generate the streamed agent response(s) +async for response in agent.invoke_stream(thread_id=thread_id): + # Process streamed response(s)... +``` +::: zone-end + +::: zone pivot="programming-language-java" + +> Agents are currently unavailable in Java. + +::: zone-end + + +## Streaming with _Agent Chat_ + +When using [_Agent Chat_](./agent-chat.md), the full conversation history is always preserved and can be accessed directly through the [_Agent Chat_](./agent-chat.md) instance. Therefore, the key difference between streamed and non-streamed invocations lies in the delivery method and the resulting content type. In both cases, users can still access the complete history, but streamed responses provide real-time updates as the conversation progresses. This allows for greater flexibility in handling interactions, depending on the application's needs. + +::: zone pivot="programming-language-csharp" +```csharp +// Define agents +ChatCompletionAgent agent1 = ...; +OpenAIAssistantAgent agent2 = ...; + +// Create chat with participating agents. +AgentGroupChat chat = + new(agent1, agent2) + { + // Override default execution settings + ExecutionSettings = + { + TerminationStrategy = { MaximumIterations = 10 } + } + }; + +// Invoke agents +string lastAgent = string.Empty; +await foreach (StreamingChatMessageContent response in chat.InvokeStreamingAsync()) +{ + if (!lastAgent.Equals(response.AuthorName, StringComparison.Ordinal)) + { + // Process begining of agent response + lastAgent = response.AuthorName; + } + + // Process streamed content... +} +``` +::: zone-end + +::: zone pivot="programming-language-python" +```python +# Define agents +agent1 = ChatCompletionAgent(...) +agent2 = OpenAIAssistantAgent(...) + +# Create chat with participating agents +chat = AgentGroupChat( + agents=[agent1, agent2], + termination_strategy=DefaultTerminationStrategy(maximum_iterations=10), +) + +# Invoke agents +last_agent = None +async for response in chat.invoke_stream(): + if message.content is not None: + if last_agent != response.name: + # Process beginning of agent response + last_agent = message.name + # Process streamed content +``` +::: zone-end + +::: zone pivot="programming-language-java" + +> Agents are currently unavailable in Java. + +::: zone-end diff --git a/semantic-kernel/Frameworks/agent/agent-templates.md b/semantic-kernel/Frameworks/agent/agent-templates.md new file mode 100644 index 00000000..6c618f41 --- /dev/null +++ b/semantic-kernel/Frameworks/agent/agent-templates.md @@ -0,0 +1,239 @@ +--- +title: Create an Agent from a Semantic Kernel Template (Experimental) +description: Describes how to use a Semantic Kernel template to define an agent. +zone_pivot_groups: programming-languages +author: crickman +ms.topic: tutorial +ms.author: crickman +ms.date: 09/13/2024 +ms.service: semantic-kernel +--- +# Create an Agent from a Semantic Kernel Template (Experimental) + +> [!WARNING] +> The _Semantic Kernel Agent Framework_ is experimental, still in development and is subject to change. + +## Prompt Templates in Semantic Kernel + +An agent's role is primarily shaped by the instructions it receives, which dictate its behavior and actions. Similar to invoking a `Kernel` [prompt](../../concepts/prompts/index.md), an agent's instructions can include templated parameters—both values and functions—that are dynamically substituted during execution. This enables flexible, context-aware responses, allowing the agent to adjust its output based on real-time input. + +Additionally, an agent can be configured directly using a _Prompt Template Configuration_, providing developers with a structured and reusable way to define its behavior. This approach offers a powerful tool for standardizing and customizing agent instructions, ensuring consistency across various use cases while still maintaining dynamic adaptability. + +#### Related API's: + +::: zone pivot="programming-language-csharp" + +- [`PromptTemplateConfig`](/dotnet/api/microsoft.semantickernel.prompttemplateconfig) +- [`KernelFunctionYaml.FromPromptYaml`](/dotnet/api/microsoft.semantickernel.kernelfunctionyaml.frompromptyaml#microsoft-semantickernel-kernelfunctionyaml-frompromptyaml(system-string-microsoft-semantickernel-iprompttemplatefactory-microsoft-extensions-logging-iloggerfactory)) +- [`IPromptTemplateFactory`](/dotnet/api/microsoft.semantickernel.iprompttemplatefactory) +- [`KernelPromptTemplateFactory`](/dotnet/api/microsoft.semantickernel.kernelprompttemplatefactory) +- [_Handlebars_](/dotnet/api/microsoft.semantickernel.prompttemplates.handlebars) +- [_Prompty_](/dotnet/api/microsoft.semantickernel.prompty) +- [_Liquid_](/dotnet/api/microsoft.semantickernel.prompttemplates.liquid) + +::: zone-end + +::: zone pivot="programming-language-python" + +**Coming Soon** + +::: zone-end + +::: zone pivot="programming-language-java" + +> Agents are currently unavailable in Java. + +::: zone-end + + +## Agent Instructions as a Template + +Creating an agent with template parameters provides greater flexibility by allowing its instructions to be easily customized based on different scenarios or requirements. This approach enables the agent's behavior to be tailored by substituting specific values or functions into the template, making it adaptable to a variety of tasks or contexts. By leveraging template parameters, developers can design more versatile agents that can be configured to meet diverse use cases without needing to modify the core logic. + +#### Chat Completion Agent +::: zone pivot="programming-language-csharp" +```csharp +// Initialize a Kernel with a chat-completion service +Kernel kernel = ...; + +ChatCompletionAgent agent = + new() + { + Kernel = kernel, + Name = "StoryTeller", + Instructions = "Tell a story about {{$topic}} that is {{$length}} sentences long.", + Arguments = new KernelArguments() + { + { "topic", "Dog" }, + { "length", "3" }, + } + }; +``` +::: zone-end + +::: zone pivot="programming-language-python" + +**Coming soon** + +::: zone-end + +::: zone pivot="programming-language-java" + +> Agents are currently unavailable in Java. + +::: zone-end + +#### Open AI Assistant Agent + +Templated instructions are especially powerful when working with an [_Open AI Assistant Agent_](./assistant-agent.md). With this approach, a single assistant definition can be created and reused multiple times, each time with different parameter values tailored to specific tasks or contexts. This enables a more efficient setup, allowing the same assistant framework to handle a wide range of scenarios while maintaining consistency in its core behavior. + +::: zone pivot="programming-language-csharp" +```csharp +// Retrieve an existing assistant definition by identifier +OpenAIAssistantAgent agent = + await OpenAIAssistantAgent.RetrieveAsync( + this.GetClientProvider(), + "", + new Kernel(), + new KernelArguments() + { + { "topic", "Dog" }, + { "length", "3" }, + }); +``` +::: zone-end + +::: zone pivot="programming-language-python" + +**Coming Soon** + +::: zone-end + +::: zone pivot="programming-language-java" + +> Agents are currently unavailable in Java. + +::: zone-end + + +## Agent Definition from a _Prompt Template_ + +The same _Prompt Template Config_ used to create a _Kernel Prompt Function_ can also be leveraged to define an agent. This allows for a unified approach in managing both prompts and agents, promoting consistency and reuse across different components. By externalizing agent definitions from the codebase, this method simplifies the management of multiple agents, making them easier to update and maintain without requiring changes to the underlying logic. This separation also enhances flexibility, enabling developers to modify agent behavior or introduce new agents by simply updating the configuration, rather than adjusting the code itself. + +#### YAML Template + +```yaml +name: GenerateStory +template: | + Tell a story about {{$topic}} that is {{$length}} sentences long. +template_format: semantic-kernel +description: A function that generates a story about a topic. +input_variables: + - name: topic + description: The topic of the story. + is_required: true + - name: length + description: The number of sentences in the story. + is_required: true +``` + +#### Agent Initialization +::: zone pivot="programming-language-csharp" +```csharp +// Read YAML resource +string generateStoryYaml = File.ReadAllText("./GenerateStory.yaml"); +// Convert to a prompt template config +PromptTemplateConfig templateConfig = KernelFunctionYaml.ToPromptTemplateConfig(generateStoryYaml); + +// Create agent with Instructions, Name and Description +// provided by the template config. +ChatCompletionAgent agent = + new(templateConfig) + { + Kernel = this.CreateKernelWithChatCompletion(), + // Provide default values for template parameters + Arguments = new KernelArguments() + { + { "topic", "Dog" }, + { "length", "3" }, + } + }; +``` +::: zone-end + +::: zone pivot="programming-language-python" + +**Coming Soon** + +::: zone-end + +::: zone pivot="programming-language-java" + +> Agents are currently unavailable in Java. + +::: zone-end + + +### Overriding Template Values for Direct Invocation + +When invoking an agent directly, without using [_Agent Chat_](./agent-chat.md), the agent's parameters can be overridden as needed. This allows for greater control and customization of the agent's behavior during specific tasks, enabling you to modify its instructions or settings on the fly to suit particular requirements. + +::: zone pivot="programming-language-csharp" +```csharp +// Initialize a Kernel with a chat-completion service +Kernel kernel = ...; + +ChatCompletionAgent agent = + new() + { + Kernel = kernel, + Name = "StoryTeller", + Instructions = "Tell a story about {{$topic}} that is {{$length}} sentences long.", + Arguments = new KernelArguments() + { + { "topic", "Dog" }, + { "length", "3" }, + } + }; + +// Create a ChatHistory object to maintain the conversation state. +ChatHistory chat = []; + +KernelArguments overrideArguments = + new() + { + { "topic", "Cat" }, + { "length", "3" }, + }); + +// Generate the agent response(s) +await foreach (ChatMessageContent response in agent.InvokeAsync(chat, overrideArguments)) +{ + // Process agent response(s)... +} +``` +::: zone-end + +::: zone pivot="programming-language-python" + +**Coming Soon** + +::: zone-end + +::: zone pivot="programming-language-java" + +> Agents are currently unavailable in Java. + +::: zone-end + + +## How-To + +For an end-to-end example for creating an agent from a _pmompt-template_, see: + +- [How-To: _Chat Completion Agent_](./examples/example-chat-agent.md) + + +> [!div class="nextstepaction"] +> [Configuring Agents with Plugins](./agent-functions.md) + diff --git a/semantic-kernel/Frameworks/agent/assistant-agent.md b/semantic-kernel/Frameworks/agent/assistant-agent.md new file mode 100644 index 00000000..f75b6d90 --- /dev/null +++ b/semantic-kernel/Frameworks/agent/assistant-agent.md @@ -0,0 +1,228 @@ +--- +title: Exploring the Semantic Kernel Open AI Assistant Agent (Experimental) +description: An exploration of the definition, behaviors, and usage patterns for a `OpenAIAssistantAgent` +zone_pivot_groups: programming-languages +author: crickman +ms.topic: tutorial +ms.author: crickman +ms.date: 09/13/2024 +ms.service: semantic-kernel +--- +# Exploring the _Semantic Kernel_ _Open AI Assistant Agent_ (Experimental) + +> [!WARNING] +> The _Semantic Kernel Agent Framework_ is experimental, still in development and is subject to change. + +Detailed API documentation related to this discussion is available at: + +::: zone pivot="programming-language-csharp" +- [`OpenAIAssistantAgent`](/dotnet/api/microsoft.semantickernel.agents.openai.openaiassistantagent) +- [`OpenAIAssistantDefinition`](/dotnet/api/microsoft.semantickernel.agents.openai.openaiassistantdefinition) +- [`OpenAIClientProvider`](/dotnet/api/microsoft.semantickernel.agents.openai.openaiclientprovider) + +::: zone-end + +::: zone pivot="programming-language-python" + +- [`open_ai_assistant_base`](/python/api/semantic-kernel/semantic_kernel.agents.open_ai.open_ai_assistant_base) +- [`azure_assistant_agent`](/python/api/semantic-kernel/semantic_kernel.agents.open_ai.azure_assistant_agent) +- [`open_ai_assistant_agent`](/python/api/semantic-kernel/semantic_kernel.agents.open_ai.open_ai_assistant_agent) + +::: zone-end + +::: zone pivot="programming-language-java" + +> Agents are currently unavailable in Java. + +::: zone-end + + +## What is an Assistant? + +The _OpenAI Assistant API_ is a specialized interface designed for more advanced and interactive AI capabilities, enabling developers to create personalized and multi-step task-oriented agents. Unlike the Chat Completion API, which focuses on simple conversational exchanges, the Assistant API allows for dynamic, goal-driven interactions with additional features like code-interpreter and file-search. + +- [Open AI Assistant Guide](https://platform.openai.com/docs/assistants) +- [Open AI Assistant API](https://platform.openai.com/docs/api-reference/assistants) +- [Assistant API in Azure](/azure/ai-services/openai/assistants-quickstart) + + +## Creating an _Open AI Assistant Agent_ + +Creating an _Open AI Assistant_ requires invoking a remote service, which is handled asynchronously. To manage this, the _Open AI Assistant Agent_ is instantiated through a static factory method, ensuring the process occurs in a non-blocking manner. This method abstracts the complexity of the asynchronous call, returning a promise or future once the assistant is fully initialized and ready for use. + +::: zone pivot="programming-language-csharp" +```csharp +OpenAIAssistantAgent agent = + await OpenAIAssistantAgent.CreateAsync( + OpenAIClientProvider.ForAzureOpenAI(/*<...service configuration>*/), + new OpenAIAssistantDefinition("") + { + Name = "", + Instructions = "", + }, + new Kernel()); +``` +::: zone-end + +::: zone pivot="programming-language-python" +```python +azure_agent = await AzureAssistantAgent.create( + kernel=kernel, + service_id=service_id, + name="", + instructions="" +) + +# or + +openai_agent = await OpenAIAssistantAgent.create( + kernel=kernel, + service_id=service_id, + name="", + instructions="" +) +``` +::: zone-end + +::: zone pivot="programming-language-java" + +> Agents are currently unavailable in Java. + +::: zone-end + + +## Retrieving an _Open AI Assistant Agent_ + +Once created, the identifier of the assistant may be access via its identifier. This identifier may be used to create an _Open AI Assistant Agent_ from an existing assistant definition. + +::: zone pivot="programming-language-csharp" + +For .NET, the agent identifier is exposed as a `string` via the property defined by any agent. + +```csharp +OpenAIAssistantAgent agent = + await OpenAIAssistantAgent.RetrieveAsync( + OpenAIClientProvider.ForAzureOpenAI(/*<...service configuration>*/), + "", + new Kernel()); +``` +::: zone-end + +::: zone pivot="programming-language-python" +```python +agent = await OpenAIAssistantAgent.retrieve(id=agent_id, kernel=kernel) + +# or + +agent = await AzureAssistantAgent.retrieve(id=agent_id, kernel=kernel) +``` +::: zone-end + +::: zone pivot="programming-language-java" + +> Agents are currently unavailable in Java. + +::: zone-end + + +## Using an _Open AI Assistant Agent_ + +As with all aspects of the _Assistant API_, conversations are stored remotely. Each conversation is referred to as a _thread_ and identified by a unique `string` identifier. Interactions with your _OpenAI Assistant Agent_ are tied to this specific thread identifier which must be specified when calling the agent/ + +::: zone pivot="programming-language-csharp" +```csharp +// Define agent +OpenAIAssistantAgent agent = ...; + +// Create a thread for the agent conversation. +string threadId = await agent.CreateThreadAsync(); + +// Add a user message to the conversation +chat.Add(threadId, new ChatMessageContent(AuthorRole.User, "")); + +// Generate the agent response(s) +await foreach (ChatMessageContent response in agent.InvokeAsync(threadId)) +{ + // Process agent response(s)... +} + +// Delete the thread when it is no longer needed +await agent.DeleteThreadAsync(threadId); +``` +::: zone-end + +::: zone pivot="programming-language-python" +```python +# Define agent +openai_agent = await ... + +# Create a thread for the agent conversation +thread_id = await agent.create_thread() + +# Add a user message to the conversation +await agent.add_chat_message( + thread_id=thread_id, + message=ChatMessageContent(role=AuthorRole.USER, content=""), +) + +# Generate the agent response(s) +async for response in agent.invoke(thread_id=thread_id): + # process agent response(s)... + +# Delete the thread when it is no longer needed +await agent.delete_thread(thread_id) +``` +::: zone-end + +::: zone pivot="programming-language-java" + +> Agents are currently unavailable in Java. + +::: zone-end + + +## Deleting an _Open AI Assistant Agent_ + +Since the assistant's definition is stored remotely, it supports the capability to self-delete. This enables the agent to be removed from the system when it is no longer needed. + +> Note: Attempting to use an agent instance after being deleted results in an exception. + +::: zone pivot="programming-language-csharp" + +For .NET, the agent identifier is exposed as a `string` via the [`Agent.Id`](/dotnet/api/microsoft.semantickernel.agents.agent.id) property defined by any agent. + +```csharp +// Perform the deletion +await agent.DeleteAsync(); + +// Inspect whether an agent has been deleted +bool isDeleted = agent.IsDeleted(); +``` +::: zone-end + +::: zone pivot="programming-language-python" +```python +await agent.delete() + +is_deleted = agent._is_deleted +``` +::: zone-end + +::: zone pivot="programming-language-java" + +> Agents are currently unavailable in Java. + +::: zone-end + + +## How-To + +For an end-to-end example for a _Open AI Assistant Agent_, see: + +- [How-To: _Open AI Assistant Agent_ Code Interpreter](./examples/example-assistant-code.md) +- [How-To: _Open AI Assistant Agent_ File Search](./examples/example-assistant-search.md) + + +> [!div class="nextstepaction"] +> [Agent Collaboration in _Agent Chat_](./agent-chat.md) + diff --git a/semantic-kernel/Frameworks/agent/chat-completion-agent.md b/semantic-kernel/Frameworks/agent/chat-completion-agent.md new file mode 100644 index 00000000..6af0cccd --- /dev/null +++ b/semantic-kernel/Frameworks/agent/chat-completion-agent.md @@ -0,0 +1,237 @@ +--- +title: Exploring the Semantic Kernel Chat Completion Agent (Experimental) +description: An exploration of the definition, behaviors, and usage patterns for a Chat Completion Agent +zone_pivot_groups: programming-languages +author: crickman +ms.topic: tutorial +ms.author: crickman +ms.date: 09/13/2024 +ms.service: semantic-kernel +--- +# Exploring the _Semantic Kernel_ Chat Completion Agent (Experimental) + +> [!WARNING] +> The _Semantic Kernel Agent Framework_ is experimental, still in development and is subject to change. + +Detailed API documentation related to this discussion is available at: + +::: zone pivot="programming-language-csharp" +- [`ChatCompletionAgent`](/dotnet/api/microsoft.semantickernel.agents.chatcompletionagent) +- [`Microsoft.SemanticKernel.Agents`](/dotnet/api/microsoft.semantickernel.agents) +- [`IChatCompletionService`](/dotnet/api/microsoft.semantickernel.chatcompletion.ichatcompletionservice) +- [`Microsoft.SemanticKernel.ChatCompletion`](/dotnet/api/microsoft.semantickernel.chatcompletion ) + +::: zone-end + +::: zone pivot="programming-language-python" + +- [`chat_completion_agent`](/python/api/semantic-kernel/semantic_kernel.agents.chat_completion.chat_completion_agent) +- [`chat_completion_client_base`](/python/api/semantic-kernel/semantic_kernel.connectors.ai.chat_completion_client_base) + +::: zone-end + +::: zone pivot="programming-language-java" + +> Agents are currently unavailable in Java. + +::: zone-end + + +## Chat Completion in _Semantic Kernel_ + +[_Chat Completion_](../../concepts/ai-services/chat-completion/index.md) is fundamentally a protocol for a chat-based interaction with an AI model where the chat-history maintained and presented to the model with each request. _Semantic Kernel_ [AI services](../../concepts/ai-services/index.md) offer a unified framework for integrating the chat-completion capabilities of various AI models. + +A _chat completion agent_ can leverage any of these [AI services](../../concepts/ai-services/index.md) to generate responses, whether directed to a user or another agent. + +::: zone pivot="programming-language-csharp" + +For .NET, _chat-completion_ AI Services are based on the [`IChatCompletionService`](/dotnet/api/microsoft.semantickernel.chatcompletion.ichatcompletionservice) interface. + +For .NET, some of AI services that support models with chat-completion include: + +Model|_Semantic Kernel_ AI Service +--|-- +Azure Open AI|[`Microsoft.SemanticKernel.Connectors.AzureOpenAI`](/dotnet/api/microsoft.semantickernel.connectors.azureopenai) +Gemini|[`Microsoft.SemanticKernel.Connectors.Google`](/dotnet/api/microsoft.semantickernel.connectors.google) +HuggingFace|[`Microsoft.SemanticKernel.Connectors.HuggingFace`](/dotnet/api/microsoft.semantickernel.connectors.huggingface) +Mistral|[`Microsoft.SemanticKernel.Connectors.MistralAI`](/dotnet/api/microsoft.semantickernel.connectors.mistralai) +OpenAI|[`Microsoft.SemanticKernel.Connectors.OpenAI`](/dotnet/api/microsoft.semantickernel.connectors.openai) +Onnx|[`Microsoft.SemanticKernel.Connectors.Onnx`](/dotnet/api/microsoft.semantickernel.connectors.onnx) + +::: zone-end + +::: zone pivot="programming-language-python" + +- [`azure_chat_completion`](/python/api/semantic-kernel/semantic_kernel.connectors.ai.open_ai.services.azure_chat_completion) +- [`open_ai_chat_completion`](/python/api/semantic-kernel/semantic_kernel.connectors.ai.open_ai.services.open_ai_chat_completion) + +::: zone-end + +::: zone pivot="programming-language-java" + +> Agents are currently unavailable in Java. + +::: zone-end + + +## Creating a Chat Completion Agent + +A _chat completion agent_ is fundamentally based on an [AI services](../../concepts/ai-services/index.md). As such, creating an _chat completion agent_ starts with creating a [_Kernel_](../../concepts/kernel.md) instance that contains one or more chat-completion services and then instantiating the agent with a reference to that [_Kernel_](../../concepts/kernel.md) instance. + +::: zone pivot="programming-language-csharp" +```csharp +// Initialize a Kernel with a chat-completion service +IKernelBuilder builder = Kernel.CreateBuilder(); + +builder.AddAzureOpenAIChatCompletion(/*<...configuration parameters>*/); + +Kernel kernel = builder.Build(); + +// Create the agent +ChatCompletionAgent agent = + new() + { + Name = "SummarizationAgent", + Instructions = "Summarize user input", + Kernel = kernel + }; +``` +::: zone-end + +::: zone pivot="programming-language-python" +```python +# Define the Kernel +kernel = Kernel() + +# Add the AzureChatCompletion AI Service to the Kernel +kernel.add_service(AzureChatCompletion(service_id="")) + +# Create the agent +agent = ChatCompletionAgent( + service_id="agent", + kernel=kernel, + name="", + instructions="", +) +``` +::: zone-end + +::: zone pivot="programming-language-java" + +> Agents are currently unavailable in Java. + +::: zone-end + + +## AI Service Selection + +No different from using _Semantic Kernel_ [AI services](../../concepts/ai-services/index.md) directly, a _chat completion agent_ support the specification of a _service-selector_. A _service-selector_ indentifies which [AI service](../../concepts/ai-services/index.md) to target when the [_Kernel_](../../concepts/kernel.md) contains more than one. + +> Note: If multiple [AI services](../../concepts/ai-services/index.md) are present and no _service-selector_ is provided, the same _default_ logic is applied for the agent that you'd find when using an [AI services](../../concepts/ai-services/index.md) outside of the _Agent Framework_ + +::: zone pivot="programming-language-csharp" +```csharp +IKernelBuilder builder = Kernel.CreateBuilder(); + +// Initialize multiple chat-completion services. +builder.AddAzureOpenAIChatCompletion(/*<...service configuration>*/, serviceId: "service-1"); +builder.AddAzureOpenAIChatCompletion(/*<...service configuration>*/, serviceId: "service-2"); + +Kernel kernel = builder.Build(); + +ChatCompletionAgent agent = + new() + { + Name = "", + Instructions = "", + Kernel = kernel, + Arguments = // Specify the service-identifier via the KernelArguments + new KernelArguments( + new OpenAIPromptExecutionSettings() + { + ServiceId = "service-2" // The target service-identifier. + }); + }; +``` +::: zone-end + +::: zone pivot="programming-language-python" +```python +# Define the Kernel +kernel = Kernel() + +# Add the AzureChatCompletion AI Service to the Kernel +kernel.add_service(AzureChatCompletion(service_id="")) + +# Create the agent +agent = ChatCompletionAgent( + service_id="agent", + kernel=kernel, + name="", + instructions="", +) +``` +::: zone-end + +::: zone pivot="programming-language-java" + +> Agents are currently unavailable in Java. + +::: zone-end + +## Conversing with _Chat Completion Agent_ + +Conversing with your _Chat Completion Agent_ is based on a _Chat History_ instance, no different from interacting with a _Chat Completion_ [AI service](../../concepts/ai-services/index.md). + +::: zone pivot="programming-language-csharp" +```csharp +// Define agent +ChatCompletionAgent agent = ...; + +// Create a ChatHistory object to maintain the conversation state. +ChatHistory chat = []; + +// Add a user message to the conversation +chat.Add(new ChatMessageContent(AuthorRole.User, "")); + +// Generate the agent response(s) +await foreach (ChatMessageContent response in agent.InvokeAsync(chat)) +{ + // Process agent response(s)... +} +``` +::: zone-end + +::: zone pivot="programming-language-python" +```python +# Define agent +agent = ChatCompletionAgent(...) + +# Define the chat history +chat = ChatHistory() + +# Add the user message +chat.add_message(ChatMessageContent(role=AuthorRole.USER, content=input)) + +# Generate the agent response(s) +async for response in agent.invoke(chat): + # process agent response(s) +``` +::: zone-end + +::: zone pivot="programming-language-java" + +> Agents are currently unavailable in Java. + +::: zone-end + + +#### How-To: + +For an end-to-end example for a _Chat Completion Agent_, see: + +- [How-To: _Chat Completion Agent_](./examples/example-chat-agent.md) + + +> [!div class="nextstepaction"] +> [Exploring _Open AI Assistant Agent_](./assistant-agent.md) diff --git a/semantic-kernel/Frameworks/agent/examples/TOC.yml b/semantic-kernel/Frameworks/agent/examples/TOC.yml new file mode 100644 index 00000000..afec67ee --- /dev/null +++ b/semantic-kernel/Frameworks/agent/examples/TOC.yml @@ -0,0 +1,8 @@ +- name: How-To use Chat Completion Agent + href: example-chat-agent.md +- name: How-To use Assistant Agent - Code Interpreter + href: example-assistant-code.md +- name: How-To use Assistant Agent - File Search + href: example-assistant-search.md +- name: How-To use Agent Group Chat + href: example-agent-collaboration.md diff --git a/semantic-kernel/Frameworks/agent/examples/example-agent-collaboration.md b/semantic-kernel/Frameworks/agent/examples/example-agent-collaboration.md new file mode 100644 index 00000000..4a088037 --- /dev/null +++ b/semantic-kernel/Frameworks/agent/examples/example-agent-collaboration.md @@ -0,0 +1,1297 @@ +--- +title: How-To: Coordinate Agent Collaboration using Agent Group Chat (Experimental) +description: A step-by-step walk-through for coordinating agent collaboration using Agent Group Chat. +zone_pivot_groups: programming-languages +author: crickman +ms.topic: tutorial +ms.author: crickman +ms.date: 09/13/2024 +ms.service: semantic-kernel +--- +# How-To: Coordinate Agent Collaboration using Agent Group Chat (Experimental) + +> [!WARNING] +> The _Semantic Kernel Agent Framework_ is experimental, still in development and is subject to change. + +## Overview + +In this sample, we will explore how to use _Agent Group Chat_ to coordinate collboration of two different agents working to review and rewrite user provided content. Each agent is assigned a distinct role: + +- **Reviewer**: Reviews and provides direction to _Writer_. +- **Writer**: Updates user content based on _Reviewer_ input. + +The approach will be broken down step-by-step to high-light the key parts of the coding process. + +## Getting Started + +Before proceeding with feature coding, make sure your development environment is fully set up and configured. + +::: zone pivot="programming-language-csharp" + +Start by creating a _Console_ project. Then, include the following package references to ensure all required dependencies are available. + +To add package dependencies from the command-line use the `dotnet` command: + +```powershell +dotnet add package Azure.Identity +dotnet add package Microsoft.Extensions.Configuration +dotnet add package Microsoft.Extensions.Configuration.Binder +dotnet add package Microsoft.Extensions.Configuration.UserSecrets +dotnet add package Microsoft.Extensions.Configuration.EnvironmentVariables +dotnet add package Microsoft.SemanticKernel.Connectors.AzureOpenAI +dotnet add package Microsoft.SemanticKernel.Agents.Core --prerelease +``` + +> If managing _NuGet_ packages in _Visual Studio_, ensure `Include prerelease` is checked. + +The project file (`.csproj`) should contain the following `PackageReference` definitions: + +```xml + + + + + + + + + +``` + +The _Agent Framework_ is experimental and requires warning suppression. This may addressed in as a property in the project file (`.csproj`): + +```xml + + $(NoWarn);CA2007;IDE1006;SKEXP0001;SKEXP0110;OPENAI001 + +``` +::: zone-end + +::: zone pivot="programming-language-python" +```python +import asyncio +import os +import copy +import pyperclip # Install via pip + +from semantic_kernel.agents import AgentGroupChat, ChatCompletionAgent +from semantic_kernel.agents.strategies.selection.kernel_function_selection_strategy import ( + KernelFunctionSelectionStrategy, +) +from semantic_kernel.agents.strategies.termination.kernel_function_termination_strategy import ( + KernelFunctionTerminationStrategy, +) +from semantic_kernel.connectors.ai.open_ai.services.azure_chat_completion import AzureChatCompletion +from semantic_kernel.contents.chat_message_content import ChatMessageContent +from semantic_kernel.contents.utils.author_role import AuthorRole +from semantic_kernel.functions.kernel_function_decorator import kernel_function +from semantic_kernel.functions.kernel_function_from_prompt import KernelFunctionFromPrompt +from semantic_kernel.kernel import Kernel +``` +::: zone-end + +::: zone pivot="programming-language-java" + +> Agents are currently unavailable in Java. + +::: zone-end + + +## Configuration + +This sample requires configuration setting in order to connect to remote services. You will need to define settings for either _Open AI_ or _Azure Open AI_. + +::: zone pivot="programming-language-csharp" + +```powershell +# Open AI +dotnet user-secrets set "OpenAISettings:ApiKey" "" +dotnet user-secrets set "OpenAISettings:ChatModel" "gpt-4o" + +# Azure Open AI +dotnet user-secrets set "AzureOpenAISettings:ApiKey" "" # Not required if using token-credential +dotnet user-secrets set "AzureOpenAISettings:Endpoint" "" +dotnet user-secrets set "AzureOpenAISettings:ChatModelDeployment" "gpt-4o" +``` + +The following class is used in all of the Agent examples. Be sure to include it in your project to ensure proper functionality. This class serves as a foundational component for the examples that follow. + +```c# +using System.Reflection; +using Microsoft.Extensions.Configuration; + +namespace AgentsSample; + +public class Settings +{ + private readonly IConfigurationRoot configRoot; + + private AzureOpenAISettings azureOpenAI; + private OpenAISettings openAI; + + public AzureOpenAISettings AzureOpenAI => this.azureOpenAI ??= this.GetSettings(); + public OpenAISettings OpenAI => this.openAI ??= this.GetSettings(); + + public class OpenAISettings + { + public string ChatModel { get; set; } = string.Empty; + public string ApiKey { get; set; } = string.Empty; + } + + public class AzureOpenAISettings + { + public string ChatModelDeployment { get; set; } = string.Empty; + public string Endpoint { get; set; } = string.Empty; + public string ApiKey { get; set; } = string.Empty; + } + + public TSettings GetSettings() => + this.configRoot.GetRequiredSection(typeof(TSettings).Name).Get()!; + + public Settings() + { + this.configRoot = + new ConfigurationBuilder() + .AddEnvironmentVariables() + .AddUserSecrets(Assembly.GetExecutingAssembly(), optional: true) + .Build(); + } +} +``` +::: zone-end + +::: zone pivot="programming-language-python" +The quickest way to get started with the proper configuration to run the sample code is to create a `.env` file at the root of your project (where your script is run). + +Configure the following settings in your `.env` file for either Azure OpenAI or OpenAI: + +```python +AZURE_OPENAI_API_KEY="..." +AZURE_OPENAI_ENDPOINT="https://..." +AZURE_OPENAI_CHAT_DEPLOYMENT_NAME="..." +AZURE_OPENAI_API_VERSION="..." + +OPENAI_API_KEY="sk-..." +OPENAI_ORG_ID="" +OPENAI_CHAT_MODEL_ID="" +``` + +Once configured, the respective AI service classes will pick up the required variables and use them during instantiation. +::: zone-end + +::: zone pivot="programming-language-java" + +> Agents are currently unavailable in Java. + +::: zone-end + + +## Coding + +The coding process for this sample involves: + +1. [Setup](#setup) - Initializing settings and the plug-in. +2. [_Agent_ Definition](#agent-definition) - Create the two _Chat Completion Agent_ instances (_Reviewer_ and _Writer_). +3. [_Chat_ Definition](#chat-definition) - Create the _Agent Group Chat_ and associated strategies. +4. [The _Chat_ Loop](#the-chat-loop) - Write the loop that drives user / agent interaction. + +The full example code is provided in the [Final](#final) section. Refer to that section for the complete implementation. + +### Setup + +Prior to creating any _Chat Completion Agent_, the configuration settings, plugins, and _Kernel_ must be initialized. + +::: zone pivot="programming-language-csharp" + +Instantiate the the `Settings` class referenced in the previous [Configuration](#configuration) section. + +```csharp +Settings settings = new(); +``` +::: zone-end + +::: zone pivot="programming-language-java" + +> Agents are currently unavailable in Java. + +::: zone-end + +::: zone pivot="programming-language-csharp" + +Now initialize a `Kernel` instance with an `IChatCompletionService`. +```csharp +IKernelBuilder builder = Kernel.CreateBuilder(); + +builder.AddAzureOpenAIChatCompletion( + settings.AzureOpenAI.ChatModelDeployment, + settings.AzureOpenAI.Endpoint, + new AzureCliCredential()); + +Kernel kernel = builder.Build(); +``` +::: zone-end + +::: zone pivot="programming-language-python" +Initialize the kernel object: + +```python +kernel = Kernel() +``` +::: zone-end + +::: zone pivot="programming-language-java" + +> Agents are currently unavailable in Java. + +::: zone-end + +Let's also create a second _Kernel_ instance via _cloning_ and add a plug-in that will allow the reivew to place updated content on the clip-board. + +::: zone pivot="programming-language-csharp" +```csharp +Kernel toolKernel = kernel.Clone(); +toolKernel.Plugins.AddFromType(); +``` +::: zone-end + +::: zone pivot="programming-language-python" +```python +tool_kernel = copy.deepcopy(kernel) +tool_kernel.add_plugin(ClipboardAccess(), plugin_name="clipboard") +``` +::: zone-end + +::: zone pivot="programming-language-java" + +> Agents are currently unavailable in Java. + +::: zone-end + +The _Clipboard_ plugin may be defined as part of the sample. + +::: zone pivot="programming-language-csharp" +```csharp +private sealed class ClipboardAccess +{ + [KernelFunction] + [Description("Copies the provided content to the clipboard.")] + public static void SetClipboard(string content) + { + if (string.IsNullOrWhiteSpace(content)) + { + return; + } + + using Process clipProcess = Process.Start( + new ProcessStartInfo + { + FileName = "clip", + RedirectStandardInput = true, + UseShellExecute = false, + }); + + clipProcess.StandardInput.Write(content); + clipProcess.StandardInput.Close(); + } +} +``` +::: zone-end + +::: zone pivot="programming-language-python" + +Note: we are leveraging a Python package called pyperclip. Please install is using pip. + +```python +class ClipboardAccess: + @kernel_function + def set_clipboard(content: str): + if not content.strip(): + return + + pyperclip.copy(content) +``` +::: zone-end + +::: zone pivot="programming-language-java" + +> Agents are currently unavailable in Java. + +::: zone-end + +### Agent Definition + +Let's declare the agent names as `const` so they might be referenced in _Agent Group Chat_ strategies: + +::: zone pivot="programming-language-csharp" +```csharp +const string ReviewerName = "Reviewer"; +const string WriterName = "Writer"; +``` +::: zone-end + +::: zone pivot="programming-language-python" +```python +REVIEWER_NAME = "Reviewer" +WRITER_NAME = "Writer" +``` +::: zone-end + +::: zone pivot="programming-language-java" + +> Agents are currently unavailable in Java. + +::: zone-end + +Defining the _Reviewer_ agent uses the pattern explored in [How-To: Chat Completion Agent](./example-chat-agent.md). + +Here the _Reviewer_ is given the role of responding to user input, providing direction to the _Writer_ agent, and verifying result of the _Writer_ agent. + +::: zone pivot="programming-language-csharp" +```csharp +ChatCompletionAgent agentReviewer = + new() + { + Name = ReviewerName, + Instructions = + """ + Your responsiblity is to review and identify how to improve user provided content. + If the user has providing input or direction for content already provided, specify how to address this input. + Never directly perform the correction or provide example. + Once the content has been updated in a subsequent response, you will review the content again until satisfactory. + Always copy satisfactory content to the clipboard using available tools and inform user. + + RULES: + - Only identify suggestions that are specific and actionable. + - Verify previous suggestions have been addressed. + - Never repeat previous suggestions. + """, + Kernel = toolKernel, + Arguments = + new KernelArguments( + new AzureOpenAIPromptExecutionSettings() + { + FunctionChoiceBehavior = FunctionChoiceBehavior.Auto() + }) + }; +``` +::: zone-end + +::: zone pivot="programming-language-python" +```python +agent_reviewer = ChatCompletionAgent( + service_id=REVIEWER_NAME, + kernel=_create_kernel_with_chat_completion(REVIEWER_NAME), + name=REVIEWER_NAME, + instructions=""" + Your responsiblity is to review and identify how to improve user provided content. + If the user has providing input or direction for content already provided, specify how to + address this input. + Never directly perform the correction or provide example. + Once the content has been updated in a subsequent response, you will review the content + again until satisfactory. + Always copy satisfactory content to the clipboard using available tools and inform user. + + RULES: + - Only identify suggestions that are specific and actionable. + - Verify previous suggestions have been addressed. + - Never repeat previous suggestions. + """, +) +``` +::: zone-end + +::: zone pivot="programming-language-java" + +> Agents are currently unavailable in Java. + +::: zone-end + +The _Writer_ agent is is similiar, but doesn't require the specification of _Execution Settings_ since it isn't configured with a plug-in. + +Here the _Writer_ is given a single-purpose task, follow direction and rewrite the content. + +::: zone pivot="programming-language-csharp" +```csharp +ChatCompletionAgent agentWriter = + new() + { + Name = WriterName, + Instructions = + """ + Your sole responsiblity is to rewrite content according to review suggestions. + + - Always apply all review direction. + - Always revise the content in its entirety without explanation. + - Never address the user. + """, + Kernel = kernel, + }; +``` +::: zone-end + +::: zone pivot="programming-language-python" +```python +agent_writer = ChatCompletionAgent( + service_id=COPYWRITER_NAME, + kernel=_create_kernel_with_chat_completion(COPYWRITER_NAME), + name=COPYWRITER_NAME, + instructions=""" + Your sole responsiblity is to rewrite content according to review suggestions. + + - Always apply all review direction. + - Always revise the content in its entirety without explanation. + - Never address the user. + """, +) +``` +::: zone-end + +::: zone pivot="programming-language-java" + +> Agents are currently unavailable in Java. + +::: zone-end + +### Chat Definition + +Defining the _Agent Group Chat_ requires considering the strategies for selecting the _Agent_ turn and determining when to exit the _Chat_ loop. For both of these considerations, we will define a _Kernel Prompt Function_. + +The first to reason over _Agent_ selection: + +::: zone pivot="programming-language-csharp" + +Using `AgentGroupChat.CreatePromptFunctionForStrategy` provides a convenient mechanism to avoid _HTML encoding_ the message paramter. + +```csharp +KernelFunction selectionFunction = + AgentGroupChat.CreatePromptFunctionForStrategy( + $$$""" + Examine the provided RESPONSE and choose the next participant. + State only the name of the chosen participant without explanation. + Never choose the participant named in the RESPONSE. + + Choose only from these participants: + - {{{ReviewerName}}} + - {{{WriterName}}} + + Always follow these rules when choosing the next participant: + - If RESPONSE is user input, it is {{{ReviewerName}}}'s turn. + - If RESPONSE is by {{{ReviewerName}}}, it is {{{WriterName}}}'s turn. + - If RESPONSE is by {{{WriterName}}}, it is {{{ReviewerName}}}'s turn. + + RESPONSE: + {{$lastmessage}} + """, + safeParameterNames: "lastmessage"); +``` +::: zone-end + +::: zone pivot="programming-language-python" +```python +selection_function = KernelFunctionFromPrompt( + function_name="selection", + prompt=f""" + Determine which participant takes the next turn in a conversation based on the the most recent participant. + State only the name of the participant to take the next turn. + No participant should take more than one turn in a row. + + Choose only from these participants: + - {REVIEWER_NAME} + - {COPYWRITER_NAME} + + Always follow these rules when selecting the next participant: + - After user input, it is {COPYWRITER_NAME}'s turn. + - After {COPYWRITER_NAME} replies, it is {REVIEWER_NAME}'s turn. + - After {REVIEWER_NAME} provides feedback, it is {COPYWRITER_NAME}'s turn. + + History: + {{{{$history}}}} + """, +) +``` +::: zone-end + +::: zone pivot="programming-language-java" + +> Agents are currently unavailable in Java. + +::: zone-end + +The second will evaluate when to exit the _Chat_ loop: + +::: zone pivot="programming-language-csharp" +```csharp +const string TerminationToken = "yes"; + +KernelFunction terminationFunction = + AgentGroupChat.CreatePromptFunctionForStrategy( + $$$""" + Examine the RESPONSE and determine whether the content has been deemed satisfactory. + If content is satisfactory, respond with a single word without explanation: {{{TerminationToken}}}. + If specific suggestions are being provided, it is not satisfactory. + If no correction is suggested, it is satisfactory. + + RESPONSE: + {{$lastmessage}} + """, + safeParameterNames: "lastmessage"); +``` +::: zone-end + +::: zone pivot="programming-language-python" +```python +TERMINATION_KEYWORD = "yes" + +termination_function = KernelFunctionFromPrompt( + function_name="termination", + prompt=f""" + Examine the RESPONSE and determine whether the content has been deemed satisfactory. + If content is satisfactory, respond with a single word without explanation: {TERMINATION_KEYWORD}. + If specific suggestions are being provided, it is not satisfactory. + If no correction is suggested, it is satisfactory. + + RESPONSE: + {{{{$history}}}} + """, +) +``` +::: zone-end + +::: zone pivot="programming-language-java" + +> Agents are currently unavailable in Java. + +::: zone-end + +Both of these _Strategies_ will only require knowledge of the most recent _Chat_ message. This will reduce token usage and help improve performance: + +::: zone pivot="programming-language-csharp" +```csharp +ChatHistoryTruncationReducer historyReducer = new(1); +``` +::: zone-end + +::: zone pivot="programming-language-python" +```python +**ChatHistoryReducer is coming soon to Python.** +``` +::: zone-end + +::: zone pivot="programming-language-java" + +> Agents are currently unavailable in Java. + +::: zone-end + +Finally we are ready to bring everything together in our _Agent Group Chat_ definition. + +::: zone pivot="programming-language-csharp" + +Creating `AgentGroupChat` involves: + +1. Include both agents in the constructor. +2. Define a `KernelFunctionSelectionStrategy` using the previously defined `KernelFunction` and `Kernel` instance. +3. Define a `KernelFunctionTerminationStrategy` using the previously defined `KernelFunction` and `Kernel` instance. + +Notice that each strategy is responsible for parsing the `KernelFunction` result. + +```csharp +AgentGroupChat chat = + new(agentReviewer, agentWriter) + { + ExecutionSettings = new AgentGroupChatSettings + { + SelectionStrategy = + new KernelFunctionSelectionStrategy(selectionFunction, kernel) + { + // Always start with the editor agent. + InitialAgent = agentReviewer, + // Save tokens by only including the final response + HistoryReducer = historyReducer, + // The prompt variable name for the history argument. + HistoryVariableName = "lastmessage", + // Returns the entire result value as a string. + ResultParser = (result) => result.GetValue() ?? agentReviewer.Name + }, + TerminationStrategy = + new KernelFunctionTerminationStrategy(terminationFunction, kernel) + { + // Only evaluate for editor's response + Agents = [agentReviewer], + // Save tokens by only including the final response + HistoryReducer = historyReducer, + // The prompt variable name for the history argument. + HistoryVariableName = "lastmessage", + // Limit total number of turns + MaximumIterations = 12, + // Customer result parser to determine if the response is "yes" + ResultParser = (result) => result.GetValue()?.Contains(TerminationToken, StringComparison.OrdinalIgnoreCase) ?? false + } + } + }; + +Console.WriteLine("Ready!"); +``` +::: zone-end + +::: zone pivot="programming-language-python" +Creating `AgentGroupChat` involves: + +1. Include both agents in the constructor. +2. Define a `KernelFunctionSelectionStrategy` using the previously defined `KernelFunction` and `Kernel` instance. +3. Define a `KernelFunctionTerminationStrategy` using the previously defined `KernelFunction` and `Kernel` instance. + +Notice that each strategy is responsible for parsing the `KernelFunction` result. +```python +chat = AgentGroupChat( + agents=[agent_writer, agent_reviewer], + selection_strategy=KernelFunctionSelectionStrategy( + function=selection_function, + kernel=_create_kernel_with_chat_completion("selection"), + result_parser=lambda result: str(result.value[0]) if result.value is not None else COPYWRITER_NAME, + agent_variable_name="agents", + history_variable_name="history", + ), + termination_strategy=KernelFunctionTerminationStrategy( + agents=[agent_reviewer], + function=termination_function, + kernel=_create_kernel_with_chat_completion("termination"), + result_parser=lambda result: TERMINATION_KEYWORD in str(result.value[0]).lower(), + history_variable_name="history", + maximum_iterations=10, + ), +) +``` +::: zone-end + +::: zone pivot="programming-language-java" + +> Agents are currently unavailable in Java. + +::: zone-end + +### The _Chat_ Loop + +At last, we are able to coordinate the interaction between the user and the _Agent Group Chat_. Start by creating creating an empty loop. + +> Note: Unlike the other examples, no external history or _thread_ is managed. _Agent Group Chat_ manages the conversation history internally. + +::: zone pivot="programming-language-csharp" +```csharp +bool isComplete = false; +do +{ + +} while (!isComplete); +``` +::: zone-end + +::: zone pivot="programming-language-python" +```python +is_complete: bool = False +while not is_complete: + # operational logic +``` +::: zone-end + +::: zone pivot="programming-language-java" + +> Agents are currently unavailable in Java. + +::: zone-end + +Now let's capture user input within the previous loop. In this case: +- Empty input will be ignored +- The term `EXIT` will signal that the conversation is completed +- The term `RESET` will clear the _Agent Group Chat_ history +- Any term starting with `@` will be treated as a file-path whose content will be provided as input +- Valid input will be added to the _Agent Group Chaty_ as a _User_ message. + + +::: zone pivot="programming-language-csharp" +```csharp +Console.WriteLine(); +Console.Write("> "); +string input = Console.ReadLine(); +if (string.IsNullOrWhiteSpace(input)) +{ + continue; +} +input = input.Trim(); +if (input.Equals("EXIT", StringComparison.OrdinalIgnoreCase)) +{ + isComplete = true; + break; +} + +if (input.Equals("RESET", StringComparison.OrdinalIgnoreCase)) +{ + await chat.ResetAsync(); + Console.WriteLine("[Converation has been reset]"); + continue; +} + +if (input.StartsWith("@", StringComparison.Ordinal) && input.Length > 1) +{ + string filePath = input.Substring(1); + try + { + if (!File.Exists(filePath)) + { + Console.WriteLine($"Unable to access file: {filePath}"); + continue; + } + input = File.ReadAllText(filePath); + } + catch (Exception) + { + Console.WriteLine($"Unable to access file: {filePath}"); + continue; + } +} + +chat.AddChatMessage(new ChatMessageContent(AuthorRole.User, input)); +``` +::: zone-end + +::: zone pivot="programming-language-python" +```python +user_input = input("User:> ") +if not user_input: + continue + +if user_input.lower() == "exit": + is_complete = True + break + +if user_input.lower() == "reset": + await chat.reset() + print("[Conversation has been reset]") + continue + +if user_input.startswith("@") and len(input) > 1: + file_path = input[1:] + try: + if not os.path.exists(file_path): + print(f"Unable to access file: {file_path}") + continue + with open(file_path) as file: + user_input = file.read() + except Exception: + print(f"Unable to access file: {file_path}") + continue + +await chat.add_chat_message(ChatMessageContent(role=AuthorRole.USER, content=user_input)) +``` +::: zone-end + +::: zone pivot="programming-language-java" + +> Agents are currently unavailable in Java. + +::: zone-end + +To initate the _Agent_ collaboration in response to user input and display the _Agent_ responses, invoke the _Agent Group Chat_; however, first be sure to reset the _Completion_ state from any prior invocation. + +> Note: Service failures are being caught and displayed to avoid crashing the conversation loop. + +::: zone pivot="programming-language-csharp" +```csharp +chat.IsComplete = false; + +try +{ + await foreach (ChatMessageContent response in chat.InvokeAsync()) + { + Console.WriteLine(); + Console.WriteLine($"{response.AuthorName.ToUpperInvariant()}:{Environment.NewLine}{response.Content}"); + } +} +catch (HttpOperationException exception) +{ + Console.WriteLine(exception.Message); + if (exception.InnerException != null) + { + Console.WriteLine(exception.InnerException.Message); + if (exception.InnerException.Data.Count > 0) + { + Console.WriteLine(JsonSerializer.Serialize(exception.InnerException.Data, new JsonSerializerOptions() { WriteIndented = true })); + } + } +} +``` +::: zone-end + +::: zone pivot="programming-language-python" +```python +chat.is_complete = False +async for response in chat.invoke(): + print(f"# {response.role} - {response.name or '*'}: '{response.content}'") + +if chat.is_complete: + is_complete = True + break +``` +::: zone-end + +::: zone pivot="programming-language-java" + +> Agents are currently unavailable in Java. + +::: zone-end + + +## Final + +Bringing all the steps together, we have the final code for this example. The complete implementation is provided below. + +Try using these suggested inputs: + +1. Hi +2. {"message: "hello world"} +3. {"message": "hello world"} +4. Semantic Kernel (SK) is an open-source SDK that enables developers to build and orchestrate complex AI workflows that involve natural language processing (NLP) and machine learning models. It provies a flexible platform for integrating AI capabilities such as semantic search, text summarization, and dialogue systems into applications. With SK, you can easily combine different AI services and models, define thei relationships, and orchestrate interactions between them. +5. make this two paragraphs +6. thank you +7. @.\WomensSuffrage.txt +8. its good, but is it ready for my college professor? + + +::: zone pivot="programming-language-csharp" +```csharp +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.ComponentModel; +using System.Diagnostics; +using System.IO; +using System.Text.Json; +using System.Threading.Tasks; +using Azure.Identity; +using Microsoft.SemanticKernel; +using Microsoft.SemanticKernel.Agents; +using Microsoft.SemanticKernel.Agents.Chat; +using Microsoft.SemanticKernel.Agents.History; +using Microsoft.SemanticKernel.ChatCompletion; +using Microsoft.SemanticKernel.Connectors.AzureOpenAI; + +namespace AgentsSample; + +public static class Program +{ + public static async Task Main() + { + // Load configuration from environment variables or user secrets. + Settings settings = new(); + + Console.WriteLine("Creating kernel..."); + IKernelBuilder builder = Kernel.CreateBuilder(); + + builder.AddAzureOpenAIChatCompletion( + settings.AzureOpenAI.ChatModelDeployment, + settings.AzureOpenAI.Endpoint, + new AzureCliCredential()); + + Kernel kernel = builder.Build(); + + Kernel toolKernel = kernel.Clone(); + toolKernel.Plugins.AddFromType(); + + + Console.WriteLine("Defining agents..."); + + const string ReviewerName = "Reviewer"; + const string WriterName = "Writer"; + + ChatCompletionAgent agentReviewer = + new() + { + Name = ReviewerName, + Instructions = + """ + Your responsiblity is to review and identify how to improve user provided content. + If the user has providing input or direction for content already provided, specify how to address this input. + Never directly perform the correction or provide example. + Once the content has been updated in a subsequent response, you will review the content again until satisfactory. + Always copy satisfactory content to the clipboard using available tools and inform user. + + RULES: + - Only identify suggestions that are specific and actionable. + - Verify previous suggestions have been addressed. + - Never repeat previous suggestions. + """, + Kernel = toolKernel, + Arguments = new KernelArguments(new AzureOpenAIPromptExecutionSettings() { FunctionChoiceBehavior = FunctionChoiceBehavior.Auto() }) + }; + + ChatCompletionAgent agentWriter = + new() + { + Name = WriterName, + Instructions = + """ + Your sole responsiblity is to rewrite content according to review suggestions. + + - Always apply all review direction. + - Always revise the content in its entirety without explanation. + - Never address the user. + """, + Kernel = kernel, + }; + + KernelFunction selectionFunction = + AgentGroupChat.CreatePromptFunctionForStrategy( + $$$""" + Examine the provided RESPONSE and choose the next participant. + State only the name of the chosen participant without explanation. + Never choose the participant named in the RESPONSE. + + Choose only from these participants: + - {{{ReviewerName}}} + - {{{WriterName}}} + + Always follow these rules when choosing the next participant: + - If RESPONSE is user input, it is {{{ReviewerName}}}'s turn. + - If RESPONSE is by {{{ReviewerName}}}, it is {{{WriterName}}}'s turn. + - If RESPONSE is by {{{WriterName}}}, it is {{{ReviewerName}}}'s turn. + + RESPONSE: + {{$lastmessage}} + """, + safeParameterNames: "lastmessage"); + + const string TerminationToken = "yes"; + + KernelFunction terminationFunction = + AgentGroupChat.CreatePromptFunctionForStrategy( + $$$""" + Examine the RESPONSE and determine whether the content has been deemed satisfactory. + If content is satisfactory, respond with a single word without explanation: {{{TerminationToken}}}. + If specific suggestions are being provided, it is not satisfactory. + If no correction is suggested, it is satisfactory. + + RESPONSE: + {{$lastmessage}} + """, + safeParameterNames: "lastmessage"); + + ChatHistoryTruncationReducer historyReducer = new(1); + + AgentGroupChat chat = + new(agentReviewer, agentWriter) + { + ExecutionSettings = new AgentGroupChatSettings + { + SelectionStrategy = + new KernelFunctionSelectionStrategy(selectionFunction, kernel) + { + // Always start with the editor agent. + InitialAgent = agentReviewer, + // Save tokens by only including the final response + HistoryReducer = historyReducer, + // The prompt variable name for the history argument. + HistoryVariableName = "lastmessage", + // Returns the entire result value as a string. + ResultParser = (result) => result.GetValue() ?? agentReviewer.Name + }, + TerminationStrategy = + new KernelFunctionTerminationStrategy(terminationFunction, kernel) + { + // Only evaluate for editor's response + Agents = [agentReviewer], + // Save tokens by only including the final response + HistoryReducer = historyReducer, + // The prompt variable name for the history argument. + HistoryVariableName = "lastmessage", + // Limit total number of turns + MaximumIterations = 12, + // Customer result parser to determine if the response is "yes" + ResultParser = (result) => result.GetValue()?.Contains(TerminationToken, StringComparison.OrdinalIgnoreCase) ?? false + } + } + }; + + Console.WriteLine("Ready!"); + + bool isComplete = false; + do + { + Console.WriteLine(); + Console.Write("> "); + string input = Console.ReadLine(); + if (string.IsNullOrWhiteSpace(input)) + { + continue; + } + input = input.Trim(); + if (input.Equals("EXIT", StringComparison.OrdinalIgnoreCase)) + { + isComplete = true; + break; + } + + if (input.Equals("RESET", StringComparison.OrdinalIgnoreCase)) + { + await chat.ResetAsync(); + Console.WriteLine("[Converation has been reset]"); + continue; + } + + if (input.StartsWith("@", StringComparison.Ordinal) && input.Length > 1) + { + string filePath = input.Substring(1); + try + { + if (!File.Exists(filePath)) + { + Console.WriteLine($"Unable to access file: {filePath}"); + continue; + } + input = File.ReadAllText(filePath); + } + catch (Exception) + { + Console.WriteLine($"Unable to access file: {filePath}"); + continue; + } + } + + chat.AddChatMessage(new ChatMessageContent(AuthorRole.User, input)); + + chat.IsComplete = false; + + try + { + await foreach (ChatMessageContent response in chat.InvokeAsync()) + { + Console.WriteLine(); + Console.WriteLine($"{response.AuthorName.ToUpperInvariant()}:{Environment.NewLine}{response.Content}"); + } + } + catch (HttpOperationException exception) + { + Console.WriteLine(exception.Message); + if (exception.InnerException != null) + { + Console.WriteLine(exception.InnerException.Message); + if (exception.InnerException.Data.Count > 0) + { + Console.WriteLine(JsonSerializer.Serialize(exception.InnerException.Data, new JsonSerializerOptions() { WriteIndented = true })); + } + } + } + } while (!isComplete); + } + + private sealed class ClipboardAccess + { + [KernelFunction] + [Description("Copies the provided content to the clipboard.")] + public static void SetClipboard(string content) + { + if (string.IsNullOrWhiteSpace(content)) + { + return; + } + + using Process clipProcess = Process.Start( + new ProcessStartInfo + { + FileName = "clip", + RedirectStandardInput = true, + UseShellExecute = false, + }); + + clipProcess.StandardInput.Write(content); + clipProcess.StandardInput.Close(); + } + } +} + +``` +::: zone-end + +::: zone pivot="programming-language-python" +```python +# Copyright (c) Microsoft. All rights reserved. + +import asyncio +import os + +from semantic_kernel.agents import AgentGroupChat, ChatCompletionAgent +from semantic_kernel.agents.strategies.selection.kernel_function_selection_strategy import ( + KernelFunctionSelectionStrategy, +) +from semantic_kernel.agents.strategies.termination.kernel_function_termination_strategy import ( + KernelFunctionTerminationStrategy, +) +from semantic_kernel.connectors.ai.open_ai.services.azure_chat_completion import AzureChatCompletion +from semantic_kernel.contents.chat_message_content import ChatMessageContent +from semantic_kernel.contents.utils.author_role import AuthorRole +from semantic_kernel.functions.kernel_function_decorator import kernel_function +from semantic_kernel.functions.kernel_function_from_prompt import KernelFunctionFromPrompt +from semantic_kernel.kernel import Kernel + +################################################################### +# The following sample demonstrates how to create a simple, # +# agent group chat that utilizes a Reviewer Chat Completion # +# Agent along with a Writer Chat Completion Agent to # +# complete a user's task. # +################################################################### + + +class ClipboardAccess: + @kernel_function + def set_clipboard(content: str): + if not content.strip(): + return + + pyperclip.copy(content) + + +REVIEWER_NAME = "Reviewer" +COPYWRITER_NAME = "Writer" + + +def _create_kernel_with_chat_completion(service_id: str) -> Kernel: + kernel = Kernel() + kernel.add_service(AzureChatCompletion(service_id=service_id)) + return kernel + + +async def main(): + agent_reviewer = ChatCompletionAgent( + service_id=REVIEWER_NAME, + kernel=_create_kernel_with_chat_completion(REVIEWER_NAME), + name=REVIEWER_NAME, + instructions=""" + Your responsiblity is to review and identify how to improve user provided content. + If the user has providing input or direction for content already provided, specify how to + address this input. + Never directly perform the correction or provide example. + Once the content has been updated in a subsequent response, you will review the content + again until satisfactory. + Always copy satisfactory content to the clipboard using available tools and inform user. + + RULES: + - Only identify suggestions that are specific and actionable. + - Verify previous suggestions have been addressed. + - Never repeat previous suggestions. + """, + ) + + agent_writer = ChatCompletionAgent( + service_id=COPYWRITER_NAME, + kernel=_create_kernel_with_chat_completion(COPYWRITER_NAME), + name=COPYWRITER_NAME, + instructions=""" + Your sole responsiblity is to rewrite content according to review suggestions. + + - Always apply all review direction. + - Always revise the content in its entirety without explanation. + - Never address the user. + """, + ) + + selection_function = KernelFunctionFromPrompt( + function_name="selection", + prompt=f""" + Determine which participant takes the next turn in a conversation based on the the most recent participant. + State only the name of the participant to take the next turn. + No participant should take more than one turn in a row. + + Choose only from these participants: + - {REVIEWER_NAME} + - {COPYWRITER_NAME} + + Always follow these rules when selecting the next participant: + - After user input, it is {COPYWRITER_NAME}'s turn. + - After {COPYWRITER_NAME} replies, it is {REVIEWER_NAME}'s turn. + - After {REVIEWER_NAME} provides feedback, it is {COPYWRITER_NAME}'s turn. + + History: + {{{{$history}}}} + """, + ) + + TERMINATION_KEYWORD = "yes" + + termination_function = KernelFunctionFromPrompt( + function_name="termination", + prompt=f""" + Examine the RESPONSE and determine whether the content has been deemed satisfactory. + If content is satisfactory, respond with a single word without explanation: {TERMINATION_KEYWORD}. + If specific suggestions are being provided, it is not satisfactory. + If no correction is suggested, it is satisfactory. + + RESPONSE: + {{{{$history}}}} + """, + ) + + chat = AgentGroupChat( + agents=[agent_writer, agent_reviewer], + selection_strategy=KernelFunctionSelectionStrategy( + function=selection_function, + kernel=_create_kernel_with_chat_completion("selection"), + result_parser=lambda result: str(result.value[0]) if result.value is not None else COPYWRITER_NAME, + agent_variable_name="agents", + history_variable_name="history", + ), + termination_strategy=KernelFunctionTerminationStrategy( + agents=[agent_reviewer], + function=termination_function, + kernel=_create_kernel_with_chat_completion("termination"), + result_parser=lambda result: TERMINATION_KEYWORD in str(result.value[0]).lower(), + history_variable_name="history", + maximum_iterations=10, + ), + ) + + is_complete: bool = False + while not is_complete: + user_input = input("User:> ") + if not user_input: + continue + + if user_input.lower() == "exit": + is_complete = True + break + + if user_input.lower() == "reset": + await chat.reset() + print("[Conversation has been reset]") + continue + + if user_input.startswith("@") and len(input) > 1: + file_path = input[1:] + try: + if not os.path.exists(file_path): + print(f"Unable to access file: {file_path}") + continue + with open(file_path) as file: + user_input = file.read() + except Exception: + print(f"Unable to access file: {file_path}") + continue + + await chat.add_chat_message(ChatMessageContent(role=AuthorRole.USER, content=user_input)) + + async for response in chat.invoke(): + print(f"# {response.role} - {response.name or '*'}: '{response.content}'") + + if chat.is_complete: + is_complete = True + break + + +if __name__ == "__main__": + asyncio.run(main()) +``` +::: zone-end + +::: zone pivot="programming-language-java" + +> Agents are currently unavailable in Java. + +::: zone-end \ No newline at end of file diff --git a/semantic-kernel/Frameworks/agent/examples/example-assistant-code.md b/semantic-kernel/Frameworks/agent/examples/example-assistant-code.md new file mode 100644 index 00000000..92ff9fe5 --- /dev/null +++ b/semantic-kernel/Frameworks/agent/examples/example-assistant-code.md @@ -0,0 +1,848 @@ +--- +title: How-To: _Open AI Assistant Agent_ Code Interpreter (Experimental) +description: A step-by-step walk-through of defining and utilizing the code-interpreter tool of an Open AI Assistant Agent. +zone_pivot_groups: programming-languages +author: crickman +ms.topic: tutorial +ms.author: crickman +ms.date: 09/13/2024 +ms.service: semantic-kernel +--- +# How-To: _Open AI Assistant Agent_ Code Interpreter (Experimental) + +> [!WARNING] +> The _Semantic Kernel Agent Framework_ is experimental, still in development and is subject to change. + +## Overview + +In this sample, we will explore how to use the _code-interpreter_ tool of an [_Open AI Assistant Agent_](../assistant-agent.md) to complete data-analysis tasks. The approach will be broken down step-by-step to high-light the key parts of the coding process. As part of the task, the agent will generate both image and text responses. This will demonstrate the versatility of this tool in performing quantitative analysis. + +Streaming will be used to deliver the agent's responses. This will provide real-time updates as the task progresses. + + +## Getting Started + +Before proceeding with feature coding, make sure your development environment is fully set up and configured. + +::: zone pivot="programming-language-csharp" + +Start by creating a _Console_ project. Then, include the following package references to ensure all required dependencies are available. + +To add package dependencies from the command-line use the `dotnet` command: + +```powershell +dotnet add package Azure.Identity +dotnet add package Microsoft.Extensions.Configuration +dotnet add package Microsoft.Extensions.Configuration.Binder +dotnet add package Microsoft.Extensions.Configuration.UserSecrets +dotnet add package Microsoft.Extensions.Configuration.EnvironmentVariables +dotnet add package Microsoft.SemanticKernel +dotnet add package Microsoft.SemanticKernel.Agents.OpenAI --prerelease +``` + +> If managing _NuGet_ packages in _Visual Studio_, ensure `Include prerelease` is checked. + +The project file (`.csproj`) should contain the following `PackageReference` definitions: + +```xml + + + + + + + + + +``` + +The _Agent Framework_ is experimental and requires warning suppression. This may addressed in as a property in the project file (`.csproj`): + +```xml + + $(NoWarn);CA2007;IDE1006;SKEXP0001;SKEXP0110;OPENAI001 + +``` + +Additionally, copy the `PopulationByAdmin1.csv` and `PopulationByCountry.csv` data files from [_Semantic Kernel_ `LearnResources` Project](https://github.com/microsoft/semantic-kernel/tree/main/dotnet/samples/LearnResources/Resources). Add these files in your project folder and configure to have them copied to the output directory: + +```xml + + + Always + + + Always + + +``` + +::: zone-end +::: zone pivot="programming-language-python" +Start by creating a folder that will hold your script (`.py` file) and the sample resources. Include the following imports at the top of your `.py` file: + +```python +import asyncio +import os + +from semantic_kernel.agents.open_ai.azure_assistant_agent import AzureAssistantAgent +from semantic_kernel.contents.chat_message_content import ChatMessageContent +from semantic_kernel.contents.streaming_file_reference_content import StreamingFileReferenceContent +from semantic_kernel.contents.utils.author_role import AuthorRole +from semantic_kernel.kernel import Kernel +``` + +Additionally, copy the `PopulationByAdmin1.csv` and `PopulationByCountry.csv` data files from [_Semantic Kernel_ `LearnResources` Project](https://github.com/microsoft/semantic-kernel/tree/main/python/samples/learn_resources/resources). Add these files in your project folder. + +::: zone-end +::: zone pivot="programming-language-java" + +> Agents are currently unavailable in Java. + +::: zone-end + + +## Configuration + +This sample requires configuration setting in order to connect to remote services. You will need to define settings for either _Open AI_ or _Azure Open AI_. + +::: zone pivot="programming-language-csharp" + +```powershell +# Open AI +dotnet user-secrets set "OpenAISettings:ApiKey" "" +dotnet user-secrets set "OpenAISettings:ChatModel" "gpt-4o" + +# Azure Open AI +dotnet user-secrets set "AzureOpenAISettings:ApiKey" "" # Not required if using token-credential +dotnet user-secrets set "AzureOpenAISettings:Endpoint" "" +dotnet user-secrets set "AzureOpenAISettings:ChatModelDeployment" "gpt-4o" +``` + +The following class is used in all of the Agent examples. Be sure to include it in your project to ensure proper functionality. This class serves as a foundational component for the examples that follow. + +```c# +using System.Reflection; +using Microsoft.Extensions.Configuration; + +namespace AgentsSample; + +public class Settings +{ + private readonly IConfigurationRoot configRoot; + + private AzureOpenAISettings azureOpenAI; + private OpenAISettings openAI; + + public AzureOpenAISettings AzureOpenAI => this.azureOpenAI ??= this.GetSettings(); + public OpenAISettings OpenAI => this.openAI ??= this.GetSettings(); + + public class OpenAISettings + { + public string ChatModel { get; set; } = string.Empty; + public string ApiKey { get; set; } = string.Empty; + } + + public class AzureOpenAISettings + { + public string ChatModelDeployment { get; set; } = string.Empty; + public string Endpoint { get; set; } = string.Empty; + public string ApiKey { get; set; } = string.Empty; + } + + public TSettings GetSettings() => + this.configRoot.GetRequiredSection(typeof(TSettings).Name).Get()!; + + public Settings() + { + this.configRoot = + new ConfigurationBuilder() + .AddEnvironmentVariables() + .AddUserSecrets(Assembly.GetExecutingAssembly(), optional: true) + .Build(); + } +} +``` +::: zone-end +::: zone pivot="programming-language-python" + +The quickest way to get started with the proper configuration to run the sample code is to create a `.env` file at the root of your project (where your script is run). + +Configure the following settings in your `.env` file for either Azure OpenAI or OpenAI: + +```python +AZURE_OPENAI_API_KEY="..." +AZURE_OPENAI_ENDPOINT="https://..." +AZURE_OPENAI_CHAT_DEPLOYMENT_NAME="..." +AZURE_OPENAI_API_VERSION="..." + +OPENAI_API_KEY="sk-..." +OPENAI_ORG_ID="" +OPENAI_CHAT_MODEL_ID="" +``` + +Once configured, the respective AI service classes will pick up the required variables and use them during instantiation. +::: zone-end + +::: zone pivot="programming-language-java" + +> Agents are currently unavailable in Java. + +::: zone-end + +## Coding + +The coding process for this sample involves: + +1. [Setup](#setup) - Initializing settings and the plug-in. +2. [Agent Definition](#agent-definition) - Create the _OpenAI_Assistant_Agent_ with templatized instructions and plug-in. +3. [The _Chat_ Loop](#the-chat-loop) - Write the loop that drives user / agent interaction. + +The full example code is provided in the [Final](#final) section. Refer to that section for the complete implementation. + +### Setup + +Prior to creating an _Open AI Assistant Agent_, ensure the configuration settings are available and prepare the file resources. + +::: zone pivot="programming-language-csharp" + +Instantiate the `Settings` class referenced in the previous [Configuration](#configuration) section. Use the settings to also create an `OpenAIClientProvider` that will be used for the [Agent Definition](#agent-definition) as well as file-upload. + +```csharp +Settings settings = new(); + +OpenAIClientProvider clientProvider = + OpenAIClientProvider.ForAzureOpenAI(new AzureCliCredential(), new Uri(settings.AzureOpenAI.Endpoint)); +``` +::: zone-end + +::: zone pivot="programming-language-java" + +> Agents are currently unavailable in Java. + +::: zone-end + +::: zone pivot="programming-language-csharp" + +Use the `OpenAIClientProvider` to access an `OpenAIFileClient` and upload the two data-files described in the previous [Configuration](#configuration) section, preserving the _File Reference_ for final clean-up. + +```csharp +Console.WriteLine("Uploading files..."); +OpenAIFileClient fileClient = clientProvider.Client.GetOpenAIFileClient(); +OpenAIFile fileDataCountryDetail = await fileClient.UploadFileAsync("PopulationByAdmin1.csv", FileUploadPurpose.Assistants); +OpenAIFile fileDataCountryList = await fileClient.UploadFileAsync("PopulationByCountry.csv", FileUploadPurpose.Assistants); +``` +::: zone-end + +::: zone pivot="programming-language-python" +```python +# Let's form the file paths that we will later pass to the assistant +csv_file_path_1 = os.path.join( + os.path.dirname(os.path.dirname(os.path.realpath(__file__))), + "PopulationByAdmin1.csv", +) + +csv_file_path_2 = os.path.join( + os.path.dirname(os.path.dirname(os.path.realpath(__file__))), + "PopulationByCountry.csv", +) +``` +::: zone-end + +::: zone pivot="programming-language-java" + +> Agents are currently unavailable in Java. + +::: zone-end + +### Agent Definition + +We are now ready to instantiate an _OpenAI Assistant Agent_. The agent is configured with its target model, _Instructions_, and the _Code Interpreter_ tool enabled. Additionally, we explicitly associate the two data files with the _Code Interpreter_ tool. + +::: zone pivot="programming-language-csharp" +```csharp +Console.WriteLine("Defining agent..."); +OpenAIAssistantAgent agent = + await OpenAIAssistantAgent.CreateAsync( + clientProvider, + new OpenAIAssistantDefinition(settings.AzureOpenAI.ChatModelDeployment) + { + Name = "SampleAssistantAgent", + Instructions = + """ + Analyze the available data to provide an answer to the user's question. + Always format response using markdown. + Always include a numerical index that starts at 1 for any lists or tables. + Always sort lists in ascending order. + """, + EnableCodeInterpreter = true, + CodeInterpreterFileIds = [fileDataCountryList.Id, fileDataCountryDetail.Id], + }, + new Kernel()); +``` +::: zone-end + +::: zone pivot="programming-language-python" +```python +agent = await AzureAssistantAgent.create( + kernel=Kernel(), + service_id="agent", + name="SampleAssistantAgent", + instructions=""" + Analyze the available data to provide an answer to the user's question. + Always format response using markdown. + Always include a numerical index that starts at 1 for any lists or tables. + Always sort lists in ascending order. + """, + enable_code_interpreter=True, + code_interpreter_filenames=[csv_file_path_1, csv_file_path_2], + ) +``` +::: zone-end + +::: zone pivot="programming-language-java" + +> Agents are currently unavailable in Java. + +::: zone-end + +### The _Chat_ Loop + +At last, we are able to coordinate the interaction between the user and the _Agent_. Start by creating an _Assistant Thread_ to maintain the conversation state and creating an empty loop. + +Let's also ensure the resources are removed at the end of execution to minimize unnecessary charges. + +::: zone pivot="programming-language-csharp" +```csharp +Console.WriteLine("Creating thread..."); +string threadId = await agent.CreateThreadAsync(); + +Console.WriteLine("Ready!"); + +try +{ + bool isComplete = false; + List fileIds = []; + do + { + + } while (!isComplete); +} +finally +{ + Console.WriteLine(); + Console.WriteLine("Cleaning-up..."); + await Task.WhenAll( + [ + agent.DeleteThreadAsync(threadId), + agent.DeleteAsync(), + fileClient.DeleteFileAsync(fileDataCountryList.Id), + fileClient.DeleteFileAsync(fileDataCountryDetail.Id), + ]); +} +``` +::: zone-end + +::: zone pivot="programming-language-python" +```python +print("Creating thread...") +thread_id = await agent.create_thread() + +try: + is_complete: bool = False + file_ids: list[str] = [] + while not is_complete: + # agent interaction logic here +finally: + print("Cleaning up resources...") + if agent is not None: + [await agent.delete_file(file_id) for file_id in agent.code_interpreter_file_ids] + await agent.delete_thread(thread_id) + await agent.delete() +``` +::: zone-end + +::: zone pivot="programming-language-java" + +> Agents are currently unavailable in Java. + +::: zone-end + +Now let's capture user input within the previous loop. In this case, empty input will be ignored and the term `EXIT` will signal that the conversation is completed. Valid input will be added to the _Assistant Thread_ as a _User_ message. + +::: zone pivot="programming-language-csharp" +```csharp +Console.WriteLine(); +Console.Write("> "); +string input = Console.ReadLine(); +if (string.IsNullOrWhiteSpace(input)) +{ + continue; +} +if (input.Trim().Equals("EXIT", StringComparison.OrdinalIgnoreCase)) +{ + isComplete = true; + break; +} + +await agent.AddChatMessageAsync(threadId, new ChatMessageContent(AuthorRole.User, input)); + +Console.WriteLine(); +``` +::: zone-end + +::: zone pivot="programming-language-python" +```python +user_input = input("User:> ") +if not user_input: + continue + +if user_input.lower() == "exit": + is_complete = True + break + +await agent.add_chat_message(thread_id=thread_id, message=ChatMessageContent(role=AuthorRole.USER, content=user_input)) +``` +::: zone-end + +::: zone pivot="programming-language-java" + +> Agents are currently unavailable in Java. + +::: zone-end + +Before invoking the _Agent_ response, let's add some helper methods to download any files that may be produced by the _Agent_. + +::: zone pivot="programming-language-csharp" +Here we're place file content in the system defined temporary directory and then launching the system defined viewer application. + +```csharp +private static async Task DownloadResponseImageAsync(OpenAIFileClient client, ICollection fileIds) +{ + if (fileIds.Count > 0) + { + Console.WriteLine(); + foreach (string fileId in fileIds) + { + await DownloadFileContentAsync(client, fileId, launchViewer: true); + } + } +} + +private static async Task DownloadFileContentAsync(OpenAIFileClient client, string fileId, bool launchViewer = false) +{ + OpenAIFile fileInfo = client.GetFile(fileId); + if (fileInfo.Purpose == FilePurpose.AssistantsOutput) + { + string filePath = + Path.Combine( + Path.GetTempPath(), + Path.GetFileName(Path.ChangeExtension(fileInfo.Filename, ".png"))); + + BinaryData content = await client.DownloadFileAsync(fileId); + await using FileStream fileStream = new(filePath, FileMode.CreateNew); + await content.ToStream().CopyToAsync(fileStream); + Console.WriteLine($"File saved to: {filePath}."); + + if (launchViewer) + { + Process.Start( + new ProcessStartInfo + { + FileName = "cmd.exe", + Arguments = $"/C start {filePath}" + }); + } + } +} +``` +::: zone-end + +::: zone pivot="programming-language-python" +```python +import os + +async def download_file_content(agent, file_id: str): + try: + # Fetch the content of the file using the provided method + response_content = await agent.client.files.content(file_id) + + # Get the current working directory of the file + current_directory = os.path.dirname(os.path.abspath(__file__)) + + # Define the path to save the image in the current directory + file_path = os.path.join( + current_directory, # Use the current directory of the file + f"{file_id}.png" # You can modify this to use the actual filename with proper extension + ) + + # Save content to a file asynchronously + with open(file_path, "wb") as file: + file.write(response_content.content) + + print(f"File saved to: {file_path}") + except Exception as e: + print(f"An error occurred while downloading file {file_id}: {str(e)}") + +async def download_response_image(agent, file_ids: list[str]): + if file_ids: + # Iterate over file_ids and download each one + for file_id in file_ids: + await download_file_content(agent, file_id) +``` +::: zone-end + +::: zone pivot="programming-language-java" + +> Agents are currently unavailable in Java. + +::: zone-end + +To generate an _Agent_ response to user input, invoke the agent by specifying the _Assistant Thread_. In this example, we choose a streamed response and capture any generated _File References_ for download and review at the end of the response cycle. It's important to note that generated code is identified by the presence of a _Metadata_ key in the response message, distinguishing it from the conversational reply. + +::: zone pivot="programming-language-csharp" +```csharp +bool isCode = false; +await foreach (StreamingChatMessageContent response in agent.InvokeStreamingAsync(threadId)) +{ + if (isCode != (response.Metadata?.ContainsKey(OpenAIAssistantAgent.CodeInterpreterMetadataKey) ?? false)) + { + Console.WriteLine(); + isCode = !isCode; + } + + // Display response. + Console.Write($"{response.Content}"); + + // Capture file IDs for downloading. + fileIds.AddRange(response.Items.OfType().Select(item => item.FileId)); +} +Console.WriteLine(); + +// Download any files referenced in the response. +await DownloadResponseImageAsync(fileClient, fileIds); +fileIds.Clear(); +``` +::: zone-end + +::: zone pivot="programming-language-python" +```python +is_code: bool = False +async for response in agent.invoke(stream(thread_id=thread_id): + if is_code != metadata.get("code"): + print() + is_code = not is_code + + print(f"{response.content}) + + file_ids.extend( + [item.file_id for item in response.items if isinstance(item, StreamingFileReferenceContent)] + ) + +print() + +await download_response_image(agent, file_ids) +file_ids.clear() +``` +::: zone-end + +::: zone pivot="programming-language-java" + +> Agents are currently unavailable in Java. + +::: zone-end + + +## Final + +Bringing all the steps together, we have the final code for this example. The complete implementation is provided below. + +Try using these suggested inputs: + +1. Compare the files to determine the number of countries do not have a state or province defined compared to the total count +2. Create a table for countries with state or province defined. Include the count of states or provinces and the total population +3. Provide a bar chart for countries whose names start with the same letter and sort the x axis by highest count to lowest (include all countries) + +::: zone pivot="programming-language-csharp" +```csharp +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using Azure.Identity; +using Microsoft.SemanticKernel; +using Microsoft.SemanticKernel.Agents.OpenAI; +using Microsoft.SemanticKernel.ChatCompletion; +using OpenAI.Files; + +namespace AgentsSample; + +public static class Program +{ + public static async Task Main() + { + // Load configuration from environment variables or user secrets. + Settings settings = new(); + + OpenAIClientProvider clientProvider = + OpenAIClientProvider.ForAzureOpenAI(new AzureCliCredential(), new Uri(settings.AzureOpenAI.Endpoint)); + + Console.WriteLine("Uploading files..."); + OpenAIFileClient fileClient = clientProvider.Client.GetOpenAIFileClient(); + OpenAIFile fileDataCountryDetail = await fileClient.UploadFileAsync("PopulationByAdmin1.csv", FileUploadPurpose.Assistants); + OpenAIFile fileDataCountryList = await fileClient.UploadFileAsync("PopulationByCountry.csv", FileUploadPurpose.Assistants); + + Console.WriteLine("Defining agent..."); + OpenAIAssistantAgent agent = + await OpenAIAssistantAgent.CreateAsync( + clientProvider, + new OpenAIAssistantDefinition(settings.AzureOpenAI.ChatModelDeployment) + { + Name = "SampleAssistantAgent", + Instructions = + """ + Analyze the available data to provide an answer to the user's question. + Always format response using markdown. + Always include a numerical index that starts at 1 for any lists or tables. + Always sort lists in ascending order. + """, + EnableCodeInterpreter = true, + CodeInterpreterFileIds = [fileDataCountryList.Id, fileDataCountryDetail.Id], + }, + new Kernel()); + + Console.WriteLine("Creating thread..."); + string threadId = await agent.CreateThreadAsync(); + + Console.WriteLine("Ready!"); + + try + { + bool isComplete = false; + List fileIds = []; + do + { + Console.WriteLine(); + Console.Write("> "); + string input = Console.ReadLine(); + if (string.IsNullOrWhiteSpace(input)) + { + continue; + } + if (input.Trim().Equals("EXIT", StringComparison.OrdinalIgnoreCase)) + { + isComplete = true; + break; + } + + await agent.AddChatMessageAsync(threadId, new ChatMessageContent(AuthorRole.User, input)); + + Console.WriteLine(); + + bool isCode = false; + await foreach (StreamingChatMessageContent response in agent.InvokeStreamingAsync(threadId)) + { + if (isCode != (response.Metadata?.ContainsKey(OpenAIAssistantAgent.CodeInterpreterMetadataKey) ?? false)) + { + Console.WriteLine(); + isCode = !isCode; + } + + // Display response. + Console.Write($"{response.Content}"); + + // Capture file IDs for downloading. + fileIds.AddRange(response.Items.OfType().Select(item => item.FileId)); + } + Console.WriteLine(); + + // Download any files referenced in the response. + await DownloadResponseImageAsync(fileClient, fileIds); + fileIds.Clear(); + + } while (!isComplete); + } + finally + { + Console.WriteLine(); + Console.WriteLine("Cleaning-up..."); + await Task.WhenAll( + [ + agent.DeleteThreadAsync(threadId), + agent.DeleteAsync(), + fileClient.DeleteFileAsync(fileDataCountryList.Id), + fileClient.DeleteFileAsync(fileDataCountryDetail.Id), + ]); + } + } + + private static async Task DownloadResponseImageAsync(OpenAIFileClient client, ICollection fileIds) + { + if (fileIds.Count > 0) + { + Console.WriteLine(); + foreach (string fileId in fileIds) + { + await DownloadFileContentAsync(client, fileId, launchViewer: true); + } + } + } + + private static async Task DownloadFileContentAsync(OpenAIFileClient client, string fileId, bool launchViewer = false) + { + OpenAIFile fileInfo = client.GetFile(fileId); + if (fileInfo.Purpose == FilePurpose.AssistantsOutput) + { + string filePath = + Path.Combine( + Path.GetTempPath(), + Path.GetFileName(Path.ChangeExtension(fileInfo.Filename, ".png"))); + + BinaryData content = await client.DownloadFileAsync(fileId); + await using FileStream fileStream = new(filePath, FileMode.CreateNew); + await content.ToStream().CopyToAsync(fileStream); + Console.WriteLine($"File saved to: {filePath}."); + + if (launchViewer) + { + Process.Start( + new ProcessStartInfo + { + FileName = "cmd.exe", + Arguments = $"/C start {filePath}" + }); + } + } + } +} +``` +::: zone-end + +::: zone pivot="programming-language-python" +```python +import asyncio +import os + +from semantic_kernel.agents.open_ai.azure_assistant_agent import AzureAssistantAgent +from semantic_kernel.contents.chat_message_content import ChatMessageContent +from semantic_kernel.contents.streaming_file_reference_content import StreamingFileReferenceContent +from semantic_kernel.contents.utils.author_role import AuthorRole +from semantic_kernel.kernel import Kernel + +# Let's form the file paths that we will later pass to the assistant +csv_file_path_1 = os.path.join( + os.path.dirname(os.path.dirname(os.path.realpath(__file__))), + "PopulationByAdmin1.csv", +) + +csv_file_path_2 = os.path.join( + os.path.dirname(os.path.dirname(os.path.realpath(__file__))), + "PopulationByCountry.csv", +) + + +async def download_file_content(agent, file_id: str): + try: + # Fetch the content of the file using the provided method + response_content = await agent.client.files.content(file_id) + + # Get the current working directory of the file + current_directory = os.path.dirname(os.path.abspath(__file__)) + + # Define the path to save the image in the current directory + file_path = os.path.join( + current_directory, # Use the current directory of the file + f"{file_id}.png", # You can modify this to use the actual filename with proper extension + ) + + # Save content to a file asynchronously + with open(file_path, "wb") as file: + file.write(response_content.content) + + print(f"File saved to: {file_path}") + except Exception as e: + print(f"An error occurred while downloading file {file_id}: {str(e)}") + + +async def download_response_image(agent, file_ids: list[str]): + if file_ids: + # Iterate over file_ids and download each one + for file_id in file_ids: + await download_file_content(agent, file_id) + + +async def main(): + agent = await AzureAssistantAgent.create( + kernel=Kernel(), + service_id="agent", + name="SampleAssistantAgent", + instructions=""" + Analyze the available data to provide an answer to the user's question. + Always format response using markdown. + Always include a numerical index that starts at 1 for any lists or tables. + Always sort lists in ascending order. + """, + enable_code_interpreter=True, + code_interpreter_filenames=[csv_file_path_1, csv_file_path_2], + ) + + print("Creating thread...") + thread_id = await agent.create_thread() + + try: + is_complete: bool = False + file_ids: list[str] = [] + while not is_complete: + user_input = input("User:> ") + if not user_input: + continue + + if user_input.lower() == "exit": + is_complete = True + break + + await agent.add_chat_message( + thread_id=thread_id, message=ChatMessageContent(role=AuthorRole.USER, content=user_input) + ) + is_code: bool = False + async for response in agent.invoke_stream(thread_id=thread_id): + if is_code != response.metadata.get("code"): + print() + is_code = not is_code + + print(f"{response.content}", end="", flush=True) + + file_ids.extend([ + item.file_id for item in response.items if isinstance(item, StreamingFileReferenceContent) + ]) + + print() + + await download_response_image(agent, file_ids) + file_ids.clear() + + finally: + print("Cleaning up resources...") + if agent is not None: + [await agent.delete_file(file_id) for file_id in agent.code_interpreter_file_ids] + await agent.delete_thread(thread_id) + await agent.delete() + + +if __name__ == "__main__": + asyncio.run(main()) +``` +::: zone-end + +::: zone pivot="programming-language-java" + +> Agents are currently unavailable in Java. + +::: zone-end + + +> [!div class="nextstepaction"] +> [How-To: _Open AI Assistant Agent_ Code File Search](./example-assistant-search.md) + diff --git a/semantic-kernel/Frameworks/agent/examples/example-assistant-search.md b/semantic-kernel/Frameworks/agent/examples/example-assistant-search.md new file mode 100644 index 00000000..cb39596e --- /dev/null +++ b/semantic-kernel/Frameworks/agent/examples/example-assistant-search.md @@ -0,0 +1,794 @@ +--- +title: How-To: _Open AI Assistant Agent_ File Search (Experimental) +description: A step-by-step walk-through of defining and utilizing the file-search tool of an Open AI Assistant Agent. +zone_pivot_groups: programming-languages +author: crickman +ms.topic: tutorial +ms.author: crickman +ms.date: 09/13/2024 +ms.service: semantic-kernel +--- +# How-To: _Open AI Assistant Agent_ File Search (Experimental) + +> [!WARNING] +> The _Semantic Kernel Agent Framework_ is experimental, still in development and is subject to change. + +## Overview + +In this sample, we will explore how to use the _file-search_ tool of an [_Open AI Assistant Agent_](../assistant-agent.md) to complete comprehension tasks. The approach will be step-by-step, ensuring clarity and precision throughout the process. As part of the task, the agent will provide document citations within the response. + +Streaming will be used to deliver the agent's responses. This will provide real-time updates as the task progresses. + + +## Getting Started + +Before proceeding with feature coding, make sure your development environment is fully set up and configured. + +::: zone pivot="programming-language-csharp" + +To add package dependencies from the command-line use the `dotnet` command: + +```powershell +dotnet add package Azure.Identity +dotnet add package Microsoft.Extensions.Configuration +dotnet add package Microsoft.Extensions.Configuration.Binder +dotnet add package Microsoft.Extensions.Configuration.UserSecrets +dotnet add package Microsoft.Extensions.Configuration.EnvironmentVariables +dotnet add package Microsoft.SemanticKernel +dotnet add package Microsoft.SemanticKernel.Agents.OpenAI --prerelease +``` + +> If managing _NuGet_ packages in _Visual Studio_, ensure `Include prerelease` is checked. + +The project file (`.csproj`) should contain the following `PackageReference` definitions: + +```xml + + + + + + + + + +``` + +The _Agent Framework_ is experimental and requires warning suppression. This may addressed in as a property in the project file (`.csproj`): + +```xml + + $(NoWarn);CA2007;IDE1006;SKEXP0001;SKEXP0110;OPENAI001 + +``` + +Additionally, copy the `Grimms-The-King-of-the-Golden-Mountain.txt`, `Grimms-The-Water-of-Life.txt` and `Grimms-The-White-Snake.txt` public domain content from [_Semantic Kernel_ `LearnResources` Project](https://github.com/microsoft/semantic-kernel/tree/main/dotnet/samples/LearnResources/Resources). Add these files in your project folder and configure to have them copied to the output directory: + +```xml + + + Always + + + Always + + + Always + + +``` +::: zone-end + +::: zone pivot="programming-language-python" +Start by creating a folder that will hold your script (`.py` file) and the sample resources. Include the following imports at the top of your `.py` file: + +```python +import asyncio +import os + +from semantic_kernel.agents.open_ai.azure_assistant_agent import AzureAssistantAgent +from semantic_kernel.contents.chat_message_content import ChatMessageContent +from semantic_kernel.contents.streaming_annotation_content import StreamingAnnotationContent +from semantic_kernel.contents.utils.author_role import AuthorRole +from semantic_kernel.kernel import Kernel +``` + +Additionally, copy the `Grimms-The-King-of-the-Golden-Mountain.txt`, `Grimms-The-Water-of-Life.txt` and `Grimms-The-White-Snake.txt` public domain content from [_Semantic Kernel_ `LearnResources` Project](https://github.com/microsoft/semantic-kernel/tree/main/python/samples/learn_resources/resources). Add these files in your project folder. + +::: zone-end + +::: zone pivot="programming-language-java" + +> Agents are currently unavailable in Java. + +::: zone-end + + +## Configuration + +This sample requires configuration setting in order to connect to remote services. You will need to define settings for either _Open AI_ or _Azure Open AI_. + +::: zone pivot="programming-language-csharp" + +```powershell +# Open AI +dotnet user-secrets set "OpenAISettings:ApiKey" "" +dotnet user-secrets set "OpenAISettings:ChatModel" "gpt-4o" + +# Azure Open AI +dotnet user-secrets set "AzureOpenAISettings:ApiKey" "" # Not required if using token-credential +dotnet user-secrets set "AzureOpenAISettings:Endpoint" "https://lightspeed-team-shared-openai-eastus.openai.azure.com/" +dotnet user-secrets set "AzureOpenAISettings:ChatModelDeployment" "gpt-4o" +``` + +The following class is used in all of the Agent examples. Be sure to include it in your project to ensure proper functionality. This class serves as a foundational component for the examples that follow. + +```c# +using System.Reflection; +using Microsoft.Extensions.Configuration; + +namespace AgentsSample; + +public class Settings +{ + private readonly IConfigurationRoot configRoot; + + private AzureOpenAISettings azureOpenAI; + private OpenAISettings openAI; + + public AzureOpenAISettings AzureOpenAI => this.azureOpenAI ??= this.GetSettings(); + public OpenAISettings OpenAI => this.openAI ??= this.GetSettings(); + + public class OpenAISettings + { + public string ChatModel { get; set; } = string.Empty; + public string ApiKey { get; set; } = string.Empty; + } + + public class AzureOpenAISettings + { + public string ChatModelDeployment { get; set; } = string.Empty; + public string Endpoint { get; set; } = string.Empty; + public string ApiKey { get; set; } = string.Empty; + } + + public TSettings GetSettings() => + this.configRoot.GetRequiredSection(typeof(TSettings).Name).Get()!; + + public Settings() + { + this.configRoot = + new ConfigurationBuilder() + .AddEnvironmentVariables() + .AddUserSecrets(Assembly.GetExecutingAssembly(), optional: true) + .Build(); + } +} +``` +::: zone-end + +::: zone pivot="programming-language-python" +The quickest way to get started with the proper configuration to run the sample code is to create a `.env` file at the root of your project (where your script is run). + +Configure the following settings in your `.env` file for either Azure OpenAI or OpenAI: + +```python +AZURE_OPENAI_API_KEY="..." +AZURE_OPENAI_ENDPOINT="https://..." +AZURE_OPENAI_CHAT_DEPLOYMENT_NAME="..." +AZURE_OPENAI_API_VERSION="..." + +OPENAI_API_KEY="sk-..." +OPENAI_ORG_ID="" +OPENAI_CHAT_MODEL_ID="" +``` + +Once configured, the respective AI service classes will pick up the required variables and use them during instantiation. +::: zone-end + +::: zone pivot="programming-language-java" + +> Agents are currently unavailable in Java. + +::: zone-end + + +## Coding + +The coding process for this sample involves: + +1. [Setup](#setup) - Initializing settings and the plug-in. +2. [Agent Definition](#agent-definition) - Create the _Chat_Completion_Agent_ with templatized instructions and plug-in. +3. [The _Chat_ Loop](#the-chat-loop) - Write the loop that drives user / agent interaction. + +The full example code is provided in the [Final](#final) section. Refer to that section for the complete implementation. + +### Setup + +Prior to creating an _Open AI Assistant Agent_, ensure the configuration settings are available and prepare the file resources. + +::: zone pivot="programming-language-csharp" + +Instantiate the `Settings` class referenced in the previous [Configuration](#configuration) section. Use the settings to also create an `OpenAIClientProvider` that will be used for the [Agent Definition](#agent-definition) as well as file-upload and the creation of a `VectorStore`. + +```csharp + +Settings settings = new(); + +OpenAIClientProvider clientProvider = + OpenAIClientProvider.ForAzureOpenAI( + new AzureCliCredential(), + new Uri(settings.AzureOpenAI.Endpoint)); +``` +::: zone-end + +::: zone pivot="programming-language-java" + +> Agents are currently unavailable in Java. + +::: zone-end + +Now create an empty _Vector Store for use with the _File Search_ tool: + +::: zone pivot="programming-language-csharp" + +Use the `OpenAIClientProvider` to access a `VectorStoreClient` and create a `VectorStore`. + +```csharp +Console.WriteLine("Creating store..."); +VectorStoreClient storeClient = clientProvider.Client.GetVectorStoreClient(); +CreateVectorStoreOperation operation = await storeClient.CreateVectorStoreAsync(waitUntilCompleted: true); +string storeId = operation.VectorStoreId; +``` +::: zone-end + +::: zone pivot="programming-language-python" +```python +def get_filepath_for_filename(filename: str) -> str: + base_directory = os.path.dirname(os.path.realpath(__file__)) + return os.path.join(base_directory, filename) +``` +::: zone-end + +::: zone pivot="programming-language-java" + +> Agents are currently unavailable in Java. + +::: zone-end + +Let's declare the the three content-files described in the previous [Configuration](#configuration) section: + +::: zone pivot="programming-language-csharp" + +```csharp +private static readonly string[] _fileNames = + [ + "Grimms-The-King-of-the-Golden-Mountain.txt", + "Grimms-The-Water-of-Life.txt", + "Grimms-The-White-Snake.txt", + ]; +``` +::: zone-end + +::: zone pivot="programming-language-python" +```python +filenames = [ + "Grimms-The-King-of-the-Golden-Mountain.txt", + "Grimms-The-Water-of-Life.txt", + "Grimms-The-White-Snake.txt", +] +``` +::: zone-end + +::: zone pivot="programming-language-java" + +> Agents are currently unavailable in Java. + +::: zone-end + +::: zone pivot="programming-language-csharp" + +Now upload those files and add them to the _Vector Store_ by using the previously created `VectorStoreClient` clients to upload each file with a `OpenAIFileClient` and add it to the _Vector Store_, preserving the resulting _File References_. + +```csharp +Dictionary fileReferences = []; + +Console.WriteLine("Uploading files..."); +OpenAIFileClient fileClient = clientProvider.Client.GetOpenAIFileClient(); +foreach (string fileName in _fileNames) +{ + OpenAIFile fileInfo = await fileClient.UploadFileAsync(fileName, FileUploadPurpose.Assistants); + await storeClient.AddFileToVectorStoreAsync(storeId, fileInfo.Id, waitUntilCompleted: true); + fileReferences.Add(fileInfo.Id, fileInfo); +} +``` +::: zone-end + +::: zone pivot="programming-language-java" + +> Agents are currently unavailable in Java. + +::: zone-end + +### Agent Definition + +We are now ready to instantiate an _OpenAI Assistant Agent_. The agent is configured with its target model, _Instructions_, and the _File Search_ tool enabled. Additionally, we explicitly associate the _Vector Store_ with the _File Search_ tool. + +::: zone pivot="programming-language-csharp" + +We will utilize the `OpenAIClientProvider` again as part of creating the `OpenAIAssistantAgent`: + +```csharp +Console.WriteLine("Defining agent..."); +OpenAIAssistantAgent agent = + await OpenAIAssistantAgent.CreateAsync( + clientProvider, + new OpenAIAssistantDefinition(settings.AzureOpenAI.ChatModelDeployment) + { + Name = "SampleAssistantAgent", + Instructions = + """ + The document store contains the text of fictional stories. + Always analyze the document store to provide an answer to the user's question. + Never rely on your knowledge of stories not included in the document store. + Always format response using markdown. + """, + EnableFileSearch = true, + VectorStoreId = storeId, + }, + new Kernel()); +``` +::: zone-end + +::: zone pivot="programming-language-python" +```python +agent = await AzureAssistantAgent.create( + kernel=Kernel(), + service_id="agent", + name="SampleAssistantAgent", + instructions=""" + The document store contains the text of fictional stories. + Always analyze the document store to provide an answer to the user's question. + Never rely on your knowledge of stories not included in the document store. + Always format response using markdown. + """, + enable_file_search=True, + vector_store_filenames=[get_filepath_for_filename(filename) for filename in filenames], +) +``` +::: zone-end + +::: zone pivot="programming-language-java" + +> Agents are currently unavailable in Java. + +::: zone-end + +### The _Chat_ Loop + +At last, we are able to coordinate the interaction between the user and the _Agent_. Start by creating an _Assistant Thread_ to maintain the conversation state and creating an empty loop. + +Let's also ensure the resources are removed at the end of execution to minimize unnecessary charges. + +::: zone pivot="programming-language-csharp" +```csharp +Console.WriteLine("Creating thread..."); +string threadId = await agent.CreateThreadAsync(); + +Console.WriteLine("Ready!"); + +try +{ + bool isComplete = false; + do + { + // Processing occurrs here + } while (!isComplete); +} +finally +{ + Console.WriteLine(); + Console.WriteLine("Cleaning-up..."); + await Task.WhenAll( + [ + agent.DeleteThreadAsync(threadId), + agent.DeleteAsync(), + storeClient.DeleteVectorStoreAsync(storeId), + ..fileReferences.Select(fileReference => fileClient.DeleteFileAsync(fileReference.Key)) + ]); +} +``` +::: zone-end + +::: zone pivot="programming-language-python" +```python +print("Creating thread...") +thread_id = await agent.create_thread() + +try: + is_complete: bool = False + while not is_complete: + # Processing occurs here + +finally: + print("Cleaning up resources...") + if agent is not None: + [await agent.delete_file(file_id) for file_id in agent.file_search_file_ids] + await agent.delete_thread(thread_id) + await agent.delete() +``` +::: zone-end + +::: zone pivot="programming-language-java" + +> Agents are currently unavailable in Java. + +::: zone-end + +Now let's capture user input within the previous loop. In this case, empty input will be ignored and the term `EXIT` will signal that the conversation is completed. Valid nput will be added to the _Assistant Thread_ as a _User_ message. + +::: zone pivot="programming-language-csharp" +```csharp +Console.WriteLine(); +Console.Write("> "); +string input = Console.ReadLine(); +if (string.IsNullOrWhiteSpace(input)) +{ + continue; +} +if (input.Trim().Equals("EXIT", StringComparison.OrdinalIgnoreCase)) +{ + isComplete = true; + break; +} + +await agent.AddChatMessageAsync(threadId, new ChatMessageContent(AuthorRole.User, input)); +Console.WriteLine(); +``` +::: zone-end + +::: zone pivot="programming-language-python" +```python +user_input = input("User:> ") +if not user_input: + continue + +if user_input.lower() == "exit": + is_complete = True + +await agent.add_chat_message( + thread_id=thread_id, message=ChatMessageContent(role=AuthorRole.USER, content=user_input) +) +``` +::: zone-end + +::: zone pivot="programming-language-java" + +> Agents are currently unavailable in Java. + +::: zone-end + +Before invoking the _Agent_ response, let's add a helper method to reformat the unicode annotation brackets to ANSI brackets. + +::: zone pivot="programming-language-csharp" +```csharp +private static string ReplaceUnicodeBrackets(this string content) => + content?.Replace('【', '[').Replace('】', ']'); +``` +::: zone-end + +::: zone pivot="programming-language-python" +```python +# No special handling required. +``` +::: zone-end + +::: zone pivot="programming-language-java" + +> Agents are currently unavailable in Java. + +::: zone-end + +To generate an _Agent_ response to user input, invoke the agent by specifying the _Assistant Thread_. In this example, we choose a streamed response and capture any associated _Citation Annotations_ for display at the end of the response cycle. Note each streamed chunk is being reformatted using the previous helper method. + +::: zone pivot="programming-language-csharp" +```csharp +List footnotes = []; +await foreach (StreamingChatMessageContent chunk in agent.InvokeStreamingAsync(threadId)) +{ + // Capture annotations for footnotes + footnotes.AddRange(chunk.Items.OfType()); + + // Render chunk with replacements for unicode brackets. + Console.Write(chunk.Content.ReplaceUnicodeBrackets()); +} + +Console.WriteLine(); + +// Render footnotes for captured annotations. +if (footnotes.Count > 0) +{ + Console.WriteLine(); + foreach (StreamingAnnotationContent footnote in footnotes) + { + Console.WriteLine($"#{footnote.Quote.ReplaceUnicodeBrackets()} - {fileReferences[footnote.FileId!].Filename} (Index: {footnote.StartIndex} - {footnote.EndIndex})"); + } +} +``` +::: zone-end + +::: zone pivot="programming-language-python" +```python +footnotes: list[StreamingAnnotationContent] = [] +async for response in agent.invoke_stream(thread_id=thread_id): + footnotes.extend([item for item in response.items if isinstance(item, StreamingAnnotationContent)]) + + print(f"{response.content}", end="", flush=True) + +print() + +if len(footnotes) > 0: + for footnote in footnotes: + print( + f"\n`{footnote.quote}` => {footnote.file_id} " + f"(Index: {footnote.start_index} - {footnote.end_index})" + ) +``` +::: zone-end + +::: zone pivot="programming-language-java" + +> Agents are currently unavailable in Java. + +::: zone-end + + +## Final + +Bringing all the steps together, we have the final code for this example. The complete implementation is provided below. + +Try using these suggested inputs: + +1. What is the paragraph count for each of the stories? +2. Create a table that identifies the protagonist and antagonist for each story. +3. What is the moral in The White Snake? + +::: zone pivot="programming-language-csharp" +```csharp +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Azure.Identity; +using Microsoft.SemanticKernel; +using Microsoft.SemanticKernel.Agents.OpenAI; +using Microsoft.SemanticKernel.ChatCompletion; +using OpenAI.Files; +using OpenAI.VectorStores; + +namespace AgentsSample; + +public static class Program +{ + private static readonly string[] _fileNames = + [ + "Grimms-The-King-of-the-Golden-Mountain.txt", + "Grimms-The-Water-of-Life.txt", + "Grimms-The-White-Snake.txt", + ]; + + /// + /// The main entry point for the application. + /// + /// A representing the asynchronous operation. + public static async Task Main() + { + // Load configuration from environment variables or user secrets. + Settings settings = new(); + + OpenAIClientProvider clientProvider = + OpenAIClientProvider.ForAzureOpenAI( + new AzureCliCredential(), + new Uri(settings.AzureOpenAI.Endpoint)); + + Console.WriteLine("Creating store..."); + VectorStoreClient storeClient = clientProvider.Client.GetVectorStoreClient(); + CreateVectorStoreOperation operation = await storeClient.CreateVectorStoreAsync(waitUntilCompleted: true); + string storeId = operation.VectorStoreId; + + // Retain file references. + Dictionary fileReferences = []; + + Console.WriteLine("Uploading files..."); + OpenAIFileClient fileClient = clientProvider.Client.GetOpenAIFileClient(); + foreach (string fileName in _fileNames) + { + OpenAIFile fileInfo = await fileClient.UploadFileAsync(fileName, FileUploadPurpose.Assistants); + await storeClient.AddFileToVectorStoreAsync(storeId, fileInfo.Id, waitUntilCompleted: true); + fileReferences.Add(fileInfo.Id, fileInfo); + } + + + Console.WriteLine("Defining agent..."); + OpenAIAssistantAgent agent = + await OpenAIAssistantAgent.CreateAsync( + clientProvider, + new OpenAIAssistantDefinition(settings.AzureOpenAI.ChatModelDeployment) + { + Name = "SampleAssistantAgent", + Instructions = + """ + The document store contains the text of fictional stories. + Always analyze the document store to provide an answer to the user's question. + Never rely on your knowledge of stories not included in the document store. + Always format response using markdown. + """, + EnableFileSearch = true, + VectorStoreId = storeId, + }, + new Kernel()); + + Console.WriteLine("Creating thread..."); + string threadId = await agent.CreateThreadAsync(); + + Console.WriteLine("Ready!"); + + try + { + bool isComplete = false; + do + { + Console.WriteLine(); + Console.Write("> "); + string input = Console.ReadLine(); + if (string.IsNullOrWhiteSpace(input)) + { + continue; + } + if (input.Trim().Equals("EXIT", StringComparison.OrdinalIgnoreCase)) + { + isComplete = true; + break; + } + + await agent.AddChatMessageAsync(threadId, new ChatMessageContent(AuthorRole.User, input)); + Console.WriteLine(); + + List footnotes = []; + await foreach (StreamingChatMessageContent chunk in agent.InvokeStreamingAsync(threadId)) + { + // Capture annotations for footnotes + footnotes.AddRange(chunk.Items.OfType()); + + // Render chunk with replacements for unicode brackets. + Console.Write(chunk.Content.ReplaceUnicodeBrackets()); + } + + Console.WriteLine(); + + // Render footnotes for captured annotations. + if (footnotes.Count > 0) + { + Console.WriteLine(); + foreach (StreamingAnnotationContent footnote in footnotes) + { + Console.WriteLine($"#{footnote.Quote.ReplaceUnicodeBrackets()} - {fileReferences[footnote.FileId!].Filename} (Index: {footnote.StartIndex} - {footnote.EndIndex})"); + } + } + } while (!isComplete); + } + finally + { + Console.WriteLine(); + Console.WriteLine("Cleaning-up..."); + await Task.WhenAll( + [ + agent.DeleteThreadAsync(threadId), + agent.DeleteAsync(), + storeClient.DeleteVectorStoreAsync(storeId), + ..fileReferences.Select(fileReference => fileClient.DeleteFileAsync(fileReference.Key)) + ]); + } + } + + private static string ReplaceUnicodeBrackets(this string content) => + content?.Replace('【', '[').Replace('】', ']'); +} +``` +::: zone-end + +::: zone pivot="programming-language-python" +```python +import asyncio +import os + +from semantic_kernel.agents.open_ai.azure_assistant_agent import AzureAssistantAgent +from semantic_kernel.contents.chat_message_content import ChatMessageContent +from semantic_kernel.contents.streaming_annotation_content import StreamingAnnotationContent +from semantic_kernel.contents.utils.author_role import AuthorRole +from semantic_kernel.kernel import Kernel + + +def get_filepath_for_filename(filename: str) -> str: + base_directory = os.path.dirname(os.path.realpath(__file__)) + return os.path.join(base_directory, filename) + + +filenames = [ + "Grimms-The-King-of-the-Golden-Mountain.txt", + "Grimms-The-Water-of-Life.txt", + "Grimms-The-White-Snake.txt", +] + + +async def main(): + agent = await AzureAssistantAgent.create( + kernel=Kernel(), + service_id="agent", + name="SampleAssistantAgent", + instructions=""" + The document store contains the text of fictional stories. + Always analyze the document store to provide an answer to the user's question. + Never rely on your knowledge of stories not included in the document store. + Always format response using markdown. + """, + enable_file_search=True, + vector_store_filenames=[get_filepath_for_filename(filename) for filename in filenames], + ) + + print("Creating thread...") + thread_id = await agent.create_thread() + + try: + is_complete: bool = False + while not is_complete: + user_input = input("User:> ") + if not user_input: + continue + + if user_input.lower() == "exit": + is_complete = True + break + + await agent.add_chat_message( + thread_id=thread_id, message=ChatMessageContent(role=AuthorRole.USER, content=user_input) + ) + + footnotes: list[StreamingAnnotationContent] = [] + async for response in agent.invoke_stream(thread_id=thread_id): + footnotes.extend([item for item in response.items if isinstance(item, StreamingAnnotationContent)]) + + print(f"{response.content}", end="", flush=True) + + print() + + if len(footnotes) > 0: + for footnote in footnotes: + print( + f"\n`{footnote.quote}` => {footnote.file_id} " + f"(Index: {footnote.start_index} - {footnote.end_index})" + ) + + finally: + print("Cleaning up resources...") + if agent is not None: + [await agent.delete_file(file_id) for file_id in agent.file_search_file_ids] + await agent.delete_thread(thread_id) + await agent.delete() + + +if __name__ == "__main__": + asyncio.run(main()) +``` +::: zone-end + +::: zone pivot="programming-language-java" + +> Agents are currently unavailable in Java. + +::: zone-end + + +> [!div class="nextstepaction"] +> [How to Coordinate Agent Collaboration using _Agent Group Chat_](./example-agent-collaboration.md) + diff --git a/semantic-kernel/Frameworks/agent/examples/example-chat-agent.md b/semantic-kernel/Frameworks/agent/examples/example-chat-agent.md new file mode 100644 index 00000000..7f52d04f --- /dev/null +++ b/semantic-kernel/Frameworks/agent/examples/example-chat-agent.md @@ -0,0 +1,669 @@ +--- +title: How-To: _Chat Completion Agent_ (Experimental) +description: A step-by-step walk-through of defining and utilizing the features of a Chat Completion Agent. +zone_pivot_groups: programming-languages +author: crickman +ms.topic: tutorial +ms.author: crickman +ms.date: 09/13/2024 +ms.service: semantic-kernel +--- +# How-To: _Chat Completion Agent_ (Experimental) + +> [!WARNING] +> The _Semantic Kernel Agent Framework_ is experimental, still in development and is subject to change. + +## Overview + +In this sample, we will explore configuring a plugin to access _GitHub_ API and provide templatized instructions to a [_Chat Completion Agent_](../chat-completion-agent.md) to answer questions about a _GitHub_ repository. The approach will be broken down step-by-step to high-light the key parts of the coding process. As part of the task, the agent will provide document citations within the response. + +Streaming will be used to deliver the agent's responses. This will provide real-time updates as the task progresses. + +## Getting Started + +Before proceeding with feature coding, make sure your development environment is fully set up and configured. + +::: zone pivot="programming-language-csharp" + +Start by creating a _Console_ project. Then, include the following package references to ensure all required dependencies are available. + +To add package dependencies from the command-line use the `dotnet` command: + +```powershell +dotnet add package Azure.Identity +dotnet add package Microsoft.Extensions.Configuration +dotnet add package Microsoft.Extensions.Configuration.Binder +dotnet add package Microsoft.Extensions.Configuration.UserSecrets +dotnet add package Microsoft.Extensions.Configuration.EnvironmentVariables +dotnet add package Microsoft.SemanticKernel.Connectors.AzureOpenAI +dotnet add package Microsoft.SemanticKernel.Agents.Core --prerelease +``` + +> If managing _NuGet_ packages in _Visual Studio_, ensure `Include prerelease` is checked. + +The project file (`.csproj`) should contain the following `PackageReference` definitions: + +```xml + + + + + + + + + +``` + +The _Agent Framework_ is experimental and requires warning suppression. This may addressed in as a property in the project file (`.csproj`): + +```xml + + $(NoWarn);CA2007;IDE1006;SKEXP0001;SKEXP0110;OPENAI001 + +``` + +Additionally, copy the GitHub plug-in and models (`GitHubPlugin.cs` and `GitHubModels.cs`) from [_Semantic Kernel_ `LearnResources` Project](https://github.com/microsoft/semantic-kernel/tree/main/dotnet/samples/LearnResources/Plugins/GitHub). Add these files in your project folder. + +::: zone-end + +::: zone pivot="programming-language-python" +Start by creating a folder that will hold your script (`.py` file) and the sample resources. Include the following imports at the top of your `.py` file: +```python +import asyncio +import os +import sys +from datetime import datetime + +from semantic_kernel.agents import ChatCompletionAgent +from semantic_kernel.connectors.ai.function_choice_behavior import FunctionChoiceBehavior +from semantic_kernel.connectors.ai.open_ai import AzureChatCompletion +from semantic_kernel.contents.chat_history import ChatHistory +from semantic_kernel.contents.chat_message_content import ChatMessageContent +from semantic_kernel.contents.utils.author_role import AuthorRole +from semantic_kernel.kernel import Kernel + +# Adjust the sys.path so we can use the GitHubPlugin and GitHubSettings classes +# This is so we can run the code from the samples/learn_resources/agent_docs directory +# If you are running code from your own project, you may not need need to do this. +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))) + +from plugins.GithubPlugin.github import GitHubPlugin, GitHubSettings # noqa: E402 +``` + +Additionally, copy the GitHub plug-in and models (`github.py`) from [_Semantic Kernel_ `LearnResources` Project](https://github.com/microsoft/semantic-kernel/tree/main/python/samples/learn_resources/plugins/GithubPlugin). Add these files in your project folder. +::: zone-end + +::: zone pivot="programming-language-java" + +> Agents are currently unavailable in Java. + +::: zone-end + +## Configuration + +This sample requires configuration setting in order to connect to remote services. You will need to define settings for either _Open AI_ or _Azure Open AI_ and also for _GitHub_. + +> Note: For information on GitHub _Personal Access Tokens_, see: [Managing your personal access tokens](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens). + +::: zone pivot="programming-language-csharp" + +```powershell +# Open AI +dotnet user-secrets set "OpenAISettings:ApiKey" "" +dotnet user-secrets set "OpenAISettings:ChatModel" "gpt-4o" + +# Azure Open AI +dotnet user-secrets set "AzureOpenAISettings:ApiKey" "" # Not required if using token-credential +dotnet user-secrets set "AzureOpenAISettings:Endpoint" "" +dotnet user-secrets set "AzureOpenAISettings:ChatModelDeployment" "gpt-4o" + +# GitHub +dotnet user-secrets set "GitHubSettings:BaseUrl" "https://api.github.com" +dotnet user-secrets set "GitHubSettings:Token" "" +``` + +The following class is used in all of the Agent examples. Be sure to include it in your project to ensure proper functionality. This class serves as a foundational component for the examples that follow. + +```c# +using System.Reflection; +using Microsoft.Extensions.Configuration; + +namespace AgentsSample; + +public class Settings +{ + private readonly IConfigurationRoot configRoot; + + private AzureOpenAISettings azureOpenAI; + private OpenAISettings openAI; + + public AzureOpenAISettings AzureOpenAI => this.azureOpenAI ??= this.GetSettings(); + public OpenAISettings OpenAI => this.openAI ??= this.GetSettings(); + + public class OpenAISettings + { + public string ChatModel { get; set; } = string.Empty; + public string ApiKey { get; set; } = string.Empty; + } + + public class AzureOpenAISettings + { + public string ChatModelDeployment { get; set; } = string.Empty; + public string Endpoint { get; set; } = string.Empty; + public string ApiKey { get; set; } = string.Empty; + } + + public TSettings GetSettings() => + this.configRoot.GetRequiredSection(typeof(TSettings).Name).Get()!; + + public Settings() + { + this.configRoot = + new ConfigurationBuilder() + .AddEnvironmentVariables() + .AddUserSecrets(Assembly.GetExecutingAssembly(), optional: true) + .Build(); + } +} +``` +::: zone-end + +::: zone pivot="programming-language-python" +The quickest way to get started with the proper configuration to run the sample code is to create a `.env` file at the root of your project (where your script is run). + +Configure the following settings in your `.env` file for either Azure OpenAI or OpenAI: + +```python +AZURE_OPENAI_API_KEY="..." +AZURE_OPENAI_ENDPOINT="https://..." +AZURE_OPENAI_CHAT_DEPLOYMENT_NAME="..." +AZURE_OPENAI_API_VERSION="..." + +OPENAI_API_KEY="sk-..." +OPENAI_ORG_ID="" +OPENAI_CHAT_MODEL_ID="" +``` + +Once configured, the respective AI service classes will pick up the required variables and use them during instantiation. +::: zone-end + +::: zone pivot="programming-language-java" + +> Agents are currently unavailable in Java. + +::: zone-end + + +## Coding + +The coding process for this sample involves: + +1. [Setup](#setup) - Initializing settings and the plug-in. +2. [_Agent_ Definition](#agent-definition) - Create the _Chat Completion Agent_ with templatized instructions and plug-in. +3. [The _Chat_ Loop](#the-chat-loop) - Write the loop that drives user / agent interaction. + +The full example code is provided in the [Final](#final) section. Refer to that section for the complete implementation. + +### Setup + +Prior to creating a _Chat Completion Agent_, the configuration settings, plugins, and _Kernel_ must be initialized. + +::: zone pivot="programming-language-csharp" + +Initialize the `Settings` class referenced in the previous [Configuration](#configuration) section. + +```csharp +Settings settings = new(); +``` +::: zone-end + +::: zone pivot="programming-language-java" + +> Agents are currently unavailable in Java. + +::: zone-end + +Initialize the plug-in using its settings. + +::: zone pivot="programming-language-csharp" + +Here, a message is displaying to indicate progress. + +```csharp +Console.WriteLine("Initialize plugins..."); +GitHubSettings githubSettings = settings.GetSettings(); +GitHubPlugin githubPlugin = new(githubSettings); +``` +::: zone-end + +::: zone pivot="programming-language-python" +```python +gh_settings = GitHubSettings( + token="" +) +kernel.add_plugin(GitHubPlugin(settings=gh_settings), plugin_name="github") +``` +::: zone-end + +::: zone pivot="programming-language-java" + +> Agents are currently unavailable in Java. + +::: zone-end + +Now initialize a `Kernel` instance with an `IChatCompletionService` and the `GitHubPlugin` previously created. + +::: zone pivot="programming-language-csharp" +```csharp +Console.WriteLine("Creating kernel..."); +IKernelBuilder builder = Kernel.CreateBuilder(); + +builder.AddAzureOpenAIChatCompletion( + settings.AzureOpenAI.ChatModelDeployment, + settings.AzureOpenAI.Endpoint, + new AzureCliCredential()); + +builder.Plugins.AddFromObject(githubPlugin); + +Kernel kernel = builder.Build(); +``` +::: zone-end + +::: zone pivot="programming-language-python" +```python +kernel = Kernel() + +# Add the AzureChatCompletion AI Service to the Kernel +service_id = "agent" +kernel.add_service(AzureChatCompletion(service_id=service_id)) + +settings = kernel.get_prompt_execution_settings_from_service_id(service_id=service_id) +# Configure the function choice behavior to auto invoke kernel functions +settings.function_choice_behavior = FunctionChoiceBehavior.Auto() +``` +::: zone-end + +::: zone pivot="programming-language-java" + +> Agents are currently unavailable in Java. + +::: zone-end + +### Agent Definition + +Finally we are ready to instantiate a _Chat Completion Agent_ with its _Instructions_, associated _Kernel_, and the default _Arguments_ and _Execution Settings_. In this case, we desire to have the any plugin functions automatically executed. + +::: zone pivot="programming-language-csharp" +```csharp +Console.WriteLine("Defining agent..."); +ChatCompletionAgent agent = + new() + { + Name = "SampleAssistantAgent", + Instructions = + """ + You are an agent designed to query and retrieve information from a single GitHub repository in a read-only manner. + You are also able to access the profile of the active user. + + Use the current date and time to provide up-to-date details or time-sensitive responses. + + The repository you are querying is a public repository with the following name: {{$repository}} + + The current date and time is: {{$now}}. + """, + Kernel = kernel, + Arguments = + new KernelArguments(new AzureOpenAIPromptExecutionSettings() { FunctionChoiceBehavior = FunctionChoiceBehavior.Auto() }) + { + { "repository", "microsoft/semantic-kernel" } + } + }; + +Console.WriteLine("Ready!"); +``` +::: zone-end + +::: zone pivot="programming-language-python" +```python +agent = ChatCompletionAgent( + service_id="agent", + kernel=kernel, + name="SampleAssistantAgent", + instructions=f""" + You are an agent designed to query and retrieve information from a single GitHub repository in a read-only + manner. + You are also able to access the profile of the active user. + + Use the current date and time to provide up-to-date details or time-sensitive responses. + + The repository you are querying is a public repository with the following name: microsoft/semantic-kernel + + The current date and time is: {current_time}. + """, + execution_settings=settings, +) +``` +::: zone-end + +::: zone pivot="programming-language-java" + +> Agents are currently unavailable in Java. + +::: zone-end + +### The _Chat_ Loop + +At last, we are able to coordinate the interaction between the user and the _Agent_. Start by creating a _Chat History_ object to maintain the conversation state and creating an empty loop. + +::: zone pivot="programming-language-csharp" +```csharp +ChatHistory history = []; +bool isComplete = false; +do +{ + // processing logic here +} while (!isComplete); +``` +::: zone-end + +::: zone pivot="programming-language-python" +```python +history = ChatHistory() +is_complete: bool = False +while not is_complete: + # processing logic here +``` +::: zone-end + +::: zone pivot="programming-language-java" + +> Agents are currently unavailable in Java. + +::: zone-end + +Now let's capture user input within the previous loop. In this case, empty input will be ignored and the term `EXIT` will signal that the conversation is completed. Valid input will be added to the _Chat History_ as a _User_ message. + +::: zone pivot="programming-language-csharp" +```csharp +Console.WriteLine(); +Console.Write("> "); +string input = Console.ReadLine(); +if (string.IsNullOrWhiteSpace(input)) +{ + continue; +} +if (input.Trim().Equals("EXIT", StringComparison.OrdinalIgnoreCase)) +{ + isComplete = true; + break; +} + +history.Add(new ChatMessageContent(AuthorRole.User, input)); + +Console.WriteLine(); +``` +::: zone-end + +::: zone pivot="programming-language-python" +```python +user_input = input("User:> ") +if not user_input: + continue + +if user_input.lower() == "exit": + is_complete = True + break + +history.add_message(ChatMessageContent(role=AuthorRole.USER, content=user_input)) +``` +::: zone-end + +::: zone pivot="programming-language-java" + +> Agents are currently unavailable in Java. + +::: zone-end + +To generate a _Agent_ response to user input, invoke the agent using _Arguments_ to provide the final template parameter that specifies the current date and time. + +The _Agent_ response is then then displayed to the user. + +::: zone pivot="programming-language-csharp" +```csharp +DateTime now = DateTime.Now; +KernelArguments arguments = + new() + { + { "now", $"{now.ToShortDateString()} {now.ToShortTimeString()}" } + }; +await foreach (ChatMessageContent response in agent.InvokeAsync(history, arguments)) +{ + Console.WriteLine($"{response.Content}"); +} +``` +::: zone-end + +::: zone pivot="programming-language-python" +**Coming soon** +::: zone-end + +::: zone pivot="programming-language-java" + +> Agents are currently unavailable in Java. + +::: zone-end + +## Final + +Bringing all the steps together, we have the final code for this example. The complete implementation is provided below. + +Try using these suggested inputs: + +1. What is my username? +2. Describe the repo. +3. Describe the newest issue created in the repo. +4. List the top 10 issues closed within the last week. +5. How were these issues labeled? +6. List the 5 most recently opened issues with the "Agents" label + +::: zone pivot="programming-language-csharp" +```csharp +using System; +using System.Threading.Tasks; +using Azure.Identity; +using Microsoft.SemanticKernel; +using Microsoft.SemanticKernel.Agents; +using Microsoft.SemanticKernel.ChatCompletion; +using Microsoft.SemanticKernel.Connectors.AzureOpenAI; +using Plugins; + +namespace AgentsSample; + +public static class Program +{ + public static async Task Main() + { + // Load configuration from environment variables or user secrets. + Settings settings = new(); + + Console.WriteLine("Initialize plugins..."); + GitHubSettings githubSettings = settings.GetSettings(); + GitHubPlugin githubPlugin = new(githubSettings); + + Console.WriteLine("Creating kernel..."); + IKernelBuilder builder = Kernel.CreateBuilder(); + + builder.AddAzureOpenAIChatCompletion( + settings.AzureOpenAI.ChatModelDeployment, + settings.AzureOpenAI.Endpoint, + new AzureCliCredential()); + + builder.Plugins.AddFromObject(githubPlugin); + + Kernel kernel = builder.Build(); + + Console.WriteLine("Defining agent..."); + ChatCompletionAgent agent = + new() + { + Name = "SampleAssistantAgent", + Instructions = + """ + You are an agent designed to query and retrieve information from a single GitHub repository in a read-only manner. + You are also able to access the profile of the active user. + + Use the current date and time to provide up-to-date details or time-sensitive responses. + + The repository you are querying is a public repository with the following name: {{$repository}} + + The current date and time is: {{$now}}. + """, + Kernel = kernel, + Arguments = + new KernelArguments(new AzureOpenAIPromptExecutionSettings() { FunctionChoiceBehavior = FunctionChoiceBehavior.Auto() }) + { + { "repository", "microsoft/semantic-kernel" } + } + }; + + Console.WriteLine("Ready!"); + + ChatHistory history = []; + bool isComplete = false; + do + { + Console.WriteLine(); + Console.Write("> "); + string input = Console.ReadLine(); + if (string.IsNullOrWhiteSpace(input)) + { + continue; + } + if (input.Trim().Equals("EXIT", StringComparison.OrdinalIgnoreCase)) + { + isComplete = true; + break; + } + + history.Add(new ChatMessageContent(AuthorRole.User, input)); + + Console.WriteLine(); + + DateTime now = DateTime.Now; + KernelArguments arguments = + new() + { + { "now", $"{now.ToShortDateString()} {now.ToShortTimeString()}" } + }; + await foreach (ChatMessageContent response in agent.InvokeAsync(history, arguments)) + { + // Display response. + Console.WriteLine($"{response.Content}"); + } + + } while (!isComplete); + } +} +``` +::: zone-end + +::: zone pivot="programming-language-python" +```python +import asyncio +import os +import sys +from datetime import datetime + +from semantic_kernel.agents import ChatCompletionAgent +from semantic_kernel.connectors.ai.function_choice_behavior import FunctionChoiceBehavior +from semantic_kernel.connectors.ai.open_ai import AzureChatCompletion +from semantic_kernel.contents.chat_history import ChatHistory +from semantic_kernel.contents.chat_message_content import ChatMessageContent +from semantic_kernel.contents.utils.author_role import AuthorRole +from semantic_kernel.kernel import Kernel + +# Adjust the sys.path so we can use the GitHubPlugin and GitHubSettings classes +# This is so we can run the code from the samples/learn_resources/agent_docs directory +# If you are running code from your own project, you may not need need to do this. +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))) + +from plugins.GithubPlugin.github import GitHubPlugin, GitHubSettings # noqa: E402 + +################################################################### +# The following sample demonstrates how to create a simple, # +# ChatCompletionAgent to use a GitHub plugin to interact # +# with the GitHub API. # +################################################################### + + +async def main(): + kernel = Kernel() + + # Add the AzureChatCompletion AI Service to the Kernel + service_id = "agent" + kernel.add_service(AzureChatCompletion(service_id=service_id)) + + settings = kernel.get_prompt_execution_settings_from_service_id(service_id=service_id) + # Configure the function choice behavior to auto invoke kernel functions + settings.function_choice_behavior = FunctionChoiceBehavior.Auto() + + # Set your GitHub Personal Access Token (PAT) value here + gh_settings = GitHubSettings(token="") + kernel.add_plugin(plugin=GitHubPlugin(gh_settings), plugin_name="GithubPlugin") + + current_time = datetime.now().isoformat() + + # Create the agent + agent = ChatCompletionAgent( + service_id="agent", + kernel=kernel, + name="SampleAssistantAgent", + instructions=f""" + You are an agent designed to query and retrieve information from a single GitHub repository in a read-only + manner. + You are also able to access the profile of the active user. + + Use the current date and time to provide up-to-date details or time-sensitive responses. + + The repository you are querying is a public repository with the following name: microsoft/semantic-kernel + + The current date and time is: {current_time}. + """, + execution_settings=settings, + ) + + history = ChatHistory() + is_complete: bool = False + while not is_complete: + user_input = input("User:> ") + if not user_input: + continue + + if user_input.lower() == "exit": + is_complete = True + break + + history.add_message(ChatMessageContent(role=AuthorRole.USER, content=user_input)) + + async for response in agent.invoke(history=history): + print(f"{response.content}") + + +if __name__ == "__main__": + asyncio.run(main()) +``` +::: zone-end + +::: zone pivot="programming-language-java" + +> Agents are currently unavailable in Java. + +::: zone-end + + + +> [!div class="nextstepaction"] +> [How-To: _Open AI Assistant Agent_ Code Interpreter](./example-assistant-code.md) + + diff --git a/semantic-kernel/Frameworks/agent/index.md b/semantic-kernel/Frameworks/agent/index.md new file mode 100644 index 00000000..b64931a8 --- /dev/null +++ b/semantic-kernel/Frameworks/agent/index.md @@ -0,0 +1,89 @@ +--- +title: Semantic Kernel Agent Framework (Experimental) +description: Introducing the Semantic Kernel Agent Framework +zone_pivot_groups: programming-languages +author: crickman +ms.topic: tutorial +ms.author: crickman +ms.date: 09/13/2024 +ms.service: semantic-kernel +--- +# Semantic Kernel Agent Framework (Experimental) + +> [!WARNING] +> The _Semantic Kernel Agent Framework_ is experimental, still in development and is subject to change. + +The _Semantic Kernel Agent Framework_ provides a platform within the Semantic Kernel eco-system that allow for the creation of AI **agents** and the ability to incorporate **agentic patterns** into any application based on the same patterns and features that exist in the core _Semantic Kernel_ framework. + +## What is an AI agent? + +An **AI agent** is a software entity designed to perform tasks autonomously or semi-autonomously by recieving input, processing information, and taking actions to achieve specific goals. + +Agents can send and receive messages, generating responses using a combination of models, tools, human inputs, or other customizable components. + +Agents are designed to work collaboratively, enabling complex workflows by interacting with each other. The _Agent Framework_ allows for the creation of both simple and sophisticated agents, enhancing modularity and ease of maintenance + + +## What problems do AI agents solve? + +AI agents offers several advantages for application development, particularly by enabling the creation of modular AI components that are able to collaborate to reduce manual intervention in complex tasks. AI agents can operate autonomously or semi-autonomously, making them powerful tools for a range of applications. + +Here are some of the key benefits: + +- **Modular Components**: Allows developers to define various types of agents for specific tasks (e.g., data scraping, API interaction, or natural language processing). This makes it easier to adapt the application as requirements evolve or new technologies emerge. + +- **Collaboration**: Multiple agents may "collaborate" on tasks. For example, one agent might handle data collection while another analyzes it and yet another uses the results to make decisions, creating a more sophisticated system with distributed intelligence. + +- **Human-Agent Collaboration**: Human-in-the-loop interactions allow agents to work alongside humans to augment decision-making processes. For instance, agents might prepare data analyses that humans can review and fine-tune, thus improving productivity. + +- **Process Orchestration**: Agents can coordinate different tasks across systems, tools, and APIs, helping to automate end-to-end processes like application deployments, cloud orchestration, or even creative processes like writing and design. + + +## When to use an AI agent? + +Using an agent framework for application development provides advantages that are especially beneficial for certain types of applications. While traditional AI models are often used as tools to perform specific tasks (e.g., classification, prediction, or recognition), agents introduce more autonomy, flexibility, and interactivity into the development process. + +- **Autonomy and Decision-Making**: If your application requires entities that can make independent decisions and adapt to changing conditions (e.g., robotic systems, autonomous vehicles, smart environments), an agent framework is preferable. + +- **Multi-Agent Collaboration**: If your application involves complex systems that require multiple independent components to work together (e.g., supply chain management, distributed computing, or swarm robotics), agents provide built-in mechanisms for coordination and communication. + +- **Interactive and Goal-Oriented**: If your application involves goal-driven behavior (e.g., completing tasks autonomously or interacting with users to achieve specific objectives), agent-based frameworks are a better choice. Examples include virtual assistants, game AI, and task planners. + + +## How do I install the _Semantic Kernel Agent Framework_? + +Installing the _Agent Framework SDK_ is specific to the distribution channel associated with your programming language. + +::: zone pivot="programming-language-csharp" + +For .NET SDK, serveral NuGet packages are available. + +> Note: The core _Semantic Kernel SDK_ is required in addition to any agent packages. + + +Package|Description +--|-- +[Microsoft.SemanticKernel](https://www.nuget.org/packages/Microsoft.SemanticKernel)|This contains the core _Semantic Kernel_ libraries for getting started with the _Agent Framework_. This must be explicitly referenced by your application. +[Microsoft.SemanticKernel.Agents.Abstractions](https://www.nuget.org/packages/Microsoft.SemanticKernel.Agents.Abstractions)|Defines the core agent abstractions for the _Agent Framework_. Generally not required to be specified as it is included in both the `Microsoft.SemanticKernel.Agents.Core` and `Microsoft.SemanticKernel.Agents.OpenAI` packages. +[Microsoft.SemanticKernel.Agents.Core](https://www.nuget.org/packages/Microsoft.SemanticKernel.Agents.Core)|Includes the [`ChatCompletionAgent`](./chat-completion-agent.md) and [`AgentGroupChat`](./agent-chat.md) classes. +[Microsoft.SemanticKernel.Agents.OpenAI](https://www.nuget.org/packages/Microsoft.SemanticKernel.Agents.OpenAI)|Provides ability to use the [Open AI Assistant API](https://platform.openai.com/docs/assistants) via the [`OpenAIAssistantAgent`](./assistant-agent.md). + +::: zone-end + +::: zone pivot="programming-language-python" + +Module|Description +--|-- +[semantic-kernel.agents](https://pypi.org/project/semantic-kernel/)|This is the _Semantic Kernel_ library for getting started with the _Agent Framework_. This must be explicitly referenced by your application. This module contains the [`ChatCompletionAgent`](./chat-completion-agent.md) and [`AgentGroupChat`](./agent-chat.md) classes, as well as the ability to use the [Open AI Assistant API](https://platform.openai.com/docs/assistants) via the [`OpenAIAssistantAgent` or `AzureOpenAssistant`](./assistant-agent.md). + +::: zone-end + +::: zone pivot="programming-language-java" + +> Agents are currently unavailable in Java. + +::: zone-end + + +> [!div class="nextstepaction"] +> [Agent Architecture](./agent-architecture.md) diff --git a/semantic-kernel/Frameworks/process/TOC.yml b/semantic-kernel/Frameworks/process/TOC.yml new file mode 100644 index 00000000..a92ee4a7 --- /dev/null +++ b/semantic-kernel/Frameworks/process/TOC.yml @@ -0,0 +1,8 @@ +- name: Overview + href: process-framework.md +- name: Core Components + href: process-core-components.md +- name: Deployment + href: process-deployment.md +- name: Best Practices + href: process-best-practices.md diff --git a/semantic-kernel/Frameworks/process/process-best-practices.md b/semantic-kernel/Frameworks/process/process-best-practices.md new file mode 100644 index 00000000..c9325713 --- /dev/null +++ b/semantic-kernel/Frameworks/process/process-best-practices.md @@ -0,0 +1,35 @@ +--- +title: Process Framework Best Practices +description: Details on the Best Practices in the Processes Framework from Semantic Kernel +zone_pivot_groups: +author: evchaki +ms.topic: tutorial +ms.author: evchaki +ms.date: 09/28/2024 +ms.service: semantic-kernel +--- + +# Best Practices for the Process Framework + +Utilizing the Process Framework effectively can significantly enhance your workflow automation. Here are some best practices to help you optimize your implementation and avoid common pitfalls. + +### File and Folder Layout Structure +Organizing your project files in a logical and maintainable structure is crucial for collaboration and scalability. A recommended file layout may include: + +- **Processes/:** A directory for all defined processes. +- **Steps/:** A dedicated directory for reusable Steps. +- **Functions/:** A folder containing your Kernel Function definitions. + +An organized structure not only simplifies navigation within the project but also enhances code reusability and facilitates collaboration among team members. + + +### Common Pitfalls +To ensure smooth implementation and operation of the Process Framework, be mindful of these common pitfalls to avoid: + +- **Overcomplicating Steps:** Keep Steps focused on a single responsibility. Avoid creating complex Steps that perform multiple tasks, as this can complicate debugging and maintenance. + +- **Ignoring Event Handling:** Events are vital for smooth communication between Steps. Ensure that you handle all potential events and errors within the process to prevent unexpected behavior or crashes. + +- **Performance and Quality:** As processes scale, it’s crucial to continuously monitor performance. Leverage telemetry from your Steps to gain insights into how Processes are functioning. + +By following these best practices, you can maximize the effectiveness of the Process Framework, enabling more robust and manageable workflows. Keeping organization, simplicity, and performance in mind will lead to a smoother development experience and higher-quality applications. diff --git a/semantic-kernel/Frameworks/process/process-core-components.md b/semantic-kernel/Frameworks/process/process-core-components.md new file mode 100644 index 00000000..e3816318 --- /dev/null +++ b/semantic-kernel/Frameworks/process/process-core-components.md @@ -0,0 +1,81 @@ +--- +title: Process Framework Core Components +description: Details on the Core Components in the Processes Framework from Semantic Kernel +zone_pivot_groups: +author: evchaki +ms.topic: tutorial +ms.author: evchaki +ms.date: 09/28/2024 +ms.service: semantic-kernel +--- + +# Core Components of the Process Framework +The Process Framework is built upon a modular architecture that enables developers to construct sophisticated workflows through its core components. Understanding these components is essential for effectively leveraging the framework. + +## Process + +A Process serves as the overarching container that orchestrates the execution of Steps. It defines the flow and routing of data between Steps, ensuring that process goals are achieved efficiently. Processes handle inputs and outputs, providing flexibility and scalability across various workflows. + +### Process Features + +- **Stateful:** Supports querying information such as tracking status and percent completion, as well as the ability to pause and resume. +- **Reusable:** A Process can be invoked within other processes, promoting modularity and reusability. +- **Event Driven:** Employs event-based flow with listeners to route data to Steps and other Processes. +- **Scalable:** Utilizes well-established runtimes for global scalability and rollouts. +- **Cloud Event Integrated:** Incorporates industry-standard eventing for triggering a Process or Step. + +### Creating A Process + +To create a new Process, add the Process Package to your project and define a name for your process. + + + +## Step + +Steps are the fundamental building blocks within a Process. Each Step corresponds to a discrete unit of work and encapsulates one or more Kernel Functions. Steps can be created independently of their use in specific Processes, enhancing their reusability. They emit events based on the work performed, which can trigger subsequent Steps. + +### Step Features + +- **Stateful:** Facilitates tracking information such as status and defined tags. +- **Reusable:** Steps can be employed across multiple Processes. +- **Dynamic:** Steps can be created dynamically by a Process as needed, depending on the required pattern. +- **Flexible:** Offers different types of Steps for developers by leveraging Kernel Functions, including Code-only, API calls, AI Agents, and Human-in-the-loop. +- **Auditable:** Telemetry is enabled across both Steps and Processes. + +### Defining a Step + +To create a Step, define a public class to name the Step and add it to the KernelStepBase. Within your class, you can incorporate one or multiple Kernel Functions. + + + +### Register a Step into a Process + +Once your class is created, you need to register it within your Process. For the first Step in the Process, add `isEntryPoint: true` so the Process knows where to start. + + + + +### Step Events + +Steps have several events available, including: + +- **OnEvent:** Triggered when the class completes its execution. +- **OnFunctionResult:** Activated when the defined Kernel Function emits results, allowing output to be sent to one or many Steps. +- **SendOutputTo:** Defines the Step and Input for sending results to a subsequent Step. + + + +## Pattern + +Patterns standardize common process flows, simplifying the implementation of frequently used operations. They promote a consistent approach to solving recurring problems across various implementations, enhancing both maintainability and readability. + +### Pattern Types + +- **Fan In:** The input for the next Step is supported by multiple outputs from previous Steps. +- **Fan Out:** The output of previous Steps is directed into multiple Steps further down the Process. +- **Cycle:** Steps continue to loop until completion based on input and output. +- **Map Reduce:** Outputs from a Step are consolidated into a smaller amount and directed to the next Step's input. + +### Setting up a Pattern + +Once your class is created for your Step and registered within the Process, you can define the events that should be sent downstream to other Steps or set conditions for Steps to be restarted based on the output from your Step. diff --git a/semantic-kernel/Frameworks/process/process-deployment.md b/semantic-kernel/Frameworks/process/process-deployment.md new file mode 100644 index 00000000..0302799f --- /dev/null +++ b/semantic-kernel/Frameworks/process/process-deployment.md @@ -0,0 +1,32 @@ +--- +title: Process Framework Deployments +description: Details on the Deployment Options in the Processes Framework from Semantic Kernel +zone_pivot_groups: +author: evchaki +ms.topic: tutorial +ms.author: evchaki +ms.date: 09/28/2024 +ms.service: semantic-kernel +--- + +# Deployment of the Process Framework +Deploying workflows built with the Process Framework can be done seamlessly across local development environments and cloud runtimes. This flexibility enables developers to choose the best approach tailored to their specific use cases. + +## Local Development + +The Process Framework provides an in-process runtime that allows developers to run processes directly on their local machines or servers without requiring complex setups or additional infrastructure. This runtime supports both memory and file-based persistence, ideal for rapid development and debugging. You can quickly test processes with immediate feedback, accelerating the development cycle and enhancing efficiency. + +## Cloud Runtimes + +For scenarios requiring scalability and distributed processing, the Process Framework supports cloud runtimes such as [**Orleans**](https://learn.microsoft.com/dotnet/orleans/overview) and [**Dapr**](https://dapr.io/). These options empower developers to deploy processes in a distributed manner, facilitating high availability and load balancing across multiple instances. By leveraging these cloud runtimes, organizations can streamline their operations and manage substantial workloads with ease. + +- **Orleans Runtime:** This framework provides a programming model for building distributed applications and is particularly well-suited for handling virtual actors in a resilient manner, complementing the Process Framework’s event-driven architecture. + +- **Dapr (Distributed Application Runtime):** Dapr simplifies microservices development by providing a foundational framework for building distributed systems. It supports state management, service invocation, and pub/sub messaging, making it easier to connect various components within a cloud environment. + +Using either runtime, developers can scale applications according to demand, ensuring that processes run smoothly and efficiently, regardless of workload. + +With the flexibility to choose between local testing environments and robust cloud platforms, the Process Framework is designed to meet diverse deployment needs. This enables developers to concentrate on building innovative AI-powered processes without the burden of infrastructure complexities. + +> [!NOTE] +> Orleans will be supported first with the .NET Process Framework, followed by Dapr in the upcoming release of the Python version of the Process Framework. diff --git a/semantic-kernel/Frameworks/process/process-framework.md b/semantic-kernel/Frameworks/process/process-framework.md new file mode 100644 index 00000000..ed804b6b --- /dev/null +++ b/semantic-kernel/Frameworks/process/process-framework.md @@ -0,0 +1,62 @@ +--- +title: Process Framework +description: Details on the Processes Framework from Semantic Kernel +zone_pivot_groups: +author: evchaki +ms.topic: tutorial +ms.author: evchaki +ms.date: 09/28/2024 +ms.service: semantic-kernel +--- + +# Overview of the Process Framework + +Welcome to the Process Framework within Microsoft's Semantic Kernel—a cutting-edge approach designed to optimize AI integration with your business processes. This framework empowers developers to efficiently create, manage, and deploy business processes while leveraging the powerful capabilities of AI, alongside your existing code and systems. + +A Process is a structured sequence of activities or tasks that deliver a service or product, adding value in alignment with specific business goals for customers. + +> [!NOTE] +> Process Framework package is currently experimental and is subject to change until it is moved to preview and GA. + +## Introduction to the Process Framework + +The Process Framework provides a robust solution for automating complex workflows. Each step within the framework performs tasks by invoking user-defined Kernel Functions, utilizing an event-driven model to manage workflow execution. + +By embedding AI into your business processes, you can significantly enhance productivity and decision-making capabilities. With the Process Framework, you benefit from seamless AI integration, facilitating smarter and more responsive workflows. This framework streamlines operations, fosters improved collaboration between business units, and boosts overall efficiency. + + +## Key Features + +- **Leverage Semantic Kernel:** Steps can utilize one or multiple Kernel Functions, enabling you to tap into all aspects of Semantic Kernel within your processes. +- **Reusability & Flexibility:** Steps and processes can be reused across different applications, promoting modularity and scalability. +- **Event-Driven Architecture:** Utilize events and metadata to trigger actions and transitions between process steps effectively. +- **Full Control and Auditability:** Maintain control of processes in a defined and repeatable manner, complete with audit capabilities through Open Telemetry. + + +## Core Concepts + +- **Process:** A collection of steps arranged to achieve a specific business goal for customers. +- **Step:** An activity within a process that has defined inputs and outputs, contributing to a larger goal. +- **Pattern:** The specific sequence type that dictates how steps are executed for the process to be fully completed. + + +## Business Process Examples + +Business processes are a part of our daily routines. Here are three examples you might have encountered this week: + +1. **Account Opening:** This process includes multiple steps such as credit pulls and ratings, fraud detection, creating customer accounts in core systems, and sending welcome information to the customer, including their customer ID. +2. **Food Delivery:** Ordering food for delivery is a familiar process. From receiving the order via phone, website, or app, to preparing each food item, ensuring quality control, driver assignment, and final delivery, there are many steps in this process that can be streamlined. +3. **Support Ticket:** We have all submitted support tickets—whether for new services, IT support, or other needs. This process can involve multiple subprocesses based on business and customer requirements, ultimately aiming for satisfaction by addressing customer needs effectively. + + +## Getting Started + +Are you ready to harness the power of the Process Framework? + +Begin your journey by exploring our [.NET samples](https://aka.ms/sk/process/dotnet) on GitHub. While Python support is on the horizon, the .NET examples provide an excellent starting point for understanding the framework's capabilities and applications. + +> [!NOTE] +> Process Framework is avaialble for .NET currently. The Process Framework for Python is in progress. + + +By diving into the Process Framework, developers can transform traditional workflows into intelligent, adaptive systems. Start building with the tools at your disposal and redefine what's possible with AI-driven business processes. diff --git a/semantic-kernel/TOC.yml b/semantic-kernel/TOC.yml index 2fe11148..e9eee636 100644 --- a/semantic-kernel/TOC.yml +++ b/semantic-kernel/TOC.yml @@ -6,10 +6,12 @@ - name: Concepts href: concepts/TOC.yml expanded: true -- name: Integrations - href: integrations/index.md +- name: Frameworks + href: frameworks/TOC.yml + expanded: true - name: Getting Support href: support/TOC.yml + expanded: true - name: API Reference href: api-docs/TOC.yml expanded: true diff --git a/semantic-kernel/concepts/TOC.yml b/semantic-kernel/concepts/TOC.yml index 73d24ce8..d606e84d 100644 --- a/semantic-kernel/concepts/TOC.yml +++ b/semantic-kernel/concepts/TOC.yml @@ -1,12 +1,16 @@ -- name: Agents - href: agents.md - name: Kernel href: kernel.md - name: AI Services href: ai-services/TOC.yml +- name: Enterprise Components + href: enterprise-readiness/TOC.yml +- name: Memory (Vector Stores) + href: vector-store-connectors/TOC.yml +- name: Prompt Engineering + href: prompts/TOC.yml - name: Plugins href: plugins/TOC.yml +- name: Text Search (RAG) + href: text-search/TOC.yml - name: Planning href: planning.md -- name: Personas - href: personas.md \ No newline at end of file diff --git a/semantic-kernel/concepts/agents.md b/semantic-kernel/concepts/agents.md deleted file mode 100644 index b699400f..00000000 --- a/semantic-kernel/concepts/agents.md +++ /dev/null @@ -1,448 +0,0 @@ ---- -title: Building Agents with Semantic Kernel -description: Learn about agents and how to build them with Semantic Kernel. -author: sophialagerkranspandey -zone_pivot_groups: programming-languages -ms.topic: overview -ms.author: sopand -ms.date: 07/11/2023 -ms.service: semantic-kernel -ms.custom: build-2023, build-2023-dataai ---- - -# What are agents? -:::row::: - :::column span="3"::: - Agents are software based entities that leverage AI models to do works for you. They are built to perform a wide range of tasks and called different names based on the jobs they do. - - For example, an agent built to answer questions is referred to as a chatbot due to it’s chat-based experience. These agents are typically grounded in your own data, like company documents. - - An agent built to work along side you is typically called a copilot. These agents help you complete tasks like writing emails or creating other office artifacts by providing recommendations that you choose to either accept or reject. - - An agent built to handle repetitive tasks are usually labeled as fully autonomous. They respond to events and perform actions without needing intervention. This is different from a copilot in that it’s doing work for you instead of alongside you. - - Semantic Kernel provides you with the infrastructure to build any kind of agent you need without being an expert in AI. - :::column-end::: - :::column span="1"::: - ![3 kinds of agents: Chatbot Copilot and Fully autonomous](../media/what-are-agents.png) - :::column-end::: -:::row-end::: - -## Building your first agent - -:::row::: - :::column span="1"::: - An agent is made up of three core building blocks that harmoniously work together to complete tasks for you. Let’s take a closer look by building a copilot that helps you write and send an email. - :::column-end::: - :::column span="2"::: - ![Building Blocks of an Agent](../media/building-blocks.png) - :::column-end::: -:::row-end::: - - -### A copilot to write and send emails -To create this copilot you’ll need to create a plugin that can send emails, a planner that generates a plan to write an email, and a persona that interacts with you to get the information needed to send the email. - -| Step | Description | -|------|---------------------------------------| -| 1 | Get the user's email address and name | -| 2 | Get the email address of the recipient| -| 3 | Get the topic of the email | -| 4 | Generate the subject and body of the email | -| 5 | Review the email with the user | -| 6 | Send the email | - -To enable this scenario, we would need to create a plugin that can send emails, a planner that can generate a plan to write an email, and a persona that can interact with the user to get the necessary information. - -The following sections will walk you through the conceptual building blocks and how to put them together to build your first agent. Afterwards, you can refer to the specific guides for each building block to learn more about how to build them. - -### Plugins: giving your agent skills -To generate the above plan, the copilot first needs the capabilities necessary to perform these steps. This is where plugins come in. Plugins allow you to give your agent skills via code. For example, you could create a plugin that sends emails, retrieves information from a database, asks for help, or even saves and retrieves memories from previous conversations. - -In our example, we can build a simple plugin that sends emails using [native code](./plugins/adding-native-plugins.md). Our plugin just has a single function, `send_email`, that takes in the email address, subject, and body of the email. It would then use this information to send the email. - -::: zone pivot="programming-language-csharp" -```csharp -using System.ComponentModel; -using Microsoft.SemanticKernel; - -public class EmailPlugin -{ - [KernelFunction("send_email")] - [Description("Sends an email to a recipient.")] - public async Task SendEmailAsync( - Kernel kernel, - List recipientEmails, - string subject, - string body - ) - { - // Add logic to send an email using the recipientEmails, subject, and body - // For now, we'll just print out a success message to the console - Console.WriteLine("Email sent!"); - } -} -``` -::: zone-end - -::: zone pivot="programming-language-python" -```python -from semantic_kernel.functions import kernel_function - -class EmailPlugin: - @kernel_function( - name="send_email", - description="Sends an email to a recipient." - ) - async def send_email(self, recipient_emails: str|List[str], subject: str, body: str): - # Add logic to send an email using the recipient_emails, subject, and body - # For now, we'll just print out a success message to the console - print("Email sent!") -``` -::: zone-end - -::: zone pivot="programming-language-java" - -> [!NOTE] -> Agents are currently unavailable in Java. -::: zone-end - -There are other ways to create plugins. For example, if you have a RestAPI that can send emails, you can automatically create a plugin using its [OpenAPI specification](./plugins/adding-openapi-plugins.md). To learn more about other ways to author plugins, see the [plugins section](./plugins/index.md). - -### Planning: completing multi-step tasks -To actually use this plugin (and to wire them up with other steps), the copilot would then need to generate a plan. This is where planning comes in. Planning comes from the built-in ability of LLMs to determine how to iteratively complete a task. - -> [!NOTE] -> In the past, special prompts were created by AI app developers to guide the AI in generating a plan that could be consumed by an SDK like Semantic Kernel. However, with the advent of LLMs, AIs can now generate plans directly from a conversation with a user with the aid of function calling. - -With built-in planning support form LLMs, using Semantic Kernel is now as easy as invoking a chat completion service with auto function calling enabled. - -::: zone pivot="programming-language-csharp" -```csharp -using Microsoft.SemanticKernel; -using Microsoft.SemanticKernel.ChatCompletion; -using Microsoft.SemanticKernel.Connectors.OpenAI; - -// Create kernel with an email plugin -var builder = Kernel.CreateBuilder(); -builder.Plugins.AddFromType(); -Kernel kernel = builder.Build(); - -// Create chat completion service -AzureOpenAIChatCompletionService chatCompletionService = new ( - deploymentName: "gpt-4", - apiKey: "YOUR_API_KEY", - endpoint: "YOUR_AZURE_ENDPOINT" -); - -// Enable planning -OpenAIPromptExecutionSettings openAIPromptExecutionSettings = new()  -{ - ToolCallBehavior = ToolCallBehavior.AutoInvokeKernelFunctions -}; - -// Create chat history -var history = new ChatHistory(); -history.AddUserMessage("Can you help me write an email for my boss?"); - -// Get the response from the AI -var result = await chatCompletionService.GetChatMessageContentAsync( - history, - executionSettings: openAIPromptExecutionSettings, - kernel: kernel -); -``` -::: zone-end - -::: zone pivot="programming-language-python" -```python -import asyncio -import logging - -from semantic_kernel import Kernel -from semantic_kernel.functions import kernel_function -from semantic_kernel.connectors.ai.open_ai import AzureChatCompletion -from semantic_kernel.connectors.ai.function_call_behavior import FunctionCallBehavior -from semantic_kernel.connectors.ai.chat_completion_client_base import ChatCompletionClientBase -from semantic_kernel.contents.chat_history import ChatHistory -from semantic_kernel.functions.kernel_arguments import KernelArguments - -from semantic_kernel.connectors.ai.open_ai.prompt_execution_settings.azure_chat_prompt_execution_settings import ( - AzureChatPromptExecutionSettings, -) - -async def main(): - # Initialize the kernel - kernel = Kernel() - - # Add Azure OpenAI chat completion - kernel.add_service(AzureChatCompletion( - deployment_name="your_models_deployment_name", - api_key="your_api_key", - base_url="your_base_url", - )) - - # Set the logging level for semantic_kernel.kernel to DEBUG. - logging.basicConfig( - format="[%(asctime)s - %(name)s:%(lineno)d - %(levelname)s] %(message)s", - datefmt="%Y-%m-%d %H:%M:%S", - ) - logging.getLogger("kernel").setLevel(logging.DEBUG) - - # Add a plugin (the EmailPlugin class is defined above) - kernel.add_plugin( - EmailPlugin(), - plugin_name="Email", - ) - - chat_completion : AzureChatCompletion = kernel.get_service(type=ChatCompletionClientBase) - - # Enable planning - execution_settings = AzureChatPromptExecutionSettings(tool_choice="auto") - execution_settings.function_call_behavior = FunctionCallBehavior.EnableFunctions(auto_invoke=True, filters={}) - - # Create a history of the conversation - history = ChatHistory() - history.add_user_message("Can you help me write an email for my boss?") - - result = (await chat_completion.get_chat_message_contents( - chat_history=history, - settings=execution_settings, - kernel=kernel, - arguments=KernelArguments(), - ))[0] - print(result) - -# Run the main function -if __name__ == "__main__": - asyncio.run(main()) -``` -::: zone-end - -::: zone pivot="programming-language-java" - -> [!NOTE] -> Agents are currently unavailable in Java. -::: zone-end - - -To learn more about planning with Semantic Kernel, see the [planning article](./planning.md). - -### Personas: giving your agent a job description -In most cases, using the built-in planning capabilities of LLMs with plugins is sufficient to building an agent, but as you build more domain-specific agents, you may want to add a persona to your agent. - -A persona is the instructions that you provide your agent so they can more effectively perform the role you want them to play. At its simplest, the persona could instruct the AI to be polite, ask for clarification when needed, or role-play as a specific job title (e.g., a customer service representative). - -For enterprise scenarios, however, you may want to provide more detailed instructions to your agent. For example, you may want to instruct your agent to follow specific rules, ask for approval before taking actions, or escalate to a human if the AI is unable to complete a task. - -To provide a persona to your agent, simply pre-pend a system message to the chat history that describes the persona. The AI will then use this persona to guide its interactions with the user. - -::: zone pivot="programming-language-csharp" -```csharp -// Create chat history -ChatHistory chatMessages = new ChatHistory(""" - You are a friendly assistant who likes to follow the rules. You will complete required steps - and request approval before taking any consequential actions. If the user doesn't provide - enough information for you to complete a task, you will keep asking questions until you have - enough information to complete the task. - """); -chatMessages.AddUserMessage("Can you help me write an email for my boss?"); - -// Get the response from the AI -var result = await chatCompletionService.GetChatMessageContentAsync( - history, - executionSettings: openAIPromptExecutionSettings, - kernel: kernel -); -``` -::: zone-end - -::: zone pivot="programming-language-python" -```python -# Create a history of the conversation -history = ChatHistory(system_message=""" - You are a friendly assistant who likes to follow the rules. You will complete required steps - and request approval before taking any consequential actions. If the user doesn't provide - enough information for you to complete a task, you will keep asking questions until you have - enough information to complete the task. - """) -history.add_user_message("Can you help me write an email for my boss?") - -# Get the response from the AI -result = (await chat_completion.get_chat_message_contents( - chat_history=history, - settings=execution_settings, - kernel=kernel, - arguments=KernelArguments(), -))[0] -``` -::: zone-end - -::: zone pivot="programming-language-java" - -> [!NOTE] -> Agents are currently unavailable in Java. -::: zone-end - -To learn more about authoring effective personas, refer to the [personas article](./personas.md). - -## Putting the pieces together -Now that we understand the core building blocks of an agent, we can now combine them together to build our first agent. To do so, we'll initialize our `Kernel` object with our plugins, planners, and persona. Afterwards, we'll use the `Kernel` object to generate a plan and then execute that plan. - -::: zone pivot="programming-language-csharp" -```csharp -// Create the kernel -var builder = Kernel.CreateBuilder(); -builder.Services.AddLogging(c => c.SetMinimumLevel(LogLevel.Trace).AddDebug()); -builder.Services.AddChatCompletionService(kernelSettings); -builder.Plugins.AddFromType(); -Kernel kernel = builder.Build(); - -// Retrieve the chat completion service from the kernel -IChatCompletionService chatCompletionService = kernel.GetRequiredService(); - -// Create the chat history -ChatHistory chatMessages = new ChatHistory(""" - You are a friendly assistant who likes to follow the rules. You will complete required steps - and request approval before taking any consequential actions. If the user doesn't provide - enough information for you to complete a task, you will keep asking questions until you have - enough information to complete the task. - """); - -// Start the conversation -while (true) -{ - // Get user input - System.Console.Write("User > "); - chatMessages.AddUserMessage(Console.ReadLine()!); - - // Get the chat completions - OpenAIPromptExecutionSettings openAIPromptExecutionSettings = new() - { - FunctionCallBehavior = FunctionCallBehavior.AutoInvokeKernelFunctions - }; - var result = chatCompletionService.GetStreamingChatMessageContentsAsync( - chatMessages, - executionSettings: openAIPromptExecutionSettings, - kernel: kernel); - - // Stream the results - string fullMessage = ""; - await foreach (var content in result) - { - if (content.Role.HasValue) - { - System.Console.Write("Assistant > "); - } - System.Console.Write(content.Content); - fullMessage += content.Content; - } - System.Console.WriteLine(); - - // Add the message from the agent to the chat history - chatMessages.AddAssistantMessage(fullMessage); -} -``` -::: zone-end - -::: zone pivot="programming-language-python" -```python -import asyncio - -from semantic_kernel import Kernel -from semantic_kernel.contents.chat_history import ChatHistory -from semantic_kernel.connectors.ai.open_ai import AzureChatCompletion -from semantic_kernel.connectors.ai.function_call_behavior import FunctionCallBehavior -from semantic_kernel.connectors.ai.chat_completion_client_base import ChatCompletionClientBase -from semantic_kernel.connectors.ai.open_ai.prompt_execution_settings.azure_chat_prompt_execution_settings import ( - AzureChatPromptExecutionSettings, -) -from semantic_kernel.functions.kernel_arguments import KernelArguments - -async def main(): - # Initialize the kernel - kernel = Kernel() - - # Add Azure OpenAI chat completion - kernel.add_service(AzureChatCompletion( - deployment_name="your_models_deployment_name", - api_key="your_api_key", - base_url="your_base_url", - )) - - # Add a plugin (the EmailPlugin class is defined above) - kernel.add_plugin( - EmailPlugin(), - plugin_name="Email", - ) - - chat_completion : AzureChatCompletion = kernel.get_service(type=ChatCompletionClientBase) - - # Enable planning - execution_settings = AzureChatPromptExecutionSettings(tool_choice="auto") - execution_settings.function_call_behavior = FunctionCallBehavior.EnableFunctions(auto_invoke=True, filters={}) - - # Create a history of the conversation - history = ChatHistory() - - # Start the conversation - while True: - # Get user input - user_input = input("User > ") - history.add_user_message(user_input) - - # Get the response from the AI - result = (await chat_completion.get_chat_message_contents( - chat_history=history, - settings=execution_settings, - kernel=kernel, - arguments=KernelArguments(), - ))[0] - - # Print the response - print("Assistant > " + str(result)) - history.add_assistant_message(str(result)) - -# Run the main function -if __name__ == "__main__": - asyncio.run(main()) -``` -::: zone-end - -::: zone pivot="programming-language-java" - -> [!NOTE] -> Agents are currently unavailable in Java. -::: zone-end - - -## Trying out your agent -Now that we've completed our program, we can give it a test run. Below is a sample conversation with our agent while using gpt-4 model. - -| Role | Message | -| ----------------------------- | ------------------------------------------------------------------------------------------------------------------------------------- | -| 🔵 **User** | Can you help me write an email for my boss? | -| 🔴 **Assistant** | Of course, I'd be happy to help you write an email to your boss. Could you please provide me with more details? Specifically, I need to know:
1. The topic or purpose of the email.
2. Your boss's email address. | -| 🔵 **User** | I want to give her an update on last months sales. We broke a bunch of records that I want to share with her, but we did have a challenge selling the X4321 model. | -| 🔴 **Assistant** | That sounds like a great topic for an email. Could you please provide me with your boss's email address so I can assist you further? | -| 🔵 **User** | Sure! It's sarah@contoso.com | -| 🔴 **Assistant** | Great! Here's a plan for your email:
1. **Start with a Clear Subject Line and Greeting**: The subject line should be concise and informative, such as "Update on Last Month's Sales & X4321 Model Challenges". Start the email with a professional greeting, like "Dear Sarah,".
2. **Highlight Achievements and Challenges in Separate Paragraphs**: Begin with the positive news about the record-breaking achievements. Use specific data and examples to illustrate the success. Then, transition to the challenges with the X4321 model. Be honest and clear about the issues, but also suggest potential solutions or steps being taken to address these challenges.
3. **End with a Call to Action and Professional Closing**: If you need Sarah to take any actions, state this clearly towards the end of the email. This could be asking for her input, a meeting to discuss further, or any other relevant action. Close the email professionally with a phrase like "Best regards" or "Sincerely", followed by your name.
Remember to proofread the email for any grammatical errors or unclear statements before sending it. This will help ensure the message is professional and easy to understand.
Would you like me to draft the email for you based on these steps? | -| 🔵 **User** | Yes please! | -| 🔴 **Assistant** | Sure, here's a draft based on the steps we discussed:
**Subject**: Update on Last Month's Sales & X4321 Model Challenges
**Dear Sarah,**
I hope this email finds you well. I am writing to update you on our sales performance for the past month.
I am pleased to report that we broke several sales records last month. Our team's hard work and dedication have paid off, resulting in significant growth in our overall sales figures. [Here, you can add specific data or examples of the records broken]
However, we did face some challenges with the X4321 model. Despite our best efforts, the sales for this model did not meet our expectations. [Here, you can explain more about the challenges faced]
We are currently working on strategies to improve the sales of the X4321 model. [Here, you can mention any specific steps or plans you have to address the issue]
I would appreciate your input on this matter. If you would like, we can schedule a meeting to discuss this further.
Thank you for your continued support and guidance.
**Best regards,**
[Your Name]
Please review this draft and let me know if you would like to make any changes before I send it. | -| 🔵 **User** | Can you add that we increased our revenue by 20%? You can also share that the X4321 didn't sell as well since most customers upgraded to the Y5000 model, but this isn't too bad since it's what helped increase our revenue. | -| 🔴 **Assistant** | Absolutely, here's the updated draft:
**Subject**: Update on Last Month's Sales & X4321 Model Challenges
**Dear Sarah,**
I hope this email finds you well. I am writing to update you on our sales performance for the past month.
I am pleased to report that we broke several sales records last month. Our team's hard work and dedication have paid off, resulting in a significant 20% increase in our overall revenue. This growth is a testament to our team's commitment and the quality of our products.
However, we did face some challenges with the X4321 model. Despite our best efforts, the sales for this model did not meet our expectations. Interestingly, most customers upgraded to the Y5000 model, which contributed significantly to our increased revenue. While we aim to improve the sales of all our models, this shift in customer preference is not entirely negative.
We are currently analyzing this trend and working on strategies to leverage it for future sales. Your insights and suggestions would be greatly appreciated.
Thank you for your continued support and guidance.
**Best regards,**
**Stephen**
Please review this updated draft and let me know if you would like to make any changes before I send it. | -| 🔵 **User** | Please sign it with Stephen and then you can go ahead and send it to Sarah | -| 🟢 **Tool** | Email sent! | -| 🔴 **Assistant** | The email has been sent to Sarah. If there's anything else you need assistance with, feel free to ask! | - - -## Next steps -Now that you've built your first agent, you can now deep dive into the different components of Semantic Kernel. - -| Section | Description | -| ------------------------------------------- | ---------------------------------------------------------- | -| [Kernel](./kernel.md) | Setting up and using the core Kernel object | -| [AI Services](./ai-services/index.md) | Integration and usage of various AI services | -| [Plugins](./plugins/index.md) | Integrate native code and APIs with Semantic Kernel | -| [Planners](./planning.md) | Enabling your AI agents to complete multi-step tasks | -| [Personas](./personas.md) | Customize your AI agents to better collaborate with others | diff --git a/semantic-kernel/concepts/ai-services/TOC.yml b/semantic-kernel/concepts/ai-services/TOC.yml index ac44a7a6..d2d469f5 100644 --- a/semantic-kernel/concepts/ai-services/TOC.yml +++ b/semantic-kernel/concepts/ai-services/TOC.yml @@ -2,4 +2,6 @@ href: index.md - name: Chat completion href: chat-completion/TOC.yml - expanded: true \ No newline at end of file + expanded: true +- name: AI Integrations + href: integrations.md \ No newline at end of file diff --git a/semantic-kernel/concepts/ai-services/chat-completion/TOC.yml b/semantic-kernel/concepts/ai-services/chat-completion/TOC.yml index c4e11a15..4a955be4 100644 --- a/semantic-kernel/concepts/ai-services/chat-completion/TOC.yml +++ b/semantic-kernel/concepts/ai-services/chat-completion/TOC.yml @@ -3,4 +3,4 @@ - name: Chat history href: chat-history.md - name: Function calling - href: function-calling.md \ No newline at end of file + href: function-calling/TOC.yml \ No newline at end of file diff --git a/semantic-kernel/concepts/ai-services/chat-completion/chat-history.md b/semantic-kernel/concepts/ai-services/chat-completion/chat-history.md index 04bc9c51..24042c5c 100644 --- a/semantic-kernel/concepts/ai-services/chat-completion/chat-history.md +++ b/semantic-kernel/concepts/ai-services/chat-completion/chat-history.md @@ -373,12 +373,11 @@ chat_history = ChatHistory([ current_chat_history_length = len(chat_history) # Get the chat message content -results = (await chat_completion.get_chat_message_contents( +results = await chat_completion.get_chat_message_content( chat_history=history, settings=execution_settings, kernel=kernel, - arguments=KernelArguments(), -))[0] +) # Get the new messages added to the chat history object for i in range(current_chat_history_length, len(chat_history)): @@ -417,7 +416,7 @@ chatHistory.addAll(results); ::: zone-end ## Next steps -Now that you know how to create and manage a chat history object, you can learn more about function calling in the [Function calling](./function-calling.md) topic. +Now that you know how to create and manage a chat history object, you can learn more about function calling in the [Function calling](./function-calling/index.md) topic. > [!div class="nextstepaction"] -> [Learn how function calling works](./function-calling.md) \ No newline at end of file +> [Learn how function calling works](./function-calling/index.md) \ No newline at end of file diff --git a/semantic-kernel/concepts/ai-services/chat-completion/function-calling/TOC.yml b/semantic-kernel/concepts/ai-services/chat-completion/function-calling/TOC.yml new file mode 100644 index 00000000..644fa098 --- /dev/null +++ b/semantic-kernel/concepts/ai-services/chat-completion/function-calling/TOC.yml @@ -0,0 +1,6 @@ +- name: Overview + href: index.md +- name: Function Choice Behaviors + href: function-choice-behaviors.md +- name: Function Invocation + href: function-invocation.md \ No newline at end of file diff --git a/semantic-kernel/concepts/ai-services/chat-completion/function-calling/function-choice-behaviors.md b/semantic-kernel/concepts/ai-services/chat-completion/function-calling/function-choice-behaviors.md new file mode 100644 index 00000000..b74009f2 --- /dev/null +++ b/semantic-kernel/concepts/ai-services/chat-completion/function-calling/function-choice-behaviors.md @@ -0,0 +1,286 @@ +--- +title: Function Choice Behavior +description: Describes function choice behavior types Semantic Kernel supports. +zone_pivot_groups: programming-languages +author: SergeyMenshykh +ms.topic: conceptual +ms.author: semenshi +ms.service: semantic-kernel +--- + +::: zone pivot="programming-language-csharp" +# Function Choice Behaviors + +Function choice behaviors are bits of configuration that allows a developer to configure: +1. Which functions are advertised to AI models. +2. How the models should choose them for invocation. +3. How Semantic Kernel might invoke those functions. + +As of today, the function choice behaviors are represented by three static methods of the `FunctionChoiceBehavior` class: +- **Auto**: Allows the AI model to decide to choose from zero or more of the provided function(s) for invocation. +- **Required**: Forces the AI model to choose provided function(s). +- **None**: Instructs the AI model not to choose any function(s). + +> [!WARNING] +> The function-calling capability is experimental and subject to change. It is expected to reach general availability (GA) by mid-November 2024. +> Please refer to the [migration guide](../../../../support/migration/function-calling-migration-guide.md) to migrate your code to the latest function-calling capabilities. + +> [!NOTE] +> The function-calling capabilities is only supported by a few AI connectors so far, see the [Supported AI Connectors](./function-choice-behaviors.md#supported-ai-connectors) section below for more details. + +## Function Advertising + +Function advertising is the process of providing functions to AI models for further calling and invocation. All three function choice behaviors accept a list of functions to advertise as a `functions` parameter. By default, it is null, which means all functions from plugins registered on the Kernel are provided to the AI model. + +```csharp +using Microsoft.SemanticKernel; + +IKernelBuilder builder = Kernel.CreateBuilder(); +builder.AddOpenAIChatCompletion("", ""); +builder.Plugins.AddFromType(); +builder.Plugins.AddFromType(); + +Kernel kernel = builder.Build(); + +// All functions from the DateTimeUtils and WeatherForecastUtils plugins will be sent to AI model together with the prompt. +PromptExecutionSettings settings = new() { FunctionChoiceBehavior = FunctionChoiceBehavior.Auto() }; + +await kernel.InvokePromptAsync("Given the current time of day and weather, what is the likely color of the sky in Boston?", new(settings)); +``` + +If a list of functions is provided, only those functions are sent to the AI model: +```csharp +using Microsoft.SemanticKernel; + +IKernelBuilder builder = Kernel.CreateBuilder(); +builder.AddOpenAIChatCompletion("", ""); +builder.Plugins.AddFromType(); +builder.Plugins.AddFromType(); + +Kernel kernel = builder.Build(); + +KernelFunction getWeatherForCity = kernel.Plugins.GetFunction("WeatherForecastUtils", "GetWeatherForCity"); +KernelFunction getCurrentTime = kernel.Plugins.GetFunction("DateTimeUtils", "GetCurrentUtcDateTime"); + +// Only the specified getWeatherForCity and getCurrentTime functions will be sent to AI model alongside the prompt. +PromptExecutionSettings settings = new() { FunctionChoiceBehavior = FunctionChoiceBehavior.Auto(functions: [getWeatherForCity, getCurrentTime]) }; + +await kernel.InvokePromptAsync("Given the current time of day and weather, what is the likely color of the sky in Boston?", new(settings)); +``` + +An empty list of functions means no functions are provided to the AI model, which is equivalent to disabling function calling. +```csharp +using Microsoft.SemanticKernel; + +IKernelBuilder builder = Kernel.CreateBuilder(); +builder.AddOpenAIChatCompletion("", ""); +builder.Plugins.AddFromType(); +builder.Plugins.AddFromType(); + +Kernel kernel = builder.Build(); + +// Disables function calling. Equivalent to var settings = new() { FunctionChoiceBehavior = null } or var settings = new() { }. +PromptExecutionSettings settings = new() { FunctionChoiceBehavior = FunctionChoiceBehavior.Auto(functions: []) }; + +await kernel.InvokePromptAsync("Given the current time of day and weather, what is the likely color of the sky in Boston?", new(settings)); +``` +## Using Auto Function Choice Behavior + +The `Auto` function choice behavior instructs the AI model to decide to choose from zero or more of the provided function(s) for invocation. + +In this example, all the functions from the `DateTimeUtils` and `WeatherForecastUtils` plugins will be provided to the AI model alongside the prompt. +The model will first choose `GetCurrentTime` function for invocation to obtain the current date and time, as this information is needed as input for the `GetWeatherForCity` function. +Next, it will choose `GetWeatherForCity` function for invocation to get the weather forecast for the city of Boston using the obtained date and time. +With this information, the model will be able to determine the likely color of the sky in Boston. +```csharp +using Microsoft.SemanticKernel; + +IKernelBuilder builder = Kernel.CreateBuilder(); +builder.AddOpenAIChatCompletion("", ""); +builder.Plugins.AddFromType(); +builder.Plugins.AddFromType(); + +Kernel kernel = builder.Build(); + +// All functions from the DateTimeUtils and WeatherForecastUtils plugins will be provided to AI model alongside the prompt. +PromptExecutionSettings settings = new() { FunctionChoiceBehavior = FunctionChoiceBehavior.Auto() }; + +await kernel.InvokePromptAsync("Given the current time of day and weather, what is the likely color of the sky in Boston?", new(settings)); +``` + +The same example can be easily modeled in a YAML prompt template configuration: +```csharp +using Microsoft.SemanticKernel; + +IKernelBuilder builder = Kernel.CreateBuilder(); +builder.AddOpenAIChatCompletion("", ""); +builder.Plugins.AddFromType(); +builder.Plugins.AddFromType(); + +Kernel kernel = builder.Build(); + +string promptTemplateConfig = """ + template_format: semantic-kernel + template: Given the current time of day and weather, what is the likely color of the sky in Boston? + execution_settings: + default: + function_choice_behavior: + type: auto + """; + +KernelFunction promptFunction = KernelFunctionYaml.FromPromptYaml(promptTemplateConfig); + +Console.WriteLine(await kernel.InvokeAsync(promptFunction)); +``` + +## Using Required Function Choice Behavior + +The `Required` behavior forces the model to choose the provided function(s) for for invocation. This is useful for scenarios when the AI model must obtain required information from the specified +functions rather than from it's own knowledge. + +> [!NOTE] +> The behavior advertises functions in the first request to the AI model only and stops sending them in subsequent requests to prevent an infinite loop where the model keeps choosing the same functions for invocation repeatedly. + +Here, we specify that the AI model must choose the `GetWeatherForCity` function for invocation to obtain the weather forecast for the city of Boston, rather than guessing it based on its own knowledge. +The model will first choose the `GetWeatherForCity` function for invocation to retrieve the weather forecast. +With this information, the model can then determine the likely color of the sky in Boston using the response from the call to `GetWeatherForCity`. +```csharp +using Microsoft.SemanticKernel; + +IKernelBuilder builder = Kernel.CreateBuilder(); +builder.AddOpenAIChatCompletion("", ""); +builder.Plugins.AddFromType(); + +Kernel kernel = builder.Build(); + +KernelFunction getWeatherForCity = kernel.Plugins.GetFunction("WeatherForecastUtils", "GetWeatherForCity"); + +PromptExecutionSettings settings = new() { FunctionChoiceBehavior = FunctionChoiceBehavior.Required(functions: [getWeatherFunction]) }; + +await kernel.InvokePromptAsync("Given that it is now the 10th of September 2024, 11:29 AM, what is the likely color of the sky in Boston?", new(settings)); +``` + +An identical example in a YAML template configuration: +```csharp +using Microsoft.SemanticKernel; + +IKernelBuilder builder = Kernel.CreateBuilder(); +builder.AddOpenAIChatCompletion("", ""); +builder.Plugins.AddFromType(); + +Kernel kernel = builder.Build(); + +string promptTemplateConfig = """ + template_format: semantic-kernel + template: Given that it is now the 10th of September 2024, 11:29 AM, what is the likely color of the sky in Boston? + execution_settings: + default: + function_choice_behavior: + type: auto + functions: + - WeatherForecastUtils.GetWeatherForCity + """; + +KernelFunction promptFunction = KernelFunctionYaml.FromPromptYaml(promptTemplateConfig); + +Console.WriteLine(await kernel.InvokeAsync(promptFunction)); +``` + +Alternatively, all functions registered in the kernel can be provided to the AI model as required. However, only the ones chosen by the AI model as a result of the first request will be invoked by the Semantic Kernel. +The functions will not be sent to the AI model in subsequent requests to prevent an infinite loop, as mentioned above. +```csharp +using Microsoft.SemanticKernel; + +IKernelBuilder builder = Kernel.CreateBuilder(); +builder.AddOpenAIChatCompletion("", ""); +builder.Plugins.AddFromType(); + +Kernel kernel = builder.Build(); + +PromptExecutionSettings settings = new() { FunctionChoiceBehavior = FunctionChoiceBehavior.Required() }; + +await kernel.InvokePromptAsync("Given that it is now the 10th of September 2024, 11:29 AM, what is the likely color of the sky in Boston?", new(settings)); +``` + +## Using None Function Choice Behavior + +The `None` behavior instructs the AI model to use the provided function(s) without choosing any of them for invocation to generate a response. This is useful for dry runs when the caller may want to see which functions the model would choose without actually invoking them. +It is also useful when you want the AI model to extract information from a user ask e.g. in the sample below the AI model correctly worked out that Boston was the city name. + +Here, we advertise all functions from the `DateTimeUtils` and `WeatherForecastUtils` plugins to the AI model but instruct it not to choose any of them. +Instead, the model will provide a response describing which functions it would choose to determine the color of the sky in Boston on a specified date. +```csharp +using Microsoft.SemanticKernel; + +IKernelBuilder builder = Kernel.CreateBuilder(); +builder.AddOpenAIChatCompletion("", ""); +builder.Plugins.AddFromType(); +builder.Plugins.AddFromType(); + +Kernel kernel = builder.Build(); + +KernelFunction getWeatherForCity = kernel.Plugins.GetFunction("WeatherForecastUtils", "GetWeatherForCity"); + +PromptExecutionSettings settings = new() { FunctionChoiceBehavior = FunctionChoiceBehavior.None() }; + +await kernel.InvokePromptAsync("Specify which provided functions are needed to determine the color of the sky in Boston on a specified date.", new(settings)) + +// Sample response: To determine the color of the sky in Boston on a specified date, first call the DateTimeUtils-GetCurrentUtcDateTime function to obtain the +// current date and time in UTC. Next, use the WeatherForecastUtils-GetWeatherForCity function, providing 'Boston' as the city name and the retrieved UTC date and time. +// These functions do not directly provide the sky's color, but the GetWeatherForCity function offers weather data, which can be used to infer the general sky condition (e.g., clear, cloudy, rainy). +``` + +A corresponding example in a YAML prompt template configuration: +```csharp +using Microsoft.SemanticKernel; + +IKernelBuilder builder = Kernel.CreateBuilder(); +builder.AddOpenAIChatCompletion("", ""); +builder.Plugins.AddFromType(); +builder.Plugins.AddFromType(); + +Kernel kernel = builder.Build(); + +string promptTemplateConfig = """ + template_format: semantic-kernel + template: Specify which provided functions are needed to determine the color of the sky in Boston on a specified date. + execution_settings: + default: + function_choice_behavior: + type: none + """; + +KernelFunction promptFunction = KernelFunctionYaml.FromPromptYaml(promptTemplateConfig); + +Console.WriteLine(await kernel.InvokeAsync(promptFunction)); +``` + +## Function Invocation + +Function invocation is the process whereby Sematic Kernel invokes functions chosen by the AI model. For more details on function invocation see [function invocation article](./function-invocation.md). + +## Supported AI Connectors + +As of today, the following AI connectors in Semantic Kernel support the function calling model: + +| AI Connector | FunctionChoiceBehavior | ToolCallBehavior | +|------------------------|------------------------|------------------| +| Anthropic | Planned | ❌ | +| AzureAIInference | Coming soon | ❌ | +| AzureOpenAI | ✔️ | ✔️ | +| Gemini | Planned | ✔️ | +| HuggingFace | Planned | ❌ | +| Mistral | Planned | ✔️ | +| Ollama | Coming soon | ❌ | +| Onnx | Coming soon | ❌ | +| OpenAI | ✔️ | ✔️ | +::: zone-end + +::: zone pivot="programming-language-python" +## Coming soon +More info coming soon. +::: zone-end +::: zone pivot="programming-language-java" +## Coming soon +More info coming soon. +::: zone-end diff --git a/semantic-kernel/concepts/ai-services/chat-completion/function-calling/function-invocation.md b/semantic-kernel/concepts/ai-services/chat-completion/function-calling/function-invocation.md new file mode 100644 index 00000000..1c683222 --- /dev/null +++ b/semantic-kernel/concepts/ai-services/chat-completion/function-calling/function-invocation.md @@ -0,0 +1,126 @@ +--- +title: Function Invocation +description: Describes function invocation types SK supports. +zone_pivot_groups: programming-languages +author: SergeyMenshykh +ms.topic: conceptual +ms.author: semenshi +ms.service: semantic-kernel +--- +::: zone pivot="programming-language-csharp" +# Function Invocation Modes +When the AI model receives a prompt containing a list of functions, it may choose one or more of them for invocation to complete the prompt. When a function is chosen by the model, it needs be **invoked** by Semantic Kernel. + +The function calling subsystem in Semantic Kernel has two modes of function invocation: **auto** and **manual**. + +Depending on the invocation mode, Semantic Kernel either does end-to-end function invocation or gives the caller control over the function invocation process. + +## Auto Function Invocation +Auto function invocation is the default mode of the Semantic Kernel function-calling subsystem. When the AI model chooses one or more functions, Semantic Kernel automatically invokes the chosen functions. +The results of these function invocations are added to the chat history and sent to the model automatically in subsequent requests. +The model then reasons about the chat history, chooses additional functions if needed, or generates the final response. +This approach is fully automated and requires no manual intervention from the caller. + +This example demonstrates how to use the auto function invocation in Semantic Kernel. AI model decides which functions to call to complete the prompt and Semantic Kernel does the rest and invokes them automatically. +```csharp +using Microsoft.SemanticKernel; + +IKernelBuilder builder = Kernel.CreateBuilder(); +builder.AddOpenAIChatCompletion("", ""); +builder.Plugins.AddFromType(); +builder.Plugins.AddFromType(); + +Kernel kernel = builder.Build(); + +// By default, functions are set to be automatically invoked. +// If you want to explicitly enable this behavior, you can do so with the following code: +// PromptExecutionSettings settings = new() { FunctionChoiceBehavior = FunctionChoiceBehavior.Auto(autoInvoke: true) }; +PromptExecutionSettings settings = new() { FunctionChoiceBehavior = FunctionChoiceBehavior.Auto() }; + +await kernel.InvokePromptAsync("Given the current time of day and weather, what is the likely color of the sky in Boston?", new(settings)); +``` + +## Manual Function Invocation +In cases when the caller wants to have more control over the function invocation process, manual function invocation can be used. + +When manual function invocation is enabled, Semantic Kernel does not automatically invoke the functions chosen by the AI model. +Instead, it returns a list of chosen functions to the caller, who can then decide which functions to invoke, invoke them sequentially or in parallel, handle exceptions, and so on. +The function invocation results need to be added to the chat history and returned to the model, which reasons about them and decides whether to choose additional functions or generate the final response. + +The example below demonstrates how to use manual function invocation. +```csharp +using Microsoft.SemanticKernel; +using Microsoft.SemanticKernel.ChatCompletion; + +IKernelBuilder builder = Kernel.CreateBuilder(); +builder.AddOpenAIChatCompletion("", ""); +builder.Plugins.AddFromType(); +builder.Plugins.AddFromType(); + +Kernel kernel = builder.Build(); + +IChatCompletionService chatCompletionService = kernel.GetRequiredService(); + +// Manual function invocation needs to be enabled explicitly by setting autoInvoke to false. +PromptExecutionSettings settings = new() { FunctionChoiceBehavior = Microsoft.SemanticKernel.FunctionChoiceBehavior.Auto(autoInvoke: false) }; + +ChatHistory chatHistory = []; +chatHistory.AddUserMessage("Given the current time of day and weather, what is the likely color of the sky in Boston?"); + +while (true) +{ + ChatMessageContent result = await chatCompletionService.GetChatMessageContentAsync(chatHistory, settings, kernel); + + // Check if the AI model has generated a response. + if (result.Content is not null) + { + Console.Write(result.Content); + // Sample output: "Considering the current weather conditions in Boston with a tornado watch in effect resulting in potential severe thunderstorms, + // the sky color is likely unusual such as green, yellow, or dark gray. Please stay safe and follow instructions from local authorities." + break; + } + + // Adding AI model response containing chosen functions to chat history as it's required by the models to preserve the context. + chatHistory.Add(result); + + // Check if the AI model has chosen any function for invocation. + IEnumerable functionCalls = FunctionCallContent.GetFunctionCalls(result); + if (!functionCalls.Any()) + { + break; + } + + // Sequentially iterating over each chosen function, invoke it, and add the result to the chat history. + foreach (FunctionCallContent functionCall in functionCalls) + { + try + { + // Invoking the function + FunctionResultContent resultContent = await functionCall.InvokeAsync(kernel); + + // Adding the function result to the chat history + chatHistory.Add(resultContent.ToChatMessage()); + } + catch (Exception ex) + { + // Adding function exception to the chat history. + chatHistory.Add(new FunctionResultContent(functionCall, ex).ToChatMessage()); + // or + //chatHistory.Add(new FunctionResultContent(functionCall, "Error details that the AI model can reason about.").ToChatMessage()); + } + } +} + +``` +> [!NOTE] +> The FunctionCallContent and FunctionResultContent classes are used to represent AI model function calls and Semantic Kernel function invocation results, respectively. +> They contain information about chosen function, such as the function ID, name, and arguments, and function invocation results, such as function call ID and result. +::: zone-end +::: zone pivot="programming-language-python" +## Coming soon +More info coming soon. +::: zone-end +::: zone pivot="programming-language-java" +## Coming soon +More info coming soon. +::: zone-end \ No newline at end of file diff --git a/semantic-kernel/concepts/ai-services/chat-completion/function-calling.md b/semantic-kernel/concepts/ai-services/chat-completion/function-calling/index.md similarity index 96% rename from semantic-kernel/concepts/ai-services/chat-completion/function-calling.md rename to semantic-kernel/concepts/ai-services/chat-completion/function-calling/index.md index cdf88a04..7c7c27ff 100644 --- a/semantic-kernel/concepts/ai-services/chat-completion/function-calling.md +++ b/semantic-kernel/concepts/ai-services/chat-completion/function-calling/index.md @@ -33,7 +33,7 @@ When you make a request to a model with function calling enabled, Semantic Kerne The following diagram illustrates the process of function calling: -![Semantic Kernel function calling](../../../media/FunctionCalling.png) +![Semantic Kernel function calling](../../../../media/FunctionCalling.png) The following section will use a concrete example to illustrate how function calling works in practice. @@ -141,12 +141,11 @@ class OrderPizzaPlugin: self.user_context = user_context self.payment_service = payment_service - @kernel_function(name="get_pizza_menu") + @kernel_function async def get_pizza_menu(self): return await self.pizza_service.get_menu() @kernel_function( - name="add_pizza_to_cart", description="Add a pizza to the user's cart; returns the new item and updated cart" ) async def add_pizza_to_cart(self, size: PizzaSize, toppings: List[PizzaToppings], quantity: int = 1, special_instructions: str = ""): @@ -154,7 +153,6 @@ class OrderPizzaPlugin: return await self.pizza_service.add_pizza_to_cart(cart_id, size, toppings, quantity, special_instructions) @kernel_function( - name="remove_pizza_from_cart", description="Remove a pizza from the user's cart; returns the updated cart" ) async def remove_pizza_from_cart(self, pizza_id: int): @@ -162,7 +160,6 @@ class OrderPizzaPlugin: return await self.pizza_service.remove_pizza_from_cart(cart_id, pizza_id) @kernel_function( - name="get_pizza_from_cart", description="Returns the specific details of a pizza in the user's cart; use this instead of relying on previous messages since the cart may have changed since then." ) async def get_pizza_from_cart(self, pizza_id: int): @@ -170,7 +167,6 @@ class OrderPizzaPlugin: return await self.pizza_service.get_pizza_from_cart(cart_id, pizza_id) @kernel_function( - name="get_cart", description="Returns the user's current cart, including the total price and items in the cart." ) async def get_cart(self): @@ -178,7 +174,6 @@ class OrderPizzaPlugin: return await self.pizza_service.get_cart(cart_id) @kernel_function( - name="checkout", description="Checkouts the user's cart; this function will retrieve the payment from the user and complete the order." ) async def checkout(self): @@ -511,7 +506,7 @@ IChatCompletionService chatCompletion = kernel.GetRequiredService responses = chatCompletion.getChatMessageContentsAsync( ::: zone-end +::: zone pivot="programming-language-csharp" +> [!NOTE] +> This example uses the `FunctionChoiceBehavior.Auto()` behavior, one of the few available ones. For more information about other function choice behaviors, check out the [function choice behaviors article](./function-choice-behaviors.md). +::: zone-end + ### 3) Model processes the input With both the chat history and the serialized functions, the model can determine the best way to respond. In this case, the model recognizes that the user wants to order a pizza. The model would likely _want_ to call the `add_pizza_to_cart` function, but because we specified the size and toppings as required parameters, the model will ask the user for this information: @@ -722,6 +722,9 @@ Not all functions will succeed, however. If the function fails, Semantic Kernel > [!TIP] > To ensure a model can self-correct, it's important to provide error messages that clearly communicate what went wrong and how to fix it. This can help the model retry the function call with the correct information. +> [!NOTE] +> Semantic Kernel automatically invokes functions by default. However, if you prefer to manage function invocation manually, you can enable manual function invocation mode. For more details on how to do this, please refer to the [function invocation article](./function-invocation.md). + ### 6) Return the function result After the function has been invoked, the function result is sent back to the model as part of the chat history. This allows the model to understand the context of the conversation and generate a subsequent response. @@ -817,7 +820,7 @@ For example, if a user wants to order multiple pizzas, the LLM can call the `add ## Next steps -Now that you understand how function calling works, you can now learn how to actually use function calling in Semantic Kernel by referring to the [planning article](../../planning.md) +Now that you understand how function calling works, you can proceed to learn how to configure various aspects of function calling that better correspond to your specific scenarios by referring to the [function choice behavior article](./function-choice-behaviors.md) > [!div class="nextstepaction"] -> [Learn more about planning](../../planning.md) +> [Function Choice Behavior](./function-choice-behaviors.md) diff --git a/semantic-kernel/concepts/ai-services/chat-completion/index.md b/semantic-kernel/concepts/ai-services/chat-completion/index.md index 8cd73862..013ecaf4 100644 --- a/semantic-kernel/concepts/ai-services/chat-completion/index.md +++ b/semantic-kernel/concepts/ai-services/chat-completion/index.md @@ -57,6 +57,12 @@ dotnet add package Microsoft.SemanticKernel.Connectors.Google dotnet add package Microsoft.SemanticKernel.Connectors.HuggingFace ``` +# [Azure AI Inference](#tab/csharp-AzureAIInference) + +```bash +dotnet add package Microsoft.SemanticKernel.Connectors.AzureAIInference +``` + # [Other](#tab/csharp-other) For other AI service providers that support the OpenAI chat completion API (e.g., LLM Studio), you can use the OpenAI chat completion connector. @@ -170,6 +176,25 @@ kernelBuilder.AddHuggingFaceChatCompletion( ); Kernel kernel = kernelBuilder.Build(); ``` +# [Azure AI Inference](#tab/csharp-AzureAIInference) + +> [!IMPORTANT] +> The Azure AI Inference chat completion connector is currently experimental. To use it, you will need to add `#pragma warning disable SKEXP0070`. + +```csharp +using Microsoft.SemanticKernel; + +#pragma warning disable SKEXP0070 +IKernelBuilder kernelBuilder = Kernel.CreateBuilder(); +kernelBuilder.AddAzureAIInferenceChatCompletion( + model: "NAME_OF_MODEL", + apiKey: "API_KEY", + endpoint: new Uri("YOUR_ENDPOINT"), // Optional + serviceId: "SERVICE_ID", // Optional; for targeting specific services within Semantic Kernel + httpClient: new HttpClient() // Optional; for customizing HTTP client +); +Kernel kernel = kernelBuilder.Build(); +``` # [Other](#tab/csharp-other) For other AI service providers that support the OpenAI chat completion API (e.g., LLM Studio), you can use the following code to reuse the existing OpenAI chat completion connector. @@ -307,6 +332,30 @@ builder.Services.AddTransient((serviceProvider)=> { }); ``` +# [Azure AI Inference](#tab/csharp-AzureAIInference) + +> [!IMPORTANT] +> The Azure AI Inference chat completion connector is currently experimental. To use it, you will need to add `#pragma warning disable SKEXP0070`. + +```csharp +using Microsoft.SemanticKernel; +using Microsoft.SemanticKernel.Connectors.AzureAIInference; + +var builder = Host.CreateApplicationBuilder(args); + +#pragma warning disable SKEXP0070 +builder.Services.AddAzureAIInferenceChatCompletion( + model: "NAME_OF_MODEL", + apiKey: "API_KEY", + endpoint: new Uri("YOUR_ENDPOINT"), // Optional + serviceId: "SERVICE_ID" // Optional; for targeting specific services within Semantic Kernel +); + +builder.Services.AddTransient((serviceProvider)=> { + return new Kernel(serviceProvider); +}); +``` + # [Other](#tab/csharp-other) For other AI service providers that support the OpenAI chat completion API (e.g., LLM Studio), you can use the following code to reuse the existing OpenAI chat completion connector. @@ -415,6 +464,22 @@ OpenAIChatCompletionService chatCompletionService = new ( ); ``` +# [Azure AI Inference](#tab/csharp-AzureAIInference) + +> [!IMPORTANT] +> The Azure AI Inference chat completion connector is currently experimental. To use it, you will need to add `#pragma warning disable SKEXP0070`. + +```csharp +using Microsoft.SemanticKernel.Connectors.AzureAIInference; + +AzureAIInferenceChatCompletionService chatCompletionService = new ( + modelId: "YOUR_MODEL_ID", + apiKey: "YOUR_API_KEY", + endpoint: new Uri("YOUR_ENDPOINT"), // Used to point to your service + httpClient: new HttpClient() // Optional; if not provided, the HttpClient from the kernel will be used +); +``` + # [Other](#tab/csharp-other) For other AI service providers that support the OpenAI chat completion API (e.g., LLM Studio), you can use the following code to reuse the existing OpenAI chat completion connector. @@ -658,10 +723,10 @@ var response = await chatCompletionService.GetChatMessageContentAsync( chat_history = ChatHistory() chat_history.add_user_message("Hello, how are you?") -response = (await chat_completion.get_chat_message_contents( +response = await chat_completion.get_chat_message_content( chat_history=history, kernel=kernel, -))[0] +) ``` ::: zone-end @@ -708,7 +773,7 @@ await foreach (var chunk in response) chat_history = ChatHistory() chat_history.add_user_message("Hello, how are you?") -response = chat_completion.get_streaming_chat_message_contents( +response = chat_completion.get_streaming_chat_message_content( chat_history=history, kernel=kernel, ) diff --git a/semantic-kernel/concepts/ai-services/integrations.md b/semantic-kernel/concepts/ai-services/integrations.md new file mode 100644 index 00000000..dce91654 --- /dev/null +++ b/semantic-kernel/concepts/ai-services/integrations.md @@ -0,0 +1,38 @@ +--- +title: AI Integrations for Semantic Kernel +description: Learn which features are available for C#, Python, and Java through integrations. +author: sophialagerkranspandey +ms.topic: reference +ms.author: sopand +ms.date: 07/11/2023 +ms.service: semantic-kernel +--- + +# AI Integrations for Semantic Kernel + +Semantic Kernel provides a wide range of AI service integrations to help you build powerful AI agents. Additionally, Semantic Kernel integrates with other Microsoft services to provide additional functionality via plugins. + +## Out-of-the-box integrations + +With the available AI connectors, developers can easily build AI agents with swappable components. This allows you to experiment with different AI services to find the best combination for your use case. + +### AI Services + +| Services | C# | Python | Java | Notes | +|-----------------------------------|:----:|:------:|:----:|-------| +| Text Generation | ✅ | ✅ | ✅ | Example: Text-Davinci-003 | +| Chat Completion | ✅ | ✅ | ✅ | Example: GPT4, Chat-GPT | +| Text Embeddings (Experimental) | ✅ | ✅ | ✅ | Example: Text-Embeddings-Ada-002 | +| Text to Image (Experimental) | ✅ | ❌ | ❌ | Example: Dall-E | +| Image to Text (Experimental) | ✅ | ❌ | ❌ | Example: Pix2Struct | +| Text to Audio (Experimental) | ✅ | ❌ | ❌ | Example: Text-to-speech | +| Audio to Text (Experimental) | ✅ | ❌ | ❌ | Example: Whisper | + +## Additional plugins + +If you want to extend the functionality of your AI agent, you can use plugins to integrate with other Microsoft services. Here are some of the plugins that are available for Semantic Kernel: + +| Plugin | C# | Python | Java | Description | +| ---------- | :-: | :----: | :--: | ----------- | +| Logic Apps | ✅ | ✅ | ✅ | Build workflows within Logic Apps using its available connectors and import them as plugins in Semantic Kernel. [Learn more](../plugins/adding-logic-apps-as-plugins.md). | +| Azure Container Apps Dynamic Sessions | ✅ | ✅ | ❌ | With dynamic sessions, you can recreate the Code Interpreter experience from the Assistants API by effortlessly spinning up Python containers where AI agents can execute Python code. [Learn more](/azure/container-apps/sessions). | diff --git a/SECURITY.md b/semantic-kernel/concepts/enterprise-readiness/SECURITY.md similarity index 100% rename from SECURITY.md rename to semantic-kernel/concepts/enterprise-readiness/SECURITY.md diff --git a/semantic-kernel/concepts/enterprise-readiness/TOC.yml b/semantic-kernel/concepts/enterprise-readiness/TOC.yml new file mode 100644 index 00000000..6ad0d62e --- /dev/null +++ b/semantic-kernel/concepts/enterprise-readiness/TOC.yml @@ -0,0 +1,6 @@ +- name: Filters + href: filters.md +- name: Observability + href: observability/TOC.yml +- name: Security + href: SECURITY.md \ No newline at end of file diff --git a/semantic-kernel/concepts/enterprise-readiness/filters.md b/semantic-kernel/concepts/enterprise-readiness/filters.md new file mode 100644 index 00000000..1be39673 --- /dev/null +++ b/semantic-kernel/concepts/enterprise-readiness/filters.md @@ -0,0 +1,91 @@ +--- +title: Semantic Kernel Filters +description: Learn about filters in Semantic Kernel. +author: dmytrostruk +ms.topic: conceptual +ms.author: sopand +ms.date: 09/10/2024 +ms.service: semantic-kernel +--- + +# What are Filters? + +Filters enhance security by providing control and visibility over how and when functions run. This is needed to instill responsible AI principles into your work so that you feel confident your solution is enterprise ready. + +For example, filters are leveraged to validate permissions before an approval flow begins. The `IFunctionInvocationFilter` is run to check the permissions of the person that’s looking to submit an approval. This means that only a select group of people will be able to kick off the process. + +A good example of filters is provided [here](https://devblogs.microsoft.com/semantic-kernel/filters-in-semantic-kernel/) in our detailed Semantic Kernel blog post on Filters. +  + ![Semantic Kernel Filters](../../media/WhatAreFilters.png) + +There are 3 types of filters: + +- Function invocation filter - it's executed every time `KernelFunction` is invoked. Allows to get information about function which is going to be executed, its arguments, catch an exception during function execution, override function result, retry function execution in case of failure (can be used to [switch to other AI model](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/Filtering/RetryWithFilters.cs)). +- Prompt render filter - it's executed before prompt rendering operation. Allows to see what prompt is going to be sent to AI, modify prompt (e.g. RAG, [PII redaction](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/Filtering/PIIDetection.cs) scenarios) and prevent the prompt from being sent to AI with function result override (can be used for [Semantic Caching](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/Caching/SemanticCachingWithFilters.cs)). +- Auto function invocation filter - similar to function invocation filter, but it is executed in a scope of `automatic function calling` operation, so it has more information available in a context, including chat history, list of all functions that will be executed and request iteration counters. It also allows to terminate auto function calling process (e.g. there are 3 functions to execute, but there is already the desired result from the second function). + +Each filter has `context` object that contains all information related to function execution or prompt rendering. Together with context, there is also a `next` delegate/callback, which executes next filter in pipeline or function itself. This provides more control, and it is useful in case there are some reasons to avoid function execution (e.g. malicious prompt or function arguments). It is possible to register multiple filters of the same type, where each filter will have different responsibility. + +Example of function invocation filter to perform logging before and after function invocation: + +```csharp +public sealed class LoggingFilter(ILogger logger) : IFunctionInvocationFilter +{ + public async Task OnFunctionInvocationAsync(FunctionInvocationContext context, Func next) + { + logger.LogInformation("FunctionInvoking - {PluginName}.{FunctionName}", context.Function.PluginName, context.Function.Name); + + await next(context); + + logger.LogInformation("FunctionInvoked - {PluginName}.{FunctionName}", context.Function.PluginName, context.Function.Name); + } +} +``` + +Example of prompt render filter which overrides rendered prompt before sending it to AI: + +```csharp +public class SafePromptFilter : IPromptRenderFilter +{ + public async Task OnPromptRenderAsync(PromptRenderContext context, Func next) + { + // Example: get function information + var functionName = context.Function.Name; + + await next(context); + + // Example: override rendered prompt before sending it to AI + context.RenderedPrompt = "Safe prompt"; + } +} +``` + +Example of auto function invocation filter which terminates function calling process as soon as we have the desired result: + +```csharp +public sealed class EarlyTerminationFilter : IAutoFunctionInvocationFilter +{ + public async Task OnAutoFunctionInvocationAsync(AutoFunctionInvocationContext context, Func next) + { + await next(context); + + var result = context.Result.GetValue(); + + if (result == "desired result") + { + context.Terminate = true; + } + } +} +``` + +## More information + +C#/.NET: +* [Function invocation filter examples](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/Filtering/FunctionInvocationFiltering.cs) +* [Prompt render filter examples](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/Filtering/PromptRenderFiltering.cs) +* [Auto function invocation filter examples](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/Filtering/AutoFunctionInvocationFiltering.cs) +* [PII detection and redaction with filters](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/Filtering/PIIDetection.cs) +* [Semantic Caching with filters](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/Caching/SemanticCachingWithFilters.cs) +* [Content Safety with filters](https://github.com/microsoft/semantic-kernel/tree/main/dotnet/samples/Demos/ContentSafety) +* [Text summarization and translation quality check with filters](https://github.com/microsoft/semantic-kernel/tree/main/dotnet/samples/Demos/QualityCheck) \ No newline at end of file diff --git a/semantic-kernel/concepts/enterprise-readiness/observability/TOC.yml b/semantic-kernel/concepts/enterprise-readiness/observability/TOC.yml new file mode 100644 index 00000000..375dd736 --- /dev/null +++ b/semantic-kernel/concepts/enterprise-readiness/observability/TOC.yml @@ -0,0 +1,10 @@ +- name: Overview + href: index.md +- name: 'Example: Console' + href: telemetry-with-console.md +- name: 'Example: Application Insights' + href: telemetry-with-app-insights.md +- name: 'Example: Aspire Dashboard' + href: telemetry-with-aspire-dashboard.md +- name: 'Advanced telemetry with Semantic Kernel' + href: telemetry-advanced.md \ No newline at end of file diff --git a/semantic-kernel/concepts/enterprise-readiness/observability/image.png b/semantic-kernel/concepts/enterprise-readiness/observability/image.png new file mode 100644 index 00000000..37b44aa6 Binary files /dev/null and b/semantic-kernel/concepts/enterprise-readiness/observability/image.png differ diff --git a/semantic-kernel/concepts/enterprise-readiness/observability/index.md b/semantic-kernel/concepts/enterprise-readiness/observability/index.md new file mode 100644 index 00000000..41583f59 --- /dev/null +++ b/semantic-kernel/concepts/enterprise-readiness/observability/index.md @@ -0,0 +1,89 @@ +--- +title: Observability in Semantic Kernel +description: Introduction to observability in Semantic Kernel +author: taochen +zone_pivot_groups: programming-languages +ms.topic: conceptual +ms.author: taochen +ms.date: 09/11/2024 +ms.service: semantic-kernel +--- + +# Observability in Semantic Kernel + +## Brief introduction to observability + +When you build AI solutions, you want to be able to observe the behavior of your services. Observability is the ability to monitor and analyze the internal state of components within a distributed system. It is a key requirement for building enterprise-ready AI solutions. + +Observability is typically achieved through logging, metrics, and tracing. They are often referred to as the three pillars of observability. You will also hear the term "telemetry" used to describe the data collected by these three pillars. Unlike debugging, observability provides an ongoing overview of the system's health and performance. + +Useful materials for further reading: + +- [Observability defined by Cloud Native Computing Foundation](https://glossary.cncf.io/observability/) +- [Distributed tracing](https://learn.microsoft.com/dotnet/core/diagnostics/distributed-tracing) +- [Observability in .Net](https://learn.microsoft.com/dotnet/core/diagnostics/observability-with-otel) +- [OpenTelemetry](https://opentelemetry.io/docs/what-is-opentelemetry/) + +## Observability in Semantic Kernel + +Semantic Kernel is designed to be observable. It emits logs, metrics, and traces that are compatible to the OpenTelemetry standard. You can use your favorite observability tools to monitor and analyze the behavior of your services built on Semantic Kernel. + +Specifically, Semantic Kernel provides the following observability features: + +- **Logging**: Semantic Kernel logs meaningful events and errors from the kernel, kernel plugins and functions, as well as the AI connectors. + ![Logs and events](../../../media/telemetry-log-events-overview-app-insights.png) + > [!IMPORTANT] + > [Traces in Application Insights](https://learn.microsoft.com/azure/azure-monitor/app/data-model-complete#trace) represent traditional log entries and [OpenTelemetry span events](https://opentelemetry.io/docs/concepts/signals/traces/#span-events). They are not the same as distributed traces. +- **Metrics**: Semantic Kernel emits metrics from kernel functions and AI connectors. You will be able to monitor metrics such as the kernel function execution time, the token consumption of AI connectors, etc. + ![Metrics](../../../media/telemetry-metrics-overview-app-insights.png) +- **Tracing**: Semantic Kernel supports distributed tracing. You can track activities across different services and within Semantic Kernel. + ![Complete end-to-end transaction of a request](../../media/telemetry-trace-overview-app-insights.png) + + +::: zone pivot="programming-language-csharp" + +| Telemetry | Description | +|-----------|---------------------------------------| +| Log | Logs are recorded throughout the Kernel. For more information on Logging in .Net, please refer to this [document](https://learn.microsoft.com/dotnet/core/extensions/logging). Sensitive data, such as kernel function arguments and results, are logged at the trace level. Please refer to this [table](https://learn.microsoft.com/dotnet/core/extensions/logging?tabs=command-line#log-level) for more information on log levels. | +| Activity | Each kernel function execution and each call to an AI model are recorded as an activity. All activities are generated by an activity source named "Microsoft.SemanticKernel". | +| Metric | Semantic Kernel captures the following metrics from kernel functions:
  • `semantic_kernel.function.invocation.duration` (Histogram) - function execution time (in seconds)
  • `semantic_kernel.function.streaming.duration` (Histogram) - function streaming execution time (in seconds)
  • `semantic_kernel.function.invocation.token_usage.prompt` (Histogram) - number of prompt token usage (only for `KernelFunctionFromPrompt`)
  • `semantic_kernel.function.invocation.token_usage.completion` (Histogram) - number of completion token usage (only for `KernelFunctionFromPrompt`)
    • | + +::: zone-end + +::: zone pivot="programming-language-python" + +| Telemetry | Description | +|-----------|---------------------------------------| +| Log | Logs are recorded throughout the Kernel. For more information on Logging in Python, please refer to this [document](https://docs.python.org/3/howto/logging.html). Sensitive data, such as kernel function arguments and results, are logged at the debug level.| +| Span | Each auto function invocation loop, each kernel function execution, and each call to an AI model are recorded as a span. | +| Metric | Semantic Kernel captures the following metrics from kernel functions:
    • `semantic_kernel.function.invocation.duration` (Histogram) - function execution time (in seconds)
    • `semantic_kernel.function.streaming.duration` (Histogram) - function streaming execution time (in seconds)
      • | + +::: zone-end + +::: zone pivot="programming-language-java" + +> [!NOTE] +> Semantic Kernel Observability is not yet available for Java. + +::: zone-end + + +## OpenTelemetry Semantic Convention + +Semantic Kernel follows the [OpenTelemetry Semantic Convention](https://opentelemetry.io/docs/concepts/semantic-conventions/) for Observability. This means that the logs, metrics, and traces emitted by Semantic Kernel are structured and follow a common schema. This ensures that you can more effectively analyze the telemetry data emitted by Semantic Kernel. + +> [!Note] +> Currently, the [Semantic Conventions for Generative AI](https://github.com/open-telemetry/semantic-conventions/blob/main/docs/gen-ai/README.md) are in experimental status. Semantic Kernel strives to follow the OpenTelemetry Semantic Convention as closely as possible, and provide a consistent and meaningful observability experience for AI solutions. + +## Next steps + +Now that you have a basic understanding of observability in Semantic Kernel, you can learn more about how to output telemetry data to the console or use APM tools to visualize and analyze telemetry data. + +> [!div class="nextstepaction"] +> [Console](telemetry-with-console.md) + +> [!div class="nextstepaction"] +> [Application Insights](telemetry-with-app-insights.md) + +> [!div class="nextstepaction"] +> [Aspire Dashboard](telemetry-with-aspire-dashboard.md) diff --git a/semantic-kernel/concepts/enterprise-readiness/observability/telemetry-advanced.md b/semantic-kernel/concepts/enterprise-readiness/observability/telemetry-advanced.md new file mode 100644 index 00000000..4246dd6a --- /dev/null +++ b/semantic-kernel/concepts/enterprise-readiness/observability/telemetry-advanced.md @@ -0,0 +1,603 @@ +--- +title: More advanced scenarios for telemetry +description: More advanced scenarios for telemetry +zone_pivot_groups: programming-languages +author: taochen +ms.topic: conceptual +ms.author: taochen +ms.date: 09/12/2024 +ms.service: semantic-kernel +--- + +# More advanced scenarios for telemetry + +> [!NOTE] +> This article will use [Aspire Dashboard](https://learn.microsoft.com/dotnet/aspire/fundamentals/dashboard/overview?tabs=bash) for illustration. If you prefer to use other tools, please refer to the documentation of the tool you are using on setup instructions. + +## Auto Function Calling + +Auto Function Calling is a Semantic Kernel feature that allows the kernel to automatically execute functions when the model responds with function calls, and provide the results back to the model. This feature is useful for scenarios where a query requires multiple iterations of function calls to get a final natural language response. For more details, please see these GitHub [samples](https://github.com/microsoft/semantic-kernel/tree/main/python/samples/concepts/auto_function_calling). + +> [!NOTE] +> Function calling is not supported by all models. + +> [!TIP] +> You will hear the term "tools" and "tool calling" sometimes used interchangeably with "functions" and "function calling". + +### Prerequisites + +::: zone pivot="programming-language-csharp" + +- An Azure OpenAI chat completion deployment that supports function calling. +- Docker +- The latest [.Net SDK](https://dotnet.microsoft.com/download/dotnet) for your operating system. + +::: zone-end + +::: zone pivot="programming-language-python" + +- An Azure OpenAI chat completion deployment that supports function calling. +- Docker +- [Python 3.10, 3.11, or 3.12](https://www.python.org/downloads/) installed on your machine. + +::: zone-end + +::: zone pivot="programming-language-java" + +> [!NOTE] +> Semantic Kernel Observability is not yet available for Java. + +::: zone-end + +### Setup + +::: zone pivot="programming-language-csharp" + +#### Create a new console application + +In a terminal, run the following command to create a new console application in C#: + +```console +dotnet new console -n TelemetryAutoFunctionCallingQuickstart +``` + +Navigate to the newly created project directory after the command completes. + +#### Install required packages + +- Semantic Kernel + + ```console + dotnet add package Microsoft.SemanticKernel + ``` + +- OpenTelemetry Console Exporter + ```console + dotnet add package OpenTelemetry.Exporter.OpenTelemetryProtocol + ``` + +### Create a simple application with Semantic Kernel + +From the project directory, open the `Program.cs` file with your favorite editor. We are going to create a simple application that uses Semantic Kernel to send a prompt to a chat completion model. Replace the existing content with the following code and fill in the required values for `deploymentName`, `endpoint`, and `apiKey`: + +```csharp +using System.ComponentModel; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.SemanticKernel; +using Microsoft.SemanticKernel.Connectors.OpenAI; +using OpenTelemetry; +using OpenTelemetry.Logs; +using OpenTelemetry.Metrics; +using OpenTelemetry.Resources; +using OpenTelemetry.Trace; + +namespace TelemetryAutoFunctionCallingQuickstart +{ + class BookingPlugin + { + [KernelFunction("FindAvailableRooms")] + [Description("Finds available conference rooms for today.")] + public async Task> FindAvailableRoomsAsync() + { + // Simulate a remote call to a booking system. + await Task.Delay(1000); + return ["Room 101", "Room 201", "Room 301"]; + } + + [KernelFunction("BookRoom")] + [Description("Books a conference room.")] + public async Task BookRoomAsync(string room) + { + // Simulate a remote call to a booking system. + await Task.Delay(1000); + return $"Room {room} booked."; + } + } + + class Program + { + static async Task Main(string[] args) + { + // Endpoint to the Aspire Dashboard + var endpoint = "http://localhost:4317"; + + var resourceBuilder = ResourceBuilder + .CreateDefault() + .AddService("TelemetryAspireDashboardQuickstart"); + + // Enable model diagnostics with sensitive data. + AppContext.SetSwitch("Microsoft.SemanticKernel.Experimental.GenAI.EnableOTelDiagnosticsSensitive", true); + + using var traceProvider = Sdk.CreateTracerProviderBuilder() + .SetResourceBuilder(resourceBuilder) + .AddSource("Microsoft.SemanticKernel*") + .AddOtlpExporter(options => options.Endpoint = new Uri(endpoint)) + .Build(); + + using var meterProvider = Sdk.CreateMeterProviderBuilder() + .SetResourceBuilder(resourceBuilder) + .AddMeter("Microsoft.SemanticKernel*") + .AddOtlpExporter(options => options.Endpoint = new Uri(endpoint)) + .Build(); + + using var loggerFactory = LoggerFactory.Create(builder => + { + // Add OpenTelemetry as a logging provider + builder.AddOpenTelemetry(options => + { + options.SetResourceBuilder(resourceBuilder); + options.AddOtlpExporter(options => options.Endpoint = new Uri(endpoint)); + // Format log messages. This is default to false. + options.IncludeFormattedMessage = true; + options.IncludeScopes = true; + }); + builder.SetMinimumLevel(LogLevel.Information); + }); + + IKernelBuilder builder = Kernel.CreateBuilder(); + builder.Services.AddSingleton(loggerFactory); + builder.AddAzureOpenAIChatCompletion( + deploymentName: "your-deployment-name", + endpoint: "your-azure-openai-endpoint", + apiKey: "your-azure-openai-api-key" + ); + builder.Plugins.AddFromType(); + + Kernel kernel = builder.Build(); + + var answer = await kernel.InvokePromptAsync( + "Reserve a conference room for me today.", + new KernelArguments( + new OpenAIPromptExecutionSettings { + ToolCallBehavior = ToolCallBehavior.AutoInvokeKernelFunctions + } + ) + ); + + Console.WriteLine(answer); + } + } +} +``` + +In the code above, we first define a mock conference room booking plugin with two functions: `FindAvailableRoomsAsync` and `BookRoomAsync`. We then create a simple console application that registers the plugin to the kernel, and ask the kernel to automatically call the functions when needed. + +::: zone-end + +::: zone pivot="programming-language-python" + +#### Create a new Python virtual environment + +```console +python -m venv telemetry-auto-function-calling-quickstart +``` + +Activate the virtual environment. + +```console +telemetry-auto-function-calling-quickstart\Scripts\activate +``` + +#### Install required packages + +```console +pip install semantic-kernel opentelemetry-exporter-otlp-proto-grpc +``` + +### Create a simple Python script with Semantic Kernel + +Create a new Python script and open it with your favorite editor. + +# [Powershell](#tab/Powershell) + +```PowerShell +New-Item -Path telemetry_auto_function_calling_quickstart.py -ItemType file +``` + +# [Bash](#tab/Bash) + +```bash +touch telemetry_auto_function_calling_quickstart.py +``` + +--- + +We are going to create a simple Python script that uses Semantic Kernel to send a prompt to a chat completion model. Replace the existing content with the following code and fill in the required values for `deployment_name`, `endpoint`, and `api_key`: + +```python +import asyncio +import logging +from typing import Annotated + +from opentelemetry._logs import set_logger_provider +from opentelemetry.exporter.otlp.proto.grpc._log_exporter import OTLPLogExporter +from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import OTLPMetricExporter +from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter +from opentelemetry.metrics import set_meter_provider +from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler +from opentelemetry.sdk._logs.export import BatchLogRecordProcessor +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader +from opentelemetry.sdk.metrics.view import DropAggregation, View +from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import BatchSpanProcessor +from opentelemetry.semconv.resource import ResourceAttributes +from opentelemetry.trace import set_tracer_provider + +from semantic_kernel import Kernel +from semantic_kernel.connectors.ai.function_choice_behavior import FunctionChoiceBehavior +from semantic_kernel.connectors.ai.open_ai import AzureChatCompletion +from semantic_kernel.connectors.ai.prompt_execution_settings import PromptExecutionSettings +from semantic_kernel.functions.kernel_arguments import KernelArguments +from semantic_kernel.functions.kernel_function_decorator import kernel_function + + +class BookingPlugin: + @kernel_function( + name="find_available_rooms", + description="Find available conference rooms for today.", + ) + def find_available_rooms(self,) -> Annotated[list[str], "A list of available rooms."]: + return ["Room 101", "Room 201", "Room 301"] + + @kernel_function( + name="book_room", + description="Book a conference room.", + ) + def book_room(self, room: str) -> Annotated[str, "A confirmation message."]: + return f"Room {room} booked." + + +# Endpoint to the Aspire Dashboard +endpoint = "http://localhost:4317" + +# Create a resource to represent the service/sample +resource = Resource.create({ResourceAttributes.SERVICE_NAME: "telemetry-aspire-dashboard-quickstart"}) + + +def set_up_logging(): + exporter = OTLPLogExporter(endpoint=endpoint) + + # Create and set a global logger provider for the application. + logger_provider = LoggerProvider(resource=resource) + # Log processors are initialized with an exporter which is responsible + # for sending the telemetry data to a particular backend. + logger_provider.add_log_record_processor(BatchLogRecordProcessor(exporter)) + # Sets the global default logger provider + set_logger_provider(logger_provider) + + # Create a logging handler to write logging records, in OTLP format, to the exporter. + handler = LoggingHandler() + # Add filters to the handler to only process records from semantic_kernel. + handler.addFilter(logging.Filter("semantic_kernel")) + # Attach the handler to the root logger. `getLogger()` with no arguments returns the root logger. + # Events from all child loggers will be processed by this handler. + logger = logging.getLogger() + logger.addHandler(handler) + logger.setLevel(logging.INFO) + + +def set_up_tracing(): + exporter = OTLPSpanExporter(endpoint=endpoint) + + # Initialize a trace provider for the application. This is a factory for creating tracers. + tracer_provider = TracerProvider(resource=resource) + # Span processors are initialized with an exporter which is responsible + # for sending the telemetry data to a particular backend. + tracer_provider.add_span_processor(BatchSpanProcessor(exporter)) + # Sets the global default tracer provider + set_tracer_provider(tracer_provider) + + +def set_up_metrics(): + exporter = OTLPMetricExporter(endpoint=endpoint) + + # Initialize a metric provider for the application. This is a factory for creating meters. + meter_provider = MeterProvider( + metric_readers=[PeriodicExportingMetricReader(exporter, export_interval_millis=5000)], + resource=resource, + views=[ + # Dropping all instrument names except for those starting with "semantic_kernel" + View(instrument_name="*", aggregation=DropAggregation()), + View(instrument_name="semantic_kernel*"), + ], + ) + # Sets the global default meter provider + set_meter_provider(meter_provider) + + +# This must be done before any other telemetry calls +set_up_logging() +set_up_tracing() +set_up_metrics() + + +async def main(): + # Create a kernel and add a service + kernel = Kernel() + kernel.add_service(AzureChatCompletion( + api_key="your-azure-openai-api-key", + endpoint="your-azure-openai-endpoint", + deployment_name="your-deployment-name" + )) + kernel.add_plugin(BookingPlugin(), "BookingPlugin") + + answer = await kernel.invoke_prompt( + "Reserve a conference room for me today.", + arguments=KernelArguments( + settings=PromptExecutionSettings( + function_choice_behavior=FunctionChoiceBehavior.Auto(), + ), + ), + ) + print(answer) + + +if __name__ == "__main__": + asyncio.run(main()) +``` + +In the code above, we first define a mock conference room booking plugin with two functions: `find_available_rooms` and `book_room`. We then create a simple Python script that registers the plugin to the kernel, and ask the kernel to automatically call the functions when needed. + +#### Environment variables + +Please refer to this [article](./telemetry-with-console.md#environment-variables) for more information on setting up the required environment variables to enable the kernel to emit spans for AI connectors. + +::: zone-end + +::: zone pivot="programming-language-java" + +> [!NOTE] +> Semantic Kernel Observability is not yet available for Java. + +::: zone-end + +### Start the Aspire Dashboard + +Follow the instructions [here](https://learn.microsoft.com/dotnet/aspire/fundamentals/dashboard/standalone?tabs=bash#start-the-dashboard) to start the dashboard. Once the dashboard is running, open a browser and navigate to `http://localhost:18888` to access the dashboard. + +### Run + +::: zone pivot="programming-language-csharp" + +Run the console application with the following command: + +```console +dotnet run +``` + +::: zone-end + +::: zone pivot="programming-language-python" + +Run the Python script with the following command: + +```console +python telemetry_auto_function_calling_quickstart.py +``` + +::: zone-end + +::: zone pivot="programming-language-java" + +> [!NOTE] +> Semantic Kernel Observability is not yet available for Java. + +::: zone-end + +You should see an output similar to the following: + +```console +Room 101 has been successfully booked for you today. +``` + +### Inspect telemetry data + +After running the application, head over to the dashboard to inspect the telemetry data. + +::: zone pivot="programming-language-csharp" + +Find the trace for the application in the **Traces** tab. You should five spans in the trace: + +![TracesAdvancedScenarioDotNet](../../../media/telemetry-advanced-scenarios-trace-detail-dotnet.png) + +These 5 spans represent the internal operations of the kernel with auto function calling enabled. It first invokes the model, which requests a function call. Then the kernel automatically executes the function `FindAvailableRoomsAsync` and returns the result to the model. The model then requests another function call to make a reservation, and the kernel automatically executes the function `BookRoomAsync` and returns the result to the model. Finally, the model returns a natural language response to the user. + +And if you click on the last span, and look for the prompt in the `gen_ai.content.prompt` event, you should see something similar to the following: + +```json +[ + { "role": "user", "content": "Reserve a conference room for me today." }, + { + "role": "Assistant", + "content": null, + "tool_calls": [ + { + "id": "call_NtKi0OgOllJj1StLkOmJU8cP", + "function": { "arguments": {}, "name": "FindAvailableRooms" }, + "type": "function" + } + ] + }, + { + "role": "tool", + "content": "[\u0022Room 101\u0022,\u0022Room 201\u0022,\u0022Room 301\u0022]" + }, + { + "role": "Assistant", + "content": null, + "tool_calls": [ + { + "id": "call_mjQfnZXLbqp4Wb3F2xySds7q", + "function": { "arguments": { "room": "Room 101" }, "name": "BookRoom" }, + "type": "function" + } + ] + }, + { "role": "tool", "content": "Room Room 101 booked." } +] +``` + +This is the chat history that gets built up as the model and the kernel interact with each other. This is sent to the model in the last iteration to get a natural language response. + +::: zone-end + +::: zone pivot="programming-language-python" + +Find the trace for the application in the **Traces** tab. You should five spans in the trace grouped under the `AutoFunctionInvocationLoop` span: + +![TracesAdvancedScenarioPython](../../../media/telemetry-advanced-scenarios-trace-detail-python.png) + +These 5 spans represent the internal operations of the kernel with auto function calling enabled. It first invokes the model, which requests a function call. Then the kernel automatically executes the function `find_available_rooms` and returns the result to the model. The model then requests another function call to make a reservation, and the kernel automatically executes the function `book_room` and returns the result to the model. Finally, the model returns a natural language response to the user. + +And if you click on the last span, and look for the prompt in the `gen_ai.content.prompt` event, you should see something similar to the following: + +```json +[ + { "role": "user", "content": "Reserve a conference room for me today." }, + { + "role": "assistant", + "tool_calls": [ + { + "id": "call_ypqO5v6uTRlYH9sPTjvkGec8", + "type": "function", + "function": { + "name": "BookingPlugin-find_available_rooms", + "arguments": "{}" + } + } + ] + }, + { + "role": "tool", + "content": "['Room 101', 'Room 201', 'Room 301']", + "tool_call_id": "call_ypqO5v6uTRlYH9sPTjvkGec8" + }, + { + "role": "assistant", + "tool_calls": [ + { + "id": "call_XDZGeTfNiWRpYKoHoH9TZRoX", + "type": "function", + "function": { + "name": "BookingPlugin-book_room", + "arguments": "{\"room\":\"Room 101\"}" + } + } + ] + }, + { + "role": "tool", + "content": "Room Room 101 booked.", + "tool_call_id": "call_XDZGeTfNiWRpYKoHoH9TZRoX" + } +] +``` + +This is the chat history that gets built up as the model and the kernel interact with each other. This is sent to the model in the last iteration to get a natural language response. + +::: zone-end + +::: zone pivot="programming-language-java" + +> [!NOTE] +> Semantic Kernel Observability is not yet available for Java. + +::: zone-end + +## Error handling + +If an error occurs during the execution of a function, the kernel will automatically catch the error and return an error message to the model. The model can then use this error message to provide a natural language response to the user. + +::: zone pivot="programming-language-csharp" + +Modify the `BookRoomAsync` function in the C# code to simulate an error: + +```csharp +[KernelFunction("BookRoom")] +[Description("Books a conference room.")] +public async Task BookRoomAsync(string room) +{ + // Simulate a remote call to a booking system. + await Task.Delay(1000); + + throw new Exception("Room is not available."); +} +``` + +Run the application again and observe the trace in the dashboard. You should see the span representing the kernel function call with an error: + +![TracesAdvancedScenarioErrorDotNet](../../../media/telemetry-advanced-scenarios-trace-detail-error-handling-dotnet.png) + +> [!NOTE] +> It is very likely that the model responses to the error may vary each time you run the application, because the model is stochastic. You may see the model reserving all three rooms at the same time, or reserving one the first time then reserving the other two the second time, etc. + +::: zone-end + + +::: zone pivot="programming-language-python" + +Modify the `book_room` function in the Python code to simulate an error: + +```python +@kernel_function( + name="book_room", + description="Book a conference room.", +) +async def book_room(self, room: str) -> Annotated[str, "A confirmation message."]: + # Simulate a remote call to a booking system + await asyncio.sleep(1) + + raise Exception("Room is not available.") +``` + +Run the application again and observe the trace in the dashboard. You should see the span representing the kernel function call with an error and the stack trace: + +![TracesAdvancedScenarioErrorPython](../../../media/telemetry-advanced-scenarios-trace-detail-error-handling-python.png) + +> [!NOTE] +> It is very likely that the model responses to the error may vary each time you run the application, because the model is stochastic. You may see the model reserving all three rooms at the same time, or reserving one the first time then reserving the other two the second time, etc. + +::: zone-end + + +::: zone pivot="programming-language-java" + +> [!NOTE] +> Semantic Kernel Observability is not yet available for Java. + +::: zone-end + +## Next steps and further reading + +In production, your services may get a large number of requests. Semantic Kernel will generate a large amount of telemetry data. some of which may not be useful for your use case and will introduce unnecessary costs to store the data. You can use the [sampling](https://opentelemetry.io/docs/concepts/sampling/) feature to reduce the amount of telemetry data that is collected. + +Observability in Semantic Kernel is constantly improving. You can find the latest updates and new features in the [GitHub repository](https://github.com/microsoft/semantic-kernel). + + + + + + diff --git a/semantic-kernel/concepts/enterprise-readiness/observability/telemetry-with-app-insights.md b/semantic-kernel/concepts/enterprise-readiness/observability/telemetry-with-app-insights.md new file mode 100644 index 00000000..cf461d9a --- /dev/null +++ b/semantic-kernel/concepts/enterprise-readiness/observability/telemetry-with-app-insights.md @@ -0,0 +1,444 @@ +--- +title: Inspection of telemetry data with Application Insights +description: Send telemetry data to Application Insights +zone_pivot_groups: programming-languages +author: taochen +ms.topic: conceptual +ms.author: taochen +ms.date: 09/12/2024 +ms.service: semantic-kernel +--- + +# Inspection of telemetry data with Application Insights + +[Application Insights](https://learn.microsoft.com/azure/azure-monitor/app/app-insights-overview) is part of [Azure Monitor](https://learn.microsoft.com/azure/azure-monitor/overview), which is a comprehensive solution for collecting, analyzing, and acting on telemetry data from your cloud and on-premises environments. With Application Insights, you can monitor your application's performance, detect issues, and diagnose problems. + +In this example, we will learn how to export telemetry data to Application Insights, and inspect the data in the Application Insights portal. + +## Exporter + +Exporters are responsible for sending telemetry data to a destination. Read more about exporters [here](https://opentelemetry.io/docs/concepts/components/#exporters). In this example, we use the Azure Monitor exporter to output telemetry data to an Application Insights instance. + +## Prerequisites + +::: zone pivot="programming-language-csharp" + +- An Azure OpenAI chat completion deployment. +- An Application Insights instance. Follow the [instructions](https://learn.microsoft.com/azure/azure-monitor/app/create-workspace-resource?tabs=bicep#create-a-workspace-based-resource) here to create a resource if you don't have one. Copy the [connection string](https://learn.microsoft.com/azure/azure-monitor/app/sdk-connection-string?tabs=dotnet5#find-your-connection-string) for later use. +- The latest [.Net SDK](https://dotnet.microsoft.com/download/dotnet) for your operating system. + +::: zone-end + +::: zone pivot="programming-language-python" + +- An Azure OpenAI chat completion deployment. +- An Application Insights instance. Follow the [instructions](https://learn.microsoft.com/azure/azure-monitor/app/create-workspace-resource?tabs=bicep#create-a-workspace-based-resource) here to create a resource if you don't have one. Copy the [connection string](https://learn.microsoft.com/azure/azure-monitor/app/sdk-connection-string?tabs=dotnet5#find-your-connection-string) for later use. +- [Python 3.10, 3.11, or 3.12](https://www.python.org/downloads/) installed on your machine. + +::: zone-end + +::: zone pivot="programming-language-java" + +> [!NOTE] +> Semantic Kernel Observability is not yet available for Java. + +::: zone-end + +## Setup + +::: zone pivot="programming-language-csharp" + +### Create a new console application + +In a terminal, run the following command to create a new console application in C#: + +```console +dotnet new console -n TelemetryApplicationInsightsQuickstart +``` + +Navigate to the newly created project directory after the command completes. + + +### Install required packages + +- Semantic Kernel + ```console + dotnet add package Microsoft.SemanticKernel + ``` + +- OpenTelemetry Console Exporter + ```console + dotnet add package Azure.Monitor.OpenTelemetry.Exporter + ``` + +### Create a simple application with Semantic Kernel + +From the project directory, open the `Program.cs` file with your favorite editor. We are going to create a simple application that uses Semantic Kernel to send a prompt to a chat completion model. Replace the existing content with the following code and fill in the required values for `deploymentName`, `endpoint`, and `apiKey`: + +```csharp +using Azure.Monitor.OpenTelemetry.Exporter; + +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.SemanticKernel; +using OpenTelemetry; +using OpenTelemetry.Logs; +using OpenTelemetry.Metrics; +using OpenTelemetry.Resources; +using OpenTelemetry.Trace; + +namespace TelemetryApplicationInsightsQuickstart +{ + class Program + { + static async Task Main(string[] args) + { + // Telemetry setup code goes here + + IKernelBuilder builder = Kernel.CreateBuilder(); + // builder.Services.AddSingleton(loggerFactory); + builder.AddAzureOpenAIChatCompletion( + deploymentName: "your-deployment-name", + endpoint: "your-azure-openai-endpoint", + apiKey: "your-azure-openai-api-key" + ); + + Kernel kernel = builder.Build(); + + var answer = await kernel.InvokePromptAsync( + "Why is the sky blue in one sentence?" + ); + + Console.WriteLine(answer); + } + } +} +``` + +### Add telemetry + +If you run the console app now, you should expect to see a sentence explaining why the sky is blue. To observe the kernel via telemetry, replace the `// Telemetry setup code goes here` comment with the following code: + +```csharp +// Replace the connection string with your Application Insights connection string +var connectionString = "your-application-insights-connection-string"; + +var resourceBuilder = ResourceBuilder + .CreateDefault() + .AddService("TelemetryApplicationInsightsQuickstart"); + +// Enable model diagnostics with sensitive data. +AppContext.SetSwitch("Microsoft.SemanticKernel.Experimental.GenAI.EnableOTelDiagnosticsSensitive", true); + +using var traceProvider = Sdk.CreateTracerProviderBuilder() + .SetResourceBuilder(resourceBuilder) + .AddSource("Microsoft.SemanticKernel*") + .AddAzureMonitorTraceExporter(options => options.ConnectionString = connectionString) + .Build(); + +using var meterProvider = Sdk.CreateMeterProviderBuilder() + .SetResourceBuilder(resourceBuilder) + .AddMeter("Microsoft.SemanticKernel*") + .AddAzureMonitorMetricExporter(options => options.ConnectionString = connectionString) + .Build(); + +using var loggerFactory = LoggerFactory.Create(builder => +{ + // Add OpenTelemetry as a logging provider + builder.AddOpenTelemetry(options => + { + options.SetResourceBuilder(resourceBuilder); + options.AddAzureMonitorLogExporter(options => options.ConnectionString = connectionString); + // Format log messages. This is default to false. + options.IncludeFormattedMessage = true; + options.IncludeScopes = true; + }); + builder.SetMinimumLevel(LogLevel.Information); +}); +``` + +Finally Uncomment the line `// builder.Services.AddSingleton(loggerFactory);` to add the logger factory to the builder. + +Please refer to this [article](./telemetry-with-console.md#add-telemetry) for more information on the telemetry setup code. The only difference here is that we are using `AddAzureMonitor[Trace|Metric|Log]Exporter` to export telemetry data to Application Insights. + +::: zone-end + +::: zone pivot="programming-language-python" + +### Create a new Python virtual environment + +```console +python -m venv telemetry-application-insights-quickstart +``` + +Activate the virtual environment. +```console +telemetry-application-insights-quickstart\Scripts\activate +``` + +### Install required packages + +```console +pip install semantic-kernel azure-monitor-opentelemetry-exporter +``` + +### Create a simple Python script with Semantic Kernel + +Create a new Python script and open it with your favorite editor. + +# [Powershell](#tab/Powershell) + +```PowerShell +New-Item -Path telemetry_application_insights_quickstart.py -ItemType file +``` + +# [Bash](#tab/Bash) + +```bash +touch telemetry_application_insights_quickstart.py +``` + +--- + +We are going to create a simple Python script that uses Semantic Kernel to send a prompt to a chat completion model. Replace the existing content with the following code and fill in the required values for `deployment_name`, `endpoint`, and `api_key`: + +```python +import asyncio +import logging + +from azure.monitor.opentelemetry.exporter import ( + AzureMonitorLogExporter, + AzureMonitorMetricExporter, + AzureMonitorTraceExporter, +) + +from opentelemetry._logs import set_logger_provider +from opentelemetry.metrics import set_meter_provider +from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler +from opentelemetry.sdk._logs.export import BatchLogRecordProcessor +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader +from opentelemetry.sdk.metrics.view import DropAggregation, View +from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import BatchSpanProcessor +from opentelemetry.semconv.resource import ResourceAttributes +from opentelemetry.trace import set_tracer_provider + +from semantic_kernel import Kernel +from semantic_kernel.connectors.ai.open_ai import AzureChatCompletion + + +# Telemetry setup code goes here + +async def main(): + # Create a kernel and add a service + kernel = Kernel() + kernel.add_service(AzureChatCompletion( + api_key="your-azure-openai-api-key", + endpoint="your-azure-openai-endpoint", + deployment_name="your-deployment-name" + )) + + answer = await kernel.invoke_prompt("Why is the sky blue in one sentence?") + print(answer) + + +if __name__ == "__main__": + asyncio.run(main()) +``` + +### Add telemetry + +#### Environment variables + +Please refer to this [article](./telemetry-with-console.md#environment-variables) for more information on setting up the required environment variables to enable the kernel to emit spans for AI connectors. + +#### Code +If you run the script now, you should expect to see a sentence explaining why the sky is blue. To observe the kernel via telemetry, replace the `# Telemetry setup code goes here` comment with the following code: + +```python +# Replace the connection string with your Application Insights connection string +connection_string = "your-application-insights-connection-string" + +# Create a resource to represent the service/sample +resource = Resource.create({ResourceAttributes.SERVICE_NAME: "telemetry-application-insights-quickstart"}) + + +def set_up_logging(): + exporter = AzureMonitorLogExporter(connection_string=connection_string) + + # Create and set a global logger provider for the application. + logger_provider = LoggerProvider(resource=resource) + # Log processors are initialized with an exporter which is responsible + # for sending the telemetry data to a particular backend. + logger_provider.add_log_record_processor(BatchLogRecordProcessor(exporter)) + # Sets the global default logger provider + set_logger_provider(logger_provider) + + # Create a logging handler to write logging records, in OTLP format, to the exporter. + handler = LoggingHandler() + # Add filters to the handler to only process records from semantic_kernel. + handler.addFilter(logging.Filter("semantic_kernel")) + # Attach the handler to the root logger. `getLogger()` with no arguments returns the root logger. + # Events from all child loggers will be processed by this handler. + logger = logging.getLogger() + logger.addHandler(handler) + logger.setLevel(logging.INFO) + + +def set_up_tracing(): + exporter = AzureMonitorTraceExporter(connection_string=connection_string) + + # Initialize a trace provider for the application. This is a factory for creating tracers. + tracer_provider = TracerProvider(resource=resource) + # Span processors are initialized with an exporter which is responsible + # for sending the telemetry data to a particular backend. + tracer_provider.add_span_processor(BatchSpanProcessor(exporter)) + # Sets the global default tracer provider + set_tracer_provider(tracer_provider) + + +def set_up_metrics(): + exporter = AzureMonitorMetricExporter(connection_string=connection_string) + + # Initialize a metric provider for the application. This is a factory for creating meters. + meter_provider = MeterProvider( + metric_readers=[PeriodicExportingMetricReader(exporter, export_interval_millis=5000)], + resource=resource, + views=[ + # Dropping all instrument names except for those starting with "semantic_kernel" + View(instrument_name="*", aggregation=DropAggregation()), + View(instrument_name="semantic_kernel*"), + ], + ) + # Sets the global default meter provider + set_meter_provider(meter_provider) + + +# This must be done before any other telemetry calls +set_up_logging() +set_up_tracing() +set_up_metrics() +``` + +Please refer to this [article](./telemetry-with-console.md#add-telemetry-1) for more information on the telemetry setup code. The only difference here is that we are using `AzureMonitor[Trace|Metric|Log]Exporter` to export telemetry data to Application Insights. + +::: zone-end + +::: zone pivot="programming-language-java" + +> [!NOTE] +> Semantic Kernel Observability is not yet available for Java. + +::: zone-end + +## Run + +::: zone pivot="programming-language-csharp" + +Run the console application with the following command: + +```console +dotnet run +``` + +::: zone-end + +::: zone pivot="programming-language-python" + +Run the Python script with the following command: + +```console +python telemetry_application_insights_quickstart.py +``` + +::: zone-end + +::: zone pivot="programming-language-java" + +> [!NOTE] +> Semantic Kernel Observability is not yet available for Java. + +::: zone-end + +## Inspect telemetry data + +After running the application, head over to the Application Insights portal to inspect the telemetry data. It may take a few minutes for the data to appear in the portal. + +### Transaction search + +Navigate to the **Transaction search** tab to view the transactions that have been recorded. + +![AppInsights Left Panel](../../../media/telemetry-app-insights-left-panel-transaction-search-highlight.png) + +Hit refresh to see the latest transactions. When results appear, click on one of them to see more details. + +![AppInsights Transaction Search](../../../media/telemetry-app-insights-transaction-search.png) + +Toggle between the **View all** and **View timeline** button to see all traces and dependencies of the transaction in different views. + +> [!IMPORTANT] +> [Traces](https://learn.microsoft.com/azure/azure-monitor/app/data-model-complete#trace) represent traditional log entries and [OpenTelemetry span events](https://opentelemetry.io/docs/concepts/signals/traces/#span-events). They are not the same as distributed traces. Dependencies represent the calls to (internal and external) components. Please refer to this [article](https://learn.microsoft.com/azure/azure-monitor/app/data-model-complete) for more information on the data model in Application Insights. + +For this particular example, you should see two dependencies and multiple traces. The first dependency represents a kernel function that is created from the prompt. The second dependency represents the call to the Azure OpenAI chat completion model. When you expand the `chat.completion {your-deployment-name}` dependency, you should see the details of the call. A set of `gen_ai` attributes are attached to the dependency, which provides additional context about the call. + +![GenAI Attributes](../../../media/telemetry-app-insights-gen-ai-attributes.png) + +::: zone pivot="programming-language-csharp" + +If you have the switch `Microsoft.SemanticKernel.Experimental.GenAI.EnableOTelDiagnosticsSensitive` set to `true`, you will also see two traces that carry the sensitive data of the prompt and the completion result. + +![GenAI Sensitive Attributes](../../../media/telemetry-app-insights-gen-ai-attributes-sensitive.png) + +Click on them and you will see the prompt and the completion result under the custom properties section. + +::: zone-end + +::: zone pivot="programming-language-python" + +If you have the environment variable `SEMANTICKERNEL_EXPERIMENTAL_GENAI_ENABLE_OTEL_DIAGNOSTICS_SENSITIVE` set to `true`, you will also see two traces that carry the sensitive data of the prompt and the completion result. + +![GenAI Sensitive Attributes](../../../media/telemetry-app-insights-gen-ai-attributes-sensitive.png) + +Click on them and you will see the prompt and the completion result under the custom properties section. + +::: zone-end + +### Log analytics + +Transaction search is not the only way to inspect telemetry data. You can also use [**Log analytics**](https://learn.microsoft.com/azure/azure-monitor/logs/log-analytics-overview) to query and analyze the data. Navigate to the **Logs** under **Monitoring** to start. + +Follow this [document](https://learn.microsoft.com/azure/azure-monitor/logs/log-analytics-overview#log-analytics-interface) to start exploring the log analytics interface. + +Below are some sample queries you can use for this example: + +```kusto +// Retrieves the total number of completion and prompt tokens used for the model if you run the application multiple times. +dependencies +| where name startswith "chat" +| project model = customDimensions["gen_ai.request.model"], completion_token = toint(customDimensions["gen_ai.response.completion_tokens"]), prompt_token = toint(customDimensions["gen_ai.response.prompt_tokens"]) +| where model == "gpt-4o" +| project completion_token, prompt_token +| summarize total_completion_tokens = sum(completion_token), total_prompt_tokens = sum(prompt_token) +``` + + +```kusto +// Retrieves all the prompts and completions and their corresponding token usage. +dependencies +| where name startswith "chat" +| project timestamp, operation_Id, name, completion_token = customDimensions["gen_ai.response.completion_tokens"], prompt_token = customDimensions["gen_ai.response.prompt_tokens"] +| join traces on operation_Id +| where message startswith "gen_ai" +|project timestamp, messages = customDimensions, token=iff(customDimensions contains "gen_ai.prompt", prompt_token, completion_token) +``` + +![Query Result](../../../media/telemetry-app-insights-kusto-token-usage-per-message.png) + +## Next steps + +Now that you have successfully output telemetry data to Application Insights, you can explore more features of Semantic Kernel that can help you monitor and diagnose your application: + +> [!div class="nextstepaction"] +> [Advanced telemetry with Semantic Kernel](./telemetry-advanced.md) diff --git a/semantic-kernel/concepts/enterprise-readiness/observability/telemetry-with-aspire-dashboard.md b/semantic-kernel/concepts/enterprise-readiness/observability/telemetry-with-aspire-dashboard.md new file mode 100644 index 00000000..b83abbfb --- /dev/null +++ b/semantic-kernel/concepts/enterprise-readiness/observability/telemetry-with-aspire-dashboard.md @@ -0,0 +1,393 @@ +--- +title: Inspection of telemetry data with Aspire Dashboard +description: Send telemetry data to Aspire Dashboard +zone_pivot_groups: programming-languages +author: taochen +ms.topic: conceptual +ms.author: taochen +ms.date: 09/12/2024 +ms.service: semantic-kernel +--- + +# Inspection of telemetry data with Aspire Dashboard + +[Aspire Dashboard](https://learn.microsoft.com/dotnet/aspire/fundamentals/dashboard/overview?tabs=bash) is part of the [.NET Aspire](https://learn.microsoft.com/dotnet/aspire/get-started/aspire-overview) offering. The dashboard allows developers to monitor and inspect their distributed applications. + +In this example, we will use the [standalone mode](https://learn.microsoft.com/dotnet/aspire/fundamentals/dashboard/standalone?tabs=bash) and learn how to export telemetry data to Aspire Dashboard, and inspect the data there. + +## Exporter + +Exporters are responsible for sending telemetry data to a destination. Read more about exporters [here](https://opentelemetry.io/docs/concepts/components/#exporters). In this example, we use the [OpenTelemetry Protocol (OTLP)](https://opentelemetry.io/docs/specs/otlp/) exporter to send telemetry data to Aspire Dashboard. + +## Prerequisites + +::: zone pivot="programming-language-csharp" + +- An Azure OpenAI chat completion deployment. +- Docker +- The latest [.Net SDK](https://dotnet.microsoft.com/download/dotnet) for your operating system. + +::: zone-end + +::: zone pivot="programming-language-python" + +- An Azure OpenAI chat completion deployment. +- Docker +- [Python 3.10, 3.11, or 3.12](https://www.python.org/downloads/) installed on your machine. + +::: zone-end + +::: zone pivot="programming-language-java" + +> [!NOTE] +> Semantic Kernel Observability is not yet available for Java. + +::: zone-end + +## Setup + +::: zone pivot="programming-language-csharp" + +### Create a new console application + +In a terminal, run the following command to create a new console application in C#: + +```console +dotnet new console -n TelemetryAspireDashboardQuickstart +``` + +Navigate to the newly created project directory after the command completes. + + +### Install required packages + +- Semantic Kernel + ```console + dotnet add package Microsoft.SemanticKernel + ``` + +- OpenTelemetry Console Exporter + ```console + dotnet add package OpenTelemetry.Exporter.OpenTelemetryProtocol + ``` + +### Create a simple application with Semantic Kernel + +From the project directory, open the `Program.cs` file with your favorite editor. We are going to create a simple application that uses Semantic Kernel to send a prompt to a chat completion model. Replace the existing content with the following code and fill in the required values for `deploymentName`, `endpoint`, and `apiKey`: + +```csharp +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.SemanticKernel; +using OpenTelemetry; +using OpenTelemetry.Logs; +using OpenTelemetry.Metrics; +using OpenTelemetry.Resources; +using OpenTelemetry.Trace; + +namespace TelemetryAspireDashboardQuickstart +{ + class Program + { + static async Task Main(string[] args) + { + // Telemetry setup code goes here + + IKernelBuilder builder = Kernel.CreateBuilder(); + // builder.Services.AddSingleton(loggerFactory); + builder.AddAzureOpenAIChatCompletion( + deploymentName: "your-deployment-name", + endpoint: "your-azure-openai-endpoint", + apiKey: "your-azure-openai-api-key" + ); + + Kernel kernel = builder.Build(); + + var answer = await kernel.InvokePromptAsync( + "Why is the sky blue in one sentence?" + ); + + Console.WriteLine(answer); + } + } +} +``` + +### Add telemetry + +If you run the console app now, you should expect to see a sentence explaining why the sky is blue. To observe the kernel via telemetry, replace the `// Telemetry setup code goes here` comment with the following code: + +```csharp +// Endpoint to the Aspire Dashboard +var endpoint = "http://localhost:4317"; + +var resourceBuilder = ResourceBuilder + .CreateDefault() + .AddService("TelemetryAspireDashboardQuickstart"); + +// Enable model diagnostics with sensitive data. +AppContext.SetSwitch("Microsoft.SemanticKernel.Experimental.GenAI.EnableOTelDiagnosticsSensitive", true); + +using var traceProvider = Sdk.CreateTracerProviderBuilder() + .SetResourceBuilder(resourceBuilder) + .AddSource("Microsoft.SemanticKernel*") + .AddOtlpExporter(options => options.Endpoint = new Uri(endpoint)) + .Build(); + +using var meterProvider = Sdk.CreateMeterProviderBuilder() + .SetResourceBuilder(resourceBuilder) + .AddMeter("Microsoft.SemanticKernel*") + .AddOtlpExporter(options => options.Endpoint = new Uri(endpoint)) + .Build(); + +using var loggerFactory = LoggerFactory.Create(builder => +{ + // Add OpenTelemetry as a logging provider + builder.AddOpenTelemetry(options => + { + options.SetResourceBuilder(resourceBuilder); + options.AddOtlpExporter(options => options.Endpoint = new Uri(endpoint)); + // Format log messages. This is default to false. + options.IncludeFormattedMessage = true; + options.IncludeScopes = true; + }); + builder.SetMinimumLevel(LogLevel.Information); +}); +``` + +Finally Uncomment the line `// builder.Services.AddSingleton(loggerFactory);` to add the logger factory to the builder. + +Please refer to this [article](./telemetry-with-console.md#add-telemetry) for more information on the telemetry setup code. The only difference here is that we are using `AddOtlpExporter` to export telemetry data to Aspire Dashboard. + +::: zone-end + +::: zone pivot="programming-language-python" + +### Create a new Python virtual environment + +```console +python -m venv telemetry-aspire-dashboard-quickstart +``` + +Activate the virtual environment. +```console +telemetry-aspire-dashboard-quickstart\Scripts\activate +``` + +### Install required packages + +```console +pip install semantic-kernel opentelemetry-exporter-otlp-proto-grpc +``` + +### Create a simple Python script with Semantic Kernel + +Create a new Python script and open it with your favorite editor. + +# [Powershell](#tab/Powershell) + +```PowerShell +New-Item -Path telemetry_aspire_dashboard_quickstart.py -ItemType file +``` + +# [Bash](#tab/Bash) + +```bash +touch telemetry_aspire_dashboard_quickstart.py +``` + +--- + +We are going to create a simple Python script that uses Semantic Kernel to send a prompt to a chat completion model. Replace the existing content with the following code and fill in the required values for `deployment_name`, `endpoint`, and `api_key`: + +```python +import asyncio +import logging + +from opentelemetry._logs import set_logger_provider +from opentelemetry.exporter.otlp.proto.grpc._log_exporter import OTLPLogExporter +from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import OTLPMetricExporter +from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter +from opentelemetry.metrics import set_meter_provider +from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler +from opentelemetry.sdk._logs.export import BatchLogRecordProcessor +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader +from opentelemetry.sdk.metrics.view import DropAggregation, View +from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import BatchSpanProcessor +from opentelemetry.semconv.resource import ResourceAttributes +from opentelemetry.trace import set_tracer_provider + +from semantic_kernel import Kernel +from semantic_kernel.connectors.ai.open_ai import AzureChatCompletion + + +# Telemetry setup code goes here + +async def main(): + # Create a kernel and add a service + kernel = Kernel() + kernel.add_service(AzureChatCompletion( + api_key="your-azure-openai-api-key", + endpoint="your-azure-openai-endpoint", + deployment_name="your-deployment-name" + )) + + answer = await kernel.invoke_prompt("Why is the sky blue in one sentence?") + print(answer) + + +if __name__ == "__main__": + asyncio.run(main()) +``` + +### Add telemetry + +#### Environment variables + +Please refer to this [article](./telemetry-with-console.md#environment-variables) for more information on setting up the required environment variables to enable the kernel to emit spans for AI connectors. + +#### Code +If you run the script now, you should expect to see a sentence explaining why the sky is blue. To observe the kernel via telemetry, replace the `# Telemetry setup code goes here` comment with the following code: + +```python +# Endpoint to the Aspire Dashboard +endpoint = "http://localhost:4317" + +# Create a resource to represent the service/sample +resource = Resource.create({ResourceAttributes.SERVICE_NAME: "telemetry-aspire-dashboard-quickstart"}) + + +def set_up_logging(): + exporter = OTLPLogExporter(endpoint=endpoint) + + # Create and set a global logger provider for the application. + logger_provider = LoggerProvider(resource=resource) + # Log processors are initialized with an exporter which is responsible + # for sending the telemetry data to a particular backend. + logger_provider.add_log_record_processor(BatchLogRecordProcessor(exporter)) + # Sets the global default logger provider + set_logger_provider(logger_provider) + + # Create a logging handler to write logging records, in OTLP format, to the exporter. + handler = LoggingHandler() + # Add filters to the handler to only process records from semantic_kernel. + handler.addFilter(logging.Filter("semantic_kernel")) + # Attach the handler to the root logger. `getLogger()` with no arguments returns the root logger. + # Events from all child loggers will be processed by this handler. + logger = logging.getLogger() + logger.addHandler(handler) + logger.setLevel(logging.INFO) + + +def set_up_tracing(): + exporter = OTLPSpanExporter(endpoint=endpoint) + + # Initialize a trace provider for the application. This is a factory for creating tracers. + tracer_provider = TracerProvider(resource=resource) + # Span processors are initialized with an exporter which is responsible + # for sending the telemetry data to a particular backend. + tracer_provider.add_span_processor(BatchSpanProcessor(exporter)) + # Sets the global default tracer provider + set_tracer_provider(tracer_provider) + + +def set_up_metrics(): + exporter = OTLPMetricExporter(endpoint=endpoint) + + # Initialize a metric provider for the application. This is a factory for creating meters. + meter_provider = MeterProvider( + metric_readers=[PeriodicExportingMetricReader(exporter, export_interval_millis=5000)], + resource=resource, + views=[ + # Dropping all instrument names except for those starting with "semantic_kernel" + View(instrument_name="*", aggregation=DropAggregation()), + View(instrument_name="semantic_kernel*"), + ], + ) + # Sets the global default meter provider + set_meter_provider(meter_provider) + + +# This must be done before any other telemetry calls +set_up_logging() +set_up_tracing() +set_up_metrics() +``` + +Please refer to this [article](./telemetry-with-console.md#add-telemetry-1) for more information on the telemetry setup code. The only difference here is that we are using `OTLP[Span|Metric|Log]Exporter` to export telemetry data to Aspire Dashboard. + +::: zone-end + +::: zone pivot="programming-language-java" + +> [!NOTE] +> Semantic Kernel Observability is not yet available for Java. + +::: zone-end + +## Start the Aspire Dashboard + +Follow the instructions [here](https://learn.microsoft.com/dotnet/aspire/fundamentals/dashboard/standalone?tabs=bash#start-the-dashboard) to start the dashboard. Once the dashboard is running, open a browser and navigate to `http://localhost:18888` to access the dashboard. + +## Run + +::: zone pivot="programming-language-csharp" + +Run the console application with the following command: + +```console +dotnet run +``` + +::: zone-end + +::: zone pivot="programming-language-python" + +Run the Python script with the following command: + +```console +python telemetry_aspire_dashboard_quickstart.py +``` + +::: zone-end + +::: zone pivot="programming-language-java" + +> [!NOTE] +> Semantic Kernel Observability is not yet available for Java. + +::: zone-end + +## Inspect telemetry data + +After running the application, head over to the dashboard to inspect the telemetry data. + +> [!TIP] +> Follow this [guide](https://learn.microsoft.com/dotnet/aspire/fundamentals/dashboard/explore) to explore the Aspire Dashboard interface. + +### Traces + +If this is your first time running the application after starting the dashboard, you should see a one trace is the `Traces` tab. Click on the trace to view more details. + +![TracesOverview](../../../media/telemetry-aspire-dashboard-traces-overview.png) + +In the trace details, you can see the span that represents the prompt function and the span that represents the chat completion model. Click on the chat completion span to see details about the request and response. + +> [!TIP] +> You can filter the attributes of the spans to find the one you are interested in. + +![TracesDetails](../../../media/telemetry-aspire-dashboard-trace-detail.png) + +### Logs + +Head over to the `Structured` tab to view the logs emitted by the application. Please refer to this [guide](https://learn.microsoft.com/dotnet/aspire/fundamentals/dashboard/explore#structured-logs-page) on how to work with structured logs in the dashboard. + +## Next steps + +Now that you have successfully output telemetry data to Aspire Dashboard, you can explore more features of Semantic Kernel that can help you monitor and diagnose your application: + +> [!div class="nextstepaction"] +> [Advanced telemetry with Semantic Kernel](./telemetry-advanced.md) diff --git a/semantic-kernel/concepts/enterprise-readiness/observability/telemetry-with-console.md b/semantic-kernel/concepts/enterprise-readiness/observability/telemetry-with-console.md new file mode 100644 index 00000000..56354c16 --- /dev/null +++ b/semantic-kernel/concepts/enterprise-readiness/observability/telemetry-with-console.md @@ -0,0 +1,689 @@ +--- +title: Inspection of telemetry data with the console +description: Output telemetry data to the console for inspection +zone_pivot_groups: programming-languages +author: taochen +ms.topic: conceptual +ms.author: taochen +ms.date: 09/12/2024 +ms.service: semantic-kernel +--- + +# Inspection of telemetry data with the console + +Although the console is not a recommended way to inspect telemetry data, it is a simple and quick way to get started. This article shows you how to output telemetry data to the console for inspection with a minimal Kernel setup. + +## Exporter + +Exporters are responsible for sending telemetry data to a destination. Read more about exporters [here](https://opentelemetry.io/docs/concepts/components/#exporters). In this example, we use the console exporter to output telemetry data to the console. + +## Prerequisites + +::: zone pivot="programming-language-csharp" + +- An Azure OpenAI chat completion deployment. +- The latest [.Net SDK](https://dotnet.microsoft.com/download/dotnet) for your operating system. + +::: zone-end + +::: zone pivot="programming-language-python" + +- An Azure OpenAI chat completion deployment. +- [Python 3.10, 3.11, or 3.12](https://www.python.org/downloads/) installed on your machine. + +::: zone-end + +::: zone pivot="programming-language-java" + +> [!NOTE] +> Semantic Kernel Observability is not yet available for Java. + +::: zone-end + +## Setup + +::: zone pivot="programming-language-csharp" + +### Create a new console application + +In a terminal, run the following command to create a new console application in C#: + +```console +dotnet new console -n TelemetryConsoleQuickstart +``` + +Navigate to the newly created project directory after the command completes. + + +### Install required packages + +- Semantic Kernel + ```console + dotnet add package Microsoft.SemanticKernel + ``` + +- OpenTelemetry Console Exporter + ```console + dotnet add package OpenTelemetry.Exporter.Console + ``` + +### Create a simple application with Semantic Kernel + +From the project directory, open the `Program.cs` file with your favorite editor. We are going to create a simple application that uses Semantic Kernel to send a prompt to a chat completion model. Replace the existing content with the following code and fill in the required values for `deploymentName`, `endpoint`, and `apiKey`: + +```csharp +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.SemanticKernel; +using OpenTelemetry; +using OpenTelemetry.Logs; +using OpenTelemetry.Metrics; +using OpenTelemetry.Resources; +using OpenTelemetry.Trace; + +namespace TelemetryConsoleQuickstart +{ + class Program + { + static async Task Main(string[] args) + { + // Telemetry setup code goes here + + IKernelBuilder builder = Kernel.CreateBuilder(); + // builder.Services.AddSingleton(loggerFactory); + builder.AddAzureOpenAIChatCompletion( + deploymentName: "your-deployment-name", + endpoint: "your-azure-openai-endpoint", + apiKey: "your-azure-openai-api-key" + ); + + Kernel kernel = builder.Build(); + + var answer = await kernel.InvokePromptAsync( + "Why is the sky blue in one sentence?" + ); + + Console.WriteLine(answer); + } + } +} +``` + +### Add telemetry + +If you run the console app now, you should expect to see a sentence explaining why the sky is blue. To observe the kernel via telemetry, replace the `// Telemetry setup code goes here` comment with the following code: + +```csharp +var resourceBuilder = ResourceBuilder + .CreateDefault() + .AddService("TelemetryConsoleQuickstart"); + +// Enable model diagnostics with sensitive data. +AppContext.SetSwitch("Microsoft.SemanticKernel.Experimental.GenAI.EnableOTelDiagnosticsSensitive", true); + +using var traceProvider = Sdk.CreateTracerProviderBuilder() + .SetResourceBuilder(resourceBuilder) + .AddSource("Microsoft.SemanticKernel*") + .AddConsoleExporter() + .Build(); + +using var meterProvider = Sdk.CreateMeterProviderBuilder() + .SetResourceBuilder(resourceBuilder) + .AddMeter("Microsoft.SemanticKernel*") + .AddConsoleExporter() + .Build(); + +using var loggerFactory = LoggerFactory.Create(builder => +{ + // Add OpenTelemetry as a logging provider + builder.AddOpenTelemetry(options => + { + options.SetResourceBuilder(resourceBuilder); + options.AddConsoleExporter(); + // Format log messages. This is default to false. + options.IncludeFormattedMessage = true; + options.IncludeScopes = true; + }); + builder.SetMinimumLevel(LogLevel.Information); +}); +``` + +Finally Uncomment the line `// builder.Services.AddSingleton(loggerFactory);` to add the logger factory to the builder. + +In the above code snippet, we first create a resource builder for building resource instances. A resource represents the entity that produces telemetry data. You can read more about resources [here](https://opentelemetry.io/docs/concepts/resources/). The resource builder to the providers is optional. If not provided, the default resource with default attributes is used. + +Next, we turn on diagnostics with sensitive data. This is an experimental feature that allows you to enable diagnostics for the AI services in the Semantic Kernel. With this turned on, you will see additional telemetry data such as the prompts sent to and the responses received from the AI models, which are considered sensitive data. If you don't want to include sensitive data in your telemetry, you can use another switch `Microsoft.SemanticKernel.Experimental.GenAI.EnableOTelDiagnostics` to enable diagnostics with non-sensitive data, such as the model name, the operation name, and token usage, etc. + +Then, we create a tracer provider builder and a meter provider builder. A provider is responsible for processing telemetry data and piping it to exporters. We subscribe to the `Microsoft.SemanticKernel*` source to receive telemetry data from the Semantic Kernel namespaces. We add a console exporter to both the tracer provider and the meter provider. The console exporter sends telemetry data to the console. + +Finally, we create a logger factory and add OpenTelemetry as a logging provider that sends log data to the console. We set the minimum log level to `Information` and include formatted messages and scopes in the log output. The logger factory is then added to the builder. + +> [!IMPORTANT] +> A provider should be a singleton and should be alive for the entire application lifetime. The provider should be disposed of when the application is shutting down. + +::: zone-end + +::: zone pivot="programming-language-python" + +### Create a new Python virtual environment + +```console +python -m venv telemetry-console-quickstart +``` + +Activate the virtual environment. +```console +telemetry-console-quickstart\Scripts\activate +``` + +### Install required packages + +```console +pip install semantic-kernel +``` + +### Create a simple Python script with Semantic Kernel + +Create a new Python script and open it with your favorite editor. + +# [Powershell](#tab/Powershell-CreateFile) + +```PowerShell +New-Item -Path telemetry_console_quickstart.py -ItemType file +``` + +# [Bash](#tab/Bash-CreatFile) + +```bash +touch telemetry_console_quickstart.py +``` + +--- + +We are going to create a simple Python script that uses Semantic Kernel to send a prompt to a chat completion model. Replace the existing content with the following code and fill in the required values for `deployment_name`, `endpoint`, and `api_key`: + +```python +import asyncio +import logging + +from opentelemetry._logs import set_logger_provider +from opentelemetry.metrics import set_meter_provider +from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler +from opentelemetry.sdk._logs.export import BatchLogRecordProcessor, ConsoleLogExporter +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.metrics.export import ConsoleMetricExporter, PeriodicExportingMetricReader +from opentelemetry.sdk.metrics.view import DropAggregation, View +from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import BatchSpanProcessor, ConsoleSpanExporter +from opentelemetry.semconv.resource import ResourceAttributes +from opentelemetry.trace import set_tracer_provider + +from semantic_kernel import Kernel +from semantic_kernel.connectors.ai.open_ai import AzureChatCompletion + + +# Telemetry setup code goes here + +async def main(): + # Create a kernel and add a service + kernel = Kernel() + kernel.add_service(AzureChatCompletion( + api_key="your-azure-openai-api-key", + endpoint="your-azure-openai-endpoint", + deployment_name="your-deployment-name" + )) + + answer = await kernel.invoke_prompt("Why is the sky blue in one sentence?") + print(answer) + + +if __name__ == "__main__": + asyncio.run(main()) +``` + +### Add telemetry + +#### Environment variables + +By default, the kernel doesn't emit spans for the AI connectors, because these spans carry `gen_ai` attributes that are considered experimental. To enable the feature, set the environment variable `SEMANTICKERNEL_EXPERIMENTAL_GENAI_ENABLE_OTEL_DIAGNOSTICS` or `SEMANTICKERNEL_EXPERIMENTAL_GENAI_ENABLE_OTEL_DIAGNOSTICS_SENSITIVE` to `true`. + +> [!IMPORTANT] +> Prompts and completions are considered sensitive data. Semantic Kernel will not emit these data from the AI connectors unless the `SEMANTICKERNEL_EXPERIMENTAL_GENAI_ENABLE_OTEL_DIAGNOSTICS_SENSITIVE` environment variable is set to `true`. Setting `SEMANTICKERNEL_EXPERIMENTAL_GENAI_ENABLE_OTEL_DIAGNOSTICS` to `true` will only emit non-sensitive data such as the model name, the operation name, and token usage. + +# [Environment fIel](#tab/EnvironmentFile) + +Create a new file named `.env` in the same directory as your script and add the following content: + +```env +SEMANTICKERNEL_EXPERIMENTAL_GENAI_ENABLE_OTEL_DIAGNOSTICS_SENSITIVE=true +``` + +# [Powershell](#tab/Powershell-EnvironmentVariable) + +```PowerShell +$Env:SEMANTICKERNEL_EXPERIMENTAL_GENAI_ENABLE_OTEL_DIAGNOSTICS_SENSITIVE = 'true' +``` + +> [!NOTE] +> To remove the environment variable, run `Remove-Item Env:SEMANTICKERNEL_EXPERIMENTAL_GENAI_ENABLE_OTEL_DIAGNOSTICS_SENSITIVE`. + +# [Bash](#tab/Bash-EnvironmentVariable) + +```bash +export SEMANTICKERNEL_EXPERIMENTAL_GENAI_ENABLE_OTEL_DIAGNOSTICS_SENSITIVE=true +``` + +> [!NOTE] +> To remove the environment variable, run `unset SEMANTICKERNEL_EXPERIMENTAL_GENAI_ENABLE_OTEL_DIAGNOSTICS_SENSITIVE`. + +--- + +#### Code +If you run the script now, you should expect to see a sentence explaining why the sky is blue. To observe the kernel via telemetry, replace the `# Telemetry setup code goes here` comment with the following code: + +```python +# Create a resource to represent the service/sample +resource = Resource.create({ResourceAttributes.SERVICE_NAME: "telemetry-console-quickstart"}) + + +def set_up_logging(): + exporter = ConsoleLogExporter() + + # Create and set a global logger provider for the application. + logger_provider = LoggerProvider(resource=resource) + # Log processors are initialized with an exporter which is responsible + # for sending the telemetry data to a particular backend. + logger_provider.add_log_record_processor(BatchLogRecordProcessor(exporter)) + # Sets the global default logger provider + set_logger_provider(logger_provider) + + # Create a logging handler to write logging records, in OTLP format, to the exporter. + handler = LoggingHandler() + # Add filters to the handler to only process records from semantic_kernel. + handler.addFilter(logging.Filter("semantic_kernel")) + # Attach the handler to the root logger. `getLogger()` with no arguments returns the root logger. + # Events from all child loggers will be processed by this handler. + logger = logging.getLogger() + logger.addHandler(handler) + logger.setLevel(logging.INFO) + + +def set_up_tracing(): + exporter = ConsoleSpanExporter() + + # Initialize a trace provider for the application. This is a factory for creating tracers. + tracer_provider = TracerProvider(resource=resource) + # Span processors are initialized with an exporter which is responsible + # for sending the telemetry data to a particular backend. + tracer_provider.add_span_processor(BatchSpanProcessor(exporter)) + # Sets the global default tracer provider + set_tracer_provider(tracer_provider) + + +def set_up_metrics(): + exporter = ConsoleMetricExporter() + + # Initialize a metric provider for the application. This is a factory for creating meters. + meter_provider = MeterProvider( + metric_readers=[PeriodicExportingMetricReader(exporter, export_interval_millis=5000)], + resource=resource, + views=[ + # Dropping all instrument names except for those starting with "semantic_kernel" + View(instrument_name="*", aggregation=DropAggregation()), + View(instrument_name="semantic_kernel*"), + ], + ) + # Sets the global default meter provider + set_meter_provider(meter_provider) + + +# This must be done before any other telemetry calls +set_up_logging() +set_up_tracing() +set_up_metrics() +``` + +In the above code snippet, we first create a resource to represent the service. A resource represents the entity that produces telemetry data. You can read more about resources [here](https://opentelemetry.io/docs/concepts/resources/). We then create three functions to set up logging, tracing, and metrics. Each function creates a provider for the respective telemetry data and adds a console exporter to the provider. + +Finally, we call the three functions to set up logging, tracing, and metrics. This must be done before any other telemetry calls. + +::: zone-end + +::: zone pivot="programming-language-java" + +> [!NOTE] +> Semantic Kernel Observability is not yet available for Java. + +::: zone-end + +## Run + +::: zone pivot="programming-language-csharp" + +Run the console application with the following command: + +```console +dotnet run +``` + +::: zone-end + +::: zone pivot="programming-language-python" + +Run the Python script with the following command: + +```console +python telemetry_console_quickstart.py +``` + +::: zone-end + +::: zone pivot="programming-language-java" + +> [!NOTE] +> Semantic Kernel Observability is not yet available for Java. + +::: zone-end + +## Inspect telemetry data + +::: zone pivot="programming-language-csharp" + +### Log records + +You should see multiple log records in the console output. They look similar to the following: + +```console +LogRecord.Timestamp: 2024-09-12T21:48:35.2295938Z +LogRecord.TraceId: 159d3f07664838f6abdad7af6a892cfa +LogRecord.SpanId: ac79a006da8a6215 +LogRecord.TraceFlags: Recorded +LogRecord.CategoryName: Microsoft.SemanticKernel.KernelFunction +LogRecord.Severity: Info +LogRecord.SeverityText: Information +LogRecord.FormattedMessage: Function InvokePromptAsync_290eb9bece084b00aea46b569174feae invoking. +LogRecord.Body: Function {FunctionName} invoking. +LogRecord.Attributes (Key:Value): + FunctionName: InvokePromptAsync_290eb9bece084b00aea46b569174feae + OriginalFormat (a.k.a Body): Function {FunctionName} invoking. + +Resource associated with LogRecord: +service.name: TelemetryConsoleQuickstart +service.instance.id: a637dfc9-0e83-4435-9534-fb89902e64f8 +telemetry.sdk.name: opentelemetry +telemetry.sdk.language: dotnet +telemetry.sdk.version: 1.9.0 +``` + +There are two parts to each log record: +- The log record itself: contains the timestamp and namespace at which the log record was generated, the severity and body of the log record, and any attributes associated with the log record. +- The resource associated with the log record: contains information about the service, instance, and SDK used to generate the log record. + +### Activities + +> [!NOTE] +> Activities in .Net are similar to spans in OpenTelemetry. They are used to represent a unit of work in the application. + +You should see multiple activities in the console output. They look similar to the following: + +```console +Activity.TraceId: 159d3f07664838f6abdad7af6a892cfa +Activity.SpanId: 8c7c79bc1036eab3 +Activity.TraceFlags: Recorded +Activity.ParentSpanId: ac79a006da8a6215 +Activity.ActivitySourceName: Microsoft.SemanticKernel.Diagnostics +Activity.DisplayName: chat.completions gpt-4o +Activity.Kind: Client +Activity.StartTime: 2024-09-12T21:48:35.5717463Z +Activity.Duration: 00:00:02.3992014 +Activity.Tags: + gen_ai.operation.name: chat.completions + gen_ai.system: openai + gen_ai.request.model: gpt-4o + gen_ai.response.prompt_tokens: 16 + gen_ai.response.completion_tokens: 29 + gen_ai.response.finish_reason: Stop + gen_ai.response.id: chatcmpl-A6lxz14rKuQpQibmiCpzmye6z9rxC +Activity.Events: + gen_ai.content.prompt [9/12/2024 9:48:35 PM +00:00] + gen_ai.prompt: [{"role": "user", "content": "Why is the sky blue in one sentence?"}] + gen_ai.content.completion [9/12/2024 9:48:37 PM +00:00] + gen_ai.completion: [{"role": "Assistant", "content": "The sky appears blue because shorter blue wavelengths of sunlight are scattered in all directions by the gases and particles in the Earth\u0027s atmosphere more than other colors."}] +Resource associated with Activity: + service.name: TelemetryConsoleQuickstart + service.instance.id: a637dfc9-0e83-4435-9534-fb89902e64f8 + telemetry.sdk.name: opentelemetry + telemetry.sdk.language: dotnet + telemetry.sdk.version: 1.9.0 +``` + +There are two parts to each activity: +- The activity itself: contains the span ID and parent span ID that APM tools use to build the traces, the duration of the activity, and any tags and events associated with the activity. +- The resource associated with the activity: contains information about the service, instance, and SDK used to generate the activity. + +> [!IMPORTANT] +> The attributes to pay extra attention to are the ones that start with `gen_ai`. These are the attributes specified in the [GenAI Semantic Conventions](https://github.com/open-telemetry/semantic-conventions/blob/main/docs/gen-ai/README.md). + +### Metrics + +You should see multiple metric records in the console output. They look similar to the following: + +```console +Metric Name: semantic_kernel.connectors.openai.tokens.prompt, Number of prompt tokens used, Unit: {token}, Meter: Microsoft.SemanticKernel.Connectors.OpenAI +(2024-09-12T21:48:37.9531072Z, 2024-09-12T21:48:38.0966737Z] LongSum +Value: 16 +``` + +Here you can see the name, the description, the unit, the time range, the type, the value of the metric, and the meter that the metric belongs to. + +> [!NOTE] +> The above metric is a Counter metric. For a full list of metric types, see [here](https://learn.microsoft.com/dotnet/core/diagnostics/metrics-instrumentation#types-of-instruments). Depending on the type of metric, the output may vary. + +::: zone-end + +::: zone pivot="programming-language-python" + +### Logs + +You should see multiple log records in the console output. They look similar to the following: + +```console +{ + "body": "Function SyVCcBjaULqEhItH invoking.", + "severity_number": "", + "severity_text": "INFO", + "attributes": { + "code.filepath": "C:\\tmp\\telemetry-console-quickstart\\Lib\\site-packages\\semantic_kernel\\functions\\kernel_function_log_messages.py", + "code.function": "log_function_invoking", + "code.lineno": 19 + }, + "dropped_attributes": 0, + "timestamp": "2024-09-13T17:55:45.504983Z", + "observed_timestamp": "2024-09-13T17:55:45.504983Z", + "trace_id": "0xe23e2c10785ea61ffc9f28be19482a80", + "span_id": "0x686bd592e27661d7", + "trace_flags": 1, + "resource": { + "attributes": { + "telemetry.sdk.language": "python", + "telemetry.sdk.name": "opentelemetry", + "telemetry.sdk.version": "1.27.0", + "service.name": "telemetry-console-quickstart" + }, + "schema_url": "" + } +} +``` + +### Spans + +You should see multiple spans in the console output. They look similar to the following: + +```console +{ + "name": "chat.completions gpt-4o", + "context": { + "trace_id": "0xe23e2c10785ea61ffc9f28be19482a80", + "span_id": "0x8b20e9655610c3c9", + "trace_state": "[]" + }, + "kind": "SpanKind.INTERNAL", + "parent_id": "0x686bd592e27661d7", + "start_time": "2024-09-13T17:55:45.515198Z", + "end_time": "2024-09-13T17:55:46.469471Z", + "status": { + "status_code": "UNSET" + }, + "attributes": { + "gen_ai.operation.name": "chat.completions", + "gen_ai.system": "openai", + "gen_ai.request.model": "gpt-4o", + "gen_ai.response.id": "chatcmpl-A74oD7WGDjawnZ44SJZrj9fKrEv1B", + "gen_ai.response.finish_reason": "FinishReason.STOP", + "gen_ai.response.prompt_tokens": 16, + "gen_ai.response.completion_tokens": 29 + }, + "events": [ + { + "name": "gen_ai.content.prompt", + "timestamp": "2024-09-13T17:55:45.515198Z", + "attributes": { + "gen_ai.prompt": "[{\"role\": \"user\", \"content\": \"Why is the sky blue in one sentence?\"}]" + } + }, + { + "name": "gen_ai.content.completion", + "timestamp": "2024-09-13T17:55:46.469471Z", + "attributes": { + "gen_ai.completion": "[{\"role\": \"assistant\", \"content\": \"The sky appears blue because shorter blue wavelengths of sunlight are scattered in all directions by the molecules and particles in the atmosphere more effectively than other colors.\"}]" + } + } + ], + "links": [], + "resource": { + "attributes": { + "telemetry.sdk.language": "python", + "telemetry.sdk.name": "opentelemetry", + "telemetry.sdk.version": "1.27.0", + "service.name": "telemetry-console-quickstart" + }, + "schema_url": "" + } +} +``` + +Pay attention to the attributes that start with `gen_ai`. These are the attributes specified in the [GenAI Semantic Conventions](https://github.com/open-telemetry/semantic-conventions/blob/main/docs/gen-ai/README.md). They provide useful information about the requests sent to and the responses received from the AI models. + +### Metrics + +You should see multiple metric records in the console output. They look similar to the following: + +```console +{ + "resource_metrics": [ + { + "resource": { + "attributes": { + "telemetry.sdk.language": "python", + "telemetry.sdk.name": "opentelemetry", + "telemetry.sdk.version": "1.27.0", + "service.name": "telemetry-console-quickstart" + }, + "schema_url": "" + }, + "scope_metrics": [ + { + "scope": { + "name": "semantic_kernel.functions.kernel_function", + "version": null, + "schema_url": "", + "attributes": null + }, + "metrics": [ + { + "name": "semantic_kernel.function.invocation.duration", + "description": "Measures the duration of a function's execution", + "unit": "s", + "data": { + "data_points": [ + { + "attributes": { + "semantic_kernel.function.name": "SyVCcBjaULqEhItH" + }, + "start_time_unix_nano": 1726250146470468300, + "time_unix_nano": 1726250146478526600, + "count": 1, + "sum": 0.9650602999900002, + "bucket_counts": [ + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "explicit_bounds": [ + 0.0, + 5.0, + 10.0, + 25.0, + 50.0, + 75.0, + 100.0, + 250.0, + 500.0, + 750.0, + 1000.0, + 2500.0, + 5000.0, + 7500.0, + 10000.0 + ], + "min": 0.9650602999900002, + "max": 0.9650602999900002 + } + ], + "aggregation_temporality": 2 + } + } + ], + "schema_url": "" + } + ], + "schema_url": "" + } + ] +} +``` + +The measurement shown above is a histogram metric. For a full list of metric types, see [here](https://opentelemetry.io/docs/specs/otel/metrics/api/#general-operations). + +::: zone-end + +::: zone pivot="programming-language-java" + +> [!NOTE] +> Semantic Kernel Observability is not yet available for Java. + +::: zone-end + +## Next steps + +Now that you have successfully output telemetry data to the console, you can learn more about how to use APM tools to visualize and analyze telemetry data. + +> [!div class="nextstepaction"] +> [Application Insights](telemetry-with-app-insights.md) + +> [!div class="nextstepaction"] +> [Aspire Dashboard](telemetry-with-aspire-dashboard.md) \ No newline at end of file diff --git a/semantic-kernel/concepts/kernel.md b/semantic-kernel/concepts/kernel.md index 9ad2bee0..69de031c 100644 --- a/semantic-kernel/concepts/kernel.md +++ b/semantic-kernel/concepts/kernel.md @@ -13,7 +13,7 @@ ms.service: semantic-kernel The kernel is the central component of Semantic Kernel. At its simplest, the kernel is a Dependency Injection container that manages all of the services and plugins necessary to run your AI application. If you provide all of your services and plugins to the kernel, they will then be seamlessly used by the AI as needed. -## The kernel is at the center of your agents +## The kernel is at the center Because the kernel has all of the services and plugins necessary to run both native code and AI services, it is used by nearly every component within the Semantic Kernel SDK to power your agents. This means that if you run any prompt or code in Semantic Kernel, the kernel will always be available to retrieve the necessary services and plugins. ![The kernel is at the center of everything in Semantic Kernel](../media/the-kernel-is-at-the-center-of-everything.png) diff --git a/semantic-kernel/concepts/personas.md b/semantic-kernel/concepts/personas.md deleted file mode 100644 index 4efaee36..00000000 --- a/semantic-kernel/concepts/personas.md +++ /dev/null @@ -1,114 +0,0 @@ ---- -title: Personas in Semantic Kernel -description: Learn about personas in Semantic Kernel. -author: sophialagerkranspandey -zone_pivot_groups: programming-languages -ms.topic: conceptual -ms.author: sopand -ms.date: 07/12/2023 -ms.service: semantic-kernel ---- - -# Personas: Giving your Agent a Role - - -:::row::: - :::column span="3"::: - Often called a "meta prompt" or "instruction", the persona is a prompt that is used to influence how the agent responds to stimuli. This allows you to influence how your agents plan tasks, generate responses, and interact with users. For example, you can use the persona to explicitly tell the agent to ask for help if it doesn't know what to do, or to be more verbose when it is explaining something. - - In Semantic Kernel, we often describe these prompts as "personas" because they're very similar to persona in user experience design. Just like how designers and UX researchers create personas to represent different types of users and the jobs-to-be-done they have, you can create personas to represent different types of agents and the tasks they are responsible for. - - By doing so, you can create agents that are more consistent, more reliable, and more predictable. This is especially important when you're building agents that are responsible for critical tasks or that interact with users in sensitive situations. - :::column-end::: - :::column span="2"::: - ![Semantic Kernel Persona](../media/personas.png) - :::column-end::: -:::row-end::: - -The rest of this article will provide tips and best practices when creating personas in Semantic Kernel. - -## Best practices for creating personas - -Creating effective personas involves careful consideration of the agent's intended role, the target audience, and the specific tasks the agent will perform. Here are some best practices to keep in mind: - -| **Step** | Description | -|-----------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------| -| **1. Define clear objectives** | Before creating a persona, clearly define the objectives you want the agent to achieve. This includes understanding the tasks the agent will perform and the type of interactions you expect it to have with users. | -| **2. Reference specific roles or public figures** | A well-defined persona should include specific details about the agent's behavior, tone, and style. Avoid vague descriptions by referencing specific roles or public figures so that the LLM can better mimic their behavior. | -| **3. Provide examples and scenarios** | To help the agent understand its role better, provide examples and scenarios that illustrate the persona in action. This can include sample conversations, user stories, or use cases that demonstrate how the agent should respond in different situations. | -| **4. Include fallback instructions** | Provide instructions on what the agent should do when it encounters an unknown situation or is unable to provide a response. This can include asking for help, providing alternative solutions, or simply acknowledging that it doesn't have an answer. | -| **5. Iterate and Refine** | Personas are not static. Continuously test and refine your personas based on user feedback and performance data. This helps in improving the agent's effectiveness over time. | - - -## Using system messages to set personas - -The most common system messages used within Semantic Kernel are the user, assistant, and tool messages. These messages are used to simulate a conversation between the user and the agent. - -A forth message type, the system message, can be used to provide the agent with additional context or instructions that isn't shown to the user. This can be used to set the persona for the agent. - -In Semantic Kernel, providing an initial system message for the agent is as simple as adding it in the `ChatHistory` constructor. For example: - -::: zone pivot="programming-language-csharp" -```csharp -ChatHistory chatHistory = new(""" - You are a technical support specialist for a software company. - Your primary task is to assist users with technical issues, - such as installation problems, software bugs, and feature - inquiries. Use technical jargon appropriately, but ensure that - explanations are easy to understand. If a problem is too complex, - suggest advanced troubleshooting steps or escalate to a higher-level - support team using the escalate tool. - """) -``` -::: zone-end - -::: zone pivot="programming-language-python" -```python -chat_history = ChatHistory(""" - You are a technical support specialist for a software company. - Your primary task is to assist users with technical issues, - such as installation problems, software bugs, and feature - inquiries. Use technical jargon appropriately, but ensure that - explanations are easy to understand. If a problem is too complex, - suggest advanced troubleshooting steps or escalate to a higher-level - support team using the escalate tool. - """) -``` -::: zone-end - -::: zone pivot="programming-language-java" -```java - ChatHistory chatHistory = new ChatHistory(""" - You are a technical support specialist for a software company. - Your primary task is to assist users with technical issues, - such as installation problems, software bugs, and feature - inquiries. Use technical jargon appropriately, but ensure that - explanations are easy to understand. If a problem is too complex, - suggest advanced troubleshooting steps or escalate to a higher-level - support team using the escalate tool. - """); -``` -::: zone-end - -If you need greater control over the persona (for example, you want to change the instructions as the conversation progresses), you can use the `AddSystemMessage` method to add additional system messages to the chat history. - -::: zone pivot="programming-language-csharp" -```csharp -chatHistory.AddSystemMessage("Remember to ask for help if you're unsure how to proceed.") -``` -::: zone-end - -::: zone pivot="programming-language-python" -```python -chat_history.add_system_message("Remember to ask for help if you're unsure how to proceed.") -``` -::: zone-end - -::: zone pivot="programming-language-java" -```java -chatHistory.addSystemMessage("Remember to ask for help if you're unsure how to proceed."); -``` -::: zone-end - - -By reminding the LLM of its personas in this way, you can ensure that your agent is consistent, reliable, and predictable in its interactions with users. diff --git a/semantic-kernel/concepts/planning.md b/semantic-kernel/concepts/planning.md index 76128896..0707bcd1 100644 --- a/semantic-kernel/concepts/planning.md +++ b/semantic-kernel/concepts/planning.md @@ -13,7 +13,7 @@ ms.service: semantic-kernel Once you have multiple plugins, you then need a way for your AI agent to use them together to solve a user’s need. This is where planning comes in. -Early on, Semantic Kernel introduced the concept of planners that used prompts to request the AI to choose which functions to invoke. Since Semantic Kernel was introduced, however, OpenAI introduced a native way for the model to invoke or “call” a function: [function calling](../concepts/ai-services/chat-completion/function-calling.md). Other AI models like Gemini, Claude, and Mistral have since adopted function calling as a core capability, making it a cross-model supported feature. +Early on, Semantic Kernel introduced the concept of planners that used prompts to request the AI to choose which functions to invoke. Since Semantic Kernel was introduced, however, OpenAI introduced a native way for the model to invoke or “call” a function: [function calling](../concepts/ai-services/chat-completion/function-calling/index.md). Other AI models like Gemini, Claude, and Mistral have since adopted function calling as a core capability, making it a cross-model supported feature. Because of these advancements, Semantic Kernel has evolved to use function calling as the primary way to plan and execute tasks. @@ -65,7 +65,7 @@ Supporting function calling without Semantic Kernel is relatively complex. You w In Semantic Kernel, we make it easy to use function calling by automating this loop for you. This allows you to focus on building the plugins needed to solve your user's needs. > [!NOTE] -> Understanding how the function calling loop works is essential for building performant and reliable AI agents. For an in-depth look at how the loop works, see the [function calling](./ai-services/chat-completion/function-calling.md) article. +> Understanding how the function calling loop works is essential for building performant and reliable AI agents. For an in-depth look at how the loop works, see the [function calling](./ai-services/chat-completion/function-calling/index.md) article. ## Using automatic function calling @@ -92,7 +92,7 @@ var chatCompletionService = kernel.GetRequiredService(); // 2. Enable automatic function calling OpenAIPromptExecutionSettings openAIPromptExecutionSettings = new()  { - ToolCallBehavior = ToolCallBehavior.AutoInvokeKernelFunctions + FunctionChoiceBehavior = FunctionChoiceBehavior.Auto() }; var history = new ChatHistory(); @@ -267,13 +267,10 @@ When you use automatic function calling, all of the steps in the automatic plann The Stepwise and Handlebars planners are still available in Semantic Kernel. However, we recommend using function calling for most tasks as it is more powerful and easier to use. Both the Stepwise and Handlebars planners will be deprecated in a future release of Semantic Kernel. -Before we deprecate these planners, we will provide guidance on how to migrate your existing planners to function calling. If you have any questions about this process, please reach out to us on the [discussions board](https://github.com/microsoft/semantic-kernel/discussions) in the Semantic Kernel GitHub repository. +Learn how to [migrate Stepwise Planner to Auto Function Calling](../support/migration/stepwise-planner-migration-guide.md). > [!CAUTION] > If you are building a new AI agent, we recommend that you _not_ use the Stepwise or Handlebars planners. Instead, use function calling as it is more powerful and easier to use. ## Next steps Now that you understand how planners work in Semantic Kernel, you can learn more about how influence your AI agent so that they best plan and execute tasks on behalf of your users. - -> [!div class="nextstepaction"] -> [Learn about personas](./personas.md) \ No newline at end of file diff --git a/semantic-kernel/concepts/plugins/adding-native-plugins.md b/semantic-kernel/concepts/plugins/adding-native-plugins.md index b3dd5a92..246837ec 100644 --- a/semantic-kernel/concepts/plugins/adding-native-plugins.md +++ b/semantic-kernel/concepts/plugins/adding-native-plugins.md @@ -84,18 +84,12 @@ class LightsPlugin: def __init__(self, lights: List[LightModel]): self._lights = lights - @kernel_function( - name="get_lights", - description="Gets a list of lights and their current state", - ) + @kernel_function async def get_lights(self) -> Annotated[List[LightModel], "An array of lights"]: """Gets a list of lights and their current state.""" return self._lights - @kernel_function( - name="change_state", - description="Changes the state of the light", - ) + @kernel_function async def change_state( self, change_state: LightModel @@ -157,14 +151,14 @@ public enum Brightness ::: zone pivot="programming-language-python" ```python -from typing import TypedDict, Optional +from typing import TypedDict class LightModel(TypedDict): id: int name: str - is_on: Optional[bool] - brightness: Optional[int] - hex: Optional[str] + is_on: bool | None + brightness: int | None + hex: str | None ``` ::: zone-end diff --git a/semantic-kernel/concepts/plugins/index.md b/semantic-kernel/concepts/plugins/index.md index b32ccfc7..ffcb390d 100644 --- a/semantic-kernel/concepts/plugins/index.md +++ b/semantic-kernel/concepts/plugins/index.md @@ -159,48 +159,39 @@ public class LightModel ::: zone pivot="programming-language-python" ```python -from typing import List, Optional, TypedDict, Annotated +from typing import TypedDict, Annotated class LightModel(TypedDict): id: int name: str - is_on: Optional[bool] - brightness: Optional[int] - hex: Optional[str] + is_on: bool | None + brightness: int | None + hex: str | None class LightsPlugin: - lights: List[LightModel] = [ + lights: list[LightModel] = [ {"id": 1, "name": "Table Lamp", "is_on": False, "brightness": 100, "hex": "FF0000"}, {"id": 2, "name": "Porch light", "is_on": False, "brightness": 50, "hex": "00FF00"}, {"id": 3, "name": "Chandelier", "is_on": True, "brightness": 75, "hex": "0000FF"}, ] - @kernel_function( - name="get_lights", - description="Gets a list of lights and their current state", - ) - async def get_lights(self) -> Annotated[List[LightModel], "An array of lights"]: + @kernel_function + async def get_lights(self) -> Annotated[list[LightModel], "An array of lights"]: """Gets a list of lights and their current state.""" return self.lights - @kernel_function( - name="get_state", - description="Gets the state of a particular light", - ) + @kernel_function async def get_state( self, id: Annotated[int, "The ID of the light"] - ) -> Annotated[Optional[LightModel], "The state of the light"]: + ) -> Annotated[LightModel | None], "The state of the light"]: """Gets the state of a particular light.""" for light in self.lights: if light["id"] == id: return light return None - @kernel_function( - name="change_state", - description="Changes the state of the light", - ) + @kernel_function async def change_state( self, id: Annotated[int, "The ID of the light"], @@ -284,7 +275,7 @@ kernel.Plugins.AddFromType("Lights"); // Enable planning OpenAIPromptExecutionSettings openAIPromptExecutionSettings = new()  { - ToolCallBehavior = ToolCallBehavior.AutoInvokeKernelFunctions + FunctionChoiceBehavior = FunctionChoiceBehavior.Auto() }; // Create a history store the conversation @@ -326,11 +317,12 @@ async def main(): kernel = Kernel() # Add Azure OpenAI chat completion - kernel.add_service(AzureChatCompletion( + chat_completion = AzureChatCompletion( deployment_name="your_models_deployment_name", api_key="your_api_key", base_url="your_base_url", - )) + ) + kernel.add_service(chat_completion) # Add a plugin (the LightsPlugin class is defined below) kernel.add_plugin( @@ -338,8 +330,6 @@ async def main(): plugin_name="Lights", ) - chat_completion : AzureChatCompletion = kernel.get_service(type=ChatCompletionClientBase) - # Enable planning execution_settings = AzureChatPromptExecutionSettings(tool_choice="auto") execution_settings.function_call_behavior = FunctionCallBehavior.EnableFunctions(auto_invoke=True, filters={}) @@ -349,12 +339,11 @@ async def main(): history.add_message("Please turn on the lamp") # Get the response from the AI - result = (await chat_completion.get_chat_message_contents( + result = await chat_completion.get_chat_message_content( chat_history=history, settings=execution_settings, kernel=kernel, - arguments=KernelArguments(), - ))[0] + ) # Print the results print("Assistant > " + str(result)) diff --git a/semantic-kernel/concepts/plugins/using-data-retrieval-functions-for-rag.md b/semantic-kernel/concepts/plugins/using-data-retrieval-functions-for-rag.md index da9e07ee..3f5463ce 100644 --- a/semantic-kernel/concepts/plugins/using-data-retrieval-functions-for-rag.md +++ b/semantic-kernel/concepts/plugins/using-data-retrieval-functions-for-rag.md @@ -150,7 +150,7 @@ When developing plugins for Retrieval Augmented Generation (RAG), you must also ### Dynamic data retrieval -In most cases, the user query will determine the data that the AI agent needs to retrieve. For example, a user might ask for the difference between two different products. The AI agent would then need to dynamically retrieve the product information from a database or API to generate a response using [function calling](../ai-services/chat-completion/function-calling.md). It would be impractical to pre-fetch all possible product information ahead of time and give it to the AI agent. +In most cases, the user query will determine the data that the AI agent needs to retrieve. For example, a user might ask for the difference between two different products. The AI agent would then need to dynamically retrieve the product information from a database or API to generate a response using [function calling](../ai-services/chat-completion/function-calling/index.md). It would be impractical to pre-fetch all possible product information ahead of time and give it to the AI agent. Below is an example of a back-and-forth chat between a user and an AI agent where dynamic data retrieval is necessary. diff --git a/semantic-kernel/concepts/plugins/using-task-automation-functions.md b/semantic-kernel/concepts/plugins/using-task-automation-functions.md index 302dee9a..a90ff55f 100644 --- a/semantic-kernel/concepts/plugins/using-task-automation-functions.md +++ b/semantic-kernel/concepts/plugins/using-task-automation-functions.md @@ -1,6 +1,7 @@ --- title: Allow agents to automate tasks description: Learn how you safely allow agents to automate tasks on behalf of users in Semantic Kernel. +zone_pivot_groups: programming-languages author: sophialagerkranspandey ms.topic: conceptual ms.author: sopand @@ -22,6 +23,7 @@ In Semantic Kernel, you can use the function invocation filter. This filter is a Here's an example of a function invocation filter that requires user consent: +::: zone pivot="programming-language-csharp" ```csharp public class ApprovalFilterExample() : IFunctionInvocationFilter { @@ -51,6 +53,38 @@ IKernelBuilder builder = Kernel.CreateBuilder(); builder.Services.AddSingleton(); Kernel kernel = builder.Build(); ``` +::: zone-end + +::: zone pivot="programming-language-python" +```python +from typing import Any, Coroutine +from semantic_kernel.filters.filter_types import FilterTypes +from semantic_kernel.filters.functions.function_invocation_context import FunctionInvocationContext +from semantic_kernel.functions.function_result import FunctionResult + +# The `filter` decorator within kernel, creates and adds the filter in one go. +@kernel.filter(filter_type=FilterTypes.FUNCTION_INVOCATION) +async def approval_filter_example( + context: FunctionInvocationContext, next: Coroutine[FunctionInvocationContext, Any, None] +): + if context.function.plugin_name == "DynamicsPlugin" and context.function.name == "create_order": + should_proceed = input("System > The agent wants to create an approval, do you want to proceed? (Y/N)") + if should_proceed.lower() != "y": + context.result = FunctionResult( + function=context.function.metadata, value="The order creation was not approved by the user" + ) + return + + await next(context) + +``` +::: zone-end + +::: zone pivot="programming-language-java" + +> Java sample is coming soon. + +::: zone-end Now, whenever the AI agent tries to create an order using the `DynamicsPlugin`, the user will be prompted to approve the action. diff --git a/semantic-kernel/concepts/prompts/TOC.yml b/semantic-kernel/concepts/prompts/TOC.yml new file mode 100644 index 00000000..b0fe1ef1 --- /dev/null +++ b/semantic-kernel/concepts/prompts/TOC.yml @@ -0,0 +1,2 @@ +- name: Prompts + href: index.md diff --git a/semantic-kernel/concepts/prompts/index.md b/semantic-kernel/concepts/prompts/index.md new file mode 100644 index 00000000..838059d6 --- /dev/null +++ b/semantic-kernel/concepts/prompts/index.md @@ -0,0 +1,56 @@ +--- +title: Prompt engineering with Semantic Kernel +description: Understanding the importance of prompts in directing the behavior of AIs +author: johnmaeda +ms.topic: prompt-engineering +ms.author: johnmaeda +ms.date: 09/27/2024 +ms.service: semantic-kernel +--- +# What are prompts? + +Prompts play a crucial role in communicating and directing the behavior of Large Language Models (LLMs) AI. They serve as inputs or queries that users can provide to elicit specific responses from a model. + + +## The subtleties of prompting +Effective prompt design is essential to achieving desired outcomes with LLM AI models. Prompt engineering, also known as prompt design, is an emerging field that requires creativity and attention to detail. It involves selecting the right words, phrases, symbols, and formats that guide the model in generating high-quality and relevant texts. + +If you've already experimented with ChatGPT, you can see how the model's behavior changes dramatically based on the inputs you provide. For example, the following prompts produce very different outputs: + +```Prompt +Please give me the history of humans. +``` + +```Prompt +Please give me the history of humans in 3 sentences. +``` + +The first prompt produces a long report, while the second prompt produces a concise response. If you were building a UI with limited space, the second prompt would be more suitable for your needs. Further refined behavior can be achieved by adding even more details to the prompt, but its possible to go too far and produce irrelevant outputs. As a prompt engineer, you must find the right balance between specificity and relevance. + +When you work directly with LLM models, you can also use other controls to influence the model's behavior. For example, you can use the `temperature` parameter to control the randomness of the model's output. Other parameters like top-k, top-p, frequency penalty, and presence penalty also influence the model's behavior. + +## Prompt engineering: a new career +Because of the amount of control that exists, prompt engineering is a critical skill for anyone working with LLM AI models. It's also a skill that's in high demand as more organizations adopt LLM AI models to automate tasks and improve productivity. A good prompt engineer can help organizations get the most out of their LLM AI models by designing prompts that produce the desired outputs. + +### Becoming a great prompt engineer with Semantic Kernel +Semantic Kernel is a valuable tool for prompt engineering because it allows you to experiment with different prompts and parameters across multiple different models using a common interface. This allows you to quickly compare the outputs of different models and parameters, and iterate on prompts to achieve the desired results. + +Once you've become familiar with prompt engineering, you can also use Semantic Kernel to apply your skills to real-world scenarios. By combining your prompts with native functions and connectors, you can build powerful AI-powered applications. + +Lastly, by deeply integrating with Visual Studio Code, Semantic Kernel also makes it easy for you to integrate prompt engineering into your existing development processes. + +> [!div class="checklist"] +> * Create prompts directly in your preferred code editor. +> * Write tests for them using your existing testing frameworks. +> * And deploy them to production using your existing CI/CD pipelines. + +### Additional tips for prompt engineering +Becoming a skilled prompt engineer requires a combination of technical knowledge, creativity, and experimentation. Here are some tips to excel in prompt engineering: + +- **Understand LLM AI models:** Gain a deep understanding of how LLM AI models work, including their architecture, training processes, and behavior. +- **Domain knowledge:** Acquire domain-specific knowledge to design prompts that align with the desired outputs and tasks. +- **Experimentation:** Explore different parameters and settings to fine-tune prompts and optimize the model's behavior for specific tasks or domains. +- **Feedback and iteration:** Continuously analyze the outputs generated by the model and iterate on prompts based on user feedback to improve their quality and relevance. +- **Stay updated:** Keep up with the latest advancements in prompt engineering techniques, research, and best practices to enhance your skills and stay ahead in the field. + +Prompt engineering is a dynamic and evolving field, and skilled prompt engineers play a crucial role in harnessing the capabilities of LLM AI models effectively. diff --git a/semantic-kernel/concepts/text-search/TOC.yml b/semantic-kernel/concepts/text-search/TOC.yml new file mode 100644 index 00000000..df21acea --- /dev/null +++ b/semantic-kernel/concepts/text-search/TOC.yml @@ -0,0 +1,12 @@ +- name: Overview + href: index.md +- name: Text Search Abstractions + href: text-search-abstractions.md +- name: Text Search Plugins + href: text-search-plugins.md +- name: Text Search Function Calling + href: text-search-function-calling.md +- name: Text Search with Vector Stores + href: text-search-vector-stores.md +- name: Out-of-the-box Text Search + href: out-of-the-box-textsearch/TOC.yml \ No newline at end of file diff --git a/semantic-kernel/concepts/text-search/index.md b/semantic-kernel/concepts/text-search/index.md new file mode 100644 index 00000000..0a4c9390 --- /dev/null +++ b/semantic-kernel/concepts/text-search/index.md @@ -0,0 +1,196 @@ +--- +title: Text Search with Semantic Kernel (Preview) +description: Describes what a Semantic Kernel Text Search is, an provides a basic example of how to use one and how to get started. +zone_pivot_groups: programming-languages +author: markwallace +ms.topic: conceptual +ms.author: markwallace +ms.date: 07/10/2024 +ms.service: semantic-kernel +--- + +# What is Semantic Kernel Text Search? + +> [!WARNING] +> The Semantic Kernel Text Search functionality is preview, and improvements that require breaking changes may still occur in limited circumstances before release. + +Semantic Kernel provides capabilities that allow developers to integrate search when calling a Large Language Model (LLM). +This is important because LLM's are trained on fixed data sets and may need access to additional data to accurately respond to a user ask. + +The process of providing additional context when prompting a LLM is called Retrieval-Augmented Generation (RAG). +RAG typically involves retrieving additional data that is relevant to the current user ask and augmenting the prompt sent to the LLM with this data. +The LLM can use its training plus the additional context to provide a more accurate response. + +A simple example of when this becomes important is when the user's ask is related to up-to-date information not included in the LLM's training data set. +By performing an appropriate text search and including the results with the user's ask, more accurate responses will be achieved. + +Semantic Kernel provides a set of Text Search capabilities that allow developers to perform searches using Web Search or Vector Databases and easily add RAG to their applications. + +::: zone pivot="programming-language-csharp" + +## Implementing RAG using web text search + +In the following sample code you can choose between using Bing or Google to perform web search operations. + +> [!TIP] +> To run the samples shown on this page go to [GettingStartedWithTextSearch/Step1_Web_Search.cs](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/GettingStartedWithTextSearch/Step1_Web_Search.cs). + +### Create text search instance + +Each sample creates a text search instance and then performs a search operation to get results for the provided query. +The search results will contain a snippet of text from the webpage that describes its contents. +This provides only a limited context i.e., a subset of the web page contents and no link to the source of the information. +Later samples show how to address these limitations. + +> [!TIP] +> The following sample code uses the Semantic Kernel OpenAI connector and Web plugins, install using the following commands: +> +> `dotnet add package Microsoft.SemanticKernel`
      +> `dotnet add package Microsoft.SemanticKernel.Plugins.Web` + +#### Bing web search + +```csharp +using Microsoft.SemanticKernel.Data; +using Microsoft.SemanticKernel.Plugins.Web.Bing; + +// Create an ITextSearch instance using Bing search +var textSearch = new BingTextSearch(apiKey: ""); + +var query = "What is the Semantic Kernel?"; + +// Search and return results +KernelSearchResults searchResults = await textSearch.SearchAsync(query, new() { Top = 4 }); +await foreach (string result in searchResults.Results) +{ + Console.WriteLine(result); +} +``` + +#### Google web search + +```csharp +using Microsoft.SemanticKernel.Data; +using Microsoft.SemanticKernel.Plugins.Web.Google; + +// Create an ITextSearch instance using Google search +var textSearch = new GoogleTextSearch( + searchEngineId: "", + apiKey: ""); + +var query = "What is the Semantic Kernel?"; + +// Search and return results +KernelSearchResults searchResults = await textSearch.SearchAsync(query, new() { Top = 4 }); +await foreach (string result in searchResults.Results) +{ + Console.WriteLine(result); +} +``` + +> [!TIP] +> For more information on what types of search results can be retrieved, refer to [the documentation on Text Search Plugins](./text-search-plugins.md). + +### Use text search results to augment a prompt + +Next steps are to create a Plugin from the web text search and invoke the Plugin to add the search results to the prompt. + +The sample code below shows how to achieve this: + +1. Create a `Kernel` that has an OpenAI service registered. This will be used to call the `gpt-4o` model with the prompt. +2. Create a text search instance. +3. Create a Search Plugin from the text search instance. +4. Create a prompt template that will invoke the Search Plugin with the query and include search results in the prompt along with the original query. +5. Invoke the prompt and display the response. + +The model will provide a response that is grounded in the latest information available from a web search. + +#### Bing web search + +```csharp +using Microsoft.SemanticKernel.Data; +using Microsoft.SemanticKernel.Plugins.Web.Bing; + +// Create a kernel with OpenAI chat completion +IKernelBuilder kernelBuilder = Kernel.CreateBuilder(); +kernelBuilder.AddOpenAIChatCompletion( + modelId: "gpt-4o", + apiKey: ""); +Kernel kernel = kernelBuilder.Build(); + +// Create a text search using Bing search +var textSearch = new BingTextSearch(apiKey: ""); + +// Build a text search plugin with Bing search and add to the kernel +var searchPlugin = textSearch.CreateWithSearch("SearchPlugin"); +kernel.Plugins.Add(searchPlugin); + +// Invoke prompt and use text search plugin to provide grounding information +var query = "What is the Semantic Kernel?"; +var prompt = "{{SearchPlugin.Search $query}}. {{$query}}"; +KernelArguments arguments = new() { { "query", query } }; +Console.WriteLine(await kernel.InvokePromptAsync(prompt, arguments)); +``` + +#### Google web search + +```csharp +using Microsoft.SemanticKernel.Data; +using Microsoft.SemanticKernel.Plugins.Web.Google; + +// Create a kernel with OpenAI chat completion +IKernelBuilder kernelBuilder = Kernel.CreateBuilder(); +kernelBuilder.AddOpenAIChatCompletion( + modelId: "gpt-4o", + apiKey: ""); +Kernel kernel = kernelBuilder.Build(); + +// Create an ITextSearch instance using Google search +var textSearch = new GoogleTextSearch( + searchEngineId: "", + apiKey: ""); + +// Build a text search plugin with Google search and add to the kernel +var searchPlugin = textSearch.CreateWithSearch("SearchPlugin"); +kernel.Plugins.Add(searchPlugin); + +// Invoke prompt and use text search plugin to provide grounding information +var query = "What is the Semantic Kernel?"; +var prompt = "{{SearchPlugin.Search $query}}. {{$query}}"; +KernelArguments arguments = new() { { "query", query } }; +Console.WriteLine(await kernel.InvokePromptAsync(prompt, arguments)); +``` + +There are a number of issues with the above sample: + +1. The response does not include citations showing the web pages that were used to provide grounding context. +2. The response will include data from any web site, it would be better to limit this to trusted sites. +3. Only a snippet of each web page is being used to provide grounding context to the model, the snippet may not contain the data required to provide an accurate response. + +See the page which describes [Text Search Plugins](./text-search-plugins.md) for solutions to these issues. + +Next we recommend looking at [Text Search Abstractions](./text-search-abstractions.md). + +::: zone-end +::: zone pivot="programming-language-python" + +## Coming soon + +More coming soon. + +::: zone-end +::: zone pivot="programming-language-java" + +## Coming soon + +More coming soon. + +::: zone-end + +## Next steps + +> [!div class="nextstepaction"] +> [Text Search Abstractions](./text-search-abstractions.md) +> [Text Search Plugins](./text-search-plugins.md) +> [Text Search Function Calling](./text-search-function-calling.md) +> [Text Search with Vector Stores](./text-search-vector-stores.md) diff --git a/semantic-kernel/concepts/text-search/out-of-the-box-textsearch/TOC.yml b/semantic-kernel/concepts/text-search/out-of-the-box-textsearch/TOC.yml new file mode 100644 index 00000000..bac96627 --- /dev/null +++ b/semantic-kernel/concepts/text-search/out-of-the-box-textsearch/TOC.yml @@ -0,0 +1,9 @@ +- name: Overview + href: index.md +- name: Bing text search + href: bing-textsearch.md +- name: Google text search + href: google-textsearch.md +- name: Vector store text search + href: vectorstore-textsearch.md + diff --git a/semantic-kernel/concepts/text-search/out-of-the-box-textsearch/bing-textsearch.md b/semantic-kernel/concepts/text-search/out-of-the-box-textsearch/bing-textsearch.md new file mode 100644 index 00000000..7c7bc43b --- /dev/null +++ b/semantic-kernel/concepts/text-search/out-of-the-box-textsearch/bing-textsearch.md @@ -0,0 +1,105 @@ +--- +title: Using the Semantic Kernel Bing text search (Preview) +description: Contains information on how to use a Semantic Kernel Text Search with Bing. +zone_pivot_groups: programming-languages +author: markwallace +ms.topic: conceptual +ms.author: markwallace +ms.date: 10/21/2024 +ms.service: semantic-kernel +--- +# Using the Bing Text Search (Preview) + +> [!WARNING] +> The Semantic Kernel Text Search functionality is in preview, and improvements that require breaking changes may still occur in limited circumstances before release. + +## Overview + +The Bing Text Search implementation uses the [Bing Web Search API](https://www.microsoft.com/bing/apis/bing-web-search-api) to retrieve search results. You must provide your own Bing Search Api Key to use this component. + +## Limitations + +| Feature Area | Support | +|-----------------------------------|---------------------------------------------------------------------------------------| +| Search API | [Bing Web Search API](https://www.microsoft.com/bing/apis/bing-web-search-api) only. | +| Supported filter clauses | Only "equal to" filter clauses are supported. | +| Supported filter keys | The [responseFilter](/bing/search-apis/bing-web-search/reference/query-parameters#responsefilter) query parameter and [advanced search keywords](https://support.microsoft.com/topic/advanced-search-keywords-ea595928-5d63-4a0b-9c6b-0b769865e78a) are supported. | + +> [!TIP] +> Follow this link for more information on how to [filter the answers that Bing returns](/bing/search-apis/bing-web-search/filter-answers#getting-results-from-a-specific-site). +> Follow this link for more information on using [advanced search keywords](https://support.microsoft.com/topic/advanced-search-keywords-ea595928-5d63-4a0b-9c6b-0b769865e78a) + +## Getting started + +The sample below shows how to create a `BingTextSearch` and use it to perform a text search. + +::: zone pivot="programming-language-csharp" + +```csharp +using Microsoft.SemanticKernel.Data; +using Microsoft.SemanticKernel.Plugins.Web.Bing; + +// Create an ITextSearch instance using Bing search +var textSearch = new BingTextSearch(apiKey: ""); + +var query = "What is the Semantic Kernel?"; + +// Search and return results as a string items +KernelSearchResults stringResults = await textSearch.SearchAsync(query, new() { Top = 4, Skip = 0 }); +Console.WriteLine("--- String Results ---\n"); +await foreach (string result in stringResults.Results) +{ + Console.WriteLine(result); +} + +// Search and return results as TextSearchResult items +KernelSearchResults textResults = await textSearch.GetTextSearchResultsAsync(query, new() { Top = 4, Skip = 4 }); +Console.WriteLine("\n--- Text Search Results ---\n"); +await foreach (TextSearchResult result in textResults.Results) +{ + Console.WriteLine($"Name: {result.Name}"); + Console.WriteLine($"Value: {result.Value}"); + Console.WriteLine($"Link: {result.Link}"); +} + +// Search and return s results as BingWebPage items +KernelSearchResults fullResults = await textSearch.GetSearchResultsAsync(query, new() { Top = 4, Skip = 8 }); +Console.WriteLine("\n--- Bing Web Page Results ---\n"); +await foreach (BingWebPage result in fullResults.Results) +{ + Console.WriteLine($"Name: {result.Name}"); + Console.WriteLine($"Snippet: {result.Snippet}"); + Console.WriteLine($"Url: {result.Url}"); + Console.WriteLine($"DisplayUrl: {result.DisplayUrl}"); + Console.WriteLine($"DateLastCrawled: {result.DateLastCrawled}"); +} +``` + +::: zone-end +::: zone pivot="programming-language-python" + +## Coming soon + +More coming soon. + +::: zone-end +::: zone pivot="programming-language-java" + +## Coming soon + +More coming soon. + +::: zone-end + +## Next steps +The following sections of the documentation show you how to: + +1. Create a [plugin](../text-search-plugins.md) and use it for Retrieval Augmented Generation (RAG). +2. Use text search together with [function calling](../text-search-function-calling.md). +3. Learn more about using [vector stores](../text-search-vector-stores.md) for text search. + +> [!div class="nextstepaction"] +> [Text Search Abstractions](../text-search-abstractions.md) +> [Text Search Plugins](../text-search-plugins.md) +> [Text Search Function Calling](../text-search-function-calling.md) +> [Text Search with Vector Stores](../text-search-vector-stores.md) diff --git a/semantic-kernel/concepts/text-search/out-of-the-box-textsearch/google-textsearch.md b/semantic-kernel/concepts/text-search/out-of-the-box-textsearch/google-textsearch.md new file mode 100644 index 00000000..5f8fb42a --- /dev/null +++ b/semantic-kernel/concepts/text-search/out-of-the-box-textsearch/google-textsearch.md @@ -0,0 +1,107 @@ +--- +title: Using the Semantic Kernel Google text search (Preview) +description: Contains information on how to use a Semantic Kernel Text Search with Google. +zone_pivot_groups: programming-languages +author: markwallace +ms.topic: conceptual +ms.author: markwallace +ms.date: 10/21/2024 +ms.service: semantic-kernel +--- +# Using the Google Text Search (Preview) + +> [!WARNING] +> The Semantic Kernel Text Search functionality is in preview, and improvements that require breaking changes may still occur in limited circumstances before release. + +## Overview + +The Google Text Search implementation uses [Google Custom Search](https://developers.google.com/custom-search) to retrieve search results. You must provide your own Google Search Api Key and Search Engine Id to use this component. + +## Limitations + +| Feature Area | Support | +|-----------------------------------|---------------------------------------------------------------------------------------| +| Search API | [Google Custom Search API](https://developers.google.com/custom-search/v1/reference/rest/v1/cse) only. | +| Supported filter clauses | Only "equal to" filter clauses are supported. | +| Supported filter keys | Following parameters are supported: "cr", "dateRestrict", "exactTerms", "excludeTerms", "filter", "gl", "hl", "linkSite", "lr", "orTerms", "rights", "siteSearch". For more information see [parameters](https://developers.google.com/custom-search/v1/reference/rest/v1/cse/list). | + +> [!TIP] +> Follow this link for more information on how [search is performed](https://developers.google.com/custom-search/v1/reference/rest/v1/cse/list) + +## Getting started + +The sample below shows how to create a `GoogleTextSearch` and use it to perform a text search. + +::: zone pivot="programming-language-csharp" + +```csharp +using Google.Apis.Http; +using Microsoft.SemanticKernel.Data; +using Microsoft.SemanticKernel.Plugins.Web.Google; + +// Create an ITextSearch instance using Google search +var textSearch = new GoogleTextSearch( + initializer: new() { ApiKey = "", HttpClientFactory = new CustomHttpClientFactory(this.Output) }, + searchEngineId: ""); + +var query = "What is the Semantic Kernel?"; + +// Search and return results as string items +KernelSearchResults stringResults = await textSearch.SearchAsync(query, new() { Top = 4, Skip = 0 }); +Console.WriteLine("——— String Results ———\n"); +await foreach (string result in stringResults.Results) +{ + Console.WriteLine(result); +} + +// Search and return results as TextSearchResult items +KernelSearchResults textResults = await textSearch.GetTextSearchResultsAsync(query, new() { Top = 4, Skip = 4 }); +Console.WriteLine("\n——— Text Search Results ———\n"); +await foreach (TextSearchResult result in textResults.Results) +{ + Console.WriteLine($"Name: {result.Name}"); + Console.WriteLine($"Value: {result.Value}"); + Console.WriteLine($"Link: {result.Link}"); +} + +// Search and return results as Google.Apis.CustomSearchAPI.v1.Data.Result items +KernelSearchResults fullResults = await textSearch.GetSearchResultsAsync(query, new() { Top = 4, Skip = 8 }); +Console.WriteLine("\n——— Google Web Page Results ———\n"); +await foreach (Google.Apis.CustomSearchAPI.v1.Data.Result result in fullResults.Results) +{ + Console.WriteLine($"Title: {result.Title}"); + Console.WriteLine($"Snippet: {result.Snippet}"); + Console.WriteLine($"Link: {result.Link}"); + Console.WriteLine($"DisplayLink: {result.DisplayLink}"); + Console.WriteLine($"Kind: {result.Kind}"); +} +``` + +::: zone-end +::: zone pivot="programming-language-python" + +## Coming soon + +More coming soon. + +::: zone-end +::: zone pivot="programming-language-java" + +## Coming soon + +More coming soon. + +::: zone-end + +## Next steps +The following sections of the documentation show you how to: + +1. Create a [plugin](../text-search-plugins.md) and use it for Retrieval Augmented Generation (RAG). +2. Use text search together with [function calling](../text-search-function-calling.md). +3. Learn more about using [vector stores](../text-search-vector-stores.md) for text search. + +> [!div class="nextstepaction"] +> [Text Search Abstractions](../text-search-abstractions.md) +> [Text Search Plugins](../text-search-plugins.md) +> [Text Search Function Calling](../text-search-function-calling.md) +> [Text Search with Vector Stores](../text-search-vector-stores.md) diff --git a/semantic-kernel/concepts/text-search/out-of-the-box-textsearch/index.md b/semantic-kernel/concepts/text-search/out-of-the-box-textsearch/index.md new file mode 100644 index 00000000..47563652 --- /dev/null +++ b/semantic-kernel/concepts/text-search/out-of-the-box-textsearch/index.md @@ -0,0 +1,21 @@ +--- +title: Out-of-the-box Text Search (Preview) +description: Out-of-the-box Text Search +author: markwallace +ms.topic: conceptual +ms.author: markwallace +ms.date: 10/21/2024 +ms.service: semantic-kernel +--- +# Out-of-the-box Text Search (Preview) + +> [!WARNING] +> The Semantic Kernel Text Search functionality is in preview, and improvements that require breaking changes may still occur in limited circumstances before release. + +Semantic Kernel provides a number of out-of-the-box Text Search integrations making it easy to get started with using Text Search. + +| Text Search | C# | Python | Java | +|------------------------------------------------|-----------|:---------------:|:---------------:| +| [Bing](./bing-textsearch.md) | ✅ | In Development | In Development | +| [Google](./google-textsearch.md) | ✅ | In Development | In Development | +| [Vector Store](./vectorstore-textsearch.md) | ✅ | In Development | In Development | diff --git a/semantic-kernel/concepts/text-search/out-of-the-box-textsearch/vectorstore-textsearch.md b/semantic-kernel/concepts/text-search/out-of-the-box-textsearch/vectorstore-textsearch.md new file mode 100644 index 00000000..efaa4a2a --- /dev/null +++ b/semantic-kernel/concepts/text-search/out-of-the-box-textsearch/vectorstore-textsearch.md @@ -0,0 +1,96 @@ +--- +title: Using the Semantic Kernel Vector Store text search (Preview) +description: Contains information on how to use a Semantic Kernel Text Search with Vector Store. +zone_pivot_groups: programming-languages +author: markwallace +ms.topic: conceptual +ms.author: markwallace +ms.date: 10/21/2024 +ms.service: semantic-kernel +--- +# Using the Vector Store Text Search (Preview) + +> [!WARNING] +> The Semantic Kernel Text Search functionality is in preview, and improvements that require breaking changes may still occur in limited circumstances before release. + +::: zone pivot="programming-language-csharp" + +## Overview + +The Vector Store Text Search implementation uses the [Vector Store Connectors](../../vector-store-connectors/out-of-the-box-connectors/index.md) to retrieve search results. This means you can use Vector Store Text Search with any Vector Store which Semantic Kernel supports and any implementation of [Microsoft.Extensions.VectorData.Abstractions](https://www.nuget.org/packages/Microsoft.Extensions.VectorData.Abstractions). + +## Limitations + +See the limitations listed for the [Vector Store connector](../../vector-store-connectors/out-of-the-box-connectors/index.md) you are using. + +## Getting started + +The sample below shows how to use an in-memory vector store to create a `VectorStoreTextSearch` and use it to perform a text search. + +```csharp +using Microsoft.Extensions.VectorData; +using Microsoft.SemanticKernel.Connectors.InMemory; +using Microsoft.SemanticKernel.Connectors.OpenAI; +using Microsoft.SemanticKernel.Data; +using Microsoft.SemanticKernel.Embeddings; + +// Create an embedding generation service. +var textEmbeddingGeneration = new OpenAITextEmbeddingGenerationService( + modelId: TestConfiguration.OpenAI.EmbeddingModelId, + apiKey: TestConfiguration.OpenAI.ApiKey); + +// Construct an InMemory vector store. +var vectorStore = new InMemoryVectorStore(); +var collectionName = "records"; + +// Get and create collection if it doesn't exist. +var recordCollection = vectorStore.GetCollection(collectionName); +await recordCollection.CreateCollectionIfNotExistsAsync().ConfigureAwait(false); + +// TODO populate the record collection with your test data +// Example https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/Search/VectorStore_TextSearch.cs + +// Create a text search instance using the InMemory vector store. +var textSearch = new VectorStoreTextSearch(recordCollection, textEmbeddingGeneration); + +// Search and return results as TextSearchResult items +var query = "What is the Semantic Kernel?"; +KernelSearchResults textResults = await textSearch.GetTextSearchResultsAsync(query, new() { Top = 2, Skip = 0 }); +Console.WriteLine("\n--- Text Search Results ---\n"); +await foreach (TextSearchResult result in textResults.Results) +{ + Console.WriteLine($"Name: {result.Name}"); + Console.WriteLine($"Value: {result.Value}"); + Console.WriteLine($"Link: {result.Link}"); +} +``` + +::: zone-end +::: zone pivot="programming-language-python" + +## Coming soon + +More coming soon. + +::: zone-end +::: zone pivot="programming-language-java" + +## Coming soon + +More coming soon. + +::: zone-end + +## Next steps + +The following sections of the documentation show you how to: + +1. Create a [plugin](../text-search-plugins.md) and use it for Retrieval Augmented Generation (RAG). +2. Use text search together with [function calling](../text-search-function-calling.md). +3. Learn more about using [vector stores](../text-search-vector-stores.md) for text search. + +> [!div class="nextstepaction"] +> [Text Search Abstractions](../text-search-abstractions.md) +> [Text Search Plugins](../text-search-plugins.md) +> [Text Search Function Calling](../text-search-function-calling.md) +> [Text Search with Vector Stores](../text-search-vector-stores.md) diff --git a/semantic-kernel/concepts/text-search/text-search-abstractions.md b/semantic-kernel/concepts/text-search/text-search-abstractions.md new file mode 100644 index 00000000..a95db7f6 --- /dev/null +++ b/semantic-kernel/concepts/text-search/text-search-abstractions.md @@ -0,0 +1,102 @@ +--- +title: Semantic Kernel Text Search Abstractions (Preview) +description: Provides a detailed look at the Semantic Kernel Text Search abstractions. +zone_pivot_groups: programming-languages +author: markwallace +ms.topic: conceptual +ms.author: markwallace +ms.date: 07/10/2024 +ms.service: semantic-kernel +--- + +# Why are Text Search abstractions needed? + +When dealing with text prompts or text content in chat history a common requirement is to provide additional relevant information related to this text. +This provides the AI model with relevant context which helps it to provide more accurate responses. +To meet this requirement the Semantic Kernel provides a Text Search abstraction which allows using text inputs from various sources, e.g. Web search engines, vector stores, etc., and provide results in a few standardized formats. + +> [!NOTE] +> Search for image content or audio content is not currently supported. + +::: zone pivot="programming-language-csharp" +## Text search abstraction + +The Semantic Kernel text search abstractions provides three methods: + +1. `Search` +2. `GetSearchResults` +3. `GetTextSearchResults` + +### `Search` +Performs a search for content related to the specified query and returns string values representing the search results. `Search` can be used in the most basic use cases e.g., when augmenting a `semantic-kernel` format prompt template with search results. `Search` always returns just a single string value per search result so is not suitable if citations are required. + +### `GetSearchResults` +Performs a search for content related to the specified query and returns search results in the format defined by the implementation. `GetSearchResults` returns the full search result as defined by the underlying search service. This provides the most versatility at the cost of tying your code to a specific search service implementation. + +### `GetTextSearchResults` +Performs a search for content related to the specified query and returns a normalized data model representing the search results. This normalized data model includes a string value and optionally a name and link. `GetTextSearchResults` allows your code to be isolated from the a specific search service implementation, so the same prompt can be used with multiple different search services. + +> [!TIP] +> To run the samples shown on this page go to [GettingStartedWithTextSearch/Step1_Web_Search.cs](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/GettingStartedWithTextSearch/Step1_Web_Search.cs). + +The sample code below shows each of the text search methods in action. + +```csharp +using Microsoft.SemanticKernel.Data; +using Microsoft.SemanticKernel.Plugins.Web.Bing; + +// Create an ITextSearch instance using Bing search +var textSearch = new BingTextSearch(apiKey: ""); + +var query = "What is the Semantic Kernel?"; + +// Search and return results +KernelSearchResults searchResults = await textSearch.SearchAsync(query, new() { Top = 4 }); +await foreach (string result in searchResults.Results) +{ + Console.WriteLine(result); +} + +// Search and return results as BingWebPage items +KernelSearchResults webPages = await textSearch.GetSearchResultsAsync(query, new() { Top = 4 }); +await foreach (BingWebPage webPage in webPages.Results) +{ + Console.WriteLine($"Name: {webPage.Name}"); + Console.WriteLine($"Snippet: {webPage.Snippet}"); + Console.WriteLine($"Url: {webPage.Url}"); + Console.WriteLine($"DisplayUrl: {webPage.DisplayUrl}"); + Console.WriteLine($"DateLastCrawled: {webPage.DateLastCrawled}"); +} + +// Search and return results as TextSearchResult items +KernelSearchResults textResults = await textSearch.GetTextSearchResultsAsync(query, new() { Top = 4 }); +await foreach (TextSearchResult result in textResults.Results) +{ + Console.WriteLine($"Name: {result.Name}"); + Console.WriteLine($"Value: {result.Value}"); + Console.WriteLine($"Link: {result.Link}"); +} +``` + +::: zone-end +::: zone pivot="programming-language-python" + +## Coming soon + +More coming soon. + +::: zone-end +::: zone pivot="programming-language-java" + +## Coming soon + +More coming soon. + +::: zone-end + +## Next steps + +> [!div class="nextstepaction"] +> [Text Search Plugins](./text-search-plugins.md) +> [Text Search Function Calling](./text-search-function-calling.md) +> [Text Search with Vector Stores](./text-search-vector-stores.md) diff --git a/semantic-kernel/concepts/text-search/text-search-function-calling.md b/semantic-kernel/concepts/text-search/text-search-function-calling.md new file mode 100644 index 00000000..f5468da0 --- /dev/null +++ b/semantic-kernel/concepts/text-search/text-search-function-calling.md @@ -0,0 +1,169 @@ +--- +title: Semantic Kernel Text Search Function Calling (Preview) +description: Describes how to use Semantic Kernel search plugins with function calling. +zone_pivot_groups: programming-languages +author: markwallace +ms.topic: conceptual +ms.author: markwallace +ms.date: 07/10/2024 +ms.service: semantic-kernel +--- + +# Why use function calling with Semantic Kernel Text Search? + +In the previous Retrieval-Augmented Generation (RAG) based samples the user ask has been used as the search query when retrieving relevant information. +The user ask could be long and may span multiple topics or there may be multiple different search implementations available which provide specialized results. +For either of these scenarios it can be useful to allow the AI model to extract the search query or queries from the user ask and use function calling to retrieve the relevant information it needs. + +::: zone pivot="programming-language-csharp" +> [!TIP] +> To run the samples shown on this page go to [GettingStartedWithTextSearch/Step3_Search_With_FunctionCalling.cs](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/GettingStartedWithTextSearch/Step3_Search_With_FunctionCalling.cs). + +## Function calling with Bing text search + +> [!TIP] +> The samples in this section use an `IFunctionInvocationFilter` filter to log the function that the model calls and what parameters it sends. +> It is interesting to see what the model uses as a search query when calling the `SearchPlugin`. + +Here is the `IFunctionInvocationFilter` filter implementation. + +```csharp +private sealed class FunctionInvocationFilter(TextWriter output) : IFunctionInvocationFilter +{ + public async Task OnFunctionInvocationAsync(FunctionInvocationContext context, Func next) + { + if (context.Function.PluginName == "SearchPlugin") + { + output.WriteLine($"{context.Function.Name}:{JsonSerializer.Serialize(context.Arguments)}\n"); + } + await next(context); + } +} +``` + +The sample below creates a `SearchPlugin` using Bing web search. +This plugin will be advertised to the AI model for use with automatic function calling, using the `FunctionChoiceBehavior` in the prompt execution settings. +When you run this sample check the console output to see what the model used as the search query. + +```csharp +using Microsoft.SemanticKernel; +using Microsoft.SemanticKernel.Connectors.OpenAI; +using Microsoft.SemanticKernel.Data; +using Microsoft.SemanticKernel.Plugins.Web.Bing; + +// Create a kernel with OpenAI chat completion +IKernelBuilder kernelBuilder = Kernel.CreateBuilder(); +kernelBuilder.AddOpenAIChatCompletion( + modelId: "gpt-4o", + apiKey: ""); +kernelBuilder.Services.AddSingleton(output); +kernelBuilder.Services.AddSingleton(); +Kernel kernel = kernelBuilder.Build(); + +// Create a search service with Bing search +var textSearch = new BingTextSearch(apiKey: ""); + +// Build a text search plugin with Bing search and add to the kernel +var searchPlugin = textSearch.CreateWithSearch("SearchPlugin"); +kernel.Plugins.Add(searchPlugin); + +// Invoke prompt and use text search plugin to provide grounding information +OpenAIPromptExecutionSettings settings = new() { FunctionChoiceBehavior = FunctionChoiceBehavior.Auto() }; +KernelArguments arguments = new(settings); +Console.WriteLine(await kernel.InvokePromptAsync("What is the Semantic Kernel?", arguments)); +``` + +## Function calling with Bing text search and citations + +The sample below includes the required changes to include citations: + +1. Use `CreateWithGetTextSearchResults` to create the `SearchPlugin`, this will include the link to the original source of the information. +2. Modify the prompt to instruct the model to include citations in it's response. + +```csharp +using Microsoft.SemanticKernel; +using Microsoft.SemanticKernel.Connectors.OpenAI; +using Microsoft.SemanticKernel.Data; +using Microsoft.SemanticKernel.Plugins.Web.Bing; + +// Create a kernel with OpenAI chat completion +IKernelBuilder kernelBuilder = Kernel.CreateBuilder(); +kernelBuilder.AddOpenAIChatCompletion( + modelId: "gpt-4o", + apiKey: ""); +kernelBuilder.Services.AddSingleton(output); +kernelBuilder.Services.AddSingleton(); +Kernel kernel = kernelBuilder.Build(); + +// Create a search service with Bing search +var textSearch = new BingTextSearch(apiKey: ""); + +// Build a text search plugin with Bing search and add to the kernel +var searchPlugin = textSearch.CreateWithGetTextSearchResults("SearchPlugin"); +kernel.Plugins.Add(searchPlugin); + +// Invoke prompt and use text search plugin to provide grounding information +OpenAIPromptExecutionSettings settings = new() { FunctionChoiceBehavior = FunctionChoiceBehavior.Auto() }; +KernelArguments arguments = new(settings); +Console.WriteLine(await kernel.InvokePromptAsync("What is the Semantic Kernel? Include citations to the relevant information where it is referenced in the response.", arguments)); +``` + +## Function calling with Bing text search and filtering + +The final sample in this section shows how to use a filter with function calling. +For this sample only search results from the Microsoft Developer Blogs site will be included. +An instance of `TextSearchFilter` is created and an equality clause is added to match the `devblogs.microsoft.com` site. +Ths filter will be used when the function is invoked in response to a function calling request from the model. + +```csharp +using Microsoft.SemanticKernel; +using Microsoft.SemanticKernel.Connectors.OpenAI; +using Microsoft.SemanticKernel.Data; +using Microsoft.SemanticKernel.Plugins.Web.Bing; + +// Create a kernel with OpenAI chat completion +IKernelBuilder kernelBuilder = Kernel.CreateBuilder(); +kernelBuilder.AddOpenAIChatCompletion( + modelId: "gpt-4o", + apiKey: ""); +kernelBuilder.Services.AddSingleton(output); +kernelBuilder.Services.AddSingleton(); +Kernel kernel = kernelBuilder.Build(); + +// Create a search service with Bing search +var textSearch = new BingTextSearch(apiKey: ""); + +// Build a text search plugin with Bing search and add to the kernel +var filter = new TextSearchFilter().Equality("site", "devblogs.microsoft.com"); +var searchOptions = new TextSearchOptions() { Filter = filter }; +var searchPlugin = KernelPluginFactory.CreateFromFunctions( + "SearchPlugin", "Search Microsoft Developer Blogs site only", + [textSearch.CreateGetTextSearchResults(searchOptions: searchOptions)]); +kernel.Plugins.Add(searchPlugin); + +// Invoke prompt and use text search plugin to provide grounding information +OpenAIPromptExecutionSettings settings = new() { FunctionChoiceBehavior = FunctionChoiceBehavior.Auto() }; +KernelArguments arguments = new(settings); +Console.WriteLine(await kernel.InvokePromptAsync("What is the Semantic Kernel? Include citations to the relevant information where it is referenced in the response.", arguments)); +``` + +::: zone-end +::: zone pivot="programming-language-python" + +## Coming soon + +More coming soon. + +::: zone-end +::: zone pivot="programming-language-java" + +## Coming soon + +More coming soon. + +::: zone-end + +## Next steps + +> [!div class="nextstepaction"] +> [Text Search with Vector Stores](./text-search-vector-stores.md) diff --git a/semantic-kernel/concepts/text-search/text-search-plugins.md b/semantic-kernel/concepts/text-search/text-search-plugins.md new file mode 100644 index 00000000..a193ff36 --- /dev/null +++ b/semantic-kernel/concepts/text-search/text-search-plugins.md @@ -0,0 +1,280 @@ +--- +title: Semantic Kernel Text Search Plugins (Preview) +description: Describes how to create and use Semantic Kernel text search plugins. Includes samples showing how to use filters and how to create a custom search plugin. +zone_pivot_groups: programming-languages +author: markwallace +ms.topic: conceptual +ms.author: markwallace +ms.date: 07/10/2024 +ms.service: semantic-kernel +--- + +# What are Semantic Kernel Text Search plugins? + +Semantic Kernel uses [Plugins](../plugins/index.md) to connect existing APIs with AI. +These Plugins have functions that can be used to add relevant data or examples to prompts, or to allow the AI to perform actions automatically. + +To integrate Text Search with Semantic Kernel, we need to turn it into a Plugin. +Once we have a Text Search plugin, we can use it to add relevant information to prompts or to retrieve information as needed. +Creating a plugin from Text Search is a simple process, which we will explain below. + +::: zone pivot="programming-language-csharp" + +> [!TIP] +> To run the samples shown on this page go to [GettingStartedWithTextSearch/Step2_Search_For_RAG.cs](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/GettingStartedWithTextSearch/Step2_Search_For_RAG.cs). + +## Basic search plugin + +Semantic Kernel provides a default template implementation that supports variable substitution and function calling. +By including an expression such as `{{MyPlugin.Function $arg1}}` in a prompt template, the specified function i.e., `MyPlugin.Function` will be invoked with the provided argument `arg1` (which is resolved from `KernelArguments`). +The return value from the function invocation is inserted into the prompt. This technique can be used to inject relevant information into a prompt. + +The sample below shows how to create a plugin named `SearchPlugin` from an instance of `BingTextSearch`. +Using `CreateWithSearch` creates a new plugin with a single `Search` function that calls the underlying text search implementation. +The `SearchPlugin` is added to the `Kernel` which makes it available to be called during prompt rendering. +The prompt template includes a call to `{{SearchPlugin.Search $query}}` which will invoke the `SearchPlugin` to retrieve results related to the current query. +The results are then inserted into the rendered prompt before it is sent to the AI model. + +```csharp +using Microsoft.SemanticKernel.Data; +using Microsoft.SemanticKernel.Plugins.Web.Bing; + +// Create a kernel with OpenAI chat completion +IKernelBuilder kernelBuilder = Kernel.CreateBuilder(); +kernelBuilder.AddOpenAIChatCompletion( + modelId: "gpt-4o", + apiKey: ""); +Kernel kernel = kernelBuilder.Build(); + +// Create a text search using Bing search +var textSearch = new BingTextSearch(apiKey: ""); + +// Build a text search plugin with Bing search and add to the kernel +var searchPlugin = textSearch.CreateWithSearch("SearchPlugin"); +kernel.Plugins.Add(searchPlugin); + +// Invoke prompt and use text search plugin to provide grounding information +var query = "What is the Semantic Kernel?"; +var prompt = "{{SearchPlugin.Search $query}}. {{$query}}"; +KernelArguments arguments = new() { { "query", query } }; +Console.WriteLine(await kernel.InvokePromptAsync(prompt, arguments)); +``` + +## Search plugin with citations + +The sample below repeats the pattern described in the previous section with a few notable changes: + +1. `CreateWithGetTextSearchResults` is used to create a `SearchPlugin` which calls the `GetTextSearchResults` method from the underlying text search implementation. +2. The prompt template uses Handlebars syntax. This allows the template to iterate over the search results and render the name, value and link for each result. +3. The prompt includes an instruction to include citations, so the AI model will do the work of adding citations to the response. + +```csharp +using Microsoft.SemanticKernel.Data; +using Microsoft.SemanticKernel.PromptTemplates.Handlebars; +using Microsoft.SemanticKernel.Plugins.Web.Bing; + +// Create a kernel with OpenAI chat completion +IKernelBuilder kernelBuilder = Kernel.CreateBuilder(); +kernelBuilder.AddOpenAIChatCompletion( + modelId: "gpt-4o", + apiKey: ""); +Kernel kernel = kernelBuilder.Build(); + +// Create a text search using Bing search +var textSearch = new BingTextSearch(apiKey: ""); + +// Build a text search plugin with Bing search and add to the kernel +var searchPlugin = textSearch.CreateWithGetTextSearchResults("SearchPlugin"); +kernel.Plugins.Add(searchPlugin); + +// Invoke prompt and use text search plugin to provide grounding information +var query = "What is the Semantic Kernel?"; +string promptTemplate = """ +{{#with (SearchPlugin-GetTextSearchResults query)}} + {{#each this}} + Name: {{Name}} + Value: {{Value}} + Link: {{Link}} + ----------------- + {{/each}} +{{/with}} + +{{query}} + +Include citations to the relevant information where it is referenced in the response. +"""; +KernelArguments arguments = new() { { "query", query } }; +HandlebarsPromptTemplateFactory promptTemplateFactory = new(); +Console.WriteLine(await kernel.InvokePromptAsync( + promptTemplate, + arguments, + templateFormat: HandlebarsPromptTemplateFactory.HandlebarsTemplateFormat, + promptTemplateFactory: promptTemplateFactory +)); +``` + +## Search plugin with a filter + +The samples shown so far will use the top ranked web search results to provide the grounding data. To provide more reliability in the data the web search can be restricted to only return results from a specified site. + +The sample below builds on the previous one to add filtering of the search results. +A `TextSearchFilter` with an equality clause is used to specify that only results from the Microsoft Developer Blogs site (`site == 'devblogs.microsoft.com'`) are to be included in the search results. + +The sample uses `KernelPluginFactory.CreateFromFunctions` to create the `SearchPlugin`. +A custom description is provided for the plugin. +The `ITextSearch.CreateGetTextSearchResults` extension method is used to create the `KernelFunction` which invokes the text search service. + +> [!TIP] +> The `site` filter is Bing specific. For Google web search use `siteSearch`. + +```csharp +using Microsoft.SemanticKernel.Data; +using Microsoft.SemanticKernel.Plugins.Web.Bing; + +// Create a kernel with OpenAI chat completion +IKernelBuilder kernelBuilder = Kernel.CreateBuilder(); +kernelBuilder.AddOpenAIChatCompletion( + modelId: "gpt-4o", + apiKey: ""); +Kernel kernel = kernelBuilder.Build(); + +// Create a text search using Bing search +var textSearch = new BingTextSearch(apiKey: ""); + +// Create a filter to search only the Microsoft Developer Blogs site +var filter = new TextSearchFilter().Equality("site", "devblogs.microsoft.com"); +var searchOptions = new TextSearchOptions() { Filter = filter }; + +// Build a text search plugin with Bing search and add to the kernel +var searchPlugin = KernelPluginFactory.CreateFromFunctions( + "SearchPlugin", "Search Microsoft Developer Blogs site only", + [textSearch.CreateGetTextSearchResults(searchOptions: searchOptions)]); +kernel.Plugins.Add(searchPlugin); + +// Invoke prompt and use text search plugin to provide grounding information +var query = "What is the Semantic Kernel?"; +string promptTemplate = """ +{{#with (SearchPlugin-GetTextSearchResults query)}} + {{#each this}} + Name: {{Name}} + Value: {{Value}} + Link: {{Link}} + ----------------- + {{/each}} +{{/with}} + +{{query}} + +Include citations to the relevant information where it is referenced in the response. +"""; +KernelArguments arguments = new() { { "query", query } }; +HandlebarsPromptTemplateFactory promptTemplateFactory = new(); +Console.WriteLine(await kernel.InvokePromptAsync( + promptTemplate, + arguments, + templateFormat: HandlebarsPromptTemplateFactory.HandlebarsTemplateFormat, + promptTemplateFactory: promptTemplateFactory +)); +``` + +> [!TIP] +> Follow the link for more information on how to [filter the answers that Bing returns](/bing/search-apis/bing-web-search/filter-answers#getting-results-from-a-specific-site). + +## Custom search plugin + +In the previous sample a static site filter was applied to the search operations. +What if you need this filter to be dynamic? + +The next sample shows how you can perform more customization of the `SearchPlugin` so that the filter value can be dynamic. +The sample uses `KernelFunctionFromMethodOptions` to specify the following for the `SearchPlugin`: + +- `FunctionName`: The search function is named `GetSiteResults` because it will apply a site filter if the query includes a domain. +- `Description`: The description describes how this specialized search function works. +- `Parameters`: The parameters include an additional optional parameter for the `site` so the domain can be specified. + +Customizing the search function is required if you want to provide multiple specialized search functions. +In prompts you can use the function names to make the template more readable. +If you use function calling then the model will use the function name and description to select the best search function to invoke. + +When this sample is executed, the response will use [techcommunity.microsoft.com](https://techcommunity.microsoft.com/) as the source for relevant data. + +```csharp +using Microsoft.SemanticKernel.Data; +using Microsoft.SemanticKernel.PromptTemplates.Handlebars; +using Microsoft.SemanticKernel.Plugins.Web.Bing; + +// Create a kernel with OpenAI chat completion +IKernelBuilder kernelBuilder = Kernel.CreateBuilder(); +kernelBuilder.AddOpenAIChatCompletion( + modelId: "gpt-4o", + apiKey: ""); +Kernel kernel = kernelBuilder.Build(); + +// Create a text search using Bing search +var textSearch = new BingTextSearch(apiKey: ""); + +// Build a text search plugin with Bing search and add to the kernel +var options = new KernelFunctionFromMethodOptions() +{ + FunctionName = "GetSiteResults", + Description = "Perform a search for content related to the specified query and optionally from the specified domain.", + Parameters = + [ + new KernelParameterMetadata("query") { Description = "What to search for", IsRequired = true }, + new KernelParameterMetadata("top") { Description = "Number of results", IsRequired = false, DefaultValue = 5 }, + new KernelParameterMetadata("skip") { Description = "Number of results to skip", IsRequired = false, DefaultValue = 0 }, + new KernelParameterMetadata("site") { Description = "Only return results from this domain", IsRequired = false }, + ], + ReturnParameter = new() { ParameterType = typeof(KernelSearchResults) }, +}; +var searchPlugin = KernelPluginFactory.CreateFromFunctions("SearchPlugin", "Search specified site", [textSearch.CreateGetTextSearchResults(options)]); +kernel.Plugins.Add(searchPlugin); + +// Invoke prompt and use text search plugin to provide grounding information +var query = "What is the Semantic Kernel?"; +string promptTemplate = """ + {{#with (SearchPlugin-GetSiteResults query)}} + {{#each this}} + Name: {{Name}} + Value: {{Value}} + Link: {{Link}} + ----------------- + {{/each}} + {{/with}} + + {{query}} + + Only include results from techcommunity.microsoft.com. + Include citations to the relevant information where it is referenced in the response. + """; +KernelArguments arguments = new() { { "query", query } }; +HandlebarsPromptTemplateFactory promptTemplateFactory = new(); +Console.WriteLine(await kernel.InvokePromptAsync( + promptTemplate, + arguments, + templateFormat: HandlebarsPromptTemplateFactory.HandlebarsTemplateFormat, + promptTemplateFactory: promptTemplateFactory +)); +``` + +::: zone-end +::: zone pivot="programming-language-python" + +## Coming soon + +More coming soon. + +::: zone-end +::: zone pivot="programming-language-java" + +## Coming soon + +More coming soon. + +::: zone-end + +## Next steps + +> [!div class="nextstepaction"] +> [Text Search Function Calling](./text-search-function-calling.md) +> [Text Search with Vector Stores](./text-search-vector-stores.md) diff --git a/semantic-kernel/concepts/text-search/text-search-vector-stores.md b/semantic-kernel/concepts/text-search/text-search-vector-stores.md new file mode 100644 index 00000000..8d0c24cf --- /dev/null +++ b/semantic-kernel/concepts/text-search/text-search-vector-stores.md @@ -0,0 +1,356 @@ +--- +title: Semantic Kernel Text Search with Vector Stores (Preview) +description: Describes how to use a Vector Store with Semantic Text search. Includes samples show how to create a search plugin which searches a Vector Store. +zone_pivot_groups: programming-languages +author: markwallace +ms.topic: conceptual +ms.author: markwallace +ms.date: 07/10/2024 +ms.service: semantic-kernel +--- + +# How to use Vector Stores with Semantic Kernel Text Search + +All of the Vector Store [connectors](../vector-store-connectors/out-of-the-box-connectors/index.md) can be used for text search. + +1. Use the Vector Store connector to retrieve the record collection you want to search. +2. Wrap the record collection with `VectorStoreTextSearch`. +3. Convert to a plugin for use in RAG and/or function calling scenarios. + +It's very likely that you will want to customize the plugin search function so that its description reflects the type of data available in the record collection. +For example if the record collection contains information about hotels the plugin search function description should mention this. +This will allow you to register multiple plugins e.g., one to search for hotels, another for restaurants and another for things to do. + +The [text search abstractions](./text-search-abstractions.md) include a function to return a normalized search result i.e., an instance of `TextSearchResult`. +This normalized search result contains a value and optionally a name and link. +The [text search abstractions](./text-search-abstractions.md) include a function to return a string value e.g., one of the data model properties will be returned as the search result. +For text search to work correctly you need to provide a way to map from the Vector Store data model to an instance of `TextSearchResult`. +The next section describes the two options you can use to perform this mapping. + +::: zone pivot="programming-language-csharp" + +> [!TIP] +> To run the samples shown on this page go to [GettingStartedWithTextSearch/Step4_Search_With_VectorStore.cs](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/GettingStartedWithTextSearch/Step4_Search_With_VectorStore.cs). + +## Using a vector store model with text search + +The mapping from a Vector Store data model to a `TextSearchResult` can be done declaratively using attributes. + +1. `[TextSearchResultValue]` - Add this attribute to the property of the data model which will be the value of the `TextSearchResult`, e.g. the textual data that the AI model will use to answer questions. +2. `[TextSearchResultName]` - Add this attribute to the property of the data model which will be the name of the `TextSearchResult`. +3. `[TextSearchResultLink]` - Add this attribute to the property of the data model which will be the link to the `TextSearchResult`. + +The following sample shows an data model which has the text search result attributes applied. + +```csharp +using Microsoft.Extensions.VectorData; +using Microsoft.SemanticKernel.Data; + +public sealed class DataModel +{ + [VectorStoreRecordKey] + [TextSearchResultName] + public Guid Key { get; init; } + + [VectorStoreRecordData] + [TextSearchResultValue] + public string Text { get; init; } + + [VectorStoreRecordData] + [TextSearchResultLink] + public string Link { get; init; } + + [VectorStoreRecordData(IsFilterable = true)] + public required string Tag { get; init; } + + [VectorStoreRecordVector(1536)] + public ReadOnlyMemory Embedding { get; init; } +} +``` + +The mapping from a Vector Store data model to a `string` or a `TextSearchResult` can also be done by providing implementations of `ITextSearchStringMapper` and `ITextSearchResultMapper` respectively. + +You may decide to create custom mappers for the following scenarios: + +1. Multiple properties from the data model need to be combined together e.g., if multiple properties need to be combined to provide the value. +2. Additional logic is required to generate one of the properties e.g., if the link property needs to be computed from the data model properties. + +The following sample shows a data model and two example mapper implementations that can be used with the data model. + +```csharp +using Microsoft.Extensions.VectorData; +using Microsoft.SemanticKernel.Data; + +protected sealed class DataModel +{ + [VectorStoreRecordKey] + public Guid Key { get; init; } + + [VectorStoreRecordData] + public required string Text { get; init; } + + [VectorStoreRecordData] + public required string Link { get; init; } + + [VectorStoreRecordData(IsFilterable = true)] + public required string Tag { get; init; } + + [VectorStoreRecordVector(1536)] + public ReadOnlyMemory Embedding { get; init; } +} + +/// +/// String mapper which converts a DataModel to a string. +/// +protected sealed class DataModelTextSearchStringMapper : ITextSearchStringMapper +{ + /// + public string MapFromResultToString(object result) + { + if (result is DataModel dataModel) + { + return dataModel.Text; + } + throw new ArgumentException("Invalid result type."); + } +} + +/// +/// Result mapper which converts a DataModel to a TextSearchResult. +/// +protected sealed class DataModelTextSearchResultMapper : ITextSearchResultMapper +{ + /// + public TextSearchResult MapFromResultToTextSearchResult(object result) + { + if (result is DataModel dataModel) + { + return new TextSearchResult(value: dataModel.Text) { Name = dataModel.Key.ToString(), Link = dataModel.Link }; + } + throw new ArgumentException("Invalid result type."); + } +} +``` + +The mapper implementations can be provided as parameters when creating the `VectorStoreTextSearch` as shown below: + +```csharp +using Microsoft.Extensions.VectorData; +using Microsoft.SemanticKernel.Data; + +// Create custom mapper to map a to a +var stringMapper = new DataModelTextSearchStringMapper(); + +// Create custom mapper to map a to a +var resultMapper = new DataModelTextSearchResultMapper(); + +// Add code to create instances of IVectorStoreRecordCollection and ITextEmbeddingGenerationService + +// Create a text search instance using the vector store record collection. +var result = new VectorStoreTextSearch(vectorStoreRecordCollection, textEmbeddingGeneration, stringMapper, resultMapper); +``` + +## Using a vector store with text search + +The sample below shows how to create an instance of `VectorStoreTextSearch` using a Vector Store record collection. + +> [!TIP] +> The following samples require instances of `IVectorStoreRecordCollection` and `ITextEmbeddingGenerationService`. +> To create an instance of `IVectorStoreRecordCollection` refer to [the documentation for each connector](../vector-store-connectors/out-of-the-box-connectors/index.md). +> To create an instance of `ITextEmbeddingGenerationService` select the service you wish to use e.g., Azure OpenAI, OpenAI, ... or use a local model ONNX, Ollama, ... and create an instance of the corresponding `ITextEmbeddingGenerationService` implementation. + +> [!TIP] +> A `VectorStoreTextSearch` can also be constructed from an instance of `IVectorizableTextSearch`. In this case no `ITextEmbeddingGenerationService` is needed. + +```csharp +using Microsoft.Extensions.VectorData; +using Microsoft.SemanticKernel; +using Microsoft.SemanticKernel.Connectors.OpenAI; +using Microsoft.SemanticKernel.Data; +using Microsoft.SemanticKernel.PromptTemplates.Handlebars; + +// Add code to create instances of IVectorStoreRecordCollection and ITextEmbeddingGenerationService + +// Create a text search instance using the vector store record collection. +var textSearch = new VectorStoreTextSearch(vectorStoreRecordCollection, textEmbeddingGeneration); + +// Search and return results as TextSearchResult items +var query = "What is the Semantic Kernel?"; +KernelSearchResults textResults = await textSearch.GetTextSearchResultsAsync(query, new() { Top = 2, Skip = 0 }); +Console.WriteLine("\n--- Text Search Results ---\n"); +await foreach (TextSearchResult result in textResults.Results) +{ + Console.WriteLine($"Name: {result.Name}"); + Console.WriteLine($"Value: {result.Value}"); + Console.WriteLine($"Link: {result.Link}"); +} +``` + +## Creating a search plugin from a vector store + +The sample below shows how to create a plugin named `SearchPlugin` from an instance of `VectorStoreTextSearch`. +Using `CreateWithGetTextSearchResults` creates a new plugin with a single `GetTextSearchResults` function that calls the underlying Vector Store record collection search implementation. +The `SearchPlugin` is added to the `Kernel` which makes it available to be called during prompt rendering. +The prompt template includes a call to `{{SearchPlugin.Search $query}}` which will invoke the `SearchPlugin` to retrieve results related to the current query. +The results are then inserted into the rendered prompt before it is sent to the model. + +```csharp +using Microsoft.Extensions.VectorData; +using Microsoft.SemanticKernel; +using Microsoft.SemanticKernel.Connectors.OpenAI; +using Microsoft.SemanticKernel.Data; +using Microsoft.SemanticKernel.PromptTemplates.Handlebars; + +// Create a kernel with OpenAI chat completion +IKernelBuilder kernelBuilder = Kernel.CreateBuilder(); +kernelBuilder.AddOpenAIChatCompletion( + modelId: TestConfiguration.OpenAI.ChatModelId, + apiKey: TestConfiguration.OpenAI.ApiKey); +Kernel kernel = kernelBuilder.Build(); + +// Add code to create instances of IVectorStoreRecordCollection and ITextEmbeddingGenerationService + +// Create a text search instance using the vector store record collection. +var textSearch = new VectorStoreTextSearch(vectorStoreRecordCollection, textEmbeddingGeneration); + +// Build a text search plugin with vector store search and add to the kernel +var searchPlugin = textSearch.CreateWithGetTextSearchResults("SearchPlugin"); +kernel.Plugins.Add(searchPlugin); + +// Invoke prompt and use text search plugin to provide grounding information +var query = "What is the Semantic Kernel?"; +string promptTemplate = """ + {{#with (SearchPlugin-GetTextSearchResults query)}} + {{#each this}} + Name: {{Name}} + Value: {{Value}} + Link: {{Link}} + ----------------- + {{/each}} + {{/with}} + + {{query}} + + Include citations to the relevant information where it is referenced in the response. + """; +KernelArguments arguments = new() { { "query", query } }; +HandlebarsPromptTemplateFactory promptTemplateFactory = new(); +Console.WriteLine(await kernel.InvokePromptAsync( + promptTemplate, + arguments, + templateFormat: HandlebarsPromptTemplateFactory.HandlebarsTemplateFormat, + promptTemplateFactory: promptTemplateFactory +)); +``` + +## Using a vector store with function calling + +The sample below also creates a `SearchPlugin` from an instance of `VectorStoreTextSearch`. +This plugin will be advertised to the model for use with automatic function calling using the `FunctionChoiceBehavior` in the prompt execution settings. +When you run this sample the model will invoke the search function to retrieve additional information to respond to the question. +It will likely just search for "Semantic Kernel" rather than the entire query. + +```csharp +using Microsoft.Extensions.VectorData; +using Microsoft.SemanticKernel; +using Microsoft.SemanticKernel.Connectors.OpenAI; +using Microsoft.SemanticKernel.Data; +using Microsoft.SemanticKernel.PromptTemplates.Handlebars; + +// Create a kernel with OpenAI chat completion +IKernelBuilder kernelBuilder = Kernel.CreateBuilder(); +kernelBuilder.AddOpenAIChatCompletion( + modelId: TestConfiguration.OpenAI.ChatModelId, + apiKey: TestConfiguration.OpenAI.ApiKey); +Kernel kernel = kernelBuilder.Build(); + +// Add code to create instances of IVectorStoreRecordCollection and ITextEmbeddingGenerationService + +// Create a text search instance using the vector store record collection. +var textSearch = new VectorStoreTextSearch(vectorStoreRecordCollection, textEmbeddingGeneration); + +// Build a text search plugin with vector store search and add to the kernel +var searchPlugin = textSearch.CreateWithGetTextSearchResults("SearchPlugin"); +kernel.Plugins.Add(searchPlugin); + +// Invoke prompt and use text search plugin to provide grounding information +OpenAIPromptExecutionSettings settings = new() { FunctionChoiceBehavior = FunctionChoiceBehavior.Auto() }; +KernelArguments arguments = new(settings); +Console.WriteLine(await kernel.InvokePromptAsync("What is the Semantic Kernel?", arguments)); +``` + +## Customizing the search function + +The sample below how to customize the description of the search function that is added to the `SearchPlugin`. +Some things you might want to do are: + +1. Change the name of the search function to reflect what is in the associated record collection e.g., you might want to name the function `SearchForHotels` if the record collection contains hotel information. +2. Change the description of the function. An accurate function description helps the AI model to select the best function to call. This is especially important if you are adding multiple search functions. +3. Add an additional parameter to the search function. If the record collection contain hotel information and one of the properties is the city name you could add a property to the search function to specify the city. A filter will be automatically added and it will filter search results by city. + +> [!TIP] +> The sample below uses the default implementation of search. You can opt to provide your own implementation which calls the underlying Vector Store record collection with additional options to fine tune your searches. + +```csharp +using Microsoft.Extensions.VectorData; +using Microsoft.SemanticKernel; +using Microsoft.SemanticKernel.Connectors.OpenAI; +using Microsoft.SemanticKernel.Data; +using Microsoft.SemanticKernel.PromptTemplates.Handlebars; + +// Create a kernel with OpenAI chat completion +IKernelBuilder kernelBuilder = Kernel.CreateBuilder(); +kernelBuilder.AddOpenAIChatCompletion( + modelId: TestConfiguration.OpenAI.ChatModelId, + apiKey: TestConfiguration.OpenAI.ApiKey); +Kernel kernel = kernelBuilder.Build(); + +// Add code to create instances of IVectorStoreRecordCollection and ITextEmbeddingGenerationService + +// Create a text search instance using the vector store record collection. +var textSearch = new VectorStoreTextSearch(vectorStoreRecordCollection, textEmbeddingGeneration); + +// Create options to describe the function I want to register. +var options = new KernelFunctionFromMethodOptions() +{ + FunctionName = "Search", + Description = "Perform a search for content related to the specified query from a record collection.", + Parameters = + [ + new KernelParameterMetadata("query") { Description = "What to search for", IsRequired = true }, + new KernelParameterMetadata("top") { Description = "Number of results", IsRequired = false, DefaultValue = 2 }, + new KernelParameterMetadata("skip") { Description = "Number of results to skip", IsRequired = false, DefaultValue = 0 }, + ], + ReturnParameter = new() { ParameterType = typeof(KernelSearchResults) }, +}; + +// Build a text search plugin with vector store search and add to the kernel +var searchPlugin = textSearch.CreateWithGetTextSearchResults("SearchPlugin", "Search a record collection", [textSearch.CreateSearch(options)]); +kernel.Plugins.Add(searchPlugin); + +// Invoke prompt and use text search plugin to provide grounding information +OpenAIPromptExecutionSettings settings = new() { FunctionChoiceBehavior = FunctionChoiceBehavior.Auto() }; +KernelArguments arguments = new(settings); +Console.WriteLine(await kernel.InvokePromptAsync("What is the Semantic Kernel?", arguments)); +``` + +::: zone-end +::: zone pivot="programming-language-python" + +## Coming soon + +More coming soon. + +::: zone-end +::: zone pivot="programming-language-java" + +## Coming soon + +More coming soon. + +::: zone-end + +## Next steps + +> [!div class="nextstepaction"] +> [Vector Stores](../vector-store-connectors/index.md) diff --git a/semantic-kernel/concepts/vector-store-connectors/TOC.yml b/semantic-kernel/concepts/vector-store-connectors/TOC.yml new file mode 100644 index 00000000..b7cfa81b --- /dev/null +++ b/semantic-kernel/concepts/vector-store-connectors/TOC.yml @@ -0,0 +1,24 @@ +- name: Overview + href: index.md +- name: Data architecture + href: data-architecture.md +- name: Defining your data model + href: defining-your-data-model.md +- name: Defining your storage schema using a record definition + href: schema-with-record-definition.md +- name: Generic data model + href: generic-data-model.md +- name: Embedding Generation + href: embedding-generation.md +- name: Vector Search + href: vector-search.md +- name: Serialization of data models + href: serialization.md +- name: Legacy Memory Stores + href: memory-stores.md +- name: Code samples + href: code-samples.md +- name: Out-of-the-box connectors + href: out-of-the-box-connectors/TOC.yml +- name: How to + href: how-to/TOC.yml diff --git a/semantic-kernel/concepts/vector-store-connectors/code-samples.md b/semantic-kernel/concepts/vector-store-connectors/code-samples.md new file mode 100644 index 00000000..5c78944f --- /dev/null +++ b/semantic-kernel/concepts/vector-store-connectors/code-samples.md @@ -0,0 +1,99 @@ +--- +title: Semantic Kernel Vector Store code samples (Preview) +description: Lists code samples for the Semantic Kernel Vector Store abstractions and implementations +zone_pivot_groups: programming-languages +author: westey-m +ms.topic: conceptual +ms.author: westey +ms.date: 07/08/2024 +ms.service: semantic-kernel +--- +# Semantic Kernel Vector Store code samples (Preview) + +> [!WARNING] +> The Semantic Kernel Vector Store functionality is in preview, and improvements that require breaking changes may still occur in limited circumstances before release. + +::: zone pivot="programming-language-csharp" + +## End to end RAG sample with Vector Stores + +This example is a standalone console application that demonstrates RAG using Semantic Kernel. The sample has the following characteristics: + +1. Allows a choice of chat and embedding services +1. Allows a choice of vector databases +1. Reads the contents of one or more PDF files and creates a chunks for each section +1. Generates embeddings for each text chunk and upserts it to the chosen vector database +1. Registers the Vector Store as a Text Search plugin with the kernel +1. Invokes the plugin to augment the prompt provided to the AI model with more context + +- [End to end RAG demo](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Demos/VectorStoreRAG/README.md) + +## Simple Data Ingestion and Vector Search + +For two very simple examples of how to do data ingestion into a vector store and do vector search, check out these +two examples, which use Qdrant and InMemory vector stores to demonstrate their usage. + +- [Simple Vector Search](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/Memory/VectorStore_VectorSearch_Simple.cs) +- [Simple Data Ingestion](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/Memory/VectorStore_DataIngestion_Simple.cs) + +## Common code with multiple stores + +Vector stores may different in certain aspects, e.g. with regards to the types of their keys or the types of fields each support. +Even so, it is possible to write code that is agnostic to these differences. + +For a data ingestion sample that demonstrates this, see: +- [MultiStore Data Ingestion](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/Memory/VectorStore_DataIngestion_MultiStore.cs) + +For a vector search sample demonstrating the same concept see the following samples. +Each of these samples are referencing the same common code, and just differ on the type of +vector store they create to use with the common code. + +- [Azure AI Search vector search with common code](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/Memory/VectorStore_VectorSearch_MultiStore_AzureAISearch.cs) +- [InMemory vector search with common code](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/Memory/VectorStore_VectorSearch_MultiStore_InMemory.cs) +- [Qdrant vector search with common code](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/Memory/VectorStore_VectorSearch_MultiStore_Qdrant.cs) +- [Redis vector search with common code](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/Memory/VectorStore_VectorSearch_MultiStore_Redis.cs) + +## Supporting multiple vectors in the same record + +The Vector Store abstractions support multiple vectors in the same record, for vector databases that support this. +The following sample shows how to create some records with multiple vectors, and pick the desired target vector +when doing a vector search. + +- [Choosing a vector for search on a record with multiple vectors](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/Memory/VectorStore_VectorSearch_MultiVector.cs) + +## Vector search with paging + +When doing vector search with the Vector Store abstractions it's possible to use Top and Skip parameters to support paging, where e.g. +you need to build a service that reponds with a small set of results per request. + +- [Vector search with paging](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/Memory/VectorStore_VectorSearch_Paging.cs) + +> [!WARNING] +> Not all vector databases support Skip functionality natively for vector searches, so some connectors may have to fetch Skip + Top records and skip +> on the client side to simulate this behavior. + +## Using the generic data model vs using a custom data model + +It's possible to use the the Vector Store abstractions without defining a data model and defining your schema via a record definition instead. +This example shows how you can create a vector store using a custom model and read using the generic data model or vice versa. + +- [Generic data model interop](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/Memory/VectorStore_GenericDataModel_Interop.cs) + +> [!TIP] +> For more information about using the generic data model, refer to [using Vector Store abstractions without defining your own data model](./generic-data-model.md). + +::: zone-end +::: zone pivot="programming-language-python" + +## Coming soon + +More coming soon. + +::: zone-end +::: zone pivot="programming-language-java" + +## Coming soon + +More coming soon. + +::: zone-end diff --git a/semantic-kernel/concepts/vector-store-connectors/data-architecture.md b/semantic-kernel/concepts/vector-store-connectors/data-architecture.md new file mode 100644 index 00000000..3e3251fe --- /dev/null +++ b/semantic-kernel/concepts/vector-store-connectors/data-architecture.md @@ -0,0 +1,29 @@ +--- +title: The Semantic Kernel Vector Store data architecture (Preview) +description: Defines the data architecture for Semantic Kernel, including the relationship between vector stores, collections and records. +author: westey-m +ms.topic: conceptual +ms.author: westey +ms.date: 07/08/2024 +ms.service: semantic-kernel +--- +# The Semantic Kernel Vector Store data architecture (Preview) + +> [!WARNING] +> The Semantic Kernel Vector Store functionality is in preview, and improvements that require breaking changes may still occur in limited circumstances before release. + +Vector Store abstractions in Semantic Kernel are based on three main components: **vector stores**, **collections** and **records**. +**Records** are contained by **collections**, and **collections** are contained by **vector stores**. + +- A **vector store** maps to an instance of a database +- A **collection** is a collection of **records** including any index required to query or filter those **records** +- A **record** is an individual data entry in the database + +## Collections in different databases + +The underlying implementation of what a collection is, will vary by connector and is influenced by how each database groups and indexes records. +Most databases have a concept of a collection of records and there is a natural mapping between this concept and the Vector Store abstraction collection. +Note that this concept may not always be referred to as a `collection` in the underlying database. + +> [!TIP] +> For more information on what the underlying implementation of a collection is per connector, refer to [the documentation for each connector](./out-of-the-box-connectors/index.md). diff --git a/semantic-kernel/concepts/vector-store-connectors/defining-your-data-model.md b/semantic-kernel/concepts/vector-store-connectors/defining-your-data-model.md new file mode 100644 index 00000000..7dc42327 --- /dev/null +++ b/semantic-kernel/concepts/vector-store-connectors/defining-your-data-model.md @@ -0,0 +1,240 @@ +--- +title: Defining your Semantic Kernel Vector Store data model (Preview) +description: Describes how to create a data model with Semantic Kernel to use when writing to or reading from a Vector Store. +zone_pivot_groups: programming-languages +author: westey-m +ms.topic: reference +ms.author: westey +ms.date: 07/08/2024 +ms.service: semantic-kernel +--- +# Defining your data model (Preview) + +> [!WARNING] +> The Semantic Kernel Vector Store functionality is in preview, and improvements that require breaking changes may still occur in limited circumstances before release. + +## Overview + +The Semantic Kernel Vector Store connectors use a model first approach to interacting with databases. + +::: zone pivot="programming-language-csharp" +All methods to upsert or get records use strongly typed model classes. +The properties on these classes are decorated with attributes that indicate the purpose of each property. + +> [!TIP] +> For an alternative to using attributes, refer to [defining your schema with a record definition](./schema-with-record-definition.md). +> [!TIP] +> For an alternative to defining your own data model, refer to [using Vector Store abstractions without defining your own data model](./generic-data-model.md). + +Here is an example of a model that is decorated with these attributes. + +```csharp +using Microsoft.Extensions.VectorData; + +public class Hotel +{ + [VectorStoreRecordKey] + public ulong HotelId { get; set; } + + [VectorStoreRecordData(IsFilterable = true)] + public string HotelName { get; set; } + + [VectorStoreRecordData(IsFullTextSearchable = true)] + public string Description { get; set; } + + [VectorStoreRecordVector(4, DistanceFunction.CosineDistance, IndexKind.Hnsw)] + public ReadOnlyMemory? DescriptionEmbedding { get; set; } + + [VectorStoreRecordData(IsFilterable = true)] + public string[] Tags { get; set; } +} +``` + +## Attributes + +### VectorStoreRecordKeyAttribute + +Use this attribute to indicate that your property is the key of the record. + +```csharp +[VectorStoreRecordKey] +public ulong HotelId { get; set; } +``` + +#### VectorStoreRecordKeyAttribute parameters + +| Parameter | Required | Description | +|---------------------------|:--------:|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| StoragePropertyName | No | Can be used to supply an alternative name for the property in the database. Note that this parameter is not supported by all connectors, e.g. where alternatives like `JsonPropertyNameAttribute` is supported. | + +> [!TIP] +> For more information on which connectors support StoragePropertyName and what alternatives are available, refer to [the documentation for each connector](./out-of-the-box-connectors/index.md). + +### VectorStoreRecordDataAttribute + +Use this attribute to indicate that your property contains general data that is not a key or a vector. + +```csharp +[VectorStoreRecordData(IsFilterable = true)] +public string HotelName { get; set; } +``` + +#### VectorStoreRecordDataAttribute parameters + +| Parameter | Required | Description | +|---------------------------|:--------:|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| IsFilterable | No | Indicates whether the property should be indexed for filtering in cases where a database requires opting in to indexing per property. Default is false. | +| IsFullTextSearchable | No | Indicates whether the property should be indexed for full text search for databases that support full text search. Default is false. | +| StoragePropertyName | No | Can be used to supply an alternative name for the property in the database. Note that this parameter is not supported by all connectors, e.g. where alternatives like `JsonPropertyNameAttribute` is supported. | + +> [!TIP] +> For more information on which connectors support StoragePropertyName and what alternatives are available, refer to [the documentation for each connector](./out-of-the-box-connectors/index.md). + +### VectorStoreRecordVectorAttribute + +Use this attribute to indicate that your property contains a vector. + +```csharp +[VectorStoreRecordVector(Dimensions: 4, DistanceFunction.CosineDistance, IndexKind.Hnsw)] +public ReadOnlyMemory? DescriptionEmbedding { get; set; } +``` + +#### VectorStoreRecordVectorAttribute parameters + +| Parameter | Required | Description | +|---------------------------|:--------:|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| Dimensions | Yes for collection create, optional otherwise | The number of dimensions that the vector has. This is typically required when creating a vector index for a collection. | +| IndexKind | No | The type of index to index the vector with. Default varies by vector store type. | +| DistanceFunction | No | The type of distance function to use when doing vector comparison during vector search over this vector. Default varies by vector store type. | +| StoragePropertyName | No | Can be used to supply an alternative name for the property in the database. Note that this parameter is not supported by all connectors, e.g. where alternatives like `JsonPropertyNameAttribute` is supported. | + +Common index kinds and distance function types are supplied as static values on the `Microsoft.SemanticKernel.Data.IndexKind` and `Microsoft.SemanticKernel.Data.DistanceFunction` classes. +Individual Vector Store implementations may also use their own index kinds and distance functions, where the database supports unusual types. + +> [!TIP] +> For more information on which connectors support StoragePropertyName and what alternatives are available, refer to [the documentation for each connector](./out-of-the-box-connectors/index.md). + +::: zone-end + +::: zone pivot="programming-language-python" + +All methods to upsert or get records use a class and a vector store record definition. + +This can be done by defining your own class with annotations for the fields, or by using a class/type in combination with a record definition. Two things need to be done for a class, the first is to add the annotations with the field types, the second is to decorate the class with the `vectorstoremodel` decorator. + +> [!TIP] +> For the alternative approach using a record definition, refer to [definining your schema with a record definition](./schema-with-record-definition.md). + +Here is an example of a model that is decorated with these annotations. + +```python +from dataclasses import dataclass, field +from typing import Annotated +from semantic_kernel.data import ( + DistanceFunction, + IndexKind, + VectorStoreRecordDataField, + VectorStoreRecordDefinition, + VectorStoreRecordKeyField, + VectorStoreRecordVectorField, + vectorstoremodel, +) + +@vectorstoremodel +@dataclass +class Hotel: + hotel_id: Annotated[str, VectorStoreRecordKeyField()] = field(default_factory=lambda: str(uuid4())) + hotel_name: Annotated[str, VectorStoreRecordDataField(is_filterable=True)] + description: Annotated[str, VectorStoreRecordDataField(is_full_text_searchable=True)] + description_embedding: Annotated[list[float], VectorStoreRecordVectorField(dimensions=4, distance_function=DistanceFunction.COSINE, index_kind=IndexKind.HNSW)] + tags: Annotated[list[str], VectorStoreRecordDataField(is_filterable=True)] +``` + +> [!TIP] +> Defining a class with these annotations can be done in multiple ways, one of which is using the `dataclasses` module in Python, shown here. This [sample](https://github.com/microsoft/semantic-kernel/blob/main/python/samples/concepts/memory/data_models.py) shows other approaches (using Pydantic BaseModels and vanilla python classes) as well. + +## Annotations + +There are three types of annotations to be used, and they have a common base class. + +### VectorStoreRecordField + +This is the base class for all annotations, it is not meant to be used directly. + +### VectorStoreRecordField parameters + +| Parameter | Required | Description | +|---------------------------|:--------:|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| name | No | Can be added directly but will be set during parsing of the model. | +| property_type | No | Should be a string, will also be derived during parsing. | + +> [!TIP] +> The annotations are parsed by the `vectorstoremodel` decorator and one of the things it does is to create a record definition for the class, it is therefore not necessary to instantiate a field class when no parameters are set, the field can be annotated with just the class, like this: +> ```python +> hotel_id: Annotated[str, VectorStoreRecordKeyField] +> ``` + +### VectorStoreRecordKeyField + +Use this annotation to indicate that this attribute is the key of the record. + +```python +VectorStoreRecordKeyField() +``` + +#### VectorStoreRecordKeyField parameters +No other parameters outside of the base class are defined. + + +### VectorStoreRecordDataField + +Use this annotation to indicate that your attribute contains general data that is not a key or a vector. + +```python +VectorStoreRecordDataField(is_filterable=True) +``` + +#### VectorStoreRecordDataField parameters + +| Parameter | Required | Description | +|---------------------------|:--------:|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| has_embedding | No | Indicates whether the property has a embedding associated with it, default is None. | +| embedding_property_name | No | The name of the property that contains the embedding, default is None. | +| is_filterable | No | Indicates whether the property should be indexed for filtering in cases where a database requires opting in to indexing per property. Default is false. | +| is_full_text_searchable | No | Indicates whether the property should be indexed for full text search for databases that support full text search. Default is false. | + + +### VectorStoreRecordVectorField + +Use this annotation to indicate that your attribute contains a vector. + +```python +VectorStoreRecordVectorField(dimensions=4, distance_function=DistanceFunction.COSINE, index_kind=IndexKind.HNSW) +``` + +#### VectorStoreRecordVectorField parameters + +| Parameter | Required | Description | +|---------------------------|:--------:|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| dimensions | Yes for collection create, optional otherwise | The number of dimensions that the vector has. This is typically required when creating a vector index for a collection. | +| index_kind | No | The type of index to index the vector with. Default varies by vector store type. | +| distance_function | No | The type of distance function to use when doing vector comparison during vector search over this vector. Default varies by vector store type. | +| local_embedding | No | Indicates whether the property has a local embedding associated with it, default is None. | +| embedding_settings | No | The settings for the embedding, in the form of a dict with service_id as key and PromptExecutionSettings as value, default is None. | +| serialize_function | No | The function to use to serialize the vector, if the type is not a list[float \| int] this function is needed, or the whole model needs to be serialized. | +| deserialize_function | No | The function to use to deserialize the vector, if the type is not a list[float \| int] this function is needed, or the whole model needs to be deserialized. | + + +Common index kinds and distance function types are supplied as static values on the `semantic_kernel.data.IndexKind` and `semantic_kernel.data.DistanceFunction` classes. +Individual Vector Store implementations may also use their own index kinds and distance functions, where the database supports unusual types. + + +::: zone-end + +::: zone pivot="programming-language-java" + +## Coming soon + +More info coming soon. + +::: zone-end diff --git a/semantic-kernel/concepts/vector-store-connectors/embedding-generation.md b/semantic-kernel/concepts/vector-store-connectors/embedding-generation.md new file mode 100644 index 00000000..43a9ec36 --- /dev/null +++ b/semantic-kernel/concepts/vector-store-connectors/embedding-generation.md @@ -0,0 +1,153 @@ +--- +title: Generating embeddings for Semantic Kernel Vector Store connectors +description: Describes how you can generate embeddings to use with Semantic Kernel vector store connectors. +zone_pivot_groups: programming-languages +author: westey-m +ms.topic: conceptual +ms.author: westey +ms.date: 09/23/2024 +ms.service: semantic-kernel +--- + +# Generating embeddings for Semantic Kernel Vector Store connectors + +> [!WARNING] +> The Semantic Kernel Vector Store functionality is in preview, and improvements that require breaking changes may still occur in limited circumstances before release. + +Semantic Kernel supports generating embeddings using many popular AI services out of the box. + +::: zone pivot="programming-language-csharp" + +These services can be constructed directly or added to a dependency injection container and resolved from there. + +## Constructing an embedding generator + +You can construct instances of the text embedding services provided by Semantic Kernel directly. +They all implement the `ITextEmbeddingGenerationService` interface. + +```csharp +// Constructing an Azure Open AI embedding generation service directly. +ITextEmbeddingGenerationService azureOpenAITES = new AzureOpenAITextEmbeddingGenerationService( + "text-embedding-ada-002", + "https://{myservice}.openai.azure.com/", + "apikey"); + +// Constructing an Olama embedding generation service directly. +ITextEmbeddingGenerationService olamaTES = new OllamaTextEmbeddingGenerationService( + "mxbai-embed-large", + new Uri("http://localhost:11434")); +``` + +You can also use helpers to register them with a dependency injection container. + +```csharp +// Registering Google AI embedding generation service with a service collection. +var services = new ServiceCollection(); +services.AddGoogleAIEmbeddingGeneration("text-embedding-004", "apiKey"); + +// Registering Mistral AI embedding generation service with the dependency injection container on +// the kernel builder. +var kernelBuilder = Kernel + .CreateBuilder() + .AddMistralTextEmbeddingGeneration("mistral-embed", "apiKey"); +``` + +## Generating embeddings + +To use the `ITextEmbeddingGenerationService` you created, just call the `GenerateEmbeddingAsync` method +on it. + +Here is an example of generating embeddings when uploading records. + +```csharp +public async Task GenerateEmbeddingsAndUpsertAsync( + ITextEmbeddingGenerationService textEmbeddingGenerationService, + IVectorStoreRecordCollection collection) +{ + // Upsert a record. + string descriptionText = "A place where everyone can be happy."; + ulong hotelId = 1; + + // Generate the embedding. + ReadOnlyMemory embedding = + await textEmbeddingGenerationService.GenerateEmbeddingAsync(descriptionText); + + // Create a record and upsert with the already generated embedding. + await collection.UpsertAsync(new Hotel + { + HotelId = hotelId, + HotelName = "Hotel Happy", + Description = descriptionText, + DescriptionEmbedding = embedding, + Tags = new[] { "luxury", "pool" } + }); +} +``` + +Here is an example of generating embeddings when searching. + +```csharp +public async Task GenerateEmbeddingsAndSearchAsync( + ITextEmbeddingGenerationService textEmbeddingGenerationService, + IVectorStoreRecordCollection collection) +{ + // Upsert a record. + string descriptionText = "Find me a hotel with happiness in mind."; + + // Generate the embedding. + ReadOnlyMemory searchEmbedding = + await textEmbeddingGenerationService.GenerateEmbeddingAsync(descriptionText); + + // Search using the already generated embedding. + List> searchResult = await collection.VectorizedSearchAsync(searchEmbedding).ToListAsync(); + + // Print the first search result. + Console.WriteLine("Score for first result: " + searchResult.FirstOrDefault()?.Score); + Console.WriteLine("Hotel description for first result: " + searchResult.FirstOrDefault()?.Record.Description); +} +``` + +## Embedding dimensions + +Vector databases typically require you to specify the number of dimensions that each vector has when creating the collection. +Different embedding models typically support generating vectors with different dimension sizes. E.g. Open AI `text-embedding-ada-002` +generates vectors with 1536 dimensions. Some models also allow a developer to choose the number of dimensions they want in the +output vector, e.g. Google `text-embedding-004` produces vectors with 768 dimension by default, but allows a developer to +choose any number of dimensions between 1 and 768. + +It is important to ensure that the vectors generated by the embedding model have the same number of dimensions as the +matching vector in the database. + +If creating a collection using the Semantic Kernel Vector Store abstractions, you need to specify the number of dimensions +required for each vector property either via annotations or via the record definition. Here are examples of both setting +the number of dimensions to 1536. + +```csharp +[VectorStoreRecordVector(Dimensions: 1536)] +public ReadOnlyMemory? DescriptionEmbedding { get; set; } +``` + +```csharp +new VectorStoreRecordVectorProperty("DescriptionEmbedding", typeof(float)) { Dimensions = 1536 } +``` + +> [!TIP] +> For more information on how to annotate your data model, refer to [defining your data model](./defining-your-data-model.md). +> [!TIP] +> For more information on creating a record definition, refer to [defining your schema with a record definition](./schema-with-record-definition.md). + +::: zone-end +::: zone pivot="programming-language-python" + +## Coming soon + +More info coming soon. + +::: zone-end +::: zone pivot="programming-language-java" + +## Coming soon + +More info coming soon. + +::: zone-end diff --git a/semantic-kernel/concepts/vector-store-connectors/generic-data-model.md b/semantic-kernel/concepts/vector-store-connectors/generic-data-model.md new file mode 100644 index 00000000..ca3e2948 --- /dev/null +++ b/semantic-kernel/concepts/vector-store-connectors/generic-data-model.md @@ -0,0 +1,110 @@ +--- +title: Using Vector Store abstractions without defining your own data model (Preview) +description: Describes how to use Vector Store abstractions without defining your own data model. +zone_pivot_groups: programming-languages +author: westey-m +ms.topic: reference +ms.author: westey +ms.date: 10/16/2024 +ms.service: semantic-kernel +--- +# Using Vector Store abstractions without defining your own data model (Preview) + +> [!WARNING] +> The Semantic Kernel Vector Store functionality is in preview, and improvements that require breaking changes may still occur in limited circumstances before release. + +::: zone pivot="programming-language-csharp" + +## Overview + +The Semantic Kernel Vector Store connectors use a model first approach to interacting with databases. This makes using the connectors easy and simple, since +your data model reflects the schema of your database records and to add any additional schema information required, you can simply add attributes to your data model properties. + +There are cases though where it is not desirable or possible to define your own data model. E.g. let's say that you do not know at compile time what your +database schema looks like, and the schema is only provided via configuration. Creating a data model that reflects the schema would be impossible in this case. + +To cater for this scenario, we provide a generic data model. + +## Generic Data Model + +The generic data model is a class named `VectorStoreGenericDataModel` and is available in the `Microsoft.Extensions.VectorData.Abstractions` package. + +To support any type of database, the type of the key of the `VectorStoreGenericDataModel` is specified via a generic parameter. + +All other properties are divided into `Data` and `Vector` properties. Any property that is not a vector or a key is considered a data property. +`Data` and `Vector` property sets are stored as string-keyed dictionaries of objects. + +## Supplying schema information when using the Generic Data Model + +When using the generic data model, connectors still need to know what the database schema looks like. Without the schema information +the connector would not be able to create a collection, or know how to map to and from the storage representation that each database uses. + +A record definition can be used to provide the schema information. Unlike a data model, a record definition can be created from configuration +at runtime, providing a solution for when schema information is not known at compile time. + +> [!TIP] +> To see how to create a record definition, refer to [defining your schema with a record definition](./schema-with-record-definition.md). + +## Example + +To use the generic data model with a connector, simply specify it as your data model when creating a collection, and simultaneously provide a record definition. + +```csharp +// Create the definition to define the schema. +VectorStoreRecordDefinition vectorStoreRecordDefinition = new() +{ + Properties = new List + { + new VectorStoreRecordKeyProperty("Key", typeof(string)), + new VectorStoreRecordDataProperty("Term", typeof(string)), + new VectorStoreRecordDataProperty("Definition", typeof(string)), + new VectorStoreRecordVectorProperty("DefinitionEmbedding", typeof(ReadOnlyMemory)) { Dimensions = 1536 } + } +}; + +// When getting your collection instance from a vector store instance +// specify the generic data model, using the appropriate key type for your database +// and also pass your record definition. +var genericDataModelCollection = vectorStore.GetCollection>( + "glossary", + vectorStoreRecordDefinition); + +// Since we have schema information available from the record definition +// it's possible to create a collection with the right vectors, dimensions, +// indexes and distance functions. +await genericDataModelCollection.CreateCollectionIfNotExistsAsync(); + +// When retrieving a record from the collection, data and vectors can +// now be accessed via the Data and Vector dictionaries respectively. +var record = await genericDataModelCollection.GetAsync("SK"); +Console.WriteLine(record.Data["Definition"]) +``` + +When constructing a collection instance directly, the record definition +is passed as an option. E.g. here is an example of constructing +an Azure AI Search collection instance with the generic data model. + +```csharp +new AzureAISearchVectorStoreRecordCollection>( + searchIndexClient, + "glossary", + new() { VectorStoreRecordDefinition = vectorStoreRecordDefinition }); +``` + +::: zone-end + +::: zone pivot="programming-language-python" + +## Coming soon + +More info coming soon. + +::: zone-end + +::: zone pivot="programming-language-java" + +## Coming soon + +More info coming soon. + +::: zone-end diff --git a/semantic-kernel/concepts/vector-store-connectors/how-to/TOC.yml b/semantic-kernel/concepts/vector-store-connectors/how-to/TOC.yml new file mode 100644 index 00000000..e68ac352 --- /dev/null +++ b/semantic-kernel/concepts/vector-store-connectors/how-to/TOC.yml @@ -0,0 +1,4 @@ +- name: How to ingest data into a Vector Store + href: vector-store-data-ingestion.md +- name: How to build a custom mapper for a Vector Store connector + href: vector-store-custom-mapper.md diff --git a/semantic-kernel/concepts/vector-store-connectors/how-to/vector-store-custom-mapper.md b/semantic-kernel/concepts/vector-store-connectors/how-to/vector-store-custom-mapper.md new file mode 100644 index 00000000..d4efdc4c --- /dev/null +++ b/semantic-kernel/concepts/vector-store-connectors/how-to/vector-store-custom-mapper.md @@ -0,0 +1,291 @@ +--- +title: How to build a custom mapper for a Semantic Kernel Vector Store connector (Preview) +description: Describes how to build a custom mapper for a Semantic Kernel Vector Store connector +zone_pivot_groups: programming-languages +author: westey-m +ms.topic: tutorial +ms.author: westey +ms.date: 07/08/2024 +ms.service: semantic-kernel +--- +# How to build a custom mapper for a Vector Store connector (Preview) + +> [!WARNING] +> The Semantic Kernel Vector Store functionality is in preview, and improvements that require breaking changes may still occur in limited circumstances before release. + +In this how to, we will show how you can replace the default mapper for a vector store record collection with your own mapper. + +We will use Qdrant to demonstrate this functionality, but the concepts will be similar for other connectors. + +## Background + +Each Vector Store connector includes a default mapper that can map from the provided data model to the storage schema supported by the underlying store. +Some stores allow a lot of freedom with regards to how data is stored while other stores require a more structured approach, e.g. where all vectors have +to be added to a dictionary of vectors and all non-vector fields to a dictionary of data fields. Therefore, mapping is an important part of abstracting +away the differences of each data store implementation. + +In some cases, the developer may want to replace the default mapper if e.g. + +1. they want to use a data model that differs from the storage schema. +2. they want to build a performance optimized mapper for their scenario. +3. the default mapper doesn't support a storage structure that the developer requires. + +All Vector Store connector implementations allow you to provide a custom mapper. + +## Differences by vector store type + +The underlying data stores of each Vector Store connector have different ways of storing data. Therefore what you are mapping to on the storage side may +differ for each connector. + +::: zone pivot="programming-language-csharp" +E.g. if using the Qdrant connector, the storage type is a `PointStruct` class provided by the Qdrant SDK. If using the Redis JSON connector, the storage type +is a `string` key and a `JsonNode`, while if using a JSON HashSet connector, the storage type is a `string` key and a `HashEntry` array. +::: zone-end +::: zone pivot="programming-language-python" +::: zone-end +::: zone pivot="programming-language-java" +::: zone-end + +If you want to do custom mapping, and you want to use multiple connector types, you will therefore need to implement a mapper for each connector type. + +::: zone pivot="programming-language-csharp" + +## Creating the data model + +Our first step is to create a data model. In this case we will not annotate the data model with attributes, since we will provide a separate record definition +that describes what the database schema will look like. + +Also note that this model is complex, with seperate classes for vectors and additional product info. + +```csharp +public class Product +{ + public ulong Id { get; set; } + public string Name { get; set; } + public string Description { get; set; } + public ProductVectors Vectors { get; set; } + public ProductInfo ProductInfo { get; set; } +} + +public class ProductInfo +{ + public double Price { get; set; } + public string SupplierId { get; set; } +} + +public class ProductVectors +{ + public ReadOnlyMemory NameEmbedding { get; set; } + public ReadOnlyMemory DescriptionEmbedding { get; set; } +} +``` + +## Creating the record definition + +We need to create a record definition instance to define what the database schema will look like. Normally a connector will require this information to do +mapping when using the default mapper. Since we are creating a custom mapper, this is not required for mapping, however, the connector will still require +this information for creating collections in the data store. + +Note that the definition here is different to the data model above. To store `ProductInfo` we have a string property called `ProductInfoJson`, and +the two vectors are defined at the same level as the `Id`, `Name` and `Description` fields. + +```csharp +using Microsoft.Extensions.VectorData; + +var productDefinition = new VectorStoreRecordDefinition +{ + Properties = new List + { + new VectorStoreRecordKeyProperty("Id", typeof(ulong)), + new VectorStoreRecordDataProperty("Name", typeof(string)) { IsFilterable = true }, + new VectorStoreRecordDataProperty("Description", typeof(string)), + new VectorStoreRecordDataProperty("ProductInfoJson", typeof(string)), + new VectorStoreRecordVectorProperty("NameEmbedding", typeof(ReadOnlyMemory)) { Dimensions = 1536 }, + new VectorStoreRecordVectorProperty("DescriptionEmbedding", typeof(ReadOnlyMemory)) { Dimensions = 1536 } + } +}; +``` + +> [!IMPORTANT] +> For this scenario, it would not be possible to use attributes instead of a record definition since the storage schema does +> not resemble the data model. + +## Creating the custom mapper + +All mappers implement the generic interface `Microsoft.SemanticKernel.Data.IVectorStoreRecordMapper`. +`TRecordDataModel` will differ depending on what data model the developer wants to use, and `TStorageModel` will be determined by the type of Vector Store. + +For Qdrant `TStorageModel` is `Qdrant.Client.Grpc.PointStruct`. + +We therefore have to implement a mapper that will map between our `Product` data model and a Qdrant `PointStruct`. + +```csharp +using Microsoft.Extensions.VectorData; +using Qdrant.Client.Grpc; + +public class ProductMapper : IVectorStoreRecordMapper +{ + public PointStruct MapFromDataToStorageModel(Product dataModel) + { + // Create a Qdrant PointStruct to map our data to. + var pointStruct = new PointStruct + { + Id = new PointId { Num = dataModel.Id }, + Vectors = new Vectors(), + Payload = { }, + }; + + // Add the data fields to the payload dictionary and serialize the product info into a json string. + pointStruct.Payload.Add("Name", dataModel.Name); + pointStruct.Payload.Add("Description", dataModel.Description); + pointStruct.Payload.Add("ProductInfoJson", JsonSerializer.Serialize(dataModel.ProductInfo)); + + // Add the vector fields to the vector dictionary. + var namedVectors = new NamedVectors(); + namedVectors.Vectors.Add("NameEmbedding", dataModel.Vectors.NameEmbedding.ToArray()); + namedVectors.Vectors.Add("DescriptionEmbedding", dataModel.Vectors.DescriptionEmbedding.ToArray()); + pointStruct.Vectors.Vectors_ = namedVectors; + + return pointStruct; + } + + public Product MapFromStorageToDataModel(PointStruct storageModel, StorageToDataModelMapperOptions options) + { + var product = new Product + { + Id = storageModel.Id.Num, + + // Retrieve the data fields from the payload dictionary and deserialize the product info from the json string that it was stored as. + Name = storageModel.Payload["Name"].StringValue, + Description = storageModel.Payload["Description"].StringValue, + ProductInfo = JsonSerializer.Deserialize(storageModel.Payload["ProductInfoJson"].StringValue)!, + + // Retrieve the vector fields from the vector dictionary. + Vectors = new ProductVectors + { + NameEmbedding = new ReadOnlyMemory(storageModel.Vectors.Vectors_.Vectors["NameEmbedding"].Data.ToArray()), + DescriptionEmbedding = new ReadOnlyMemory(storageModel.Vectors.Vectors_.Vectors["DescriptionEmbedding"].Data.ToArray()) + } + }; + + return product; + } +} +``` + +## Using your custom mapper with a record collection + +To use the custom mapper that we have created, we need to pass it to the record collection at construction time. +We also need to pass the record definition that we created earlier, so that collections are created in the +data store using the right schema. +One more setting that is important here, is Qdrant's named vectors mode, since we have more than one +vector. Without this mode switched on, only one vector is supported. + +```csharp +using Microsoft.SemanticKernel; +using Microsoft.SemanticKernel.Connectors.Qdrant; +using Qdrant.Client; + +var productMapper = new ProductMapper(); +var collection = new QdrantVectorStoreRecordCollection( + new QdrantClient("localhost"), + "skproducts", + new() + { + HasNamedVectors = true, + PointStructCustomMapper = productMapper, + VectorStoreRecordDefinition = productDefinition + }); +``` + +## Using a custom mapper with IVectorStore + +When using `IVectorStore` to get `IVectorStoreRecordCollection` object instances, it is not possible to provide a custom mapper directly to +the `GetCollection` method. This is because custom mappers differ for each Vector Store type, and would make it impossible to use `IVectorStore` +to communicate with any vector store implementation. + +It is however possible to provide a factory when constructing a Vector Store implementation. This can be used to customize `IVectorStoreRecordCollection` +instances as they are created. + +Here is an example of such a factory, which checks if `CreateCollection` was called with the product definition and data type, and if so +injects the custom mapper and switches on named vectors mode. + +```csharp +public class QdrantCollectionFactory(VectorStoreRecordDefinition productDefinition) : IQdrantVectorStoreRecordCollectionFactory +{ + public IVectorStoreRecordCollection CreateVectorStoreRecordCollection(QdrantClient qdrantClient, string name, VectorStoreRecordDefinition? vectorStoreRecordDefinition) + where TKey : notnull + where TRecord : class + { + // If the record definition is the product definition and the record type is the product data + // model, inject the custom mapper into the collection options. + if (vectorStoreRecordDefinition == productDefinition && typeof(TRecord) == typeof(Product)) + { + var customCollection = new QdrantVectorStoreRecordCollection( + qdrantClient, + name, + new() + { + HasNamedVectors = true, + PointStructCustomMapper = new ProductMapper(), + VectorStoreRecordDefinition = vectorStoreRecordDefinition + }) as IVectorStoreRecordCollection; + return customCollection!; + } + + // Otherwise, just create a standard collection with the default mapper. + var collection = new QdrantVectorStoreRecordCollection( + qdrantClient, + name, + new() + { + VectorStoreRecordDefinition = vectorStoreRecordDefinition + }) as IVectorStoreRecordCollection; + return collection!; + } +} +``` + +To use the collection factory, pass it to the Vector Store when constructing it, or when registering it with the dependency injection container. + +```csharp +// When registering with the dependency injection container on the kernel builder. +kernelBuilder.AddQdrantVectorStore( + "localhost", + options: new() + { + VectorStoreCollectionFactory = new QdrantCollectionFactory(productDefinition) + }); +``` + +```csharp +// When constructing the Vector Store instance directly. +var vectorStore = new QdrantVectorStore( + new QdrantClient("localhost"), + new() + { + VectorStoreCollectionFactory = new QdrantCollectionFactory(productDefinition) + }); +``` + +Now you can use the vector store as normal to get a collection. + +```csharp +var collection = vectorStore.GetCollection("skproducts", productDefinition); +``` + +::: zone-end +::: zone pivot="programming-language-python" + +## Coming Soon + +More info coming soon. + +::: zone-end +::: zone pivot="programming-language-java" + +## Coming Soon + +More info coming soon. + +::: zone-end diff --git a/semantic-kernel/concepts/vector-store-connectors/how-to/vector-store-data-ingestion.md b/semantic-kernel/concepts/vector-store-connectors/how-to/vector-store-data-ingestion.md new file mode 100644 index 00000000..be9d84d3 --- /dev/null +++ b/semantic-kernel/concepts/vector-store-connectors/how-to/vector-store-data-ingestion.md @@ -0,0 +1,300 @@ +--- +title: How to ingest data into a Semantic Kernel Vector Store (Preview) +description: Step by step instructions on how to ingest data into a Vector Store using Semantic Kernel +zone_pivot_groups: programming-languages +author: westey-m +ms.topic: tutorial +ms.author: westey +ms.date: 07/08/2024 +ms.service: semantic-kernel +--- +# How to ingest data into a Vector Store using Semantic Kernel (Preview) + +> [!WARNING] +> The Semantic Kernel Vector Store functionality is in preview, and improvements that require breaking changes may still occur in limited circumstances before release. + +This article will demonstrate how to create an application to + +1. Take text from each paragraph in a Microsoft Word document +2. Generate an embedding for each paragraph +3. Upsert the text, embedding and a reference to the original location into a Redis instance. + +## Prerequisites + +For this sample you will need + +1. An embedding generation model hosted in Azure or another provider of your choice. +2. An instance of Redis or Docker Desktop so that you can run Redis locally. +3. A Word document to parse and load. Here is a zip containing a sample Word document you can download and use: [vector-store-data-ingestion-input.zip](../../../media/vector-store-data-ingestion-input.zip). + +## Setup Redis + +If you already have a Redis instance you can use that. If you prefer to test your project locally +you can easily start a Redis container using docker. + +``` +docker run -d --name redis-stack -p 6379:6379 -p 8001:8001 redis/redis-stack:latest +``` + +To verify that it is running succesfully, visit [http://localhost:8001/redis-stack/browser](http://localhost:8001/redis-stack/browser) in your browser. + +The rest of these instructions will assume that you are using this container using the above settings. + +::: zone pivot="programming-language-csharp" + +## Create your project + +Create a new project and add nuget package references for the Redis connector from Semantic Kernel, the open xml package to read the word +document with and the OpenAI connector from Semantic Kernel for generating embeddings. + +```dotnetcli +dotnet new console --framework net8.0 --name SKVectorIngest +cd SKVectorIngest +dotnet add package Microsoft.SemanticKernel.Connectors.AzureOpenAI +dotnet add package Microsoft.SemanticKernel.Connectors.Redis --prerelease +dotnet add package DocumentFormat.OpenXml +``` + +## Add a data model + +To upload data we need to first describe what format the data should have in the database. +We can do this by creating a data model with attributes that describe the function of each property. + +Add a new file to the project called `TextParagraph.cs` and add the following model to it. + +```csharp +using Microsoft.Extensions.VectorData; + +namespace SKVectorIngest; + +internal class TextParagraph +{ + /// A unique key for the text paragraph. + [VectorStoreRecordKey] + public required string Key { get; init; } + + /// A uri that points at the original location of the document containing the text. + [VectorStoreRecordData] + public required string DocumentUri { get; init; } + + /// The id of the paragraph from the document containing the text. + [VectorStoreRecordData] + public required string ParagraphId { get; init; } + + /// The text of the paragraph. + [VectorStoreRecordData] + public required string Text { get; init; } + + /// The embedding generated from the Text. + [VectorStoreRecordVector(1536)] + public ReadOnlyMemory TextEmbedding { get; set; } +} +``` + +Note that we are passing the value `1536` to the `VectorStoreRecordVectorAttribute`. This is the dimension size of the vector and has to match the size of vector that your chosen embedding generator produces. + +> [!TIP] +> For more information on how to annotate your data model and what additional options are available for each attribute, refer to [definining your data model](../../../concepts/vector-store-connectors/defining-your-data-model.md). + +## Read the paragraphs in the document + +We need some code to read the word document and find the text of each paragraph in it. + +Add a new file to the project called `DocumentReader.cs` and add the following class to read the paragraphs from a document. + +```csharp +using System.Text; +using System.Xml; +using DocumentFormat.OpenXml.Packaging; + +namespace SKVectorIngest; + +internal class DocumentReader +{ + public static IEnumerable ReadParagraphs(Stream documentContents, string documentUri) + { + // Open the document. + using WordprocessingDocument wordDoc = WordprocessingDocument.Open(documentContents, false); + if (wordDoc.MainDocumentPart == null) + { + yield break; + } + + // Create an XmlDocument to hold the document contents and load the document contents into the XmlDocument. + XmlDocument xmlDoc = new XmlDocument(); + XmlNamespaceManager nsManager = new XmlNamespaceManager(xmlDoc.NameTable); + nsManager.AddNamespace("w", "http://schemas.openxmlformats.org/wordprocessingml/2006/main"); + nsManager.AddNamespace("w14", "http://schemas.microsoft.com/office/word/2010/wordml"); + + xmlDoc.Load(wordDoc.MainDocumentPart.GetStream()); + + // Select all paragraphs in the document and break if none found. + XmlNodeList? paragraphs = xmlDoc.SelectNodes("//w:p", nsManager); + if (paragraphs == null) + { + yield break; + } + + // Iterate over each paragraph. + foreach (XmlNode paragraph in paragraphs) + { + // Select all text nodes in the paragraph and continue if none found. + XmlNodeList? texts = paragraph.SelectNodes(".//w:t", nsManager); + if (texts == null) + { + continue; + } + + // Combine all non-empty text nodes into a single string. + var textBuilder = new StringBuilder(); + foreach (XmlNode text in texts) + { + if (!string.IsNullOrWhiteSpace(text.InnerText)) + { + textBuilder.Append(text.InnerText); + } + } + + // Yield a new TextParagraph if the combined text is not empty. + var combinedText = textBuilder.ToString(); + if (!string.IsNullOrWhiteSpace(combinedText)) + { + Console.WriteLine("Found paragraph:"); + Console.WriteLine(combinedText); + Console.WriteLine(); + + yield return new TextParagraph + { + Key = Guid.NewGuid().ToString(), + DocumentUri = documentUri, + ParagraphId = paragraph.Attributes?["w14:paraId"]?.Value ?? string.Empty, + Text = combinedText + }; + } + } + } +} +``` + +## Generate embeddings and upload the data + +We will need some code to generate embeddings and upload the paragraphs to Redis. Let's do this in a separate class. + +Add a new file called `DataUploader.cs` and add the following class to it. + +```csharp +#pragma warning disable SKEXP0001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. + +using Microsoft.Extensions.VectorData; +using Microsoft.SemanticKernel.Embeddings; + +namespace SKVectorIngest; + +internal class DataUploader(IVectorStore vectorStore, ITextEmbeddingGenerationService textEmbeddingGenerationService) +{ + /// + /// Generate an embedding for each text paragraph and upload it to the specified collection. + /// + /// The name of the collection to upload the text paragraphs to. + /// The text paragraphs to upload. + /// An async task. + public async Task GenerateEmbeddingsAndUpload(string collectionName, IEnumerable textParagraphs) + { + var collection = vectorStore.GetCollection(collectionName); + await collection.CreateCollectionIfNotExistsAsync(); + + foreach (var paragraph in textParagraphs) + { + // Generate the text embedding. + Console.WriteLine($"Generating embedding for paragraph: {paragraph.ParagraphId}"); + paragraph.TextEmbedding = await textEmbeddingGenerationService.GenerateEmbeddingAsync(paragraph.Text); + + // Upload the text paragraph. + Console.WriteLine($"Upserting paragraph: {paragraph.ParagraphId}"); + await collection.UpsertAsync(paragraph); + + Console.WriteLine(); + } + } +} +``` + +## Put it all together + +Finally, we need to put together the different pieces. +In this example, we will use the Semantic Kernel dependency injection container but it is also possible to use any `IServiceCollection` based container. + +Add the following code to your `Program.cs` file to create the container, register the Redis vector store and register the embedding service. +Make sure to replace the text embedding generation settings with your own values. + +```csharp +#pragma warning disable SKEXP0010 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. +#pragma warning disable SKEXP0020 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. + +using Microsoft.Extensions.DependencyInjection; +using Microsoft.SemanticKernel; +using SKVectorIngest; + +// Replace with your values. +var deploymentName = "text-embedding-ada-002"; +var endpoint = "https://sksample.openai.azure.com/"; +var apiKey = "your-api-key"; + +// Register Azure Open AI text embedding generation service and Redis vector store. +var builder = Kernel.CreateBuilder() + .AddAzureOpenAITextEmbeddingGeneration(deploymentName, endpoint, apiKey) + .AddRedisVectorStore("localhost:6379"); + +// Register the data uploader. +builder.Services.AddSingleton(); + +// Build the kernel and get the data uploader. +var kernel = builder.Build(); +var dataUploader = kernel.Services.GetRequiredService(); +``` + +As a last step, we want to read the paragraphs from our word document, and call the data uploader +to generate the embeddings and upload the paragraphs. + +```csharp +// Load the data. +var textParagraphs = DocumentReader.ReadParagraphs( + new FileStream( + "vector-store-data-ingestion-input.docx", + FileMode.Open), + "file:///c:/vector-store-data-ingestion-input.docx"); + +await dataUploader.GenerateEmbeddingsAndUpload( + "sk-documentation", + textParagraphs); +``` + +## See your data in Redis + +Navigate to the Redis stack browser, e.g. [http://localhost:8001/redis-stack/browser](http://localhost:8001/redis-stack/browser) where you should now be able to see +your uploaded paragraphs. Here is an example of what you should see for one of the uploaded paragraphs. + +```json +{ + "DocumentUri" : "file:///c:/vector-store-data-ingestion-input.docx", + "ParagraphId" : "14CA7304", + "Text" : "Version 1.0+ support across C#, Python, and Java means it’s reliable, committed to non breaking changes. Any existing chat-based APIs are easily expanded to support additional modalities like voice and video.", + "TextEmbedding" : [...] +} +``` + +::: zone-end +::: zone pivot="programming-language-python" + +## Coming soon + +Further instructions coming soon + +::: zone-end +::: zone pivot="programming-language-java" + +## Coming soon + +Further instructions coming soon + +::: zone-end diff --git a/semantic-kernel/concepts/vector-store-connectors/index.md b/semantic-kernel/concepts/vector-store-connectors/index.md new file mode 100644 index 00000000..7d71297b --- /dev/null +++ b/semantic-kernel/concepts/vector-store-connectors/index.md @@ -0,0 +1,299 @@ +--- +title: What are Semantic Kernel Vector Store connectors? (Preview) +description: Describes what a Semantic Kernal Vector Store is, an provides a basic example of how to use one and how to get started. +zone_pivot_groups: programming-languages +author: westey-m +ms.topic: conceptual +ms.author: westey +ms.date: 07/08/2024 +ms.service: semantic-kernel +--- +# What are Semantic Kernel Vector Store connectors? (Preview) + +> [!WARNING] +> The Semantic Kernel Vector Store functionality is in preview, and improvements that require breaking changes may still occur in limited circumstances before release. +> [!TIP] +> If you are looking for information about the legacy Memory Store connectors, refer to the [Memory Stores page](./memory-stores.md). + +Vector databases have many use cases across different domains and applications that involve natural language processing (NLP), computer vision (CV), recommendation systems (RS), and other areas that require semantic understanding and matching of data. + +One use case for storing information in a vector database is to enable large language models (LLMs) to generate more relevant and coherent responses. Large language models often face challenges such as generating inaccurate or irrelevant information; lacking factual consistency or common sense; repeating or contradicting themselves; being biased or offensive. To help overcome these challenges, you can use a vector database to store information about different topics, keywords, facts, opinions, and/or sources related to your desired domain or genre. The vector database allows you to efficiently find the subset of information related to a specific question or topic. You can then pass information from the vector database with your prompt to your large language model to generate more accurate and relevant content. + +For example, if you want to write a blog post about the latest trends in AI, you can use a vector database to store the latest information about that topic and pass the information along with the ask to a LLM in order to generate a blog post that leverages the latest information. + +Semantic Kernel and .net provides an abstraction for interacting with Vector Stores and a list of out-of-the-box connectors that implement these abstractions. Features include creating, listing and deleting collections of records, and uploading, retrieving and deleting records. The abstraction makes it easy to experiment with a free or locally hosted Vector Store and then switch to a service when needing to scale up. + +::: zone pivot="programming-language-csharp" + +## The Vector Store Abstraction + +The main interfaces in the Vector Store abstraction are the following. + +### Microsoft.Extensions.VectorData.IVectorStore + +`IVectorStore` contains operations that spans across all collections in the vector store, e.g. ListCollectionNames. +It also provides the ability to get `IVectorStoreRecordCollection` instances. + +### Microsoft.Extensions.VectorData.IVectorStoreRecordCollection\ + +`IVectorStoreRecordCollection` represents a collection. +This collection may or may not exist, and the interface provides methods to check if the collection exists, create it or delete it. +The interface also provides methods to upsert, get and delete records. +Finally, the interface inherits from `IVectorizedSearch` providing vector search capabilities. + +### Microsoft.Extensions.VectorData.IVectorizedSearch\ + +`IVectorizedSearch` contains a method for doing vector searches. +`IVectorStoreRecordCollection` inherits from `IVectorizedSearch` making it possible to use +`IVectorizedSearch` on its own in cases where only search is needed and no record or collection management is needed. + +### IVectorizableTextSearch\ + +`IVectorizableTextSearch` contains a method for doing vector searches where the vector database has the ability to +generate embeddings automatically. E.g. you can call this method with a text string and the database will generate the embedding +for you and search against a vector field. This is not supported by all vector databases and is therefore only implemented +by select connectors. + +::: zone-end +::: zone pivot="programming-language-python" +::: zone-end +::: zone pivot="programming-language-java" +::: zone-end + +## Getting started with Vector Store connectors + +::: zone pivot="programming-language-csharp" + +### Import the necessary nuget packages + +All the vector store interfaces and any abstraction related classes are available in the `Microsoft.Extensions.VectorData.Abstractions` nuget package. +Each vector store implementation is available in its own nuget package. For a list of known implementations, see the [Out-of-the-box connectors page](./out-of-the-box-connectors/index.md). + +The abstractions package can be added like this. + +```dotnetcli +dotnet add package Microsoft.Extensions.VectorData.Abstractions --prerelease +``` + +> [!WARNING] +> From version 1.23.0 of Semantic Kernel, the Vector Store abstractions have been removed from `Microsoft.SemanticKernel.Abstractions` +> and are available in the new dedicated `Microsoft.Extensions.VectorData.Abstractions` package. +> +> Note that from version 1.23.0, `Microsoft.SemanticKernel.Abstractions` has a dependency on `Microsoft.Extensions.VectorData.Abstractions`, +> therefore there is no need to reference additional packages. +> The abstractions will however now be in the new `Microsoft.Extensions.VectorData` namespace. +> +> When upgrading from 1.22.0 or earlier to 1.23.0 or later, you will need to add an additional `using Microsoft.Extensions.VectorData;` +> clause in files where any of the Vector Store abstraction types are used e.g. `IVectorStore`, `IVectorStoreRecordCollection`, `VectorStoreRecordDataAttribute`, `VectorStoreRecordKeyProperty`, etc. +> +> This change has been made to support vector store providers when creating their own implementations. A provider only has to reference +> the `Microsoft.Extensions.VectorData.Abstractions` package. This reduces potential version conflicts and allows Semantic Kernel +> to continue to evolve fast without impacting vector store providers. + +::: zone-end +::: zone pivot="programming-language-python" +::: zone-end +::: zone pivot="programming-language-java" +::: zone-end + +### Define your data model + +The Semantic Kernel Vector Store connectors use a model first approach to interacting with databases. This means that the first step is to define a data model that maps to the storage schema. To help the connectors create collections of records and map to the storage schema, the model can be annotated to indicate the function of each property. + +::: zone pivot="programming-language-csharp" + +```csharp +using Microsoft.Extensions.VectorData; + +public class Hotel +{ + [VectorStoreRecordKey] + public ulong HotelId { get; set; } + + [VectorStoreRecordData(IsFilterable = true)] + public string HotelName { get; set; } + + [VectorStoreRecordData(IsFullTextSearchable = true)] + public string Description { get; set; } + + [VectorStoreRecordVector(Dimensions: 4, DistanceFunction.CosineDistance, IndexKind.Hnsw)] + public ReadOnlyMemory? DescriptionEmbedding { get; set; } + + [VectorStoreRecordData(IsFilterable = true)] + public string[] Tags { get; set; } +} +``` + +::: zone-end +::: zone pivot="programming-language-python" +```python +from dataclasses import dataclass, field +from typing import Annotated +from semantic_kernel.data import ( + DistanceFunction, + IndexKind, + VectorStoreRecordDataField, + VectorStoreRecordDefinition, + VectorStoreRecordKeyField, + VectorStoreRecordVectorField, + vectorstoremodel, +) + +@vectorstoremodel +@dataclass +class Hotel: + hotel_id: Annotated[str, VectorStoreRecordKeyField()] = field(default_factory=lambda: str(uuid4())) + hotel_name: Annotated[str, VectorStoreRecordDataField(is_filterable=True)] + description: Annotated[str, VectorStoreRecordDataField(is_full_text_searchable=True)] + description_embedding: Annotated[list[float], VectorStoreRecordVectorField(dimensions=4, distance_function=DistanceFunction.COSINE, index_kind=IndexKind.HNSW)] + tags: Annotated[list[str], VectorStoreRecordDataField(is_filterable=True)] +``` +::: zone-end +::: zone pivot="programming-language-java" +::: zone-end + +> [!TIP] +> For more information on how to annotate your data model, refer to [defining your data model](./defining-your-data-model.md). +> [!TIP] +> For an alternative to annotating your data model, refer to [defining your schema with a record definition](./schema-with-record-definition.md). + +### Connect to your database and select a collection + +Once you have defined your data model, the next step is to create a VectorStore instance for the database of your choice and select a collection of records. + +::: zone pivot="programming-language-csharp" + +In this example, we'll use Qdrant. You will therefore need to import the Qdrant nuget package. + +```dotnetcli +dotnet add package Microsoft.SemanticKernel.Connectors.Qdrant --prerelease +``` + +Since databases support many different types of keys and records, we allow you to specify the type of the key and record for your collection using generics. +In our case, the type of record will be the `Hotel` class we already defined, and the type of key will be `ulong`, since the `HotelId` property is a `ulong` and Qdrant only supports `Guid` or `ulong` keys. + +```csharp +using Microsoft.SemanticKernel.Connectors.Qdrant; +using Qdrant.Client; + +// Create a Qdrant VectorStore object +var vectorStore = new QdrantVectorStore(new QdrantClient("localhost")); + +// Choose a collection from the database and specify the type of key and record stored in it via Generic parameters. +var collection = vectorStore.GetCollection("skhotels"); +``` + +::: zone-end +::: zone pivot="programming-language-python" + +Since databases support many different types of keys and records, we allow you to specify the type of the key and record for your collection using generics. +In our case, the type of record will be the `Hotel` class we already defined, and the type of key will be `str`, since the `HotelId` property is a `str` and Qdrant only supports `str` or `int` keys. + +```python +from semantic_kernel.connectors.memory.qdrant import QdrantStore + +# Create a Qdrant VectorStore object, this will look in the environment for Qdrant related settings, and will fall back to the default, which is to run in-memory. +vector_store = QdrantStore() + +# Choose a collection from the database and specify the type of key and record stored in it via Generic parameters. +collection = vector_store.get_collection( + collection_name="skhotels", + data_model_type=Hotel +) +``` +::: zone-end +::: zone pivot="programming-language-java" +::: zone-end + +> [!TIP] +> For more information on what key and field types each Vector Store connector supports, refer to [the documentation for each connector](./out-of-the-box-connectors/index.md). + +::: zone pivot="programming-language-csharp" + +### Create the collection and add records + +```csharp +// Create the collection if it doesn't exist yet. +await collection.CreateCollectionIfNotExistsAsync(); + +// Upsert a record. +string descriptionText = "A place where everyone can be happy."; +ulong hotelId = 1; + +// Create a record and generate a vector for the description using your chosen embedding generation implementation. +// Just showing a placeholder embedding generation method here for brevity. +await collection.UpsertAsync(new Hotel +{ + HotelId = hotelId, + HotelName = "Hotel Happy", + Description = descriptionText, + DescriptionEmbedding = await GenerateEmbeddingAsync(descriptionText), + Tags = new[] { "luxury", "pool" } +}); + +// Retrieve the upserted record. +Hotel? retrievedHotel = await collection.GetAsync(hotelId); +``` + +::: zone-end +::: zone pivot="programming-language-python" + +### Create the collection and add records + +```python +# Create the collection if it doesn't exist yet. +await collection.create_collection_if_not_exists() + +# Upsert a record. +description = "A place where everyone can be happy." +hotel_id = "1" + +await collection.upsert(Hotel( + hotel_id = hotel_id, + hotel_name = "Hotel Happy", + description = description, + description_embedding = await GenerateEmbeddingAsync(description), + tags = ["luxury", "pool"] +)) + +# Retrieve the upserted record. +retrieved_hotel = await collection.get(hotel_id) +``` + +::: zone-end +::: zone pivot="programming-language-java" +::: zone-end + +::: zone pivot="programming-language-csharp" + +> [!TIP] +> For more information on how to generate embeddings see [embedding generation](./embedding-generation.md). + +### Do a vector search + +```csharp +// Generate a vector for your search text, using your chosen embedding generation implementation. +// Just showing a placeholder method here for brevity. +var searchVector = await GenerateEmbeddingAsync("I'm looking for a hotel where customer happiness is the priority."); +// Do the search. +var searchResult = await collection.VectorizedSearchAsync(searchVector, new() { Top = 1 }).Results.ToListAsync() + +// Inspect the returned hotels. +Hotel hotel = searchResult.First().Record; +Console.WriteLine("Found hotel description: " + hotel.Description); +``` + +::: zone-end +::: zone pivot="programming-language-python" +::: zone-end +::: zone pivot="programming-language-java" +::: zone-end + +> [!TIP] +> For more information on how to generate embeddings see [embedding generation](./embedding-generation.md). + +## Next steps + +> [!div class="nextstepaction"] +> [Learn about the Vector Store data architecture](./data-architecture.md) +> [How to ingest data into a Vector Store](./how-to/vector-store-data-ingestion.md) diff --git a/semantic-kernel/concepts/vector-store-connectors/memory-stores.md b/semantic-kernel/concepts/vector-store-connectors/memory-stores.md new file mode 100644 index 00000000..6e646944 --- /dev/null +++ b/semantic-kernel/concepts/vector-store-connectors/memory-stores.md @@ -0,0 +1,137 @@ +--- +title: Legacy Semantic Kernel Memory Stores +description: Describes the legacy Semantic Kernel Memory Stores and the benefits of moving to Vector Stores +zone_pivot_groups: programming-languages +author: westey-m +ms.topic: conceptual +ms.author: westey +ms.date: 10/15/2024 +ms.service: semantic-kernel +--- +# Legacy Semantic Kernel Memory Stores + +> [!TIP] +> We recommend using the Vector Store abstractions instead of the legacy Memory Stores. For more information on how to use the Vector Store abstractions start [here](./index.md). + +Semantic Kernel provides a set of Memory Store abstractions where the primary interface is `Microsoft.SemanticKernel.Memory.IMemoryStore`. + +## Memory Store vs Vector Store abstractions + +As part of an effort to evolve and expand the vector storage and search capbilities of Semantic Kernel, we have released a new set of abstractions to replace the Memory Store abstractions. +We are calling the replacement abstractions Vector Store abstractions. +The purpose of both are similar, but their interfaces differ and the Vector Store abstractions provide expanded functionality. + +::: zone pivot="programming-language-csharp" + +|Characteristic|Legacy Memory Stores|Vector Stores| +|-|-|-| +|Main Interface|IMemoryStore|IVectorStore| +|Abstractions nuget package|Microsoft.SemanticKernel.Abstractions|Microsoft.Extensions.VectorData.Abstractions| +|Naming Convention|{Provider}MemoryStore, e.g. RedisMemoryStore|{Provider}VectorStore, e.g. RedisVectorStore| +|Supports record upsert, get and delete|Yes|Yes| +|Supports collection create and delete|Yes|Yes| +|Supports vector search|Yes|Yes| +|Supports choosing your preferred vector search index and distance function|No|Yes| +|Supports multiple vectors per record|No|Yes| +|Supports custom schemas|No|Yes| +|Supports metadata pre-filtering for vector search|No|Yes| +|Supports vector search on non-vector databases by downloading the entire dataset onto the client and doing a local vector search|Yes|No| + +::: zone-end +::: zone pivot="programming-language-python" +::: zone-end +::: zone pivot="programming-language-java" +::: zone-end + +## Available Memory Store connectors + +Semantic Kernel offers several Memory Store connectors to vector databases that you can use to store and retrieve information. These include: + +| Service | C# | Python | +|--------------------------|:----:|:------:| +| Vector Database in Azure Cosmos DB for NoSQL | [C#](https://github.com/microsoft/semantic-kernel/tree/main/dotnet/src/Connectors/Connectors.Memory.AzureCosmosDBNoSQL) | [Python](https://github.com/microsoft/semantic-kernel/tree/main/python/semantic_kernel/connectors/memory/azure_cosmosdb_no_sql) +| Vector Database in vCore-based Azure Cosmos DB for MongoDB | [C#](https://github.com/microsoft/semantic-kernel/tree/main/dotnet/src/Connectors/Connectors.Memory.AzureCosmosDBMongoDB) | [Python](https://github.com/microsoft/semantic-kernel/tree/main/python/semantic_kernel/connectors/memory/azure_cosmosdb) | +| Azure AI Search | [C#](https://github.com/microsoft/semantic-kernel/tree/main/dotnet/src/Connectors/Connectors.Memory.AzureAISearch) | [Python](https://github.com/microsoft/semantic-kernel/tree/main/python/semantic_kernel/connectors/memory/azure_cognitive_search) | +| Azure PostgreSQL Server | [C#](https://github.com/microsoft/semantic-kernel/tree/main/dotnet/src/Connectors/Connectors.Memory.Postgres) | +| Azure SQL Database | [C#](https://github.com/kbeaugrand/SemanticKernel.Connectors.Memory.SqlServer) | +| Chroma | [C#](https://github.com/microsoft/semantic-kernel/tree/main/dotnet/src/Connectors/Connectors.Memory.Chroma) | [Python](https://github.com/microsoft/semantic-kernel/tree/main/python/semantic_kernel/connectors/memory/chroma) | +| DuckDB | [C#](https://github.com/microsoft/semantic-kernel/tree/main/dotnet/src/Connectors/Connectors.Memory.DuckDB) | | +| Milvus | [C#](https://github.com/microsoft/semantic-kernel/tree/main/dotnet/src/Connectors/Connectors.Memory.Milvus) | [Python](https://github.com/microsoft/semantic-kernel/tree/main/python/semantic_kernel/connectors/memory/milvus) | +| MongoDB Atlas Vector Search | [C#](https://github.com/microsoft/semantic-kernel/tree/main/dotnet/src/Connectors/Connectors.Memory.MongoDB) | [Python](https://github.com/microsoft/semantic-kernel/tree/main/python/semantic_kernel/connectors/memory/mongodb_atlas) | +| Pinecone | [C#](https://github.com/microsoft/semantic-kernel/tree/main/dotnet/src/Connectors/Connectors.Memory.Pinecone) | [Python](https://github.com/microsoft/semantic-kernel/tree/main/python/semantic_kernel/connectors/memory/pinecone) | +| Postgres | [C#](https://github.com/microsoft/semantic-kernel/tree/main/dotnet/src/Connectors/Connectors.Memory.Postgres) | [Python](https://github.com/microsoft/semantic-kernel/tree/main/python/semantic_kernel/connectors/memory/postgres) | +| Qdrant | [C#](https://github.com/microsoft/semantic-kernel/tree/main/dotnet/src/Connectors/Connectors.Memory.Qdrant) | | +| Redis | [C#](https://github.com/microsoft/semantic-kernel/tree/main/dotnet/src/Connectors/Connectors.Memory.Redis) | | +| Sqlite | [C#](https://github.com/microsoft/semantic-kernel/tree/main/dotnet/src/Connectors/Connectors.Memory.Sqlite) | | +| Weaviate | [C#](https://github.com/microsoft/semantic-kernel/tree/main/dotnet/src/Connectors/Connectors.Memory.Weaviate) | [Python](https://github.com/microsoft/semantic-kernel/tree/main/python/semantic_kernel/connectors/memory/weaviate) | + +::: zone pivot="programming-language-csharp" + +## Migrating from Memory Stores to Vector Stores + +If you wanted to migrate from using the Memory Store abstractions to the Vector Store abtractions there are various ways in which you can do this. + +### Use the existing collection with the Vector Store abstractions + +The simplest way in many cases could be to just use the Vector Store abstractions to access a collection that was created using the Memory Store abstractions. +In many cases this is possible, since the Vector Store abstraction allows you to choose the schema that you would like to use. +The main requirement is to create a data model that matches the schema that the legacy Memory Store implementation used. + +E.g. to access a collection created by the Azure AI Search Memory Store, you can use the following Vector Store data model. + +```csharp +using Microsoft.Extensions.VectorData; + +class VectorStoreRecord +{ + [VectorStoreRecordKey] + public string Id { get; set; } + + [VectorStoreRecordData] + public string Description { get; set; } + + [VectorStoreRecordData] + public string Text { get; set; } + + [VectorStoreRecordData] + public bool IsReference { get; set; } + + [VectorStoreRecordData] + public string ExternalSourceName { get; set; } + + [VectorStoreRecordData] + public string AdditionalMetadata { get; set; } + + [VectorStoreRecordVector(VectorSize)] + public ReadOnlyMemory Embedding { get; set; } +} +``` + +> [!TIP] +> For more detailed examples on how to use the Vector Store abstractions to access collections created using a Memory Store, see [here](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/Memory/VectorStore_ConsumeFromMemoryStore_Common.cs). + +### Create a new collection + +In some cases migrating to a new collection may be preferable than using the existing collection directly. The schema that was chosen by the Memory Store may not match your requirements, especially with regards to filtering. + +E.g. The Redis Memory store uses a schema with three fields: + +- string metadata +- long timestamp +- float[] embedding + +All data other than the embedding or timestamp is stored as a serialized json string in the Metadata field. This means that it is not possible to index the individual values and filter on them. +E.g. perhaps you may want to filter using the ExternalSourceName, but this is not possible while it is inside a json string. + +In this case, it may be better to migrate the data to a new collection with a flat schema. +There are two options here. You could create a new collection from your source data or simply map and copy the data from the old to the new. +The first option may be more costly as you will need to regenerate the embeddings from the source data. + +> [!TIP] +> For an example using Redis showing how to copy data from a collection created using the Memory Store abstractions to one created using the Vector Store abstractions see [here](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/Memory/VectorStore_MigrateFromMemoryStore_Redis.cs). + +::: zone-end +::: zone pivot="programming-language-python" +::: zone-end +::: zone pivot="programming-language-java" +::: zone-end diff --git a/semantic-kernel/concepts/vector-store-connectors/out-of-the-box-connectors/TOC.yml b/semantic-kernel/concepts/vector-store-connectors/out-of-the-box-connectors/TOC.yml new file mode 100644 index 00000000..bec70816 --- /dev/null +++ b/semantic-kernel/concepts/vector-store-connectors/out-of-the-box-connectors/TOC.yml @@ -0,0 +1,20 @@ +- name: Overview + href: index.md +- name: Azure AI Search connector + href: azure-ai-search-connector.md +- name: Azure CosmosDB MongoDB connector + href: azure-cosmosdb-mongodb-connector.md +- name: Azure CosmosDB NoSQL connector + href: azure-cosmosdb-nosql-connector.md +- name: In-memory connector + href: inmemory-connector.md +- name: Pinecone connector + href: pinecone-connector.md +- name: Qdrant connector + href: qdrant-connector.md +- name: Redis connector + href: redis-connector.md +- name: Volatile (in-memory) connector + href: volatile-connector.md +- name: Weaviate connector + href: weaviate-connector.md diff --git a/semantic-kernel/concepts/vector-store-connectors/out-of-the-box-connectors/azure-ai-search-connector.md b/semantic-kernel/concepts/vector-store-connectors/out-of-the-box-connectors/azure-ai-search-connector.md new file mode 100644 index 00000000..e489842a --- /dev/null +++ b/semantic-kernel/concepts/vector-store-connectors/out-of-the-box-connectors/azure-ai-search-connector.md @@ -0,0 +1,205 @@ +--- +title: Using the Semantic Kernel Azure AI Search Vector Store connector (Preview) +description: Contains information on how to use a Semantic Kernel Vector store connector to access and manipulate data in Azure AI Search. +zone_pivot_groups: programming-languages +author: westey-m +ms.topic: conceptual +ms.author: westey +ms.date: 07/08/2024 +ms.service: semantic-kernel +--- +# Using the Azure AI Search Vector Store connector (Preview) + +> [!WARNING] +> The Semantic Kernel Vector Store functionality is in preview, and improvements that require breaking changes may still occur in limited circumstances before release. + +## Overview + +The Azure AI Search Vector Store connector can be used to access and manage data in Azure AI Search. The connector has the following characteristics. + +| Feature Area | Support | +|-----------------------------------|----------------------------------------------------------------------------------------------------------------------------------| +| Collection maps to | Azure AI Search Index | +| Supported key property types | string | +| Supported data property types |
      • string
      • int
      • long
      • double
      • float
      • bool
      • DateTimeOffset
      • *and enumerables of each of these types*
      | +| Supported vector property types | ReadOnlyMemory\ | +| Supported index types |
      • Hnsw
      • Flat
      | +| Supported distance functions |
      • CosineSimilarity
      • DotProductSimilarity
      • EuclideanDistance
      | +| Supports multiple vectors in a record | Yes | +| IsFilterable supported? | Yes | +| IsFullTextSearchable supported? | Yes | +| StoragePropertyName supported? | No, use `JsonSerializerOptions` and `JsonPropertyNameAttribute` instead. [See here for more info.](#data-mapping) | + +## Limitations + +Notable Azure AI Search connector functionality limitations. + +| Feature Area | Workaround | +|--------------------------------------------------------------------------------------| -----------------------------------------------------------------------------------------------| +| Configuring full text search analyzers during collection creation is not supported. | Use the Azure AI Search Client SDK directly for collection creation | + +::: zone pivot="programming-language-csharp" + +## Getting started + +Add the Azure AI Search Vector Store connector NuGet package to your project. + +```dotnetcli +dotnet add package Microsoft.SemanticKernel.Connectors.AzureAISearch --prerelease +``` + +You can add the vector store to the dependency injection container available on the `KernelBuilder` or to the `IServiceCollection` dependency injection container using extension methods provided by Semantic Kernel. + +```csharp +using Azure; +using Microsoft.SemanticKernel; + +// Using Kernel Builder. +var kernelBuilder = Kernel + .CreateBuilder() + .AddAzureAISearchVectorStore(new Uri(azureAISearchUri), new AzureKeyCredential(secret)); +``` + +```csharp +using Azure; +using Microsoft.SemanticKernel; + +// Using IServiceCollection with ASP.NET Core. +var builder = WebApplication.CreateBuilder(args); +builder.Services.AddAzureAISearchVectorStore(new Uri(azureAISearchUri), new AzureKeyCredential(secret)); +``` + +Extension methods that take no parameters are also provided. These require an instance of the Azure AI Search `SearchIndexClient` to be separately registered with the dependency injection container. + +```csharp +using Azure; +using Azure.Search.Documents.Indexes; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.SemanticKernel; + +// Using Kernel Builder. +var kernelBuilder = Kernel.CreateBuilder(); +kernelBuilder.Services.AddSingleton( + sp => new SearchIndexClient( + new Uri(azureAISearchUri), + new AzureKeyCredential(secret))); +kernelBuilder.AddAzureAISearchVectorStore(); +``` + +```csharp +using Azure; +using Azure.Search.Documents.Indexes; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.SemanticKernel; + +// Using IServiceCollection with ASP.NET Core. +var builder = WebApplication.CreateBuilder(args); +builder.Services.AddSingleton( + sp => new SearchIndexClient( + new Uri(azureAISearchUri), + new AzureKeyCredential(secret))); +builder.Services.AddAzureAISearchVectorStore(); +``` + +You can construct an Azure AI Search Vector Store instance directly. + +```csharp +using Azure; +using Azure.Search.Documents.Indexes; +using Microsoft.SemanticKernel.Connectors.AzureAISearch; + +var vectorStore = new AzureAISearchVectorStore( + new SearchIndexClient( + new Uri(azureAISearchUri), + new AzureKeyCredential(secret))); +``` + +It is possible to construct a direct reference to a named collection. + +```csharp +using Azure; +using Azure.Search.Documents.Indexes; +using Microsoft.SemanticKernel.Connectors.AzureAISearch; + +var collection = new AzureAISearchVectorStoreRecordCollection( + new SearchIndexClient(new Uri(azureAISearchUri), new AzureKeyCredential(secret)), + "skhotels"); +``` + +::: zone-end +::: zone pivot="programming-language-python" + +## Getting started + +Install semantic kernel with the azure extras, which includes the Azure AI Search SDK. + +```cli +pip install semantic-kernel[azure] +``` + +You can then create a vector store instance using the `AzureAISearchStore` class, this will use the environment variables `AZURE_AI_SEARCH_ENDPOINT` and `AZURE_AI_SEARCH_API_KEY` to connect to the Azure AI Search instance, those values can also be supplied directly. You can also supply azure credentials or token credentials instead of a API key. + +```python + +from semantic_kernel.connectors.memory.azure_ai_search import AzureAISearchStore + +vector_store = AzureAISearchStore() +``` + +You can also create the vector store with your own instance of the azure search client. + +```python +from azure.search.documents.indexes import SearchIndexClient +from semantic_kernel.connectors.memory.azure_ai_search import AzureAISearchStore + +search_client = SearchIndexClient(endpoint="https://.search.windows.net", credential="") +vector_store = AzureAISearchStore(search_index_client=search_client) +``` + +You can also create a collection directly. + +```python +from semantic_kernel.connectors.memory.azure_ai_search import AzureAISearchCollection + +collection = AzureAISearchCollection(collection_name="skhotels", data_model_type=hotel) +``` + +## Serialization + +Since the Azure AI Search connector needs a simple dict with the fields corresponding to the index as the input, the serialization is quite easy, just return a dict with the values with the keys corresponding to the index fields, the built-in step from dict to the the store model is a straight passthrough of the created dict. + +For more details on this concept see the [serialization documentation](./../serialization.md). + +::: zone-end +::: zone pivot="programming-language-java" +::: zone-end + +::: zone pivot="programming-language-csharp" + +## Data mapping + +The default mapper used by the Azure AI Search connector when mapping data from the data model to storage is the one provided by the Azure AI Search SDK. + +This mapper does a direct conversion of the list of properties on the data model to the fields in Azure AI Search and uses `System.Text.Json.JsonSerializer` +to convert to the storage schema. This means that usage of the `JsonPropertyNameAttribute` is supported if a different storage name to the +data model property name is required. + +It is also possible to use a custom `JsonSerializerOptions` instance with a customized property naming policy. To enable this, the `JsonSerializerOptions` +must be passed to both the `SearchIndexClient` and the `AzureAISearchVectorStoreRecordCollection` on construction. + +```csharp +var jsonSerializerOptions = new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseUpper }; +var collection = new AzureAISearchVectorStoreRecordCollection( + new SearchIndexClient( + new Uri(azureAISearchUri), + new AzureKeyCredential(secret), + new() { Serializer = new JsonObjectSerializer(jsonSerializerOptions) }), + "skhotels", + new() { JsonSerializerOptions = jsonSerializerOptions }); +``` + +::: zone-end +::: zone pivot="programming-language-python" +::: zone-end +::: zone pivot="programming-language-java" +::: zone-end diff --git a/semantic-kernel/concepts/vector-store-connectors/out-of-the-box-connectors/azure-cosmosdb-mongodb-connector.md b/semantic-kernel/concepts/vector-store-connectors/out-of-the-box-connectors/azure-cosmosdb-mongodb-connector.md new file mode 100644 index 00000000..40b58201 --- /dev/null +++ b/semantic-kernel/concepts/vector-store-connectors/out-of-the-box-connectors/azure-cosmosdb-mongodb-connector.md @@ -0,0 +1,174 @@ +--- +title: Using the Semantic Kernel Azure CosmosDB MongoDB Vector Store connector (Preview) +description: Contains information on how to use a Semantic Kernel Vector store connector to access and manipulate data in Azure CosmosDB MongoDB. +zone_pivot_groups: programming-languages +author: westey-m +ms.topic: conceptual +ms.author: westey +ms.date: 09/23/2024 +ms.service: semantic-kernel +--- +# Using the Azure CosmosDB MongoDB Vector Store connector (Preview) + +> [!WARNING] +> The Semantic Kernel Vector Store functionality is in preview, and improvements that require breaking changes may still occur in limited circumstances before release. + +::: zone pivot="programming-language-csharp" + +## Overview + +The Azure CosmosDB MongoDB Vector Store connector can be used to access and manage data in Azure CosmosDB MongoDB. The connector has the following characteristics. + +| Feature Area | Support | +|-----------------------------------|----------------------------------------------------------------------------------------------------------------------------------| +| Collection maps to | Azure Cosmos DB MongoDB Collection + Index | +| Supported key property types | string | +| Supported data property types |
      • string
      • int
      • long
      • double
      • float
      • decimal
      • bool
      • DateTime
      • *and enumerables of each of these types*
      | +| Supported vector property types |
      • ReadOnlyMemory\
      • ReadOnlyMemory\
      | +| Supported index types |
      • Hnsw
      • IvfFlat
      | +| Supported distance functions |
      • CosineDistance
      • DotProductSimilarity
      • EuclideanDistance
      | +| Supports multiple vectors in a record | Yes | +| IsFilterable supported? | Yes | +| IsFullTextSearchable supported? | No | +| StoragePropertyName supported? | No, use BsonElementAttribute instead. [See here for more info.](#data-mapping) | + +## Getting started + +Add the Azure CosmosDB MongoDB Vector Store connector NuGet package to your project. + +```dotnetcli +dotnet add package Microsoft.SemanticKernel.Connectors.AzureCosmosDBMongoDB --prerelease +``` + +You can add the vector store to the dependency injection container available on the `KernelBuilder` or to the `IServiceCollection` dependency injection container using extension methods provided by Semantic Kernel. + +```csharp +using Microsoft.SemanticKernel; + +// Using Kernel Builder. +var kernelBuilder = Kernel + .CreateBuilder() + .AddAzureCosmosDBMongoDBVectorStore(connectionString, databaseName); +``` + +```csharp +using Microsoft.SemanticKernel; + +// Using IServiceCollection with ASP.NET Core. +var builder = WebApplication.CreateBuilder(args); +builder.Services.AddAzureCosmosDBMongoDBVectorStore(connectionString, databaseName); +``` + +Extension methods that take no parameters are also provided. These require an instance of `MongoDB.Driver.IMongoDatabase` to be separately registered with the dependency injection container. + +```csharp +using Microsoft.Extensions.DependencyInjection; +using Microsoft.SemanticKernel; +using MongoDB.Driver; + +// Using Kernel Builder. +var kernelBuilder = Kernel.CreateBuilder(); +kernelBuilder.Services.AddSingleton( + sp => + { + var mongoClient = new MongoClient(connectionString); + return mongoClient.GetDatabase(databaseName); + }); +kernelBuilder.AddAzureCosmosDBMongoDBVectorStore(); +``` + +```csharp +using Microsoft.Extensions.DependencyInjection; +using Microsoft.SemanticKernel; +using MongoDB.Driver; + +// Using IServiceCollection with ASP.NET Core. +var builder = WebApplication.CreateBuilder(args); +builder.Services.AddSingleton( + sp => + { + var mongoClient = new MongoClient(connectionString); + return mongoClient.GetDatabase(databaseName); + }); +builder.Services.AddAzureCosmosDBMongoDBVectorStore(); +``` + +You can construct an Azure CosmosDB MongoDB Vector Store instance directly. + +```csharp +using Microsoft.SemanticKernel.Connectors.AzureCosmosDBMongoDB; +using MongoDB.Driver; + +var mongoClient = new MongoClient(connectionString); +var database = mongoClient.GetDatabase(databaseName); +var vectorStore = new AzureCosmosDBMongoDBVectorStore(database); +``` + +It is possible to construct a direct reference to a named collection. + +```csharp +using Microsoft.SemanticKernel.Connectors.AzureCosmosDBMongoDB; +using MongoDB.Driver; + +var mongoClient = new MongoClient(connectionString); +var database = mongoClient.GetDatabase(databaseName); +var collection = new AzureCosmosDBMongoDBVectorStoreRecordCollection( + database, + "skhotels"); +``` + +## Data mapping + +The Azure CosmosDB MognoDB Vector Store connector provides a default mapper when mapping data from the data model to storage. + +This mapper does a direct conversion of the list of properties on the data model to the fields in Azure CosmosDB MongoDB and uses `MongoDB.Bson.Serialization` +to convert to the storage schema. This means that usage of the `MongoDB.Bson.Serialization.Attributes.BsonElement` is supported if a different storage name to the +data model property name is required. The only exception is the key of the record which is mapped to a database field named `_id`, since all CosmosDB MongoDB +records must use this name for ids. + +### Property name override + +For data properties and vector properties, you can provide override field names to use in storage that is different to the +property names on the data model. This is not supported for keys, since a key has a fixed name in MongoDB. + +The property name override is done by setting the `BsonElement` attribute on the data model properties. + +Here is an example of a data model with `BsonElement` set. + +```csharp +using Microsoft.Extensions.VectorData; + +public class Hotel +{ + [VectorStoreRecordKey] + public ulong HotelId { get; set; } + + [BsonElement("hotel_name")] + [VectorStoreRecordData(IsFilterable = true)] + public string HotelName { get; set; } + + [BsonElement("hotel_description")] + [VectorStoreRecordData(IsFullTextSearchable = true)] + public string Description { get; set; } + + [BsonElement("hotel_description_embedding")] + [VectorStoreRecordVector(4, DistanceFunction.CosineDistance, IndexKind.Hnsw)] + public ReadOnlyMemory? DescriptionEmbedding { get; set; } +} +``` + +::: zone-end +::: zone pivot="programming-language-python" + +## Coming soon + +More info coming soon. + +::: zone-end +::: zone pivot="programming-language-java" + +## Coming soon + +More info coming soon. + +::: zone-end diff --git a/semantic-kernel/concepts/vector-store-connectors/out-of-the-box-connectors/azure-cosmosdb-nosql-connector.md b/semantic-kernel/concepts/vector-store-connectors/out-of-the-box-connectors/azure-cosmosdb-nosql-connector.md new file mode 100644 index 00000000..962855db --- /dev/null +++ b/semantic-kernel/concepts/vector-store-connectors/out-of-the-box-connectors/azure-cosmosdb-nosql-connector.md @@ -0,0 +1,194 @@ +--- +title: Using the Semantic Kernel Azure CosmosDB NoSQL Vector Store connector (Preview) +description: Contains information on how to use a Semantic Kernel Vector store connector to access and manipulate data in Azure CosmosDB NoSQL. +zone_pivot_groups: programming-languages +author: westey-m +ms.topic: conceptual +ms.author: westey +ms.date: 09/23/2024 +ms.service: semantic-kernel +--- +# Using the Azure CosmosDB NoSQL Vector Store connector (Preview) + +> [!WARNING] +> The Semantic Kernel Vector Store functionality is in preview, and improvements that require breaking changes may still occur in limited circumstances before release. + +::: zone pivot="programming-language-csharp" + +## Overview + +The Azure CosmosDB NoSQL Vector Store connector can be used to access and manage data in Azure CosmosDB NoSQL. The connector has the following characteristics. + +| Feature Area | Support | +|-----------------------------------|----------------------------------------------------------------------------------------------------------------------------------| +| Collection maps to | Azure Cosmos DB NoSQL Container | +| Supported key property types |
      • string
      • AzureCosmosDBNoSQLCompositeKey
      | +| Supported data property types |
      • string
      • int
      • long
      • double
      • float
      • bool
      • DateTimeOffset
      • *and enumerables of each of these types*
      | +| Supported vector property types |
      • ReadOnlyMemory\
      • ReadOnlyMemory\
      • ReadOnlyMemory\
      • ReadOnlyMemory\
      | +| Supported index types |
      • Flat
      • QuantizedFlat
      • DiskAnn
      | +| Supported distance functions |
      • CosineSimilarity
      • DotProductSimilarity
      • EuclideanDistance
      | +| Supports multiple vectors in a record | Yes | +| IsFilterable supported? | Yes | +| IsFullTextSearchable supported? | Yes | +| StoragePropertyName supported? | No, use `JsonSerializerOptions` and `JsonPropertyNameAttribute` instead. [See here for more info.](#data-mapping) | + +## Getting started + +Add the Azure CosmosDB NoSQL Vector Store connector NuGet package to your project. + +```dotnetcli +dotnet add package Microsoft.SemanticKernel.Connectors.AzureCosmosDBNoSQL --prerelease +``` + +You can add the vector store to the dependency injection container available on the `KernelBuilder` or to the `IServiceCollection` dependency injection container using extension methods provided by Semantic Kernel. + +```csharp +using Microsoft.SemanticKernel; + +// Using Kernel Builder. +var kernelBuilder = Kernel + .CreateBuilder() + .AddAzureCosmosDBNoSQLVectorStore(connectionString, databaseName); +``` + +```csharp +using Microsoft.SemanticKernel; + +// Using IServiceCollection with ASP.NET Core. +var builder = WebApplication.CreateBuilder(args); +builder.Services.AddAzureCosmosDBNoSQLVectorStore(connectionString, databaseName); +``` + +Extension methods that take no parameters are also provided. These require an instance of `Microsoft.Azure.Cosmos.Database` to be separately registered with the dependency injection container. + +```csharp +using Microsoft.Azure.Cosmos; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.SemanticKernel; + +// Using Kernel Builder. +var kernelBuilder = Kernel.CreateBuilder(); +kernelBuilder.Services.AddSingleton( + sp => + { + var cosmosClient = new CosmosClient(connectionString); + return cosmosClient.GetDatabase(databaseName); + }); +kernelBuilder.AddAzureCosmosDBNoSQLVectorStore(); +``` + +```csharp +using Microsoft.Azure.Cosmos; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.SemanticKernel; + +// Using IServiceCollection with ASP.NET Core. +var builder = WebApplication.CreateBuilder(args); +builder.Services.AddSingleton( + sp => + { + var cosmosClient = new CosmosClient(connectionString); + return cosmosClient.GetDatabase(databaseName); + }); +builder.Services.AddAzureCosmosDBNoSQLVectorStore(); +``` + +You can construct an Azure CosmosDB NoSQL Vector Store instance directly. + +```csharp +using Microsoft.Azure.Cosmos; +using Microsoft.SemanticKernel.Connectors.AzureCosmosDBNoSQL; + +var cosmosClient = new CosmosClient(connectionString); +var database = cosmosClient.GetDatabase(databaseName); +var vectorStore = new AzureCosmosDBNoSQLVectorStore(database); +``` + +It is possible to construct a direct reference to a named collection. + +```csharp +using Microsoft.Azure.Cosmos; +using Microsoft.SemanticKernel.Connectors.AzureCosmosDBNoSQL; + +var cosmosClient = new CosmosClient(connectionString); +var database = cosmosClient.GetDatabase(databaseName); +var collection = new AzureCosmosDBNoSQLVectorStoreRecordCollection( + database, + "skhotels"); +``` + +## Data mapping + +The Azure CosmosDB NoSQL Vector Store connector provides a default mapper when mapping from the data model to storage. + +This mapper does a direct conversion of the list of properties on the data model to the fields in Azure CosmosDB NoSQL and uses `System.Text.Json.JsonSerializer` +to convert to the storage schema. This means that usage of the `JsonPropertyNameAttribute` is supported if a different storage name to the +data model property name is required. The only exception is the key of the record which is mapped to a database field named `id`, since all CosmosDB NoSQL +records must use this name for ids. + +It is also possible to use a custom `JsonSerializerOptions` instance with a customized property naming policy. To enable this, the `JsonSerializerOptions` +must be passed to the `AzureCosmosDBNoSQLVectorStoreRecordCollection` on construction. + +```csharp +using System.Text.Json; +using Microsoft.Azure.Cosmos; +using Microsoft.SemanticKernel.Connectors.AzureCosmosDBNoSQL; + +var jsonSerializerOptions = new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseUpper }; + +var cosmosClient = new CosmosClient(connectionString); + +var database = cosmosClient.GetDatabase(databaseName); +var collection = new AzureCosmosDBNoSQLVectorStoreRecordCollection( + database, + "skhotels", + new() { JsonSerializerOptions = jsonSerializerOptions }); +``` + +Using the above custom `JsonSerializerOptions` which is using `SnakeCaseUpper`, the following data model will be mapped to the below json. + +```csharp +using System.Text.Json.Serialization; +using Microsoft.Extensions.VectorData; + +public class Hotel +{ + [VectorStoreRecordKey] + public ulong HotelId { get; set; } + + [VectorStoreRecordData(IsFilterable = true)] + public string HotelName { get; set; } + + [VectorStoreRecordData(IsFullTextSearchable = true)] + public string Description { get; set; } + + [JsonPropertyName("HOTEL_DESCRIPTION_EMBEDDING")] + [VectorStoreRecordVector(4, DistanceFunction.EuclideanDistance, IndexKind.QuantizedFlat)] + public ReadOnlyMemory? DescriptionEmbedding { get; set; } +} +``` + +```json +{ + "id": 1, + "HOTEL_NAME": "Hotel Happy", + "DESCRIPTION": "A place where everyone can be happy.", + "HOTEL_DESCRIPTION_EMBEDDING": [0.9, 0.1, 0.1, 0.1], +} +``` + +::: zone-end +::: zone pivot="programming-language-python" + +## Coming soon + +More info coming soon. + +::: zone-end +::: zone pivot="programming-language-java" + +## Coming soon + +More info coming soon. + +::: zone-end diff --git a/semantic-kernel/concepts/vector-store-connectors/out-of-the-box-connectors/index.md b/semantic-kernel/concepts/vector-store-connectors/out-of-the-box-connectors/index.md new file mode 100644 index 00000000..1b7319ea --- /dev/null +++ b/semantic-kernel/concepts/vector-store-connectors/out-of-the-box-connectors/index.md @@ -0,0 +1,30 @@ +--- +title: Out-of-the-box Vector Store connectors (Preview) +description: Out-of-the-box Vector Store connectors +author: westey-m +ms.topic: conceptual +ms.author: westey +ms.date: 07/08/2024 +ms.service: semantic-kernel +--- +# Out-of-the-box Vector Store connectors (Preview) + +> [!WARNING] +> The Semantic Kernel Vector Store functionality is in preview, and improvements that require breaking changes may still occur in limited circumstances before release. + +Semantic Kernel provides a number of out-of-the-box Vector Store integrations making it easy to get started with using Vector Stores. It also allows you to experiment with a free or locally hosted Vector Store and then easily switch to a service when scale requires it. + +> [!IMPORTANT] +> Some connectors are using SDKs that are not officially supported by Microsoft or by the Database provider. The *Officially supported SDK* column lists which are using officially supported SDKs and which are not. + +| Vector Store Connectors | C# | Python | Java | Officially supported SDK | +|------------------------------------------------------------|:--------------:|:---------------:|:--------------:|:----------------------------------:| +| [Azure AI Search](./azure-ai-search-connector.md) | ✅ | ✅ | In Development | ✅ | +| [Cosmos DB MongoDB](./azure-cosmosdb-mongodb-connector.md) | ✅ | In Development | In Development | ✅ | +| [Cosmos DB No SQL](./azure-cosmosdb-nosql-connector.md) | ✅ | In Development | In Development | ✅ | +| [In-Memory](./inmemory-connector.md) | ✅ | In Development | In Development | N/A | +| [Pinecone](./pinecone-connector.md) | ✅ | In Development | In Development | C#: ❌ Python: ✅ | +| [Qdrant](./qdrant-connector.md) | ✅ | ✅ | In Development | ✅ | +| [Redis](./redis-connector.md) | ✅ | ✅ | In Development | ✅ | +| [Volatile (In-Memory)](./volatile-connector.md) | Deprecated (use In-Memory) | ✅ | In Development | N/A | +| [Weaviate](./weaviate-connector.md) | ✅ | In Development | In Development | N/A | diff --git a/semantic-kernel/concepts/vector-store-connectors/out-of-the-box-connectors/inmemory-connector.md b/semantic-kernel/concepts/vector-store-connectors/out-of-the-box-connectors/inmemory-connector.md new file mode 100644 index 00000000..d4038b47 --- /dev/null +++ b/semantic-kernel/concepts/vector-store-connectors/out-of-the-box-connectors/inmemory-connector.md @@ -0,0 +1,95 @@ +--- +title: Using the Semantic Kernel In-Memory Vector Store connector (Preview) +description: Contains information on how to use a Semantic Kernel Vector store connector to access and manipulate data in an in-memory Semantic Kernel supplied vector store. +zone_pivot_groups: programming-languages +author: westey-m +ms.topic: conceptual +ms.author: westey +ms.date: 11/10/2024 +ms.service: semantic-kernel +--- +# Using the In-Memory connector (Preview) + +> [!WARNING] +> The Semantic Kernel Vector Store functionality is in preview, and improvements that require breaking changes may still occur in limited circumstances before release. + +::: zone pivot="programming-language-csharp" + +## Overview + +The In-Memory Vector Store connector is a Vector Store implementation provided by Semantic Kernel that uses no external database and stores data in memory. +This Vector Store is useful for prototyping scenarios or where high-speed in-memory operations are required. + +The connector has the following characteristics. + +| Feature Area | Support | +|-----------------------------------|----------------------------------------------------------------------------------------------------------------------------------| +| Collection maps to | In-memory dictionary | +| Supported key property types | Any type that can be compared | +| Supported data property types | Any type | +| Supported vector property types | ReadOnlyMemory\ | +| Supported index types | N/A | +| Supported distance functions | N/A | +| Supports multiple vectors in a record | Yes | +| IsFilterable supported? | Yes | +| IsFullTextSearchable supported? | Yes | +| StoragePropertyName supported? | No, since storage is in-memory and data reuse is therefore not possible, custom naming is not applicable. | + +## Getting started + +Add the Semantic Kernel Core nuget package to your project. + +```dotnetcli +dotnet add package Microsoft.SemanticKernel.Connectors.InMemory --prerelease +``` + +You can add the vector store to the dependency injection container available on the `KernelBuilder` or to the `IServiceCollection` dependency injection container using extension methods provided by Semantic Kernel. + +```csharp +using Microsoft.SemanticKernel; + +// Using Kernel Builder. +var kernelBuilder = Kernel + .CreateBuilder() + .AddInMemoryVectorStore(); +``` + +```csharp +using Microsoft.SemanticKernel; + +// Using IServiceCollection with ASP.NET Core. +var builder = WebApplication.CreateBuilder(args); +builder.Services.AddInMemoryVectorStore(); +``` + +You can construct an InMemory Vector Store instance directly. + +```csharp +using Microsoft.SemanticKernel.Connectors.InMemory; + +var vectorStore = new InMemoryVectorStore(); +``` + +It is possible to construct a direct reference to a named collection. + +```csharp +using Microsoft.SemanticKernel.Connectors.InMemory; + +var collection = new InMemoryVectorStoreRecordCollection("skhotels"); +``` + +::: zone-end +::: zone pivot="programming-language-python" + +## Coming soon + +More info coming soon. + +::: zone-end +::: zone pivot="programming-language-java" + +## Coming soon + +More info coming soon. + +::: zone-end diff --git a/semantic-kernel/concepts/vector-store-connectors/out-of-the-box-connectors/pinecone-connector.md b/semantic-kernel/concepts/vector-store-connectors/out-of-the-box-connectors/pinecone-connector.md new file mode 100644 index 00000000..6a44bf3d --- /dev/null +++ b/semantic-kernel/concepts/vector-store-connectors/out-of-the-box-connectors/pinecone-connector.md @@ -0,0 +1,191 @@ +--- +title: Using the Semantic Kernel Pinecone Vector Store connector (Preview) +description: Contains information on how to use a Semantic Kernel Vector store connector to access and manipulate data in Pinecone. +zone_pivot_groups: programming-languages +author: westey-m +ms.topic: conceptual +ms.author: westey +ms.date: 07/08/2024 +ms.service: semantic-kernel +--- +# Using the Pinecone connector (Preview) + +> [!WARNING] +> The Semantic Kernel Vector Store functionality is in preview, and improvements that require breaking changes may still occur in limited circumstances before release. + +::: zone pivot="programming-language-csharp" + +## Overview + +The Pinecone Vector Store connector can be used to access and manage data in Pinecone. The connector has the following characteristics. + +| Feature Area | Support | +|-----------------------------------|----------------------------------------------------------------------------------------------------------------------------------| +| Collection maps to | Pinecone serverless Index | +| Supported key property types | string | +| Supported data property types |
      • string
      • int
      • long
      • double
      • float
      • bool
      • decimal
      • *enumerables of type* string
      | +| Supported vector property types | ReadOnlyMemory\ | +| Supported index types | PGA (Pinecone Graph Algorithm) | +| Supported distance functions |
      • CosineSimilarity
      • DotProductSimilarity
      • EuclideanDistance
      | +| Supports multiple vectors in a record | No | +| IsFilterable supported? | Yes | +| IsFullTextSearchable supported? | No | +| StoragePropertyName supported? | Yes | + +## Limitations + +Notable Pinecone connector functionality limitations. + +| Feature Area | Workaround | +|--------------------------------------------------------------------------------------| -----------------------------------------------------------------------------------------------| +| Vector Search is not yet implemented | No workaround at this stage, implementation to follow soon | + +## Getting started + +Add the Pinecone Vector Store connector NuGet package to your project. + +```dotnetcli +dotnet add package Microsoft.SemanticKernel.Connectors.Pinecone --prerelease +``` + +You can add the vector store to the dependency injection container available on the `KernelBuilder` or to the `IServiceCollection` dependency injection container using extension methods provided by Semantic Kernel. + +```csharp +using Microsoft.SemanticKernel; + +// Using Kernel Builder. +var kernelBuilder = Kernel + .CreateBuilder() + .AddPineconeVectorStore(pineconeApiKey); +``` + +```csharp +using Microsoft.SemanticKernel; + +// Using IServiceCollection with ASP.NET Core. +var builder = WebApplication.CreateBuilder(args); +builder.Services.AddPineconeVectorStore(pineconeApiKey); +``` + +Extension methods that take no parameters are also provided. These require an instance of the `PineconeClient` to be separately registered with the dependency injection container. + +```csharp +using Microsoft.Extensions.DependencyInjection; +using Microsoft.SemanticKernel; +using PineconeClient = Pinecone.PineconeClient; + +// Using Kernel Builder. +var kernelBuilder = Kernel.CreateBuilder(); +kernelBuilder.Services.AddSingleton( + sp => new PineconeClient(pineconeApiKey)); +kernelBuilder.AddPineconeVectorStore(); +``` + +```csharp +using Microsoft.Extensions.DependencyInjection; +using Microsoft.SemanticKernel; +using PineconeClient = Pinecone.PineconeClient; + +// Using IServiceCollection with ASP.NET Core. +var builder = WebApplication.CreateBuilder(args); +builder.Services.AddSingleton( + sp => new PineconeClient(pineconeApiKey)); +builder.Services.AddPineconeVectorStore(); +``` + +You can construct a Pinecone Vector Store instance directly. + +```csharp +using Microsoft.SemanticKernel.Connectors.Pinecone; +using PineconeClient = Pinecone.PineconeClient; + +var vectorStore = new PineconeVectorStore( + new PineconeClient(pineconeApiKey)); +``` + +It is possible to construct a direct reference to a named collection. + +```csharp +using Microsoft.SemanticKernel.Connectors.Pinecone; +using PineconeClient = Pinecone.PineconeClient; + +var collection = new PineconeVectorStoreRecordCollection( + new PineconeClient(pineconeApiKey), + "skhotels"); +``` + +## Index Namespace + +The Vector Store abstraction does not support a multi tiered record grouping mechanism. Collections in the abstraction map to a Pinecone serverless index +and no second level exists in the abstraction. Pinecone does support a second level of grouping called namespaces. + +By default the Pinecone connector will pass null as the namespace for all operations. However it is possible to pass a single namespace to the +Pinecone collection when constructing it and use this instead for all operations. + +```csharp +using Microsoft.SemanticKernel.Connectors.Pinecone; +using PineconeClient = Pinecone.PineconeClient; + +var collection = new PineconeVectorStoreRecordCollection( + new PineconeClient(pineconeApiKey), + "skhotels", + new() { IndexNamespace = "seasidehotels" }); +``` + +## Data mapping + +The Pinecone connector provides a default mapper when mapping data from the data model to storage. +Pinecone requires properties to be mapped into id, metadata and values groupings. +The default mapper uses the model annotations or record definition to determine the type of each property and to do this mapping. + +- The data model property annotated as a key will be mapped to the Pinecone id property. +- The data model properties annotated as data will be mapped to the Pinecone metadata object. +- The data model property annotated as a vector will be mapped to the Pinecone vector property. + +### Property name override + +For data properties, you can provide override field names to use in storage that is different to the +property names on the data model. This is not supported for keys, since a key has a fixed name in Pinecone. +It is also not supported for vectors, since the vector is stored under a fixed name `values`. +The property name override is done by setting the `StoragePropertyName` option via the data model attributes or record definition. + +Here is an example of a data model with `StoragePropertyName` set on its attributes and how that will be represented in Pinecone. + +```csharp +using Microsoft.Extensions.VectorData; + +public class Hotel +{ + [VectorStoreRecordKey] + public ulong HotelId { get; set; } + + [VectorStoreRecordData(IsFilterable = true, StoragePropertyName = "hotel_name")] + public string HotelName { get; set; } + + [VectorStoreRecordData(IsFullTextSearchable = true, StoragePropertyName = "hotel_description")] + public string Description { get; set; } + + [VectorStoreRecordVector(Dimensions: 4, DistanceFunction.CosineDistance, IndexKind.Hnsw)] + public ReadOnlyMemory? DescriptionEmbedding { get; set; } +} +``` + +```json +{ + "id": "h1", + "values": [0.9, 0.1, 0.1, 0.1], + "metadata": { "hotel_name": "Hotel Happy", "hotel_description": "A place where everyone can be happy." } +} +``` + +::: zone-end +::: zone pivot="programming-language-python" + +The Pinecone connector is not yet available in Python. + +::: zone-end +::: zone pivot="programming-language-java" + +The Pinecone connector is not yet available in Java. + +::: zone-end diff --git a/semantic-kernel/concepts/vector-store-connectors/out-of-the-box-connectors/qdrant-connector.md b/semantic-kernel/concepts/vector-store-connectors/out-of-the-box-connectors/qdrant-connector.md new file mode 100644 index 00000000..164e58ce --- /dev/null +++ b/semantic-kernel/concepts/vector-store-connectors/out-of-the-box-connectors/qdrant-connector.md @@ -0,0 +1,355 @@ +--- +title: Using the Semantic Kernel Qdrant Vector Store connector (Preview) +description: Contains information on how to use a Semantic Kernel Vector store connector to access and manipulate data in Qdrant. +zone_pivot_groups: programming-languages +author: westey-m +ms.topic: conceptual +ms.author: westey +ms.date: 07/08/2024 +ms.service: semantic-kernel +--- +# Using the Qdrant connector (Preview) + +> [!WARNING] +> The Semantic Kernel Vector Store functionality is in preview, and improvements that require breaking changes may still occur in limited circumstances before release. + +## Overview + +The Qdrant Vector Store connector can be used to access and manage data in Qdrant. The connector has the following characteristics. + +| Feature Area | Support | +|-----------------------------------|----------------------------------------------------------------------------------------------------------------------------------| +| Collection maps to | Qdrant collection with payload indices for filterable data fields | +| Supported key property types |
      • ulong
      • Guid
      | +| Supported data property types |
      • string
      • int
      • long
      • double
      • float
      • bool
      • *and enumerables of each of these types*
      | +| Supported vector property types | ReadOnlyMemory\ | +| Supported index types | Hnsw | +| Supported distance functions |
      • CosineSimilarity
      • DotProductSimilarity
      • EuclideanDistance
      • ManhattanDistance
      | +| Supports multiple vectors in a record | Yes (configurable) | +| IsFilterable supported? | Yes | +| IsFullTextSearchable supported? | Yes | +| StoragePropertyName supported? | Yes | + +::: zone pivot="programming-language-csharp" + +## Getting started + +Add the Qdrant Vector Store connector NuGet package to your project. + +```dotnetcli +dotnet add package Microsoft.SemanticKernel.Connectors.Qdrant --prerelease +``` + +You can add the vector store to the dependency injection container available on the `KernelBuilder` or to the `IServiceCollection` dependency injection container using extension methods provided by Semantic Kernel. + +```csharp +using Microsoft.SemanticKernel; + +// Using Kernel Builder. +var kernelBuilder = Kernel + .CreateBuilder() + .AddQdrantVectorStore("localhost"); +``` + +```csharp +using Microsoft.SemanticKernel; + +// Using IServiceCollection with ASP.NET Core. +var builder = WebApplication.CreateBuilder(args); +builder.Services.AddQdrantVectorStore("localhost"); +``` + +Extension methods that take no parameters are also provided. These require an instance of the `Qdrant.Client.QdrantClient` class to be separately registered with the dependency injection container. + +```csharp +using Microsoft.Extensions.DependencyInjection; +using Microsoft.SemanticKernel; +using Qdrant.Client; + +// Using Kernel Builder. +var kernelBuilder = Kernel.CreateBuilder(); +kernelBuilder.Services.AddSingleton(sp => new QdrantClient("localhost")); +kernelBuilder.AddQdrantVectorStore(); +``` + +```csharp +using Microsoft.Extensions.DependencyInjection; +using Microsoft.SemanticKernel; +using Qdrant.Client; + +// Using IServiceCollection with ASP.NET Core. +var builder = WebApplication.CreateBuilder(args); +builder.Services.AddSingleton(sp => new QdrantClient("localhost")); +builder.Services.AddQdrantVectorStore(); +``` + +You can construct a Qdrant Vector Store instance directly. + +```csharp +using Microsoft.SemanticKernel.Connectors.Qdrant; +using Qdrant.Client; + +var vectorStore = new QdrantVectorStore(new QdrantClient("localhost")); +``` + +It is possible to construct a direct reference to a named collection. + +```csharp +using Microsoft.SemanticKernel.Connectors.Qdrant; +using Qdrant.Client; + +var collection = new QdrantVectorStoreRecordCollection( + new QdrantClient("localhost"), + "skhotels"); +``` + +## Data mapping + +The Qdrant connector provides a default mapper when mapping data from the data model to storage. +Qdrant requires properties to be mapped into id, payload and vector(s) groupings. +The default mapper uses the model annotations or record definition to determine the type of each property and to do this mapping. + +- The data model property annotated as a key will be mapped to the Qdrant point id. +- The data model properties annotated as data will be mapped to the Qdrant point payload object. +- The data model properties annotated as vectors will be mapped to the Qdrant point vector object. + +### Property name override + +For data properties and vector properties (if using named vectors mode), you can provide override field names to use in storage that is different to the +property names on the data model. This is not supported for keys, since a key has a fixed name in Qdrant. It is also not supported for vectors in *single +unnamed vector* mode, since the vector is stored under a fixed name. + +The property name override is done by setting the `StoragePropertyName` option via the data model attributes or record definition. + +Here is an example of a data model with `StoragePropertyName` set on its attributes and how that will be represented in Qdrant. + +```csharp +using Microsoft.Extensions.VectorData; + +public class Hotel +{ + [VectorStoreRecordKey] + public ulong HotelId { get; set; } + + [VectorStoreRecordData(IsFilterable = true, StoragePropertyName = "hotel_name")] + public string HotelName { get; set; } + + [VectorStoreRecordData(IsFullTextSearchable = true, StoragePropertyName = "hotel_description")] + public string Description { get; set; } + + [VectorStoreRecordVector(4, DistanceFunction.CosineDistance, IndexKind.Hnsw, StoragePropertyName = "hotel_description_embedding")] + public ReadOnlyMemory? DescriptionEmbedding { get; set; } +} +``` + +```json +{ + "id": 1, + "payload": { "hotel_name": "Hotel Happy", "hotel_description": "A place where everyone can be happy." }, + "vector": { + "hotel_description_embedding": [0.9, 0.1, 0.1, 0.1], + } +} +``` + +::: zone-end +::: zone pivot="programming-language-python" + +## Getting started + +Install semantic kernel with the qdrant extras, which includes the [qdrant client](https://github.com/qdrant/qdrant-client). + +```cli +pip install semantic-kernel[qdrant] +``` + +You can then create a vector store instance using the `QdrantStore` class, this will create a AsyncQdrantClient using the environment variables `QDRANT_URL`, `QDRANT_API_KEY`, `QDRANT_HOST`, `QDRANT_PORT`, `QDRANT_GRPC_PORT`, `QDRANT_PATH`, `QDRANT_LOCATION` and `QDRANT_PREFER_GRPS` to connect to the Qdrant instance, those values can also be supplied directly. If nothing is supplied it falls back to `location=:memory:`. + +```python + +from semantic_kernel.connectors.memory.qdrant import QdrantStore + +vector_store = QdrantStore() +``` + +You can also create the vector store with your own instance of the qdrant client. + +```python +from qdrant_client.async_qdrant_client import AsyncQdrantClient +from semantic_kernel.connectors.memory.qdrant import QdrantStore + +client = AsyncQdrantClient(host='localhost', port=6333) +vector_store = QdrantStore(client=client) +``` + +You can also create a collection directly. + +```python +from semantic_kernel.connectors.memory.qdrant import QdrantCollection + +collection = QdrantCollection(collection_name="skhotels", data_model_type=hotel) +``` + +## Serialization + +The Qdrant connector uses a model called `PointStruct` for reading and writing to the store. This can be imported from `from qdrant_client.models import PointStruct`. The serialization methods expects a output of a list of PointStruct objects, and the deserialization method recieves a list of PointStruct objects. + +There are some special considerations for this that have to do with named or unnamed vectors, see below. + +For more details on this concept see the [serialization documentation](./../serialization.md). + +::: zone-end +::: zone pivot="programming-language-java" +::: zone-end + +### Qdrant vector modes + +Qdrant supports two modes for vector storage and the Qdrant Connector with default mapper supports both modes. +The default mode is *single unnamed vector*. + +#### Single unnamed vector + +With this option a collection may only contain a single vector and it will be unnamed in the storage model in Qdrant. +Here is an example of how an object is represented in Qdrant when using *single unnamed vector* mode: + +::: zone pivot="programming-language-csharp" + +```csharp +new Hotel +{ + HotelId = 1, + HotelName = "Hotel Happy", + Description = "A place where everyone can be happy.", + DescriptionEmbedding = new float[4] { 0.9f, 0.1f, 0.1f, 0.1f } +}; +``` + +```json +{ + "id": 1, + "payload": { "HotelName": "Hotel Happy", "Description": "A place where everyone can be happy." }, + "vector": [0.9, 0.1, 0.1, 0.1] +} +``` + +::: zone-end +::: zone pivot="programming-language-python" + +```python +Hotel( + hotel_id = 1, + hotel_name = "Hotel Happy", + description = "A place where everyone can be happy.", + description_embedding = [0.9f, 0.1f, 0.1f, 0.1f], +) +``` + +```python +from qdrant_client.models import PointStruct + +PointStruct( + id=1, + payload={ "hotel_name": "Hotel Happy", "description": "A place where everyone can be happy." }, + vector=[0.9, 0.1, 0.1, 0.1], +) +``` +::: zone-end +::: zone pivot="programming-language-java" +::: zone-end + + +#### Named vectors + +If using the named vectors mode, it means that each point in a collection may contain more than one vector, and each will be named. +Here is an example of how an object is represented in Qdrant when using *named vectors* mode: + +::: zone pivot="programming-language-csharp" + +```csharp +new Hotel +{ + HotelId = 1, + HotelName = "Hotel Happy", + Description = "A place where everyone can be happy.", + HotelNameEmbedding = new float[4] { 0.9f, 0.5f, 0.5f, 0.5f } + DescriptionEmbedding = new float[4] { 0.9f, 0.1f, 0.1f, 0.1f } +}; +``` + +```json +{ + "id": 1, + "payload": { "HotelName": "Hotel Happy", "Description": "A place where everyone can be happy." }, + "vector": { + "HotelNameEmbedding": [0.9, 0.5, 0.5, 0.5], + "DescriptionEmbedding": [0.9, 0.1, 0.1, 0.1], + } +} +``` + +::: zone-end +::: zone pivot="programming-language-python" + +```python +Hotel( + hotel_id = 1, + hotel_name = "Hotel Happy", + description = "A place where everyone can be happy.", + hotel_name_embedding = [0.9f, 0.5f, 0.5f, 0.5f], + description_embedding = [0.9f, 0.1f, 0.1f, 0.1f], +) +``` + +```python +from qdrant_client.models import PointStruct + +PointStruct( + id=1, + payload={ "hotel_name": "Hotel Happy", "description": "A place where everyone can be happy." }, + vector={ + "hotel_name_embedding": [0.9, 0.5, 0.5, 0.5], + "description_embedding": [0.9, 0.1, 0.1, 0.1], + }, +) +``` + +::: zone-end +::: zone pivot="programming-language-java" +::: zone-end + +To enable named vectors mode, pass this as an option when constructing a Vector Store or collection. +The same options can also be passed to any of the provided dependency injection container extension methods. + +::: zone pivot="programming-language-csharp" + +```csharp +using Microsoft.SemanticKernel.Connectors.Qdrant; +using Qdrant.Client; + +var vectorStore = new QdrantVectorStore( + new QdrantClient("localhost"), + new() { HasNamedVectors = true }); + +var collection = new QdrantVectorStoreRecordCollection( + new QdrantClient("localhost"), + "skhotels", + new() { HasNamedVectors = true }); +``` + +::: zone-end +::: zone pivot="programming-language-python" + +In python the default value for `named_vectors` is True, but you can also disable this as shown below. + +```python +from semantic_kernel.connectors.memory.qdrant import QdrantCollection + +collection = QdrantCollection( + collection_name="skhotels", + data_model_type=Hotel, + named_vectors=False, +) +``` +::: zone-end +::: zone pivot="programming-language-java" +::: zone-end diff --git a/semantic-kernel/concepts/vector-store-connectors/out-of-the-box-connectors/redis-connector.md b/semantic-kernel/concepts/vector-store-connectors/out-of-the-box-connectors/redis-connector.md new file mode 100644 index 00000000..a58e0c6b --- /dev/null +++ b/semantic-kernel/concepts/vector-store-connectors/out-of-the-box-connectors/redis-connector.md @@ -0,0 +1,330 @@ +--- +title: Using the Semantic Kernel Redis Vector Store connector (Preview) +description: Contains information on how to use a Semantic Kernel Vector store connector to access and manipulate data in Redis. +zone_pivot_groups: programming-languages +author: westey-m +ms.topic: conceptual +ms.author: westey +ms.date: 07/08/2024 +ms.service: semantic-kernel +--- +# Using the Redis connector (Preview) + +> [!WARNING] +> The Semantic Kernel Vector Store functionality is in preview, and improvements that require breaking changes may still occur in limited circumstances before release. + +## Overview + +The Redis Vector Store connector can be used to access and manage data in Redis. The connector supports both Hashes and JSON modes and which mode you pick will determine what other features are supported. + +The connector has the following characteristics. + +| Feature Area | Support | +|-----------------------------------|----------------------------------------------------------------------------------------------------------------------------------| +| Collection maps to | Redis index with prefix set to `:` | +| Supported key property types | string | +| Supported data property types | **When using Hashes:**
      • string
      • int
      • uint
      • long
      • ulong
      • double
      • float
      • bool
      **When using JSON:**
      Any types serializable to JSON| +| Supported vector property types |
      • ReadOnlyMemory\
      • ReadOnlyMemory\
      | +| Supported index types |
      • Hnsw
      • Flat
      | +| Supported distance functions |
      • CosineSimilarity
      • DotProductSimilarity
      • EuclideanDistance
      | +| Supports multiple vectors in a record | Yes | +| IsFilterable supported? | Yes | +| IsFullTextSearchable supported? | Yes | +| StoragePropertyName supported? | **When using Hashes:** Yes
      **When using JSON:** No, use `JsonSerializerOptions` and `JsonPropertyNameAttribute` instead. [See here for more info.](#data-mapping) | + +::: zone pivot="programming-language-csharp" + +## Getting started + +Add the Redis Vector Store connector nuget package to your project. + +```dotnetcli +dotnet add package Microsoft.SemanticKernel.Connectors.Redis --prerelease +``` + +You can add the vector store to the dependency injection container available on the `KernelBuilder` or to the `IServiceCollection` dependency injection container using extension methods provided by Semantic Kernel. + +```csharp +using Microsoft.SemanticKernel; + +// Using Kernel Builder. +var kernelBuilder = Kernel + .CreateBuilder() + .AddRedisVectorStore("localhost:6379"); +``` + +```csharp +using Microsoft.SemanticKernel; + +// Using IServiceCollection with ASP.NET Core. +var builder = WebApplication.CreateBuilder(args); +builder.Services.AddRedisVectorStore("localhost:6379"); +``` + +Extension methods that take no parameters are also provided. These require an instance of the Redis `IDatabase` to be separately registered with the dependency injection container. + +```csharp +using Microsoft.Extensions.DependencyInjection; +using Microsoft.SemanticKernel; +using StackExchange.Redis; + +// Using Kernel Builder. +var kernelBuilder = Kernel.CreateBuilder(); +kernelBuilder.Services.AddSingleton(sp => ConnectionMultiplexer.Connect("localhost:6379").GetDatabase()); +kernelBuilder.AddRedisVectorStore(); +``` + +```csharp +using Microsoft.Extensions.DependencyInjection; +using Microsoft.SemanticKernel; +using StackExchange.Redis; + +// Using IServiceCollection with ASP.NET Core. +var builder = WebApplication.CreateBuilder(args); +builder.Services.AddSingleton(sp => ConnectionMultiplexer.Connect("localhost:6379").GetDatabase()); +builder.Services.AddRedisVectorStore(); +``` + +You can construct a Redis Vector Store instance directly. + +```csharp +using Microsoft.SemanticKernel.Connectors.Redis; +using StackExchange.Redis; + +var vectorStore = new RedisVectorStore(ConnectionMultiplexer.Connect("localhost:6379").GetDatabase()); +``` + +It is possible to construct a direct reference to a named collection. +When doing so, you have to choose between the JSON or Hashes instance depending on how you wish to store data in Redis. + +```csharp +using Microsoft.SemanticKernel.Connectors.Redis; +using StackExchange.Redis; + +// Using Hashes. +var hashesCollection = new RedisHashSetVectorStoreRecordCollection( + ConnectionMultiplexer.Connect("localhost:6379").GetDatabase(), + "skhotelshashes"); +``` + +```csharp +using Microsoft.SemanticKernel.Connectors.Redis; +using StackExchange.Redis; + +// Using JSON. +var jsonCollection = new RedisJsonVectorStoreRecordCollection( + ConnectionMultiplexer.Connect("localhost:6379").GetDatabase(), + "skhotelsjson"); +``` + +When constructing a `RedisVectorStore` or registering it with the dependency injection container, it's possible to pass a `RedisVectorStoreOptions` instance +that configures the preferred storage type / mode used: Hashes or JSON. If not specified, the default is JSON. + +```csharp +using Microsoft.SemanticKernel.Connectors.Redis; +using StackExchange.Redis; + +var vectorStore = new RedisVectorStore( + ConnectionMultiplexer.Connect("localhost:6379").GetDatabase(), + new() { StorageType = RedisStorageType.HashSet }); +``` + +::: zone-end +::: zone pivot="programming-language-python" + +## Getting started + +Install semantic kernel with the redis extras, which includes the redis client. + +```cli +pip install semantic-kernel[redis] +``` + +You can then create a vector store instance using the `RedisStore` class, this will use the environment variables `REDIS_CONNECTION_STRING` to connect to a Redis instance, those values can also be supplied directly. + +```python + +from semantic_kernel.connectors.memory.redis import RedisStore + +vector_store = RedisStore() +``` + +You can also create the vector store with your own instance of the redis database client. + +```python +from redis.asyncio.client import Redis +from semantic_kernel.connectors.memory.redis import RedisStore + +redis_database = Redis.from_url(url="https://") +vector_store = RedisStore(redis_database=redis_database) +``` + +You can also create a collection directly, but there are two types of collections, one for Hashes and one for JSON. + +```python +from semantic_kernel.connectors.memory.redis import RedisHashsetCollection, RedisJsonCollection + +hash_collection = RedisHashsetCollection(collection_name="skhotels", data_model_type=Hotel) +json_collection = RedisJsonCollection(collection_name="skhotels", data_model_type=Hotel) +``` + +When creating a collection from the vector store, you can pass in the collection type, as a enum: `RedisCollectionTypes`, the default is a hash collection. + +```python +from semantic_kernel.connectors.memory.redis import RedisStore, RedisCollectionTypes + +vector_store = RedisStore() +collection = vector_store.get_collection( + collection_name="skhotels", + data_model_type=Hotel, + collection_type=RedisCollectionTypes.JSON, +) + +``` + +## Serialization + +The redis collections both use a dict as the data format when upserting, however the structure of the dicts are different between them. + +For JSON collections see [redis docs](https://redis-py.readthedocs.io/en/stable/examples/search_json_examples.html) for a example. + +For Hashset collections, it uses the hset command with the key field as `name`, data fields as `mapping -> metadata` and vectors as `mapping -> [vector_field_name]` , see [here](https://redis-py.readthedocs.io/en/stable/commands.html#redis.commands.core.CoreCommands.hset) for more information. + +For more details on this concept see the [serialization documentation](./../serialization.md). + +::: zone-end +::: zone pivot="programming-language-java" +::: zone-end + +## Index prefixes + +Redis uses a system of key prefixing to associate a record with an index. +When creating an index you can specify one or more prefixes to use with that index. +If you want to associate a record with that index, you have to add the prefix to the key of that record. + +E.g. If you create a index called `skhotelsjson` with a prefix of `skhotelsjson:`, when setting a record +with key `h1`, the record key will need to be prefixed like this `skhotelsjson:h1` to be added to the index. + +When creating a new collection using the Redis connector, the connector will create an index in Redis with a +prefix consisting of the collection name and a colon, like this `:`. +By default, the connector will also prefix all keys with the this prefix when doing record operations like Get, Upsert, and Delete. + +If you didn't want to use a prefix consisting of the collection name and a colon, it is possible to switch +off the prefixing behavior and pass in the fully prefixed key to the record operations. + +::: zone pivot="programming-language-csharp" + +```csharp +using Microsoft.SemanticKernel.Connectors.Redis; +using StackExchange.Redis; + +var collection = new RedisJsonVectorStoreRecordCollection( + ConnectionMultiplexer.Connect("localhost:6379").GetDatabase(), + "skhotelsjson", + new() { PrefixCollectionNameToKeyNames = false }); + +await collection.GetAsync("myprefix_h1"); +``` + +::: zone-end +::: zone pivot="programming-language-python" +```python +from semantic_kernel.connectors.memory.redis import RedisJsonCollection + +collection = RedisJsonCollection(collection_name="skhotels", data_model_type=hotel, prefix_collection_name_to_key_names=False) + +await collection.get("myprefix_h1") +``` +::: zone-end +::: zone pivot="programming-language-java" +::: zone-end + +::: zone pivot="programming-language-csharp" +## Data mapping + +Redis supports two modes for storing data: JSON and Hashes. The Redis connector supports both storage types, and mapping differs depending on the chosen storage type. + +### Data mapping when using the JSON storage type + +When using the JSON storage type, the Redis connector will use `System.Text.Json.JsonSerializer` to do mapping. +Since Redis stores records with a separate key and value, the mapper will serialize all properties except for the key to a JSON object +and use that as the value. + +Usage of the `JsonPropertyNameAttribute` is supported if a different storage name to the +data model property name is required. It is also possible to use a custom `JsonSerializerOptions` instance with a customized property naming policy. To enable this, the `JsonSerializerOptions` +must be passed to the `RedisJsonVectorStoreRecordCollection` on construction. + +```csharp +var jsonSerializerOptions = new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseUpper }; +var collection = new RedisJsonVectorStoreRecordCollection( + ConnectionMultiplexer.Connect("localhost:6379").GetDatabase(), + "skhotelsjson", + new() { JsonSerializerOptions = jsonSerializerOptions }); +``` + +Since a naming policy of snake case upper was chosen, here is an example of how this data type will be set in Redis. +Also note the use of `JsonPropertyNameAttribute` on the `Description` property to further customize the storage naming. + +```csharp +using System.Text.Json.Serialization; +using Microsoft.Extensions.VectorData; + +public class Hotel +{ + [VectorStoreRecordKey] + public ulong HotelId { get; set; } + + [VectorStoreRecordData(IsFilterable = true)] + public string HotelName { get; set; } + + [JsonPropertyName("HOTEL_DESCRIPTION")] + [VectorStoreRecordData(IsFullTextSearchable = true)] + public string Description { get; set; } + + [VectorStoreRecordVector(Dimensions: 4, DistanceFunction.CosineDistance, IndexKind.Hnsw)] + public ReadOnlyMemory? DescriptionEmbedding { get; set; } +} +``` + +```redis +JSON.SET skhotelsjson:h1 $ '{ "HOTEL_NAME": "Hotel Happy", "HOTEL_DESCRIPTION": "A place where everyone can be happy.", "DESCRIPTION_EMBEDDING": [0.9, 0.1, 0.1, 0.1] }' +``` + +### Data mapping when using the Hashes storage type + +When using the Hashes storage type, the Redis connector provides its own mapper to do mapping. +This mapper will map each property to a field-value pair as supported by the Redis `HSET` command. + +For data properties and vector properties, you can provide override field names to use in storage that is different to the +property names on the data model. This is not supported for keys, since keys cannot be named in Redis. + +Property name overriding is done by setting the `StoragePropertyName` option via the data model attributes or record definition. + +Here is an example of a data model with `StoragePropertyName` set on its attributes and how these are set in Redis. + +```csharp +using Microsoft.Extensions.VectorData; + +public class Hotel +{ + [VectorStoreRecordKey] + public ulong HotelId { get; set; } + + [VectorStoreRecordData(IsFilterable = true, StoragePropertyName = "hotel_name")] + public string HotelName { get; set; } + + [VectorStoreRecordData(IsFullTextSearchable = true, StoragePropertyName = "hotel_description")] + public string Description { get; set; } + + [VectorStoreRecordVector(Dimensions: 4, DistanceFunction.CosineDistance, IndexKind.Hnsw, StoragePropertyName = "hotel_description_embedding")] + public ReadOnlyMemory? DescriptionEmbedding { get; set; } +} +``` + +```redis +HSET skhotelshashes:h1 hotel_name "Hotel Happy" hotel_description 'A place where everyone can be happy.' hotel_description_embedding +``` + +::: zone-end +::: zone pivot="programming-language-java" +::: zone-end diff --git a/semantic-kernel/concepts/vector-store-connectors/out-of-the-box-connectors/volatile-connector.md b/semantic-kernel/concepts/vector-store-connectors/out-of-the-box-connectors/volatile-connector.md new file mode 100644 index 00000000..c56856bb --- /dev/null +++ b/semantic-kernel/concepts/vector-store-connectors/out-of-the-box-connectors/volatile-connector.md @@ -0,0 +1,133 @@ +--- +title: Using the Semantic Kernel Volatile (In-Memory) Vector Store connector (Preview) +description: Contains information on how to use a Semantic Kernel Vector store connector to access and manipulate data using the Volatile (in-memory) Semantic Kernel supplied vector store. +zone_pivot_groups: programming-languages +author: westey-m +ms.topic: conceptual +ms.author: westey +ms.date: 07/08/2024 +ms.service: semantic-kernel +--- +# Using the Volatile (In-Memory) connector (Preview) + +::: zone pivot="programming-language-csharp" + +> [!WARNING] +> The C# VolatileVectorStore is obsolete and has been replaced with a new package. See [InMemory Connector](./inmemory-connector.md) + +::: zone-end +::: zone pivot="programming-language-python" + +> [!WARNING] +> The Semantic Kernel Vector Store functionality is in preview, and improvements that require breaking changes may still occur in limited circumstances before release. + +::: zone-end +::: zone pivot="programming-language-java" + +> [!WARNING] +> The Semantic Kernel Vector Store functionality is in preview, and improvements that require breaking changes may still occur in limited circumstances before release. + +::: zone-end + +## Overview + +The Volatile Vector Store connector is a Vector Store implementation provided by Semantic Kernel that uses no external database and stores data in memory. +This Vector Store is useful for prototyping scenarios or where high-speed in-memory operations are required. + +The connector has the following characteristics. + +| Feature Area | Support | +|-----------------------------------|----------------------------------------------------------------------------------------------------------------------------------| +| Collection maps to | In-memory dictionary | +| Supported key property types | Any type that can be compared | +| Supported data property types | Any type | +| Supported vector property types | ReadOnlyMemory\ | +| Supported index types | N/A | +| Supported distance functions | N/A | +| Supports multiple vectors in a record | Yes | +| IsFilterable supported? | Yes | +| IsFullTextSearchable supported? | Yes | +| StoragePropertyName supported? | No, since storage is volatile and data reuse is therefore not possible, custom naming is not useful and not supported. | + +::: zone pivot="programming-language-csharp" + +## Getting started + +Add the Semantic Kernel Core nuget package to your project. + +```dotnetcli +dotnet add package Microsoft.SemanticKernel.Core +``` + +You can add the vector store to the dependency injection container available on the `KernelBuilder` or to the `IServiceCollection` dependency injection container using extension methods provided by Semantic Kernel. + +```csharp +using Microsoft.SemanticKernel; + +// Using Kernel Builder. +var kernelBuilder = Kernel + .CreateBuilder() + .AddVolatileVectorStore(); +``` + +```csharp +using Microsoft.SemanticKernel; + +// Using IServiceCollection with ASP.NET Core. +var builder = WebApplication.CreateBuilder(args); +builder.Services.AddVolatileVectorStore(); +``` + +You can construct a Volatile Vector Store instance directly. + +```csharp +using Microsoft.SemanticKernel.Data; + +var vectorStore = new VolatileVectorStore(); +``` + +It is possible to construct a direct reference to a named collection. + +```csharp +using Microsoft.SemanticKernel.Data; + +var collection = new VolatileVectorStoreRecordCollection("skhotels"); +``` + +::: zone-end +::: zone pivot="programming-language-python" + +## Getting started + +Install semantic kernel. + +```cli +pip install semantic-kernel +``` + +You can then create a vector store instance using the `VolatileStore` class. + +```python + +from semantic_kernel.connectors.memory.volatile import VolatileStore + +vector_store = VolatileStore() +``` + +You can also create a collection directly. + +```python +from semantic_kernel.connectors.memory.volatile import VolatileCollection + +collection = VolatileCollection(collection_name="skhotels", data_model_type=Hotel) +``` + +## Serialization + +Since the Volatile connector has a simple dict as the internal storage mechanism it can store any data model that can be serialized to a dict. + +For more details on this concept see the [serialization documentation](./../serialization.md). + +::: zone-end +::: zone pivot="programming-language-java" +::: zone-end diff --git a/semantic-kernel/concepts/vector-store-connectors/out-of-the-box-connectors/weaviate-connector.md b/semantic-kernel/concepts/vector-store-connectors/out-of-the-box-connectors/weaviate-connector.md new file mode 100644 index 00000000..49fdcfb9 --- /dev/null +++ b/semantic-kernel/concepts/vector-store-connectors/out-of-the-box-connectors/weaviate-connector.md @@ -0,0 +1,191 @@ +--- +title: Using the Semantic Kernel Weaviate Vector Store connector (Preview) +description: Contains information on how to use a Semantic Kernel Vector store connector to access and manipulate data in Weaviate. +zone_pivot_groups: programming-languages +author: westey-m +ms.topic: conceptual +ms.author: westey +ms.date: 09/23/2024 +ms.service: semantic-kernel +--- +# Using the Weaviate Vector Store connector (Preview) + +> [!WARNING] +> The Semantic Kernel Vector Store functionality is in preview, and improvements that require breaking changes may still occur in limited circumstances before release. + +::: zone pivot="programming-language-csharp" + +## Overview + +The Weaviate Vector Store connector can be used to access and manage data in Weaviate. The connector has the following characteristics. + +| Feature Area | Support | +|-----------------------------------|----------------------------------------------------------------------------------------------------------------------------------| +| Collection maps to | Weaviate Collection | +| Supported key property types | Guid | +| Supported data property types |
      • string
      • byte
      • short
      • int
      • long
      • double
      • float
      • decimal
      • bool
      • DateTime
      • DateTimeOffset
      • Guid
      • *and enumerables of each of these types*
      | +| Supported vector property types |
      • ReadOnlyMemory\
      • ReadOnlyMemory\
      | +| Supported index types |
      • Hnsw
      • Flat
      • Dynamic
      | +| Supported distance functions |
      • CosineDistance
      • DotProductSimilarity
      • EuclideanSquaredDistance
      • Hamming
      • ManhattanDistance
      | +| Supports multiple vectors in a record | Yes | +| IsFilterable supported? | Yes | +| IsFullTextSearchable supported? | Yes | +| StoragePropertyName supported? | No, use `JsonSerializerOptions` and `JsonPropertyNameAttribute` instead. [See here for more info.](#data-mapping) | + +## Limitations + +Notable Weaviate connector functionality limitations. + +| Feature Area | Workaround | +|------------------------------------------------------------------------| -----------------------------------------------------------------------------------------------| +| Using the 'vector' property for single vector objects is not supported | Use of the 'vectors' property is supported instead. | + +## Getting started + +Add the Weaviate Vector Store connector NuGet package to your project. + +```dotnetcli +dotnet add package Microsoft.SemanticKernel.Connectors.Weaviate --prerelease +``` + +You can add the vector store to the dependency injection container available on the `KernelBuilder` or to the `IServiceCollection` dependency injection container using extension methods provided by Semantic Kernel. +The Weaviate vector store uses an `HttpClient` to communicate with the Weaviate service. There are two options for providing the URL/endpoint for the Weaviate service. +It can be provided via options or by setting the base address of the `HttpClient`. + +This first example shows how to set the service URL via options. +Also note that these methods will retrieve an `HttpClient` instance for making calls to the Weaviate service from the dependency injection service provider. + +```csharp +using Microsoft.SemanticKernel; + +// Using Kernel Builder. +var kernelBuilder = Kernel + .CreateBuilder() + .AddWeaviateVectorStore(options: new() { Endpoint = new Uri("http://localhost:8080/v1/") }); +``` + +```csharp +using Microsoft.SemanticKernel; + +// Using IServiceCollection with ASP.NET Core. +var builder = WebApplication.CreateBuilder(args); +builder.Services.AddWeaviateVectorStore(options: new() { Endpoint = new Uri("http://localhost:8080/v1/") }); +``` + +Overloads where you can specify your own `HttpClient` are also provided. +In this case it's possible to set the service url via the `HttpClient` `BaseAddress` option. + +```csharp +using System.Net.Http; +using Microsoft.SemanticKernel; + +// Using Kernel Builder. +var kernelBuilder = Kernel.CreateBuilder(); +using HttpClient client = new HttpClient { BaseAddress = new Uri("http://localhost:8080/v1/") }; +kernelBuilder.AddWeaviateVectorStore(client); +``` + +```csharp +using System.Net.Http; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.SemanticKernel; + +// Using IServiceCollection with ASP.NET Core. +var builder = WebApplication.CreateBuilder(args); +using HttpClient client = new HttpClient { BaseAddress = new Uri("http://localhost:8080/v1/") }; +builder.Services.AddWeaviateVectorStore(client); +``` + +You can construct a Weaviate Vector Store instance directly as well. + +```csharp +using System.Net.Http; +using Microsoft.SemanticKernel.Connectors.Weaviate; + +var vectorStore = new WeaviateVectorStore( + new HttpClient { BaseAddress = new Uri("http://localhost:8080/v1/") }); +``` + +It is possible to construct a direct reference to a named collection. + +```csharp +using System.Net.Http; +using Microsoft.SemanticKernel.Connectors.Weaviate; + +var collection = new WeaviateVectorStoreRecordCollection( + new HttpClient { BaseAddress = new Uri("http://localhost:8080/v1/") }, + "skhotels"); +``` + +If needed, it is possible to pass an Api Key, as an option, when using any of the above mentioned mechanisms, e.g. + +```csharp +using Microsoft.SemanticKernel; + +var kernelBuilder = Kernel + .CreateBuilder() + .AddWeaviateVectorStore(options: new() { Endpoint = new Uri("http://localhost:8080/v1/"), ApiKey = secretVar }); +``` + +## Data mapping + +The Weaviate Vector Store connector provides a default mapper when mapping from the data model to storage. +Weaviate requires properties to be mapped into id, payload and vectors groupings. +The default mapper uses the model annotations or record definition to determine the type of each property and to do this mapping. + +- The data model property annotated as a key will be mapped to the Weaviate `id` property. +- The data model properties annotated as data will be mapped to the Weaviate `properties` object. +- The data model properties annotated as vectors will be mapped to the Weaviate `vectors` object. + +The default mapper uses `System.Text.Json.JsonSerializer` to convert to the storage schema. +This means that usage of the `JsonPropertyNameAttribute` is supported if a different storage name to the +data model property name is required. + +Here is an example of a data model with `JsonPropertyNameAttribute` set and how that will be represented in Weaviate. + +```csharp +using System.Text.Json.Serialization; +using Microsoft.Extensions.VectorData; + +public class Hotel +{ + [VectorStoreRecordKey] + public ulong HotelId { get; set; } + + [VectorStoreRecordData(IsFilterable = true)] + public string HotelName { get; set; } + + [VectorStoreRecordData(IsFullTextSearchable = true)] + public string Description { get; set; } + + [JsonPropertyName("HOTEL_DESCRIPTION_EMBEDDING")] + [VectorStoreRecordVector(4, DistanceFunction.EuclideanDistance, IndexKind.QuantizedFlat)] + public ReadOnlyMemory? DescriptionEmbedding { get; set; } +} +``` + +```json +{ + "id": 1, + "properties": { "HotelName": "Hotel Happy", "Description": "A place where everyone can be happy." }, + "vectors": { + "HOTEL_DESCRIPTION_EMBEDDING": [0.9, 0.1, 0.1, 0.1], + } +} +``` + +::: zone-end +::: zone pivot="programming-language-python" + +## Coming soon + +More info coming soon. + +::: zone-end +::: zone pivot="programming-language-java" + +## Coming soon + +More info coming soon. + +::: zone-end diff --git a/semantic-kernel/concepts/vector-store-connectors/schema-with-record-definition.md b/semantic-kernel/concepts/vector-store-connectors/schema-with-record-definition.md new file mode 100644 index 00000000..f94cb86e --- /dev/null +++ b/semantic-kernel/concepts/vector-store-connectors/schema-with-record-definition.md @@ -0,0 +1,173 @@ +--- +title: Defining your Semantic Kernel storage schema using a record definition (Preview) +description: Describes how to create a record definition with Semantic Kernel to use when writing to or reading from a Vector Store. +zone_pivot_groups: programming-languages +author: westey-m +ms.topic: reference +ms.author: westey +ms.date: 07/08/2024 +ms.service: semantic-kernel +--- +# Defining your storage schema using a record definition (Preview) + +> [!WARNING] +> The Semantic Kernel Vector Store functionality is in preview, and improvements that require breaking changes may still occur in limited circumstances before release. + +## Overview + +The Semantic Kernel Vector Store connectors use a model first approach to interacting with databases and allows annotating data +models with information that is needed for creating indexes or mapping data to the database schema. + +Another way of providing this information is via record definitions, that can be defined and supplied separately to the data model. +This can be useful in multiple scenarios: + +- There may be a case where a developer wants to use the same data model with more than one configuration. +- There may be a case where the developer wants to store data using a very different schema to the model and wants to supply a custom mapper for converting between the data model and storage schema. +- There may be a case where a developer wants to use a built-in type, like a dict, or a optimized format like a dataframe and still wants to leverage the vector store functionality. + +::: zone pivot="programming-language-csharp" + +Here is an example of how to create a record definition. + +```csharp +using Microsoft.Extensions.VectorData; + +var hotelDefinition = new VectorStoreRecordDefinition +{ + Properties = new List + { + new VectorStoreRecordKeyProperty("HotelId", typeof(ulong)), + new VectorStoreRecordDataProperty("HotelName", typeof(string)) { IsFilterable = true }, + new VectorStoreRecordDataProperty("Description", typeof(string)) { IsFullTextSearchable = true }, + new VectorStoreRecordVectorProperty("DescriptionEmbedding", typeof(float)) { Dimensions = 4, DistanceFunction = DistanceFunction.CosineDistance, IndexKind = IndexKind.Hnsw }, + } +}; +``` + +When creating a definition you always have to provide a name and type for each property in your schema, since this is required for index creation and data mapping. + +To use the definition, pass it to the GetCollection method. + +```csharp +var collection = vectorStore.GetCollection("skhotels", hotelDefinition); +``` + +## Record Property configuration classes + +### VectorStoreRecordKeyProperty + +Use this class to incidate that your property is the key of the record. + +```csharp +new VectorStoreRecordKeyProperty("HotelId", typeof(ulong)), +``` + +#### VectorStoreRecordKeyProperty configuration settings + +| Parameter | Required | Description | +|---------------------------|:--------:|-------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| DataModelPropertyName | Yes | The name of the property on the data model. Used by the built in mappers to automatically map between the storage schema and data model and for creating indexes. | +| PropertyType | Yes | The type of the property on the data model. Used by the built in mappers to automatically map between the storage schema and data model and for creating indexes. | +| StoragePropertyName | No | Can be used to supply an alternative name for the property in the database. Note that this parameter is not supported by all connectors, e.g. where alternatives like `JsonPropertyNameAttribute` is supported. | + +> [!TIP] +> For more information on which connectors support StoragePropertyName and what alternatives are available, refer to [the documentation for each connector](./out-of-the-box-connectors/index.md). + +### VectorStoreRecordDataProperty + +Use this class to incidate that your property contains general data that is not a key or a vector. + +```csharp +new VectorStoreRecordDataProperty("HotelName", typeof(string)) { IsFilterable = true }, +``` + +#### VectorStoreRecordDataProperty configuration settings + +| Parameter | Required | Description | +|---------------------------|:--------:|-------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| DataModelPropertyName | Yes | The name of the property on the data model. Used by the built in mappers to automatically map between the storage schema and data model and for creating indexes. | +| PropertyType | Yes | The type of the property on the data model. Used by the built in mappers to automatically map between the storage schema and data model and for creating indexes. | +| IsFilterable | No | Indicates whether the property should be indexed for filtering in cases where a database requires opting in to indexing per property. Default is false. | +| IsFullTextSearchable | No | Indicates whether the property should be indexed for full text search for databases that support full text search. Default is false. | +| StoragePropertyName | No | Can be used to supply an alternative name for the property in the database. Note that this parameter is not supported by all connectors, e.g. where alternatives like `JsonPropertyNameAttribute` is supported. | + +> [!TIP] +> For more information on which connectors support StoragePropertyName and what alternatives are available, refer to [the documentation for each connector](./out-of-the-box-connectors/index.md). + +### VectorStoreRecordVectorProperty + +Use this class to incidate that your property contains a vector. + +```csharp +new VectorStoreRecordVectorProperty("DescriptionEmbedding", typeof(float)) { Dimensions = 4, DistanceFunction = DistanceFunction.CosineDistance, IndexKind = IndexKind.Hnsw }, +``` + +#### VectorStoreRecordVectorProperty configuration settings + +| Parameter | Required | Description | +|---------------------------|:--------:|-------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| DataModelPropertyName | Yes | The name of the property on the data model. Used by the built in mappers to automatically map between the storage schema and data model and for creating indexes. | +| PropertyType | Yes | The type of the property on the data model. Used by the built in mappers to automatically map between the storage schema and data model and for creating indexes. | +| Dimensions | Yes for collection create, optional otherwise | The number of dimensions that the vector has. This is typically required when creating a vector index for a collection. | +| IndexKind | No | The type of index to index the vector with. Default varies by vector store type. | +| DistanceFunction | No | The type of distance function to use when doing vector comparison during vector search over this vector. Default varies by vector store type. | +| StoragePropertyName | No | Can be used to supply an alternative name for the property in the database. Note that this parameter is not supported by all connectors, e.g. where alternatives like `JsonPropertyNameAttribute` is supported. | + +> [!TIP] +> For more information on which connectors support StoragePropertyName and what alternatives are available, refer to [the documentation for each connector](./out-of-the-box-connectors/index.md). + +::: zone-end +::: zone pivot="programming-language-python" + +Here is an example of how to create a record definition, for use with a [pandas DataFrame](https://pandas.pydata.org/docs/reference/frame.html). + +> [!Note] +> The same fields as in the [data model definition](./defining-your-data-model.md) are used here, for a datamodel they are added as annotations, here as a dict with the name. + +There are a couple of important things to note, other then the fields definitions themselves. The first is the `container_mode` parameter. When set to True, this indicates that the data model is a container type, like a DataFrame, and that the data model is therefore a container of records, instead of a single one, a container record can be used in the exact same way, the main difference is that `get` and `get_batch` will return the same data type, with a single record for a `get` and one or more for a `get_batch`. When you want to do a upsert, `upsert` and `upsert_batch` can be used interchangeably, in other words, passing a container to `upsert` will result in multiple upserts, instead of a single one. + +The second is the addition of the `to_dict` and `from_dict` methods, which are used to convert between the data model and the storage schema. In this case, the `to_dict` method is used to convert the DataFrame to a list of records, and the `from_dict` method is used to convert a list of records to a DataFrame. There can also be a `serialize` and `deserialize` method (not shown in the example below), for details on the difference between those see the [serialization documentation](./serialization.md). + +```python +from semantic_kernel.data import ( + VectorStoreRecordDataField, + VectorStoreRecordDefinition, + VectorStoreRecordKeyField, + VectorStoreRecordVectorField, +) + +hotel_definition = VectorStoreRecordDefinition( + fields={ + "hotel_id": VectorStoreRecordKeyField(property_type="str"), + "hotel_name": VectorStoreRecordDataField(property_type="str", is_filterable=True), + "description": VectorStoreRecordDataField( + property_type="str", has_embedding=True, embedding_property_name="description_embedding" + ), + "description_embedding": VectorStoreRecordVectorField(property_type="list[float]"), + }, + container_mode=True, + to_dict=lambda record, **_: record.to_dict(orient="records"), + from_dict=lambda records, **_: DataFrame(records), +) +``` + +When creating a definition you always have to provide a name (as the key in the `fields` dict) and type for each property in your schema, since this is required for index creation and data mapping. + +To use the definition, pass it to the GetCollection method or a collection constructor, together with the data model type. + +```python +collection = vector_store.get_collection( + collection_name="skhotels", + data_model_type=pd.DataFrame, + data_model_definition=hotel_definition, +) +``` + +::: zone-end +::: zone pivot="programming-language-java" + +## Coming soon + +More info coming soon. + +::: zone-end diff --git a/semantic-kernel/concepts/vector-store-connectors/serialization.md b/semantic-kernel/concepts/vector-store-connectors/serialization.md new file mode 100644 index 00000000..56a48fc4 --- /dev/null +++ b/semantic-kernel/concepts/vector-store-connectors/serialization.md @@ -0,0 +1,73 @@ +--- +title: Serialization of your data model to and from different stores (Preview) +description: How does Semantic Kernel serialize your data model to and from different stores +author: edvan +zone_pivot_groups: programming-languages +ms.topic: conceptual +ms.author: edvan +ms.date: 08/15/2024 +ms.service: semantic-kernel +--- +# Serialization of your data model to and from different stores (Preview) + +::: zone pivot="programming-language-csharp" + +## Coming soon + +More info coming soon. + +::: zone-end +::: zone pivot="programming-language-python" + +In order for your data model defined either as a [class](./defining-your-data-model.md) or a [definition](./schema-with-record-definition.md) to be stored in a database, it needs to be serialized to a format that the database can understand. + +There are two ways that can be done, either by using the built-in serialization provided by the Semantic Kernel or by providing your own serialization logic. + +## Serialization options + +### Built-in serialization + +The built-in serialization is done by first converting the data model to a dictionary and then serializing it to the model that that store understands, for each store that is different and defined as part of the built-in connector. Deserialization is done in the reverse order. + +#### Custom to and from dict methods + +The built-in serialization can also use custom methods to go from the data model to a dictionary and from a dictionary to the data model. This can be done by implementing methods from the `VectorStoreModelToDictFromDictProtocol` for a class or functions following the `ToDictProtocol` and `FromDictProtocol` protocols in your record definition, both can be found in `semantic_kernel/data/vector_store_model_protocols.py`. + +This is especially useful when you want to use a optimized, container format in your code, but still want to be able to move between stores easily. + +#### Pydantic models +When you define you model using a Pydantic BaseModel, it will use the `model_dump` and `model_validate` methods to serialize and deserialize the data model to and from a dict. + +### Custom serialization +You can also define the serialization to be done directly from your model into the model of the data store. + +This can be done by implementing the `VectorStoreModelFunctionSerdeProtocol` protocol, or by adding functions that follow the `SerializeProtocol` and `DeserializeProtocol` in your record definition, both can be found in `semantic_kernel/data/vector_store_model_protocols.py`. + +## Serialization of vectors + +When you have a vector in your data model, it needs to either be a list of floats or list of ints, since that is what most stores need, if you want your class to store the vector in a different format, you can use the `serialize_function` and `deserialize_function` defined in the `VectorStoreRecordVectorField` annotation. For instance for a numpy array you can use the following annotation: + +```python +import numpy as np + +vector: Annotated[ + np.ndarray | None, + VectorStoreRecordVectorField( + dimensions=1536, + serialize_function=np.ndarray.tolist, + deserialize_function=np.array, + ), +] = None +``` + +If you do use a vector store that can handle native numpy arrays and you don't want to have them converted back and forth, you should setup the direct serialization and deserialization for the model and that store. + +::: zone-end +::: zone pivot="programming-language-java" + +## Coming soon + +More info coming soon. + +::: zone-end + diff --git a/semantic-kernel/concepts/vector-store-connectors/vector-search.md b/semantic-kernel/concepts/vector-store-connectors/vector-search.md new file mode 100644 index 00000000..b8c494d2 --- /dev/null +++ b/semantic-kernel/concepts/vector-store-connectors/vector-search.md @@ -0,0 +1,241 @@ +--- +title: Vector seearch using Semantic Kernel Vector Store connectors (Preview) +description: Describes the different options you can use when doing a vector search using Semantic Kernel vector store connectors. +zone_pivot_groups: programming-languages +author: westey-m +ms.topic: conceptual +ms.author: westey +ms.date: 09/23/2024 +ms.service: semantic-kernel +--- +# Vector search using Semantic Kernel Vector Store connectors (Preview) + +> [!WARNING] +> The Semantic Kernel Vector Store functionality is in preview, and improvements that require breaking changes may still occur in limited circumstances before release. + +Semantic Kernel provides vector search capabilities as part of its Vector Store abstractions. This supports filtering and many other options, which this article will explain in more detail. + +::: zone pivot="programming-language-csharp" + +## Vector Search + +The `VectorizedSearchAsync` method allows searching using data that has already been vectorized. This method takes a vector and an optional `VectorSearchOptions` class as input. +This method is available on the following interfaces: + +1. `IVectorizedSearch` +2. `IVectorStoreRecordCollection` + +Note that `IVectorStoreRecordCollection` inherits from `IVectorizedSearch`. + +Assuming you have a collection that already contains data, you can easily search it. Here is an example using Qdrant. + +```csharp +using Microsoft.SemanticKernel.Connectors.Qdrant; +using Microsoft.Extensions.VectorData; +using Qdrant.Client; + +// Create a Qdrant VectorStore object and choose an existing collection that already contains records. +IVectorStore vectorStore = new QdrantVectorStore(new QdrantClient("localhost")); +IVectorStoreRecordCollection collection = vectorStore.GetCollection("skhotels"); + +// Generate a vector for your search text, using your chosen embedding generation implementation. +// Just showing a placeholder method here for brevity. +var searchVector = await GenerateEmbeddingAsync("I'm looking for a hotel where customer happiness is the priority."); + +// Do the search, passing an options object with a Top value to limit resulst to the single top match. +var searchResult = await collection.VectorizedSearchAsync(searchVector, new() { Top = 1 }).Results.ToListAsync(); + +// Inspect the returned hotel. +Hotel hotel = searchResult.First().Record; +Console.WriteLine("Found hotel description: " + hotel.Description); +``` + +> [!TIP] +> For more information on how to generate embeddings see [embedding generation](./embedding-generation.md). + +## Supported Vector Types + +`VectorizedSearchAsync` takes a generic type as the vector parameter. +The types of vectors supported y each data store vary. +See [the documentation for each connector](./out-of-the-box-connectors/index.md) for the list of supported vector types. + +It is also important for the search vector type to match the target vector that is being searched, e.g. if you have two vectors +on the same record with different vector types, make sure that the search vector you supply matches the type of the specific vector +you are targeting. +See [VectorPropertyName](#vectorpropertyname) for how to pick a target vector if you have more than one per record. + +## Vector Search Options + +The following options can be provided using the `VectorSearchOptions` class. + +### VectorPropertyName + +The `VectorPropertyName` option can be used to specify the name of the vector property to target during the search. +If none is provided, the first vector found on the data model or specified in the record definition will be used. + +Note that when specifying the `VectorPropertyName`, use the name of the property as defined on the data model or in the record definition. +Use this property name even if the property may be stored under a different name in the vector store. The storage name may e.g. be different +because of custom serialization settings. + +```csharp +using Microsoft.Extensions.VectorData; +using Microsoft.Connectors.Memory.InMemory; + +var vectorStore = new InMemoryVectorStore(); +var collection = vectorStore.GetCollection("skproducts"); + +// Create the vector search options and indicate that we want to search the FeatureListEmbedding property. +var vectorSearchOptions = new VectorSearchOptions +{ + VectorPropertyName = nameof(Product.FeatureListEmbedding) +}; + +// This snippet assumes searchVector is already provided, having been created using the embedding model of your choice. +var searchResult = await collection.VectorizedSearchAsync(searchVector, vectorSearchOptions).Results.ToListAsync(); + +public sealed class Product +{ + [VectorStoreRecordKey] + public int Key { get; set; } + + [VectorStoreRecordData] + public string Description { get; set; } + + [VectorStoreRecordData] + public List FeatureList { get; set; } + + [VectorStoreRecordVector(1536)] + public ReadOnlyMemory DescriptionEmbedding { get; set; } + + [VectorStoreRecordVector(1536)] + public ReadOnlyMemory FeatureListEmbedding { get; set; } +} +``` + +### Top and Skip + +The `Top` and `Skip` options allow you to limit the number of results to the Top n results and +to skip a number of results from the top of the resultset. +Top and Skip can be used to do paging if you wish to retrieve a large number of results using separate calls. + +```csharp +// Create the vector search options and indicate that we want to skip the first 40 results and then get the next 20. +var vectorSearchOptions = new VectorSearchOptions +{ + Top = 20, + Skip = 40 +}; + +// This snippet assumes searchVector is already provided, having been created using the embedding model of your choice. +var searchResult = await collection.VectorizedSearchAsync(searchVector, vectorSearchOptions).Results.ToListAsync(); +``` + +The default values for `Top` is 3 and `Skip` is 0. + +### IncludeVectors + +The `IncludeVectors` option allows you to specify whether you wish to return vectors in the search results. +If `false`, the vector properties on the returned model will be left null. +Using `false` can significantly reduce the amount of data retrieved from the vector store during search, +making searches more efficient. + +The default value for `IncludeVectors` is `false`. + +```csharp +// Create the vector search options and indicate that we want to include vectors in the search results. +var vectorSearchOptions = new VectorSearchOptions +{ + IncludeVectors = true +} +// This snippet assumes searchVector is already provided, having been created using the embedding model of your choice. +var searchResult = await collection.VectorizedSearchAsync(searchVector, vectorSearchOptions).Results.ToListAsync() +``` + +### VectorSearchFilter + +The `VectorSearchFilter` option can be used to provide a filter for filtering the records in the chosen collection +before applying the vector search. + +This has multiple benefits: + +- Reduce latency and processing cost, since only records remaining after filtering need to be compared with the search vector and therefore fewer vector comparisons have to be done. +- Limit the resultset for e.g. access control purposes, by excluding data that the user shouldn't have access to. + +Note that in order for fields to be used for filtering, many vector stores require those fields to be indexed first. +Some vector stores will allow filtering using any field, but may optionally allow indexing to improve filtering performance. + +If creating a collection via the Semantic Kernel vector store abstractions and you wish to enable filtering on a field, +set the `IsFilterable` property to true when defining your data model or when creating your record definition. + +> [!TIP] +> For more information on how to set the `IsFilterable` property, refer to [VectorStoreRecordDataAttribute parameters](./defining-your-data-model.md#vectorstorerecorddataattribute-parameters) or [VectorStoreRecordDataProperty configuration settings](./schema-with-record-definition.md#vectorstorerecorddataproperty-configuration-settings). + +To create a filter use the `VectorSearchFilter` class. You can combine multiple filter clauses together in one `VectorSearchFilter`. +All filter clauses are combined with `and`. +Note that when providing a property name when constructing the filter, use the name of the property as defined on the data model or in the record definition. +Use this property name even if the property may be stored under a different name in the vector store. The storage name may e.g. be different +because of custom serialization settings. + +```csharp +// Filter where Category == 'External Definitions' and Tags contain 'memory'. +var filter = new VectorSearchFilter() + .EqualTo(nameof(Glossary.Category), "External Definitions") + .AnyTagEqualTo(nameof(Glossary.Tags), "memory"); + +// Create the vector search options and set the filter on the options. +var vectorSearchOptions = new VectorSearchOptions +{ + Filter = filter +}; + +// This snippet assumes searchVector is already provided, having been created using the embedding model of your choice. +searchResult = await collection.VectorizedSearchAsync(searchVector, vectorSearchOptions).Results.ToListAsync(); + +private sealed class Glossary +{ + [VectorStoreRecordKey] + public ulong Key { get; set; } + + // Category is marked as filterable, since we want to filter using this property. + [VectorStoreRecordData(IsFilterable = true)] + public string Category { get; set; } + + // Tags is marked as filterable, since we want to filter using this property. + [VectorStoreRecordData(IsFilterable = true)] + public List Tags { get; set; } + + [VectorStoreRecordData] + public string Term { get; set; } + + [VectorStoreRecordData] + public string Definition { get; set; } + + [VectorStoreRecordVector(1536)] + public ReadOnlyMemory DefinitionEmbedding { get; set; } +} +``` + +#### EqualTo filter clause + +Use `EqualTo` for a direct comparison between property and value. + +#### AnyTagEqualTo filter clause + +Use `AnyTagEqualTo` to check if any of the strings, stored in a tag property in the vector store, contains a provided value. +For a property to be considered a tag property, it needs to be a List, array or other enumerable of string. + +::: zone-end +::: zone pivot="programming-language-python" + +## Coming soon + +More info coming soon. + +::: zone-end +::: zone pivot="programming-language-java" + +## Coming soon + +More info coming soon. + +::: zone-end diff --git a/semantic-kernel/docfx.json b/semantic-kernel/docfx.json index 175f9271..7e93b1f4 100644 --- a/semantic-kernel/docfx.json +++ b/semantic-kernel/docfx.json @@ -27,7 +27,8 @@ "**/*.svg", "**/*.png", "**/*.jpg", - "**/*.gif" + "**/*.gif", + "media/vector-store-data-ingestion-input.zip" ], "exclude": [ "**/obj/**", diff --git a/semantic-kernel/get-started/TOC.yml b/semantic-kernel/get-started/TOC.yml index 80e3bac6..94175c79 100644 --- a/semantic-kernel/get-started/TOC.yml +++ b/semantic-kernel/get-started/TOC.yml @@ -4,7 +4,3 @@ href: detailed-samples.md - name: Supported languages href: supported-languages.md -- name: Contributing to Semantic Kernel - href: contributing.md -- name: Hackathon materials - href: hackathon-materials.md \ No newline at end of file diff --git a/semantic-kernel/get-started/detailed-samples.md b/semantic-kernel/get-started/detailed-samples.md index 49324627..df4ed756 100644 --- a/semantic-kernel/get-started/detailed-samples.md +++ b/semantic-kernel/get-started/detailed-samples.md @@ -42,128 +42,6 @@ If you want to dive into deeper into Semantic Kernel and learn how to use more a :::column-end::: :::row-end::: -::: zone pivot="programming-language-csharp" -| Category | Sample Name | Link | -| ------------------------------ | --------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------ | -| **Agents** | ComplexChat_NestedShopper | [Link](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/Agents/ComplexChat_NestedShopper.cs) | -| | Legacy_AgentAuthoring | [Link](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/Agents/Legacy_AgentAuthoring.cs) | -| | Legacy_AgentCharts | [Link](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/Agents/Legacy_AgentCharts.cs) | -| | Legacy_AgentCollaboration | [Link](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/Agents/Legacy_AgentCollaboration.cs) | -| | Legacy_AgentDelegation | [Link](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/Agents/Legacy_AgentDelegation.cs) | -| | Legacy_AgentTools | [Link](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/Agents/Legacy_AgentTools.cs) | -| | Legacy_Agents | [Link](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/Agents/Legacy_Agents.cs) | -| | Legacy_ChatCompletionAgent | [Link](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/Agents/Legacy_ChatCompletionAgent.cs) | -| | MixedChat_Agents | [Link](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/Agents/MixedChat_Agents.cs) | -| | OpenAIAssistant_ChartMaker | [Link](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/Agents/OpenAIAssistant_ChartMaker.cs) | -| | OpenAIAssistant_CodeInterpreter | [Link](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/Agents/OpenAIAssistant_CodeInterpreter.cs) | -| | OpenAIAssistant_Retrieval | [Link](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/Agents/OpenAIAssistant_Retrieval.cs) | -| **Audio to text** | OpenAI_AudioToText | [Link](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/AudioToText/OpenAI_AudioToText.cs) | -| Automatic function calling | Gemini_FunctionCalling | [Link](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/AutoFunctionCalling/Gemini_FunctionCalling.cs) | -| | OpenAI_FunctionCalling | [Link](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/AutoFunctionCalling/OpenAI_FunctionCalling.cs) | -| **Semantic caching** | SemanticCachingWithFilters | [Link](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/Caching/SemanticCachingWithFilters.cs) | -| **Chat completion** | AzureOpenAIWithData_ChatCompletion | [Link](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/ChatCompletion/AzureOpenAIWithData_ChatCompletion.cs) | -| | ChatHistoryAuthorName | [Link](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/ChatCompletion/ChatHistoryAuthorName.cs) | -| | ChatHistorySerialization | [Link](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/ChatCompletion/ChatHistorySerialization.cs) | -| | Connectors_CustomHttpClient | [Link](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/ChatCompletion/Connectors_CustomHttpClient.cs) | -| | Connectors_KernelStreaming | [Link](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/ChatCompletion/Connectors_KernelStreaming.cs) | -| | Connectors_WithMultipleLLMs | [Link](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/ChatCompletion/Connectors_WithMultipleLLMs.cs) | -| | Google_GeminiChatCompletion | [Link](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/ChatCompletion/Google_GeminiChatCompletion.cs) | -| | Google_GeminiChatCompletionStreaming | [Link](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/ChatCompletion/Google_GeminiChatCompletionStreaming.cs) | -| | Google_GeminiGetModelResult | [Link](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/ChatCompletion/Google_GeminiGetModelResult.cs) | -| | Google_GeminiVision | [Link](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/ChatCompletion/Google_GeminiVision.cs) | -| | OpenAI_ChatCompletion | [Link](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/ChatCompletion/OpenAI_ChatCompletion.cs) | -| | OpenAI_ChatCompletionMultipleChoices | [Link](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/ChatCompletion/OpenAI_ChatCompletionMultipleChoices.cs) | -| | OpenAI_ChatCompletionStreaming | [Link](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/ChatCompletion/OpenAI_ChatCompletionStreaming.cs) | -| | OpenAI_ChatCompletionStreamingMultipleChoices | [Link](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/ChatCompletion/OpenAI_ChatCompletionStreamingMultipleChoices.cs) | -| | OpenAI_ChatCompletionWithVision | [Link](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/ChatCompletion/OpenAI_ChatCompletionWithVision.cs) | -| | OpenAI_CustomAzureOpenAIClient | [Link](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/ChatCompletion/OpenAI_CustomAzureOpenAIClient.cs) | -| | OpenAI_UsingLogitBias | [Link](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/ChatCompletion/OpenAI_UsingLogitBias.cs) | -| | OpenAI_FunctionCalling | [Link](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/ChatCompletion/OpenAI_FunctionCalling.cs) | -| | MistralAI_ChatPrompt | [Link](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/ChatCompletion/MistralAI_ChatPrompt.cs) | -| | MistralAI_FunctionCalling | [Link](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/ChatCompletion/MistralAI_FunctionCalling.cs) | -| | MistralAI_StreamingFunctionCalling | [Link](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/ChatCompletion/MistralAI_StreamingFunctionCalling.cs) | -| **Dependency injection** | HttpClient_Registration | [Link](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/DependencyInjection/HttpClient_Registration.cs) | -| | HttpClient_Resiliency | [Link](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/DependencyInjection/HttpClient_Resiliency.cs) | -| | Kernel_Building | [Link](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/DependencyInjection/Kernel_Building.cs) | -| | Kernel_Injecting | [Link](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/DependencyInjection/Kernel_Injecting.cs) | -| **Filtering** | AutoFunctionInvocationFiltering | [Link](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/Filtering/AutoFunctionInvocationFiltering.cs) | -| | FunctionInvocationFiltering | [Link](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/Filtering/FunctionInvocationFiltering.cs) | -| | Legacy_KernelHooks | [Link](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/Filtering/Legacy_KernelHooks.cs) | -| | PromptRenderFiltering | [Link](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/Filtering/PromptRenderFiltering.cs) | -| | RetryWithFilters | [Link](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/Filtering/RetryWithFilters.cs) | -| | PIIDetectionWithFilters | [Link](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/Filtering/PIIDetectionWithFilters.cs) | -| **Plugin functions** | Arguments | [Link](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/Functions/Arguments.cs) | -| | FunctionResult_Metadata | [Link](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/Functions/FunctionResult_Metadata.cs) | -| | FunctionResult_StronglyTyped | [Link](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/Functions/FunctionResult_StronglyTyped.cs) | -| | MethodFunctions | [Link](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/Functions/MethodFunctions.cs) | -| | MethodFunctions_Advanced | [Link](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/Functions/MethodFunctions_Advanced.cs) | -| | MethodFunctions_Types | [Link](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/Functions/MethodFunctions_Types.cs) | -| | PromptFunctions_Inline | [Link](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/Functions/PromptFunctions_Inline.cs) | -| | PromptFunctions_MultipleArguments | [Link](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/Functions/PromptFunctions_MultipleArguments.cs) | -| **Image to text** | HuggingFace_ImageToText | [Link](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/ImageToText/HuggingFace_ImageToText.cs) | -| **Local models** | HuggingFace_ChatCompletionWithTGI | [Link](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/LocalModels/HuggingFace_ChatCompletionWithTGI.cs) | -| | MultipleProviders_ChatCompletion | [Link](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/LocalModels/MultipleProviders_ChatCompletion.cs) | -| **Memory** | HuggingFace_EmbeddingGeneration | [Link](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/Memory/HuggingFace_EmbeddingGeneration.cs) | -| | MemoryStore_CustomReadOnly | [Link](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/Memory/MemoryStore_CustomReadOnly.cs) | -| | SemanticTextMemory_Building | [Link](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/Memory/SemanticTextMemory_Building.cs) | -| | TextChunkerUsage | [Link](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/Memory/TextChunkerUsage.cs) | -| | TextChunkingAndEmbedding | [Link](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/Memory/TextChunkingAndEmbedding.cs) | -| | TextMemoryPlugin_GeminiEmbeddingGeneration | [Link](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/Memory/TextMemoryPlugin_GeminiEmbeddingGeneration.cs) | -| | TextMemoryPlugin_MultipleMemoryStore | [Link](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/Memory/TextMemoryPlugin_MultipleMemoryStore.cs) | -| **Planning** | FunctionCallStepwisePlanning | [Link](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/Planners/FunctionCallStepwisePlanning.cs) | -| | HandlebarsPlanning | [Link](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/Planners/HandlebarsPlanning.cs) | -| **Plugins** | ApiManifestBasedPlugins | [Link](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/Plugins/ApiManifestBasedPlugins.cs) | -| | ConversationSummaryPlugin | [Link](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/Plugins/ConversationSummaryPlugin.cs) | -| | CreatePluginFromOpenAI_AzureKeyVault | [Link](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/Plugins/CreatePluginFromOpenAI_AzureKeyVault.cs) | -| | CreatePluginFromOpenApiSpec_Github | [Link](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/Plugins/CreatePluginFromOpenApiSpec_Github.cs) | -| | CreatePluginFromOpenApiSpec_Jira | [Link](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/Plugins/CreatePluginFromOpenApiSpec_Jira.cs) | -| | CustomMutablePlugin | [Link](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/Plugins/CustomMutablePlugin.cs) | -| | DescribeAllPluginsAndFunctions | [Link](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/Plugins/DescribeAllPluginsAndFunctions.cs) | -| | GroundednessChecks | [Link](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/Plugins/GroundednessChecks.cs) | -| | ImportPluginFromGrpc | [Link](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/Plugins/ImportPluginFromGrpc.cs) | -| | OpenAIPlugins | [Link](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/Plugins/OpenAIPlugins.cs) | -| **Prompt templates** | ChatCompletionPrompts | [Link](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/PromptTemplates/ChatCompletionPrompts.cs) | -| | ChatWithPrompts | [Link](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/PromptTemplates/ChatWithPrompts.cs) | -| | LiquidPrompts | [Link](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/PromptTemplates/LiquidPrompts.cs) | -| | MultiplePromptTemplates | [Link](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/PromptTemplates/MultiplePromptTemplates.cs) | -| | PromptFunctionsWithChatGPT | [Link](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/PromptTemplates/PromptFunctionsWithChatGPT.cs) | -| | TemplateLanguage | [Link](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/PromptTemplates/TemplateLanguage.cs) | -| | PromptyFunction | [Link](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/PromptTemplates/PromptyFunction.cs) | -| **Retrieval augmented generation** | WithFunctionCallingStepwisePlanner | [Link](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/RAG/WithFunctionCallingStepwisePlanner.cs) | -| | WithPlugins | [Link](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/RAG/WithPlugins.cs) | -| Search | BingAndGooglePlugins | [Link](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/Search/BingAndGooglePlugins.cs) | -| | MyAzureAISearchPlugin | [Link](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/Search/MyAzureAISearchPlugin.cs) | -| | WebSearchQueriesPlugin | [Link](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/Search/WebSearchQueriesPlugin.cs) | -| **Text generation** | Custom_TextGenerationService | [Link](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/TextGeneration/Custom_TextGenerationService.cs) | -| | HuggingFace_TextGeneration | [Link](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/TextGeneration/HuggingFace_TextGeneration.cs) | -| | OpenAI_TextGenerationStreaming | [Link](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/TextGeneration/OpenAI_TextGenerationStreaming.cs) | -| **Text to audio** | OpenAI_TextToAudio | [Link](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/TextToAudio/OpenAI_TextToAudio.cs) | -| **Text to image** | OpenAI_TextToImage | [Link](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/TextToImage/OpenAI_TextToImageDalle3.cs) | - -::: zone-end - -::: zone pivot="programming-language-python" -| Features | Description | Link | -| -------- | ----------- | ---- | -| AutoFunctionCalling | Using `Auto Function Calling` to allow function call capable models to invoke Kernel Functions automatically | [Link](https://github.com/microsoft/semantic-kernel/tree/main/python/samples/concepts/auto_function_calling) | -| ChatCompletion | Using `ChatCompletion` messaging capable service with models | [Link](https://github.com/microsoft/semantic-kernel/tree/main/python/samples/concepts/chat_completion) | -| Filtering | Creating and using Filters | [Link](https://github.com/microsoft/semantic-kernel/tree/main/python/samples/concepts/filtering) | -| Functions | Invoking `Method` or `Prompt` functions with `Kernel` | [Link](https://github.com/microsoft/semantic-kernel/tree/main/python/samples/concepts/functions) | -| Grounding | An example of how to perform LLM grounding | [Link](https://github.com/microsoft/semantic-kernel/tree/main/python/samples/concepts/grounding) | -| Logging | Showing how to set up logging | [Link](https://github.com/microsoft/semantic-kernel/tree/main/python/samples/concepts/logging) | -| Memory | Using `Memory` AI concepts | [Link](https://github.com/microsoft/semantic-kernel/tree/main/python/samples/concepts/memory) | -| On Your Data | Examples of using AzureOpenAI `On Your Data` | [Link](https://github.com/microsoft/semantic-kernel/tree/main/python/samples/concepts/on_your_data) | -| Planners | Showing the uses of `Planners` | [Link](https://github.com/microsoft/semantic-kernel/tree/main/python/samples/concepts/planners) | -| Plugins | Different ways of creating and using `Plugins` | [Link](https://github.com/microsoft/semantic-kernel/tree/main/python/samples/concepts/plugins) | -| PromptTemplates | Using `Templates` with parametrization for `Prompt` rendering | [Link](https://github.com/microsoft/semantic-kernel/tree/main/python/samples/concepts/prompt_templates) | -| RAG | Different ways of `RAG` (Retrieval-Augmented Generation) | [Link](https://github.com/microsoft/semantic-kernel/tree/main/python/samples/concepts/rag) | -| Search | Using search services information | [Link](https://github.com/microsoft/semantic-kernel/tree/main/python/samples/concepts/search) | -| Service Selector | Shows how to create and use a custom service selector class. | [Link](https://github.com/microsoft/semantic-kernel/tree/main/python/samples/concepts/service_selector) | -| Setup | How to setup environment variables for Semantic Kernel | [Link](https://github.com/microsoft/semantic-kernel/tree/main/python/samples/concepts/setup) | -| TextGeneration | Using [`TextGeneration`](https://github.com/microsoft/semantic-kernel/blob/main/python/semantic_kernel/connectors/ai/text_completion_client_base.py) capable service with models | [Link](https://github.com/microsoft/semantic-kernel/tree/main/python/samples/concepts/text_generation) | - -::: zone-end - ::: zone pivot="programming-language-java" - [All Java concept samples](https://github.com/microsoft/semantic-kernel-java/tree/main/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples) diff --git a/semantic-kernel/get-started/quick-start-guide.md b/semantic-kernel/get-started/quick-start-guide.md index adc3c9bb..a130b7cd 100644 --- a/semantic-kernel/get-started/quick-start-guide.md +++ b/semantic-kernel/get-started/quick-start-guide.md @@ -80,7 +80,7 @@ Instructions for accessing the `SemanticKernel` Java package is available [here] ## Quickly get started with notebooks If you're a Python or C# developer, you can quickly get started with our notebooks. These notebooks provide step-by-step guides on how to use Semantic Kernel to build AI agents. -![Semantic Kernel notebooks](../media/notebook-image.png) +![Semantic Kernel notebooks](../media/python_getting_started_notebooks.png) ::: zone-end @@ -92,7 +92,7 @@ To get started, follow these steps: 4. Open _00-getting-started.ipynb_ to get started setting your environment and creating your first AI agent! ::: zone-end -::: zone pivot="programming-language-python" +::: zone pivot="programming-language-csharp" To get started, follow these steps: 1. Clone the [Semantic Kernel repo](https://github.com/microsoft/semantic-kernel) 2. Open the repo in Visual Studio Code @@ -150,7 +150,7 @@ kernel.Plugins.AddFromType("Lights"); // Enable planning OpenAIPromptExecutionSettings openAIPromptExecutionSettings = new()  { - ToolCallBehavior = ToolCallBehavior.AutoInvokeKernelFunctions + FunctionChoiceBehavior = FunctionChoiceBehavior.Auto() }; // Create a history store the conversation @@ -189,6 +189,7 @@ do { import asyncio from semantic_kernel import Kernel +from semantic_kernel.utils.logging import setup_logging from semantic_kernel.functions import kernel_function from semantic_kernel.connectors.ai.open_ai import AzureChatCompletion from semantic_kernel.connectors.ai.function_call_behavior import FunctionCallBehavior @@ -205,17 +206,15 @@ async def main(): kernel = Kernel() # Add Azure OpenAI chat completion - kernel.add_service(AzureChatCompletion( + chat_completion = AzureChatCompletion( deployment_name="your_models_deployment_name", api_key="your_api_key", base_url="your_base_url", - )) + ) + kernel.add_service(chat_completion) # Set the logging level for semantic_kernel.kernel to DEBUG. - logging.basicConfig( - format="[%(asctime)s - %(name)s:%(lineno)d - %(levelname)s] %(message)s", - datefmt="%Y-%m-%d %H:%M:%S", - ) + setup_logging() logging.getLogger("kernel").setLevel(logging.DEBUG) # Add a plugin (the LightsPlugin class is defined below) @@ -224,8 +223,6 @@ async def main(): plugin_name="Lights", ) - chat_completion : AzureChatCompletion = kernel.get_service(type=ChatCompletionClientBase) - # Enable planning execution_settings = AzureChatPromptExecutionSettings(tool_choice="auto") execution_settings.function_call_behavior = FunctionCallBehavior.EnableFunctions(auto_invoke=True, filters={}) @@ -247,12 +244,11 @@ async def main(): history.add_user_message(userInput) # Get the response from the AI - result = (await chat_completion.get_chat_message_contents( + result = await chat_completion.get_chat_message_content( chat_history=history, settings=execution_settings, kernel=kernel, - arguments=KernelArguments(), - ))[0] + ) # Print the results print("Assistant > " + str(result)) @@ -291,7 +287,7 @@ If you're interested in understanding more about the code above, we'll break it To make it easier to get started building enterprise apps with Semantic Kernel, we've created a step-by-step that guides you through the process of creating a kernel and using it to interact with AI services. ::: zone pivot="programming-language-python" -![Semantic Kernel Python map](../media/PythonMap.png) +![Semantic Kernel Python map](../media/UpdatedPythonMap.png) ::: zone-end ::: zone pivot="programming-language-csharp" @@ -627,7 +623,7 @@ kernel.add_plugin( ### 9) Planning -Semantic Kernel leverages [function calling](../concepts/ai-services/chat-completion/function-calling.md)–a native feature of most LLMs–to provide [planning](../concepts/planning.md). With function calling, LLMs can request (or call) a particular function to satisfy a user's request. Semantic Kernel then marshals the request to the appropriate function in your codebase and returns the results back to the LLM so the AI agent can generate a final response. +Semantic Kernel leverages [function calling](../concepts/ai-services/chat-completion/function-calling/index.md)–a native feature of most LLMs–to provide [planning](../concepts/planning.md). With function calling, LLMs can request (or call) a particular function to satisfy a user's request. Semantic Kernel then marshals the request to the appropriate function in your codebase and returns the results back to the LLM so the AI agent can generate a final response. To enable automatic function calling, we first need to create the appropriate execution settings so that Semantic Kernel knows to automatically invoke the functions in the kernel when the AI agent requests them. @@ -636,7 +632,7 @@ To enable automatic function calling, we first need to create the appropriate ex ```csharp OpenAIPromptExecutionSettings openAIPromptExecutionSettings = new() { - ToolCallBehavior = ToolCallBehavior.AutoInvokeKernelFunctions + FunctionChoiceBehavior = FunctionChoiceBehavior.Auto() }; ``` diff --git a/semantic-kernel/get-started/supported-languages.md b/semantic-kernel/get-started/supported-languages.md index d8f5f1a1..e50698db 100644 --- a/semantic-kernel/get-started/supported-languages.md +++ b/semantic-kernel/get-started/supported-languages.md @@ -56,6 +56,12 @@ In Python, there's a single package that includes everything you need to get sta pip install semantic-kernel ``` +On [PyPI](https://pypi.org/project/semantic-kernel/) under `Provides-Extra` the additional extras you can install are also listed and when used that will install the packages needed for using SK with that specific connector or service, you can install those with the square bracket syntax for instance: +```bash +pip install semantic-kernel[azure] +``` +This will install Semantic Kernel, as well as specific tested versions of: `azure-ai-inference`, `azure-search-documents`, `azure-core`, `azure-identity`, `azure-cosmos` and `msgraph-sdk` (and any dependencies of those packages). Similarly the extra `hugging_face` will install `transformers` and `sentence-transformers`. + ### Java packages For Java, Semantic Kernel has the following packages; all are under the group Id `com.microsoft.semantic-kernel`, and can be imported @@ -152,19 +158,19 @@ Once you've created a prompt, you can serialize it so that it can be stored or s | YAML | ✅ | ✅ | ✅ | | | Prompty | ❌ | ✅ | ❌ | | -### AI Services +### AI Services Modalities | Services | C# | Python | Java | Notes | |-----------------------------------|:----:|:------:|:----:|-------| | Text Generation | ✅ | ✅ | ✅ | Example: Text-Davinci-003 | | Chat Completion | ✅ | ✅ | ✅ | Example: GPT4, Chat-GPT | | Text Embeddings (Experimental) | ✅ | ✅ | ✅ | Example: Text-Embeddings-Ada-002 | -| Text to Image (Experimental) | ✅ | ❌ | ❌ | Example: Dall-E | -| Image to Text (Experimental) | ✅ | ❌ | ❌ | Example: Pix2Struct | +| Text to Image (Experimental) | ✅ | ✅ | ❌ | Example: Dall-E | +| Image to Text (Experimental) | ✅ | ✅ | ❌ | Example: Pix2Struct | | Text to Audio (Experimental) | ✅ | ❌ | ❌ | Example: Text-to-speech | | Audio to Text (Experimental) | ✅ | ❌ | ❌ | Example: Whisper | -### AI service endpoints +### AI Service Connectors | Endpoints | C# | Python | Java | Notes | |-------------------------------------------|:----:|:------:|:----:|-------| @@ -176,8 +182,7 @@ Once you've created a prompt, you can serialize it so that it can be stored or s ### Memory Connectors (Experimental) > [!IMPORTANT] -> All of the existing memory connectors are currently experimental and are undergoing active development to improve the experience of using them. To provide feedback on the latest proposal, please refer to the active [Search](https://github.com/microsoft/semantic-kernel/pull/6012) and [Memory Connector](https://github.com/microsoft/semantic-kernel/pull/6364) ADRs. - +> All of the existing memory connectors are currently experimental and will be replaced by Vector Store connectors. These will provide more functionality via an updated abstraction layer. | Memory Connectors | C# | Python | Java | Notes | |--------------------------|:----:|:------:|:----:|-------| @@ -192,3 +197,9 @@ Once you've created a prompt, you can serialize it so that it can be stored or s | Sqlite | ✅ | ❌ | 🔄 | | | Weaviate | ✅ | ✅ | ❌ | | +### Vector Store Connectors (Experimental) + +> [!IMPORTANT] +> All of the existing Vector Store connectors are currently experimental and are undergoing active development to improve the experience of using them. To provide feedback on the latest proposal, please refer to the active [Search](https://github.com/microsoft/semantic-kernel/pull/6012) and [Memory Connector](https://github.com/microsoft/semantic-kernel/pull/6364) ADRs. + +For the list of out of the box vector store connectors and the language support for each, refer to [out of the box connectors](../concepts/vector-store-connectors/out-of-the-box-connectors/index.md). diff --git a/semantic-kernel/index.yml b/semantic-kernel/index.yml index 4b3b967f..3f94c798 100644 --- a/semantic-kernel/index.yml +++ b/semantic-kernel/index.yml @@ -21,32 +21,40 @@ productDirectory: links: - url: /semantic-kernel/overview/index text: Introduction to Semantic Kernel - - url: /semantic-kernel/get-started/contributing - text: Contributing to Semantic Kernel + - url: /semantic-kernel/get-started/supported-languages + text: Supported Languages Semantic Kernel - title: Quick Start imageSrc: /semantic-kernel/media/Quick-start.svg links: - url: /semantic-kernel/get-started/quick-start-guide text: Learn how to use Semantic Kernel - links: - url: /semantic-kernel/get-started/detailed-samples text: Check out samples for Semantic Kernel - title: Concepts imageSrc: /semantic-kernel/media/Concepts.svg links: - - url: /semantic-kernel/concepts/agents - text: Agents + - url: /semantic-kernel/concepts/kernel + text: Kernel - url: /semantic-kernel/concepts/plugins/index text: Plugins - - url: /semantic-kernel/concepts/planning - text: Planning - - url: /semantic-kernel/concepts/personas - text: Personas - - title: Integrations + - url: /semantic-kernel/concepts/vector-store-connectors/index + text: Memory + - title: Frameworks imageSrc: /semantic-kernel/media/Integrations.svg links: - - url: /semantic-kernel/integrations/ - text: Browse available features by language + - url: /semantic-kernel/frameworks/process/process-framework + text: Process Framework + - url: /semantic-kernel/frameworks/agent/ + text: Agent Framework + - title: Enterprise Components + imageSrc: /semantic-kernel/media/Security.svg + links: + - url: /semantic-kernel/concepts/enterprise-readiness/observability/index + text: Observability + - url: /semantic-kernel/concepts/enterprise-readiness/SECURITY + text: Security + - url: /semantic-kernel/concepts/enterprise-readiness/filters + text: Filters - title: Connect with us imageSrc: /semantic-kernel/media/whats-new.svg links: diff --git a/semantic-kernel/integrations/index.md b/semantic-kernel/integrations/index.md deleted file mode 100644 index 37657c3b..00000000 --- a/semantic-kernel/integrations/index.md +++ /dev/null @@ -1,83 +0,0 @@ ---- -title: integrations -description: Learn which features are available for C#, Python, and Java through integrations. -author: sophialagerkranspandey -ms.topic: reference -ms.author: sopand -ms.date: 07/11/2023 -ms.service: semantic-kernel ---- - -# Integrations for Semantic Kernel - -Semantic Kernel provides a wide range of integrations to help you build powerful AI agents. These integrations include AI services, memory connectors. Additionally, Semantic Kernel integrates with other Microsoft services to provide additional functionality via plugins. - -## Out-of-the-box integrations - -With the available AI and memory connectors, developers can easily build AI agents with swappable components. This allows you to experiment with different AI services and memory connectors to find the best combination for your use case. - -### AI Services - -| Services | C# | Python | Java | Notes | -|-----------------------------------|:----:|:------:|:----:|-------| -| Text Generation | ✅ | ✅ | ✅ | Example: Text-Davinci-003 | -| Chat Completion | ✅ | ✅ | ✅ | Example: GPT4, Chat-GPT | -| Text Embeddings (Experimental) | ✅ | ✅ | ✅ | Example: Text-Embeddings-Ada-002 | -| Text to Image (Experimental) | ✅ | ❌ | ❌ | Example: Dall-E | -| Image to Text (Experimental) | ✅ | ❌ | ❌ | Example: Pix2Struct | -| Text to Audio (Experimental) | ✅ | ❌ | ❌ | Example: Text-to-speech | -| Audio to Text (Experimental) | ✅ | ❌ | ❌ | Example: Whisper | - - -### Memory Connectors (Experimental) - -Vector databases have many use cases across different domains and applications that involve natural language processing (NLP), computer vision (CV), recommendation systems (RS), and other areas that require semantic understanding and matching of data. - -One use case for storing information in a vector database is to enable large language models (LLMs) to generate more relevant and coherent text based on an [AI plugin](../create-plugins/index.md). - -However, large language models often face challenges such as generating inaccurate or irrelevant information; lacking factual consistency or common sense; repeating or contradicting themselves; being biased or offensive. To overcome these challenges, you can use a vector database to store information about different topics, keywords, facts, opinions, and/or sources related to your desired domain or genre. -Then, you can use a large language model and pass information from the vector database with your AI plugin to generate more informative and engaging content that matches your intent and style. - -For example, if you want to write a blog post about the latest trends in AI, you can use a vector database to store the latest information about that topic and pass the information along with the ask to a LLM in order to generate a blog post that leverages the latest information. - -## Available connectors to vector databases -Today, Semantic Kernel offers several connectors to vector databases that you can use to store and retrieve information. These include: - - -| Service | C# | Python | -|--------------------------|:----:|:------:| -| Vector Database in Azure Cosmos DB for NoSQL | [C#](https://github.com/microsoft/semantic-kernel/tree/main/dotnet/src/Connectors/Connectors.Memory.AzureCosmosDBNoSQL) | [Python](https://github.com/microsoft/semantic-kernel/tree/main/python/semantic_kernel/connectors/memory/azure_cosmosdb_no_sql) -| Vector Database in vCore-based Azure Cosmos DB for MongoDB | [C#](https://github.com/microsoft/semantic-kernel/tree/main/dotnet/src/Connectors/Connectors.Memory.AzureCosmosDBMongoDB) | [Python](https://github.com/microsoft/semantic-kernel/tree/main/python/semantic_kernel/connectors/memory/azure_cosmosdb) | -| Azure AI Search | [C#](https://github.com/microsoft/semantic-kernel/tree/main/dotnet/src/Connectors/Connectors.Memory.AzureAISearch) | [Python](https://github.com/microsoft/semantic-kernel/tree/main/python/semantic_kernel/connectors/memory/azure_cognitive_search) | -| Azure PostgreSQL Server | [C#](https://github.com/microsoft/semantic-kernel/tree/main/dotnet/src/Connectors/Connectors.Memory.Postgres) | -| Azure SQL Database | [C#](https://github.com/kbeaugrand/SemanticKernel.Connectors.Memory.SqlServer) | -| Chroma | [C#](https://github.com/microsoft/semantic-kernel/tree/main/dotnet/src/Connectors/Connectors.Memory.Chroma) | [Python](https://github.com/microsoft/semantic-kernel/tree/main/python/semantic_kernel/connectors/memory/chroma) | -| DuckDB | [C#](https://github.com/microsoft/semantic-kernel/tree/main/dotnet/src/Connectors/Connectors.Memory.DuckDB) | | -| Milvus | [C#](https://github.com/microsoft/semantic-kernel/tree/main/dotnet/src/Connectors/Connectors.Memory.Milvus) | [Python](https://github.com/microsoft/semantic-kernel/tree/main/python/semantic_kernel/connectors/memory/milvus) | -| MongoDB Atlas Vector Search | [C#](https://github.com/microsoft/semantic-kernel/tree/main/dotnet/src/Connectors/Connectors.Memory.MongoDB) | [Python](https://github.com/microsoft/semantic-kernel/tree/main/python/semantic_kernel/connectors/memory/mongodb_atlas) | -| Pinecone | [C#](https://github.com/microsoft/semantic-kernel/tree/main/dotnet/src/Connectors/Connectors.Memory.Pinecone) | [Python](https://github.com/microsoft/semantic-kernel/tree/main/python/semantic_kernel/connectors/memory/pinecone) | -| Postgres | [C#](https://github.com/microsoft/semantic-kernel/tree/main/dotnet/src/Connectors/Connectors.Memory.Postgres) | [Python](https://github.com/microsoft/semantic-kernel/tree/main/python/semantic_kernel/connectors/memory/postgres) | -| Qdrant | [C#](https://github.com/microsoft/semantic-kernel/tree/main/dotnet/src/Connectors/Connectors.Memory.Qdrant) | | -| Redis | [C#](https://github.com/microsoft/semantic-kernel/tree/main/dotnet/src/Connectors/Connectors.Memory.Redis) | | -| Sqlite | [C#](https://github.com/microsoft/semantic-kernel/tree/main/dotnet/src/Connectors/Connectors.Memory.Sqlite) | | -| Weaviate | [C#](https://github.com/microsoft/semantic-kernel/tree/main/dotnet/src/Connectors/Connectors.Memory.Weaviate) | [Python](https://github.com/microsoft/semantic-kernel/tree/main/python/semantic_kernel/connectors/memory/weaviate) | - -### Vector database solutions -- [Azure Cosmos DB for NoSQL](/azure/cosmos-db/nosql/vector-search) Integrated Vector Database with DiskANN -- [Azure Cosmos DB for MongoDB](/azure/cosmos-db/mongodb/vcore/vector-search) Integrated Vector Database -- [Azure SQL Database](/azure/azure-sql/database/ai-artificial-intelligence-intelligent-applications?&preserve-view=true#vector-search) -- [Azure PostgreSQL Server pgvector Extension](/azure/postgresql/flexible-server/how-to-use-pgvector) -- [Azure AI Search](/azure/search/search-what-is-azure-search) -- [Open-source vector databases](/azure/cosmos-db/mongodb/vcore/vector-search-ai) - -:::image type="content" source="../media/decision-guide-databases-and-ai-search.png" lightbox="../media/decision-guide-databases-and-ai-search.png" alt-text="Vector indexing service decision guide"::: - - -## Additional plugins - -If you want to extend the functionality of your AI agent, you can use plugins to integrate with other Microsoft services. Here are some of the plugins that are available for Semantic Kernel: - -| Plugin | C# | Python | Java | Description | -| ---------- | :-: | :----: | :--: | ----------- | -| Logic Apps | ✅ | ✅ | ✅ | Build workflows within Logic Apps using its available connectors and import them as plugins in Semantic Kernel. [Learn more](../concepts/plugins/adding-logic-apps-as-plugins.md). | -| Azure Container Apps Dynamic Sessions | ✅ | ✅ | ❌ | With dynamic sessions, you can recreate the Code Interpreter experience from the Assistants API by effortlessly spinning up Python containers where AI agents can execute Python code. [Learn more](/azure/container-apps/sessions). | diff --git a/semantic-kernel/media/UpdatedPythonMap.png b/semantic-kernel/media/UpdatedPythonMap.png new file mode 100644 index 00000000..9a6116c2 Binary files /dev/null and b/semantic-kernel/media/UpdatedPythonMap.png differ diff --git a/semantic-kernel/media/notebook-image.png b/semantic-kernel/media/notebook-image.png deleted file mode 100644 index ff243fa6..00000000 Binary files a/semantic-kernel/media/notebook-image.png and /dev/null differ diff --git a/semantic-kernel/media/python_getting_started_notebooks.png b/semantic-kernel/media/python_getting_started_notebooks.png new file mode 100644 index 00000000..aa6e5552 Binary files /dev/null and b/semantic-kernel/media/python_getting_started_notebooks.png differ diff --git a/semantic-kernel/media/telemetry-advanced-scenarios-trace-detail-dotnet.png b/semantic-kernel/media/telemetry-advanced-scenarios-trace-detail-dotnet.png new file mode 100644 index 00000000..bf5afcc8 Binary files /dev/null and b/semantic-kernel/media/telemetry-advanced-scenarios-trace-detail-dotnet.png differ diff --git a/semantic-kernel/media/telemetry-advanced-scenarios-trace-detail-error-handling-dotnet.png b/semantic-kernel/media/telemetry-advanced-scenarios-trace-detail-error-handling-dotnet.png new file mode 100644 index 00000000..02d4bcd0 Binary files /dev/null and b/semantic-kernel/media/telemetry-advanced-scenarios-trace-detail-error-handling-dotnet.png differ diff --git a/semantic-kernel/media/telemetry-advanced-scenarios-trace-detail-error-handling-python.png b/semantic-kernel/media/telemetry-advanced-scenarios-trace-detail-error-handling-python.png new file mode 100644 index 00000000..ee91c5a1 Binary files /dev/null and b/semantic-kernel/media/telemetry-advanced-scenarios-trace-detail-error-handling-python.png differ diff --git a/semantic-kernel/media/telemetry-advanced-scenarios-trace-detail-python.png b/semantic-kernel/media/telemetry-advanced-scenarios-trace-detail-python.png new file mode 100644 index 00000000..37b44aa6 Binary files /dev/null and b/semantic-kernel/media/telemetry-advanced-scenarios-trace-detail-python.png differ diff --git a/semantic-kernel/media/telemetry-app-insights-gen-ai-attributes-sensitive.png b/semantic-kernel/media/telemetry-app-insights-gen-ai-attributes-sensitive.png new file mode 100644 index 00000000..f75b8e98 Binary files /dev/null and b/semantic-kernel/media/telemetry-app-insights-gen-ai-attributes-sensitive.png differ diff --git a/semantic-kernel/media/telemetry-app-insights-gen-ai-attributes.png b/semantic-kernel/media/telemetry-app-insights-gen-ai-attributes.png new file mode 100644 index 00000000..717666c3 Binary files /dev/null and b/semantic-kernel/media/telemetry-app-insights-gen-ai-attributes.png differ diff --git a/semantic-kernel/media/telemetry-app-insights-kusto-token-usage-per-message.png b/semantic-kernel/media/telemetry-app-insights-kusto-token-usage-per-message.png new file mode 100644 index 00000000..d3d9dccc Binary files /dev/null and b/semantic-kernel/media/telemetry-app-insights-kusto-token-usage-per-message.png differ diff --git a/semantic-kernel/media/telemetry-app-insights-left-panel-transaction-search-highlight.png b/semantic-kernel/media/telemetry-app-insights-left-panel-transaction-search-highlight.png new file mode 100644 index 00000000..a8dc599f Binary files /dev/null and b/semantic-kernel/media/telemetry-app-insights-left-panel-transaction-search-highlight.png differ diff --git a/semantic-kernel/media/telemetry-app-insights-transaction-search.png b/semantic-kernel/media/telemetry-app-insights-transaction-search.png new file mode 100644 index 00000000..b72cba26 Binary files /dev/null and b/semantic-kernel/media/telemetry-app-insights-transaction-search.png differ diff --git a/semantic-kernel/media/telemetry-aspire-dashboard-trace-detail.png b/semantic-kernel/media/telemetry-aspire-dashboard-trace-detail.png new file mode 100644 index 00000000..85c46665 Binary files /dev/null and b/semantic-kernel/media/telemetry-aspire-dashboard-trace-detail.png differ diff --git a/semantic-kernel/media/telemetry-aspire-dashboard-traces-overview.png b/semantic-kernel/media/telemetry-aspire-dashboard-traces-overview.png new file mode 100644 index 00000000..608b24ab Binary files /dev/null and b/semantic-kernel/media/telemetry-aspire-dashboard-traces-overview.png differ diff --git a/semantic-kernel/media/telemetry-log-events-overview-app-insights.png b/semantic-kernel/media/telemetry-log-events-overview-app-insights.png new file mode 100644 index 00000000..8a43c313 Binary files /dev/null and b/semantic-kernel/media/telemetry-log-events-overview-app-insights.png differ diff --git a/semantic-kernel/media/telemetry-metrics-overview-app-insights.png b/semantic-kernel/media/telemetry-metrics-overview-app-insights.png new file mode 100644 index 00000000..50994b25 Binary files /dev/null and b/semantic-kernel/media/telemetry-metrics-overview-app-insights.png differ diff --git a/semantic-kernel/media/telemetry-trace-overview-app-insights.png b/semantic-kernel/media/telemetry-trace-overview-app-insights.png new file mode 100644 index 00000000..676192a5 Binary files /dev/null and b/semantic-kernel/media/telemetry-trace-overview-app-insights.png differ diff --git a/semantic-kernel/media/vector-store-data-ingestion-input.zip b/semantic-kernel/media/vector-store-data-ingestion-input.zip new file mode 100644 index 00000000..7ca13e68 Binary files /dev/null and b/semantic-kernel/media/vector-store-data-ingestion-input.zip differ diff --git a/semantic-kernel/get-started/contributing.md b/semantic-kernel/support/contributing.md similarity index 98% rename from semantic-kernel/get-started/contributing.md rename to semantic-kernel/support/contributing.md index 35eaebb5..9fb5e042 100644 --- a/semantic-kernel/get-started/contributing.md +++ b/semantic-kernel/support/contributing.md @@ -107,4 +107,4 @@ The continuous integration (CI) system will automatically perform the required b If the CI build fails for any reason, the PR issue will be updated with a link that can be used to determine the cause of the failure so that it can be addressed. ### Contributing to documentation -We also accept contributions to the [Semantic Kernel documentation repository](https://github.com/MicrosoftDocs/semantic-kernel-docs/issues). To learn how to make contributions, please start with the Microsoft [docs contributor guide](/contribute). \ No newline at end of file +We also accept contributions to the [Semantic Kernel documentation repository](https://github.com/MicrosoftDocs/semantic-kernel-docs/issues). \ No newline at end of file diff --git a/semantic-kernel/support/glossary.md b/semantic-kernel/support/glossary.md index added3f7..c11cd625 100644 --- a/semantic-kernel/support/glossary.md +++ b/semantic-kernel/support/glossary.md @@ -31,6 +31,6 @@ ms.service: semantic-kernel ## More support information * [Frequently Asked Questions (FAQs)](/semantic-kernel/support/faqs) -* [Hackathon Materials](/semantic-kernel/support/hackathon) +* [Hackathon Materials](/semantic-kernel/support/hackathon-materials) * [Code of Conduct](/semantic-kernel/support/CodeofConduct) diff --git a/semantic-kernel/get-started/hackathon-materials.md b/semantic-kernel/support/hackathon-materials.md similarity index 100% rename from semantic-kernel/get-started/hackathon-materials.md rename to semantic-kernel/support/hackathon-materials.md diff --git a/semantic-kernel/support/index.md b/semantic-kernel/support/index.md index c05b3a93..d825d098 100644 --- a/semantic-kernel/support/index.md +++ b/semantic-kernel/support/index.md @@ -22,9 +22,10 @@ ms.service: semantic-kernel ## More support information -* [Frequently Asked Questions (FAQs)](/semantic-kernel/support/faqs) -* [Hackathon Materials](/semantic-kernel/support/hackathon) +* [Frequently Asked Questions (FAQs)](/semantic-kernel/support/glossary) +* [Hackathon Materials](/semantic-kernel/support/hackathon-materials) * [Code of Conduct](/semantic-kernel/support/CodeofConduct) +* [Transparency Documentation](https://github.com/microsoft/semantic-kernel/blob/main/TRANSPARENCY_FAQS.md) ## Next step diff --git a/semantic-kernel/support/migration/function-calling-migration-guide.md b/semantic-kernel/support/migration/function-calling-migration-guide.md new file mode 100644 index 00000000..76fd00a1 --- /dev/null +++ b/semantic-kernel/support/migration/function-calling-migration-guide.md @@ -0,0 +1,160 @@ +--- +title: Migrating to the new Function Calling capabilities +description: Describes the steps for SK caller code to migrate from the current function calling capabilities, represented by the `ToolCallBehavior` class, to the new one represented by the `FunctionChoiceBehavior` class. +zone_pivot_groups: programming-languages +author: SergeyMenshykh +ms.topic: conceptual +ms.author: semenshi +ms.service: semantic-kernel +--- +::: zone pivot="programming-language-csharp" +# Function Calling Migration Guide +Semantic Kernel is gradually transitioning from the current function calling capabilities, represented by the `ToolCallBehavior` class, to the new enhanced capabilities, represented by the `FunctionChoiceBehavior` class. +The new capability is service-agnostic and is not tied to any specific AI service, unlike the current model. Therefore, it resides in Semantic Kernel abstractions and will be used by all AI connectors working with function-calling capable AI models. + + +This guide is intended to help you to migrate your code to the new function calling capabilities. + +## Migrate ToolCallBehavior.AutoInvokeKernelFunctions behavior +The `ToolCallBehavior.AutoInvokeKernelFunctions` behavior is equivalent to the `FunctionChoiceBehavior.Auto` behavior in the new model. +```csharp +// Before +var executionSettings = new OpenAIPromptExecutionSettings { ToolCallBehavior = ToolCallBehavior.AutoInvokeKernelFunctions }; + +// After +var executionSettings = new OpenAIPromptExecutionSettings { FunctionChoiceBehavior = FunctionChoiceBehavior.Auto() }; +``` + +## Migrate ToolCallBehavior.EnableKernelFunctions behavior +The `ToolCallBehavior.EnableKernelFunctions` behavior is equivalent to the `FunctionChoiceBehavior.Auto` behavior with disabled auto invocation. +```csharp +// Before +var executionSettings = new OpenAIPromptExecutionSettings { ToolCallBehavior = ToolCallBehavior.EnableKernelFunctions }; + +// After +var executionSettings = new OpenAIPromptExecutionSettings { FunctionChoiceBehavior = FunctionChoiceBehavior.Auto(autoInvoke: false) }; +``` + +## Migrate ToolCallBehavior.EnableFunctions behavior +The `ToolCallBehavior.EnableFunctions` behavior is equivalent to the `FunctionChoiceBehavior.Auto` behavior that configured with list of functions with disabled auto invocation. +```csharp +var function = kernel.CreateFunctionFromMethod(() => DayOfWeek.Friday, "GetDayOfWeek", "Returns the current day of the week."); + +// Before +var executionSettings = new OpenAIPromptExecutionSettings() { ToolCallBehavior = ToolCallBehavior.EnableFunctions(functions: [function.Metadata.ToOpenAIFunction()]) }; + +// After +var executionSettings = new OpenAIPromptExecutionSettings { FunctionChoiceBehavior = FunctionChoiceBehavior.Auto(functions: [function], autoInvoke: false) }; +``` + +## Migrate ToolCallBehavior.RequireFunction behavior +The `ToolCallBehavior.RequireFunction` behavior is equivalent to the `FunctionChoiceBehavior.Required` behavior that configured with list of functions with disabled auto invocation. +```csharp +var function = kernel.CreateFunctionFromMethod(() => DayOfWeek.Friday, "GetDayOfWeek", "Returns the current day of the week."); + +// Before +var executionSettings = new OpenAIPromptExecutionSettings() { ToolCallBehavior = ToolCallBehavior.RequireFunction(functions: [function.Metadata.ToOpenAIFunction()]) }; + +// After +var executionSettings = new OpenAIPromptExecutionSettings { FunctionChoiceBehavior = FunctionChoiceBehavior.Required(functions: [function], autoInvoke: false) }; +``` + +## Replace the usage of connector-specific function call classes +Function calling functionality in Semantic Kernel allows developers to access a list of functions chosen by the AI model in two ways: +- Using connector-specific function call classes like `ChatToolCall` or `ChatCompletionsFunctionToolCall`, available via the `ToolCalls` property of the OpenAI-specific `OpenAIChatMessageContent` item in chat history. +- Using connector-agnostic function call classes like `FunctionCallContent`, available via the `Items` property of the connector-agnostic `ChatMessageContent` item in chat history. + +Both ways are supported at the moment by the current and new models. However, we strongly recommend using the connector-agnostic approach to access function calls, as it is more flexible and allows your code to work with any AI connector that supports the new function-calling model. +Moreover, considering that the current model will be deprecated soon, now is a good time to migrate your code to the new model to avoid breaking changes in the future. + +So, if you use [Manual Function Invocation](../concepts/ai-services/chat-completion/function-calling/function-invocation.md#manual-function-invocation) with the connector-specific function call classes like in this code snippet: +```csharp +using System.Text.Json; +using Microsoft.SemanticKernel; +using Microsoft.SemanticKernel.ChatCompletion; +using Microsoft.SemanticKernel.Connectors.OpenAI; +using OpenAI.Chat; + +var chatHistory = new ChatHistory(); + +var settings = new OpenAIPromptExecutionSettings() { ToolCallBehavior = ToolCallBehavior.EnableKernelFunctions }; + +var result = await chatCompletionService.GetChatMessageContentAsync(chatHistory, settings, kernel); + +// Current way of accessing function calls using connector specific classes. +var toolCalls = ((OpenAIChatMessageContent)result).ToolCalls.OfType().ToList(); + +while (toolCalls.Count > 0) +{ + // Adding function call from AI model to chat history + chatHistory.Add(result); + + // Iterating over the requested function calls and invoking them + foreach (var toolCall in toolCalls) + { + string content = kernel.Plugins.TryGetFunctionAndArguments(toolCall, out KernelFunction? function, out KernelArguments? arguments) ? + JsonSerializer.Serialize((await function.InvokeAsync(kernel, arguments)).GetValue()) : + "Unable to find function. Please try again!"; + + // Adding the result of the function call to the chat history + chatHistory.Add(new ChatMessageContent( + AuthorRole.Tool, + content, + metadata: new Dictionary(1) { { OpenAIChatMessageContent.ToolIdProperty, toolCall.Id } })); + } + + // Sending the functions invocation results back to the AI model to get the final response + result = await chatCompletionService.GetChatMessageContentAsync(chatHistory, settings, kernel); + toolCalls = ((OpenAIChatMessageContent)result).ToolCalls.OfType().ToList(); +} +``` + +You can refactor it to use the connector-agnostic classes: +```csharp +using Microsoft.SemanticKernel; +using Microsoft.SemanticKernel.ChatCompletion; + +var chatHistory = new ChatHistory(); + +var settings = new PromptExecutionSettings() { FunctionChoiceBehavior = FunctionChoiceBehavior.Auto(autoInvoke: false) }; + +var messageContent = await chatCompletionService.GetChatMessageContentAsync(chatHistory, settings, kernel); + +// New way of accessing function calls using connector agnostic function calling model classes. +var functionCalls = FunctionCallContent.GetFunctionCalls(messageContent).ToArray(); + +while (functionCalls.Length != 0) +{ + // Adding function call from AI model to chat history + chatHistory.Add(messageContent); + + // Iterating over the requested function calls and invoking them + foreach (var functionCall in functionCalls) + { + var result = await functionCall.InvokeAsync(kernel); + + chatHistory.Add(result.ToChatMessage()); + } + + // Sending the functions invocation results to the AI model to get the final response + messageContent = await chatCompletionService.GetChatMessageContentAsync(chatHistory, settings, kernel); + functionCalls = FunctionCallContent.GetFunctionCalls(messageContent).ToArray(); +} +``` + +The code snippets above demonstrate how to migrate your code that uses the OpenAI AI connector. +A similar migration process can be applied to the Gemini and Mistral AI connectors when they are updated to support the new function calling model. + +## Next steps +Now after you have migrated your code to the new function calling model, you can proceed to learn how to configure various aspects of the model that might better correspond to your specific scenarios by referring to the function calling behaviors section. + + +::: zone-end +::: zone pivot="programming-language-python" +## Coming soon +More info coming soon. +::: zone-end +::: zone pivot="programming-language-java" +## Coming soon +More info coming soon. +::: zone-end diff --git a/semantic-kernel/support/migration/stepwise-planner-migration-guide.md b/semantic-kernel/support/migration/stepwise-planner-migration-guide.md new file mode 100644 index 00000000..b6fe74ad --- /dev/null +++ b/semantic-kernel/support/migration/stepwise-planner-migration-guide.md @@ -0,0 +1,127 @@ +--- +title: Migrating from Stepwise Planner to Auto Function Calling +description: Describes the steps for SK caller code to migrate from Stepwise Planner to Auto Function Calling. +zone_pivot_groups: programming-languages +author: dmytrostruk +ms.topic: conceptual +ms.author: dmytrostruk +ms.service: semantic-kernel +--- +::: zone pivot="programming-language-csharp" +# Stepwise Planner Migration Guide +This migration guide shows how to migrate from `FunctionCallingStepwisePlanner` to a new recommended approach for planning capability - [Auto Function Calling](../../concepts/ai-services/chat-completion/function-calling/index.md). The new approach produces the results more reliably and uses fewer tokens compared to `FunctionCallingStepwisePlanner`. + +## Plan generation +Following code shows how to generate a new plan with Auto Function Calling by using `FunctionChoiceBehavior = FunctionChoiceBehavior.Auto()`. After sending a request to AI model, the plan will be located in `ChatHistory` object where a message with `Assistant` role will contain a list of functions (steps) to call. + +Old approach: +```csharp +Kernel kernel = Kernel + .CreateBuilder() + .AddOpenAIChatCompletion("gpt-4", Environment.GetEnvironmentVariable("OpenAI__ApiKey")) + .Build(); + +FunctionCallingStepwisePlanner planner = new(); + +FunctionCallingStepwisePlannerResult result = await planner.ExecuteAsync(kernel, "Check current UTC time and return current weather in Boston city."); + +ChatHistory generatedPlan = result.ChatHistory; +``` + +New approach: +```csharp +Kernel kernel = Kernel + .CreateBuilder() + .AddOpenAIChatCompletion("gpt-4", Environment.GetEnvironmentVariable("OpenAI__ApiKey")) + .Build(); + +IChatCompletionService chatCompletionService = kernel.GetRequiredService(); + +ChatHistory chatHistory = []; +chatHistory.AddUserMessage("Check current UTC time and return current weather in Boston city."); + +OpenAIPromptExecutionSettings executionSettings = new() { FunctionChoiceBehavior = FunctionChoiceBehavior.Auto() }; + +await chatCompletionService.GetChatMessageContentAsync(chatHistory, executionSettings, kernel); + +ChatHistory generatedPlan = chatHistory; +``` + +## Execution of the new plan +Following code shows how to execute a new plan with Auto Function Calling by using `FunctionChoiceBehavior = FunctionChoiceBehavior.Auto()`. This approach is useful when only result is needed without plan steps. In this case, `Kernel` object can be used to pass a goal to `InvokePromptAsync` method. The result of plan execution will be located in `FunctionResult` object. + +Old approach: +```csharp +Kernel kernel = Kernel + .CreateBuilder() + .AddOpenAIChatCompletion("gpt-4", Environment.GetEnvironmentVariable("OpenAI__ApiKey")) + .Build(); + +FunctionCallingStepwisePlanner planner = new(); + +FunctionCallingStepwisePlannerResult result = await planner.ExecuteAsync(kernel, "Check current UTC time and return current weather in Boston city."); + +string planResult = result.FinalAnswer; +``` + +New approach: +```csharp +Kernel kernel = Kernel + .CreateBuilder() + .AddOpenAIChatCompletion("gpt-4", Environment.GetEnvironmentVariable("OpenAI__ApiKey")) + .Build(); + +OpenAIPromptExecutionSettings executionSettings = new() { FunctionChoiceBehavior = FunctionChoiceBehavior.Auto() }; + +FunctionResult result = await kernel.InvokePromptAsync("Check current UTC time and return current weather in Boston city.", new(executionSettings)); + +string planResult = result.ToString(); +``` + +## Execution of the existing plan +Following code shows how to execute an existing plan with Auto Function Calling by using `FunctionChoiceBehavior = FunctionChoiceBehavior.Auto()`. This approach is useful when `ChatHistory` is already present (e.g. stored in cache) and it should be re-executed again and final result should be provided by AI model. + +Old approach: +```csharp +Kernel kernel = Kernel + .CreateBuilder() + .AddOpenAIChatCompletion("gpt-4", Environment.GetEnvironmentVariable("OpenAI__ApiKey")) + .Build(); + +FunctionCallingStepwisePlanner planner = new(); +ChatHistory existingPlan = GetExistingPlan(); // plan can be stored in database or cache for reusability. + +FunctionCallingStepwisePlannerResult result = await planner.ExecuteAsync(kernel, "Check current UTC time and return current weather in Boston city.", existingPlan); + +string planResult = result.FinalAnswer; +``` + +New approach: +```csharp +Kernel kernel = Kernel + .CreateBuilder() + .AddOpenAIChatCompletion("gpt-4", Environment.GetEnvironmentVariable("OpenAI__ApiKey")) + .Build(); + +IChatCompletionService chatCompletionService = kernel.GetRequiredService(); + +ChatHistory existingPlan = GetExistingPlan(); // plan can be stored in database or cache for reusability. + +OpenAIPromptExecutionSettings executionSettings = new() { FunctionChoiceBehavior = FunctionChoiceBehavior.Auto() }; + +ChatMessageContent result = await chatCompletionService.GetChatMessageContentAsync(existingPlan, executionSettings, kernel); + +string planResult = result.Content; +``` + +The code snippets above demonstrate how to migrate your code that uses Stepwise Planner to use Auto Function Calling. Learn more about [Function Calling with chat completion](../../concepts/ai-services/chat-completion/function-calling/index.md). + +::: zone-end +::: zone pivot="programming-language-python" +## Coming soon +More info coming soon. +::: zone-end +::: zone pivot="programming-language-java" +## Coming soon +More info coming soon. +::: zone-end diff --git a/semantic-kernel/support/migration/toc.yml b/semantic-kernel/support/migration/toc.yml new file mode 100644 index 00000000..b58c3e0c --- /dev/null +++ b/semantic-kernel/support/migration/toc.yml @@ -0,0 +1,8 @@ +- name: .NET Migration to v1+ + href: v1-migration-guide.md +- name: .NET Migration to 1.18.0+ + href: v2-openai-migration-guide.md +- name: Function Calling Migration + href: function-calling-migration-guide.md +- name: Stepwise Planner Migration + href: stepwise-planner-migration-guide.md \ No newline at end of file diff --git a/semantic-kernel/support/v1-migration-guide.md b/semantic-kernel/support/migration/v1-migration-guide.md similarity index 100% rename from semantic-kernel/support/v1-migration-guide.md rename to semantic-kernel/support/migration/v1-migration-guide.md diff --git a/semantic-kernel/support/migration/v2-openai-migration-guide.md b/semantic-kernel/support/migration/v2-openai-migration-guide.md new file mode 100644 index 00000000..cdee8c3b --- /dev/null +++ b/semantic-kernel/support/migration/v2-openai-migration-guide.md @@ -0,0 +1,240 @@ +--- +title: OpenAI Connector Migration Guide +description: Learn how to migrate .Net OpenAI Connectors after 1.18.0. +author: rogerbarreto +ms.topic: tutorial +ms.author: rbarreto +ms.date: 08/13/2024 +ms.service: semantic-kernel +--- + +# OpenAI Connector Migration Guide + +Coming as part of the new **1.18** version of Semantic Kernel we migrated our `OpenAI` and `AzureOpenAI` services to use the new `OpenAI SDK v2.0` and `Azure OpenAI SDK v2.0` SDKs. + +As those changes were major breaking changes when implementing ours we looked forward to break as minimal as possible the dev experience. + +This guide prepares you for the migration that you may need to do to use our new OpenAI Connector is a complete rewrite of the existing OpenAI Connector and is designed to be more efficient, reliable, and scalable. This manual will guide you through the migration process and help you understand the changes that have been made to the OpenAI Connector. + +Those changes are needed for anyone using `OpenAI` or `AzureOpenAI` connectors with Semantic Kernel version `1.18.0-rc` or above. + +## 1. Package Setup when using Azure only services + +If you are working with Azure services you will need to change the package from `Microsoft.SemanticKernel.Connectors.OpenAI` to `Microsoft.SemanticKernel.Connectors.AzureOpenAI`. This is necessary as we created two distinct connectors for each. + +> [!IMPORTANT] +> The `Microsoft.SemanticKernel.Connectors.AzureOpenAI` package depends on the `Microsoft.SemanticKernel.Connectors.OpenAI` package so there's no need to add both to your project when using `OpenAI` related types. + +```diff +Before +- using Microsoft.SemanticKernel.Connectors.OpenAI; +After ++ using Microsoft.SemanticKernel.Connectors.AzureOpenAI; +``` + +### 1.1 AzureOpenAIClient + +When using Azure with OpenAI, before where you were using `OpenAIClient` you will need to update your code to use the new `AzureOpenAIClient` type. + +### 1.2 Services + +All services below now belong to the `Microsoft.SemanticKernel.Connectors.AzureOpenAI` namespace. + +- `AzureOpenAIAudioToTextService` +- `AzureOpenAIChatCompletionService` +- `AzureOpenAITextEmbeddingGenerationService` +- `AzureOpenAITextToAudioService` +- `AzureOpenAITextToImageService` + +## 2. Text Generation Deprecation + +The latest `OpenAI` SDK does not support text generation modality, when migrating to their underlying SDK we had to drop support as well and remove `TextGeneration` specific services. + +If you were using OpenAI's `gpt-3.5-turbo-instruct` legacy model with any of the `OpenAITextGenerationService` or `AzureOpenAITextGenerationService` you will need to update your code to target a chat completion model instead, using `OpenAIChatCompletionService` or `AzureOpenAIChatCompletionService` instead. + +> [!NOTE] +> OpenAI and AzureOpenAI `ChatCompletion` services also implement the `ITextGenerationService` interface and that may not require any changes to your code if you were targeting the `ITextGenerationService` interface. + +tags: `AddOpenAITextGeneration`, `AddAzureOpenAITextGeneration` + +## 3. ChatCompletion Multiple Choices Deprecated + +The latest `OpenAI` SDK does not support multiple choices, when migrating to their underlying SDK we had to drop the support and remove `ResultsPerPrompt` also from the `OpenAIPromptExecutionSettings`. + +tags: `results_per_prompt` + +## 4. OpenAI File Service Deprecation + +The `OpenAIFileService` was deprecated in the latest version of the OpenAI Connector. We strongly recommend to update your code to use the new `OpenAIClient.GetFileClient()` for file management operations. + +## 5. OpenAI ChatCompletion custom endpoint + +The `OpenAIChatCompletionService` **experimental** constructor for custom endpoints will not attempt to auto-correct the endpoint and use it as is. + +We have the two only specific cases where we attempted to auto-correct the endpoint. + +1. If you provided `chat/completions` path before. Now those need to be removed as they are added automatically to the end of your original endpoint by `OpenAI SDK`. + + ```diff + - http://any-host-and-port/v1/chat/completions + + http://any-host-and-port/v1 + ``` + +2. If you provided a custom endpoint without any path. We won't be adding the `v1/` as the first path. Now the `v1` path needs to provided as part of your endpoint. + + ```diff + - http://any-host-and-port/ + + http://any-host-and-port/v1 + ``` + +## 6. SemanticKernel MetaPackage + +To be retrocompatible with the new OpenAI and AzureOpenAI Connectors, our `Microsoft.SemanticKernel` meta package changed its dependency to use the new `Microsoft.SemanticKernel.Connectors.AzureOpenAI` package that depends on the `Microsoft.SemanticKernel.Connectors.OpenAI` package. This way if you are using the metapackage, no change is needed to get access to `Azure` related types. + +## 7. Chat Message Content Changes + +### 7.1 OpenAIChatMessageContent + +- The `Tools` property type has changed from `IReadOnlyList` to `IReadOnlyList`. + +- Inner content type has changed from `ChatCompletionsFunctionToolCall` to `ChatToolCall`. + +- Metadata type `FunctionToolCalls` has changed from `IEnumerable` to `IEnumerable`. + +### 7.2 OpenAIStreamingChatMessageContent + +- The `FinishReason` property type has changed from `CompletionsFinishReason` to `FinishReason`. +- The `ToolCallUpdate` property has been renamed to `ToolCallUpdates` and its type has changed from `StreamingToolCallUpdate?` to `IReadOnlyList?`. +- The `AuthorName` property is not initialized because it's not provided by the underlying library anymore. + +## 8. Metrics for AzureOpenAI Connector + +The meter `s_meter = new("Microsoft.SemanticKernel.Connectors.OpenAI");` and the relevant counters still have old names that contain "openai" in them, such as: + +- `semantic_kernel.connectors.openai.tokens.prompt` +- `semantic_kernel.connectors.openai.tokens.completion` +- `semantic_kernel.connectors.openai.tokens.total` + +## 9. Using Azure with your data (Data Sources) + +With the new `AzureOpenAIClient`, you can now specify your datasource thru the options and that requires a small change in your code to the new type. + +Before + +```csharp +var promptExecutionSettings = new OpenAIPromptExecutionSettings +{ + AzureChatExtensionsOptions = new AzureChatExtensionsOptions + { + Extensions = [ new AzureSearchChatExtensionConfiguration + { + SearchEndpoint = new Uri(TestConfiguration.AzureAISearch.Endpoint), + Authentication = new OnYourDataApiKeyAuthenticationOptions(TestConfiguration.AzureAISearch.ApiKey), + IndexName = TestConfiguration.AzureAISearch.IndexName + }] + }; +}; +``` + +After + +```csharp +var promptExecutionSettings = new AzureOpenAIPromptExecutionSettings +{ + AzureChatDataSource = new AzureSearchChatDataSource + { + Endpoint = new Uri(TestConfiguration.AzureAISearch.Endpoint), + Authentication = DataSourceAuthentication.FromApiKey(TestConfiguration.AzureAISearch.ApiKey), + IndexName = TestConfiguration.AzureAISearch.IndexName + } +}; +``` +Tags: `WithData`, `AzureOpenAIChatCompletionWithDataConfig`, `AzureOpenAIChatCompletionWithDataService` + +## 10. Breaking glass scenarios + +Breaking glass scenarios are scenarios where you may need to update your code to use the new OpenAI Connector. Below are some of the breaking changes that you may need to be aware of. + +#### 10.1 KernelContent Metadata + +Some of the keys in the content metadata dictionary have changed and removed. + +- Changed: `Created` -> `CreatedAt` +- Changed: `LogProbabilityInfo` -> `ContentTokenLogProbabilities` +- Changed: `PromptFilterResults` -> `ContentFilterResultForPrompt` +- Changed: `ContentFilterResultsForPrompt` -> `ContentFilterResultForResponse` +- Removed: `FinishDetails` +- Removed: `Index` +- Removed: `Enhancements` + +#### 10.2 Prompt Filter Results + +The `PromptFilterResults` metadata type has changed from `IReadOnlyList` to `ContentFilterResultForPrompt`. + +#### 10.3 Content Filter Results + +The `ContentFilterResultsForPrompt` type has changed from `ContentFilterResultsForChoice` to `ContentFilterResultForResponse`. + +#### 10.4 Finish Reason + +The FinishReason metadata string value has changed from `stop` to `Stop` + +#### 10.5 Tool Calls + +The ToolCalls metadata string value has changed from `tool_calls` to `ToolCalls` + +#### 10.6 LogProbs / Log Probability Info + +The `LogProbabilityInfo` type has changed from `ChatChoiceLogProbabilityInfo` to `IReadOnlyList`. + +#### 10.7 Token Usage + +The Token usage naming convention from `OpenAI` changed from `Completion`, `Prompt` tokens to `Output` and `Input` respectively. You will need to update your code to use the new naming. + +The type also changed from `CompletionsUsage` to `ChatTokenUsage`. + +[Example of Token Usage Metadata Changes](https://github.com/microsoft/semantic-kernel/pull/7151/files#diff-a323107b9f8dc8559a83e50080c6e34551ddf6d9d770197a473f249589e8fb47) + +```diff +Before +- var usage = FunctionResult.Metadata?["Usage"] as CompletionsUsage; +- var completionTokesn = usage?.CompletionTokens; +- var promptTokens = usage?.PromptTokens; + +After ++ var usage = FunctionResult.Metadata?["Usage"] as ChatTokenUsage; ++ var promptTokens = usage?.InputTokens; ++ var completionTokens = completionTokens: usage?.OutputTokens; +``` + +#### 10.8 OpenAIClient + +The `OpenAIClient` type previously was a Azure specific namespace type but now it is an `OpenAI` SDK namespace type, you will need to update your code to use the new `OpenAIClient` type. + +When using Azure, you will need to update your code to use the new `AzureOpenAIClient` type. + +#### 10.9 OpenAIClientOptions + +The `OpenAIClientOptions` type previously was a Azure specific namespace type but now it is an `OpenAI` SDK namespace type, you will need to update your code to use the new `AzureOpenAIClientOptions` type if you are using the new `AzureOpenAIClient` with any of the specific options for the Azure client. + +#### 10.10 Pipeline Configuration + +The new `OpenAI` SDK uses a different pipeline configuration, and has a dependency on `System.ClientModel` package. You will need to update your code to use the new `HttpClientPipelineTransport` transport configuration where before you were using `HttpClientTransport` from `Azure.Core.Pipeline`. + +[Example of Pipeline Configuration](https://github.com/microsoft/semantic-kernel/pull/7151/files#diff-fab02d9a75bf43cb57f71dddc920c3f72882acf83fb125d8cad963a643d26eb3) + +```diff +var clientOptions = new OpenAIClientOptions +{ +Before: From Azure.Core.Pipeline +- Transport = new HttpClientTransport(httpClient), +- RetryPolicy = new RetryPolicy(maxRetries: 0), // Disable Azure SDK retry policy if and only if a custom HttpClient is provided. +- Retry = { NetworkTimeout = Timeout.InfiniteTimeSpan } // Disable Azure SDK default timeout + + +After: From OpenAI SDK -> System.ClientModel ++ Transport = new HttpClientPipelineTransport(httpClient), ++ RetryPolicy = new ClientRetryPolicy(maxRetries: 0); // Disable retry policy if and only if a custom HttpClient is provided. ++ NetworkTimeout = Timeout.InfiniteTimeSpan; // Disable default timeout +}; +``` diff --git a/semantic-kernel/support/toc.yml b/semantic-kernel/support/toc.yml index 12aabca5..878f992d 100644 --- a/semantic-kernel/support/toc.yml +++ b/semantic-kernel/support/toc.yml @@ -1,6 +1,10 @@ - name: Overview href: index.md -- name: Migrating to v1+ - href: v1-migration-guide.md +- name: Contributing to Semantic Kernel + href: contributing.md +- name: Hackathon materials + href: hackathon-materials.md - name: Glossary href: glossary.md +- name: Upgrading + href: migration/toc.yml diff --git a/semantic-kernel/zone-pivot-groups.yml b/semantic-kernel/zone-pivot-groups.yml index cd152fd5..977596dd 100644 --- a/semantic-kernel/zone-pivot-groups.yml +++ b/semantic-kernel/zone-pivot-groups.yml @@ -9,4 +9,4 @@ groups: - id: programming-language-python title: Python - id: programming-language-java - title: Java \ No newline at end of file + title: Java