-
Notifications
You must be signed in to change notification settings - Fork 3.4k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add StepwisePlanner extension for MRKL style planning (#1468)
### Motivation and Context This pull request adds a new extension for semantic planning using a stepwise approach. The StepwisePlanner extension allows users to create and execute plans that consist of a sequence of semantic and native functions, each with a goal and a set of inputs and outputs. The extension uses a semantic search engine to find relevant functions for each step, and a plan creation service to generate a plan that satisfies the user's ask. The extension also provides a system step function that executes the plan and returns the final answer and intermediate observations. The extension can be configured with various parameters, such as the relevancy threshold, the maximum number of relevant functions, the excluded and included functions and skills, and the maximum number of tokens, iterations, and time for the plan. Regarding #1472 ### Description - Add StepwisePlanner.cs, which registers the planner native functions and the system step function - Add StepwisePlannerConfig.cs, which defines the configuration options for the StepwisePlanner extension - Add SystemStep.cs, which represents a step in a Stepwise plan, with properties for the thought, action, action variables, observation, final answer, and original response - Add helper methods for formatting and validating function views, generating plan requests, and invoking the plan. - Add logging and error handling for the planner extension - Add unit tests for the planner extension and the native functions ### Related - Majority of work initially started from @kaza in #992 ### Changes in other PRs to merge separately - #1464 - #1465 - #1466 ### Contribution Checklist <!-- Before submitting this PR, please make sure: --> - [x] The code builds clean without any errors or warnings - [x] The PR follows SK Contribution Guidelines (https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md) - [x] The code follows the .NET coding conventions (https://learn.microsoft.com/dotnet/csharp/fundamentals/coding-style/coding-conventions) verified with `dotnet format` - [x] All unit tests pass, and I have added new tests where possible - [x] I didn't break anyone 😄 --------- Co-authored-by: Almir Kazazic <kaza@users.noreply.github.com> --------- Co-authored-by: Lee Miller <lemillermicrosoft@users.noreply.github.com>
- Loading branch information
1 parent
85d420f
commit 49e2010
Showing
16 changed files
with
1,176 additions
and
11 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
189 changes: 189 additions & 0 deletions
189
dotnet/samples/KernelSyntaxExamples/Example51_StepwisePlanner.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,189 @@ | ||
| // Copyright (c) Microsoft. All rights reserved. | ||
|
|
||
| using System; | ||
| using System.Diagnostics; | ||
| using System.Threading.Tasks; | ||
| using Microsoft.SemanticKernel; | ||
| using Microsoft.SemanticKernel.Planning; | ||
| using Microsoft.SemanticKernel.Reliability; | ||
| using Microsoft.SemanticKernel.Skills.Core; | ||
| using Microsoft.SemanticKernel.Skills.Web; | ||
| using Microsoft.SemanticKernel.Skills.Web.Bing; | ||
| using NCalcSkills; | ||
| using RepoUtils; | ||
|
|
||
| /** | ||
| * This example shows how to use Stepwise Planner to create a plan for a given goal. | ||
| */ | ||
|
|
||
| // ReSharper disable once InconsistentNaming | ||
| public static class Example51_StepwisePlanner | ||
| { | ||
| public static async Task RunAsync() | ||
| { | ||
| string[] questions = new string[] | ||
| { | ||
| "Who is the current president of the United States? What is his current age divided by 2", | ||
| // "Who is Leo DiCaprio's girlfriend? What is her current age raised to the (his current age)/100 power?", | ||
| // "What is the capital of France? Who is that cities current mayor? What percentage of their life has been in the 21st century as of today?", | ||
| // "What is the current day of the calendar year? Using that as an angle in degrees, what is the area of a unit circle with that angle?" | ||
| }; | ||
|
|
||
| foreach (var question in questions) | ||
| { | ||
| await RunTextCompletion(question); | ||
| await RunChatCompletion(question); | ||
| } | ||
| } | ||
|
|
||
| public static async Task RunTextCompletion(string question) | ||
| { | ||
| Console.WriteLine("RunTextCompletion"); | ||
| var kernel = GetKernel(); | ||
| await RunWithQuestion(kernel, question); | ||
| } | ||
|
|
||
| public static async Task RunChatCompletion(string question) | ||
| { | ||
| Console.WriteLine("RunChatCompletion"); | ||
| var kernel = GetKernel(true); | ||
| await RunWithQuestion(kernel, question); | ||
| } | ||
|
|
||
| public static async Task RunWithQuestion(IKernel kernel, string question) | ||
| { | ||
| using var bingConnector = new BingConnector(Env.Var("BING_API_KEY")); | ||
| var webSearchEngineSkill = new WebSearchEngineSkill(bingConnector); | ||
|
|
||
| kernel.ImportSkill(webSearchEngineSkill, "WebSearch"); | ||
| kernel.ImportSkill(new LanguageCalculatorSkill(kernel), "advancedCalculator"); | ||
| // kernel.ImportSkill(new SimpleCalculatorSkill(kernel), "basicCalculator"); | ||
| kernel.ImportSkill(new TimeSkill(), "time"); | ||
|
|
||
| Console.WriteLine("*****************************************************"); | ||
| Stopwatch sw = new(); | ||
| Console.WriteLine("Question: " + question); | ||
|
|
||
| var config = new Microsoft.SemanticKernel.Planning.Stepwise.StepwisePlannerConfig(); | ||
| config.ExcludedFunctions.Add("TranslateMathProblem"); | ||
| config.MinIterationTimeMs = 1500; | ||
| config.MaxTokens = 4000; | ||
|
|
||
| StepwisePlanner planner = new(kernel, config); | ||
| sw.Start(); | ||
| var plan = planner.CreatePlan(question); | ||
|
|
||
| var result = await plan.InvokeAsync(kernel.CreateNewContext()); | ||
| Console.WriteLine("Result: " + result); | ||
| if (result.Variables.TryGetValue("stepCount", out string? stepCount)) | ||
| { | ||
| Console.WriteLine("Steps Taken: " + stepCount); | ||
| } | ||
|
|
||
| if (result.Variables.TryGetValue("skillCount", out string? skillCount)) | ||
| { | ||
| Console.WriteLine("Skills Used: " + skillCount); | ||
| } | ||
|
|
||
| Console.WriteLine("Time Taken: " + sw.Elapsed); | ||
| Console.WriteLine("*****************************************************"); | ||
| } | ||
|
|
||
| private static IKernel GetKernel(bool useChat = false) | ||
| { | ||
| var builder = new KernelBuilder(); | ||
| if (useChat) | ||
| { | ||
| builder.WithAzureChatCompletionService( | ||
| Env.Var("AZURE_OPENAI_CHAT_DEPLOYMENT_NAME"), | ||
| Env.Var("AZURE_OPENAI_ENDPOINT"), | ||
| Env.Var("AZURE_OPENAI_KEY"), | ||
| alsoAsTextCompletion: true, | ||
| setAsDefault: true); | ||
| } | ||
| else | ||
| { | ||
| builder.WithAzureTextCompletionService( | ||
| Env.Var("AZURE_OPENAI_DEPLOYMENT_NAME"), | ||
| Env.Var("AZURE_OPENAI_ENDPOINT"), | ||
| Env.Var("AZURE_OPENAI_KEY")); | ||
| } | ||
|
|
||
| var kernel = builder | ||
| .WithLogger(ConsoleLogger.Log) | ||
| .Configure(c => c.SetDefaultHttpRetryConfig(new HttpRetryConfig | ||
| { | ||
| MaxRetryCount = 3, | ||
| UseExponentialBackoff = true, | ||
| MinRetryDelay = TimeSpan.FromSeconds(3), | ||
| })) | ||
| .Build(); | ||
|
|
||
| return kernel; | ||
| } | ||
| } | ||
|
|
||
| // RunTextCompletion | ||
| // ***************************************************** | ||
| // Question: Who is the current president of the United States? What is his current age divided by 2 | ||
| // Result: The current president of the United States is Joe Biden. His current age divided by 2 is 40. | ||
| // Steps Taken: 10 | ||
| // Skills Used: 4 (WebSearch.Search(2), time.Date(1), advancedCalculator.Calculator(1)) | ||
| // Time Taken: 00:00:53.6331324 | ||
| // ***************************************************** | ||
| // RunChatCompletion | ||
| // ***************************************************** | ||
| // Question: Who is the current president of the United States? What is his current age divided by 2 | ||
| // Result: The current president of the United States is Joe Biden. His current age divided by 2 is 40.5. | ||
| // Steps Taken: 9 | ||
| // Skills Used: 7 (WebSearch.Search(4), time.Year(1), time.Date(1), advancedCalculator.Calculator(1)) | ||
| // Time Taken: 00:01:13.3766860 | ||
| // ***************************************************** | ||
| // RunTextCompletion | ||
| // ***************************************************** | ||
| // Question: Who is Leo DiCaprio's girlfriend? What is her current age raised to the (his current age)/100 power? | ||
| // Result: Leo DiCaprio's girlfriend is Camila Morrone. Her current age raised to the (his current age)/100 power is 4.935565735151678. | ||
| // Steps Taken: 6 | ||
| // Skills Used: 5 (WebSearch.Search(3), time.Year(1), advancedCalculator.Calculator(1)) | ||
| // Time Taken: 00:00:37.8941510 | ||
| // ***************************************************** | ||
| // RunChatCompletion | ||
| // ***************************************************** | ||
| // Question: Who is Leo DiCaprio's girlfriend? What is her current age raised to the (his current age)/100 power? | ||
| // Result: Leo DiCaprio's girlfriend is Camila Morrone. Her current age raised to the power of (his current age)/100 is approximately 4.94. | ||
| // Steps Taken: 9 | ||
| // Skills Used: 5 (WebSearch.Search(3), time.Year(1), advancedCalculator.Calculator(1)) | ||
| // Time Taken: 00:01:17.6742136 | ||
| // ***************************************************** | ||
| // RunTextCompletion | ||
| // ***************************************************** | ||
| // Question: What is the capital of France? Who is that cities current mayor? What percentage of their life has been in the 21st century as of today? | ||
| // Result: The capital of France is Paris. The current mayor of Paris is Anne Hidalgo. She has spent 36.51% of her life in the 21st century as of 2023. | ||
| // Steps Taken: 7 | ||
| // Skills Used: 4 (WebSearch.Search(3), advancedCalculator.Calculator(1)) | ||
| // Time Taken: 00:00:41.6837628 | ||
| // ***************************************************** | ||
| // RunChatCompletion | ||
| // ***************************************************** | ||
| // Question: What is the capital of France? Who is that cities current mayor? What percentage of their life has been in the 21st century as of today? | ||
| // Result: The capital of France is Paris. The current mayor of Paris is Anne Hidalgo, who was born on June 19, 1959. As of today, she has lived for 64 years, with 23 of those years in the 21st century. Therefore, 35.94% of her life has been spent in the 21st century. | ||
| // Steps Taken: 14 | ||
| // Skills Used: 12 (WebSearch.Search(8), time.Year(1), advancedCalculator.Calculator(3)) | ||
| // Time Taken: 00:02:06.6682909 | ||
| // ***************************************************** | ||
| // RunTextCompletion | ||
| // ***************************************************** | ||
| // Question: What is the current day of the calendar year? Using that as an angle in degrees, what is the area of a unit circle with that angle? | ||
| // Result: The current day of the calendar year is 177. The angle in degrees corresponding to this day is 174.6. The area of a unit circle with that angle is 0.764 * pi. | ||
| // Steps Taken: 16 | ||
| // Skills Used: 2 (time.DayOfYear(1), time.Date(1)) | ||
| // Time Taken: 00:01:29.9931039 | ||
| // ***************************************************** | ||
| // RunChatCompletion | ||
| // ***************************************************** | ||
| // Question: What is the current day of the calendar year? Using that as an angle in degrees, what is the area of a unit circle with that angle? | ||
| // Result: The current day of the year is 177. Using that as an angle in degrees (approximately 174.58), the area of a unit circle with that angle is approximately 1.523 square units. | ||
| // Steps Taken: 11 | ||
| // Skills Used: 9 (time.Now(1), time.DayOfYear(1), time.DaysBetween(1), time.MonthNumber(1), time.Day(1), advancedCalculator.Calculator(4)) | ||
| // Time Taken: 00:01:41.5585861 | ||
| // ***************************************************** |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
74 changes: 74 additions & 0 deletions
74
dotnet/src/Extensions/Extensions.UnitTests/Planning/StepwisePlanner/ParseResultTests.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,74 @@ | ||
| // Copyright (c) Microsoft. All rights reserved. | ||
|
|
||
| using System.Collections.Generic; | ||
| using Microsoft.Extensions.Logging; | ||
| using Microsoft.SemanticKernel; | ||
| using Microsoft.SemanticKernel.SkillDefinition; | ||
| using Moq; | ||
| using Xunit; | ||
|
|
||
| namespace SemanticKernel.Extensions.UnitTests.Planning.StepwisePlanner; | ||
|
|
||
| public sealed class ParseResultTests | ||
| { | ||
| [Theory] | ||
| [InlineData("[FINAL ANSWER] 42", "42")] | ||
| [InlineData("[FINAL ANSWER]42", "42")] | ||
| [InlineData("I think I have everything I need.\n[FINAL ANSWER] 42", "42")] | ||
| [InlineData("I think I have everything I need.\n[FINAL ANSWER] 42\n", "42")] | ||
| [InlineData("I think I have everything I need.\n[FINAL ANSWER] 42\n\n", "42")] | ||
| [InlineData("I think I have everything I need.\n[FINAL ANSWER]42\n\n\n", "42")] | ||
| [InlineData("I think I have everything I need.\n[FINAL ANSWER]\n 42\n\n\n", "42")] | ||
| public void WhenInputIsFinalAnswerReturnsFinalAnswer(string input, string expected) | ||
| { | ||
| // Arrange | ||
| var kernel = new Mock<IKernel>(); | ||
| kernel.Setup(x => x.Log).Returns(new Mock<ILogger>().Object); | ||
|
|
||
| var planner = new Microsoft.SemanticKernel.Planning.StepwisePlanner(kernel.Object); | ||
|
|
||
| // Act | ||
| var result = planner.ParseResult(input); | ||
|
|
||
| // Assert | ||
| Assert.Equal(expected, result.FinalAnswer); | ||
| } | ||
|
|
||
| [Theory] | ||
| [InlineData("To answer the first part of the question, I need to search for Leo DiCaprio's girlfriend on the web. To answer the second part, I need to find her current age and use a calculator to raise it to the 0.43 power.\n[ACTION]\n{\n \"action\": \"Search\",\n \"action_variables\": {\"input\": \"Leo DiCaprio's girlfriend\"}\n}", "Search", "input", "Leo DiCaprio's girlfriend")] | ||
| [InlineData("To answer the first part of the question, I need to search the web for Leo DiCaprio's girlfriend. To answer the second part, I need to find her current age and use the calculator tool to raise it to the 0.43 power.\n[ACTION]\n```\n{\n \"action\": \"Search\",\n \"action_variables\": {\"input\": \"Leo DiCaprio's girlfriend\"}\n}\n```", "Search", "input", "Leo DiCaprio's girlfriend")] | ||
| [InlineData("The web search result is a snippet from a Wikipedia article that says Leo DiCaprio's girlfriend is Camila Morrone, an Argentine-American model and actress. I need to find out her current age, which might be in the same article or another source. I can use the WebSearch.Search function again to search for her name and age.\n\n[ACTION] {\n \"action\": \"WebSearch.Search\",\n \"action_variables\": {\"input\": \"Camila Morrone age\", \"count\": \"1\"}\n}", "WebSearch.Search", "input", | ||
| "Camila Morrone age", "count", "1")] | ||
| public void ParseActionReturnsAction(string input, string expectedAction, params string[] expectedVariables) | ||
| { | ||
| Dictionary<string, string>? expectedDictionary = null; | ||
| for (int i = 0; i < expectedVariables.Length; i += 2) | ||
| { | ||
| expectedDictionary ??= new Dictionary<string, string>(); | ||
| expectedDictionary.Add(expectedVariables[i], expectedVariables[i + 1]); | ||
| } | ||
|
|
||
| // Arrange | ||
| var kernel = new Mock<IKernel>(); | ||
| kernel.Setup(x => x.Log).Returns(new Mock<ILogger>().Object); | ||
|
|
||
| var planner = new Microsoft.SemanticKernel.Planning.StepwisePlanner(kernel.Object); | ||
|
|
||
| // Act | ||
| var result = planner.ParseResult(input); | ||
|
|
||
| // Assert | ||
| Assert.Equal(expectedAction, result.Action); | ||
| Assert.Equal(expectedDictionary, result.ActionVariables); | ||
| } | ||
|
|
||
| // Method to create Mock<ISKFunction> objects | ||
| private static Mock<ISKFunction> CreateMockFunction(FunctionView functionView) | ||
| { | ||
| var mockFunction = new Mock<ISKFunction>(); | ||
| mockFunction.Setup(x => x.Describe()).Returns(functionView); | ||
| mockFunction.Setup(x => x.Name).Returns(functionView.Name); | ||
| mockFunction.Setup(x => x.SkillName).Returns(functionView.SkillName); | ||
| return mockFunction; | ||
| } | ||
| } |
23 changes: 23 additions & 0 deletions
23
dotnet/src/Extensions/Planning.StepwisePlanner/EmbeddedResource.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,23 @@ | ||
| // Copyright (c) Microsoft. All rights reserved. | ||
|
|
||
| using System.IO; | ||
| using System.Reflection; | ||
|
|
||
| namespace Microsoft.SemanticKernel.Planning.Stepwise; | ||
|
|
||
| internal static class EmbeddedResource | ||
| { | ||
| private static readonly string? s_namespace = typeof(EmbeddedResource).Namespace; | ||
|
|
||
| internal static string Read(string name) | ||
| { | ||
| var assembly = typeof(EmbeddedResource).GetTypeInfo().Assembly; | ||
| if (assembly == null) { throw new PlanningException(PlanningException.ErrorCodes.InvalidConfiguration, $"[{s_namespace}] {name} assembly not found"); } | ||
|
|
||
| using Stream? resource = assembly.GetManifestResourceStream($"{s_namespace}." + name); | ||
| if (resource == null) { throw new PlanningException(PlanningException.ErrorCodes.InvalidConfiguration, $"[{s_namespace}] {name} resource not found"); } | ||
|
|
||
| using var reader = new StreamReader(resource); | ||
| return reader.ReadToEnd(); | ||
| } | ||
| } |
Oops, something went wrong.