Skip to content

Commit

Permalink
.Net: Fix function calling integration tests (#8948)
Browse files Browse the repository at this point in the history 
### Motivation, Context and Description
Some of the function-calling integration tests were failing occasionally
due to various reasons not related to the function-calling functionality
in SK. For example, the AI model could decide to call the same function
twice instead of once in one out of ten integration test runs.
Additionally, Azure may have activated the content management policy by
default, causing a few integration tests to fail with the error: "Error:
Exception while invoking function. HTTP 400 (content_filter), Parameter:
prompt. The response was filtered due to the prompt triggering Azure
OpenAI's content management policy. Please modify your prompt and
retry."
   
Lastly, some of the integration tests were disabled to unblock the
latest release because the AI model started to call functions that were
not related to the prompt but were advertised to the model.
   
This PR fixes the intermittently failing tests and re-enables the
disabled ones.
  • Loading branch information
@SergeyMenshykh
SergeyMenshykh authored Sep 23, 2024
1 parent be90d23 commit 8924bdc
Show file tree
Hide file tree
Showing 6 changed files with 80 additions and 90 deletions.
91 changes: 54 additions & 37 deletions .../IntegrationTests/Connectors/AzureOpenAI/AzureOpenAIChatCompletionFunctionCallingTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,20 @@ public async Task CanAutoInvokeKernelFunctionsStreamingAsync()
public async Task CanAutoInvokeKernelFunctionsWithComplexTypeParametersAsync()
{
// Arrange
var kernel = this.CreateAndInitializeKernel(importHelperPlugin: true);
var kernel = this.CreateAndInitializeKernel();
kernel.ImportPluginFromFunctions("HelperFunctions",
[
kernel.CreateFunctionFromMethod((WeatherParameters parameters) =>
{
if (parameters.City.Name.Equals("Dublin", StringComparison.OrdinalIgnoreCase) &&
(parameters.City.Country.Equals("Ireland", StringComparison.OrdinalIgnoreCase) || parameters.City.Country.Equals("IE", StringComparison.OrdinalIgnoreCase)))
{
return Task.FromResult(42.8); // 42.8 Fahrenheit.
}

throw new NotSupportedException($"Weather in {parameters.City.Name} ({parameters.City.Country}) is not supported.");
}, "Get_Current_Temperature", "Get current temperature."),
]);

AzureOpenAIPromptExecutionSettings settings = new() { ToolCallBehavior = ToolCallBehavior.AutoInvokeKernelFunctions };

Expand Down Expand Up @@ -129,10 +142,19 @@ public async Task CanAutoInvokeKernelFunctionsWithEnumTypeParametersAsync()
public async Task CanAutoInvokeKernelFunctionFromPromptAsync()
{
// Arrange
var invokedFunctions = new List<string>();

var filter = new FakeFunctionFilter(async (context, next) =>
{
invokedFunctions.Add(context.Function.Name);
await next(context);
});

var kernel = this.CreateAndInitializeKernel();
kernel.FunctionInvocationFilters.Add(filter);

var promptFunction = KernelFunctionFactory.CreateFromPrompt(
"Your role is always to return this text - 'A Game-Changer for the Transportation Industry'. Don't ask for more details or context.",
"Hey LLM, give me one news title that's hot off the press!",
functionName: "FindLatestNews",
description: "Searches for the latest news.");

Expand All @@ -144,21 +166,30 @@ public async Task CanAutoInvokeKernelFunctionFromPromptAsync()
AzureOpenAIPromptExecutionSettings settings = new() { ToolCallBehavior = ToolCallBehavior.AutoInvokeKernelFunctions };

// Act
var result = await kernel.InvokePromptAsync("Show me the latest news as they are.", new(settings));
var result = await kernel.InvokePromptAsync("Show me the latest news.", new(settings));

// Assert
Assert.NotNull(result);
Assert.Contains("Transportation", result.GetValue<string>(), StringComparison.InvariantCultureIgnoreCase);
Assert.Contains(invokedFunctions, functionName => functionName.Contains("InvokePromptAsync"));
Assert.Contains(invokedFunctions, functionName => functionName.Contains("FindLatestNews"));
}

[Fact]
public async Task CanAutoInvokeKernelFunctionFromPromptStreamingAsync()
{
// Arrange
var invokedFunctions = new List<string>();

var filter = new FakeFunctionFilter(async (context, next) =>
{
invokedFunctions.Add(context.Function.Name);
await next(context);
});

var kernel = this.CreateAndInitializeKernel();
kernel.FunctionInvocationFilters.Add(filter);

var promptFunction = KernelFunctionFactory.CreateFromPrompt(
"Your role is always to return this text - 'A Game-Changer for the Transportation Industry'. Don't ask for more details or context.",
"Hey LLM, give me one news title that's hot off the press!",
functionName: "FindLatestNews",
description: "Searches for the latest news.");

Expand All @@ -170,20 +201,14 @@ public async Task CanAutoInvokeKernelFunctionFromPromptStreamingAsync()
AzureOpenAIPromptExecutionSettings settings = new() { ToolCallBehavior = ToolCallBehavior.AutoInvokeKernelFunctions };

// Act
var streamingResult = kernel.InvokePromptStreamingAsync("Show me the latest news as they are.", new(settings));

var builder = new StringBuilder();

var streamingResult = kernel.InvokePromptStreamingAsync("Show me the latest news.", new(settings));
await foreach (var update in streamingResult)
{
builder.Append(update.ToString());
}

var result = builder.ToString();

// Assert
Assert.NotNull(result);
Assert.Contains("Transportation", result, StringComparison.InvariantCultureIgnoreCase);
Assert.Contains(invokedFunctions, functionName => functionName.Contains("InvokePromptStreamingAsync"));
Assert.Contains(invokedFunctions, functionName => functionName.Contains("FindLatestNews"));
}

[Fact]
Expand Down Expand Up @@ -482,7 +507,7 @@ public async Task ConnectorAgnosticFunctionCallingModelClassesCanBeUsedForAutoFu
Assert.NotNull(getWeatherForCityFunctionCallResult.Result);
}

[Fact(Skip = "Weather in Boston (USA) is not supported.")]
[Fact]
public async Task ConnectorAgnosticFunctionCallingModelClassesCanBeUsedForManualFunctionCallingForStreamingAsync()
{
// Arrange
Expand Down Expand Up @@ -784,10 +809,13 @@ public async Task ItShouldSupportOldFunctionCallingModelSerializedIntoChatHistor
string? emailBody = null, emailRecipient = null;

var kernel = this.CreateAndInitializeKernel(importHelperPlugin: true);
kernel.ImportPluginFromFunctions("EmailPlugin", [KernelFunctionFactory.CreateFromMethod((string body, string recipient) => { emailBody = body; emailRecipient = recipient; }, "SendEmail")]);
kernel.ImportPluginFromFunctions("EmailPlugin", [
KernelFunctionFactory.CreateFromMethod((string body, string recipient) => { emailBody = body; emailRecipient = recipient; }, "SendEmail"),
KernelFunctionFactory.CreateFromMethod(() => "abc@domain.com", "GetMyEmail")
]);

// The deserialized chat history contains a list of function calls and the final answer to the question regarding the color of the sky in Boston.
chatHistory.AddUserMessage("Send the exact answer to my email: abc@domain.com");
chatHistory.AddUserMessage("Send the exact answer to my email.");

var settings = new AzureOpenAIPromptExecutionSettings() { ToolCallBehavior = ToolCallBehavior.AutoInvokeKernelFunctions };

Expand Down Expand Up @@ -820,10 +848,13 @@ public async Task ItShouldSupportNewFunctionCallingModelSerializedIntoChatHistor
string? emailBody = null, emailRecipient = null;

var kernel = this.CreateAndInitializeKernel(importHelperPlugin: true);
kernel.ImportPluginFromFunctions("EmailPlugin", [KernelFunctionFactory.CreateFromMethod((string body, string recipient) => { emailBody = body; emailRecipient = recipient; }, "SendEmail")]);
kernel.ImportPluginFromFunctions("EmailPlugin", [
KernelFunctionFactory.CreateFromMethod((string body, string recipient) => { emailBody = body; emailRecipient = recipient; }, "SendEmail"),
KernelFunctionFactory.CreateFromMethod(() => "abc@domain.com", "GetMyEmail")
]);

// The deserialized chat history contains a list of function calls and the final answer to the question regarding the color of the sky in Boston.
chatHistory.AddUserMessage("Send the exact answer to my email: abc@domain.com");
chatHistory.AddUserMessage("Send the exact answer to my email.");

var settings = new AzureOpenAIPromptExecutionSettings() { ToolCallBehavior = ToolCallBehavior.AutoInvokeKernelFunctions };

Expand All @@ -832,7 +863,7 @@ public async Task ItShouldSupportNewFunctionCallingModelSerializedIntoChatHistor

// Assert
Assert.Equal("abc@domain.com", emailRecipient);
Assert.Contains("61\u00B0F", emailBody);
Assert.Contains("61", emailBody);
}

/// <summary>
Expand Down Expand Up @@ -867,7 +898,7 @@ public async Task SubsetOfFunctionsCanBeUsedForFunctionCallingAsync()
// Arrange
var kernel = this.CreateAndInitializeKernel();

var function = kernel.CreateFunctionFromMethod(() => DayOfWeek.Friday, "GetDayOfWeek", "Retrieves the current day of the week.");
var function = kernel.CreateFunctionFromMethod(() => DayOfWeek.Friday.ToString(), "GetDayOfWeek", "Retrieves the current day of the week.");
kernel.ImportPluginFromFunctions("HelperFunctions", [function]);

var chatHistory = new ChatHistory();
Expand All @@ -891,7 +922,7 @@ public async Task RequiredFunctionShouldBeCalledAsync()
// Arrange
var kernel = this.CreateAndInitializeKernel();

var function = kernel.CreateFunctionFromMethod(() => DayOfWeek.Friday, "GetDayOfWeek", "Retrieves the current day of the week.");
var function = kernel.CreateFunctionFromMethod(() => DayOfWeek.Friday.ToString(), "GetDayOfWeek", "Retrieves the current day of the week.");
kernel.ImportPluginFromFunctions("HelperFunctions", [function]);

var chatHistory = new ChatHistory();
Expand Down Expand Up @@ -939,20 +970,6 @@ private Kernel CreateAndInitializeKernel(bool importHelperPlugin = false)
_ => "31 and snowing",
};
}, "Get_Weather_For_City", "Gets the current weather for the specified city"),
kernel.CreateFunctionFromMethod((WeatherParameters parameters) =>
{
if (parameters.City.Name == "Dublin" && (parameters.City.Country == "Ireland" || parameters.City.Country == "IE"))
{
return Task.FromResult(42.8); // 42.8 Fahrenheit.
}

throw new NotSupportedException($"Weather in {parameters.City.Name} ({parameters.City.Country}) is not supported.");
}, "Get_Current_Temperature", "Get current temperature."),
kernel.CreateFunctionFromMethod((double temperatureInFahrenheit) =>
{
double temperatureInCelsius = (temperatureInFahrenheit - 32) * 5 / 9;
return Task.FromResult(temperatureInCelsius);
}, "Convert_Temperature_From_Fahrenheit_To_Celsius", "Convert temperature from Fahrenheit to Celsius.")
]);
}

Expand Down
12 changes: 3 additions & 9 deletions ...Tests/Connectors/AzureOpenAI/AzureOpenAIChatCompletion_AutoFunctionChoiceBehaviorTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,6 @@ public async Task SpecifiedInCodeInstructsConnectorToInvokeKernelFunctionAutomat
// Assert
Assert.NotNull(result);

Assert.Single(invokedFunctions);
Assert.Contains("GetCurrentDate", invokedFunctions);
}

Expand Down Expand Up @@ -94,7 +93,6 @@ public async Task SpecifiedInPromptInstructsConnectorToInvokeKernelFunctionAutom
// Assert
Assert.NotNull(result);

Assert.Single(invokedFunctions);
Assert.Contains("GetCurrentDate", invokedFunctions);
}

Expand Down Expand Up @@ -127,7 +125,7 @@ public async Task SpecifiedInCodeInstructsConnectorToInvokeKernelFunctionManuall

var functionCalls = FunctionCallContent.GetFunctionCalls(result);
Assert.NotNull(functionCalls);
Assert.Single(functionCalls);
Assert.NotEmpty(functionCalls);

var functionCall = functionCalls.First();
Assert.Equal("DateTimeUtils", functionCall.PluginName);
Expand Down Expand Up @@ -164,7 +162,6 @@ public async Task SpecifiedInCodeInstructsConnectorToInvokeKernelFunctionAutomat
// Assert
Assert.NotNull(result);

Assert.Single(invokedFunctions);
Assert.Contains("GetCurrentDate", invokedFunctions);
}

Expand Down Expand Up @@ -205,11 +202,10 @@ public async Task SpecifiedInPromptInstructsConnectorToInvokeKernelFunctionAutom
// Assert
Assert.NotNull(result);

Assert.Single(invokedFunctions);
Assert.Contains("GetCurrentDate", invokedFunctions);
}

[Fact(Skip = "Temporarily disabled to unblock PR pipeline.")]
[Fact]
public async Task SpecifiedInCodeInstructsConnectorToInvokeKernelFunctionManuallyForStreamingAsync()
{
// Arrange
Expand Down Expand Up @@ -240,7 +236,6 @@ public async Task SpecifiedInCodeInstructsConnectorToInvokeKernelFunctionManuall
}

// Assert
Assert.Single(functionsForManualInvocation);
Assert.Contains("DateTimeUtils-GetCurrentDate", functionsForManualInvocation);

Assert.Empty(invokedFunctions);
Expand Down Expand Up @@ -275,7 +270,7 @@ public async Task SpecifiedInCodeInstructsConnectorToInvokeNonKernelFunctionManu

var functionCalls = FunctionCallContent.GetFunctionCalls(result);
Assert.NotNull(functionCalls);
Assert.Single(functionCalls);
Assert.NotEmpty(functionCalls);

var functionCall = functionCalls.First();
Assert.Equal("DateTimeUtils", functionCall.PluginName);
Expand Down Expand Up @@ -313,7 +308,6 @@ public async Task SpecifiedInCodeInstructsConnectorToInvokeNonKernelFunctionManu
}

// Assert
Assert.Single(functionsForManualInvocation);
Assert.Contains("DateTimeUtils-GetCurrentDate", functionsForManualInvocation);

Assert.Empty(invokedFunctions);
Expand Down
10 changes: 2 additions & 8 deletions ...s/Connectors/AzureOpenAI/AzureOpenAIChatCompletion_RequiredFunctionChoiceBehaviorTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,6 @@ public async Task SpecifiedInCodeInstructsConnectorToInvokeKernelFunctionAutomat
// Assert
Assert.NotNull(result);

Assert.Single(invokedFunctions);
Assert.Contains("GetCurrentDate", invokedFunctions);
}

Expand Down Expand Up @@ -134,7 +133,6 @@ public async Task SpecifiedInPromptInstructsConnectorToInvokeKernelFunctionAutom
// Assert
Assert.NotNull(result);

Assert.Single(invokedFunctions);
Assert.Contains("GetCurrentDate", invokedFunctions);
}

Expand Down Expand Up @@ -167,7 +165,7 @@ public async Task SpecifiedInCodeInstructsConnectorToInvokeKernelFunctionManuall

var functionCalls = FunctionCallContent.GetFunctionCalls(result);
Assert.NotNull(functionCalls);
Assert.Single(functionCalls);
Assert.NotEmpty(functionCalls);

var functionCall = functionCalls.First();
Assert.Equal("DateTimeUtils", functionCall.PluginName);
Expand Down Expand Up @@ -250,7 +248,6 @@ public async Task SpecifiedInCodeInstructsConnectorToInvokeKernelFunctionAutomat
// Assert
Assert.NotNull(result);

Assert.Single(invokedFunctions);
Assert.Contains("GetCurrentDate", invokedFunctions);
}

Expand Down Expand Up @@ -291,7 +288,6 @@ public async Task SpecifiedInPromptInstructsConnectorToInvokeKernelFunctionAutom
// Assert
Assert.NotNull(result);

Assert.Single(invokedFunctions);
Assert.Contains("GetCurrentDate", invokedFunctions);
}

Expand Down Expand Up @@ -326,7 +322,6 @@ public async Task SpecifiedInCodeInstructsConnectorToInvokeKernelFunctionManuall
}

// Assert
Assert.Single(functionsForManualInvocation);
Assert.Contains("DateTimeUtils-GetCurrentDate", functionsForManualInvocation);

Assert.Empty(invokedFunctions);
Expand Down Expand Up @@ -361,7 +356,7 @@ public async Task SpecifiedInCodeInstructsConnectorToInvokeNonKernelFunctionManu

var functionCalls = FunctionCallContent.GetFunctionCalls(result);
Assert.NotNull(functionCalls);
Assert.Single(functionCalls);
Assert.NotEmpty(functionCalls);

var functionCall = functionCalls.First();
Assert.Equal("DateTimeUtils", functionCall.PluginName);
Expand Down Expand Up @@ -399,7 +394,6 @@ public async Task SpecifiedInCodeInstructsConnectorToInvokeNonKernelFunctionManu
}

// Assert
Assert.Single(functionsForManualInvocation);
Assert.Contains("DateTimeUtils-GetCurrentDate", functionsForManualInvocation);

Assert.Empty(invokedFunctions);
Expand Down
10 changes: 2 additions & 8 deletions ...ntegrationTests/Connectors/OpenAI/OpenAIChatCompletion_AutoFunctionChoiceBehaviorTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,6 @@ public async Task SpecifiedInCodeInstructsConnectorToInvokeKernelFunctionAutomat
// Assert
Assert.NotNull(result);

Assert.Single(invokedFunctions);
Assert.Contains("GetCurrentDate", invokedFunctions);
}

Expand Down Expand Up @@ -91,7 +90,6 @@ public async Task SpecifiedInPromptInstructsConnectorToInvokeKernelFunctionAutom
// Assert
Assert.NotNull(result);

Assert.Single(invokedFunctions);
Assert.Contains("GetCurrentDate", invokedFunctions);
}

Expand Down Expand Up @@ -124,7 +122,7 @@ public async Task SpecifiedInCodeInstructsConnectorToInvokeKernelFunctionManuall

var functionCalls = FunctionCallContent.GetFunctionCalls(result);
Assert.NotNull(functionCalls);
Assert.Single(functionCalls);
Assert.NotEmpty(functionCalls);

var functionCall = functionCalls.First();
Assert.Equal("DateTimeUtils", functionCall.PluginName);
Expand Down Expand Up @@ -161,7 +159,6 @@ public async Task SpecifiedInCodeInstructsConnectorToInvokeKernelFunctionAutomat
// Assert
Assert.NotNull(result);

Assert.Single(invokedFunctions);
Assert.Contains("GetCurrentDate", invokedFunctions);
}

Expand Down Expand Up @@ -202,7 +199,6 @@ public async Task SpecifiedInPromptInstructsConnectorToInvokeKernelFunctionAutom
// Assert
Assert.NotNull(result);

Assert.Single(invokedFunctions);
Assert.Contains("GetCurrentDate", invokedFunctions);
}

Expand Down Expand Up @@ -237,7 +233,6 @@ public async Task SpecifiedInCodeInstructsConnectorToInvokeKernelFunctionManuall
}

// Assert
Assert.Single(functionsForManualInvocation);
Assert.Contains("DateTimeUtils-GetCurrentDate", functionsForManualInvocation);

Assert.Empty(invokedFunctions);
Expand Down Expand Up @@ -272,7 +267,7 @@ public async Task SpecifiedInCodeInstructsConnectorToInvokeNonKernelFunctionManu

var functionCalls = FunctionCallContent.GetFunctionCalls(result);
Assert.NotNull(functionCalls);
Assert.Single(functionCalls);
Assert.NotEmpty(functionCalls);

var functionCall = functionCalls.First();
Assert.Equal("DateTimeUtils", functionCall.PluginName);
Expand Down Expand Up @@ -310,7 +305,6 @@ public async Task SpecifiedInCodeInstructsConnectorToInvokeNonKernelFunctionManu
}

// Assert
Assert.Single(functionsForManualInvocation);
Assert.Contains("DateTimeUtils-GetCurrentDate", functionsForManualInvocation);

Assert.Empty(invokedFunctions);
Expand Down
Loading

0 comments on commit 8924bdc

Please sign in to comment.