From 870bae73c952cc1639c45f2ef06ac930c8cda669 Mon Sep 17 00:00:00 2001 From: Yiannis Charalambous Date: Fri, 19 Apr 2024 12:35:32 +0100 Subject: [PATCH 01/13] Removed unused imports --- esbmc_ai/commands/fix_code_command.py | 1 - 1 file changed, 1 deletion(-) diff --git a/esbmc_ai/commands/fix_code_command.py b/esbmc_ai/commands/fix_code_command.py index c624ea3..00f8517 100644 --- a/esbmc_ai/commands/fix_code_command.py +++ b/esbmc_ai/commands/fix_code_command.py @@ -3,7 +3,6 @@ import sys from typing import Any, Tuple from typing_extensions import override -from langchain.schema import AIMessage, HumanMessage from esbmc_ai.chat_response import FinishReason From 236bef7dee3e1e934e113f25359f2d156b1c2c57 Mon Sep 17 00:00:00 2001 From: Yiannis Charalambous Date: Fri, 19 Apr 2024 13:11:31 +0100 Subject: [PATCH 02/13] SolutionGenerator: Allow ChatPromptSettings to be supplied as well --- esbmc_ai/solution_generator.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/esbmc_ai/solution_generator.py b/esbmc_ai/solution_generator.py index f22f54e..a3ca769 100644 --- a/esbmc_ai/solution_generator.py +++ b/esbmc_ai/solution_generator.py @@ -82,7 +82,7 @@ def get_esbmc_output_formatted(esbmc_output_type: str, esbmc_output: str) -> str class SolutionGenerator(BaseChatInterface): def __init__( self, - ai_model_agent: DynamicAIModelAgent, + ai_model_agent: DynamicAIModelAgent | ChatPromptSettings, llm: BaseLanguageModel, source_code: str, esbmc_output: str, @@ -91,10 +91,16 @@ def __init__( source_code_format: str = "full", esbmc_output_type: str = "full", ) -> None: - # Convert to chat prompt - chat_prompt: ChatPromptSettings = DynamicAIModelAgent.to_chat_prompt_settings( - ai_model_agent=ai_model_agent, scenario=scenario - ) + """Initializes the solution generator. This ModelChat provides Dynamic + Prompting. Will get the correct scenario from the DynamicAIModelAgent + supplied and create a ChatPrompt.""" + + chat_prompt: ChatPromptSettings = ai_model_agent + if isinstance(ai_model_agent, DynamicAIModelAgent): + # Convert to chat prompt + chat_prompt = DynamicAIModelAgent.to_chat_prompt_settings( + ai_model_agent=ai_model_agent, scenario=scenario + ) super().__init__( ai_model_agent=chat_prompt, @@ -102,8 +108,6 @@ def __init__( llm=llm, ) - self.initial_prompt = ai_model_agent.initial_prompt - self.esbmc_output_type: str = esbmc_output_type self.source_code_format: str = source_code_format self.source_code_raw: str = source_code @@ -162,7 +166,9 @@ def update_state( self.esbmc_output = esbmc_output def generate_solution(self) -> tuple[str, FinishReason]: - self.push_to_message_stack(HumanMessage(content=self.initial_prompt)) + self.push_to_message_stack( + HumanMessage(content=self.ai_model_agent.initial_prompt) + ) # Format source code source_code_formatted: str = get_source_code_formatted( From 32021ee2f8cf1dc52314e41ad0a2f6c949961ec9 Mon Sep 17 00:00:00 2001 From: Yiannis Charalambous Date: Fri, 19 Apr 2024 13:12:12 +0100 Subject: [PATCH 03/13] Created LatestStateSolutionGenerator --- esbmc_ai/latest_state_solution_generator.py | 23 +++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 esbmc_ai/latest_state_solution_generator.py diff --git a/esbmc_ai/latest_state_solution_generator.py b/esbmc_ai/latest_state_solution_generator.py new file mode 100644 index 0000000..391bdd9 --- /dev/null +++ b/esbmc_ai/latest_state_solution_generator.py @@ -0,0 +1,23 @@ +# Author: Yiannis Charalambous + +from typing_extensions import override +from langchain import BaseMessage +from esbmc_ai.solution_generator import SolutionGenerator +from esbmc_ai.chat_response import FinishReason + + +class LatestStateSolutionGenerator(SolutionGenerator): + """SolutionGenerator that only shows the latest source code and verifier + output state.""" + + @override + def generate_solution(self) -> tuple[str, FinishReason]: + # Backup message stack and clear before sending base message. We want + # to keep the message stack intact because we will print it with + # print_raw_conversation. + messages: list[BaseMessage] = self.messages + self.messages: list[BaseMessage] = [] + solution, finish_reason = super().generate_solution() + # Restore + self.messages = messages + return solution, finish_reason From 7d30d471c42138cb266ff0df53666af5a48c13cd Mon Sep 17 00:00:00 2001 From: Yiannis Charalambous Date: Fri, 19 Apr 2024 13:27:26 +0100 Subject: [PATCH 04/13] Added enabling LatestStateSolutionGenerator from config. --- config.json | 1 + esbmc_ai/commands/fix_code_command.py | 51 +++++++++++++++++++-------- esbmc_ai/config.py | 12 +++++++ 3 files changed, 49 insertions(+), 15 deletions(-) diff --git a/config.json b/config.json index b53172d..2324bae 100644 --- a/config.json +++ b/config.json @@ -72,6 +72,7 @@ "generate_solution": { "max_attempts": 5, "temperature": 1.3, + "message_history": "normal", "scenarios": { "division by zero": { "system": [ diff --git a/esbmc_ai/commands/fix_code_command.py b/esbmc_ai/commands/fix_code_command.py index 00f8517..09c0226 100644 --- a/esbmc_ai/commands/fix_code_command.py +++ b/esbmc_ai/commands/fix_code_command.py @@ -5,6 +5,7 @@ from typing_extensions import override from esbmc_ai.chat_response import FinishReason +from esbmc_ai.latest_state_solution_generator import LatestStateSolutionGenerator from .chat_command import ChatCommand from .. import config @@ -61,21 +62,41 @@ def print_raw_conversation() -> None: ) try: - solution_generator = SolutionGenerator( - ai_model_agent=config.chat_prompt_generator_mode, - source_code=source_code, - esbmc_output=esbmc_output, - ai_model=config.ai_model, - llm=config.ai_model.create_llm( - api_keys=config.api_keys, - temperature=config.chat_prompt_generator_mode.temperature, - requests_max_tries=config.requests_max_tries, - requests_timeout=config.requests_timeout, - ), - scenario=scenario, - source_code_format=config.source_code_format, - esbmc_output_type=config.esbmc_output_type, - ) + match config.fix_code_message_history: + case "normal": + solution_generator = SolutionGenerator( + ai_model_agent=config.chat_prompt_generator_mode, + source_code=source_code, + esbmc_output=esbmc_output, + ai_model=config.ai_model, + llm=config.ai_model.create_llm( + api_keys=config.api_keys, + temperature=config.chat_prompt_generator_mode.temperature, + requests_max_tries=config.requests_max_tries, + requests_timeout=config.requests_timeout, + ), + scenario=scenario, + source_code_format=config.source_code_format, + esbmc_output_type=config.esbmc_output_type, + ) + case "latest_only": + solution_generator = LatestStateSolutionGenerator( + ai_model_agent=config.chat_prompt_generator_mode, + source_code=source_code, + esbmc_output=esbmc_output, + ai_model=config.ai_model, + llm=config.ai_model.create_llm( + api_keys=config.api_keys, + temperature=config.chat_prompt_generator_mode.temperature, + requests_max_tries=config.requests_max_tries, + requests_timeout=config.requests_timeout, + ), + scenario=scenario, + source_code_format=config.source_code_format, + esbmc_output_type=config.esbmc_output_type, + ) + case _: + raise ValueError() except ESBMCTimedOutException: print("error: ESBMC has timed out...") sys.exit(1) diff --git a/esbmc_ai/config.py b/esbmc_ai/config.py index 1506178..a15f3f6 100644 --- a/esbmc_ai/config.py +++ b/esbmc_ai/config.py @@ -44,6 +44,7 @@ source_code_format: str = "full" fix_code_max_attempts: int = 5 +fix_code_message_history: str = "" requests_max_tries: int = 5 requests_timeout: float = 60 @@ -384,6 +385,17 @@ def load_config(file_path: str) -> None: f"ESBMC output type in the config is not valid: {esbmc_output_type}" ) + global fix_code_message_history + fix_code_message_history, _ = _load_config_value( + config_file=config_file["chat_modes"]["generate_solution"], + name="message_history", + ) + + if fix_code_message_history not in ["normal", "latest_only", "reverse"]: + raise ValueError( + f"error: fix code mode message history not valid: {fix_code_message_history}" + ) + global requests_max_tries requests_max_tries = int( _load_config_real_number( From 932f60ecb5d2d534635a74d5e213dd92d52afee9 Mon Sep 17 00:00:00 2001 From: Yiannis Charalambous Date: Fri, 19 Apr 2024 13:27:50 +0100 Subject: [PATCH 05/13] FCM Latest State: Fixed bug that did not actually store message history --- esbmc_ai/latest_state_solution_generator.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/esbmc_ai/latest_state_solution_generator.py b/esbmc_ai/latest_state_solution_generator.py index 391bdd9..81d15a5 100644 --- a/esbmc_ai/latest_state_solution_generator.py +++ b/esbmc_ai/latest_state_solution_generator.py @@ -1,10 +1,12 @@ # Author: Yiannis Charalambous from typing_extensions import override -from langchain import BaseMessage +from langchain_core.messages import BaseMessage from esbmc_ai.solution_generator import SolutionGenerator from esbmc_ai.chat_response import FinishReason +# TODO Test me + class LatestStateSolutionGenerator(SolutionGenerator): """SolutionGenerator that only shows the latest source code and verifier @@ -18,6 +20,8 @@ def generate_solution(self) -> tuple[str, FinishReason]: messages: list[BaseMessage] = self.messages self.messages: list[BaseMessage] = [] solution, finish_reason = super().generate_solution() + # Append last messages to the messages stack + messages.extend(self.messages) # Restore self.messages = messages return solution, finish_reason From 50f55c0b3e4b30a3de3dc872c00382b58db2a4d3 Mon Sep 17 00:00:00 2001 From: Yiannis Charalambous Date: Fri, 19 Apr 2024 14:07:24 +0100 Subject: [PATCH 06/13] Added Reverse Order Solution Generator --- esbmc_ai/commands/fix_code_command.py | 21 +++++- esbmc_ai/reverse_order_solution_generator.py | 71 ++++++++++++++++++++ 2 files changed, 91 insertions(+), 1 deletion(-) create mode 100644 esbmc_ai/reverse_order_solution_generator.py diff --git a/esbmc_ai/commands/fix_code_command.py b/esbmc_ai/commands/fix_code_command.py index 09c0226..75b59e3 100644 --- a/esbmc_ai/commands/fix_code_command.py +++ b/esbmc_ai/commands/fix_code_command.py @@ -6,6 +6,7 @@ from esbmc_ai.chat_response import FinishReason from esbmc_ai.latest_state_solution_generator import LatestStateSolutionGenerator +from esbmc_ai.reverse_order_solution_generator import ReverseOrderSolutionGenerator from .chat_command import ChatCommand from .. import config @@ -95,8 +96,26 @@ def print_raw_conversation() -> None: source_code_format=config.source_code_format, esbmc_output_type=config.esbmc_output_type, ) + case "reverse": + solution_generator = ReverseOrderSolutionGenerator( + ai_model_agent=config.chat_prompt_generator_mode, + source_code=source_code, + esbmc_output=esbmc_output, + ai_model=config.ai_model, + llm=config.ai_model.create_llm( + api_keys=config.api_keys, + temperature=config.chat_prompt_generator_mode.temperature, + requests_max_tries=config.requests_max_tries, + requests_timeout=config.requests_timeout, + ), + scenario=scenario, + source_code_format=config.source_code_format, + esbmc_output_type=config.esbmc_output_type, + ) case _: - raise ValueError() + raise NotImplementedError( + f"error: {config.fix_code_message_history} has not been implemented in the Fix Code Command" + ) except ESBMCTimedOutException: print("error: ESBMC has timed out...") sys.exit(1) diff --git a/esbmc_ai/reverse_order_solution_generator.py b/esbmc_ai/reverse_order_solution_generator.py new file mode 100644 index 0000000..15dcad8 --- /dev/null +++ b/esbmc_ai/reverse_order_solution_generator.py @@ -0,0 +1,71 @@ +# Author: Yiannis Charalambous + +from langchain.schema import BaseMessage, HumanMessage +from typing_extensions import override, Optional +from esbmc_ai.solution_generator import ( + SolutionGenerator, + get_source_code_formatted, + get_source_code_err_line_idx, + get_clang_err_line_index, + apply_line_patch, +) +from esbmc_ai.chat_response import FinishReason, ChatResponse + +# TODO Test me + + +class ReverseOrderSolutionGenerator(SolutionGenerator): + """SolutionGenerator that shows the source code and verifier output state in + reverse order.""" + + @override + def generate_solution(self) -> tuple[str, FinishReason]: + self.push_to_message_stack( + HumanMessage(content=self.ai_model_agent.initial_prompt) + ) + + # Format source code + source_code_formatted: str = get_source_code_formatted( + source_code_format=self.source_code_format, + source_code=self.source_code_raw, + esbmc_output=self.esbmc_output, + ) + + # Apply template substitution to message stack + self.apply_template_value( + source_code=source_code_formatted, + esbmc_output=self.esbmc_output, + ) + + # Reverse the messages + messages: list[BaseMessage] = self.messages.copy() + self.messages.reverse() + + # Generate the solution + response: ChatResponse = self.send_message() + + # Add to the reversed message the new message received by the LLM. + messages.append(self.messages[-1]) + # Restore + self.messages = messages + + solution: str = str(response.message.content) + + solution = SolutionGenerator.get_code_from_solution(solution) + + # If source code passed to LLM is formatted then we need to recombine to + # full source code before giving to ESBMC + match self.source_code_format: + case "single": + # Get source code error line from esbmc output + line: Optional[int] = get_source_code_err_line_idx(self.esbmc_output) + if not line: + # Check if it parses + line = get_clang_err_line_index(self.esbmc_output) + + assert ( + line + ), "fix code command: error line could not be found to apply brutal patch replacement" + solution = apply_line_patch(self.source_code_raw, solution, line, line) + + return solution, response.finish_reason From 900c66028f38f1b0cd593c19b51e1590f1b63c00 Mon Sep 17 00:00:00 2001 From: Yiannis Charalambous Date: Fri, 19 Apr 2024 17:50:57 +0100 Subject: [PATCH 07/13] Update --- esbmc_ai/config.py | 1 + 1 file changed, 1 insertion(+) diff --git a/esbmc_ai/config.py b/esbmc_ai/config.py index a15f3f6..3ce23fa 100644 --- a/esbmc_ai/config.py +++ b/esbmc_ai/config.py @@ -58,6 +58,7 @@ cfg_path: str +# TODO Get rid of this class as soon as ConfigTool with the pyautoconfig class AIAgentConversation(NamedTuple): """Immutable class describing the conversation definition for an AI agent. The class represents the system messages of the AI agent defined and contains a load From 3242fbebc2955ff72c2513aa6686c98be91b9642 Mon Sep 17 00:00:00 2001 From: Yiannis Charalambous Date: Fri, 19 Apr 2024 17:52:02 +0100 Subject: [PATCH 08/13] SolutionGenerator: Moved template formatting to update_state --- esbmc_ai/commands/fix_code_command.py | 123 +++++++++++--------------- esbmc_ai/solution_generator.py | 63 ++++++------- 2 files changed, 87 insertions(+), 99 deletions(-) diff --git a/esbmc_ai/commands/fix_code_command.py b/esbmc_ai/commands/fix_code_command.py index 75b59e3..d051641 100644 --- a/esbmc_ai/commands/fix_code_command.py +++ b/esbmc_ai/commands/fix_code_command.py @@ -19,8 +19,6 @@ from ..solution_generator import ( ESBMCTimedOutException, SolutionGenerator, - SourceCodeParseError, - get_esbmc_output_formatted, ) from ..logging import print_horizontal_line, printv, printvv @@ -62,60 +60,59 @@ def print_raw_conversation() -> None: else "Using generic prompt..." ) + match config.fix_code_message_history: + case "normal": + solution_generator = SolutionGenerator( + ai_model_agent=config.chat_prompt_generator_mode, + ai_model=config.ai_model, + llm=config.ai_model.create_llm( + api_keys=config.api_keys, + temperature=config.chat_prompt_generator_mode.temperature, + requests_max_tries=config.requests_max_tries, + requests_timeout=config.requests_timeout, + ), + scenario=scenario, + source_code_format=config.source_code_format, + esbmc_output_type=config.esbmc_output_type, + ) + case "latest_only": + solution_generator = LatestStateSolutionGenerator( + ai_model_agent=config.chat_prompt_generator_mode, + ai_model=config.ai_model, + llm=config.ai_model.create_llm( + api_keys=config.api_keys, + temperature=config.chat_prompt_generator_mode.temperature, + requests_max_tries=config.requests_max_tries, + requests_timeout=config.requests_timeout, + ), + scenario=scenario, + source_code_format=config.source_code_format, + esbmc_output_type=config.esbmc_output_type, + ) + case "reverse": + solution_generator = ReverseOrderSolutionGenerator( + ai_model_agent=config.chat_prompt_generator_mode, + ai_model=config.ai_model, + llm=config.ai_model.create_llm( + api_keys=config.api_keys, + temperature=config.chat_prompt_generator_mode.temperature, + requests_max_tries=config.requests_max_tries, + requests_timeout=config.requests_timeout, + ), + scenario=scenario, + source_code_format=config.source_code_format, + esbmc_output_type=config.esbmc_output_type, + ) + case _: + raise NotImplementedError( + f"error: {config.fix_code_message_history} has not been implemented in the Fix Code Command" + ) + try: - match config.fix_code_message_history: - case "normal": - solution_generator = SolutionGenerator( - ai_model_agent=config.chat_prompt_generator_mode, - source_code=source_code, - esbmc_output=esbmc_output, - ai_model=config.ai_model, - llm=config.ai_model.create_llm( - api_keys=config.api_keys, - temperature=config.chat_prompt_generator_mode.temperature, - requests_max_tries=config.requests_max_tries, - requests_timeout=config.requests_timeout, - ), - scenario=scenario, - source_code_format=config.source_code_format, - esbmc_output_type=config.esbmc_output_type, - ) - case "latest_only": - solution_generator = LatestStateSolutionGenerator( - ai_model_agent=config.chat_prompt_generator_mode, - source_code=source_code, - esbmc_output=esbmc_output, - ai_model=config.ai_model, - llm=config.ai_model.create_llm( - api_keys=config.api_keys, - temperature=config.chat_prompt_generator_mode.temperature, - requests_max_tries=config.requests_max_tries, - requests_timeout=config.requests_timeout, - ), - scenario=scenario, - source_code_format=config.source_code_format, - esbmc_output_type=config.esbmc_output_type, - ) - case "reverse": - solution_generator = ReverseOrderSolutionGenerator( - ai_model_agent=config.chat_prompt_generator_mode, - source_code=source_code, - esbmc_output=esbmc_output, - ai_model=config.ai_model, - llm=config.ai_model.create_llm( - api_keys=config.api_keys, - temperature=config.chat_prompt_generator_mode.temperature, - requests_max_tries=config.requests_max_tries, - requests_timeout=config.requests_timeout, - ), - scenario=scenario, - source_code_format=config.source_code_format, - esbmc_output_type=config.esbmc_output_type, - ) - case _: - raise NotImplementedError( - f"error: {config.fix_code_message_history} has not been implemented in the Fix Code Command" - ) + solution_generator.update_state( + source_code=source_code, + esbmc_output=esbmc_output, + ) except ESBMCTimedOutException: print("error: ESBMC has timed out...") sys.exit(1) @@ -132,9 +129,7 @@ def print_raw_conversation() -> None: llm_solution, finish_reason = solution_generator.generate_solution() self.anim.stop() if finish_reason == FinishReason.length: - self.anim.start("Compressing message stack... Please Wait") solution_generator.compress_message_stack() - self.anim.stop() else: source_code = llm_solution break @@ -174,16 +169,9 @@ def print_raw_conversation() -> None: return False, source_code - # TODO Move this process into Solution Generator since have (beginning) is done - # inside, and the other half is done here. - # Get formatted ESBMC output try: - esbmc_output = get_esbmc_output_formatted( - esbmc_output_type=config.esbmc_output_type, - esbmc_output=esbmc_output, - ) - except SourceCodeParseError: - pass + # Update state + solution_generator.update_state(source_code, esbmc_output) except ESBMCTimedOutException: print("error: ESBMC has timed out...") sys.exit(1) @@ -191,9 +179,6 @@ def print_raw_conversation() -> None: # Failure case print(f"ESBMC-AI Notice: Failure {idx+1}/{max_retries}: Retrying...") - # Update state - solution_generator.update_state(source_code, esbmc_output) - if config.raw_conversation: print_raw_conversation() diff --git a/esbmc_ai/solution_generator.py b/esbmc_ai/solution_generator.py index a3ca769..95b8eec 100644 --- a/esbmc_ai/solution_generator.py +++ b/esbmc_ai/solution_generator.py @@ -84,8 +84,6 @@ def __init__( self, ai_model_agent: DynamicAIModelAgent | ChatPromptSettings, llm: BaseLanguageModel, - source_code: str, - esbmc_output: str, ai_model: AIModel, scenario: str = "", source_code_format: str = "full", @@ -110,26 +108,17 @@ def __init__( self.esbmc_output_type: str = esbmc_output_type self.source_code_format: str = source_code_format - self.source_code_raw: str = source_code - # Used for resetting state. - self._original_source_code: str = source_code - # Format ESBMC output - try: - self.esbmc_output = get_esbmc_output_formatted( - esbmc_output_type=self.esbmc_output_type, - esbmc_output=esbmc_output, - ) - except SourceCodeParseError: - # When clang output is displayed, show it entirely as it doesn't get very - # big. - self.esbmc_output = esbmc_output + self.source_code_raw: Optional[str] = None + self.source_code_formatted: Optional[str] = None + self.esbmc_output: Optional[str] = None @override def compress_message_stack(self) -> None: # Resets the conversation - cannot summarize code + # If generate_solution is called after this point, it will start new + # with the currently set state. self.messages: list[BaseMessage] = [] - self.source_code_raw = self._original_source_code @classmethod def get_code_from_solution(cls, solution: str) -> str: @@ -157,29 +146,43 @@ def get_code_from_solution(cls, solution: str) -> str: pass return solution - def update_state( - self, source_code: Optional[str] = None, esbmc_output: Optional[str] = None - ) -> None: - if source_code: - self.source_code_raw = source_code - if esbmc_output: - self.esbmc_output = esbmc_output + def update_state(self, source_code: str, esbmc_output: str) -> None: + """Updates the latest state of the code and ESBMC output. This should be + called before generate_solution.""" + self.source_code_raw = source_code - def generate_solution(self) -> tuple[str, FinishReason]: - self.push_to_message_stack( - HumanMessage(content=self.ai_model_agent.initial_prompt) - ) + # Format ESBMC output + try: + self.esbmc_output = get_esbmc_output_formatted( + esbmc_output_type=self.esbmc_output_type, + esbmc_output=esbmc_output, + ) + except SourceCodeParseError: + # When clang output is displayed, show it entirely as it doesn't get very + # big. + self.esbmc_output = esbmc_output # Format source code - source_code_formatted: str = get_source_code_formatted( + self.source_code_formatted = get_source_code_formatted( source_code_format=self.source_code_format, - source_code=self.source_code_raw, + source_code=source_code, esbmc_output=self.esbmc_output, ) + def generate_solution(self) -> tuple[str, FinishReason]: + assert ( + self.source_code_raw is not None + and self.source_code_formatted is not None + and self.esbmc_output is not None + ), "Call update_state before calling generate_solution." + + self.push_to_message_stack( + HumanMessage(content=self.ai_model_agent.initial_prompt) + ) + # Apply template substitution to message stack self.apply_template_value( - source_code=source_code_formatted, + source_code=self.source_code_formatted, esbmc_output=self.esbmc_output, ) From 92086a83ff57e9aa1503689e29f47a4b6f787ecd Mon Sep 17 00:00:00 2001 From: Yiannis Charalambous Date: Fri, 19 Apr 2024 17:52:39 +0100 Subject: [PATCH 09/13] SolutionGenerator: Updated Reverse order to override send message --- esbmc_ai/reverse_order_solution_generator.py | 43 +-------- .../test_reverse_order_solution_generator.py | 92 +++++++++++++++++++ 2 files changed, 95 insertions(+), 40 deletions(-) create mode 100644 tests/test_reverse_order_solution_generator.py diff --git a/esbmc_ai/reverse_order_solution_generator.py b/esbmc_ai/reverse_order_solution_generator.py index 15dcad8..6d47938 100644 --- a/esbmc_ai/reverse_order_solution_generator.py +++ b/esbmc_ai/reverse_order_solution_generator.py @@ -19,53 +19,16 @@ class ReverseOrderSolutionGenerator(SolutionGenerator): reverse order.""" @override - def generate_solution(self) -> tuple[str, FinishReason]: - self.push_to_message_stack( - HumanMessage(content=self.ai_model_agent.initial_prompt) - ) - - # Format source code - source_code_formatted: str = get_source_code_formatted( - source_code_format=self.source_code_format, - source_code=self.source_code_raw, - esbmc_output=self.esbmc_output, - ) - - # Apply template substitution to message stack - self.apply_template_value( - source_code=source_code_formatted, - esbmc_output=self.esbmc_output, - ) - + def send_message(self, message: Optional[str] = None) -> ChatResponse: # Reverse the messages messages: list[BaseMessage] = self.messages.copy() self.messages.reverse() - # Generate the solution - response: ChatResponse = self.send_message() + response: ChatResponse = super().send_message(message) # Add to the reversed message the new message received by the LLM. messages.append(self.messages[-1]) # Restore self.messages = messages - solution: str = str(response.message.content) - - solution = SolutionGenerator.get_code_from_solution(solution) - - # If source code passed to LLM is formatted then we need to recombine to - # full source code before giving to ESBMC - match self.source_code_format: - case "single": - # Get source code error line from esbmc output - line: Optional[int] = get_source_code_err_line_idx(self.esbmc_output) - if not line: - # Check if it parses - line = get_clang_err_line_index(self.esbmc_output) - - assert ( - line - ), "fix code command: error line could not be found to apply brutal patch replacement" - solution = apply_line_patch(self.source_code_raw, solution, line, line) - - return solution, response.finish_reason + return response diff --git a/tests/test_reverse_order_solution_generator.py b/tests/test_reverse_order_solution_generator.py new file mode 100644 index 0000000..65bb69d --- /dev/null +++ b/tests/test_reverse_order_solution_generator.py @@ -0,0 +1,92 @@ +# Author: Yiannis Charalambous + +import pytest + +from langchain.schema import HumanMessage, AIMessage, SystemMessage +from langchain_community.llms.fake import FakeListLLM + +from esbmc_ai.ai_models import AIModel +from esbmc_ai.config import AIAgentConversation, ChatPromptSettings +from esbmc_ai.reverse_order_solution_generator import ReverseOrderSolutionGenerator + + +@pytest.fixture(scope="function") +def setup_llm_model(): + llm = FakeListLLM( + responses=[ + "This is a test response", + "Another test response", + "One more!", + ], + ) + model = AIModel("test model", 1000) + return llm, model + + +def test_call_update_state_first(setup_llm_model) -> None: + llm, model = setup_llm_model + + chat_settings = ChatPromptSettings( + system_messages=AIAgentConversation( + messages=( + SystemMessage(content="Test message 1"), + HumanMessage(content="Test message 2"), + AIMessage(content="Test message 3"), + ), + ), + initial_prompt="Initial test message", + temperature=1.0, + ) + + solution_generator = ReverseOrderSolutionGenerator( + llm=llm, + ai_model=model, + ai_model_agent=chat_settings, + ) + + with pytest.raises(AssertionError): + solution_generator.generate_solution() + + +def test_message_stack(setup_llm_model) -> None: + llm, model = setup_llm_model + + chat_settings = ChatPromptSettings( + system_messages=AIAgentConversation( + messages=( + SystemMessage(content="Test message 1"), + HumanMessage(content="Test message 2"), + AIMessage(content="Test message 3"), + ), + ), + initial_prompt="Initial test message", + temperature=1.0, + ) + + solution_generator = ReverseOrderSolutionGenerator( + llm=llm, + ai_model=model, + ai_model_agent=chat_settings, + ) + + with pytest.raises(AssertionError): + solution_generator.generate_solution() + + solution_generator.update_state("", "") + + solution, _ = solution_generator.generate_solution() + assert solution == llm.responses[0] + solution_generator.ai_model_agent.initial_prompt = "Test message 2" + solution, _ = solution_generator.generate_solution() + assert solution == llm.responses[1] + solution_generator.ai_model_agent.initial_prompt = "Test message 3" + solution, _ = solution_generator.generate_solution() + assert solution == llm.responses[2] + + # Test history is intact + assert solution_generator.messages[0].content == "Initial test message" + assert solution_generator.messages[1].content == "This is a test response" + assert solution_generator.messages[2].content == "Test message 2" + assert solution_generator.messages[3].content == "Another test response" + assert solution_generator.messages[4].content == "Test message 3" + assert solution_generator.messages[5].content == "One more!" From 13460799330c36e31e8adc384e0bdd2f3250ba4b Mon Sep 17 00:00:00 2001 From: Yiannis Charalambous Date: Fri, 19 Apr 2024 17:52:48 +0100 Subject: [PATCH 10/13] Write tests --- tests/test_latest_state_solution_generator.py | 92 +++++++++++++++++++ 1 file changed, 92 insertions(+) create mode 100644 tests/test_latest_state_solution_generator.py diff --git a/tests/test_latest_state_solution_generator.py b/tests/test_latest_state_solution_generator.py new file mode 100644 index 0000000..6bf4631 --- /dev/null +++ b/tests/test_latest_state_solution_generator.py @@ -0,0 +1,92 @@ +# Author: Yiannis Charalambous + +import pytest + +from langchain.schema import HumanMessage, AIMessage, SystemMessage +from langchain_community.llms.fake import FakeListLLM + +from esbmc_ai.ai_models import AIModel +from esbmc_ai.config import AIAgentConversation, ChatPromptSettings +from esbmc_ai.latest_state_solution_generator import LatestStateSolutionGenerator + + +@pytest.fixture(scope="function") +def setup_llm_model(): + llm = FakeListLLM( + responses=[ + "This is a test response", + "Another test response", + "One more!", + ], + ) + model = AIModel("test model", 1000) + return llm, model + + +def test_call_update_state_first(setup_llm_model) -> None: + llm, model = setup_llm_model + + chat_settings = ChatPromptSettings( + system_messages=AIAgentConversation( + messages=( + SystemMessage(content="Test message 1"), + HumanMessage(content="Test message 2"), + AIMessage(content="Test message 3"), + ), + ), + initial_prompt="Initial test message", + temperature=1.0, + ) + + solution_generator = LatestStateSolutionGenerator( + llm=llm, + ai_model=model, + ai_model_agent=chat_settings, + ) + + with pytest.raises(AssertionError): + solution_generator.generate_solution() + + +def test_message_stack(setup_llm_model) -> None: + llm, model = setup_llm_model + + chat_settings = ChatPromptSettings( + system_messages=AIAgentConversation( + messages=( + SystemMessage(content="Test message 1"), + HumanMessage(content="Test message 2"), + AIMessage(content="Test message 3"), + ), + ), + initial_prompt="Initial test message", + temperature=1.0, + ) + + solution_generator = LatestStateSolutionGenerator( + llm=llm, + ai_model=model, + ai_model_agent=chat_settings, + ) + + with pytest.raises(AssertionError): + solution_generator.generate_solution() + + solution_generator.update_state("", "") + + solution, _ = solution_generator.generate_solution() + assert solution == llm.responses[0] + solution_generator.ai_model_agent.initial_prompt = "Test message 2" + solution, _ = solution_generator.generate_solution() + assert solution == llm.responses[1] + solution_generator.ai_model_agent.initial_prompt = "Test message 3" + solution, _ = solution_generator.generate_solution() + assert solution == llm.responses[2] + + # Test history is intact + assert solution_generator.messages[0].content == "Initial test message" + assert solution_generator.messages[1].content == "This is a test response" + assert solution_generator.messages[2].content == "Test message 2" + assert solution_generator.messages[3].content == "Another test response" + assert solution_generator.messages[4].content == "Test message 3" + assert solution_generator.messages[5].content == "One more!" From dd791b8e4a7c1c93e8e217ac709c9682cc204137 Mon Sep 17 00:00:00 2001 From: Yiannis Charalambous Date: Sat, 20 Apr 2024 18:03:22 +0100 Subject: [PATCH 11/13] Removed unused imports in ReverseOrderSolutionGenerator --- esbmc_ai/latest_state_solution_generator.py | 2 - esbmc_ai/reverse_order_solution_generator.py | 12 ++---- tests/test_latest_state_solution_generator.py | 39 +++++++++++++++++++ 3 files changed, 42 insertions(+), 11 deletions(-) diff --git a/esbmc_ai/latest_state_solution_generator.py b/esbmc_ai/latest_state_solution_generator.py index 81d15a5..90e8185 100644 --- a/esbmc_ai/latest_state_solution_generator.py +++ b/esbmc_ai/latest_state_solution_generator.py @@ -5,8 +5,6 @@ from esbmc_ai.solution_generator import SolutionGenerator from esbmc_ai.chat_response import FinishReason -# TODO Test me - class LatestStateSolutionGenerator(SolutionGenerator): """SolutionGenerator that only shows the latest source code and verifier diff --git a/esbmc_ai/reverse_order_solution_generator.py b/esbmc_ai/reverse_order_solution_generator.py index 6d47938..bd13d46 100644 --- a/esbmc_ai/reverse_order_solution_generator.py +++ b/esbmc_ai/reverse_order_solution_generator.py @@ -1,15 +1,9 @@ # Author: Yiannis Charalambous -from langchain.schema import BaseMessage, HumanMessage +from langchain.schema import BaseMessage from typing_extensions import override, Optional -from esbmc_ai.solution_generator import ( - SolutionGenerator, - get_source_code_formatted, - get_source_code_err_line_idx, - get_clang_err_line_index, - apply_line_patch, -) -from esbmc_ai.chat_response import FinishReason, ChatResponse +from esbmc_ai.solution_generator import SolutionGenerator +from esbmc_ai.chat_response import ChatResponse # TODO Test me diff --git a/tests/test_latest_state_solution_generator.py b/tests/test_latest_state_solution_generator.py index 6bf4631..88554e7 100644 --- a/tests/test_latest_state_solution_generator.py +++ b/tests/test_latest_state_solution_generator.py @@ -1,11 +1,13 @@ # Author: Yiannis Charalambous +from typing import Optional import pytest from langchain.schema import HumanMessage, AIMessage, SystemMessage from langchain_community.llms.fake import FakeListLLM from esbmc_ai.ai_models import AIModel +from esbmc_ai.chat_response import ChatResponse from esbmc_ai.config import AIAgentConversation, ChatPromptSettings from esbmc_ai.latest_state_solution_generator import LatestStateSolutionGenerator @@ -48,6 +50,43 @@ def test_call_update_state_first(setup_llm_model) -> None: solution_generator.generate_solution() +def test_functionality(setup_llm_model) -> None: + """Test functionality to see if the latest state along with system messages + are passed to send_message only.""" + + def mocked_send_message(message: Optional[str] = None) -> ChatResponse: + assert len(solution_generator.messages) == 1 + assert solution_generator.messages[0] == HumanMessage( + content="Initial test message" + ) + return ChatResponse() + + llm, model = setup_llm_model + + chat_settings = ChatPromptSettings( + system_messages=AIAgentConversation( + messages=( + SystemMessage(content="Test message 1"), + HumanMessage(content="Test message 2"), + AIMessage(content="Test message 3"), + ), + ), + initial_prompt="Initial test message", + temperature=1.0, + ) + + solution_generator = LatestStateSolutionGenerator( + llm=llm, + ai_model=model, + ai_model_agent=chat_settings, + ) + + solution_generator.update_state("", "") + + solution_generator.send_message = mocked_send_message + solution_generator.generate_solution() + + def test_message_stack(setup_llm_model) -> None: llm, model = setup_llm_model From f4b7a5095bebedaa5812fe71a22707e29737cba7 Mon Sep 17 00:00:00 2001 From: Yiannis Charalambous Date: Sat, 20 Apr 2024 18:05:09 +0100 Subject: [PATCH 12/13] Removed unused imports --- esbmc_ai/reverse_order_solution_generator.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/esbmc_ai/reverse_order_solution_generator.py b/esbmc_ai/reverse_order_solution_generator.py index bd13d46..6d47938 100644 --- a/esbmc_ai/reverse_order_solution_generator.py +++ b/esbmc_ai/reverse_order_solution_generator.py @@ -1,9 +1,15 @@ # Author: Yiannis Charalambous -from langchain.schema import BaseMessage +from langchain.schema import BaseMessage, HumanMessage from typing_extensions import override, Optional -from esbmc_ai.solution_generator import SolutionGenerator -from esbmc_ai.chat_response import ChatResponse +from esbmc_ai.solution_generator import ( + SolutionGenerator, + get_source_code_formatted, + get_source_code_err_line_idx, + get_clang_err_line_index, + apply_line_patch, +) +from esbmc_ai.chat_response import FinishReason, ChatResponse # TODO Test me From 6617c99b9ab357c39d262c800fd506b53df0e2b3 Mon Sep 17 00:00:00 2001 From: Yiannis Charalambous Date: Sat, 20 Apr 2024 18:05:28 +0100 Subject: [PATCH 13/13] Finished writing tests for LatestStateSolutionGenerator --- esbmc_ai/latest_state_solution_generator.py | 2 + tests/test_latest_state_solution_generator.py | 39 ------------------- 2 files changed, 2 insertions(+), 39 deletions(-) diff --git a/esbmc_ai/latest_state_solution_generator.py b/esbmc_ai/latest_state_solution_generator.py index 90e8185..81d15a5 100644 --- a/esbmc_ai/latest_state_solution_generator.py +++ b/esbmc_ai/latest_state_solution_generator.py @@ -5,6 +5,8 @@ from esbmc_ai.solution_generator import SolutionGenerator from esbmc_ai.chat_response import FinishReason +# TODO Test me + class LatestStateSolutionGenerator(SolutionGenerator): """SolutionGenerator that only shows the latest source code and verifier diff --git a/tests/test_latest_state_solution_generator.py b/tests/test_latest_state_solution_generator.py index 88554e7..6bf4631 100644 --- a/tests/test_latest_state_solution_generator.py +++ b/tests/test_latest_state_solution_generator.py @@ -1,13 +1,11 @@ # Author: Yiannis Charalambous -from typing import Optional import pytest from langchain.schema import HumanMessage, AIMessage, SystemMessage from langchain_community.llms.fake import FakeListLLM from esbmc_ai.ai_models import AIModel -from esbmc_ai.chat_response import ChatResponse from esbmc_ai.config import AIAgentConversation, ChatPromptSettings from esbmc_ai.latest_state_solution_generator import LatestStateSolutionGenerator @@ -50,43 +48,6 @@ def test_call_update_state_first(setup_llm_model) -> None: solution_generator.generate_solution() -def test_functionality(setup_llm_model) -> None: - """Test functionality to see if the latest state along with system messages - are passed to send_message only.""" - - def mocked_send_message(message: Optional[str] = None) -> ChatResponse: - assert len(solution_generator.messages) == 1 - assert solution_generator.messages[0] == HumanMessage( - content="Initial test message" - ) - return ChatResponse() - - llm, model = setup_llm_model - - chat_settings = ChatPromptSettings( - system_messages=AIAgentConversation( - messages=( - SystemMessage(content="Test message 1"), - HumanMessage(content="Test message 2"), - AIMessage(content="Test message 3"), - ), - ), - initial_prompt="Initial test message", - temperature=1.0, - ) - - solution_generator = LatestStateSolutionGenerator( - llm=llm, - ai_model=model, - ai_model_agent=chat_settings, - ) - - solution_generator.update_state("", "") - - solution_generator.send_message = mocked_send_message - solution_generator.generate_solution() - - def test_message_stack(setup_llm_model) -> None: llm, model = setup_llm_model