autogen
autogen copied to clipboard
microsoft
[Issue]: need some help on multimodal multi agent, function call
Describe the issue
Hi guys, I am currently using autogen multi-agent, multimodal, and function call for my project, my goal is to 1. convert code to image 2. get gpt-4 vision image feedback 3. improve the code based on image feedback.
I am currently using autogen.GroupChatManager for my task, my project has several agents:
coder: trigger the python code executor and convert python code to image bytes or extract error message (this agent uses function call so it uses gpt-4 text model)
image_critic: given image feedback (this agent uses gpt-4 vision)
current workflow:
step 1. coder triggers the function call to get image
step 2. coder sends the image to image_critic for feedback
step 3. image_critic send feedback to gpt-4 vision and request code update
continue step 1-3 until max iteration or all feedback has been addressed.
My local env: docker container with pip installing following version
autogen==0.2.15
jupyter_client==8.6.0
here is my self contained code
import autogen
import jupyter_client
import base64
from base64 import b64decode
from autogen.agentchat.contrib.multimodal_conversable_agent import MultimodalConversableAgent
from autogen.agentchat.contrib.img_utils import gpt4v_formatter
from autogen import Agent, AssistantAgent
from typing import List, Dict, Union, Optional
config_list = autogen.config_list_from_json(
"env.json",
filter_dict={
"model": [
"gpt-4-vision-preview",
# "gpt-4-1106-preview"
],
},
)
config_list_4v = autogen.config_list_from_json(
"env.json",
filter_dict={
"model": ["gpt-4-vision-preview", #"gpt-4-1106-preview"
],
},
)
config_list_gpt4 = autogen.config_list_from_json(
"env.json",
filter_dict={
"model": ["gpt-4-vision-preview", "gpt-4-1106-preview"],
},
)
# gpt4_llm_config = {"config_list": config_list_gpt4, "cache_seed": 42}
criteria_prompt = """Evaluate the image against the provided criteria. For each criterion, assign a score on a scale from 1 to 5, where 1 indicates the worst performance and 5 indicates the best.
Criteria:
Q0. How helpful is the image?
Q1. How easy is it to follow the image?
use the following format in your output:
Q 1 | Q 2 score | your reason 1
Q 2 | Q 2 score | your reason 2
...
"""
# sample python code for testing, use this to save image
py_code = """
import matplotlib.pyplot as plt
import numpy as np
# Create a figure with high dpi
plt.figure(dpi=400)
# Define the x values
x = np.linspace(0.1, 2*np.pi, 400)
x = x[x != np.pi] # Remove pi to avoid division by zero in cot(x) and csc(x)
# Calculate the functions
u = 1/np.tan(x) # g(x) = cot(x)
y = np.sin(u) # f(g(x)) = sin(cot(x))
g_prime = -1/(np.sin(x)**2) # g'(x) = -csc^2(x)
f_prime_of_g = np.cos(u) # f'(g(x)) = cos(cot(x))
dy_dx = f_prime_of_g * g_prime # The derivative
# Plotting
plt.subplot(2, 3, 1)
plt.plot(x, u, label='$g(x) = cot(x)$')
plt.title('Inner Function $g(x)$')
plt.xlabel('$x$')
plt.ylabel('$g(x)$')
plt.legend()
plt.subplot(2, 3, 2)
plt.plot(x, y, label='$f(g(x)) = sin(cot(x))$')
plt.title('Composite Function $f(g(x))$')
plt.xlabel('$x$')
plt.ylabel('$f(g(x))$')
plt.legend()
plt.subplot(2, 3, 3)
plt.plot(x, g_prime, label="$g'(x) = -csc^2(x)$")
plt.title("Derivative of Inner Function $g'(x)$")
plt.xlabel('$x$')
plt.ylabel("$g'(x)$")
plt.legend()
plt.subplot(2, 3, 4)
plt.plot(x, f_prime_of_g, label="$f'(g(x)) = cos(cot(x))$")
plt.title("Derivative of Outer Function $f'(u)$")
plt.xlabel('$x$')
plt.ylabel("$f'(g(x))$")
plt.legend()
plt.subplot(2, 3, 5)
plt.plot(x, dy_dx, label="$dy/dx = -cos(cot(x)) \\cdot csc^2(x)$")
plt.title("Derivative of Composite Function $dy/dx$")
plt.xlabel('$x$')
plt.ylabel("$dy/dx$")
plt.legend()
# Adjust layout to prevent overlap
plt.tight_layout()
# Show the plot
plt.show()
"""
# jupyter kernel to execute python code and output a tuple of image bytes and error message
def call_python(code: str):
# Create a new kernel manager
km = jupyter_client.KernelManager()
km.start_kernel()
# Create a client for the kernel
kc = km.client()
kc.start_channels()
image_data, error_message = None, None
try:
# Send code to be executed
kc.execute(code)
# Wait for and process the execution result
while True:
try:
msg = kc.get_iopub_msg(timeout=90)
if msg['header']['msg_type'] in ['execute_result', 'display_data']:
data = msg['content']['data']
# change the type
if 'image/png' in data:
# Decode the image data
image_data = b64decode(data['image/png'])
break
if msg['header']['msg_type'] == 'error':
error_content = msg['content']
error_message = f"Error: {error_content['ename']}: {error_content['evalue']}\n"
error_message += "\n".join(error_content['traceback'])
break
if msg['header']['msg_type'] == 'status':
if msg['content']['execution_state'] == 'idle':
break
except Exception as e:
error_message = f"Exception while waiting for execute result: {e}"
break
finally:
# Stop the channels and kernel
kc.stop_channels()
km.shutdown_kernel()
if error_message:
return None, error_message
return image_data, None
def code_to_image_bytes_or_error(code: str):
image_data, error = call_python(code)
if error:
return error
else:
image_bytes = base64.b64encode(image_data).decode('utf-8')
msg = f"""<img data:image/jpeg;base64,{image_bytes}>"""
return msg
# # Modified MultimodalConversableAgent to support passing image bytes
import copy
from autogen.code_utils import content_str
class ImageAgent(MultimodalConversableAgent):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
# modified this to support image bytes in str message """<img data:image/jpeg;base64,{image_bytes}>"""
@staticmethod
def _message_to_dict(message: Union[Dict, List, str]) -> Dict:
"""Convert a message to a dictionary. This implementation
handles the GPT-4V formatting for easier prompts.
The message can be a string, a dictionary, or a list of dictionaries:
- If it's a string, it will be cast into a list and placed in the 'content' field.
- If it's a list, it will be directly placed in the 'content' field.
- If it's a dictionary, it is already in message dict format. The 'content' field of this dictionary
will be processed using the gpt4v_formatter.
"""
if isinstance(message, str):
# important: changed from pil to url
return {"content": gpt4v_formatter(message, img_format="url")}
if isinstance(message, list):
return {"content": message}
if isinstance(message, dict):
assert "content" in message, "The message dict must have a `content` field"
if isinstance(message["content"], str):
message = copy.deepcopy(message)
message["content"] = gpt4v_formatter(message["content"], img_format="pil")
try:
content_str(message["content"])
except (TypeError, ValueError) as e:
print("The `content` field should be compatible with the content_str function!")
raise e
return message
raise ValueError(f"Unsupported message type: {type(message)}")
# verify image bytes work fine in ImageAgent
test_image_agent = MultimodalConversableAgent(
name="image-explainer",
max_consecutive_auto_reply=10,
llm_config={"config_list": config_list_4v, "temperature": 0, "max_tokens": 4096},
)
test_user_proxy = ImageAgent(
name="User_proxy",
system_message="A human admin.",
human_input_mode="NEVER", # Try between ALWAYS or NEVER
max_consecutive_auto_reply=0,
code_execution_config={
"use_docker": False
},
llm_config={"config_list": config_list_4v, "temperature": 0, "max_tokens": 4096},
)
image_tag = code_to_image_bytes_or_error(py_code)
test_user_proxy.initiate_chat(
test_image_agent,
message=f""" what is this image about? {image_tag}""",
)
# MultimodalConversableAgent or ImageAgent
image_critic = MultimodalConversableAgent(
name="image-explainer",
system_message=criteria_prompt,
max_consecutive_auto_reply=10,
llm_config={"config_list": config_list_4v, "temperature": 0, "max_tokens": 4096},
)
# MultimodalConversableAgent or ImageAgent or autogen.UserProxyAgent?
user_proxy = autogen.UserProxyAgent(
name="User_proxy",
system_message="A human admin.",
human_input_mode="TERMINATE",
code_execution_config={
"use_docker": False
},
llm_config={"config_list": config_list_gpt4, #config_list_4v,
"temperature": 0, "max_tokens": 4096,
"functions": [
{
"name": "code_to_image_bytes_or_error",
"description": "python code executor",
"parameters": {
"type": "object",
"properties": {
"code": {
"type": "string",
"description": "python code str",
}
},
"required": ["code"],
},
},
]
},
function_map={"code_to_image_bytes_or_error": code_to_image_bytes_or_error}
)
# MultimodalConversableAgent or ImageAgent
coder = ImageAgent(
name="Coder",
system_message="your goal is to trigger the function call for code",
llm_config={"config_list": config_list_gpt4, "temperature": 0, "max_tokens": 4096,
"functions": [
{
"name": "code_to_image_bytes_or_error",
"description": "python code executor",
"parameters": {
"type": "object",
"properties": {
"code": {
"type": "string",
"description": "python code str",
}
},
"required": ["code"],
},
},
]
},
function_map={"code_to_image_bytes_or_error": code_to_image_bytes_or_error}
)
groupchat = autogen.GroupChat(agents=[user_proxy, image_critic, coder], messages=[], max_round=12)
manager = autogen.GroupChatManager(
groupchat=groupchat,
llm_config={"config_list": config_list_gpt4, "temperature": 0, "max_tokens": 4096}
)
user_proxy.initiate_chat(
manager,
message=f""" let's improve the code by using the following flow:
step 1. trigger the function call for given code (I will execute the code and provide image to you)
step 2. send the image to critics for feedback
step 3. use the critics feedback to update the code
continue step 1-3 until all feedback has been addressed.
let's start the the initial python code
{py_code}
"""
)
changes I have made:
ImageAgent._message_to_dict has been modified to support image bytes, verified to work in test_user_proxy.initiate_chat, check out the notebook cell
Errors I got in last block:
User_proxy (to chat_manager):
let's improve the code by using the following flow:
step 1. trigger the function call for given code (I will execute the code and provide image to you)
step 2. send the image to critics for feedback
step 3. use the critics feedback to update the code
continue step 1-3 until all feedback has been addressed.
let's start the the initial python code
import matplotlib.pyplot as plt
import numpy as np
# Create a figure with high dpi
plt.figure(dpi=400)
# Define the x values
x = np.linspace(0.1, 2*np.pi, 400)
x = x[x != np.pi] # Remove pi to avoid division by zero in cot(x) and csc(x)
# Calculate the functions
u = 1/np.tan(x) # g(x) = cot(x)
y = np.sin(u) # f(g(x)) = sin(cot(x))
g_prime = -1/(np.sin(x)**2) # g'(x) = -csc^2(x)
f_prime_of_g = np.cos(u) # f'(g(x)) = cos(cot(x))
dy_dx = f_prime_of_g * g_prime # The derivative
# Plotting
plt.subplot(2, 3, 1)
plt.plot(x, u, label='$g(x) = cot(x)$')
plt.title('Inner Function $g(x)$')
plt.xlabel('$x$')
plt.ylabel('$g(x)$')
plt.legend()
plt.subplot(2, 3, 2)
plt.plot(x, y, label='$f(g(x)) = sin(cot(x))$')
plt.title('Composite Function $f(g(x))$')
plt.xlabel('$x$')
plt.ylabel('$f(g(x))$')
plt.legend()
plt.subplot(2, 3, 3)
plt.plot(x, g_prime, label="$g'(x) = -csc^2(x)$")
plt.title("Derivative of Inner Function $g'(x)$")
plt.xlabel('$x$')
plt.ylabel("$g'(x)$")
plt.legend()
plt.subplot(2, 3, 4)
plt.plot(x, f_prime_of_g, label="$f'(g(x)) = cos(cot(x))$")
plt.title("Derivative of Outer Function $f'(u)$")
plt.xlabel('$x$')
plt.ylabel("$f'(g(x))$")
plt.legend()
plt.subplot(2, 3, 5)
plt.plot(x, dy_dx, label="$dy/dx = -cos(cot(x)) \cdot csc^2(x)$")
plt.title("Derivative of Composite Function $dy/dx$")
plt.xlabel('$x$')
plt.ylabel("$dy/dx$")
plt.legend()
# Adjust layout to prevent overlap
plt.tight_layout()
# Show the plot
plt.show()
--------------------------------------------------------------------------------
>>>>>>>> USING AUTO REPLY...
User_proxy (to chat_manager):
***** Suggested function Call: code_to_image_bytes_or_error *****
Arguments:
{"code":"import matplotlib.pyplot as plt\nimport numpy as np\n\n# Create a figure with high dpi\nplt.figure(dpi=400)\n\n# Define the x values\nx = np.linspace(0.1, 2*np.pi, 400)\nx = x[x != np.pi] # Remove pi to avoid division by zero in cot(x) and csc(x)\n\n# Calculate the functions\nu = 1/np.tan(x) # g(x) = cot(x)\ny = np.sin(u) # f(g(x)) = sin(cot(x))\ng_prime = -1/(np.sin(x)**2) # g'(x) = -csc^2(x)\nf_prime_of_g = np.cos(u) # f'(g(x)) = cos(cot(x))\ndy_dx = f_prime_of_g * g_prime # The derivative\n\n# Plotting\nplt.subplot(2, 3, 1)\nplt.plot(x, u, label='$g(x) = cot(x)$')\nplt.title('Inner Function $g(x)$')\nplt.xlabel('$x$')\nplt.ylabel('$g(x)$')\nplt.legend()\n\nplt.subplot(2, 3, 2)\nplt.plot(x, y, label='$f(g(x)) = sin(cot(x))$')\nplt.title('Composite Function $f(g(x))$')\nplt.xlabel('$x$')\nplt.ylabel('$f(g(x))$')\nplt.legend()\n\nplt.subplot(2, 3, 3)\nplt.plot(x, g_prime, label=\"$g'(x) = -csc^2(x)$\")\nplt.title(\"Derivative of Inner Function $g'(x)$\")\nplt.xlabel('$x$')\nplt.ylabel(\"$g'(x)$\")\nplt.legend()\n\nplt.subplot(2, 3, 4)\nplt.plot(x, f_prime_of_g, label=\"$f'(g(x)) = cos(cot(x))$\")\nplt.title(\"Derivative of Outer Function $f'(u)$\")\nplt.xlabel('$x$')\nplt.ylabel(\"$f'(g(x))$\")\nplt.legend()\n\nplt.subplot(2, 3, 5)\nplt.plot(x, dy_dx, label=\"$dy/dx = -cos(cot(x)) \\cdot csc^2(x)$\")\nplt.title(\"Derivative of Composite Function $dy/dx$\")\nplt.xlabel('$x$')\nplt.ylabel(\"$dy/dx$\")\nplt.legend()\n\n# Adjust layout to prevent overlap\nplt.tight_layout()\n\n# Show the plot\nplt.show()\n"}
*****************************************************************
--------------------------------------------------------------------------------
---------------------------------------------------------------------------
BadRequestError Traceback (most recent call last)
Cell In[6], line 69
64 groupchat = autogen.GroupChat(agents=[user_proxy, image_critic, coder], messages=[], max_round=12)
65 manager = autogen.GroupChatManager(
66 groupchat=groupchat,
67 llm_config={"config_list": config_list_gpt4, "temperature": 0, "max_tokens": 4096}
68 )
---> 69 user_proxy.initiate_chat(
70 manager,
71 message=f""" let's improve the code by using the following flow:
72 step 1. trigger the function call for given code (I will execute the code and provide image to you)
73 step 2. send the image to critics for feedback
74 step 3. use the critics feedback to update the code
75 continue step 1-3 until all feedback has been addressed.
76
77 let's start the the initial python code
78 {py_code}
79 """
80 )
File /usr/local/lib/python3.11/site-packages/autogen/agentchat/conversable_agent.py:838, in ConversableAgent.initiate_chat(self, recipient, clear_history, silent, cache, max_turns, **context)
836 else:
837 self._prepare_chat(recipient, clear_history)
--> 838 self.send(self.generate_init_message(**context), recipient, silent=silent)
839 summary = self._summarize_chat(
840 context.get("summary_method", ConversableAgent.DEFAULT_summary_method),
841 recipient,
842 prompt=context.get("summary_prompt"),
843 cache=cache,
844 )
845 for agent in [self, recipient]:
File /usr/local/lib/python3.11/site-packages/autogen/agentchat/conversable_agent.py:530, in ConversableAgent.send(self, message, recipient, request_reply, silent)
528 valid = self._append_oai_message(message, "assistant", recipient)
529 if valid:
--> 530 recipient.receive(message, self, request_reply, silent)
531 else:
532 raise ValueError(
533 "Message can't be converted into a valid ChatCompletion message. Either content or function_call must be provided."
534 )
File /usr/local/lib/python3.11/site-packages/autogen/agentchat/conversable_agent.py:692, in ConversableAgent.receive(self, message, sender, request_reply, silent)
690 if request_reply is False or request_reply is None and self.reply_at_receive[sender] is False:
691 return
--> 692 reply = self.generate_reply(messages=self.chat_messages[sender], sender=sender)
693 if reply is not None:
694 self.send(reply, sender, silent=silent)
File /usr/local/lib/python3.11/site-packages/autogen/agentchat/conversable_agent.py:1665, in ConversableAgent.generate_reply(self, messages, sender, **kwargs)
1663 continue
1664 if self._match_trigger(reply_func_tuple["trigger"], sender):
-> 1665 final, reply = reply_func(self, messages=messages, sender=sender, config=reply_func_tuple["config"])
1666 if final:
1667 return reply
File /usr/local/lib/python3.11/site-packages/autogen/agentchat/groupchat.py:575, in GroupChatManager.run_chat(self, messages, sender, config)
572 break
573 try:
574 # select the next speaker
--> 575 speaker = groupchat.select_speaker(speaker, self)
576 # let the speaker speak
577 reply = speaker.generate_reply(sender=self)
File /usr/local/lib/python3.11/site-packages/autogen/agentchat/groupchat.py:395, in GroupChat.select_speaker(self, last_speaker, selector)
393 # auto speaker selection
394 selector.update_system_message(self.select_speaker_msg(agents))
--> 395 final, name = selector.generate_oai_reply(messages)
396 return self._finalize_speaker(last_speaker, final, name, agents)
File /usr/local/lib/python3.11/site-packages/autogen/agentchat/conversable_agent.py:1081, in ConversableAgent.generate_oai_reply(self, messages, sender, config)
1079 if messages is None:
1080 messages = self._oai_messages[sender]
-> 1081 extracted_response = self._generate_oai_reply_from_client(
1082 client, self._oai_system_message + messages, self.client_cache
1083 )
1084 return (False, None) if extracted_response is None else (True, extracted_response)
File /usr/local/lib/python3.11/site-packages/autogen/agentchat/conversable_agent.py:1100, in ConversableAgent._generate_oai_reply_from_client(self, llm_client, messages, cache)
1097 all_messages.append(message)
1099 # TODO: #1143 handle token limit exceeded error
-> 1100 response = llm_client.create(
1101 context=messages[-1].pop("context", None),
1102 messages=all_messages,
1103 cache=cache,
1104 )
1105 extracted_response = llm_client.extract_text_or_completion_object(response)[0]
1107 if extracted_response is None:
File /usr/local/lib/python3.11/site-packages/autogen/oai/client.py:624, in OpenAIWrapper.create(self, **config)
622 try:
623 request_ts = get_current_ts()
--> 624 response = client.create(params)
625 except APITimeoutError as err:
626 logger.debug(f"config {i} timed out", exc_info=True)
File /usr/local/lib/python3.11/site-packages/autogen/oai/client.py:278, in OpenAIClient.create(self, params)
276 params = params.copy()
277 params["stream"] = False
--> 278 response = completions.create(**params)
280 return response
File /usr/local/lib/python3.11/site-packages/openai/_utils/_utils.py:270, in required_args.<locals>.inner.<locals>.wrapper(*args, **kwargs)
268 msg = f"Missing required argument: {quote(missing[0])}"
269 raise TypeError(msg)
--> 270 return func(*args, **kwargs)
File /usr/local/lib/python3.11/site-packages/openai/resources/chat/completions.py:645, in Completions.create(self, messages, model, frequency_penalty, function_call, functions, logit_bias, logprobs, max_tokens, n, presence_penalty, response_format, seed, stop, stream, temperature, tool_choice, tools, top_logprobs, top_p, user, extra_headers, extra_query, extra_body, timeout)
596 @required_args(["messages", "model"], ["messages", "model", "stream"])
597 def create(
598 self,
(...)
643 timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN,
644 ) -> ChatCompletion | Stream[ChatCompletionChunk]:
--> 645 return self._post(
646 "/chat/completions",
647 body=maybe_transform(
648 {
649 "messages": messages,
650 "model": model,
651 "frequency_penalty": frequency_penalty,
652 "function_call": function_call,
653 "functions": functions,
654 "logit_bias": logit_bias,
655 "logprobs": logprobs,
656 "max_tokens": max_tokens,
657 "n": n,
658 "presence_penalty": presence_penalty,
659 "response_format": response_format,
660 "seed": seed,
661 "stop": stop,
662 "stream": stream,
663 "temperature": temperature,
664 "tool_choice": tool_choice,
665 "tools": tools,
666 "top_logprobs": top_logprobs,
667 "top_p": top_p,
668 "user": user,
669 },
670 completion_create_params.CompletionCreateParams,
671 ),
672 options=make_request_options(
673 extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout
674 ),
675 cast_to=ChatCompletion,
676 stream=stream or False,
677 stream_cls=Stream[ChatCompletionChunk],
678 )
File /usr/local/lib/python3.11/site-packages/openai/_base_client.py:1088, in SyncAPIClient.post(self, path, cast_to, body, options, files, stream, stream_cls)
1074 def post(
1075 self,
1076 path: str,
(...)
1083 stream_cls: type[_StreamT] | None = None,
1084 ) -> ResponseT | _StreamT:
1085 opts = FinalRequestOptions.construct(
1086 method="post", url=path, json_data=body, files=to_httpx_files(files), **options
1087 )
-> 1088 return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))
File /usr/local/lib/python3.11/site-packages/openai/_base_client.py:853, in SyncAPIClient.request(self, cast_to, options, remaining_retries, stream, stream_cls)
844 def request(
845 self,
846 cast_to: Type[ResponseT],
(...)
851 stream_cls: type[_StreamT] | None = None,
852 ) -> ResponseT | _StreamT:
--> 853 return self._request(
854 cast_to=cast_to,
855 options=options,
856 stream=stream,
857 stream_cls=stream_cls,
858 remaining_retries=remaining_retries,
859 )
File /usr/local/lib/python3.11/site-packages/openai/_base_client.py:930, in SyncAPIClient._request(self, cast_to, options, remaining_retries, stream, stream_cls)
927 if not err.response.is_closed:
928 err.response.read()
--> 930 raise self._make_status_error_from_response(err.response) from None
932 return self._process_response(
933 cast_to=cast_to,
934 options=options,
(...)
937 stream_cls=stream_cls,
938 )
BadRequestError: Error code: 400 - {'error': {'message': "None is not of type 'object' - 'messages.2.function_call'", 'type': 'invalid_request_error', 'param': None, 'code': None}}
I am not sure if I used autogen incorrectly or this feature is not yet supported, and need some help on my code.
A few questions during learning autogen: In multi-agent setup, I saw some examples using self.register_reply like https://github.com/microsoft/autogen/blob/f0d5808fefe3f7a248ed0123e4a30da51534db90/notebook/agentchat_dalle_and_gpt4v.ipynb while some does not like https://github.com/microsoft/autogen/blob/main/notebook/agentchat_groupchat_vis.ipynb ? how do each agent interact with other other? for example, in 3 agents A/B/C env, when agent A output X, does llm choose the next agent to respond or I need to implement the logic myself? what if I want the coder to debug if the generated code does not work? how should I do that?
Appreciate any hints/code/guidance!
Steps to reproduce
use notebook to replicate the error.
Screenshots and logs
No response
Additional Information
No response
@tomorrmato, thank you for the detailed bug report!
As you already mentioned, for instance at the end of the LMM Notebook, my implementation with three agents (Commander, Critics, and Coder) to perform the same plotting task uses a predefined dataflow rather than a non-deterministic GroupChat.
Therefore, using "self.register_reply" is preferred when you can simply define a deterministic data flow, because this can save API calls and provide better data control; otherwise, if it is too hard to manually define how the agents should interact with each other, use GroupChat and let the manager (an LLM) decide which agent to call next.
I understand that you wanted to integrate these agents into GroupChat, and I will test your implementation and get back to you soon.
@BeibinLi does the following code in the PR notebook fix my issue?
vision_capability = VisionCapability(lmm_config={"config_list": config_list_4v, "temperature": 0.5, "max_tokens": 300})
group_chat_manager = autogen.GroupChatManager(groupchat=groupchat, llm_config=gpt4_llm_config)
vision_capability.add_to_agent(group_chat_manager)
If so, I can update my autogen version and give it a try.
@tomorrmato Yes, it should resolve your issue. The PR is not merged into the main branch yet, but you can checkout to the working branch to test your code. Or, you can wait the PR to be merged (hopefully soon).
Hey @BeibinLi, thank you very much for your PR, I just gave it a try and had some errors, not sure how to fix the errors.
here is my latest script based on your PR branch
import autogen
import jupyter_client
import base64
from base64 import b64decode
from autogen.agentchat.contrib.multimodal_conversable_agent import MultimodalConversableAgent
from autogen.agentchat.contrib.capabilities.vision_capability import VisionCapability
from autogen import Agent, AssistantAgent
from typing import List, Dict, Union, Optional
config_list = autogen.config_list_from_json(
"env.json",
filter_dict={
"model": [
"gpt-4-vision-preview",
"gpt-4-1106-preview"
],
},
)
config_list_4v = autogen.config_list_from_json(
"env.json",
filter_dict={
"model": ["gpt-4-vision-preview", #"gpt-4-1106-preview"
],
},
)
config_list_gpt4 = autogen.config_list_from_json(
"env.json",
filter_dict={
"model": ["gpt-4-vision-preview", "gpt-4-1106-preview"],
},
)
gpt4_llm_config = {"config_list": config_list_gpt4, "cache_seed": 42}
criteria_prompt = """Evaluate the image against the provided criteria. For each criterion, assign a score on a scale from 1 to 5, where 1 indicates the worst performance and 5 indicates the best.
Criteria:
Q0. How helpful is the image?
Q1. How easy is it to follow the image?
use the following format in your output:
Q 1 | Q 2 score | your reason 1
Q 2 | Q 2 score | your reason 2
...
"""
# sample python code for testing, use this to save image
py_code = """
import matplotlib.pyplot as plt
import numpy as np
# Create a figure with high dpi
plt.figure(dpi=400)
# Define the x values
x = np.linspace(0.1, 2*np.pi, 400)
x = x[x != np.pi] # Remove pi to avoid division by zero in cot(x) and csc(x)
# Calculate the functions
u = 1/np.tan(x) # g(x) = cot(x)
y = np.sin(u) # f(g(x)) = sin(cot(x))
g_prime = -1/(np.sin(x)**2) # g'(x) = -csc^2(x)
f_prime_of_g = np.cos(u) # f'(g(x)) = cos(cot(x))
dy_dx = f_prime_of_g * g_prime # The derivative
# Plotting
plt.subplot(2, 3, 1)
plt.plot(x, u, label='$g(x) = cot(x)$')
plt.title('Inner Function $g(x)$')
plt.xlabel('$x$')
plt.ylabel('$g(x)$')
plt.legend()
plt.subplot(2, 3, 2)
plt.plot(x, y, label='$f(g(x)) = sin(cot(x))$')
plt.title('Composite Function $f(g(x))$')
plt.xlabel('$x$')
plt.ylabel('$f(g(x))$')
plt.legend()
plt.subplot(2, 3, 3)
plt.plot(x, g_prime, label="$g'(x) = -csc^2(x)$")
plt.title("Derivative of Inner Function $g'(x)$")
plt.xlabel('$x$')
plt.ylabel("$g'(x)$")
plt.legend()
plt.subplot(2, 3, 4)
plt.plot(x, f_prime_of_g, label="$f'(g(x)) = cos(cot(x))$")
plt.title("Derivative of Outer Function $f'(u)$")
plt.xlabel('$x$')
plt.ylabel("$f'(g(x))$")
plt.legend()
plt.subplot(2, 3, 5)
plt.plot(x, dy_dx, label="$dy/dx = -cos(cot(x)) \\cdot csc^2(x)$")
plt.title("Derivative of Composite Function $dy/dx$")
plt.xlabel('$x$')
plt.ylabel("$dy/dx$")
plt.legend()
# Adjust layout to prevent overlap
plt.tight_layout()
# Show the plot
plt.show()
"""
# jupyter kernel to execute python code and output a tuple of image bytes and error message
def call_python(code: str):
# Create a new kernel manager
km = jupyter_client.KernelManager()
km.start_kernel()
# Create a client for the kernel
kc = km.client()
kc.start_channels()
image_data, error_message = None, None
try:
# Send code to be executed
kc.execute(code)
# Wait for and process the execution result
while True:
try:
msg = kc.get_iopub_msg(timeout=90)
if msg['header']['msg_type'] in ['execute_result', 'display_data']:
data = msg['content']['data']
# change the type
if 'image/png' in data:
# Decode the image data
image_data = b64decode(data['image/png'])
break
if msg['header']['msg_type'] == 'error':
error_content = msg['content']
error_message = f"Error: {error_content['ename']}: {error_content['evalue']}\n"
error_message += "\n".join(error_content['traceback'])
break
if msg['header']['msg_type'] == 'status':
if msg['content']['execution_state'] == 'idle':
break
except Exception as e:
error_message = f"Exception while waiting for execute result: {e}"
break
finally:
# Stop the channels and kernel
kc.stop_channels()
km.shutdown_kernel()
if error_message:
return None, error_message
return image_data, None
def code_to_image_bytes_or_error(code: str):
image_data, error = call_python(code)
if error:
return error
else:
image_bytes = base64.b64encode(image_data).decode('utf-8')
msg = f"""<img data:image/jpeg;base64,{image_bytes}>"""
return msg
image_critic = MultimodalConversableAgent(
name="image-explainer",
system_message=criteria_prompt,
max_consecutive_auto_reply=10,
llm_config={"config_list": config_list_4v, "temperature": 0, "max_tokens": 4096},
)
user_proxy = autogen.UserProxyAgent(
name="User_proxy",
system_message="A human admin.",
human_input_mode="TERMINATE",
code_execution_config={
"use_docker": False
},
llm_config={"config_list": config_list_gpt4, #config_list_4v,
"temperature": 0, "max_tokens": 4096,
# "functions": [
# {
# "name": "code_to_image_bytes_or_error",
# "description": "python code executor",
# "parameters": {
# "type": "object",
# "properties": {
# "code": {
# "type": "string",
# "description": "python code str",
# }
# },
# "required": ["code"],
# },
# },
# ]
},
# function_map={"code_to_image_bytes_or_error": code_to_image_bytes_or_error}
)
coder = MultimodalConversableAgent(
name="Coder",
system_message="your goal is to trigger the function call for code",
llm_config={"config_list": config_list_gpt4, "temperature": 0, "max_tokens": 4096,
"functions": [
{
"name": "code_to_image_bytes_or_error",
"description": "python code executor",
"parameters": {
"type": "object",
"properties": {
"code": {
"type": "string",
"description": "python code str",
}
},
"required": ["code"],
},
},
]
},
function_map={"code_to_image_bytes_or_error": code_to_image_bytes_or_error}
)
groupchat = autogen.GroupChat(agents=[user_proxy, image_critic, coder], messages=[], max_round=12)
vision_capability = VisionCapability(lmm_config={"config_list": config_list_4v, "temperature": 0.5, "max_tokens": 300})
group_chat_manager = autogen.GroupChatManager(groupchat=groupchat, llm_config=gpt4_llm_config)
vision_capability.add_to_agent(group_chat_manager)
user_proxy.initiate_chat(
group_chat_manager,
message=f""" let's improve the code by using the following flow:
step 1. trigger the function call for given code (I will execute the code and provide image to you)
step 2. send the image to critics for feedback
step 3. use the critics feedback to update the code
continue step 1-3 until all feedback has been addressed.
let's start with the the initial python code with no function call in `user_proxy` agent,
{py_code}
"""
)
It does not trigger function call and User_proxy agent seems to get stuck
User_proxy (to chat_manager):
let's improve the code by using the following flow:
step 1. trigger the function call for given code (I will execute the code and provide image to you)
step 2. send the image to critics for feedback
step 3. use the critics feedback to update the code
continue step 1-3 until all feedback has been addressed.
let's start with the the initial python code
import matplotlib.pyplot as plt
import numpy as np
# Create a figure with high dpi
plt.figure(dpi=400)
# Define the x values
x = np.linspace(0.1, 2*np.pi, 400)
x = x[x != np.pi] # Remove pi to avoid division by zero in cot(x) and csc(x)
# Calculate the functions
u = 1/np.tan(x) # g(x) = cot(x)
y = np.sin(u) # f(g(x)) = sin(cot(x))
g_prime = -1/(np.sin(x)**2) # g'(x) = -csc^2(x)
f_prime_of_g = np.cos(u) # f'(g(x)) = cos(cot(x))
dy_dx = f_prime_of_g * g_prime # The derivative
# Plotting
plt.subplot(2, 3, 1)
plt.plot(x, u, label='$g(x) = cot(x)$')
plt.title('Inner Function $g(x)$')
plt.xlabel('$x$')
plt.ylabel('$g(x)$')
plt.legend()
plt.subplot(2, 3, 2)
plt.plot(x, y, label='$f(g(x)) = sin(cot(x))$')
plt.title('Composite Function $f(g(x))$')
plt.xlabel('$x$')
plt.ylabel('$f(g(x))$')
plt.legend()
plt.subplot(2, 3, 3)
plt.plot(x, g_prime, label="$g'(x) = -csc^2(x)$")
plt.title("Derivative of Inner Function $g'(x)$")
plt.xlabel('$x$')
plt.ylabel("$g'(x)$")
plt.legend()
plt.subplot(2, 3, 4)
plt.plot(x, f_prime_of_g, label="$f'(g(x)) = cos(cot(x))$")
plt.title("Derivative of Outer Function $f'(u)$")
plt.xlabel('$x$')
plt.ylabel("$f'(g(x))$")
plt.legend()
plt.subplot(2, 3, 5)
plt.plot(x, dy_dx, label="$dy/dx = -cos(cot(x)) \cdot csc^2(x)$")
plt.title("Derivative of Composite Function $dy/dx$")
plt.xlabel('$x$')
plt.ylabel("$dy/dx$")
plt.legend()
# Adjust layout to prevent overlap
plt.tight_layout()
# Show the plot
plt.show()
--------------------------------------------------------------------------------
>>>>>>>> USING AUTO REPLY...
User_proxy (to chat_manager):
As an AI language model, I'm unable to execute code or provide images. However, I can guide you through the process of improving the code based on hypothetical feedback from critics.
Let's assume the critics have provided the following feedback:
1. The plots are too small and difficult to read.
2. The colors of the plots are not distinct enough for people with color vision deficiency.
3. The legend overlaps with the plot in some cases.
4. The x-axis should include the value of pi for better understanding of the function behavior around that point.
5. The title of the fourth subplot is incorrect; it should be "Derivative of Outer Function $f'(g(x))$".
Based on this feedback, here are the improvements we can make:
1. Increase the size of the figure to make the plots larger and more readable.
2. Use a color palette that is friendly for color vision deficiency.
3. Move the legend to a location where it does not overlap with the plot.
4. Include the value of pi in the x-axis.
5. Correct the title of the fourth subplot.
Here's the updated code:
python
import matplotlib.pyplot as plt
import numpy as np
# Create a larger figure with high dpi
plt.figure(figsize=(12, 8), dpi=400)
# Define the x values
x = np.linspace(0, 2*np.pi, 400)
x = x[x != np.pi] # Remove pi to avoid division by zero in cot(x) and csc(x)
# Calculate the functions
u = 1/np.tan(x) # g(x) = cot(x)
y = np.sin(u) # f(g(x)) = sin(cot(x))
g_prime = -1/(np.sin(x)**2) # g'(x) = -csc^2(x)
f_prime_of_g = np.cos(u) # f'(g(x)) = cos(cot(x))
dy_dx = f_prime_of_g * g_prime # The derivative
# Define a colorblind-friendly color palette
colors = ['tab:blue', 'tab:orange', 'tab:green', 'tab:red', 'tab:purple']
# Plotting
plt.subplot(2, 3, 1)
plt.plot(x, u, label='$g(x) = cot(x)$', color=colors[0])
plt.title('Inner Function $g(x)$')
plt.xlabel('$x$')
plt.ylabel('$g(x)$')
plt.legend(loc='upper right')
plt.subplot(2, 3, 2)
plt.plot(x, y, label='$f(g(x)) = sin(cot(x))$', color=colors[1])
plt.title('Composite Function $f(g(x))$')
plt.xlabel('$x$')
plt.ylabel('$f(g(x))$')
plt.legend(loc='upper right')
plt.subplot(2, 3, 3)
plt.plot(x, g_prime, label="$g'(x) = -csc^2(x)$", color=colors[2])
plt.title("Derivative of Inner Function $g'(x)$")
plt.xlabel('$x$')
plt.ylabel("$g'(x)$")
plt.legend(loc='upper right')
plt.subplot(2, 3, 4)
plt.plot(x, f_prime_of_g, label="$f'(g(x)) = cos(cot(x))$", color=colors[3])
plt.title("Derivative of Outer Function $f'(g(x))$")
plt.xlabel('$x$')
plt.ylabel("$f'(g(x))$")
plt.legend(loc='upper right')
plt.subplot(2, 3, 5)
plt.plot(x, dy_dx, label="$dy/dx = -cos(cot(x)) \cdot csc^2(x)$", color=colors[4])
plt.title("Derivative of Composite Function $dy/dx$")
plt.xlabel('$x$')
plt.ylabel("$dy/dx$")
plt.legend(loc='upper right')
# Adjust layout to prevent overlap
plt.tight_layout()
# Show the plot
plt.show()
This updated code addresses the feedback provided by the critics. If you have the ability to execute the code and generate the image, you can then send the updated image to the critics for further feedback. Repeat the process until all feedback has been addressed satisfactorily.
--------------------------------------------------------------------------------
>>>>>>>> USING AUTO REPLY...
User_proxy (to chat_manager):
As an AI language model, I'm unable to execute code or provide images. However, I can guide you through the process of improving the code based on hypothetical feedback from critics.
Let's assume the critics have provided the following feedback:
1. The plots are too small and difficult to read.
2. The colors of the plots are not distinct enough for people with color vision deficiency.
3. The legend overlaps with the plot in some cases.
4. The x-axis should include the value of pi for better understanding of the function behavior around that point.
5. The title of the fourth subplot is incorrect; it should be "Derivative of Outer Function $f'(g(x))$".
Based on this feedback, here are the improvements we can make:
1. Increase the size of the figure to make the plots larger and more readable.
2. Use a color palette that is friendly for color vision deficiency.
3. Move the legend to a location where it does not overlap with the plot.
4. Include the value of pi in the x-axis.
5. Correct the title of the fourth subplot.
Here's the updated code:
python
import matplotlib.pyplot as plt
import numpy as np
# Create a larger figure with high dpi
plt.figure(figsize=(12, 8), dpi=400)
# Define the x values
x = np.linspace(0, 2*np.pi, 400)
x = x[x != np.pi] # Remove pi to avoid division by zero in cot(x) and csc(x)
# Calculate the functions
u = 1/np.tan(x) # g(x) = cot(x)
y = np.sin(u) # f(g(x)) = sin(cot(x))
g_prime = -1/(np.sin(x)**2) # g'(x) = -csc^2(x)
f_prime_of_g = np.cos(u) # f'(g(x)) = cos(cot(x))
dy_dx = f_prime_of_g * g_prime # The derivative
# Define a colorblind-friendly color palette
colors = ['tab:blue', 'tab:orange', 'tab:green', 'tab:red', 'tab:purple']
# Plotting
plt.subplot(2, 3, 1)
plt.plot(x, u, label='$g(x) = cot(x)$', color=colors[0])
plt.title('Inner Function $g(x)$')
plt.xlabel('$x$')
plt.ylabel('$g(x)$')
plt.legend(loc='upper right')
plt.subplot(2, 3, 2)
plt.plot(x, y, label='$f(g(x)) = sin(cot(x))$', color=colors[1])
plt.title('Composite Function $f(g(x))$')
plt.xlabel('$x$')
plt.ylabel('$f(g(x))$')
plt.legend(loc='upper right')
plt.subplot(2, 3, 3)
plt.plot(x, g_prime, label="$g'(x) = -csc^2(x)$", color=colors[2])
plt.title("Derivative of Inner Function $g'(x)$")
plt.xlabel('$x$')
plt.ylabel("$g'(x)$")
plt.legend(loc='upper right')
plt.subplot(2, 3, 4)
plt.plot(x, f_prime_of_g, label="$f'(g(x)) = cos(cot(x))$", color=colors[3])
plt.title("Derivative of Outer Function $f'(g(x))$")
plt.xlabel('$x$')
plt.ylabel("$f'(g(x))$")
plt.legend(loc='upper right')
plt.subplot(2, 3, 5)
plt.plot(x, dy_dx, label="$dy/dx = -cos(cot(x)) \cdot csc^2(x)$", color=colors[4])
plt.title("Derivative of Composite Function $dy/dx$")
plt.xlabel('$x$')
plt.ylabel("$dy/dx$")
plt.legend(loc='upper right')
# Adjust layout to prevent overlap
plt.tight_layout()
# Show the plot
plt.show()
This updated code addresses the feedback provided by the critics. If you have the ability to execute the code and generate the image, you can then send the updated image to the critics for further feedback. Repeat the process until all feedback has been addressed satisfactorily.
--------------------------------------------------------------------------------
>>>>>>>> USING AUTO REPLY...
User_proxy (to chat_manager):
I apologize for the confusion, but as an AI developed by OpenAI, I don't have the capability to execute code or generate images. However, I can help you refine the code based on the hypothetical feedback you might receive from critics. If you have specific feedback or areas of improvement you'd like to address, please let me know, and I can assist you in updating the code accordingly.
--------------------------------------------------------------------------------
>>>>>>>> USING AUTO REPLY...
User_proxy (to chat_manager):
I apologize for the confusion, but as an AI developed by OpenAI, I don't have the capability to execute code or generate images. However, I can help you refine the code based on the hypothetical feedback you might receive from critics. If you have specific feedback or areas of improvement you'd like to address, please let me know, and I can assist you in updating the code accordingly.
...
I also tried to uncomment out the function call parameters in user_proxy agent, and it raised the following error
User_proxy (to chat_manager):
let's improve the code by using the following flow:
step 1. trigger the function call for given code (I will execute the code and provide image to you)
step 2. send the image to critics for feedback
step 3. use the critics feedback to update the code
continue step 1-3 until all feedback has been addressed.
let's start with the the initial python code
import matplotlib.pyplot as plt
import numpy as np
# Create a figure with high dpi
plt.figure(dpi=400)
# Define the x values
x = np.linspace(0.1, 2*np.pi, 400)
x = x[x != np.pi] # Remove pi to avoid division by zero in cot(x) and csc(x)
# Calculate the functions
u = 1/np.tan(x) # g(x) = cot(x)
y = np.sin(u) # f(g(x)) = sin(cot(x))
g_prime = -1/(np.sin(x)**2) # g'(x) = -csc^2(x)
f_prime_of_g = np.cos(u) # f'(g(x)) = cos(cot(x))
dy_dx = f_prime_of_g * g_prime # The derivative
# Plotting
plt.subplot(2, 3, 1)
plt.plot(x, u, label='$g(x) = cot(x)$')
plt.title('Inner Function $g(x)$')
plt.xlabel('$x$')
plt.ylabel('$g(x)$')
plt.legend()
plt.subplot(2, 3, 2)
plt.plot(x, y, label='$f(g(x)) = sin(cot(x))$')
plt.title('Composite Function $f(g(x))$')
plt.xlabel('$x$')
plt.ylabel('$f(g(x))$')
plt.legend()
plt.subplot(2, 3, 3)
plt.plot(x, g_prime, label="$g'(x) = -csc^2(x)$")
plt.title("Derivative of Inner Function $g'(x)$")
plt.xlabel('$x$')
plt.ylabel("$g'(x)$")
plt.legend()
plt.subplot(2, 3, 4)
plt.plot(x, f_prime_of_g, label="$f'(g(x)) = cos(cot(x))$")
plt.title("Derivative of Outer Function $f'(u)$")
plt.xlabel('$x$')
plt.ylabel("$f'(g(x))$")
plt.legend()
plt.subplot(2, 3, 5)
plt.plot(x, dy_dx, label="$dy/dx = -cos(cot(x)) \cdot csc^2(x)$")
plt.title("Derivative of Composite Function $dy/dx$")
plt.xlabel('$x$')
plt.ylabel("$dy/dx$")
plt.legend()
# Adjust layout to prevent overlap
plt.tight_layout()
# Show the plot
plt.show()
--------------------------------------------------------------------------------
>>>>>>>> USING AUTO REPLY...
User_proxy (to chat_manager):
***** Suggested function Call: code_to_image_bytes_or_error *****
Arguments:
{"code":"import matplotlib.pyplot as plt\nimport numpy as np\n\n# Create a figure with high dpi\nplt.figure(dpi=400)\n\n# Define the x values\nx = np.linspace(0.1, 2*np.pi, 400)\nx = x[x != np.pi] # Remove pi to avoid division by zero in cot(x) and csc(x)\n\n# Calculate the functions\nu = 1/np.tan(x) # g(x) = cot(x)\ny = np.sin(u) # f(g(x)) = sin(cot(x))\ng_prime = -1/(np.sin(x)**2) # g'(x) = -csc^2(x)\nf_prime_of_g = np.cos(u) # f'(g(x)) = cos(cot(x))\ndy_dx = f_prime_of_g * g_prime # The derivative\n\n# Plotting\nplt.subplot(2, 3, 1)\nplt.plot(x, u, label='$g(x) = cot(x)$')\nplt.title('Inner Function $g(x)$')\nplt.xlabel('$x$')\nplt.ylabel('$g(x)$')\nplt.legend()\n\nplt.subplot(2, 3, 2)\nplt.plot(x, y, label='$f(g(x)) = sin(cot(x))$')\nplt.title('Composite Function $f(g(x))$')\nplt.xlabel('$x$')\nplt.ylabel('$f(g(x))$')\nplt.legend()\n\nplt.subplot(2, 3, 3)\nplt.plot(x, g_prime, label=\"$g'(x) = -csc^2(x)$\")\nplt.title(\"Derivative of Inner Function $g'(x)$\")\nplt.xlabel('$x$')\nplt.ylabel(\"$g'(x)$\")\nplt.legend()\n\nplt.subplot(2, 3, 4)\nplt.plot(x, f_prime_of_g, label=\"$f'(g(x)) = cos(cot(x))$\")\nplt.title(\"Derivative of Outer Function $f'(u)$\")\nplt.xlabel('$x$')\nplt.ylabel(\"$f'(g(x))$\")\nplt.legend()\n\nplt.subplot(2, 3, 5)\nplt.plot(x, dy_dx, label=\"$dy/dx = -cos(cot(x)) \\cdot csc^2(x)$\")\nplt.title(\"Derivative of Composite Function $dy/dx$\")\nplt.xlabel('$x$')\nplt.ylabel(\"$dy/dx$\")\nplt.legend()\n\n# Adjust layout to prevent overlap\nplt.tight_layout()\n\n# Show the plot\nplt.show()\n"}
*****************************************************************
--------------------------------------------------------------------------------
Traceback (most recent call last):
File "/app/temp/autogen/notebook/test_multimodal_ex.py", line 241, in <module>
user_proxy.initiate_chat(
File "/usr/local/lib/python3.11/site-packages/autogen/agentchat/conversable_agent.py", line 977, in initiate_chat
self.send(msg2send, recipient, silent=silent)
File "/usr/local/lib/python3.11/site-packages/autogen/agentchat/conversable_agent.py", line 624, in send
recipient.receive(message, self, request_reply, silent)
File "/usr/local/lib/python3.11/site-packages/autogen/agentchat/conversable_agent.py", line 783, in receive
reply = self.generate_reply(messages=self.chat_messages[sender], sender=sender)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/autogen/agentchat/conversable_agent.py", line 1866, in generate_reply
final, reply = reply_func(self, messages=messages, sender=sender, config=reply_func_tuple["config"])
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/autogen/agentchat/groupchat.py", line 614, in run_chat
speaker = groupchat.select_speaker(speaker, self)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/autogen/agentchat/groupchat.py", line 430, in select_speaker
final, name = selector.generate_oai_reply(messages)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/autogen/agentchat/conversable_agent.py", line 1265, in generate_oai_reply
extracted_response = self._generate_oai_reply_from_client(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/autogen/agentchat/conversable_agent.py", line 1284, in _generate_oai_reply_from_client
response = llm_client.create(
^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/autogen/oai/client.py", line 625, in create
response = client.create(params)
^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/autogen/oai/client.py", line 278, in create
response = completions.create(**params)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/openai/_utils/_utils.py", line 270, in wrapper
return func(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/openai/resources/chat/completions.py", line 645, in create
return self._post(
^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/openai/_base_client.py", line 1088, in post
return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/openai/_base_client.py", line 853, in request
return self._request(
^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/site-packages/openai/_base_client.py", line 930, in _request
raise self._make_status_error_from_response(err.response) from None
openai.BadRequestError: Error code: 400 - {'error': {'message': "None is not of type 'object' - 'messages.2.function_call'", 'type': 'invalid_request_error', 'param': None, 'code': None}}
based on this post, it seems to be that function call array is not supported, do you know how to fix this issue? Once again, thank you very much for your help!
@tomorrmato
- You should NOT provide a "llm_config" into user_proxy; otherwise, it will not run code for you.
- You need to wrap your code in markdown if you want the UserProxyAgent to run code.
So, the last line of your code should be
user_proxy.initiate_chat(
group_chat_manager,
message=f""" let's improve the code by using the following flow:
step 1. trigger the function call for given code (I will execute the code and provide image to you)
step 2. send the image to critics for feedback
step 3. use the critics feedback to update the code
continue step 1-3 until all feedback has been addressed.
let's start with the the initial python code with no function call in `user_proxy` agent,
```python
{py_code}
""" )
Because you have a manually-defined workflow, you don't need a groupchat manager.
You can either use sequential chat or the FigureCreator directly from here.
I can create an example later with plotting improvements.
@tomorrmato
- You should NOT provide a "llm_config" into user_proxy; otherwise, it will not run code for you.
- You need to wrap your code in markdown if you want the UserProxyAgent to run code.
So, the last line of your code should be
user_proxy.initiate_chat( group_chat_manager, message=f""" let's improve the code by using the following flow: step 1. trigger the function call for given code (I will execute the code and provide image to you) step 2. send the image to critics for feedback step 3. use the critics feedback to update the code continue step 1-3 until all feedback has been addressed. let's start with the the initial python code with no function call in `user_proxy` agent, ```python {py_code}""" )
This is not True: "You should NOT provide a "llm_config" into user_proxy; otherwise, it will not run code for you." code execution based reply proceeds llm-based reply.
@sonichi @BeibinLi I tried to
- comment out
llm_configin user_proxy and - update markdown format by using the following
```python
{py_code}
```
here is the terminal output
```
User_proxy (to chat_manager):
let's improve the code by using the following flow:
step 1. trigger the function call for given code (I will execute the code and provide image to you)
step 2. send the image to critics for feedback
step 3. use the critics feedback to update the code
continue step 1-3 until all feedback has been addressed.
let's start with the the initial python code
```python
import matplotlib.pyplot as plt
import numpy as np
# Create a figure with high dpi
plt.figure(dpi=400)
# Define the x values
x = np.linspace(0.1, 2*np.pi, 400)
x = x[x != np.pi] # Remove pi to avoid division by zero in cot(x) and csc(x)
# Calculate the functions
u = 1/np.tan(x) # g(x) = cot(x)
y = np.sin(u) # f(g(x)) = sin(cot(x))
g_prime = -1/(np.sin(x)**2) # g'(x) = -csc^2(x)
f_prime_of_g = np.cos(u) # f'(g(x)) = cos(cot(x))
dy_dx = f_prime_of_g * g_prime # The derivative
# Plotting
plt.subplot(2, 3, 1)
plt.plot(x, u, label='$g(x) = cot(x)$')
plt.title('Inner Function $g(x)$')
plt.xlabel('$x$')
plt.ylabel('$g(x)$')
plt.legend()
plt.subplot(2, 3, 2)
plt.plot(x, y, label='$f(g(x)) = sin(cot(x))$')
plt.title('Composite Function $f(g(x))$')
plt.xlabel('$x$')
plt.ylabel('$f(g(x))$')
plt.legend()
plt.subplot(2, 3, 3)
plt.plot(x, g_prime, label="$g'(x) = -csc^2(x)$")
plt.title("Derivative of Inner Function $g'(x)$")
plt.xlabel('$x$')
plt.ylabel("$g'(x)$")
plt.legend()
plt.subplot(2, 3, 4)
plt.plot(x, f_prime_of_g, label="$f'(g(x)) = cos(cot(x))$")
plt.title("Derivative of Outer Function $f'(u)$")
plt.xlabel('$x$')
plt.ylabel("$f'(g(x))$")
plt.legend()
plt.subplot(2, 3, 5)
plt.plot(x, dy_dx, label="$dy/dx = -cos(cot(x)) \cdot csc^2(x)$")
plt.title("Derivative of Composite Function $dy/dx$")
plt.xlabel('$x$')
plt.ylabel("$dy/dx$")
plt.legend()
# Adjust layout to prevent overlap
plt.tight_layout()
# Show the plot
plt.show()
```
--------------------------------------------------------------------------------
>>>>>>>> USING AUTO REPLY...
User_proxy (to chat_manager):
--------------------------------------------------------------------------------
>>>>>>>> USING AUTO REPLY...
User_proxy (to chat_manager):
--------------------------------------------------------------------------------
>>>>>>>> USING AUTO REPLY...
User_proxy (to chat_manager):
--------------------------------------------------------------------------------
>>>>>>>> USING AUTO REPLY...
User_proxy (to chat_manager):
--------------------------------------------------------------------------------
>>>>>>>> USING AUTO REPLY...
User_proxy (to chat_manager):
--------------------------------------------------------------------------------
>>>>>>>> USING AUTO REPLY...
User_proxy (to chat_manager):
--------------------------------------------------------------------------------
>>>>>>>> USING AUTO REPLY...
User_proxy (to chat_manager):
--------------------------------------------------------------------------------
>>>>>>>> USING AUTO REPLY...
User_proxy (to chat_manager):
--------------------------------------------------------------------------------
>>>>>>>> USING AUTO REPLY...
User_proxy (to chat_manager):
--------------------------------------------------------------------------------
>>>>>>>> USING AUTO REPLY...
User_proxy (to chat_manager):
--------------------------------------------------------------------------------
>>>>>>>> USING AUTO REPLY...
User_proxy (to chat_manager):
```
it does not seem to do anything. I also tried to comment out `function_map` of `user_proxy`, the terminal output is similar, I am not sure if there is anything I may miss, please let me know, thank you!
