OpenAdapt
OpenAdapt copied to clipboard
Published
20 hours ago •
OpenAdaptAI
OpenAdaptAI
[Bug]: handle OpenAI API image errors
Describe the bug
python -m openadapt.replay VisualReplayStrategy --instructions="enter the word 'goodbye' in cell D5"
...
File "/Users/abrichr/oa/OpenAdapt/openadapt/replay.py", line 84, in replay
strategy = strategy_class(recording, **kwargs)
| | -> {'instructions': "enter the word 'goodbye' in cell D5"}
| -> Recording(id=79, timestamp=1720741542.6430058, monitor_width=1512, monitor_height=982, double_click_interval_seconds=0.5, dou...
-> <class 'openadapt.strategies.visual.VisualReplayStrategy'>
File "/Users/abrichr/oa/OpenAdapt/openadapt/strategies/visual.py", line 182, in __init__
add_active_segment_descriptions(recording.processed_action_events)
| | -> <property object at 0x12fb0f880>
| -> Recording(id=79, timestamp=1720741542.6430058, monitor_width=1512, monitor_height=982, double_click_interval_seconds=0.5, dou...
-> <function add_active_segment_descriptions at 0x2a904e710>
File "/Users/abrichr/oa/OpenAdapt/openadapt/strategies/visual.py", line 109, in add_active_segment_descriptions
window_segmentation = get_window_segmentation(action)
| -> ActionEvent(name='move', timestamp=1720741549.2011595, recording_timestamp=1720741542.6430058, screenshot_timestamp=172074155...
-> <function get_window_segmentation at 0x2cecd5d80>
File "/Users/abrichr/oa/OpenAdapt/openadapt/strategies/visual.py", line 431, in get_window_segmentation
descriptions = prompt_for_descriptions(
-> <function prompt_for_descriptions at 0x2cecd5e10>
File "/Users/abrichr/oa/OpenAdapt/openadapt/strategies/visual.py", line 512, in prompt_for_descriptions
descriptions_json = driver.prompt(
| -> <function prompt at 0x16cb51480>
-> <module 'openadapt.drivers.openai' from '/Users/abrichr/oa/OpenAdapt/openadapt/drivers/openai.py'>
File "/Users/abrichr/oa/OpenAdapt/openadapt/drivers/openai.py", line 209, in prompt
result = get_completion(payload)
| -> {'model': 'gpt-4o', 'messages': [{'role': 'system', 'content': [{'type': 'text', 'text': "You are the cognitive engine poweri...
-> <function get_completion at 0x16cb51240>
File "/Users/abrichr/oa/OpenAdapt/openadapt/drivers/openai.py", line 155, in get_completion
raise exc
File "/Users/abrichr/oa/OpenAdapt/openadapt/drivers/openai.py", line 145, in get_completion
result = get_response(payload)
| -> {'model': 'gpt-4o', 'messages': [{'role': 'system', 'content': [{'type': 'text', 'text': "You are the cognitive engine poweri...
-> <function get_response at 0x16cb51360>
File "/Users/abrichr/oa/OpenAdapt/openadapt/cache.py", line 85, in wrapper
rval = fn(*args, **kwargs)
| | -> {}
| -> ({'model': 'gpt-4o', 'messages': [{'role': 'system', 'content': [{'type': 'text', 'text': "You are the cognitive engine power...
-> MemorizedFunc(func=<function get_response at 0x16cb512d0>, location=.cache/joblib)
File "/Users/abrichr/Library/Caches/pypoetry/virtualenvs/openadapt-VBXg4jpm-py3.10/lib/python3.10/site-packages/joblib/memory.py", line 573, in __call__
return self._cached_call(args, kwargs, shelving=False)
| | | -> {}
| | -> ({'model': 'gpt-4o', 'messages': [{'role': 'system', 'content': [{'type': 'text', 'text': "You are the cognitive engine power...
| -> <function MemorizedFunc._cached_call at 0x168a2a440>
-> MemorizedFunc(func=<function get_response at 0x16cb512d0>, location=.cache/joblib)
File "/Users/abrichr/Library/Caches/pypoetry/virtualenvs/openadapt-VBXg4jpm-py3.10/lib/python3.10/site-packages/joblib/memory.py", line 530, in _cached_call
return self._call(call_id, args, kwargs, shelving)
| | | | | -> False
| | | | -> {}
| | | -> ({'model': 'gpt-4o', 'messages': [{'role': 'system', 'content': [{'type': 'text', 'text': "You are the cognitive engine power...
| | -> ('openadapt/drivers/openai/get_response', '59a23667cfdc4ce3d6fff1877495bcc5')
| -> <function MemorizedFunc._call at 0x168a2ab00>
-> MemorizedFunc(func=<function get_response at 0x16cb512d0>, location=.cache/joblib)
File "/Users/abrichr/Library/Caches/pypoetry/virtualenvs/openadapt-VBXg4jpm-py3.10/lib/python3.10/site-packages/joblib/memory.py", line 762, in _call
output = self.func(*args, **kwargs)
| | | -> {}
| | -> ({'model': 'gpt-4o', 'messages': [{'role': 'system', 'content': [{'type': 'text', 'text': "You are the cognitive engine power...
| -> <function get_response at 0x16cb512d0>
-> MemorizedFunc(func=<function get_response at 0x16cb512d0>, location=.cache/joblib)
File "/Users/abrichr/oa/OpenAdapt/openadapt/drivers/openai.py", line 130, in get_response
raise Exception(message)
-> "You uploaded an unsupported image. Please make sure your image is below 20 MB in size and is of one the following formats: [...
Exception: You uploaded an unsupported image. Please make sure your image is below 20 MB in size and is of one the following formats: ['png', 'jpeg', 'gif', 'webp'].
To Reproduce
Create a recording involving a spreadsheet, then run python -m openadapt.replay VisualReplayStrategy --instructions="enter the word 'goodbye' in cell D5"
