langserve
langserve copied to clipboard
Published
20 hours ago •
langchain-ai
langchain-ai
Sample code in RunnableWithMessageHistory not working through langserve
trafficstars
I am using the code from the sample section of the RunnableWithMessageHistory. it works perfectly if I invoke the chain manually by calling in python code directly. It DOES NOT work when calling through the langserve API. hopefully I am not mistaken but i believe the dictionary parsing in langserve probably not able to recognize the dict properly. Attach is the error I am encountering also.
This works:
print(chain_with_history.invoke( # noqa: T201
{"ability": "math", "question": "What's its inverse"},
config={"configurable": {"session_id": "foo"}}
))
with the samples from the RunnableWithMessageHistory file
from typing import List
from langchain_openai.chat_models import ChatOpenAI
from langchain_core.chat_history import BaseChatMessageHistory
from langchain_core.messages import BaseMessage, AIMessage
from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder
from langchain_core.pydantic_v1 import BaseModel, Field
from langchain_core.runnables.history import RunnableWithMessageHistory
def example_history(settings):
class InMemoryHistory(BaseChatMessageHistory, BaseModel):
messages: List[BaseMessage] = Field(default_factory=list)
def add_messages(self, messages: List[BaseMessage]) -> None:
self.messages.extend(messages)
def clear(self) -> None:
self.messages = []
# Here we use a global variable to store the chat message history.
# This will make it easier to inspect it to see the underlying results.
store = {}
def get_by_session_id(session_id: str) -> BaseChatMessageHistory:
if session_id not in store:
store[session_id] = InMemoryHistory()
return store[session_id]
history = get_by_session_id("1")
history.add_message(AIMessage(content="hello"))
# print(store) # noqa: T201
prompt = ChatPromptTemplate.from_messages([
("system", "You're an assistant who's good at {ability}"),
MessagesPlaceholder(variable_name="history"),
("human", "{question}"),
])
model = ChatOpenAI(temperature=0, openai_api_key=settings.OPENAI_API_KEY)
chain = prompt | model
chain_with_history = RunnableWithMessageHistory(
chain,
# Uses the get_by_session_id function defined in the example
# above.
get_by_session_id,
input_messages_key="question",
history_messages_key="history",
)
return chain_with_history
However, it will fail if call through the api through langserve.
await fetch(`localhost:8000/example_history/invoke`, {
method: 'POST',
headers: { accept: 'application/json', 'Content-Type': 'application/json' },
body: JSON.stringify({
input: { ability: 'math', question: "What's its inverse?" },
config: {
configurable: {
session_id: "something_random",
}
}
}),
});
This produces the following errors:
ne 65, in __call__
await wrap_app_handling_exceptions(self.app, conn)(scope, receive, send)
File "/Users/harris/code/meow/.venv/lib/python3.10/site-packages/starlette/_exception_handler.py", line 64, in wrapped_app
raise exc
File "/Users/harris/code/meow/.venv/lib/python3.10/site-packages/starlette/_exception_handler.py", line 53, in wrapped_app
await app(scope, receive, sender)
File "/Users/harris/code/meow/.venv/lib/python3.10/site-packages/starlette/routing.py", line 756, in __call__
await self.middleware_stack(scope, receive, send)
File "/Users/harris/code/meow/.venv/lib/python3.10/site-packages/starlette/routing.py", line 776, in app
await route.handle(scope, receive, send)
File "/Users/harris/code/meow/.venv/lib/python3.10/site-packages/starlette/routing.py", line 297, in handle
await self.app(scope, receive, send)
File "/Users/harris/code/meow/.venv/lib/python3.10/site-packages/starlette/routing.py", line 77, in app
await wrap_app_handling_exceptions(app, request)(scope, receive, send)
File "/Users/harris/code/meow/.venv/lib/python3.10/site-packages/starlette/_exception_handler.py", line 64, in wrapped_app
raise exc
File "/Users/harris/code/meow/.venv/lib/python3.10/site-packages/starlette/_exception_handler.py", line 53, in wrapped_app
await app(scope, receive, sender)
File "/Users/harris/code/meow/.venv/lib/python3.10/site-packages/starlette/routing.py", line 72, in app
response = await func(request)
File "/Users/harris/code/meow/.venv/lib/python3.10/site-packages/fastapi/routing.py", line 278, in app
raw_response = await run_endpoint_function(
File "/Users/harris/code/meow/.venv/lib/python3.10/site-packages/fastapi/routing.py", line 191, in run_endpoint_function
return await dependant.call(**values)
File "/Users/harris/code/meow/.venv/lib/python3.10/site-packages/langserve/server.py", line 530, in invoke
return await api_handler.invoke(request)
File "/Users/harris/code/meow/.venv/lib/python3.10/site-packages/langserve/api_handler.py", line 833, in invoke
output = await invoke_coro
File "/Users/harris/code/meow/.venv/lib/python3.10/site-packages/langchain_core/runnables/base.py", line 4529, in ainvoke
return await self.bound.ainvoke(
File "/Users/harris/code/meow/.venv/lib/python3.10/site-packages/langchain_core/runnables/base.py", line 4529, in ainvoke
return await self.bound.ainvoke(
File "/Users/harris/code/meow/.venv/lib/python3.10/site-packages/langchain_core/runnables/base.py", line 2536, in ainvoke
input = await step.ainvoke(
File "/Users/harris/code/meow/.venv/lib/python3.10/site-packages/langchain_core/runnables/base.py", line 4529, in ainvoke
return await self.bound.ainvoke(
File "/Users/harris/code/meow/.venv/lib/python3.10/site-packages/langchain_core/runnables/base.py", line 2536, in ainvoke
input = await step.ainvoke(
File "/Users/harris/code/meow/.venv/lib/python3.10/site-packages/langchain_core/prompts/base.py", line 143, in ainvoke
return await self._acall_with_config(
File "/Users/harris/code/meow/.venv/lib/python3.10/site-packages/langchain_core/runnables/base.py", line 1677, in _acall_with_config
output = await coro
File "/Users/harris/code/meow/.venv/lib/python3.10/site-packages/langchain_core/prompts/base.py", line 117, in _aformat_prompt_with_error_handling
_inner_input = self._validate_input(inner_input)
File "/Users/harris/code/meow/.venv/lib/python3.10/site-packages/langchain_core/prompts/base.py", line 103, in _validate_input
raise KeyError(
KeyError: "Input to ChatPromptTemplate is missing variables {'ability'}. Expected: ['ability', 'history', 'question'] Received: ['question', 'history']"
one thing I notice is the input in BasePromptTemplate::invoke would get parsed properly with ability while BasePromptTemplate::ainvoke would not include it.
def invoke(
self, input: Dict, config: Optional[RunnableConfig] = None
) -> PromptValue:
config = ensure_config(config)
if self.metadata:
config["metadata"] = {**config["metadata"], **self.metadata}
if self.tags:
config["tags"] = config["tags"] + self.tags
return self._call_with_config(
self._format_prompt_with_error_handling,
input,
config,
run_type="prompt",
)
async def ainvoke(
self, input: Dict, config: Optional[RunnableConfig] = None, **kwargs: Any
) -> PromptValue:
config = ensure_config(config)
if self.metadata:
config["metadata"].update(self.metadata)
if self.tags:
config["tags"].extend(self.tags)
return await self._acall_with_config(
self._aformat_prompt_with_error_handling,
input,
config,
run_type="prompt",
)
I am able to bypass the input_schema validation so the data would flow through in this PR: https://github.com/langchain-ai/langserve/pull/631
please let me know the proper fix.
looking for the solution!!! Anyone able to manage session id based chat history(each user specific) or pipeline state(langgraph) if we deploy through langserve
