Streaming chat

[kfatehi]

This pull request implements the streaming chat API per the documentation seen here in this notebook:

https://github.com/openai/openai-cookbook/blob/b92d7e7b9204ecf914a91a2781dd967aa7c52be1/examples/How_to_stream_completions.ipynb

Here is example code to test with:

import requests
import json

url = "http://localhost:8000/v1/chat/completions"
headers = {"Content-Type": "application/json"}
data = {
    "model": "vicuna-13b",
    "messages": [{"role": "user", "content": "Hello!"}],
    "stream": True,
    "n": 3
}

def pretty_print_json(json_string):
    json_obj = json.loads(json_string)
    print(json.dumps(json_obj, indent=4, sort_keys=True))

response = requests.post(url, headers=headers, data=json.dumps(data), stream=True)

for chunk in response.iter_content(chunk_size=None, decode_unicode=True):
    if chunk:
        pretty_print_json(chunk.strip())
print("Done")

Here is the responses you will see if you run the above code:

{
    "choices": [
        {
            "delta": {
                "role": "assistant"
            },
            "finish_reason": null,
            "index": 0
        }
    ],
    "object": "chat.completion.chunk"
}
{
    "choices": [
        {
            "delta": "Hello",
            "finish_reason": null,
            "index": 0
        }
    ],
    "object": "chat.completion.chunk"
}
{
    "choices": [
        {
            "delta": "! It",
            "finish_reason": null,
            "index": 0
        }
    ],
    "object": "chat.completion.chunk"
}
{
    "choices": [
        {
            "delta": "'s",
            "finish_reason": null,
            "index": 0
        }
    ],
    "object": "chat.completion.chunk"
}
{
    "choices": [
        {
            "delta": " great to",
            "finish_reason": null,
            "index": 0
        }
    ],
    "object": "chat.completion.chunk"
}
{
    "choices": [
        {
            "delta": " meet you",
            "finish_reason": null,
            "index": 0
        }
    ],
    "object": "chat.completion.chunk"
}
{
    "choices": [
        {
            "delta": ". Is",
            "finish_reason": null,
            "index": 0
        }
    ],
    "object": "chat.completion.chunk"
}
{
    "choices": [
        {
            "delta": " there anything",
            "finish_reason": null,
            "index": 0
        }
    ],
    "object": "chat.completion.chunk"
}
{
    "choices": [
        {
            "delta": " I can",
            "finish_reason": null,
            "index": 0
        }
    ],
    "object": "chat.completion.chunk"
}
{
    "choices": [
        {
            "delta": " help you",
            "finish_reason": null,
            "index": 0
        }
    ],
    "object": "chat.completion.chunk"
}
{
    "choices": [
        {
            "delta": " with today",
            "finish_reason": null,
            "index": 0
        }
    ],
    "object": "chat.completion.chunk"
}
{
    "choices": [
        {
            "delta": "? I",
            "finish_reason": null,
            "index": 0
        }
    ],
    "object": "chat.completion.chunk"
}
{
    "choices": [
        {
            "delta": "'m",
            "finish_reason": null,
            "index": 0
        }
    ],
    "object": "chat.completion.chunk"
}
{
    "choices": [
        {
            "delta": " here to",
            "finish_reason": null,
            "index": 0
        }
    ],
    "object": "chat.completion.chunk"
}
{
    "choices": [
        {
            "delta": " answer any",
            "finish_reason": null,
            "index": 0
        }
    ],
    "object": "chat.completion.chunk"
}
{
    "choices": [
        {
            "delta": " questions you",
            "finish_reason": null,
            "index": 0
        }
    ],
    "object": "chat.completion.chunk"
}
{
    "choices": [
        {
            "delta": " might have",
            "finish_reason": null,
            "index": 0
        }
    ],
    "object": "chat.completion.chunk"
}
{
    "choices": [
        {
            "delta": ".",
            "finish_reason": null,
            "index": 0
        }
    ],
    "object": "chat.completion.chunk"
}
{
    "choices": [
        {
            "delta": {},
            "finish_reason": "stop",
            "index": 0
        }
    ],
    "object": "chat.completion.chunk"
}
{
    "choices": [
        {
            "delta": {
                "role": "assistant"
            },
            "finish_reason": null,
            "index": 1
        }
    ],
    "object": "chat.completion.chunk"
}
{
    "choices": [
        {
            "delta": "Hello",
            "finish_reason": null,
            "index": 1
        }
    ],
    "object": "chat.completion.chunk"
}
{
    "choices": [
        {
            "delta": "! How",
            "finish_reason": null,
            "index": 1
        }
    ],
    "object": "chat.completion.chunk"
}
{
    "choices": [
        {
            "delta": " can I",
            "finish_reason": null,
            "index": 1
        }
    ],
    "object": "chat.completion.chunk"
}
{
    "choices": [
        {
            "delta": " help you",
            "finish_reason": null,
            "index": 1
        }
    ],
    "object": "chat.completion.chunk"
}
{
    "choices": [
        {
            "delta": " today?",
            "finish_reason": null,
            "index": 1
        }
    ],
    "object": "chat.completion.chunk"
}
{
    "choices": [
        {
            "delta": " Is there",
            "finish_reason": null,
            "index": 1
        }
    ],
    "object": "chat.completion.chunk"
}
{
    "choices": [
        {
            "delta": " something you",
            "finish_reason": null,
            "index": 1
        }
    ],
    "object": "chat.completion.chunk"
}
{
    "choices": [
        {
            "delta": " want to",
            "finish_reason": null,
            "index": 1
        }
    ],
    "object": "chat.completion.chunk"
}
{
    "choices": [
        {
            "delta": " talk about",
            "finish_reason": null,
            "index": 1
        }
    ],
    "object": "chat.completion.chunk"
}
{
    "choices": [
        {
            "delta": " or learn",
            "finish_reason": null,
            "index": 1
        }
    ],
    "object": "chat.completion.chunk"
}
{
    "choices": [
        {
            "delta": " more about",
            "finish_reason": null,
            "index": 1
        }
    ],
    "object": "chat.completion.chunk"
}
{
    "choices": [
        {
            "delta": "? I",
            "finish_reason": null,
            "index": 1
        }
    ],
    "object": "chat.completion.chunk"
}
{
    "choices": [
        {
            "delta": "'m",
            "finish_reason": null,
            "index": 1
        }
    ],
    "object": "chat.completion.chunk"
}
{
    "choices": [
        {
            "delta": " here to",
            "finish_reason": null,
            "index": 1
        }
    ],
    "object": "chat.completion.chunk"
}
{
    "choices": [
        {
            "delta": " assist you",
            "finish_reason": null,
            "index": 1
        }
    ],
    "object": "chat.completion.chunk"
}
{
    "choices": [
        {
            "delta": " with any",
            "finish_reason": null,
            "index": 1
        }
    ],
    "object": "chat.completion.chunk"
}
{
    "choices": [
        {
            "delta": " questions you",
            "finish_reason": null,
            "index": 1
        }
    ],
    "object": "chat.completion.chunk"
}
{
    "choices": [
        {
            "delta": " might have",
            "finish_reason": null,
            "index": 1
        }
    ],
    "object": "chat.completion.chunk"
}
{
    "choices": [
        {
            "delta": ".",
            "finish_reason": null,
            "index": 1
        }
    ],
    "object": "chat.completion.chunk"
}
{
    "choices": [
        {
            "delta": {},
            "finish_reason": "stop",
            "index": 1
        }
    ],
    "object": "chat.completion.chunk"
}
{
    "choices": [
        {
            "delta": {
                "role": "assistant"
            },
            "finish_reason": null,
            "index": 2
        }
    ],
    "object": "chat.completion.chunk"
}
{
    "choices": [
        {
            "delta": "Hello",
            "finish_reason": null,
            "index": 2
        }
    ],
    "object": "chat.completion.chunk"
}
{
    "choices": [
        {
            "delta": "! How",
            "finish_reason": null,
            "index": 2
        }
    ],
    "object": "chat.completion.chunk"
}
{
    "choices": [
        {
            "delta": " can I",
            "finish_reason": null,
            "index": 2
        }
    ],
    "object": "chat.completion.chunk"
}
{
    "choices": [
        {
            "delta": " assist you",
            "finish_reason": null,
            "index": 2
        }
    ],
    "object": "chat.completion.chunk"
}
{
    "choices": [
        {
            "delta": " today?",
            "finish_reason": null,
            "index": 2
        }
    ],
    "object": "chat.completion.chunk"
}
{
    "choices": [
        {
            "delta": {},
            "finish_reason": "stop",
            "index": 2
        }
    ],
    "object": "chat.completion.chunk"
}
Done