openai-go icon indicating copy to clipboard operation
openai-go copied to clipboard

Support for streaming tool call for Ollama

Open gluonfield opened this issue 6 months ago • 1 comments

Currently streaming tool calls for Ollama doesn't work. (While it works for other libraries). Since Ollama supports works on all other OpenAI compatible SDKs (including main python and JS SDKs, it would be good to be able to handle it on go side too.

Example code

package main

import (
	"context"

	"github.com/openai/openai-go"
	"github.com/openai/openai-go/option"
)

func main() {
	model := "qwen3:8b"
	client := openai.NewClient(option.WithBaseURL("http://localhost:11434/v1"))
	ctx := context.Background()

	sysprompt := "Share only a brief description of the place in 50 words. Then immediately make some tool calls and announce them. /no_think"

	question := "Tell me about Greece's largest city."

	messages := []openai.ChatCompletionMessageParamUnion{
		openai.SystemMessage(sysprompt),
		openai.UserMessage(question),
	}

	print("> ")
	println(question)
	println()

	params := openai.ChatCompletionNewParams{
		Messages: messages,
		Seed:     openai.Int(0),
		Model:    model,
		Tools:    tools,
	}

	stream := client.Chat.Completions.NewStreaming(ctx, params)
	acc := openai.ChatCompletionAccumulator{}

	for stream.Next() {
		chunk := stream.Current()

		acc.AddChunk(chunk)

		// When this fires, the current chunk value will not contain content data
		if _, ok := acc.JustFinishedContent(); ok {
			println()
			println("finish-event: Content stream finished")
		}

		if refusal, ok := acc.JustFinishedRefusal(); ok {
			println()
			println("finish-event: refusal stream finished:", refusal)
			println()
		}

		if tool, ok := acc.JustFinishedToolCall(); ok {
			println("finish-event: tool call stream finished:", tool.Index, tool.Name, tool.Arguments)
		}

		// It's best to use chunks after handling JustFinished events.
		// Here we print the delta of the content, if it exists.
		if len(chunk.Choices) > 0 && chunk.Choices[0].Delta.Content != "" {
			print(chunk.Choices[0].Delta.Content)
		}
	}

	if err := stream.Err(); err != nil {
		panic(err)
	}

	if acc.Usage.TotalTokens > 0 {
		println("Total Tokens:", acc.Usage.TotalTokens)
	}

	println()
	println("Count of Tool Calls:", len(acc.Choices[0].Message.ToolCalls))
}

var tools = []openai.ChatCompletionToolParam{
	{
		Function: openai.FunctionDefinitionParam{
			Name:        "get_live_weather",
			Description: openai.String("Get weather at the given location"),
			Parameters: openai.FunctionParameters{
				"type": "object",
				"properties": map[string]interface{}{
					"location": map[string]string{
						"type": "string",
					},
				},
				"required": []string{"location"},
			},
		},
	},
	{
		Function: openai.FunctionDefinitionParam{
			Name:        "get_population",
			Description: openai.String("Get population of a given town"),
			Parameters: openai.FunctionParameters{
				"type": "object",
				"properties": map[string]interface{}{
					"town": map[string]string{
						"type": "string",
					},
					"nation": map[string]string{
						"type": "string",
					},
					"rounding": map[string]string{
						"type":        "integer",
						"description": "Nearest base 10 to round to, e.g. 1000 or 1000000",
					},
				},
				"required": []string{"town", "nation"},
			},
		},
	},
}

// Mock function to simulate weather data retrieval
func getWeather(location string) string {
	// In a real implementation, this function would call a weather API
	return "Sunny, 25°C"
}

// Mock function to simulate population data retrieval
func getPopulation(town, nation string, rounding int) string {
	// In a real implementation, this function would call a population API
	return "Athens, Greece: 664,046"
}

gluonfield avatar Jun 06 '25 18:06 gluonfield

Implemented fix here https://github.com/openai/openai-go/pull/418

gluonfield avatar Jun 08 '25 17:06 gluonfield