langchaingo icon indicating copy to clipboard operation
langchaingo copied to clipboard

StreamingReasoningFunc或者StreamingFunc中期望返回原始的Choices信息,主要考虑到在遇到一些特定代码的时候,给前端返回不用的错误代码。而不是仅暴露内容出去。

Open mdsq1 opened this issue 8 months ago • 1 comments

StreamingReasoningFunc或者StreamingFunc中期望返回原始的Choices信息,主要考虑到在遇到一些特定代码的时候,给前端返回不用的错误代码。而不是仅暴露内容出去。

比如我想在遇到content_filter的时候,返回给前端特定的错误代码。这样的话,就可以更方便。

此处我为了避免影响原来的方案,实现了StreamedChatResponsePayloadFunc,这样更加灵活一些,也兼容代码的原有逻辑。

实现方案: llms/openai/internal/openaiclient/chat.go

type ChatRequest struct {
    // ... existing fields ...

    // StreamingFunc is a function to be called for each chunk of a streaming response.
    // Return an error to stop streaming early.
    StreamingFunc func(ctx context.Context, chunk []byte) error `json:"-"`

    // StreamingReasoningFunc is a function to be called for each reasoning and content chunk of a streaming response.
    // Return an error to stop streaming early.
    StreamingReasoningFunc func(ctx context.Context, reasoningChunk, chunk []byte) error `json:"-"`

    // StreamedChatResponsePayloadFunc is a function to be called for each streaming response payload.
    // Return an error to stop streaming early.
    StreamedChatResponsePayloadFunc func(ctx context.Context, payload StreamedChatResponsePayload) error `json:"-"`

    // ... existing fields ...
}
func combineStreamingChatResponse(
    ctx context.Context,
    payload *ChatRequest,
    responseChan chan StreamedChatResponsePayload,
) (*ChatCompletionResponse, error) {
    response := ChatCompletionResponse{
        Choices: []*ChatCompletionChoice{
            {},
        },
    }

    for streamResponse := range responseChan {
        if streamResponse.Error != nil {
            return nil, streamResponse.Error
        }

        // Call StreamingReasoningPayloadFunc if it's set
        if payload.StreamedChatResponsePayload != nil {
            err := payload.StreamingReasoningPayloadFunc(ctx, streamResponse)
            if err != nil {
                return nil, fmt.Errorf("streaming reasoning payload func returned an error: %w", err)
            }
        }

        if streamResponse.Usage != nil {
            response.Usage.CompletionTokens = streamResponse.Usage.CompletionTokens
            response.Usage.PromptTokens = streamResponse.Usage.PromptTokens
            response.Usage.TotalTokens = streamResponse.Usage.TotalTokens
            response.Usage.CompletionTokensDetails.ReasoningTokens = streamResponse.Usage.CompletionTokensDetails.ReasoningTokens
        }

        if len(streamResponse.Choices) == 0 {
            continue
        }
        choice := streamResponse.Choices[0]
        chunk := []byte(choice.Delta.Content)
        reasoningChunk := []byte(choice.Delta.ReasoningContent)
        response.Choices[0].Message.Content += choice.Delta.Content
        response.Choices[0].FinishReason = choice.FinishReason
        response.Choices[0].Message.ReasoningContent += choice.Delta.ReasoningContent

        if choice.Delta.FunctionCall != nil {
            chunk = updateFunctionCall(response.Choices[0].Message, choice.Delta.FunctionCall)
        }

        if len(choice.Delta.ToolCalls) > 0 {
            chunk, response.Choices[0].Message.ToolCalls = updateToolCalls(response.Choices[0].Message.ToolCalls,
                choice.Delta.ToolCalls)
        }

        if payload.StreamingFunc != nil {
            err := payload.StreamingFunc(ctx, chunk)
            if err != nil {
                return nil, fmt.Errorf("streaming func returned an error: %w", err)
            }
        }
        if payload.StreamingReasoningFunc != nil {
            err := payload.StreamingReasoningFunc(ctx, reasoningChunk, chunk)
            if err != nil {
                return nil, fmt.Errorf("streaming reasoning func returned an error: %w", err)
            }
        }
    }
    return &response, nil
}

mdsq1 avatar Apr 11 '25 02:04 mdsq1

@tmc Please prioritize looking at this issue. Thanks大佬,如果可以的话,我可以提一个PR,解决这个问题。

mdsq1 avatar Apr 12 '25 08:04 mdsq1