Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 31 additions & 2 deletions haystack/components/generators/chat/openai.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
)
from openai.types.chat.chat_completion import Choice
from openai.types.chat.chat_completion_chunk import Choice as ChunkChoice
from openai.types.chat.chat_completion_chunk import ChoiceDeltaToolCall
from pydantic import BaseModel

from haystack import component, default_from_dict, default_to_dict, logging
Expand Down Expand Up @@ -775,6 +776,32 @@ def _convert_chat_completion_to_chat_message(
return ChatMessage.from_assistant(text=text, tool_calls=tool_calls, meta=meta)


def _resolve_tool_call_index(
tool_call: ChoiceDeltaToolCall, previous_chunks: list[StreamingChunk], current: list[ToolCallDelta]
) -> int:
"""
Returns the index of a streamed tool call delta.

OpenAI always sends `index`. Some OpenAI-compatible servers do not: Gemini's OpenAI-compatible endpoint and
Mistral send each tool call whole with only an `id`. For those, a delta with an id not seen yet starts a new
tool call and a delta without an id continues the latest one.

:param tool_call: The tool call delta from the OpenAI API.
:param previous_chunks: The StreamingChunks received so far.
:param current: The ToolCallDeltas already built from the current chunk.
:returns: The index of the tool call.
"""
if tool_call.index is not None:
return tool_call.index
seen = [tc for c in previous_chunks for tc in (c.tool_calls or [])] + current
if not seen:
return 0
last = max(tc.index for tc in seen)
if tool_call.id and tool_call.id not in {tc.id for tc in seen if tc.id}:
return last + 1
return last


def _convert_chat_completion_chunk_to_streaming_chunk(
chunk: ChatCompletionChunk, previous_chunks: list[StreamingChunk], component_info: ComponentInfo | None = None
) -> StreamingChunk:
Expand Down Expand Up @@ -816,12 +843,14 @@ def _convert_chat_completion_chunk_to_streaming_chunk(

# create a list of ToolCallDelta objects from the tool calls
if choice.delta and choice.delta.tool_calls:
tool_calls_deltas = []
tool_calls_deltas: list[ToolCallDelta] = []
for tool_call in choice.delta.tool_calls:
function = tool_call.function
tool_calls_deltas.append(
ToolCallDelta(
index=tool_call.index,
index=_resolve_tool_call_index(
tool_call=tool_call, previous_chunks=previous_chunks, current=tool_calls_deltas
),
id=tool_call.id,
tool_name=function.name if function else None,
arguments=function.arguments if function and function.arguments else None,
Expand Down
5 changes: 3 additions & 2 deletions haystack/components/generators/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -143,8 +143,9 @@ def _convert_streaming_chunks_to_chat_message(chunks: list[StreamingChunk]) -> C
if tool_call.index not in tool_call_data:
tool_call_data[tool_call.index] = {"id": "", "name": "", "arguments": ""}

# Save the ID if present
if tool_call.id is not None:
# Save the ID if present. Some providers (e.g. Alibaba Cloud Model Studio) send an empty id on the
# later chunks of a tool call, which must not overwrite the id from the first chunk.
if tool_call.id:
tool_call_data[tool_call.index]["id"] = tool_call.id

if tool_call.tool_name is not None:
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
---
fixes:
- |
``OpenAIChatGenerator`` with a ``streaming_callback`` no longer fails on tool calls from OpenAI-compatible
servers that leave out the tool call ``index`` (Gemini's OpenAI-compatible endpoint, Mistral). Before, the stream
raised ``ValueError: If `tool_call`, `tool_call_result` or `reasoning` is set, `index` must also be set``; now
each tool call gets its own index, so parallel calls stay separate.
Streamed tool calls also keep their id when a provider sends an empty ``id`` on the later chunks of the same call
(Alibaba Cloud Model Studio); before, the reply's ``ToolCall.id`` came back as ``""``.
139 changes: 138 additions & 1 deletion test/components/generators/chat/test_openai.py
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@
_convert_chat_completion_to_chat_message,
_make_schema_strict,
)
from haystack.components.generators.utils import print_streaming_chunk
from haystack.components.generators.utils import _convert_streaming_chunks_to_chat_message, print_streaming_chunk
from haystack.dataclasses import (
ChatMessage,
ChatRole,
Expand Down Expand Up @@ -1950,6 +1950,143 @@ def test_convert_chat_completion_chunk_to_streaming_chunk(
assert stream_chunk == haystack_chunk
previous_chunks.append(stream_chunk)

@staticmethod
def _stream(raw_chunks: list[dict[str, Any]]) -> ChatMessage:
# The OpenAI SDK builds stream chunks without validation, so a field the server omitted (like a tool call
# `index`) arrives as None; ChatCompletionChunk.construct does the same.
previous_chunks: list[StreamingChunk] = []
for raw in raw_chunks:
previous_chunks.append(
_convert_chat_completion_chunk_to_streaming_chunk(
chunk=ChatCompletionChunk.construct(**raw), previous_chunks=previous_chunks
)
)
return _convert_streaming_chunks_to_chat_message(chunks=previous_chunks)

def test_tool_calls_without_index_stay_separate(self) -> None:
# Recorded from Gemini's OpenAI-compatible endpoint (pydantic-ai cassette
# test_compatible_api_gemini_3_tool_call_thought_signature[True]): two parallel calls, each whole in its own
# chunk, and no `index` on either.
base = {
"id": "3tXOasW2I-2KmtkPrtulEA",
"object": "chat.completion.chunk",
"created": 1791940062,
"model": "gemini-3.1-flash-lite",
}
message = self._stream(
[
{
**base,
"choices": [
{
"index": 0,
"delta": {
"role": "assistant",
"tool_calls": [
{
"id": "call_108766",
"type": "function",
"function": {"name": "get_weather", "arguments": '{"city":"Taipei"}'},
}
],
},
}
],
},
{
**base,
"choices": [
{
"index": 0,
"delta": {
"role": "assistant",
"tool_calls": [
{
"id": "call_108772",
"type": "function",
"function": {"name": "get_weather", "arguments": '{"city":"Kaohsiung"}'},
}
],
},
}
],
},
{**base, "choices": [{"index": 0, "delta": {"role": "assistant"}, "finish_reason": "stop"}]},
]
)
assert message.tool_calls == [
ToolCall(id="call_108766", tool_name="get_weather", arguments={"city": "Taipei"}),
ToolCall(id="call_108772", tool_name="get_weather", arguments={"city": "Kaohsiung"}),
]

def test_empty_tool_call_id_on_later_chunks_keeps_the_first_id(self) -> None:
# Recorded from Alibaba Cloud Model Studio, qwen3-max (vercel/ai fixture alibaba-tool-call.chunks.txt): the
# chunks after the first one repeat `"id": ""`.
base = {
"id": "chatcmpl-8e243c57-23b3-9db2-a02e-e3c53929c368",
"object": "chat.completion.chunk",
"created": 1770764938,
"model": "qwen3-max",
}
message = self._stream(
[
{
**base,
"choices": [
{
"index": 0,
"delta": {
"role": "assistant",
"tool_calls": [
{
"index": 0,
"id": "call_eee11723464a4b9eb8cee71d",
"type": "function",
"function": {"name": "weather", "arguments": ""},
}
],
},
}
],
},
{
**base,
"choices": [
{
"index": 0,
"delta": {
"tool_calls": [
{
"index": 0,
"id": "",
"type": "function",
"function": {"arguments": '{"location": "San Francisco'},
}
]
},
}
],
},
{
**base,
"choices": [
{
"index": 0,
"delta": {
"tool_calls": [
{"index": 0, "id": "", "type": "function", "function": {"arguments": '"}'}}
]
},
}
],
},
{**base, "choices": [{"index": 0, "delta": {}, "finish_reason": "tool_calls"}]},
]
)
assert message.tool_calls == [
ToolCall(id="call_eee11723464a4b9eb8cee71d", tool_name="weather", arguments={"location": "San Francisco"})
]

def test_convert_chat_completion_chunk_with_empty_tool_calls(self) -> None:

# This can happen with some LLM providers where tool calls are not present but the pydantic models are still
Expand Down