diff --git a/haystack/components/generators/chat/openai.py b/haystack/components/generators/chat/openai.py index 764fb740d2e..42ddf328f48 100644 --- a/haystack/components/generators/chat/openai.py +++ b/haystack/components/generators/chat/openai.py @@ -20,6 +20,7 @@ ) from openai.types.chat.chat_completion import Choice from openai.types.chat.chat_completion_chunk import Choice as ChunkChoice +from openai.types.chat.chat_completion_chunk import ChoiceDeltaToolCall from pydantic import BaseModel from haystack import component, default_from_dict, default_to_dict, logging @@ -775,6 +776,32 @@ def _convert_chat_completion_to_chat_message( return ChatMessage.from_assistant(text=text, tool_calls=tool_calls, meta=meta) +def _resolve_tool_call_index( + tool_call: ChoiceDeltaToolCall, previous_chunks: list[StreamingChunk], current: list[ToolCallDelta] +) -> int: + """ + Returns the index of a streamed tool call delta. + + OpenAI always sends `index`. Some OpenAI-compatible servers do not: Gemini's OpenAI-compatible endpoint and + Mistral send each tool call whole with only an `id`. For those, a delta with an id not seen yet starts a new + tool call and a delta without an id continues the latest one. + + :param tool_call: The tool call delta from the OpenAI API. + :param previous_chunks: The StreamingChunks received so far. + :param current: The ToolCallDeltas already built from the current chunk. + :returns: The index of the tool call. + """ + if tool_call.index is not None: + return tool_call.index + seen = [tc for c in previous_chunks for tc in (c.tool_calls or [])] + current + if not seen: + return 0 + last = max(tc.index for tc in seen) + if tool_call.id and tool_call.id not in {tc.id for tc in seen if tc.id}: + return last + 1 + return last + + def _convert_chat_completion_chunk_to_streaming_chunk( chunk: ChatCompletionChunk, previous_chunks: list[StreamingChunk], component_info: ComponentInfo | None = None ) -> StreamingChunk: @@ -816,12 +843,14 @@ def _convert_chat_completion_chunk_to_streaming_chunk( # create a list of ToolCallDelta objects from the tool calls if choice.delta and choice.delta.tool_calls: - tool_calls_deltas = [] + tool_calls_deltas: list[ToolCallDelta] = [] for tool_call in choice.delta.tool_calls: function = tool_call.function tool_calls_deltas.append( ToolCallDelta( - index=tool_call.index, + index=_resolve_tool_call_index( + tool_call=tool_call, previous_chunks=previous_chunks, current=tool_calls_deltas + ), id=tool_call.id, tool_name=function.name if function else None, arguments=function.arguments if function and function.arguments else None, diff --git a/haystack/components/generators/utils.py b/haystack/components/generators/utils.py index 950852812b2..4162073432c 100644 --- a/haystack/components/generators/utils.py +++ b/haystack/components/generators/utils.py @@ -143,8 +143,9 @@ def _convert_streaming_chunks_to_chat_message(chunks: list[StreamingChunk]) -> C if tool_call.index not in tool_call_data: tool_call_data[tool_call.index] = {"id": "", "name": "", "arguments": ""} - # Save the ID if present - if tool_call.id is not None: + # Save the ID if present. Some providers (e.g. Alibaba Cloud Model Studio) send an empty id on the + # later chunks of a tool call, which must not overwrite the id from the first chunk. + if tool_call.id: tool_call_data[tool_call.index]["id"] = tool_call.id if tool_call.tool_name is not None: diff --git a/releasenotes/notes/openai-stream-tool-calls-without-index-4c1f8e2a9b7d6035.yaml b/releasenotes/notes/openai-stream-tool-calls-without-index-4c1f8e2a9b7d6035.yaml new file mode 100644 index 00000000000..4b84b6cad5e --- /dev/null +++ b/releasenotes/notes/openai-stream-tool-calls-without-index-4c1f8e2a9b7d6035.yaml @@ -0,0 +1,9 @@ +--- +fixes: + - | + ``OpenAIChatGenerator`` with a ``streaming_callback`` no longer fails on tool calls from OpenAI-compatible + servers that leave out the tool call ``index`` (Gemini's OpenAI-compatible endpoint, Mistral). Before, the stream + raised ``ValueError: If `tool_call`, `tool_call_result` or `reasoning` is set, `index` must also be set``; now + each tool call gets its own index, so parallel calls stay separate. + Streamed tool calls also keep their id when a provider sends an empty ``id`` on the later chunks of the same call + (Alibaba Cloud Model Studio); before, the reply's ``ToolCall.id`` came back as ``""``. diff --git a/test/components/generators/chat/test_openai.py b/test/components/generators/chat/test_openai.py index 9cd09f4c3f3..78767830731 100644 --- a/test/components/generators/chat/test_openai.py +++ b/test/components/generators/chat/test_openai.py @@ -46,7 +46,7 @@ _convert_chat_completion_to_chat_message, _make_schema_strict, ) -from haystack.components.generators.utils import print_streaming_chunk +from haystack.components.generators.utils import _convert_streaming_chunks_to_chat_message, print_streaming_chunk from haystack.dataclasses import ( ChatMessage, ChatRole, @@ -1950,6 +1950,143 @@ def test_convert_chat_completion_chunk_to_streaming_chunk( assert stream_chunk == haystack_chunk previous_chunks.append(stream_chunk) + @staticmethod + def _stream(raw_chunks: list[dict[str, Any]]) -> ChatMessage: + # The OpenAI SDK builds stream chunks without validation, so a field the server omitted (like a tool call + # `index`) arrives as None; ChatCompletionChunk.construct does the same. + previous_chunks: list[StreamingChunk] = [] + for raw in raw_chunks: + previous_chunks.append( + _convert_chat_completion_chunk_to_streaming_chunk( + chunk=ChatCompletionChunk.construct(**raw), previous_chunks=previous_chunks + ) + ) + return _convert_streaming_chunks_to_chat_message(chunks=previous_chunks) + + def test_tool_calls_without_index_stay_separate(self) -> None: + # Recorded from Gemini's OpenAI-compatible endpoint (pydantic-ai cassette + # test_compatible_api_gemini_3_tool_call_thought_signature[True]): two parallel calls, each whole in its own + # chunk, and no `index` on either. + base = { + "id": "3tXOasW2I-2KmtkPrtulEA", + "object": "chat.completion.chunk", + "created": 1791940062, + "model": "gemini-3.1-flash-lite", + } + message = self._stream( + [ + { + **base, + "choices": [ + { + "index": 0, + "delta": { + "role": "assistant", + "tool_calls": [ + { + "id": "call_108766", + "type": "function", + "function": {"name": "get_weather", "arguments": '{"city":"Taipei"}'}, + } + ], + }, + } + ], + }, + { + **base, + "choices": [ + { + "index": 0, + "delta": { + "role": "assistant", + "tool_calls": [ + { + "id": "call_108772", + "type": "function", + "function": {"name": "get_weather", "arguments": '{"city":"Kaohsiung"}'}, + } + ], + }, + } + ], + }, + {**base, "choices": [{"index": 0, "delta": {"role": "assistant"}, "finish_reason": "stop"}]}, + ] + ) + assert message.tool_calls == [ + ToolCall(id="call_108766", tool_name="get_weather", arguments={"city": "Taipei"}), + ToolCall(id="call_108772", tool_name="get_weather", arguments={"city": "Kaohsiung"}), + ] + + def test_empty_tool_call_id_on_later_chunks_keeps_the_first_id(self) -> None: + # Recorded from Alibaba Cloud Model Studio, qwen3-max (vercel/ai fixture alibaba-tool-call.chunks.txt): the + # chunks after the first one repeat `"id": ""`. + base = { + "id": "chatcmpl-8e243c57-23b3-9db2-a02e-e3c53929c368", + "object": "chat.completion.chunk", + "created": 1770764938, + "model": "qwen3-max", + } + message = self._stream( + [ + { + **base, + "choices": [ + { + "index": 0, + "delta": { + "role": "assistant", + "tool_calls": [ + { + "index": 0, + "id": "call_eee11723464a4b9eb8cee71d", + "type": "function", + "function": {"name": "weather", "arguments": ""}, + } + ], + }, + } + ], + }, + { + **base, + "choices": [ + { + "index": 0, + "delta": { + "tool_calls": [ + { + "index": 0, + "id": "", + "type": "function", + "function": {"arguments": '{"location": "San Francisco'}, + } + ] + }, + } + ], + }, + { + **base, + "choices": [ + { + "index": 0, + "delta": { + "tool_calls": [ + {"index": 0, "id": "", "type": "function", "function": {"arguments": '"}'}} + ] + }, + } + ], + }, + {**base, "choices": [{"index": 0, "delta": {}, "finish_reason": "tool_calls"}]}, + ] + ) + assert message.tool_calls == [ + ToolCall(id="call_eee11723464a4b9eb8cee71d", tool_name="weather", arguments={"location": "San Francisco"}) + ] + def test_convert_chat_completion_chunk_with_empty_tool_calls(self) -> None: # This can happen with some LLM providers where tool calls are not present but the pydantic models are still