Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 18 additions & 8 deletions haystack/components/evaluators/context_relevance.py
Original file line number Diff line number Diff line change
Expand Up @@ -220,14 +220,24 @@ def _postprocess_results(self, result: dict[str, Any]) -> dict[str, Any]:
The updated dictionary containing final scores and tracking metrics.
"""
for idx, res in enumerate(result["results"]):
if res is None:
result["results"][idx] = {"relevant_statements": [], "score": float("nan"), "status": "error"}
continue
res["status"] = "evaluated"
if len(res["relevant_statements"]) > 0:
res["score"] = 1
else:
res["score"] = 0
if res is not None:
statements = res["relevant_statements"]
if isinstance(statements, list):
res["score"] = 1 if statements else 0
res["status"] = "evaluated"
continue

value_type = type(statements).__name__
if self.raise_on_failure:
msg = f"Input {idx}: expected `relevant_statements` to be a list, got {value_type}."
raise ValueError(msg)
logger.warning(
"Input {idx}: expected `relevant_statements` to be a list, got {value_type}.",
idx=idx,
value_type=value_type,
)

result["results"][idx] = {"relevant_statements": [], "score": float("nan"), "status": "error"}

# calculate average context relevance score over all queries
scores = [res["score"] for res in result["results"]]
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
---
fixes:
- |
Fixed two issues in how ``ContextRelevanceEvaluator`` scores malformed responses. When ``relevant_statements`` was
``null``, evaluation stopped with a ``TypeError`` even when ``raise_on_failure=False``. When it was a non-empty
string, the evaluator incorrectly assigned a relevance score of 1 and included it in the average. Both cases now
raise a ``ValueError`` when ``raise_on_failure=True``. Otherwise, the evaluator marks the input as failed and
excludes it from the average.
22 changes: 22 additions & 0 deletions test/components/evaluators/test_context_relevance_evaluator.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,15 @@
#
# SPDX-License-Identifier: Apache-2.0

import json
import math
import os

import pytest

from haystack import Pipeline
from haystack.components.evaluators import ContextRelevanceEvaluator
from haystack.components.generators.chat import MockChatGenerator
from haystack.components.generators.chat.openai import OpenAIChatGenerator
from haystack.dataclasses.chat_message import ChatMessage
from haystack.utils.auth import Secret
Expand Down Expand Up @@ -269,6 +271,26 @@ def chat_generator_run(self, *args, **kwargs):
assert all(math.isnan(score) for score in results["individual_scores"])
assert [result["status"] for result in results["results"]] == ["error", "error"]

@pytest.mark.parametrize("statements", [None, "unexpected"])
def test_run_invalid_statements_value_raise_on_failure_false(self, statements):
generator = MockChatGenerator(
responses=[json.dumps({"relevant_statements": statements}), '{"relevant_statements": ["c2"]}']
)
component = ContextRelevanceEvaluator(chat_generator=generator, raise_on_failure=False)

results = component.run(questions=["q1", "q2"], contexts=[["c1"], ["c2"]])

assert results["results"][0]["status"] == "error"
assert math.isnan(results["results"][0]["score"])
assert results["score"] == 1

def test_run_invalid_statements_value_raise_on_failure_true(self):
generator = MockChatGenerator(responses='{"relevant_statements": null}')
component = ContextRelevanceEvaluator(chat_generator=generator, raise_on_failure=True)

with pytest.raises(ValueError, match="expected `relevant_statements` to be a list"):
component.run(questions=["q1"], contexts=[["c1"]])

@pytest.mark.skipif(
not os.environ.get("OPENAI_API_KEY", None),
reason="Export an env var called OPENAI_API_KEY containing the OpenAI API key to run this test.",
Expand Down
Loading