Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 22 additions & 1 deletion haystack/dataclasses/document.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,27 @@
_LEGACY_FIELDS = ["content_type", "id_hash_keys", "dataframe"]


def _meta_for_id(value: Any) -> Any:
"""
Recursively turn the values of `meta` into something that hashes to a stable and unique string.

`json.dumps` alone loses the distinction between types that it can't represent natively: a tuple becomes a list, and
a set goes through `default=str`. Two documents with `{"tags": [1, 2]}` and `{"tags": (1, 2)}` then end up with the
same ID, which makes document stores reject the second one or quietly overwrite it. Sets are converted to a sorted
list so that iteration order doesn't leak into the ID either.
"""
if isinstance(value, dict):
return {key: _meta_for_id(item) for key, item in value.items()}
if isinstance(value, list):
return [_meta_for_id(item) for item in value]
if isinstance(value, tuple):
return {"__tuple__": [_meta_for_id(item) for item in value]}
if isinstance(value, (set, frozenset)):
items = [_meta_for_id(item) for item in value]
return {"__set__": sorted(items, key=lambda item: json.dumps(item, sort_keys=True, default=str))}
return value


class _RemoveLegacyFields(type):
def __call__(cls, *args: Any, **kwargs: Any) -> Any:
"""
Expand Down Expand Up @@ -138,7 +159,7 @@ def _create_id(self) -> str:
blob = self.blob.data if self.blob is not None else None
mime_type = self.blob.mime_type if self.blob is not None else None
# Sort keys so meta order doesn't affect the ID. Keep "{}" for empty meta so existing IDs stay stable.
meta = json.dumps(self.meta, sort_keys=True, default=str) if self.meta else "{}"
meta = json.dumps(_meta_for_id(self.meta), sort_keys=True, default=str) if self.meta else "{}"
embedding = self.embedding if self.embedding is not None else None
sparse_embedding = self.sparse_embedding.to_dict() if self.sparse_embedding is not None else ""
data = f"{text}{dataframe}{blob!r}{mime_type}{meta}{embedding}{sparse_embedding}"
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
---
fixes:
- |
``Document`` no longer generates the same ID for metadata values that only differ in type. ``_create_id`` hashed the
metadata with ``json.dumps(..., default=str)``, and both that and JSON itself flatten the type away: a tuple became
a list, and a set went through ``str()``. ``Document(content="x", meta={"tags": [1, 2]})`` and
``Document(content="x", meta={"tags": (1, 2)})`` ended up with the same ID, so writing both either raised
``DuplicateDocumentError`` or, with ``DuplicatePolicy.OVERWRITE``, silently kept only one of them.

Tuples and sets are now serialized with a type marker for the ID. Sets are sorted by their JSON representation, so
building one in a different order still gives the same ID, while the element order of a tuple still matters. IDs of
documents whose metadata only uses types JSON supports are unchanged.
40 changes: 40 additions & 0 deletions test/dataclasses/test_document.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
from haystack import Document
from haystack.dataclasses.byte_stream import ByteStream
from haystack.dataclasses.sparse_embedding import SparseEmbedding
from haystack.document_stores.in_memory import InMemoryDocumentStore


@pytest.mark.parametrize(
Expand Down Expand Up @@ -167,6 +168,45 @@ def test_id_is_independent_of_nested_meta_key_order():
assert doc1.id == doc2.id


@pytest.mark.parametrize(
"meta_a,meta_b",
[
pytest.param({"tags": [1, 2]}, {"tags": (1, 2)}, id="list_vs_tuple"),
pytest.param({"tags": {1, 2}}, {"tags": "{1, 2}"}, id="set_vs_its_str"),
pytest.param({"tags": {1, 2}}, {"tags": [1, 2]}, id="set_vs_list"),
pytest.param({"a": ([1], [2, 3])}, {"a": [(1,), (2, 3)]}, id="nested"),
pytest.param({"a": (1, (2, 3))}, {"a": ((1, 2), 3)}, id="nested_tuple_order"),
],
)
def test_id_distinguishes_meta_value_types(meta_a, meta_b):
doc_a = Document(content="same content", meta=meta_a)
doc_b = Document(content="same content", meta=meta_b)

assert doc_a.meta != doc_b.meta
assert doc_a.id != doc_b.id


def test_id_of_set_meta_is_independent_of_iteration_order():
doc1 = Document(content="hello", meta={"tags": {1, 2, 3}})
doc2 = Document(content="hello", meta={"tags": {3, 2, 1}})

assert doc1.id == doc2.id


def test_id_of_tuple_meta_depends_on_element_order():
doc1 = Document(content="hello", meta={"point": (1, 2)})
doc2 = Document(content="hello", meta={"point": (2, 1)})

assert doc1.id != doc2.id


def test_documents_with_different_meta_types_can_be_stored_together():
store = InMemoryDocumentStore()
store.write_documents([Document(content="x", meta={"tags": [1, 2]}), Document(content="x", meta={"tags": (1, 2)})])

assert store.count_documents() == 2


def test_to_dict():
doc = Document()
assert doc.to_dict() == {
Expand Down
Loading