Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 30 additions & 3 deletions haystack/components/preprocessors/document_cleaner.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,13 +8,16 @@
from dataclasses import replace
from functools import partial, reduce
from itertools import chain
from typing import Literal
from typing import Any, Literal
from unicodedata import normalize

from haystack import Document, component, logging

logger = logging.getLogger(__name__)

# metadata written by the splitters that is only valid for the text they produced
_SPLIT_OFFSET_FIELDS = ("split_idx_start", "_split_overlap")


@component
class DocumentCleaner:
Expand Down Expand Up @@ -123,7 +126,8 @@ def run(self, documents: list[Document]) -> dict[str, list[Document]]:
if not isinstance(documents, list) or documents and not isinstance(documents[0], Document):
raise TypeError("DocumentCleaner expects a List of Documents as input.")

cleaned_docs = []
cleaned_docs: list[Document] = []
rewritten_sources: set[Any] = set()
for doc in documents:
if doc.content is None:
logger.warning(
Expand Down Expand Up @@ -166,10 +170,33 @@ def run(self, documents: list[Document]) -> dict[str, list[Document]]:
)
continue

if text != doc.content:
rewritten_sources.add(doc.meta.get("source_id"))

clean_doc = replace(doc, id=doc.id if self.keep_id else "", content=text, meta=deepcopy(doc.meta))
cleaned_docs.append(clean_doc)

return {"documents": cleaned_docs}
return {"documents": self._drop_stale_split_offsets(cleaned_docs, rewritten_sources)}

@staticmethod
def _drop_stale_split_offsets(documents: list[Document], rewritten_sources: set[Any]) -> list[Document]:
"""
Remove split offsets that no longer describe the document they are attached to.

Splitters record `split_idx_start` and `_split_overlap` against the text of the whole source document, so the
values of one chunk only make sense next to the values of its siblings. Once the cleaner changes the text of
any chunk, the offsets of the whole source are stale and components that read them, like
`SentenceWindowRetriever`, would merge the chunks at the wrong places.
"""
if not rewritten_sources:
return documents

for doc in documents:
if "split_id" in doc.meta and doc.meta.get("source_id") in rewritten_sources:
for field in _SPLIT_OFFSET_FIELDS:
doc.meta.pop(field, None)

return documents

def _normalize_unicode(self, text: str, form: Literal["NFC", "NFKC", "NFD", "NFKD"]) -> str:
"""
Expand Down
5 changes: 3 additions & 2 deletions haystack/components/retrievers/sentence_window_retriever.py
Original file line number Diff line number Diff line change
Expand Up @@ -128,8 +128,9 @@ def merge_documents_text(documents: list[Document]) -> str:
:param documents: List of Documents to merge.
"""
if any("split_idx_start" not in doc.meta for doc in documents):
# If any of the documents is missing the 'split_idx_start' metadata we just concatenate their content.
return "".join(doc.content for doc in documents if doc.content)
# Without the offsets we can't tell where one chunk ends and the next one begins, so join the
# contents with a space instead of gluing the last word of one chunk to the first of the next.
return " ".join(doc.content for doc in documents if doc.content)

sorted_docs = sorted(documents, key=lambda doc: doc.meta["split_idx_start"])
merged_text = ""
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
---
fixes:
- |
``DocumentCleaner`` now drops ``split_idx_start`` and ``_split_overlap`` from the chunks it rewrites. Those values
are computed by the splitters over the text of the whole source document, and cleaning a chunk changes its length,
so after cleaning they no longer matched the chunks they were attached to. ``SentenceWindowRetriever`` read them to
merge neighbouring chunks and produced windows with the text glued together, for example
``'Sentence 1.Sentence 2.Sentence 3.'`` instead of ``'Sentence 1. Sentence 2. Sentence 3.'``. This affected
``DocumentPreprocessor`` out of the box, since it cleans the chunks produced by the splitter.

The other split metadata (``split_id``, ``source_id``, ``page_number``) is kept, so the retriever still groups the
chunks by source and orders them. Chunks that the cleaner leaves untouched keep their offsets, unless another chunk
of the same source was rewritten, because in that case the offsets of the whole source are off.
47 changes: 46 additions & 1 deletion test/components/preprocessors/test_document_cleaner.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
import pytest

from haystack import Document
from haystack.components.preprocessors import DocumentCleaner
from haystack.components.preprocessors import DocumentCleaner, DocumentSplitter
from haystack.dataclasses import ByteStream, SparseEmbedding


Expand Down Expand Up @@ -211,6 +211,51 @@ def test_keep_id_does_not_alter_document_ids(self):
assert result["documents"][0].id == "1"
assert result["documents"][1].id == "2"

def test_drops_split_offsets_of_rewritten_chunks(self):
splitter = DocumentSplitter(split_by="sentence", split_length=2, split_overlap=1, split_threshold=0)
chunks = splitter.run(documents=[Document(content="First one. Second one. Third one.")])["documents"]
assert all("split_idx_start" in doc.meta and "_split_overlap" in doc.meta for doc in chunks)

cleaned = DocumentCleaner().run(documents=chunks)["documents"]

# the offsets were computed on the text before cleaning, so they can't be used to merge the chunks anymore
for doc in cleaned:
assert "split_idx_start" not in doc.meta
assert "_split_overlap" not in doc.meta
# everything else the splitter wrote is still there
assert "split_id" in doc.meta
assert "source_id" in doc.meta

def test_drops_split_offsets_for_all_chunks_of_a_rewritten_source(self):
source_id = "source"
documents = [
Document(content="Already clean.", meta={"source_id": source_id, "split_id": 0, "split_idx_start": 0}),
Document(content="Needs cleaning.", meta={"source_id": source_id, "split_id": 1, "split_idx_start": 14}),
]

cleaned = DocumentCleaner().run(documents=documents)["documents"]

# the first chunk didn't change, but its offset only makes sense together with the ones of its siblings
assert all("split_idx_start" not in doc.meta for doc in cleaned)
assert [doc.content for doc in cleaned] == ["Already clean.", "Needs cleaning."]

def test_keeps_split_offsets_of_untouched_chunks(self):
splitter = DocumentSplitter(split_by="sentence", split_length=1, split_overlap=0, split_threshold=0)
documents = splitter.run(documents=[Document(content="First one. Second one.")])["documents"]

cleaned = DocumentCleaner(remove_extra_whitespaces=False).run(documents=documents)["documents"]

assert [doc.meta["split_idx_start"] for doc in cleaned] == [0, 11]

def test_split_offsets_survive_on_documents_without_split_id(self):
# metadata we don't know the meaning of is left alone
documents = [Document(content="Some text.", meta={"split_idx_start": 7})]

cleaned = DocumentCleaner().run(documents=documents)["documents"]

assert cleaned[0].content == "Some text."
assert cleaned[0].meta["split_idx_start"] == 7

def test_unicode_normalization(self):
text = """\
アイウエオ
Expand Down
21 changes: 17 additions & 4 deletions test/components/retrievers/test_sentence_window_retriever.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
import pytest

from haystack import DeserializationError, Document, Pipeline
from haystack.components.preprocessors import DocumentSplitter
from haystack.components.preprocessors import DocumentCleaner, DocumentSplitter
from haystack.components.retrievers import InMemoryBM25Retriever
from haystack.components.retrievers.sentence_window_retriever import SentenceWindowRetriever
from haystack.document_stores.in_memory import InMemoryDocumentStore
Expand Down Expand Up @@ -316,9 +316,22 @@ def test_run_custom_fields(self, in_memory_doc_store):
assert len(result["context_documents"]) == 7
assert [doc.meta["split_id_test"] for doc in result["context_documents"]] == [1, 2, 3, 4, 5, 6, 7]
assert result["context_windows"] == [
"Sentence 1.Sentence 2.Sentence 3.Sentence 4.Sentence 5.Sentence 6.Sentence 7."
"Sentence 1. Sentence 2. Sentence 3. Sentence 4. Sentence 5. Sentence 6. Sentence 7."
]

def test_run_after_cleaning_does_not_glue_chunks_together(self, in_memory_doc_store):
text = "Alpha beta. Gamma delta. Epsilon zeta."
chunks = DocumentSplitter(split_by="sentence", split_length=1, split_overlap=0, split_threshold=0).run(
documents=[Document(content=text)]
)["documents"]
cleaned = DocumentCleaner().run(documents=chunks)["documents"]
in_memory_doc_store.write_documents(cleaned)
retriever = SentenceWindowRetriever(document_store=in_memory_doc_store, window_size=1)

result = retriever.run(retrieved_documents=[cleaned[1]])

assert result["context_windows"] == ["Alpha beta. Gamma delta. Epsilon zeta."]

def test_run_with_multiple_source_ids(self, in_memory_doc_store):
docs = [
Document(content="This is the first chunk.", meta={"section": "1", "split_id": 0, "source_id": "source1"}),
Expand Down Expand Up @@ -362,7 +375,7 @@ def test_run_queries_document_store_once_for_all_retrieved_documents(self, in_me

filter_documents.assert_called_once()
assert len(filter_documents.call_args.args[0]["conditions"]) == 2
assert result["context_windows"] == ["a1.a2.a3.", "b4.b5.b6.", "a1.a2.a3."]
assert result["context_windows"] == ["a1. a2. a3.", "b4. b5. b6.", "a1. a2. a3."]
assert [doc.content for doc in result["context_documents"]] == [
"a1.",
"a2.",
Expand Down Expand Up @@ -391,7 +404,7 @@ def test_run_with_documents_missing_metadata_queries_document_store_once(self, i
result = retriever.run(retrieved_documents=[doc_without_meta, docs[2]])

filter_documents.assert_called_once()
assert result["context_windows"] == ["No metadata.", "1.2.3."]
assert result["context_windows"] == ["No metadata.", "1. 2. 3."]
assert result["context_documents"] == [doc_without_meta, docs[1], docs[2], docs[3]]

def test_run_does_not_query_document_store_without_documents_to_expand(self, in_memory_doc_store):
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -179,7 +179,7 @@ async def test_run_async_custom_fields(self, in_memory_doc_store):
assert len(result["context_documents"]) == 7
assert [doc.meta["split_id_test"] for doc in result["context_documents"]] == [1, 2, 3, 4, 5, 6, 7]
assert result["context_windows"] == [
"Sentence 1.Sentence 2.Sentence 3.Sentence 4.Sentence 5.Sentence 6.Sentence 7."
"Sentence 1. Sentence 2. Sentence 3. Sentence 4. Sentence 5. Sentence 6. Sentence 7."
]

@pytest.mark.asyncio
Expand Down Expand Up @@ -227,7 +227,7 @@ async def test_run_async_queries_document_store_once_for_all_retrieved_documents

filter_documents_async.assert_awaited_once()
assert len(filter_documents_async.call_args.kwargs["filters"]["conditions"]) == 2
assert result["context_windows"] == ["a1.a2.a3.", "b4.b5.b6.", "a1.a2.a3."]
assert result["context_windows"] == ["a1. a2. a3.", "b4. b5. b6.", "a1. a2. a3."]
assert [doc.content for doc in result["context_documents"]] == [
"a1.",
"a2.",
Expand Down Expand Up @@ -257,7 +257,7 @@ async def test_run_async_with_documents_missing_metadata_queries_document_store_
result = await retriever.run_async(retrieved_documents=[doc_without_meta, docs[2]])

filter_documents_async.assert_awaited_once()
assert result["context_windows"] == ["No metadata.", "1.2.3."]
assert result["context_windows"] == ["No metadata.", "1. 2. 3."]
assert result["context_documents"] == [doc_without_meta, docs[1], docs[2], docs[3]]

@pytest.mark.asyncio
Expand Down
Loading