From a94525a7ac7e7a63c5e2ea2c88400c9c41d19da8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=AE=B8=E6=9C=AC=E5=BC=BA?= <820396428@qq.com> Date: Tue, 27 Oct 2026 20:22:43 +0800 Subject: [PATCH 1/3] [FIX] Fail indexing when the existence check fails is_document_indexed swallowed every error from the "already indexed?" query and returned False, so a failed check indexed a second copy of the document next to the first one and the run reported success. Treat only a 404 as "nothing indexed yet" - llama-index creates the Qdrant collection on the first write - and re-raise anything else. Applies to both copies of the check: the legacy executor and sdk1's Index. Fixes #2348 --- unstract/sdk1/src/unstract/sdk1/exceptions.py | 32 ++++++++++ unstract/sdk1/src/unstract/sdk1/index.py | 14 ++++- .../sdk1/tests/test_vector_store_not_found.py | 62 +++++++++++++++++++ workers/executor/executors/index.py | 16 +++-- 4 files changed, 115 insertions(+), 9 deletions(-) create mode 100644 unstract/sdk1/tests/test_vector_store_not_found.py diff --git a/unstract/sdk1/src/unstract/sdk1/exceptions.py b/unstract/sdk1/src/unstract/sdk1/exceptions.py index d272d5a3eb..872b7b4fb3 100644 --- a/unstract/sdk1/src/unstract/sdk1/exceptions.py +++ b/unstract/sdk1/src/unstract/sdk1/exceptions.py @@ -8,6 +8,38 @@ logger = logging.getLogger(__name__) +def _status_code_of(error: BaseException) -> int | None: + """Return the HTTP status carried by a provider exception, if any.""" + status = getattr(error, "status_code", None) + if status is None: + response = getattr(error, "response", None) + status = getattr(response, "status_code", None) + if status is None: + return None + try: + return int(status) + except (TypeError, ValueError): + return None + + +def is_vector_store_not_found(error: BaseException) -> bool: + """True when a vector-DB query failed only because the index is not there yet. + + llama-index creates a Qdrant collection on the first write, so the very first + existence check legitimately answers 404. Every other failure means the check + could not be answered, and treating that as "not indexed yet" indexes a + second copy of the document next to the first one. + """ + seen: set[int] = set() + current: BaseException | None = error + while current is not None and id(current) not in seen: + seen.add(id(current)) + if _status_code_of(current) == 404: + return True + current = current.__cause__ or current.__context__ + return False + + class SdkError(Exception): DEFAULT_MESSAGE = "Something went wrong" actual_err: Exception | None = None diff --git a/unstract/sdk1/src/unstract/sdk1/index.py b/unstract/sdk1/src/unstract/sdk1/index.py index 92220f0a87..8a742518cd 100644 --- a/unstract/sdk1/src/unstract/sdk1/index.py +++ b/unstract/sdk1/src/unstract/sdk1/index.py @@ -21,7 +21,13 @@ from unstract.sdk1.adapters.x2text.llm_whisperer_v2.src import LLMWhispererV2 from unstract.sdk1.constants import LogLevel from unstract.sdk1.embedding import EmbeddingCompat -from unstract.sdk1.exceptions import IndexingError, SdkError, VectorDBError, X2TextError +from unstract.sdk1.exceptions import ( + IndexingError, + SdkError, + VectorDBError, + X2TextError, + is_vector_store_not_found, +) from unstract.sdk1.file_storage import FileStorage, FileStorageProvider from unstract.sdk1.platform import PlatformHelper from unstract.sdk1.utils.common import Utils, capture_metrics, log_elapsed @@ -311,9 +317,11 @@ def index( else: self.tool.stream_log(f"No nodes found for {doc_id}") except Exception as e: + if not is_vector_store_not_found(e): + raise self.tool.stream_log( - f"Error querying {vector_db_instance_id}: {e}, proceeding to index", - level=LogLevel.ERROR, + f"No index yet for {vector_db_instance_id}: {e}, proceeding to index", + level=LogLevel.INFO, ) if doc_id_found and not reindex: diff --git a/unstract/sdk1/tests/test_vector_store_not_found.py b/unstract/sdk1/tests/test_vector_store_not_found.py new file mode 100644 index 0000000000..dd18b7601c --- /dev/null +++ b/unstract/sdk1/tests/test_vector_store_not_found.py @@ -0,0 +1,62 @@ +"""is_vector_store_not_found: only a missing index may be treated as "not indexed". + +A Qdrant collection is created on the first write, so the first existence check +legitimately answers 404. Any other failure means the check could not be +answered; treating that as "not indexed yet" indexes a second copy of the +document and the run still reports success. +""" + +from unstract.sdk1.exceptions import is_vector_store_not_found + + +class _HttpError(Exception): + def __init__(self, status_code=None, response=None): + super().__init__(f"HTTP {status_code}") + if status_code is not None: + self.status_code = status_code + if response is not None: + self.response = response + + +class _Response: + def __init__(self, status_code): + self.status_code = status_code + + +def test_status_code_404_is_not_found(): + assert is_vector_store_not_found(_HttpError(status_code=404)) + + +def test_response_status_code_404_is_not_found(): + assert is_vector_store_not_found(_HttpError(response=_Response(404))) + + +def test_other_status_code_is_not_treated_as_missing_index(): + assert not is_vector_store_not_found(_HttpError(status_code=500)) + + +def test_timeout_is_not_treated_as_missing_index(): + assert not is_vector_store_not_found(TimeoutError("read timed out")) + + +def test_wrapped_404_is_not_found(): + try: + try: + raise _HttpError(status_code=404) + except _HttpError as cause: + raise RuntimeError("vector store query failed") from cause + except RuntimeError as error: + assert is_vector_store_not_found(error) + + +def test_plain_error_is_not_missing_index(): + assert not is_vector_store_not_found(ValueError("bad query")) + + +def test_cyclic_cause_chain_terminates(): + first = RuntimeError("first") + second = RuntimeError("second") + first.__cause__ = second + second.__cause__ = first + + assert not is_vector_store_not_found(first) diff --git a/workers/executor/executors/index.py b/workers/executor/executors/index.py index 4bbbaf8839..2f6fb8dec1 100644 --- a/workers/executor/executors/index.py +++ b/workers/executor/executors/index.py @@ -20,9 +20,12 @@ InstanceIdentifiers, ProcessingOptions, ) - from unstract.sdk1.constants import LogLevel -from unstract.sdk1.exceptions import SdkError, parse_litellm_err +from unstract.sdk1.exceptions import ( + SdkError, + is_vector_store_not_found, + parse_litellm_err, +) from unstract.sdk1.file_storage.impl import FileStorage from unstract.sdk1.file_storage.provider import FileStorageProvider from unstract.sdk1.platform import PlatformHelper as ToolAdapter @@ -123,10 +126,11 @@ def is_document_indexed( else: self.tool.stream_log(f"No nodes found for {doc_id}") except Exception as e: - logger.warning( - f"Error querying {self.instance_identifiers.vector_db_instance_id}:" - f" {str(e)}, proceeding to index", - exc_info=True, + if not is_vector_store_not_found(e): + raise + logger.info( + f"No index yet for {self.instance_identifiers.vector_db_instance_id}:" + f" {e}, proceeding to index" ) if doc_id_found and not self.processing_options.reindex: From 36bad414ef08a134d0098459b95e37108b25edd8 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 27 Oct 2026 12:23:41 +0000 Subject: [PATCH 2/3] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- workers/executor/executors/index.py | 1 + 1 file changed, 1 insertion(+) diff --git a/workers/executor/executors/index.py b/workers/executor/executors/index.py index 2f6fb8dec1..1fff12dd3d 100644 --- a/workers/executor/executors/index.py +++ b/workers/executor/executors/index.py @@ -20,6 +20,7 @@ InstanceIdentifiers, ProcessingOptions, ) + from unstract.sdk1.constants import LogLevel from unstract.sdk1.exceptions import ( SdkError, From 5fb5004b8ef05a381f751da735dd36a5bf42dc5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=AE=B8=E6=9C=AC=E5=BC=BA?= <820396428@qq.com> Date: Tue, 27 Oct 2026 20:24:55 +0800 Subject: [PATCH 3/3] [FIX] Annotate the existence-check tests pre-commit's ruff run enables flake8-annotations for tests too; add the return and argument annotations. --- .../sdk1/tests/test_vector_store_not_found.py | 30 +++++++++++-------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/unstract/sdk1/tests/test_vector_store_not_found.py b/unstract/sdk1/tests/test_vector_store_not_found.py index dd18b7601c..e681fa537a 100644 --- a/unstract/sdk1/tests/test_vector_store_not_found.py +++ b/unstract/sdk1/tests/test_vector_store_not_found.py @@ -9,8 +9,17 @@ from unstract.sdk1.exceptions import is_vector_store_not_found +class _Response: + def __init__(self, status_code: int) -> None: + self.status_code = status_code + + class _HttpError(Exception): - def __init__(self, status_code=None, response=None): + def __init__( + self, + status_code: int | None = None, + response: _Response | None = None, + ) -> None: super().__init__(f"HTTP {status_code}") if status_code is not None: self.status_code = status_code @@ -18,28 +27,23 @@ def __init__(self, status_code=None, response=None): self.response = response -class _Response: - def __init__(self, status_code): - self.status_code = status_code - - -def test_status_code_404_is_not_found(): +def test_status_code_404_is_not_found() -> None: assert is_vector_store_not_found(_HttpError(status_code=404)) -def test_response_status_code_404_is_not_found(): +def test_response_status_code_404_is_not_found() -> None: assert is_vector_store_not_found(_HttpError(response=_Response(404))) -def test_other_status_code_is_not_treated_as_missing_index(): +def test_other_status_code_is_not_treated_as_missing_index() -> None: assert not is_vector_store_not_found(_HttpError(status_code=500)) -def test_timeout_is_not_treated_as_missing_index(): +def test_timeout_is_not_treated_as_missing_index() -> None: assert not is_vector_store_not_found(TimeoutError("read timed out")) -def test_wrapped_404_is_not_found(): +def test_wrapped_404_is_not_found() -> None: try: try: raise _HttpError(status_code=404) @@ -49,11 +53,11 @@ def test_wrapped_404_is_not_found(): assert is_vector_store_not_found(error) -def test_plain_error_is_not_missing_index(): +def test_plain_error_is_not_missing_index() -> None: assert not is_vector_store_not_found(ValueError("bad query")) -def test_cyclic_cause_chain_terminates(): +def test_cyclic_cause_chain_terminates() -> None: first = RuntimeError("first") second = RuntimeError("second") first.__cause__ = second