[FIX] Fail indexing when the existence check fails

unstract/sdk1/src/unstract/sdk1/exceptions.py

@@ -8,6 +8,38 @@
 logger = logging.getLogger(__name__)
 
 
+def _status_code_of(error: BaseException) -> int | None:
+    """Return the HTTP status carried by a provider exception, if any."""
+    status = getattr(error, "status_code", None)
+    if status is None:
+        response = getattr(error, "response", None)
+        status = getattr(response, "status_code", None)
+    if status is None:
+        return None
+    try:
+        return int(status)
+    except (TypeError, ValueError):
+        return None
+
+
+def is_vector_store_not_found(error: BaseException) -> bool:
+    """True when a vector-DB query failed only because the index is not there yet.
+
+    llama-index creates a Qdrant collection on the first write, so the very first
+    existence check legitimately answers 404. Every other failure means the check
+    could not be answered, and treating that as "not indexed yet" indexes a
+    second copy of the document next to the first one.
+    """
+    seen: set[int] = set()
+    current: BaseException | None = error
+    while current is not None and id(current) not in seen:
+        seen.add(id(current))
+        if _status_code_of(current) == 404:
+            return True
+        current = current.__cause__ or current.__context__
+    return False
+
+
 class SdkError(Exception):
     DEFAULT_MESSAGE = "Something went wrong"
     actual_err: Exception | None = None

unstract/sdk1/src/unstract/sdk1/index.py

@@ -21,7 +21,13 @@
 from unstract.sdk1.adapters.x2text.llm_whisperer_v2.src import LLMWhispererV2
 from unstract.sdk1.constants import LogLevel
 from unstract.sdk1.embedding import EmbeddingCompat
-from unstract.sdk1.exceptions import IndexingError, SdkError, VectorDBError, X2TextError
+from unstract.sdk1.exceptions import (
+    IndexingError,
+    SdkError,
+    VectorDBError,
+    X2TextError,
+    is_vector_store_not_found,
+)
 from unstract.sdk1.file_storage import FileStorage, FileStorageProvider
 from unstract.sdk1.platform import PlatformHelper
 from unstract.sdk1.utils.common import Utils, capture_metrics, log_elapsed
@@ -311,9 +317,11 @@ def index(
                 else:
                     self.tool.stream_log(f"No nodes found for {doc_id}")
             except Exception as e:
+                if not is_vector_store_not_found(e):
+                    raise
                 self.tool.stream_log(
-                    f"Error querying {vector_db_instance_id}: {e}, proceeding to index",
-                    level=LogLevel.ERROR,
+                    f"No index yet for {vector_db_instance_id}: {e}, proceeding to index",
+                    level=LogLevel.INFO,
                 )
 
             if doc_id_found and not reindex:

unstract/sdk1/tests/test_vector_store_not_found.py

@@ -0,0 +1,66 @@
+"""is_vector_store_not_found: only a missing index may be treated as "not indexed".
+
+A Qdrant collection is created on the first write, so the first existence check
+legitimately answers 404. Any other failure means the check could not be
+answered; treating that as "not indexed yet" indexes a second copy of the
+document and the run still reports success.
+"""
+
+from unstract.sdk1.exceptions import is_vector_store_not_found
+
+
+class _Response:
+    def __init__(self, status_code: int) -> None:
+        self.status_code = status_code
+
+
+class _HttpError(Exception):
+    def __init__(
+        self,
+        status_code: int | None = None,
+        response: _Response | None = None,
+    ) -> None:
+        super().__init__(f"HTTP {status_code}")
+        if status_code is not None:
+            self.status_code = status_code
+        if response is not None:
+            self.response = response
+
+
+def test_status_code_404_is_not_found() -> None:
+    assert is_vector_store_not_found(_HttpError(status_code=404))
+
+
+def test_response_status_code_404_is_not_found() -> None:
+    assert is_vector_store_not_found(_HttpError(response=_Response(404)))
+
+
+def test_other_status_code_is_not_treated_as_missing_index() -> None:
+    assert not is_vector_store_not_found(_HttpError(status_code=500))
+
+
+def test_timeout_is_not_treated_as_missing_index() -> None:
+    assert not is_vector_store_not_found(TimeoutError("read timed out"))
+
+
+def test_wrapped_404_is_not_found() -> None:
+    try:
+        try:
+            raise _HttpError(status_code=404)
+        except _HttpError as cause:
+            raise RuntimeError("vector store query failed") from cause
+    except RuntimeError as error:
+        assert is_vector_store_not_found(error)
+
+
+def test_plain_error_is_not_missing_index() -> None:
+    assert not is_vector_store_not_found(ValueError("bad query"))
+
+
+def test_cyclic_cause_chain_terminates() -> None:
+    first = RuntimeError("first")
+    second = RuntimeError("second")
+    first.__cause__ = second
+    second.__cause__ = first
+
+    assert not is_vector_store_not_found(first)

workers/executor/executors/index.py

@@ -22,7 +22,11 @@
 )
 
 from unstract.sdk1.constants import LogLevel
-from unstract.sdk1.exceptions import SdkError, parse_litellm_err
+from unstract.sdk1.exceptions import (
+    SdkError,
+    is_vector_store_not_found,
+    parse_litellm_err,
+)
 from unstract.sdk1.file_storage.impl import FileStorage
 from unstract.sdk1.file_storage.provider import FileStorageProvider
 from unstract.sdk1.platform import PlatformHelper as ToolAdapter
@@ -123,10 +127,11 @@ def is_document_indexed(
             else:
                 self.tool.stream_log(f"No nodes found for {doc_id}")
         except Exception as e:
-            logger.warning(
-                f"Error querying {self.instance_identifiers.vector_db_instance_id}:"
-                f" {str(e)}, proceeding to index",
-                exc_info=True,
+            if not is_vector_store_not_found(e):
+                raise
+            logger.info(
+                f"No index yet for {self.instance_identifiers.vector_db_instance_id}:"
+                f" {e}, proceeding to index"
             )
 
         if doc_id_found and not self.processing_options.reindex: