[wip] Optimize test durations

docs/changes/newsfragments/xxxx.breaking

@@ -0,0 +1,10 @@
+The ``InterDependencies_`` of a ``DataSet`` or ``DataSetInMem`` are now frozen once the
+dataset has been started, also after the measurement has completed and when loading a
+started dataset, since the parameters of a dataset can not change at that point.
+``dataset.description.interdeps`` is therefore a ``FrozenInterDependencies_`` for such
+datasets and modifying it in place raises a ``TypeError``. Methods that return a new
+object such as ``extend`` and ``remove`` continue to work, and ``FrozenInterDependencies_``
+now compares equal to an ``InterDependencies_`` with the same parameters.
+The ``override`` argument of ``DataSet.set_interdependencies`` is deprecated, since the
+interdependencies of a started dataset are fixed and ``override`` never updated the run
+description stored in the database.

docs/changes/newsfragments/xxxx.improved

@@ -0,0 +1,6 @@
+Loading data from a ``DataSet`` is now faster. Array and complex valued data is read
+several times faster since the header of the stored arrays is only parsed once per
+parameter rather than once per stored value. Repeated calls to ``get_parameter_data``
+on a started ``DataSet``, e.g. when the cache is updated during a measurement, are also
+faster as the run description held by the dataset is used rather than reading and
+parsing it from the database on every call.

docs/changes/newsfragments/xxxx.underthehood

@@ -0,0 +1,4 @@
+The test suite now runs considerably faster. Tests are distributed between pytest-xdist
+workers using ``worksteal`` rather than ``loadfile``, tests using the simulated Keysight
+B220x and B1500 instruments no longer first try to connect to a real instrument, and a
+number of unneeded sleeps and waits have been removed or shortened.

pyproject.toml

@@ -228,7 +228,7 @@ executionEnvironments = [
 [tool.pytest.ini_options]
 minversion = "7.2"
 testpaths = "tests"
-addopts = "-n auto --dist=loadfile --cov-config=pyproject.toml"
+addopts = "-n auto --dist=worksteal --cov-config=pyproject.toml"
 asyncio_default_fixture_loop_scope = "function"
 markers = "serial"
 

src/qcodes/dataset/_results_backend.py

@@ -166,15 +166,47 @@ def read_parameter_data(
         callback: Callable[[float], None] | None,
     ) -> ParameterData:
         """Read parameter data for the given parameters from the backend."""
-        return get_parameter_data(
-            self.results_conn,
-            self._dataset.table_name,
-            valid_param_names,
-            start,
-            end,
-            callback,
+        if self._dataset.pristine:
+            # The parameters of a pristine dataset may still change and their
+            # columns do not exist yet, so read with the description stored
+            # in the database, which finds no data.
+            return get_parameter_data(
+                self.results_conn,
+                self._dataset.table_name,
+                valid_param_names,
+                start,
+                end,
+                callback,
+            )
+        return self._read_parameter_data_with_own_rundescriber(
+            valid_param_names, start, end, callback
         )
 
+    def _read_parameter_data_with_own_rundescriber(
+        self,
+        valid_param_names: list[str],
+        start: int | None,
+        end: int | None,
+        callback: Callable[[float], None] | None,
+    ) -> ParameterData:
+        # The run description can not change once a dataset has been started,
+        # so use the one held by the dataset (with frozen interdependencies
+        # that cache dependency lookups) rather than reading and parsing it
+        # from the database on every call.
+        ds = self._dataset
+        output: ParameterData = {}
+        for param_name in valid_param_names:
+            output[param_name] = get_shaped_parameter_data_for_one_paramtree(
+                self.results_conn,
+                ds.table_name,
+                ds._rundescriber,
+                param_name,
+                start,
+                end,
+                callback,
+            )
+        return output
+
 
 class MainDatabaseResultsBackend(ResultsBackend):
     """Store results in the main QCoDeS database (the default).
@@ -268,26 +300,13 @@ def read_parameter_data(
         end: int | None,
         callback: Callable[[float], None] | None,
     ) -> ParameterData:
-        ds = self._dataset
         if self._conn is None:
             # Not started yet / no separate file: defer to the default reader,
             # which will find no results table and return empty data.
             return super().read_parameter_data(valid_param_names, start, end, callback)
-        # When results live in a separate DB, bypass get_parameter_data (which
-        # looks up the rundescriber from the main DB) and call the lower-level
-        # function directly with the rundescriber we already hold.
-        output: ParameterData = {}
-        for param_name in valid_param_names:
-            output[param_name] = get_shaped_parameter_data_for_one_paramtree(
-                self._conn,
-                ds.table_name,
-                ds._rundescriber,
-                param_name,
-                start,
-                end,
-                callback,
-            )
-        return output
+        return self._read_parameter_data_with_own_rundescriber(
+            valid_param_names, start, end, callback
+        )
 
     def close(self) -> None:
         if self._conn is not None:

src/qcodes/dataset/data_set.py

@@ -6,15 +6,17 @@
 import tempfile
 import time
 import uuid
+import warnings
 from dataclasses import dataclass
 from pathlib import Path
 from queue import Queue
 from threading import Thread
-from typing import TYPE_CHECKING, Any, Literal
+from typing import TYPE_CHECKING, Any, Literal, overload
 
 import numpy
 import numpy.typing as npt
 from tqdm.auto import trange
+from typing_extensions import deprecated
 
 import qcodes
 from qcodes.dataset.data_set_protocol import (
@@ -27,7 +29,10 @@
     ValuesType,
 )
 from qcodes.dataset.descriptions.dependencies import InterDependencies_
-from qcodes.dataset.descriptions.rundescriber import RunDescriber
+from qcodes.dataset.descriptions.rundescriber import (
+    RunDescriber,
+    _freeze_rundescriber,
+)
 from qcodes.dataset.descriptions.versioning.converters import new_to_old, old_to_new
 from qcodes.dataset.descriptions.versioning.v0 import InterDependencies
 from qcodes.dataset.experiment_settings import get_default_experiment_id
@@ -81,6 +86,7 @@
 )
 from qcodes.utils import (
     NumpyJSONEncoder,
+    QCoDeSDeprecationWarning,
 )
 
 from ._raw_data_storage import (
@@ -120,6 +126,12 @@
 
 log = logging.getLogger(__name__)
 
+_SET_INTERDEPENDENCIES_OVERRIDE_DEPRECATION = (
+    "The `override` argument of `DataSet.set_interdependencies` is deprecated "
+    "and will be removed in a future release. The interdependencies of a "
+    "dataset can not be changed once it has been started."
+)
+
 
 # TODO: storing parameters in separate table as an extension (dropping
 # the column parameters would be much nicer
@@ -314,6 +326,8 @@ def __init__(
             self._rundescriber = run_desc
             self._metadata = get_metadata_from_run_id(self.conn, self.run_id)
             self._started = self.run_timestamp_raw is not None
+            if self._started:
+                self._rundescriber = _freeze_rundescriber(self._rundescriber)
             self._parent_dataset_links = str_to_links(
                 get_parent_dataset_links(self.conn, self.run_id)
             )
@@ -629,18 +643,65 @@ def toggle_debug(self) -> None:
         self.conn.close()
         self.conn = connect(path_to_db, self._debug)
 
+    @overload
     def set_interdependencies(
         self,
         interdeps: InterDependencies_,
         shapes: Shapes | None = None,
-        override: bool = False,
+    ) -> None: ...
+
+    @overload
+    @deprecated(_SET_INTERDEPENDENCIES_OVERRIDE_DEPRECATION)
+    def set_interdependencies(
+        self,
+        interdeps: InterDependencies_,
+        shapes: Shapes | None = None,
+        *,
+        override: bool,
+    ) -> None: ...
+
+    @overload
+    @deprecated(_SET_INTERDEPENDENCIES_OVERRIDE_DEPRECATION)
+    def set_interdependencies(
+        self,
+        interdeps: InterDependencies_,
+        shapes: Shapes | None,
+        override: bool,
+    ) -> None: ...
+
+    def set_interdependencies(
+        self,
+        interdeps: InterDependencies_,
+        shapes: Shapes | None = None,
+        override: bool | None = None,
     ) -> None:
         """
         Set the interdependencies object (which holds all added
         parameters and their relationships) of this dataset and
         optionally the shapes object that holds information about
         the shape of the data to be measured.
+
+        Args:
+            interdeps: The interdependencies of the parameters of the dataset.
+            shapes: The expected shapes of the data of the parameters.
+            override: Deprecated. Allows replacing the interdependencies of a
+                dataset that has already been started. The interdependencies
+                of a started dataset are fixed. Replacing them only changes
+                this object (and data exported from it) but not the run
+                description stored in the database, so the two can end up
+                inconsistent.
+
+        Raises:
+            TypeError: If interdeps is not an InterDependencies_ instance.
+            RuntimeError: If the dataset has already been started.
+
         """
+        if override is not None:
+            warnings.warn(
+                _SET_INTERDEPENDENCIES_OVERRIDE_DEPRECATION,
+                QCoDeSDeprecationWarning,
+                stacklevel=2,
+            )
         if not isinstance(interdeps, InterDependencies_):
             raise TypeError(
                 f"Wrong input type. Expected InterDepencies_, got {type(interdeps)}"
@@ -650,6 +711,8 @@ def set_interdependencies(
             mssg = "Can not set interdependencies on a DataSet that has been started."
             raise RuntimeError(mssg)
         self._rundescriber = RunDescriber(interdeps, shapes=shapes)
+        if not self.pristine:
+            self._rundescriber = _freeze_rundescriber(self._rundescriber)
 
     def add_metadata(self, tag: str, metadata: Any) -> None:
         """
@@ -748,6 +811,7 @@ def _perform_start_actions(self, start_bg_writer: bool) -> None:
         """
         Perform the actions that must take place once the run has been started
         """
+        self._rundescriber = _freeze_rundescriber(self._rundescriber)
         paramspecs = new_to_old(self._rundescriber.interdeps).paramspecs
 
         # Register all parameters in a single transaction (each add_parameter

src/qcodes/dataset/data_set_in_memory.py

@@ -19,7 +19,10 @@
     ParameterData,
 )
 from qcodes.dataset.descriptions.dependencies import InterDependencies_
-from qcodes.dataset.descriptions.rundescriber import RunDescriber
+from qcodes.dataset.descriptions.rundescriber import (
+    RunDescriber,
+    _freeze_rundescriber,
+)
 from qcodes.dataset.descriptions.versioning.converters import new_to_old
 from qcodes.dataset.export_config import DataExportType
 from qcodes.dataset.guids import generate_guid
@@ -115,6 +118,8 @@ def __init__(
             rundescriber = RunDescriber(interdeps, shapes=None)
 
         self._rundescriber = rundescriber
+        if not self.pristine:
+            self._rundescriber = _freeze_rundescriber(self._rundescriber)
         if parent_dataset_links is not None:
             self._parent_dataset_links = list(parent_dataset_links)
         else:
@@ -752,7 +757,6 @@ def _set_interdependencies(
         self,
         interdeps: InterDependencies_,
         shapes: Shapes | None = None,
-        override: bool = False,
     ) -> None:
         """
         Set the interdependencies object (which holds all added
@@ -765,7 +769,7 @@ def _set_interdependencies(
                 f"Wrong input type. Expected InterDepencies_, got {type(interdeps)}"
             )
 
-        if not self.pristine and not override:
+        if not self.pristine:
             mssg = "Can not set interdependencies on a DataSet that has been started."
             raise RuntimeError(mssg)
         self._rundescriber = RunDescriber(interdeps, shapes=shapes)
@@ -786,6 +790,7 @@ def _perform_start_actions(self) -> None:
         """
         Perform the actions that must take place once the run has been started
         """
+        self._rundescriber = _freeze_rundescriber(self._rundescriber)
 
         with contextlib.closing(
             conn_from_dbpath_or_conn(conn=None, path_to_db=self._path_to_db)

src/qcodes/dataset/descriptions/dependencies.py

@@ -629,7 +629,8 @@ class FrozenInterDependencies_(InterDependencies_):  # noqa: PLW1641
     # todo: not clear if this should implement __hash__.
     """
     A frozen version of InterDependencies_ that is immutable and caches
-    expensive lookups. This is used exclusively while running a measurement
+    expensive lookups. Datasets freeze their interdependencies once they are
+    started, since the parameters of a dataset can not change after that point,
     to minimize the overhead of dependency lookups for each data operation.
 
     Args:
@@ -666,7 +667,11 @@ def add_paramspecs(self, paramspecs: Sequence[ParamSpecBase]) -> None:
         raise TypeError("FrozenInterDependencies_ is immutable")
 
     def remove(self, paramspec: ParamSpecBase) -> InterDependencies_:
-        raise TypeError("FrozenInterDependencies_ is immutable")
+        """
+        Create a new :class:`InterDependencies_` object that is similar
+        to this instance, but has the given parameter removed.
+        """
+        return self.to_interdependencies().remove(paramspec)
 
     def extend(
         self,
@@ -767,7 +772,7 @@ def __repr__(self) -> str:
         return rep
 
     def __eq__(self, other: object) -> bool:
-        if not isinstance(other, FrozenInterDependencies_):
+        if not isinstance(other, InterDependencies_):
             return False
         return nx.utils.graphs_equal(self.graph, other.graph)
 

src/qcodes/dataset/descriptions/rundescriber.py

@@ -2,7 +2,10 @@
 
 from typing import TYPE_CHECKING, cast
 
-from qcodes.dataset.descriptions.dependencies import InterDependencies_
+from qcodes.dataset.descriptions.dependencies import (
+    FrozenInterDependencies_,
+    InterDependencies_,
+)
 
 from .versioning.converters import new_to_old, old_to_new
 from .versioning.v0 import InterDependencies
@@ -112,3 +115,17 @@ def __eq__(self, other: object) -> bool:
 
     def __repr__(self) -> str:
         return f"RunDescriber({self.interdeps}, Shapes: {self._shapes})"
+
+
+def _freeze_rundescriber(rundescriber: RunDescriber) -> RunDescriber:
+    """
+    Return a :class:`RunDescriber` with frozen interdependencies. The
+    interdependencies of a dataset must not change once it has been started,
+    and freezing them allows caching the dependency lookups that are needed
+    when adding and reading data.
+    """
+    if isinstance(rundescriber.interdeps, FrozenInterDependencies_):
+        return rundescriber
+    return RunDescriber(
+        FrozenInterDependencies_(rundescriber.interdeps), shapes=rundescriber.shapes
+    )

src/qcodes/dataset/measurements.py

@@ -768,28 +768,6 @@ def __exit__(
                     self._span.record_exception(exception_value)
                 self.ds.add_metadata("measurement_exception", exception_string)
 
-            # for now we set the interdependencies back to the
-            # not frozen state, so that further modifications are possible
-            # this is not recommended but we want to minimize the changes for now
-
-            if isinstance(self.ds.description.interdeps, FrozenInterDependencies_):
-                intedeps = self.ds.description.interdeps.to_interdependencies()
-            else:
-                intedeps = self.ds.description.interdeps
-
-            if isinstance(self.ds, DataSet):
-                self.ds.set_interdependencies(
-                    shapes=self.ds.description.shapes,
-                    interdeps=intedeps,
-                    override=True,
-                )
-            elif isinstance(self.ds, DataSetInMem):
-                self.ds._set_interdependencies(
-                    shapes=self.ds.description.shapes,
-                    interdeps=intedeps,
-                    override=True,
-                )
-
             # and finally mark the dataset as closed, thus
             # finishing the measurement
             # Note that the completion of a dataset entails waiting for the