From f361e699e1eb8b26fa68e068dbc23c2f8387188f Mon Sep 17 00:00:00 2001 From: "Jens H. Nielsen" Date: Sat, 17 Oct 2026 09:05:16 +0200 Subject: [PATCH 1/5] Speed up test suite by removing unneeded waits Distribute tests with worksteal rather than loadfile so that a single slow file no longer determines the wall time of the run. Use simulated Keysight B220x/B1500 instruments directly instead of first probing a placeholder VISA address, simulate faster heating in the Lakeshore mocks, poll subscriber state more often and drop sleeps that are not needed. --- pyproject.toml | 2 +- .../test_measurement_context_manager.py | 2 +- tests/dataset/test_nested_measurements.py | 2 +- tests/dataset/test_subscribing.py | 2 +- .../b1500_driver_tests/conftest.py | 13 +++------- tests/drivers/test_keysight_b220x.py | 17 ++++--------- tests/drivers/test_lakeshore_335.py | 9 ++++--- tests/drivers/test_lakeshore_336.py | 9 ++++--- tests/drivers/test_lakeshore_372.py | 25 ++++++++++++++++--- tests/parameter/test_parameter_cache.py | 4 +-- tests/test_interactive_widget.py | 8 ------ tests/test_monitor.py | 1 + 12 files changed, 49 insertions(+), 45 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index e9b1bbca5ad7..e899619cba61 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -228,7 +228,7 @@ executionEnvironments = [ [tool.pytest.ini_options] minversion = "7.2" testpaths = "tests" -addopts = "-n auto --dist=loadfile --cov-config=pyproject.toml" +addopts = "-n auto --dist=worksteal --cov-config=pyproject.toml" asyncio_default_fixture_loop_scope = "function" markers = "serial" diff --git a/tests/dataset/measurement/test_measurement_context_manager.py b/tests/dataset/measurement/test_measurement_context_manager.py index bac839bb6287..836039d03e3d 100644 --- a/tests/dataset/measurement/test_measurement_context_manager.py +++ b/tests/dataset/measurement/test_measurement_context_manager.py @@ -579,7 +579,7 @@ def collect_values_larger_than_7(results, length, state): # wrap the assertions. This is going to ensure that some time is # given to the Subscriber threads to finish exhausting the queue. @retry_until_does_not_throw( - exception_class_to_expect=AssertionError, delay=0.5, tries=20 + exception_class_to_expect=AssertionError, delay=0.05, tries=200 ) def assert_states_updated_from_callbacks( num: int, expected_values: list[int] diff --git a/tests/dataset/test_nested_measurements.py b/tests/dataset/test_nested_measurements.py index dc832f7cf191..e4df7073b123 100644 --- a/tests/dataset/test_nested_measurements.py +++ b/tests/dataset/test_nested_measurements.py @@ -221,7 +221,7 @@ def test_subscription_on_dual_datasets(experiment, basic_subscriber) -> None: expected_state_2[x + 1] = [(x, y2)] @retry_until_does_not_throw( - exception_class_to_expect=AssertionError, delay=0.5, tries=10 + exception_class_to_expect=AssertionError, delay=0.05, tries=100 ) def assert_expected_state(): assert dataset1.subscribers[sub_id_1].state == expected_state_1 diff --git a/tests/dataset/test_subscribing.py b/tests/dataset/test_subscribing.py index 07750119070d..932fb24ada71 100644 --- a/tests/dataset/test_subscribing.py +++ b/tests/dataset/test_subscribing.py @@ -142,7 +142,7 @@ def test_basic_subscription(dataset, basic_subscriber) -> None: expected_state[x + 1] = [(x, y)] @retry_until_does_not_throw( - exception_class_to_expect=AssertionError, delay=0.5, tries=10 + exception_class_to_expect=AssertionError, delay=0.05, tries=100 ) def assert_expected_state(): assert dataset.subscribers[sub_id].state == expected_state diff --git a/tests/drivers/keysight_b1500/b1500_driver_tests/conftest.py b/tests/drivers/keysight_b1500/b1500_driver_tests/conftest.py index 8db81b04ee5d..7f5019b079c8 100644 --- a/tests/drivers/keysight_b1500/b1500_driver_tests/conftest.py +++ b/tests/drivers/keysight_b1500/b1500_driver_tests/conftest.py @@ -3,7 +3,6 @@ import pytest from pytest import FixtureRequest -from pyvisa import VisaIOError from qcodes.instrument_drivers.Keysight.keysightb1500.KeysightB1500_base import ( KeysightB1500, @@ -17,15 +16,9 @@ def _make_b1500(request: FixtureRequest) -> "Generator[KeysightB1500, None, None]": request.addfinalizer(KeysightB1500.close_all) - try: - resource_name = "insert_Keysight_B2200_VISA_resource_name_here" - instance = KeysightB1500("SPA", address=resource_name) - except (ValueError, VisaIOError): - # Either there is no VISA lib installed or there was no real - # instrument found at the specified address => use simulated instrument - instance = KeysightB1500( - "SPA", address="GPIB::1::INSTR", pyvisa_sim_file="keysight_b1500.yaml" - ) + instance = KeysightB1500( + "SPA", address="GPIB::1::INSTR", pyvisa_sim_file="keysight_b1500.yaml" + ) instance.get_status() instance.reset() diff --git a/tests/drivers/test_keysight_b220x.py b/tests/drivers/test_keysight_b220x.py index 8df739cfcff3..5047ba3a3f3c 100644 --- a/tests/drivers/test_keysight_b220x.py +++ b/tests/drivers/test_keysight_b220x.py @@ -2,7 +2,6 @@ from typing import TYPE_CHECKING import pytest -from pyvisa.errors import VisaIOError from qcodes.instrument_drivers.Keysight.keysight_b220x import KeysightB220X @@ -12,17 +11,11 @@ @pytest.fixture def uut() -> "Generator[KeysightB220X, None, None]": - try: - resource_name = "insert_Keysight_B2200_VISA_resource_name_here" - instance = KeysightB220X("switch_matrix", address=resource_name) - except (ValueError, VisaIOError): - # Either there is no VISA lib installed or there was no real - # instrument found at the specified address => use simulated instrument - instance = KeysightB220X( - "switch_matrix", - address="GPIB::1::INSTR", - pyvisa_sim_file="keysight_b220x.yaml", - ) + instance = KeysightB220X( + "switch_matrix", + address="GPIB::1::INSTR", + pyvisa_sim_file="keysight_b220x.yaml", + ) instance.get_status() instance.clear_status() diff --git a/tests/drivers/test_lakeshore_335.py b/tests/drivers/test_lakeshore_335.py index ab3c866780a8..2f94681ecce3 100644 --- a/tests/drivers/test_lakeshore_335.py +++ b/tests/drivers/test_lakeshore_335.py @@ -5,10 +5,12 @@ from qcodes.instrument_drivers.Lakeshore import LakeshoreModel335 from .test_lakeshore_372 import ( + SIMULATED_RAMP_RATE, DictClass, MockVisaInstrument, instrument_fixture, query, + speed_up_outputs, ) log = logging.getLogger(__name__) @@ -63,6 +65,8 @@ def __init__(self, *args, **kwargs) -> None: for i in self.channel_name_command } + speed_up_outputs(self) + # simulate delayed heating self.simulate_heating = False self.start_heating_time = time.perf_counter() @@ -76,9 +80,8 @@ def get_t_when_heating(self): Simply define a fixed setpoint of 4 k for now """ delta = abs(time.perf_counter() - self.start_heating_time) - # make it simple to start with: linear ramp 1K per second - # start at 7K. - return max(4, 7 - delta) + # make it simple to start with: linear ramp starting at 7K. + return max(4, 7 - SIMULATED_RAMP_RATE * delta) @query("KRDG?") def temperature(self, output): diff --git a/tests/drivers/test_lakeshore_336.py b/tests/drivers/test_lakeshore_336.py index 62486b1b47b0..5a03b55f7f62 100644 --- a/tests/drivers/test_lakeshore_336.py +++ b/tests/drivers/test_lakeshore_336.py @@ -7,10 +7,12 @@ from qcodes.instrument_drivers.Lakeshore import LakeshoreModel336 from .test_lakeshore_372 import ( + SIMULATED_RAMP_RATE, DictClass, MockVisaInstrument, instrument_fixture, query, + speed_up_outputs, ) log = logging.getLogger(__name__) @@ -85,6 +87,8 @@ def __init__(self, *args, **kwargs) -> None: for i in self.channel_name_command } + speed_up_outputs(self) + # simulate delayed heating self.simulate_heating = False self.start_heating_time = time.perf_counter() @@ -98,9 +102,8 @@ def get_t_when_heating(self): Simply define a fixed setpoint of 4 k for now """ delta = abs(time.perf_counter() - self.start_heating_time) - # make it simple to start with: linear ramp 1K per second - # start at 7K. - return max(4, 7 - delta) + # make it simple to start with: linear ramp starting at 7K. + return max(4, 7 - SIMULATED_RAMP_RATE * delta) @query("KRDG?") def temperature(self, output): diff --git a/tests/drivers/test_lakeshore_372.py b/tests/drivers/test_lakeshore_372.py index 76ff8497f225..2878d6241197 100644 --- a/tests/drivers/test_lakeshore_372.py +++ b/tests/drivers/test_lakeshore_372.py @@ -11,6 +11,7 @@ from qcodes.instrument import InstrumentBase from qcodes.instrument_drivers.Lakeshore import LakeshoreModel372 from qcodes.instrument_drivers.Lakeshore.lakeshore_base import ( + LakeshoreBaseOutput, LakeshoreBaseSensorChannel, ) from qcodes.logger import get_instrument_logger @@ -23,6 +24,23 @@ VISA_LOGGER = f"{InstrumentBase.__module__}.com.visa" +# The mocks simulate a linear temperature ramp that is much faster than on a +# real instrument to keep the tests that wait for the setpoint short. +SIMULATED_RAMP_RATE = 10 # K/s + + +def speed_up_outputs(instrument: InstrumentBase) -> None: + """ + Scale the waiting parameters of all outputs to match the simulated ramp + rate and remove the setpoint ``post_delay`` that is only needed for real + instruments. + """ + for submodule in instrument.submodules.values(): + if isinstance(submodule, LakeshoreBaseOutput): + submodule.setpoint.post_delay = 0 + submodule.wait_cycle_time(0.01) + submodule.wait_equilibration_time(0.05) + class MockVisaInstrument: """ @@ -186,6 +204,8 @@ def __init__(self, *args, **kwargs) -> None: for i in range(1, 17) } + speed_up_outputs(self) + # simulate delayed heating self.simulate_heating = False self.start_heating_time = time.perf_counter() @@ -199,9 +219,8 @@ def get_t_when_heating(self): Simply define a fixed setpoint of 4 k for now """ delta = abs(time.perf_counter() - self.start_heating_time) - # make it simple to start with: linear ramp 1K per second - # start at 7K. - return max(4, 7 - delta) + # make it simple to start with: linear ramp starting at 7K. + return max(4, 7 - SIMULATED_RAMP_RATE * delta) @query("KRDG?") def temperature(self, output): diff --git a/tests/parameter/test_parameter_cache.py b/tests/parameter/test_parameter_cache.py index ca1220f063ad..7ddf6113db08 100644 --- a/tests/parameter/test_parameter_cache.py +++ b/tests/parameter/test_parameter_cache.py @@ -21,11 +21,11 @@ def test_get_from_cache_does_not_trigger_real_get_if_get_if_invalid_false() -> N not trigger a get of the parameter when parameter has expired due to max_val_age """ - param = BetterGettableParam(name="param", max_val_age=1) + param = BetterGettableParam(name="param", max_val_age=0.1) param.get() assert param._get_count == 1 # let the cache expire - time.sleep(2) + time.sleep(0.2) param.cache.get(get_if_invalid=False) assert param._get_count == 1 diff --git a/tests/test_interactive_widget.py b/tests/test_interactive_widget.py index 283665bcf5a7..ebc8ce3bed15 100644 --- a/tests/test_interactive_widget.py +++ b/tests/test_interactive_widget.py @@ -1,4 +1,3 @@ -import time from unittest.mock import patch # importing ipykernel has the side effect @@ -58,11 +57,9 @@ def test_button_to_text( box = interactive_widget.button_to_text("title", "body") (button,) = box.children button.click() - time.sleep(0.5) # after click text_area, back_button = box.children assert "body" in text_area.value back_button.click() - time.sleep(0.5) # after click assert len(box.children) == 1 @@ -70,7 +67,6 @@ def test_snapshot_button(tab, standalone_parameters_dataset) -> None: ds = standalone_parameters_dataset snapshot_button = interactive_widget._get_snapshot_button(ds, tab) snapshot_button.click() - time.sleep(0.5) # after click # maybe use https://github.com/jupyter-widgets/ipywidgets/issues/2417 assert "snapshot" in tab.get_title(1) @@ -79,7 +75,6 @@ def test_export_button(tab, standalone_parameters_dataset) -> None: ds = standalone_parameters_dataset export_button = interactive_widget._get_export_button(ds, tab) export_button.click() - time.sleep(0.5) # after click @patch("matplotlib.pyplot.show") @@ -87,7 +82,6 @@ def test_plot_button(tab, standalone_parameters_dataset) -> None: ds = standalone_parameters_dataset plot_button = interactive_widget._get_plot_button(ds, tab) plot_button.click() - time.sleep(0.5) # after click @pytest.mark.parametrize( @@ -107,7 +101,6 @@ def test_get_experiment_button( box = get_button_function(ds) snapshot_button = box.children[0] snapshot_button.click() - time.sleep(0.5) # after click assert len(box.children) == 2 @@ -132,7 +125,6 @@ def test_editable_metadata( test_test = "test value" text_area.value = test_test save_button.click() - time.sleep(0.5) # after click # Test if metadata is saved. assert ds.metadata[interactive_widget._META_DATA_KEY] == test_test diff --git a/tests/test_monitor.py b/tests/test_monitor.py index b0772e63b678..e0a5ebdbeb9b 100644 --- a/tests/test_monitor.py +++ b/tests/test_monitor.py @@ -52,6 +52,7 @@ def _make_channel_instr_monitor(channel_instr, request: FixtureRequest): m = monitor.Monitor( channel_instr.A.dummy_start, channel_instr.B.dummy_start, + interval=0.1, use_root_instrument=request.param, ) try: From 5043d4f5dd55dcc3d5934a1a330ae602c531e21d Mon Sep 17 00:00:00 2001 From: "Jens H. Nielsen" Date: Sat, 17 Oct 2026 09:24:26 +0200 Subject: [PATCH 2/5] Speed up reading array and complex data from sqlite Parsing the npy header of every stored value dominated the time it takes to load array and complex data. Cache the parsed header, since all rows of a parameter share it, and fall back to numpy's read_array for anything the fast path does not handle. --- src/qcodes/dataset/sqlite/database.py | 101 +++++++++++---- tests/dataset/test_sqlite_converters.py | 163 ++++++++++++++++++++++++ 2 files changed, 242 insertions(+), 22 deletions(-) create mode 100644 tests/dataset/test_sqlite_converters.py diff --git a/src/qcodes/dataset/sqlite/database.py b/src/qcodes/dataset/sqlite/database.py index cb39c378c09d..7dad29bf3a9d 100644 --- a/src/qcodes/dataset/sqlite/database.py +++ b/src/qcodes/dataset/sqlite/database.py @@ -6,13 +6,14 @@ from __future__ import annotations +import functools import io import math import sqlite3 import sys from contextlib import contextmanager from os.path import expanduser, normpath -from typing import TYPE_CHECKING, Literal +from typing import TYPE_CHECKING, Any, Literal import numpy as np import numpy.typing as npt @@ -52,16 +53,82 @@ def _adapt_array(arr: npt.NDArray) -> sqlite3.Binary: def _convert_array(text: bytes) -> npt.NDArray: - # Using np.lib.format.read_array (counterpart of np.lib.format.write_array) - # npy format version 3.0 is 3 times faster than previous verions (no clean up step - # for python 2 backward compatibility) - return np.lib.format.read_array(io.BytesIO(text), allow_pickle=False) + """ + Convert the ``.npy`` serialization of an array (as written by + :func:`_adapt_array`) back into an array. + + This is equivalent to :func:`numpy.lib.format.read_array` but caches the + parsing of the header, which otherwise dominates the time it takes to + load small and medium sized arrays. All rows of a given array parameter + typically share the same header. Input that cannot be handled by the fast + path is passed on to :func:`numpy.lib.format.read_array`. + """ + header_info = _npy_header_info(text) + if header_info is None: + return np.lib.format.read_array(io.BytesIO(text), allow_pickle=False) + offset, shape, fortran_order, dtype = header_info + count = math.prod(shape) + if len(text) - offset < count * dtype.itemsize: + return np.lib.format.read_array(io.BytesIO(text), allow_pickle=False) + # copy since frombuffer returns a read only view of the bytes + array = np.frombuffer(text, dtype=dtype, count=count, offset=offset).copy() + if fortran_order: + return array.reshape(shape[::-1]).transpose() + return array.reshape(shape) + + +_NPY_MAGIC_PREFIX = b"\x93NUMPY" + + +def _npy_header_info( + text: bytes, +) -> tuple[int, tuple[int, ...], bool, np.dtype[Any]] | None: + """ + Return the offset of the data, shape, fortran order and dtype of an + ``.npy`` serialized array or None if the array should be read with + :func:`numpy.lib.format.read_array`. + """ + if len(text) < 12 or not text.startswith(_NPY_MAGIC_PREFIX): + return None + major_version = text[6] + if major_version == 1: + offset = 10 + int.from_bytes(text[8:10], "little") + elif major_version in (2, 3): + offset = 12 + int.from_bytes(text[8:12], "little") + else: + return None + header_info = _parse_npy_header(text[:offset]) + if header_info is None: + return None + return (offset, *header_info) + + +@functools.lru_cache(maxsize=128) +def _parse_npy_header( + header: bytes, +) -> tuple[tuple[int, ...], bool, np.dtype[Any]] | None: + # numpy's public header readers decode the header as latin1 while + # version 3.0 headers are utf8 encoded. These are only identical for + # ascii, which covers all headers written by QCoDeS. + header_text_start = 10 if header[6] == 1 else 12 + if not header[header_text_start:].isascii(): + return None + fp = io.BytesIO(header) + try: + major_version, _ = np.lib.format.read_magic(fp) + if major_version == 1: + shape, fortran_order, dtype = np.lib.format.read_array_header_1_0(fp) + else: + shape, fortran_order, dtype = np.lib.format.read_array_header_2_0(fp) + except ValueError: + return None + if dtype.hasobject or dtype.itemsize == 0: + return None + return shape, fortran_order, dtype def _convert_complex(text: bytes) -> np.complexfloating: - out = io.BytesIO(text) - out.seek(0) - return np.load(out)[0] + return _convert_array(text)[0] this_session_default_encoding = sys.getdefaultencoding() @@ -89,20 +156,10 @@ def _convert_numeric(value: bytes) -> float | int | str: # Let string casting fail if bytes encoding is invalid return str(value, encoding=this_session_default_encoding) - # If that worked, e.g. did not raise an exception, then we check if the outcome is - # either an infinity or a NaN - # For a single value, math.isfinite is 10 times faster than np.isinfinite (or - # combining np.isnan and np.isinf) - if not math.isfinite(numeric): - return numeric - - # If it is not 'nan' and not 'inf', then we need to see if the value is really an - # integer or with floating point digits - numeric_int = int(numeric) - if numeric != numeric_int: - return numeric - else: - return numeric_int + # is_integer is False for NaN and infinity so these are returned as floats + if numeric.is_integer(): + return int(numeric) + return numeric def _adapt_float(fl: float) -> float | str: diff --git a/tests/dataset/test_sqlite_converters.py b/tests/dataset/test_sqlite_converters.py new file mode 100644 index 000000000000..7cbd541e0789 --- /dev/null +++ b/tests/dataset/test_sqlite_converters.py @@ -0,0 +1,163 @@ +from __future__ import annotations + +import io +import math +from typing import TYPE_CHECKING, Any, Literal + +import numpy as np +import pytest + +from qcodes.dataset.sqlite.database import ( + _adapt_array, + _adapt_complex, + _convert_array, + _convert_complex, + _convert_numeric, +) + +if TYPE_CHECKING: + import numpy.typing as npt + from pytest_mock import MockerFixture + + +def _serialize( + array: npt.NDArray, version: tuple[int, int] | None, allow_pickle: bool = False +) -> bytes: + out = io.BytesIO() + if version is None: + np.save(out, array, allow_pickle=allow_pickle) + else: + np.lib.format.write_array( + out, array, version=version, allow_pickle=allow_pickle + ) + return out.getvalue() + + +def _read_with_numpy(serialized: bytes) -> npt.NDArray: + return np.lib.format.read_array(io.BytesIO(serialized), allow_pickle=False) + + +@pytest.mark.parametrize("version", [(1, 0), (2, 0), (3, 0), None]) +@pytest.mark.parametrize("order", ["C", "F"]) +@pytest.mark.parametrize("shape", [(), (0,), (1,), (7,), (3, 4), (2, 3, 4), (0, 3)]) +@pytest.mark.parametrize( + "dtype", + [ + "f8", + ">f8", + "f4", + "i1", + ">i4", + "u8", + "c8", + "c16", + "?", + "U3", + "S4", + "M8[ns]", + [("a", "f8"), ("b", "i4")], + ], +) +def test_convert_array_matches_numpy( + mocker: MockerFixture, + dtype: Any, + shape: tuple[int, ...], + order: Literal["C", "F"], + version: tuple[int, int] | None, +) -> None: + array = np.asarray(np.zeros(shape, dtype=np.dtype(dtype)), order=order) + if array.dtype.fields is None: + array.flat = np.arange(array.size) % 100 + serialized = _serialize(array, version) + expected = _read_with_numpy(serialized) + + read_array_spy = mocker.spy(np.lib.format, "read_array") + result = _convert_array(serialized) + + read_array_spy.assert_not_called() + assert result.dtype == expected.dtype + assert result.shape == expected.shape + assert result.flags.c_contiguous == expected.flags.c_contiguous + assert result.flags.f_contiguous == expected.flags.f_contiguous + assert result.flags.writeable + assert result.tobytes() == expected.tobytes() + + +def test_convert_array_round_trips_adapted_array() -> None: + array = np.random.default_rng().random((3, 5)) + result = _convert_array(bytes(_adapt_array(array))) + np.testing.assert_array_equal(result, array) + # the returned array must not share memory with the immutable input bytes + result[0, 0] = 1.0 + + +def test_convert_array_non_ascii_header_falls_back_to_numpy( + mocker: MockerFixture, +) -> None: + serialized = _serialize(np.zeros(2, dtype=[("\u00e5", "f8")]), (3, 0)) + read_array_spy = mocker.spy(np.lib.format, "read_array") + + result = _convert_array(serialized) + + read_array_spy.assert_called_once() + assert result.dtype == _read_with_numpy(serialized).dtype + + +def test_convert_array_object_array_raises() -> None: + serialized = _serialize(np.array([1, "a"], dtype=object), (3, 0), True) + with pytest.raises(ValueError, match="Object arrays cannot be loaded"): + _convert_array(serialized) + + +@pytest.mark.parametrize("serialized", [b"", b"not an array", b"\x93NUMP"]) +def test_convert_array_invalid_input_raises(serialized: bytes) -> None: + with pytest.raises(ValueError): + _convert_array(serialized) + + +def test_convert_array_truncated_data_raises() -> None: + serialized = bytes(_adapt_array(np.arange(10.0)))[:-8] + with pytest.raises(ValueError, match="array data"): + _convert_array(serialized) + + +@pytest.mark.parametrize( + "value", + [1 + 2j, np.complex64(3 - 1j), np.complex128(-0.5 + 0.25j), complex("nan+infj")], +) +def test_convert_complex(value: complex | np.complexfloating) -> None: + serialized = bytes(_adapt_complex(value)) + expected = np.load(io.BytesIO(serialized))[0] + + result = _convert_complex(serialized) + + assert type(result) is type(expected) + np.testing.assert_array_equal(result, expected) + + +@pytest.mark.parametrize( + ("value", "expected"), + [ + (b"1.5", 1.5), + (b"-2.25e-3", -2.25e-3), + (b"3", 3), + (b"3.0", 3), + (b"-0.0", 0), + (b"1e20", 10**20), + (b"inf", math.inf), + (b"-Infinity", -math.inf), + (b"some text", "some text"), + ], +) +def test_convert_numeric(value: bytes, expected: float | str) -> None: + result = _convert_numeric(value) + assert type(result) is type(expected) + assert result == expected + + +@pytest.mark.parametrize("value", [b"nan", b"NaN", b"-nan"]) +def test_convert_numeric_nan(value: bytes) -> None: + result = _convert_numeric(value) + assert isinstance(result, float) + assert math.isnan(result) From 80b05adbaf6eb1666aab5c5b48e1b3ee0c892f7f Mon Sep 17 00:00:00 2001 From: "Jens H. Nielsen" Date: Sat, 17 Oct 2026 09:58:17 +0200 Subject: [PATCH 3/5] Freeze interdependencies once a dataset is started The interdependencies of a dataset can not change once it has been started, so freeze them at that point and when loading a started run rather than only during a measurement. Reads now use the dataset's own frozen description instead of parsing it from the database every call. Unfreezing at the end of a measurement was only needed because frozen and non frozen interdependencies never compared equal. Make equality symmetric and let remove on a frozen instance return a new object like extend does. --- src/qcodes/dataset/_results_backend.py | 65 ++++++++++++------- src/qcodes/dataset/data_set.py | 10 ++- src/qcodes/dataset/data_set_in_memory.py | 10 ++- .../dataset/descriptions/dependencies.py | 11 +++- .../dataset/descriptions/rundescriber.py | 19 +++++- src/qcodes/dataset/measurements.py | 22 ------- .../test_measurement_context_manager.py | 12 ++-- tests/dataset/test_dataset_basic.py | 43 +++++++++++- tests/dataset/test_dataset_in_memory.py | 29 ++++++++- tests/dataset/test_dependencies.py | 24 ++++++- 10 files changed, 181 insertions(+), 64 deletions(-) diff --git a/src/qcodes/dataset/_results_backend.py b/src/qcodes/dataset/_results_backend.py index c94f01248849..12cf63c5d57f 100644 --- a/src/qcodes/dataset/_results_backend.py +++ b/src/qcodes/dataset/_results_backend.py @@ -166,15 +166,47 @@ def read_parameter_data( callback: Callable[[float], None] | None, ) -> ParameterData: """Read parameter data for the given parameters from the backend.""" - return get_parameter_data( - self.results_conn, - self._dataset.table_name, - valid_param_names, - start, - end, - callback, + if self._dataset.pristine: + # The parameters of a pristine dataset may still change and their + # columns do not exist yet, so read with the description stored + # in the database, which finds no data. + return get_parameter_data( + self.results_conn, + self._dataset.table_name, + valid_param_names, + start, + end, + callback, + ) + return self._read_parameter_data_with_own_rundescriber( + valid_param_names, start, end, callback ) + def _read_parameter_data_with_own_rundescriber( + self, + valid_param_names: list[str], + start: int | None, + end: int | None, + callback: Callable[[float], None] | None, + ) -> ParameterData: + # The run description can not change once a dataset has been started, + # so use the one held by the dataset (with frozen interdependencies + # that cache dependency lookups) rather than reading and parsing it + # from the database on every call. + ds = self._dataset + output: ParameterData = {} + for param_name in valid_param_names: + output[param_name] = get_shaped_parameter_data_for_one_paramtree( + self.results_conn, + ds.table_name, + ds._rundescriber, + param_name, + start, + end, + callback, + ) + return output + class MainDatabaseResultsBackend(ResultsBackend): """Store results in the main QCoDeS database (the default). @@ -268,26 +300,13 @@ def read_parameter_data( end: int | None, callback: Callable[[float], None] | None, ) -> ParameterData: - ds = self._dataset if self._conn is None: # Not started yet / no separate file: defer to the default reader, # which will find no results table and return empty data. return super().read_parameter_data(valid_param_names, start, end, callback) - # When results live in a separate DB, bypass get_parameter_data (which - # looks up the rundescriber from the main DB) and call the lower-level - # function directly with the rundescriber we already hold. - output: ParameterData = {} - for param_name in valid_param_names: - output[param_name] = get_shaped_parameter_data_for_one_paramtree( - self._conn, - ds.table_name, - ds._rundescriber, - param_name, - start, - end, - callback, - ) - return output + return self._read_parameter_data_with_own_rundescriber( + valid_param_names, start, end, callback + ) def close(self) -> None: if self._conn is not None: diff --git a/src/qcodes/dataset/data_set.py b/src/qcodes/dataset/data_set.py index fe5339339625..6788de0a197f 100644 --- a/src/qcodes/dataset/data_set.py +++ b/src/qcodes/dataset/data_set.py @@ -27,7 +27,10 @@ ValuesType, ) from qcodes.dataset.descriptions.dependencies import InterDependencies_ -from qcodes.dataset.descriptions.rundescriber import RunDescriber +from qcodes.dataset.descriptions.rundescriber import ( + RunDescriber, + _freeze_rundescriber, +) from qcodes.dataset.descriptions.versioning.converters import new_to_old, old_to_new from qcodes.dataset.descriptions.versioning.v0 import InterDependencies from qcodes.dataset.experiment_settings import get_default_experiment_id @@ -314,6 +317,8 @@ def __init__( self._rundescriber = run_desc self._metadata = get_metadata_from_run_id(self.conn, self.run_id) self._started = self.run_timestamp_raw is not None + if self._started: + self._rundescriber = _freeze_rundescriber(self._rundescriber) self._parent_dataset_links = str_to_links( get_parent_dataset_links(self.conn, self.run_id) ) @@ -650,6 +655,8 @@ def set_interdependencies( mssg = "Can not set interdependencies on a DataSet that has been started." raise RuntimeError(mssg) self._rundescriber = RunDescriber(interdeps, shapes=shapes) + if not self.pristine: + self._rundescriber = _freeze_rundescriber(self._rundescriber) def add_metadata(self, tag: str, metadata: Any) -> None: """ @@ -748,6 +755,7 @@ def _perform_start_actions(self, start_bg_writer: bool) -> None: """ Perform the actions that must take place once the run has been started """ + self._rundescriber = _freeze_rundescriber(self._rundescriber) paramspecs = new_to_old(self._rundescriber.interdeps).paramspecs # Register all parameters in a single transaction (each add_parameter diff --git a/src/qcodes/dataset/data_set_in_memory.py b/src/qcodes/dataset/data_set_in_memory.py index 241b789ba3bb..82de47d2fb13 100644 --- a/src/qcodes/dataset/data_set_in_memory.py +++ b/src/qcodes/dataset/data_set_in_memory.py @@ -19,7 +19,10 @@ ParameterData, ) from qcodes.dataset.descriptions.dependencies import InterDependencies_ -from qcodes.dataset.descriptions.rundescriber import RunDescriber +from qcodes.dataset.descriptions.rundescriber import ( + RunDescriber, + _freeze_rundescriber, +) from qcodes.dataset.descriptions.versioning.converters import new_to_old from qcodes.dataset.export_config import DataExportType from qcodes.dataset.guids import generate_guid @@ -115,6 +118,8 @@ def __init__( rundescriber = RunDescriber(interdeps, shapes=None) self._rundescriber = rundescriber + if not self.pristine: + self._rundescriber = _freeze_rundescriber(self._rundescriber) if parent_dataset_links is not None: self._parent_dataset_links = list(parent_dataset_links) else: @@ -769,6 +774,8 @@ def _set_interdependencies( mssg = "Can not set interdependencies on a DataSet that has been started." raise RuntimeError(mssg) self._rundescriber = RunDescriber(interdeps, shapes=shapes) + if not self.pristine: + self._rundescriber = _freeze_rundescriber(self._rundescriber) def _get_paramspecs(self) -> SPECS: old_interdeps = new_to_old(self.description.interdeps) @@ -786,6 +793,7 @@ def _perform_start_actions(self) -> None: """ Perform the actions that must take place once the run has been started """ + self._rundescriber = _freeze_rundescriber(self._rundescriber) with contextlib.closing( conn_from_dbpath_or_conn(conn=None, path_to_db=self._path_to_db) diff --git a/src/qcodes/dataset/descriptions/dependencies.py b/src/qcodes/dataset/descriptions/dependencies.py index 254067252ae9..1a6f8a4e6041 100644 --- a/src/qcodes/dataset/descriptions/dependencies.py +++ b/src/qcodes/dataset/descriptions/dependencies.py @@ -629,7 +629,8 @@ class FrozenInterDependencies_(InterDependencies_): # noqa: PLW1641 # todo: not clear if this should implement __hash__. """ A frozen version of InterDependencies_ that is immutable and caches - expensive lookups. This is used exclusively while running a measurement + expensive lookups. Datasets freeze their interdependencies once they are + started, since the parameters of a dataset can not change after that point, to minimize the overhead of dependency lookups for each data operation. Args: @@ -666,7 +667,11 @@ def add_paramspecs(self, paramspecs: Sequence[ParamSpecBase]) -> None: raise TypeError("FrozenInterDependencies_ is immutable") def remove(self, paramspec: ParamSpecBase) -> InterDependencies_: - raise TypeError("FrozenInterDependencies_ is immutable") + """ + Create a new :class:`InterDependencies_` object that is similar + to this instance, but has the given parameter removed. + """ + return self.to_interdependencies().remove(paramspec) def extend( self, @@ -767,7 +772,7 @@ def __repr__(self) -> str: return rep def __eq__(self, other: object) -> bool: - if not isinstance(other, FrozenInterDependencies_): + if not isinstance(other, InterDependencies_): return False return nx.utils.graphs_equal(self.graph, other.graph) diff --git a/src/qcodes/dataset/descriptions/rundescriber.py b/src/qcodes/dataset/descriptions/rundescriber.py index 81a7837208d3..1d7b5b441fee 100644 --- a/src/qcodes/dataset/descriptions/rundescriber.py +++ b/src/qcodes/dataset/descriptions/rundescriber.py @@ -2,7 +2,10 @@ from typing import TYPE_CHECKING, cast -from qcodes.dataset.descriptions.dependencies import InterDependencies_ +from qcodes.dataset.descriptions.dependencies import ( + FrozenInterDependencies_, + InterDependencies_, +) from .versioning.converters import new_to_old, old_to_new from .versioning.v0 import InterDependencies @@ -112,3 +115,17 @@ def __eq__(self, other: object) -> bool: def __repr__(self) -> str: return f"RunDescriber({self.interdeps}, Shapes: {self._shapes})" + + +def _freeze_rundescriber(rundescriber: RunDescriber) -> RunDescriber: + """ + Return a :class:`RunDescriber` with frozen interdependencies. The + interdependencies of a dataset must not change once it has been started, + and freezing them allows caching the dependency lookups that are needed + when adding and reading data. + """ + if isinstance(rundescriber.interdeps, FrozenInterDependencies_): + return rundescriber + return RunDescriber( + FrozenInterDependencies_(rundescriber.interdeps), shapes=rundescriber.shapes + ) diff --git a/src/qcodes/dataset/measurements.py b/src/qcodes/dataset/measurements.py index 53e687107b54..c5092950bb6c 100644 --- a/src/qcodes/dataset/measurements.py +++ b/src/qcodes/dataset/measurements.py @@ -768,28 +768,6 @@ def __exit__( self._span.record_exception(exception_value) self.ds.add_metadata("measurement_exception", exception_string) - # for now we set the interdependencies back to the - # not frozen state, so that further modifications are possible - # this is not recommended but we want to minimize the changes for now - - if isinstance(self.ds.description.interdeps, FrozenInterDependencies_): - intedeps = self.ds.description.interdeps.to_interdependencies() - else: - intedeps = self.ds.description.interdeps - - if isinstance(self.ds, DataSet): - self.ds.set_interdependencies( - shapes=self.ds.description.shapes, - interdeps=intedeps, - override=True, - ) - elif isinstance(self.ds, DataSetInMem): - self.ds._set_interdependencies( - shapes=self.ds.description.shapes, - interdeps=intedeps, - override=True, - ) - # and finally mark the dataset as closed, thus # finishing the measurement # Note that the completion of a dataset entails waiting for the diff --git a/tests/dataset/measurement/test_measurement_context_manager.py b/tests/dataset/measurement/test_measurement_context_manager.py index 836039d03e3d..931c519be4fd 100644 --- a/tests/dataset/measurement/test_measurement_context_manager.py +++ b/tests/dataset/measurement/test_measurement_context_manager.py @@ -21,10 +21,7 @@ import qcodes as qc import qcodes.validators as vals from qcodes.dataset.data_set import DataSet, load_by_id -from qcodes.dataset.descriptions.dependencies import ( - FrozenInterDependencies_, - InterDependencies_, -) +from qcodes.dataset.descriptions.dependencies import FrozenInterDependencies_ from qcodes.dataset.experiment_container import Experiment, new_experiment from qcodes.dataset.export_config import DataExportType from qcodes.dataset.measurements import Measurement @@ -743,13 +740,12 @@ def test_datasaver_scalars( ds = datasaver.dataset assert isinstance(ds, DataSet) - assert isinstance(ds.description.interdeps, InterDependencies_) - assert not isinstance(ds.description.interdeps, FrozenInterDependencies_) + assert isinstance(ds.description.interdeps, FrozenInterDependencies_) loaded_ds = load_by_id(ds.run_id) - assert isinstance(loaded_ds.description.interdeps, InterDependencies_) - assert not isinstance(loaded_ds.description.interdeps, FrozenInterDependencies_) + assert isinstance(loaded_ds.description.interdeps, FrozenInterDependencies_) + assert loaded_ds.description == ds.description # More assertions of setpoints, labels and units in the DB! diff --git a/tests/dataset/test_dataset_basic.py b/tests/dataset/test_dataset_basic.py index 1b35782d8ca0..1116bba91350 100644 --- a/tests/dataset/test_dataset_basic.py +++ b/tests/dataset/test_dataset_basic.py @@ -23,7 +23,10 @@ ) from qcodes.dataset.data_set import DataSet from qcodes.dataset.data_set_protocol import CompletedError -from qcodes.dataset.descriptions.dependencies import InterDependencies_ +from qcodes.dataset.descriptions.dependencies import ( + FrozenInterDependencies_, + InterDependencies_, +) from qcodes.dataset.descriptions.rundescriber import RunDescriber from qcodes.dataset.guids import parse_guid from qcodes.dataset.sqlite.connection import atomic, path_to_dbfile @@ -218,6 +221,44 @@ def test_mark_completed_twice(start_bg_writer) -> None: ds.mark_completed() +@pytest.mark.parametrize("start_bg_writer", (True, False)) +@pytest.mark.usefixtures("experiment") +def test_interdependencies_frozen_once_started(start_bg_writer) -> None: + x = ParamSpecBase("x", "numeric") + y = ParamSpecBase("y", "numeric") + idps = InterDependencies_(dependencies={y: (x,)}) + ds = DataSet() + ds.set_interdependencies(idps) + assert not isinstance(ds.description.interdeps, FrozenInterDependencies_) + + ds.mark_started(start_bg_writer=start_bg_writer) + assert isinstance(ds.description.interdeps, FrozenInterDependencies_) + assert ds.description.interdeps == idps + + ds.add_results([{"x": 1, "y": 2}]) + ds.mark_completed() + assert isinstance(ds.description.interdeps, FrozenInterDependencies_) + + loaded = load_by_id(ds.run_id) + assert isinstance(loaded.description.interdeps, FrozenInterDependencies_) + assert loaded.description == ds.description + assert loaded.the_same_dataset_as(ds) + + ds.set_interdependencies(idps, override=True) + assert isinstance(ds.description.interdeps, FrozenInterDependencies_) + + +@pytest.mark.usefixtures("experiment") +def test_interdependencies_of_pristine_loaded_dataset_not_frozen() -> None: + x = ParamSpecBase("x", "numeric") + ds = DataSet(specs=InterDependencies_(standalones=(x,))) + + loaded = load_by_id(ds.run_id) + + assert loaded.pristine + assert not isinstance(loaded.description.interdeps, FrozenInterDependencies_) + + @pytest.mark.usefixtures("experiment") def test_timestamps_are_none() -> None: ds = DataSet() diff --git a/tests/dataset/test_dataset_in_memory.py b/tests/dataset/test_dataset_in_memory.py index df00b01c0cb3..446cbc36ee28 100644 --- a/tests/dataset/test_dataset_in_memory.py +++ b/tests/dataset/test_dataset_in_memory.py @@ -16,7 +16,10 @@ from qcodes.dataset import Measurement, load_by_id, load_by_run_spec from qcodes.dataset.data_set_in_memory import DataSetInMem, load_from_file from qcodes.dataset.data_set_protocol import DataSetType -from qcodes.dataset.descriptions.dependencies import InterDependencies_ +from qcodes.dataset.descriptions.dependencies import ( + FrozenInterDependencies_, + InterDependencies_, +) from qcodes.dataset.sqlite.connection import AtomicConnection, atomic_transaction from qcodes.parameters import ManualParameter, Parameter, ParamSpecBase from qcodes.station import Station @@ -459,6 +462,30 @@ def test_load_from_db(meas_with_registered_param, DMM, DAC, tmp_path) -> None: compare_datasets(ds, loaded_ds) +def test_interdependencies_frozen_after_measurement_and_load( + meas_with_registered_param, DMM, DAC, tmp_path +) -> None: + with meas_with_registered_param.run( + dataset_class=DataSetType.DataSetInMem + ) as datasaver: + datasaver.add_result((DAC.ch1, 1.0), (DMM.v1, 2.0)) + + ds = datasaver.dataset + assert isinstance(ds, DataSetInMem) + assert isinstance(ds.description.interdeps, FrozenInterDependencies_) + ds.export(export_type="netcdf", path=tmp_path) + + loaded_from_db = load_by_id(ds.run_id) + assert isinstance(loaded_from_db, DataSetInMem) + assert isinstance(loaded_from_db.description.interdeps, FrozenInterDependencies_) + assert loaded_from_db.description == ds.description + + export_path = ds.export_info.export_paths["nc"] + loaded_from_file = load_from_file(export_path) + assert isinstance(loaded_from_file.description.interdeps, FrozenInterDependencies_) + assert loaded_from_file.description == ds.description + + def test_load_from_file(meas_with_registered_param, DMM, DAC, tmp_path) -> None: qcodes.config["dataset"]["export_prefix"] = "my_export_prefix" qcodes.config["dataset"]["export_type"] = "netcdf" diff --git a/tests/dataset/test_dependencies.py b/tests/dataset/test_dependencies.py index d43699c8c9d3..9cc2b7416a1d 100644 --- a/tests/dataset/test_dependencies.py +++ b/tests/dataset/test_dependencies.py @@ -499,12 +499,16 @@ def test_frozen_interdependencies(some_paramspecbases) -> None: with pytest.raises(TypeError, match="FrozenInterDependencies_ is immutable"): frozen.add_standalones((ps4,)) - with pytest.raises(TypeError, match="FrozenInterDependencies_ is immutable"): - frozen.remove(ps1) - with pytest.raises(TypeError, match="FrozenInterDependencies_ is immutable"): frozen.add_paramspecs((ps1,)) + # Test remove returns a new InterDependencies_ (mutable) + removed = frozen.remove(ps1) + assert isinstance(removed, InterDependencies_) + assert not isinstance(removed, FrozenInterDependencies_) + assert ps1 not in removed.paramspecs + assert ps1 in frozen.paramspecs + # Test extend returns InterDependencies_ (mutable) ps5 = ParamSpecBase("psb5", "numeric", "number", "") extended = frozen.extend(standalones=(ps5,)) @@ -533,3 +537,17 @@ def test_frozen_from_dict(some_paramspecbases) -> None: frozen = FrozenInterDependencies_._from_dict(ser) assert isinstance(frozen, FrozenInterDependencies_) assert frozen == FrozenInterDependencies_(idps) + + +def test_frozen_equality_is_symmetric(some_paramspecbases) -> None: + ps1, ps2, ps3, ps4 = some_paramspecbases + idps = InterDependencies_(dependencies={ps1: (ps2, ps3)}) + frozen = FrozenInterDependencies_(idps) + + assert frozen == idps + assert idps == frozen + + other = InterDependencies_(dependencies={ps1: (ps2, ps4)}) + assert frozen != other + assert other != frozen + assert frozen != "not interdependencies" From 36e8d864fd71381cac692e685111d858501eed9c Mon Sep 17 00:00:00 2001 From: "Jens H. Nielsen" Date: Sat, 17 Oct 2026 10:05:42 +0200 Subject: [PATCH 4/5] Deprecate override argument of set_interdependencies The interdependencies of a started dataset are fixed and override never updated the run description in the database, only the in memory object and data exported from it. Nothing uses it anymore since measurements no longer unfreeze the interdependencies on exit. Remove the argument from the private DataSetInMem method outright. --- src/qcodes/dataset/data_set.py | 60 +++++++++++++++++++++++- src/qcodes/dataset/data_set_in_memory.py | 5 +- tests/dataset/test_dataset_basic.py | 28 ++++++++++- 3 files changed, 86 insertions(+), 7 deletions(-) diff --git a/src/qcodes/dataset/data_set.py b/src/qcodes/dataset/data_set.py index 6788de0a197f..dee9257fe1c7 100644 --- a/src/qcodes/dataset/data_set.py +++ b/src/qcodes/dataset/data_set.py @@ -6,15 +6,17 @@ import tempfile import time import uuid +import warnings from dataclasses import dataclass from pathlib import Path from queue import Queue from threading import Thread -from typing import TYPE_CHECKING, Any, Literal +from typing import TYPE_CHECKING, Any, Literal, overload import numpy import numpy.typing as npt from tqdm.auto import trange +from typing_extensions import deprecated import qcodes from qcodes.dataset.data_set_protocol import ( @@ -84,6 +86,7 @@ ) from qcodes.utils import ( NumpyJSONEncoder, + QCoDeSDeprecationWarning, ) from ._raw_data_storage import ( @@ -123,6 +126,12 @@ log = logging.getLogger(__name__) +_SET_INTERDEPENDENCIES_OVERRIDE_DEPRECATION = ( + "The `override` argument of `DataSet.set_interdependencies` is deprecated " + "and will be removed in a future release. The interdependencies of a " + "dataset can not be changed once it has been started." +) + # TODO: storing parameters in separate table as an extension (dropping # the column parameters would be much nicer @@ -634,18 +643,65 @@ def toggle_debug(self) -> None: self.conn.close() self.conn = connect(path_to_db, self._debug) + @overload + def set_interdependencies( + self, + interdeps: InterDependencies_, + shapes: Shapes | None = None, + ) -> None: ... + + @overload + @deprecated(_SET_INTERDEPENDENCIES_OVERRIDE_DEPRECATION) def set_interdependencies( self, interdeps: InterDependencies_, shapes: Shapes | None = None, - override: bool = False, + *, + override: bool, + ) -> None: ... + + @overload + @deprecated(_SET_INTERDEPENDENCIES_OVERRIDE_DEPRECATION) + def set_interdependencies( + self, + interdeps: InterDependencies_, + shapes: Shapes | None, + override: bool, + ) -> None: ... + + def set_interdependencies( + self, + interdeps: InterDependencies_, + shapes: Shapes | None = None, + override: bool | None = None, ) -> None: """ Set the interdependencies object (which holds all added parameters and their relationships) of this dataset and optionally the shapes object that holds information about the shape of the data to be measured. + + Args: + interdeps: The interdependencies of the parameters of the dataset. + shapes: The expected shapes of the data of the parameters. + override: Deprecated. Allows replacing the interdependencies of a + dataset that has already been started. The interdependencies + of a started dataset are fixed. Replacing them only changes + this object (and data exported from it) but not the run + description stored in the database, so the two can end up + inconsistent. + + Raises: + TypeError: If interdeps is not an InterDependencies_ instance. + RuntimeError: If the dataset has already been started. + """ + if override is not None: + warnings.warn( + _SET_INTERDEPENDENCIES_OVERRIDE_DEPRECATION, + QCoDeSDeprecationWarning, + stacklevel=2, + ) if not isinstance(interdeps, InterDependencies_): raise TypeError( f"Wrong input type. Expected InterDepencies_, got {type(interdeps)}" diff --git a/src/qcodes/dataset/data_set_in_memory.py b/src/qcodes/dataset/data_set_in_memory.py index 82de47d2fb13..f4c3738fbbad 100644 --- a/src/qcodes/dataset/data_set_in_memory.py +++ b/src/qcodes/dataset/data_set_in_memory.py @@ -757,7 +757,6 @@ def _set_interdependencies( self, interdeps: InterDependencies_, shapes: Shapes | None = None, - override: bool = False, ) -> None: """ Set the interdependencies object (which holds all added @@ -770,12 +769,10 @@ def _set_interdependencies( f"Wrong input type. Expected InterDepencies_, got {type(interdeps)}" ) - if not self.pristine and not override: + if not self.pristine: mssg = "Can not set interdependencies on a DataSet that has been started." raise RuntimeError(mssg) self._rundescriber = RunDescriber(interdeps, shapes=shapes) - if not self.pristine: - self._rundescriber = _freeze_rundescriber(self._rundescriber) def _get_paramspecs(self) -> SPECS: old_interdeps = new_to_old(self.description.interdeps) diff --git a/tests/dataset/test_dataset_basic.py b/tests/dataset/test_dataset_basic.py index 1116bba91350..e216291ec1fc 100644 --- a/tests/dataset/test_dataset_basic.py +++ b/tests/dataset/test_dataset_basic.py @@ -33,6 +33,7 @@ from qcodes.dataset.sqlite.database import _convert_array, get_DB_location from qcodes.dataset.sqlite.queries import _rewrite_timestamps, _unicode_categories from qcodes.parameters import ParamSpecBase +from qcodes.utils import QCoDeSDeprecationWarning from qcodes.utils.types import complex_types, numpy_complex, numpy_floats, numpy_ints from tests.common import error_caused_by from tests.dataset.helper_functions import verify_data_dict @@ -244,8 +245,33 @@ def test_interdependencies_frozen_once_started(start_bg_writer) -> None: assert loaded.description == ds.description assert loaded.the_same_dataset_as(ds) - ds.set_interdependencies(idps, override=True) + +@pytest.mark.usefixtures("experiment") +def test_set_interdependencies_override_is_deprecated() -> None: + x = ParamSpecBase("x", "numeric") + y = ParamSpecBase("y", "numeric") + idps = InterDependencies_(dependencies={y: (x,)}) + ds = DataSet() + ds.set_interdependencies(idps) + ds.mark_started() + + with pytest.raises(RuntimeError, match="has been started"): + ds.set_interdependencies(idps) + + match = "The `override` argument of `DataSet.set_interdependencies` is deprecated" + with pytest.warns(QCoDeSDeprecationWarning, match=match): + ds.set_interdependencies(idps, override=True) # pyright: ignore[reportDeprecated] assert isinstance(ds.description.interdeps, FrozenInterDependencies_) + assert ds.description.interdeps == idps + + with pytest.warns(QCoDeSDeprecationWarning, match=match): + ds.set_interdependencies(idps, None, True) # pyright: ignore[reportDeprecated] + + with ( + pytest.warns(QCoDeSDeprecationWarning, match=match), + pytest.raises(RuntimeError, match="has been started"), + ): + ds.set_interdependencies(idps, override=False) # pyright: ignore[reportDeprecated] @pytest.mark.usefixtures("experiment") From e1333b2bf78a3042f1d88f833cb3da5397dc4e20 Mon Sep 17 00:00:00 2001 From: "Jens H. Nielsen" Date: Sat, 17 Oct 2026 10:15:25 +0200 Subject: [PATCH 5/5] Add newsfragments --- docs/changes/newsfragments/xxxx.breaking | 10 ++++++++++ docs/changes/newsfragments/xxxx.improved | 6 ++++++ docs/changes/newsfragments/xxxx.underthehood | 4 ++++ 3 files changed, 20 insertions(+) create mode 100644 docs/changes/newsfragments/xxxx.breaking create mode 100644 docs/changes/newsfragments/xxxx.improved create mode 100644 docs/changes/newsfragments/xxxx.underthehood diff --git a/docs/changes/newsfragments/xxxx.breaking b/docs/changes/newsfragments/xxxx.breaking new file mode 100644 index 000000000000..2c679842c12b --- /dev/null +++ b/docs/changes/newsfragments/xxxx.breaking @@ -0,0 +1,10 @@ +The ``InterDependencies_`` of a ``DataSet`` or ``DataSetInMem`` are now frozen once the +dataset has been started, also after the measurement has completed and when loading a +started dataset, since the parameters of a dataset can not change at that point. +``dataset.description.interdeps`` is therefore a ``FrozenInterDependencies_`` for such +datasets and modifying it in place raises a ``TypeError``. Methods that return a new +object such as ``extend`` and ``remove`` continue to work, and ``FrozenInterDependencies_`` +now compares equal to an ``InterDependencies_`` with the same parameters. +The ``override`` argument of ``DataSet.set_interdependencies`` is deprecated, since the +interdependencies of a started dataset are fixed and ``override`` never updated the run +description stored in the database. diff --git a/docs/changes/newsfragments/xxxx.improved b/docs/changes/newsfragments/xxxx.improved new file mode 100644 index 000000000000..118d5e281d36 --- /dev/null +++ b/docs/changes/newsfragments/xxxx.improved @@ -0,0 +1,6 @@ +Loading data from a ``DataSet`` is now faster. Array and complex valued data is read +several times faster since the header of the stored arrays is only parsed once per +parameter rather than once per stored value. Repeated calls to ``get_parameter_data`` +on a started ``DataSet``, e.g. when the cache is updated during a measurement, are also +faster as the run description held by the dataset is used rather than reading and +parsing it from the database on every call. diff --git a/docs/changes/newsfragments/xxxx.underthehood b/docs/changes/newsfragments/xxxx.underthehood new file mode 100644 index 000000000000..6401239448f7 --- /dev/null +++ b/docs/changes/newsfragments/xxxx.underthehood @@ -0,0 +1,4 @@ +The test suite now runs considerably faster. Tests are distributed between pytest-xdist +workers using ``worksteal`` rather than ``loadfile``, tests using the simulated Keysight +B220x and B1500 instruments no longer first try to connect to a real instrument, and a +number of unneeded sleeps and waits have been removed or shortened.