diff --git a/CHANGELOG b/CHANGELOG index 81779b7..60b2030 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -44,6 +44,9 @@ older servers (#108) - enh: do not retry API requests when the server name cannot be resolved (DCOR-Aid froze for minutes on Windows with a wrong server) + - ref: remove the fallback to uploading resources via CKAN + (`resource_add_upload_legacy_indirect_ckan`); DCOR only accepts + uploads to S3 - ref: convert `dcoraid.cli` module into a package; the upload task CLI now lives in `dcoraid.cli.task_upload_task` - ref: use `super()`, PEP 604 optional type hints, and builtin/ @@ -64,6 +67,8 @@ - ref: new `dcoraid.api.create_dcor_dev_user` and `dcoraid.api.is_valid_api_key` used by the setup wizard and the tests - setup: add tool.ruff.lint section to pyproject.toml + - setup: remove `requests_toolbelt` dependency (was only used for + uploads via CKAN) - tests: automatically create test users on DCOR-dev if `DCOR_API_KEY` is not set or `tests/api_key` is missing or invalid - tests: replace tests that required access to the `figshare-import` diff --git a/build-recipes/hook-dcoraid.py b/build-recipes/hook-dcoraid.py index 4942025..5660e4b 100644 --- a/build-recipes/hook-dcoraid.py +++ b/build-recipes/hook-dcoraid.py @@ -17,4 +17,4 @@ # Add the Zstandard library used by dclab datas += collect_data_files("hdf5plugin", includes=["plugins/libh5zstd.*"]) -hiddenimports = ["dclab.cli", "requests_toolbelt"] +hiddenimports = ["dclab.cli"] diff --git a/dcoraid/api/ckan_api.py b/dcoraid/api/ckan_api.py index 8e9bf60..5d56825 100644 --- a/dcoraid/api/ckan_api.py +++ b/dcoraid/api/ckan_api.py @@ -435,7 +435,7 @@ def post(self, api_call, data, dump_json=True, headers=None, ---------- api_call: str An API call function (e.g. "package_create") - data: dict, MultipartEncoder, ... + data: dict The data connected to the post request. For "package_create", this would be a dictionary with the dataset name, author, license, etc. diff --git a/dcoraid/api/dataset.py b/dcoraid/api/dataset.py index 27bd903..b95a2c1 100644 --- a/dcoraid/api/dataset.py +++ b/dcoraid/api/dataset.py @@ -5,12 +5,8 @@ import functools import logging import pathlib -import time from collections.abc import Callable -import requests -from requests_toolbelt import MultipartEncoder, MultipartEncoderMonitor - from .errors import ( APIBadRequest, APIConflictError, APINotFoundError, NoS3UploadAvailableError ) @@ -198,16 +194,18 @@ def resource_add(dataset_id, path, api, resource_name=None, it, only update the resource_dict. monitor_callback: None or callable This callable is used to monitor the upload progress. It must - accept one argument: a - `requests_toolbelt.MultipartEncoderMonitor` object. + accept one argument: an object with the attributes `len` (file + size) and `bytes_read` (number of bytes uploaded). Returns ------- srv_time: float - Total time the server (nginx+uwsgi) needed to process the upload + Always 0 (kept for backwards compatibility; this was the time the + server needed to process uploads via CKAN, which are not supported + anymore) etag: str - ETag of the resource on S3; returns None if the upload was not - performed via S3 + ETag of the resource on S3; returns None if the resource already + existed and was not uploaded See Also -------- @@ -230,31 +228,16 @@ def resource_add(dataset_id, path, api, resource_name=None, resource_name=resource_name, api=api): - # Perform the upload - try: - # Uploading directly to S3 is the preferred option. - etag = resource_add_upload_direct_s3( - api=api, - resource_path=path, - dataset_id=dataset_id, - resource_name=resource_name, - monitor_callback=monitor_callback, - logger=logger, - timeout=27.9 - ) - except NoS3UploadAvailableError as e: - # This is the fall-back option that causes a performance hit - # on the DCOR server and will be unsupported in the future. - logger.warning(str(e)) - resource_add_upload_legacy_indirect_ckan( - api=api, - resource_path=path, - dataset_id=dataset_id, - resource_name=resource_name, - monitor_callback=monitor_callback, - logger=logger, - timeout=500, - ) + # Perform the upload (DCOR only accepts uploads to S3) + etag = resource_add_upload_direct_s3( + api=api, + resource_path=path, + dataset_id=dataset_id, + resource_name=resource_name, + monitor_callback=monitor_callback, + logger=logger, + timeout=27.9 + ) # If we are here, then the upload was successful logger.info(f"Finished upload of {dataset_id}/{resource_name}") @@ -358,77 +341,6 @@ def resource_add_upload_direct_s3( return etag -def resource_add_upload_legacy_indirect_ckan( - api: CKANAPI, - resource_path: str | pathlib.Path, - dataset_id: str, - resource_name: str, - monitor_callback: Callable | None = None, - logger: logging.Logger | None = None, - timeout: float = 27.9): - """Legacy upload procedure for resources - - This method uses the legacy path (nginx -> uwsgi -> CKAN API) to - upload resources to the block storage of the DCOR instance. It is - termed legacy, because it is very inefficient to process resources - like this. A better approach is to upload resources directly to S3. - DCOR supports uploading resources directly to S3, which is facilitated - via the `resource_upload_s3_url` API action providing a presigned - upload link. (see :func:`resource_add_upload_direct_s3`) - """ - upload_id = f"{dataset_id}/{resource_name}" - if logger is not None: - logger.info(f"Commencing legacy upload of {upload_id}") - - with pathlib.Path(resource_path).open("rb") as fd: - # use package_revise to upload the resource - # https://github.com/DCOR-dev/DCOR-Aid/issues/28 - e = MultipartEncoder(fields={ - 'match__id': dataset_id, - 'update__resources__extend': f'[{{"name":"{resource_name}"}}]', - 'update__resources__-1__upload': (resource_name, fd)}) - m = MultipartEncoderMonitor(e, monitor_callback) - # Increase the read size to speed-up upload (the default chunk - # size for uploads in urllib is 8k which results in a lot of - # Python code being involved in uploading a 20GB file; Setting - # the chunk size to 4MB should increase the upload speed): - # https://github.com/requests/toolbelt/issues/75 - # #issuecomment-237189952 - m._read = m.read - m.read = lambda size: m._read(4 * 1024 * 1024) - - try: - api.post("package_revise", - data=m, - dump_json=False, - headers={"Content-Type": m.content_type}, - timeout=timeout) - except requests.exceptions.Timeout: - # This means that the server does not respond. This is ok, - # because we can just check whether the resource was - # processed correctly. - if logger is not None: - logger.info(f"Timeout for upload {upload_id}") - start_wait_srv = time.monotonic() - wait_time_minutes = 60 - for ii in range(wait_time_minutes): - if resource_exists(dataset_id=dataset_id, - resource_name=resource_name, - api=api): - srv_time = timeout + time.monotonic() - start_wait_srv - if logger is not None: - logger.info( - f"Waited {srv_time / 60:.1f} min for {upload_id}") - break - else: - if logger is not None: - logger.info(f"Waiting {ii + 1} min for {upload_id}") - time.sleep(60) - else: - raise ValueError(f"Timeout or {upload_id} not processed after " - + f"{wait_time_minutes} minutes!") - - def resource_exists(dataset_id: str, resource_name: str, api: CKANAPI, diff --git a/dcoraid/gui/main.py b/dcoraid/gui/main.py index 41d95b1..9875928 100644 --- a/dcoraid/gui/main.py +++ b/dcoraid/gui/main.py @@ -13,7 +13,6 @@ import dclab import requests import requests_cache -import requests_toolbelt import urllib3 from PyQt6 import QtCore, QtGui, QtTest, QtWidgets @@ -342,7 +341,6 @@ def on_action_software(self): libs = [dclab, requests, requests_cache, - requests_toolbelt, urllib3, ] sw_text = f"DCOR-Aid {__version__}\n\n" diff --git a/pyproject.toml b/pyproject.toml index 0890e14..592367b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,9 +31,8 @@ dependencies = [ "dclab[dcor]>=0.64.2", "numpy>=1.21", "requests>=2.31", # CVE-2023-32681 - "urllib3>=2.0", # requests_toolbelt and general compatibility + "urllib3>=2.0", # general compatibility "requests_cache", # caching API-'get' requests - "requests_toolbelt>=1.0.0", # multipart uploads with progress ] dynamic = ["version"]