tests: upload resources to S3 in package_revise tests

tests/helper.py

@@ -5,6 +5,7 @@
 import time
 
 from dcoraid.api import CKANAPI, create_dcor_dev_user, is_valid_api_key
+from dcoraid.api import dataset, s3_api
 
 
 SERVER = "dcor-dev.mpl.mpg.de"
@@ -71,3 +72,21 @@ def make_dataset_dict(circle=None, hint=None):
         "authors": "Guybrush Ulysses Threepwood",
     }
     return dataset_dict
+
+
+def upload_resource_to_s3(api, dataset_id, path):
+    """Upload a file to S3 without adding it to the dataset
+
+    Returns the resource ID, which can then be used to add the
+    resource to the dataset with `package_revise`.
+    """
+    path = pathlib.Path(path)
+    org_id = dataset.get_organization_id_for_dataset(api=api,
+                                                     dataset_id=dataset_id)
+    upload_info = api.get("resource_upload_s3_urls",
+                          organization_id=org_id,
+                          file_size=path.stat().st_size)
+    s3_api.upload_s3_presigned(path=path,
+                               upload_urls=upload_info["upload_urls"],
+                               complete_url=upload_info["complete_url"])
+    return upload_info["resource_id"]

tests/test_schemas_basic.py

@@ -7,7 +7,9 @@
 import dcoraid
 from dcoraid.api import dataset
 
-from helper import CKANAPI, get_api, make_dataset_dict
+from helper import (
+    CKANAPI, get_api, make_dataset_dict, upload_resource_to_s3
+)
 
 data_path = pathlib.Path(__file__).parent / "data"
 
@@ -211,16 +213,18 @@ def test_package_revise_upload_not_allowed_active_dataset():
                          )
     dataset.dataset_activate(dataset_id=data["id"],
                              api=get_api())
-    # now try to add a resource with package_revise
-    path = data_path / "calibration_beads_47.rtdc"
-    with path.open("rb") as fd:
-        req = requests.post(
-            api.api_url + "package_revise",
-            data={"match__id": data["id"],
-                  "update__resources__extend": '[{"name":"peter.rtdc"}]',
-                  },
-            files=[("update__resources__-1__upload", ("peter.rtdc", fd))],
-            headers=api.headers)
+    # now try to add a resource (uploaded to S3) with package_revise
+    rid = upload_resource_to_s3(api=api,
+                                dataset_id=data["id"],
+                                path=data_path / "calibration_beads_47.rtdc")
+    req = requests.post(
+        api.api_url + "package_revise",
+        json={"match": {"id": data["id"]},
+              "update__resources__extend": [{"id": rid,
+                                             "name": "peter.rtdc",
+                                             "s3_available": True}],
+              },
+        headers=api.headers)
     assert not req.ok
     assert not req.json()["success"]
     error = req.json()["error"]
@@ -275,18 +279,19 @@ def test_package_revise_upload_not_allowed_same_resource_name():
                          api=api,
                          )
 
-    # now try to add a resource with package_revise
-    path = data_path / "calibration_beads_47.rtdc"
-    with path.open("rb") as fd:
-        req = requests.post(
-            api.api_url + "package_revise",
-            data={"match__id": data["id"],
-                  "update__resources__extend":
-                      '[{"name":"calibration_beads_47.rtdc"}]',
-                  },
-            files=[("update__resources__-1__upload",
-                    ("calibration_beads_47.rtdc", fd))],
-            headers=api.headers)
+    # now try to add a resource (uploaded to S3) with the same name
+    rid = upload_resource_to_s3(api=api,
+                                dataset_id=data["id"],
+                                path=data_path / "calibration_beads_47.rtdc")
+    req = requests.post(
+        api.api_url + "package_revise",
+        json={"match": {"id": data["id"]},
+              "update__resources__extend": [
+                  {"id": rid,
+                   "name": "calibration_beads_47.rtdc",
+                   "s3_available": True}],
+              },
+        headers=api.headers)
     assert not req.ok
     assert not req.json()["success"]
     assert req.reason == "CONFLICT"