From 18eb84a691800e44e732637e93c744d3d2b49336 Mon Sep 17 00:00:00 2001 From: Chris Burr Date: Mon, 19 Oct 2026 13:35:03 +0200 Subject: [PATCH] fix (DMS): presigned URLs from the S3Gateway Accessing an S3 storage element through the S3Gateway was broken with S3 implementations which validate presigned requests: - the gateway looked up urls.get("Fields") instead of the fields of each URL, so the checksum metadata was never stored on upload - boto does not add the Fields of generate_presigned_post to the policy, so once passed S3 rejected them as "Extra input fields" - exists() sent a GET to a URL signed for HEAD, which is rejected as Forbidden --- src/DIRAC/DataManagementSystem/Service/S3GatewayHandler.py | 6 +++--- src/DIRAC/Resources/Storage/S3Storage.py | 7 +++++-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/DIRAC/DataManagementSystem/Service/S3GatewayHandler.py b/src/DIRAC/DataManagementSystem/Service/S3GatewayHandler.py index de9cdde6899..3a45262c5c0 100644 --- a/src/DIRAC/DataManagementSystem/Service/S3GatewayHandler.py +++ b/src/DIRAC/DataManagementSystem/Service/S3GatewayHandler.py @@ -150,9 +150,9 @@ def export_createPresignedUrl(self, storageName, s3_method, urls, expiration): failed[url] = "Permission denied" continue - res = returnSingleResult( - s3Plugin.createPresignedUrl({url: urls.get("Fields")}, s3_method, expiration=expiration) - ) + # For put_object, urls maps each url to the fields (metadata) of the file + fields = urls[url] if s3_method == "put_object" else None + res = returnSingleResult(s3Plugin.createPresignedUrl({url: fields}, s3_method, expiration=expiration)) log.debug(f"Presigned URL for {url}: {res}") if res["OK"]: diff --git a/src/DIRAC/Resources/Storage/S3Storage.py b/src/DIRAC/Resources/Storage/S3Storage.py index ac20e294c34..6551e9bfa8e 100644 --- a/src/DIRAC/Resources/Storage/S3Storage.py +++ b/src/DIRAC/Resources/Storage/S3Storage.py @@ -268,7 +268,8 @@ def _presigned_exists(self, urls): # and perform it with requests for url, presignedURL in presignedURLs.items(): try: - response = requests.get(presignedURL, timeout=30) + # The URL is signed for head_object so it must be used with HEAD + response = requests.head(presignedURL, timeout=30) if response.status_code == 200: successful[url] = True elif response.status_code == 404: # not found @@ -768,8 +769,10 @@ def createPresignedUrl(self, urls, s3_method, expiration=3600): fields = keys.get(key) if not isinstance(fields, dict): fields = None + # boto does not add the fields to the policy, without which S3 rejects them + conditions = [{k: v} for k, v in fields.items()] if fields else None response = self.s3_client.generate_presigned_post( - self.bucketName, key, Fields=fields, ExpiresIn=expiration + self.bucketName, key, Fields=fields, Conditions=conditions, ExpiresIn=expiration ) successful[key] = response