feat(storage): move media to S3 and remove DVC versioning

.env.example

@@ -23,13 +23,16 @@ GITHUB_TOKEN=
 PUBLIC_SITE_URL=https://civicdataspace.in
 PUBLIC_API_URL=https://api.datakeep.civicdays.in
 
-# --- s3 only ---
-# STORAGE_BUCKET=dataspace-dev
-# STORAGE_REGION=ap-south-1
-# STORAGE_ENV=dev
-# STORAGE_PUBLIC_BASE_URL=
-# STORAGE_ACCESS_KEY_ID=
-# STORAGE_SECRET_ACCESS_KEY=
+# Media storage (S3 only; required). STORAGE_ENV prefixes every key: use
+# "local" for local development against the dev bucket. Leave the access keys
+# empty on ECS/EC2 so the task role is used. STORAGE_PUBLIC_BASE_URL is the
+# base for public asset URLs (CDN or bucket URL); empty uses the S3 bucket URL.
+STORAGE_BUCKET=dataspace-dev
+STORAGE_REGION=ap-south-1
+STORAGE_ENV=local
+STORAGE_PUBLIC_BASE_URL=
+STORAGE_ACCESS_KEY_ID=
+STORAGE_SECRET_ACCESS_KEY=
 
 # Per-IP request throttling (api/middleware/rate_limit.py). False on dev only.
 RATE_LIMIT_ENABLED=True

DataSpace/settings.py

@@ -255,8 +255,39 @@
 # https://docs.djangoproject.com/en/4.0/howto/static-files/
 
 # This STATIC_URL setting is overridden below
+# Uploaded media lives only in S3 (see STORAGES below). MEDIA_URL is kept as the
+# prefix of the legacy /public/<name> redirect route for links issued before S3.
 MEDIA_URL = "public/"
-MEDIA_ROOT = os.path.join(BASE_DIR, "files", "public")
+
+# S3 media storage. STORAGE_ENV prefixes every key (local, dev, prod) so one
+# bucket can hold several environments. Empty keys fall back to the boto3
+# credential chain, which is what ECS/EC2 task roles use.
+STORAGE_BUCKET = env("STORAGE_BUCKET")
+STORAGE_REGION = env("STORAGE_REGION")
+STORAGE_ENV = env("STORAGE_ENV")
+STORAGE_PUBLIC_BASE_URL = env("STORAGE_PUBLIC_BASE_URL", default="")
+
+_S3_OPTIONS = {
+    "bucket_name": STORAGE_BUCKET,
+    "region_name": STORAGE_REGION,
+    "access_key": env("STORAGE_ACCESS_KEY_ID", default="") or None,
+    "secret_key": env("STORAGE_SECRET_ACCESS_KEY", default="") or None,
+    "file_overwrite": False,
+    # The bucket enforces object ownership; access is granted by bucket policy, not ACLs.
+    "default_acl": None,
+}
+_S3_PUBLIC_OPTIONS = {
+    **_S3_OPTIONS,
+    "location": f"{STORAGE_ENV}/public",
+    "querystring_auth": False,
+    "custom_domain": STORAGE_PUBLIC_BASE_URL.split("://", 1)[-1].rstrip("/") or None,
+}
+_S3_PRIVATE_OPTIONS = {
+    **_S3_OPTIONS,
+    "location": f"{STORAGE_ENV}/private",
+    "querystring_auth": True,
+    "querystring_expire": 300,
+}
 
 # Default primary key field type
 # https://docs.djangoproject.com/en/4.0/ref/settings/#default-auto-field
@@ -308,11 +339,6 @@
 PUBLIC_SITE_URL = os.getenv("PUBLIC_SITE_URL", "https://civicdataspace.in")
 PUBLIC_API_URL = os.getenv("PUBLIC_API_URL", "https://api.datakeep.civicdays.in")
 
-# DVC settings
-DVC_REPO_PATH = os.path.join(BASE_DIR, "dvc")
-DVC_REMOTE_NAME = os.getenv("DVC_REMOTE_NAME", None)
-DVC_REMOTE_URL = os.getenv("DVC_REMOTE_URL", None)
-
 # Django REST Framework settings
 REST_FRAMEWORK = {
     "DEFAULT_PERMISSION_CLASSES": [
@@ -377,6 +403,9 @@
             "level": "DEBUG" if DEBUG else "INFO",
             "propagate": True,
         },
+        # At DEBUG the AWS SDK logs every signed request, Authorization header
+        # included, and floods the log on each S3 call.
+        **{name: {"level": "WARNING"} for name in ("boto3", "botocore", "s3transfer", "urllib3")},
     },
 }
 
@@ -490,8 +519,16 @@
     "whitenoise.middleware.WhiteNoiseMiddleware",
 )
 
-# Use the simplest WhiteNoise storage configuration to avoid MIME type issues
-STATICFILES_STORAGE = "whitenoise.storage.StaticFilesStorage"
+# "public"/"private" are what model fields name through api.utils.storage. The
+# default is private so anything that falls back to default_storage is never
+# world-readable by accident.
+STORAGES = {
+    "default": {"BACKEND": "storages.backends.s3.S3Storage", "OPTIONS": _S3_PRIVATE_OPTIONS},
+    "public": {"BACKEND": "storages.backends.s3.S3Storage", "OPTIONS": _S3_PUBLIC_OPTIONS},
+    "private": {"BACKEND": "storages.backends.s3.S3Storage", "OPTIONS": _S3_PRIVATE_OPTIONS},
+    # Use the simplest WhiteNoise storage configuration to avoid MIME type issues
+    "staticfiles": {"BACKEND": "whitenoise.storage.StaticFilesStorage"},
+}
 
 # Disable compression to avoid MIME type issues
 WHITENOISE_ENABLE_COMPRESSION = False

DataSpace/urls.py

@@ -23,7 +23,7 @@
 from drf_yasg import openapi
 from drf_yasg.views import get_schema_view
 
-from api.views import health
+from api.views import health, legacy_media
 
 # Type alias for URL patterns
 URLPatternsList = List[Union[URLPattern, URLResolver]]
@@ -49,16 +49,22 @@
     # Health check endpoint
     path("health/", health.health_check, name="health_check"),
     # API documentation
-    path(
-        "swagger<format>/", schema_view.without_ui(cache_timeout=0), name="schema-json"
-    ),
+    path("swagger<format>/", schema_view.without_ui(cache_timeout=0), name="schema-json"),
     path(
         "swagger/",
         schema_view.with_ui("swagger", cache_timeout=0),
         name="schema-swagger-ui",
     ),
     path("redoc/", schema_view.with_ui("redoc", cache_timeout=0), name="schema-redoc"),
-] + static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)
+    # Links issued before media moved to S3; see api/views/legacy_media.py.
+    path(
+        # Django prefixes a relative MEDIA_URL with the script name ("/public/"),
+        # which a URL pattern never matches.
+        f"{settings.MEDIA_URL.lstrip('/')}<path:name>",
+        legacy_media.legacy_media_redirect,
+        name="legacy_media",
+    ),
+]
 
 # In debug mode, add static URL patterns and debug toolbar
 if settings.DEBUG:

SECURITY.md

@@ -134,21 +134,20 @@ When contributing to this project, please follow these security guidelines:
 - **GraphQL**: Implements query complexity limits and depth restrictions
 - **REST API**: Rate limiting and throttling enabled
 - **File Uploads**: Validated for type, size, and content
-- **Data Versioning**: DVC integration for data versioning and tracking
 
 ### Data Protection
 - **Encryption**: All data encrypted in transit (TLS 1.2+)
 - **Database**: PostgreSQL with proper access controls
 - **Search**: Elasticsearch with authentication enabled
 - **Caching**: Redis with secure configuration
-- **File Storage**: Secure file handling and validation
+- **File Storage**: Media in S3; public assets via unsigned URLs, private files via presigned URLs or gated views
 
 ### Third-Party Services
 - **Keycloak**: Ensure proper realm and client configuration
 - **Elasticsearch**: Enable security features and authentication
 - **Redis**: Use password protection and disable dangerous commands
 - **OpenTelemetry**: Configure to exclude sensitive data
-- **DVC**: Secure data versioning and storage
+- **S3**: Block public access except the `*/public/*` prefix; least-privilege task role
 
 ## Security Checklist for Pull Requests
 

api/management/commands/backfill_file_hashes.py

@@ -3,7 +3,7 @@
     python manage.py backfill_file_hashes            # fill every missing hash
     python manage.py backfill_file_hashes --dry-run  # only count
 
-Writes through a queryset update, so the DVC versioning signal does not run.
+Writes through a queryset update, so no model save signals run.
 """
 
 from django.core.management.base import BaseCommand

api/management/commands/migrate_media_to_s3.py

@@ -0,0 +1,144 @@
+"""Bring every stored file into S3 under its current, prefix-free name.
+
+    python manage.py migrate_media_to_s3                                 # dry run
+    python manage.py migrate_media_to_s3 --execute                       # copy + rename
+    python manage.py migrate_media_to_s3 --source /code/files/public --execute
+
+Rows created on disk carry a ``files/public/`` or ``files/`` prefix that the S3
+keys no longer use (see ``normalize_media_name``). For each stored name this
+puts the file at the normalised key in the storage its field names, taking it
+from the old S3 key or, with ``--source``, from the old disk tree, then points
+the row at the new name. Safe to re-run: names already in place are skipped.
+Nothing is deleted, neither local files nor old S3 objects.
+"""
+
+from collections import Counter
+from pathlib import Path
+from typing import Any, Iterator, List, Optional, Set, Tuple, Type
+
+import structlog
+from django.apps import apps
+from django.core.files import File
+from django.core.files.storage import Storage
+from django.core.management.base import BaseCommand, CommandError, CommandParser
+from django.db import models
+
+from api.utils.storage import normalize_media_name
+
+logger = structlog.get_logger("dataspace.commands.migrate_media_to_s3")
+
+
+def iter_file_fields() -> Iterator[Tuple[Type[models.Model], models.FileField]]:
+    """Yield every concrete file field across installed models."""
+    for model in apps.get_models():
+        for field in model._meta.get_fields():
+            if isinstance(field, models.FileField):
+                yield model, field
+
+
+def stored_names(model: Type[models.Model], field: models.FileField) -> List[str]:
+    """Distinct non-empty names stored in ``field``."""
+    return list(
+        model._default_manager.exclude(**{field.name: ""})
+        .exclude(**{f"{field.name}__isnull": True})
+        .values_list(field.name, flat=True)
+        .distinct()
+    )
+
+
+def copy_to(storage: Storage, target: str, fh: Any) -> None:
+    """Save ``fh`` under exactly ``target``; the row will point at that name."""
+    saved = storage.save(target, File(fh))
+    if saved != target:
+        storage.delete(saved)
+        raise RuntimeError(f"storage renamed {target} to {saved}")
+
+
+class Command(BaseCommand):
+    help = "Copy media into S3 under prefix-free names and point DB rows at them."
+
+    def add_arguments(self, parser: CommandParser) -> None:
+        parser.add_argument(
+            "--source",
+            help="Old MEDIA_ROOT that legacy names are relative to (e.g. /code/files/public)",
+        )
+        parser.add_argument(
+            "--execute", action="store_true", help="Copy and rename; without it, only report."
+        )
+
+    def handle(self, *args: Any, **options: Any) -> None:
+        source: Optional[Path] = None
+        if options["source"]:
+            source = Path(options["source"]).resolve()
+            if not source.is_dir():
+                raise CommandError(f"Source directory not found: {source}")
+        execute: bool = options["execute"]
+
+        counts: Counter = Counter()
+        seen: Set[str] = set()
+        for model, field in iter_file_fields():
+            storage = field.storage
+            for name in stored_names(model, field):
+                target = normalize_media_name(name)
+                seen.add(target)
+                try:
+                    present = self._place(storage, name, target, source, execute, counts)
+                    if present and target != name:
+                        counts["rows_renamed"] += 1
+                        if execute:
+                            model._default_manager.filter(**{field.name: name}).update(
+                                **{field.name: target}
+                            )
+                except Exception as exc:
+                    counts["failed"] += 1
+                    logger.error("media_migration_failed", name=name, error=str(exc))
+
+        if source:
+            counts["orphaned"] = sum(
+                1
+                for p in source.rglob("*")
+                # Disk files keep their legacy names; rows may already use the clean ones.
+                if p.is_file()
+                and normalize_media_name(p.relative_to(source).as_posix()) not in seen
+            )
+
+        summary = ", ".join(f"{key}={counts[key]}" for key in sorted(counts))
+        logger.info("media_migration_summary", execute=execute, **dict(counts))
+        self.stdout.write(f"{'Done' if execute else 'Dry run'}: {summary}")
+        if counts["failed"]:
+            raise CommandError(f"{counts['failed']} file(s) failed")
+
+    def _place(
+        self,
+        storage: Storage,
+        name: str,
+        target: str,
+        source: Optional[Path],
+        execute: bool,
+        counts: Counter,
+    ) -> bool:
+        """Make sure ``target`` exists in ``storage``; return whether it does (or would)."""
+        if storage.exists(target):
+            counts["in_place"] += 1
+            return True
+
+        if target != name and storage.exists(name):
+            counts["moved_in_s3"] += 1
+            if execute:
+                with storage.open(name, "rb") as fh:
+                    copy_to(storage, target, fh)
+            return True
+
+        if source is not None:
+            local = (source / name).resolve()
+            # Names come from the DB; never read outside the source tree.
+            if local.is_relative_to(source) and local.is_file():
+                counts["copied_from_disk"] += 1
+                if execute:
+                    with local.open("rb") as fh:
+                        copy_to(storage, target, fh)
+                return True
+
+        counts["missing"] += 1
+        logger.warning("media_source_missing", name=name)
+        return False