diff --git a/simplyblock_cli/cli-reference.yaml b/simplyblock_cli/cli-reference.yaml index 3ae79e74aa..1c1fc9c989 100644 --- a/simplyblock_cli/cli-reference.yaml +++ b/simplyblock_cli/cli-reference.yaml @@ -3046,6 +3046,17 @@ commands: dest: into type: str required: false + - name: backup-create + help: "Back up one generation of a consistency group to S3 as one all-or-nothing unit. Takes a fresh crash-consistent generation unless --group-seq names an existing one." + arguments: + - name: "group_id" + help: "Consistency group id (or uuid)." + dest: group_id + type: str + - name: "--group-seq" + help: "Back up an existing generation by its sequence number. Omit to take a fresh generation." + dest: group_seq + type: int - name: "backup" help: "Backup Commands" weight: 650 @@ -3267,6 +3278,168 @@ commands: help: "The target id (storage pool or logical volume id)." dest: target_id type: str + - name: "group-backup" + help: "Consistency-group backup commands" + aliases: + - gb + weight: 655 + subcommands: + - name: list + help: "List the cluster's consistency-group backups." + arguments: + - name: "cluster_id" + help: "Cluster UUID." + dest: cluster_id + type: str + - name: "--json" + aliases: + - "-j" + help: "Print output in JSON format." + dest: json + type: bool + action: store_true + - name: get + help: "Show one consistency-group backup and its members." + arguments: + - name: "group_backup_id" + help: "Group backup id (or uuid)." + dest: group_backup_id + type: str + - name: "--json" + aliases: + - "-j" + help: "Print output in JSON format." + dest: json + type: bool + action: store_true + - name: restore + help: "Restore a consistency-group backup into a NEW consistency group, all-or-nothing. Each member volume recovers asynchronously." + arguments: + - name: "group_backup_id" + help: "Group backup id (or uuid)." + dest: group_backup_id + type: str + - name: "--name" + help: "Name of the NEW consistency group to form from the restored volumes. Must not already exist." + dest: into_group_name + type: str + required: true + - name: "--pool" + help: "The target pool name or id." + dest: pool + type: str + required: true + - name: "--node" + help: "The target storage node id. Omit to auto-select; every member is pinned to one store." + dest: node + type: str + - name: "--access-key-id" + help: "Access key for the backup's bucket, when it is not this cluster's own." + dest: access_key_id + type: secret + - name: "--secret-access-key" + help: "Secret key for the backup's bucket, when it is not this cluster's own." + dest: secret_access_key + type: secret + - name: delete + help: "Delete a consistency-group backup: its group manifest and record. Its member backups are retired: merged into the copy of the next generation while that chains against them, deleted once nothing does." + arguments: + - name: "group_backup_id" + help: "Group backup id (or uuid)." + dest: group_backup_id + type: str + - name: export + help: "Export a consistency-group backup's manifests as a unit, to a JSON file for cross-cluster restore." + arguments: + - name: "group_backup_id" + help: "Group backup id (or uuid)." + dest: group_backup_id + type: str + - name: "--cluster-id" + help: "The cluster id." + dest: cluster_id + type: str + - name: "-o" + aliases: + - "--output" + help: "The output file path." + dest: output + type: str + - name: discover + help: "List the consistency-group backups a bucket contains, reading its group manifests. Needs no cluster." + arguments: + - name: "--bucket" + help: "The bucket holding the group backups." + dest: bucket + type: str + required: true + - name: "--region" + help: "The bucket's region. Omit to let the AWS SDK resolve it." + dest: region + type: str + - name: "--endpoint" + help: "Endpoint of an S3-compatible store, e.g. http://minio:9000. Omit for AWS." + dest: endpoint + type: str + - name: "--access-key-id" + help: "Access key for the bucket. Omit to use the node's instance role." + dest: access_key_id + type: secret + - name: "--secret-access-key" + help: "Secret key for the bucket. Omit to use the node's instance role." + dest: secret_access_key + type: secret + - name: "--no-verify-tls" + help: "Skip certificate verification for the endpoint." + dest: no_verify_tls + type: bool + action: store_true + - name: "--path-style" + help: "Use path-style addressing, as MinIO and most S3-compatible stores need." + dest: path_style + type: bool + action: store_true + - name: import + help: "Register the consistency-group backups held in a bucket, or in an export file, into this cluster." + arguments: + - name: "--cluster-id" + help: "The target cluster to import into (required for cross-cluster restore)." + dest: cluster_id + type: str + - name: "--bucket" + help: "Import every group backup in this bucket. Give this or --from-file, not both." + dest: bucket + type: str + - name: "--from-file" + help: "Import the group backup in this file, from 'group-backup export'. It records which bucket its members live in, so --bucket is neither needed nor accepted." + dest: from_file + type: str + - name: "--region" + help: "The bucket's region. Omit to let the AWS SDK resolve it." + dest: region + type: str + - name: "--endpoint" + help: "Endpoint of an S3-compatible store, e.g. http://minio:9000. Omit for AWS." + dest: endpoint + type: str + - name: "--access-key-id" + help: "Access key for the bucket. Omit to use the node's instance role." + dest: access_key_id + type: secret + - name: "--secret-access-key" + help: "Secret key for the bucket. Omit to use the node's instance role." + dest: secret_access_key + type: secret + - name: "--no-verify-tls" + help: "Skip certificate verification for the endpoint." + dest: no_verify_tls + type: bool + action: store_true + - name: "--path-style" + help: "Use path-style addressing, as MinIO and most S3-compatible stores need." + dest: path_style + type: bool + action: store_true - name: "qos" help: "QoS Commands" weight: 700 diff --git a/simplyblock_cli/cli.py b/simplyblock_cli/cli.py index fe37a2631d..7442c21575 100755 --- a/simplyblock_cli/cli.py +++ b/simplyblock_cli/cli.py @@ -31,6 +31,7 @@ def __init__(self): self.init_snapshot() self.init_consistency_group() self.init_backup() + self.init_group_backup() self.init_qos() self.init_db_backup() super().__init__() @@ -1165,6 +1166,7 @@ def init_consistency_group(self): self.init_consistency_group__snapshot_list(subparser) self.init_consistency_group__snapshot_delete(subparser) self.init_consistency_group__clone(subparser) + self.init_consistency_group__backup_create(subparser) def init_consistency_group__list(self, subparser): @@ -1211,6 +1213,11 @@ def init_consistency_group__clone(self, subparser): subcommand.add_argument('seq', help='The generation number (group_seq) to clone.', type=int) subcommand.add_argument('--into', help='Name of the new consistency group to form from the clones. When omitted, the clones are independent volumes.', type=str, dest='into', required=False) + def init_consistency_group__backup_create(self, subparser): + subcommand = self.add_sub_command(subparser, 'backup-create', 'Back up one generation of a consistency group to S3 as one all-or-nothing unit. Takes a fresh crash-consistent generation unless --group-seq names an existing one.') + subcommand.add_argument('group_id', help='Consistency group id (or uuid).', type=str) + subcommand.add_argument('--group-seq', help='Back up an existing generation by its sequence number. Omit to take a fresh generation.', type=int, dest='group_seq') + def init_backup(self): subparser = self.add_command('backup', 'Backup Commands') @@ -1303,6 +1310,69 @@ def init_backup__policy_detach(self, subparser): subcommand.add_argument('target_id', help='The target id (storage pool or logical volume id).', type=str) + def init_group_backup(self): + subparser = self.add_command('group-backup', 'Consistency-group backup commands', aliases=['gb',]) + self.init_group_backup__list(subparser) + self.init_group_backup__get(subparser) + self.init_group_backup__restore(subparser) + self.init_group_backup__delete(subparser) + self.init_group_backup__export(subparser) + self.init_group_backup__discover(subparser) + self.init_group_backup__import(subparser) + + + def init_group_backup__list(self, subparser): + subcommand = self.add_sub_command(subparser, 'list', 'List the cluster\'s consistency-group backups.') + subcommand.add_argument('cluster_id', help='Cluster UUID.', type=str) + subcommand.add_argument('--json', '-j', help='Print output in JSON format.', dest='json', action='store_true') + + def init_group_backup__get(self, subparser): + subcommand = self.add_sub_command(subparser, 'get', 'Show one consistency-group backup and its members.') + subcommand.add_argument('group_backup_id', help='Group backup id (or uuid).', type=str) + subcommand.add_argument('--json', '-j', help='Print output in JSON format.', dest='json', action='store_true') + + def init_group_backup__restore(self, subparser): + subcommand = self.add_sub_command(subparser, 'restore', 'Restore a consistency-group backup into a NEW consistency group, all-or-nothing. Each member volume recovers asynchronously.') + subcommand.add_argument('group_backup_id', help='Group backup id (or uuid).', type=str) + subcommand.add_argument('--name', help='Name of the NEW consistency group to form from the restored volumes. Must not already exist.', type=str, dest='into_group_name', required=True) + subcommand.add_argument('--pool', help='The target pool name or id.', type=str, dest='pool', required=True) + subcommand.add_argument('--node', help='The target storage node id. Omit to auto-select; every member is pinned to one store.', type=str, dest='node') + subcommand.add_argument('--access-key-id', help='Access key for the backup\'s bucket, when it is not this cluster\'s own.', type=SecretStr, dest='access_key_id') + subcommand.add_argument('--secret-access-key', help='Secret key for the backup\'s bucket, when it is not this cluster\'s own.', type=SecretStr, dest='secret_access_key') + + def init_group_backup__delete(self, subparser): + subcommand = self.add_sub_command(subparser, 'delete', 'Delete a consistency-group backup: its group manifest and record. Its member backups are retired: merged into the copy of the next generation while that chains against them, deleted once nothing does.') + subcommand.add_argument('group_backup_id', help='Group backup id (or uuid).', type=str) + + def init_group_backup__export(self, subparser): + subcommand = self.add_sub_command(subparser, 'export', 'Export a consistency-group backup\'s manifests as a unit, to a JSON file for cross-cluster restore.') + subcommand.add_argument('group_backup_id', help='Group backup id (or uuid).', type=str) + subcommand.add_argument('--cluster-id', help='The cluster id.', type=str, dest='cluster_id') + subcommand.add_argument('-o', '--output', help='The output file path.', type=str, dest='output') + + def init_group_backup__discover(self, subparser): + subcommand = self.add_sub_command(subparser, 'discover', 'List the consistency-group backups a bucket contains, reading its group manifests. Needs no cluster.') + subcommand.add_argument('--bucket', help='The bucket holding the group backups.', type=str, dest='bucket', required=True) + subcommand.add_argument('--region', help='The bucket\'s region. Omit to let the AWS SDK resolve it.', type=str, dest='region') + subcommand.add_argument('--endpoint', help='Endpoint of an S3-compatible store, e.g. http://minio:9000. Omit for AWS.', type=str, dest='endpoint') + subcommand.add_argument('--access-key-id', help='Access key for the bucket. Omit to use the node\'s instance role.', type=SecretStr, dest='access_key_id') + subcommand.add_argument('--secret-access-key', help='Secret key for the bucket. Omit to use the node\'s instance role.', type=SecretStr, dest='secret_access_key') + subcommand.add_argument('--no-verify-tls', help='Skip certificate verification for the endpoint.', dest='no_verify_tls', action='store_true') + subcommand.add_argument('--path-style', help='Use path-style addressing, as MinIO and most S3-compatible stores need.', dest='path_style', action='store_true') + + def init_group_backup__import(self, subparser): + subcommand = self.add_sub_command(subparser, 'import', 'Register the consistency-group backups held in a bucket, or in an export file, into this cluster.') + subcommand.add_argument('--cluster-id', help='The target cluster to import into (required for cross-cluster restore).', type=str, dest='cluster_id') + subcommand.add_argument('--bucket', help='Import every group backup in this bucket. Give this or --from-file, not both.', type=str, dest='bucket') + subcommand.add_argument('--from-file', help='Import the group backup in this file, from \'group-backup export\'. It records which bucket its members live in, so --bucket is neither needed nor accepted.', type=str, dest='from_file') + subcommand.add_argument('--region', help='The bucket\'s region. Omit to let the AWS SDK resolve it.', type=str, dest='region') + subcommand.add_argument('--endpoint', help='Endpoint of an S3-compatible store, e.g. http://minio:9000. Omit for AWS.', type=str, dest='endpoint') + subcommand.add_argument('--access-key-id', help='Access key for the bucket. Omit to use the node\'s instance role.', type=SecretStr, dest='access_key_id') + subcommand.add_argument('--secret-access-key', help='Secret key for the bucket. Omit to use the node\'s instance role.', type=SecretStr, dest='secret_access_key') + subcommand.add_argument('--no-verify-tls', help='Skip certificate verification for the endpoint.', dest='no_verify_tls', action='store_true') + subcommand.add_argument('--path-style', help='Use path-style addressing, as MinIO and most S3-compatible stores need.', dest='path_style', action='store_true') + + def init_qos(self): subparser = self.add_command('qos', 'QoS Commands') self.init_qos__add(subparser) @@ -1827,6 +1897,8 @@ def run(self): ret = self.consistency_group__snapshot_delete(sub_command, args) elif sub_command in ['clone']: ret = self.consistency_group__clone(sub_command, args) + elif sub_command in ['backup-create']: + ret = self.consistency_group__backup_create(sub_command, args) else: self.parser.print_help() @@ -1857,6 +1929,25 @@ def run(self): else: self.parser.print_help() + elif args.command in ['group-backup', 'gb']: + sub_command = args_dict['group-backup'] + if sub_command in ['list']: + ret = self.group_backup__list(sub_command, args) + elif sub_command in ['get']: + ret = self.group_backup__get(sub_command, args) + elif sub_command in ['restore']: + ret = self.group_backup__restore(sub_command, args) + elif sub_command in ['delete']: + ret = self.group_backup__delete(sub_command, args) + elif sub_command in ['export']: + ret = self.group_backup__export(sub_command, args) + elif sub_command in ['discover']: + ret = self.group_backup__discover(sub_command, args) + elif sub_command in ['import']: + ret = self.group_backup__import(sub_command, args) + else: + self.parser.print_help() + elif args.command in ['qos']: sub_command = args_dict['qos'] if sub_command in ['add']: diff --git a/simplyblock_cli/clibase.py b/simplyblock_cli/clibase.py index c2b33dad6b..0468e592ab 100755 --- a/simplyblock_cli/clibase.py +++ b/simplyblock_cli/clibase.py @@ -27,7 +27,12 @@ from simplyblock_core.controllers.backup import controller as backup_controller from simplyblock_core.controllers.backup import policy as backup_policy from simplyblock_core.controllers.backup.chain import BackupChain -from simplyblock_core.controllers.backup.manifest import ManifestError, parse_export +from simplyblock_core.controllers.backup.manifest import ( + ManifestError, + list_all, + parse_export, + parse_group_export, +) from simplyblock_core.exceptions import MigrationConflictError, PreconditionError from simplyblock_core.models.backup_config import BackupConfig, S3Credentials from simplyblock_core.models.cluster import Cluster, HashicorpVaultSettings @@ -780,11 +785,13 @@ def cluster__replication_policy_add(self, sub_command, args): def cluster__replication_policy_snapshot(self, sub_command, args): from simplyblock_core.controllers import consistency_group_controller - snap_ids, err = consistency_group_controller.create_group_snapshot( + group_seq, err = consistency_group_controller.create_group_snapshot( args.policy_id) if err: return f"Group snapshot failed: {err}" - return utils.print_table([{"Snapshot": s} for s in snap_ids]) + group = db_controller.DBController().get_consistency_group_for_policy( + args.policy_id) + return self._cg_generation_table(group, group_seq) def cluster__replication_policy_list(self, sub_command, args): data = [{ @@ -1294,6 +1301,13 @@ def _cg_resolve(self, group_id): return g raise + def _cg_generation_table(self, group, group_seq): + """The member snapshots of the generation a take answered with.""" + from simplyblock_core.controllers import consistency_group_controller as cgc + return utils.print_table([ + {"Gen": group_seq, "Snapshot": s.get_id(), "Volume": s.lvol.get_id()} + for s in cgc.generation_snapshots(group, group_seq)]) + def consistency_group__list(self, sub_command, args): db = db_controller.DBController() data = [{ @@ -1354,10 +1368,11 @@ def consistency_group__remove_member(self, sub_command, args): def consistency_group__snapshot_take(self, sub_command, args): from simplyblock_core.controllers import consistency_group_controller as cgc - ids, err = cgc.create_group_snapshot_for_group(self._cg_resolve(args.group_id)) + group = self._cg_resolve(args.group_id) + group_seq, err = cgc.create_group_snapshot_for_group(group) if err: return f"Group snapshot failed: {err}" - return utils.print_table([{"Snapshot": s} for s in ids]) + return self._cg_generation_table(group, group_seq) def consistency_group__snapshot_list(self, sub_command, args): from simplyblock_core.controllers import consistency_group_controller as cgc @@ -1386,6 +1401,140 @@ def consistency_group__clone(self, sub_command, args): return f"Clone failed: {err}" return utils.print_table([{"Volume": v} for v in created]) + def consistency_group__backup_create(self, sub_command, args): + from simplyblock_core.controllers.backup import group_backup + group = self._cg_resolve(args.group_id) + group_backup_id, err = group_backup.create_group_backup( + group.get_id(), group_seq=getattr(args, 'group_seq', None)) + if err: + return f"Group backup failed: {err}" + return f"Group backup {group_backup_id} started" + + def _group_backup_row(self, gb): + return { + "ID": gb.get_id(), + "Group": gb.consistency_group_name or "-", + "Gen": gb.group_seq, + "Status": gb.status, + "Members": len(gb.members), + "Size": utils.humanbytes(gb.size), + } + + def group_backup__list(self, sub_command, args): + db = db_controller.DBController() + group_backups = sorted(db.get_group_backups(args.cluster_id), + key=lambda g: (g.created_at, g.uuid), reverse=True) + return _format_result([self._group_backup_row(g) for g in group_backups], + json=args.json) + + def group_backup__get(self, sub_command, args): + db = db_controller.DBController() + try: + gb = db.get_group_backup_by_id(args.group_backup_id) + except KeyError: + return f"Group backup {args.group_backup_id} not found" + if args.json: + return _format_json(gb.get_clean_dict()) + return utils.print_table([self._group_backup_row(gb)]) + + def group_backup__restore(self, sub_command, args): + from simplyblock_core.controllers.backup import group_backup + group_id, lvol_ids, err = group_backup.restore_group_backup( + args.group_backup_id, args.into_group_name, args.pool, + target_node_id=getattr(args, 'node', None), + s3_credentials=_s3_credentials(args)) + if err: + return f"Group restore failed: {err}" + return (f"Restoring group backup into consistency group {group_id} " + f"({len(lvol_ids)} volumes)") + + def group_backup__delete(self, sub_command, args): + from simplyblock_core.controllers.backup import group_backup + try: + group_backup.delete_group_backup(args.group_backup_id) + except KeyError: + return f"Delete failed: group backup {args.group_backup_id} not found" + except (PreconditionError, ManifestError) as e: + return f"Delete failed: {e}" + return (f"Group backup {args.group_backup_id} is being deleted; its " + f"member backups are retired in the background") + + def group_backup__export(self, sub_command, args): + from simplyblock_core.controllers.backup import group_backup + try: + export = group_backup.export_group_backup( + args.group_backup_id, cluster_id=getattr(args, 'cluster_id', None)) + except (PreconditionError, ValueError) as e: + print(f"Error: {e}") + return False + + output = _format_json(export.model_dump(mode="json")) + output_file = getattr(args, 'output', None) + if output_file: + with open(output_file, 'w') as f: + f.write(output) + print(f"Exported group backup {args.group_backup_id} to {output_file}") + else: + print(output) + return True + + def group_backup__discover(self, sub_command, args): + from simplyblock_core.controllers.backup import group_backup + config = _bucket_config(args) + try: + manifests = group_backup.discover_group_backups(config) + # "Complete" means the bucket still holds every member's manifest, + # which the group manifest alone cannot say -- a torn generation is + # worth showing rather than hiding, as that IS the finding a + # discover is for. + member_manifests = list_all(config) if manifests else [] + except (ManifestError, ValueError) as e: + print(f"Error: {e}") + return False + if not manifests: + print(f"No group backups found in {args.bucket}") + return False + # str, not UUID: this row goes through `utils.dump_json` for --json output, + # and json.dumps has no encoder for a UUID. + return [{ + "ID": str(m.group_backup_id), + "Group": m.source.consistency_group_name, + "Gen": m.source.generation, + "Members": len(m.member_backup_ids), + "Complete": "yes" if m.is_complete(member_manifests) else "no", + "Size": m.size, + "Encrypted": "yes" if m.encrypted else "no", + "Created": _format_timestamp(m.created_at.timestamp()), + } for m in manifests] + + def group_backup__import(self, sub_command, args): + from simplyblock_core.controllers.backup import group_backup + from_file = getattr(args, 'from_file', None) + bucket = getattr(args, 'bucket', None) + + # An export file records the location of everything in it, so naming a + # bucket alongside one could only contradict it. Reading a bucket needs + # the opposite: the name is how the manifests are found at all. + if bool(from_file) == bool(bucket): + print("Error: give exactly one of --from-file or --bucket") + return False + + cluster_id = getattr(args, 'cluster_id', None) + try: + if bucket: + ids = group_backup.import_group_backup_from_bucket( + _bucket_config(args), cluster_id=cluster_id) + else: + with open(str(from_file), 'rb') as f: + export = parse_group_export(f.read(), str(from_file)) + ids = [group_backup.import_group_backup(export, cluster_id=cluster_id)] + except (ManifestError, PreconditionError, ValueError, OSError) as e: + print(f"Error: {e}") + return False + + print(f"Imported {len(ids)} group backup(s)") + return True + def snapshot__replication_status(self, sub_command, args): return snapshot_controller.list_replication_tasks(args.cluster_id) diff --git a/simplyblock_core/controllers/backup/controller.py b/simplyblock_core/controllers/backup/controller.py index c2b077fe3a..293bed4b8c 100644 --- a/simplyblock_core/controllers/backup/controller.py +++ b/simplyblock_core/controllers/backup/controller.py @@ -19,7 +19,7 @@ lvol_dek_path, pool_kek_name, ) -from simplyblock_core.models.backup import Backup +from simplyblock_core.models.backup import Backup, GroupBackup from simplyblock_core.models.backup_config import ( BackupConfig, BackupLocation, @@ -28,6 +28,7 @@ from simplyblock_core.models.cluster import Cluster from simplyblock_core.models.snapshot import SnapShot from simplyblock_core.models.storage_node import StorageNode +from simplyblock_core.rpc_client import RPCException logger = logging.getLogger() @@ -168,6 +169,7 @@ def build_manifest(backup: Backup) -> backup_manifest.BackupManifest: "snapshot_id": backup.snapshot_id, "snapshot_name": backup.snapshot_name, "size": backup.size, + "max_namespace_per_subsys": backup.max_namespace_per_subsys or None, **settings, }) @@ -483,6 +485,8 @@ def create_single_backup(snapshot, lvol, node_id, cluster_id, prev_backup, locat backup.created_at = int(time.time()) backup.status = Backup.STATUS_PENDING backup.encrypted = bool(lvol.crypto_bdev) + # The subsystem sharing the restore reapplies (Backup.max_namespace_per_subsys). + backup.max_namespace_per_subsys = int(getattr(lvol, "max_namespace_per_subsys", 0) or 1) if backup.encrypted: cluster = db_controller.get_cluster_by_id(cluster_id) @@ -800,7 +804,8 @@ def backup_snapshot(snapshot_id, cluster_id=None): def restore_backup(backup_id: str, lvol_name: str, pool_id_or_name: str, target_node_id: str | None = None, - s3_credentials: S3Credentials | None = None): + s3_credentials: S3Credentials | None = None, + consistency_group: str | None = None): """Restore a backup chain into a new fully-accessible lvol. Creates the volume (with subsystem, listeners, namespace) via @@ -808,7 +813,18 @@ def restore_backup(backup_id: str, lvol_name: str, pool_id_or_name: str, fill in the data from S3. The volume is in STATUS_RESTORING until the data transfer completes. + The volume is created with the subsystem sharing its source had + (``Backup.max_namespace_per_subsys``): a member of a shared subsystem comes + back namespaced under the same cap, a volume that owned its subsystem comes + back owning one. A record from before the field was kept takes + add_lvol_ha's default. + Args: + consistency_group: Name of the consistency group the restored volume + joins AT CREATION, which is what lets a namespaced member take the + group's own subsystem (cg_colocation) rather than any subsystem of + the pool with a free slot. A group restore passes the group it is + forming; a single-volume restore leaves it unset. s3_credentials: Credentials for the backup's bucket, when that is not the cluster's own. Omit to use the nodes' instance role. target_node_id: Optional node to restore onto. If not provided, a node @@ -861,6 +877,9 @@ def restore_backup(backup_id: str, lvol_name: str, pool_id_or_name: str, # cannot run. s3_config = foreign_bucket_config(backup, cluster, s3_credentials) + # 0 is a record that predates the field: leave both unset, and add_lvol_ha + # gives the volume a subsystem of its own as it always did. + ns_cap = backup.max_namespace_per_subsys or 0 lvol_id, error = lvol_controller.add_lvol_ha( name=lvol_name, size=size, @@ -878,6 +897,9 @@ def restore_backup(backup_id: str, lvol_name: str, pool_id_or_name: str, distr_vuid=0, lvol_priority_class=0, fabric="tcp", + namespaced=ns_cap > 1, + max_namespace_per_subsys=ns_cap if ns_cap > 0 else None, + consistency_group=consistency_group, ) if error or not lvol_id: raise RuntimeError(f"Failed to create restore volume: {error}") @@ -972,9 +994,60 @@ def delete_backups(lvol_id): return True, None +def delete_backup(backup_id): + """Delete ONE backup: its record, its KMS keys and, best effort, its S3 + objects -- the single-record form of :func:`delete_backups`, for a backup + that nothing chains against any more (a pruned group generation's leaf + copy, design-consistency-group-backup.md §9). + + Raises: + KeyError: no backup has this id. + PreconditionError: another backup names this one as its predecessor, + so deleting it would make every descendant unrestorable -- merge it + into its successor instead (``trigger_merge``), after which it is + MERGED, not deleted, and its objects are the survivor's; or it is a + member of a group backup that still exists, which deleting it would + tear. + """ + backup = db_controller.get_backup_by_id(backup_id) + dependents = [b.uuid for b in db_controller.get_backups_by_lvol_id(backup.lvol_id) + if b.prev_backup_id == backup.uuid] + if dependents: + raise PreconditionError( + f"backup {backup.uuid} is the predecessor of " + f"{', '.join(d[:8] for d in dependents)}; merge it into its " + "successor instead of deleting it") + if backup.group_backup_id: + try: + owner = db_controller.get_group_backup_by_id(backup.group_backup_id) + except KeyError: + owner = None # its group backup is gone: an orphaned member copy + # A group backup being deleted retires its own members. + if owner is not None and owner.status != GroupBackup.STATUS_DELETING: + raise PreconditionError( + f"backup {backup.uuid} is a member of group backup " + f"{backup.group_backup_id}; delete the group backup instead") + + _cleanup_backup_kms_keys([backup]) + if backup.status == Backup.STATUS_COMPLETED: + try: + snode = db_controller.get_storage_node_by_id(backup.node_id) + except KeyError: + snode = None + if snode is not None and snode.status == StorageNode.STATUS_ONLINE: + try: + snode.rpc_client().bdev_lvol_s3_delete([backup.s3_id]) + except RPCException as e: + # Best effort, as delete_backups: the data plane does not + # implement the S3 delete yet, and the record goes regardless. + logger.error(f"Error deleting S3 objects of backup {backup.uuid}: {e}") + backup_events.backup_deleted(backup.cluster_id, backup.node_id, backup) + backup.remove(db_controller.kv_store) + + def list_backups(cluster_id=None, show_all=False): """List all backups, optionally filtered by cluster. - + Merged backups are hidden unless `show_all` is set -- a merge folds a backup's data into its successor, so the record stays for chain history but is no longer something an operator restores from. @@ -1154,6 +1227,7 @@ def import_backups(export: backup_manifest.BackupExport, cluster_id=None) -> int backup.node_id = str(manifest.source.node_id) backup.prev_backup_id = str(manifest.prev_backup_id) if manifest.prev_backup_id else "" backup.size = manifest.size + backup.max_namespace_per_subsys = manifest.volume.max_namespace_per_subsys or 0 backup.created_at = manifest.created_at backup.completed_at = manifest.completed_at backup.status = Backup.STATUS_COMPLETED @@ -1183,3 +1257,26 @@ def import_from_bucket(config: BackupConfig, cluster_id=None) -> int: manifests=discover_backups(config)), ]), cluster_id=cluster_id) + + +def trigger_merge(keep_backup, old_backup): + """Merge ``old_backup`` into ``keep_backup``, its successor in one volume's + chain: mark the old one merging and queue the merge task. + + Both must be completed; otherwise nothing happens and a later pass retries. + Used by the single-volume retention (policy) and by the group backup's + retirement of a pruned generation's members (group_backup). + """ + if old_backup.status != Backup.STATUS_COMPLETED: + return + if keep_backup.status != Backup.STATUS_COMPLETED: + return + + old_backup.status = Backup.STATUS_MERGING + old_backup.write_to_db() + + tasks_controller.add_backup_merge_task( + keep_backup.cluster_id, + keep_backup.node_id, + keep_backup.uuid, + old_backup.uuid) diff --git a/simplyblock_core/controllers/backup/group_backup.py b/simplyblock_core/controllers/backup/group_backup.py new file mode 100644 index 0000000000..ce4f40d80c --- /dev/null +++ b/simplyblock_core/controllers/backup/group_backup.py @@ -0,0 +1,998 @@ +"""Consistency-group backup: back up the member snapshots of one group +generation to S3 as one all-or-nothing unit (design-consistency-group-backup.md). + +This module holds the group-level decisions that sit on top of the per-member +single-volume backup path (``controllers/backup/controller.py``): the +homogeneity gate that the first version requires (§3.4) and the all-or-nothing +commit decision that makes a group backup visible only when every member +completed (§3.3). The orchestration around them -- :func:`create_group_backup` +resolving or taking a generation, creating the ``GroupBackup`` record, and +fanning out one single-volume backup per member, and +:func:`try_commit_group_backup` publishing the group manifest last once every +member finished -- lives here too and calls into the single-volume create flow +rather than reimplementing it. +""" +import time +from datetime import UTC, datetime +from enum import Enum +from uuid import UUID, uuid4 + +from simplyblock_core import db_controller, utils +from simplyblock_core.controllers import consistency_group_controller as cg +from simplyblock_core.controllers import lvol_controller, tasks_controller +from simplyblock_core.controllers.backup import chain as backup_chain +from simplyblock_core.controllers.backup import controller as backup_controller +from simplyblock_core.controllers.backup import manifest as backup_manifest +from simplyblock_core.exceptions import PreconditionError +from simplyblock_core.models.backup import Backup, GroupBackup + +logger = utils.get_logger(__name__) + +class CommitState(Enum): + """The commit decision for a group backup, from its members' statuses.""" + + #: At least one member is not yet terminal. + PENDING = "pending" + #: Every member completed: publish the group manifest. + READY = "ready" + #: A member failed terminally: fail the group as a unit. + FAILED = "failed" + + +def check_members_homogeneous(members: list) -> list: + """The members that break the first version's homogeneity requirement (§3.4). + + ``members`` is a list of ``{"lvol_id", "location", "encrypted"}``. Every + member must share one S3 location and one encryption disposition, which is + the existing per-chain coherence gate (a chain cannot span buckets or + encodings) applied across the group. Returns ``[(lvol_id, reason), ...]`` for + the members that do not match the first, named so a rejection points at the + offenders rather than failing anonymously. Empty means homogeneous. + """ + if not members: + return [] + + reference = members[0] + problems = [] + for member in members[1:]: + if member["location"] != reference["location"]: + problems.append((member["lvol_id"], "different backup location")) + elif member["encrypted"] != reference["encrypted"]: + problems.append((member["lvol_id"], "different encryption disposition")) + return problems + + +def group_commit_state(member_statuses: list) -> CommitState: + """The commit decision for a group backup, from its members' backup statuses. + + All-or-nothing (§3.3): a group backup commits (its group manifest is + published) only when every member backup has completed, and one terminal + member failure fails the whole group even while others are still uploading. + + * ``CommitState.FAILED`` -- any member is terminally failed. + * ``CommitState.READY`` -- there is at least one member and every member completed. + * ``CommitState.PENDING`` -- otherwise (a member is still pending or in progress, + or there are no members yet). + """ + if any(status == Backup.STATUS_FAILED for status in member_statuses): + return CommitState.FAILED + if member_statuses and all( + status == Backup.STATUS_COMPLETED for status in member_statuses): + return CommitState.READY + return CommitState.PENDING + + +def build_group_manifest(group_backup, member_backups: dict): + """Assemble the group manifest from the ``GroupBackup`` record and its member + ``Backup`` records (design §4.2). + + Pure: the caller supplies the records. ``member_backups`` maps + ``Backup.get_id()`` to the member ``Backup`` for every member in + ``group_backup.members``. Called at the commit barrier, once every member is + completed, so a missing member is a programming error, not a partial group. + + The group manifest is written LAST, after every member manifest exists, so + its presence is the all-or-nothing commit barrier (§3.3). It names the member + backups by id; a reader then reads each ``manifests/.json`` to + assemble that member's chain. The encryption keys are per-member and live in + those member manifests; the group manifest records only the shared + ``encrypted`` disposition (§3.4). + + Raises: + ValueError: a member backup is missing, or an id is not a UUID -- the + manifest types ids as UUID, and this is the seam where a malformed + one is caught while committing rather than during a recovery. + """ + member_backup_ids = [] + total_size = 0 + encrypted = False + for entry in group_backup.members: + backup = member_backups.get(entry["backup_id"]) + if backup is None: + raise ValueError( + f"member backup {entry['backup_id']} missing while committing " + f"group backup {group_backup.uuid}") + member_backup_ids.append(UUID(backup.uuid)) + total_size += backup.size + encrypted = encrypted or backup.encrypted + + return backup_manifest.GroupBackupManifest( + group_backup_id=UUID(group_backup.uuid), + created_at=datetime.fromtimestamp(group_backup.created_at, tz=UTC), + completed_at=datetime.fromtimestamp(group_backup.completed_at, tz=UTC), + size=total_size, + source=backup_manifest.GroupManifestSource( + cluster_id=UUID(group_backup.cluster_id), + consistency_group_id=group_backup.consistency_group_id, + consistency_group_name=group_backup.consistency_group_name, + generation=group_backup.group_seq, + ), + member_backup_ids=member_backup_ids, + encrypted=encrypted, + ) + + +def _config_for_group(group_backup): + """Credentials for the bucket a group backup's members live in (design §3.4). + + The group record keeps no location of its own: each member backup records + one, and they share it, so the group manifest's bucket is theirs. Each member + goes through the single-volume ``_config_for``, so one outside the cluster's + configured bucket is refused exactly as it would be on its own. + + Raises: + PreconditionError: no member backup record is left to locate the group + backup by, or a member lives in a bucket other than the cluster's + configured one -- the cluster's backup configuration was repointed + while the group backup was in flight. + """ + + db = db_controller.DBController() + config = None + for backup_id in group_backup.member_backup_ids(): + try: + member = db.get_backup_by_id(backup_id) + except KeyError: + continue + config = backup_controller._config_for(member) + if config is None: + raise PreconditionError( + f"Group backup {group_backup.uuid} has no member backup left to " + f"locate its bucket by") + return config + + +def create_group_backup(consistency_group_id, group_seq=None): + """Back up one generation of a consistency group as an all-or-nothing unit. + + Resolves the generation -- taking a fresh crash-consistent group snapshot + when ``group_seq`` is None -- gates its members on homogeneity (§3.4), + records a :class:`GroupBackup`, and fans out one single-volume backup per + member stamped with the group backup id and generation. The group manifest + that makes the backup visible is published later, by the commit task this + enqueues, once every member backup has completed (§3.3). Each member backup + is an ordinary single-volume backup, so the existing backup runner drives it + to a terminal state and the commit task only watches. + + Returns ``(group_backup_id, None)`` or ``(None, error_message)``. + """ + + db = db_controller.DBController() + try: + group = db.get_consistency_group_by_id(consistency_group_id) + except KeyError as e: + return None, str(e) + + cluster_id = group.cluster_id + if group_seq is None: + group_seq, err = cg.create_group_snapshot_for_group(group) + if err is not None: + return None, err + + # A torn generation is refused rather than backed up as a group that + # silently lacks a volume (§5). + try: + member_snaps = cg.resolve_complete_generation(group, group_seq) + except cg.ConsistencyGroupError as e: + return None, str(e) + + try: + location = db.get_cluster_by_id(cluster_id).get_backup_config().location() + except (KeyError, ValueError) as e: + return None, str(e) + + # Homogeneity gate (§3.4): every member must share one location and one + # encryption disposition, the per-chain coherence rule applied across the + # group. The location is the cluster's own and shared by construction; the + # encryption disposition is per volume, so this is where a group mixing an + # encrypted and a plaintext member is refused before anything is written. + location_dict = location.model_dump(mode="json") + member_facts = [ + {"lvol_id": s.lvol.get_id(), "location": location_dict, + "encrypted": bool(s.lvol.crypto_bdev)} + for s in member_snaps + ] + problems = check_members_homogeneous(member_facts) + if problems: + detail = ", ".join(f"{lvol_id} ({reason})" for lvol_id, reason in problems) + return None, (f"consistency group {consistency_group_id} is not " + f"homogeneous and cannot be backed up as one unit: {detail}") + + group_backup = GroupBackup() + group_backup.uuid = str(uuid4()) + group_backup.cluster_id = cluster_id + group_backup.consistency_group_id = group.get_id() + # group_name, not name: BaseModel.name is the class name (replication.py). + group_backup.consistency_group_name = group.group_name + group_backup.group_seq = group_seq + group_backup.created_at = int(time.time()) + group_backup.status = GroupBackup.STATUS_IN_PROGRESS + group_backup.write_to_db() + + members = [] + total_size = 0 + for snap in member_snaps: + backup_id, err = backup_controller.backup_snapshot(snap.get_id(), cluster_id) + if err is not None: + group_backup.status = GroupBackup.STATUS_FAILED + group_backup.error_message = ( + f"member {snap.lvol.get_id()} could not be backed up: {err}") + group_backup.write_to_db() + return None, group_backup.error_message + + backup = db.get_backup_by_id(backup_id) + # Store the group backup's get_id() ("/"), like the + # sibling lvol_id/snapshot_id fields, so get_backups_by_group_backup_id + # resolves the members of a group backup from its id. + backup.group_backup_id = group_backup.get_id() + backup.group_seq = group_seq + backup.write_to_db() + members.append({"lvol_id": snap.lvol.get_id(), "backup_id": backup.get_id()}) + total_size += backup.size + + group_backup.members = members + group_backup.size = total_size + group_backup.write_to_db() + + tasks_controller.add_group_backup_commit_task(group_backup) + return group_backup.get_id(), None + + +def try_commit_group_backup(group_backup_id) -> CommitState: + """The all-or-nothing commit barrier for a group backup (design §3.3). + + Reads the member backups, decides with :func:`group_commit_state`, and on + the first version's three outcomes: + + * ``CommitState.PENDING`` -- a member is still uploading; the caller defers. + * ``CommitState.FAILED`` -- a member failed terminally (or its record is gone); + the group is marked failed as a unit. + * ``CommitState.READY`` -- every member completed, so the group manifest is + published and only then is the record marked completed, which keeps the + manifest-before-``COMPLETED`` invariant the single-volume path keeps: a + ``COMPLETED`` group backup is always identifiable from the bucket alone. + + Idempotent: a group backup already in a terminal state returns that outcome + without touching S3, so the commit task is safe to re-run. Returns one of the + ``CommitState`` values. + + Raises: + ManifestError, PreconditionError: the group manifest could not be + published; the record is left in progress so the caller can retry. + """ + + db = db_controller.DBController() + try: + group_backup = db.get_group_backup_by_id(group_backup_id) + except KeyError: + return CommitState.FAILED + + if group_backup.status == GroupBackup.STATUS_COMPLETED: + return CommitState.READY + if group_backup.status == GroupBackup.STATUS_FAILED: + return CommitState.FAILED + + member_backups = {} + for entry in group_backup.members: + try: + backup = db.get_backup_by_id(entry["backup_id"]) + except KeyError: + continue + member_backups[backup.get_id()] = backup + + if len(member_backups) != len(group_backup.members): + return _fail_group(group_backup, "a member backup record is missing") + + state = group_commit_state([b.status for b in member_backups.values()]) + if state == CommitState.PENDING: + return CommitState.PENDING + if state == CommitState.FAILED: + return _fail_group(group_backup, "a member backup failed") + + # CommitState.READY: publish the group manifest LAST, before marking completed, + # so the record is never COMPLETED without its manifest in the bucket. + group_backup.completed_at = int(time.time()) + group_backup.size = sum(b.size for b in member_backups.values()) + manifest = build_group_manifest(group_backup, member_backups) + backup_manifest.write_group(_config_for_group(group_backup), manifest) + + group_backup.status = GroupBackup.STATUS_COMPLETED + group_backup.write_to_db() + return CommitState.READY + + +def _fail_group(group_backup, reason: str) -> CommitState: + group_backup.status = GroupBackup.STATUS_FAILED + group_backup.error_message = reason + group_backup.write_to_db() + return CommitState.FAILED + + +def _unwind_restored(lvol_ids: list) -> None: + """Tear down the volumes a failed group restore already created (design §6.5). + + Best-effort: every volume is attempted even if one teardown fails, so a + failed group restore leaves no half-group behind. The volumes are still + STATUS_RESTORING, so the teardown is forced. + """ + + db = db_controller.DBController() + for lvol_id in lvol_ids: + try: + lvol_controller.delete_lvol(db.get_lvol_by_id(lvol_id), force_delete=True) + except (KeyError, ValueError, PreconditionError) as e: + logger.warning("Group restore unwind: could not delete volume %s: %s", + lvol_id, e) + + +def restore_group_backup(group_backup_id, into_group_name, pool_id_or_name, + target_node_id=None, s3_credentials=None): + """Restore a group backup into a NEW consistency group (design §6). + + Recreates the N member volumes of one group backup from S3 and re-forms them + into a new standalone consistency group named ``into_group_name``, the + ``clone_generation(into_name=...)`` shape read from a backup rather than live + snapshots. Each member is an ordinary single-volume restore: it comes up + STATUS_RESTORING and its data recovery runs asynchronously, exactly as a + standalone restore does, so "all members online" is each member's restore + task reaching ONLINE, watched by the caller. + + All-or-nothing (design §6.5): the member chains are all validated restorable + before any volume is created, every restored volume is pinned to one store so + the group can form (§3.5), and if any member fails to restore or join, the + volumes already created are torn down so a failed group restore leaves no + half-group. + + Returns ``(new_group_id, members, None)`` or ``(None, None, error_message)``, + where ``members`` is ``[{"lvol_id", "member_name"}, ...]``, one per restored + volume, in member order. + """ + + db = db_controller.DBController() + try: + group_backup = db.get_group_backup_by_id(group_backup_id) + except KeyError as e: + return None, None, str(e) + + # Only a committed group backup is restorable: an in-progress or failed one + # is not a crash-consistent generation (design §3.3). + if group_backup.status != GroupBackup.STATUS_COMPLETED: + return None, None, ( + f"group backup {group_backup_id} is {group_backup.status}, not " + f"completed; only a committed group backup can be restored") + + if not into_group_name: + return None, None, "a restored consistency group needs a name" + + cluster_id = group_backup.cluster_id + # A group restore always produces a new group, and never deletes one that + # holds the name: an empty group is not known to be meant for replacing. + if db.get_consistency_group_by_name(cluster_id, into_group_name) is not None: + return None, None, ( + f"consistency group {into_group_name} already exists in cluster " + f"{cluster_id}; a group restore always produces a new group") + + # Resolve the member backups and validate every chain BEFORE creating any + # volume, so a doomed restore leaves nothing behind (design §6.2). The + # membership is complete by construction -- a group backup is COMPLETED only + # when every member committed -- but a member whose Backup record was since + # deleted is caught here. + try: + member_backups = [db.get_backup_by_id(entry["backup_id"]) + for entry in group_backup.members] + except KeyError as e: + return None, None, f"a member backup of the group is missing: {e}" + + all_backups = db.get_backups() + for backup in member_backups: + try: + chain = backup_chain.BackupChain.of_backups(UUID(backup.uuid), all_backups) + chain.require_restorable( + completed=True, + what=f"member {backup.lvol_name} of group backup {group_backup.uuid}") + except (PreconditionError, ValueError) as e: + return None, None, str(e) + + # Create the member volumes, pinned to one store so they can form a group. + # The first restore may auto-select a node; the rest are pinned to it, which + # add_member_to_group then enforces (design §3.5). Each member is created + # INTO the new group (add_lvol_ha's consistency_group): that is what lets a + # member whose source shared a subsystem take the group's own subsystem + # (cg_colocation), so the restored group is laid out as its source was -- + # one shared subsystem, members at NSIDs 1..N -- rather than one subsystem + # per member (observed 2026-10-20). + pin_node = target_node_id + created_ids: list = [] + members_out: list = [] + for backup in member_backups: + lvol_name = f"{into_group_name}_{backup.lvol_name}" + try: + lvol_id = backup_controller.restore_backup( + backup.uuid, lvol_name, pool_id_or_name, + target_node_id=pin_node, s3_credentials=s3_credentials, + consistency_group=into_group_name) + except (PreconditionError, RuntimeError, ValueError) as e: + _unwind_restored(created_ids) + return None, None, ( + f"member {backup.lvol_name} could not be restored: {e}") + created_ids.append(lvol_id) + # member_name is the source member's volume name, which the caller forms + # the restored claim's name from (the operator names it + # -) and uses to tell the members apart. + members_out.append({"lvol_id": lvol_id, "member_name": backup.lvol_name}) + if pin_node is None: + pin_node = db.get_lvol_by_id(lvol_id).node_id + + # Re-form the consistency group from the restored volumes. add_member_to_group + # enforces single-store placement and fails loudly on an off-store member, + # which the unwind then cleans up (design §6.4). The group we just created is + # removed on failure too, so the half-formed name does not block a retry. + group = cg.ensure_group(cluster_id, into_group_name) + for lvol_id in created_ids: + try: + lvol = db.get_lvol_by_id(lvol_id) + if getattr(lvol, "group_id", "") == group.get_id(): + continue # joined at creation; re-adding would open a fresh epoch + group = cg.add_member_to_group(group, lvol) + # Stamp the volume too: add_member_to_group writes only the group, + # and a member without group_id cannot be detached when it is + # deleted, so its entry outlives it and holds the group "in use". + lvol.group_id = group.get_id() + lvol.write_to_db(db.kv_store) + except (cg.ConsistencyGroupError, KeyError) as e: + _unwind_restored(created_ids) + db.get_consistency_group_by_id(group.get_id()).remove(db.kv_store) + return None, None, ( + f"restored volumes could not form group {into_group_name}: {e}") + + logger.info("Restored group backup %s into consistency group %s (%d members)", + group_backup.uuid, into_group_name, len(created_ids)) + return group.get_id(), members_out, None + + +class DeleteState(Enum): + """Where one pass of a group backup's delete left it.""" + #: A member is still being retired: a merge in flight, or a successor or + #: the member itself not yet completed. The task comes back. + PENDING = "pending" + #: Every member is retired and the manifest and record are gone, or the + #: group backup is no longer being deleted. + DONE = "done" + + +def delete_group_backup(group_backup_id): + """Start deleting a group backup (design §9), metadata first. + + A group backup is created data first and committed last; it is deleted the + other way round. This marks the group manifest and then the record as being + deleted, before any member is touched, and queues the task that retires the + members (:func:`advance_group_backup_delete`). The manifest and the record + are removed only once every member is retired. A member backup that the next + generation chains against is MERGED into it, so that generation stays + restorable; one nothing chains against is deleted. + + Deleting a group backup already being deleted re-queues its task if none is + open, and deleting a broken one resumes the retirement where it stopped. + + Raises: + KeyError: no group backup has this id. + PreconditionError: the group backup is still committing, or the bucket + its members live in is not the cluster's configured one. + ManifestError: the group manifest could not be marked; nothing changed. + """ + + db = db_controller.DBController() + group_backup = db.get_group_backup_by_id(group_backup_id) + if group_backup.status in (GroupBackup.STATUS_PENDING, GroupBackup.STATUS_IN_PROGRESS): + raise PreconditionError( + f"Group backup {group_backup.uuid} is still committing; delete it once " + f"it has completed or failed") + + if group_backup.status != GroupBackup.STATUS_DELETING: + # The bucket first: a crash after it leaves a manifest that says + # deleting over a record that does not, which the next delete repairs, + # rather than a record nothing will ever finish deleting. + if group_backup.manifest_published(): + _mark_manifest(group_backup, _config_for_deleting(db, group_backup), + backup_manifest.GroupManifestState.DELETING) + group_backup.status = GroupBackup.STATUS_DELETING + group_backup.error_message = "" + group_backup.write_to_db() + tasks_controller.add_group_backup_delete_task(group_backup) + + +def advance_group_backup_delete(group_backup_id) -> DeleteState: + """One pass of a group backup's delete: retire what members can be retired + now, and once all are, remove the group manifest and then the record. + + Idempotent, so the delete task re-runs it until it answers ``DONE``. + + Raises: + ManifestError, PreconditionError: the group manifest could not be + removed, or a member could not be retired; retryable. + """ + + db = db_controller.DBController() + try: + group_backup = db.get_group_backup_by_id(group_backup_id) + except KeyError: + return DeleteState.DONE + if group_backup.status != GroupBackup.STATUS_DELETING: + return DeleteState.DONE + + # Resolved before this pass retires anything: a leaf deleted below takes the + # record the bucket is found by with it. + config = _config_for_deleting(db, group_backup) + if not _retire_members(db, group_backup): + return DeleteState.PENDING + + if group_backup.manifest_published(): + backup_manifest.delete_retired_group(config, UUID(group_backup.uuid)) + group_backup.remove(db.kv_store) + logger.info("Group backup %s deleted: every member retired", group_backup.uuid) + return DeleteState.DONE + + +def abandon_group_backup_delete(group_backup_id, reason: str) -> None: + """Settle a group backup whose delete task ended unfinished: a timeout, the + retry ceiling, or a cancellation, none of which the task's handler sees. + + If no member was touched, the group backup is restored as it was, completed + with its manifest committed again. If one was, what is left cannot restore as + a group: it is marked broken, in the record and in the bucket, and deleting + it again resumes the retirement. Best effort on the manifest: the record is + written regardless, and a manifest left saying ``deleting`` is never read as + committed. + """ + + db = db_controller.DBController() + try: + group_backup = db.get_group_backup_by_id(group_backup_id) + except KeyError: + return + if group_backup.status != GroupBackup.STATUS_DELETING: + return + + published = group_backup.manifest_published() + if not published: + group_backup.status = GroupBackup.STATUS_FAILED + elif _members_touched(db, group_backup): + group_backup.status = GroupBackup.STATUS_BROKEN + else: + group_backup.status = GroupBackup.STATUS_COMPLETED + group_backup.error_message = f"delete did not finish: {reason}" + + if published: + state = (backup_manifest.GroupManifestState.BROKEN + if group_backup.status == GroupBackup.STATUS_BROKEN + else backup_manifest.GroupManifestState.COMMITTED) + try: + _mark_manifest(group_backup, _config_for_deleting(db, group_backup), state) + except (backup_manifest.ManifestError, PreconditionError) as e: + logger.error("Group backup %s is %s, but its manifest could not be marked " + "%s and still says deleting: %s", + group_backup.uuid, group_backup.status, state, e) + group_backup.write_to_db() + logger.warning("Group backup %s delete abandoned (%s): now %s", + group_backup.uuid, reason, group_backup.status) + + +def _mark_manifest(group_backup, config, state) -> None: + """Rewrite the group manifest in the bucket with ``state``, from the copy + the bucket holds, so a broken group backup's manifest keeps naming the + members it was committed with.""" + manifest = backup_manifest.read_group(config, UUID(group_backup.uuid)) + if manifest.state != state: + backup_manifest.write_group(config, manifest.model_copy(update={"state": state})) + + +def _config_for_deleting(db, group_backup): + """Credentials for a group backup's bucket while it is deleted. + + Its members' bucket, as :func:`_config_for_group` resolves it, while any + member record is left. Retiring the last leaf removes the last record, and + then it is the cluster's configured bucket: the one every member was checked + against when the delete was marked. + """ + for backup_id in group_backup.member_backup_ids(): + try: + db.get_backup_by_id(backup_id) + except KeyError: + continue + return _config_for_group(group_backup) + return db.get_cluster_by_id(group_backup.cluster_id).get_backup_config() + + +def _members_touched(db, group_backup) -> bool: + """Whether retiring a member has begun: a member deleted, merging, or merged.""" + for backup_id in group_backup.member_backup_ids(): + try: + member = db.get_backup_by_id(backup_id) + except KeyError: + return True + if member.status in (Backup.STATUS_MERGING, Backup.STATUS_MERGED): + return True + return False + + +def _retire_members(db, group_backup) -> bool: + """Retire what members of a deleting group backup can be retired now. + + A member nothing chains against is deleted. One the next generation's copy + chains against is merged into it once both are completed, one merge per + volume chain at a time, as the single-volume retention keeps it. Returns + whether every member is retired (deleted or merged). + """ + + retired = True + for backup_id in group_backup.member_backup_ids(): + try: + member = db.get_backup_by_id(backup_id) + except KeyError: + continue # deleted + if member.status == Backup.STATUS_MERGED: + continue + chain = db.get_backups_by_lvol_id(member.lvol_id) + successor = next((b for b in chain if b.prev_backup_id == member.uuid), None) + busy = (member.status in (Backup.STATUS_PENDING, Backup.STATUS_IN_PROGRESS) + or any(b.status == Backup.STATUS_MERGING for b in chain)) + if successor is None and not busy: + try: + backup_controller.delete_backup(member.uuid) + except KeyError: + pass # deleted concurrently + logger.info("Group backup %s: member %s deleted, nothing chains against it", + group_backup.uuid, member.uuid[:8]) + continue + retired = False + if (successor is not None and not busy + and member.status == Backup.STATUS_COMPLETED + and successor.status == Backup.STATUS_COMPLETED): + backup_controller.trigger_merge(successor, member) + logger.info("Group backup %s: member %s merging into its successor %s", + group_backup.uuid, member.uuid[:8], successor.uuid[:8]) + return retired + + +def retire_orphaned_member_backups(lvol_ids, db=None): + """Retire the member backups of group backups that no longer exist (design + §9): for each of *lvol_ids*, every completed backup whose ``group_backup_id`` + names a GroupBackup that is gone (pruned, or deleted by hand). + + * A backup the next generation chains against (a successor names it as + ``prev_backup_id``) is MERGED into that successor, the same + ``backup_controller.trigger_merge`` the single-volume schedule runs: the chain + stays restorable and the base's objects fold into the survivor. One + merge per volume chain, and none while one is already in flight there, + as the single-volume schedule keeps it; the next cycle takes the rest. + * A backup nothing chains against is deleted outright. + + Called by the group schedule every cycle, for member copies orphaned before + a group backup's delete retired its own members (:func:`delete_group_backup`). + A member of a group backup being deleted is not an orphan: its group backup + still exists, and its delete task retires it. + + Returns ``(merges_started, deleted)``. + """ + + if db is None: + db = db_controller.DBController() + merges = deleted = 0 + for lvol_id in lvol_ids: + chain = db.get_backups_by_lvol_id(lvol_id) + if any(b.status == Backup.STATUS_MERGING for b in chain): + continue + orphans = [] + for b in chain: + if b.status != Backup.STATUS_COMPLETED or not b.group_backup_id: + continue + try: + db.get_group_backup_by_id(b.group_backup_id) + except KeyError: + orphans.append(b) + by_prev = {b.prev_backup_id: b for b in chain if b.prev_backup_id} + # Oldest first: the base of the chain is the one whose merge unblocks + # the next, and a leaf is only a leaf once nothing is below it. + for old in sorted(orphans, key=lambda b: (b.created_at, b.uuid)): + successor = by_prev.get(old.uuid) + if successor is None: + try: + backup_controller.delete_backup(old.uuid) + except (KeyError, PreconditionError) as e: + logger.warning("Orphaned member backup %s of volume %s not deleted: %s", + old.uuid[:8], lvol_id[:8], e) + continue + deleted += 1 + logger.info("Retired orphaned member backup %s of volume %s: " + "nothing chains against it, deleted", old.uuid[:8], lvol_id[:8]) + continue + if successor.status != Backup.STATUS_COMPLETED: + break # let it finish; the next cycle merges + backup_controller.trigger_merge(successor, old) + merges += 1 + logger.info("Retiring orphaned member backup %s of volume %s: merging into " + "its successor %s", old.uuid[:8], lvol_id[:8], successor.uuid[:8]) + break # one merge per chain per cycle + return merges, deleted + + +def export_group_backup(group_backup_id, cluster_id=None): + """Export a group backup's manifests as a unit, for import into another + cluster (design §6.5). + + Emits the group manifest together with the member backup chains it names, as + one :class:`backup_manifest.GroupBackupExport`. The member chains are grouped + by the location they live in exactly as the single-volume + :func:`controller.export_backups` groups its manifests, so the same import + machinery lands the member ``Backup`` records; the group manifest is what ties + them back into a ``GroupBackup`` on the way in. + + The member manifests are each member's whole restorable chain, not just the + member backup itself: an import has to land every ancestor for the group to be + restorable, the unit ``export_backups(backup_id=...)`` produces for one volume. + + Args: + cluster_id: scopes the backup population the chains are walked over, as in + :func:`controller.export_backups`. The group backup's own cluster by + construction; passed through so a caller can state it. + + Returns a :class:`backup_manifest.GroupBackupExport`. + + Raises: + PreconditionError: the group backup is unknown, is not a committed unit, a + member backup record is missing, or a member's chain is broken. + ValueError: one of the selected backups records no usable location, so it + cannot be described. + """ + + db = db_controller.DBController() + try: + group_backup = db.get_group_backup_by_id(group_backup_id) + except KeyError as e: + raise PreconditionError(str(e)) from e + + # Only a committed group backup is a unit worth exporting: an in-progress or + # failed one has no published group manifest and is not crash-consistent + # (design §3.3), the same rule that gates restore. + if group_backup.status != GroupBackup.STATUS_COMPLETED: + raise PreconditionError( + f"group backup {group_backup_id} is {group_backup.status}, not " + f"completed; only a committed group backup can be exported") + + member_by_id = {} + for entry in group_backup.members: + try: + backup = db.get_backup_by_id(entry["backup_id"]) + except KeyError as e: + raise PreconditionError( + f"a member backup of group backup {group_backup_id} is " + f"missing: {e}") from e + member_by_id[backup.get_id()] = backup + + group_manifest = build_group_manifest(group_backup, member_by_id) + + # Each member's whole chain, collected and deduplicated across members: an + # import lands every ancestor so the group is restorable, the unit + # export_backups(backup_id=...) produces for one volume. + all_backups = db.get_backups(cluster_id) + chain_records = {} + for backup in member_by_id.values(): + try: + chain = backup_chain.BackupChain.of_backups(UUID(backup.uuid), all_backups) + except ValueError as e: + raise PreconditionError(str(e)) from e + for record in chain.records(): + chain_records[record.uuid] = record + + return backup_manifest.GroupBackupExport( + group_manifest=group_manifest, + members=backup_controller._group_by_location(chain_records.values()), + ) + + +def discover_group_backups(config): + """Every group backup in a bucket, read from its group manifests alone + (design §6.5). + + The disaster-recovery entry point for group backups, parallel to the + single-volume :func:`controller.discover_backups`: given a bucket and + credentials for it, this answers "what group backups are in here" with no + reference to any cluster, live or dead. Reads the ``groups/`` prefix through + :func:`backup_manifest.list_all_groups`. + + Raises: + ManifestError: the bucket could not be listed, or one of its group + manifests could not be parsed. + """ + return backup_manifest.list_all_groups(config) + + +def import_group_backup(export, cluster_id=None): + """Register a group backup described by a ``GroupBackupExport`` into this + cluster, preserving ids and all-or-nothing across the group (design §6.5). + + Recreates both the member ``Backup`` records AND the ``GroupBackup`` record + that ties them together, keeping every id: the member uuids and s3_ids (which + name the objects in the bucket and so cannot be reassigned), the group + backup's own uuid, and the generation. The whole group imports or none of it + does -- a group id collision, a member id collision, or a broken member chain + refuses the batch before the group record is written. + + The member backups go in through the single-volume + :func:`controller.import_backups`, which is where the chain validation and the + member-id-collision rule live; this adds the group record on top and stamps + each imported member with the group backup id and generation that the member + manifests do not themselves carry. + + Args: + export: a validated :class:`backup_manifest.GroupBackupExport`, from + :func:`export_group_backup`, a bucket read, or a file parsed into one. + cluster_id: the cluster to import into, so the records are visible in its + namespace. The group manifest's own source cluster when omitted. + + Returns the imported group backup's composite ``cluster_id/uuid`` id. + + Raises: + PreconditionError: the group backup id is already known, or the member + import's own preconditions fail (a member id collision, a broken or + over-long chain). + ValueError: the group names a member whose manifest the export does not + carry, so the group would import incomplete. + """ + + db = db_controller.DBController() + group_manifest = export.group_manifest + target_cluster = cluster_id or str(group_manifest.source.cluster_id) + uuid = str(group_manifest.group_backup_id) + composite_id = f"{target_cluster}/{uuid}" + + # Only a committed group backup is whole. One being deleted on its source, + # or broken by a delete that did not finish, has members that are retired + # or about to be. + if group_manifest.state != backup_manifest.GroupManifestState.COMMITTED: + raise PreconditionError( + f"Group backup {uuid} is {group_manifest.state} in its bucket, not " + f"committed; it cannot be imported") + + # Reject a group id collision before writing anything -- the group analog of + # import_backups' per-backup collision check. Group-backup lookups resolve by + # uuid, so a reused uuid would make either record unaddressable. + try: + existing = db.get_group_backup_by_id(composite_id) + except KeyError: + pass + else: + raise PreconditionError( + f"Group backup {uuid} already exists in cluster {existing.cluster_id}") + + # Every member the group names has to be present in the export: a group that + # imported without one of its members would be a torn generation that looks + # complete in `group-backup list` and fails only at restore. + exported = { + manifest.backup_id + for group in export.members.groups for manifest in group.manifests + } + missing = [backup_id for backup_id in group_manifest.member_backup_ids + if backup_id not in exported] + if missing: + raise ValueError( + "group export is missing member manifests for " + + ", ".join(str(b) for b in missing)) + + # Land the member backups all-or-nothing (chains, id collisions) on the + # single-volume path. Nothing above this wrote a record, so a refusal here + # leaves the import empty. + backup_controller.import_backups(export.members, cluster_id=target_cluster) + + # Stamp each imported member with the group it belongs to -- the member + # manifests do not carry it, so import_backups left it blank -- and the + # generation, both preserved from the group manifest. + members = [] + for backup_id in group_manifest.member_backup_ids: + backup = db.get_backup_by_id(str(backup_id)) + backup.group_backup_id = composite_id + backup.group_seq = group_manifest.source.generation + backup.write_to_db() + # The member's volume comes from its own manifest, which import_backups + # recorded on the backup. + members.append({"lvol_id": backup.lvol_id, "backup_id": backup.get_id()}) + + # Recreate the GroupBackup record, preserving its uuid and generation. It is + # imported COMPLETED: only a committed group backup is exportable, so what + # arrives is always a committed one. + group_backup = GroupBackup() + group_backup.uuid = uuid + group_backup.cluster_id = target_cluster + group_backup.consistency_group_id = group_manifest.source.consistency_group_id + group_backup.consistency_group_name = group_manifest.source.consistency_group_name + group_backup.group_seq = group_manifest.source.generation + group_backup.members = members + group_backup.size = group_manifest.size + group_backup.created_at = int(group_manifest.created_at.timestamp()) + group_backup.completed_at = int(group_manifest.completed_at.timestamp()) + group_backup.status = GroupBackup.STATUS_COMPLETED + group_backup.write_to_db() + + return group_backup.get_id() + + +def import_group_backup_from_bucket(config, cluster_id=None): + """Import every group backup a bucket holds (design §6.5). + + The disaster-recovery path: given a bucket and credentials for it, read its + group manifests and the member manifests they name, and import each group + backup. Reuses the single-volume bucket read for the member manifests, since a + group's members live in the same bucket (§3.4), and walks each member's chain + over them so a group imports exactly its own members' chains rather than every + backup the bucket happens to hold. + + Each group backup is imported atomically (:func:`import_group_backup`); a + collision or broken chain refuses that group backup, and the ones imported + before it remain. + + Returns the composite ids of the group backups imported. + + Raises: + ManifestError: the bucket could not be read. + PreconditionError: a group backup it holds cannot be imported. + ValueError: a group manifest names a member the bucket does not hold. + """ + + group_manifests = backup_manifest.list_all_groups(config) + member_manifests = backup_manifest.list_all(config) + location = config.location() + by_id = {manifest.backup_id: manifest for manifest in member_manifests} + + imported = [] + for group_manifest in group_manifests: + # A bucket holds group backups being deleted on their source too; they + # are not part of what it can restore, so they are left, not refused. + if group_manifest.state != backup_manifest.GroupManifestState.COMMITTED: + logger.info("Group backup %s is %s in the bucket; not importing it", + group_manifest.group_backup_id, group_manifest.state) + continue + # Each named member's whole chain, gathered from the bucket's member + # manifests. Walked rather than taking every manifest in the bucket, which + # would pull in other group backups' and standalone backups' chains. + chain_manifests = {} + for backup_id in group_manifest.member_backup_ids: + head = by_id.get(backup_id) + if head is None: + raise ValueError( + f"group backup {group_manifest.group_backup_id} names member " + f"backup {backup_id}, which the bucket does not hold") + # of_manifests builds its links from manifests; the isinstance narrows + # the Backup|BackupManifest link type back to the manifest it is. + for link in backup_chain.BackupChain.of_manifests(head, member_manifests, location).links: + if isinstance(link, backup_manifest.BackupManifest): + chain_manifests[link.backup_id] = link + + export = backup_manifest.GroupBackupExport( + group_manifest=group_manifest, + members=backup_manifest.BackupExport(groups=[ + backup_manifest.LocatedManifests( + location=location, manifests=list(chain_manifests.values()))]), + ) + imported.append(import_group_backup(export, cluster_id=cluster_id)) + + return imported diff --git a/simplyblock_core/controllers/backup/manifest.py b/simplyblock_core/controllers/backup/manifest.py index 91e3255786..4e13d0c797 100644 --- a/simplyblock_core/controllers/backup/manifest.py +++ b/simplyblock_core/controllers/backup/manifest.py @@ -41,14 +41,25 @@ """ import json import logging +from collections.abc import Iterable +from enum import StrEnum from typing import Annotated, Literal, Union from uuid import UUID import boto3 from botocore.config import Config as BotoConfig from botocore.exceptions import BotoCoreError, ClientError -from pydantic import BaseModel, ConfigDict, Field, HttpUrl, SecretStr, TypeAdapter - +from pydantic import ( + AwareDatetime, + BaseModel, + ConfigDict, + Field, + HttpUrl, + SecretStr, + TypeAdapter, +) + +from simplyblock_core.exceptions import PreconditionError from simplyblock_core.kms import KMS from simplyblock_core.models.backup_config import BackupConfig, BackupLocation from simplyblock_core.utils.secrets import unwrap_secret @@ -120,6 +131,12 @@ class ManifestVolume(BaseModel): ha_type: Literal["single", "ha"] | None = None fabric: Literal["tcp", "rdma", "tcp,rdma"] | None = None + #: The namespace cap of the subsystem the volume was created with: 1 is a + #: subsystem of its own, more is a namespaced volume sharing one. Unlike the + #: settings below it comes off the backup record, so it survives the volume + #: and a restore -- including an imported one -- recreates the same sharing. + #: Absent for a backup recorded before the field existed. + max_namespace_per_subsys: int | None = None lvol_priority_class: int | None = None max_size: int | None = None rw_ios_per_sec: int | None = None @@ -457,3 +474,310 @@ def parse_export(body: bytes, source: str) -> BackupExport: return BackupExport.model_validate(data) except ValueError as e: raise ManifestError(f"{source} is malformed: {e}") from e + + +# --------------------------------------------------------------------------- +# Group backup manifest +# +# The self-describing part of a CONSISTENCY-GROUP backup: one JSON document per +# group backup, under a "groups/" prefix, naming the N member backups of one +# generation as a unit. It is written LAST, after every member manifest exists, +# so its presence is the group backup's all-or-nothing commit barrier +# (design-consistency-group-backup.md §3.3, §4.2). A reader resolves a group +# backup from this document, then reads each member's manifests/.json +# to assemble that member's chain. Like the per-backup manifest, it records +# identity and encoding, never the bucket, region, endpoint, or credentials. +# --------------------------------------------------------------------------- + +GROUP_MANIFEST_PREFIX = "groups/" +GROUP_MANIFEST_SCHEMA_VERSION = 1 + + +def group_manifest_key(group_backup_id: UUID) -> str: + return f"{GROUP_MANIFEST_PREFIX}{group_backup_id}.json" + + +class GroupManifestSource(BaseModel): + """Where a group backup's generation came from. Provenance for a reader.""" + model_config = ConfigDict(extra="forbid") + + cluster_id: UUID + #: ``ConsistencyGroup.get_id()``: the composite "/". + consistency_group_id: str + consistency_group_name: str + #: The generation (``group_seq``) this backup was taken from. + generation: int + #: Absent when the cluster's own record of its name was unreadable. + cluster_name: str | None = None + + +class GroupManifestState(StrEnum): + """Whether a group manifest's member backups can be relied on (design §9). + + A group manifest is written ``committed``, at the commit barrier. A delete + marks it ``deleting`` before it touches any member, and removes it only once + every member is retired. A delete that ended after retiring a member marks it + ``broken``: some members are gone, so the group backup cannot restore, and + the bucket says so rather than presenting it as whole. Only a ``committed`` + group backup is restorable or importable. + """ + COMMITTED = "committed" + DELETING = "deleting" + BROKEN = "broken" + + +class GroupBackupManifest(BaseModel): + """The self-describing document for one consistency-group backup.""" + model_config = ConfigDict(extra="forbid") + + schema_version: int = GROUP_MANIFEST_SCHEMA_VERSION + group_backup_id: UUID + created_at: AwareDatetime + #: When the group backup committed. A group manifest is written only at + #: commit, so this is always set. + completed_at: AwareDatetime + #: The sum of the member backups' sizes, in bytes. + size: Annotated[int, Field(ge=0)] + + source: GroupManifestSource + #: The member backups, one per member volume of the generation. Everything + #: about a member -- its volume, s3 id, chain link, key -- is in its own + #: manifest, ``manifests/.json``. + member_backup_ids: list[UUID] + + #: Whether the member backups can be relied on; see GroupManifestState. A + #: manifest written before deletes marked it reads as committed, which is + #: what it was. + state: GroupManifestState = GroupManifestState.COMMITTED + + #: Whether the members are encrypted. Homogeneous (design §3.4): every member + #: shares this disposition. The actual per-member keys are NOT here -- each + #: member has its own DEK, recorded in its own manifest + #: (``manifests/.json``) -- so this is only the group-level + #: statement that a reader must resolve per-member keys before a restore. + encrypted: bool = False + + def is_complete(self, member_manifests: Iterable[BackupManifest]) -> bool: + """Whether a bucket holding ``member_manifests`` holds this group backup + whole: a manifest for every member it names. + + The group manifest is written only once every member completed, so a + torn group backup is one whose member manifests have since gone missing, + which the group manifest alone cannot show. + """ + present = {m.backup_id for m in member_manifests} + return all(backup_id in present for backup_id in self.member_backup_ids) + + +def write_group(config: BackupConfig, manifest: GroupBackupManifest) -> None: + """Store a group manifest, next to the member backups it names. + + Written LAST, after every member manifest exists: its presence is the group + backup's commit barrier (design §3.3). + + Raises: + ManifestError: it could not be stored. A group backup with member + manifests but no group manifest is not a committed group backup. + """ + try: + s3_client(config).put_object( + Bucket=config.bucket_name, + Key=group_manifest_key(manifest.group_backup_id), + Body=manifest.model_dump_json().encode(), + ContentType="application/json", + ) + except (BotoCoreError, ClientError) as e: + raise ManifestError( + "Failed to write group manifest for group backup " + f"{manifest.group_backup_id}") from e + + logger.info("Wrote group manifest for group backup %s to %s", + manifest.group_backup_id, config.bucket_name) + + +def read_group(config: BackupConfig, group_backup_id: UUID) -> GroupBackupManifest: + """Load one group manifest by group backup id. + + Raises: + ManifestError: it is absent, unreadable, or a schema version this build + does not understand. + """ + try: + body = s3_client(config).get_object( + Bucket=config.bucket_name, + Key=group_manifest_key(group_backup_id))["Body"].read() + except (BotoCoreError, ClientError) as e: + raise ManifestError( + f"Failed to read group manifest for group backup {group_backup_id} " + f"from {config.bucket_name}") from e + + return _parse_group(body, group_manifest_key(group_backup_id)) + + +def list_all_groups(config: BackupConfig) -> list[GroupBackupManifest]: + """Every group manifest in the bucket, newest first. + + The disaster-recovery entry point for group backups, parallel to + :func:`list_all`. + + Raises: + ManifestError: the bucket could not be listed, or one group manifest + could not be parsed. Not best-effort, for :func:`list_all`'s reason. + """ + client = s3_client(config) + manifests = [] + + try: + pages = client.get_paginator("list_objects_v2").paginate( + Bucket=config.bucket_name, Prefix=GROUP_MANIFEST_PREFIX) + for page in pages: + for entry in page.get("Contents", []): + key = entry["Key"] + if not key.endswith(".json"): + continue + body = client.get_object(Bucket=config.bucket_name, Key=key)["Body"].read() + manifests.append(_parse_group(body, key)) + except (BotoCoreError, ClientError) as e: + raise ManifestError( + f"Failed to list group manifests in {config.bucket_name}") from e + + manifests.sort(key=lambda m: (m.created_at, m.group_backup_id), reverse=True) + return manifests + + +def delete_retired_group(config: BackupConfig, group_backup_id: UUID) -> None: + """Remove the group manifest of a group backup whose delete has retired + every member: the last bucket step of deleting a group backup (design §9). + + The group manifest is not a link in any backup chain. A member manifest + names its own predecessor, never a group manifest, so removing one breaks + no chain. What it does is un-commit the group backup: the bucket stops + naming its members as one restorable generation, and that cannot be undone + from the bucket. So only a manifest a delete has marked ``deleting`` + (``GroupManifestState.DELETING``) is removed. The delete marks it before it + touches any member and calls this once every member is retired, which the + control plane's records, not the bucket, can tell (a deleted member's + manifest stays in the bucket as the only record of its objects). A + committed or broken manifest is refused and left in place. An absent one + is already removed, which is what a retried delete pass finds. + + Raises: + PreconditionError: the manifest is not marked ``deleting``. + ManifestError: the manifest could not be read, parsed, or removed. + """ + manifest = _read_group_if_present(config, group_backup_id) + if manifest is None: + return + if manifest.state != GroupManifestState.DELETING: + raise PreconditionError( + f"Group manifest of group backup {group_backup_id} is {manifest.state}, " + f"not {GroupManifestState.DELETING}; only a group backup whose delete " + f"has retired its members may lose its manifest") + _delete_group_object(config, group_backup_id) + + +def _read_group_if_present(config: BackupConfig, group_backup_id: UUID) -> GroupBackupManifest | None: + """:func:`read_group`, but None for a manifest the bucket does not hold.""" + try: + body = s3_client(config).get_object( + Bucket=config.bucket_name, + Key=group_manifest_key(group_backup_id))["Body"].read() + except ClientError as e: + if e.response.get("Error", {}).get("Code") in ("NoSuchKey", "404"): + return None + raise ManifestError( + f"Failed to read group manifest for group backup {group_backup_id} " + f"from {config.bucket_name}") from e + except BotoCoreError as e: + raise ManifestError( + f"Failed to read group manifest for group backup {group_backup_id} " + f"from {config.bucket_name}") from e + return _parse_group(body, group_manifest_key(group_backup_id)) + + +def _delete_group_object(config: BackupConfig, group_backup_id: UUID) -> None: + """Remove a group manifest object, unconditionally. + + Private: removing a group manifest un-commits its group backup, and nothing + here checks that may happen. :func:`delete_retired_group` is the only + caller, after it has. + """ + try: + s3_client(config).delete_object( + Bucket=config.bucket_name, + Key=group_manifest_key(group_backup_id)) + except (BotoCoreError, ClientError) as e: + raise ManifestError( + "Failed to delete group manifest for group backup " + f"{group_backup_id}") from e + + +def _parse_group(body: bytes, key: str) -> GroupBackupManifest: + try: + data = json.loads(body) + except ValueError as e: + raise ManifestError(f"Group manifest {key} is not valid JSON") from e + + version = data.get("schema_version") if isinstance(data, dict) else None + if version != GROUP_MANIFEST_SCHEMA_VERSION: + # Refuse rather than guess, for the reason `_parse` refuses a manifest. + raise ManifestError( + f"Group manifest {key} has schema version {version}, " + f"this build understands {GROUP_MANIFEST_SCHEMA_VERSION}") + + try: + return GroupBackupManifest.model_validate(data) + except ValueError as e: + raise ManifestError(f"Group manifest {key} is malformed: {e}") from e + + +GROUP_EXPORT_SCHEMA_VERSION = 1 + + +class GroupBackupExport(BaseModel): + """A group backup carried out of a cluster in a file: the group manifest plus + the member backup manifests it names, as one unit (design §6.5). + + This is to the group manifest what ``BackupExport`` is to a single-volume + manifest -- the one document that does record a location, because it is not + stored in the bucket it describes. The group manifest names the generation's + members; ``members`` carries each member's whole restorable chain, grouped by + the location it lives in exactly as a single-volume export groups its + manifests. A group backup is homogeneous -- its members share one location + (§3.4) -- so there is one group in practice, but reusing ``BackupExport`` keeps + the member records on the single-volume import path and describes a member's + chain the same way everywhere. + """ + model_config = ConfigDict(extra="forbid") + + schema_version: int = GROUP_EXPORT_SCHEMA_VERSION + group_manifest: GroupBackupManifest + members: BackupExport + + +def parse_group_export(body: bytes, source: str) -> GroupBackupExport: + """Read a group export document, naming *source* in anything it rejects. + + The group analog of :func:`parse_export`. + + Raises: + ManifestError: It is not valid JSON, states a schema version this build + does not understand, or is not shaped like a group export. + """ + try: + data = json.loads(body) + except ValueError as e: + raise ManifestError(f"{source} is not valid JSON") from e + + version = data.get("schema_version") if isinstance(data, dict) else None + if version != GROUP_EXPORT_SCHEMA_VERSION: + # Refused rather than guessed at, for the reason `_parse` refuses a + # manifest: the field names may not mean the same thing. + raise ManifestError( + f"{source} has group export schema version {version}, " + f"this build understands {GROUP_EXPORT_SCHEMA_VERSION}") + + try: + return GroupBackupExport.model_validate(data) + except ValueError as e: + raise ManifestError(f"{source} is malformed: {e}") from e diff --git a/simplyblock_core/controllers/backup/policy.py b/simplyblock_core/controllers/backup/policy.py index f2858855d9..83c12717d4 100644 --- a/simplyblock_core/controllers/backup/policy.py +++ b/simplyblock_core/controllers/backup/policy.py @@ -9,11 +9,17 @@ import time import uuid -from simplyblock_core.controllers import tasks_controller from simplyblock_core.controllers.backup import controller as backup_controller +from simplyblock_core.controllers.backup import group_backup +from simplyblock_core.controllers.backup.manifest import ManifestError from simplyblock_core.db_controller import DBController from simplyblock_core.exceptions import PreconditionError -from simplyblock_core.models.backup import Backup, BackupPolicy, BackupPolicyAttachment +from simplyblock_core.models.backup import ( + Backup, + BackupPolicy, + BackupPolicyAttachment, + GroupBackup, +) from simplyblock_core.models.cluster import Cluster logger = logging.getLogger() @@ -129,13 +135,16 @@ def attach_policy(policy_id, target_type, target_id): except KeyError as e: return None, str(e) - if target_type not in ("pool", "lvol"): - return None, f"Invalid target_type: {target_type}. Use 'pool' or 'lvol'" + if target_type not in ("pool", "lvol", "consistency_group"): + return None, (f"Invalid target_type: {target_type}. " + f"Use 'pool', 'lvol', or 'consistency_group'") # Validate target exists try: if target_type == "pool": db_controller.get_pool_by_id(target_id) + elif target_type == "consistency_group": + db_controller.get_consistency_group_by_id(target_id) else: db_controller.get_lvol_by_id(target_id) except KeyError as e: @@ -214,7 +223,7 @@ def evaluate_policy(lvol): if versions_exceeded or age_exceeded: oldest = completed[0] second = completed[1] - _trigger_merge(second, oldest) + backup_controller.trigger_merge(second, oldest) def evaluate_schedule(lvol): @@ -288,10 +297,81 @@ def evaluate_schedule(lvol): tier_backups.sort(key=lambda b: b.created_at) oldest = tier_backups[0] second = tier_backups[1] - _trigger_merge(second, oldest) + backup_controller.trigger_merge(second, oldest) return # One merge per cycle to avoid conflicts +def evaluate_group_backups(group): + """Back up and prune a consistency group per its backup policy (design §9). + + One pass, three steps. Member copies orphaned by a delete from before group + backups retired their own members are swept. A generation is taken when the + schedule's smallest interval has elapsed since the last one, and the pass + ends there so the take commits before anything is pruned. Otherwise the + generations either retention rule selects are deleted: the count/age rule + (``max_versions``, ``max_age``) and the schedule's tiers (``keep_count`` per + age band). A group backup is the retention unit, so the rules count + generations, and the delete retires each one's member backups. + """ + policy = db_controller.get_policy_for_group(group) + if not policy: + return + + try: + tiers = _parse_schedule(policy.backup_schedule) if policy.backup_schedule else [] + except ValueError: + tiers = [] + now = int(time.time()) + group_backups = db_controller.get_group_backups_by_consistency_group(group.get_id()) + completed = [g for g in group_backups if g.status == GroupBackup.STATUS_COMPLETED] + in_flight = [g for g in group_backups + if g.status in (GroupBackup.STATUS_PENDING, GroupBackup.STATUS_IN_PROGRESS)] + + group_backup.retire_orphaned_member_backups( + [lid for lid, m in (group.members or {}).items() if m.get("removed_seq", 0) == 0]) + + # One group backup at a time: none while one is still committing, the same + # gate the single-volume schedule keeps. + if tiers and not in_flight: + latest = max((g.created_at for g in completed), default=None) + if latest is None or now - latest >= tiers[0][0]: + _, err = group_backup.create_group_backup(group.get_id()) + if err: + logger.warning("Scheduled group backup skipped for group %s: %s", + group.uuid, err) + return + + for gb in _group_backups_to_prune(completed, policy, tiers, now): + try: + group_backup.delete_group_backup(gb.get_id()) + except (KeyError, PreconditionError, ManifestError) as e: + logger.warning("Group retention could not delete group backup %s: %s", gb.uuid, e) + + +def _group_backups_to_prune(completed, policy, tiers, now): + """The completed group backups either retention rule deletes, oldest first. + + The count/age rule selects every generation beyond ``max_versions`` and every + one older than ``max_age``. The tiers select one generation per pass: the + oldest of the first age band holding more than its ``keep_count``. + """ + completed = sorted(completed, key=lambda g: g.created_at) + doomed = [] + if policy.max_versions > 0 and len(completed) > policy.max_versions: + doomed.extend(completed[:len(completed) - policy.max_versions]) + if policy.max_age_seconds > 0: + doomed.extend(gb for gb in completed if (now - gb.created_at) > policy.max_age_seconds) + for tier_idx, (_interval, keep_count) in enumerate(tiers): + lower_age = 0 if tier_idx == 0 else tiers[tier_idx - 1][0] + upper_age = tiers[tier_idx + 1][0] if tier_idx + 1 < len(tiers) else float('inf') + band = [g for g in completed if lower_age <= (now - g.created_at) < upper_age] + if len(band) > keep_count: + doomed.append(band[0]) + break + unique = {gb.uuid: gb for gb in doomed} + return sorted(unique.values(), key=lambda g: g.created_at) + + def _auto_backup_lvol(lvol): """Create an automatic snapshot, then back up its whole unbacked ancestry. @@ -309,18 +389,3 @@ def _auto_backup_lvol(lvol): logger.warning(f"Auto-backup skipped for lvol {lvol.get_id()}: {e}") -def _trigger_merge(keep_backup, old_backup): - """Trigger a merge of old_backup into keep_backup.""" - if old_backup.status != Backup.STATUS_COMPLETED: - return - if keep_backup.status != Backup.STATUS_COMPLETED: - return - - old_backup.status = Backup.STATUS_MERGING - old_backup.write_to_db() - - tasks_controller.add_backup_merge_task( - keep_backup.cluster_id, - keep_backup.node_id, - keep_backup.uuid, - old_backup.uuid) diff --git a/simplyblock_core/controllers/consistency_group_controller.py b/simplyblock_core/controllers/consistency_group_controller.py index 22103c92d5..9831ba0efe 100644 --- a/simplyblock_core/controllers/consistency_group_controller.py +++ b/simplyblock_core/controllers/consistency_group_controller.py @@ -51,6 +51,11 @@ class ConsistencyGroupError(Exception): pass +class IncompleteGenerationError(ConsistencyGroupError): + """A generation lacks an online snapshot of a volume that was a member at + that sequence.""" + + # --------------------------------------------------------------------------- # # Group lifecycle (driven by the policy controller) # --------------------------------------------------------------------------- # @@ -93,14 +98,34 @@ def delete_group(group): fail-back could resolve no members. Raises ConsistencyGroupError when the group still has an open member. + + An open member whose volume record no longer exists is dangling, not live: + a volume that joined through :func:`add_member_to_group` alone carried no + ``group_id``, so its deletion could not close its epoch, and the entry + outlived the volume (restored members, live 2026-10-20: "still has 3 + member(s)" for three volumes long gone). Dangling entries do not hold the + group; they are dropped with it. """ open_members = [lid for lid, m in (group.members or {}).items() if m.get("removed_seq", 0) == 0] - if open_members: + live, dangling = [], [] + for lid in open_members: + try: + db.get_lvol_by_id(lid) + except KeyError: + dangling.append(lid) + else: + live.append(lid) + if live: raise ConsistencyGroupError( f"consistency group {group.group_name or group.uuid[:8]} still has " - f"{len(open_members)} member(s); detach or hand them off before " + f"{len(live)} member(s); detach or hand them off before " f"deleting it") + if dangling: + logger.warning("Consistency group %s (%s): %d open member(s) have no volume " + "record any more (%s); dropping them with the group", + group.uuid[:8], group.group_name or "-", len(dangling), + ", ".join(d[:8] for d in dangling)) group.remove(db.kv_store) logger.info("Deleted consistency group %s (%s)", group.uuid[:8], group.group_name or "-") @@ -886,20 +911,14 @@ def demote_group(group): lvol_controller.replication_failback(m.get_id()) group = db.get_consistency_group_by_id(group.get_id()) - created_ids, err = create_group_snapshot_for_group(group) + group_seq, err = create_group_snapshot_for_group(group) if err: return {"demoted": False, "error": err, "members": [{"lvol_id": m.get_id(), "demoted": False, "error": err} for m in members]} - by_lvol = {} - for sid in created_ids or []: - try: - s = db.get_snapshot_by_id(sid) - except KeyError: - continue - if s.lvol: - by_lvol[s.lvol.get_id()] = sid + by_lvol = {s.lvol.get_id(): s.get_id() + for s in generation_snapshots(group, group_seq) if s.lvol} for m in members: sid = by_lvol.get(m.get_id()) if not sid: @@ -1030,14 +1049,38 @@ def create_group_snapshot(policy_id, snap_type=SnapShot.TYPE_INTERNAL, lock=True return create_group_snapshot_for_group(group, snap_type=snap_type, lock=lock) -def create_group_snapshot_for_group(group, snap_type=SnapShot.TYPE_INTERNAL, lock=True): +def generation_for_request(group, request_id): + """The sequence of the generation taken with ``request_id``. + + None when no generation of this group carries the id, and always None for + an empty id, so a take without an id never matches one. + """ + if not request_id: + return None + return next((s.group_seq for s in _group_snapshots(group) + if s.group_request_id == request_id), None) + + +def create_group_snapshot_for_group(group, snap_type=SnapShot.TYPE_INTERNAL, lock=True, + request_id=""): """Take ONE crash-consistent snapshot of every group member. - Returns (list_of_snapshot_ids, None) or (None, error). All-or-nothing: + Returns (group_seq, None) or (None, error): the generation is what was + taken, and generation_snapshots(group, group_seq) its member snapshots. + All-or-nothing: a failure anywhere rolls back every snapshot bdev of this tick (SPDK already GC'd if the failure was inside the RPC; registration/record failures are rolled back here) and the generation counter does not move. + + ``request_id`` makes the take idempotent for a caller that may retry it + after losing the answer: it is stamped on every member snapshot, and a take + with an id some generation of this group already carries returns that + generation without taking another. """ + taken = generation_for_request(group, request_id) + if taken is not None: + return taken, None + members, precheck_err = _precheck_members(group) if precheck_err is not None: return None, precheck_err @@ -1080,6 +1123,11 @@ def create_group_snapshot_for_group(group, snap_type=SnapShot.TYPE_INTERNAL, loc # so a take that just released it is seen. with object_mutation_lock(pool.cluster_id, group.get_id(), enabled=lock): group = db.get_consistency_group_by_id(group.get_id()) + # Checked again under the lock: a concurrent take with the same id may + # have committed between the check above and acquiring it. + taken = generation_for_request(group, request_id) + if taken is not None: + return taken, None group_seq = group.last_group_seq + 1 now_ts = int(time.time()) plan = [] @@ -1195,6 +1243,7 @@ def _rollback_all(): snap.snap_type = snap_type snap.group_id = group.get_id() snap.group_seq = group_seq + snap.group_request_id = request_id snap.create_dt = str(datetime.now()) snap.write_to_db(db.kv_store) @@ -1225,7 +1274,7 @@ def _rollback_all(): logger.info("Consistency group %s: generation %d complete (%d snapshots)", group.uuid[:8], group_seq, len(created_ids)) - return created_ids, None + return group_seq, None # --------------------------------------------------------------------------- # @@ -1233,9 +1282,11 @@ def _rollback_all(): # --------------------------------------------------------------------------- # def _group_snapshots(group): - """Every member snapshot that belongs to this group, any generation.""" - return [s for s in db.get_snapshots(group.cluster_id) - if s.group_id == group.get_id() and s.group_seq] + """Every member snapshot that belongs to this group, any generation, on the + group's own cluster. A replicated copy on the peer cluster carries the same + group id and is not one of this group's snapshots.""" + return [s for s in db.get_group_snapshots(group.get_id()) + if s.group_seq and s.cluster_id == group.cluster_id] def list_members(group): @@ -1265,6 +1316,40 @@ def list_members(group): return rows +def generation_snapshots(group, seq): + """The member snapshots of generation ``seq`` of ``group``, in any status. + + The one lookup of a generation's snapshots. It does not check that the + generation is whole; :func:`resolve_complete_generation` does. + """ + return [s for s in _group_snapshots(group) if s.group_seq == seq] + + +def resolve_complete_generation(group, seq): + """Resolve generation ``seq`` of ``group`` to its member snapshots, requiring + it whole: an ONLINE snapshot of every volume that was a member at ``seq`` + (the membership epochs, ``included_in_seq``). Returns those snapshots. + + Raises: + ConsistencyGroupError: the generation has no member snapshots. + IncompleteGenerationError: a member at ``seq`` has no online snapshot in + it; the error names the missing volumes. + """ + snaps = generation_snapshots(group, seq) + if not snaps: + raise ConsistencyGroupError( + f"consistency group {group.get_id()} has no member snapshots at generation {seq}") + online = [s for s in snaps if s.status == SnapShot.STATUS_ONLINE and s.lvol] + present = {s.lvol.get_id() for s in online} + expected = {lvol_id for lvol_id in (group.members or {}) if group.included_in_seq(lvol_id, seq)} + missing = sorted(expected - present) + if missing: + raise IncompleteGenerationError( + f"generation {seq} of consistency group {group.get_id()} has no online " + f"snapshot of member(s) {', '.join(missing)}") + return online + + def list_generations(group): """Historical per-generation view (design §6.3): one row per ``group_seq``. @@ -1307,7 +1392,7 @@ def delete_generation(group, seq): replication_recovery_points, snapshot_controller, ) - targets = [s for s in _group_snapshots(group) if s.group_seq == seq] + targets = generation_snapshots(group, seq) if not targets: return None, f"generation {seq} of group {group.uuid[:8]} not found" newest_seq, _, _ = replication_recovery_points.newest_group_generation(group.get_id(), db=db) @@ -1337,7 +1422,7 @@ def clone_generation(group, seq, into_name=None): :func:`add_member_to_group`. Returns (new_lvol_ids, None) or (None, error). """ from simplyblock_core.controllers import snapshot_controller - targets = [s for s in _group_snapshots(group) if s.group_seq == seq] + targets = generation_snapshots(group, seq) if not targets: return None, f"generation {seq} of group {group.uuid[:8]} not found" new_group = ensure_group(group.cluster_id, into_name) if into_name else None diff --git a/simplyblock_core/controllers/tasks_controller.py b/simplyblock_core/controllers/tasks_controller.py index d18563af2a..86081cec8a 100644 --- a/simplyblock_core/controllers/tasks_controller.py +++ b/simplyblock_core/controllers/tasks_controller.py @@ -1282,6 +1282,51 @@ def add_backup_task(backup): ) +def add_group_backup_delete_task(group_backup): + """Create the task that retires a deleting group backup's members, unless + one is already open for it. + + Like the commit task it does no data-plane RPC of its own (a member's merge + is its own FN_BACKUP_MERGE task), so it carries no node. Returns the task + id, or False when one is already open. + """ + for task in db.get_job_tasks_by_function(group_backup.cluster_id, + JobSchedule.FN_GROUP_BACKUP_DELETE): + if ((task.function_params or {}).get("group_backup_id") == group_backup.uuid + and not task.canceled and task.status != JobSchedule.STATUS_DONE): + logger.info(f"Task found, skip adding new task: {task.uuid}") + return False + return _add_task( + JobSchedule.FN_GROUP_BACKUP_DELETE, + group_backup.cluster_id, + "", + "", + max_retry=constants.BACKUP_TASK_MAX_RETRIES, + function_params={ + "group_backup_id": group_backup.uuid, + }, + ) + + +def add_group_backup_commit_task(group_backup): + """Create the task that commits a consistency-group backup. + + The task does no data-plane RPC: it watches the member backups (which the + FN_BACKUP runner drives) and publishes the group manifest once every member + has completed, so it carries no node and only the group backup id. + """ + return _add_task( + JobSchedule.FN_GROUP_BACKUP_COMMIT, + group_backup.cluster_id, + "", + "", + max_retry=constants.BACKUP_TASK_MAX_RETRIES, + function_params={ + "group_backup_id": group_backup.uuid, + }, + ) + + def add_backup_restore_task(cluster_id, node_id, backup_id, lvol_name, chain_ids, lvol_id="", s3_config=None): """Create the task that restores an S3 backup chain into a new lvol. diff --git a/simplyblock_core/db_controller.py b/simplyblock_core/db_controller.py index fc64734519..5a8f9b3515 100644 --- a/simplyblock_core/db_controller.py +++ b/simplyblock_core/db_controller.py @@ -17,6 +17,7 @@ BackupChainLock, BackupPolicy, BackupPolicyAttachment, + GroupBackup, ) from simplyblock_core.models.base_model import BaseModel from simplyblock_core.models.cluster import Cluster, DeployConfig, PortReservation @@ -509,6 +510,17 @@ def get_hostnames_by_pool_id(self, pool_id: str) -> list[str]: hostnames.append(lv.hostname) return hostnames + def get_group_snapshots(self, group_id: str, group_seq: int | None = None) -> list[SnapShot]: + """A consistency group's member snapshots, of every generation or of + ``group_seq`` only. A blank group id names no group, so it has none: + return empty rather than querying the index, which rejects a blank + component.""" + if not group_id: + return [] + if group_seq is None: + return self.query(SnapShot, 'group_id+group_seq', group_id) + return self.query(SnapShot, 'group_id+group_seq', group_id, group_seq) + def get_snapshots(self, cluster_id: str | None = None) -> list[SnapShot]: start_time = time.time() snaps = (self.query(SnapShot, 'cluster_id', cluster_id) if cluster_id @@ -1566,6 +1578,32 @@ def get_backups_by_lvol_id(self, lvol_id: str) -> list[Backup]: def get_backups_by_snapshot_id(self, snapshot_id: str) -> list[Backup]: return self.query(Backup, 'snapshot_id', snapshot_id) + def get_backups_by_group_backup_id(self, group_backup_id: str) -> list[Backup]: + """The member backups of a group backup. A standalone backup's blank + group_backup_id is dropped by the index, so only members are returned + (design-consistency-group-backup.md §4.1). + + A blank id names no group backup, so it has no members: return empty + rather than querying the index, which rejects a blank component (a blank + is never indexed, so it could only ever match nothing).""" + if not group_backup_id: + return [] + return self.query(Backup, 'group_backup_id', group_backup_id) + + def get_group_backups(self, cluster_id: str | None = None) -> list[GroupBackup]: + prefix = cluster_id if cluster_id else " " + return GroupBackup().read_from_db(self.kv_store, id=prefix) + + def get_group_backup_by_id(self, group_backup_id: str) -> GroupBackup: + gb = self.query_one(GroupBackup, 'uuid', group_backup_id.split('/')[-1]) + if gb is None: + raise KeyError(f'GroupBackup {group_backup_id} not found') + return gb + + def get_group_backups_by_consistency_group( + self, consistency_group_id: str) -> list[GroupBackup]: + return self.query(GroupBackup, 'consistency_group_id', consistency_group_id) + def get_replication_targets(self, cluster_id: str | None = None) -> list[ReplicationTarget]: prefix = cluster_id if cluster_id else " " return ReplicationTarget().read_from_db(self.kv_store, id=prefix) @@ -1678,3 +1716,22 @@ def get_policy_for_lvol(self, lvol) -> BackupPolicy | None: except KeyError: return None return None + + def get_policy_for_group(self, group) -> BackupPolicy | None: + """The backup policy attached to a consistency group, or None. + + A group is backed up as one unit, so its policy attaches to the group + directly rather than to a pool or an lvol (design-consistency-group-backup.md + §7.1). + """ + policy_id = next( + (att.policy_id for att in self.query( + BackupPolicyAttachment, 'target_type+target_id', + 'consistency_group', group.get_id())), + None) + if policy_id: + try: + return self.get_backup_policy_by_id(policy_id) + except KeyError: + return None + return None diff --git a/simplyblock_core/models/backup.py b/simplyblock_core/models/backup.py index 5872172c3b..b1ffdd001f 100644 --- a/simplyblock_core/models/backup.py +++ b/simplyblock_core/models/backup.py @@ -17,6 +17,9 @@ class Backup(BaseModel): Index('lvol_id'), Index('snapshot_id'), Index('prev_backup_id'), + # A standalone backup's group_backup_id is blank, which the index drops + # (models/indices.py), so only the members of a group backup are indexed. + Index('group_backup_id'), ) STATUS_PENDING = 'pending' @@ -66,6 +69,24 @@ class Backup(BaseModel): #: record predates self-describing backups. encryption: dict = default_factory(dict) + #: When this backup is a member of a group backup -- a consistency-group + #: generation backed up as one unit -- the ``GroupBackup.get_id()`` it + #: belongs to and the generation sequence. Blank/0 for a standalone + #: single-volume backup, so the ``group_backup_id`` index skips standalone + #: backups (design-consistency-group-backup.md §4.1). + group_backup_id: str = "" + group_seq: int = 0 + + #: How the source volume shared its NVMe subsystem: the namespace cap of the + #: subsystem it was created with (``LVol.max_namespace_per_subsys``; 1 is a + #: subsystem of its own, more is a namespaced volume that shares one). A + #: restore recreates the volume with the same cap, so a group restored from + #: members that shared a subsystem shares one again instead of landing each + #: member in its own (observed 2026-10-20: three restored members, three + #: subsystems, every one at NSID 1). 0 is a record that predates the field; + #: restore then falls back to add_lvol_ha's default, a subsystem of its own. + max_namespace_per_subsys: int = 0 + def get_id(self): return "%s/%s" % (self.cluster_id, self.uuid) @@ -93,6 +114,98 @@ def write_to_db(self, kv_store=None): super().write_to_db(kv_store) +class GroupBackup(BaseModel): + """A consistency-group backup: the member snapshots of one group generation + backed up to S3 as one all-or-nothing unit. + + The crash-consistency comes from the generation, not this record: the member + snapshots are a single point in time because one frozen + ``bdev_lvol_snapshot_group`` call took them (design-consistency-groups.md + §5). This record ties the N per-member :class:`Backup` records together and + is the control-plane half of the group manifest written to + ``groups/.json``. It is ``completed`` only once every member backup is + completed and the group manifest is published -- the commit barrier + (design-consistency-group-backup.md §3.3). + """ + + _WATCHED = True + + def __eq__(self, other): + if self is other: + return True + if not isinstance(other, GroupBackup): + return NotImplemented + return self.cluster_id == other.cluster_id and self.uuid == other.uuid + + def __hash__(self): + return hash((self.cluster_id, self.uuid)) + + _INDEXES: ClassVar[tuple] = ( + # uuid: a caller holding only the uuid (the mirror, a restore) can point + # read. consistency_group_id: "the group backups of this group" (listing, + # retention) without a scan. + Index('uuid'), + Index('consistency_group_id'), + ) + + STATUS_PENDING = 'pending' + STATUS_IN_PROGRESS = 'in_progress' + STATUS_COMPLETED = 'completed' + STATUS_FAILED = 'failed' + #: Being deleted: its members are being retired, the group manifest says + #: so, and it is no longer restorable (design §9). + STATUS_DELETING = 'deleting' + #: A delete ended after retiring some members: what is left cannot restore + #: as a group. Deleting it again resumes the retirement. + STATUS_BROKEN = 'broken' + + _STATUS_CODE_MAP: ClassVar[dict] = { + STATUS_PENDING: 0, + STATUS_IN_PROGRESS: 1, + STATUS_COMPLETED: 2, + STATUS_FAILED: 3, + STATUS_DELETING: 4, + STATUS_BROKEN: 5, + } + + cluster_id: str = "" + #: The source consistency group and generation. ``consistency_group_id`` is + #: ``ConsistencyGroup.get_id()`` ("/"); ``group_seq`` is the + #: generation (``last_group_seq`` at the take). + consistency_group_id: str = "" + consistency_group_name: str = "" + group_seq: int = 0 + #: The member backups, one per member snapshot of the generation. Each entry + #: is ``{"lvol_id": , "backup_id": }``. + members: list = default_factory(list) + #: Where the members live and whether they are encrypted is not repeated + #: here: each member Backup records both, and they agree (design §3.4). + size: int = 0 + created_at: int = 0 + completed_at: int = 0 + error_message: str = "" + + def get_id(self): + return "%s/%s" % (self.cluster_id, self.uuid) + + def watch_scope(self): + return (self.cluster_id,) + + def manifest_published(self) -> bool: + """Whether the group manifest was written: at the commit, which is the + only place ``completed_at`` is set, or at an import, of a committed one. + A group backup that failed before committing has none.""" + return bool(self.completed_at) + + def member_backup_ids(self) -> list: + """The ``Backup.get_id()`` of every member, in record order.""" + return [m["backup_id"] for m in self.members] + + def write_to_db(self, kv_store=None): + self.updated_at = str(datetime.datetime.now(datetime.UTC)) + super().write_to_db(kv_store) + + class BackupChainLock(BaseModel): """Short-lived lock used while enqueuing a snapshot backup chain.""" diff --git a/simplyblock_core/models/job_schedule.py b/simplyblock_core/models/job_schedule.py index 2c9b6c561c..c977eab98a 100644 --- a/simplyblock_core/models/job_schedule.py +++ b/simplyblock_core/models/job_schedule.py @@ -64,6 +64,13 @@ class JobSchedule(BaseModel): FN_BACKUP = "s3_backup" FN_BACKUP_RESTORE = "s3_backup_restore" FN_BACKUP_MERGE = "s3_backup_merge" + # Consistency-group backup commit: watch the member backups of one group + # generation and publish the group manifest once every member completed, + # the all-or-nothing barrier (design-consistency-group-backup.md §3.3). + FN_GROUP_BACKUP_COMMIT = "group_backup_commit" + # Consistency-group backup delete: retire the member backups of a group + # backup marked deleting, then remove its manifest and record (§9). + FN_GROUP_BACKUP_DELETE = "group_backup_delete" FN_CLUSTER_EXPAND = "cluster_expand" # Cross-cluster replication cutover: freeze source IO, transfer the final # lvol delta to the target, flip ANA so the client fails over. Used for diff --git a/simplyblock_core/models/snapshot.py b/simplyblock_core/models/snapshot.py index ac529f127f..1aeb0b47aa 100644 --- a/simplyblock_core/models/snapshot.py +++ b/simplyblock_core/models/snapshot.py @@ -19,6 +19,10 @@ class SnapShot(BaseModel): [(snap.lvol.get_id(),)] if snap.lvol else [] )), Unique(('cluster_id', 'snap_name')), + # A consistency group's member snapshots, and one generation's by + # prefix. A snapshot outside any group has a blank group_id, which the + # index drops, so only group snapshots are indexed. + Index(('group_id', 'group_seq')), # Creation order within one lvol: the tail is the chain predecessor, # found by a reverse range read of limit 1. Ordered, so its segments # are the fixed-width encoding `encode_value` reserves for that. @@ -74,6 +78,10 @@ class SnapShot(BaseModel): #: this snapshot belongs to (0 = not a group snapshot). group_id: str = "" group_seq: int = 0 + #: the caller's request id for the group generation this snapshot belongs + #: to, stamped on every member snapshot of it, so a retried take with the + #: same id answers with this generation instead of taking another. + group_request_id: str = "" #: The volume's own snapshot sequence, in creation order. prev_snap_uuid #: is also the blob parent: a snapshot's blob holds only the clusters #: written since that predecessor. It is empty on the first snapshot taken diff --git a/simplyblock_core/services/backup_merge_service.py b/simplyblock_core/services/backup_merge_service.py index 2f0030c258..710e34e97e 100644 --- a/simplyblock_core/services/backup_merge_service.py +++ b/simplyblock_core/services/backup_merge_service.py @@ -31,6 +31,15 @@ def main(): backup_policy.evaluate_schedule(lvol) except Exception as e: logger.error(f"Error evaluating schedule for lvol {lvol.get_id()}: {e}") + + # Consistency-group backups are policy-driven in the control plane + # too: a group attached to a policy is backed up as one unit on its + # schedule, and its generations are pruned per the policy (§7.1, §9). + for group in db.get_consistency_groups(cl.get_id()): + try: + backup_policy.evaluate_group_backups(group) + except Exception as e: + logger.error(f"Error applying the backup policy of group {group.get_id()}: {e}") except Exception as e: logger.error(f"Error processing cluster {cl.get_id()}: {e}") diff --git a/simplyblock_core/services/snapshot_monitor.py b/simplyblock_core/services/snapshot_monitor.py index d5a7993c9c..3f7fd7ffc3 100644 --- a/simplyblock_core/services/snapshot_monitor.py +++ b/simplyblock_core/services/snapshot_monitor.py @@ -670,7 +670,7 @@ def _group_id_of(lvol): group = db.get_consistency_group_by_id(group_id) logger.info("Taking consistency-group snapshot for group %s " "(%d members)", group_id, len(members)) - _ids, err = consistency_group_controller.create_group_snapshot_for_group(group) + _seq, err = consistency_group_controller.create_group_snapshot_for_group(group) if err: logger.warning("Group snapshot for group %s failed: %s", group_id, err) diff --git a/simplyblock_core/services/tasks_runner_backup.py b/simplyblock_core/services/tasks_runner_backup.py index 980fbe3090..0c96577cc8 100644 --- a/simplyblock_core/services/tasks_runner_backup.py +++ b/simplyblock_core/services/tasks_runner_backup.py @@ -16,9 +16,10 @@ from simplyblock_core.controllers import backup_events from simplyblock_core.controllers.backup import controller as backup_controller from simplyblock_core.controllers.backup import device as backup_device +from simplyblock_core.controllers.backup import group_backup from simplyblock_core.controllers.backup.manifest import ManifestError from simplyblock_core.exceptions import PreconditionError -from simplyblock_core.models.backup import Backup +from simplyblock_core.models.backup import Backup, GroupBackup from simplyblock_core.models.backup_config import BackupConfig from simplyblock_core.models.cluster import Cluster from simplyblock_core.models.job_schedule import JobSchedule @@ -413,10 +414,62 @@ def _run_merge(task): raise TaskDefer("Merge in progress") +def _run_group_backup_commit(task): + """Commit a consistency-group backup once its members are done. + + Does no data-plane RPC. It watches the member backups -- which the FN_BACKUP + runner drives independently -- and publishes the group manifest when every + one has completed, the all-or-nothing barrier (design §3.3). It defers while + any member is still in flight, aborts if a member failed, and treats a + manifest-publish failure as retryable S3 trouble rather than a group failure. + """ + group_backup_id = task.function_params.get("group_backup_id") + if not group_backup_id: + raise TaskAbort("Missing group_backup_id") + + try: + state = group_backup.try_commit_group_backup(group_backup_id) + except (ManifestError, PreconditionError) as e: + raise TaskRetry(f"Could not publish group manifest: {e}") + + if state == group_backup.CommitState.PENDING: + raise TaskDefer("Group backup members still in flight") + if state == group_backup.CommitState.FAILED: + raise TaskAbort("Group backup failed: a member backup did not complete") + + set_result(task, "Group backup completed") + + +def _run_group_backup_delete(task): + """Retire a deleting group backup's members, then remove its manifest and + record (design §9). + + Does no data-plane RPC of its own: a member's merge is its own + FN_BACKUP_MERGE task, which this waits on. Defers while a member is still + being retired, and treats a manifest or member failure as retryable; a task + that ends unfinished settles the group backup in :func:`finalize_resource`. + """ + group_backup_id = task.function_params.get("group_backup_id") + if not group_backup_id: + raise TaskAbort("Missing group_backup_id") + + try: + state = group_backup.advance_group_backup_delete(group_backup_id) + except (ManifestError, PreconditionError) as e: + raise TaskRetry(f"Could not finish deleting the group backup: {e}") + + if state == group_backup.DeleteState.PENDING: + raise TaskDefer("Group backup members still being retired") + + set_result(task, "Group backup deleted") + + _HANDLERS = { JobSchedule.FN_BACKUP: _run_backup, JobSchedule.FN_BACKUP_RESTORE: _run_restore, JobSchedule.FN_BACKUP_MERGE: _run_merge, + JobSchedule.FN_GROUP_BACKUP_COMMIT: _run_group_backup_commit, + JobSchedule.FN_GROUP_BACKUP_DELETE: _run_group_backup_delete, } @@ -480,6 +533,32 @@ def finalize_resource(task): old_backup.status = Backup.STATUS_COMPLETED old_backup.write_to_db() + elif task.function_name == JobSchedule.FN_GROUP_BACKUP_COMMIT: + group_backup_id = task.function_params.get("group_backup_id") + if not group_backup_id: + return + try: + gb = db.get_group_backup_by_id(group_backup_id) + except KeyError: + return + # The commit task ended (timeout, retry ceiling, abort, cancellation) + # while the group was still unresolved: fail it as a unit. A member + # backup that is still running is the FN_BACKUP runner's to terminate; + # this only closes the group record so nothing reads it as in progress + # forever. + if gb.status in (GroupBackup.STATUS_PENDING, GroupBackup.STATUS_IN_PROGRESS): + gb.status = GroupBackup.STATUS_FAILED + gb.error_message = reason + gb.write_to_db() + + elif task.function_name == JobSchedule.FN_GROUP_BACKUP_DELETE: + group_backup_id = task.function_params.get("group_backup_id") + if not group_backup_id: + return + # Restored if no member was touched, marked broken if one was; a no-op + # once the delete finished and the record is gone. + group_backup.abandon_group_backup_delete(group_backup_id, reason) + SPEC = RunnerSpec( name="tasks-runner-backup", @@ -496,7 +575,8 @@ def finalize_resource(task): #: "max retry reached (8/8)" and would end any other task older than the #: backup timeout (2026-09-29). BACKUP_FUNCTIONS = (JobSchedule.FN_BACKUP, JobSchedule.FN_BACKUP_RESTORE, - JobSchedule.FN_BACKUP_MERGE) + JobSchedule.FN_BACKUP_MERGE, JobSchedule.FN_GROUP_BACKUP_COMMIT, + JobSchedule.FN_GROUP_BACKUP_DELETE) def main(): diff --git a/simplyblock_core/test/test_consistency_groups.py b/simplyblock_core/test/test_consistency_groups.py index cb6b8dd716..dea22c5311 100644 --- a/simplyblock_core/test/test_consistency_groups.py +++ b/simplyblock_core/test/test_consistency_groups.py @@ -91,12 +91,20 @@ def test_no_group_or_no_seq_is_silent(): # --------------------------------------------------------------------------- # class _FakeDB: - def __init__(self, group): + def __init__(self, group, lvols=()): self._group = group + # The volume records that still exist; a member id not here is a + # dangling entry whose volume is gone. + self._lvols = set(lvols) def get_consistency_group_for_policy(self, policy_id): return self._group + def get_lvol_by_id(self, lvol_id): + if lvol_id not in self._lvols: + raise KeyError(lvol_id) + return _Lvol(lvol_id, "NODE_A", "LVS_1") + @property def kv_store(self): return None @@ -192,12 +200,38 @@ def test_delete_group_refuses_while_it_has_a_current_member(monkeypatch): g = _group({"v1": {"joined_seq": 1, "removed_seq": 0}}, last_seq=4) removed = [] g.remove = lambda kv=None: removed.append(g) - monkeypatch.setattr(cgc, "db", _FakeDB(g)) + monkeypatch.setattr(cgc, "db", _FakeDB(g, lvols=("v1",))) with pytest.raises(cgc.ConsistencyGroupError, match="still has 1 member"): cgc.delete_group(g) assert removed == [], "a group with a live member must NOT be deleted" +def test_delete_group_drops_a_dangling_member_whose_volume_is_gone(monkeypatch): + """A member joined through add_member_to_group alone carries no group_id, so + deleting its volume never closed its epoch: the entry outlived the volume + and held the group "in use" for good (live 2026-10-20, three restored + members long deleted). An open entry without a volume record is dangling, + not live, and does not block the delete.""" + g = _group({"gone": {"joined_seq": 1, "removed_seq": 0}, + "closed": {"joined_seq": 1, "removed_seq": 3}}, last_seq=4) + removed = [] + g.remove = lambda kv=None: removed.append(g) + monkeypatch.setattr(cgc, "db", _FakeDB(g)) # no volume record exists + cgc.delete_group(g) + assert removed == [g] + + +def test_delete_group_still_refuses_when_one_open_member_is_live(monkeypatch): + g = _group({"gone": {"joined_seq": 1, "removed_seq": 0}, + "live": {"joined_seq": 1, "removed_seq": 0}}, last_seq=4) + removed = [] + g.remove = lambda kv=None: removed.append(g) + monkeypatch.setattr(cgc, "db", _FakeDB(g, lvols=("live",))) + with pytest.raises(cgc.ConsistencyGroupError, match="still has 1 member"): + cgc.delete_group(g) + assert removed == [] + + def test_delete_group_removes_an_empty_group(monkeypatch): g = _group({}, last_seq=0) removed = [] @@ -492,69 +526,4 @@ def test_list_members_reports_current_only(standalone): assert rows[0]["online"] is True -def test_list_generations_reports_incomplete(standalone): - g = cgc.ensure_group("CL", "db-group") - g.members = {"a": {"joined_seq": 1, "removed_seq": 0}, - "b": {"joined_seq": 1, "removed_seq": 0}} - gid = g.get_id() - la, lb = _StandaloneLvol("a"), _StandaloneLvol("b") - # generation 1 complete (both members), generation 2 missing member b - standalone.snapshots = { - "s1a": _StandaloneSnap("s1a", gid, 1, la), - "s1b": _StandaloneSnap("s1b", gid, 1, lb), - "s2a": _StandaloneSnap("s2a", gid, 2, la), - } - rows = {r["group_seq"]: r for r in cgc.list_generations(g)} - assert rows[1]["expected"] == 2 and rows[1]["present"] == 2 and rows[1]["complete"] - assert rows[2]["expected"] == 2 and rows[2]["present"] == 1 and not rows[2]["complete"] - - -def test_delete_generation_removes_every_member_snapshot(standalone, monkeypatch): - from simplyblock_core.controllers import snapshot_controller - g = cgc.ensure_group("CL", "db-group") - gid = g.get_id() - la = _StandaloneLvol("a") - standalone.snapshots = { - "s2a": _StandaloneSnap("s2a", gid, 2, la), - "s2b": _StandaloneSnap("s2b", gid, 2, la), - "s3a": _StandaloneSnap("s3a", gid, 3, la), - } - deleted_calls = [] - - def _fake_delete(sid, *a, **k): - deleted_calls.append(sid) - return True - - monkeypatch.setattr(snapshot_controller, "delete", _fake_delete) - deleted, err = cgc.delete_generation(g, 2) - assert err is None - assert set(deleted_calls) == {"s2a", "s2b"} # generation 3 untouched - -def test_delete_generation_unknown_seq_errors(standalone): - g = cgc.ensure_group("CL", "db-group") - deleted, err = cgc.delete_generation(g, 9) - assert deleted is None and "not found" in err - - -def test_clone_generation_forms_new_group(standalone, monkeypatch): - from simplyblock_core.controllers import snapshot_controller - g = cgc.ensure_group("CL", "db-group") - g.node_id, g.lvs_name = "NODE_A", "LVS_1" - gid = g.get_id() - standalone.snapshots = { - "s1a": _StandaloneSnap("s1a", gid, 1, _StandaloneLvol("a")), - "s1b": _StandaloneSnap("s1b", gid, 1, _StandaloneLvol("b")), - } - # each clone lands on the pinned node so the new group colocates - standalone.lvols = {"clone-a": _StandaloneLvol("clone-a"), - "clone-b": _StandaloneLvol("clone-b")} - seq = iter(["clone-a", "clone-b"]) - monkeypatch.setattr(snapshot_controller, "clone", - lambda sid, name, *a, **k: (next(seq), False)) - created, err = cgc.clone_generation(g, 1, into_name="restored") - assert err is None - assert set(created) == {"clone-a", "clone-b"} - new_group = standalone.get_consistency_group_by_name("CL", "restored") - assert new_group is not None - assert set(new_group.members) == {"clone-a", "clone-b"} diff --git a/simplyblock_core/test/test_replication_policies.py b/simplyblock_core/test/test_replication_policies.py index b6d9a3cd4d..87a3c9aed5 100644 --- a/simplyblock_core/test/test_replication_policies.py +++ b/simplyblock_core/test/test_replication_policies.py @@ -1089,7 +1089,7 @@ def test_group_demote_leaves_the_serving_peer_untouched_on_reprotect(monkeypatch def _record_group_snapshot(group, **kwargs): snapped.append(group.uuid) - return [], None + return 1, None monkeypatch.setattr(cgc, "create_group_snapshot_for_group", _record_group_snapshot) def _record_demote(lvol_id): @@ -1255,13 +1255,18 @@ def test_group_demote_resolves_to_peer_primary_and_ships_home(monkeypatch): monkeypatch.setattr(lc, "replication_failback", _recording(failed_back)) monkeypatch.setattr(rfs, "fence_source_paths", lambda *a, **k: None) + cut = [] + def _group_snap_cut(group, **kw): - ids = [] for src, sid in (("PB1", "DEMOTE_PB1"), ("PB2", "DEMOTE_PB2")): - db._snapshots.append(_snap(sid, db.get_lvol_by_id(src))) - ids.append(sid) - return ids, None + snap = _snap(sid, db.get_lvol_by_id(src)) + db._snapshots.append(snap) + cut.append(snap) + return 7, None monkeypatch.setattr(cgc, "create_group_snapshot_for_group", _group_snap_cut) + # The fake has no group-snapshot lookup; the cut is generation 7. + monkeypatch.setattr(cgc, "generation_snapshots", + lambda group, seq: cut if seq == 7 else []) result = cgc.demote_group(local) diff --git a/simplyblock_web/api/v2/_dependencies.py b/simplyblock_web/api/v2/_dependencies.py index c38a550a2b..38a049848b 100644 --- a/simplyblock_web/api/v2/_dependencies.py +++ b/simplyblock_web/api/v2/_dependencies.py @@ -6,6 +6,7 @@ from simplyblock_core.db_controller import DBController from simplyblock_core.models.backup import Backup as BackupModel from simplyblock_core.models.backup import BackupPolicy +from simplyblock_core.models.backup import GroupBackup as GroupBackupModel from simplyblock_core.models.cluster import Cluster as ClusterModel from simplyblock_core.models.job_schedule import JobSchedule from simplyblock_core.models.lvol_migration import LVolMigration @@ -140,6 +141,19 @@ def _lookup_backup(backup_id: UUID, cluster: Cluster) -> BackupModel: BackupResource = Annotated[BackupModel, Depends(_lookup_backup)] +def _lookup_group_backup(group_backup_id: UUID, cluster: Cluster) -> GroupBackupModel: + try: + group_backup = _db.get_group_backup_by_id(str(group_backup_id)) + except KeyError as e: + raise HTTPException(404, str(e)) + if group_backup.cluster_id != cluster.get_id(): + raise HTTPException(404, f'GroupBackup {group_backup_id} not found') + return group_backup + + +GroupBackupResource = Annotated[GroupBackupModel, Depends(_lookup_group_backup)] + + def _lookup_backup_policy(policy_id: UUID, cluster: Cluster) -> BackupPolicy: try: policy = _db.get_backup_policy_by_id(str(policy_id)) diff --git a/simplyblock_web/api/v2/_dtos.py b/simplyblock_web/api/v2/_dtos.py index 7ff7888c79..893c93134c 100644 --- a/simplyblock_web/api/v2/_dtos.py +++ b/simplyblock_web/api/v2/_dtos.py @@ -7,9 +7,14 @@ from pydantic import BaseModel, SecretStr, field_serializer from simplyblock_core.controllers import migration_controller -from simplyblock_core.controllers.backup.manifest import BackupExport, BackupManifest +from simplyblock_core.controllers.backup.manifest import ( + BackupExport, + BackupManifest, + GroupBackupExport, + GroupBackupManifest, +) from simplyblock_core.db_controller import DBController -from simplyblock_core.models.backup import Backup, BackupPolicy +from simplyblock_core.models.backup import Backup, BackupPolicy, GroupBackup from simplyblock_core.models.backup_config import ( BackupConfig, BackupLocation, @@ -129,6 +134,8 @@ "s3_backup", "s3_backup_restore", "s3_backup_merge", + "group_backup_commit", + "group_backup_delete", "cluster_expand", "replication_final", "fdb_backup", @@ -702,6 +709,19 @@ class BackupDTO(BaseModel): completed_at: int encrypted: bool + #: The group backup this copy is a member of, when it belongs to a + #: consistency-group backup; null for a standalone single-volume backup. The + #: id is the group backup's composite ``cluster_id/uuid``, and ``group_seq`` + #: is its generation. The operator mirrors these onto + #: ``StorageBackup.status.source.group`` so the members of one generation are + #: legible as a unit. + group_backup_id: str | None = None + group_seq: int = 0 + #: The namespace cap of the subsystem the source volume shared (1: its own + #: subsystem); a restore recreates the same sharing. 0 for a record from + #: before the field was kept. + max_namespace_per_subsys: int = 0 + @staticmethod def from_model(model: Backup): return BackupDTO( @@ -718,6 +738,65 @@ def from_model(model: Backup): created_at=model.created_at, completed_at=model.completed_at, encrypted=model.encrypted, + group_backup_id=model.group_backup_id or None, + group_seq=model.group_seq, + max_namespace_per_subsys=model.max_namespace_per_subsys, + ) + + +#: A group backup's self-describing document as the API exchanges it: the body of +#: the group-backups export GET and of an inline group-backups import. An alias +#: for the same reason ``BackupExportDTO`` is one -- the wire form and the stored +#: form are one shape, and naming it here still lets the API diverge later. +GroupBackupExportDTO = GroupBackupExport + +#: A group backup's manifest as discover returns it, read from a bucket's +#: ``groups/`` prefix with no cluster reference. An alias like ``BackupManifestDTO``. +GroupBackupManifestDTO = GroupBackupManifest + + +def _bare_uuid(composite_id: str) -> UUID: + """The UUID of a ``cluster_id/uuid`` record id. A DTO names the cluster once, + in ``cluster_id``, and every other object by its bare UUID.""" + return UUID(composite_id.split('/')[-1]) + + +class GroupBackupDTO(BaseModel): + """A consistency-group backup: the member backups of one generation as one + all-or-nothing unit (design §4.1).""" + id: UUID + cluster_id: UUID + #: The consistency group the generation was taken from. + consistency_group_id: UUID + consistency_group_name: str + group_seq: int + status: str + #: The member backups, one per member volume of the generation. Each member + #: backup carries its own volume; GET .../members/ returns the records. + member_backup_ids: list[UUID] + size: int + created_at: datetime + #: When the group backup committed: every member completed and the group + #: manifest was published. None until then. + completed_at: datetime | None = None + #: Why the group backup failed. None unless it did. + error_message: str | None = None + + @staticmethod + def from_model(model: GroupBackup): + return GroupBackupDTO( + id=UUID(model.uuid), + cluster_id=UUID(model.cluster_id), + consistency_group_id=_bare_uuid(model.consistency_group_id), + consistency_group_name=model.consistency_group_name, + group_seq=model.group_seq, + status=model.status, + member_backup_ids=[_bare_uuid(m["backup_id"]) for m in model.members], + size=model.size, + created_at=datetime.fromtimestamp(model.created_at, tz=UTC), + completed_at=(datetime.fromtimestamp(model.completed_at, tz=UTC) + if model.completed_at else None), + error_message=model.error_message or None, ) diff --git a/simplyblock_web/api/v2/cluster/__init__.py b/simplyblock_web/api/v2/cluster/__init__.py index 1c8133ed0e..6a336f3b1b 100644 --- a/simplyblock_web/api/v2/cluster/__init__.py +++ b/simplyblock_web/api/v2/cluster/__init__.py @@ -27,6 +27,7 @@ from .arbitration import api as arbitration_api from .backup import api as backup_api from .consistency_group import api as consistency_group_api +from .group_backup import api as group_backup_api from .replication import api as replication_api from .storage_node import api as storage_node_api from .storage_pool import api as pool_api @@ -310,6 +311,9 @@ def rebalance_cluster( cluster: Cluster) -> Response: instance_api.include_router(storage_node_api, prefix='/storage-nodes') instance_api.include_router(task_api, prefix='/tasks') instance_api.include_router(pool_api, prefix='/storage-pools') +# Ahead of the single-volume backups, whose /backups/{backup_id}/ would +# otherwise take "groups" for a backup id. +instance_api.include_router(group_backup_api, prefix='/backups/groups') instance_api.include_router(backup_api, prefix='/backups') instance_api.include_router(subsystem_api, prefix='/subsystems') instance_api.include_router(replication_api, prefix='/replication') diff --git a/simplyblock_web/api/v2/cluster/backup.py b/simplyblock_web/api/v2/cluster/backup.py index 0e3c19e22e..aa5e123f9d 100644 --- a/simplyblock_web/api/v2/cluster/backup.py +++ b/simplyblock_web/api/v2/cluster/backup.py @@ -233,6 +233,13 @@ def _validate_attachment_target(target_type: str, target_id: str, cluster: Clust raise HTTPException(404, f'Pool {target_id} not found') elif target_type == "lvol": _lookup_lvol_in_cluster(target_id, cluster) + elif target_type == "consistency_group": + try: + group = db.get_consistency_group_by_id(target_id) + except KeyError as e: + raise HTTPException(404, str(e)) + if group.cluster_id != cluster.get_id(): + raise HTTPException(404, f'ConsistencyGroup {target_id} not found') @policy_api.delete('/{policy_id}', name='clusters:backup-policies:delete', status_code=204, responses={204: {"content": None}}) diff --git a/simplyblock_web/api/v2/cluster/consistency_group.py b/simplyblock_web/api/v2/cluster/consistency_group.py index 17ee723f6b..d399fa43c9 100644 --- a/simplyblock_web/api/v2/cluster/consistency_group.py +++ b/simplyblock_web/api/v2/cluster/consistency_group.py @@ -13,7 +13,7 @@ from fastapi import APIRouter, HTTPException, Response from fastapi.responses import JSONResponse -from pydantic import BaseModel +from pydantic import BaseModel, Field from simplyblock_core.controllers import ( cg_colocation, @@ -238,16 +238,31 @@ def list_generations(cluster: Cluster, group: ConsistencyGroupResource) -> built for r in consistency_group_controller.list_generations(group)] +class _GenerationTakeParams(BaseModel): + #: The caller's id for this take. A retry with the same id answers with the + #: generation the first call took instead of taking another. + request_id: str = Field('', max_length=128) + + @instance_api.post('/snapshots', name='clusters:consistency-groups:snapshots:take', response_model=ConsistencyGroupGenerationDTO) -def take_generation(cluster: Cluster, group: ConsistencyGroupResource) -> ConsistencyGroupGenerationDTO: - """Take one crash-consistent generation across every current member (§5).""" - _ids, err = consistency_group_controller.create_group_snapshot_for_group(group) +def take_generation(cluster: Cluster, group: ConsistencyGroupResource, + body: _GenerationTakeParams | None = None) -> ConsistencyGroupGenerationDTO: + """Take one crash-consistent generation across every current member (§5). + + With a ``request_id``, a retry of a take whose answer was lost returns the + generation already taken under that id. + """ + request_id = body.request_id if body is not None else '' + seq, err = consistency_group_controller.create_group_snapshot_for_group( + group, request_id=request_id) if err is not None: raise HTTPException(422, err) fresh = db.get_consistency_group_by_id(group.get_id()) - rows = consistency_group_controller.list_generations(fresh) - return _generation_dto(rows[-1]) + for row in consistency_group_controller.list_generations(fresh): + if row["group_seq"] == seq: + return _generation_dto(row) + raise HTTPException(404, f'generation {seq} not found') @instance_api.get('/snapshots/{seq}', name='clusters:consistency-groups:snapshots:detail', diff --git a/simplyblock_web/api/v2/cluster/group_backup.py b/simplyblock_web/api/v2/cluster/group_backup.py new file mode 100644 index 0000000000..11370cd769 --- /dev/null +++ b/simplyblock_web/api/v2/cluster/group_backup.py @@ -0,0 +1,227 @@ +"""Consistency-group backup REST surface (design-consistency-group-backup.md §8). + +Group backups are backups, so they live under the backup tree, at +`/clusters/{cluster_id}/backups/groups/`: creating one from a generation of the +consistency group the body names, listing them, reading one and its member +backups, restoring one into a new consistency group, deleting one, and the +export, import, and discover a single-volume backup has. The endpoints are thin. +Create fans out to the single-volume backup path per member, restore fans out to +the single-volume restore path per member and re-forms the group, and delete +removes the group manifest and the record, leaving the member backups to +per-volume retention (§9). + +This router is mounted ahead of the single-volume one, whose +`/backups/{backup_id}/` would otherwise take `groups` for a backup id. +""" +from typing import Union +from uuid import UUID + +from fastapi import APIRouter, HTTPException, Query, Response +from pydantic import BaseModel, ConfigDict + +from simplyblock_core.controllers.backup import group_backup as group_backup_controller +from simplyblock_core.controllers.backup.manifest import ManifestError +from simplyblock_core.db_controller import DBController +from simplyblock_core.exceptions import PreconditionError +from simplyblock_core.models.backup_config import S3Credentials + +from .._dependencies import Cluster, GroupBackupResource +from .._dtos import ( + BackupConfigDTO, + BackupDTO, + GroupBackupDTO, + GroupBackupExportDTO, + GroupBackupManifestDTO, +) + +api = APIRouter(tags=['group-backups']) +db = DBController() + + +@api.get('/', name='clusters:backups:groups:list', response_model=list[GroupBackupDTO]) +def list_group_backups(cluster: Cluster) -> list[GroupBackupDTO]: + """List the cluster's group backups, newest first.""" + group_backups = db.get_group_backups(cluster.get_id()) + group_backups = sorted(group_backups, key=lambda g: (g.created_at, g.uuid), reverse=True) + return [GroupBackupDTO.from_model(g) for g in group_backups] + + +class _GroupBackupCreateParams(BaseModel): + #: The consistency group to back up. + consistency_group_id: UUID + #: Back up an existing generation by its sequence. Omit to take a fresh + #: crash-consistent generation and back that up. + group_seq: int | None = None + + +@api.post('/', name='clusters:backups:groups:create', status_code=201, response_model=GroupBackupDTO) +def create_group_backup(cluster: Cluster, body: _GroupBackupCreateParams) -> GroupBackupDTO: + """Back up one generation of a consistency group to S3 as one all-or-nothing + unit (design §6). Takes a fresh crash-consistent generation unless + ``group_seq`` names an existing one. Returns the in-progress group backup, + which commits once every member backup completes. + """ + try: + group = db.get_consistency_group_by_id(str(body.consistency_group_id)) + except KeyError as e: + raise HTTPException(404, str(e)) + if group.cluster_id != cluster.get_id(): + raise HTTPException(404, f'ConsistencyGroup {body.consistency_group_id} not found') + group_backup_id, err = group_backup_controller.create_group_backup( + group.get_id(), group_seq=body.group_seq) + if err is not None: + # 409, not 422: FastAPI reserves 422 for its own validation error, whose + # body is a list the generated clients decode; a string detail there is + # undecodable (see restore below). + raise HTTPException(409, err) + return GroupBackupDTO.from_model(db.get_group_backup_by_id(group_backup_id)) + + +@api.get('/export', name='clusters:backups:groups:export') +def export_group_backup( + cluster: Cluster, + group_backup_id: str = Query(..., description="Export the group backup with this UUID"), +) -> GroupBackupExportDTO: + """Export a group backup's manifests as a unit, for import elsewhere (§6.5). + + Carries the group manifest plus every member backup's chain, so a hand-carried + file describes the group backup identically to a bucket read. + """ + try: + group_backup = db.get_group_backup_by_id(group_backup_id) + except KeyError: + raise HTTPException(404, f"Group backup {group_backup_id} not found") + if group_backup.cluster_id != cluster.get_id(): + raise HTTPException(404, f"Group backup {group_backup_id} not found") + return group_backup_controller.export_group_backup( + group_backup.get_id(), cluster_id=cluster.get_id()) + + +class _GroupImportManifests(BaseModel): + """A group export carried in the request itself, e.g. read from a file. + + Names nothing beside it: the export records where its member backups live, so + the caller states nothing the document has not already recorded. + """ + model_config = ConfigDict(extra="forbid") + + metadata: GroupBackupExportDTO + + +class _GroupImportFromBucket(BaseModel): + """Import whatever group backups a bucket turns out to contain. + + The disaster-recovery path: it needs a bucket and credentials for it, and + nothing from the cluster that wrote the group backups. + """ + model_config = ConfigDict(extra="forbid") + + bucket: BackupConfigDTO + + +#: The two ways to name what to import, a union rather than one model with two +#: optional fields -- the same shape the single-volume import takes, so pydantic +#: rejects a malformed body itself and the schema says "one of these two". +_GroupImportParams = Union[_GroupImportManifests, _GroupImportFromBucket] + + +@api.post('/import', name='clusters:backups:groups:import') +def import_group_backup(cluster: Cluster, parameters: _GroupImportParams): + """Import a group backup, preserving ids, all-or-nothing (§6.5).""" + try: + ids = ( + group_backup_controller.import_group_backup_from_bucket( + parameters.bucket, cluster_id=cluster.get_id()) + if isinstance(parameters, _GroupImportFromBucket) else + [group_backup_controller.import_group_backup( + parameters.metadata, cluster_id=cluster.get_id())] + ) + except ManifestError as e: + # The bucket named in the request could not be read. 400 rather than 502, + # as the single-volume import does: nothing here proxies for S3, and what + # the caller supplied is the only thing that can be wrong from here. + raise HTTPException(400, str(e)) from e + except ValueError as e: + raise HTTPException(400, str(e)) from e + return {"imported": len(ids), "group_backup_ids": ids} + + +@api.post('/discover', name='clusters:backups:groups:discover') +def discover_group_backups(parameters: BackupConfigDTO) -> list[GroupBackupManifestDTO]: + """List the group backups a bucket contains, without importing anything (§6.5). + + A POST because it carries credentials, which have no business in a query + string. Takes no cluster state at all: this is what an operator runs when the + cluster that wrote the group backups no longer exists. + """ + try: + return group_backup_controller.discover_group_backups(parameters) + except ManifestError as e: + raise HTTPException(400, str(e)) from e + + +instance_api = APIRouter(prefix='/{group_backup_id}') + + +@instance_api.get('/', name='clusters:backups:groups:detail', response_model=GroupBackupDTO) +def get_group_backup(cluster: Cluster, group_backup: GroupBackupResource) -> GroupBackupDTO: + return GroupBackupDTO.from_model(group_backup) + + +@instance_api.get('/members/', name='clusters:backups:groups:members', response_model=list[BackupDTO]) +def list_group_backup_members(cluster: Cluster, group_backup: GroupBackupResource) -> list[BackupDTO]: + """The group backup's member backups, as the single-volume backup records + they are: one per member volume of the generation.""" + return [BackupDTO.from_model(b) for b in db.get_backups_by_group_backup_id(group_backup.get_id())] + + +class _GroupRestoreParams(BaseModel): + #: Name of the NEW consistency group the restore forms; must not already + #: exist (design §6: a group restore always produces a new group). + into_group_name: str + pool: str + target_node_id: str | None = None + #: Credentials for the backup's bucket when it is not this cluster's own -- + #: the disaster-recovery case. Omit to use the nodes' instance role. + s3_credentials: S3Credentials | None = None + + +@instance_api.post('/restore', name='clusters:backups:groups:restore', status_code=202) +def restore_group_backup(cluster: Cluster, group_backup: GroupBackupResource, + parameters: _GroupRestoreParams): + """Restore a group backup into a NEW consistency group, all-or-nothing + (design §6). Returns the restored group id and one entry per restored member + volume (its id and the source member's name); each volume recovers its data + asynchronously, the same as a single-volume restore. + """ + group_id, members, error = group_backup_controller.restore_group_backup( + group_backup.get_id(), parameters.into_group_name, parameters.pool, + target_node_id=parameters.target_node_id, + s3_credentials=parameters.s3_credentials) + if error: + # 409, not 422: FastAPI reserves 422 for its own request-validation + # error, whose body is a LIST of field errors, and every generated + # client types it so. A string detail under 422 is undecodable to the + # operator ("cannot unmarshal string into ... HTTPValidationError.detail", + # live 2026-10-20), which hid the refusal behind a parse error. + raise HTTPException(409, error) + return {"group_id": group_id, "members": members} + + +@instance_api.delete('/', name='clusters:backups:groups:delete', + status_code=202, responses={202: {"content": None}}) +def delete_group_backup(cluster: Cluster, group_backup: GroupBackupResource) -> Response: + """Start deleting a group backup (design §9). Its record and manifest are + marked deleting, and a task retires its member backups and then removes + both; the record reads ``deleting`` until then, and ``broken`` if the delete + could not finish after retiring a member.""" + try: + group_backup_controller.delete_group_backup(group_backup.get_id()) + except PreconditionError as e: + raise HTTPException(409, str(e)) from e + except ManifestError as e: + raise HTTPException(400, str(e)) from e + return Response(status_code=202) + + +api.include_router(instance_api) diff --git a/tests/integration/backup/test_delete_backup.py b/tests/integration/backup/test_delete_backup.py new file mode 100644 index 0000000000..c0cc3b7a5d --- /dev/null +++ b/tests/integration/backup/test_delete_backup.py @@ -0,0 +1,124 @@ +"""delete_backup removes one backup record, and refuses what would break a restore, +against real FoundationDB. + +Deleting a backup another one chains against makes every descendant +unrestorable, and deleting a member of a group backup that still exists tears +that group backup. Both are refused with an exception rather than reported in a +return value, so a caller cannot forget to look. + +The database is real. The only boundary patched is the storage node's RPC +client, which the integration tier never reaches a real node for. +""" +import uuid as uuid_mod +import zlib +from unittest.mock import patch + +import pytest + +from simplyblock_core.controllers.backup import controller as backup_controller +from simplyblock_core.db_controller import DBController +from simplyblock_core.exceptions import PreconditionError +from simplyblock_core.models.backup import Backup, GroupBackup +from simplyblock_core.models.storage_node import StorageNode +from simplyblock_core.rpc_client import RPCException + + +def _uuid(name: str) -> str: + return f"{zlib.crc32(name.encode()):08x}-0000-4000-8000-000000000000" + + +CLUSTER = _uuid("cluster") +LVOL = _uuid("lvol") + + +@pytest.fixture +def db(): + return DBController() + + +@pytest.fixture +def node(db): + """An online node whose RPC client is patched: the node has no reachable + address, and the S3 delete is the only call made to it.""" + n = StorageNode() + n.uuid = str(uuid_mod.uuid4()) + n.cluster_id = CLUSTER + n.status = StorageNode.STATUS_ONLINE + n.hostname = "delete-backup-host" + n.mgmt_ip = "127.0.0.1" + n.write_to_db(db.kv_store) + with patch.object(StorageNode, "rpc_client") as rpc: + n.rpc = rpc + yield n + + +def _backup(db, node, name, prev="", group_backup_id=""): + b = Backup() + b.uuid = _uuid(name) + b.cluster_id = CLUSTER + b.lvol_id = LVOL + b.node_id = node.uuid + b.s3_id = zlib.crc32(name.encode()) % 1000 + b.prev_backup_id = _uuid(prev) if prev else "" + b.group_backup_id = group_backup_id + b.status = Backup.STATUS_COMPLETED + b.write_to_db(db.kv_store) + return b + + +def _group_backup(db): + gb = GroupBackup() + gb.uuid = _uuid("gb") + gb.cluster_id = CLUSTER + gb.status = GroupBackup.STATUS_COMPLETED + gb.write_to_db(db.kv_store) + return gb + + +def _exists(db, backup): + try: + db.get_backup_by_id(backup.uuid) + return True + except KeyError: + return False + + +def test_a_backup_nothing_depends_on_is_deleted(db, node): + leaf = _backup(db, node, "leaf") + assert backup_controller.delete_backup(leaf.uuid) is None + assert not _exists(db, leaf) + + +def test_a_backup_another_chains_against_is_refused(db, node): + base = _backup(db, node, "base") + _backup(db, node, "next", prev="base") + with pytest.raises(PreconditionError, match="predecessor"): + backup_controller.delete_backup(base.uuid) + assert _exists(db, base) + + +def test_a_member_of_a_group_backup_that_exists_is_refused(db, node): + gb = _group_backup(db) + member = _backup(db, node, "member", group_backup_id=gb.get_id()) + with pytest.raises(PreconditionError, match="group backup"): + backup_controller.delete_backup(member.uuid) + assert _exists(db, member) + + +def test_a_member_whose_group_backup_is_gone_is_deleted(db, node): + orphan = _backup(db, node, "orphan", group_backup_id=f"{CLUSTER}/{_uuid('gone')}") + backup_controller.delete_backup(orphan.uuid) + assert not _exists(db, orphan) + + +def test_an_unknown_backup_raises(db): + with pytest.raises(KeyError): + backup_controller.delete_backup(_uuid("nope")) + + +def test_a_failed_s3_delete_still_deletes_the_record(db, node): + """The S3 delete is best effort: the data plane does not implement it yet.""" + leaf = _backup(db, node, "leaf") + node.rpc.return_value.bdev_lvol_s3_delete.side_effect = RPCException("no such method") + backup_controller.delete_backup(leaf.uuid) + assert not _exists(db, leaf) diff --git a/tests/integration/backup/test_group_backup_commit.py b/tests/integration/backup/test_group_backup_commit.py new file mode 100644 index 0000000000..c38e0a5ea9 --- /dev/null +++ b/tests/integration/backup/test_group_backup_commit.py @@ -0,0 +1,152 @@ +"""The all-or-nothing commit barrier for a consistency-group backup, against +real FoundationDB (design-consistency-group-backup.md §3.3). + +The database is real; nothing here mocks the DB layer. The only boundaries +patched are the ones above it: the S3 manifest write and the bucket-credential +lookup, which the integration tier never reaches a real endpoint for. The +member backups are seeded as real Backup records and driven by setting their +status, standing in for what the FN_BACKUP runner does. +""" +import zlib +from unittest.mock import patch + +import pytest + +from simplyblock_core.controllers.backup import group_backup +from simplyblock_core.db_controller import DBController +from simplyblock_core.models.backup import Backup, GroupBackup + + +def _uuid(name: str) -> str: + # A deterministic, valid UUID string: the commit path converts member and + # group ids with UUID(...) while building the manifest, so the fixtures must + # be real UUIDs, not readable labels. + return f"{zlib.crc32(name.encode()):08x}-0000-4000-8000-000000000000" + + +CLUSTER = _uuid("cluster") + + +@pytest.fixture +def db(): + return DBController() + + +def _member(name, group_backup_id, status, size=100): + b = Backup() + b.uuid = _uuid(name) + b.cluster_id = CLUSTER + b.lvol_id = _uuid("lv-" + name) + b.lvol_name = name + b.s3_id = size + b.size = size + b.group_backup_id = group_backup_id + b.group_seq = 7 + b.status = status + return b + + +def _group_backup(members, status=GroupBackup.STATUS_IN_PROGRESS): + gb = GroupBackup() + gb.uuid = _uuid("gb") + gb.cluster_id = CLUSTER + gb.consistency_group_id = f"{CLUSTER}/{_uuid('cg')}" + gb.consistency_group_name = "gitea" + gb.group_seq = 7 + gb.created_at = 100 + gb.status = status + gb.members = [{"lvol_id": m.lvol_id, "backup_id": m.get_id()} for m in members] + return gb + + +def _seed(db, members, status=GroupBackup.STATUS_IN_PROGRESS): + for m in members: + m.write_to_db(db.kv_store) + gb = _group_backup(members, status=status) + gb.write_to_db(db.kv_store) + return gb + + +def test_pending_member_defers_without_touching_s3(db): + gb = _seed(db, [ + _member("b-db", "", Backup.STATUS_COMPLETED), + _member("b-data", "", Backup.STATUS_IN_PROGRESS), + ]) + # group_backup_id on the members is irrelevant here: try_commit reads the + # members by the ids recorded on the group backup, not by the index. + with patch.object(group_backup.backup_manifest, "write_group") as write_group: + state = group_backup.try_commit_group_backup(gb.get_id()) + + assert state == group_backup.CommitState.PENDING + write_group.assert_not_called() + assert db.get_group_backup_by_id(gb.get_id()).status == GroupBackup.STATUS_IN_PROGRESS + + +def test_failed_member_fails_the_group_without_touching_s3(db): + gb = _seed(db, [ + _member("b-db", "", Backup.STATUS_COMPLETED), + _member("b-data", "", Backup.STATUS_FAILED), + ]) + with patch.object(group_backup.backup_manifest, "write_group") as write_group: + state = group_backup.try_commit_group_backup(gb.get_id()) + + assert state == group_backup.CommitState.FAILED + write_group.assert_not_called() + stored = db.get_group_backup_by_id(gb.get_id()) + assert stored.status == GroupBackup.STATUS_FAILED + assert "failed" in stored.error_message + + +def test_all_completed_publishes_the_manifest_then_completes(db): + members = [ + _member("b-db", "", Backup.STATUS_COMPLETED, size=100), + _member("b-data", "", Backup.STATUS_COMPLETED, size=50), + ] + gb = _seed(db, members) + + with patch.object(group_backup.backup_manifest, "write_group") as write_group, \ + patch.object(group_backup, "_config_for_group", return_value="cfg"): + state = group_backup.try_commit_group_backup(gb.get_id()) + + assert state == group_backup.CommitState.READY + # The manifest is published exactly once, before the record is completed. + write_group.assert_called_once() + _, manifest = write_group.call_args[0] + assert {str(i) for i in manifest.member_backup_ids} == {_uuid("b-db"), _uuid("b-data")} + assert manifest.size == 150 + + stored = db.get_group_backup_by_id(gb.get_id()) + assert stored.status == GroupBackup.STATUS_COMPLETED + assert stored.size == 150 + assert stored.completed_at > 0 + + +def test_a_missing_member_record_fails_the_group(db): + members = [_member("b-db", "", Backup.STATUS_COMPLETED)] + gb = _group_backup(members) + # Record a second member on the group backup whose Backup record was never + # written (e.g. hard-deleted): the group cannot be committed. + gb.members.append({"lvol_id": f"{CLUSTER}/lv-gone", "backup_id": f"{CLUSTER}/b-gone"}) + members[0].write_to_db(db.kv_store) + gb.write_to_db(db.kv_store) + + with patch.object(group_backup.backup_manifest, "write_group") as write_group: + state = group_backup.try_commit_group_backup(gb.get_id()) + + assert state == group_backup.CommitState.FAILED + write_group.assert_not_called() + assert db.get_group_backup_by_id(gb.get_id()).status == GroupBackup.STATUS_FAILED + + +def test_commit_is_idempotent_once_completed(db): + gb = _seed(db, [_member("b-db", "", Backup.STATUS_COMPLETED)], + status=GroupBackup.STATUS_COMPLETED) + with patch.object(group_backup.backup_manifest, "write_group") as write_group: + state = group_backup.try_commit_group_backup(gb.get_id()) + + assert state == group_backup.CommitState.READY + write_group.assert_not_called() + + +def test_commit_of_an_unknown_group_backup_is_terminal(db): + assert group_backup.try_commit_group_backup(f"{CLUSTER}/nope") == group_backup.CommitState.FAILED diff --git a/tests/integration/backup/test_group_backup_db.py b/tests/integration/backup/test_group_backup_db.py new file mode 100644 index 0000000000..e3a968e7ac --- /dev/null +++ b/tests/integration/backup/test_group_backup_db.py @@ -0,0 +1,80 @@ +"""GroupBackup records round-trip and the group indexes resolve, against real +FoundationDB (design-consistency-group-backup.md §4.1). + +The database is real; nothing here mocks the DB layer. Nodes are not involved. +""" +import pytest + +from simplyblock_core.db_controller import DBController +from simplyblock_core.models.backup import Backup, GroupBackup + + +@pytest.fixture +def db(): + return DBController() + + +def _group_backup(uuid="gb-1", cluster="cl-1", cg="cl-1/cg-1", seq=7): + gb = GroupBackup() + gb.uuid = uuid + gb.cluster_id = cluster + gb.consistency_group_id = cg + gb.consistency_group_name = "gitea" + gb.group_seq = seq + gb.members = [{"lvol_id": "lv-db", "backup_id": f"{cluster}/b-db"}] + return gb + + +def _member_backup(uuid, cluster, group_backup_id, lvol="lv-db"): + b = Backup() + b.uuid = uuid + b.cluster_id = cluster + b.lvol_id = lvol + b.group_backup_id = group_backup_id + b.group_seq = 7 + return b + + +def test_group_backup_round_trips_and_reads_by_id(db): + gb = _group_backup() + gb.write_to_db(db.kv_store) + + got = db.get_group_backup_by_id(gb.get_id()) + assert got.get_id() == gb.get_id() + assert got.consistency_group_id == "cl-1/cg-1" + assert got.members == gb.members + + +def test_get_group_backup_by_id_missing_raises(db): + with pytest.raises(KeyError): + db.get_group_backup_by_id("cl-1/nope") + + +def test_group_backups_by_consistency_group(db): + a = _group_backup(uuid="gb-a", seq=1) + b = _group_backup(uuid="gb-b", seq=2) + other = _group_backup(uuid="gb-c", cg="cl-1/cg-2", seq=1) + for g in (a, b, other): + g.write_to_db(db.kv_store) + + got = db.get_group_backups_by_consistency_group("cl-1/cg-1") + assert {g.uuid for g in got} == {"gb-a", "gb-b"} + + +def test_backups_by_group_backup_id_finds_members_only(db): + gid = "cl-1/gb-1" + m1 = _member_backup("b-db", "cl-1", gid, lvol="lv-db") + m2 = _member_backup("b-data", "cl-1", gid, lvol="lv-data") + standalone = _member_backup("b-solo", "cl-1", "", lvol="lv-x") + for b in (m1, m2, standalone): + b.write_to_db(db.kv_store) + + got = db.get_backups_by_group_backup_id(gid) + assert {b.uuid for b in got} == {"b-db", "b-data"} + + +def test_standalone_backup_is_not_indexed_under_the_empty_group(db): + # A blank group_backup_id is dropped by the index, so a standalone backup + # does not pile onto a "" key (models/indices.py). + _member_backup("b-solo", "cl-1", "", lvol="lv-x").write_to_db(db.kv_store) + assert db.get_backups_by_group_backup_id("") == [] diff --git a/tests/integration/backup/test_group_backup_delete.py b/tests/integration/backup/test_group_backup_delete.py new file mode 100644 index 0000000000..802ca0b3b5 --- /dev/null +++ b/tests/integration/backup/test_group_backup_delete.py @@ -0,0 +1,347 @@ +"""Deleting a group backup, against real FoundationDB (design §9). + +A group backup is deleted the way it is created, metadata outermost: the group +manifest and the record are first marked as being deleted, then the member +backups are retired (a leaf deleted, a base merged into the next generation's +copy), and only once every member is retired are the manifest and the record +removed. A delete that fails before any member was touched restores the group +backup as it was; one that fails after says so, by marking it broken, in the +record and in the bucket. + +The database is real. The bucket is the one boundary faked: the S3 client the +group manifest functions use is an in-memory one. +""" +import io +from uuid import UUID + +import pytest +from botocore.exceptions import ClientError + +from simplyblock_core.controllers.backup import group_backup +from simplyblock_core.controllers.backup import manifest as backup_manifest +from simplyblock_core.controllers.backup.manifest import GroupManifestState +from simplyblock_core.db_controller import DBController +from simplyblock_core.exceptions import PreconditionError +from simplyblock_core.models.backup import Backup, GroupBackup +from simplyblock_core.models.backup_config import BackupConfig +from simplyblock_core.models.cluster import Cluster +from simplyblock_core.models.job_schedule import JobSchedule + +CLUSTER = "c3000000-0000-4000-8000-000000000001" +NODE = "d3000000-0000-4000-8000-000000000001" +BUCKET = "simplyblock-backup-primary" +GEN_7 = "9d000000-0000-4000-8000-000000000007" +GEN_8 = "9d000000-0000-4000-8000-000000000008" + + +def _config(): + return BackupConfig.model_validate({"bucket_name": BUCKET, "region": "eu-central-1"}) + + +@pytest.fixture +def db(): + db = DBController() + c = Cluster() + c.uuid = CLUSTER + c.backup_config = _config().model_dump(exclude_none=True) + c.write_to_db(db.kv_store) + return db + + +class _GroupManifests: + """An in-memory bucket, behind the three object calls the group manifest + functions make, read and written by group backup uuid.""" + + def __init__(self): + self.objects: dict = {} + + def put_object(self, Bucket, Key, Body, **_): + self.objects[Key] = Body + + def get_object(self, Bucket, Key): + if Key not in self.objects: + raise ClientError({"Error": {"Code": "NoSuchKey"}}, "GetObject") + return {"Body": io.BytesIO(self.objects[Key])} + + def delete_object(self, Bucket, Key): + self.objects.pop(Key, None) + + @staticmethod + def _key(uuid): + return backup_manifest.group_manifest_key(UUID(uuid)) + + def __getitem__(self, uuid): + return backup_manifest._parse_group(self.objects[self._key(uuid)], self._key(uuid)) + + def __setitem__(self, uuid, manifest): + self.objects[self._key(uuid)] = manifest.model_dump_json().encode() + + def __contains__(self, uuid): + return self._key(uuid) in self.objects + + def __len__(self): + return len(self.objects) + + +@pytest.fixture +def bucket(monkeypatch): + """The group manifests in the bucket. The manifest functions run for real + over it; only the S3 client is faked.""" + b = _GroupManifests() + monkeypatch.setattr(backup_manifest, "s3_client", lambda config: b) + return b + + +def _backup_uuid(gen, lvol): + return f"0ae0{gen:04d}-0000-4000-8000-{lvol:012d}" + + +def _member(db, gen_uuid, gen, lvol, prev_gen=None, status=Backup.STATUS_COMPLETED): + b = Backup() + b.uuid = _backup_uuid(gen, lvol) + b.s3_id = gen * 100 + lvol + b.cluster_id = CLUSTER + b.lvol_id = f"lv-{lvol}" + b.node_id = NODE + b.snapshot_id = f"5a000000-0000-4000-8000-{gen * 100 + lvol:012d}" + b.prev_backup_id = _backup_uuid(prev_gen, lvol) if prev_gen is not None else "" + b.group_backup_id = f"{CLUSTER}/{gen_uuid}" + b.group_seq = gen + b.size = 4096 + b.created_at = gen + b.status = status + b.location = _config().location().model_dump(mode="json") + b.write_to_db(db.kv_store) + return b + + +def _group_backup(db, bucket, gen_uuid, gen, members, status=GroupBackup.STATUS_COMPLETED): + gb = GroupBackup() + gb.uuid = gen_uuid + gb.cluster_id = CLUSTER + gb.consistency_group_id = f"{CLUSTER}/cccc0000-0000-4000-8000-000000000001" + gb.consistency_group_name = "gitea" + gb.group_seq = gen + gb.members = [{"lvol_id": m.lvol_id, "backup_id": m.get_id()} for m in members] + gb.created_at = 100 + gb.status = status + if status == GroupBackup.STATUS_COMPLETED: + gb.completed_at = 200 + bucket[gen_uuid] = group_backup.build_group_manifest( + gb, {m.get_id(): m for m in members}) + gb.write_to_db(db.kv_store) + return gb + + +def _two_generations(db, bucket): + """Generation 7 of two volumes, and generation 8 of one of them. + + lv-1's generation-7 copy is the base its generation-8 copy chains against, + so retiring it is a merge; lv-2's has no successor, so it is a leaf. + """ + base = _member(db, GEN_7, 7, 1) + leaf = _member(db, GEN_7, 7, 2) + gb = _group_backup(db, bucket, GEN_7, 7, [base, leaf]) + successor = _member(db, GEN_8, 8, 1, prev_gen=7) + _group_backup(db, bucket, GEN_8, 8, [successor]) + return gb, base, leaf, successor + + +def _status(db, backup): + try: + return db.get_backup_by_id(backup.get_id()).status + except KeyError: + return None + + +def _delete_tasks(db): + return [t for t in db.get_job_tasks_by_function(CLUSTER, JobSchedule.FN_GROUP_BACKUP_DELETE) + if t.status != JobSchedule.STATUS_DONE and not t.canceled] + + +class TestMarkFirst: + + def test_the_record_and_manifest_are_marked_before_any_member_is_touched(self, db, bucket): + gb, base, leaf, _ = _two_generations(db, bucket) + + group_backup.delete_group_backup(gb.get_id()) + + assert db.get_group_backup_by_id(gb.get_id()).status == GroupBackup.STATUS_DELETING + assert bucket[GEN_7].state == GroupManifestState.DELETING + assert _status(db, base) == Backup.STATUS_COMPLETED + assert _status(db, leaf) == Backup.STATUS_COMPLETED + assert len(_delete_tasks(db)) == 1 + + def test_a_manifest_that_cannot_be_marked_refuses_the_delete_and_changes_nothing( + self, db, bucket, monkeypatch): + gb, base, leaf, _ = _two_generations(db, bucket) + + def unreachable(config, manifest): + raise backup_manifest.ManifestError("bucket unreachable") + monkeypatch.setattr(backup_manifest, "write_group", unreachable) + + with pytest.raises(backup_manifest.ManifestError): + group_backup.delete_group_backup(gb.get_id()) + + assert db.get_group_backup_by_id(gb.get_id()).status == GroupBackup.STATUS_COMPLETED + assert bucket[GEN_7].state == GroupManifestState.COMMITTED + assert _delete_tasks(db) == [] + + def test_a_group_backup_still_committing_is_refused(self, db, bucket): + member = _member(db, GEN_7, 7, 1, status=Backup.STATUS_IN_PROGRESS) + gb = _group_backup(db, bucket, GEN_7, 7, [member], status=GroupBackup.STATUS_IN_PROGRESS) + + with pytest.raises(PreconditionError, match="committing"): + group_backup.delete_group_backup(gb.get_id()) + + assert db.get_group_backup_by_id(gb.get_id()).status == GroupBackup.STATUS_IN_PROGRESS + + def test_deleting_a_group_backup_already_being_deleted_queues_no_second_task(self, db, bucket): + gb, *_ = _two_generations(db, bucket) + + group_backup.delete_group_backup(gb.get_id()) + group_backup.delete_group_backup(gb.get_id()) + + assert len(_delete_tasks(db)) == 1 + + def test_an_unknown_group_backup_raises(self, db, bucket): + with pytest.raises(KeyError): + group_backup.delete_group_backup(f"{CLUSTER}/{GEN_7}") + + +class TestRetireMembers: + + def test_a_leaf_is_deleted_and_a_base_merged_while_the_group_stays(self, db, bucket): + gb, base, leaf, _ = _two_generations(db, bucket) + group_backup.delete_group_backup(gb.get_id()) + + state = group_backup.advance_group_backup_delete(gb.get_id()) + + assert state == group_backup.DeleteState.PENDING + assert _status(db, leaf) is None + assert _status(db, base) == Backup.STATUS_MERGING + assert db.get_group_backup_by_id(gb.get_id()).status == GroupBackup.STATUS_DELETING + assert bucket[GEN_7].state == GroupManifestState.DELETING + + def test_the_manifest_and_record_go_only_once_every_member_is_retired(self, db, bucket): + gb, base, leaf, _ = _two_generations(db, bucket) + group_backup.delete_group_backup(gb.get_id()) + group_backup.advance_group_backup_delete(gb.get_id()) + merged = db.get_backup_by_id(base.get_id()) + merged.status = Backup.STATUS_MERGED # the merge task finished + merged.write_to_db(db.kv_store) + + state = group_backup.advance_group_backup_delete(gb.get_id()) + + assert state == group_backup.DeleteState.DONE + assert GEN_7 not in bucket + with pytest.raises(KeyError): + db.get_group_backup_by_id(gb.get_id()) + + def test_the_next_generation_is_left_whole(self, db, bucket): + gb, base, leaf, successor = _two_generations(db, bucket) + group_backup.delete_group_backup(gb.get_id()) + + group_backup.advance_group_backup_delete(gb.get_id()) + + assert _status(db, successor) == Backup.STATUS_COMPLETED + assert db.get_group_backup_by_id(f"{CLUSTER}/{GEN_8}").status == GroupBackup.STATUS_COMPLETED + assert bucket[GEN_8].state == GroupManifestState.COMMITTED + + def test_a_failed_group_backup_has_no_manifest_to_mark(self, db, bucket): + member = _member(db, GEN_7, 7, 1, status=Backup.STATUS_FAILED) + gb = _group_backup(db, bucket, GEN_7, 7, [member], status=GroupBackup.STATUS_FAILED) + + group_backup.delete_group_backup(gb.get_id()) + state = group_backup.advance_group_backup_delete(gb.get_id()) + + assert state == group_backup.DeleteState.DONE + assert len(bucket) == 0 + assert _status(db, member) is None + + +class TestFailure: + + def test_a_delete_that_touched_no_member_restores_the_group_backup(self, db, bucket): + gb, base, leaf, _ = _two_generations(db, bucket) + group_backup.delete_group_backup(gb.get_id()) + + group_backup.abandon_group_backup_delete(gb.get_id(), "timeout after 3600s") + + restored = db.get_group_backup_by_id(gb.get_id()) + assert restored.status == GroupBackup.STATUS_COMPLETED + assert "timeout" in restored.error_message + assert bucket[GEN_7].state == GroupManifestState.COMMITTED + + def test_a_delete_that_retired_a_member_marks_the_group_backup_broken(self, db, bucket): + gb, base, leaf, _ = _two_generations(db, bucket) + group_backup.delete_group_backup(gb.get_id()) + group_backup.advance_group_backup_delete(gb.get_id()) # leaf deleted, base merging + + group_backup.abandon_group_backup_delete(gb.get_id(), "timeout after 3600s") + + broken = db.get_group_backup_by_id(gb.get_id()) + assert broken.status == GroupBackup.STATUS_BROKEN + assert "timeout" in broken.error_message + assert bucket[GEN_7].state == GroupManifestState.BROKEN + + def test_a_broken_group_backup_is_not_restorable(self, db, bucket): + gb, *_ = _two_generations(db, bucket) + group_backup.delete_group_backup(gb.get_id()) + group_backup.advance_group_backup_delete(gb.get_id()) + group_backup.abandon_group_backup_delete(gb.get_id(), "timeout") + + _, _, err = group_backup.restore_group_backup(gb.get_id(), "restored", "pool") + + assert err is not None + + def test_a_broken_group_backup_can_be_deleted_again(self, db, bucket): + gb, base, *_ = _two_generations(db, bucket) + group_backup.delete_group_backup(gb.get_id()) + group_backup.advance_group_backup_delete(gb.get_id()) + group_backup.abandon_group_backup_delete(gb.get_id(), "timeout") + for task in _delete_tasks(db): + task.status = JobSchedule.STATUS_DONE + task.write_to_db(db.kv_store) + + group_backup.delete_group_backup(gb.get_id()) + + assert db.get_group_backup_by_id(gb.get_id()).status == GroupBackup.STATUS_DELETING + assert bucket[GEN_7].state == GroupManifestState.DELETING + assert len(_delete_tasks(db)) == 1 + + +class TestTask: + """The FN_GROUP_BACKUP_DELETE task drives the retirement across cycles.""" + + def _task(self, db, gb): + return [t for t in _delete_tasks(db) + if t.function_params.get("group_backup_id") == gb.uuid][0] + + def test_it_defers_while_a_member_is_still_being_retired(self, db, bucket): + from simplyblock_core.services import tasks_runner_backup as runner + from simplyblock_core.services.task_runner_base import TaskDefer + gb, *_ = _two_generations(db, bucket) + group_backup.delete_group_backup(gb.get_id()) + + with pytest.raises(TaskDefer): + runner._run_group_backup_delete(self._task(db, gb)) + + def test_it_owns_the_delete_function(self): + from simplyblock_core.services import tasks_runner_backup as runner + assert JobSchedule.FN_GROUP_BACKUP_DELETE in runner.SPEC.function_names + assert JobSchedule.FN_GROUP_BACKUP_DELETE in runner.BACKUP_FUNCTIONS + + def test_a_task_that_ends_unfinished_settles_the_group_backup(self, db, bucket): + """A timeout, the retry ceiling, or a cancellation the handler never sees: + the group backup is restored or marked broken, never left deleting.""" + from simplyblock_core.services import tasks_runner_backup as runner + gb, *_ = _two_generations(db, bucket) + group_backup.delete_group_backup(gb.get_id()) + task = self._task(db, gb) + task.function_result = "timeout after 3600s" + + runner.finalize_resource(task) + + assert db.get_group_backup_by_id(gb.get_id()).status == GroupBackup.STATUS_COMPLETED + assert bucket[GEN_7].state == GroupManifestState.COMMITTED diff --git a/tests/integration/backup/test_group_backup_export.py b/tests/integration/backup/test_group_backup_export.py new file mode 100644 index 0000000000..bf00b4a204 --- /dev/null +++ b/tests/integration/backup/test_group_backup_export.py @@ -0,0 +1,330 @@ +"""Group-backup export, import and discover against real FoundationDB +(design-consistency-group-backup.md §6.5). + +A group backup is portable the way a single backup is: export emits the group +manifest together with the member backup chains it names, import preserves ids +(the member uuids and s3_ids, the group backup's own uuid, and the generation) +all-or-nothing, and discover reads the ``groups/`` prefix with no cluster +reference. These prove the round-trip the way the single-volume +``test_manifest_flow.py`` does: build records, throw the database away, rebuild +from the export alone. + +The database is real; nothing here mocks the DB layer. Only boto3 (reached +through ``backup_manifest.list_all_groups`` / ``list_all``) is patched, since it +is an external service client. +""" +from unittest.mock import patch + +import pytest + +from simplyblock_core.controllers.backup import group_backup +from simplyblock_core.controllers.backup import manifest as backup_manifest +from simplyblock_core.db_controller import DBController +from simplyblock_core.exceptions import PreconditionError +from simplyblock_core.models.backup import Backup, GroupBackup +from simplyblock_core.models.backup_config import BackupConfig +from simplyblock_core.models.cluster import Cluster + +# Every id a manifest carries is a UUID, so the records these tests build get real +# ones rather than readable stand-ins. +SOURCE_CLUSTER = "c1000000-0000-4000-8000-000000000001" +TARGET_CLUSTER = "c1000000-0000-4000-8000-000000000002" +NODE_ID = "d0de0000-0000-4000-8000-000000000001" +GROUP_BACKUP_UUID = "9b000000-0000-4000-8000-000000000001" +CG_ID = f"{SOURCE_CLUSTER}/cccc0000-0000-4000-8000-000000000001" + +BUCKET = "simplyblock-backup-primary" + + +def _backup_id(index: int) -> str: + return f"0ac00000-0000-4000-8000-{index:012d}" + + +def _lvol_id(index: int) -> str: + return f"10101000-0000-4000-8000-{index:012d}" + + +def _snapshot_id(index: int) -> str: + return f"50a50000-0000-4000-8000-{index:012d}" + + +def _config(**overrides): + return BackupConfig.model_validate({ + "bucket_name": BUCKET, "region": "eu-central-1", **overrides}) + + +@pytest.fixture +def db(): + return DBController() + + +@pytest.fixture +def source_cluster(db): + c = Cluster() + c.uuid = SOURCE_CLUSTER + c.cluster_name = "primary" + c.backup_config = _config().model_dump(exclude_none=True) + c.write_to_db(db.kv_store) + return c + + +def _member_backup(db, index, cluster=SOURCE_CLUSTER, prev=None, + group_backup_id="", status=Backup.STATUS_COMPLETED): + b = Backup() + b.uuid = _backup_id(index) + b.s3_id = index + b.cluster_id = cluster + b.lvol_id = _lvol_id(index) + b.lvol_name = f"vol-{index}" + b.snapshot_id = _snapshot_id(index) + b.snapshot_name = f"snap-{index}" + b.node_id = NODE_ID + b.prev_backup_id = _backup_id(prev) if prev is not None else "" + b.size = 4096 + b.created_at = 1000 + index + b.completed_at = 2000 + index + b.status = status + b.location = _config().location().model_dump(mode="json") + b.group_backup_id = group_backup_id + b.group_seq = 7 if group_backup_id else 0 + b.write_to_db(db.kv_store) + return b + + +def _group_backup(db, members, cluster=SOURCE_CLUSTER, + status=GroupBackup.STATUS_COMPLETED): + gb = GroupBackup() + gb.uuid = GROUP_BACKUP_UUID + gb.cluster_id = cluster + gb.consistency_group_id = CG_ID + gb.consistency_group_name = "gitea" + gb.group_seq = 7 + gb.status = status + gb.members = [{"lvol_id": m.lvol_id, "backup_id": m.get_id()} for m in members] + gb.size = sum(m.size for m in members) + gb.created_at = 100 + gb.completed_at = 200 + gb.write_to_db(db.kv_store) + return gb + + +def _seed_two_member_group(db): + """A committed group backup of two members, one of which has a two-link chain. + + Member 1 is a full backup; member 2 chains onto its own ancestor (backup 3), + so the export has to carry that ancestor for the group to be restorable. + """ + composite = f"{SOURCE_CLUSTER}/{GROUP_BACKUP_UUID}" + ancestor = _member_backup(db, 3) # member 2's parent + m1 = _member_backup(db, 1, group_backup_id=composite) + m2 = _member_backup(db, 2, prev=3, group_backup_id=composite) + return _group_backup(db, [m1, m2]), ancestor, m1, m2 + + +class TestExport: + + def test_export_carries_the_group_manifest_and_member_chains( + self, db, source_cluster): + gb, ancestor, m1, m2 = _seed_two_member_group(db) + + export = group_backup.export_group_backup(gb.get_id(), cluster_id=SOURCE_CLUSTER) + + assert str(export.group_manifest.group_backup_id) == GROUP_BACKUP_UUID + assert export.group_manifest.source.generation == 7 + # Member 2's ancestor travels too, so the group is restorable. + exported = {str(m.backup_id) + for group in export.members.groups for m in group.manifests} + assert exported == {_backup_id(1), _backup_id(2), _backup_id(3)} + + def test_export_refuses_an_uncommitted_group_backup(self, db, source_cluster): + gb, *_ = _seed_two_member_group(db) + gb.status = GroupBackup.STATUS_IN_PROGRESS + gb.write_to_db(db.kv_store) + + with pytest.raises(PreconditionError, match="not.*completed"): + group_backup.export_group_backup(gb.get_id(), cluster_id=SOURCE_CLUSTER) + + def test_export_refuses_when_a_member_record_is_missing(self, db, source_cluster): + gb, ancestor, m1, m2 = _seed_two_member_group(db) + db.get_backup_by_id(m2.get_id()).remove(db.kv_store) + + with pytest.raises(PreconditionError, match="missing"): + group_backup.export_group_backup(gb.get_id(), cluster_id=SOURCE_CLUSTER) + + +class TestExportImportRoundTrip: + """Export, wipe the database, import into another cluster.""" + + def _export_and_wipe(self, db): + gb, ancestor, m1, m2 = _seed_two_member_group(db) + export = group_backup.export_group_backup(gb.get_id(), cluster_id=SOURCE_CLUSTER) + for record in db.get_backups(): + record.remove(db.kv_store) + db.get_group_backup_by_id(gb.get_id()).remove(db.kv_store) + assert db.get_backups() == [] + return export + + def test_group_backup_survives_losing_the_database(self, db, source_cluster): + export = self._export_and_wipe(db) + + composite = group_backup.import_group_backup(export, cluster_id=TARGET_CLUSTER) + + assert composite == f"{TARGET_CLUSTER}/{GROUP_BACKUP_UUID}" + restored = db.get_group_backup_by_id(composite) + assert restored.status == GroupBackup.STATUS_COMPLETED + assert restored.group_seq == 7 + assert restored.consistency_group_name == "gitea" + assert {m["backup_id"] for m in restored.members} == { + f"{TARGET_CLUSTER}/{_backup_id(1)}", f"{TARGET_CLUSTER}/{_backup_id(2)}"} + # Where the group lives is its members': each imported member records it. + for member_id in restored.member_backup_ids(): + assert db.get_backup_by_id(member_id).get_location().bucket_name == BUCKET + + def test_member_ids_and_group_stamp_are_preserved(self, db, source_cluster): + export = self._export_and_wipe(db) + + group_backup.import_group_backup(export, cluster_id=TARGET_CLUSTER) + + member = db.get_backup_by_id(_backup_id(2)) + assert member.s3_id == 2 # names the objects; never reassigned + assert member.cluster_id == TARGET_CLUSTER + assert member.group_backup_id == f"{TARGET_CLUSTER}/{GROUP_BACKUP_UUID}" + assert member.group_seq == 7 + # The index resolves "the members of this group backup". + found = db.get_backups_by_group_backup_id(f"{TARGET_CLUSTER}/{GROUP_BACKUP_UUID}") + assert {b.uuid for b in found} == {_backup_id(1), _backup_id(2)} + + def test_the_ancestor_lands_as_a_standalone_backup(self, db, source_cluster): + """A member's chain ancestor is not itself a group member, so it imports + without a group stamp -- but it has to be present for the restore.""" + export = self._export_and_wipe(db) + + group_backup.import_group_backup(export, cluster_id=TARGET_CLUSTER) + + ancestor = db.get_backup_by_id(_backup_id(3)) + assert ancestor.group_backup_id == "" + assert ancestor.cluster_id == TARGET_CLUSTER + + def test_a_group_id_collision_rejects_the_whole_batch(self, db, source_cluster): + export = group_backup.export_group_backup( + _seed_two_member_group(db)[0].get_id(), cluster_id=SOURCE_CLUSTER) + + # Importing back into the SAME cluster collides on the group backup id. + with pytest.raises(PreconditionError, match="already exists"): + group_backup.import_group_backup(export, cluster_id=SOURCE_CLUSTER) + + def test_a_member_id_collision_rejects_the_whole_batch(self, db, source_cluster): + """import_backups' per-backup collision check refuses the member, and the + GroupBackup record is never written -- the whole group imports or none.""" + export = self._export_and_wipe(db) + # A pre-existing member record in the target cluster. + _member_backup(db, 1, cluster=TARGET_CLUSTER) + + with pytest.raises(PreconditionError, match="already exists"): + group_backup.import_group_backup(export, cluster_id=TARGET_CLUSTER) + + with pytest.raises(KeyError): + db.get_group_backup_by_id(f"{TARGET_CLUSTER}/{GROUP_BACKUP_UUID}") + + @pytest.mark.parametrize("state", [backup_manifest.GroupManifestState.DELETING, + backup_manifest.GroupManifestState.BROKEN]) + def test_a_group_backup_being_deleted_or_broken_is_refused( + self, db, source_cluster, state): + """Its members are, or may already be, retired on the source: importing it + would land a group backup that cannot restore.""" + export = self._export_and_wipe(db) + export.group_manifest = export.group_manifest.model_copy(update={"state": state}) + + with pytest.raises(PreconditionError, match=str(state)): + group_backup.import_group_backup(export, cluster_id=TARGET_CLUSTER) + + with pytest.raises(KeyError): + db.get_group_backup_by_id(f"{TARGET_CLUSTER}/{GROUP_BACKUP_UUID}") + + def test_a_torn_export_is_refused(self, db, source_cluster): + """The group names a member whose manifest the export does not carry.""" + export = self._export_and_wipe(db) + # Drop member 2's manifest from the members, leaving the group naming it. + for group in export.members.groups: + group.manifests = [m for m in group.manifests + if str(m.backup_id) != _backup_id(2)] + + with pytest.raises(ValueError, match="missing member manifests"): + group_backup.import_group_backup(export, cluster_id=TARGET_CLUSTER) + + with pytest.raises(KeyError): + db.get_group_backup_by_id(f"{TARGET_CLUSTER}/{GROUP_BACKUP_UUID}") + + +class TestDiscoverAndImportFromBucket: + """The disaster-recovery path: nothing but a bucket and credentials.""" + + def test_discover_reads_the_group_prefix(self, db, source_cluster): + gb, *_ = _seed_two_member_group(db) + group_manifest = group_backup.export_group_backup( + gb.get_id(), cluster_id=SOURCE_CLUSTER).group_manifest + + with patch.object(backup_manifest, "list_all_groups", + return_value=[group_manifest]) as list_groups: + found = group_backup.discover_group_backups(_config()) + + list_groups.assert_called_once() + assert [str(m.group_backup_id) for m in found] == [GROUP_BACKUP_UUID] + assert all(isinstance(m, backup_manifest.GroupBackupManifest) for m in found) + + def test_import_from_bucket_needs_no_prior_records(self, db, source_cluster): + export = group_backup.export_group_backup( + _seed_two_member_group(db)[0].get_id(), cluster_id=SOURCE_CLUSTER) + member_manifests = [m for group in export.members.groups for m in group.manifests] + for record in db.get_backups(): + record.remove(db.kv_store) + db.get_group_backup_by_id(f"{SOURCE_CLUSTER}/{GROUP_BACKUP_UUID}").remove(db.kv_store) + + with patch.object(backup_manifest, "list_all_groups", + return_value=[export.group_manifest]), \ + patch.object(backup_manifest, "list_all", + return_value=member_manifests): + imported = group_backup.import_group_backup_from_bucket( + _config(), cluster_id=TARGET_CLUSTER) + + assert imported == [f"{TARGET_CLUSTER}/{GROUP_BACKUP_UUID}"] + restored = db.get_group_backup_by_id(f"{TARGET_CLUSTER}/{GROUP_BACKUP_UUID}") + assert {m["backup_id"] for m in restored.members} == { + f"{TARGET_CLUSTER}/{_backup_id(1)}", f"{TARGET_CLUSTER}/{_backup_id(2)}"} + + def test_import_from_bucket_skips_a_group_backup_being_deleted(self, db, source_cluster): + """A bucket can hold a group backup whose deletion is in flight on its + source; the disaster-recovery import leaves it and imports the rest.""" + export = group_backup.export_group_backup( + _seed_two_member_group(db)[0].get_id(), cluster_id=SOURCE_CLUSTER) + member_manifests = [m for group in export.members.groups for m in group.manifests] + for record in db.get_backups(): + record.remove(db.kv_store) + db.get_group_backup_by_id(f"{SOURCE_CLUSTER}/{GROUP_BACKUP_UUID}").remove(db.kv_store) + deleting = export.group_manifest.model_copy( + update={"state": backup_manifest.GroupManifestState.DELETING}) + + with patch.object(backup_manifest, "list_all_groups", return_value=[deleting]), \ + patch.object(backup_manifest, "list_all", return_value=member_manifests): + imported = group_backup.import_group_backup_from_bucket( + _config(), cluster_id=TARGET_CLUSTER) + + assert imported == [] + assert db.get_backups() == [] + + def test_import_from_bucket_refuses_a_group_missing_its_members( + self, db, source_cluster): + """A group manifest in the bucket names a member whose manifest is not + there -- the bucket is torn, and the import says so rather than landing a + group that cannot restore.""" + export = group_backup.export_group_backup( + _seed_two_member_group(db)[0].get_id(), cluster_id=SOURCE_CLUSTER) + for record in db.get_backups(): + record.remove(db.kv_store) + + with patch.object(backup_manifest, "list_all_groups", + return_value=[export.group_manifest]), \ + patch.object(backup_manifest, "list_all", return_value=[]): + with pytest.raises(ValueError, match="does not hold"): + group_backup.import_group_backup_from_bucket( + _config(), cluster_id=TARGET_CLUSTER) diff --git a/tests/integration/backup/test_group_backup_location.py b/tests/integration/backup/test_group_backup_location.py new file mode 100644 index 0000000000..ddf1fb9d37 --- /dev/null +++ b/tests/integration/backup/test_group_backup_location.py @@ -0,0 +1,83 @@ +"""Where a group backup's manifest lives comes from its member backups, against +real FoundationDB. + +A group backup's members share one location (design-consistency-group-backup.md +§3.4), and each member Backup records it. The group record keeps no copy of its +own: the bucket its group manifest is written to and deleted from is the one its +members live in, and only the credentials come from the cluster. +""" +import pytest + +from simplyblock_core.controllers.backup import group_backup +from simplyblock_core.db_controller import DBController +from simplyblock_core.exceptions import PreconditionError +from simplyblock_core.models.backup import Backup, GroupBackup +from simplyblock_core.models.backup_config import BackupConfig +from simplyblock_core.models.cluster import Cluster + +CLUSTER = "c2000000-0000-4000-8000-000000000001" +GROUP_BACKUP_UUID = "9c000000-0000-4000-8000-000000000001" +BUCKET = "simplyblock-backup-primary" + + +def _config(bucket=BUCKET): + return BackupConfig.model_validate({"bucket_name": bucket, "region": "eu-central-1"}) + + +@pytest.fixture +def db(): + db = DBController() + c = Cluster() + c.uuid = CLUSTER + c.backup_config = _config().model_dump(exclude_none=True) + c.write_to_db(db.kv_store) + return db + + +def _member(db, index, bucket=BUCKET): + b = Backup() + b.uuid = f"0ad00000-0000-4000-8000-{index:012d}" + b.cluster_id = CLUSTER + b.lvol_id = f"lv-{index}" + b.group_backup_id = f"{CLUSTER}/{GROUP_BACKUP_UUID}" + b.status = Backup.STATUS_COMPLETED + b.location = _config(bucket).location().model_dump(mode="json") + b.write_to_db(db.kv_store) + return b + + +def _group_backup(db, members): + gb = GroupBackup() + gb.uuid = GROUP_BACKUP_UUID + gb.cluster_id = CLUSTER + gb.members = [{"lvol_id": m.lvol_id, "backup_id": m.get_id()} for m in members] + gb.status = GroupBackup.STATUS_IN_PROGRESS + gb.write_to_db(db.kv_store) + return gb + + +def test_the_group_record_keeps_no_location_of_its_own(): + assert "location" not in GroupBackup().get_attrs_map() + + +def test_the_group_manifest_goes_to_the_bucket_its_members_live_in(db): + gb = _group_backup(db, [_member(db, 1), _member(db, 2)]) + + assert group_backup._config_for_group(gb).bucket_name == BUCKET + + +def test_a_member_outside_the_configured_bucket_is_refused(db): + """The cluster's credentials reach only its configured bucket: a member that + lives elsewhere (the configuration was repointed mid-flight) is refused, as + the single-volume path refuses it.""" + gb = _group_backup(db, [_member(db, 1), _member(db, 2, bucket="the-old-bucket")]) + + with pytest.raises(PreconditionError, match="the-old-bucket"): + group_backup._config_for_group(gb) + + +def test_a_group_backup_with_no_member_record_left_has_no_location(db): + gb = _group_backup(db, []) + + with pytest.raises(PreconditionError, match="no member backup"): + group_backup._config_for_group(gb) diff --git a/tests/integration/backup/test_group_backup_restore.py b/tests/integration/backup/test_group_backup_restore.py new file mode 100644 index 0000000000..90fa202b56 --- /dev/null +++ b/tests/integration/backup/test_group_backup_restore.py @@ -0,0 +1,192 @@ +"""Group restore re-forms a consistency group from a group backup, all-or-nothing, +against real FoundationDB (design-consistency-group-backup.md §6). + +The database is real; nothing here mocks the DB layer. The boundaries patched are +the ones above it: the single-volume restore (which creates the volume and drives +the data-plane recovery), the chain validation (its own unit covers it), and the +volume teardown used to unwind. The group forming (ensure_group / +add_member_to_group) runs for real against FDB, which is what the all-or-nothing +and single-store rules turn on. +""" +import zlib +from unittest.mock import patch + +import pytest + +from simplyblock_core.controllers import consistency_group_controller as cg +from simplyblock_core.controllers.backup import group_backup +from simplyblock_core.db_controller import DBController +from simplyblock_core.models.backup import Backup, GroupBackup +from simplyblock_core.models.lvol_model import LVol + + +def _uuid(name: str) -> str: + # Restore validates each member's chain with UUID(backup.uuid), so member + # backup ids must be real UUIDs, not readable labels. + return f"{zlib.crc32(name.encode()):08x}-0000-4000-8000-000000000000" + + +CLUSTER = _uuid("cluster") +RESTORE_BACKUP = "simplyblock_core.controllers.backup.controller.restore_backup" +BACKUP_CHAIN = "simplyblock_core.controllers.backup.chain.BackupChain" +DELETE_LVOL = "simplyblock_core.controllers.lvol_controller.delete_lvol" + + +@pytest.fixture +def db(): + return DBController() + + +def _member(name, lvol_name, status=Backup.STATUS_COMPLETED): + b = Backup() + b.uuid = _uuid(name) + b.cluster_id = CLUSTER + b.lvol_id = _uuid("lv-" + name) + b.lvol_name = lvol_name + b.size = 100 + b.status = status + return b + + +def _seed_group(db, members, status=GroupBackup.STATUS_COMPLETED): + for m in members: + m.write_to_db(db.kv_store) + gb = GroupBackup() + gb.uuid = "gb-1" + gb.cluster_id = CLUSTER + gb.consistency_group_id = f"{CLUSTER}/cg-1" + gb.consistency_group_name = "gitea" + gb.group_seq = 7 + gb.status = status + gb.members = [{"lvol_id": m.lvol_id, "backup_id": m.get_id()} for m in members] + gb.write_to_db(db.kv_store) + return gb + + +def _restored_lvol(db, uuid, node="node-1", lvs="LVS_1"): + """A volume standing in for what the (mocked) single-volume restore creates.""" + lv = LVol() + lv.uuid = uuid + lv.cluster_id = CLUSTER + lv.pool_uuid = "pool-1" + lv.node_id = node + lv.lvs_name = lvs + lv.lvol_name = uuid + lv.status = LVol.STATUS_RESTORING + lv.write_to_db(db.kv_store) + return lv + + +def test_restore_forms_a_new_group_from_all_members(db): + gb = _seed_group(db, [_member("b-db", "db"), _member("b-data", "data")]) + _restored_lvol(db, "rv-db") + _restored_lvol(db, "rv-data") + + with patch(RESTORE_BACKUP, side_effect=["rv-db", "rv-data"]) as restore, \ + patch(BACKUP_CHAIN): + group_id, members, err = group_backup.restore_group_backup( + gb.get_id(), "restored-cg", "pool-1") + + assert err is None + assert members == [ + {"lvol_id": "rv-db", "member_name": "db"}, + {"lvol_id": "rv-data", "member_name": "data"}] + # The second member is pinned to the first member's node, not auto-placed. + assert restore.call_args_list[1].kwargs["target_node_id"] == "node-1" + + group = db.get_consistency_group_by_name(CLUSTER, "restored-cg") + assert group is not None + assert group.get_id() == group_id + assert set(group.members) == {"rv-db", "rv-data"} + + +def test_a_member_restore_failure_unwinds_and_forms_no_group(db): + gb = _seed_group(db, [_member("b-db", "db"), _member("b-data", "data")]) + _restored_lvol(db, "rv-db") + + with patch(RESTORE_BACKUP, side_effect=["rv-db", RuntimeError("recovery RPC failed")]), \ + patch(BACKUP_CHAIN), \ + patch(DELETE_LVOL) as delete_lvol: + group_id, lvol_ids, err = group_backup.restore_group_backup( + gb.get_id(), "restored-cg", "pool-1") + + assert group_id is None and lvol_ids is None + assert "could not be restored" in err + # The one volume already created is torn down, and no group was formed. + assert delete_lvol.call_count == 1 + assert db.get_consistency_group_by_name(CLUSTER, "restored-cg") is None + + +def test_an_off_store_member_unwinds_including_the_half_formed_group(db): + gb = _seed_group(db, [_member("b-db", "db"), _member("b-data", "data")]) + _restored_lvol(db, "rv-db", node="node-1", lvs="LVS_1") + _restored_lvol(db, "rv-data", node="node-2", lvs="LVS_2") # a different store + + with patch(RESTORE_BACKUP, side_effect=["rv-db", "rv-data"]), \ + patch(BACKUP_CHAIN), \ + patch(DELETE_LVOL) as delete_lvol: + group_id, lvol_ids, err = group_backup.restore_group_backup( + gb.get_id(), "restored-cg", "pool-1") + + assert group_id is None + assert "could not form group" in err + # Both restored volumes are torn down and the half-formed group is removed, + # so the name does not block a retry. + assert delete_lvol.call_count == 2 + assert db.get_consistency_group_by_name(CLUSTER, "restored-cg") is None + + +def test_refuses_an_uncommitted_group_backup(db): + gb = _seed_group(db, [_member("b-db", "db")], + status=GroupBackup.STATUS_IN_PROGRESS) + group_id, lvol_ids, err = group_backup.restore_group_backup( + gb.get_id(), "restored-cg", "pool-1") + assert group_id is None and lvol_ids is None + assert "not" in err and "completed" in err + + +def test_refuses_a_name_held_by_an_empty_group_and_keeps_it(db): + """A group restore always produces a new group. An empty group holding the + name is not known to be meant for replacing, so it is refused, never + deleted.""" + gb = _seed_group(db, [_member("b-db", "db")]) + existing = cg.ensure_group(CLUSTER, "taken") + group_id, lvol_ids, err = group_backup.restore_group_backup( + gb.get_id(), "taken", "pool-1") + assert group_id is None and lvol_ids is None + assert "already exists" in err + assert db.get_consistency_group_by_name(CLUSTER, "taken").uuid == existing.uuid + + +def test_refuses_when_the_target_group_name_is_in_use(db): + """A group with a live member is not a leftover: a group restore always + produces a NEW group, never one that is serving volumes.""" + gb = _seed_group(db, [_member("b-db", "db")]) + taken = cg.ensure_group(CLUSTER, "taken") + # A LIVE member: its volume record exists. An open entry whose volume is + # gone is dangling and would not hold the name (see delete_group). + live = LVol() + live.uuid = _uuid("live-member") + live.lvol_name = "live" + live.group_id = taken.get_id() + live.write_to_db(db.kv_store) + taken.members = {live.uuid: {"joined_seq": 1, "removed_seq": 0}} + taken.write_to_db(db.kv_store) + group_id, lvol_ids, err = group_backup.restore_group_backup( + gb.get_id(), "taken", "pool-1") + assert group_id is None + assert "already exists" in err + assert db.get_consistency_group_by_name(CLUSTER, "taken") is not None + + +def test_refuses_when_a_member_backup_record_is_missing(db): + members = [_member("b-db", "db")] + gb = _seed_group(db, members) + # Record a second member whose Backup record was never written. + gb.members.append({"lvol_id": f"{CLUSTER}/lv-gone", "backup_id": f"{CLUSTER}/b-gone"}) + gb.write_to_db(db.kv_store) + + group_id, lvol_ids, err = group_backup.restore_group_backup( + gb.get_id(), "restored-cg", "pool-1") + assert group_id is None + assert "missing" in err diff --git a/tests/integration/backup/test_group_policy_evaluators.py b/tests/integration/backup/test_group_policy_evaluators.py new file mode 100644 index 0000000000..fab5f3765e --- /dev/null +++ b/tests/integration/backup/test_group_policy_evaluators.py @@ -0,0 +1,261 @@ +"""A consistency group's backup policy applied against real FoundationDB +(design-consistency-group-backup.md §9). + +These are the integration-tier counterpart of the pure-logic +``tests/unit/backup/test_group_retention.py``: there the retention arithmetic is +tested in isolation, here the whole flow of ``evaluate_group_backups`` runs -- +policy resolution through the ``BackupPolicyAttachment`` index, the group-backup +listing, and the take / prune decision -- for real against FDB. + +The database is real; nothing here mocks the DB layer. The only boundary patched +is the data plane: ``group_backup.create_group_backup`` and +``group_backup.delete_group_backup`` are what reach snapshots, the backup runner, +and S3, which the integration tier never drives to a real endpoint. The create +returns a ``(result, error)`` tuple, so its mock returns one too. +""" +import time +import zlib +from unittest.mock import patch + +import pytest + +from simplyblock_core.controllers.backup import policy +from simplyblock_core.db_controller import DBController +from simplyblock_core.models.backup import ( + BackupPolicy, + BackupPolicyAttachment, + GroupBackup, +) +from simplyblock_core.models.replication import ConsistencyGroup + + +def _uuid(name: str) -> str: + # A deterministic, valid UUID string, matching the sibling group-backup + # suites. Nothing in the evaluators converts these ids with UUID(...) -- the + # data-plane boundary that would is patched -- but keeping real UUIDs matches + # the fixture style and stays safe if a path ever does. + return f"{zlib.crc32(name.encode()):08x}-0000-4000-8000-000000000000" + + +CLUSTER = _uuid("cluster") +CREATE = "simplyblock_core.controllers.backup.group_backup.create_group_backup" +DELETE = "simplyblock_core.controllers.backup.group_backup.delete_group_backup" + + +@pytest.fixture +def db(): + return DBController() + + +def _group(db): + group = ConsistencyGroup() + group.uuid = _uuid("cg") + group.cluster_id = CLUSTER + group.group_name = "gitea" + group.write_to_db(db.kv_store) + return group + + +def _policy(db, max_versions=0, max_age_seconds=0, schedule=""): + p = BackupPolicy() + p.uuid = _uuid("policy") + p.cluster_id = CLUSTER + p.policy_name = "nightly" + p.max_versions = max_versions + p.max_age_seconds = max_age_seconds + p.backup_schedule = schedule + p.status = BackupPolicy.STATUS_ACTIVE + p.write_to_db(db.kv_store) + return p + + +def _attach(db, group, p): + att = BackupPolicyAttachment() + att.uuid = _uuid("attachment") + att.cluster_id = CLUSTER + att.policy_id = p.get_id() + att.target_type = "consistency_group" + att.target_id = group.get_id() + att.write_to_db(db.kv_store) + return att + + +def _group_backup(db, group, name, age_seconds, status=GroupBackup.STATUS_COMPLETED): + """A group backup of ``group`` created ``age_seconds`` ago.""" + gb = GroupBackup() + gb.uuid = _uuid(name) + gb.cluster_id = CLUSTER + gb.consistency_group_id = group.get_id() + gb.consistency_group_name = group.group_name + gb.group_seq = 1 + gb.created_at = int(time.time()) - age_seconds + gb.status = status + gb.write_to_db(db.kv_store) + return gb + + +class TestGroupSchedule: + """A schedule: take a generation when due, prune each tier's excess.""" + + def test_no_prior_group_backup_takes_one(self, db): + group = _group(db) + _attach(db, group, _policy(db, schedule="15m,4 60m,11")) + + with patch(CREATE, return_value=(None, None)) as create, \ + patch(DELETE) as delete: + policy.evaluate_group_backups(group) + + create.assert_called_once_with(group.get_id()) + delete.assert_not_called() + + def test_recent_group_backup_within_interval_takes_none(self, db): + group = _group(db) + _attach(db, group, _policy(db, schedule="15m,4")) + # One completed generation a minute ago: well inside the 15m smallest + # tier, so nothing is due, and one generation is under the keep_count. + _group_backup(db, group, "gb-recent", age_seconds=60) + + with patch(CREATE, return_value=(None, None)) as create, \ + patch(DELETE) as delete: + policy.evaluate_group_backups(group) + + create.assert_not_called() + delete.assert_not_called() + + def test_in_flight_group_backup_takes_none(self, db): + group = _group(db) + _attach(db, group, _policy(db, schedule="15m,4")) + # A generation still committing gates a second one, even though the + # smallest interval has long elapsed (created an hour ago). + _group_backup(db, group, "gb-flight", age_seconds=3600, + status=GroupBackup.STATUS_IN_PROGRESS) + + with patch(CREATE, return_value=(None, None)) as create, \ + patch(DELETE) as delete: + policy.evaluate_group_backups(group) + + create.assert_not_called() + delete.assert_not_called() + + def test_tier_over_keep_count_prunes_the_oldest(self, db): + group = _group(db) + # Smallest tier keeps 2; its age band is [0, 3600). Four generations all + # under a minute old sit in that band -- the latest is recent enough that + # nothing is due, and the pruning pass runs. + _attach(db, group, _policy(db, schedule="15m,2 60m,11")) + oldest = _group_backup(db, group, "gb-1", age_seconds=400) + _group_backup(db, group, "gb-2", age_seconds=300) + _group_backup(db, group, "gb-3", age_seconds=200) + _group_backup(db, group, "gb-4", age_seconds=100) + + with patch(CREATE, return_value=(None, None)) as create, \ + patch(DELETE) as delete: + policy.evaluate_group_backups(group) + + create.assert_not_called() + # One delete per cycle, and it is the oldest generation in the tier. + delete.assert_called_once_with(oldest.get_id()) + + +class TestGroupPolicy: + """Count/age retention: the generations over the count or past the age are deleted.""" + + def test_over_max_versions_deletes_the_oldest_excess(self, db): + group = _group(db) + _attach(db, group, _policy(db, max_versions=2)) + gb_a = _group_backup(db, group, "gb-a", age_seconds=400) + gb_b = _group_backup(db, group, "gb-b", age_seconds=300) + _group_backup(db, group, "gb-c", age_seconds=200) + _group_backup(db, group, "gb-d", age_seconds=100) + + with patch(CREATE, return_value=(None, None)) as create, \ + patch(DELETE) as delete: + policy.evaluate_group_backups(group) + + create.assert_not_called() + # Keep the newest two of four: the two oldest are deleted, oldest first. + assert [c.args[0] for c in delete.call_args_list] == [gb_a.get_id(), gb_b.get_id()] + + def test_older_than_max_age_is_deleted(self, db): + group = _group(db) + _attach(db, group, _policy(db, max_age_seconds=100)) + old = _group_backup(db, group, "gb-old", age_seconds=1000) + _group_backup(db, group, "gb-recent", age_seconds=10) + + with patch(CREATE, return_value=(None, None)) as create, \ + patch(DELETE) as delete: + policy.evaluate_group_backups(group) + + create.assert_not_called() + delete.assert_called_once_with(old.get_id()) + + +class TestNoAttachedPolicy: + """A group with no attached policy is neither backed up nor pruned.""" + + def test_nothing_is_taken_or_pruned(self, db): + group = _group(db) + # A policy exists in the cluster but is NOT attached to this group, and + # there are generations that would be pruned if one were. + _policy(db, max_versions=1, schedule="15m,1") + _group_backup(db, group, "gb-1", age_seconds=400) + _group_backup(db, group, "gb-2", age_seconds=100) + + with patch(CREATE, return_value=(None, None)) as create, \ + patch(DELETE) as delete: + policy.evaluate_group_backups(group) + + create.assert_not_called() + delete.assert_not_called() + + +class TestBothRules: + """A policy with a count limit and a schedule is applied as one: one pass + prunes what either rule selects, and none in the pass that takes.""" + + def test_one_pass_prunes_the_union_of_both_rules(self, db): + group = _group(db) + # The schedule keeps 3 in [0, 3600) and so selects nothing; the count + # keeps 2 of 3 and selects the oldest. The newest is recent: nothing due. + _attach(db, group, _policy(db, max_versions=2, schedule="15m,3 60m,11")) + oldest = _group_backup(db, group, "gb-1", age_seconds=300) + _group_backup(db, group, "gb-2", age_seconds=200) + _group_backup(db, group, "gb-3", age_seconds=100) + + with patch(CREATE, return_value=(None, None)) as create, \ + patch(DELETE) as delete: + policy.evaluate_group_backups(group) + + create.assert_not_called() + delete.assert_called_once_with(oldest.get_id()) + + def test_a_generation_both_rules_select_is_deleted_once(self, db): + group = _group(db) + _attach(db, group, _policy(db, max_versions=2, schedule="15m,2 60m,11")) + oldest = _group_backup(db, group, "gb-1", age_seconds=300) + _group_backup(db, group, "gb-2", age_seconds=200) + _group_backup(db, group, "gb-3", age_seconds=100) + + with patch(CREATE, return_value=(None, None)), patch(DELETE) as delete: + policy.evaluate_group_backups(group) + + delete.assert_called_once_with(oldest.get_id()) + + def test_the_pass_that_takes_a_generation_prunes_none(self, db): + group = _group(db) + # Due (the newest is an hour old against a 15m interval) and over the + # count: the take goes first, and pruning waits for it to commit. + _attach(db, group, _policy(db, max_versions=1, schedule="15m,4 60m,11")) + _group_backup(db, group, "gb-1", age_seconds=7200) + _group_backup(db, group, "gb-2", age_seconds=3600) + + with patch(CREATE, return_value=(None, None)) as create, \ + patch(DELETE) as delete: + policy.evaluate_group_backups(group) + + create.assert_called_once_with(group.get_id()) + delete.assert_not_called() + + def test_the_old_pair_is_gone(self): + assert not hasattr(policy, "evaluate_group_policy") + assert not hasattr(policy, "evaluate_group_schedule") diff --git a/tests/integration/backup/test_legacy.py b/tests/integration/backup/test_legacy.py index c5bcd273c9..cab7454441 100644 --- a/tests/integration/backup/test_legacy.py +++ b/tests/integration/backup/test_legacy.py @@ -969,7 +969,7 @@ def test_not_found(self, mock_db): class TestEvaluatePolicy(unittest.TestCase): - @patch("simplyblock_core.controllers.backup.policy.tasks_controller") + @patch("simplyblock_core.controllers.backup.controller.tasks_controller") @patch("simplyblock_core.controllers.backup.policy.db_controller") def test_no_policy(self, mock_db, mock_tasks): mock_db.get_policy_for_lvol.return_value = None @@ -981,7 +981,7 @@ def test_no_policy(self, mock_db, mock_tasks): mock_tasks.add_backup_merge_task.assert_not_called() @patch.object(Backup, 'write_to_db') - @patch("simplyblock_core.controllers.backup.policy.tasks_controller") + @patch("simplyblock_core.controllers.backup.controller.tasks_controller") @patch("simplyblock_core.controllers.backup.policy.db_controller") def test_version_limit_exceeded(self, mock_db, mock_tasks, _mock_write): policy = _policy(max_versions=2, max_age_seconds=0) @@ -999,7 +999,7 @@ def test_version_limit_exceeded(self, mock_db, mock_tasks, _mock_write): mock_tasks.add_backup_merge_task.assert_called_once() - @patch("simplyblock_core.controllers.backup.policy.tasks_controller") + @patch("simplyblock_core.controllers.backup.controller.tasks_controller") @patch("simplyblock_core.controllers.backup.policy.db_controller") def test_under_version_limit(self, mock_db, mock_tasks): policy = _policy(max_versions=5, max_age_seconds=0) @@ -1017,7 +1017,7 @@ def test_under_version_limit(self, mock_db, mock_tasks): mock_tasks.add_backup_merge_task.assert_not_called() @patch.object(Backup, 'write_to_db') - @patch("simplyblock_core.controllers.backup.policy.tasks_controller") + @patch("simplyblock_core.controllers.backup.controller.tasks_controller") @patch("simplyblock_core.controllers.backup.policy.db_controller") def test_age_limit_exceeded(self, mock_db, mock_tasks, _mock_write): policy = _policy(max_versions=0) @@ -1036,7 +1036,7 @@ def test_age_limit_exceeded(self, mock_db, mock_tasks, _mock_write): mock_tasks.add_backup_merge_task.assert_called_once() @patch.object(Backup, 'write_to_db') - @patch("simplyblock_core.controllers.backup.policy.tasks_controller") + @patch("simplyblock_core.controllers.backup.controller.tasks_controller") @patch("simplyblock_core.controllers.backup.policy.db_controller") def test_both_conditions_required(self, mock_db, mock_tasks, _mock_write): """When both versions and age are set, either limit can trigger a merge.""" @@ -1056,7 +1056,7 @@ def test_both_conditions_required(self, mock_db, mock_tasks, _mock_write): mock_tasks.add_backup_merge_task.assert_called_once() @patch.object(Backup, 'write_to_db') - @patch("simplyblock_core.controllers.backup.policy.tasks_controller") + @patch("simplyblock_core.controllers.backup.controller.tasks_controller") @patch("simplyblock_core.controllers.backup.policy.db_controller") def test_both_conditions_met(self, mock_db, mock_tasks, _mock_write): """When both limits set and both exceeded, merge triggers.""" @@ -1076,7 +1076,7 @@ def test_both_conditions_met(self, mock_db, mock_tasks, _mock_write): mock_tasks.add_backup_merge_task.assert_called_once() - @patch("simplyblock_core.controllers.backup.policy.tasks_controller") + @patch("simplyblock_core.controllers.backup.controller.tasks_controller") @patch("simplyblock_core.controllers.backup.policy.db_controller") def test_fewer_than_two_backups(self, mock_db, mock_tasks): """Never merge with fewer than 2 completed backups.""" @@ -1283,31 +1283,31 @@ def test_location_without_a_region_is_accepted(self): # =========================================================================== -# 20. _trigger_merge +# 20. trigger_merge # =========================================================================== class TestTriggerMerge(unittest.TestCase): - @patch("simplyblock_core.controllers.backup.policy.tasks_controller") + @patch("simplyblock_core.controllers.backup.controller.tasks_controller") def test_trigger_merge_marks_old_as_merging(self, mock_tasks): - from simplyblock_core.controllers.backup.policy import _trigger_merge + from simplyblock_core.controllers.backup.controller import trigger_merge keep = _backup(uuid="keep") old = _backup(uuid="old") old.write_to_db = MagicMock() - _trigger_merge(keep, old) + trigger_merge(keep, old) self.assertEqual(old.status, Backup.STATUS_MERGING) old.write_to_db.assert_called_once() mock_tasks.add_backup_merge_task.assert_called_once() - @patch("simplyblock_core.controllers.backup.policy.tasks_controller") + @patch("simplyblock_core.controllers.backup.controller.tasks_controller") def test_skip_if_not_completed(self, mock_tasks): - from simplyblock_core.controllers.backup.policy import _trigger_merge + from simplyblock_core.controllers.backup.controller import trigger_merge keep = _backup(uuid="keep") old = _backup(uuid="old", status=Backup.STATUS_PENDING) - _trigger_merge(keep, old) + trigger_merge(keep, old) mock_tasks.add_backup_merge_task.assert_not_called() diff --git a/tests/integration/backup/test_manifest_flow.py b/tests/integration/backup/test_manifest_flow.py index e7e5866081..80453307ca 100644 --- a/tests/integration/backup/test_manifest_flow.py +++ b/tests/integration/backup/test_manifest_flow.py @@ -225,7 +225,7 @@ def test_says_nothing_about_who_may_attach(self, db, cluster, lvol): def test_survives_a_deleted_volume(self, db, cluster, lvol): """A backup outlives its volume; that must not stop the manifest.""" - backup = _backup(db, 1) + backup = _backup(db, 1, max_namespace_per_subsys=10) lvol.remove(db.kv_store) manifest = backup_controller.build_manifest(backup) @@ -234,6 +234,16 @@ def test_survives_a_deleted_volume(self, db, cluster, lvol): assert manifest.volume.pool_name is None # only on the volume # Absent, not 0 -- which for a QoS cap would read as "unlimited". assert manifest.volume.rw_ios_per_sec is None + # The subsystem sharing is carried on the backup, so it outlives the + # volume too: it is what a restore reapplies. + assert manifest.volume.max_namespace_per_subsys == 10 + + def test_records_the_subsystem_sharing_off_the_backup(self, db, cluster, lvol): + assert backup_controller.build_manifest( + _backup(db, 1, max_namespace_per_subsys=10)).volume.max_namespace_per_subsys == 10 + # A record from before the field: absent, not 0. + assert backup_controller.build_manifest( + _backup(db, 2, max_namespace_per_subsys=0)).volume.max_namespace_per_subsys is None def test_records_source_as_provenance(self, db, cluster, lvol): manifest = backup_controller.build_manifest(_backup(db, 1)) @@ -301,6 +311,18 @@ def test_backups_survive_losing_the_database(self, db, cluster, lvol): assert restored.get_location().bucket_name == BUCKET assert restored.encrypted is True + def test_import_keeps_the_subsystem_sharing(self, db, cluster, lvol): + """A restore on the importing cluster recreates the source's sharing.""" + _backup(db, 1, max_namespace_per_subsys=10) + exported = _manifests(backup_controller.export_backups(cluster_id=CLUSTER_ID)) + for backup in db.get_backups(): + backup.remove(db.kv_store) + + backup_controller.import_backups( + _export(exported, _config().location()), cluster_id=OTHER_CLUSTER_ID) + + assert db.get_backup_by_id(_backup_id(1)).max_namespace_per_subsys == 10 + def test_encrypted_flag_survives(self, db, cluster, lvol): """It used to be dropped, restoring a plaintext volume over ciphertext.""" _backup(db, 1, encrypted=True, encryption=_encryption(_backup_id(1))) diff --git a/tests/integration/models/indices/test_group_snapshot_index.py b/tests/integration/models/indices/test_group_snapshot_index.py new file mode 100644 index 0000000000..93f0107938 --- /dev/null +++ b/tests/integration/models/indices/test_group_snapshot_index.py @@ -0,0 +1,65 @@ +"""The SnapShot (group_id, group_seq) index: a consistency group's snapshots, and +one generation's, without scanning every snapshot of the cluster. + +The answers must be the same while the index builds (the filtered-scan fallback) +and once it is ready, and a snapshot outside any group, whose group_id is blank, +has no entry at all. +""" +from simplyblock_core.models.lvol_model import LVol +from simplyblock_core.models.snapshot import SnapShot + +from .helpers import CLUSTER, index_keys, ready + +GROUP_A = f"{CLUSTER}/group-a" +GROUP_B = f"{CLUSTER}/group-b" + + +def _snap(db, uuid, group_id="", group_seq=0): + snap = SnapShot() + snap.uuid = uuid + snap.cluster_id = CLUSTER + snap.group_id = group_id + snap.group_seq = group_seq + snap.lvol = LVol() + snap.lvol.uuid = f"lv-{uuid}" + snap.write_to_db(db.kv_store) + return snap + + +def _seed(db): + _snap(db, "a-3-x", GROUP_A, 3) + _snap(db, "a-3-y", GROUP_A, 3) + _snap(db, "a-4-x", GROUP_A, 4) + _snap(db, "b-3-x", GROUP_B, 3) + _snap(db, "loose") + + +def _uuids(snaps): + return sorted(s.uuid for s in snaps) + + +def _answers(db): + return (_uuids(db.get_group_snapshots(GROUP_A)), + _uuids(db.get_group_snapshots(GROUP_A, 3)), + _uuids(db.get_group_snapshots(GROUP_B, 4))) + + +def test_group_snapshots_agree_across_the_flip(db): + _seed(db) + building = _answers(db) + ready(SnapShot) + assert _answers(db) == building == ( + ["a-3-x", "a-3-y", "a-4-x"], ["a-3-x", "a-3-y"], []) + + +def test_a_snapshot_outside_any_group_is_not_indexed(db): + _seed(db) + ready(SnapShot) + keys = [k for k in index_keys(db, SnapShot) if b"/group_id+group_seq/" in k] + assert keys and not any(k.endswith(b"loose") for k in keys) + + +def test_a_blank_group_has_no_snapshots(db): + _seed(db) + ready(SnapShot) + assert db.get_group_snapshots("") == [] diff --git a/tests/integration/test_consistency_group_generation_resolve.py b/tests/integration/test_consistency_group_generation_resolve.py new file mode 100644 index 0000000000..29c57eeb12 --- /dev/null +++ b/tests/integration/test_consistency_group_generation_resolve.py @@ -0,0 +1,158 @@ +"""Resolving a consistency group at a generation to its member snapshots, against +real FoundationDB. + +`generation_snapshots(group, seq)` is the one lookup of a generation's member +snapshots. `resolve_complete_generation(group, seq)` also requires the +generation whole: an ONLINE snapshot of every volume that was a member at that +sequence (the membership epochs, `ConsistencyGroup.included_in_seq`). A group +backup is taken through the strict form, so a generation missing a member is +refused rather than backed up as a group that silently lacks a volume +(design-consistency-group-backup.md §5). +""" +import pytest + +from simplyblock_core.controllers import consistency_group_controller as cgc +from simplyblock_core.controllers.backup import group_backup +from simplyblock_core.db_controller import DBController +from simplyblock_core.models.lvol_model import LVol +from simplyblock_core.models.replication import ConsistencyGroup +from simplyblock_core.models.snapshot import SnapShot + +CLUSTER_ID = "cgr-cluster-1" + + +@pytest.fixture +def db(): + return DBController() + + +def _lvol(db, uuid): + lvol = LVol() + lvol.uuid = uuid + lvol.cluster_id = CLUSTER_ID + lvol.lvol_name = f"VOL_{uuid}" + lvol.status = LVol.STATUS_ONLINE + lvol.write_to_db(db.kv_store) + return lvol + + +def _group(db, members, uuid="cgr-group-1"): + """A group whose members map lvol uuid -> (joined_seq, removed_seq).""" + group = ConsistencyGroup() + group.uuid = uuid + group.cluster_id = CLUSTER_ID + group.group_name = uuid + group.members = {lv: {"joined_seq": j, "removed_seq": r} for lv, (j, r) in members.items()} + group.last_group_seq = 9 + group.write_to_db(db.kv_store) + return group + + +def _snap(db, group, lvol, seq, status=SnapShot.STATUS_ONLINE, uuid=None): + snap = SnapShot() + snap.uuid = uuid or f"cgr-snap-{group.uuid}-{lvol.uuid}-{seq}" + snap.cluster_id = CLUSTER_ID + snap.lvol = lvol + snap.group_id = group.get_id() + snap.group_seq = seq + snap.status = status + snap.write_to_db(db.kv_store) + return snap + + +def _ids(snaps): + return sorted(s.get_id() for s in snaps) + + +def test_generation_snapshots_are_this_groups_at_this_sequence(db): + a, b = _lvol(db, "lv-a"), _lvol(db, "lv-b") + group = _group(db, {"lv-a": (1, 0), "lv-b": (1, 0)}) + other = _group(db, {"lv-a": (1, 0)}, uuid="cgr-group-2") + want = [_snap(db, group, a, 3), _snap(db, group, b, 3)] + _snap(db, group, a, 4) + _snap(db, other, a, 3) + + assert _ids(cgc.generation_snapshots(group, 3)) == _ids(want) + + +def test_a_complete_generation_resolves_to_its_member_snapshots(db): + a, b = _lvol(db, "lv-a"), _lvol(db, "lv-b") + group = _group(db, {"lv-a": (1, 0), "lv-b": (1, 0)}) + want = [_snap(db, group, a, 3), _snap(db, group, b, 3)] + + assert _ids(cgc.resolve_complete_generation(group, 3)) == _ids(want) + + +def test_a_generation_missing_a_member_is_refused_naming_it(db): + a, _ = _lvol(db, "lv-a"), _lvol(db, "lv-b") + group = _group(db, {"lv-a": (1, 0), "lv-b": (1, 0)}) + _snap(db, group, a, 3) + + with pytest.raises(cgc.IncompleteGenerationError, match="lv-b"): + cgc.resolve_complete_generation(group, 3) + + +def test_a_member_snapshot_being_deleted_counts_as_missing(db): + a, b = _lvol(db, "lv-a"), _lvol(db, "lv-b") + group = _group(db, {"lv-a": (1, 0), "lv-b": (1, 0)}) + _snap(db, group, a, 3) + _snap(db, group, b, 3, status=SnapShot.STATUS_IN_DELETION) + + with pytest.raises(cgc.IncompleteGenerationError, match="lv-b"): + cgc.resolve_complete_generation(group, 3) + + +def test_only_the_members_at_that_sequence_are_expected(db): + """A volume that joined later, or left earlier, is not part of the generation.""" + a = _lvol(db, "lv-a") + _lvol(db, "lv-late"), _lvol(db, "lv-gone") + group = _group(db, {"lv-a": (1, 0), "lv-late": (5, 0), "lv-gone": (1, 2)}) + want = [_snap(db, group, a, 3)] + + assert _ids(cgc.resolve_complete_generation(group, 3)) == _ids(want) + + +def test_a_generation_with_no_snapshots_is_refused(db): + _lvol(db, "lv-a") + group = _group(db, {"lv-a": (1, 0)}) + + with pytest.raises(cgc.ConsistencyGroupError, match="no member snapshots"): + cgc.resolve_complete_generation(group, 3) + + +def test_a_group_backup_of_an_incomplete_generation_is_refused(db): + a, _ = _lvol(db, "lv-a"), _lvol(db, "lv-b") + group = _group(db, {"lv-a": (1, 0), "lv-b": (1, 0)}) + _snap(db, group, a, 3) + + group_backup_id, err = group_backup.create_group_backup(group.get_id(), group_seq=3) + + assert group_backup_id is None + assert "lv-b" in err + + +def test_a_replicated_copy_on_the_peer_cluster_is_not_a_member_snapshot(db): + """A copy replicated to the peer carries the same group id, but belongs to + the peer: deleting or counting a generation must not reach it.""" + a = _lvol(db, "lv-a") + group = _group(db, {"lv-a": (1, 0)}) + mine = _snap(db, group, a, 3) + copy = _snap(db, group, a, 3, uuid="cgr-copy-on-peer") + copy.cluster_id = "cgr-cluster-peer" + copy.write_to_db(db.kv_store) + + assert _ids(cgc.generation_snapshots(group, 3)) == _ids([mine]) + + +def test_a_group_backup_backs_up_the_generation_its_take_answered(db, monkeypatch): + """Without a sequence, the backup takes a generation and backs up the one + the take answers with, not one read back off a snapshot it created.""" + a, _ = _lvol(db, "lv-a"), _lvol(db, "lv-b") + group = _group(db, {"lv-a": (1, 0), "lv-b": (1, 0)}) + _snap(db, group, a, 3) + monkeypatch.setattr(cgc, "create_group_snapshot_for_group", lambda g: (3, None)) + + group_backup_id, err = group_backup.create_group_backup(group.get_id()) + + assert group_backup_id is None + assert "lv-b" in err diff --git a/tests/integration/test_consistency_group_generations.py b/tests/integration/test_consistency_group_generations.py new file mode 100644 index 0000000000..44e2f14d6d --- /dev/null +++ b/tests/integration/test_consistency_group_generations.py @@ -0,0 +1,107 @@ +"""Listing, deleting, and cloning one generation of a consistency group, against +real FoundationDB. + +Moved from simplyblock_core/test/test_consistency_groups.py, whose fake database +stood in for the snapshot lookups (tests/AGENTS.md): the generation's snapshots +are now read through the (group_id, group_seq) index, which the fake did not +implement. The groups, volumes, and snapshots are real records; only the +snapshot delete and clone, which reach storage nodes, are patched. +""" +import pytest + +from simplyblock_core.controllers import consistency_group_controller as cgc +from simplyblock_core.controllers import snapshot_controller +from simplyblock_core.db_controller import DBController +from simplyblock_core.models.lvol_model import LVol +from simplyblock_core.models.snapshot import SnapShot + +CLUSTER = "cgg-cluster" + + +@pytest.fixture +def db(): + return DBController() + + +def _lvol(db, uuid, node="NODE_A", lvs="LVS_1"): + lvol = LVol() + lvol.uuid = uuid + lvol.cluster_id = CLUSTER + lvol.node_id = node + lvol.lvs_name = lvs + lvol.lvol_name = f"VOL_{uuid}" + lvol.status = LVol.STATUS_ONLINE + lvol.write_to_db(db.kv_store) + return lvol + + +def _snap(db, uuid, group, seq, lvol): + snap = SnapShot() + snap.uuid = uuid + snap.cluster_id = CLUSTER + snap.lvol = lvol + snap.group_id = group.get_id() + snap.group_seq = seq + snap.status = SnapShot.STATUS_ONLINE + snap.write_to_db(db.kv_store) + return snap + + +def test_list_generations_reports_incomplete(db): + g = cgc.ensure_group(CLUSTER, "db-group") + g.members = {"a": {"joined_seq": 1, "removed_seq": 0}, + "b": {"joined_seq": 1, "removed_seq": 0}} + g.write_to_db(db.kv_store) + la, lb = _lvol(db, "a"), _lvol(db, "b") + # generation 1 complete (both members), generation 2 missing member b + _snap(db, "s1a", g, 1, la) + _snap(db, "s1b", g, 1, lb) + _snap(db, "s2a", g, 2, la) + + rows = {r["group_seq"]: r for r in cgc.list_generations(g)} + assert rows[1]["expected"] == 2 and rows[1]["present"] == 2 and rows[1]["complete"] + assert rows[2]["expected"] == 2 and rows[2]["present"] == 1 and not rows[2]["complete"] + + +def test_delete_generation_removes_every_member_snapshot(db, monkeypatch): + g = cgc.ensure_group(CLUSTER, "db-group") + la = _lvol(db, "a") + s2a, s2b = _snap(db, "s2a", g, 2, la), _snap(db, "s2b", g, 2, la) + _snap(db, "s3a", g, 3, la) + deleted_calls = [] + + def _fake_delete(sid, *a, **k): + deleted_calls.append(sid) + return True + + monkeypatch.setattr(snapshot_controller, "delete", _fake_delete) + deleted, err = cgc.delete_generation(g, 2) + assert err is None + assert set(deleted_calls) == {s2a.get_id(), s2b.get_id()} # generation 3 untouched + + +def test_delete_generation_unknown_seq_errors(db): + g = cgc.ensure_group(CLUSTER, "db-group") + deleted, err = cgc.delete_generation(g, 9) + assert deleted is None and "not found" in err + + +def test_clone_generation_forms_new_group(db, monkeypatch): + g = cgc.ensure_group(CLUSTER, "db-group") + g.node_id, g.lvs_name = "NODE_A", "LVS_1" + g.write_to_db(db.kv_store) + _snap(db, "s1a", g, 1, _lvol(db, "a")) + _snap(db, "s1b", g, 1, _lvol(db, "b")) + # each clone lands on the pinned node so the new group colocates + _lvol(db, "clone-a") + _lvol(db, "clone-b") + seq = iter(["clone-a", "clone-b"]) + monkeypatch.setattr(snapshot_controller, "clone", + lambda sid, name, *a, **k: (next(seq), False)) + + created, err = cgc.clone_generation(g, 1, into_name="restored") + assert err is None + assert set(created) == {"clone-a", "clone-b"} + new_group = db.get_consistency_group_by_name(CLUSTER, "restored") + assert new_group is not None + assert set(new_group.members) == {"clone-a", "clone-b"} diff --git a/tests/integration/test_group_replication_attach.py b/tests/integration/test_group_replication_attach.py index c65b06957d..a4fc0b2d1e 100644 --- a/tests/integration/test_group_replication_attach.py +++ b/tests/integration/test_group_replication_attach.py @@ -141,14 +141,15 @@ def test_demote_group_ship_home_takes_ONE_group_snapshot(db, monkeypatch): def _grp_snap(g, **k): grp_calls.append(g.get_id()) - ids = [] for mid in ("v1", "v2"): s = SnapShot() s.uuid = f"snap-{mid}" + s.cluster_id = g.cluster_id s.lvol = db.get_lvol_by_id(mid) + s.group_id = g.get_id() + s.group_seq = 1 s.write_to_db(db.kv_store) - ids.append(s.uuid) - return ids, None + return 1, None monkeypatch.setattr(cgc, "create_group_snapshot_for_group", _grp_snap) result = cgc.demote_group(group) @@ -169,7 +170,7 @@ def test_demote_group_superseded_source_does_not_take_a_group_snapshot(db, monke monkeypatch.setattr(cgc, "_members_are_superseded_source", lambda members: True) grp_calls = [] monkeypatch.setattr(cgc, "create_group_snapshot_for_group", - lambda g, **k: grp_calls.append(g) or ([], None)) + lambda g, **k: grp_calls.append(g) or (1, None)) monkeypatch.setattr(lc, "demote_lvol", lambda lvol_id: {"demoted": True}) result = cgc.demote_group(group) diff --git a/tests/unit/backup/test_backup_import_structure.py b/tests/unit/backup/test_backup_import_structure.py new file mode 100644 index 0000000000..21da1af278 --- /dev/null +++ b/tests/unit/backup/test_backup_import_structure.py @@ -0,0 +1,60 @@ +"""The backup controllers import at module level, and each imports cleanly as +the first module loaded. + +group_backup and policy once depended on each other: policy calls the group +backup lifecycle, and group_backup reached back into policy for the merge +trigger. That cycle was survivable only through function-local imports, which +hide a module's real dependencies and break as soon as somebody hoists one. +The merge trigger now lives in the single-volume backup controller both +depend on, so neither needs a function-local import, and these tests keep it +that way. +""" +import ast +import pathlib +import subprocess +import sys + +import pytest + +from tests.unit.test_controller_import_order import _FDB_STUB + +_ROOT = pathlib.Path(__file__).resolve().parents[3] + +#: New backup code that must declare every dependency at module level. +MODULE_LEVEL_ONLY = [ + "simplyblock_core/controllers/backup/group_backup.py", + "simplyblock_core/controllers/backup/policy.py", +] + +BACKUP_MODULES = [ + "simplyblock_core.controllers.backup.group_backup", + "simplyblock_core.controllers.backup.policy", + "simplyblock_core.controllers.backup.controller", + "simplyblock_core.services.tasks_runner_backup", +] + + +def _function_local_imports(path): + tree = ast.parse(path.read_text()) + found = [] + for fn in ast.walk(tree): + if isinstance(fn, (ast.FunctionDef, ast.AsyncFunctionDef)): + for node in ast.walk(fn): + if isinstance(node, (ast.Import, ast.ImportFrom)): + found.append(f"{path.name}:{node.lineno} {ast.unparse(node)}") + return found + + +@pytest.mark.parametrize("relpath", MODULE_LEVEL_ONLY) +def test_imports_are_at_module_level(relpath): + found = _function_local_imports(_ROOT / relpath) + assert not found, "function-local imports:\n" + "\n".join(found) + + +@pytest.mark.parametrize("module", BACKUP_MODULES) +def test_backup_module_imports_first(module): + proc = subprocess.run( + [sys.executable, "-c", _FDB_STUB + f"\nimport {module}"], + capture_output=True, text=True, timeout=120) + assert proc.returncode == 0, ( + f"{module} does not import as the first module loaded:\n{proc.stderr}") diff --git a/tests/unit/backup/test_cli.py b/tests/unit/backup/test_cli.py index c7cbbece1b..c121d84187 100644 --- a/tests/unit/backup/test_cli.py +++ b/tests/unit/backup/test_cli.py @@ -13,10 +13,14 @@ from simplyblock_cli import clibase from simplyblock_core.controllers.backup import controller as backup_controller +from simplyblock_core.controllers.backup import group_backup as group_backup_controller from simplyblock_core.controllers.backup.manifest import ( BackupExport, BackupManifest, FDBKeyDescriptor, + GroupBackupExport, + GroupBackupManifest, + GroupManifestSource, LocatedManifests, ManifestDataPlane, ManifestSource, @@ -300,4 +304,124 @@ def test_a_file_from_a_later_build_is_refused(self, tmp_path, capsys): assert not ok assert "schema version 99" in capsys.readouterr().out - imported.assert_not_called() + + +def _group_manifest(): + return GroupBackupManifest( + group_backup_id=_id("gb-1"), + created_at=100, + completed_at=200, + size=4096, + source=GroupManifestSource( + cluster_id=_id("cluster"), + consistency_group_id=f"{_id('cluster')}/{_id('cg')}", + consistency_group_name="gitea", generation=7), + member_backup_ids=[_id("b-1")], + ) + + +def _group_export(): + return GroupBackupExport( + group_manifest=_group_manifest(), + members=BackupExport(groups=[LocatedManifests( + location=BackupLocation(bucket_name="backups", region="eu-central-1"), + manifests=[_manifest()])])) + + +class TestGroupExportImportFile: + """What `group-backup export` writes and `group-backup import --from-file` + reads back -- the single-volume contract (TestExportImportFile) lifted to a + group backup carried as one unit (design §6.5).""" + + def _export(self, tmp_path, **args): + out = tmp_path / "group.json" + with patch.object(group_backup_controller, "export_group_backup", + return_value=_group_export()) as export: + written = clibase.CLIWrapperBase.group_backup__export( + clibase.CLIWrapperBase.__new__(clibase.CLIWrapperBase), + "export", _Args(group_backup_id="gb-1", cluster_id=None, + output=str(out), **args)) + return written, out, export + + def _import(self, path, **args): + with patch.object(group_backup_controller, "import_group_backup", + return_value="cl/gb-1") as imported: + ok = clibase.CLIWrapperBase.group_backup__import( + clibase.CLIWrapperBase.__new__(clibase.CLIWrapperBase), + "import", _Args(from_file=str(path), bucket=None, + cluster_id=None, **args)) + return ok, imported + + def test_the_file_carries_the_group_manifest_and_its_members(self, tmp_path): + written, out, _ = self._export(tmp_path) + + assert written + document = json.loads(out.read_text()) + assert document["group_manifest"]["source"]["consistency_group_name"] == "gitea" + assert document["members"]["groups"][0]["location"]["bucket_name"] == "backups" + + def test_a_file_imports_without_naming_a_bucket(self, tmp_path): + _, out, _ = self._export(tmp_path) + + ok, imported = self._import(out) + + assert ok + (export,), _ = imported.call_args + assert str(export.group_manifest.group_backup_id) == str(_id("gb-1")) + + def test_naming_a_bucket_alongside_a_file_is_refused(self, tmp_path, capsys): + _, out, _ = self._export(tmp_path) + + ok = clibase.CLIWrapperBase.group_backup__import( + clibase.CLIWrapperBase.__new__(clibase.CLIWrapperBase), + "import", _Args(from_file=str(out), bucket="elsewhere", cluster_id=None)) + + assert not ok + assert "exactly one" in capsys.readouterr().out + + def test_no_credential_reaches_the_file(self, tmp_path): + _, out, _ = self._export(tmp_path) + + body = out.read_text() + assert "access_key" not in body and "secret" not in body + + def test_a_file_from_a_later_build_is_refused(self, tmp_path, capsys): + out = tmp_path / "future.json" + out.write_text(json.dumps({"schema_version": 99, "group_manifest": {}, + "members": {"schema_version": 1, "groups": []}})) + + ok, imported = self._import(out) + + assert not ok + assert "group export schema version 99" in capsys.readouterr().out + + +class TestGroupDiscoverRendering: + """`group-backup discover` reports the group backups a bucket holds, including + a torn generation (design §6.5).""" + + def _run(self, manifests, member_manifests=()): + with patch.object(group_backup_controller, "discover_group_backups", + return_value=manifests), \ + patch.object(clibase, "list_all", return_value=list(member_manifests)): + return clibase.CLIWrapperBase.group_backup__discover( + clibase.CLIWrapperBase.__new__(clibase.CLIWrapperBase), + "discover", _Args(bucket="backups", region="eu-central-1", + endpoint=None, access_key_id=None, + secret_access_key=None, no_verify_tls=False, + path_style=False)) + + def test_lists_a_complete_group_backup(self): + rows = self._run([_group_manifest()], [_manifest("b-1")]) + assert rows[0]["ID"] == str(_id("gb-1")) + assert rows[0]["Members"] == 1 + assert rows[0]["Complete"] == "yes" + + def test_shows_a_torn_generation_as_incomplete(self): + """The bucket no longer holds a member's own manifest.""" + rows = self._run([_group_manifest()], []) + assert rows[0]["Complete"] == "no" + + def test_an_empty_bucket_is_reported(self, capsys): + assert self._run([]) is False + assert "No group backups found" in capsys.readouterr().out diff --git a/tests/unit/backup/test_group_backup_logic.py b/tests/unit/backup/test_group_backup_logic.py new file mode 100644 index 0000000000..2087de5392 --- /dev/null +++ b/tests/unit/backup/test_group_backup_logic.py @@ -0,0 +1,141 @@ +"""The decision core of consistency-group backup: the homogeneity gate and the +all-or-nothing commit decision (design-consistency-group-backup.md §3.3, §3.4). + +Pure logic -- the orchestration around it (resolving a generation, creating the +record, enqueuing member backups, publishing the group manifest) touches the +database and the data plane and is the integration tier's. +""" +import zlib +from datetime import UTC, datetime +from uuid import UUID + +import pytest + +from simplyblock_core.controllers.backup import group_backup +from simplyblock_core.models.backup import Backup, GroupBackup + + +def _uuid(name: str) -> str: + return f"{zlib.crc32(name.encode()):08x}-0000-4000-8000-000000000000" + +LOC_A = {"bucket_name": "backups", "region": "us-east-1"} +LOC_B = {"bucket_name": "other", "region": "us-east-1"} + + +def _m(lvol_id, location=None, encrypted=False): + return {"lvol_id": lvol_id, "location": location or LOC_A, "encrypted": encrypted} + + +class TestHomogeneity: + def test_matching_members_are_homogeneous(self): + members = [_m("db"), _m("data"), _m("web")] + assert group_backup.check_members_homogeneous(members) == [] + + def test_a_different_bucket_is_an_offender(self): + members = [_m("db"), _m("data", location=LOC_B)] + problems = group_backup.check_members_homogeneous(members) + assert [p[0] for p in problems] == ["data"] + assert "location" in problems[0][1] + + def test_a_different_encryption_disposition_is_an_offender(self): + members = [_m("db"), _m("data", encrypted=True)] + problems = group_backup.check_members_homogeneous(members) + assert [p[0] for p in problems] == ["data"] + assert "encryption" in problems[0][1] + + def test_a_single_member_is_trivially_homogeneous(self): + assert group_backup.check_members_homogeneous([_m("db")]) == [] + + def test_no_members_is_homogeneous(self): + assert group_backup.check_members_homogeneous([]) == [] + + +class TestCommitDecision: + def test_all_completed_is_ready_to_commit(self): + statuses = [Backup.STATUS_COMPLETED, Backup.STATUS_COMPLETED] + assert group_backup.group_commit_state(statuses) == group_backup.CommitState.READY + + def test_any_failed_fails_the_group(self): + statuses = [Backup.STATUS_COMPLETED, Backup.STATUS_FAILED] + assert group_backup.group_commit_state(statuses) == group_backup.CommitState.FAILED + + def test_an_in_flight_member_holds_the_group(self): + statuses = [Backup.STATUS_COMPLETED, Backup.STATUS_IN_PROGRESS] + assert group_backup.group_commit_state(statuses) == group_backup.CommitState.PENDING + + def test_a_pending_member_holds_the_group(self): + statuses = [Backup.STATUS_PENDING, Backup.STATUS_PENDING] + assert group_backup.group_commit_state(statuses) == group_backup.CommitState.PENDING + + def test_failure_wins_over_still_in_flight(self): + # All-or-nothing: one terminal failure fails the group even while others + # are still uploading (design §3.3). + statuses = [Backup.STATUS_FAILED, Backup.STATUS_IN_PROGRESS] + assert group_backup.group_commit_state(statuses) == group_backup.CommitState.FAILED + + def test_no_members_is_not_ready(self): + assert group_backup.group_commit_state([]) == group_backup.CommitState.PENDING + + def test_the_decision_is_a_commit_state_not_a_bare_string(self): + """A closed set: a caller comparing against a string literal is a bug + the type makes visible rather than one that silently never matches.""" + state = group_backup.group_commit_state([Backup.STATUS_COMPLETED]) + assert isinstance(state, group_backup.CommitState) + assert state != "ready" + assert set(group_backup.CommitState) == { + group_backup.CommitState.PENDING, group_backup.CommitState.READY, group_backup.CommitState.FAILED} + + +class TestBuildGroupManifest: + """build_group_manifest assembles the group manifest from the GroupBackup + record and its member Backup records (design §4.2).""" + + def _group_and_members(self, encrypted=False): + cluster = _uuid("cluster") + cg = f"{cluster}/{_uuid('cg')}" + gb = GroupBackup() + gb.uuid = _uuid("gb") + gb.cluster_id = cluster + gb.consistency_group_id = cg + gb.consistency_group_name = "gitea" + gb.group_seq = 7 + gb.created_at = 100 + gb.completed_at = 200 + + members, by_id = [], {} + for name, size in (("db", 100), ("data", 50)): + b = Backup() + b.uuid = _uuid("b-" + name) + b.cluster_id = cluster + b.lvol_id = _uuid("lv-" + name) + b.lvol_name = name + b.s3_id = size + b.size = size + b.encrypted = encrypted + members.append({"lvol_id": b.lvol_id, "backup_id": b.get_id()}) + by_id[b.get_id()] = b + gb.members = members + return gb, by_id + + def test_builds_from_records(self): + gb, by_id = self._group_and_members() + manifest = group_backup.build_group_manifest(gb, by_id) + + assert manifest.group_backup_id == UUID(gb.uuid) + assert manifest.source.consistency_group_id == gb.consistency_group_id + assert manifest.source.generation == 7 + assert manifest.size == 150 + assert manifest.member_backup_ids == [UUID(by_id[e["backup_id"]].uuid) for e in gb.members] + assert manifest.created_at == datetime.fromtimestamp(gb.created_at, tz=UTC) + assert manifest.completed_at == datetime.fromtimestamp(gb.completed_at, tz=UTC) + assert manifest.encrypted is False + + def test_carries_the_encrypted_disposition(self): + gb, by_id = self._group_and_members(encrypted=True) + assert group_backup.build_group_manifest(gb, by_id).encrypted is True + + def test_a_missing_member_backup_is_an_error(self): + gb, by_id = self._group_and_members() + by_id.pop(gb.members[0]["backup_id"]) # drop one member's record + with pytest.raises(ValueError, match="missing"): + group_backup.build_group_manifest(gb, by_id) diff --git a/tests/unit/backup/test_group_manifest.py b/tests/unit/backup/test_group_manifest.py new file mode 100644 index 0000000000..48f7ad71af --- /dev/null +++ b/tests/unit/backup/test_group_manifest.py @@ -0,0 +1,238 @@ +"""Group backup manifest schema: serialization, version handling, completeness. + +Pure logic -- the S3 plumbing (write_group/read_group/list_all_groups) is the +integration tier's, like the per-backup manifest (test_manifest.py). +""" +import json +import zlib +from datetime import UTC, datetime +from uuid import UUID + +import pytest + +from simplyblock_core.controllers.backup import manifest as backup_manifest +from simplyblock_core.controllers.backup.manifest import ( + GROUP_EXPORT_SCHEMA_VERSION, + GROUP_MANIFEST_SCHEMA_VERSION, + BackupExport, + BackupManifest, + GroupBackupExport, + GroupBackupManifest, + GroupManifestSource, + LocatedManifests, + ManifestDataPlane, + ManifestError, + ManifestSource, + ManifestVolume, +) +from simplyblock_core.models.backup_config import BackupLocation + + +def _id(name: str) -> UUID: + return UUID(f"{zlib.crc32(name.encode()):08x}-0000-4000-8000-000000000000") + + +CLUSTER_ID = _id("cluster") +CREATED = datetime.fromtimestamp(1700000000, tz=UTC) +COMPLETED = datetime.fromtimestamp(1700000100, tz=UTC) + + +def _group_manifest(**overrides): + names = overrides.pop("names", ["db", "data"]) + fields = { + "group_backup_id": _id("gb-1"), + "created_at": CREATED, + "completed_at": COMPLETED, + "size": 8192, + "source": GroupManifestSource( + cluster_id=CLUSTER_ID, + consistency_group_id=f"{CLUSTER_ID}/{_id('cg')}", + consistency_group_name="gitea", + generation=7), + "member_backup_ids": [_id("b-" + n) for n in names], + } + fields.update(overrides) + return GroupBackupManifest(**fields) + + +class TestGroupManifestKey: + def test_key_is_under_its_own_prefix(self): + """A group manifest lives under groups/, disjoint from manifests/ and the + data plane's decimal keyspace.""" + key = backup_manifest.group_manifest_key(_id("gb-1")) + assert key.startswith("groups/") + assert not key.split("/")[0].isdigit() + + +class TestSchema: + def test_round_trip(self): + original = _group_manifest() + restored = backup_manifest._parse_group(original.model_dump_json().encode(), "k") + assert restored == original + + def test_round_trip_keeps_the_encrypted_disposition(self): + original = _group_manifest(encrypted=True) + restored = backup_manifest._parse_group(original.model_dump_json().encode(), "k") + assert restored.encrypted is True + assert restored == original + + def test_lists_member_backup_ids(self): + data = json.loads(_group_manifest().model_dump_json()) + assert data["member_backup_ids"] == [str(_id("b-db")), str(_id("b-data"))] + + def test_names_members_by_backup_id_only(self): + """Each member's volume, s3 id, and the rest are in its own manifest, + manifests/.json; the group manifest only links to it.""" + data = json.loads(_group_manifest().model_dump_json()) + assert "members" not in data + assert "membership" not in data + + def test_says_nothing_about_how_to_reach_the_bucket(self): + data = json.dumps(json.loads(_group_manifest().model_dump_json())) + assert "bucket_name" not in data + assert "endpoint" not in data + assert "region" not in data + + def test_carries_no_credential_field(self): + assert "access_key_id" not in json.dumps( + json.loads(_group_manifest().model_dump_json())) + + def test_an_unencrypted_group_is_marked_plain(self): + assert _group_manifest().encrypted is False + + def test_times_are_utc_datetimes_on_the_wire(self): + data = json.loads(_group_manifest().model_dump_json()) + assert datetime.fromisoformat(data["created_at"]) == CREATED + assert datetime.fromisoformat(data["completed_at"]) == COMPLETED + + def test_a_time_without_a_timezone_is_rejected(self): + with pytest.raises(ValueError): + _group_manifest(created_at=datetime(2026, 10, 21, 12, 0, 0)) + + def test_a_negative_size_is_rejected(self): + with pytest.raises(ValueError): + _group_manifest(size=-1) + + def test_unknown_field_is_rejected(self): + with pytest.raises(ValueError): + GroupBackupManifest.model_validate({ + **json.loads(_group_manifest().model_dump_json()), "extra": 1}) + + +class TestCompleteness: + """Completeness is a property of the bucket, not of the group manifest alone: + the group manifest is written only once every member completed, so what can + go missing afterwards is a member's own manifest.""" + + def test_complete_when_every_member_manifest_is_present(self): + members = [_backup_manifest("db"), _backup_manifest("data")] + assert _group_manifest().is_complete(members) is True + + def test_incomplete_when_a_member_manifest_is_missing(self): + assert _group_manifest().is_complete([_backup_manifest("db")]) is False + + def test_other_manifests_in_the_bucket_do_not_count(self): + members = [_backup_manifest("db"), _backup_manifest("web")] + assert _group_manifest().is_complete(members) is False + + +class TestParse: + def test_rejects_a_newer_schema_version(self): + data = json.loads(_group_manifest().model_dump_json()) + data["schema_version"] = GROUP_MANIFEST_SCHEMA_VERSION + 1 + with pytest.raises(ManifestError, match="schema version"): + backup_manifest._parse_group(json.dumps(data).encode(), "k") + + def test_rejects_invalid_json(self): + with pytest.raises(ManifestError, match="not valid JSON"): + backup_manifest._parse_group(b"{not json", "k") + + def test_rejects_a_malformed_manifest(self): + with pytest.raises(ManifestError, match="malformed"): + backup_manifest._parse_group( + json.dumps({"schema_version": GROUP_MANIFEST_SCHEMA_VERSION}).encode(), "k") + + def test_names_the_key_it_could_not_read(self): + with pytest.raises(ManifestError, match="groups/gb-9.json"): + backup_manifest._parse_group(b"{not json", "groups/gb-9.json") + + +def _backup_manifest(name): + """A single-volume member manifest for the member named *name*.""" + return BackupManifest( + backup_id=_id("b-" + name), + s3_id=zlib.crc32(name.encode()), + created_at=100, + completed_at=200, + size=4096, + source=ManifestSource(cluster_id=CLUSTER_ID, node_id=_id("node")), + volume=ManifestVolume( + lvol_id=_id(name), lvol_name=name, + snapshot_id=_id("snap-" + name), snapshot_name="snap", size=4096), + dataplane=ManifestDataPlane(), + ) + + +LOCATION = BackupLocation(bucket_name="backups", region="eu-central-1") + + +def _group_export(**overrides): + """A group export: the group manifest plus its member backup manifests.""" + members = overrides.pop("members", [_backup_manifest("db"), _backup_manifest("data")]) + fields = { + "group_manifest": _group_manifest(), + "members": BackupExport(groups=[ + LocatedManifests(location=LOCATION, manifests=members)]), + } + fields.update(overrides) + return GroupBackupExport(**fields) + + +class TestGroupExportSchema: + """GroupBackupExport is the unit of transfer: the group manifest together with + the member backup manifests it names (design §6.5).""" + + def test_round_trip(self): + original = _group_export() + restored = backup_manifest.parse_group_export( + original.model_dump_json().encode(), "k") + assert restored == original + + def test_carries_the_group_manifest_and_the_member_manifests(self): + export = _group_export() + assert export.group_manifest.source.consistency_group_name == "gitea" + assert [m.volume.lvol_name for m in export.members.groups[0].manifests] == [ + "db", "data"] + + def test_names_where_the_members_live(self): + """Unlike a manifest, an export records a location -- it is not stored in + the bucket it describes, so the reader has nothing else to go on.""" + export = _group_export() + assert export.members.groups[0].location.bucket_name == "backups" + + def test_unknown_field_is_rejected(self): + with pytest.raises(ValueError): + GroupBackupExport.model_validate({ + **json.loads(_group_export().model_dump_json()), "extra": 1}) + + +class TestParseGroupExport: + def test_rejects_a_newer_schema_version(self): + data = json.loads(_group_export().model_dump_json()) + data["schema_version"] = GROUP_EXPORT_SCHEMA_VERSION + 1 + with pytest.raises(ManifestError, match="group export schema version"): + backup_manifest.parse_group_export(json.dumps(data).encode(), "file.json") + + def test_rejects_invalid_json(self): + with pytest.raises(ManifestError, match="not valid JSON"): + backup_manifest.parse_group_export(b"{not json", "file.json") + + def test_rejects_a_malformed_export(self): + with pytest.raises(ManifestError, match="malformed"): + backup_manifest.parse_group_export( + json.dumps({"schema_version": GROUP_EXPORT_SCHEMA_VERSION}).encode(), + "file.json") + + def test_names_the_source_it_could_not_read(self): + with pytest.raises(ManifestError, match="file.json"): + backup_manifest.parse_group_export(b"{not json", "file.json") diff --git a/tests/unit/backup/test_group_manifest_delete.py b/tests/unit/backup/test_group_manifest_delete.py new file mode 100644 index 0000000000..ac8080b8a5 --- /dev/null +++ b/tests/unit/backup/test_group_manifest_delete.py @@ -0,0 +1,123 @@ +"""Removing a group manifest from its bucket (design §9). + +Removing the group manifest un-commits its group backup: the bucket stops naming +its members as one restorable generation. The manifest module therefore offers +no unguarded delete. ``delete_retired_group`` removes only a manifest that a +group backup delete has marked ``deleting``, and refuses a committed or broken +one, which leaves it in place. + +The S3 client is the one boundary faked, as an in-memory bucket: the manifest +functions under test run for real over it, and no database is involved. +""" +import io +import zlib +from datetime import UTC, datetime +from uuid import UUID + +import pytest +from botocore.exceptions import ClientError + +from simplyblock_core.controllers.backup import manifest as backup_manifest +from simplyblock_core.controllers.backup.manifest import ( + GroupBackupManifest, + GroupManifestSource, + GroupManifestState, + ManifestError, +) +from simplyblock_core.exceptions import PreconditionError +from simplyblock_core.models.backup_config import BackupConfig + + +def _id(name: str) -> UUID: + return UUID(f"{zlib.crc32(name.encode()):08x}-0000-4000-8000-000000000000") + + +GROUP_BACKUP = _id("gb-1") +CONFIG = BackupConfig.model_validate({"bucket_name": "backups", "region": "eu-central-1"}) + + +class _Bucket: + """The three object calls the group manifest functions make.""" + + def __init__(self, fail=False): + self.objects: dict = {} + self.fail = fail + + def _check(self, op): + if self.fail: + raise ClientError({"Error": {"Code": "InternalError"}}, op) + + def put_object(self, Bucket, Key, Body, **_): + self._check("PutObject") + self.objects[Key] = Body + + def get_object(self, Bucket, Key): + self._check("GetObject") + if Key not in self.objects: + raise ClientError({"Error": {"Code": "NoSuchKey"}}, "GetObject") + return {"Body": io.BytesIO(self.objects[Key])} + + def delete_object(self, Bucket, Key): + self._check("DeleteObject") + self.objects.pop(Key, None) + + +@pytest.fixture +def bucket(monkeypatch): + b = _Bucket() + monkeypatch.setattr(backup_manifest, "s3_client", lambda config: b) + return b + + +def _store(state): + backup_manifest.write_group(CONFIG, GroupBackupManifest( + group_backup_id=GROUP_BACKUP, + created_at=datetime.fromtimestamp(100, tz=UTC), + completed_at=datetime.fromtimestamp(200, tz=UTC), + size=4096, + source=GroupManifestSource( + cluster_id=_id("cluster"), consistency_group_id=f"{_id('cluster')}/{_id('cg')}", + consistency_group_name="gitea", generation=7), + member_backup_ids=[_id("b-db")], + state=state, + )) + + +KEY = backup_manifest.group_manifest_key(GROUP_BACKUP) + + +def test_the_module_offers_no_unguarded_group_manifest_delete(): + assert not hasattr(backup_manifest, "delete_group") + + +def test_a_manifest_marked_deleting_is_removed(bucket): + _store(GroupManifestState.DELETING) + + backup_manifest.delete_retired_group(CONFIG, GROUP_BACKUP) + + assert KEY not in bucket.objects + + +@pytest.mark.parametrize("state", [GroupManifestState.COMMITTED, GroupManifestState.BROKEN]) +def test_a_manifest_not_marked_deleting_is_refused_and_kept(bucket, state): + _store(state) + + with pytest.raises(PreconditionError, match=str(state)): + backup_manifest.delete_retired_group(CONFIG, GROUP_BACKUP) + + assert KEY in bucket.objects + + +def test_an_absent_manifest_is_already_removed(bucket): + """A retried delete pass that removed the manifest but not yet the record.""" + backup_manifest.delete_retired_group(CONFIG, GROUP_BACKUP) + + assert bucket.objects == {} + + +def test_a_bucket_that_cannot_be_read_is_a_manifest_error(monkeypatch): + b = _Bucket(fail=True) + monkeypatch.setattr(backup_manifest, "s3_client", lambda config: b) + + with pytest.raises(ManifestError): + backup_manifest.delete_retired_group(CONFIG, GROUP_BACKUP) diff --git a/tests/unit/backup/test_group_retention.py b/tests/unit/backup/test_group_retention.py new file mode 100644 index 0000000000..26449fa4c2 --- /dev/null +++ b/tests/unit/backup/test_group_retention.py @@ -0,0 +1,167 @@ +"""Which group backups a count/age policy prunes (design-consistency-group-backup.md §9). + +Pure logic -- ``evaluate_group_backups`` around it touches the database and the +data plane and is the integration tier's. Retention is per generation: a +group backup is the unit kept or deleted. Its member copies are retired under +it: merged into the next generation's copy while that chains against them, +deleted once nothing does (TestOrphanedMemberBackups). +""" +from types import SimpleNamespace + +from simplyblock_core.controllers.backup import controller as backup_controller +from simplyblock_core.controllers.backup import group_backup, policy +from simplyblock_core.models.backup import Backup, GroupBackup + + +def _gb(uuid, created_at): + gb = GroupBackup() + gb.uuid = uuid + gb.created_at = created_at + return gb + + +def _policy(max_versions=0, max_age_seconds=0): + return SimpleNamespace(max_versions=max_versions, max_age_seconds=max_age_seconds) + + +NOW = 1_000_000 + + +class TestMaxVersions: + def test_keeps_the_newest_and_prunes_the_oldest(self): + completed = [_gb("a", NOW - 400), _gb("b", NOW - 300), + _gb("c", NOW - 200), _gb("d", NOW - 100)] + doomed = policy._group_backups_to_prune(completed, _policy(max_versions=2), [], NOW) + assert [g.uuid for g in doomed] == ["a", "b"] + + def test_within_the_limit_prunes_nothing(self): + completed = [_gb("a", NOW - 200), _gb("b", NOW - 100)] + assert policy._group_backups_to_prune(completed, _policy(max_versions=2), [], NOW) == [] + + def test_zero_versions_is_no_count_limit(self): + completed = [_gb("a", NOW - 200), _gb("b", NOW - 100)] + assert policy._group_backups_to_prune(completed, _policy(max_versions=0), [], NOW) == [] + + +class TestMaxAge: + def test_prunes_those_older_than_the_age(self): + completed = [_gb("old", NOW - 1000), _gb("recent", NOW - 10)] + doomed = policy._group_backups_to_prune(completed, _policy(max_age_seconds=100), [], NOW) + assert [g.uuid for g in doomed] == ["old"] + + +class TestBoth: + def test_count_and_age_union_without_double_listing(self): + # a and b exceed the count (keep 2 of 4); a is also past the age. a must + # appear once, not twice. + completed = [_gb("a", NOW - 1000), _gb("b", NOW - 300), + _gb("c", NOW - 200), _gb("d", NOW - 100)] + doomed = policy._group_backups_to_prune( + completed, _policy(max_versions=2, max_age_seconds=500), [], NOW) + assert [g.uuid for g in doomed] == ["a", "b"] + + +class TestTiers: + """A schedule's tiers keep ``keep_count`` per age band. One generation per + pass: the oldest of the first band over its count.""" + + TIERS = ((900, 2), (3600, 11)) # "15m,2 60m,11" + + def test_the_oldest_of_a_band_over_its_count(self): + completed = [_gb("a", NOW - 400), _gb("b", NOW - 300), _gb("c", NOW - 200)] + doomed = policy._group_backups_to_prune(completed, _policy(), self.TIERS, NOW) + assert [g.uuid for g in doomed] == ["a"] + + def test_a_band_within_its_count_selects_nothing(self): + completed = [_gb("a", NOW - 300), _gb("b", NOW - 200)] + assert policy._group_backups_to_prune(completed, _policy(), self.TIERS, NOW) == [] + + def test_the_tiers_and_the_count_are_one_union(self): + # The band keeps 2 of 3 (selects a); the count keeps 1 of 3 (a and b). + completed = [_gb("a", NOW - 400), _gb("b", NOW - 300), _gb("c", NOW - 200)] + doomed = policy._group_backups_to_prune( + completed, _policy(max_versions=1), self.TIERS, NOW) + assert [g.uuid for g in doomed] == ["a", "b"] + + +def _backup(uuid, lvol, prev="", group="gone", status=Backup.STATUS_COMPLETED, created_at=0): + b = Backup() + b.uuid, b.lvol_id, b.prev_backup_id = uuid, lvol, prev + b.group_backup_id, b.status, b.created_at = group, status, created_at + return b + + +class _DB: + """Backups by volume, and the group backups that still exist.""" + + def __init__(self, backups, group_backups=()): + self._backups = list(backups) + self._groups = set(group_backups) + + def get_backups_by_lvol_id(self, lvol_id): + return [b for b in self._backups if b.lvol_id == lvol_id] + + def get_group_backup_by_id(self, group_backup_id): + if group_backup_id not in self._groups: + raise KeyError(group_backup_id) + return SimpleNamespace(uuid=group_backup_id) + + +class TestOrphanedMemberBackups: + """A pruned generation's member copies must not outlive it in the bucket: + with a 5m,2 schedule the oldest generation's three full copies stayed in + `backup list` for good while the chain grew under them (live 2026-10-20).""" + + def _retire(self, monkeypatch, db, lvols=("v1",)): + merged, deleted = [], [] + monkeypatch.setattr(backup_controller, "trigger_merge", lambda keep, old: merged.append((old.uuid, keep.uuid))) + monkeypatch.setattr(backup_controller, "delete_backup", deleted.append) + result = group_backup.retire_orphaned_member_backups(list(lvols), db=db) + return result, merged, deleted + + def test_a_base_the_next_generation_chains_against_is_merged_into_it(self, monkeypatch): + db = _DB([_backup("g9", "v1", created_at=1), + _backup("g10", "v1", prev="g9", group="kept", created_at=2)], group_backups={"kept"}) + (merges, dels), merged, deleted = self._retire(monkeypatch, db) + assert merged == [("g9", "g10")] and deleted == [] + assert (merges, dels) == (1, 0) + + def test_a_leaf_nothing_chains_against_is_deleted(self, monkeypatch): + db = _DB([_backup("g9", "v1", created_at=1)]) + (merges, dels), merged, deleted = self._retire(monkeypatch, db) + assert deleted == ["g9"] and merged == [] + assert (merges, dels) == (0, 1) + + def test_a_member_of_a_living_group_backup_is_left_alone(self, monkeypatch): + db = _DB([_backup("g10", "v1", group="kept", created_at=2)], group_backups={"kept"}) + _, merged, deleted = self._retire(monkeypatch, db) + assert merged == [] and deleted == [] + + def test_one_merge_per_chain_oldest_first(self, monkeypatch): + """Two pruned generations at once: the base merges first; the next cycle + takes the rest, as the single-volume schedule paces its merges.""" + db = _DB([_backup("g8", "v1", created_at=1), + _backup("g9", "v1", prev="g8", created_at=2), + _backup("g10", "v1", prev="g9", group="kept", created_at=3)], group_backups={"kept"}) + _, merged, deleted = self._retire(monkeypatch, db) + assert merged == [("g8", "g9")] and deleted == [] + + def test_a_chain_with_a_merge_in_flight_is_skipped(self, monkeypatch): + db = _DB([_backup("g8", "v1", status=Backup.STATUS_MERGING, created_at=1), + _backup("g9", "v1", prev="g8", created_at=2), + _backup("g10", "v1", prev="g9", group="kept", created_at=3)], group_backups={"kept"}) + _, merged, deleted = self._retire(monkeypatch, db) + assert merged == [] and deleted == [] + + def test_a_successor_still_uploading_defers_the_merge(self, monkeypatch): + db = _DB([_backup("g9", "v1", created_at=1), + _backup("g10", "v1", prev="g9", group="kept", + status=Backup.STATUS_IN_PROGRESS, created_at=2)], group_backups={"kept"}) + _, merged, deleted = self._retire(monkeypatch, db) + assert merged == [] and deleted == [] + + def test_each_member_volume_is_retired_independently(self, monkeypatch): + db = _DB([_backup("a9", "v1", created_at=1), _backup("a10", "v1", prev="a9", group="kept", created_at=2), + _backup("b9", "v2", created_at=1)], group_backups={"kept"}) + _, merged, deleted = self._retire(monkeypatch, db, lvols=("v1", "v2")) + assert merged == [("a9", "a10")] and deleted == ["b9"] diff --git a/tests/unit/backup/test_manifest.py b/tests/unit/backup/test_manifest.py index 9a48c15d7d..50bfb39557 100644 --- a/tests/unit/backup/test_manifest.py +++ b/tests/unit/backup/test_manifest.py @@ -137,6 +137,16 @@ def test_volume_settings_are_absent_rather_than_zero(self): assert volume.rw_ios_per_sec is None assert volume.pool_name is None + def test_subsystem_sharing_round_trips(self): + """The namespace cap is what a restore reapplies, so it has to survive + the bucket; absent when the backup predates the field.""" + assert _manifest().volume.max_namespace_per_subsys is None + shared = _manifest(volume=ManifestVolume( + lvol_id=VOLUME_ID, lvol_name="vol", snapshot_id=SNAPSHOT_ID, + snapshot_name="snap", size=4096, max_namespace_per_subsys=10)) + parsed = backup_manifest._parse(shared.model_dump_json().encode(), "k") + assert parsed.volume.max_namespace_per_subsys == 10 + class TestParse: def test_rejects_a_newer_schema_version(self): diff --git a/tests/unit/backup/test_restore_node_selection.py b/tests/unit/backup/test_restore_node_selection.py index 342ec59f02..c7d8298186 100644 --- a/tests/unit/backup/test_restore_node_selection.py +++ b/tests/unit/backup/test_restore_node_selection.py @@ -206,3 +206,49 @@ def test_task_creation_failure_is_a_runtime_error(self, db, add_lvol_ha, tasks): with pytest.raises(RuntimeError, match="Failed to create restore task"): _restore() + + +class TestVolumeShape: + """Restore recreates the subsystem sharing the source had. + + Three members of a consistency group shared one subsystem (NSIDs 1..3) and + came back as three subsystems at NSID 1 each: the record carried no + namespacing and restore created every volume as a bare add_lvol does. The + backup records the cap now and restore passes it through. + """ + + def _restore_with_cap(self, db, cap, **kwargs): + backup = _backup(node_id="target-node") + backup.max_namespace_per_subsys = cap + db.get_backup_by_id.return_value = backup + db.get_backups.return_value = [backup] + return _restore(**kwargs) + + def test_a_shared_subsystem_member_comes_back_namespaced(self, db, add_lvol_ha, tasks): + self._restore_with_cap(db, 10) + kwargs = add_lvol_ha.call_args.kwargs + assert kwargs["namespaced"] is True + assert kwargs["max_namespace_per_subsys"] == 10 + + def test_a_volume_owning_its_subsystem_comes_back_owning_one(self, db, add_lvol_ha, tasks): + self._restore_with_cap(db, 1) + kwargs = add_lvol_ha.call_args.kwargs + assert kwargs["namespaced"] is False + assert kwargs["max_namespace_per_subsys"] == 1 + + def test_a_record_without_the_field_takes_the_default(self, db, add_lvol_ha, tasks): + """0 predates the field: neither forced, add_lvol_ha decides as before.""" + self._restore_with_cap(db, 0) + kwargs = add_lvol_ha.call_args.kwargs + assert kwargs["namespaced"] is False + assert kwargs["max_namespace_per_subsys"] is None + + def test_the_group_is_joined_at_creation(self, db, add_lvol_ha, tasks): + """A group restore names the group it forms, so a namespaced member takes + the group's subsystem rather than any of the pool's with a free slot.""" + self._restore_with_cap(db, 10, consistency_group="restored-cg") + assert add_lvol_ha.call_args.kwargs["consistency_group"] == "restored-cg" + + def test_a_single_volume_restore_joins_no_group(self, db, add_lvol_ha, tasks): + self._restore_with_cap(db, 10) + assert add_lvol_ha.call_args.kwargs["consistency_group"] is None diff --git a/tests/unit/models/test_group_backup_model.py b/tests/unit/models/test_group_backup_model.py new file mode 100644 index 0000000000..4c4478d14c --- /dev/null +++ b/tests/unit/models/test_group_backup_model.py @@ -0,0 +1,73 @@ +"""The GroupBackup model and the group fields on Backup. + +A GroupBackup ties the N per-member Backup records of one consistency-group +generation together as one all-or-nothing unit +(design-consistency-group-backup.md §4). This test pins its field shape, its +serialization round-trip, the non-shared mutable default, and the group fields +added to Backup. +""" +from simplyblock_core.models.backup import Backup, GroupBackup + + +def _group_backup(): + gb = GroupBackup() + gb.uuid = "gb-1" + gb.cluster_id = "cl-1" + gb.consistency_group_id = "cl-1/cg-1" + gb.consistency_group_name = "gitea" + gb.group_seq = 7 + gb.members = [ + {"lvol_id": "lv-db", "backup_id": "cl-1/b-db"}, + {"lvol_id": "lv-data", "backup_id": "cl-1/b-data"}, + ] + gb.size = 3 * 1024 ** 3 + return gb + + +def test_group_backup_round_trips(): + gb = _group_backup() + restored = GroupBackup(gb.to_dict()) + assert restored.uuid == "gb-1" + assert restored.get_id() == "cl-1/gb-1" + assert restored.consistency_group_id == "cl-1/cg-1" + assert restored.group_seq == 7 + assert restored.members == gb.members + assert restored.member_backup_ids() == ["cl-1/b-db", "cl-1/b-data"] + + +def test_the_group_record_does_not_repeat_its_members_encryption(): + """Each member Backup records whether it is encrypted; the group record does + not keep a copy that would have to agree with them.""" + assert "encrypted" not in GroupBackup().get_attrs_map() + + +def test_group_backup_members_default_is_not_shared(): + a, b = GroupBackup(), GroupBackup() + assert a.members is not b.members + a.members.append({"lvol_id": "x", "backup_id": "y"}) + assert b.members == [] + + +def test_group_backup_has_its_own_statuses_and_does_not_merge(): + assert GroupBackup.STATUS_PENDING in GroupBackup._STATUS_CODE_MAP + assert GroupBackup.STATUS_COMPLETED in GroupBackup._STATUS_CODE_MAP + # A group backup does not merge: merging is a per-member single-volume + # concern on Backup, not a group one. + assert not hasattr(GroupBackup, "STATUS_MERGING") + + +def test_backup_gains_group_fields_defaulting_to_standalone(): + b = Backup() + assert b.group_backup_id == "" # standalone single-volume backup by default + assert b.group_seq == 0 + b.group_backup_id = "cl-1/gb-1" + b.group_seq = 7 + restored = Backup(b.to_dict()) + assert restored.group_backup_id == "cl-1/gb-1" + assert restored.group_seq == 7 + + +def test_backup_indexes_group_backup_id(): + # The blank group_backup_id of a standalone backup is auto-dropped by the + # index, so only group members are indexed (models/indices.py Index). + assert "group_backup_id" in [ix.name for ix in Backup._INDEXES] diff --git a/tests/unit/test_group_snapshot_request_id.py b/tests/unit/test_group_snapshot_request_id.py new file mode 100644 index 0000000000..36304b13d3 --- /dev/null +++ b/tests/unit/test_group_snapshot_request_id.py @@ -0,0 +1,75 @@ +"""A group generation taken with a request id is taken once. + +A caller that takes a generation and loses the answer (a crash between the +call and recording its sequence) retries with the same request id. The take +must answer with the generation it already took rather than freeze the group +and take another, which would leave one unused generation behind per lost +answer. The id is stamped on every member snapshot of the generation, so the +lookup reads the snapshots the take wrote and needs no other record. A take +answers with its generation's sequence, the one identifier of what it took. +""" +import pytest + +from simplyblock_core.controllers import consistency_group_controller as cgc +from simplyblock_core.db_controller import DBController +from simplyblock_core.models.lvol_model import LVol +from simplyblock_core.models.snapshot import SnapShot +from tests.unit.test_consistency_group_store import _FakeKV + +CLUSTER = "aaaaaaaa-0000-0000-0000-000000000001" + + +@pytest.fixture +def db(monkeypatch): + instance = DBController() + instance.kv_store = _FakeKV() + monkeypatch.setattr(cgc, 'db', instance) + return instance + + +def _member_snapshot(db, group, uuid, lvol_uuid, seq, request_id=""): + lvol = LVol() + lvol.uuid = lvol_uuid + snap = SnapShot() + snap.uuid = uuid + snap.cluster_id = CLUSTER + snap.lvol = lvol + snap.status = SnapShot.STATUS_ONLINE + snap.group_id = group.get_id() + snap.group_seq = seq + snap.group_request_id = request_id + snap.write_to_db(db.kv_store) + return snap + + +@pytest.fixture +def group(db): + group = cgc.ensure_group(CLUSTER, "orders") + _member_snapshot(db, group, "s-3a", "lv-a", 3, request_id="req-1") + _member_snapshot(db, group, "s-3b", "lv-b", 3, request_id="req-1") + _member_snapshot(db, group, "s-4a", "lv-a", 4, request_id="req-2") + _member_snapshot(db, group, "s-5a", "lv-a", 5) + return group + + +def test_the_generation_a_request_id_took_is_found(group): + assert cgc.generation_for_request(group, "req-1") == 3 + assert cgc.generation_for_request(group, "req-2") == 4 + + +def test_an_unknown_or_empty_request_id_finds_nothing(group): + assert cgc.generation_for_request(group, "req-9") is None + # Generation 5 carries no id; an empty id must never match it. + assert cgc.generation_for_request(group, "") is None + + +def test_a_retried_take_answers_with_the_generation_it_took(group, monkeypatch): + """The retry returns before anything is frozen: no precheck, no RPC.""" + def no_take(_group): + raise AssertionError("a retried take must not take another generation") + monkeypatch.setattr(cgc, '_precheck_members', no_take) + + seq, err = cgc.create_group_snapshot_for_group(group, request_id="req-1", lock=False) + + assert err is None + assert seq == 3 diff --git a/tests/unit/test_replication_incident_0929.py b/tests/unit/test_replication_incident_0929.py index a1f1d4ca39..cd5a1df8a2 100644 --- a/tests/unit/test_replication_incident_0929.py +++ b/tests/unit/test_replication_incident_0929.py @@ -247,7 +247,9 @@ def test_snapshot_replication_is_not_owned(self): def test_backup_family_is_owned(self): self.assertEqual( set(tasks_runner_backup.SPEC.function_names), - {JobSchedule.FN_BACKUP, JobSchedule.FN_BACKUP_RESTORE, JobSchedule.FN_BACKUP_MERGE}, + {JobSchedule.FN_BACKUP, JobSchedule.FN_BACKUP_RESTORE, + JobSchedule.FN_BACKUP_MERGE, JobSchedule.FN_GROUP_BACKUP_COMMIT, + JobSchedule.FN_GROUP_BACKUP_DELETE}, ) diff --git a/tests/unit/web/api/v2/_factories.py b/tests/unit/web/api/v2/_factories.py index ae7f6c03fe..33b9ba196f 100644 --- a/tests/unit/web/api/v2/_factories.py +++ b/tests/unit/web/api/v2/_factories.py @@ -7,7 +7,7 @@ from pydantic import SecretStr -from simplyblock_core.models.backup import Backup, BackupPolicy +from simplyblock_core.models.backup import Backup, BackupPolicy, GroupBackup from simplyblock_core.models.cluster import Cluster from simplyblock_core.models.events import EventObj from simplyblock_core.models.job_schedule import JobSchedule @@ -42,6 +42,7 @@ TARGET_POOL_ID = 'ffffffff-ffff-ffff-ffff-ffffffffffff' VOLUME_NQN = 'nqn.2023-02.io.simplyblock:volume-1' CONSISTENCY_GROUP_ID = 'c9c9c9c9-c9c9-c9c9-c9c9-c9c9c9c9c9c9' +GROUP_BACKUP_ID = 'a7a7a7a7-a7a7-a7a7-a7a7-a7a7a7a7a7a7' def _apply(model, attrs): @@ -181,6 +182,21 @@ def make_backup(**attrs) -> Backup: return _apply(backup, attrs) +def make_group_backup(**attrs) -> GroupBackup: + group_backup = GroupBackup() + group_backup.uuid = GROUP_BACKUP_ID + group_backup.cluster_id = CLUSTER_ID + group_backup.consistency_group_id = f'{CLUSTER_ID}/{CONSISTENCY_GROUP_ID}' + group_backup.consistency_group_name = 'app-group-cg' + group_backup.group_seq = 7 + group_backup.status = GroupBackup.STATUS_IN_PROGRESS + group_backup.members = [{'lvol_id': VOLUME_ID, 'backup_id': f'{CLUSTER_ID}/{BACKUP_ID}'}] + group_backup.size = 1024 ** 3 + group_backup.created_at = 1700000000 + group_backup.completed_at = 0 + return _apply(group_backup, attrs) + + def make_backup_policy(**attrs) -> BackupPolicy: policy = BackupPolicy() policy.uuid = POLICY_ID diff --git a/tests/unit/web/api/v2/conftest.py b/tests/unit/web/api/v2/conftest.py index c776be6672..156e609bec 100644 --- a/tests/unit/web/api/v2/conftest.py +++ b/tests/unit/web/api/v2/conftest.py @@ -30,6 +30,7 @@ import simplyblock_web.api.v2.cluster.arbitration as arbitration_module import simplyblock_web.api.v2.cluster.backup as backup_module import simplyblock_web.api.v2.cluster.consistency_group as consistency_group_module +import simplyblock_web.api.v2.cluster.group_backup as group_backup_module import simplyblock_web.api.v2.cluster.replication as replication_module import simplyblock_web.api.v2.cluster.storage_node as storage_node_module import simplyblock_web.api.v2.cluster.storage_node.device as device_module @@ -81,6 +82,7 @@ def db(monkeypatch): cluster_module, backup_module, consistency_group_module, + group_backup_module, replication_module, storage_node_module, device_module, @@ -227,6 +229,15 @@ def backup_controller(monkeypatch): return mock +@pytest.fixture() +def group_backup_controller(monkeypatch): + """The CG-backup controller, reached as ``group_backup_controller`` from the + group-backups router.""" + mock = MagicMock() + monkeypatch.setattr(group_backup_module, 'group_backup_controller', mock) + return mock + + @pytest.fixture() def replication_policy_controller(monkeypatch): mock = MagicMock() @@ -341,6 +352,14 @@ def backup(db, cluster): return backup +@pytest.fixture() +def group_backup(db, cluster): + group_backup = factories.make_group_backup() + db.get_group_backups.return_value = [group_backup] + db.get_group_backup_by_id.return_value = group_backup + return group_backup + + @pytest.fixture() def backup_policy(db, cluster): policy = factories.make_backup_policy() diff --git a/tests/unit/web/api/v2/test_consistency_group_endpoints.py b/tests/unit/web/api/v2/test_consistency_group_endpoints.py index d3c2884fcd..0307a209dc 100644 --- a/tests/unit/web/api/v2/test_consistency_group_endpoints.py +++ b/tests/unit/web/api/v2/test_consistency_group_endpoints.py @@ -347,3 +347,41 @@ def test_group_migration_create_maps_conflicts_to_409(self, client, db, cluster, resp = client.post(f'{BASE}/migration', json={"target_node_id": "N2"}) assert resp.status_code == 409 mc.create_group_migration.assert_called_once_with(factories.VOLUME_ID, "N2", host_nqn=None) + + +class TestTakeGeneration: + """POST /snapshots takes one generation. A request id makes a retry return + the generation the first call took, so the answer is that generation, not + whichever generation happens to be the latest.""" + + @staticmethod + def _row(seq): + return {"group_seq": seq, "created_at": 1, "expected": 1, "present": 1, + "complete": True, "members": []} + + def _setup(self, db, consistency_group_controller, seq_taken): + db.get_consistency_group_by_id.return_value = factories.make_consistency_group() + consistency_group_controller.create_group_snapshot_for_group.return_value = (seq_taken, None) + consistency_group_controller.list_generations.return_value = [self._row(3), self._row(4)] + + def test_the_request_id_is_passed_and_its_generation_answered( + self, client, db, cluster, consistency_group_controller): + self._setup(db, consistency_group_controller, seq_taken=3) + + resp = client.post(f'{BASE}/snapshots', json={'request_id': 'sbk-uid-1'}) + + assert resp.status_code == 200 + assert resp.json()['group_seq'] == 3 + _, kwargs = consistency_group_controller.create_group_snapshot_for_group.call_args + assert kwargs['request_id'] == 'sbk-uid-1' + + def test_no_body_still_takes_a_generation( + self, client, db, cluster, consistency_group_controller): + self._setup(db, consistency_group_controller, seq_taken=4) + + resp = client.post(f'{BASE}/snapshots') + + assert resp.status_code == 200 + assert resp.json()['group_seq'] == 4 + _, kwargs = consistency_group_controller.create_group_snapshot_for_group.call_args + assert kwargs['request_id'] == '' diff --git a/tests/unit/web/api/v2/test_group_backup_endpoints.py b/tests/unit/web/api/v2/test_group_backup_endpoints.py new file mode 100644 index 0000000000..ceccef7af5 --- /dev/null +++ b/tests/unit/web/api/v2/test_group_backup_endpoints.py @@ -0,0 +1,407 @@ +"""v2 REST endpoints for consistency-group backup (design §8). + +The routers run through TestClient; the control-plane logic (group_backup +controller) and the database are mocked, so these assert the HTTP contract: which +controller call a request makes, and how a controller result or error maps to a +status code. +""" +from datetime import UTC, datetime +from typing import Any + +from simplyblock_core.controllers.backup.manifest import ( + GroupBackupExport, + ManifestError, +) +from simplyblock_core.exceptions import PreconditionError +from tests.unit.web.api.v2 import _factories as factories +from tests.unit.web.api.v2._factories import ( + BACKUP_ID, + CLUSTER_ID, + CONSISTENCY_GROUP_ID, + GROUP_BACKUP_ID, + SNAPSHOT_ID, + STORAGE_NODE_ID, + VOLUME_ID, +) + +BASE = f'/api/v2/clusters/{CLUSTER_ID}/backups/groups' +CG_BASE = f'/api/v2/clusters/{CLUSTER_ID}/consistency-groups/{CONSISTENCY_GROUP_ID}' + +#: The source consistency group, as the group manifest spells it. +CG_COMPOSITE = f'{CLUSTER_ID}/{CONSISTENCY_GROUP_ID}' + +#: A member backup manifest, the shape a single-volume export carries. +_MEMBER_MANIFEST: dict[str, Any] = { + 'schema_version': 1, + 'backup_id': BACKUP_ID, + 's3_id': 7, + 'created_at': 100, + 'completed_at': 200, + 'size': 4096, + 'source': {'cluster_id': CLUSTER_ID, 'node_id': STORAGE_NODE_ID}, + 'volume': {'lvol_id': VOLUME_ID, 'lvol_name': 'vol', + 'snapshot_id': SNAPSHOT_ID, 'snapshot_name': 'snap', + 'size': 4096}, + 'dataplane': {}, +} + +#: The group manifest, naming the member above. +_GROUP_MANIFEST: dict[str, Any] = { + 'schema_version': 1, + 'group_backup_id': GROUP_BACKUP_ID, + 'created_at': 100, + 'completed_at': 200, + 'size': 4096, + 'source': {'cluster_id': CLUSTER_ID, 'consistency_group_id': CG_COMPOSITE, + 'consistency_group_name': 'app-group-cg', 'generation': 7}, + 'member_backup_ids': [BACKUP_ID], + 'encrypted': False, +} + +_LOCATION: dict[str, str] = {'bucket_name': 'backups', 'region': 'eu-central-1'} + + +def _group_export() -> dict[str, Any]: + return { + 'schema_version': 1, + 'group_manifest': _GROUP_MANIFEST, + 'members': {'schema_version': 1, 'groups': [ + {'location': _LOCATION, 'manifests': [_MEMBER_MANIFEST]}]}, + } + + +def _group_export_model() -> GroupBackupExport: + """A real export model, for the export endpoint's response_model to render.""" + return GroupBackupExport.model_validate(_group_export()) + + +class TestListGroupBackups: + def test_lists_the_clusters_group_backups(self, client, db, group_backup): + response = client.get(f'{BASE}/') + assert response.status_code == 200 + assert [g['id'] for g in response.json()] == [GROUP_BACKUP_ID] + + +class TestGetGroupBackup: + def test_returns_the_group_backup_with_its_member_backup_ids(self, client, db, group_backup): + """The members are named by their backup ids only. Each member backup + carries its own volume, and GET .../members/ returns the records. Ids are + bare UUIDs, as on every other DTO: the cluster is cluster_id.""" + response = client.get(f'{BASE}/{GROUP_BACKUP_ID}/') + assert response.status_code == 200 + body = response.json() + assert body['id'] == GROUP_BACKUP_ID + assert body['group_seq'] == 7 + assert body['consistency_group_id'] == CONSISTENCY_GROUP_ID + assert body['member_backup_ids'] == [BACKUP_ID] + assert 'members' not in body + + def test_encryption_is_reported_by_the_member_backups_not_the_group( + self, client, db, group_backup): + """Each member backup carries its own ``encrypted``; the group does not + repeat it.""" + body = client.get(f'{BASE}/{GROUP_BACKUP_ID}/').json() + assert 'encrypted' not in body + + def test_times_are_utc_datetimes_and_an_uncommitted_one_has_no_completion( + self, client, db, group_backup): + body = client.get(f'{BASE}/{GROUP_BACKUP_ID}/').json() + assert datetime.fromisoformat(body['created_at']) == datetime.fromtimestamp(1700000000, tz=UTC) + assert body['completed_at'] is None + + def test_a_committed_group_backup_reports_when_it_completed(self, client, db, cluster): + db.get_group_backup_by_id.return_value = factories.make_group_backup( + status='completed', completed_at=1700000100) + body = client.get(f'{BASE}/{GROUP_BACKUP_ID}/').json() + assert datetime.fromisoformat(body['completed_at']) == datetime.fromtimestamp(1700000100, tz=UTC) + + def test_a_group_backup_without_an_error_reports_none(self, client, db, group_backup): + body = client.get(f'{BASE}/{GROUP_BACKUP_ID}/').json() + assert body['error_message'] is None + + def test_a_failed_group_backup_reports_its_error(self, client, db, cluster): + db.get_group_backup_by_id.return_value = factories.make_group_backup( + status='failed', error_message='member vol-1 could not be backed up') + body = client.get(f'{BASE}/{GROUP_BACKUP_ID}/').json() + assert body['error_message'] == 'member vol-1 could not be backed up' + + def test_404_when_the_group_backup_belongs_to_another_cluster(self, client, db, cluster): + other = factories.make_group_backup(cluster_id='00000000-0000-0000-0000-000000000000') + db.get_group_backup_by_id.return_value = other + response = client.get(f'{BASE}/{GROUP_BACKUP_ID}/') + assert response.status_code == 404 + + +class TestRestoreGroupBackup: + def test_restores_into_a_new_group_and_returns_the_members( + self, client, db, group_backup, group_backup_controller): + members = [{'lvol_id': 'lvol-a', 'member_name': 'db'}, + {'lvol_id': 'lvol-b', 'member_name': 'data'}] + group_backup_controller.restore_group_backup.return_value = ( + f'{CLUSTER_ID}/new-cg', members, None) + + response = client.post( + f'{BASE}/{GROUP_BACKUP_ID}/restore', + json={'into_group_name': 'restored-cg', 'pool': 'pool-1'}) + + assert response.status_code == 202 + assert response.json() == { + 'group_id': f'{CLUSTER_ID}/new-cg', 'members': members} + args, kwargs = group_backup_controller.restore_group_backup.call_args + assert args[0] == group_backup.get_id() + assert args[1] == 'restored-cg' + assert args[2] == 'pool-1' + + def test_a_restore_error_is_409(self, client, db, group_backup, group_backup_controller): + group_backup_controller.restore_group_backup.return_value = ( + None, None, 'consistency group restored-cg already exists') + response = client.post( + f'{BASE}/{GROUP_BACKUP_ID}/restore', + json={'into_group_name': 'restored-cg', 'pool': 'pool-1'}) + # 409, never 422: a string detail under 422 is undecodable to the + # generated clients, which type 422 as FastAPI's field-error list. + assert response.status_code == 409 + assert 'already exists' in response.json()['detail'] + + +class TestDeleteGroupBackup: + def test_a_delete_is_accepted_and_finishes_in_the_background( + self, client, db, group_backup, group_backup_controller): + """The record and manifest are marked deleting; a task retires the + members and removes both, so the answer is 202, not 204.""" + group_backup_controller.delete_group_backup.return_value = None + response = client.delete(f'{BASE}/{GROUP_BACKUP_ID}/') + assert response.status_code == 202 + group_backup_controller.delete_group_backup.assert_called_once_with( + group_backup.get_id()) + + def test_a_group_backup_still_committing_is_409( + self, client, db, group_backup, group_backup_controller): + group_backup_controller.delete_group_backup.side_effect = PreconditionError( + 'still committing') + response = client.delete(f'{BASE}/{GROUP_BACKUP_ID}/') + assert response.status_code == 409 + + def test_a_manifest_that_cannot_be_marked_is_400( + self, client, db, group_backup, group_backup_controller): + group_backup_controller.delete_group_backup.side_effect = ManifestError( + 'bucket unreachable') + response = client.delete(f'{BASE}/{GROUP_BACKUP_ID}/') + assert response.status_code == 400 + + +class TestCreateGroupBackup: + """POST /backups/groups/ backs up one generation of the group the body names.""" + + def _group(self, db, **attrs): + db.get_consistency_group_by_id.return_value = factories.make_consistency_group(**attrs) + + def test_takes_a_fresh_generation_by_default( + self, client, db, cluster, group_backup, group_backup_controller): + self._group(db) + group_backup_controller.create_group_backup.return_value = (group_backup.get_id(), None) + + response = client.post(f'{BASE}/', json={'consistency_group_id': CONSISTENCY_GROUP_ID}) + + assert response.status_code == 201 + assert response.json()['id'] == GROUP_BACKUP_ID + args, kwargs = group_backup_controller.create_group_backup.call_args + assert args[0] == CG_COMPOSITE + assert kwargs['group_seq'] is None + + def test_backs_up_a_named_generation( + self, client, db, cluster, group_backup, group_backup_controller): + self._group(db) + group_backup_controller.create_group_backup.return_value = (group_backup.get_id(), None) + + response = client.post(f'{BASE}/', json={'consistency_group_id': CONSISTENCY_GROUP_ID, 'group_seq': 3}) + + assert response.status_code == 201 + _, kwargs = group_backup_controller.create_group_backup.call_args + assert kwargs['group_seq'] == 3 + + def test_a_create_error_is_409(self, client, db, cluster, group_backup_controller): + self._group(db) + group_backup_controller.create_group_backup.return_value = ( + None, 'consistency group has no member snapshots at generation 3') + response = client.post(f'{BASE}/', json={'consistency_group_id': CONSISTENCY_GROUP_ID, 'group_seq': 3}) + assert response.status_code == 409 + + def test_an_unknown_group_is_404(self, client, db, cluster, group_backup_controller): + db.get_consistency_group_by_id.side_effect = KeyError('ConsistencyGroup not found') + response = client.post(f'{BASE}/', json={'consistency_group_id': CONSISTENCY_GROUP_ID}) + assert response.status_code == 404 + group_backup_controller.create_group_backup.assert_not_called() + + def test_a_group_of_another_cluster_is_404(self, client, db, cluster, group_backup_controller): + self._group(db, cluster_id='00000000-0000-4000-8000-000000000000') + response = client.post(f'{BASE}/', json={'consistency_group_id': CONSISTENCY_GROUP_ID}) + assert response.status_code == 404 + group_backup_controller.create_group_backup.assert_not_called() + + def test_the_group_is_required(self, client, db, cluster, group_backup_controller): + response = client.post(f'{BASE}/', json={}) + assert response.status_code == 422 + group_backup_controller.create_group_backup.assert_not_called() + + +class TestGroupBackupMembers: + def test_lists_the_member_backups(self, client, db, group_backup): + member = factories.make_backup(group_backup_id=group_backup.get_id(), group_seq=7) + db.get_backups_by_group_backup_id.return_value = [member] + + response = client.get(f'{BASE}/{GROUP_BACKUP_ID}/members/') + + assert response.status_code == 200 + body = response.json() + assert [m['id'] for m in body] == [BACKUP_ID] + assert body[0]['group_seq'] == 7 + db.get_backups_by_group_backup_id.assert_called_once_with(group_backup.get_id()) + + def test_404_for_a_group_backup_of_another_cluster(self, client, db, cluster): + db.get_group_backup_by_id.return_value = factories.make_group_backup( + cluster_id='00000000-0000-4000-8000-000000000000') + response = client.get(f'{BASE}/{GROUP_BACKUP_ID}/members/') + assert response.status_code == 404 + + +class TestGroupBackupRoutes: + """Group backups live under /backups/, beside routes whose last segment is a + backup id. /backups/groups/ must reach the group routes, not be read as a + backup id that is not a UUID.""" + + def test_groups_is_not_read_as_a_backup_id(self, client, db, group_backup): + response = client.get(f'/api/v2/clusters/{CLUSTER_ID}/backups/groups/') + assert response.status_code == 200 + assert isinstance(response.json(), list) + + def test_the_old_paths_are_gone(self, client, db, cluster, group_backup): + assert client.get(f'/api/v2/clusters/{CLUSTER_ID}/group-backups/').status_code == 404 + assert client.post(f'{CG_BASE}/backups', json={}).status_code in (404, 405) + + +class TestExportGroupBackup: + def test_exports_the_group_backup_as_a_unit( + self, client, db, group_backup, group_backup_controller): + group_backup_controller.export_group_backup.return_value = _group_export_model() + + response = client.get(f'{BASE}/export?group_backup_id={GROUP_BACKUP_ID}') + + assert response.status_code == 200 + body = response.json() + assert body['group_manifest']['group_backup_id'] == GROUP_BACKUP_ID + assert [m['backup_id'] for m in body['members']['groups'][0]['manifests']] == [ + BACKUP_ID] + group_backup_controller.export_group_backup.assert_called_once_with( + group_backup.get_id(), cluster_id=CLUSTER_ID) + + def test_404_for_an_unknown_group_backup(self, client, db, cluster): + db.get_group_backup_by_id.side_effect = KeyError('nope') + response = client.get(f'{BASE}/export?group_backup_id={GROUP_BACKUP_ID}') + assert response.status_code == 404 + + def test_404_when_the_group_backup_belongs_to_another_cluster( + self, client, db, cluster): + db.get_group_backup_by_id.return_value = factories.make_group_backup( + cluster_id='00000000-0000-0000-0000-000000000000') + response = client.get(f'{BASE}/export?group_backup_id={GROUP_BACKUP_ID}') + assert response.status_code == 404 + + +class TestImportGroupBackup: + """The body is a union of two shapes, as the single-volume import is.""" + + def test_inline_export_is_validated_by_the_body_type( + self, client, db, cluster, group_backup_controller): + group_backup_controller.import_group_backup.return_value = f'{CLUSTER_ID}/{GROUP_BACKUP_ID}' + + response = client.post(f'{BASE}/import', json={'metadata': _group_export()}) + + assert response.status_code == 200 + assert response.json() == { + 'imported': 1, 'group_backup_ids': [f'{CLUSTER_ID}/{GROUP_BACKUP_ID}']} + (export,), kwargs = group_backup_controller.import_group_backup.call_args + assert str(export.group_manifest.group_backup_id) == GROUP_BACKUP_ID + assert kwargs['cluster_id'] == CLUSTER_ID + + def test_a_bucket_reads_the_group_backups_itself( + self, client, db, cluster, group_backup_controller): + group_backup_controller.import_group_backup_from_bucket.return_value = [ + f'{CLUSTER_ID}/gb-a', f'{CLUSTER_ID}/gb-b'] + + response = client.post(f'{BASE}/import', json={'bucket': _LOCATION}) + + assert response.status_code == 200 + assert response.json()['imported'] == 2 + group_backup_controller.import_group_backup.assert_not_called() + + def test_naming_both_sources_is_rejected( + self, client, db, cluster, group_backup_controller): + response = client.post(f'{BASE}/import', json={ + 'metadata': _group_export(), 'bucket': _LOCATION}) + assert response.status_code == 422 + + def test_naming_neither_source_is_rejected( + self, client, db, cluster, group_backup_controller): + response = client.post(f'{BASE}/import', json={}) + assert response.status_code == 422 + + def test_an_unreadable_bucket_is_a_bad_request( + self, client, db, cluster, group_backup_controller): + from simplyblock_core.controllers.backup.manifest import ManifestError + group_backup_controller.import_group_backup_from_bucket.side_effect = \ + ManifestError('no such bucket') + + response = client.post(f'{BASE}/import', json={'bucket': _LOCATION}) + + assert response.status_code == 400 + assert 'no such bucket' in response.json()['detail'] + + def test_a_torn_export_is_a_bad_request( + self, client, db, cluster, group_backup_controller): + group_backup_controller.import_group_backup.side_effect = ValueError( + 'group export is missing member manifests') + + response = client.post(f'{BASE}/import', json={'metadata': _group_export()}) + + assert response.status_code == 400 + assert 'missing member manifests' in response.json()['detail'] + + def test_a_precondition_error_is_not_mapped_here( + self, client, db, cluster, group_backup_controller): + """app.py maps PreconditionError to 400 for the whole API; the endpoint + lets it through rather than mapping it a second, disagreeing way.""" + import pytest + + from simplyblock_core.exceptions import PreconditionError + group_backup_controller.import_group_backup.side_effect = PreconditionError( + 'already exists') + + with pytest.raises(PreconditionError): + client.post(f'{BASE}/import', json={'metadata': _group_export()}) + + +class TestDiscoverGroupBackups: + def test_returns_the_group_backups_the_bucket_holds( + self, client, db, group_backup_controller): + from simplyblock_core.controllers.backup.manifest import GroupBackupManifest + group_backup_controller.discover_group_backups.return_value = [ + GroupBackupManifest.model_validate(_GROUP_MANIFEST)] + + response = client.post(f'{BASE}/discover', json=_LOCATION) + + assert response.status_code == 200 + assert [g['group_backup_id'] for g in response.json()] == [GROUP_BACKUP_ID] + + def test_credentials_are_masked_in_a_failure_response( + self, client, db, group_backup_controller): + from simplyblock_core.controllers.backup.manifest import ManifestError + group_backup_controller.discover_group_backups.side_effect = \ + ManifestError('unreachable') + + response = client.post(f'{BASE}/discover', json={ + **_LOCATION, + 'credentials': {'access_key_id': 'AKIA', 'secret_access_key': 's3cr3t'}}) + + assert response.status_code == 400 + assert 's3cr3t' not in response.text