feat(backup): consistency-group backup foundation (records, manifest,…

simplyblock_cli/cli-reference.yaml

@@ -3046,6 +3046,17 @@ commands:
             dest: into
             type: str
             required: false
+      - name: backup-create
+        help: "Back up one generation of a consistency group to S3 as one all-or-nothing unit. Takes a fresh crash-consistent generation unless --group-seq names an existing one."
+        arguments:
+          - name: "group_id"
+            help: "Consistency group id (or uuid)."
+            dest: group_id
+            type: str
+          - name: "--group-seq"
+            help: "Back up an existing generation by its sequence number. Omit to take a fresh generation."
+            dest: group_seq
+            type: int
   - name: "backup"
     help: "Backup Commands"
     weight: 650
@@ -3267,6 +3278,168 @@ commands:
             help: "The target id (storage pool or logical volume id)."
             dest: target_id
             type: str
+  - name: "group-backup"
+    help: "Consistency-group backup commands"
+    aliases:
+      - gb
+    weight: 655
+    subcommands:
+      - name: list
+        help: "List the cluster's consistency-group backups."
+        arguments:
+          - name: "cluster_id"
+            help: "Cluster UUID."
+            dest: cluster_id
+            type: str
+          - name: "--json"
+            aliases:
+              - "-j"
+            help: "Print output in JSON format."
+            dest: json
+            type: bool
+            action: store_true
+      - name: get
+        help: "Show one consistency-group backup and its members."
+        arguments:
+          - name: "group_backup_id"
+            help: "Group backup id (or uuid)."
+            dest: group_backup_id
+            type: str
+          - name: "--json"
+            aliases:
+              - "-j"
+            help: "Print output in JSON format."
+            dest: json
+            type: bool
+            action: store_true
+      - name: restore
+        help: "Restore a consistency-group backup into a NEW consistency group, all-or-nothing. Each member volume recovers asynchronously."
+        arguments:
+          - name: "group_backup_id"
+            help: "Group backup id (or uuid)."
+            dest: group_backup_id
+            type: str
+          - name: "--name"
+            help: "Name of the NEW consistency group to form from the restored volumes. Must not already exist."
+            dest: into_group_name
+            type: str
+            required: true
+          - name: "--pool"
+            help: "The target pool name or id."
+            dest: pool
+            type: str
+            required: true
+          - name: "--node"
+            help: "The target storage node id. Omit to auto-select; every member is pinned to one store."
+            dest: node
+            type: str
+          - name: "--access-key-id"
+            help: "Access key for the backup's bucket, when it is not this cluster's own."
+            dest: access_key_id
+            type: secret
+          - name: "--secret-access-key"
+            help: "Secret key for the backup's bucket, when it is not this cluster's own."
+            dest: secret_access_key
+            type: secret
+      - name: delete
+        help: "Delete a consistency-group backup: its group manifest and record. Its member backups are retired: merged into the copy of the next generation while that chains against them, deleted once nothing does."
+        arguments:
+          - name: "group_backup_id"
+            help: "Group backup id (or uuid)."
+            dest: group_backup_id
+            type: str
+      - name: export
+        help: "Export a consistency-group backup's manifests as a unit, to a JSON file for cross-cluster restore."
+        arguments:
+          - name: "group_backup_id"
+            help: "Group backup id (or uuid)."
+            dest: group_backup_id
+            type: str
+          - name: "--cluster-id"
+            help: "The cluster id."
+            dest: cluster_id
+            type: str
+          - name: "-o"
+            aliases:
+              - "--output"
+            help: "The output file path."
+            dest: output
+            type: str
+      - name: discover
+        help: "List the consistency-group backups a bucket contains, reading its group manifests. Needs no cluster."
+        arguments:
+          - name: "--bucket"
+            help: "The bucket holding the group backups."
+            dest: bucket
+            type: str
+            required: true
+          - name: "--region"
+            help: "The bucket's region. Omit to let the AWS SDK resolve it."
+            dest: region
+            type: str
+          - name: "--endpoint"
+            help: "Endpoint of an S3-compatible store, e.g. http://minio:9000. Omit for AWS."
+            dest: endpoint
+            type: str
+          - name: "--access-key-id"
+            help: "Access key for the bucket. Omit to use the node's instance role."
+            dest: access_key_id
+            type: secret
+          - name: "--secret-access-key"
+            help: "Secret key for the bucket. Omit to use the node's instance role."
+            dest: secret_access_key
+            type: secret
+          - name: "--no-verify-tls"
+            help: "Skip certificate verification for the endpoint."
+            dest: no_verify_tls
+            type: bool
+            action: store_true
+          - name: "--path-style"
+            help: "Use path-style addressing, as MinIO and most S3-compatible stores need."
+            dest: path_style
+            type: bool
+            action: store_true
+      - name: import
+        help: "Register the consistency-group backups held in a bucket, or in an export file, into this cluster."
+        arguments:
+          - name: "--cluster-id"
+            help: "The target cluster to import into (required for cross-cluster restore)."
+            dest: cluster_id
+            type: str
+          - name: "--bucket"
+            help: "Import every group backup in this bucket. Give this or --from-file, not both."
+            dest: bucket
+            type: str
+          - name: "--from-file"
+            help: "Import the group backup in this file, from 'group-backup export'. It records which bucket its members live in, so --bucket is neither needed nor accepted."
+            dest: from_file
+            type: str
+          - name: "--region"
+            help: "The bucket's region. Omit to let the AWS SDK resolve it."
+            dest: region
+            type: str
+          - name: "--endpoint"
+            help: "Endpoint of an S3-compatible store, e.g. http://minio:9000. Omit for AWS."
+            dest: endpoint
+            type: str
+          - name: "--access-key-id"
+            help: "Access key for the bucket. Omit to use the node's instance role."
+            dest: access_key_id
+            type: secret
+          - name: "--secret-access-key"
+            help: "Secret key for the bucket. Omit to use the node's instance role."
+            dest: secret_access_key
+            type: secret
+          - name: "--no-verify-tls"
+            help: "Skip certificate verification for the endpoint."
+            dest: no_verify_tls
+            type: bool
+            action: store_true
+          - name: "--path-style"
+            help: "Use path-style addressing, as MinIO and most S3-compatible stores need."
+            dest: path_style
+            type: bool
+            action: store_true
   - name: "qos"
     help: "QoS Commands"
     weight: 700

simplyblock_cli/cli.py

@@ -31,6 +31,7 @@ def __init__(self):
         self.init_snapshot()
         self.init_consistency_group()
         self.init_backup()
+        self.init_group_backup()
         self.init_qos()
         self.init_db_backup()
         super().__init__()
@@ -1165,6 +1166,7 @@ def init_consistency_group(self):
         self.init_consistency_group__snapshot_list(subparser)
         self.init_consistency_group__snapshot_delete(subparser)
         self.init_consistency_group__clone(subparser)
+        self.init_consistency_group__backup_create(subparser)
 
 
     def init_consistency_group__list(self, subparser):
@@ -1211,6 +1213,11 @@ def init_consistency_group__clone(self, subparser):
         subcommand.add_argument('seq', help='The generation number (group_seq) to clone.', type=int)
         subcommand.add_argument('--into', help='Name of the new consistency group to form from the clones. When omitted, the clones are independent volumes.', type=str, dest='into', required=False)
 
+    def init_consistency_group__backup_create(self, subparser):
+        subcommand = self.add_sub_command(subparser, 'backup-create', 'Back up one generation of a consistency group to S3 as one all-or-nothing unit. Takes a fresh crash-consistent generation unless --group-seq names an existing one.')
+        subcommand.add_argument('group_id', help='Consistency group id (or uuid).', type=str)
+        subcommand.add_argument('--group-seq', help='Back up an existing generation by its sequence number. Omit to take a fresh generation.', type=int, dest='group_seq')
+
 
     def init_backup(self):
         subparser = self.add_command('backup', 'Backup Commands')
@@ -1303,6 +1310,69 @@ def init_backup__policy_detach(self, subparser):
         subcommand.add_argument('target_id', help='The target id (storage pool or logical volume id).', type=str)
 
 
+    def init_group_backup(self):
+        subparser = self.add_command('group-backup', 'Consistency-group backup commands', aliases=['gb',])
+        self.init_group_backup__list(subparser)
+        self.init_group_backup__get(subparser)
+        self.init_group_backup__restore(subparser)
+        self.init_group_backup__delete(subparser)
+        self.init_group_backup__export(subparser)
+        self.init_group_backup__discover(subparser)
+        self.init_group_backup__import(subparser)
+
+
+    def init_group_backup__list(self, subparser):
+        subcommand = self.add_sub_command(subparser, 'list', 'List the cluster\'s consistency-group backups.')
+        subcommand.add_argument('cluster_id', help='Cluster UUID.', type=str)
+        subcommand.add_argument('--json', '-j', help='Print output in JSON format.', dest='json', action='store_true')
+
+    def init_group_backup__get(self, subparser):
+        subcommand = self.add_sub_command(subparser, 'get', 'Show one consistency-group backup and its members.')
+        subcommand.add_argument('group_backup_id', help='Group backup id (or uuid).', type=str)
+        subcommand.add_argument('--json', '-j', help='Print output in JSON format.', dest='json', action='store_true')
+
+    def init_group_backup__restore(self, subparser):
+        subcommand = self.add_sub_command(subparser, 'restore', 'Restore a consistency-group backup into a NEW consistency group, all-or-nothing. Each member volume recovers asynchronously.')
+        subcommand.add_argument('group_backup_id', help='Group backup id (or uuid).', type=str)
+        subcommand.add_argument('--name', help='Name of the NEW consistency group to form from the restored volumes. Must not already exist.', type=str, dest='into_group_name', required=True)
+        subcommand.add_argument('--pool', help='The target pool name or id.', type=str, dest='pool', required=True)
+        subcommand.add_argument('--node', help='The target storage node id. Omit to auto-select; every member is pinned to one store.', type=str, dest='node')
+        subcommand.add_argument('--access-key-id', help='Access key for the backup\'s bucket, when it is not this cluster\'s own.', type=SecretStr, dest='access_key_id')
+        subcommand.add_argument('--secret-access-key', help='Secret key for the backup\'s bucket, when it is not this cluster\'s own.', type=SecretStr, dest='secret_access_key')
+
+    def init_group_backup__delete(self, subparser):
+        subcommand = self.add_sub_command(subparser, 'delete', 'Delete a consistency-group backup: its group manifest and record. Its member backups are retired: merged into the copy of the next generation while that chains against them, deleted once nothing does.')
+        subcommand.add_argument('group_backup_id', help='Group backup id (or uuid).', type=str)
+
+    def init_group_backup__export(self, subparser):
+        subcommand = self.add_sub_command(subparser, 'export', 'Export a consistency-group backup\'s manifests as a unit, to a JSON file for cross-cluster restore.')
+        subcommand.add_argument('group_backup_id', help='Group backup id (or uuid).', type=str)
+        subcommand.add_argument('--cluster-id', help='The cluster id.', type=str, dest='cluster_id')
+        subcommand.add_argument('-o', '--output', help='The output file path.', type=str, dest='output')
+
+    def init_group_backup__discover(self, subparser):
+        subcommand = self.add_sub_command(subparser, 'discover', 'List the consistency-group backups a bucket contains, reading its group manifests. Needs no cluster.')
+        subcommand.add_argument('--bucket', help='The bucket holding the group backups.', type=str, dest='bucket', required=True)
+        subcommand.add_argument('--region', help='The bucket\'s region. Omit to let the AWS SDK resolve it.', type=str, dest='region')
+        subcommand.add_argument('--endpoint', help='Endpoint of an S3-compatible store, e.g. http://minio:9000. Omit for AWS.', type=str, dest='endpoint')
+        subcommand.add_argument('--access-key-id', help='Access key for the bucket. Omit to use the node\'s instance role.', type=SecretStr, dest='access_key_id')
+        subcommand.add_argument('--secret-access-key', help='Secret key for the bucket. Omit to use the node\'s instance role.', type=SecretStr, dest='secret_access_key')
+        subcommand.add_argument('--no-verify-tls', help='Skip certificate verification for the endpoint.', dest='no_verify_tls', action='store_true')
+        subcommand.add_argument('--path-style', help='Use path-style addressing, as MinIO and most S3-compatible stores need.', dest='path_style', action='store_true')
+
+    def init_group_backup__import(self, subparser):
+        subcommand = self.add_sub_command(subparser, 'import', 'Register the consistency-group backups held in a bucket, or in an export file, into this cluster.')
+        subcommand.add_argument('--cluster-id', help='The target cluster to import into (required for cross-cluster restore).', type=str, dest='cluster_id')
+        subcommand.add_argument('--bucket', help='Import every group backup in this bucket. Give this or --from-file, not both.', type=str, dest='bucket')
+        subcommand.add_argument('--from-file', help='Import the group backup in this file, from \'group-backup export\'. It records which bucket its members live in, so --bucket is neither needed nor accepted.', type=str, dest='from_file')
+        subcommand.add_argument('--region', help='The bucket\'s region. Omit to let the AWS SDK resolve it.', type=str, dest='region')
+        subcommand.add_argument('--endpoint', help='Endpoint of an S3-compatible store, e.g. http://minio:9000. Omit for AWS.', type=str, dest='endpoint')
+        subcommand.add_argument('--access-key-id', help='Access key for the bucket. Omit to use the node\'s instance role.', type=SecretStr, dest='access_key_id')
+        subcommand.add_argument('--secret-access-key', help='Secret key for the bucket. Omit to use the node\'s instance role.', type=SecretStr, dest='secret_access_key')
+        subcommand.add_argument('--no-verify-tls', help='Skip certificate verification for the endpoint.', dest='no_verify_tls', action='store_true')
+        subcommand.add_argument('--path-style', help='Use path-style addressing, as MinIO and most S3-compatible stores need.', dest='path_style', action='store_true')
+
+
     def init_qos(self):
         subparser = self.add_command('qos', 'QoS Commands')
         self.init_qos__add(subparser)
@@ -1827,6 +1897,8 @@ def run(self):
                     ret = self.consistency_group__snapshot_delete(sub_command, args)
                 elif sub_command in ['clone']:
                     ret = self.consistency_group__clone(sub_command, args)
+                elif sub_command in ['backup-create']:
+                    ret = self.consistency_group__backup_create(sub_command, args)
                 else:
                     self.parser.print_help()
 
@@ -1857,6 +1929,25 @@ def run(self):
                 else:
                     self.parser.print_help()
 
+            elif args.command in ['group-backup', 'gb']:
+                sub_command = args_dict['group-backup']
+                if sub_command in ['list']:
+                    ret = self.group_backup__list(sub_command, args)
+                elif sub_command in ['get']:
+                    ret = self.group_backup__get(sub_command, args)
+                elif sub_command in ['restore']:
+                    ret = self.group_backup__restore(sub_command, args)
+                elif sub_command in ['delete']:
+                    ret = self.group_backup__delete(sub_command, args)
+                elif sub_command in ['export']:
+                    ret = self.group_backup__export(sub_command, args)
+                elif sub_command in ['discover']:
+                    ret = self.group_backup__discover(sub_command, args)
+                elif sub_command in ['import']:
+                    ret = self.group_backup__import(sub_command, args)
+                else:
+                    self.parser.print_help()
+
             elif args.command in ['qos']:
                 sub_command = args_dict['qos']
                 if sub_command in ['add']: