MM-5 Enhance Delinea Secret Server Migration: Phase one

keepercommander/commands/helpers/team.py

@@ -0,0 +1,68 @@
+from ... import api, crypto, utils
+
+
+class MissingTeamEncryptionKey(ValueError):
+    """The enterprise tree key or data key needed to create a team is missing."""
+
+
+def generate_team_add_key_fields(team_key, allow_rsa=True):
+    # type: (bytes, bool) -> dict
+    """Generate the EC (and, when allowed, RSA) key fields for a team_add request.
+
+    The EC private key is encrypted with AES-v2 and the RSA private key with AES-v1,
+    both under ``team_key``. Every value is base64url-encoded. Returned names match
+    the request fields: ``ecc_private_key``, ``ecc_public_key``, and when RSA is
+    allowed ``private_key`` and ``public_key``.
+    """
+    ec_private_key, ec_public_key = crypto.generate_ec_key()
+    fields = {
+        'ecc_private_key': utils.base64_url_encode(
+            crypto.encrypt_aes_v2(crypto.unload_ec_private_key(ec_private_key), team_key)),
+        'ecc_public_key': utils.base64_url_encode(crypto.unload_ec_public_key(ec_public_key)),
+    }
+    if allow_rsa:
+        rsa_private_key, rsa_public_key = crypto.generate_rsa_key()
+        fields['private_key'] = utils.base64_url_encode(
+            crypto.encrypt_aes_v1(crypto.unload_rsa_private_key(rsa_private_key), team_key))
+        fields['public_key'] = utils.base64_url_encode(crypto.unload_rsa_public_key(rsa_public_key))
+    return fields
+
+
+def build_team_add_request(params, team_name, node_id,
+                           restrict_edit=False, restrict_share=False, restrict_view=False,
+                           manage_only=True, team_uid=None, team_key=None):
+    # type: (object, str, int, bool, bool, bool, bool, str, bytes) -> tuple
+    """Build a team_add request and return ``(request, team_key)``.
+
+    ``team_uid`` and the team AES key are generated when the caller does not
+    supply them. The team key is encrypted with the data key (AES-v1) and the
+    enterprise tree key (AES-v2). RSA key fields are omitted when
+    ``params.forbid_rsa`` is set. Raises MissingTeamEncryptionKey when the tree
+    key or data key is missing. The returned team key is the raw AES key, for
+    callers that still need it to add members.
+    """
+    tree_key = (getattr(params, 'enterprise', None) or {}).get('unencrypted_tree_key')
+    data_key = getattr(params, 'data_key', None)
+    if not tree_key or not data_key:
+        raise MissingTeamEncryptionKey('Enterprise tree key or data key is not available')
+
+    if team_uid is None:
+        team_uid = api.generate_record_uid()
+    if team_key is None:
+        team_key = api.generate_aes_key()
+
+    request = {
+        'command': 'team_add',
+        'team_uid': team_uid,
+        'team_name': team_name,
+        'restrict_edit': restrict_edit,
+        'restrict_share': restrict_share,
+        'restrict_view': restrict_view,
+        'node_id': node_id,
+        'team_key': utils.base64_url_encode(crypto.encrypt_aes_v1(team_key, data_key)),
+        'encrypted_team_key': utils.base64_url_encode(crypto.encrypt_aes_v2(team_key, tree_key)),
+        'manage_only': manage_only,
+    }
+    request.update(generate_team_add_key_fields(
+        team_key, allow_rsa=not getattr(params, 'forbid_rsa', False)))
+    return request, team_key

keepercommander/importer/commands.py

@@ -19,7 +19,7 @@
 
 from . import imp_exp
 from .. import api, record_types
-from .importer import SharedFolder, Team, Permission, PathDelimiter, replace_email_domain, BaseDownloadMembership, BaseDownloadRecordType, RecordType
+from .importer import SharedFolder, Team, Permission, Record, PathDelimiter, normalize_folder_path, replace_email_domain, BaseDownloadMembership, BaseDownloadRecordType, RecordType
 from .json.json import KeeperJsonImporter, KeeperJsonExporter
 from ..commands.base import dump_report_data, raise_parse_exception, suppress_exit, user_choice, Command
 from ..commands.enterprise_common import EnterpriseCommand
@@ -63,6 +63,17 @@ def _cyberark_skip_arg(value):
     return ",".join(skip_targets)
 
 
+
+def _workers_arg(value):
+    try:
+        workers = int(value)
+    except (TypeError, ValueError):
+        raise argparse.ArgumentTypeError('must be an integer')
+    if not 1 <= workers <= 16:
+        raise argparse.ArgumentTypeError('--workers must be between 1 and 16')
+    return workers
+
+
 def _positive_int_arg(value):
     try:
         depth = int(value)
@@ -120,9 +131,14 @@ def _positive_int_arg(value):
 import_parser.add_argument('--file-cache', dest='tmpdir', action='store',
                            help='temp directory used to cache encrypted attachment imports')
 import_parser.add_argument('--show-skipped', dest='show_skipped', action='store_true',
-                           help='display skipped records (always enabled for CyberArk imports)')
+                           help='display skipped records (always enabled for CyberArk and Thycotic imports)')
 import_parser.add_argument('--secret-ids', dest='secret_ids', action='store',
                            help='Comma separated list of secret IDs to fetch (Thycotic)')
+import_parser.add_argument('--add-users', dest='add_users', action='store_true',
+                           help='provision source users as pre-activated Keeper enterprise users')
+import_parser.add_argument('--add-teams', dest='add_teams', action='store_true',
+                           help='create Keeper teams from source groups and add members '
+                                'who already exist as Keeper users')
 import_parser.add_argument('--target-node', '--node', dest='target_node', action='store',
                            help='node name or ID for CyberArk-provisioned users, teams, and roles (default: root node)')
 import_parser.add_argument('--skip', dest='skip', action='store', type=_cyberark_skip_arg,
@@ -171,6 +187,9 @@ def _positive_int_arg(value):
 download_membership_parser.add_argument('name', type=str, nargs='?', help='Output file name. "shared_folder_membership.json" if omitted.')
 download_membership_parser.add_argument('--timeout', dest='timeout', action='store', type=int, default=10, help='Timeout in seconds for network requests. Currently, supported for cyberark. Default: 10')
 download_membership_parser.add_argument('--api-page-size-limit', dest='api_page_size_limit', action='store', type=int, default=200, help='Page size limit for API requests. Currently, supported for cyberark. Default: 200')
+download_membership_parser.add_argument('--workers', dest='workers', action='store', type=_workers_arg, default=None,
+                                        help='Parallel requests when loading secret permissions (1-16). '
+                                             'Currently, supported for thycotic. Default: 4')
 download_membership_parser.error = raise_parse_exception
 download_membership_parser.exit = suppress_exit
 
@@ -460,6 +479,12 @@ def set_permission(perm, user, permit, restrict, perm_name):
     setattr(perm, perm_name, p)
 
 
+def _share_record_key(record):
+    """Identity of a share record in the membership file: all its folder paths plus title."""
+    paths = tuple(sorted({normalize_folder_path(folder.path) for folder in record.folders or []}))
+    return (paths or ('',), (record.title or '').casefold())
+
+
 class DownloadMembershipCommand(Command):
     def get_parser(self):  # type: () -> Optional[argparse.ArgumentParser]
         return download_membership_parser
@@ -495,6 +520,7 @@ def execute(self, params, **kwargs):  # type: (KeeperParams, **any) -> any
 
         added_folders = []  # type: List[SharedFolder]
         added_teams = []    # type: List[Team]
+        added_records = []  # type: List[Record]
 
         try:
             full_name = f'keepercommander.importer.{"json" if source == "keeper" else source}'
@@ -508,7 +534,7 @@ def execute(self, params, **kwargs):  # type: (KeeperParams, **any) -> any
             logging.warning('Error loading membership plugin: %s', source)
             return
 
-        download_membership = plugin.download_membership(params, folders_only=folders_only,timeout=kwargs.get('timeout'), api_page_size_limit=kwargs.get('api_page_size_limit'))
+        download_membership = plugin.download_membership(params, folders_only=folders_only,timeout=kwargs.get('timeout'), api_page_size_limit=kwargs.get('api_page_size_limit'), workers=kwargs.get('workers'))
 
         for obj in download_membership:
             if isinstance(obj, SharedFolder):
@@ -535,6 +561,14 @@ def execute(self, params, **kwargs):  # type: (KeeperParams, **any) -> any
                     obj.members = [replace_email_domain(x, old_domain, new_domain) for x in obj.members]
                 if obj.name:
                     added_teams.append(obj)
+            elif isinstance(obj, Record) and obj.shares:
+                if old_domain and new_domain:
+                    for share in obj.shares:
+                        share.name = replace_email_domain(share.name, old_domain, new_domain)
+                if import_into:
+                    for folder in obj.folders or []:
+                        folder.path = PathDelimiter.join([import_into, folder.path]) if folder.path else import_into
+                added_records.append(obj)
 
         # process shared sub folders
         sub_folder_action = kwargs.get('sub_folder') or 'ignore'
@@ -562,6 +596,7 @@ def execute(self, params, **kwargs):  # type: (KeeperParams, **any) -> any
 
         shared_folders = {}  # type: Dict[str, SharedFolder]
         teams = {}           # type: Dict[str, Team]
+        existing_records = []  # type: List[Record]
 
         json_importer = KeeperJsonImporter()
         if os.path.exists(file_name):
@@ -573,28 +608,36 @@ def execute(self, params, **kwargs):  # type: (KeeperParams, **any) -> any
                     elif isinstance(obj, Team):
                         if obj.uid:
                             teams[obj.uid] = obj
+                    elif isinstance(obj, Record) and obj.shares:
+                        existing_records.append(obj)
             except:
                 pass
 
-        if added_folders or added_teams:
+        if added_folders or added_teams or added_records:
             for sf in added_folders:
                 if sf.uid and sf.uid in shared_folders:
                     del shared_folders[sf.uid]
             for t in added_teams:
                 if t.uid and t.uid in teams:
                     del teams[t.uid]
+            replaced = {_share_record_key(record) for record in added_records}
+            existing_records = [record for record in existing_records if _share_record_key(record) not in replaced]
 
             memberships = []
             memberships.extend(shared_folders.values())
             memberships.extend(teams.values())
+            memberships.extend(existing_records)
             memberships.extend(added_folders)
             memberships.extend(added_teams)
+            memberships.extend(added_records)
             json_exporter = KeeperJsonExporter()
             json_exporter.do_export(file_name, memberships)
             if len(added_folders) > 0:
                 logging.info('%d shared folder memberships added.', len(added_folders))
             if len(added_teams) > 0:
                 logging.info('%d team memberships added.', len(added_teams))
+            if len(added_records) > 0:
+                logging.info('%d secret permission lists added.', len(added_records))
         else:
             logging.info('No folder memberships downloaded.')
 
@@ -611,13 +654,16 @@ def execute(self, params, **kwargs):
 
         shared_folders = []  # type: List[SharedFolder]
         teams = []     # type: List[Team]
+        secret_records = []  # type: List[Record]
 
         json_importer = KeeperJsonImporter()
         for obj in json_importer.do_import(file_name, users_only=True):
             if isinstance(obj, SharedFolder):
                 shared_folders.append(obj)
             if isinstance(obj, Team):
                 teams.append(obj)
+            if isinstance(obj, Record) and obj.shares:
+                secret_records.append(obj)
 
         full_sync = kwargs.get('full_sync') is True
         unsafe = kwargs.get('unsafe') is True
@@ -627,6 +673,9 @@ def execute(self, params, **kwargs):
         if len(teams) > 0:
             imp_exp.import_teams(params, teams, full_sync)
 
+        if len(secret_records) > 0:
+            imp_exp.apply_secret_share_records(params, secret_records)
+
 
 class DownloadRecordTypeCommand(EnterpriseCommand):
     def get_parser(self):
@@ -790,4 +839,4 @@ def importer_for_format(input_format):
             return module.Importer
         raise Exception('Cannot resolve importer for format {}'.format(input_format))
     except ModuleNotFoundError as e:
-        raise CommandError('', f'The required module is not installed:\n\tpip install {e.name}')
+        raise CommandError('', f'The required module is not installed:\n\tpip install {e.name}')