diff --git a/keepercommander/commands/helpers/team.py b/keepercommander/commands/helpers/team.py new file mode 100644 index 000000000..a42d984c8 --- /dev/null +++ b/keepercommander/commands/helpers/team.py @@ -0,0 +1,68 @@ +from ... import api, crypto, utils + + +class MissingTeamEncryptionKey(ValueError): + """The enterprise tree key or data key needed to create a team is missing.""" + + +def generate_team_add_key_fields(team_key, allow_rsa=True): + # type: (bytes, bool) -> dict + """Generate the EC (and, when allowed, RSA) key fields for a team_add request. + + The EC private key is encrypted with AES-v2 and the RSA private key with AES-v1, + both under ``team_key``. Every value is base64url-encoded. Returned names match + the request fields: ``ecc_private_key``, ``ecc_public_key``, and when RSA is + allowed ``private_key`` and ``public_key``. + """ + ec_private_key, ec_public_key = crypto.generate_ec_key() + fields = { + 'ecc_private_key': utils.base64_url_encode( + crypto.encrypt_aes_v2(crypto.unload_ec_private_key(ec_private_key), team_key)), + 'ecc_public_key': utils.base64_url_encode(crypto.unload_ec_public_key(ec_public_key)), + } + if allow_rsa: + rsa_private_key, rsa_public_key = crypto.generate_rsa_key() + fields['private_key'] = utils.base64_url_encode( + crypto.encrypt_aes_v1(crypto.unload_rsa_private_key(rsa_private_key), team_key)) + fields['public_key'] = utils.base64_url_encode(crypto.unload_rsa_public_key(rsa_public_key)) + return fields + + +def build_team_add_request(params, team_name, node_id, + restrict_edit=False, restrict_share=False, restrict_view=False, + manage_only=True, team_uid=None, team_key=None): + # type: (object, str, int, bool, bool, bool, bool, str, bytes) -> tuple + """Build a team_add request and return ``(request, team_key)``. + + ``team_uid`` and the team AES key are generated when the caller does not + supply them. The team key is encrypted with the data key (AES-v1) and the + enterprise tree key (AES-v2). RSA key fields are omitted when + ``params.forbid_rsa`` is set. Raises MissingTeamEncryptionKey when the tree + key or data key is missing. The returned team key is the raw AES key, for + callers that still need it to add members. + """ + tree_key = (getattr(params, 'enterprise', None) or {}).get('unencrypted_tree_key') + data_key = getattr(params, 'data_key', None) + if not tree_key or not data_key: + raise MissingTeamEncryptionKey('Enterprise tree key or data key is not available') + + if team_uid is None: + team_uid = api.generate_record_uid() + if team_key is None: + team_key = api.generate_aes_key() + + request = { + 'command': 'team_add', + 'team_uid': team_uid, + 'team_name': team_name, + 'restrict_edit': restrict_edit, + 'restrict_share': restrict_share, + 'restrict_view': restrict_view, + 'node_id': node_id, + 'team_key': utils.base64_url_encode(crypto.encrypt_aes_v1(team_key, data_key)), + 'encrypted_team_key': utils.base64_url_encode(crypto.encrypt_aes_v2(team_key, tree_key)), + 'manage_only': manage_only, + } + request.update(generate_team_add_key_fields( + team_key, allow_rsa=not getattr(params, 'forbid_rsa', False))) + return request, team_key \ No newline at end of file diff --git a/keepercommander/importer/commands.py b/keepercommander/importer/commands.py index d065c8776..268f02c04 100644 --- a/keepercommander/importer/commands.py +++ b/keepercommander/importer/commands.py @@ -136,6 +136,9 @@ def _positive_int_arg(value): help='Comma separated list of secret IDs to fetch (Thycotic)') import_parser.add_argument('--add-users', dest='add_users', action='store_true', help='provision source users as pre-activated Keeper enterprise users') +import_parser.add_argument('--add-teams', dest='add_teams', action='store_true', + help='create Keeper teams from source groups and add members ' + 'who already exist as Keeper users') import_parser.add_argument('--target-node', '--node', dest='target_node', action='store', help='node name or ID for CyberArk-provisioned users, teams, and roles (default: root node)') import_parser.add_argument('--skip', dest='skip', action='store', type=_cyberark_skip_arg, diff --git a/keepercommander/importer/imp_exp.py b/keepercommander/importer/imp_exp.py index c54eda32c..d6e7d7813 100644 --- a/keepercommander/importer/imp_exp.py +++ b/keepercommander/importer/imp_exp.py @@ -852,6 +852,7 @@ def _import(params, file_format, filename, **kwargs): skip_role = kwargs.get('skip_role') is True or 'role' in cyberark_skip skip_user = kwargs.get('skip_user') is True or 'user' in cyberark_skip add_users = kwargs.get('add_users') is True + add_teams = kwargs.get('add_teams') is True import_into_raw = kwargs.get('import_into') or '' import_into = import_into_raw @@ -864,6 +865,9 @@ def _import(params, file_format, filename, **kwargs): if add_users and not importer.support_add_users(): logging.warning('--add-users is not supported for --format=%s; ignoring', file_format) add_users = False + if add_teams and not importer.support_add_teams(): + logging.warning('--add-teams is not supported for --format=%s; ignoring', file_format) + add_teams = False verbose_import_summary = importer.verbose_import_summary show_skipped = show_skipped or verbose_import_summary @@ -884,7 +888,8 @@ def _import(params, file_format, filename, **kwargs): old_domain=old_domain, new_domain=new_domain, tmpdir=tmpdir, secret_ids=secret_ids, dry_run=dry_run, target_node=target_node, use_nsf=use_nsf, record_type=record_type, skip_team=skip_team, skip_role=skip_role, skip_user=skip_user, - add_users=add_users): + add_users=add_users, + add_teams=add_teams): if isinstance(x, ImportRecord): if filter_folder and not importer.support_folder_filter(): if not x.folders: diff --git a/keepercommander/importer/importer.py b/keepercommander/importer/importer.py index 99ee7ac07..304403125 100644 --- a/keepercommander/importer/importer.py +++ b/keepercommander/importer/importer.py @@ -352,6 +352,12 @@ def support_add_users(self): """ return False + def support_add_teams(self): + """ + Indicates if this importer can create Keeper teams from source groups + """ + return False + @staticmethod def import_field(field_type, field_value): # type: (str, str) -> any if not field_value: @@ -594,4 +600,4 @@ def download_membership(self, params, **kwargs): # type: (KeeperParams, ...) class BaseDownloadRecordType(abc.ABC): @abc.abstractmethod def download_record_type(self, params, **kwargs): # type: (KeeperParams, ...) -> Iterable[RecordType] - pass + pass \ No newline at end of file diff --git a/keepercommander/importer/thycotic/thycotic.py b/keepercommander/importer/thycotic/thycotic.py index 591bf335d..66a2b0a2a 100644 --- a/keepercommander/importer/thycotic/thycotic.py +++ b/keepercommander/importer/thycotic/thycotic.py @@ -25,6 +25,7 @@ from collections import deque from concurrent.futures import ThreadPoolExecutor from contextlib import contextmanager +from dataclasses import dataclass from urllib3.exceptions import InsecureRequestWarning import xml.etree.ElementTree as ET @@ -172,38 +173,33 @@ def url_from_tenant(tenant): # type: (Optional[str]) -> str return 'https://{0}/'.format(tenant) @staticmethod - def expand_shell_value(value, envvars): # type: (str, Optional[dict]) -> Optional[str] - """Replace ${NAME} from the Commander shell. Do not print the result.""" - from keepercommander.commands.base import expand_cmd_args, parameter_pattern - expanded = expand_cmd_args(value or '', envvars or {}) - missing = parameter_pattern.findall(expanded) - if missing: - logging.warning('Shell variable is not set: %s', ', '.join(missing)) - return None - return expanded - - @staticmethod - def connect_to_server(url=None, envvars=None): # type: (Optional[str], Optional[dict]) -> ThycoticAuth + def connect_to_server(url=None, username=None, password=None): # type: (Optional[str], Optional[str], Optional[str]) -> ThycoticAuth + url = url or os.getenv('KEEPER_THYCOTIC_URL') + username = username or os.getenv('KEEPER_THYCOTIC_USER') + password = password or os.getenv('KEEPER_THYCOTIC_PASSWORD') + if not url: url = input('...' + 'Thycotic Host or URL'.rjust(30) + ': ') if not url: logging.warning('Thycotic Host or URL is required') return - url = ThycoticMixin.expand_shell_value(url, envvars) - if not url: - return url = ThycoticMixin.url_from_tenant(url) if not url.startswith('http'): url = f'https://{url}' - username = input('...' + 'Thycotic Username'.rjust(30) + ': ') + if not username: - logging.warning('Thycotic username is required') - return - password = getpass.getpass(prompt='...' + 'Thycotic Password'.rjust(30) + ': ', stream=None) + username = input('...' + 'Thycotic Username'.rjust(30) + ': ') + if not username: + logging.warning('Thycotic username is required') + return + if not password: - logging.warning('Thycotic password is required') - return + password = getpass.getpass( + prompt='...' + 'Thycotic Password'.rjust(30) + ': ', stream=None) + if not password: + logging.warning('Thycotic password is required') + return def request_totp(): return input('...' + 'Enter TOTP Code'.rjust(30) + ': ') @@ -212,6 +208,7 @@ def request_totp(): auth.authenticate(username, password, on_totp=request_totp) return auth + @staticmethod def get_user_lookup(auth): # type: (ThycoticAuth) -> Optional[Dict[int, Dict]] try: @@ -237,14 +234,12 @@ def get_group_lookup(auth, on_progress=None): g = { 'name': group['name'], 'users': [], # type: List[int] - 'groups': [], # type: List[int] } if group.get('memberCount', 0) > 0: member_rs = auth.thycotic_search(f'/v1/groups/{group_id}/users') if on_progress: on_progress(i, total) g['users'].extend((x['userId'] for x in member_rs if 'userId' in x and x['userId'] > 0)) - g['groups'].extend((x['groupId'] for x in member_rs if 'groupId' in x and x['groupId'] > 0)) lookup[group_id] = g return lookup except: @@ -519,7 +514,7 @@ def get_folders(auth, skip_permissions=False): folder['inheritPermissions'] = True return folders - + @staticmethod def _secret_role_flags(role): @@ -952,6 +947,17 @@ def parse_thycotic_import_target(filename): return None return username, host +# Team create and member-add requests carry encrypted keys, so each execute payload stays small. +_TEAM_BATCH_SIZE = 25 + +@dataclass +class TeamMemberCounts: + """Import summary of how members were applied to one Keeper team.""" + added: int = 0 + queued: int = 0 + already: int = 0 + missing: int = 0 + failed: int = 0 class ThycoticImporter(BaseImporter, ThycoticMixin): # Same end-of-import terminal summary CyberArk prints (Import finished, skipped rows). @@ -964,10 +970,13 @@ def __init__(self): def support_folder_filter(self): return True - + def support_add_users(self): return True - + + def support_add_teams(self): + return True + @staticmethod def pop_field(fields, key): # type: (Dict[str, Dict[str, str]], str) -> Tuple[str, Optional[Dict[str, str]]] field = fields.pop(key, None) @@ -1013,11 +1022,12 @@ def _extract_user_email(user): return value.strip() return '' - def _prepare_thycotic_users(self, auth): # type: (ThycoticAuth) -> List[dict] + def _prepare_thycotic_users(self, auth, quiet=False): # type: (ThycoticAuth, bool) -> List[dict] """Fetch Delinea users that have an email address, including inactive users. Application accounts are omitted. When the user list has no email, the user detail endpoint is tried once. + quiet suppresses the summary line used while resolving group members. """ all_users = auth.thycotic_search('/v1/users?filter.includeInactive=true') result = [] @@ -1196,8 +1206,436 @@ def _migrate_thycotic_users(self, auth, params): logging.warning('Failed to fetch Thycotic users: %s', e) logging.warning('Continuing import without creating Keeper users.') return - report = self._create_enterprise_users(params, thycotic_users) - self._print_thycotic_user_report(report) + self._create_enterprise_users(params, thycotic_users) + + def _load_thycotic_groups(self, auth): # type: (ThycoticAuth) -> list + return auth.thycotic_search('/v1/groups') + + def _load_thycotic_user_email_lookup(self, auth): # type: (ThycoticAuth) -> Dict[int, str] + """Map Thycotic user ids to emails. A user-fetch failure continues with no emails.""" + try: + users = self._prepare_thycotic_users(auth, quiet=True) + except Exception as e: + logging.warning('Failed to fetch Thycotic users for group membership: %s', e) + users = [] + email_by_id = {} + for user in users or []: + if not isinstance(user, dict): + continue + user_id = user.get('id') + email = str(user.get('email') or '').strip() + if isinstance(user_id, int) and user_id > 0 and email: + email_by_id[user_id] = email + return email_by_id + + @staticmethod + def _select_thycotic_groups(groups): # type: (list) -> Tuple[List[dict], int, int] + """Keep enabled groups with an int id and a non-blank name, and count the skips.""" + included = [] + skipped_disabled = 0 + skipped_unnamed = 0 + for group in groups or []: + if not isinstance(group, dict): + continue + if group.get('enabled') is False: + skipped_disabled += 1 + continue + group_id = group.get('id') + name = str(group.get('name') or '').strip() + if not isinstance(group_id, int) or not name: + skipped_unnamed += 1 + continue + included.append({ + 'id': group_id, + 'name': name, + 'member_count': group.get('memberCount'), + }) + return included, skipped_disabled, skipped_unnamed + + @staticmethod + def _fetch_thycotic_group_member_ids(auth, group_id, name, member_count): + # type: (ThycoticAuth, int, str, Any) -> List + """Return direct member user ids. A failed member fetch aborts preparation.""" + user_ids = [] + if (member_count or 0) > 0: + try: + member_rs = auth.thycotic_search(f'/v1/groups/{group_id}/users') + except Exception as e: + raise Exception( + f'Failed to fetch members of Thycotic group "{name}": {e}' + ) from e + for member in member_rs or []: + if not isinstance(member, dict): + continue + user_ids.append(member.get('userId')) + return user_ids + + @staticmethod + def _map_thycotic_group_member_emails(name, user_ids, email_by_id): + # type: (str, list, Dict[int, str]) -> List[str] + member_emails = {} + unresolved = 0 + for user_id in user_ids: + email = email_by_id.get(user_id) or '' + if not email: + unresolved += 1 + continue + member_emails.setdefault(email.casefold(), email) + if unresolved: + logging.info( + 'Group "%s": %d member(s) have no email address and will not be added', + name, unresolved, + ) + return [member_emails[key] for key in sorted(member_emails)] + + def _assemble_thycotic_group_results(self, lookup, email_by_id, skipped_disabled, skipped_unnamed): + # type: (Dict[int, Dict], Dict[int, str], int, int) -> List[dict] + result = [] + for group_id, entry in lookup.items(): + result.append({ + 'id': group_id, + 'name': entry['name'], + 'members': self._map_thycotic_group_member_emails( + entry['name'], entry['users'], email_by_id, + ), + }) + if skipped_disabled: + logging.info('Skipped %d disabled Thycotic group(s)', skipped_disabled) + if skipped_unnamed: + logging.info('Skipped %d Thycotic group(s) with no name', skipped_unnamed) + logging.info(f'Found {len(result)} Thycotic group(s)') + return result + + def _prepare_thycotic_groups(self, auth): # type: (ThycoticAuth) -> List[dict] + """Fetch enabled Delinea groups and their direct user members. + Disabled groups are omitted. A failed member fetch aborts preparation + so Keeper teams are not created from an incomplete member list. + """ + groups = self._load_thycotic_groups(auth) + email_by_id = self._load_thycotic_user_email_lookup(auth) + included, skipped_disabled, skipped_unnamed = self._select_thycotic_groups(groups) + lookup = {} # type: Dict[int, Dict] + for group in included: + lookup[group['id']] = { + 'name': group['name'], + 'users': self._fetch_thycotic_group_member_ids( + auth, group['id'], group['name'], group['member_count'], + ), + } + return self._assemble_thycotic_group_results( + lookup, email_by_id, skipped_disabled, skipped_unnamed, + ) + + def _load_existing_keeper_teams(self, params): + # type: (KeeperParams) -> Tuple[Dict[str, dict], Dict[str, set], str] + """Return teams keyed by exact stripped name, member user ids keyed by team uid, and an error string.""" + try: + api.query_enterprise(params) + except Exception as e: + if not isinstance(getattr(params, 'enterprise', None), dict): + logging.warning('Error loading existing Keeper teams: %s', e) + return {}, {}, 'Failed to load existing Keeper teams' + logging.warning('Could not refresh Keeper teams: %s', e) + enterprise_data = getattr(params, 'enterprise', None) + if not isinstance(enterprise_data, dict): + return {}, {}, 'Failed to load existing Keeper teams' + + teams = {} + for source in ('queued_teams', 'teams'): + for team in enterprise_data.get(source) or []: + if not isinstance(team, dict) or not team.get('team_uid'): + continue + name = str(team.get('name') or '').strip() + if name: + teams[name] = team + + members = {} # type: Dict[str, set] + for team_user in enterprise_data.get('team_users') or []: + if not isinstance(team_user, dict): + continue + team_uid = team_user.get('team_uid') + user_id = team_user.get('enterprise_user_id') + if team_uid and user_id: + members.setdefault(team_uid, set()).add(user_id) + for queued in enterprise_data.get('queued_team_users') or []: + if not isinstance(queued, dict): + continue + team_uid = queued.get('team_uid') + if not team_uid: + continue + for user_id in queued.get('users') or []: + if user_id: + members.setdefault(team_uid, set()).add(user_id) + return teams, members, '' + + def _resolve_root_node_id(self, params): # type: (KeeperParams) -> Optional[int] + from ...commands.enterprise_common import EnterpriseCommand + try: + root_nodes = list(EnterpriseCommand.get_user_root_nodes(params)) + except Exception as e: + logging.warning('Failed to resolve the Keeper root node: %s', e) + return None + if not root_nodes: + logging.warning('No Keeper root node is available for new teams') + return None + return root_nodes[0] + + def _build_team_add_request(self, params, name, node_id): + # type: (KeeperParams, str, int) -> Tuple[dict, bytes] + """Build a team_add request in the same shape as enterprise-team --add.""" + from ...commands.helpers.team import build_team_add_request + return build_team_add_request(params, name, node_id) + + def _load_team_key(self, params, team_uid): # type: (KeeperParams, str) -> Optional[bytes] + try: + api.load_team_keys(params, [team_uid]) + except Exception as e: + logging.warning('Failed to load the key for Keeper team %s: %s', team_uid, e) + return None + keys = (getattr(params, 'key_cache', None) or {}).get(team_uid) + return getattr(keys, 'aes', None) or None + + def _add_team_members(self, params, team_uid, team_name, team_key, emails, already_member_ids, is_real_team): + # type: (KeeperParams, str, str, Optional[bytes], List[str], set, bool) -> TeamMemberCounts + """Add Keeper users to a team. Active users of a real team are keyed; everyone else is queued.""" + from ...commands.enterprise import EnterpriseTeamCommand + + users_by_email = {} + for user in ((getattr(params, 'enterprise', None) or {}).get('users') or []): + if not isinstance(user, dict): + continue + email = str(user.get('username') or user.get('email') or '').strip() + if email and user.get('enterprise_user_id'): + users_by_email.setdefault(email.casefold(), user) + + already = set(already_member_ids or []) + active_users = [] + queued_users = [] + missing = [] + already_count = 0 + for email in emails or []: + if not isinstance(email, str) or not email.strip(): + continue + user = users_by_email.get(email.strip().casefold()) + if not user: + missing.append(email.strip()) + continue + user_id = user.get('enterprise_user_id') + if user_id in already: + already_count += 1 + continue + already.add(user_id) + if is_real_team and user.get('status') == 'active': + active_users.append(user) + else: + queued_users.append(user) + + if missing: + logging.warning( + 'Team "%s": skipped %d member(s) who are not Keeper users', + team_name, len(missing), + ) + + if active_users and not team_key: + team_key = self._load_team_key(params, team_uid) + failed = 0 + if active_users and not team_key: + logging.warning('Cannot add active members to team "%s": team key is not available', team_name) + failed += len(active_users) + active_users = [] + + requests = [] + for user in queued_users: + requests.append({ + 'command': 'team_queue_user', + 'team_uid': team_uid, + 'enterprise_user_id': user.get('enterprise_user_id'), + }) + for user in active_users: + request = EnterpriseTeamCommand._create_add_user_request(params, team_uid, team_key, user, '') + if request: + requests.append(request) + else: + failed += 1 + + added = 0 + queued = 0 + for offset in range(0, len(requests), _TEAM_BATCH_SIZE): + chunk = requests[offset:offset + _TEAM_BATCH_SIZE] + try: + responses = api.execute_batch(params, chunk) or [] + except Exception as e: + logging.warning('Failed to add members to Keeper team "%s": %s', team_name, e) + failed += len(chunk) + continue + for index, request in enumerate(chunk): + response = responses[index] if index < len(responses) else None + # result "success" with an empty result_code is a completed add or queue. + succeeded = isinstance(response, dict) and response.get('result') == 'success' + if succeeded: + if request.get('command') == 'team_queue_user': + queued += 1 + else: + added += 1 + continue + failed += 1 + message = (response or {}).get('message') if isinstance(response, dict) else response + if request.get('command') == 'team_queue_user': + logging.warning('Failed to queue a member on team "%s": %s', team_name, message) + else: + logging.warning('Failed to add a member to team "%s": %s', team_name, message) + return TeamMemberCounts( + added=added, + queued=queued, + already=already_count, + missing=len(missing), + failed=failed, + ) + + def _create_enterprise_teams(self, params, groups): + # type: (KeeperParams, List[dict]) -> None + from ...commands.helpers.enterprise import is_valid_name_length + from ...commands.helpers.team import MissingTeamEncryptionKey + + existing_teams, members_by_team, error = self._load_existing_keeper_teams(params) + if error: + logging.warning(error) + logging.warning('Continuing import without creating Keeper teams.') + return + + pending = [] + existing = [] + seen = set() + for group in groups or []: + if not isinstance(group, dict): + continue + name = str(group.get('name') or '').strip() + # A case-only difference (Test123 vs test123) must still be attempted. + if not name or name in seen: + continue + seen.add(name) + members = [ + email.strip() for email in (group.get('members') or []) + if isinstance(email, str) and email.strip() + ] + team = existing_teams.get(name) + if team: + existing.append((team, members)) + continue + if not is_valid_name_length(name, 'Team name', 'import'): + continue + pending.append((name, members)) + + created = [] # type: List[Tuple[str, str, bytes, List[str]]] + if pending: + node_id = self._resolve_root_node_id(params) + tree_key = (getattr(params, 'enterprise', None) or {}).get('unencrypted_tree_key') + if node_id is None or not tree_key or not getattr(params, 'data_key', None): + logging.warning('Cannot create Keeper teams: root node or enterprise keys are not available') + else: + batch = [] + batch_meta = [] + for name, members in pending: + try: + request, team_key = self._build_team_add_request(params, name, node_id) + except MissingTeamEncryptionKey as e: + # The helper rejects every team when these keys are missing. + logging.warning('%s', e) + batch.clear() + batch_meta.clear() + break + except Exception as e: + logging.warning('Could not prepare Keeper team "%s": %s', name, e) + continue + batch.append(request) + batch_meta.append((name, request['team_uid'], team_key, members)) + for offset in range(0, len(batch), _TEAM_BATCH_SIZE): + chunk = batch[offset:offset + _TEAM_BATCH_SIZE] + chunk_meta = batch_meta[offset:offset + _TEAM_BATCH_SIZE] + try: + responses = api.execute_batch(params, chunk) or [] + except Exception as e: + logging.warning('Failed to create Keeper teams: %s', e) + responses = [] + for index, meta in enumerate(chunk_meta): + name, team_uid, team_key, members = meta + response = responses[index] if index < len(responses) else None + if isinstance(response, dict) and response.get('result') == 'success': + created.append(meta) + logging.info('\'%s\' team is created', name) + else: + message = response.get('message') if isinstance(response, dict) else response + logging.warning('\'%s\' team is not created: %s', name, message or 'no response') + + rows = [] + for name, team_uid, team_key, members in created: + counts = self._add_team_members( + params, team_uid, name, team_key, members, set(), True) + rows.append(self._team_result_row(name, 'created', counts)) + for team, members in existing: + team_uid = team.get('team_uid') + team_name = team.get('name') or team_uid + is_real_team = 'restrict_edit' in team + counts = self._add_team_members( + params, team_uid, team_name, None, members, + members_by_team.get(team_uid) or set(), is_real_team) + rows.append(self._team_result_row(team_name, 'already exists', counts)) + if rows: + logging.info(tabulate(rows, headers='keys')) + + @staticmethod + def _team_result_row(name, result, counts): + # type: (str, str, TeamMemberCounts) -> dict + return { + 'Team': name, + 'Result': result, + 'Added': counts.added, + 'Queued': counts.queued, + 'Already on team': counts.already, + 'Not in Keeper': counts.missing, + 'Failed': counts.failed, + } + + def _preview_thycotic_teams(self, auth): + # type: (ThycoticAuth) -> None + """Fetch Delinea groups and print them. Does not create Keeper teams.""" + logging.info('Dry run: fetching Thycotic groups (no Keeper teams will be created)...') + try: + groups = self._prepare_thycotic_groups(auth) + except Exception as e: + logging.warning('Failed to fetch Thycotic groups: %s', e) + return + if not groups: + logging.info('No Thycotic groups are available to migrate.') + return + rows = [ + { + 'Group': group.get('name') or '', + 'Members': ', '.join(group.get('members') or []) or '-', + } + for group in groups + ] + logging.info(f'Thycotic groups available to migrate ({len(rows)}):') + logging.info(tabulate(rows, headers='keys')) + + def _migrate_thycotic_teams(self, auth, params): + # type: (ThycoticAuth, KeeperParams) -> None + """Create Keeper teams from Delinea groups and add members who already exist in Keeper. + + A group-fetch failure does not abort secret migration. + Run this after user provisioning so newly created users can be added. + """ + logging.info('Migrating Thycotic groups to Keeper teams...') + try: + groups = self._prepare_thycotic_groups(auth) + except Exception as e: + logging.warning('Failed to fetch Thycotic groups: %s', e) + logging.warning('Continuing import without creating Keeper teams.') + return + if not groups: + logging.info('No Thycotic groups are available to migrate.') + return + self._create_enterprise_teams(params, groups) def do_import(self, filename, **kwargs): # type: (ThycoticImporter, str, ...) -> Iterable[Union[Record, SharedFolder]] @@ -1246,6 +1684,13 @@ def do_import(self, filename, **kwargs): else: self._migrate_thycotic_users(auth, params) + # Teams are created after users so members provisioned above can be added. + if kwargs.get('add_teams') is True: + if kwargs.get('dry_run') is True: + self._preview_thycotic_teams(auth) + else: + self._migrate_thycotic_teams(auth, params) + totp_codes = {} csv_file = 'secrets-export.csv' if not os.path.isfile('secrets-export.csv'): @@ -1329,7 +1774,7 @@ def do_import(self, filename, **kwargs): found_secrets = [x for x in stringified_debug_ids if x in stringified_secrets_ids] logging.info(f'From the specified {len(debug_ids)} secret IDs, {len(found_secrets)} were found in the secret server lookup.') logging.info(', '.join(found_secrets)) - + # Replace import list with specified IDs secrets_ids = debug_ids else: @@ -2000,8 +2445,7 @@ def prepare(self): class ThycoticMembershipDownload(BaseDownloadMembership, ThycoticMixin): def download_membership(self, params, **kwargs): - envvars = getattr(params, 'environment_variables', None) if params is not None else None - auth = ThycoticMixin.connect_to_server(envvars=envvars) + auth = ThycoticMixin.connect_to_server() if not auth: return @@ -2012,12 +2456,7 @@ def download_membership(self, params, **kwargs): for folder in folders.values(): if folder.get('inheritPermissions', True): continue - if 'permissions' not in folder: - continue - permissions = folder['permissions'] - if not isinstance(permissions, list): - continue - if len(permissions) == 0: + if 'permissions' not in folder or not isinstance(folder['permissions'], list): continue shared_folder = SharedFolder() @@ -2053,8 +2492,7 @@ def download_membership(self, params, **kwargs): sf_groups.add(group_id) yield shared_folder - folders_only = kwargs.get('folders_only') is True - if folders_only is True: + if kwargs.get('folders_only') is True: return if len(sf_groups) > 0: @@ -2091,8 +2529,7 @@ def progress(position, total): class ThycoticRecordTypeDownload(BaseDownloadRecordType, ThycoticMixin): def download_record_type(self, params, **kwargs): - envvars = getattr(params, 'environment_variables', None) if params is not None else None - auth = ThycoticMixin.connect_to_server(envvars=envvars) + auth = ThycoticMixin.connect_to_server() if not auth: return templates = auth.thycotic_search('/v1/secret-templates?filter.includeInactive=false&filter.includeSecretCount=true') @@ -2330,4 +2767,4 @@ def download_record_type(self, params, **kwargs): rf = RecordTypeField.create('fileRef', '') rt.fields.append(rf) - yield rt \ No newline at end of file + yield rt diff --git a/unit-tests/test_thycotic_add_teams.py b/unit-tests/test_thycotic_add_teams.py new file mode 100644 index 000000000..14eae65e2 --- /dev/null +++ b/unit-tests/test_thycotic_add_teams.py @@ -0,0 +1,729 @@ +import logging +import unittest +from types import SimpleNamespace +from unittest import mock + +from keepercommander.commands.helpers.team import MissingTeamEncryptionKey +from keepercommander.importer.commands import import_parser +from keepercommander.importer.thycotic.thycotic import ( + TeamMemberCounts, ThycoticImporter, _TEAM_BATCH_SIZE, +) + + +class _QuietTestCase(unittest.TestCase): + def setUp(self): + self._patch('builtins.print') + self._patch('keepercommander.importer.thycotic.thycotic.logging') + self.log = self._mocks['logging'] + + def _patch(self, target, **kwargs): + patcher = mock.patch(target, **kwargs) + name = target.rsplit('.', 1)[-1] + self.__dict__.setdefault('_mocks', {})[name] = patcher.start() + self.addCleanup(patcher.stop) + + def importer(self): + return ThycoticImporter() + + +class TestPrepareThycoticGroups(_QuietTestCase): + def _groups(self, groups, members=None, users=None, user_error=None): + auth = mock.MagicMock() + + def search(endpoint): + endpoint = str(endpoint) + if endpoint == '/v1/groups': + return groups + return (members or {}).get(endpoint, []) + + auth.thycotic_search.side_effect = search + importer = self.importer() + if user_error is not None: + importer._prepare_thycotic_users = mock.Mock(side_effect=user_error) + else: + importer._prepare_thycotic_users = mock.Mock(return_value=users or []) + found = importer._prepare_thycotic_groups(auth) + return found, importer + + def test_uses_direct_members_and_skips_disabled(self): + found, importer = self._groups( + [ + {'id': 10, 'name': 'Admins', 'enabled': True, 'memberCount': 2}, + {'id': 11, 'name': 'Empty', 'enabled': True, 'memberCount': 0}, + {'id': 12, 'name': 'Disabled', 'enabled': False, 'memberCount': 1}, + {'id': 13, 'name': ' Nested ', 'enabled': True, 'memberCount': 1}, + {'name': 'no-id', 'memberCount': 1}, + 'junk', + ], + members={ + '/v1/groups/10/users': [ + {'userId': 1, 'groupId': 0}, + {'userId': 2}, + {'userId': 9}, + ], + '/v1/groups/13/users': [{'groupId': 10, 'userId': 0}, {'userId': 2}], + }, + users=[ + {'id': 1, 'email': 'a@x.com'}, + {'id': 2, 'email': 'b@x.com'}, + ], + ) + importer._prepare_thycotic_users.assert_called_once_with(mock.ANY, quiet=True) + self.assertEqual( + [(g['name'], g['members']) for g in found], + [ + ('Admins', ['a@x.com', 'b@x.com']), + ('Empty', []), + ('Nested', ['b@x.com']), + ], + ) + self.log.info.assert_called() + + def test_same_names_stay_separate_groups(self): + found, _ = self._groups( + [ + {'id': 1, 'name': 'Ops', 'memberCount': 1}, + {'id': 2, 'name': 'ops', 'memberCount': 1}, + ], + members={ + '/v1/groups/1/users': [{'userId': 1}], + '/v1/groups/2/users': [{'userId': 2}], + }, + users=[{'id': 1, 'email': 'a@x.com'}, {'id': 2, 'email': 'B@x.com'}], + ) + self.assertEqual(found, [ + {'id': 1, 'name': 'Ops', 'members': ['a@x.com']}, + {'id': 2, 'name': 'ops', 'members': ['B@x.com']}, + ]) + + def test_member_fetch_failure_stops_preparation(self): + auth = mock.MagicMock() + error = RuntimeError('403') + + def search(endpoint): + endpoint = str(endpoint) + if endpoint == '/v1/groups': + return [ + {'id': 1, 'name': 'Admins', 'memberCount': 1}, + {'id': 3, 'name': 'Ops', 'memberCount': 4}, + ] + if endpoint == '/v1/groups/1/users': + return [{'userId': 1}] + raise error + + auth.thycotic_search.side_effect = search + importer = self.importer() + importer._prepare_thycotic_users = mock.Mock(return_value=[ + {'id': 1, 'email': 'a@x.com'}, + ]) + importer._create_enterprise_teams = mock.Mock() + + with self.assertRaises(Exception) as caught: + importer._prepare_thycotic_groups(auth) + self.assertIn('Ops', str(caught.exception)) + self.assertIn('403', str(caught.exception)) + self.assertIs(caught.exception.__cause__, error) + + importer._migrate_thycotic_teams(auth, object()) + importer._create_enterprise_teams.assert_not_called() + self.log.warning.assert_any_call( + 'Continuing import without creating Keeper teams.', + ) + fetch_failures = [ + call.args[1] for call in self.log.warning.call_args_list + if call.args and call.args[0] == 'Failed to fetch Thycotic groups: %s' + ] + self.assertTrue(fetch_failures) + self.assertIn('Ops', str(fetch_failures[-1])) + self.assertIn('403', str(fetch_failures[-1])) + + importer._preview_thycotic_teams(auth) + importer._create_enterprise_teams.assert_not_called() + info = ' '.join(str(call.args[0]) for call in self.log.info.call_args_list) + self.assertNotIn('Thycotic groups available to migrate', info) + + def test_user_lookup_failure_still_returns_groups(self): + found, _ = self._groups( + [{'id': 3, 'name': 'Ops', 'memberCount': 0}], + user_error=RuntimeError('users down'), + ) + self.assertEqual(found[0]['name'], 'Ops') + self.log.warning.assert_called() + + +class TestLoadExistingKeeperTeams(_QuietTestCase): + def setUp(self): + super().setUp() + self._patch('keepercommander.importer.thycotic.thycotic.api.query_enterprise') + self.query = self._mocks['query_enterprise'] + + def test_active_team_replaces_queued_team_of_the_same_name(self): + params = SimpleNamespace(enterprise={ + 'queued_teams': [{'name': 'Ops', 'team_uid': 'queued'}], + 'teams': [ + {'name': ' Ops ', 'team_uid': 'real', 'restrict_edit': False}, + {'name': 'ops', 'team_uid': 'lower', 'restrict_edit': False}, + {'name': 'Empty', 'team_uid': ''}, + ], + 'team_users': [ + {'team_uid': 'real', 'enterprise_user_id': 7}, + {'team_uid': 'real', 'enterprise_user_id': None}, + 'junk', + ], + 'queued_team_users': [{'team_uid': 'queued', 'users': [8, 0]}], + }) + teams, members, error = self.importer()._load_existing_keeper_teams(params) + self.assertEqual(error, '') + self.assertEqual(teams['Ops']['team_uid'], 'real') + self.assertEqual(teams['ops']['team_uid'], 'lower') + self.assertEqual(members, {'real': {7}, 'queued': {8}}) + self.query.assert_called_once() + + def test_query_failure_returns_error(self): + self.query.side_effect = RuntimeError('not an admin') + teams, members, error = self.importer()._load_existing_keeper_teams(SimpleNamespace(enterprise=None)) + self.assertEqual((teams, members, bool(error)), ({}, {}, True)) + + +class TestBuildTeamAddRequest(_QuietTestCase): + def _params(self, **extra): + params = { + 'enterprise': {'unencrypted_tree_key': b'tree'}, + 'data_key': b'data', + 'forbid_rsa': False, + } + params.update(extra) + return SimpleNamespace(**params) + + def _patches(self): + targets = { + 'keepercommander.commands.helpers.team.api.generate_record_uid': {'return_value': 'uid-1'}, + 'keepercommander.commands.helpers.team.api.generate_aes_key': {'return_value': b'k' * 32}, + 'keepercommander.commands.helpers.team.crypto.encrypt_aes_v1': {'return_value': b'v1'}, + 'keepercommander.commands.helpers.team.crypto.encrypt_aes_v2': {'return_value': b'v2'}, + 'keepercommander.commands.helpers.team.crypto.generate_ec_key': {'return_value': ('ec-priv', 'ec-pub')}, + 'keepercommander.commands.helpers.team.crypto.unload_ec_private_key': {'return_value': b'ec-priv'}, + 'keepercommander.commands.helpers.team.crypto.unload_ec_public_key': {'return_value': b'ec-pub'}, + 'keepercommander.commands.helpers.team.crypto.generate_rsa_key': {'return_value': ('rsa-priv', 'rsa-pub')}, + 'keepercommander.commands.helpers.team.crypto.unload_rsa_private_key': {'return_value': b'rsa-priv'}, + 'keepercommander.commands.helpers.team.crypto.unload_rsa_public_key': {'return_value': b'rsa-pub'}, + 'keepercommander.commands.helpers.team.utils.base64_url_encode': { + 'side_effect': lambda value: 'b64-' + value.decode()}, + } + started = {} + for target, kwargs in targets.items(): + patcher = mock.patch(target, **kwargs) + started[target.rsplit('.', 1)[-1]] = patcher.start() + self.addCleanup(patcher.stop) + return started + + def test_request_matches_enterprise_team_add(self): + self._patches() + request, team_key = self.importer()._build_team_add_request(self._params(), 'Admins', 9) + self.assertEqual(team_key, b'k' * 32) + self.assertEqual(request['command'], 'team_add') + self.assertEqual(request['team_uid'], 'uid-1') + self.assertEqual(request['team_name'], 'Admins') + self.assertEqual(request['node_id'], 9) + self.assertTrue(request['manage_only']) + self.assertIn('public_key', request) + self.assertIn('private_key', request) + self.assertIn('ecc_public_key', request) + + def test_rsa_keys_omitted_when_forbidden(self): + patched = self._patches() + request, _ = self.importer()._build_team_add_request(self._params(forbid_rsa=True), 'Admins', 9) + self.assertNotIn('public_key', request) + self.assertNotIn('private_key', request) + patched['generate_rsa_key'].assert_not_called() + + def test_missing_keys_raise(self): + with self.assertRaises(MissingTeamEncryptionKey) as caught: + self.importer()._build_team_add_request(SimpleNamespace(enterprise={}, data_key=None), 'Admins', 9) + self.assertIsInstance(caught.exception, ValueError) + self.assertEqual(str(caught.exception), 'Enterprise tree key or data key is not available') + + +class TestAddTeamMembers(_QuietTestCase): + def _params(self): + return SimpleNamespace(enterprise={'users': [ + {'username': 'A@x.com', 'enterprise_user_id': 1, 'status': 'active'}, + {'username': 'b@x.com', 'enterprise_user_id': 2, 'status': 'invited'}, + {'username': 'c@x.com', 'enterprise_user_id': 3, 'status': 'active'}, + ]}) + + def test_active_users_are_added_and_invited_users_are_queued(self): + importer = self.importer() + with mock.patch( + 'keepercommander.commands.enterprise.EnterpriseTeamCommand._create_add_user_request', + return_value={'command': 'team_enterprise_user_add', 'enterprise_user_id': 1}, + ) as add_user, mock.patch( + 'keepercommander.importer.thycotic.thycotic.api.execute_batch', + return_value=[{'result': 'success'}, {'result': 'success'}], + ) as batch: + counts = importer._add_team_members( + self._params(), 'uid', 'Ops', b'key', + ['a@x.com', 'b@x.com', 'c@x.com', 'missing@x.com', ''], + {3}, True, + ) + self.assertEqual(counts, TeamMemberCounts(added=1, queued=1, already=1, missing=1, failed=0)) + queued, active = batch.call_args[0][1] + self.assertEqual(queued['command'], 'team_queue_user') + self.assertEqual(queued['enterprise_user_id'], 2) + self.assertEqual(active['command'], 'team_enterprise_user_add') + add_user.assert_called_once() + self.log.warning.assert_called() + + def test_queued_team_does_not_encrypt_a_team_key(self): + importer = self.importer() + with mock.patch( + 'keepercommander.commands.enterprise.EnterpriseTeamCommand._create_add_user_request', + ) as add_user, mock.patch( + 'keepercommander.importer.thycotic.thycotic.api.execute_batch', + return_value=[{'result': 'success'}], + ) as batch: + counts = importer._add_team_members( + self._params(), 'uid', 'Ops', None, ['a@x.com'], set(), False, + ) + add_user.assert_not_called() + self.assertEqual(batch.call_args[0][1][0]['command'], 'team_queue_user') + self.assertEqual(counts.queued, 1) + + def test_captured_success_counts_as_added_and_duplicate_comes_from_membership(self): + importer = self.importer() + captured = { + 'result': 'success', + 'result_code': '', + 'message': '', + 'command': 'team_enterprise_user_add', + 'revision': 4027138144, + } + queued_success = { + 'result': 'success', + 'result_code': '', + 'message': '', + 'command': 'team_queue_user', + 'revision': 4027138145, + } + with mock.patch( + 'keepercommander.commands.enterprise.EnterpriseTeamCommand._create_add_user_request', + return_value={'command': 'team_enterprise_user_add', 'enterprise_user_id': 1}, + ), mock.patch( + 'keepercommander.importer.thycotic.thycotic.api.execute_batch', + side_effect=[ + [captured], + [queued_success], + [{ + 'result': 'fail', + 'result_code': 'exists', + 'message': 'The object you are trying to create already exists.', + }], + [{ + 'result': 'fail', + 'message': 'The object you are trying to create already exists.', + }], + ], + ) as batch: + added = importer._add_team_members( + self._params(), 'uid', 'Ops', b'key', ['a@x.com'], set(), True, + ) + queued = importer._add_team_members( + self._params(), 'uid', 'Ops', None, ['b@x.com'], set(), False, + ) + coded_failure = importer._add_team_members( + self._params(), 'uid', 'Ops', b'key', ['a@x.com'], set(), True, + ) + message_failure = importer._add_team_members( + self._params(), 'uid', 'Ops', b'key', ['a@x.com'], set(), True, + ) + already = importer._add_team_members( + self._params(), 'uid', 'Ops', b'key', ['a@x.com'], {1}, True, + ) + self.assertEqual(added, TeamMemberCounts(added=1, queued=0, already=0, missing=0, failed=0)) + self.assertEqual(queued, TeamMemberCounts(added=0, queued=1, already=0, missing=0, failed=0)) + self.assertEqual(coded_failure, TeamMemberCounts(added=0, queued=0, already=0, missing=0, failed=1)) + self.assertEqual(message_failure, TeamMemberCounts(added=0, queued=0, already=0, missing=0, failed=1)) + self.assertEqual(already, TeamMemberCounts(added=0, queued=0, already=1, missing=0, failed=0)) + self.assertEqual(batch.call_count, 4) + self.assertEqual( + [call.args[0] for call in self.log.warning.call_args_list], + [ + 'Failed to add a member to team "%s": %s', + 'Failed to add a member to team "%s": %s', + ], + ) + + def test_missing_team_key_counts_as_failed(self): + importer = self.importer() + importer._load_team_key = mock.Mock(return_value=None) + with mock.patch('keepercommander.importer.thycotic.thycotic.api.execute_batch') as batch: + counts = importer._add_team_members( + self._params(), 'uid', 'Ops', None, ['a@x.com'], set(), True, + ) + batch.assert_not_called() + self.assertEqual(counts.failed, 1) + self.log.warning.assert_called() + + def test_member_adds_are_chunked_and_counts_stay_aligned(self): + self.assertEqual(_TEAM_BATCH_SIZE, 25) + importer = self.importer() + params = SimpleNamespace(enterprise={'users': [ + {'username': 'q1@x.com', 'enterprise_user_id': 11, 'status': 'invited'}, + {'username': 'q2@x.com', 'enterprise_user_id': 12, 'status': 'invited'}, + {'username': 'q3@x.com', 'enterprise_user_id': 13, 'status': 'invited'}, + {'username': 'q4@x.com', 'enterprise_user_id': 14, 'status': 'invited'}, + {'username': 'a1@x.com', 'enterprise_user_id': 21, 'status': 'active'}, + {'username': 'a2@x.com', 'enterprise_user_id': 22, 'status': 'active'}, + {'username': 'a3@x.com', 'enterprise_user_id': 23, 'status': 'active'}, + ]}) + + def add_user(_params, _team_uid, _team_key, user, _role): + return { + 'command': 'team_enterprise_user_add', + 'enterprise_user_id': user.get('enterprise_user_id'), + } + + chunks = [] + + def execute(_params, chunk): + chunks.append(list(chunk)) + if len(chunks) == 1: + return [ + {'result': 'success', 'result_code': '', 'message': ''}, + {'result': 'fail', 'result_code': 'exists', 'message': ''}, + ] + if len(chunks) == 2: + raise RuntimeError('timeout') + if len(chunks) == 3: + return [{'result': 'success'}] + return [{'result': 'success'}] + + with mock.patch( + 'keepercommander.commands.enterprise.EnterpriseTeamCommand._create_add_user_request', + side_effect=add_user, + ), mock.patch( + 'keepercommander.importer.thycotic.thycotic._TEAM_BATCH_SIZE', 2, + ), mock.patch( + 'keepercommander.importer.thycotic.thycotic.api.execute_batch', + side_effect=execute, + ) as batch: + counts = importer._add_team_members( + params, 'uid', 'Ops', b'key', + ['q1@x.com', 'a1@x.com', 'q2@x.com', 'q3@x.com', 'a2@x.com', 'q4@x.com', 'a3@x.com'], + set(), True, + ) + + self.assertEqual(batch.call_count, 4) + self.assertEqual( + [[request['enterprise_user_id'] for request in chunk] for chunk in chunks], + [[11, 12], [13, 14], [21, 22], [23]], + ) + self.assertTrue(all(len(chunk) <= 2 for chunk in chunks)) + self.assertEqual(counts, TeamMemberCounts(added=2, queued=1, already=0, missing=0, failed=4)) + warnings = [call.args[0] for call in self.log.warning.call_args_list] + self.assertEqual(warnings.count('Failed to add members to Keeper team "%s": %s'), 1) + self.assertEqual(warnings.count('Failed to queue a member on team "%s": %s'), 1) + self.assertEqual(warnings.count('Failed to add a member to team "%s": %s'), 1) + + +class TestCreateEnterpriseTeams(_QuietTestCase): + def _importer(self, existing=None, error=''): + importer = self.importer() + importer._load_existing_keeper_teams = mock.Mock(return_value=(existing or {}, {}, error)) + importer._resolve_root_node_id = mock.Mock(return_value=5) + importer._build_team_add_request = mock.Mock( + return_value=({'command': 'team_add', 'team_uid': 'uid-1'}, b'key')) + importer._add_team_members = mock.Mock(return_value=TeamMemberCounts( + added=1, queued=0, already=0, missing=0, failed=0, + )) + return importer + + def _params(self): + return SimpleNamespace(enterprise={'unencrypted_tree_key': b'tree'}, data_key=b'data') + + def test_creates_a_team_then_adds_members(self): + importer = self._importer() + with mock.patch( + 'keepercommander.importer.thycotic.thycotic.api.execute_batch', + return_value=[{'result': 'success'}, {'result': 'success'}], + ) as batch: + importer._create_enterprise_teams(self._params(), [ + {'name': 'Admins', 'members': [' a@x.com ']}, + {'name': 'admins', 'members': ['other@x.com']}, + {'name': ' Admins ', 'members': ['dup@x.com']}, + {'name': ' ', 'members': ['x@x.com']}, + None, + ]) + batch.assert_called_once() + created = batch.call_args[0][1] + self.assertEqual([request['command'] for request in created], ['team_add', 'team_add']) + self.assertEqual( + [call.args[1] for call in importer._build_team_add_request.call_args_list], + ['Admins', 'admins'], + ) + self.assertEqual( + [call.args[2] for call in importer._add_team_members.call_args_list], + ['Admins', 'admins'], + ) + self.assertEqual(importer._add_team_members.call_args_list[0].args[4], ['a@x.com']) + self.assertEqual(importer._add_team_members.call_args_list[1].args[4], ['other@x.com']) + + def test_existing_team_is_not_created_and_gains_missing_members(self): + importer = self._importer(existing={ + 'Ops': {'name': 'Ops', 'team_uid': 'uid-ops', 'restrict_edit': False}, + }) + importer._load_existing_keeper_teams.return_value = ( + {'Ops': {'name': 'Ops', 'team_uid': 'uid-ops', 'restrict_edit': False}}, + {'uid-ops': {4}}, + '', + ) + with mock.patch('keepercommander.importer.thycotic.thycotic.api.execute_batch') as batch: + importer._create_enterprise_teams(self._params(), [{'name': ' Ops ', 'members': ['a@x.com']}]) + batch.assert_not_called() + importer._build_team_add_request.assert_not_called() + importer._add_team_members.assert_called_once_with( + mock.ANY, 'uid-ops', 'Ops', None, ['a@x.com'], {4}, True) + + def test_case_distinct_group_is_created_beside_existing_team(self): + importer = self._importer() + importer._load_existing_keeper_teams.return_value = ( + {'Ops': {'name': 'Ops', 'team_uid': 'uid-ops', 'restrict_edit': False}}, + {'uid-ops': {4}}, + '', + ) + with mock.patch( + 'keepercommander.importer.thycotic.thycotic.api.execute_batch', + return_value=[{'result': 'success'}], + ) as batch: + importer._create_enterprise_teams(self._params(), [ + {'name': 'ops', 'members': ['new@x.com']}, + {'name': 'Ops', 'members': ['old@x.com']}, + ]) + importer._build_team_add_request.assert_called_once_with(mock.ANY, 'ops', 5) + self.assertEqual(len(batch.call_args[0][1]), 1) + self.assertEqual( + [(call.args[2], call.args[4]) for call in importer._add_team_members.call_args_list], + [('ops', ['new@x.com']), ('Ops', ['old@x.com'])], + ) + + def test_team_creates_are_chunked_and_a_failed_chunk_does_not_stop_later_teams(self): + importer = self._importer() + names = ['TeamA', 'TeamB', 'TeamC', 'TeamD', 'TeamE'] + + def build(_params, name, _node_id): + return ({'command': 'team_add', 'team_uid': 'uid-' + name, 'team_name': name}, b'key') + + importer._build_team_add_request.side_effect = build + chunks = [] + + def execute(_params, chunk): + chunks.append(list(chunk)) + if len(chunks) == 1: + raise RuntimeError('timeout') + if len(chunks) == 2: + return [{'result': 'success'}] + return [{'result': 'success'}] + + groups = [{'name': name, 'members': [name.lower() + '@x.com']} for name in names] + with mock.patch( + 'keepercommander.importer.thycotic.thycotic._TEAM_BATCH_SIZE', 2, + ), mock.patch( + 'keepercommander.importer.thycotic.thycotic.api.execute_batch', + side_effect=execute, + ) as batch: + importer._create_enterprise_teams(self._params(), groups) + + self.assertEqual(batch.call_count, 3) + self.assertEqual( + [[request['team_name'] for request in chunk] for chunk in chunks], + [['TeamA', 'TeamB'], ['TeamC', 'TeamD'], ['TeamE']], + ) + self.assertTrue(all(len(chunk) <= 2 for chunk in chunks)) + self.assertEqual( + [(call.args[1], call.args[2], call.args[4]) for call in importer._add_team_members.call_args_list], + [('uid-TeamC', 'TeamC', ['teamc@x.com']), ('uid-TeamE', 'TeamE', ['teame@x.com'])], + ) + warnings = [call.args[0] for call in self.log.warning.call_args_list] + self.assertEqual(warnings.count('Failed to create Keeper teams: %s'), 1) + self.assertNotIn('Continuing import without creating Keeper teams.', warnings) + self.assertEqual(warnings.count('\'%s\' team is not created: %s'), 3) + + def test_missing_encryption_key_stops_remaining_teams(self): + importer = self._importer() + + def build(_params, name, _node_id): + if name == 'Admins': + raise MissingTeamEncryptionKey('Enterprise tree key or data key is not available') + return ({'command': 'team_add', 'team_uid': 'uid-' + name, 'team_name': name}, b'key') + + importer._build_team_add_request.side_effect = build + with mock.patch('keepercommander.importer.thycotic.thycotic.api.execute_batch') as batch: + importer._create_enterprise_teams(self._params(), [ + {'name': 'Admins', 'members': ['a@x.com']}, + {'name': 'Ops', 'members': ['b@x.com']}, + {'name': 'Dev', 'members': ['c@x.com']}, + ]) + self.assertEqual( + [call.args[1] for call in importer._build_team_add_request.call_args_list], + ['Admins'], + ) + batch.assert_not_called() + importer._add_team_members.assert_not_called() + self.assertEqual(self.log.warning.call_count, 1) + warning_format, warning_error = self.log.warning.call_args.args + self.assertEqual(warning_format, '%s') + self.assertIsInstance(warning_error, MissingTeamEncryptionKey) + self.assertEqual(str(warning_error), 'Enterprise tree key or data key is not available') + + def test_other_prepare_error_skips_only_that_team(self): + importer = self._importer() + + def build(_params, name, _node_id): + if name == 'Admins': + raise ValueError('team uid could not be generated') + return ({'command': 'team_add', 'team_uid': 'uid-' + name, 'team_name': name}, b'key') + + importer._build_team_add_request.side_effect = build + with mock.patch( + 'keepercommander.importer.thycotic.thycotic.api.execute_batch', + return_value=[{'result': 'success'}], + ) as batch: + importer._create_enterprise_teams(self._params(), [ + {'name': 'Admins', 'members': ['a@x.com']}, + {'name': 'Ops', 'members': ['b@x.com']}, + ]) + self.assertEqual( + [call.args[1] for call in importer._build_team_add_request.call_args_list], + ['Admins', 'Ops'], + ) + self.assertEqual( + [request['team_name'] for request in batch.call_args[0][1]], + ['Ops'], + ) + importer._add_team_members.assert_called_once() + self.assertEqual(importer._add_team_members.call_args.args[2], 'Ops') + self.assertEqual(importer._add_team_members.call_args.args[4], ['b@x.com']) + self.assertEqual(self.log.warning.call_count, 1) + warning_format, team_name, warning_error = self.log.warning.call_args.args + self.assertEqual(warning_format, 'Could not prepare Keeper team "%s": %s') + self.assertEqual(team_name, 'Admins') + self.assertIsInstance(warning_error, ValueError) + self.assertNotIsInstance(warning_error, MissingTeamEncryptionKey) + self.assertEqual(str(warning_error), 'team uid could not be generated') + + def test_failed_create_does_not_add_members(self): + importer = self._importer() + with mock.patch( + 'keepercommander.importer.thycotic.thycotic.api.execute_batch', + return_value=[{'result': 'fail', 'message': 'exists'}], + ): + importer._create_enterprise_teams(self._params(), [{'name': 'Admins', 'members': ['a@x.com']}]) + importer._add_team_members.assert_not_called() + self.log.warning.assert_called() + + def test_loader_error_skips_team_creation(self): + importer = self._importer(error='Failed to load existing Keeper teams') + with mock.patch('keepercommander.importer.thycotic.thycotic.api.execute_batch') as batch: + importer._create_enterprise_teams(self._params(), [{'name': 'Admins', 'members': []}]) + batch.assert_not_called() + importer._add_team_members.assert_not_called() + + def test_missing_root_node_skips_create(self): + importer = self._importer() + importer._resolve_root_node_id.return_value = None + with mock.patch('keepercommander.importer.thycotic.thycotic.api.execute_batch') as batch: + importer._create_enterprise_teams(self._params(), [{'name': 'Admins', 'members': []}]) + batch.assert_not_called() + self.log.warning.assert_called() + + +class TestMigrateTeamsDoesNotAbortImport(_QuietTestCase): + def test_fetch_failure_is_logged_and_swallowed(self): + importer = self.importer() + auth = mock.MagicMock() + auth.thycotic_search.side_effect = RuntimeError('401') + importer._migrate_thycotic_teams(auth, object()) + self.assertGreaterEqual(self.log.warning.call_count, 2) + + +class TestDoImportTeamGate(_QuietTestCase): + def _run(self, **kwargs): + importer = self.importer() + auth = mock.MagicMock() + auth.thycotic_search.return_value = [] + params = mock.Mock() + params.rest_context.proxies = None + with mock.patch('keepercommander.importer.thycotic.thycotic.getpass.getpass', return_value='pw'), \ + mock.patch('keepercommander.importer.thycotic.thycotic.ThycoticAuth', return_value=auth), \ + mock.patch.object(ThycoticImporter, 'get_folders', return_value={}), \ + mock.patch.object(importer, '_migrate_thycotic_users') as migrate_users, \ + mock.patch.object(importer, '_preview_thycotic_users') as preview_users, \ + mock.patch.object(importer, '_migrate_thycotic_teams') as migrate_teams, \ + mock.patch.object(importer, '_preview_thycotic_teams') as preview_teams, \ + mock.patch('os.path.isfile', return_value=False): + list(importer.do_import('user@secret.example.com', params=params, **kwargs)) + return migrate_users, preview_users, migrate_teams, preview_teams + + def test_flag_creates_teams(self): + _, _, migrate, preview = self._run(add_teams=True) + migrate.assert_called_once() + preview.assert_not_called() + + def test_flag_absent_does_not_create_teams(self): + _, _, migrate, preview = self._run() + migrate.assert_not_called() + preview.assert_not_called() + + def test_dry_run_prints_groups_without_creating_them(self): + _, _, migrate, preview = self._run(add_teams=True, dry_run=True) + migrate.assert_not_called() + preview.assert_called_once() + + def test_users_are_provisioned_before_teams(self): + importer = self.importer() + order = [] + auth = mock.MagicMock() + params = mock.Mock() + params.rest_context.proxies = None + with mock.patch('keepercommander.importer.thycotic.thycotic.getpass.getpass', return_value='pw'), \ + mock.patch('keepercommander.importer.thycotic.thycotic.ThycoticAuth', return_value=auth), \ + mock.patch.object(ThycoticImporter, 'get_folders', return_value={}), \ + mock.patch.object(importer, '_migrate_thycotic_users', + side_effect=lambda *args, **kwargs: order.append('users')), \ + mock.patch.object(importer, '_migrate_thycotic_teams', + side_effect=lambda *args, **kwargs: order.append('teams')), \ + mock.patch('os.path.isfile', return_value=False): + list(importer.do_import( + 'user@secret.example.com', params=params, + add_users=True, add_teams=True, + )) + self.assertEqual(order, ['users', 'teams']) + + def test_preview_prints_groups_without_creating_teams(self): + importer = self.importer() + importer._prepare_thycotic_groups = mock.Mock(return_value=[ + {'name': 'Ops', 'members': ['a@x.com']}, + ]) + importer._create_enterprise_teams = mock.Mock() + importer._preview_thycotic_teams(mock.Mock()) + printed = ' '.join(str(call) for call in self.log.info.call_args_list) + self.assertIn('Thycotic groups available to migrate', printed) + self.assertIn('Ops', printed) + self.assertIn('a@x.com', printed) + importer._create_enterprise_teams.assert_not_called() + + +class TestImportParser(unittest.TestCase): + def test_flag_defaults_off_and_parses(self): + off = import_parser.parse_args(['--format', 'thycotic', 'https://secret.example.com']) + self.assertFalse(off.add_teams) + on = import_parser.parse_args([ + '--format', 'thycotic', '--add-users', '--add-teams', + 'https://secret.example.com', + ]) + self.assertTrue(on.add_teams) + + +if __name__ == '__main__': + logging.basicConfig() + unittest.main() \ No newline at end of file diff --git a/unit-tests/test_thycotic_add_users.py b/unit-tests/test_thycotic_add_users.py index cca52233d..3d90b2ead 100644 --- a/unit-tests/test_thycotic_add_users.py +++ b/unit-tests/test_thycotic_add_users.py @@ -276,7 +276,8 @@ def test_flag_defaults_off_and_parses(self): off = import_parser.parse_args(['--format', 'thycotic', 'https://secret.example.com']) self.assertFalse(off.add_users) on = import_parser.parse_args([ - '--format', 'thycotic', '--add-users', 'https://secret.example.com', + '--format', 'thycotic', '--add-users', '--add-teams', + 'https://secret.example.com', ]) self.assertTrue(on.add_users)