diff --git a/keepercommander/importer/1password/one_password.py b/keepercommander/importer/1password/one_password.py index a40decbc1..05f839fe2 100644 --- a/keepercommander/importer/1password/one_password.py +++ b/keepercommander/importer/1password/one_password.py @@ -19,7 +19,7 @@ from typing import Union, Optional, Tuple, Any, Dict, Set, Iterable from ..importer import ( - BaseImporter, Record, Folder, RecordField, RecordReferences, SharedFolder, Attachment + BaseImporter, Record, Folder, EmptyFolder, RecordField, RecordReferences, SharedFolder, Attachment ) from ... import utils from ...params import KeeperParams @@ -92,6 +92,14 @@ def do_import(self, filename, **kwargs): elif vault_type == 'U': vault_name = name + if vault_name and not is_shared and not any( + x.get('trashed') is not True for x in vault.get('items', [])): + empty = EmptyFolder() + fol = Folder() + fol.path = vault_name + empty.folders.append(fol) + yield empty + vault_records = {} references = {} # type: Dict[str, Set[str]] for item in vault.get('items', []): diff --git a/keepercommander/importer/bitwarden/bitwarden.py b/keepercommander/importer/bitwarden/bitwarden.py index 42fcdd9c6..0f4fa0574 100644 --- a/keepercommander/importer/bitwarden/bitwarden.py +++ b/keepercommander/importer/bitwarden/bitwarden.py @@ -52,6 +52,25 @@ def do_import(self, filename, **kwargs): sf.path = domain yield sf + used_folders = {i.get('folderId') for i in bw_import.get('items') or []} + used_collections = {cid for i in bw_import.get('items') or [] for cid in i.get('collectionIds') or []} + for fid, name in folders.items(): + if fid not in used_folders: + empty = importer.EmptyFolder() + f = importer.Folder() + f.path = _to_keeper_path(name) + empty.folders.append(f) + yield empty + for cid, name in shared_folders.items(): + first, _, rest = name.partition('/') + if rest and cid not in used_collections: + empty = importer.EmptyFolder() + f = importer.Folder() + f.domain = first.replace('\\', '\\\\') + f.path = _to_keeper_path(rest) + empty.folders.append(f) + yield empty + for i in bw_import.get('items') or []: record = importer.Record() item_type = i.get('type', 0) diff --git a/keepercommander/importer/commands.py b/keepercommander/importer/commands.py index 7a701f988..750234df3 100644 --- a/keepercommander/importer/commands.py +++ b/keepercommander/importer/commands.py @@ -97,6 +97,9 @@ def _positive_int_arg(value): help='NSF only: maximum resulting folder depth (default 5); folders beyond this depth ' 'are attached directly at the deepest allowed level, keeping their own name and ' 'records - names are never merged or renamed') +import_parser.add_argument('--include-empty-folders', dest='include_empty_folders', action='store_true', + help='create folders that contain no records (classic and --nsf). ' + 'Requires an export made with --include-empty-folders (json, csv, keepass)') import_parser.add_argument('--output', dest='output', action='store', help='NSF only: write a report of the resulting Keeper folder structure (original path ' '-> new path -> folder UID) to CSV file; use --output=table to print a table. ' @@ -149,6 +152,8 @@ def _positive_int_arg(value): help='Create ZIP archive for file attachments. JSON only') export_parser.add_argument('--owned-only', dest='owned_only', action='store_true', help='Only export owned records') +export_parser.add_argument('--include-empty-folders', dest='include_empty_folders', action='store_true', + help='Also export folders that contain no records (classic and Nested Share Folders). json, csv, keepass') export_parser.add_argument('--save-in-vault', dest='save_in_vault', action='store_true', help='Stores exports file as a record attachment. KeePass only') export_parser.add_argument('--force', dest='force', action='store_true', help='Suppress user interaction. Assume "yes"') diff --git a/keepercommander/importer/csv/csv.py b/keepercommander/importer/csv/csv.py index c47678f57..1f3ba5d6d 100644 --- a/keepercommander/importer/csv/csv.py +++ b/keepercommander/importer/csv/csv.py @@ -13,7 +13,7 @@ import sys -from ..importer import (BaseFileImporter, BaseExporter, Record, Folder, RecordField, TWO_FACTOR_CODE, +from ..importer import (BaseFileImporter, BaseExporter, Record, Folder, EmptyFolder, RecordField, TWO_FACTOR_CODE, FIELD_TYPE_ONE_TIME_CODE) from ...recordv3 import RecordV3 from ... import record_types @@ -30,6 +30,9 @@ ''' +EMPTY_FOLDER_MARKER = '$empty_folder' + + class KeeperCsvImporter(BaseFileImporter): def do_import(self, filename, **kwargs): with open(filename, "r", encoding='utf-8-sig') as csvfile: @@ -61,6 +64,13 @@ def do_import(self, filename, **kwargs): folder.path = (row[0] or '').strip() record.folders = [folder] + if EMPTY_FOLDER_MARKER in row[7::2]: + if record.folders: + empty = EmptyFolder() + empty.folders = record.folders + yield empty + continue + if len(row) > 7: for i in range(7, len(row)-1, 2): if i+1 < len(row): @@ -104,7 +114,12 @@ def do_export(self, filename, records, **kwargs): csvfile = open(filename, 'w', encoding='utf-8', newline='') if filename else sys.stdout writer = csv.writer(csvfile) for r in records: - if type(r) == Record: + if isinstance(r, EmptyFolder): + for folder in r.folders: + if folder.domain or folder.path: + writer.writerow([folder.path or '', '', '', '', '', '', folder.domain or '', + EMPTY_FOLDER_MARKER, 'true']) + elif type(r) == Record: domain = '' path = '' if r.folders: @@ -152,6 +167,9 @@ def do_export(self, filename, records, **kwargs): def extension(self): return 'csv' + def has_empty_folders(self): + return True + def supports_stdout(self): return True diff --git a/keepercommander/importer/cyberark/cyberark.py b/keepercommander/importer/cyberark/cyberark.py index 45b124902..657e249a6 100644 --- a/keepercommander/importer/cyberark/cyberark.py +++ b/keepercommander/importer/cyberark/cyberark.py @@ -27,6 +27,7 @@ from ..importer import ( BaseDownloadMembership, BaseImporter, + EmptyFolder, Folder, PathDelimiter, Permission, @@ -1374,6 +1375,12 @@ def _do_import_inner(self, filename, **kwargs): nsf_folder = SharedFolder() nsf_folder.path = safe.replace(PathDelimiter, 2 * PathDelimiter) yield nsf_folder + elif not accounts: + empty = EmptyFolder() + empty_folder = Folder() + empty_folder.domain = safe.replace(PathDelimiter, 2 * PathDelimiter) + empty.folders.append(empty_folder) + yield empty with _suppress_progressbar_executor_noise(), ProgressBar() as pb: skip_all = {} skipped_accounts = [] diff --git a/keepercommander/importer/cyberark_portal/cyberark_portal.py b/keepercommander/importer/cyberark_portal/cyberark_portal.py index 119b0c9cc..a7b2eb519 100644 --- a/keepercommander/importer/cyberark_portal/cyberark_portal.py +++ b/keepercommander/importer/cyberark_portal/cyberark_portal.py @@ -15,7 +15,7 @@ from tabulate import tabulate -from ..importer import BaseImporter, Folder, PathDelimiter, Record, RecordField, SharedFolder +from ..importer import BaseImporter, EmptyFolder, Folder, PathDelimiter, Record, RecordField, SharedFolder import secrets import string @@ -766,6 +766,16 @@ def complete(self): nsf_folder.path = fname.replace(PathDelimiter, 2 * PathDelimiter) yield nsf_folder + used_folder_ids = {id(f) for owners in folder_index.values() for f in owners} + for folder in folders: + fname = self._folder_display_name(folder) + if fname and id(folder) not in used_folder_ids: + empty = EmptyFolder() + empty_folder = Folder() + empty_folder.path = fname.replace(PathDelimiter, 2 * PathDelimiter) + empty.folders.append(empty_folder) + yield empty + if len(apps) > 0: print_formatted_text( HTML(f"Importing {len(apps)} Applications:\n"), diff --git a/keepercommander/importer/imp_exp.py b/keepercommander/importer/imp_exp.py index f6c67317d..bae3056f8 100644 --- a/keepercommander/importer/imp_exp.py +++ b/keepercommander/importer/imp_exp.py @@ -35,7 +35,7 @@ from .encryption_reader import EncryptionReader from .importer import (importer_for_format, exporter_for_format, path_components, PathDelimiter, BaseExporter, - BaseImporter, Record as ImportRecord, RecordField as ImportRecordField, Folder as ImportFolder, + BaseImporter, Record as ImportRecord, RecordField as ImportRecordField, Folder as ImportFolder, EmptyFolder as ImportEmptyFolder, SharedFolder as ImportSharedFolder, Permission as ImportPermission, BytesAttachment, Attachment as ImportAttachment, RecordSchemaField, File as ImportFile, Team as ImportTeam, RecordReferences, FIELD_TYPE_ONE_TIME_CODE, TWO_FACTOR_CODE) @@ -518,6 +518,22 @@ def on_folder(base_folder): # type: (BaseFolderNode) -> None rec_count = len(to_export) - sf_count + if kwargs.get('include_empty_folders') and exporter.has_empty_folders(): + # Folders without records never show up through the records above; export them explicitly. + # Only leaf folders are needed, their parents are re-created implicitly on import. + parent_uids = {f.parent_uid for f in params.folder_cache.values() if f.parent_uid} + for folder_uid, f in params.folder_cache.items(): + if f.type not in (BaseFolderNode.UserFolderType, BaseFolderNode.SharedFolderFolderType, + BaseFolderNode.NestedShareFolderType): + continue + if folder_uid in parent_uids or params.subfolder_record_cache.get(folder_uid): + continue + if folder_filter and folder_uid not in folder_filter: + continue + empty = ImportEmptyFolder() + empty.folders.append(get_import_folder(params, folder_uid, None)) + to_export.append(empty) + args = {} file_password = kwargs.get('file_password') if file_password: @@ -843,6 +859,7 @@ def _import(params, file_format, filename, **kwargs): secret_ids = kwargs.get('secret_ids') target_node = kwargs.get('target_node') folder_depth = kwargs.get('folder_depth') + include_empty_folders = kwargs.get('include_empty_folders') is True cyberark_skip = { x.strip().lower() for x in str(kwargs.get('skip') or '').split(',') @@ -869,6 +886,7 @@ def _import(params, file_format, filename, **kwargs): folders = [] # type: List[ImportSharedFolder] records = [] # type: List[ImportRecord] + empty_folders = [] # type: List[ImportEmptyFolder] files = [] # type: List[ImportFile] filter_folder_lower = filter_folder.lower() if isinstance(filter_folder, str) else '' @@ -880,7 +898,9 @@ def _import(params, file_format, filename, **kwargs): old_domain=old_domain, new_domain=new_domain, tmpdir=tmpdir, secret_ids=secret_ids, dry_run=dry_run, target_node=target_node, use_nsf=use_nsf, record_type=record_type, skip_team=skip_team, skip_role=skip_role, skip_user=skip_user): - if isinstance(x, ImportRecord): + if isinstance(x, ImportEmptyFolder) and not include_empty_folders: + continue + if isinstance(x, (ImportRecord, ImportEmptyFolder)): if filter_folder and not importer.support_folder_filter(): if not x.folders: continue @@ -924,6 +944,10 @@ def _import(params, file_format, filename, **kwargs): else: f.path = import_into + if isinstance(x, ImportEmptyFolder): + empty_folders.append(x) + continue + if record_type and not x.type: x.type = record_type try: @@ -986,9 +1010,9 @@ def _import(params, file_format, filename, **kwargs): if use_nsf: from .nsf_import import ensure_nsf_record_folders, flatten_record_folder_paths - flatten_record_folder_paths(records) + flatten_record_folder_paths(records + empty_folders) default_nsf = import_into if (import_into and not nsf_base_parent) else None - ensure_nsf_record_folders(records, folders, default_nsf) + ensure_nsf_record_folders(records + empty_folders, folders, default_nsf) # Importers with verbose summaries also emit a clear message when the target # folder(s) already exist in Keeper. @@ -1144,10 +1168,10 @@ def _import(params, file_format, filename, **kwargs): if use_nsf: from .nsf_import import prepare_nsf_folders if not dry_run: - _created, folder_mapping = prepare_nsf_folders(params, folders, records, nsf_base_parent, + _created, folder_mapping = prepare_nsf_folders(params, folders, records + empty_folders, nsf_base_parent, folder_depth=folder_depth) else: - folder_add = prepare_folder_add(params, folders, records, manage_users, manage_records, can_edit, can_share) + folder_add = prepare_folder_add(params, folders, records + empty_folders, manage_users, manage_records, can_edit, can_share) if folder_add: if not dry_run: fol_rs, _ = execute_import_folder_record(params, folder_add, None) diff --git a/keepercommander/importer/importer.py b/keepercommander/importer/importer.py index 928a4e10e..a68dc423c 100644 --- a/keepercommander/importer/importer.py +++ b/keepercommander/importer/importer.py @@ -193,6 +193,12 @@ def get_folder_path(self): return path +class EmptyFolder: + """A folder that holds no records. Carried so that an export/import round trip keeps the folder structure.""" + def __init__(self): + self.folders = [] # type: List[Folder] + + class RecordField: def __init__(self, type=None, label=None, value=None): self.type = type # type: Optional[str] @@ -496,6 +502,9 @@ def do_export(self, filename, records, **kwargs): def has_shared_folders(self): return False + def has_empty_folders(self): + return False + def has_attachments(self): return False diff --git a/keepercommander/importer/json/json.py b/keepercommander/importer/json/json.py index 567cf86a2..ed5cf82fc 100644 --- a/keepercommander/importer/json/json.py +++ b/keepercommander/importer/json/json.py @@ -22,7 +22,7 @@ from .. import imp_exp from ..importer import (BaseFileImporter, BaseExporter, Record, RecordField, RecordSchemaField, RecordReferences, - Folder, SharedFolder, Permission, Team, Attachment, + Folder, SharedFolder, EmptyFolder, Permission, Team, Attachment, BaseDownloadMembership, BaseDownloadRecordType, RecordType, RecordTypeField) from ... import api, utils, record_types from ...proto import enterprise_pb2 @@ -185,6 +185,7 @@ def do_import(self, filename, **kwargs): records = None folders = None + empty_folders = None teams = None if type(export) == list: records = export @@ -192,6 +193,7 @@ def do_import(self, filename, **kwargs): elif type(export) == dict: records = export.get('records') folders = export.get('shared_folders') + empty_folders = export.get('empty_folders') teams = export.get('teams') if users_only else None if folders: @@ -225,6 +227,18 @@ def do_import(self, filename, **kwargs): yield fol + if isinstance(empty_folders, list) and not users_only: + for ef in empty_folders: + if not isinstance(ef, dict): + continue + folder = Folder() + folder.domain = ef.get('shared_folder') + folder.path = ef.get('folder') + if folder.domain or folder.path: + empty = EmptyFolder() + empty.folders.append(folder) + yield empty + if isinstance(teams, list): for t in teams: team = Team() @@ -260,6 +274,7 @@ def do_export(self, filename, items, zip_archive=None, **kwargs): shared_folders = [] # type: List[SharedFolder] records = [] # type: List[Record] teams = [] # type: List[Team] + empty_folders = [] # type: List[EmptyFolder] if zip_archive is True and not filename: raise ValueError('Please provide zip archive file name') @@ -271,6 +286,8 @@ def do_export(self, filename, items, zip_archive=None, **kwargs): shared_folders.append(item) elif isinstance(item, Team): teams.append(item) + elif isinstance(item, EmptyFolder): + empty_folders.append(item) ts = [] for t in teams: @@ -312,6 +329,17 @@ def do_export(self, filename, items, zip_archive=None, **kwargs): sfo['permissions'].append(po) sfs.append(sfo) + efs = [] + for ef in empty_folders: + for folder in ef.folders: + if folder.domain or folder.path: + efo = {} + if folder.domain: + efo['shared_folder'] = folder.domain + if folder.path: + efo['folder'] = folder.path + efs.append(efo) + rs = [] atta = {} for r in records: @@ -411,6 +439,8 @@ def do_export(self, filename, items, zip_archive=None, **kwargs): jo['teams'] = ts if sfs: jo['shared_folders'] = sfs + if efs: + jo['empty_folders'] = efs if rs: jo['records'] = rs @@ -440,6 +470,9 @@ def do_export(self, filename, items, zip_archive=None, **kwargs): def has_shared_folders(self): return True + def has_empty_folders(self): + return True + def has_attachments(self): return True diff --git a/keepercommander/importer/keepass/keepass.py b/keepercommander/importer/keepass/keepass.py index a93d6d8f6..9fab44789 100644 --- a/keepercommander/importer/keepass/keepass.py +++ b/keepercommander/importer/keepass/keepass.py @@ -26,7 +26,7 @@ from pykeepass.entry import Entry from ..importer import path_components, PathDelimiter, BaseFileImporter, BaseExporter, \ - Record, Folder, SharedFolder, BytesAttachment, RecordField + Record, Folder, SharedFolder, EmptyFolder, BytesAttachment, RecordField from ... import utils from ...display import bcolors from ...utils import parse_totp_uri @@ -93,8 +93,16 @@ def do_import(self, filename, **kwargs): pos = pos + 1 records = [] + empty_folders = [] for group in groups: entries = group.entries + if not entries and not group.subgroups and group is not root: + empty = EmptyFolder() + fol = Folder() + fol.path = KeepassImporter.get_folder(group) + if fol.path: + empty.folders.append(fol) + empty_folders.append(empty) if len(entries) > 0: folder = KeepassImporter.get_folder(group) for entry in entries: @@ -259,6 +267,8 @@ def resolve_references(text): text = text[:match.start()] + val + text[match.end():] return text + yield from empty_folders + for record in records: try: record.login = resolve_references(record.login) @@ -314,15 +324,31 @@ def do_export(self, filename, records, file_password=None, kbdx_key_file=None, * sfs = [] # type: list[SharedFolder] rs = [] # type: list[Record] + efs = [] # type: list[EmptyFolder] for x in records: if isinstance(x, Record): rs.append(x) + elif isinstance(x, EmptyFolder): + efs.append(x) elif isinstance(x, SharedFolder): sfs.append(x) with create_database(filename, password=password, keyfile=keyfile) as kdb: root = kdb.root_group + for ef in efs: + node = root + for fol in ef.folders: + for path in (fol.domain, fol.path): + for comp in path_components(path) if path else []: + comp = self.sanitize_xml_text(comp) + if not comp: + continue + sub_node = next((x for x in node.subgroups if x.name == comp), None) + if sub_node is None: + sub_node = kdb.add_group(node, comp) + node = sub_node + for r in rs: try: node = root @@ -450,6 +476,9 @@ def do_export(self, filename, records, file_password=None, kbdx_key_file=None, * def has_shared_folders(self): return True + def has_empty_folders(self): + return True + def has_attachments(self): return True diff --git a/keepercommander/importer/lastpass/lastpass.py b/keepercommander/importer/lastpass/lastpass.py index fef9131ee..481125aa5 100644 --- a/keepercommander/importer/lastpass/lastpass.py +++ b/keepercommander/importer/lastpass/lastpass.py @@ -19,7 +19,7 @@ from ...params import KeeperParams from ..importer import ( - BaseImporter, Record, Folder, RecordField, RecordReferences, SharedFolder, Permission, BaseDownloadMembership, + BaseImporter, Record, Folder, EmptyFolder, RecordField, RecordReferences, SharedFolder, Permission, BaseDownloadMembership, replace_email_domain, FIELD_TYPE_ONE_TIME_CODE) from .account import Account from .exceptions import LastPassUnknownError @@ -198,6 +198,18 @@ def do_import(self, name, users_only=False, old_domain=None, new_domain=None, tm is_secure_note = True record.login_url = None elif record.login_url == 'http://group': + # a group placeholder entry: the only trace of a folder that has no records + group = account.group + if isinstance(group, bytes): + group = group.decode('utf-8', 'ignore') + if group or account.shared_folder: + empty = EmptyFolder() + fol = Folder() + if account.shared_folder: + fol.domain = account.shared_folder.name + fol.path = group + empty.folders.append(fol) + yield empty continue record.type = 'login' diff --git a/keepercommander/importer/nordpass/nordpass.py b/keepercommander/importer/nordpass/nordpass.py index 981608b30..ff107f736 100644 --- a/keepercommander/importer/nordpass/nordpass.py +++ b/keepercommander/importer/nordpass/nordpass.py @@ -29,6 +29,13 @@ def do_import(self, filename, **kwargs): for row in reader: record_type = row.get('type') or '' if record_type == 'folder': + folder_name = row.get('folder') or row.get('name') or '' + if folder_name: + empty = importer.EmptyFolder() + f = importer.Folder() + f.path = folder_name + empty.folders.append(f) + yield empty continue record = importer.Record() diff --git a/keepercommander/importer/thycotic/thycotic.py b/keepercommander/importer/thycotic/thycotic.py index f7409609c..6d13ce1b9 100644 --- a/keepercommander/importer/thycotic/thycotic.py +++ b/keepercommander/importer/thycotic/thycotic.py @@ -28,7 +28,7 @@ from urllib.parse import urlparse, urlunparse from typing import Tuple, Optional, Iterable, Union, Dict, Callable, Any, List -from ..importer import (BaseImporter, BaseDownloadMembership, Record, SharedFolder, Folder, Attachment, Permission, +from ..importer import (BaseImporter, BaseDownloadMembership, Record, SharedFolder, Folder, EmptyFolder, Attachment, Permission, RecordField, BytesAttachment, Team, BaseDownloadRecordType, RecordType, RecordTypeField) from ...params import KeeperParams from ... import record_types, vault, api @@ -380,6 +380,20 @@ def do_import(self, filename, **kwargs): self._send_keep_alive_if_needed(params) self._send_keep_alive_if_needed(params) + parent_ids = {x.get('parentFolderId') for x in folders.values()} + for folder_id, folder in folders.items(): + # leaf folders only: parents are re-created from the path + if folder_id in parent_ids: + continue + if any(s.get('folderId') == folder_id for s in secrets): + continue + if folder.get('folderPath'): + empty = EmptyFolder() + empty_folder = Folder() + empty_folder.path = folder['folderPath'] + empty.folders.append(empty_folder) + yield empty + for secret in secrets: record = Record() record.title = secret.get('name', '')