Add --include-empty-folders flag to Import/Export empty folders

keepercommander/importer/1password/one_password.py

@@ -19,7 +19,7 @@
 from typing import Union, Optional, Tuple, Any, Dict, Set, Iterable
 
 from ..importer import (
-    BaseImporter, Record, Folder, RecordField, RecordReferences, SharedFolder, Attachment
+    BaseImporter, Record, Folder, EmptyFolder, RecordField, RecordReferences, SharedFolder, Attachment
 )
 from ... import utils
 from ...params import KeeperParams
@@ -92,6 +92,14 @@ def do_import(self, filename, **kwargs):
                             elif vault_type == 'U':
                                 vault_name = name
 
+                        if vault_name and not is_shared and not any(
+                                x.get('trashed') is not True for x in vault.get('items', [])):
+                            empty = EmptyFolder()
+                            fol = Folder()
+                            fol.path = vault_name
+                            empty.folders.append(fol)
+                            yield empty
+
                         vault_records = {}
                         references = {}   # type: Dict[str, Set[str]]
                         for item in vault.get('items', []):

keepercommander/importer/bitwarden/bitwarden.py

@@ -52,6 +52,25 @@ def do_import(self, filename, **kwargs):
                 sf.path = domain
                 yield sf
 
+        used_folders = {i.get('folderId') for i in bw_import.get('items') or []}
+        used_collections = {cid for i in bw_import.get('items') or [] for cid in i.get('collectionIds') or []}
+        for fid, name in folders.items():
+            if fid not in used_folders:
+                empty = importer.EmptyFolder()
+                f = importer.Folder()
+                f.path = _to_keeper_path(name)
+                empty.folders.append(f)
+                yield empty
+        for cid, name in shared_folders.items():
+            first, _, rest = name.partition('/')
+            if rest and cid not in used_collections:
+                empty = importer.EmptyFolder()
+                f = importer.Folder()
+                f.domain = first.replace('\\', '\\\\')
+                f.path = _to_keeper_path(rest)
+                empty.folders.append(f)
+                yield empty
+
         for i in bw_import.get('items') or []:
             record = importer.Record()
             item_type = i.get('type', 0)

keepercommander/importer/commands.py

@@ -97,6 +97,9 @@ def _positive_int_arg(value):
                            help='NSF only: maximum resulting folder depth (default 5); folders beyond this depth '
                                 'are attached directly at the deepest allowed level, keeping their own name and '
                                 'records - names are never merged or renamed')
+import_parser.add_argument('--include-empty-folders', dest='include_empty_folders', action='store_true',
+                           help='create folders that contain no records (classic and --nsf). '
+                                'Requires an export made with --include-empty-folders (json, csv, keepass)')
 import_parser.add_argument('--output', dest='output', action='store',
                            help='NSF only: write a report of the resulting Keeper folder structure (original path '
                                 '-> new path -> folder UID) to CSV file; use --output=table to print a table. '
@@ -149,6 +152,8 @@ def _positive_int_arg(value):
                            help='Create ZIP archive for file attachments. JSON only')
 export_parser.add_argument('--owned-only', dest='owned_only', action='store_true',
                            help='Only export owned records')
+export_parser.add_argument('--include-empty-folders', dest='include_empty_folders', action='store_true',
+                           help='Also export folders that contain no records (classic and Nested Share Folders). json, csv, keepass')
 export_parser.add_argument('--save-in-vault', dest='save_in_vault', action='store_true',
                            help='Stores exports file as a record attachment. KeePass only')
 export_parser.add_argument('--force', dest='force', action='store_true', help='Suppress user interaction. Assume "yes"')

keepercommander/importer/csv/csv.py

@@ -13,7 +13,7 @@
 
 import sys
 
-from ..importer import (BaseFileImporter, BaseExporter, Record, Folder, RecordField, TWO_FACTOR_CODE,
+from ..importer import (BaseFileImporter, BaseExporter, Record, Folder, EmptyFolder, RecordField, TWO_FACTOR_CODE,
                         FIELD_TYPE_ONE_TIME_CODE)
 from ...recordv3 import RecordV3
 from ... import record_types
@@ -30,6 +30,9 @@
 '''
 
 
+EMPTY_FOLDER_MARKER = '$empty_folder'
+
+
 class KeeperCsvImporter(BaseFileImporter):
     def do_import(self, filename, **kwargs):
         with open(filename, "r", encoding='utf-8-sig') as csvfile:
@@ -61,6 +64,13 @@ def do_import(self, filename, **kwargs):
                         folder.path = (row[0] or '').strip()
                         record.folders = [folder]
 
+                    if EMPTY_FOLDER_MARKER in row[7::2]:
+                        if record.folders:
+                            empty = EmptyFolder()
+                            empty.folders = record.folders
+                            yield empty
+                        continue
+
                     if len(row) > 7:
                         for i in range(7, len(row)-1, 2):
                             if i+1 < len(row):
@@ -104,7 +114,12 @@ def do_export(self, filename, records, **kwargs):
         csvfile = open(filename, 'w', encoding='utf-8', newline='') if filename else sys.stdout
         writer = csv.writer(csvfile)
         for r in records:
-            if type(r) == Record:
+            if isinstance(r, EmptyFolder):
+                for folder in r.folders:
+                    if folder.domain or folder.path:
+                        writer.writerow([folder.path or '', '', '', '', '', '', folder.domain or '',
+                                         EMPTY_FOLDER_MARKER, 'true'])
+            elif type(r) == Record:
                 domain = ''
                 path = ''
                 if r.folders:
@@ -152,6 +167,9 @@ def do_export(self, filename, records, **kwargs):
     def extension(self):
         return 'csv'
 
+    def has_empty_folders(self):
+        return True
+
     def supports_stdout(self):
         return True
 

keepercommander/importer/cyberark/cyberark.py

@@ -27,6 +27,7 @@
 from ..importer import (
     BaseDownloadMembership,
     BaseImporter,
+    EmptyFolder,
     Folder,
     PathDelimiter,
     Permission,
@@ -1374,6 +1375,12 @@ def _do_import_inner(self, filename, **kwargs):
                 nsf_folder = SharedFolder()
                 nsf_folder.path = safe.replace(PathDelimiter, 2 * PathDelimiter)
                 yield nsf_folder
+            elif not accounts:
+                empty = EmptyFolder()
+                empty_folder = Folder()
+                empty_folder.domain = safe.replace(PathDelimiter, 2 * PathDelimiter)
+                empty.folders.append(empty_folder)
+                yield empty
             with _suppress_progressbar_executor_noise(), ProgressBar() as pb:
                 skip_all = {}
                 skipped_accounts = []

keepercommander/importer/cyberark_portal/cyberark_portal.py

@@ -15,7 +15,7 @@
 from tabulate import tabulate
 
 
-from ..importer import BaseImporter, Folder, PathDelimiter, Record, RecordField, SharedFolder
+from ..importer import BaseImporter, EmptyFolder, Folder, PathDelimiter, Record, RecordField, SharedFolder
 import secrets
 import string
 
@@ -766,6 +766,16 @@ def complete(self):
                     nsf_folder.path = fname.replace(PathDelimiter, 2 * PathDelimiter)
                     yield nsf_folder
 
+        used_folder_ids = {id(f) for owners in folder_index.values() for f in owners}
+        for folder in folders:
+            fname = self._folder_display_name(folder)
+            if fname and id(folder) not in used_folder_ids:
+                empty = EmptyFolder()
+                empty_folder = Folder()
+                empty_folder.path = fname.replace(PathDelimiter, 2 * PathDelimiter)
+                empty.folders.append(empty_folder)
+                yield empty
+
         if len(apps) > 0:
             print_formatted_text(
                 HTML(f"Importing <b>{len(apps)}</b> Applications:\n"),

keepercommander/importer/imp_exp.py

@@ -35,7 +35,7 @@
 
 from .encryption_reader import EncryptionReader
 from .importer import (importer_for_format, exporter_for_format, path_components, PathDelimiter, BaseExporter,
-                       BaseImporter, Record as ImportRecord, RecordField as ImportRecordField, Folder as ImportFolder,
+                       BaseImporter, Record as ImportRecord, RecordField as ImportRecordField, Folder as ImportFolder, EmptyFolder as ImportEmptyFolder,
                        SharedFolder as ImportSharedFolder, Permission as ImportPermission, BytesAttachment,
                        Attachment as ImportAttachment, RecordSchemaField, File as ImportFile, Team as ImportTeam,
                        RecordReferences, FIELD_TYPE_ONE_TIME_CODE, TWO_FACTOR_CODE)
@@ -518,6 +518,22 @@ def on_folder(base_folder):   # type: (BaseFolderNode) -> None
 
     rec_count = len(to_export) - sf_count
 
+    if kwargs.get('include_empty_folders') and exporter.has_empty_folders():
+        # Folders without records never show up through the records above; export them explicitly.
+        # Only leaf folders are needed, their parents are re-created implicitly on import.
+        parent_uids = {f.parent_uid for f in params.folder_cache.values() if f.parent_uid}
+        for folder_uid, f in params.folder_cache.items():
+            if f.type not in (BaseFolderNode.UserFolderType, BaseFolderNode.SharedFolderFolderType,
+                              BaseFolderNode.NestedShareFolderType):
+                continue
+            if folder_uid in parent_uids or params.subfolder_record_cache.get(folder_uid):
+                continue
+            if folder_filter and folder_uid not in folder_filter:
+                continue
+            empty = ImportEmptyFolder()
+            empty.folders.append(get_import_folder(params, folder_uid, None))
+            to_export.append(empty)
+
     args = {}
     file_password = kwargs.get('file_password')
     if file_password:
@@ -843,6 +859,7 @@ def _import(params, file_format, filename, **kwargs):
     secret_ids = kwargs.get('secret_ids')
     target_node = kwargs.get('target_node')
     folder_depth = kwargs.get('folder_depth')
+    include_empty_folders = kwargs.get('include_empty_folders') is True
     cyberark_skip = {
         x.strip().lower()
         for x in str(kwargs.get('skip') or '').split(',')
@@ -869,6 +886,7 @@ def _import(params, file_format, filename, **kwargs):
 
     folders = []        # type: List[ImportSharedFolder]
     records = []        # type: List[ImportRecord]
+    empty_folders = []  # type: List[ImportEmptyFolder]
     files = []          # type: List[ImportFile]
 
     filter_folder_lower = filter_folder.lower() if isinstance(filter_folder, str) else ''
@@ -880,7 +898,9 @@ def _import(params, file_format, filename, **kwargs):
                               old_domain=old_domain, new_domain=new_domain, tmpdir=tmpdir, secret_ids=secret_ids,
                               dry_run=dry_run, target_node=target_node, use_nsf=use_nsf, record_type=record_type,
                               skip_team=skip_team, skip_role=skip_role, skip_user=skip_user):
-        if isinstance(x, ImportRecord):
+        if isinstance(x, ImportEmptyFolder) and not include_empty_folders:
+            continue
+        if isinstance(x, (ImportRecord, ImportEmptyFolder)):
             if filter_folder and not importer.support_folder_filter():
                 if not x.folders:
                     continue
@@ -924,6 +944,10 @@ def _import(params, file_format, filename, **kwargs):
                         else:
                             f.path = import_into
 
+            if isinstance(x, ImportEmptyFolder):
+                empty_folders.append(x)
+                continue
+
             if record_type and not x.type:
                 x.type = record_type
             try:
@@ -986,9 +1010,9 @@ def _import(params, file_format, filename, **kwargs):
 
     if use_nsf:
         from .nsf_import import ensure_nsf_record_folders, flatten_record_folder_paths
-        flatten_record_folder_paths(records)
+        flatten_record_folder_paths(records + empty_folders)
         default_nsf = import_into if (import_into and not nsf_base_parent) else None
-        ensure_nsf_record_folders(records, folders, default_nsf)
+        ensure_nsf_record_folders(records + empty_folders, folders, default_nsf)
 
     # Importers with verbose summaries also emit a clear message when the target
     # folder(s) already exist in Keeper.
@@ -1144,10 +1168,10 @@ def _import(params, file_format, filename, **kwargs):
     if use_nsf:
         from .nsf_import import prepare_nsf_folders
         if not dry_run:
-            _created, folder_mapping = prepare_nsf_folders(params, folders, records, nsf_base_parent,
+            _created, folder_mapping = prepare_nsf_folders(params, folders, records + empty_folders, nsf_base_parent,
                                                             folder_depth=folder_depth)
     else:
-        folder_add = prepare_folder_add(params, folders, records, manage_users, manage_records, can_edit, can_share)
+        folder_add = prepare_folder_add(params, folders, records + empty_folders, manage_users, manage_records, can_edit, can_share)
         if folder_add:
             if not dry_run:
                 fol_rs, _ = execute_import_folder_record(params, folder_add, None)

keepercommander/importer/importer.py

@@ -193,6 +193,12 @@ def get_folder_path(self):
         return path
 
 
+class EmptyFolder:
+    """A folder that holds no records. Carried so that an export/import round trip keeps the folder structure."""
+    def __init__(self):
+        self.folders = []   # type: List[Folder]
+
+
 class RecordField:
     def __init__(self, type=None, label=None, value=None):
         self.type = type    # type: Optional[str]
@@ -496,6 +502,9 @@ def do_export(self, filename, records, **kwargs):
     def has_shared_folders(self):
         return False
 
+    def has_empty_folders(self):
+        return False
+
     def has_attachments(self):
         return False
 

keepercommander/importer/json/json.py

@@ -22,7 +22,7 @@
 
 from .. import imp_exp
 from ..importer import (BaseFileImporter, BaseExporter, Record, RecordField, RecordSchemaField, RecordReferences,
-                        Folder, SharedFolder, Permission, Team, Attachment,
+                        Folder, SharedFolder, EmptyFolder, Permission, Team, Attachment,
                         BaseDownloadMembership, BaseDownloadRecordType, RecordType, RecordTypeField)
 from ... import api, utils, record_types
 from ...proto import enterprise_pb2
@@ -185,13 +185,15 @@ def do_import(self, filename, **kwargs):
 
         records = None
         folders = None
+        empty_folders = None
         teams = None
         if type(export) == list:
             records = export
 
         elif type(export) == dict:
             records = export.get('records')
             folders = export.get('shared_folders')
+            empty_folders = export.get('empty_folders')
             teams = export.get('teams') if users_only else None
 
         if folders:
@@ -225,6 +227,18 @@ def do_import(self, filename, **kwargs):
 
                 yield fol
 
+        if isinstance(empty_folders, list) and not users_only:
+            for ef in empty_folders:
+                if not isinstance(ef, dict):
+                    continue
+                folder = Folder()
+                folder.domain = ef.get('shared_folder')
+                folder.path = ef.get('folder')
+                if folder.domain or folder.path:
+                    empty = EmptyFolder()
+                    empty.folders.append(folder)
+                    yield empty
+
         if isinstance(teams, list):
             for t in teams:
                 team = Team()
@@ -260,6 +274,7 @@ def do_export(self, filename, items, zip_archive=None, **kwargs):
         shared_folders = []     # type: List[SharedFolder]
         records = []            # type: List[Record]
         teams = []              # type: List[Team]
+        empty_folders = []      # type: List[EmptyFolder]
 
         if zip_archive is True and not filename:
             raise ValueError('Please provide zip archive file name')
@@ -271,6 +286,8 @@ def do_export(self, filename, items, zip_archive=None, **kwargs):
                 shared_folders.append(item)
             elif isinstance(item, Team):
                 teams.append(item)
+            elif isinstance(item, EmptyFolder):
+                empty_folders.append(item)
 
         ts = []
         for t in teams:
@@ -312,6 +329,17 @@ def do_export(self, filename, items, zip_archive=None, **kwargs):
                     sfo['permissions'].append(po)
             sfs.append(sfo)
 
+        efs = []
+        for ef in empty_folders:
+            for folder in ef.folders:
+                if folder.domain or folder.path:
+                    efo = {}
+                    if folder.domain:
+                        efo['shared_folder'] = folder.domain
+                    if folder.path:
+                        efo['folder'] = folder.path
+                    efs.append(efo)
+
         rs = []
         atta = {}
         for r in records:
@@ -411,6 +439,8 @@ def do_export(self, filename, items, zip_archive=None, **kwargs):
             jo['teams'] = ts
         if sfs:
             jo['shared_folders'] = sfs
+        if efs:
+            jo['empty_folders'] = efs
         if rs:
             jo['records'] = rs
 
@@ -440,6 +470,9 @@ def do_export(self, filename, items, zip_archive=None, **kwargs):
     def has_shared_folders(self):
         return True
 
+    def has_empty_folders(self):
+        return True
+
     def has_attachments(self):
         return True