Store session paths relative to the session file

apps/camera_lidar_trajectory_viewer/TrajectoryViewer.cpp

@@ -14,6 +14,7 @@
 #include <Core/pfd_wrapper.hpp>
 #include <HDMapping/HDMAPPING_ConfigureInfo.hpp>
 #include <HDMapping/PoseInterpolation.h>
+#include <HDMapping/SessionPaths.h>
 #include <RaylibWidgets/CenterOfRotationWindow.h>
 #include <RaylibWidgets/CompassRuler.h>
 #include <RaylibWidgets/OrbitCamera.h>
@@ -768,20 +769,15 @@ static fs::path resolveMjs(const fs::path& p, std::string& err)
     return {};
 }
 
-//! Read the manifest's poses and chunk list. A stored path that no longer exists
-//! falls back to the same file name next to the manifest, as Session::load does.
+//! Read the manifest's poses and chunk list. Stored paths resolve as in
+//! Session::load (see HDMapping/SessionPaths.h).
 static bool readSessionFiles(const fs::path& mjs, SessionFiles& out, std::string& err)
 {
     out = SessionFiles{};
     out.mjs = mjs;
-    auto relocate = [&](std::string path) -> fs::path
+    auto relocate = [&](const std::string& path) -> fs::path
     {
-        std::replace(path.begin(), path.end(), '\\', '/');
-        fs::path p(path);
-        std::error_code ec;
-        if (path.empty() || fs::exists(p, ec))
-            return p;
-        return mjs.parent_path() / p.filename();
+        return path.empty() ? fs::path() : fs::path(resolveSessionPath(path, mjs.string()));
     };
     try
     {

apps/lidar_odometry_step_1/lidar_odometry.cpp

@@ -6,6 +6,7 @@
 
 #include <Core/system_info.hpp>
 #include <Fusion.h>
+#include <HDMapping/SessionPaths.h>
 #if defined(__GLIBC__)
 #include <malloc.h>
 #endif
@@ -1090,22 +1091,25 @@ void save_result(std::vector<WorkerData>& worker_data, LidarOdometryParams& para
     nlohmann::json jj;
     nlohmann::json j;
 
+    // Paths are stored relative to the session file, so the result folder can be moved
+    const std::string session_file = path3.string();
+
     j["offset_x"] = 0.0;
     j["offset_y"] = 0.0;
     j["offset_z"] = 0.0;
-    j["folder_name"] = outwd;
-    j["out_folder_name"] = outwd;
-    j["poses_file_name"] = path2.string();
-    j["initial_poses_file_name"] = path.string();
-    j["out_poses_file_name"] = path2.string();
+    j["folder_name"] = toSessionPath(outwd.string(), session_file);
+    j["out_folder_name"] = toSessionPath(outwd.string(), session_file);
+    j["poses_file_name"] = toSessionPath(path2.string(), session_file);
+    j["initial_poses_file_name"] = toSessionPath(path.string(), session_file);
+    j["out_poses_file_name"] = toSessionPath(path2.string(), session_file);
     j["lidar_odometry_version"] = HDMAPPING_CONFIGURE_PROJECT_VERSION;
     j["length of trajectory[m]"] = params.total_length_of_calculated_trajectory;
     j["elapsed time seconds"] = elapsed_time_s;
     if (params.save_index_pose)
     {
-        j["index_poses_path"] = index_poses_path.string();
+        j["index_poses_path"] = toSessionPath(index_poses_path.string(), session_file);
     }
-    j["point_sizes_path"] = point_sizes_path.string();
+    j["point_sizes_path"] = toSessionPath(point_sizes_path.string(), session_file);
     j["decimation"] = params.decimation;
     j["threshold_nr_poses"] = params.threshold_nr_poses;
 
@@ -1119,7 +1123,7 @@ void save_result(std::vector<WorkerData>& worker_data, LidarOdometryParams& para
         path /= filename;
         std::cout << "adding file: " << path << std::endl;
 
-        nlohmann::json jfn{ { "file_name", path.string() } };
+        nlohmann::json jfn{ { "file_name", toSessionPath(path.string(), session_file) } };
         jlaz_file_names.push_back(jfn);
     }
     jj["laz_file_names"] = jlaz_file_names;

core/src/session.cpp

@@ -5,6 +5,7 @@
 #include <nlohmann/json.hpp>
 
 #include <HDMapping/HDMAPPING_ConfigureInfo.hpp>
+#include <HDMapping/SessionPaths.h>
 
 namespace fs = std::filesystem;
 
@@ -97,29 +98,10 @@ bool Session::load(const std::string& file_name, bool is_decimate, double bucket
     std::vector<bool> vfixed_fi;
     std::vector<bool> vfixed_ka;
 
-    // Get a loaded file directory
-    std::string directory = fs::path(file_name).parent_path().string();
-
-    // Local pathUpdater lambda (keep directories unchanged, update files to be in session directory)
+    // Stored paths are relative to the session file (absolute in older sessions)
     auto getNewPath = [&](const std::string& path) -> std::string
     {
-        // Normalize Windows backslashes to forward slashes so that
-        // std::filesystem::path correctly extracts the filename on Linux.
-        std::string normalized = path;
-        std::replace(normalized.begin(), normalized.end(), '\\', '/');
-        fs::path p(normalized);
-        if (is_directory(p))
-            return p.string();
-        // Prefer the stored path when it still resolves: this keeps sessions
-        // working when they were saved to a directory other than the one that
-        // holds the referenced .laz/.csv files (e.g. session file one level up
-        // from a lio_result_* folder). Only when the stored path is gone do we
-        // fall back to a file of the same name sitting next to the session file
-        // (the "session + data moved together" case).
-        std::error_code ec;
-        if (fs::exists(p, ec))
-            return p.string();
-        return (fs::path(directory) / p.filename()).string();
+        return resolveSessionPath(path, file_name);
     };
 
     try
@@ -404,11 +386,11 @@ bool Session::save(
     j["offset_to_apply_x"] = point_clouds_container.offset_to_apply.x();
     j["offset_to_apply_y"] = point_clouds_container.offset_to_apply.y();
     j["offset_to_apply_z"] = point_clouds_container.offset_to_apply.z();
-    j["folder_name"] = point_clouds_container.folder_name;
-    j["out_folder_name"] = point_clouds_container.out_folder_name;
-    j["poses_file_name"] = poses_file_name; // point_clouds_container.poses_file_name;
-    j["initial_poses_file_name"] = initial_poses_file_name; // point_clouds_container.initial_poses_file_name;
-    j["out_poses_file_name"] = point_clouds_container.out_poses_file_name;
+    j["folder_name"] = toSessionPath(point_clouds_container.folder_name, file_name);
+    j["out_folder_name"] = toSessionPath(point_clouds_container.out_folder_name, file_name);
+    j["poses_file_name"] = toSessionPath(poses_file_name, file_name); // point_clouds_container.poses_file_name;
+    j["initial_poses_file_name"] = toSessionPath(initial_poses_file_name, file_name); // point_clouds_container.initial_poses_file_name;
+    j["out_poses_file_name"] = toSessionPath(point_clouds_container.out_poses_file_name, file_name);
     j["ground_truth"] = is_ground_truth;
     j["exporting_software_version"] = HDMAPPING_CONFIGURE_PROJECT_VERSION;
     jj["Session Settings"] = j;
@@ -450,19 +432,27 @@ bool Session::save(
         {
             if (pc.visible)
             {
-                nlohmann::json jfn{ { "file_name", pc.file_name }, { "fuse_inclination_from_IMU", pc.fuse_inclination_from_IMU },
-                                    { "fixed_x", pc.fixed_x },     { "fixed_y", pc.fixed_y },
-                                    { "fixed_z", pc.fixed_z },     { "fixed_om", pc.fixed_om },
-                                    { "fixed_fi", pc.fixed_fi },   { "fixed_ka", pc.fixed_ka } };
+                nlohmann::json jfn{ { "file_name", toSessionPath(pc.file_name, file_name) },
+                                    { "fuse_inclination_from_IMU", pc.fuse_inclination_from_IMU },
+                                    { "fixed_x", pc.fixed_x },
+                                    { "fixed_y", pc.fixed_y },
+                                    { "fixed_z", pc.fixed_z },
+                                    { "fixed_om", pc.fixed_om },
+                                    { "fixed_fi", pc.fixed_fi },
+                                    { "fixed_ka", pc.fixed_ka } };
                 jlaz_file_names.push_back(jfn);
             }
         }
         else
         {
-            nlohmann::json jfn{ { "file_name", pc.file_name }, { "fuse_inclination_from_IMU", pc.fuse_inclination_from_IMU },
-                                { "fixed_x", pc.fixed_x },     { "fixed_y", pc.fixed_y },
-                                { "fixed_z", pc.fixed_z },     { "fixed_om", pc.fixed_om },
-                                { "fixed_fi", pc.fixed_fi },   { "fixed_ka", pc.fixed_ka } };
+            nlohmann::json jfn{ { "file_name", toSessionPath(pc.file_name, file_name) },
+                                { "fuse_inclination_from_IMU", pc.fuse_inclination_from_IMU },
+                                { "fixed_x", pc.fixed_x },
+                                { "fixed_y", pc.fixed_y },
+                                { "fixed_z", pc.fixed_z },
+                                { "fixed_om", pc.fixed_om },
+                                { "fixed_fi", pc.fixed_fi },
+                                { "fixed_ka", pc.fixed_ka } };
             jlaz_file_names.push_back(jfn);
         }
     }

shared/include/HDMapping/SessionPaths.h

@@ -0,0 +1,56 @@
+#pragma once
+#include <algorithm>
+#include <filesystem>
+#include <string>
+#include <system_error>
+
+// Session files (Session::save() in core, session.mjs from lidar_odometry_step_1)
+// store the paths of the files they reference relative to the session file's
+// folder, so a session keeps working after its folder is moved or copied to
+// another computer. In memory the paths stay absolute: writers convert with
+// toSessionPath(), readers with resolveSessionPath().
+
+// Path to write into session_file for `path`: relative to the session file's
+// folder with '/' separators, or absolute when there is no relative path (e.g.
+// another drive on Windows). An empty path stays empty.
+inline std::string toSessionPath(const std::string& path, const std::string& session_file)
+{
+    namespace fs = std::filesystem;
+    if (path.empty())
+    {
+        return path;
+    }
+    std::error_code ec_path;
+    std::error_code ec_session;
+    const fs::path absolute_path = fs::absolute(path, ec_path).lexically_normal();
+    const fs::path session_dir = fs::absolute(session_file, ec_session).parent_path().lexically_normal();
+    if (ec_path || ec_session)
+    {
+        return fs::path(path).generic_string();
+    }
+    const fs::path relative_path = absolute_path.lexically_relative(session_dir);
+    return (relative_path.empty() ? absolute_path : relative_path).generic_string();
+}
+
+// Absolute path for a path read from session_file. Relative paths resolve
+// against the session file's folder, absolute ones (older sessions) are used as
+// they are. If the result doesn't exist, the file of the same name next to the
+// session file is used instead, so sessions moved together with their data keep
+// loading.
+inline std::string resolveSessionPath(const std::string& stored_path, const std::string& session_file)
+{
+    namespace fs = std::filesystem;
+    // Sessions written on Windows use '\' separators
+    std::string normalized = stored_path;
+    std::replace(normalized.begin(), normalized.end(), '\\', '/');
+    const fs::path path(normalized);
+
+    std::error_code ec;
+    const fs::path session_dir = fs::absolute(session_file, ec).parent_path();
+    const fs::path resolved = path.is_absolute() ? path : (session_dir / path).lexically_normal();
+    if (fs::exists(resolved, ec))
+    {
+        return resolved.string();
+    }
+    return (session_dir / path.filename()).string();
+}

shared/tests/CMakeLists.txt

@@ -11,6 +11,7 @@ project(hdmapping_shared_tests)
 # noticeably to build time.
 add_executable(hdmapping_shared_tests
     test_pose_interpolation.cpp
+    test_session_paths.cpp
 )
 
 target_include_directories(hdmapping_shared_tests PRIVATE

shared/tests/test_session_paths.cpp

@@ -0,0 +1,143 @@
+#include <doctest.h>
+
+#include <HDMapping/SessionPaths.h>
+
+#include <chrono>
+#include <filesystem>
+#include <fstream>
+#include <string>
+
+namespace fs = std::filesystem;
+
+namespace
+{
+    // Fresh folder under the system temp folder, removed again at the end of the test
+    struct TempDir
+    {
+        fs::path path;
+
+        TempDir()
+        {
+            const auto stamp = std::chrono::steady_clock::now().time_since_epoch().count();
+            path = fs::absolute(fs::temp_directory_path() / ("hdmapping_session_paths_" + std::to_string(stamp)));
+            fs::create_directories(path);
+        }
+
+        ~TempDir()
+        {
+            std::error_code ec;
+            fs::remove_all(path, ec);
+        }
+    };
+
+    // Switches the current folder and restores it at the end of the test
+    struct CurrentDir
+    {
+        fs::path previous = fs::current_path();
+
+        explicit CurrentDir(const fs::path& dir)
+        {
+            fs::current_path(dir);
+        }
+
+        ~CurrentDir()
+        {
+            std::error_code ec;
+            fs::current_path(previous, ec);
+        }
+    };
+
+    void touch(const fs::path& file)
+    {
+        fs::create_directories(file.parent_path());
+        std::ofstream(file) << "x";
+    }
+
+    bool samePath(const std::string& a, const fs::path& b)
+    {
+        return fs::path(a).lexically_normal() == b.lexically_normal();
+    }
+} // namespace
+
+TEST_CASE("toSessionPath: paths become relative to the session file's folder")
+{
+    TempDir tmp;
+    const std::string session = (tmp.path / "lio_result_0" / "session.mjs").string();
+
+    CHECK(toSessionPath((tmp.path / "lio_result_0" / "scan_lio_0.laz").string(), session) == "scan_lio_0.laz");
+    CHECK(toSessionPath((tmp.path / "lio_result_0" / "cache" / "0.cache").string(), session) == "cache/0.cache");
+    CHECK(toSessionPath((tmp.path / "lio_result_1" / "scan_lio_0.laz").string(), session) == "../lio_result_1/scan_lio_0.laz");
+    CHECK(toSessionPath((tmp.path / "lio_result_0").string(), session) == ".");
+}
+
+TEST_CASE("toSessionPath: an empty path stays empty")
+{
+    TempDir tmp;
+    CHECK(toSessionPath("", (tmp.path / "session.mjs").string()).empty());
+}
+
+#ifdef _WIN32
+TEST_CASE("toSessionPath: a path on another drive stays absolute")
+{
+    CHECK(toSessionPath("D:/data/scan_lio_0.laz", "C:/sessions/session.mjs") == "D:/data/scan_lio_0.laz");
+}
+#endif
+
+TEST_CASE("resolveSessionPath: relative paths resolve against the session file's folder")
+{
+    TempDir tmp;
+    const std::string session = (tmp.path / "lio_result_0" / "session.mjs").string();
+    touch(tmp.path / "lio_result_0" / "scan_lio_0.laz");
+    touch(tmp.path / "lio_result_1" / "scan_lio_0.laz");
+
+    CHECK(samePath(resolveSessionPath("scan_lio_0.laz", session), tmp.path / "lio_result_0" / "scan_lio_0.laz"));
+    CHECK(samePath(resolveSessionPath("../lio_result_1/scan_lio_0.laz", session), tmp.path / "lio_result_1" / "scan_lio_0.laz"));
+    CHECK(samePath(resolveSessionPath("..\\lio_result_1\\scan_lio_0.laz", session), tmp.path / "lio_result_1" / "scan_lio_0.laz"));
+    CHECK(fs::equivalent(resolveSessionPath(".", session), tmp.path / "lio_result_0"));
+}
+
+TEST_CASE("resolveSessionPath: the current folder doesn't matter")
+{
+    TempDir tmp;
+    const std::string session = (tmp.path / "lio_result_0" / "session.mjs").string();
+    touch(tmp.path / "lio_result_0" / "scan_lio_0.laz");
+    touch(tmp.path / "elsewhere" / "scan_lio_0.laz");
+    CurrentDir cwd(tmp.path / "elsewhere");
+
+    CHECK(samePath(resolveSessionPath("scan_lio_0.laz", session), tmp.path / "lio_result_0" / "scan_lio_0.laz"));
+}
+
+TEST_CASE("resolveSessionPath: an absolute path from an older session is used while it exists")
+{
+    TempDir tmp;
+    const fs::path data = tmp.path / "data" / "scan_lio_0.laz";
+    touch(data);
+
+    CHECK(samePath(resolveSessionPath(data.string(), (tmp.path / "session.mjs").string()), data));
+}
+
+TEST_CASE("resolveSessionPath: a missing path falls back to the same name next to the session file")
+{
+    TempDir tmp;
+    const std::string session = (tmp.path / "moved" / "session.mjs").string();
+    const fs::path next_to_session = tmp.path / "moved" / "scan_lio_0.laz";
+    touch(next_to_session);
+
+    // absolute paths from the computer the session was made on
+    CHECK(samePath(resolveSessionPath("C:\\data\\lio_result_0\\scan_lio_0.laz", session), next_to_session));
+    CHECK(samePath(resolveSessionPath((tmp.path / "gone" / "scan_lio_0.laz").string(), session), next_to_session));
+}
+
+TEST_CASE("Session paths survive moving the session folder")
+{
+    // A session file one level above its data: the old "same name next to the
+    // session file" fallback can't find these files once the folder is moved.
+    TempDir tmp;
+    const fs::path before = tmp.path / "before";
+    const fs::path after = tmp.path / "after";
+    const std::string stored = toSessionPath((before / "lio_result_0" / "scan_lio_0.laz").string(), (before / "session.mjs").string());
+    CHECK(stored == "lio_result_0/scan_lio_0.laz");
+
+    touch(after / "lio_result_0" / "scan_lio_0.laz");
+    CHECK(samePath(resolveSessionPath(stored, (after / "session.mjs").string()), after / "lio_result_0" / "scan_lio_0.laz"));
+}