Read Igor Pro 8 long data-folder names

igor2/record/__init__.py

@@ -6,7 +6,7 @@
 from .wave import WaveRecord
 from .procedure import ProcedureRecord
 from .packedfile import PackedFileRecord
-from .folder import FolderStartRecord, FolderEndRecord
+from .folder import FolderStartRecord, FolderEndRecord, LongFolderStartRecord
 
 
 # From PackedFile.h
@@ -22,4 +22,6 @@
     8: PackedFileRecord,
     9: FolderStartRecord,
     10: FolderEndRecord,
+    # Igor Pro 8 uses 101 when a folder name does not fit in 31 bytes.
+    101: LongFolderStartRecord,
 }

igor2/record/folder.py

@@ -1,3 +1,5 @@
+import struct
+
 from .base import TextRecord
 
 
@@ -7,3 +9,29 @@ class FolderStartRecord (TextRecord):
 
 class FolderEndRecord (TextRecord):
     pass
+
+
+class LongFolderStartRecord (FolderStartRecord):
+    """Folder start used when the name does not fit in 31 bytes.
+
+    Igor Pro 8 writes record type 101 instead of type 9. The payload
+    is a uint32 byte count followed by exactly that many name bytes.
+    A type 10 record closes the folder, as with a short name.
+    """
+
+    def __init__(self, *args, **kwargs):
+        super(LongFolderStartRecord, self).__init__(*args, **kwargs)
+        order = self.byte_order or '='
+        data = bytes(self.data)
+        if len(data) < 4:
+            raise ValueError(
+                'long folder name record is shorter than its length field')
+        nlen = struct.unpack_from(order + 'I', data)[0]
+        name = data[4:]
+        if nlen != len(name):
+            raise ValueError(
+                'long folder name length {} does not match {} name bytes'
+                .format(nlen, len(name)))
+        # Keep the counted bytes. TextRecord would stop at the NUL that
+        # sits inside the length field of every little-endian name.
+        self.null_terminated_text = name

tests/test_pxp.py

@@ -1,5 +1,7 @@
 import hashlib
+import struct
 import threading
+from io import BytesIO
 
 import numpy as np
 
@@ -267,3 +269,65 @@ def task(thread_id, iteration):
         iterations_per_worker=10,
         task=task,
     )
+
+
+def _pxp_record(record_type, version, payload, order):
+    header = struct.pack(
+        order + 'Hhl', record_type, version, len(payload))
+    return header + payload
+
+
+def _short_folder_name(name):
+    if len(name) > 31:
+        raise ValueError(name)
+    return name.ljust(32, b'\x00')
+
+
+def test_igor8_long_folder_name():
+    # Names taken from the Igor Pro 8 experiment attached to issue 28.
+    long_name = (
+        b'X6_This is very_long_object_name_which_is_allowed_'
+        b'AMC_25_crystalline_04_02_26_0030')
+    issue_name = b'AMC_25_crystalline_04_07_26_10min_0036'
+    name_32 = b'X7_AMC_cyrstalline_04_01_26_0031'
+    short_name = b'AMC_04_07_26_3min_0033'
+    assert len(long_name) == 82
+    assert len(issue_name) == 38
+    assert len(name_32) == 32
+    assert len(short_name) <= 31
+    expected = {long_name, issue_name, name_32, short_name}
+    for order in ('<', '>'):
+        parts = [
+            _pxp_record(9, 1, _short_folder_name(b'USAXS'), order),
+            _pxp_record(33, 0, struct.pack(order + 'I', 1), order),
+        ]
+        for name in (long_name, issue_name, name_32):
+            parts.append(_pxp_record(
+                101, 0, struct.pack(order + 'I', len(name)) + name, order))
+            parts.append(_pxp_record(
+                33, 0, struct.pack(order + 'I', 2), order))
+            parts.append(_pxp_record(10, 0, b'', order))
+        parts.append(_pxp_record(
+            9, 1, _short_folder_name(short_name), order))
+        parts.append(_pxp_record(10, 0, b'', order))
+        parts.append(_pxp_record(10, 0, b'', order))
+        filesystem = loadpxp(BytesIO(b''.join(parts)))[1]
+        children = filesystem['root'][b'USAXS']
+        assert set(children) == expected
+        assert children[long_name] == {}
+        assert children[issue_name] == {}
+        assert children[name_32] == {}
+        assert children[short_name] == {}
+
+
+def test_long_folder_name_rejects_bad_length():
+    payload = struct.pack('<I', 3) + b'abcd'
+    blob = _pxp_record(9, 1, _short_folder_name(b'short'), '<')
+    blob += _pxp_record(101, 0, payload, '<')
+    blob += _pxp_record(10, 0, b'', '<')
+    try:
+        loadpxp(BytesIO(blob))
+    except ValueError as exc:
+        assert 'long folder name length' in str(exc)
+    else:
+        raise AssertionError('mismatched long folder name was accepted')