From 71ef986be895caac959a3a181e0598ed771c717d Mon Sep 17 00:00:00 2001 From: zyansheep Date: Sat, 7 Nov 2026 18:39:27 -0800 Subject: [PATCH] Signal import: the same file twice in a message is two attachments; a placeholder for a file the hub has is not a difference, and words the hub never had are filled in Found by a trial import of a real backup into a copy of the hub: a message with one picture attached twice broke the import, 22 'differences' were pictures live sync has but the backup lacks, and 7 were stickers whose emoji live sync never got. Co-Authored-By: Claude Opus 5.5 --- packages/hub/src/imports/signal-v2.ts | 6 +++++ packages/hub/src/store.ts | 13 ++++++++-- packages/hub/test/fixtures/signal-v2.ts | 4 +-- packages/hub/test/signal-v2.test.ts | 34 +++++++++++++++++++++++-- 4 files changed, 51 insertions(+), 6 deletions(-) diff --git a/packages/hub/src/imports/signal-v2.ts b/packages/hub/src/imports/signal-v2.ts index d2a5024..2367555 100644 --- a/packages/hub/src/imports/signal-v2.ts +++ b/packages/hub/src/imports/signal-v2.ts @@ -440,6 +440,12 @@ export function mapFrames(b: Uint8Array, files: string, o: Omit keep(sub(fields(a.bytes!), 1))) .filter((a): a is P.AttachmentFact => !!a); m.kind = atts.length ? 'media' : 'text'; + // The same file twice in one message (it happens) is still two attachments. + const used = new Set(); + for (const [i, a] of atts.entries()) { + if (used.has(a.nativeId!)) a.nativeId = `${a.nativeId}#${i}`; + used.add(a.nativeId!); + } if (atts.length) m.attachments = atts; const q = sub(standard, 1); if (q) { diff --git a/packages/hub/src/store.ts b/packages/hub/src/store.ts index e09852c..35b27c1 100644 --- a/packages/hub/src/store.ts +++ b/packages/hub/src/store.ts @@ -1014,11 +1014,20 @@ export class Store { counts.same++; continue; } - if (f.body && normalizeForCompare(row.body) !== normalizeForCompare(f.body)) { - conflict('body'); + // The hub has the file and no words; the source lacks the file and says so in words: the same message. + if (!row.body && row.atts && !f.attachments?.length) { + counts.same++; continue; } let filled = false; + // Words the hub never had (a sticker's emoji, say) are filled in, not a difference. + if (!row.body && f.body) { + if (!o.dryRun) this.db.query('UPDATE message SET body = ? WHERE id = ?').run(f.body, mid); + filled = true; + } else if (f.body && normalizeForCompare(row.body) !== normalizeForCompare(f.body)) { + conflict('body'); + continue; + } if (!row.reply_to_id && f.replyTo) { if (!o.dryRun) this.db.query('UPDATE message SET reply_to_id = ? WHERE id = ?').run(hubId('msg', acc.network, f.replyTo), mid); filled = true; diff --git a/packages/hub/test/fixtures/signal-v2.ts b/packages/hub/test/fixtures/signal-v2.ts index 06d196e..394576c 100644 --- a/packages/hub/test/fixtures/signal-v2.ts +++ b/packages/hub/test/fixtures/signal-v2.ts @@ -99,8 +99,8 @@ export function writeSyntheticV2(dir: string, o: { snapshot?: string } = {}) { [[4, [[1, 11], [2, 3], [3, T + 5000], [11, [[2, text('hello club')]]]]]], [[4, [[1, 10], [2, 2], [3, T + 6000], [15, [[7, [[2, 1], [3, 1], [4, 3]]]]]]]], [[4, [[1, 11], [2, 1], [3, T + 7000], [15, [[2, [[1, []]]]]]]]], - // Note to Self. - [[4, [[1, 12], [2, 1], [3, T + 8000], [11, [[2, text('remember milk')]]]]]], + // Note to Self, with the same picture twice. + [[4, [[1, 12], [2, 1], [3, T + 8000], [11, [[2, text('remember milk')], [3, [[1, pointer]]], [3, [[1, pointer]]]]]]]], // From someone with no ACI: left out. [[4, [[1, 13], [2, 4], [3, T + 9000], [11, [[2, text('who')]]]]]], ]; diff --git a/packages/hub/test/signal-v2.test.ts b/packages/hub/test/signal-v2.test.ts index 8731200..bc42e53 100644 --- a/packages/hub/test/signal-v2.test.ts +++ b/packages/hub/test/signal-v2.test.ts @@ -56,7 +56,7 @@ describe('reading a snapshot', () => { const media = join(dir, 'media'); const lines: any[] = []; const stats = readSignalV2({ dir: src, backupKey: key(), selfId: SELF, mediaDir: media }, (l) => lines.push(l)); - expect(stats).toMatchObject({ conversations: 3, messages: 7, selfMessages: 3, duplicates: 0, files: { linked: 1, missing: 0 } }); + expect(stats).toMatchObject({ conversations: 3, messages: 7, selfMessages: 3, duplicates: 0, files: { linked: 3, missing: 0 } }); expect(stats.skipped).toEqual({ deleted: 1, sms: 1, 'chat not kept': 1 }); const convs = lines.filter((l) => l.t === 'conversation').map((l) => l.conversation); const group = `group:${groupIdFromMasterKey(MASTER_KEY).toString('base64')}`; @@ -78,6 +78,7 @@ describe('reading a snapshot', () => { expect(by(T + 6000)).toMatchObject({ kind: 'call', body: '📞 Missed voice call' }); expect(by(T + 7000)).toMatchObject({ kind: 'system', body: 'You updated the group.' }); expect(by(T + 8000)).toMatchObject({ conversation: `dm:${SELF}:${SELF}`, body: 'remember milk' }); + expect(by(T + 8000).attachments!.map((a) => a.nativeId)).toEqual([made.attachmentSha, `${made.attachmentSha}#1`]); }); }); @@ -142,9 +143,10 @@ describe('a Molly folder over HTTP', () => { await json(`/api/imports/${id}/run`, { method: 'POST', body: { dryRun: false } }); v = await until(id, (v) => !!v.run?.finishedAt); - expect(v.run).toMatchObject({ stage: 'done', counts: { messages: 7, new: 6, same: 1, conflicts: 0 }, files: { linked: 1, missing: 0 } }); + expect(v.run).toMatchObject({ stage: 'done', counts: { messages: 7, new: 6, same: 1, conflicts: 0 }, files: { linked: 3, missing: 0 } }); const n = (store.db.query(`SELECT COUNT(*) AS n FROM message WHERE network = 'signal'`).get() as any).n; expect(n).toBe(7); + expect((store.db.query(`SELECT COUNT(*) AS n FROM attachment`).get() as any).n).toBe(3); const att = store.db.query(`SELECT url FROM attachment`).get() as any; expect(att.url).toStartWith(`file://${join(hubDir, 'imports/media')}`); @@ -153,3 +155,31 @@ describe('a Molly folder over HTTP', () => { expect(v.run!.counts).toMatchObject({ new: 0, same: 7 }); }, 30_000); }); + +describe('comparing with what live sync has', () => { + test('a placeholder for a file the hub has is the same message; words the hub never had are filled in', () => { + const store = new Store(join(dir, 'cmp.sqlite'), mediaSigner('s')); + let acc = store.createAccount({ id: 'acc_s', network: 'signal', label: 'me', color: '#000', adapter: 'signal' } as any); + store.ingest(acc, { t: 'status', status: 'online', self: { nativeId: SELF, isSelf: true } }); + acc = store.account('acc_s')!; + const conversation = `dm:${SELF}:${ALICE}`; + store.ingest(acc, { t: 'message', message: { nativeId: `${ALICE}:1`, conversation, author: ALICE, ts: 1, kind: 'media', body: '', attachments: [{ kind: 'image', url: 'file:///x.png' }] }, live: false }); + store.ingest(acc, { t: 'message', message: { nativeId: `${ALICE}:2`, conversation, author: ALICE, ts: 2, kind: 'sticker', body: '' }, live: false }); + const counts = store.importBatch( + acc, + { + conversation: { nativeId: conversation, shape: 'direct', participants: [SELF, ALICE] }, + identities: [], + messages: [ + { nativeId: `${ALICE}:1`, conversation, author: ALICE, ts: 1, kind: 'media', body: '📎 Attachment (not in the backup)' }, + { nativeId: `${ALICE}:2`, conversation, author: ALICE, ts: 2, kind: 'sticker', body: '🐱' }, + ], + }, + { sourceSha: 'x', runId: 'r', dryRun: false, archiveNew: true, first: true }, + ); + expect(counts).toMatchObject({ same: 1, filled: 1, conflicts: 0 }); + expect((store.db.query(`SELECT body FROM message WHERE ts = 2`).get() as any).body).toBe('🐱'); + expect((store.db.query(`SELECT body FROM message WHERE ts = 1`).get() as any).body).toBe(''); + store.db.close(); + }); +});