From a1383a9304ff457d6671e12ded4265b135256004 Mon Sep 17 00:00:00 2001 From: nsfisis Date: Wed, 31 Dec 2025 15:25:36 +0900 Subject: feat(crdt): add crdtChanges to sync push payload MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add CRDT document generation to the sync push flow. Each pending entity is now converted to an Automerge CRDT document and included as base64- encoded binary in the push payload alongside the existing entity data. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 --- docs/dev/roadmap.md | 2 +- src/client/sync/push.test.ts | 437 ++++++++++++++++++++++++++++++++++++++++++- src/client/sync/push.ts | 102 ++++++++++ 3 files changed, 539 insertions(+), 2 deletions(-) diff --git a/docs/dev/roadmap.md b/docs/dev/roadmap.md index 656b10d..d17ba41 100644 --- a/docs/dev/roadmap.md +++ b/docs/dev/roadmap.md @@ -23,7 +23,7 @@ Replace the current Last-Write-Wins (LWW) conflict resolution with Automerge CRD ### Phase 3: Modify Sync Protocol -- [ ] Modify `src/client/sync/push.ts` - Add crdtChanges to push payload +- [x] Modify `src/client/sync/push.ts` - Add crdtChanges to push payload - [ ] Modify `src/client/sync/pull.ts` - Handle crdtChanges in pull response - [ ] Modify `src/client/sync/conflict.ts` - Replace LWW with Automerge merge - [ ] Modify `src/client/sync/manager.ts` - Integrate CRDT sync flow diff --git a/src/client/sync/push.test.ts b/src/client/sync/push.test.ts index 16198c1..bce4652 100644 --- a/src/client/sync/push.test.ts +++ b/src/client/sync/push.test.ts @@ -13,7 +13,13 @@ import { localNoteTypeRepository, localReviewLogRepository, } from "../db/repositories"; -import { PushService, pendingChangesToPushData } from "./push"; +import { base64ToBinary } from "./crdt/sync-state"; +import { CrdtEntityType } from "./crdt/types"; +import { + generateCrdtChanges, + PushService, + pendingChangesToPushData, +} from "./push"; import type { PendingChanges } from "./queue"; import { SyncQueue } from "./queue"; @@ -61,6 +67,9 @@ function createEmptyPushData(): Omit< noteFieldTypes: [], notes: [], noteFieldValues: [], + crdtChanges: expect.any( + Array, + ) as import("./crdt/sync-state").CrdtSyncPayload[], }; } @@ -1097,3 +1106,429 @@ describe("PushService", () => { }); }); }); + +describe("generateCrdtChanges", () => { + it("should generate CRDT changes for decks", () => { + const changes: PendingChanges = { + decks: [ + { + id: "deck-1", + userId: "user-1", + name: "Test Deck", + description: null, + newCardsPerDay: 20, + createdAt: new Date("2024-01-01T10:00:00Z"), + updatedAt: new Date("2024-01-02T15:30:00Z"), + deletedAt: null, + syncVersion: 0, + _synced: false, + }, + ], + cards: [], + reviewLogs: [], + noteTypes: [], + noteFieldTypes: [], + notes: [], + noteFieldValues: [], + }; + + const crdtChanges = generateCrdtChanges(changes); + + expect(crdtChanges).toHaveLength(1); + expect(crdtChanges[0]?.entityType).toBe(CrdtEntityType.Deck); + expect(crdtChanges[0]?.entityId).toBe("deck-1"); + expect(crdtChanges[0]?.documentId).toBe("deck:deck-1"); + expect(crdtChanges[0]?.binary).toBeDefined(); + // Verify it's valid base64 + expect(() => base64ToBinary(crdtChanges[0]!.binary)).not.toThrow(); + }); + + it("should generate CRDT changes for cards", () => { + const changes: PendingChanges = { + decks: [], + cards: [ + { + id: "card-1", + deckId: "deck-1", + noteId: "note-1", + isReversed: false, + front: "Question", + back: "Answer", + state: CardState.New, + due: new Date("2024-01-01T10:00:00Z"), + stability: 0, + difficulty: 0, + elapsedDays: 0, + scheduledDays: 0, + reps: 0, + lapses: 0, + lastReview: null, + createdAt: new Date("2024-01-01T10:00:00Z"), + updatedAt: new Date("2024-01-01T10:00:00Z"), + deletedAt: null, + syncVersion: 0, + _synced: false, + }, + ], + reviewLogs: [], + noteTypes: [], + noteFieldTypes: [], + notes: [], + noteFieldValues: [], + }; + + const crdtChanges = generateCrdtChanges(changes); + + expect(crdtChanges).toHaveLength(1); + expect(crdtChanges[0]?.entityType).toBe(CrdtEntityType.Card); + expect(crdtChanges[0]?.entityId).toBe("card-1"); + expect(crdtChanges[0]?.documentId).toBe("card:card-1"); + }); + + it("should generate CRDT changes for review logs", () => { + const changes: PendingChanges = { + decks: [], + cards: [], + reviewLogs: [ + { + id: "log-1", + cardId: "card-1", + userId: "user-1", + rating: Rating.Good, + state: CardState.New, + scheduledDays: 1, + elapsedDays: 0, + reviewedAt: new Date("2024-01-02T10:00:00Z"), + durationMs: 5000, + syncVersion: 0, + _synced: false, + }, + ], + noteTypes: [], + noteFieldTypes: [], + notes: [], + noteFieldValues: [], + }; + + const crdtChanges = generateCrdtChanges(changes); + + expect(crdtChanges).toHaveLength(1); + expect(crdtChanges[0]?.entityType).toBe(CrdtEntityType.ReviewLog); + expect(crdtChanges[0]?.entityId).toBe("log-1"); + expect(crdtChanges[0]?.documentId).toBe("reviewLog:log-1"); + }); + + it("should generate CRDT changes for note types", () => { + const changes: PendingChanges = { + decks: [], + cards: [], + reviewLogs: [], + noteTypes: [ + { + id: "note-type-1", + userId: "user-1", + name: "Basic", + frontTemplate: "{{Front}}", + backTemplate: "{{Back}}", + isReversible: true, + createdAt: new Date("2024-01-01T10:00:00Z"), + updatedAt: new Date("2024-01-02T15:30:00Z"), + deletedAt: null, + syncVersion: 0, + _synced: false, + }, + ], + noteFieldTypes: [], + notes: [], + noteFieldValues: [], + }; + + const crdtChanges = generateCrdtChanges(changes); + + expect(crdtChanges).toHaveLength(1); + expect(crdtChanges[0]?.entityType).toBe(CrdtEntityType.NoteType); + expect(crdtChanges[0]?.entityId).toBe("note-type-1"); + expect(crdtChanges[0]?.documentId).toBe("noteType:note-type-1"); + }); + + it("should generate CRDT changes for note field types", () => { + const changes: PendingChanges = { + decks: [], + cards: [], + reviewLogs: [], + noteTypes: [], + noteFieldTypes: [ + { + id: "field-type-1", + noteTypeId: "note-type-1", + name: "Front", + order: 0, + fieldType: FieldType.Text, + createdAt: new Date("2024-01-01T10:00:00Z"), + updatedAt: new Date("2024-01-02T15:30:00Z"), + deletedAt: null, + syncVersion: 0, + _synced: false, + }, + ], + notes: [], + noteFieldValues: [], + }; + + const crdtChanges = generateCrdtChanges(changes); + + expect(crdtChanges).toHaveLength(1); + expect(crdtChanges[0]?.entityType).toBe(CrdtEntityType.NoteFieldType); + expect(crdtChanges[0]?.entityId).toBe("field-type-1"); + expect(crdtChanges[0]?.documentId).toBe("noteFieldType:field-type-1"); + }); + + it("should generate CRDT changes for notes", () => { + const changes: PendingChanges = { + decks: [], + cards: [], + reviewLogs: [], + noteTypes: [], + noteFieldTypes: [], + notes: [ + { + id: "note-1", + deckId: "deck-1", + noteTypeId: "note-type-1", + createdAt: new Date("2024-01-01T10:00:00Z"), + updatedAt: new Date("2024-01-02T15:30:00Z"), + deletedAt: null, + syncVersion: 0, + _synced: false, + }, + ], + noteFieldValues: [], + }; + + const crdtChanges = generateCrdtChanges(changes); + + expect(crdtChanges).toHaveLength(1); + expect(crdtChanges[0]?.entityType).toBe(CrdtEntityType.Note); + expect(crdtChanges[0]?.entityId).toBe("note-1"); + expect(crdtChanges[0]?.documentId).toBe("note:note-1"); + }); + + it("should generate CRDT changes for note field values", () => { + const changes: PendingChanges = { + decks: [], + cards: [], + reviewLogs: [], + noteTypes: [], + noteFieldTypes: [], + notes: [], + noteFieldValues: [ + { + id: "field-value-1", + noteId: "note-1", + noteFieldTypeId: "field-type-1", + value: "What is 2+2?", + createdAt: new Date("2024-01-01T10:00:00Z"), + updatedAt: new Date("2024-01-02T15:30:00Z"), + syncVersion: 0, + _synced: false, + }, + ], + }; + + const crdtChanges = generateCrdtChanges(changes); + + expect(crdtChanges).toHaveLength(1); + expect(crdtChanges[0]?.entityType).toBe(CrdtEntityType.NoteFieldValue); + expect(crdtChanges[0]?.entityId).toBe("field-value-1"); + expect(crdtChanges[0]?.documentId).toBe("noteFieldValue:field-value-1"); + }); + + it("should generate CRDT changes for all entity types in correct order", () => { + const changes: PendingChanges = { + decks: [ + { + id: "deck-1", + userId: "user-1", + name: "Test Deck", + description: null, + newCardsPerDay: 20, + createdAt: new Date("2024-01-01T10:00:00Z"), + updatedAt: new Date("2024-01-02T15:30:00Z"), + deletedAt: null, + syncVersion: 0, + _synced: false, + }, + ], + cards: [ + { + id: "card-1", + deckId: "deck-1", + noteId: "note-1", + isReversed: false, + front: "Q", + back: "A", + state: CardState.New, + due: new Date("2024-01-01T10:00:00Z"), + stability: 0, + difficulty: 0, + elapsedDays: 0, + scheduledDays: 0, + reps: 0, + lapses: 0, + lastReview: null, + createdAt: new Date("2024-01-01T10:00:00Z"), + updatedAt: new Date("2024-01-01T10:00:00Z"), + deletedAt: null, + syncVersion: 0, + _synced: false, + }, + ], + reviewLogs: [ + { + id: "log-1", + cardId: "card-1", + userId: "user-1", + rating: Rating.Good, + state: CardState.New, + scheduledDays: 1, + elapsedDays: 0, + reviewedAt: new Date("2024-01-02T10:00:00Z"), + durationMs: 5000, + syncVersion: 0, + _synced: false, + }, + ], + noteTypes: [ + { + id: "note-type-1", + userId: "user-1", + name: "Basic", + frontTemplate: "{{Front}}", + backTemplate: "{{Back}}", + isReversible: false, + createdAt: new Date("2024-01-01T10:00:00Z"), + updatedAt: new Date("2024-01-02T15:30:00Z"), + deletedAt: null, + syncVersion: 0, + _synced: false, + }, + ], + noteFieldTypes: [ + { + id: "field-type-1", + noteTypeId: "note-type-1", + name: "Front", + order: 0, + fieldType: FieldType.Text, + createdAt: new Date("2024-01-01T10:00:00Z"), + updatedAt: new Date("2024-01-02T15:30:00Z"), + deletedAt: null, + syncVersion: 0, + _synced: false, + }, + ], + notes: [ + { + id: "note-1", + deckId: "deck-1", + noteTypeId: "note-type-1", + createdAt: new Date("2024-01-01T10:00:00Z"), + updatedAt: new Date("2024-01-02T15:30:00Z"), + deletedAt: null, + syncVersion: 0, + _synced: false, + }, + ], + noteFieldValues: [ + { + id: "field-value-1", + noteId: "note-1", + noteFieldTypeId: "field-type-1", + value: "What is 2+2?", + createdAt: new Date("2024-01-01T10:00:00Z"), + updatedAt: new Date("2024-01-02T15:30:00Z"), + syncVersion: 0, + _synced: false, + }, + ], + }; + + const crdtChanges = generateCrdtChanges(changes); + + expect(crdtChanges).toHaveLength(7); + + // Verify order: decks, noteTypes, noteFieldTypes, notes, noteFieldValues, cards, reviewLogs + expect(crdtChanges[0]?.entityType).toBe(CrdtEntityType.Deck); + expect(crdtChanges[1]?.entityType).toBe(CrdtEntityType.NoteType); + expect(crdtChanges[2]?.entityType).toBe(CrdtEntityType.NoteFieldType); + expect(crdtChanges[3]?.entityType).toBe(CrdtEntityType.Note); + expect(crdtChanges[4]?.entityType).toBe(CrdtEntityType.NoteFieldValue); + expect(crdtChanges[5]?.entityType).toBe(CrdtEntityType.Card); + expect(crdtChanges[6]?.entityType).toBe(CrdtEntityType.ReviewLog); + }); + + it("should return empty array for empty pending changes", () => { + const changes: PendingChanges = { + decks: [], + cards: [], + reviewLogs: [], + noteTypes: [], + noteFieldTypes: [], + notes: [], + noteFieldValues: [], + }; + + const crdtChanges = generateCrdtChanges(changes); + + expect(crdtChanges).toHaveLength(0); + }); +}); + +describe("pendingChangesToPushData with crdtChanges", () => { + it("should include crdtChanges in push data", () => { + const changes: PendingChanges = { + decks: [ + { + id: "deck-1", + userId: "user-1", + name: "Test Deck", + description: null, + newCardsPerDay: 20, + createdAt: new Date("2024-01-01T10:00:00Z"), + updatedAt: new Date("2024-01-02T15:30:00Z"), + deletedAt: null, + syncVersion: 0, + _synced: false, + }, + ], + cards: [], + reviewLogs: [], + noteTypes: [], + noteFieldTypes: [], + notes: [], + noteFieldValues: [], + }; + + const pushData = pendingChangesToPushData(changes); + + expect(pushData.crdtChanges).toHaveLength(1); + expect(pushData.crdtChanges[0]?.entityType).toBe(CrdtEntityType.Deck); + expect(pushData.crdtChanges[0]?.entityId).toBe("deck-1"); + }); + + it("should include empty crdtChanges for empty pending changes", () => { + const changes: PendingChanges = { + decks: [], + cards: [], + reviewLogs: [], + noteTypes: [], + noteFieldTypes: [], + notes: [], + noteFieldValues: [], + }; + + const pushData = pendingChangesToPushData(changes); + + expect(pushData.crdtChanges).toHaveLength(0); + }); +}); diff --git a/src/client/sync/push.ts b/src/client/sync/push.ts index b83136e..eea671b 100644 --- a/src/client/sync/push.ts +++ b/src/client/sync/push.ts @@ -7,6 +7,17 @@ import type { LocalNoteType, LocalReviewLog, } from "../db/index"; +import { + crdtCardRepository, + crdtDeckRepository, + crdtNoteFieldTypeRepository, + crdtNoteFieldValueRepository, + crdtNoteRepository, + crdtNoteTypeRepository, + crdtReviewLogRepository, +} from "./crdt"; +import type { CrdtSyncPayload } from "./crdt/sync-state"; +import { binaryToBase64 } from "./crdt/sync-state"; import type { PendingChanges, SyncQueue } from "./queue"; /** @@ -20,6 +31,8 @@ export interface SyncPushData { noteFieldTypes: SyncNoteFieldTypeData[]; notes: SyncNoteData[]; noteFieldValues: SyncNoteFieldValueData[]; + /** CRDT document changes for conflict-free sync */ + crdtChanges: CrdtSyncPayload[]; } export interface SyncDeckData { @@ -254,6 +267,94 @@ function noteFieldValueToSyncData( }; } +/** + * Generate CRDT sync payloads from pending changes + */ +export function generateCrdtChanges( + changes: PendingChanges, +): CrdtSyncPayload[] { + const crdtChanges: CrdtSyncPayload[] = []; + + // Convert decks to CRDT documents + for (const deck of changes.decks) { + const result = crdtDeckRepository.toCrdtDocument(deck); + crdtChanges.push({ + documentId: result.documentId, + entityType: crdtDeckRepository.entityType, + entityId: deck.id, + binary: binaryToBase64(result.binary), + }); + } + + // Convert note types to CRDT documents + for (const noteType of changes.noteTypes) { + const result = crdtNoteTypeRepository.toCrdtDocument(noteType); + crdtChanges.push({ + documentId: result.documentId, + entityType: crdtNoteTypeRepository.entityType, + entityId: noteType.id, + binary: binaryToBase64(result.binary), + }); + } + + // Convert note field types to CRDT documents + for (const fieldType of changes.noteFieldTypes) { + const result = crdtNoteFieldTypeRepository.toCrdtDocument(fieldType); + crdtChanges.push({ + documentId: result.documentId, + entityType: crdtNoteFieldTypeRepository.entityType, + entityId: fieldType.id, + binary: binaryToBase64(result.binary), + }); + } + + // Convert notes to CRDT documents + for (const note of changes.notes) { + const result = crdtNoteRepository.toCrdtDocument(note); + crdtChanges.push({ + documentId: result.documentId, + entityType: crdtNoteRepository.entityType, + entityId: note.id, + binary: binaryToBase64(result.binary), + }); + } + + // Convert note field values to CRDT documents + for (const fieldValue of changes.noteFieldValues) { + const result = crdtNoteFieldValueRepository.toCrdtDocument(fieldValue); + crdtChanges.push({ + documentId: result.documentId, + entityType: crdtNoteFieldValueRepository.entityType, + entityId: fieldValue.id, + binary: binaryToBase64(result.binary), + }); + } + + // Convert cards to CRDT documents + for (const card of changes.cards) { + const result = crdtCardRepository.toCrdtDocument(card); + crdtChanges.push({ + documentId: result.documentId, + entityType: crdtCardRepository.entityType, + entityId: card.id, + binary: binaryToBase64(result.binary), + }); + } + + // Convert review logs to CRDT documents + for (const reviewLog of changes.reviewLogs) { + const result = crdtReviewLogRepository.toCrdtDocument(reviewLog); + crdtChanges.push({ + documentId: result.documentId, + entityType: crdtReviewLogRepository.entityType, + entityId: reviewLog.id, + binary: binaryToBase64(result.binary), + }); + } + + return crdtChanges; +} + /** * Convert pending changes to sync push data format */ @@ -268,6 +369,7 @@ export function pendingChangesToPushData( noteFieldTypes: changes.noteFieldTypes.map(noteFieldTypeToSyncData), notes: changes.notes.map(noteToSyncData), noteFieldValues: changes.noteFieldValues.map(noteFieldValueToSyncData), + crdtChanges: generateCrdtChanges(changes), }; } -- cgit v1.2.3-70-g09d2