diff options
| -rw-r--r-- | docs/dev/roadmap.md | 2 | ||||
| -rw-r--r-- | src/client/sync/crdt/index.ts | 15 | ||||
| -rw-r--r-- | src/client/sync/crdt/migration.test.ts | 639 | ||||
| -rw-r--r-- | src/client/sync/crdt/migration.ts | 482 |
4 files changed, 1134 insertions, 4 deletions
diff --git a/docs/dev/roadmap.md b/docs/dev/roadmap.md index 1531b63..4d8e991 100644 --- a/docs/dev/roadmap.md +++ b/docs/dev/roadmap.md @@ -38,7 +38,7 @@ Replace the current Last-Write-Wins (LWW) conflict resolution with Automerge CRD ### Phase 5: Migration -- [ ] Create `src/client/sync/crdt/migration.ts` - One-time migration script +- [x] Create `src/client/sync/crdt/migration.ts` - One-time migration script - [ ] Create server migration script to convert existing data ### Phase 6: Testing and Cleanup diff --git a/src/client/sync/crdt/index.ts b/src/client/sync/crdt/index.ts index 4a3d600..2256260 100644 --- a/src/client/sync/crdt/index.ts +++ b/src/client/sync/crdt/index.ts @@ -47,7 +47,18 @@ export { saveIncremental, updateDocument, } from "./document-manager"; - +// Migration utilities +export { + clearAllCrdtState, + getMigrationStatus, + isMigrationCompleted, + MIGRATION_VERSION, + type MigrationProgress, + type MigrationResult, + resetMigration, + runMigration, + runMigrationWithBatching, +} from "./migration"; // CRDT-aware repository wrappers export { type CrdtDocumentResult, @@ -66,7 +77,6 @@ export { getRepositoryForDocumentId, mergeAndConvert, } from "./repositories"; - // Sync state management export { base64ToBinary, @@ -80,7 +90,6 @@ export { entriesToSyncPayload, syncPayloadToEntries, } from "./sync-state"; - // Type definitions export { type CrdtCardDocument, diff --git a/src/client/sync/crdt/migration.test.ts b/src/client/sync/crdt/migration.test.ts new file mode 100644 index 0000000..22f311d --- /dev/null +++ b/src/client/sync/crdt/migration.test.ts @@ -0,0 +1,639 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import type { + LocalCard, + LocalDeck, + LocalNote, + LocalNoteFieldType, + LocalNoteFieldValue, + LocalNoteType, + LocalReviewLog, +} from "../../db/index"; +import { CardState, Rating } from "../../db/index"; +import { + crdtCardRepository, + crdtDeckRepository, + crdtNoteFieldTypeRepository, + crdtNoteFieldValueRepository, + crdtNoteRepository, + crdtNoteTypeRepository, + crdtReviewLogRepository, +} from "./repositories"; +import { crdtSyncDb, crdtSyncStateManager } from "./sync-state"; +import { CrdtEntityType } from "./types"; + +// Mock Dexie databases +vi.mock("../../db/index", async () => { + const actual = + await vi.importActual<typeof import("../../db/index")>("../../db/index"); + return { + ...actual, + db: { + decks: { + toArray: vi.fn(), + count: vi.fn(), + offset: vi.fn(), + }, + noteTypes: { + toArray: vi.fn(), + count: vi.fn(), + offset: vi.fn(), + }, + noteFieldTypes: { + toArray: vi.fn(), + count: vi.fn(), + offset: vi.fn(), + }, + notes: { + toArray: vi.fn(), + count: vi.fn(), + offset: vi.fn(), + }, + noteFieldValues: { + toArray: vi.fn(), + count: vi.fn(), + offset: vi.fn(), + }, + cards: { + toArray: vi.fn(), + count: vi.fn(), + offset: vi.fn(), + }, + reviewLogs: { + toArray: vi.fn(), + count: vi.fn(), + offset: vi.fn(), + }, + }, + }; +}); + +vi.mock("./sync-state", async () => { + const actual = + await vi.importActual<typeof import("./sync-state")>("./sync-state"); + return { + ...actual, + crdtSyncDb: { + metadata: { + get: vi.fn(), + put: vi.fn(), + delete: vi.fn(), + }, + syncState: { + clear: vi.fn(), + }, + }, + crdtSyncStateManager: { + setDocumentBinary: vi.fn(), + batchSetDocuments: vi.fn(), + clearAll: vi.fn(), + }, + }; +}); + +// Get mocked db +import { db } from "../../db/index"; +// Import the module under test after mocks are set up +import { + clearAllCrdtState, + getMigrationStatus, + isMigrationCompleted, + MIGRATION_VERSION, + resetMigration, + runMigration, + runMigrationWithBatching, +} from "./migration"; + +describe("migration", () => { + const mockDeck: LocalDeck = { + id: "deck-1", + userId: "user-1", + name: "Test Deck", + description: null, + newCardsPerDay: 20, + createdAt: new Date(), + updatedAt: new Date(), + deletedAt: null, + syncVersion: 1, + _synced: true, + }; + + const mockNoteType: LocalNoteType = { + id: "note-type-1", + userId: "user-1", + name: "Basic", + frontTemplate: "{{Front}}", + backTemplate: "{{Back}}", + isReversible: false, + createdAt: new Date(), + updatedAt: new Date(), + deletedAt: null, + syncVersion: 1, + _synced: true, + }; + + const mockNoteFieldType: LocalNoteFieldType = { + id: "field-type-1", + noteTypeId: "note-type-1", + name: "Front", + order: 0, + fieldType: "text", + createdAt: new Date(), + updatedAt: new Date(), + deletedAt: null, + syncVersion: 1, + _synced: true, + }; + + const mockNote: LocalNote = { + id: "note-1", + deckId: "deck-1", + noteTypeId: "note-type-1", + createdAt: new Date(), + updatedAt: new Date(), + deletedAt: null, + syncVersion: 1, + _synced: true, + }; + + const mockNoteFieldValue: LocalNoteFieldValue = { + id: "field-value-1", + noteId: "note-1", + noteFieldTypeId: "field-type-1", + value: "Hello", + createdAt: new Date(), + updatedAt: new Date(), + syncVersion: 1, + _synced: true, + }; + + const mockCard: LocalCard = { + id: "card-1", + deckId: "deck-1", + noteId: "note-1", + isReversed: false, + front: "Front", + back: "Back", + state: CardState.New, + due: new Date(), + stability: 0, + difficulty: 0, + elapsedDays: 0, + scheduledDays: 0, + reps: 0, + lapses: 0, + lastReview: null, + createdAt: new Date(), + updatedAt: new Date(), + deletedAt: null, + syncVersion: 1, + _synced: true, + }; + + const mockReviewLog: LocalReviewLog = { + id: "review-1", + cardId: "card-1", + userId: "user-1", + rating: Rating.Good, + state: CardState.Learning, + scheduledDays: 1, + elapsedDays: 0, + reviewedAt: new Date(), + durationMs: 1000, + syncVersion: 1, + _synced: true, + }; + + beforeEach(() => { + vi.clearAllMocks(); + + // Default mock implementations for empty database + vi.mocked(db.decks.toArray).mockResolvedValue([]); + vi.mocked(db.noteTypes.toArray).mockResolvedValue([]); + vi.mocked(db.noteFieldTypes.toArray).mockResolvedValue([]); + vi.mocked(db.notes.toArray).mockResolvedValue([]); + vi.mocked(db.noteFieldValues.toArray).mockResolvedValue([]); + vi.mocked(db.cards.toArray).mockResolvedValue([]); + vi.mocked(db.reviewLogs.toArray).mockResolvedValue([]); + + // Default: no existing migration + vi.mocked(crdtSyncDb.metadata.get).mockResolvedValue(undefined); + }); + + describe("isMigrationCompleted", () => { + it("should return false when no migration status exists", async () => { + vi.mocked(crdtSyncDb.metadata.get).mockResolvedValue(undefined); + + const result = await isMigrationCompleted(); + expect(result).toBe(false); + }); + + it("should return true when migration is completed", async () => { + vi.mocked(crdtSyncDb.metadata.get).mockResolvedValue({ + key: "crdt-migration-status", + lastSyncAt: Date.now(), + syncVersionWatermark: MIGRATION_VERSION, + actorId: JSON.stringify({ + version: MIGRATION_VERSION, + completedAt: Date.now(), + counts: { + decks: 0, + noteTypes: 0, + noteFieldTypes: 0, + notes: 0, + noteFieldValues: 0, + cards: 0, + reviewLogs: 0, + }, + }), + }); + + const result = await isMigrationCompleted(); + expect(result).toBe(true); + }); + + it("should return false when migration version is outdated", async () => { + vi.mocked(crdtSyncDb.metadata.get).mockResolvedValue({ + key: "crdt-migration-status", + lastSyncAt: Date.now(), + syncVersionWatermark: 0, + actorId: JSON.stringify({ + version: 0, // Older version + completedAt: Date.now(), + counts: { + decks: 0, + noteTypes: 0, + noteFieldTypes: 0, + notes: 0, + noteFieldValues: 0, + cards: 0, + reviewLogs: 0, + }, + }), + }); + + const result = await isMigrationCompleted(); + expect(result).toBe(false); + }); + }); + + describe("getMigrationStatus", () => { + it("should return null when no status exists", async () => { + vi.mocked(crdtSyncDb.metadata.get).mockResolvedValue(undefined); + + const result = await getMigrationStatus(); + expect(result).toBeNull(); + }); + + it("should return parsed status when exists", async () => { + const expectedStatus = { + version: MIGRATION_VERSION, + completedAt: 1234567890, + counts: { + decks: 5, + noteTypes: 2, + noteFieldTypes: 4, + notes: 10, + noteFieldValues: 20, + cards: 15, + reviewLogs: 50, + }, + }; + + vi.mocked(crdtSyncDb.metadata.get).mockResolvedValue({ + key: "crdt-migration-status", + lastSyncAt: expectedStatus.completedAt, + syncVersionWatermark: expectedStatus.version, + actorId: JSON.stringify(expectedStatus), + }); + + const result = await getMigrationStatus(); + expect(result).toEqual(expectedStatus); + }); + + it("should return null for invalid JSON", async () => { + vi.mocked(crdtSyncDb.metadata.get).mockResolvedValue({ + key: "crdt-migration-status", + lastSyncAt: 0, + syncVersionWatermark: 0, + actorId: "invalid json", + }); + + const result = await getMigrationStatus(); + expect(result).toBeNull(); + }); + }); + + describe("runMigration", () => { + it("should skip migration if already completed", async () => { + const existingStatus = { + version: MIGRATION_VERSION, + completedAt: Date.now(), + counts: { + decks: 1, + noteTypes: 0, + noteFieldTypes: 0, + notes: 0, + noteFieldValues: 0, + cards: 0, + reviewLogs: 0, + }, + }; + + vi.mocked(crdtSyncDb.metadata.get).mockResolvedValue({ + key: "crdt-migration-status", + lastSyncAt: existingStatus.completedAt, + syncVersionWatermark: existingStatus.version, + actorId: JSON.stringify(existingStatus), + }); + + const result = await runMigration(); + + expect(result.wasRun).toBe(false); + expect(result.status).toEqual(existingStatus); + expect(crdtSyncStateManager.setDocumentBinary).not.toHaveBeenCalled(); + }); + + it("should migrate all entity types", async () => { + vi.mocked(db.decks.toArray).mockResolvedValue([mockDeck]); + vi.mocked(db.noteTypes.toArray).mockResolvedValue([mockNoteType]); + vi.mocked(db.noteFieldTypes.toArray).mockResolvedValue([ + mockNoteFieldType, + ]); + vi.mocked(db.notes.toArray).mockResolvedValue([mockNote]); + vi.mocked(db.noteFieldValues.toArray).mockResolvedValue([ + mockNoteFieldValue, + ]); + vi.mocked(db.cards.toArray).mockResolvedValue([mockCard]); + vi.mocked(db.reviewLogs.toArray).mockResolvedValue([mockReviewLog]); + + const result = await runMigration(); + + expect(result.wasRun).toBe(true); + expect(result.status).toBeDefined(); + expect(result.status?.counts).toEqual({ + decks: 1, + noteTypes: 1, + noteFieldTypes: 1, + notes: 1, + noteFieldValues: 1, + cards: 1, + reviewLogs: 1, + }); + + // Verify all entity types were migrated + expect(crdtSyncStateManager.setDocumentBinary).toHaveBeenCalledTimes(7); + }); + + it("should call setDocumentBinary with correct parameters for deck", async () => { + vi.mocked(db.decks.toArray).mockResolvedValue([mockDeck]); + + await runMigration(); + + expect(crdtSyncStateManager.setDocumentBinary).toHaveBeenCalledWith( + CrdtEntityType.Deck, + mockDeck.id, + expect.any(Uint8Array), + mockDeck.syncVersion, + ); + }); + + it("should save migration status on success", async () => { + await runMigration(); + + expect(crdtSyncDb.metadata.put).toHaveBeenCalledWith( + expect.objectContaining({ + key: "crdt-migration-status", + syncVersionWatermark: MIGRATION_VERSION, + }), + ); + }); + + it("should handle errors gracefully", async () => { + const error = new Error("Database error"); + vi.mocked(db.decks.toArray).mockRejectedValue(error); + + const result = await runMigration(); + + expect(result.wasRun).toBe(true); + expect(result.status).toBeNull(); + expect(result.error).toBe(error); + }); + }); + + describe("runMigrationWithBatching", () => { + it("should skip migration if already completed", async () => { + const existingStatus = { + version: MIGRATION_VERSION, + completedAt: Date.now(), + counts: { + decks: 0, + noteTypes: 0, + noteFieldTypes: 0, + notes: 0, + noteFieldValues: 0, + cards: 0, + reviewLogs: 0, + }, + }; + + vi.mocked(crdtSyncDb.metadata.get).mockResolvedValue({ + key: "crdt-migration-status", + lastSyncAt: existingStatus.completedAt, + syncVersionWatermark: existingStatus.version, + actorId: JSON.stringify(existingStatus), + }); + + const result = await runMigrationWithBatching(10); + + expect(result.wasRun).toBe(false); + expect(result.status).toEqual(existingStatus); + }); + + it("should process entities in batches", async () => { + // Create 3 decks to test batching + const decks = [ + { ...mockDeck, id: "deck-1" }, + { ...mockDeck, id: "deck-2" }, + { ...mockDeck, id: "deck-3" }, + ]; + + // Mock the offset/limit chain for batching + let callCount = 0; + const mockOffset = vi.fn().mockImplementation(() => ({ + limit: vi.fn().mockImplementation((limit: number) => ({ + toArray: vi.fn().mockImplementation(() => { + const start = callCount * limit; + callCount++; + return Promise.resolve(decks.slice(start, start + limit)); + }), + })), + })); + + // biome-ignore lint/suspicious/noExplicitAny: Test mock + (db.decks as any).offset = mockOffset; + vi.mocked(db.decks.count).mockResolvedValue(3); + + // Empty arrays for other entity types + const emptyOffset = vi.fn().mockReturnValue({ + limit: vi.fn().mockReturnValue({ + toArray: vi.fn().mockResolvedValue([]), + }), + }); + + for (const table of [ + db.noteTypes, + db.noteFieldTypes, + db.notes, + db.noteFieldValues, + db.cards, + db.reviewLogs, + ]) { + // biome-ignore lint/suspicious/noExplicitAny: Test mock + (table as any).offset = emptyOffset; + vi.mocked(table.count).mockResolvedValue(0); + } + + const result = await runMigrationWithBatching(2); // Batch size of 2 + + expect(result.wasRun).toBe(true); + expect(result.status?.counts.decks).toBe(3); + + // Should have called batchSetDocuments for deck batches + expect(crdtSyncStateManager.batchSetDocuments).toHaveBeenCalled(); + }); + + it("should call progress callback", async () => { + const deck = { ...mockDeck, id: "deck-1" }; + + let callCount = 0; + const mockOffset = vi.fn().mockReturnValue({ + limit: vi.fn().mockReturnValue({ + toArray: vi.fn().mockImplementation(() => { + if (callCount === 0) { + callCount++; + return Promise.resolve([deck]); + } + return Promise.resolve([]); + }), + }), + }); + + // biome-ignore lint/suspicious/noExplicitAny: Test mock + (db.decks as any).offset = mockOffset; + vi.mocked(db.decks.count).mockResolvedValue(1); + + // Empty arrays for other entity types + const emptyOffset = vi.fn().mockReturnValue({ + limit: vi.fn().mockReturnValue({ + toArray: vi.fn().mockResolvedValue([]), + }), + }); + + for (const table of [ + db.noteTypes, + db.noteFieldTypes, + db.notes, + db.noteFieldValues, + db.cards, + db.reviewLogs, + ]) { + // biome-ignore lint/suspicious/noExplicitAny: Test mock + (table as any).offset = emptyOffset; + vi.mocked(table.count).mockResolvedValue(0); + } + + const progressCallback = vi.fn(); + await runMigrationWithBatching(10, progressCallback); + + expect(progressCallback).toHaveBeenCalledWith( + expect.objectContaining({ + entityType: "deck", + percentage: expect.any(Number), + }), + ); + }); + }); + + describe("resetMigration", () => { + it("should delete migration status", async () => { + await resetMigration(); + + expect(crdtSyncDb.metadata.delete).toHaveBeenCalledWith( + "crdt-migration-status", + ); + }); + }); + + describe("clearAllCrdtState", () => { + it("should clear all sync state", async () => { + await clearAllCrdtState(); + + expect(crdtSyncStateManager.clearAll).toHaveBeenCalled(); + }); + }); + + describe("CRDT document conversion", () => { + it("should convert deck to valid CRDT binary", () => { + const { binary, documentId } = + crdtDeckRepository.toCrdtDocument(mockDeck); + + expect(binary).toBeInstanceOf(Uint8Array); + expect(binary.length).toBeGreaterThan(0); + expect(documentId).toBe(`deck:${mockDeck.id}`); + }); + + it("should convert noteType to valid CRDT binary", () => { + const { binary, documentId } = + crdtNoteTypeRepository.toCrdtDocument(mockNoteType); + + expect(binary).toBeInstanceOf(Uint8Array); + expect(binary.length).toBeGreaterThan(0); + expect(documentId).toBe(`noteType:${mockNoteType.id}`); + }); + + it("should convert noteFieldType to valid CRDT binary", () => { + const { binary, documentId } = + crdtNoteFieldTypeRepository.toCrdtDocument(mockNoteFieldType); + + expect(binary).toBeInstanceOf(Uint8Array); + expect(binary.length).toBeGreaterThan(0); + expect(documentId).toBe(`noteFieldType:${mockNoteFieldType.id}`); + }); + + it("should convert note to valid CRDT binary", () => { + const { binary, documentId } = + crdtNoteRepository.toCrdtDocument(mockNote); + + expect(binary).toBeInstanceOf(Uint8Array); + expect(binary.length).toBeGreaterThan(0); + expect(documentId).toBe(`note:${mockNote.id}`); + }); + + it("should convert noteFieldValue to valid CRDT binary", () => { + const { binary, documentId } = + crdtNoteFieldValueRepository.toCrdtDocument(mockNoteFieldValue); + + expect(binary).toBeInstanceOf(Uint8Array); + expect(binary.length).toBeGreaterThan(0); + expect(documentId).toBe(`noteFieldValue:${mockNoteFieldValue.id}`); + }); + + it("should convert card to valid CRDT binary", () => { + const { binary, documentId } = + crdtCardRepository.toCrdtDocument(mockCard); + + expect(binary).toBeInstanceOf(Uint8Array); + expect(binary.length).toBeGreaterThan(0); + expect(documentId).toBe(`card:${mockCard.id}`); + }); + + it("should convert reviewLog to valid CRDT binary", () => { + const { binary, documentId } = + crdtReviewLogRepository.toCrdtDocument(mockReviewLog); + + expect(binary).toBeInstanceOf(Uint8Array); + expect(binary.length).toBeGreaterThan(0); + expect(documentId).toBe(`reviewLog:${mockReviewLog.id}`); + }); + }); +}); diff --git a/src/client/sync/crdt/migration.ts b/src/client/sync/crdt/migration.ts new file mode 100644 index 0000000..83ee583 --- /dev/null +++ b/src/client/sync/crdt/migration.ts @@ -0,0 +1,482 @@ +/** + * CRDT Migration Script + * + * One-time migration script to convert existing local entities to CRDT documents. + * This migrates data from the legacy LWW (Last-Write-Wins) sync system to the + * new Automerge CRDT-based sync system. + * + * Migration process: + * 1. Check if migration has already been completed + * 2. Read all entities from local IndexedDB + * 3. Convert each entity to an Automerge CRDT document + * 4. Store the CRDT binary in the sync state database + * 5. Mark migration as complete + * + * The migration is idempotent - running it multiple times has no effect + * after the first successful run. + */ + +import { db } from "../../db/index"; +import { + crdtCardRepository, + crdtDeckRepository, + crdtNoteFieldTypeRepository, + crdtNoteFieldValueRepository, + crdtNoteRepository, + crdtNoteTypeRepository, + crdtReviewLogRepository, +} from "./repositories"; +import { crdtSyncDb, crdtSyncStateManager } from "./sync-state"; +import { CrdtEntityType } from "./types"; + +/** + * Migration status stored in IndexedDB + */ +interface MigrationStatus { + /** Migration version */ + version: number; + /** Timestamp when migration was completed */ + completedAt: number; + /** Entity counts migrated */ + counts: { + decks: number; + noteTypes: number; + noteFieldTypes: number; + notes: number; + noteFieldValues: number; + cards: number; + reviewLogs: number; + }; +} + +/** + * Current migration version + * Increment this when making breaking changes to the migration logic + */ +export const MIGRATION_VERSION = 1; + +/** + * Key used to store migration status in metadata + */ +const MIGRATION_STATUS_KEY = "crdt-migration-status"; + +/** + * Result of running the migration + */ +export interface MigrationResult { + /** Whether the migration was run (false if already completed) */ + wasRun: boolean; + /** Migration status after completion */ + status: MigrationStatus | null; + /** Error if migration failed */ + error?: Error; +} + +/** + * Check if migration has already been completed + */ +export async function isMigrationCompleted(): Promise<boolean> { + const status = await getMigrationStatus(); + return status !== null && status.version >= MIGRATION_VERSION; +} + +/** + * Get the current migration status + */ +export async function getMigrationStatus(): Promise<MigrationStatus | null> { + const entry = await crdtSyncDb.metadata.get(MIGRATION_STATUS_KEY); + if (!entry) { + return null; + } + + // Parse the status from the metadata entry + // We store it as a JSON string in the actorId field for simplicity + try { + return JSON.parse(entry.actorId) as MigrationStatus; + } catch { + return null; + } +} + +/** + * Save the migration status + */ +async function saveMigrationStatus(status: MigrationStatus): Promise<void> { + await crdtSyncDb.metadata.put({ + key: MIGRATION_STATUS_KEY, + lastSyncAt: status.completedAt, + syncVersionWatermark: status.version, + actorId: JSON.stringify(status), + }); +} + +/** + * Run the CRDT migration + * + * Converts all existing local entities to CRDT documents and stores them + * in the CRDT sync state database. + * + * @returns Migration result indicating whether migration was run and the status + */ +export async function runMigration(): Promise<MigrationResult> { + // Check if migration is already completed + if (await isMigrationCompleted()) { + const status = await getMigrationStatus(); + return { wasRun: false, status }; + } + + try { + const counts = { + decks: 0, + noteTypes: 0, + noteFieldTypes: 0, + notes: 0, + noteFieldValues: 0, + cards: 0, + reviewLogs: 0, + }; + + // Migrate Decks + const decks = await db.decks.toArray(); + for (const deck of decks) { + const { binary } = crdtDeckRepository.toCrdtDocument(deck); + await crdtSyncStateManager.setDocumentBinary( + CrdtEntityType.Deck, + deck.id, + binary, + deck.syncVersion, + ); + counts.decks++; + } + + // Migrate NoteTypes + const noteTypes = await db.noteTypes.toArray(); + for (const noteType of noteTypes) { + const { binary } = crdtNoteTypeRepository.toCrdtDocument(noteType); + await crdtSyncStateManager.setDocumentBinary( + CrdtEntityType.NoteType, + noteType.id, + binary, + noteType.syncVersion, + ); + counts.noteTypes++; + } + + // Migrate NoteFieldTypes + const noteFieldTypes = await db.noteFieldTypes.toArray(); + for (const noteFieldType of noteFieldTypes) { + const { binary } = + crdtNoteFieldTypeRepository.toCrdtDocument(noteFieldType); + await crdtSyncStateManager.setDocumentBinary( + CrdtEntityType.NoteFieldType, + noteFieldType.id, + binary, + noteFieldType.syncVersion, + ); + counts.noteFieldTypes++; + } + + // Migrate Notes + const notes = await db.notes.toArray(); + for (const note of notes) { + const { binary } = crdtNoteRepository.toCrdtDocument(note); + await crdtSyncStateManager.setDocumentBinary( + CrdtEntityType.Note, + note.id, + binary, + note.syncVersion, + ); + counts.notes++; + } + + // Migrate NoteFieldValues + const noteFieldValues = await db.noteFieldValues.toArray(); + for (const noteFieldValue of noteFieldValues) { + const { binary } = + crdtNoteFieldValueRepository.toCrdtDocument(noteFieldValue); + await crdtSyncStateManager.setDocumentBinary( + CrdtEntityType.NoteFieldValue, + noteFieldValue.id, + binary, + noteFieldValue.syncVersion, + ); + counts.noteFieldValues++; + } + + // Migrate Cards + const cards = await db.cards.toArray(); + for (const card of cards) { + const { binary } = crdtCardRepository.toCrdtDocument(card); + await crdtSyncStateManager.setDocumentBinary( + CrdtEntityType.Card, + card.id, + binary, + card.syncVersion, + ); + counts.cards++; + } + + // Migrate ReviewLogs + const reviewLogs = await db.reviewLogs.toArray(); + for (const reviewLog of reviewLogs) { + const { binary } = crdtReviewLogRepository.toCrdtDocument(reviewLog); + await crdtSyncStateManager.setDocumentBinary( + CrdtEntityType.ReviewLog, + reviewLog.id, + binary, + reviewLog.syncVersion, + ); + counts.reviewLogs++; + } + + // Save migration status + const status: MigrationStatus = { + version: MIGRATION_VERSION, + completedAt: Date.now(), + counts, + }; + await saveMigrationStatus(status); + + return { wasRun: true, status }; + } catch (error) { + return { + wasRun: true, + status: null, + error: error instanceof Error ? error : new Error(String(error)), + }; + } +} + +/** + * Run migration with batching for better performance with large datasets + * + * This version processes entities in batches to avoid memory issues + * and provide progress feedback for large migrations. + * + * @param batchSize Number of entities to process per batch + * @param onProgress Optional callback for progress updates + * @returns Migration result + */ +export async function runMigrationWithBatching( + batchSize = 100, + onProgress?: (progress: MigrationProgress) => void, +): Promise<MigrationResult> { + // Check if migration is already completed + if (await isMigrationCompleted()) { + const status = await getMigrationStatus(); + return { wasRun: false, status }; + } + + try { + const counts = { + decks: 0, + noteTypes: 0, + noteFieldTypes: 0, + notes: 0, + noteFieldValues: 0, + cards: 0, + reviewLogs: 0, + }; + + // Get total counts for progress reporting + const totalCounts = { + decks: await db.decks.count(), + noteTypes: await db.noteTypes.count(), + noteFieldTypes: await db.noteFieldTypes.count(), + notes: await db.notes.count(), + noteFieldValues: await db.noteFieldValues.count(), + cards: await db.cards.count(), + reviewLogs: await db.reviewLogs.count(), + }; + + const totalEntities = Object.values(totalCounts).reduce( + (sum, count) => sum + count, + 0, + ); + let processedEntities = 0; + + // Helper to report progress + const reportProgress = (entityType: string, current: number) => { + processedEntities++; + if (onProgress) { + onProgress({ + entityType, + current, + total: totalEntities, + processed: processedEntities, + percentage: Math.round((processedEntities / totalEntities) * 100), + }); + } + }; + + // Migrate Decks in batches + counts.decks = await migrateEntityType( + db.decks, + crdtDeckRepository, + CrdtEntityType.Deck, + batchSize, + () => reportProgress("deck", counts.decks + 1), + ); + + // Migrate NoteTypes in batches + counts.noteTypes = await migrateEntityType( + db.noteTypes, + crdtNoteTypeRepository, + CrdtEntityType.NoteType, + batchSize, + () => reportProgress("noteType", counts.noteTypes + 1), + ); + + // Migrate NoteFieldTypes in batches + counts.noteFieldTypes = await migrateEntityType( + db.noteFieldTypes, + crdtNoteFieldTypeRepository, + CrdtEntityType.NoteFieldType, + batchSize, + () => reportProgress("noteFieldType", counts.noteFieldTypes + 1), + ); + + // Migrate Notes in batches + counts.notes = await migrateEntityType( + db.notes, + crdtNoteRepository, + CrdtEntityType.Note, + batchSize, + () => reportProgress("note", counts.notes + 1), + ); + + // Migrate NoteFieldValues in batches + counts.noteFieldValues = await migrateEntityType( + db.noteFieldValues, + crdtNoteFieldValueRepository, + CrdtEntityType.NoteFieldValue, + batchSize, + () => reportProgress("noteFieldValue", counts.noteFieldValues + 1), + ); + + // Migrate Cards in batches + counts.cards = await migrateEntityType( + db.cards, + crdtCardRepository, + CrdtEntityType.Card, + batchSize, + () => reportProgress("card", counts.cards + 1), + ); + + // Migrate ReviewLogs in batches + counts.reviewLogs = await migrateEntityType( + db.reviewLogs, + crdtReviewLogRepository, + CrdtEntityType.ReviewLog, + batchSize, + () => reportProgress("reviewLog", counts.reviewLogs + 1), + ); + + // Save migration status + const status: MigrationStatus = { + version: MIGRATION_VERSION, + completedAt: Date.now(), + counts, + }; + await saveMigrationStatus(status); + + return { wasRun: true, status }; + } catch (error) { + return { + wasRun: true, + status: null, + error: error instanceof Error ? error : new Error(String(error)), + }; + } +} + +/** + * Progress information for batch migration + */ +export interface MigrationProgress { + /** Current entity type being migrated */ + entityType: string; + /** Current entity number within type */ + current: number; + /** Total entities across all types */ + total: number; + /** Total entities processed so far */ + processed: number; + /** Percentage complete (0-100) */ + percentage: number; +} + +/** + * Interface representing a table with offset/limit capabilities + * Used for batch processing of entities + */ +interface BatchableTable<T> { + offset(n: number): { limit(n: number): { toArray(): Promise<T[]> } }; +} + +/** + * Helper to migrate entities of a specific type in batches + */ +async function migrateEntityType<T extends { id: string; syncVersion: number }>( + table: BatchableTable<T>, + repository: { toCrdtDocument: (entity: T) => { binary: Uint8Array } }, + entityType: string, + batchSize: number, + onEntity?: () => void, +): Promise<number> { + let count = 0; + let offset = 0; + + for (;;) { + const batch = await table.offset(offset).limit(batchSize).toArray(); + if (batch.length === 0) { + break; + } + + // Prepare batch entries for bulk insert + const entries = batch.map((entity) => { + const { binary } = repository.toCrdtDocument(entity); + return { + entityType: entityType as import("./types").CrdtEntityTypeValue, + entityId: entity.id, + binary, + syncVersion: entity.syncVersion, + }; + }); + + // Batch insert + await crdtSyncStateManager.batchSetDocuments(entries); + + count += batch.length; + offset += batchSize; + + // Call progress callback for each entity + if (onEntity) { + for (let i = 0; i < batch.length; i++) { + onEntity(); + } + } + } + + return count; +} + +/** + * Reset migration status (for testing or retry purposes) + * + * WARNING: This should only be used for development/testing. + * In production, resetting migration could lead to duplicate CRDT documents. + */ +export async function resetMigration(): Promise<void> { + await crdtSyncDb.metadata.delete(MIGRATION_STATUS_KEY); +} + +/** + * Clear all CRDT sync state (for full reset) + * + * WARNING: This will delete all CRDT documents and require a full resync. + * Only use this for development/testing or when explicitly requested by user. + */ +export async function clearAllCrdtState(): Promise<void> { + await crdtSyncStateManager.clearAll(); +} |