aboutsummaryrefslogtreecommitdiffhomepage
diff options
context:
space:
mode:
authornsfisis <nsfisis@gmail.com>2025-12-31 16:59:55 +0900
committernsfisis <nsfisis@gmail.com>2025-12-31 16:59:55 +0900
commita7d1579d6c9463016ab9313574772aa2959363f8 (patch)
tree8697fc207226da948bb5ca002bbf7c71cdbe6f15
parent29aed156d0ec252c3ce49c5c68183aaa6d45a531 (diff)
downloadkioku-a7d1579d6c9463016ab9313574772aa2959363f8.tar.gz
kioku-a7d1579d6c9463016ab9313574772aa2959363f8.tar.zst
kioku-a7d1579d6c9463016ab9313574772aa2959363f8.zip
feat(crdt): add client-side CRDT migration script
Add one-time migration script to convert existing local IndexedDB entities to Automerge CRDT documents. Includes: - Migration with idempotency check (runs only once) - Batch processing option for large datasets - Progress callback for UI feedback - Unit tests for migration logic 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
-rw-r--r--docs/dev/roadmap.md2
-rw-r--r--src/client/sync/crdt/index.ts15
-rw-r--r--src/client/sync/crdt/migration.test.ts639
-rw-r--r--src/client/sync/crdt/migration.ts482
4 files changed, 1134 insertions, 4 deletions
diff --git a/docs/dev/roadmap.md b/docs/dev/roadmap.md
index 1531b63..4d8e991 100644
--- a/docs/dev/roadmap.md
+++ b/docs/dev/roadmap.md
@@ -38,7 +38,7 @@ Replace the current Last-Write-Wins (LWW) conflict resolution with Automerge CRD
### Phase 5: Migration
-- [ ] Create `src/client/sync/crdt/migration.ts` - One-time migration script
+- [x] Create `src/client/sync/crdt/migration.ts` - One-time migration script
- [ ] Create server migration script to convert existing data
### Phase 6: Testing and Cleanup
diff --git a/src/client/sync/crdt/index.ts b/src/client/sync/crdt/index.ts
index 4a3d600..2256260 100644
--- a/src/client/sync/crdt/index.ts
+++ b/src/client/sync/crdt/index.ts
@@ -47,7 +47,18 @@ export {
saveIncremental,
updateDocument,
} from "./document-manager";
-
+// Migration utilities
+export {
+ clearAllCrdtState,
+ getMigrationStatus,
+ isMigrationCompleted,
+ MIGRATION_VERSION,
+ type MigrationProgress,
+ type MigrationResult,
+ resetMigration,
+ runMigration,
+ runMigrationWithBatching,
+} from "./migration";
// CRDT-aware repository wrappers
export {
type CrdtDocumentResult,
@@ -66,7 +77,6 @@ export {
getRepositoryForDocumentId,
mergeAndConvert,
} from "./repositories";
-
// Sync state management
export {
base64ToBinary,
@@ -80,7 +90,6 @@ export {
entriesToSyncPayload,
syncPayloadToEntries,
} from "./sync-state";
-
// Type definitions
export {
type CrdtCardDocument,
diff --git a/src/client/sync/crdt/migration.test.ts b/src/client/sync/crdt/migration.test.ts
new file mode 100644
index 0000000..22f311d
--- /dev/null
+++ b/src/client/sync/crdt/migration.test.ts
@@ -0,0 +1,639 @@
+import { beforeEach, describe, expect, it, vi } from "vitest";
+import type {
+ LocalCard,
+ LocalDeck,
+ LocalNote,
+ LocalNoteFieldType,
+ LocalNoteFieldValue,
+ LocalNoteType,
+ LocalReviewLog,
+} from "../../db/index";
+import { CardState, Rating } from "../../db/index";
+import {
+ crdtCardRepository,
+ crdtDeckRepository,
+ crdtNoteFieldTypeRepository,
+ crdtNoteFieldValueRepository,
+ crdtNoteRepository,
+ crdtNoteTypeRepository,
+ crdtReviewLogRepository,
+} from "./repositories";
+import { crdtSyncDb, crdtSyncStateManager } from "./sync-state";
+import { CrdtEntityType } from "./types";
+
+// Mock Dexie databases
+vi.mock("../../db/index", async () => {
+ const actual =
+ await vi.importActual<typeof import("../../db/index")>("../../db/index");
+ return {
+ ...actual,
+ db: {
+ decks: {
+ toArray: vi.fn(),
+ count: vi.fn(),
+ offset: vi.fn(),
+ },
+ noteTypes: {
+ toArray: vi.fn(),
+ count: vi.fn(),
+ offset: vi.fn(),
+ },
+ noteFieldTypes: {
+ toArray: vi.fn(),
+ count: vi.fn(),
+ offset: vi.fn(),
+ },
+ notes: {
+ toArray: vi.fn(),
+ count: vi.fn(),
+ offset: vi.fn(),
+ },
+ noteFieldValues: {
+ toArray: vi.fn(),
+ count: vi.fn(),
+ offset: vi.fn(),
+ },
+ cards: {
+ toArray: vi.fn(),
+ count: vi.fn(),
+ offset: vi.fn(),
+ },
+ reviewLogs: {
+ toArray: vi.fn(),
+ count: vi.fn(),
+ offset: vi.fn(),
+ },
+ },
+ };
+});
+
+vi.mock("./sync-state", async () => {
+ const actual =
+ await vi.importActual<typeof import("./sync-state")>("./sync-state");
+ return {
+ ...actual,
+ crdtSyncDb: {
+ metadata: {
+ get: vi.fn(),
+ put: vi.fn(),
+ delete: vi.fn(),
+ },
+ syncState: {
+ clear: vi.fn(),
+ },
+ },
+ crdtSyncStateManager: {
+ setDocumentBinary: vi.fn(),
+ batchSetDocuments: vi.fn(),
+ clearAll: vi.fn(),
+ },
+ };
+});
+
+// Get mocked db
+import { db } from "../../db/index";
+// Import the module under test after mocks are set up
+import {
+ clearAllCrdtState,
+ getMigrationStatus,
+ isMigrationCompleted,
+ MIGRATION_VERSION,
+ resetMigration,
+ runMigration,
+ runMigrationWithBatching,
+} from "./migration";
+
+describe("migration", () => {
+ const mockDeck: LocalDeck = {
+ id: "deck-1",
+ userId: "user-1",
+ name: "Test Deck",
+ description: null,
+ newCardsPerDay: 20,
+ createdAt: new Date(),
+ updatedAt: new Date(),
+ deletedAt: null,
+ syncVersion: 1,
+ _synced: true,
+ };
+
+ const mockNoteType: LocalNoteType = {
+ id: "note-type-1",
+ userId: "user-1",
+ name: "Basic",
+ frontTemplate: "{{Front}}",
+ backTemplate: "{{Back}}",
+ isReversible: false,
+ createdAt: new Date(),
+ updatedAt: new Date(),
+ deletedAt: null,
+ syncVersion: 1,
+ _synced: true,
+ };
+
+ const mockNoteFieldType: LocalNoteFieldType = {
+ id: "field-type-1",
+ noteTypeId: "note-type-1",
+ name: "Front",
+ order: 0,
+ fieldType: "text",
+ createdAt: new Date(),
+ updatedAt: new Date(),
+ deletedAt: null,
+ syncVersion: 1,
+ _synced: true,
+ };
+
+ const mockNote: LocalNote = {
+ id: "note-1",
+ deckId: "deck-1",
+ noteTypeId: "note-type-1",
+ createdAt: new Date(),
+ updatedAt: new Date(),
+ deletedAt: null,
+ syncVersion: 1,
+ _synced: true,
+ };
+
+ const mockNoteFieldValue: LocalNoteFieldValue = {
+ id: "field-value-1",
+ noteId: "note-1",
+ noteFieldTypeId: "field-type-1",
+ value: "Hello",
+ createdAt: new Date(),
+ updatedAt: new Date(),
+ syncVersion: 1,
+ _synced: true,
+ };
+
+ const mockCard: LocalCard = {
+ id: "card-1",
+ deckId: "deck-1",
+ noteId: "note-1",
+ isReversed: false,
+ front: "Front",
+ back: "Back",
+ state: CardState.New,
+ due: new Date(),
+ stability: 0,
+ difficulty: 0,
+ elapsedDays: 0,
+ scheduledDays: 0,
+ reps: 0,
+ lapses: 0,
+ lastReview: null,
+ createdAt: new Date(),
+ updatedAt: new Date(),
+ deletedAt: null,
+ syncVersion: 1,
+ _synced: true,
+ };
+
+ const mockReviewLog: LocalReviewLog = {
+ id: "review-1",
+ cardId: "card-1",
+ userId: "user-1",
+ rating: Rating.Good,
+ state: CardState.Learning,
+ scheduledDays: 1,
+ elapsedDays: 0,
+ reviewedAt: new Date(),
+ durationMs: 1000,
+ syncVersion: 1,
+ _synced: true,
+ };
+
+ beforeEach(() => {
+ vi.clearAllMocks();
+
+ // Default mock implementations for empty database
+ vi.mocked(db.decks.toArray).mockResolvedValue([]);
+ vi.mocked(db.noteTypes.toArray).mockResolvedValue([]);
+ vi.mocked(db.noteFieldTypes.toArray).mockResolvedValue([]);
+ vi.mocked(db.notes.toArray).mockResolvedValue([]);
+ vi.mocked(db.noteFieldValues.toArray).mockResolvedValue([]);
+ vi.mocked(db.cards.toArray).mockResolvedValue([]);
+ vi.mocked(db.reviewLogs.toArray).mockResolvedValue([]);
+
+ // Default: no existing migration
+ vi.mocked(crdtSyncDb.metadata.get).mockResolvedValue(undefined);
+ });
+
+ describe("isMigrationCompleted", () => {
+ it("should return false when no migration status exists", async () => {
+ vi.mocked(crdtSyncDb.metadata.get).mockResolvedValue(undefined);
+
+ const result = await isMigrationCompleted();
+ expect(result).toBe(false);
+ });
+
+ it("should return true when migration is completed", async () => {
+ vi.mocked(crdtSyncDb.metadata.get).mockResolvedValue({
+ key: "crdt-migration-status",
+ lastSyncAt: Date.now(),
+ syncVersionWatermark: MIGRATION_VERSION,
+ actorId: JSON.stringify({
+ version: MIGRATION_VERSION,
+ completedAt: Date.now(),
+ counts: {
+ decks: 0,
+ noteTypes: 0,
+ noteFieldTypes: 0,
+ notes: 0,
+ noteFieldValues: 0,
+ cards: 0,
+ reviewLogs: 0,
+ },
+ }),
+ });
+
+ const result = await isMigrationCompleted();
+ expect(result).toBe(true);
+ });
+
+ it("should return false when migration version is outdated", async () => {
+ vi.mocked(crdtSyncDb.metadata.get).mockResolvedValue({
+ key: "crdt-migration-status",
+ lastSyncAt: Date.now(),
+ syncVersionWatermark: 0,
+ actorId: JSON.stringify({
+ version: 0, // Older version
+ completedAt: Date.now(),
+ counts: {
+ decks: 0,
+ noteTypes: 0,
+ noteFieldTypes: 0,
+ notes: 0,
+ noteFieldValues: 0,
+ cards: 0,
+ reviewLogs: 0,
+ },
+ }),
+ });
+
+ const result = await isMigrationCompleted();
+ expect(result).toBe(false);
+ });
+ });
+
+ describe("getMigrationStatus", () => {
+ it("should return null when no status exists", async () => {
+ vi.mocked(crdtSyncDb.metadata.get).mockResolvedValue(undefined);
+
+ const result = await getMigrationStatus();
+ expect(result).toBeNull();
+ });
+
+ it("should return parsed status when exists", async () => {
+ const expectedStatus = {
+ version: MIGRATION_VERSION,
+ completedAt: 1234567890,
+ counts: {
+ decks: 5,
+ noteTypes: 2,
+ noteFieldTypes: 4,
+ notes: 10,
+ noteFieldValues: 20,
+ cards: 15,
+ reviewLogs: 50,
+ },
+ };
+
+ vi.mocked(crdtSyncDb.metadata.get).mockResolvedValue({
+ key: "crdt-migration-status",
+ lastSyncAt: expectedStatus.completedAt,
+ syncVersionWatermark: expectedStatus.version,
+ actorId: JSON.stringify(expectedStatus),
+ });
+
+ const result = await getMigrationStatus();
+ expect(result).toEqual(expectedStatus);
+ });
+
+ it("should return null for invalid JSON", async () => {
+ vi.mocked(crdtSyncDb.metadata.get).mockResolvedValue({
+ key: "crdt-migration-status",
+ lastSyncAt: 0,
+ syncVersionWatermark: 0,
+ actorId: "invalid json",
+ });
+
+ const result = await getMigrationStatus();
+ expect(result).toBeNull();
+ });
+ });
+
+ describe("runMigration", () => {
+ it("should skip migration if already completed", async () => {
+ const existingStatus = {
+ version: MIGRATION_VERSION,
+ completedAt: Date.now(),
+ counts: {
+ decks: 1,
+ noteTypes: 0,
+ noteFieldTypes: 0,
+ notes: 0,
+ noteFieldValues: 0,
+ cards: 0,
+ reviewLogs: 0,
+ },
+ };
+
+ vi.mocked(crdtSyncDb.metadata.get).mockResolvedValue({
+ key: "crdt-migration-status",
+ lastSyncAt: existingStatus.completedAt,
+ syncVersionWatermark: existingStatus.version,
+ actorId: JSON.stringify(existingStatus),
+ });
+
+ const result = await runMigration();
+
+ expect(result.wasRun).toBe(false);
+ expect(result.status).toEqual(existingStatus);
+ expect(crdtSyncStateManager.setDocumentBinary).not.toHaveBeenCalled();
+ });
+
+ it("should migrate all entity types", async () => {
+ vi.mocked(db.decks.toArray).mockResolvedValue([mockDeck]);
+ vi.mocked(db.noteTypes.toArray).mockResolvedValue([mockNoteType]);
+ vi.mocked(db.noteFieldTypes.toArray).mockResolvedValue([
+ mockNoteFieldType,
+ ]);
+ vi.mocked(db.notes.toArray).mockResolvedValue([mockNote]);
+ vi.mocked(db.noteFieldValues.toArray).mockResolvedValue([
+ mockNoteFieldValue,
+ ]);
+ vi.mocked(db.cards.toArray).mockResolvedValue([mockCard]);
+ vi.mocked(db.reviewLogs.toArray).mockResolvedValue([mockReviewLog]);
+
+ const result = await runMigration();
+
+ expect(result.wasRun).toBe(true);
+ expect(result.status).toBeDefined();
+ expect(result.status?.counts).toEqual({
+ decks: 1,
+ noteTypes: 1,
+ noteFieldTypes: 1,
+ notes: 1,
+ noteFieldValues: 1,
+ cards: 1,
+ reviewLogs: 1,
+ });
+
+ // Verify all entity types were migrated
+ expect(crdtSyncStateManager.setDocumentBinary).toHaveBeenCalledTimes(7);
+ });
+
+ it("should call setDocumentBinary with correct parameters for deck", async () => {
+ vi.mocked(db.decks.toArray).mockResolvedValue([mockDeck]);
+
+ await runMigration();
+
+ expect(crdtSyncStateManager.setDocumentBinary).toHaveBeenCalledWith(
+ CrdtEntityType.Deck,
+ mockDeck.id,
+ expect.any(Uint8Array),
+ mockDeck.syncVersion,
+ );
+ });
+
+ it("should save migration status on success", async () => {
+ await runMigration();
+
+ expect(crdtSyncDb.metadata.put).toHaveBeenCalledWith(
+ expect.objectContaining({
+ key: "crdt-migration-status",
+ syncVersionWatermark: MIGRATION_VERSION,
+ }),
+ );
+ });
+
+ it("should handle errors gracefully", async () => {
+ const error = new Error("Database error");
+ vi.mocked(db.decks.toArray).mockRejectedValue(error);
+
+ const result = await runMigration();
+
+ expect(result.wasRun).toBe(true);
+ expect(result.status).toBeNull();
+ expect(result.error).toBe(error);
+ });
+ });
+
+ describe("runMigrationWithBatching", () => {
+ it("should skip migration if already completed", async () => {
+ const existingStatus = {
+ version: MIGRATION_VERSION,
+ completedAt: Date.now(),
+ counts: {
+ decks: 0,
+ noteTypes: 0,
+ noteFieldTypes: 0,
+ notes: 0,
+ noteFieldValues: 0,
+ cards: 0,
+ reviewLogs: 0,
+ },
+ };
+
+ vi.mocked(crdtSyncDb.metadata.get).mockResolvedValue({
+ key: "crdt-migration-status",
+ lastSyncAt: existingStatus.completedAt,
+ syncVersionWatermark: existingStatus.version,
+ actorId: JSON.stringify(existingStatus),
+ });
+
+ const result = await runMigrationWithBatching(10);
+
+ expect(result.wasRun).toBe(false);
+ expect(result.status).toEqual(existingStatus);
+ });
+
+ it("should process entities in batches", async () => {
+ // Create 3 decks to test batching
+ const decks = [
+ { ...mockDeck, id: "deck-1" },
+ { ...mockDeck, id: "deck-2" },
+ { ...mockDeck, id: "deck-3" },
+ ];
+
+ // Mock the offset/limit chain for batching
+ let callCount = 0;
+ const mockOffset = vi.fn().mockImplementation(() => ({
+ limit: vi.fn().mockImplementation((limit: number) => ({
+ toArray: vi.fn().mockImplementation(() => {
+ const start = callCount * limit;
+ callCount++;
+ return Promise.resolve(decks.slice(start, start + limit));
+ }),
+ })),
+ }));
+
+ // biome-ignore lint/suspicious/noExplicitAny: Test mock
+ (db.decks as any).offset = mockOffset;
+ vi.mocked(db.decks.count).mockResolvedValue(3);
+
+ // Empty arrays for other entity types
+ const emptyOffset = vi.fn().mockReturnValue({
+ limit: vi.fn().mockReturnValue({
+ toArray: vi.fn().mockResolvedValue([]),
+ }),
+ });
+
+ for (const table of [
+ db.noteTypes,
+ db.noteFieldTypes,
+ db.notes,
+ db.noteFieldValues,
+ db.cards,
+ db.reviewLogs,
+ ]) {
+ // biome-ignore lint/suspicious/noExplicitAny: Test mock
+ (table as any).offset = emptyOffset;
+ vi.mocked(table.count).mockResolvedValue(0);
+ }
+
+ const result = await runMigrationWithBatching(2); // Batch size of 2
+
+ expect(result.wasRun).toBe(true);
+ expect(result.status?.counts.decks).toBe(3);
+
+ // Should have called batchSetDocuments for deck batches
+ expect(crdtSyncStateManager.batchSetDocuments).toHaveBeenCalled();
+ });
+
+ it("should call progress callback", async () => {
+ const deck = { ...mockDeck, id: "deck-1" };
+
+ let callCount = 0;
+ const mockOffset = vi.fn().mockReturnValue({
+ limit: vi.fn().mockReturnValue({
+ toArray: vi.fn().mockImplementation(() => {
+ if (callCount === 0) {
+ callCount++;
+ return Promise.resolve([deck]);
+ }
+ return Promise.resolve([]);
+ }),
+ }),
+ });
+
+ // biome-ignore lint/suspicious/noExplicitAny: Test mock
+ (db.decks as any).offset = mockOffset;
+ vi.mocked(db.decks.count).mockResolvedValue(1);
+
+ // Empty arrays for other entity types
+ const emptyOffset = vi.fn().mockReturnValue({
+ limit: vi.fn().mockReturnValue({
+ toArray: vi.fn().mockResolvedValue([]),
+ }),
+ });
+
+ for (const table of [
+ db.noteTypes,
+ db.noteFieldTypes,
+ db.notes,
+ db.noteFieldValues,
+ db.cards,
+ db.reviewLogs,
+ ]) {
+ // biome-ignore lint/suspicious/noExplicitAny: Test mock
+ (table as any).offset = emptyOffset;
+ vi.mocked(table.count).mockResolvedValue(0);
+ }
+
+ const progressCallback = vi.fn();
+ await runMigrationWithBatching(10, progressCallback);
+
+ expect(progressCallback).toHaveBeenCalledWith(
+ expect.objectContaining({
+ entityType: "deck",
+ percentage: expect.any(Number),
+ }),
+ );
+ });
+ });
+
+ describe("resetMigration", () => {
+ it("should delete migration status", async () => {
+ await resetMigration();
+
+ expect(crdtSyncDb.metadata.delete).toHaveBeenCalledWith(
+ "crdt-migration-status",
+ );
+ });
+ });
+
+ describe("clearAllCrdtState", () => {
+ it("should clear all sync state", async () => {
+ await clearAllCrdtState();
+
+ expect(crdtSyncStateManager.clearAll).toHaveBeenCalled();
+ });
+ });
+
+ describe("CRDT document conversion", () => {
+ it("should convert deck to valid CRDT binary", () => {
+ const { binary, documentId } =
+ crdtDeckRepository.toCrdtDocument(mockDeck);
+
+ expect(binary).toBeInstanceOf(Uint8Array);
+ expect(binary.length).toBeGreaterThan(0);
+ expect(documentId).toBe(`deck:${mockDeck.id}`);
+ });
+
+ it("should convert noteType to valid CRDT binary", () => {
+ const { binary, documentId } =
+ crdtNoteTypeRepository.toCrdtDocument(mockNoteType);
+
+ expect(binary).toBeInstanceOf(Uint8Array);
+ expect(binary.length).toBeGreaterThan(0);
+ expect(documentId).toBe(`noteType:${mockNoteType.id}`);
+ });
+
+ it("should convert noteFieldType to valid CRDT binary", () => {
+ const { binary, documentId } =
+ crdtNoteFieldTypeRepository.toCrdtDocument(mockNoteFieldType);
+
+ expect(binary).toBeInstanceOf(Uint8Array);
+ expect(binary.length).toBeGreaterThan(0);
+ expect(documentId).toBe(`noteFieldType:${mockNoteFieldType.id}`);
+ });
+
+ it("should convert note to valid CRDT binary", () => {
+ const { binary, documentId } =
+ crdtNoteRepository.toCrdtDocument(mockNote);
+
+ expect(binary).toBeInstanceOf(Uint8Array);
+ expect(binary.length).toBeGreaterThan(0);
+ expect(documentId).toBe(`note:${mockNote.id}`);
+ });
+
+ it("should convert noteFieldValue to valid CRDT binary", () => {
+ const { binary, documentId } =
+ crdtNoteFieldValueRepository.toCrdtDocument(mockNoteFieldValue);
+
+ expect(binary).toBeInstanceOf(Uint8Array);
+ expect(binary.length).toBeGreaterThan(0);
+ expect(documentId).toBe(`noteFieldValue:${mockNoteFieldValue.id}`);
+ });
+
+ it("should convert card to valid CRDT binary", () => {
+ const { binary, documentId } =
+ crdtCardRepository.toCrdtDocument(mockCard);
+
+ expect(binary).toBeInstanceOf(Uint8Array);
+ expect(binary.length).toBeGreaterThan(0);
+ expect(documentId).toBe(`card:${mockCard.id}`);
+ });
+
+ it("should convert reviewLog to valid CRDT binary", () => {
+ const { binary, documentId } =
+ crdtReviewLogRepository.toCrdtDocument(mockReviewLog);
+
+ expect(binary).toBeInstanceOf(Uint8Array);
+ expect(binary.length).toBeGreaterThan(0);
+ expect(documentId).toBe(`reviewLog:${mockReviewLog.id}`);
+ });
+ });
+});
diff --git a/src/client/sync/crdt/migration.ts b/src/client/sync/crdt/migration.ts
new file mode 100644
index 0000000..83ee583
--- /dev/null
+++ b/src/client/sync/crdt/migration.ts
@@ -0,0 +1,482 @@
+/**
+ * CRDT Migration Script
+ *
+ * One-time migration script to convert existing local entities to CRDT documents.
+ * This migrates data from the legacy LWW (Last-Write-Wins) sync system to the
+ * new Automerge CRDT-based sync system.
+ *
+ * Migration process:
+ * 1. Check if migration has already been completed
+ * 2. Read all entities from local IndexedDB
+ * 3. Convert each entity to an Automerge CRDT document
+ * 4. Store the CRDT binary in the sync state database
+ * 5. Mark migration as complete
+ *
+ * The migration is idempotent - running it multiple times has no effect
+ * after the first successful run.
+ */
+
+import { db } from "../../db/index";
+import {
+ crdtCardRepository,
+ crdtDeckRepository,
+ crdtNoteFieldTypeRepository,
+ crdtNoteFieldValueRepository,
+ crdtNoteRepository,
+ crdtNoteTypeRepository,
+ crdtReviewLogRepository,
+} from "./repositories";
+import { crdtSyncDb, crdtSyncStateManager } from "./sync-state";
+import { CrdtEntityType } from "./types";
+
+/**
+ * Migration status stored in IndexedDB
+ */
+interface MigrationStatus {
+ /** Migration version */
+ version: number;
+ /** Timestamp when migration was completed */
+ completedAt: number;
+ /** Entity counts migrated */
+ counts: {
+ decks: number;
+ noteTypes: number;
+ noteFieldTypes: number;
+ notes: number;
+ noteFieldValues: number;
+ cards: number;
+ reviewLogs: number;
+ };
+}
+
+/**
+ * Current migration version
+ * Increment this when making breaking changes to the migration logic
+ */
+export const MIGRATION_VERSION = 1;
+
+/**
+ * Key used to store migration status in metadata
+ */
+const MIGRATION_STATUS_KEY = "crdt-migration-status";
+
+/**
+ * Result of running the migration
+ */
+export interface MigrationResult {
+ /** Whether the migration was run (false if already completed) */
+ wasRun: boolean;
+ /** Migration status after completion */
+ status: MigrationStatus | null;
+ /** Error if migration failed */
+ error?: Error;
+}
+
+/**
+ * Check if migration has already been completed
+ */
+export async function isMigrationCompleted(): Promise<boolean> {
+ const status = await getMigrationStatus();
+ return status !== null && status.version >= MIGRATION_VERSION;
+}
+
+/**
+ * Get the current migration status
+ */
+export async function getMigrationStatus(): Promise<MigrationStatus | null> {
+ const entry = await crdtSyncDb.metadata.get(MIGRATION_STATUS_KEY);
+ if (!entry) {
+ return null;
+ }
+
+ // Parse the status from the metadata entry
+ // We store it as a JSON string in the actorId field for simplicity
+ try {
+ return JSON.parse(entry.actorId) as MigrationStatus;
+ } catch {
+ return null;
+ }
+}
+
+/**
+ * Save the migration status
+ */
+async function saveMigrationStatus(status: MigrationStatus): Promise<void> {
+ await crdtSyncDb.metadata.put({
+ key: MIGRATION_STATUS_KEY,
+ lastSyncAt: status.completedAt,
+ syncVersionWatermark: status.version,
+ actorId: JSON.stringify(status),
+ });
+}
+
+/**
+ * Run the CRDT migration
+ *
+ * Converts all existing local entities to CRDT documents and stores them
+ * in the CRDT sync state database.
+ *
+ * @returns Migration result indicating whether migration was run and the status
+ */
+export async function runMigration(): Promise<MigrationResult> {
+ // Check if migration is already completed
+ if (await isMigrationCompleted()) {
+ const status = await getMigrationStatus();
+ return { wasRun: false, status };
+ }
+
+ try {
+ const counts = {
+ decks: 0,
+ noteTypes: 0,
+ noteFieldTypes: 0,
+ notes: 0,
+ noteFieldValues: 0,
+ cards: 0,
+ reviewLogs: 0,
+ };
+
+ // Migrate Decks
+ const decks = await db.decks.toArray();
+ for (const deck of decks) {
+ const { binary } = crdtDeckRepository.toCrdtDocument(deck);
+ await crdtSyncStateManager.setDocumentBinary(
+ CrdtEntityType.Deck,
+ deck.id,
+ binary,
+ deck.syncVersion,
+ );
+ counts.decks++;
+ }
+
+ // Migrate NoteTypes
+ const noteTypes = await db.noteTypes.toArray();
+ for (const noteType of noteTypes) {
+ const { binary } = crdtNoteTypeRepository.toCrdtDocument(noteType);
+ await crdtSyncStateManager.setDocumentBinary(
+ CrdtEntityType.NoteType,
+ noteType.id,
+ binary,
+ noteType.syncVersion,
+ );
+ counts.noteTypes++;
+ }
+
+ // Migrate NoteFieldTypes
+ const noteFieldTypes = await db.noteFieldTypes.toArray();
+ for (const noteFieldType of noteFieldTypes) {
+ const { binary } =
+ crdtNoteFieldTypeRepository.toCrdtDocument(noteFieldType);
+ await crdtSyncStateManager.setDocumentBinary(
+ CrdtEntityType.NoteFieldType,
+ noteFieldType.id,
+ binary,
+ noteFieldType.syncVersion,
+ );
+ counts.noteFieldTypes++;
+ }
+
+ // Migrate Notes
+ const notes = await db.notes.toArray();
+ for (const note of notes) {
+ const { binary } = crdtNoteRepository.toCrdtDocument(note);
+ await crdtSyncStateManager.setDocumentBinary(
+ CrdtEntityType.Note,
+ note.id,
+ binary,
+ note.syncVersion,
+ );
+ counts.notes++;
+ }
+
+ // Migrate NoteFieldValues
+ const noteFieldValues = await db.noteFieldValues.toArray();
+ for (const noteFieldValue of noteFieldValues) {
+ const { binary } =
+ crdtNoteFieldValueRepository.toCrdtDocument(noteFieldValue);
+ await crdtSyncStateManager.setDocumentBinary(
+ CrdtEntityType.NoteFieldValue,
+ noteFieldValue.id,
+ binary,
+ noteFieldValue.syncVersion,
+ );
+ counts.noteFieldValues++;
+ }
+
+ // Migrate Cards
+ const cards = await db.cards.toArray();
+ for (const card of cards) {
+ const { binary } = crdtCardRepository.toCrdtDocument(card);
+ await crdtSyncStateManager.setDocumentBinary(
+ CrdtEntityType.Card,
+ card.id,
+ binary,
+ card.syncVersion,
+ );
+ counts.cards++;
+ }
+
+ // Migrate ReviewLogs
+ const reviewLogs = await db.reviewLogs.toArray();
+ for (const reviewLog of reviewLogs) {
+ const { binary } = crdtReviewLogRepository.toCrdtDocument(reviewLog);
+ await crdtSyncStateManager.setDocumentBinary(
+ CrdtEntityType.ReviewLog,
+ reviewLog.id,
+ binary,
+ reviewLog.syncVersion,
+ );
+ counts.reviewLogs++;
+ }
+
+ // Save migration status
+ const status: MigrationStatus = {
+ version: MIGRATION_VERSION,
+ completedAt: Date.now(),
+ counts,
+ };
+ await saveMigrationStatus(status);
+
+ return { wasRun: true, status };
+ } catch (error) {
+ return {
+ wasRun: true,
+ status: null,
+ error: error instanceof Error ? error : new Error(String(error)),
+ };
+ }
+}
+
+/**
+ * Run migration with batching for better performance with large datasets
+ *
+ * This version processes entities in batches to avoid memory issues
+ * and provide progress feedback for large migrations.
+ *
+ * @param batchSize Number of entities to process per batch
+ * @param onProgress Optional callback for progress updates
+ * @returns Migration result
+ */
+export async function runMigrationWithBatching(
+ batchSize = 100,
+ onProgress?: (progress: MigrationProgress) => void,
+): Promise<MigrationResult> {
+ // Check if migration is already completed
+ if (await isMigrationCompleted()) {
+ const status = await getMigrationStatus();
+ return { wasRun: false, status };
+ }
+
+ try {
+ const counts = {
+ decks: 0,
+ noteTypes: 0,
+ noteFieldTypes: 0,
+ notes: 0,
+ noteFieldValues: 0,
+ cards: 0,
+ reviewLogs: 0,
+ };
+
+ // Get total counts for progress reporting
+ const totalCounts = {
+ decks: await db.decks.count(),
+ noteTypes: await db.noteTypes.count(),
+ noteFieldTypes: await db.noteFieldTypes.count(),
+ notes: await db.notes.count(),
+ noteFieldValues: await db.noteFieldValues.count(),
+ cards: await db.cards.count(),
+ reviewLogs: await db.reviewLogs.count(),
+ };
+
+ const totalEntities = Object.values(totalCounts).reduce(
+ (sum, count) => sum + count,
+ 0,
+ );
+ let processedEntities = 0;
+
+ // Helper to report progress
+ const reportProgress = (entityType: string, current: number) => {
+ processedEntities++;
+ if (onProgress) {
+ onProgress({
+ entityType,
+ current,
+ total: totalEntities,
+ processed: processedEntities,
+ percentage: Math.round((processedEntities / totalEntities) * 100),
+ });
+ }
+ };
+
+ // Migrate Decks in batches
+ counts.decks = await migrateEntityType(
+ db.decks,
+ crdtDeckRepository,
+ CrdtEntityType.Deck,
+ batchSize,
+ () => reportProgress("deck", counts.decks + 1),
+ );
+
+ // Migrate NoteTypes in batches
+ counts.noteTypes = await migrateEntityType(
+ db.noteTypes,
+ crdtNoteTypeRepository,
+ CrdtEntityType.NoteType,
+ batchSize,
+ () => reportProgress("noteType", counts.noteTypes + 1),
+ );
+
+ // Migrate NoteFieldTypes in batches
+ counts.noteFieldTypes = await migrateEntityType(
+ db.noteFieldTypes,
+ crdtNoteFieldTypeRepository,
+ CrdtEntityType.NoteFieldType,
+ batchSize,
+ () => reportProgress("noteFieldType", counts.noteFieldTypes + 1),
+ );
+
+ // Migrate Notes in batches
+ counts.notes = await migrateEntityType(
+ db.notes,
+ crdtNoteRepository,
+ CrdtEntityType.Note,
+ batchSize,
+ () => reportProgress("note", counts.notes + 1),
+ );
+
+ // Migrate NoteFieldValues in batches
+ counts.noteFieldValues = await migrateEntityType(
+ db.noteFieldValues,
+ crdtNoteFieldValueRepository,
+ CrdtEntityType.NoteFieldValue,
+ batchSize,
+ () => reportProgress("noteFieldValue", counts.noteFieldValues + 1),
+ );
+
+ // Migrate Cards in batches
+ counts.cards = await migrateEntityType(
+ db.cards,
+ crdtCardRepository,
+ CrdtEntityType.Card,
+ batchSize,
+ () => reportProgress("card", counts.cards + 1),
+ );
+
+ // Migrate ReviewLogs in batches
+ counts.reviewLogs = await migrateEntityType(
+ db.reviewLogs,
+ crdtReviewLogRepository,
+ CrdtEntityType.ReviewLog,
+ batchSize,
+ () => reportProgress("reviewLog", counts.reviewLogs + 1),
+ );
+
+ // Save migration status
+ const status: MigrationStatus = {
+ version: MIGRATION_VERSION,
+ completedAt: Date.now(),
+ counts,
+ };
+ await saveMigrationStatus(status);
+
+ return { wasRun: true, status };
+ } catch (error) {
+ return {
+ wasRun: true,
+ status: null,
+ error: error instanceof Error ? error : new Error(String(error)),
+ };
+ }
+}
+
+/**
+ * Progress information for batch migration
+ */
+export interface MigrationProgress {
+ /** Current entity type being migrated */
+ entityType: string;
+ /** Current entity number within type */
+ current: number;
+ /** Total entities across all types */
+ total: number;
+ /** Total entities processed so far */
+ processed: number;
+ /** Percentage complete (0-100) */
+ percentage: number;
+}
+
+/**
+ * Interface representing a table with offset/limit capabilities
+ * Used for batch processing of entities
+ */
+interface BatchableTable<T> {
+ offset(n: number): { limit(n: number): { toArray(): Promise<T[]> } };
+}
+
+/**
+ * Helper to migrate entities of a specific type in batches
+ */
+async function migrateEntityType<T extends { id: string; syncVersion: number }>(
+ table: BatchableTable<T>,
+ repository: { toCrdtDocument: (entity: T) => { binary: Uint8Array } },
+ entityType: string,
+ batchSize: number,
+ onEntity?: () => void,
+): Promise<number> {
+ let count = 0;
+ let offset = 0;
+
+ for (;;) {
+ const batch = await table.offset(offset).limit(batchSize).toArray();
+ if (batch.length === 0) {
+ break;
+ }
+
+ // Prepare batch entries for bulk insert
+ const entries = batch.map((entity) => {
+ const { binary } = repository.toCrdtDocument(entity);
+ return {
+ entityType: entityType as import("./types").CrdtEntityTypeValue,
+ entityId: entity.id,
+ binary,
+ syncVersion: entity.syncVersion,
+ };
+ });
+
+ // Batch insert
+ await crdtSyncStateManager.batchSetDocuments(entries);
+
+ count += batch.length;
+ offset += batchSize;
+
+ // Call progress callback for each entity
+ if (onEntity) {
+ for (let i = 0; i < batch.length; i++) {
+ onEntity();
+ }
+ }
+ }
+
+ return count;
+}
+
+/**
+ * Reset migration status (for testing or retry purposes)
+ *
+ * WARNING: This should only be used for development/testing.
+ * In production, resetting migration could lead to duplicate CRDT documents.
+ */
+export async function resetMigration(): Promise<void> {
+ await crdtSyncDb.metadata.delete(MIGRATION_STATUS_KEY);
+}
+
+/**
+ * Clear all CRDT sync state (for full reset)
+ *
+ * WARNING: This will delete all CRDT documents and require a full resync.
+ * Only use this for development/testing or when explicitly requested by user.
+ */
+export async function clearAllCrdtState(): Promise<void> {
+ await crdtSyncStateManager.clearAll();
+}