diff options
| -rw-r--r-- | README.md | 1 | ||||
| -rw-r--r-- | docs/dev/architecture.md | 1 | ||||
| -rw-r--r-- | docs/manual/features.md | 5 | ||||
| -rw-r--r-- | package.json | 3 | ||||
| -rw-r--r-- | src/server/anki/index.ts | 13 | ||||
| -rw-r--r-- | src/server/anki/parser.test.ts | 868 | ||||
| -rw-r--r-- | src/server/anki/parser.ts | 642 | ||||
| -rw-r--r-- | src/server/scripts/import-anki.ts | 117 |
8 files changed, 1 insertions, 1649 deletions
@@ -7,7 +7,6 @@ A spaced repetition learning application (Anki clone) with PWA offline support a - Spaced Repetition: FSRS algorithm for optimal learning scheduling - Offline Support: Full PWA with IndexedDB local storage - Cloud Sync: Automatic sync when online with conflict resolution -- Anki Import: Import existing .apkg decks from Anki ## Development Setup diff --git a/docs/dev/architecture.md b/docs/dev/architecture.md index 12c6fb0..912df5f 100644 --- a/docs/dev/architecture.md +++ b/docs/dev/architecture.md @@ -251,4 +251,3 @@ GET /api/sync/pull - Pull server changes ## References - [ts-fsrs](https://github.com/open-spaced-repetition/ts-fsrs) -- [Anki APKG Format](https://eikowagenknecht.de/posts/understanding-the-anki-apkg-format/) diff --git a/docs/manual/features.md b/docs/manual/features.md index 0d712cd..81c81ca 100644 --- a/docs/manual/features.md +++ b/docs/manual/features.md @@ -30,8 +30,3 @@ A list of features available in Kioku. - Sync indicator: See your sync status at a glance - Manual sync: Force sync with a button tap - Offline indicator: Know when you're working offline - -## Anki Import - -- Import existing Anki decks (`.apkg` / `.colpkg` files) -- Seamlessly migrate your flashcards to Kioku diff --git a/package.json b/package.json index f5719e6..251c47a 100644 --- a/package.json +++ b/package.json @@ -21,8 +21,7 @@ "db:migrate": "drizzle-kit migrate", "db:push": "drizzle-kit push", "db:studio": "drizzle-kit studio", - "user:add": "node dist/server/scripts/add-user.js", - "anki:import": "node dist/server/scripts/import-anki.js" + "user:add": "node dist/server/scripts/add-user.js" }, "keywords": [], "author": "nsfisis", diff --git a/src/server/anki/index.ts b/src/server/anki/index.ts deleted file mode 100644 index 13e6aa0..0000000 --- a/src/server/anki/index.ts +++ /dev/null @@ -1,13 +0,0 @@ -export { - type AnkiCard, - type AnkiDeck, - type AnkiModel, - type AnkiNote, - type AnkiPackage, - type KiokuCard, - type KiokuDeck, - type KiokuImportData, - listAnkiPackageContents, - mapAnkiToKioku, - parseAnkiPackage, -} from "./parser.js"; diff --git a/src/server/anki/parser.test.ts b/src/server/anki/parser.test.ts deleted file mode 100644 index aeee62c..0000000 --- a/src/server/anki/parser.test.ts +++ /dev/null @@ -1,868 +0,0 @@ -import { randomBytes } from "node:crypto"; -import { mkdir, rm, writeFile } from "node:fs/promises"; -import { tmpdir } from "node:os"; -import { join } from "node:path"; -import { DatabaseSync } from "node:sqlite"; -import { deflateRawSync } from "node:zlib"; -import { afterAll, beforeAll, describe, expect, it } from "vitest"; -import { - type AnkiPackage, - listAnkiPackageContents, - mapAnkiToKioku, - parseAnkiPackage, -} from "./parser.js"; - -/** - * Create a minimal ZIP file with the given entries - */ -function createZip(entries: Map<string, Buffer>): Buffer { - const chunks: Buffer[] = []; - const centralDirectory: Buffer[] = []; - let offset = 0; - - for (const [name, data] of entries) { - const nameBuffer = Buffer.from(name, "utf8"); - const compressedData = deflateRawSync(data); - - // Local file header - const localHeader = Buffer.alloc(30 + nameBuffer.length); - localHeader.writeUInt32LE(0x04034b50, 0); // signature - localHeader.writeUInt16LE(20, 4); // version needed - localHeader.writeUInt16LE(0, 6); // flags - localHeader.writeUInt16LE(8, 8); // compression method (deflate) - localHeader.writeUInt16LE(0, 10); // mod time - localHeader.writeUInt16LE(0, 12); // mod date - localHeader.writeUInt32LE(0, 14); // crc32 (not validated in our parser) - localHeader.writeUInt32LE(compressedData.length, 18); // compressed size - localHeader.writeUInt32LE(data.length, 22); // uncompressed size - localHeader.writeUInt16LE(nameBuffer.length, 26); // file name length - localHeader.writeUInt16LE(0, 28); // extra field length - nameBuffer.copy(localHeader, 30); - - // Central directory entry - const centralEntry = Buffer.alloc(46 + nameBuffer.length); - centralEntry.writeUInt32LE(0x02014b50, 0); // signature - centralEntry.writeUInt16LE(20, 4); // version made by - centralEntry.writeUInt16LE(20, 6); // version needed - centralEntry.writeUInt16LE(0, 8); // flags - centralEntry.writeUInt16LE(8, 10); // compression method - centralEntry.writeUInt16LE(0, 12); // mod time - centralEntry.writeUInt16LE(0, 14); // mod date - centralEntry.writeUInt32LE(0, 16); // crc32 - centralEntry.writeUInt32LE(compressedData.length, 20); // compressed size - centralEntry.writeUInt32LE(data.length, 24); // uncompressed size - centralEntry.writeUInt16LE(nameBuffer.length, 28); // file name length - centralEntry.writeUInt16LE(0, 30); // extra field length - centralEntry.writeUInt16LE(0, 32); // comment length - centralEntry.writeUInt16LE(0, 34); // disk number - centralEntry.writeUInt16LE(0, 36); // internal attributes - centralEntry.writeUInt32LE(0, 38); // external attributes - centralEntry.writeUInt32LE(offset, 42); // offset of local header - nameBuffer.copy(centralEntry, 46); - - centralDirectory.push(centralEntry); - chunks.push(localHeader, compressedData); - offset += localHeader.length + compressedData.length; - } - - // Central directory - const centralDirOffset = offset; - const centralDirBuffer = Buffer.concat(centralDirectory); - chunks.push(centralDirBuffer); - - // End of central directory - const endRecord = Buffer.alloc(22); - endRecord.writeUInt32LE(0x06054b50, 0); // signature - endRecord.writeUInt16LE(0, 4); // disk number - endRecord.writeUInt16LE(0, 6); // disk with central dir - endRecord.writeUInt16LE(entries.size, 8); // entries on this disk - endRecord.writeUInt16LE(entries.size, 10); // total entries - endRecord.writeUInt32LE(centralDirBuffer.length, 12); // central dir size - endRecord.writeUInt32LE(centralDirOffset, 16); // central dir offset - endRecord.writeUInt16LE(0, 20); // comment length - chunks.push(endRecord); - - return Buffer.concat(chunks); -} - -/** - * Create a test Anki SQLite database - */ -function createTestAnkiDb(dbPath: string): void { - const db = new DatabaseSync(dbPath); - - // Create tables - db.exec(` - CREATE TABLE col ( - id INTEGER PRIMARY KEY, - crt INTEGER NOT NULL, - mod INTEGER NOT NULL, - scm INTEGER NOT NULL, - ver INTEGER NOT NULL, - dty INTEGER NOT NULL, - usn INTEGER NOT NULL, - ls INTEGER NOT NULL, - conf TEXT NOT NULL, - models TEXT NOT NULL, - decks TEXT NOT NULL, - dconf TEXT NOT NULL, - tags TEXT NOT NULL - ) - `); - - db.exec(` - CREATE TABLE notes ( - id INTEGER PRIMARY KEY, - guid TEXT NOT NULL, - mid INTEGER NOT NULL, - mod INTEGER NOT NULL, - usn INTEGER NOT NULL, - tags TEXT NOT NULL, - flds TEXT NOT NULL, - sfld TEXT NOT NULL, - csum INTEGER NOT NULL, - flags INTEGER NOT NULL, - data TEXT NOT NULL - ) - `); - - db.exec(` - CREATE TABLE cards ( - id INTEGER PRIMARY KEY, - nid INTEGER NOT NULL, - did INTEGER NOT NULL, - ord INTEGER NOT NULL, - mod INTEGER NOT NULL, - usn INTEGER NOT NULL, - type INTEGER NOT NULL, - queue INTEGER NOT NULL, - due INTEGER NOT NULL, - ivl INTEGER NOT NULL, - factor INTEGER NOT NULL, - reps INTEGER NOT NULL, - lapses INTEGER NOT NULL, - left INTEGER NOT NULL, - odue INTEGER NOT NULL, - odid INTEGER NOT NULL, - flags INTEGER NOT NULL, - data TEXT NOT NULL - ) - `); - - // Insert collection data - const decks = { - "1": { id: 1, name: "Default", desc: "" }, - "1234567890123": { - id: 1234567890123, - name: "Test Deck", - desc: "A test deck", - }, - }; - - const models = { - "9876543210987": { - id: 9876543210987, - name: "Basic", - flds: [{ name: "Front" }, { name: "Back" }], - tmpls: [{ name: "Card 1", qfmt: "{{Front}}", afmt: "{{Back}}" }], - }, - }; - - const insertCol = db.prepare(` - INSERT INTO col (id, crt, mod, scm, ver, dty, usn, ls, conf, models, decks, dconf, tags) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - `); - insertCol.run( - 1, - 1600000000, - 1600000001000, - 1600000000000, - 11, - 0, - -1, - 0, - "{}", - JSON.stringify(models), - JSON.stringify(decks), - "{}", - "{}", - ); - - // Insert test notes - const insertNote = db.prepare(` - INSERT INTO notes (id, guid, mid, mod, usn, tags, flds, sfld, csum, flags, data) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - `); - - // Note 1: Simple card - insertNote.run( - 1000000000001, - "abc123", - 9876543210987, - 1600000001, - -1, - " vocabulary test ", - "Hello\x1fWorld", - "Hello", - 12345, - 0, - "", - ); - - // Note 2: Card with multiple tags - insertNote.run( - 1000000000002, - "def456", - 9876543210987, - 1600000002, - -1, - " japanese kanji n5 ", - "日本語\x1fJapanese", - "日本語", - 67890, - 0, - "", - ); - - // Note 3: Card with no tags - insertNote.run( - 1000000000003, - "ghi789", - 9876543210987, - 1600000003, - -1, - "", - "Question\x1fAnswer", - "Question", - 11111, - 0, - "", - ); - - // Insert test cards - const insertCard = db.prepare(` - INSERT INTO cards (id, nid, did, ord, mod, usn, type, queue, due, ivl, factor, reps, lapses, left, odue, odid, flags, data) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - `); - - // Card for note 1 (new card) - insertCard.run( - 2000000000001, - 1000000000001, - 1234567890123, - 0, - 1600000001, - -1, - 0, - 0, - 1, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - "", - ); - - // Card for note 2 (review card) - insertCard.run( - 2000000000002, - 1000000000002, - 1234567890123, - 0, - 1600000002, - -1, - 2, - 2, - 100, - 30, - 2500, - 5, - 1, - 0, - 0, - 0, - 0, - "", - ); - - // Card for note 3 (learning card) - insertCard.run( - 2000000000003, - 1000000000003, - 1234567890123, - 0, - 1600000003, - -1, - 1, - 1, - 1600100000, - 1, - 2500, - 1, - 0, - 1001, - 0, - 0, - 0, - "", - ); - - db.close(); -} - -describe("Anki Parser", () => { - let tempDir: string; - let testApkgPath: string; - - beforeAll(async () => { - // Create temp directory - tempDir = join(tmpdir(), `kioku-test-${randomBytes(8).toString("hex")}`); - await mkdir(tempDir, { recursive: true }); - - // Create test database - const dbPath = join(tempDir, "collection.anki2"); - createTestAnkiDb(dbPath); - - // Read the database file - const { readFile } = await import("node:fs/promises"); - const dbBuffer = await readFile(dbPath); - - // Create media file (empty JSON object) - const mediaBuffer = Buffer.from("{}", "utf8"); - - // Create ZIP with database and media - const zipEntries = new Map<string, Buffer>(); - zipEntries.set("collection.anki2", dbBuffer); - zipEntries.set("media", mediaBuffer); - - const zipBuffer = createZip(zipEntries); - - // Write the .apkg file - testApkgPath = join(tempDir, "test.apkg"); - await writeFile(testApkgPath, zipBuffer); - }); - - afterAll(async () => { - // Clean up - await rm(tempDir, { recursive: true, force: true }); - }); - - describe("listAnkiPackageContents", () => { - it("should list files in the package", async () => { - const files = await listAnkiPackageContents(testApkgPath); - - expect(files).toContain("collection.anki2"); - expect(files).toContain("media"); - }); - }); - - describe("parseAnkiPackage", () => { - let result: AnkiPackage; - - beforeAll(async () => { - result = await parseAnkiPackage(testApkgPath); - }); - - it("should parse decks correctly", () => { - expect(result.decks.length).toBe(2); - - const testDeck = result.decks.find((d) => d.name === "Test Deck"); - expect(testDeck).toBeDefined(); - expect(testDeck?.description).toBe("A test deck"); - - const defaultDeck = result.decks.find((d) => d.name === "Default"); - expect(defaultDeck).toBeDefined(); - }); - - it("should parse models correctly", () => { - expect(result.models.length).toBe(1); - - const basicModel = result.models[0]; - expect(basicModel).toBeDefined(); - expect(basicModel?.name).toBe("Basic"); - expect(basicModel?.fields).toEqual(["Front", "Back"]); - expect(basicModel?.templates.length).toBe(1); - expect(basicModel?.templates[0]?.name).toBe("Card 1"); - expect(basicModel?.templates[0]?.qfmt).toBe("{{Front}}"); - expect(basicModel?.templates[0]?.afmt).toBe("{{Back}}"); - }); - - it("should parse notes correctly", () => { - expect(result.notes.length).toBe(3); - - // Note 1 - const note1 = result.notes.find((n) => n.guid === "abc123"); - expect(note1).toBeDefined(); - expect(note1?.fields).toEqual(["Hello", "World"]); - expect(note1?.tags).toEqual(["vocabulary", "test"]); - expect(note1?.sfld).toBe("Hello"); - - // Note 2 - const note2 = result.notes.find((n) => n.guid === "def456"); - expect(note2).toBeDefined(); - expect(note2?.fields).toEqual(["日本語", "Japanese"]); - expect(note2?.tags).toEqual(["japanese", "kanji", "n5"]); - - // Note 3 (no tags) - const note3 = result.notes.find((n) => n.guid === "ghi789"); - expect(note3).toBeDefined(); - expect(note3?.tags).toEqual([]); - }); - - it("should parse cards correctly", () => { - expect(result.cards.length).toBe(3); - - // New card - const card1 = result.cards.find((c) => c.nid === 1000000000001); - expect(card1).toBeDefined(); - expect(card1?.type).toBe(0); // new - expect(card1?.reps).toBe(0); - - // Review card - const card2 = result.cards.find((c) => c.nid === 1000000000002); - expect(card2).toBeDefined(); - expect(card2?.type).toBe(2); // review - expect(card2?.ivl).toBe(30); - expect(card2?.reps).toBe(5); - expect(card2?.lapses).toBe(1); - - // Learning card - const card3 = result.cards.find((c) => c.nid === 1000000000003); - expect(card3).toBeDefined(); - expect(card3?.type).toBe(1); // learning - }); - - it("should throw error for non-existent file", async () => { - await expect(parseAnkiPackage("/non/existent/file.apkg")).rejects.toThrow( - "File not found", - ); - }); - - it("should throw error for invalid package without database", async () => { - // Create a ZIP without a database - const zipEntries = new Map<string, Buffer>(); - zipEntries.set("media", Buffer.from("{}", "utf8")); - const invalidZip = createZip(zipEntries); - - const invalidPath = join(tempDir, "invalid.apkg"); - await writeFile(invalidPath, invalidZip); - - await expect(parseAnkiPackage(invalidPath)).rejects.toThrow( - "No Anki database found", - ); - }); - }); - - describe("ZIP extraction", () => { - it("should handle uncompressed entries", async () => { - // Create a ZIP with uncompressed entries - const name = Buffer.from("test.txt", "utf8"); - const data = Buffer.from("Hello, World!", "utf8"); - - // Local file header (uncompressed) - const localHeader = Buffer.alloc(30 + name.length); - localHeader.writeUInt32LE(0x04034b50, 0); - localHeader.writeUInt16LE(20, 4); - localHeader.writeUInt16LE(0, 6); - localHeader.writeUInt16LE(0, 8); // no compression - localHeader.writeUInt16LE(0, 10); - localHeader.writeUInt16LE(0, 12); - localHeader.writeUInt32LE(0, 14); - localHeader.writeUInt32LE(data.length, 18); - localHeader.writeUInt32LE(data.length, 22); - localHeader.writeUInt16LE(name.length, 26); - localHeader.writeUInt16LE(0, 28); - name.copy(localHeader, 30); - - // Central directory - const centralEntry = Buffer.alloc(46 + name.length); - centralEntry.writeUInt32LE(0x02014b50, 0); - centralEntry.writeUInt16LE(20, 4); - centralEntry.writeUInt16LE(20, 6); - centralEntry.writeUInt16LE(0, 8); - centralEntry.writeUInt16LE(0, 10); - centralEntry.writeUInt16LE(0, 12); - centralEntry.writeUInt16LE(0, 14); - centralEntry.writeUInt32LE(0, 16); - centralEntry.writeUInt32LE(data.length, 20); - centralEntry.writeUInt32LE(data.length, 24); - centralEntry.writeUInt16LE(name.length, 28); - centralEntry.writeUInt16LE(0, 30); - centralEntry.writeUInt16LE(0, 32); - centralEntry.writeUInt16LE(0, 34); - centralEntry.writeUInt16LE(0, 36); - centralEntry.writeUInt32LE(0, 38); - centralEntry.writeUInt32LE(0, 42); - name.copy(centralEntry, 46); - - // End of central directory - const endRecord = Buffer.alloc(22); - endRecord.writeUInt32LE(0x06054b50, 0); - endRecord.writeUInt16LE(0, 4); - endRecord.writeUInt16LE(0, 6); - endRecord.writeUInt16LE(1, 8); - endRecord.writeUInt16LE(1, 10); - endRecord.writeUInt32LE(centralEntry.length, 12); - endRecord.writeUInt32LE(localHeader.length + data.length, 16); - endRecord.writeUInt16LE(0, 20); - - const zipBuffer = Buffer.concat([ - localHeader, - data, - centralEntry, - endRecord, - ]); - - const testPath = join(tempDir, "uncompressed.zip"); - await writeFile(testPath, zipBuffer); - - const files = await listAnkiPackageContents(testPath); - expect(files).toContain("test.txt"); - }); - }); - - describe("mapAnkiToKioku", () => { - let pkg: AnkiPackage; - - beforeAll(async () => { - pkg = await parseAnkiPackage(testApkgPath); - }); - - it("should map decks correctly", () => { - const result = mapAnkiToKioku(pkg); - - // Should have Test Deck (skipping Default deck by default) - expect(result.length).toBe(1); - - const testDeckData = result.find((d) => d.deck.name === "Test Deck"); - expect(testDeckData).toBeDefined(); - expect(testDeckData?.deck.description).toBe("A test deck"); - }); - - it("should include default deck when skipDefaultDeck is false", () => { - // Create a package with cards in both default and test decks - const pkgWithDefault: AnkiPackage = { - notes: [ - { - id: 1, - guid: "test1", - mid: 1, - mod: 0, - tags: [], - fields: ["Front1", "Back1"], - sfld: "Front1", - }, - { - id: 2, - guid: "test2", - mid: 1, - mod: 0, - tags: [], - fields: ["Front2", "Back2"], - sfld: "Front2", - }, - ], - cards: [ - { - id: 1, - nid: 1, - did: 1, // Default deck - ord: 0, - mod: 0, - type: 0, - queue: 0, - due: 0, - ivl: 0, - factor: 0, - reps: 0, - lapses: 0, - }, - { - id: 2, - nid: 2, - did: 2, // Test deck - ord: 0, - mod: 0, - type: 0, - queue: 0, - due: 0, - ivl: 0, - factor: 0, - reps: 0, - lapses: 0, - }, - ], - decks: [ - { id: 1, name: "Default", description: "" }, - { id: 2, name: "Test Deck", description: "" }, - ], - models: [ - { - id: 1, - name: "Basic", - fields: ["Front", "Back"], - templates: [ - { name: "Card 1", qfmt: "{{Front}}", afmt: "{{Back}}" }, - ], - }, - ], - }; - - // With skipDefaultDeck = true (default), should only have Test Deck - const resultSkip = mapAnkiToKioku(pkgWithDefault); - expect(resultSkip.length).toBe(1); - expect(resultSkip[0]?.deck.name).toBe("Test Deck"); - - // With skipDefaultDeck = false, should have both decks - const result = mapAnkiToKioku(pkgWithDefault, { skipDefaultDeck: false }); - const deckNames = result.map((d) => d.deck.name); - expect(deckNames).toContain("Default"); - expect(deckNames).toContain("Test Deck"); - }); - - it("should map note fields to front/back", () => { - const result = mapAnkiToKioku(pkg); - const testDeckData = result.find((d) => d.deck.name === "Test Deck"); - - expect(testDeckData?.cards.length).toBe(3); - - // Card 1: Hello/World - const card1 = testDeckData?.cards.find((c) => c.front === "Hello"); - expect(card1).toBeDefined(); - expect(card1?.back).toBe("World"); - - // Card 2: 日本語/Japanese - const card2 = testDeckData?.cards.find((c) => c.front === "日本語"); - expect(card2).toBeDefined(); - expect(card2?.back).toBe("Japanese"); - - // Card 3: Question/Answer - const card3 = testDeckData?.cards.find((c) => c.front === "Question"); - expect(card3).toBeDefined(); - expect(card3?.back).toBe("Answer"); - }); - - it("should map card states correctly", () => { - const result = mapAnkiToKioku(pkg); - const testDeckData = result.find((d) => d.deck.name === "Test Deck"); - - // New card (Hello/World) - const newCard = testDeckData?.cards.find((c) => c.front === "Hello"); - expect(newCard?.state).toBe(0); // New - - // Review card (日本語/Japanese) - const reviewCard = testDeckData?.cards.find((c) => c.front === "日本語"); - expect(reviewCard?.state).toBe(2); // Review - expect(reviewCard?.reps).toBe(5); - expect(reviewCard?.lapses).toBe(1); - - // Learning card (Question/Answer) - const learningCard = testDeckData?.cards.find( - (c) => c.front === "Question", - ); - expect(learningCard?.state).toBe(1); // Learning - }); - - it("should map scheduling data correctly", () => { - const result = mapAnkiToKioku(pkg); - const testDeckData = result.find((d) => d.deck.name === "Test Deck"); - - // Review card has interval of 30 days - const reviewCard = testDeckData?.cards.find((c) => c.front === "日本語"); - expect(reviewCard?.scheduledDays).toBe(30); - expect(reviewCard?.stability).toBe(30); // Stability approximates interval - expect(reviewCard?.elapsedDays).toBe(30); - - // New card has no interval - const newCard = testDeckData?.cards.find((c) => c.front === "Hello"); - expect(newCard?.scheduledDays).toBe(0); - expect(newCard?.stability).toBe(0); - expect(newCard?.elapsedDays).toBe(0); - }); - - it("should convert Anki factor to FSRS difficulty", () => { - const result = mapAnkiToKioku(pkg); - const testDeckData = result.find((d) => d.deck.name === "Test Deck"); - - // Review card has factor 2500 (default ease) - const reviewCard = testDeckData?.cards.find((c) => c.front === "日本語"); - // Factor 2500 should map to a moderate difficulty (around 5) - expect(reviewCard?.difficulty).toBeGreaterThan(0); - expect(reviewCard?.difficulty).toBeLessThan(10); - - // New card has factor 0, should have difficulty 0 - const newCard = testDeckData?.cards.find((c) => c.front === "Hello"); - expect(newCard?.difficulty).toBe(0); - }); - - it("should set due date for cards", () => { - const result = mapAnkiToKioku(pkg); - const testDeckData = result.find((d) => d.deck.name === "Test Deck"); - - // All cards should have valid due dates - for (const card of testDeckData?.cards || []) { - expect(card.due).toBeInstanceOf(Date); - expect(card.due.getTime()).not.toBeNaN(); - } - }); - - it("should set lastReview for reviewed cards", () => { - const result = mapAnkiToKioku(pkg); - const testDeckData = result.find((d) => d.deck.name === "Test Deck"); - - // Review card has been reviewed - const reviewCard = testDeckData?.cards.find((c) => c.front === "日本語"); - expect(reviewCard?.lastReview).toBeInstanceOf(Date); - - // New card has not been reviewed - const newCard = testDeckData?.cards.find((c) => c.front === "Hello"); - expect(newCard?.lastReview).toBeNull(); - }); - - it("should strip HTML tags from fields", () => { - // Create a mock package with HTML in fields - const htmlPkg: AnkiPackage = { - notes: [ - { - id: 1, - guid: "test1", - mid: 1, - mod: 0, - tags: [], - fields: ["<b>Bold</b> text", "<div>Answer</div><br/>Line 2"], - sfld: "Bold text", - }, - ], - cards: [ - { - id: 1, - nid: 1, - did: 2, - ord: 0, - mod: 0, - type: 0, - queue: 0, - due: 0, - ivl: 0, - factor: 0, - reps: 0, - lapses: 0, - }, - ], - decks: [{ id: 2, name: "HTML Test", description: "" }], - models: [ - { - id: 1, - name: "Basic", - fields: ["Front", "Back"], - templates: [ - { name: "Card 1", qfmt: "{{Front}}", afmt: "{{Back}}" }, - ], - }, - ], - }; - - const result = mapAnkiToKioku(htmlPkg); - const card = result[0]?.cards[0]; - - expect(card?.front).toBe("Bold text"); - expect(card?.back).toBe("Answer\nLine 2"); - }); - - it("should decode HTML entities", () => { - const htmlPkg: AnkiPackage = { - notes: [ - { - id: 1, - guid: "test1", - mid: 1, - mod: 0, - tags: [], - fields: [ - "<code> & "quotes"", - " spaced ", - ], - sfld: "code", - }, - ], - cards: [ - { - id: 1, - nid: 1, - did: 2, - ord: 0, - mod: 0, - type: 0, - queue: 0, - due: 0, - ivl: 0, - factor: 0, - reps: 0, - lapses: 0, - }, - ], - decks: [{ id: 2, name: "Entity Test", description: "" }], - models: [ - { - id: 1, - name: "Basic", - fields: ["Front", "Back"], - templates: [ - { name: "Card 1", qfmt: "{{Front}}", afmt: "{{Back}}" }, - ], - }, - ], - }; - - const result = mapAnkiToKioku(htmlPkg); - const card = result[0]?.cards[0]; - - expect(card?.front).toBe('<code> & "quotes"'); - expect(card?.back).toBe("spaced"); - }); - - it("should handle empty package", () => { - const emptyPkg: AnkiPackage = { - notes: [], - cards: [], - decks: [], - models: [], - }; - - const result = mapAnkiToKioku(emptyPkg); - expect(result).toEqual([]); - }); - - it("should skip cards with missing notes", () => { - const incompletePkg: AnkiPackage = { - notes: [], // No notes - cards: [ - { - id: 1, - nid: 999, // Non-existent note - did: 2, - ord: 0, - mod: 0, - type: 0, - queue: 0, - due: 0, - ivl: 0, - factor: 0, - reps: 0, - lapses: 0, - }, - ], - decks: [{ id: 2, name: "Test", description: "" }], - models: [], - }; - - const result = mapAnkiToKioku(incompletePkg); - // Should have deck but no cards - expect(result).toEqual([]); - }); - }); -}); diff --git a/src/server/anki/parser.ts b/src/server/anki/parser.ts deleted file mode 100644 index 3c961ca..0000000 --- a/src/server/anki/parser.ts +++ /dev/null @@ -1,642 +0,0 @@ -import { randomBytes } from "node:crypto"; -import { existsSync } from "node:fs"; -import { mkdir, open, rm, writeFile } from "node:fs/promises"; -import { tmpdir } from "node:os"; -import { join } from "node:path"; -import { DatabaseSync } from "node:sqlite"; -import { createInflateRaw } from "node:zlib"; - -/** - * Represents a note from an Anki database - */ -export interface AnkiNote { - id: number; - guid: string; - mid: number; // model/notetype id - mod: number; - tags: string[]; - fields: string[]; // fields separated by 0x1f in the database - sfld: string; // sort field -} - -/** - * Represents a card from an Anki database - */ -export interface AnkiCard { - id: number; - nid: number; // note id - did: number; // deck id - ord: number; // ordinal (which template/cloze) - mod: number; - type: number; // 0=new, 1=learning, 2=review, 3=relearning - queue: number; - due: number; - ivl: number; // interval - factor: number; - reps: number; - lapses: number; -} - -/** - * Represents a deck from an Anki database - */ -export interface AnkiDeck { - id: number; - name: string; - description: string; -} - -/** - * Represents a model (note type) from an Anki database - */ -export interface AnkiModel { - id: number; - name: string; - fields: string[]; - templates: { - name: string; - qfmt: string; // question format - afmt: string; // answer format - }[]; -} - -/** - * Represents the parsed contents of an Anki package - */ -export interface AnkiPackage { - notes: AnkiNote[]; - cards: AnkiCard[]; - decks: AnkiDeck[]; - models: AnkiModel[]; -} - -// Local file header signature -const LOCAL_FILE_HEADER_SIG = 0x04034b50; -const CENTRAL_DIR_SIG = 0x02014b50; -const END_CENTRAL_DIR_SIG = 0x06054b50; - -/** - * Parse a ZIP file and extract entries - * This is a minimal implementation for .apkg files - */ -async function parseZip(filePath: string): Promise<Map<string, Buffer>> { - const fileHandle = await open(filePath, "r"); - const stat = await fileHandle.stat(); - const fileSize = stat.size; - - try { - const entries = new Map<string, Buffer>(); - - // Read the entire file for simplicity (apkg files are typically small) - const buffer = Buffer.alloc(fileSize); - await fileHandle.read(buffer, 0, fileSize, 0); - - let offset = 0; - - while (offset < fileSize) { - // Read signature - const sig = buffer.readUInt32LE(offset); - - if (sig === LOCAL_FILE_HEADER_SIG) { - // Local file header - const compressionMethod = buffer.readUInt16LE(offset + 8); - const compressedSize = buffer.readUInt32LE(offset + 18); - const fileNameLength = buffer.readUInt16LE(offset + 26); - const extraFieldLength = buffer.readUInt16LE(offset + 28); - - const fileName = buffer - .subarray(offset + 30, offset + 30 + fileNameLength) - .toString("utf8"); - const dataOffset = offset + 30 + fileNameLength + extraFieldLength; - - // Extract the data - const compressedData = buffer.subarray( - dataOffset, - dataOffset + compressedSize, - ); - - let data: Buffer; - if (compressionMethod === 0) { - // Stored (no compression) - data = compressedData; - } else if (compressionMethod === 8) { - // Deflate - data = await inflateBuffer(compressedData); - } else { - throw new Error( - `Unsupported compression method: ${compressionMethod}`, - ); - } - - entries.set(fileName, data); - - offset = dataOffset + compressedSize; - } else if (sig === CENTRAL_DIR_SIG || sig === END_CENTRAL_DIR_SIG) { - // We've reached the central directory, stop parsing - break; - } else { - // Unknown signature, try to move forward - offset++; - } - } - - return entries; - } finally { - await fileHandle.close(); - } -} - -/** - * Inflate a deflate-compressed buffer - */ -function inflateBuffer(data: Buffer): Promise<Buffer> { - return new Promise((resolve, reject) => { - const chunks: Buffer[] = []; - const inflate = createInflateRaw(); - - inflate.on("data", (chunk) => chunks.push(chunk)); - inflate.on("end", () => resolve(Buffer.concat(chunks))); - inflate.on("error", reject); - - inflate.write(data); - inflate.end(); - }); -} - -/** - * Extract and parse an Anki package file (.apkg) - */ -export async function parseAnkiPackage(filePath: string): Promise<AnkiPackage> { - if (!existsSync(filePath)) { - throw new Error(`File not found: ${filePath}`); - } - - // Extract ZIP contents - const entries = await parseZip(filePath); - - // Find the database file - let dbBuffer: Buffer | undefined; - let dbFormat: "anki2" | "anki21" | "anki21b" | undefined; - - // Check for different database formats (newest first) - if (entries.has("collection.anki21b")) { - dbBuffer = entries.get("collection.anki21b"); - dbFormat = "anki21b"; - } else if (entries.has("collection.anki21")) { - dbBuffer = entries.get("collection.anki21"); - dbFormat = "anki21"; - } else if (entries.has("collection.anki2")) { - dbBuffer = entries.get("collection.anki2"); - dbFormat = "anki2"; - } - - if (!dbBuffer || !dbFormat) { - const availableFiles = Array.from(entries.keys()).join(", "); - throw new Error( - `No Anki database found in package. Available files: ${availableFiles}`, - ); - } - - // For anki21b format, the database is zstd compressed - if (dbFormat === "anki21b") { - throw new Error( - "anki21b format (zstd compressed) is not yet supported. Please export from Anki using the legacy format.", - ); - } - - // Write database to temp file (node:sqlite requires a file path) - const tempDir = join( - tmpdir(), - `kioku-anki-${randomBytes(8).toString("hex")}`, - ); - await mkdir(tempDir, { recursive: true }); - const tempDbPath = join(tempDir, "collection.db"); - - try { - await writeFile(tempDbPath, dbBuffer); - - // Parse the SQLite database - return parseAnkiDatabase(tempDbPath); - } finally { - // Clean up temp files - await rm(tempDir, { recursive: true, force: true }); - } -} - -/** - * Parse an Anki SQLite database - */ -function parseAnkiDatabase(dbPath: string): AnkiPackage { - const db = new DatabaseSync(dbPath, { open: true }); - - try { - // Parse notes - const notes = parseNotes(db); - - // Parse cards - const cards = parseCards(db); - - // Parse decks and models from the col table - const { decks, models } = parseCollection(db); - - return { notes, cards, decks, models }; - } finally { - db.close(); - } -} - -/** - * Parse notes from the database - */ -function parseNotes(db: DatabaseSync): AnkiNote[] { - const stmt = db.prepare( - "SELECT id, guid, mid, mod, tags, flds, sfld FROM notes", - ); - const rows = stmt.all() as Array<{ - id: number; - guid: string; - mid: number; - mod: number; - tags: string; - flds: string; - sfld: string; - }>; - - return rows.map((row) => ({ - id: row.id, - guid: row.guid, - mid: row.mid, - mod: row.mod, - tags: row.tags - .trim() - .split(/\s+/) - .filter((t) => t.length > 0), - fields: row.flds.split("\x1f"), - sfld: row.sfld, - })); -} - -/** - * Parse cards from the database - */ -function parseCards(db: DatabaseSync): AnkiCard[] { - const stmt = db.prepare( - "SELECT id, nid, did, ord, mod, type, queue, due, ivl, factor, reps, lapses FROM cards", - ); - const rows = stmt.all() as Array<{ - id: number; - nid: number; - did: number; - ord: number; - mod: number; - type: number; - queue: number; - due: number; - ivl: number; - factor: number; - reps: number; - lapses: number; - }>; - - return rows.map((row) => ({ - id: row.id, - nid: row.nid, - did: row.did, - ord: row.ord, - mod: row.mod, - type: row.type, - queue: row.queue, - due: row.due, - ivl: row.ivl, - factor: row.factor, - reps: row.reps, - lapses: row.lapses, - })); -} - -/** - * Parse collection metadata (decks and models) - */ -function parseCollection(db: DatabaseSync): { - decks: AnkiDeck[]; - models: AnkiModel[]; -} { - const stmt = db.prepare("SELECT decks, models FROM col LIMIT 1"); - const row = stmt.get() as { decks: string; models: string } | undefined; - - if (!row) { - throw new Error("No collection data found in database"); - } - - // Parse decks JSON - const decksJson = JSON.parse(row.decks) as Record< - string, - { id: number; name: string; desc?: string } - >; - const decks: AnkiDeck[] = Object.values(decksJson).map((d) => ({ - id: d.id, - name: d.name, - description: d.desc || "", - })); - - // Parse models JSON - const modelsJson = JSON.parse(row.models) as Record< - string, - { - id: number; - name: string; - flds: Array<{ name: string }>; - tmpls: Array<{ name: string; qfmt: string; afmt: string }>; - } - >; - const models: AnkiModel[] = Object.values(modelsJson).map((m) => ({ - id: m.id, - name: m.name, - fields: m.flds.map((f) => f.name), - templates: m.tmpls.map((t) => ({ - name: t.name, - qfmt: t.qfmt, - afmt: t.afmt, - })), - })); - - return { decks, models }; -} - -/** - * Get the list of files in a ZIP archive - */ -export async function listAnkiPackageContents( - filePath: string, -): Promise<string[]> { - const entries = await parseZip(filePath); - return Array.from(entries.keys()); -} - -/** - * Represents a Kioku deck ready for import - */ -export interface KiokuDeck { - name: string; - description: string | null; -} - -/** - * Represents a Kioku card ready for import - */ -export interface KiokuCard { - front: string; - back: string; - state: number; - due: Date; - stability: number; - difficulty: number; - elapsedDays: number; - scheduledDays: number; - reps: number; - lapses: number; - lastReview: Date | null; -} - -/** - * Represents the import data with deck and cards - */ -export interface KiokuImportData { - deck: KiokuDeck; - cards: KiokuCard[]; -} - -/** - * Strip HTML tags from a string - */ -function stripHtml(html: string): string { - // Replace <br> and <br/> with newlines - let text = html.replace(/<br\s*\/?>/gi, "\n"); - // Replace </div>, </p>, </li> with newlines - text = text.replace(/<\/(div|p|li)>/gi, "\n"); - // Remove all other HTML tags - text = text.replace(/<[^>]*>/g, ""); - // Decode common HTML entities - text = text - .replace(/ /g, " ") - .replace(/&/g, "&") - .replace(/</g, "<") - .replace(/>/g, ">") - .replace(/"/g, '"') - .replace(/'/g, "'") - .replace(/'/g, "'"); - // Normalize whitespace - text = text.replace(/\n\s*\n/g, "\n").trim(); - return text; -} - -/** - * Convert Anki factor (0-10000) to FSRS difficulty (0-10) - * Anki factor: 2500 = 250% ease = easy, lower = harder - * FSRS difficulty: higher = harder - */ -function ankiFactorToFsrsDifficulty(factor: number): number { - // Default Anki factor is 2500 (250% ease) - // Range is typically 1300-2500+ (130% to 250%+) - // FSRS difficulty is 0-10 where higher means harder - - if (factor === 0) { - // New card, use default FSRS difficulty - return 0; - } - - // Convert: high factor (easy) -> low difficulty, low factor (hard) -> high difficulty - // Map factor range [1300, 3500] to difficulty [8, 2] - const minFactor = 1300; - const maxFactor = 3500; - const minDifficulty = 2; - const maxDifficulty = 8; - - const clampedFactor = Math.max(minFactor, Math.min(maxFactor, factor)); - const normalized = (clampedFactor - minFactor) / (maxFactor - minFactor); - // Invert: high factor -> low difficulty - const difficulty = - maxDifficulty - normalized * (maxDifficulty - minDifficulty); - - return Math.round(difficulty * 100) / 100; -} - -/** - * Estimate FSRS stability from Anki interval - * Stability in FSRS roughly corresponds to the interval in days - */ -function ankiIntervalToFsrsStability(ivl: number, state: number): number { - // For new cards, stability is 0 - if (state === 0) { - return 0; - } - - // For learning/relearning cards, use a small initial stability - if (state === 1 || state === 3) { - return Math.max(0.5, ivl); - } - - // For review cards, stability approximates the interval - return Math.max(1, ivl); -} - -/** - * Convert Anki due timestamp to a Date - * Anki stores due differently based on card type: - * - New cards: due is a position in the new queue (integer) - * - Learning cards: due is Unix timestamp in seconds - * - Review cards: due is days since collection creation - */ -function ankiDueToDate( - due: number, - cardType: number, - collectionCreation?: number, -): Date { - const now = new Date(); - - if (cardType === 0) { - // New card: due is queue position, return current time - return now; - } - - if (cardType === 1 || cardType === 3) { - // Learning/Relearning: due is Unix timestamp in seconds - if (due > 100000000) { - // Sanity check for timestamp - return new Date(due * 1000); - } - return now; - } - - // Review card: due is days since collection creation - if (collectionCreation) { - const baseDate = new Date(collectionCreation * 1000); - baseDate.setDate(baseDate.getDate() + due); - return baseDate; - } - - // Fallback: treat as days from now (roughly) - const dueDate = new Date(); - dueDate.setDate(dueDate.getDate() + due); - return dueDate; -} - -/** - * Convert an Anki package to Kioku import format - * Groups cards by deck and maps note fields to front/back - * - * @param pkg The parsed Anki package - * @param options Optional configuration for the mapping - * @returns Array of decks with their cards ready for import - */ -export function mapAnkiToKioku( - pkg: AnkiPackage, - options?: { - /** Skip the default Anki deck (id: 1) */ - skipDefaultDeck?: boolean; - /** Collection creation timestamp (for accurate due date calculation) */ - collectionCreation?: number; - }, -): KiokuImportData[] { - const skipDefaultDeck = options?.skipDefaultDeck ?? true; - const collectionCreation = options?.collectionCreation; - - // Build lookup maps - const noteById = new Map<number, AnkiNote>(); - for (const note of pkg.notes) { - noteById.set(note.id, note); - } - - const modelById = new Map<number, AnkiModel>(); - for (const model of pkg.models) { - modelById.set(model.id, model); - } - - const deckById = new Map<number, AnkiDeck>(); - for (const deck of pkg.decks) { - deckById.set(deck.id, deck); - } - - // Group cards by deck - const cardsByDeck = new Map<number, AnkiCard[]>(); - for (const card of pkg.cards) { - const existing = cardsByDeck.get(card.did) || []; - existing.push(card); - cardsByDeck.set(card.did, existing); - } - - const result: KiokuImportData[] = []; - - for (const [deckId, ankiCards] of cardsByDeck) { - // Skip default deck if configured - if (skipDefaultDeck && deckId === 1) { - continue; - } - - const ankiDeck = deckById.get(deckId); - if (!ankiDeck) { - continue; - } - - const kiokuDeck: KiokuDeck = { - name: ankiDeck.name, - description: ankiDeck.description || null, - }; - - const kiokuCards: KiokuCard[] = []; - - for (const ankiCard of ankiCards) { - const note = noteById.get(ankiCard.nid); - if (!note) { - continue; - } - - const model = modelById.get(note.mid); - - // Get front and back fields - // For Basic model: fields[0] = Front, fields[1] = Back - // For other models, we try to use the first two fields - let front = note.fields[0] || ""; - let back = note.fields[1] || ""; - - // If there's a template, try to identify question/answer fields - if (model && model.templates.length > 0) { - const template = model.templates[ankiCard.ord] || model.templates[0]; - if (template) { - // Use template hints if available - // For now, we just use the first two fields as front/back - // A more sophisticated approach would parse the template - } - } - - // Strip HTML from fields - front = stripHtml(front); - back = stripHtml(back); - - // Map card state (Anki and FSRS use the same values: 0=New, 1=Learning, 2=Review, 3=Relearning) - const state = ankiCard.type; - - const kiokuCard: KiokuCard = { - front, - back, - state, - due: ankiDueToDate(ankiCard.due, ankiCard.type, collectionCreation), - stability: ankiIntervalToFsrsStability(ankiCard.ivl, ankiCard.type), - difficulty: ankiFactorToFsrsDifficulty(ankiCard.factor), - elapsedDays: ankiCard.ivl > 0 ? ankiCard.ivl : 0, - scheduledDays: ankiCard.ivl, - reps: ankiCard.reps, - lapses: ankiCard.lapses, - lastReview: ankiCard.reps > 0 ? new Date() : null, // We don't have exact last review time - }; - - kiokuCards.push(kiokuCard); - } - - if (kiokuCards.length > 0) { - result.push({ deck: kiokuDeck, cards: kiokuCards }); - } - } - - return result; -} diff --git a/src/server/scripts/import-anki.ts b/src/server/scripts/import-anki.ts deleted file mode 100644 index 739b485..0000000 --- a/src/server/scripts/import-anki.ts +++ /dev/null @@ -1,117 +0,0 @@ -import * as readline from "node:readline/promises"; -import { mapAnkiToKioku, parseAnkiPackage } from "../anki/index.js"; -import { db } from "../db/index.js"; -import { cards, decks } from "../db/schema.js"; -import { userRepository } from "../repositories/index.js"; - -async function main() { - const rl = readline.createInterface({ - input: process.stdin, - output: process.stdout, - }); - - // Get file path from command line argument or prompt - let filePath = process.argv[2]; - if (!filePath) { - filePath = await rl.question("Anki package path (.apkg): "); - } - - if (!filePath) { - console.error("Error: File path is required"); - process.exit(1); - } - - // Get username - const username = await rl.question("Username: "); - rl.close(); - - if (!username) { - console.error("Error: Username is required"); - process.exit(1); - } - - // Find user - const user = await userRepository.findByUsername(username); - if (!user) { - console.error(`Error: User "${username}" not found`); - process.exit(1); - } - - console.log(`\nParsing Anki package: ${filePath}`); - - // Parse the Anki package - const ankiPackage = await parseAnkiPackage(filePath); - - console.log(`Found ${ankiPackage.decks.length} deck(s)`); - console.log(`Found ${ankiPackage.notes.length} note(s)`); - console.log(`Found ${ankiPackage.cards.length} card(s)`); - - // Convert to Kioku format - const importData = mapAnkiToKioku(ankiPackage); - - if (importData.length === 0) { - console.log("\nNo decks to import (all decks may be empty or skipped)"); - process.exit(0); - } - - console.log(`\nImporting ${importData.length} deck(s):`); - for (const { deck, cards: deckCards } of importData) { - console.log(` - ${deck.name}: ${deckCards.length} card(s)`); - } - - // Import decks and cards - let totalCards = 0; - for (const { deck, cards: kiokuCards } of importData) { - // Create deck - const [newDeck] = await db - .insert(decks) - .values({ - userId: user.id, - name: deck.name, - description: deck.description, - newCardsPerDay: 20, - }) - .returning(); - - if (!newDeck) { - console.error(`Error: Failed to create deck "${deck.name}"`); - continue; - } - - // Create cards in batches - if (kiokuCards.length > 0) { - const cardValues = kiokuCards.map((card) => ({ - deckId: newDeck.id, - front: card.front, - back: card.back, - state: card.state, - due: card.due, - stability: card.stability, - difficulty: card.difficulty, - elapsedDays: card.elapsedDays, - scheduledDays: card.scheduledDays, - reps: card.reps, - lapses: card.lapses, - lastReview: card.lastReview, - })); - - await db.insert(cards).values(cardValues); - totalCards += kiokuCards.length; - } - - console.log( - ` Created deck "${deck.name}" with ${kiokuCards.length} cards`, - ); - } - - console.log(`\nImport complete!`); - console.log(` Decks: ${importData.length}`); - console.log(` Cards: ${totalCards}`); - - process.exit(0); -} - -main().catch((error) => { - console.error("Error:", error.message); - process.exit(1); -}); |