Add tokenizer mode support for message search

apps/desktop/src/@types/matrix-seshat.d.ts

@@ -9,6 +9,9 @@ declare module "matrix-seshat" {
     interface IConfig {
         language?: string;
         passphrase?: string;
+        tokenizerMode?: "ngram" | "language";
+        ngramMinSize?: number;
+        ngramMaxSize?: number;
     }
 
     interface IMatrixEvent {

apps/desktop/src/seshat-config.test.ts

@@ -0,0 +1,31 @@
+/*
+Copyright 2026 Hiroshi Shinaoka
+
+SPDX-License-Identifier: AGPL-3.0-only OR GPL-3.0-only OR LicenseRef-Element-Commercial
+Please see LICENSE files in the repository root for full details.
+*/
+
+import { describe, expect, it } from "vitest";
+
+import { createSeshatConfig, TokenizerMode } from "./seshat-config.js";
+
+describe("createSeshatConfig", () => {
+    it.each([
+        [
+            TokenizerMode.Ngram,
+            {
+                tokenizerMode: TokenizerMode.Ngram,
+                ngramMinSize: 2,
+                ngramMaxSize: 4,
+            },
+        ],
+        [TokenizerMode.Language, { tokenizerMode: TokenizerMode.Language }],
+        [undefined, { tokenizerMode: TokenizerMode.Language }],
+    ])("returns the expected config for tokenizerMode %s", (tokenizerMode, expectedConfig) => {
+        expect(createSeshatConfig(tokenizerMode)).toEqual(expectedConfig);
+    });
+
+    it("throws for unknown tokenizer modes", () => {
+        expect(() => createSeshatConfig("unknown")).toThrow("Unknown tokenizer mode: unknown");
+    });
+});

apps/desktop/src/seshat-config.ts

@@ -0,0 +1,38 @@
+/*
+Copyright 2025 Hiroshi Shinaoka
+
+SPDX-License-Identifier: AGPL-3.0-only OR GPL-3.0-only OR LicenseRef-Element-Commercial
+Please see LICENSE files in the repository root for full details.
+*/
+
+export enum TokenizerMode {
+    Ngram = "ngram",
+    Language = "language",
+}
+
+/**
+ * Create Seshat configuration based on tokenizer mode.
+ *
+ * @param tokenizerMode - The tokenizer mode: "ngram" for N-gram tokenization (CJK languages),
+ *                        or "language" for standard language-based tokenization.
+ * @returns Configuration object for Seshat initialization.
+ */
+export function createSeshatConfig(tokenizerMode?: string): {
+    tokenizerMode: TokenizerMode;
+    ngramMinSize?: number;
+    ngramMaxSize?: number;
+} {
+    if (tokenizerMode === TokenizerMode.Ngram) {
+        return {
+            tokenizerMode: TokenizerMode.Ngram,
+            ngramMinSize: 2,
+            ngramMaxSize: 4,
+        };
+    }
+
+    if (tokenizerMode === undefined || tokenizerMode === TokenizerMode.Language) {
+        return { tokenizerMode: TokenizerMode.Language };
+    }
+
+    throw new Error(`Unknown tokenizer mode: ${tokenizerMode}`);
+}

apps/desktop/src/seshat-index.test.ts

@@ -0,0 +1,140 @@
+/*
+Copyright 2026 Hiroshi Shinaoka
+
+SPDX-License-Identifier: AGPL-3.0-only OR GPL-3.0-only OR LicenseRef-Element-Commercial
+Please see LICENSE files in the repository root for full details.
+*/
+
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { describe, expect, it, vi } from "vitest";
+
+import { TokenizerMode } from "./seshat-config.js";
+import { initEventIndex } from "./seshat-index.js";
+
+const eventStorePath = join(tmpdir(), "element-desktop-seshat-index-test");
+const passphrase = "fixture-value";
+
+class FakeReindexError extends Error {}
+
+describe("initEventIndex", () => {
+    it("passes ngram config when opening a new index", async () => {
+        const mkdir = vi.fn().mockResolvedValue(undefined);
+        const deleteContents = vi.fn().mockResolvedValue(undefined);
+        const eventIndex = { kind: "index" };
+        const Seshat = vi.fn().mockImplementation(() => eventIndex);
+        const SeshatRecovery = vi.fn();
+
+        const result = await initEventIndex(eventStorePath, passphrase, TokenizerMode.Ngram, {
+            mkdir,
+            deleteContents,
+            createSeshat: Seshat,
+            createSeshatRecovery: SeshatRecovery,
+            isReindexError: (error) => error instanceof FakeReindexError,
+        });
+
+        expect(mkdir).toHaveBeenCalledWith(eventStorePath, { recursive: true });
+        expect(Seshat).toHaveBeenCalledWith(eventStorePath, {
+            passphrase,
+            tokenizerMode: TokenizerMode.Ngram,
+            ngramMinSize: 2,
+            ngramMaxSize: 4,
+        });
+        expect(SeshatRecovery).not.toHaveBeenCalled();
+        expect(result).toEqual({ eventIndex });
+    });
+
+    it("passes tokenizer config through the reindex recovery path", async () => {
+        const mkdir = vi.fn().mockResolvedValue(undefined);
+        const deleteContents = vi.fn().mockResolvedValue(undefined);
+        const reopenedIndex = { kind: "reopened-index" };
+        const recoveryIndex = {
+            getUserVersion: vi.fn().mockResolvedValue(1),
+            shutdown: vi.fn().mockResolvedValue(undefined),
+            reindex: vi.fn().mockResolvedValue(undefined),
+        };
+
+        const Seshat = vi
+            .fn()
+            .mockImplementationOnce(() => {
+                throw new FakeReindexError("schema changed");
+            })
+            .mockImplementationOnce(() => reopenedIndex);
+        const SeshatRecovery = vi.fn().mockImplementation(() => recoveryIndex);
+
+        const result = await initEventIndex(eventStorePath, passphrase, TokenizerMode.Language, {
+            mkdir,
+            deleteContents,
+            createSeshat: Seshat,
+            createSeshatRecovery: SeshatRecovery,
+            isReindexError: (error) => error instanceof FakeReindexError,
+        });
+
+        expect(SeshatRecovery).toHaveBeenCalledWith(eventStorePath, {
+            passphrase,
+            tokenizerMode: TokenizerMode.Language,
+        });
+        expect(recoveryIndex.reindex).toHaveBeenCalledOnce();
+        expect(Seshat).toHaveBeenNthCalledWith(2, eventStorePath, {
+            passphrase,
+            tokenizerMode: TokenizerMode.Language,
+        });
+        expect(deleteContents).not.toHaveBeenCalled();
+        expect(result).toEqual({ eventIndex: reopenedIndex });
+    });
+
+    it("marks the index as recreated when recovery deletes a version 0 database", async () => {
+        const mkdir = vi.fn().mockResolvedValue(undefined);
+        const deleteContents = vi.fn().mockResolvedValue(undefined);
+        const recreatedIndex = { kind: "recreated-index" };
+        const recoveryIndex = {
+            getUserVersion: vi.fn().mockResolvedValue(0),
+            shutdown: vi.fn().mockResolvedValue(undefined),
+            reindex: vi.fn().mockResolvedValue(undefined),
+        };
+        const Seshat = vi
+            .fn()
+            .mockImplementationOnce(() => {
+                throw new FakeReindexError("schema changed");
+            })
+            .mockImplementationOnce(() => recreatedIndex);
+        const SeshatRecovery = vi.fn().mockImplementation(() => recoveryIndex);
+
+        const result = await initEventIndex(eventStorePath, passphrase, TokenizerMode.Language, {
+            mkdir,
+            deleteContents,
+            createSeshat: Seshat,
+            createSeshatRecovery: SeshatRecovery,
+            isReindexError: (error) => error instanceof FakeReindexError,
+        });
+
+        expect(recoveryIndex.shutdown).toHaveBeenCalledOnce();
+        expect(recoveryIndex.reindex).not.toHaveBeenCalled();
+        expect(deleteContents).toHaveBeenCalledWith(eventStorePath);
+        expect(result).toEqual({ eventIndex: recreatedIndex, wasRecreated: true });
+    });
+
+    it("propagates non-reindex errors without deleting the database", async () => {
+        const mkdir = vi.fn().mockResolvedValue(undefined);
+        const deleteContents = vi.fn().mockResolvedValue(undefined);
+        const openError = new Error("filesystem lock");
+
+        const Seshat = vi.fn().mockImplementationOnce(() => {
+            throw openError;
+        });
+        const SeshatRecovery = vi.fn();
+
+        await expect(
+            initEventIndex(eventStorePath, passphrase, TokenizerMode.Ngram, {
+                mkdir,
+                deleteContents,
+                createSeshat: Seshat,
+                createSeshatRecovery: SeshatRecovery,
+                isReindexError: (error) => error instanceof FakeReindexError,
+            }),
+        ).rejects.toThrow("filesystem lock");
+
+        expect(deleteContents).not.toHaveBeenCalled();
+        expect(SeshatRecovery).not.toHaveBeenCalled();
+    });
+});

apps/desktop/src/seshat-index.ts

@@ -0,0 +1,63 @@
+/*
+Copyright 2026 Hiroshi Shinaoka
+
+SPDX-License-Identifier: AGPL-3.0-only OR GPL-3.0-only OR LicenseRef-Element-Commercial
+Please see LICENSE files in the repository root for full details.
+*/
+
+import type { Seshat as SeshatType, SeshatRecovery as SeshatRecoveryType } from "matrix-seshat";
+import { createSeshatConfig } from "./seshat-config.js";
+
+type SeshatConfig = NonNullable<ConstructorParameters<typeof SeshatType>[1]>;
+
+export interface SeshatIndexDependencies {
+    mkdir(path: string, options: { recursive: true }): Promise<string | undefined>;
+    deleteContents(path: string): Promise<void>;
+    createSeshat(path: string, config: SeshatConfig): SeshatType;
+    createSeshatRecovery(path: string, config: SeshatConfig): SeshatRecoveryType;
+    isReindexError(error: unknown): boolean;
+}
+
+export interface InitEventIndexResult {
+    eventIndex: SeshatType;
+    wasRecreated?: boolean;
+}
+
+export async function initEventIndex(
+    eventStorePath: string,
+    passphrase: string,
+    tokenizerMode: string | undefined,
+    { mkdir, deleteContents, createSeshat, createSeshatRecovery, isReindexError }: SeshatIndexDependencies,
+): Promise<InitEventIndexResult> {
+    const seshatConfig = { passphrase, ...createSeshatConfig(tokenizerMode) };
+
+    await mkdir(eventStorePath, { recursive: true });
+
+    try {
+        return { eventIndex: createSeshat(eventStorePath, seshatConfig) };
+    } catch (e) {
+        if (isReindexError(e)) {
+            // If this is a reindex error, the index schema changed. Try to open the
+            // database in recovery mode, reindex the database and finally try to
+            // open the database again.
+            const recoveryIndex = createSeshatRecovery(eventStorePath, seshatConfig);
+            const userVersion = await recoveryIndex.getUserVersion();
+
+            // If our user version is 0 we'll delete the db anyways so reindexing it is a waste of time.
+            if (userVersion === 0) {
+                await recoveryIndex.shutdown();
+                await deleteContents(eventStorePath);
+                return { eventIndex: createSeshat(eventStorePath, seshatConfig), wasRecreated: true };
+            } else {
+                await recoveryIndex.reindex();
+            }
+
+            return { eventIndex: createSeshat(eventStorePath, seshatConfig) };
+        }
+
+        // For non-reindex errors (e.g. bad passphrase, filesystem lock),
+        // propagate to the caller so the user sees an error instead of
+        // silently losing their search index.
+        throw e;
+    }
+}

apps/desktop/src/seshat.ts

@@ -17,6 +17,7 @@ import type {
 import IpcMainEvent = Electron.IpcMainEvent;
 import { randomArray } from "./utils.js";
 import Store from "./store.js";
+import { initEventIndex } from "./seshat-index.js";
 
 let seshatSupported = false;
 let Seshat: typeof SeshatType;
@@ -103,39 +104,28 @@ ipcMain.on("seshat", async function (_ev: IpcMainEvent, payload): Promise<void>
             if (eventIndex === null) {
                 const userId = args[0];
                 const deviceId = args[1];
+                const tokenizerMode = args[2] as string | undefined;
                 const passphraseKey = `seshat|${userId}|${deviceId}`;
 
                 const passphrase = await getOrCreatePassphrase(store, passphraseKey);
 
                 try {
-                    await afs.mkdir(eventStorePath, { recursive: true });
-                    eventIndex = new Seshat(eventStorePath, { passphrase });
-                } catch (e) {
-                    if (e instanceof ReindexError) {
-                        // If this is a reindex error, the index schema
-                        // changed. Try to open the database in recovery mode,
-                        // reindex the database and finally try to open the
-                        // database again.
-                        const recoveryIndex = new SeshatRecovery(eventStorePath, {
-                            passphrase,
-                        });
-
-                        const userVersion = await recoveryIndex.getUserVersion();
-
-                        // If our user version is 0 we'll delete the db
-                        // anyways so reindexing it is a waste of time.
-                        if (userVersion === 0) {
-                            await recoveryIndex.shutdown();
-                            await deleteContents(eventStorePath);
-                        } else {
-                            await recoveryIndex.reindex();
-                        }
-
-                        eventIndex = new Seshat(eventStorePath, { passphrase });
-                    } else {
-                        sendError(payload.id, <Error>e);
-                        return;
+                    const result = await initEventIndex(eventStorePath, passphrase, tokenizerMode, {
+                        mkdir: afs.mkdir,
+                        deleteContents,
+                        createSeshat: (indexPath, config) => new Seshat(indexPath, config),
+                        createSeshatRecovery: (indexPath, config) => new SeshatRecovery(indexPath, config),
+                        isReindexError: (error) => error instanceof ReindexError,
+                    });
+
+                    eventIndex = result.eventIndex;
+                    if (result.wasRecreated) {
+                        // Tell element-web to force re-adding initial checkpoints.
+                        ret = { wasRecreated: true };
                     }
+                } catch (e) {
+                    sendError(payload.id, <Error>e);
+                    return;
                 }
             }
             break;