Add Tokenizer Mode Support for Seshat Search Index

.github/workflows/static_analysis.yaml

@@ -26,6 +26,9 @@ jobs:
             - name: Typecheck
               run: "pnpm run lint:types"
 
+            - name: Unit Tests
+              run: "pnpm run test:unit"
+
     i18n_lint:
         name: "i18n Check"
         uses: matrix-org/matrix-web-i18n/.github/workflows/i18n_check.yml@6eda3835118f3bc3fb658a1a3c20b7da9d16ae42

package.json

@@ -32,9 +32,10 @@
         "lint": "pnpm lint:types && pnpm lint:js && pnpm lint:workflows",
         "lint:js": "eslint --max-warnings 0 src hak playwright scripts && prettier --check .",
         "lint:js-fix": "eslint --fix --max-warnings 0 src hak playwright scripts && prettier --log-level=warn --write .",
-        "lint:types": "pnpm lint:types:src && pnpm lint:types:test && pnpm lint:types:scripts && pnpm lint:types:hak",
+        "lint:types": "pnpm lint:types:src && pnpm lint:types:test && pnpm lint:types:unit && pnpm lint:types:scripts && pnpm lint:types:hak",
         "lint:types:src": "tsc --noEmit",
         "lint:types:test": "tsc --noEmit -p playwright/tsconfig.json",
+        "lint:types:unit": "tsc --noEmit -p test/tsconfig.json",
         "lint:types:scripts": "tsc --noEmit -p scripts/tsconfig.json",
         "lint:types:hak": "tsc --noEmit -p hak/tsconfig.json",
         "lint:workflows": "find .github/workflows -type f \\( -iname '*.yaml' -o -iname '*.yml' \\) | xargs -I {} sh -c 'echo \"Linting {}\"; action-validator \"{}\"'",
@@ -54,6 +55,7 @@
         "clean": "rimraf webapp.asar dist packages deploys lib",
         "hak": "tsx scripts/hak/index.ts",
         "test": "playwright test",
+        "test:unit": "vitest run --environment node test/unit-tests",
         "test:open": "pnpm test --ui",
         "test:screenshots:build": "docker build playwright -t element-desktop-playwright --platform linux/amd64",
         "test:screenshots:run": "docker run --rm --network host -v $(pwd):/work/element-desktop -v element-desktop-playwright:/work/element-desktop/node_modules -v /var/run/docker.sock:/var/run/docker.sock --platform linux/amd64 -it element-desktop-playwright",
@@ -110,7 +112,8 @@
         "rimraf": "^6.0.0",
         "tar": "^7.5.8",
         "tsx": "^4.19.2",
-        "typescript": "5.9.3"
+        "typescript": "5.9.3",
+        "vitest": "^2.1.9"
     },
     "hakDependencies": {
         "matrix-seshat": "^4.0.1"
@@ -124,7 +127,8 @@
     },
     "pnpm": {
         "onlyBuiltDependencies": [
-            "electron"
+            "electron",
+            "esbuild"
         ],
         "patchedDependencies": {
             "@types/auto-launch": "patches/@types__auto-launch.patch"