POC for showPointLabels with TeX

.changeset/shaggy-pumas-drum.md

@@ -0,0 +1,2 @@
+---
+---

packages/perseus-editor/src/widgets/interactive-graph-editor/locked-figures/util.ts

@@ -40,6 +40,12 @@ export function generateLockedFigureAppearanceDescription(
  * Exported for testing.
  *
  * Example: "Circle with radius $\frac{1}{2}$" ==> "Circle with radius one half"
+ *
+ * @deprecated Use `generateSpokenMathDetails` from `@khanacademy/perseus`
+ * (`packages/perseus/src/util/spoken-math.ts`) instead. This editor copy was
+ * duplicated into perseus so the renderer can convert point labels at runtime.
+ * TODO(LEMS): migrate callers (`joinLabelsAsSpokenMath` below, and the
+ * locked-figure settings) to the perseus util and delete this copy.
  */
 export async function generateSpokenMathDetails(mathString: string) {
     const engine = await SpeechRuleEngine.setup("en");

packages/perseus/package.json

@@ -57,6 +57,7 @@
         "uuid": "^10.0.0"
     },
     "devDependencies": {
+        "@khanacademy/mathjax-renderer": "catalog:devDeps",
         "@khanacademy/wonder-blocks-announcer": "catalog:devDeps",
         "@khanacademy/wonder-blocks-banner": "catalog:devDeps",
         "@khanacademy/wonder-blocks-button": "catalog:devDeps",
@@ -94,6 +95,7 @@
         "underscore": "catalog:devDeps"
     },
     "peerDependencies": {
+        "@khanacademy/mathjax-renderer": "catalog:peerDeps",
         "@khanacademy/wonder-blocks-announcer": "catalog:peerDeps",
         "@khanacademy/wonder-blocks-banner": "catalog:peerDeps",
         "@khanacademy/wonder-blocks-button": "catalog:peerDeps",
@@ -130,6 +132,6 @@
     },
     "keywords": [],
     "khan": {
-        "catalogHash": "8560a5ad63db87f8"
+        "catalogHash": "3dc0969744c5c56c"
     }
 }

packages/perseus/src/util/spoken-math.test.ts

@@ -0,0 +1,116 @@
+import {generateSpokenMathDetails} from "./spoken-math";
+
+describe("generateSpokenMathDetails", () => {
+    it("converts TeX to spoken language (root, fraction)", async () => {
+        // Arrange, Act
+        const convertedString = await generateSpokenMathDetails(
+            "$\\sqrt{\\frac{1}{2}}$",
+        );
+
+        // Assert
+        expect(convertedString).toBe("StartRoot one half EndRoot");
+    });
+
+    it("converts TeX to spoken language (exponent)", async () => {
+        // Arrange, Act
+        const convertedString = await generateSpokenMathDetails("$x^{2}$");
+
+        // Assert
+        expect(convertedString).toBe("x Superscript 2");
+    });
+
+    it("converts TeX to spoken language (negative)", async () => {
+        // Arrange, Act
+        const convertedString = await generateSpokenMathDetails("$-2$");
+
+        // Assert
+        expect(convertedString).toBe("negative 2");
+    });
+
+    it("converts TeX to spoken language (subtraction)", async () => {
+        // Arrange, Act
+        const convertedString = await generateSpokenMathDetails("$2-1$");
+
+        // Assert
+        expect(convertedString).toBe("2 minus 1");
+    });
+
+    it("converts TeX to spoken language (normal words)", async () => {
+        // Arrange, Act
+        const convertedString =
+            await generateSpokenMathDetails("$\\text{square b}$");
+
+        // Assert
+        expect(convertedString).toBe("square b");
+    });
+
+    it("converts TeX to spoken language (random letters)", async () => {
+        // Arrange, Act
+        const convertedString = await generateSpokenMathDetails("$cat$");
+
+        // Assert
+        expect(convertedString).toBe("c a t");
+    });
+
+    it("keeps non-math text as is", async () => {
+        // Arrange, Act
+        const convertedString = await generateSpokenMathDetails(
+            "Circle with radius $\\frac{1}{2}$ units",
+        );
+
+        // Assert
+        expect(convertedString).toBe("Circle with radius one half units");
+    });
+
+    it("reads dollar signs as dollars inside tex", async () => {
+        // Arrange, Act
+        const convertedString = await generateSpokenMathDetails(
+            "This sandwich costs ${$}12.34$",
+        );
+
+        // Assert
+        expect(convertedString).toBe("This sandwich costs dollar sign 12.34");
+    });
+
+    it("reads dollar signs as dollars outside tex", async () => {
+        // Arrange, Act
+        const convertedString = await generateSpokenMathDetails(
+            "This sandwich costs \\$12.34",
+        );
+
+        // Assert
+        expect(convertedString).toBe("This sandwich costs $12.34");
+    });
+
+    it("reads curly braces", async () => {
+        // Arrange, Act
+        const convertedString = await generateSpokenMathDetails("Hello}{");
+
+        // Assert
+        expect(convertedString).toBe("Hello}{");
+    });
+
+    it("reads backslashes", async () => {
+        // Arrange, Act
+        const convertedString = await generateSpokenMathDetails("\\");
+
+        // Assert
+        expect(convertedString).toBe("\\");
+    });
+
+    it("reads lone dollar signs as regular dollar signs", async () => {
+        // Arrange, Act
+        const convertedString = await generateSpokenMathDetails("$50");
+
+        // Assert
+        expect(convertedString).toBe("$50");
+    });
+
+    it("reads lone escaped dollar signs in text as regular dollar signs", async () => {
+        // Arrange, Act
+        const convertedString = await generateSpokenMathDetails("\\$50");
+
+        // Assert
+        expect(convertedString).toBe("$50");
+    });
+});

packages/perseus/src/util/spoken-math.ts

@@ -0,0 +1,48 @@
+import {SpeechRuleEngine} from "@khanacademy/mathjax-renderer";
+
+import {mathOnlyParser} from "../widgets/interactive-graphs/utils";
+
+/**
+ * Given a string that may contain math within TeX represented by $...$,
+ * returns the spoken math equivalent using the SpeechRuleEngine.
+ *
+ * Example: "Circle with radius $\frac{1}{2}$" ==> "Circle with radius one half"
+ *
+ * NOTE(LEMS): This is duplicated from perseus-editor's
+ * `interactive-graph-editor/locked-figures/util.ts` so it can run in the
+ * renderer (perseus can't import from perseus-editor). A follow-up will
+ * migrate the editor to import this copy and delete its own.
+ */
+export async function generateSpokenMathDetails(
+    mathString: string,
+): Promise<string> {
+    const engine = await SpeechRuleEngine.setup("en");
+    let convertedSpeech = "";
+
+    // All the information we need is in the first section,
+    // whether it's typed as "blockmath" or "paragraph"
+    const parsedContent = mathOnlyParser(mathString);
+
+    // If it's a paragraph, we need to iterate through the sections
+    // to look for individual math blocks.
+    for (const piece of parsedContent) {
+        switch (piece.type) {
+            case "math":
+                convertedSpeech += engine.texToSpeech(piece.content);
+                break;
+            case "specialCharacter":
+                // We don't want the backslash from special character
+                // to show up in the generated aria label.
+                convertedSpeech +=
+                    piece.content.length > 1
+                        ? piece.content.slice(1)
+                        : piece.content;
+                break;
+            default:
+                convertedSpeech += piece.content;
+                break;
+        }
+    }
+
+    return convertedSpeech;
+}

packages/perseus/src/widgets/interactive-graphs/__docs__/interactive-graph.stories.tsx

@@ -11,6 +11,7 @@ import {
     linearWithCustomLabelsQuestion,
     pointQuestion,
     pointWithCustomLabelQuestion,
+    pointWithTexLabelQuestion,
     pointWithDefaultLabelQuestion,
     polygonQuestion,
     polygonWithCustomLabelsQuestion,
@@ -175,6 +176,24 @@ export const PointWithCustomLabel: Story = {
     },
 };
 
+/**
+ * A point whose custom label is TeX: `$\left(\dfrac{1}{2}, 3\right)$`. The
+ * visible on-canvas label renders the math correctly, but the screen reader
+ * currently announces the raw TeX literally because the label string is read
+ * verbatim. Use this story with a screen reader to hear the bug that
+ * spoken-math conversion fixes.
+ */
+export const PointWithTeXLabel: Story = {
+    globals: {
+        featureFlags: ["perseus-enable-point-label-field"],
+    },
+    args: {
+        item: generateTestPerseusItem({
+            question: pointWithTexLabelQuestion,
+        }),
+    },
+};
+
 /**
  * The same reference question as `PointWithCustomLabel`, but
  * with `pointLabels` omitted so the interactive point falls back to the

packages/perseus/src/widgets/interactive-graphs/graphs/components/build-point-aria-label.test.ts

@@ -1,9 +1,31 @@
-import {mockPerseusI18nContext} from "../../../../components/i18n-context";
+import {renderHook, waitFor} from "@testing-library/react";
+import * as React from "react";
 
-import {buildPointAriaLabel, resolvePointLabel} from "./build-point-aria-label";
+import {
+    mockPerseusI18nContext,
+    PerseusI18nContextProvider,
+} from "../../../../components/i18n-context";
+
+import {
+    buildPointAriaLabel,
+    resolvePointLabel,
+    usePointAriaLabel,
+} from "./build-point-aria-label";
+
+import type {PropsWithChildren} from "react";
 
 const {strings, locale} = mockPerseusI18nContext;
 
+// Wrapper that provides the i18n context `usePointAriaLabel` reads. Built with
+// `createElement` (not JSX) so this stays a `.ts` test file.
+function I18nWrapper({children}: PropsWithChildren) {
+    return React.createElement(
+        PerseusI18nContextProvider,
+        {strings, locale},
+        children,
+    );
+}
+
 describe("resolvePointLabel", () => {
     it("returns the 1-indexed default when pointLabels is undefined", () => {
         expect(resolvePointLabel(undefined, 0)).toBe(1);
@@ -85,3 +107,45 @@ describe("buildPointAriaLabel", () => {
         ).toBeUndefined();
     });
 });
+
+describe("usePointAriaLabel", () => {
+    it("builds the aria-label from a plain-text label synchronously", () => {
+        // Arrange, Act
+        const {result} = renderHook(() => usePointAriaLabel(["T"]), {
+            wrapper: I18nWrapper,
+        });
+
+        // Assert
+        expect(result.current(0, [0, 0])).toBe("Point T at 0 comma 0.");
+    });
+
+    it("returns undefined for a point without a custom label", () => {
+        // Arrange, Act
+        const {result} = renderHook(() => usePointAriaLabel(["T"]), {
+            wrapper: I18nWrapper,
+        });
+
+        // Assert — no label at index 1, falls back to the numeric default
+        expect(result.current(1, [3, 5])).toBeUndefined();
+    });
+
+    it("reads the numeric default during the window, then the spoken math once resolved", async () => {
+        // Arrange, Act
+        const {result} = renderHook(
+            () => usePointAriaLabel(["$\\frac{1}{2}$"]),
+            {wrapper: I18nWrapper},
+        );
+
+        // Assert — before the async conversion resolves, the TeX slot is
+        // blanked so the aria-label falls back to the default rather than
+        // announcing the literal TeX.
+        expect(result.current(0, [2, 3])).toBeUndefined();
+
+        // Assert — once the spoken form resolves, the aria-label uses it.
+        await waitFor(() => {
+            expect(result.current(0, [2, 3])).toBe(
+                "Point one half at 2 comma 3.",
+            );
+        });
+    });
+});