diff --git a/.changeset/shaggy-pumas-drum.md b/.changeset/shaggy-pumas-drum.md new file mode 100644 index 0000000000..a845151cc8 --- /dev/null +++ b/.changeset/shaggy-pumas-drum.md @@ -0,0 +1,2 @@ +--- +--- diff --git a/packages/perseus-editor/src/widgets/interactive-graph-editor/locked-figures/util.ts b/packages/perseus-editor/src/widgets/interactive-graph-editor/locked-figures/util.ts index 6446e1779e..7d3d6e0925 100644 --- a/packages/perseus-editor/src/widgets/interactive-graph-editor/locked-figures/util.ts +++ b/packages/perseus-editor/src/widgets/interactive-graph-editor/locked-figures/util.ts @@ -40,6 +40,12 @@ export function generateLockedFigureAppearanceDescription( * Exported for testing. * * Example: "Circle with radius $\frac{1}{2}$" ==> "Circle with radius one half" + * + * @deprecated Use `generateSpokenMathDetails` from `@khanacademy/perseus` + * (`packages/perseus/src/util/spoken-math.ts`) instead. This editor copy was + * duplicated into perseus so the renderer can convert point labels at runtime. + * TODO(LEMS): migrate callers (`joinLabelsAsSpokenMath` below, and the + * locked-figure settings) to the perseus util and delete this copy. */ export async function generateSpokenMathDetails(mathString: string) { const engine = await SpeechRuleEngine.setup("en"); diff --git a/packages/perseus/package.json b/packages/perseus/package.json index 08ef7eba9a..0867ab6a12 100644 --- a/packages/perseus/package.json +++ b/packages/perseus/package.json @@ -57,6 +57,7 @@ "uuid": "^10.0.0" }, "devDependencies": { + "@khanacademy/mathjax-renderer": "catalog:devDeps", "@khanacademy/wonder-blocks-announcer": "catalog:devDeps", "@khanacademy/wonder-blocks-banner": "catalog:devDeps", "@khanacademy/wonder-blocks-button": "catalog:devDeps", @@ -94,6 +95,7 @@ "underscore": "catalog:devDeps" }, "peerDependencies": { + "@khanacademy/mathjax-renderer": "catalog:peerDeps", "@khanacademy/wonder-blocks-announcer": "catalog:peerDeps", "@khanacademy/wonder-blocks-banner": "catalog:peerDeps", "@khanacademy/wonder-blocks-button": "catalog:peerDeps", @@ -130,6 +132,6 @@ }, "keywords": [], "khan": { - "catalogHash": "8560a5ad63db87f8" + "catalogHash": "3dc0969744c5c56c" } } diff --git a/packages/perseus/src/util/spoken-math.test.ts b/packages/perseus/src/util/spoken-math.test.ts new file mode 100644 index 0000000000..7ecb32ee7f --- /dev/null +++ b/packages/perseus/src/util/spoken-math.test.ts @@ -0,0 +1,116 @@ +import {generateSpokenMathDetails} from "./spoken-math"; + +describe("generateSpokenMathDetails", () => { + it("converts TeX to spoken language (root, fraction)", async () => { + // Arrange, Act + const convertedString = await generateSpokenMathDetails( + "$\\sqrt{\\frac{1}{2}}$", + ); + + // Assert + expect(convertedString).toBe("StartRoot one half EndRoot"); + }); + + it("converts TeX to spoken language (exponent)", async () => { + // Arrange, Act + const convertedString = await generateSpokenMathDetails("$x^{2}$"); + + // Assert + expect(convertedString).toBe("x Superscript 2"); + }); + + it("converts TeX to spoken language (negative)", async () => { + // Arrange, Act + const convertedString = await generateSpokenMathDetails("$-2$"); + + // Assert + expect(convertedString).toBe("negative 2"); + }); + + it("converts TeX to spoken language (subtraction)", async () => { + // Arrange, Act + const convertedString = await generateSpokenMathDetails("$2-1$"); + + // Assert + expect(convertedString).toBe("2 minus 1"); + }); + + it("converts TeX to spoken language (normal words)", async () => { + // Arrange, Act + const convertedString = + await generateSpokenMathDetails("$\\text{square b}$"); + + // Assert + expect(convertedString).toBe("square b"); + }); + + it("converts TeX to spoken language (random letters)", async () => { + // Arrange, Act + const convertedString = await generateSpokenMathDetails("$cat$"); + + // Assert + expect(convertedString).toBe("c a t"); + }); + + it("keeps non-math text as is", async () => { + // Arrange, Act + const convertedString = await generateSpokenMathDetails( + "Circle with radius $\\frac{1}{2}$ units", + ); + + // Assert + expect(convertedString).toBe("Circle with radius one half units"); + }); + + it("reads dollar signs as dollars inside tex", async () => { + // Arrange, Act + const convertedString = await generateSpokenMathDetails( + "This sandwich costs ${$}12.34$", + ); + + // Assert + expect(convertedString).toBe("This sandwich costs dollar sign 12.34"); + }); + + it("reads dollar signs as dollars outside tex", async () => { + // Arrange, Act + const convertedString = await generateSpokenMathDetails( + "This sandwich costs \\$12.34", + ); + + // Assert + expect(convertedString).toBe("This sandwich costs $12.34"); + }); + + it("reads curly braces", async () => { + // Arrange, Act + const convertedString = await generateSpokenMathDetails("Hello}{"); + + // Assert + expect(convertedString).toBe("Hello}{"); + }); + + it("reads backslashes", async () => { + // Arrange, Act + const convertedString = await generateSpokenMathDetails("\\"); + + // Assert + expect(convertedString).toBe("\\"); + }); + + it("reads lone dollar signs as regular dollar signs", async () => { + // Arrange, Act + const convertedString = await generateSpokenMathDetails("$50"); + + // Assert + expect(convertedString).toBe("$50"); + }); + + it("reads lone escaped dollar signs in text as regular dollar signs", async () => { + // Arrange, Act + const convertedString = await generateSpokenMathDetails("\\$50"); + + // Assert + expect(convertedString).toBe("$50"); + }); +}); diff --git a/packages/perseus/src/util/spoken-math.ts b/packages/perseus/src/util/spoken-math.ts new file mode 100644 index 0000000000..d934104aa4 --- /dev/null +++ b/packages/perseus/src/util/spoken-math.ts @@ -0,0 +1,48 @@ +import {SpeechRuleEngine} from "@khanacademy/mathjax-renderer"; + +import {mathOnlyParser} from "../widgets/interactive-graphs/utils"; + +/** + * Given a string that may contain math within TeX represented by $...$, + * returns the spoken math equivalent using the SpeechRuleEngine. + * + * Example: "Circle with radius $\frac{1}{2}$" ==> "Circle with radius one half" + * + * NOTE(LEMS): This is duplicated from perseus-editor's + * `interactive-graph-editor/locked-figures/util.ts` so it can run in the + * renderer (perseus can't import from perseus-editor). A follow-up will + * migrate the editor to import this copy and delete its own. + */ +export async function generateSpokenMathDetails( + mathString: string, +): Promise { + const engine = await SpeechRuleEngine.setup("en"); + let convertedSpeech = ""; + + // All the information we need is in the first section, + // whether it's typed as "blockmath" or "paragraph" + const parsedContent = mathOnlyParser(mathString); + + // If it's a paragraph, we need to iterate through the sections + // to look for individual math blocks. + for (const piece of parsedContent) { + switch (piece.type) { + case "math": + convertedSpeech += engine.texToSpeech(piece.content); + break; + case "specialCharacter": + // We don't want the backslash from special character + // to show up in the generated aria label. + convertedSpeech += + piece.content.length > 1 + ? piece.content.slice(1) + : piece.content; + break; + default: + convertedSpeech += piece.content; + break; + } + } + + return convertedSpeech; +} diff --git a/packages/perseus/src/widgets/interactive-graphs/__docs__/interactive-graph.stories.tsx b/packages/perseus/src/widgets/interactive-graphs/__docs__/interactive-graph.stories.tsx index 952add6e09..2513d30d11 100644 --- a/packages/perseus/src/widgets/interactive-graphs/__docs__/interactive-graph.stories.tsx +++ b/packages/perseus/src/widgets/interactive-graphs/__docs__/interactive-graph.stories.tsx @@ -11,6 +11,7 @@ import { linearWithCustomLabelsQuestion, pointQuestion, pointWithCustomLabelQuestion, + pointWithTexLabelQuestion, pointWithDefaultLabelQuestion, polygonQuestion, polygonWithCustomLabelsQuestion, @@ -175,6 +176,24 @@ export const PointWithCustomLabel: Story = { }, }; +/** + * A point whose custom label is TeX: `$\left(\dfrac{1}{2}, 3\right)$`. The + * visible on-canvas label renders the math correctly, but the screen reader + * currently announces the raw TeX literally because the label string is read + * verbatim. Use this story with a screen reader to hear the bug that + * spoken-math conversion fixes. + */ +export const PointWithTeXLabel: Story = { + globals: { + featureFlags: ["perseus-enable-point-label-field"], + }, + args: { + item: generateTestPerseusItem({ + question: pointWithTexLabelQuestion, + }), + }, +}; + /** * The same reference question as `PointWithCustomLabel`, but * with `pointLabels` omitted so the interactive point falls back to the diff --git a/packages/perseus/src/widgets/interactive-graphs/graphs/components/build-point-aria-label.test.ts b/packages/perseus/src/widgets/interactive-graphs/graphs/components/build-point-aria-label.test.ts index b9f00c5621..2f29d967fd 100644 --- a/packages/perseus/src/widgets/interactive-graphs/graphs/components/build-point-aria-label.test.ts +++ b/packages/perseus/src/widgets/interactive-graphs/graphs/components/build-point-aria-label.test.ts @@ -1,9 +1,31 @@ -import {mockPerseusI18nContext} from "../../../../components/i18n-context"; +import {renderHook, waitFor} from "@testing-library/react"; +import * as React from "react"; -import {buildPointAriaLabel, resolvePointLabel} from "./build-point-aria-label"; +import { + mockPerseusI18nContext, + PerseusI18nContextProvider, +} from "../../../../components/i18n-context"; + +import { + buildPointAriaLabel, + resolvePointLabel, + usePointAriaLabel, +} from "./build-point-aria-label"; + +import type {PropsWithChildren} from "react"; const {strings, locale} = mockPerseusI18nContext; +// Wrapper that provides the i18n context `usePointAriaLabel` reads. Built with +// `createElement` (not JSX) so this stays a `.ts` test file. +function I18nWrapper({children}: PropsWithChildren) { + return React.createElement( + PerseusI18nContextProvider, + {strings, locale}, + children, + ); +} + describe("resolvePointLabel", () => { it("returns the 1-indexed default when pointLabels is undefined", () => { expect(resolvePointLabel(undefined, 0)).toBe(1); @@ -85,3 +107,45 @@ describe("buildPointAriaLabel", () => { ).toBeUndefined(); }); }); + +describe("usePointAriaLabel", () => { + it("builds the aria-label from a plain-text label synchronously", () => { + // Arrange, Act + const {result} = renderHook(() => usePointAriaLabel(["T"]), { + wrapper: I18nWrapper, + }); + + // Assert + expect(result.current(0, [0, 0])).toBe("Point T at 0 comma 0."); + }); + + it("returns undefined for a point without a custom label", () => { + // Arrange, Act + const {result} = renderHook(() => usePointAriaLabel(["T"]), { + wrapper: I18nWrapper, + }); + + // Assert — no label at index 1, falls back to the numeric default + expect(result.current(1, [3, 5])).toBeUndefined(); + }); + + it("reads the numeric default during the window, then the spoken math once resolved", async () => { + // Arrange, Act + const {result} = renderHook( + () => usePointAriaLabel(["$\\frac{1}{2}$"]), + {wrapper: I18nWrapper}, + ); + + // Assert — before the async conversion resolves, the TeX slot is + // blanked so the aria-label falls back to the default rather than + // announcing the literal TeX. + expect(result.current(0, [2, 3])).toBeUndefined(); + + // Assert — once the spoken form resolves, the aria-label uses it. + await waitFor(() => { + expect(result.current(0, [2, 3])).toBe( + "Point one half at 2 comma 3.", + ); + }); + }); +}); diff --git a/packages/perseus/src/widgets/interactive-graphs/graphs/components/build-point-aria-label.ts b/packages/perseus/src/widgets/interactive-graphs/graphs/components/build-point-aria-label.ts index 2fac281236..c7d2116cf5 100644 --- a/packages/perseus/src/widgets/interactive-graphs/graphs/components/build-point-aria-label.ts +++ b/packages/perseus/src/widgets/interactive-graphs/graphs/components/build-point-aria-label.ts @@ -1,4 +1,7 @@ +import * as React from "react"; + import {usePerseusI18n} from "../../../../components/i18n-context"; +import {generateSpokenMathDetails} from "../../../../util/spoken-math"; import {srFormatNumber} from "../screenreader-text"; import type {PerseusStrings} from "../../../../strings"; @@ -52,15 +55,94 @@ export function buildPointAriaLabel( }); } +// Returns true when a label contains a TeX math segment (e.g. "$\dfrac12$") +// that a screen reader would otherwise read literally. Plain-text labels are +// announced correctly as-is. The `typeof` guard mirrors `resolvePointLabel`'s +// defensiveness — non-string entries can slip past the parser via malformed +// hand-authored JSON. +function hasTeX(label: unknown): boolean { + return typeof label === "string" && label.includes("$"); +} + +// Seeds the initial spoken-label state: blanks only the TeX entries so they +// fall back to the generic "Point N" default during the async conversion +// window (never reading raw TeX), while keeping plain-text labels intact. +function seedSpokenLabels( + pointLabels: ReadonlyArray | undefined, +): ReadonlyArray | undefined { + return pointLabels?.map((label) => (hasTeX(label) ? "" : label)); +} + +/** + * Resolves the spoken-math form of each point label for screen readers. + * + * A TeX label like `$\dfrac{1}{2}$` renders fine visually but is read + * literally ("dollar dfrac one …") by screen readers, so we convert it to + * spoken text via `generateSpokenMathDetails`. The conversion is async (the + * speech engine loads on first use), so the results are resolved into state. + * + * Until a TeX label resolves, its slot stays `""` so the aria-label falls + * back to the generic "Point N" default rather than announcing raw TeX. Note + * that an in-place aria-label change is not re-announced, so a point focused + * during the (one-time, cached) window reads the generic label until the user + * re-focuses it. Plain-text labels — and the case where no label contains + * TeX — skip the engine entirely. + */ +function useSpokenPointLabels( + pointLabels: ReadonlyArray | undefined, +): ReadonlyArray | undefined { + const [spokenLabels, setSpokenLabels] = React.useState< + ReadonlyArray | undefined + >(() => seedSpokenLabels(pointLabels)); + + // Stable content key so the effect re-runs only when the label *values* + // change, not when the caller recreates the array each render. Keying on + // the array itself would loop: effect -> setState -> re-render -> new + // array ref -> effect -> ... + const labelsKey = pointLabels == null ? "" : JSON.stringify(pointLabels); + + React.useEffect(() => { + // No TeX anywhere -> nothing to convert; use the labels as-is. + if (pointLabels == null || !pointLabels.some(hasTeX)) { + setSpokenLabels(pointLabels); + return; + } + + let cancelled = false; + Promise.all( + pointLabels.map((label) => + hasTeX(label) ? generateSpokenMathDetails(label) : label, + ), + ).then((resolved) => { + if (!cancelled) { + setSpokenLabels(resolved); + } + }); + return () => { + cancelled = true; + }; + // `pointLabels` is read via the stable `labelsKey` rather than being + // listed as a dep — listing the array (a fresh ref each render) would + // re-fire this effect on every render. + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [labelsKey]); + + return spokenLabels; +} + /** * Hook that returns a point aria-label builder bound to the current locale * and the given `pointLabels`. Returns `undefined` for points without a * custom label so callers can fall back to default labels. + * + * Labels containing TeX are converted to spoken math (see + * `useSpokenPointLabels`) so screen readers don't read the raw TeX literally. */ export function usePointAriaLabel( pointLabels: ReadonlyArray | undefined, ) { const {strings, locale} = usePerseusI18n(); + const spokenLabels = useSpokenPointLabels(pointLabels); return (index: number, point: vec.Vector2) => - buildPointAriaLabel(pointLabels, index, point, strings, locale); + buildPointAriaLabel(spokenLabels, index, point, strings, locale); } diff --git a/packages/perseus/src/widgets/interactive-graphs/interactive-graph.testdata.ts b/packages/perseus/src/widgets/interactive-graphs/interactive-graph.testdata.ts index 98aef998c7..b00571419a 100644 --- a/packages/perseus/src/widgets/interactive-graphs/interactive-graph.testdata.ts +++ b/packages/perseus/src/widgets/interactive-graphs/interactive-graph.testdata.ts @@ -1272,6 +1272,32 @@ export const pointWithCustomLabelQuestion: PerseusRenderer = }), }); +// A point whose custom label contains TeX. The visible on-canvas label +// renders the math correctly via , but the screen-reader aria-label +// currently reads the raw TeX literally (e.g. "dollar left dfrac one …") +// because the label string is passed straight through. Used to demo the +// spoken-math conversion fix. +export const pointWithTexLabelQuestion: PerseusRenderer = + generateInteractiveGraphQuestion({ + content: + "**Plot the point and listen to how its label is announced.**\n\n[[☃ interactive-graph 1]]", + markings: "graph", + gridStep: [1, 1], + snapStep: [1, 1], + step: [1, 1], + range: [ + [-6, 6], + [-2, 6], + ], + correct: generateIGPointGraph({ + numPoints: 1, + startCoords: [[0, 0]], + coords: [[2, 3]], + pointLabels: ["$\\left(\\dfrac{1}{2}, 3\\right)$"], + showPointLabels: true, + }), + }); + // Same reference question as `pointWithCustomLabelQuestion` // but with `pointLabels` omitted, so the interactive point falls back to // the legacy numeric default. JAWS announces "Point 1 at …" even though diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 6b1a5304d6..9dd105895b 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -619,6 +619,9 @@ importers: specifier: ^10.0.0 version: 10.0.0 devDependencies: + '@khanacademy/mathjax-renderer': + specifier: catalog:devDeps + version: 3.0.0 '@khanacademy/wonder-blocks-announcer': specifier: catalog:devDeps version: 1.1.1(aphrodite@1.2.5)(react-dom@18.2.0(react@18.2.0))(react-router-dom-v5-compat@6.30.0(react-dom@18.2.0(react@18.2.0))(react-router-dom@5.3.4(react@18.2.0))(react@18.2.0))(react-router-dom@5.3.4(react@18.2.0))(react-router@5.3.4(react@18.2.0))(react@18.2.0)