@@ -17,14 +17,18 @@ const path = require('node:path')
1717
1818const {
1919 WHISPER_MODELS ,
20+ WHISPER_TEST_MODEL_NAMES ,
21+ buildWhisperManifest,
2022 buildWhisperStageBlock,
23+ buildSelectionCode,
2124 buildScript
2225} = require ( '../generate-prestage-block' )
26+ const { TEST_MODELS } = require ( '../generate-mobile-model-manifest' )
2327
2428// Wrap a bare whisper stage block in the same setup preamble the real script
2529// emits, so it is runnable in isolation with adb/curl stubbed out.
2630function wrapWhisperBlock ( block ) {
27- return `set -e\nPRESTAGE_DIR=/data/local/tmp/prestaged-models\nmkdir -p /tmp/prestage \n${ block }
31+ return `set -e\nPRESTAGE_DIR=/data/local/tmp/prestaged-models\nHOST_PRESTAGE_DIR=/tmp/prestage\ nmkdir -p "$HOST_PRESTAGE_DIR" \n${ block }
2832}
2933
3034function runWithStubs ( script , { adbExit = 0 , curlExit = 0 } ) {
@@ -44,6 +48,85 @@ function runWithStubs(script, { adbExit = 0, curlExit = 0 }) {
4448 }
4549}
4650
51+ function writeExecutable ( filePath , contents ) {
52+ fs . writeFileSync ( filePath , contents , { mode : 0o755 } )
53+ }
54+
55+ function runCompleteScript ( {
56+ grep = '' ,
57+ manifest = { } ,
58+ curlFailMatch = '' ,
59+ adbFailMatch = ''
60+ } = { } ) {
61+ const dir = fs . mkdtempSync ( path . join ( os . tmpdir ( ) , 'asr-prestage-complete-' ) )
62+ const binDir = path . join ( dir , 'bin' )
63+ const tmpDir = path . join ( dir , 'tmp' )
64+ const logDir = path . join ( dir , 'logs' )
65+ fs . mkdirSync ( binDir )
66+ fs . mkdirSync ( tmpDir )
67+ fs . mkdirSync ( logDir )
68+ fs . writeFileSync ( path . join ( tmpDir , 'qvacShardGrep.txt' ) , grep )
69+ writeExecutable (
70+ path . join ( binDir , 'curl' ) ,
71+ `#!/usr/bin/env bash
72+ set -euo pipefail
73+ printf '%s\\n' "$*" >> "$STUB_LOG_DIR/curl.log"
74+ if [ -n "\${CURL_FAIL_MATCH:-}" ] && [[ "$*" == *"$CURL_FAIL_MATCH"* ]]; then
75+ exit 22
76+ fi
77+ OUTPUT=""
78+ while [ "$#" -gt 0 ]; do
79+ case "$1" in
80+ -o)
81+ OUTPUT="$2"
82+ shift 2
83+ ;;
84+ *)
85+ shift
86+ ;;
87+ esac
88+ done
89+ printf 'model-data' > "$OUTPUT"
90+ `
91+ )
92+ writeExecutable (
93+ path . join ( binDir , 'adb' ) ,
94+ `#!/usr/bin/env bash
95+ set -euo pipefail
96+ printf '%s\\n' "$*" >> "$STUB_LOG_DIR/adb.log"
97+ if [ "$1" = "push" ] && [ -n "\${ADB_FAIL_MATCH:-}" ] && [[ "$2" == *"$ADB_FAIL_MATCH"* ]]; then
98+ exit 1
99+ fi
100+ exit 0
101+ `
102+ )
103+
104+ const manifestB64 = Buffer . from ( JSON . stringify ( manifest ) , 'utf8' ) . toString ( 'base64' )
105+ const result = childProcess . spawnSync ( 'bash' , [ '-c' , buildScript ( manifestB64 ) ] , {
106+ cwd : dir ,
107+ env : {
108+ ...process . env ,
109+ PATH : `${ binDir } ${ process . env . PATH } ,
110+ QVAC_PRESTAGE_TMP_DIR : tmpDir ,
111+ STUB_LOG_DIR : logDir ,
112+ CURL_FAIL_MATCH : curlFailMatch ,
113+ ADB_FAIL_MATCH : adbFailMatch
114+ } ,
115+ encoding : 'utf8'
116+ } )
117+ const readLog = ( name ) => {
118+ const logPath = path . join ( logDir , name )
119+ return fs . existsSync ( logPath ) ? fs . readFileSync ( logPath , 'utf8' ) : ''
120+ }
121+ const outcome = {
122+ ...result ,
123+ curlLog : readLog ( 'curl.log' ) ,
124+ adbLog : readLog ( 'adb.log' )
125+ }
126+ fs . rmSync ( dir , { recursive : true , force : true } )
127+ return outcome
128+ }
129+
47130test ( 'WHISPER_MODELS covers the full mobile set: functional + perf-sweep quants' , ( ) => {
48131 const names = WHISPER_MODELS . map ( ( m ) => m . name )
49132 // tiny + VAD (functional) plus the base/small q5_1/q8_0 perf-sweep quants.
@@ -61,6 +144,35 @@ test('WHISPER_MODELS covers the full mobile set: functional + perf-sweep quants'
61144 }
62145} )
63146
147+ test ( 'Whisper manifest selects only the models required by each test runner' , ( ) => {
148+ const manifest = buildWhisperManifest ( )
149+
150+ assert . deepEqual ( Object . keys ( manifest ) , Object . keys ( WHISPER_TEST_MODEL_NAMES ) )
151+ assert . deepEqual (
152+ manifest . runMobilePerfTinyCpuTest . map ( ( model ) => model . name ) ,
153+ [ 'ggml-tiny.bin' ]
154+ )
155+ assert . deepEqual (
156+ manifest . runMobilePerfSweepGpuTest . map ( ( model ) => model . name ) ,
157+ [
158+ 'ggml-base-q5_1.bin' ,
159+ 'ggml-base-q8_0.bin' ,
160+ 'ggml-small-q5_1.bin' ,
161+ 'ggml-small-q8_0.bin'
162+ ]
163+ )
164+ assert . deepEqual ( manifest . runLiveStreamSimulationTest , [ ] )
165+ } )
166+
167+ test ( 'Parakeet manifest makes model-free runners explicit and stages validation models' , ( ) => {
168+ assert . deepEqual ( TEST_MODELS . runParakeetCorruptedModelTest , [ ] )
169+ assert . deepEqual ( TEST_MODELS . runParakeetSortformerStreamingAliasTest , [ ] )
170+ assert . deepEqual (
171+ TEST_MODELS . runParakeetModelFileValidationTest . map ( ( model ) => model . name ) ,
172+ [ 'parakeet-tdt-0.6b-v3.q4_0.gguf' ]
173+ )
174+ } )
175+
64176test ( 'buildWhisperStageBlock stages every model with a .size sidecar and degrades gracefully' , ( ) => {
65177 const block = buildWhisperStageBlock ( [
66178 { name : 'a.bin' , url : 'https://example.com/a.bin' } ,
@@ -84,22 +196,124 @@ test('buildWhisperStageBlock stages every model with a .size sidecar and degrade
84196 assert . match ( failedDownload . stdout , / d e v i c e w i l l u s e n e t w o r k f a l l b a c k / )
85197} )
86198
87- test ( 'buildScript emits the parakeet manifest block and the whisper block together ' , ( ) => {
199+ test ( 'buildScript selects Parakeet and Whisper models from the explicit shard grep ' , ( ) => {
88200 const script = buildScript ( 'QkFTRTY0' )
201+ const selectionCode = buildSelectionCode ( )
89202 // Parakeet (fail-hard, manifest-driven).
203+ assert . ok ( script . startsWith ( 'set -euo pipefail\n' ) )
90204 assert . match ( script , / P R E S T A G E _ D I R = \/ d a t a \/ l o c a l \/ t m p \/ p r e s t a g e d - m o d e l s / )
91- assert . match ( script , / b a s e 6 4 - d > \/ t m p \/ m o d e l - m a n i f e s t \. j s o n / )
205+ assert . match ( script , / b a s e 6 4 - d > " \$ T M P _ R O O T \/ m o d e l - m a n i f e s t \. j s o n " / )
206+ assert . match ( script , / b a s e 6 4 - d > " \$ T M P _ R O O T \/ w h i s p e r - m a n i f e s t \. j s o n " / )
207+ assert . match ( script , / c a t " \$ T M P _ R O O T \/ q v a c S h a r d G r e p \. t x t " / )
208+ assert . doesNotMatch ( script , / w d i o \. c o n f i g \. d e v i c e f a r m \. j s / )
209+ assert . match ( selectionCode , / m i s s i n g m o d e l m a p p i n g f o r r u n n e r / )
210+ assert . match ( selectionCode , / i n v a l i d .* m o d e l m a p p i n g f o r r u n n e r / )
211+ assert . match ( selectionCode , / s e e n \[ k i n d \] \. g e t \( m o d e l \. n a m e \) / )
212+ assert . match ( script , / p a r a k e e t - p r e s t a g e - l i s t \. t s v / )
213+ assert . match ( script , / w h i s p e r - p r e s t a g e - l i s t \. t s v / )
92214 assert . match ( script , / a d b s h e l l t e s t - s / )
93215 assert . match ( script , / F A T A L / )
94216 // Whisper (graceful).
95- assert . match ( script , / s t a g e " g g m l - t i n y \. b i n " / )
96- assert . match ( script , / s t a g e " g g m l - s i l e r o - v 5 \. 1 \. 2 \. b i n " / )
97- assert . match ( script , / s t a g e " g g m l - b a s e - q 5 _ 1 \. b i n " / )
98- assert . match ( script , / s t a g e " g g m l - b a s e - q 8 _ 0 \. b i n " / )
99- assert . match ( script , / s t a g e " g g m l - s m a l l - q 5 _ 1 \. b i n " / )
100- assert . match ( script , / s t a g e " g g m l - s m a l l - q 8 _ 0 \. b i n " / )
217+ assert . match ( script , / s t a g e " \$ N A M E " " \$ U R L " / )
218+ assert . match ( script , / d e v i c e w i l l u s e n e t w o r k f a l l b a c k / )
101219 assert . match ( script , / \[ p r e s t a g e \] d o n e / )
102220
103- const syntax = childProcess . spawnSync ( 'sh ' , [ '-n' ] , { input : script , encoding : 'utf8' } )
221+ const syntax = childProcess . spawnSync ( 'bash ' , [ '-n' ] , { input : script , encoding : 'utf8' } )
104222 assert . equal ( syntax . status , 0 , syntax . stderr )
105223} )
224+
225+ test ( 'complete prestage script deduplicates selected Parakeet models' , ( ) => {
226+ const model = { name : 'shared.gguf' , url : 'https://example.com/shared.gguf' }
227+ const result = runCompleteScript ( {
228+ grep : 'runParakeetOneTest|runParakeetTwoTest' ,
229+ manifest : {
230+ runParakeetOneTest : [ model ] ,
231+ runParakeetTwoTest : [ model ]
232+ }
233+ } )
234+
235+ assert . equal ( result . status , 0 , result . stderr )
236+ assert . equal ( result . curlLog . trim ( ) . split ( '\n' ) . length , 1 )
237+ assert . match ( result . stderr , / 1 p a r a k e e t \+ 0 w h i s p e r m o d e l \( s \) f o r 2 t e s t \( s \) / )
238+ } )
239+
240+ test ( 'complete prestage script accepts an explicitly model-free runner' , ( ) => {
241+ const result = runCompleteScript ( {
242+ grep : 'runParakeetModelFreeTest' ,
243+ manifest : { runParakeetModelFreeTest : [ ] }
244+ } )
245+
246+ assert . equal ( result . status , 0 , result . stderr )
247+ assert . equal ( result . curlLog , '' )
248+ assert . match ( result . stderr , / 0 p a r a k e e t \+ 0 w h i s p e r m o d e l \( s \) f o r 1 t e s t \( s \) / )
249+ } )
250+
251+ test ( 'complete prestage script rejects missing grep and unknown mappings' , ( ) => {
252+ const missingGrep = runCompleteScript ( )
253+ assert . notEqual ( missingGrep . status , 0 )
254+ assert . match ( missingGrep . stdout , / F A T A L : s h a r d g r e p i s r e q u i r e d / )
255+
256+ const unknownMapping = runCompleteScript ( { grep : 'runRenamedParakeetTest' } )
257+ assert . notEqual ( unknownMapping . status , 0 )
258+ assert . match ( unknownMapping . stderr , / m i s s i n g m o d e l m a p p i n g f o r r u n n e r : r u n R e n a m e d P a r a k e e t T e s t / )
259+ } )
260+
261+ test ( 'complete prestage script rejects malformed manifest entries' , ( ) => {
262+ const result = runCompleteScript ( {
263+ grep : 'runMalformedParakeetTest' ,
264+ manifest : {
265+ runMalformedParakeetTest : [ { name : 'broken.gguf' } ]
266+ }
267+ } )
268+
269+ assert . notEqual ( result . status , 0 )
270+ assert . match (
271+ result . stderr ,
272+ / i n v a l i d p a r a k e e t m o d e l m a p p i n g f o r r u n n e r r u n M a l f o r m e d P a r a k e e t T e s t a t i n d e x 0 /
273+ )
274+ } )
275+
276+ test ( 'complete prestage script keeps Parakeet staging fail-hard' , ( ) => {
277+ const result = runCompleteScript ( {
278+ grep : 'runRequiredParakeetTest' ,
279+ manifest : {
280+ runRequiredParakeetTest : [
281+ { name : 'required.gguf' , url : 'https://example.com/required.gguf' }
282+ ]
283+ } ,
284+ curlFailMatch : 'required.gguf'
285+ } )
286+
287+ assert . notEqual ( result . status , 0 )
288+ assert . match ( result . curlLog , / r e q u i r e d \. g g u f / )
289+ } )
290+
291+ for ( const fallback of [
292+ {
293+ name : 'download failure' ,
294+ curlFailMatch : 'ggml-tiny.bin' ,
295+ expected : / h o s t d o w n l o a d f a i l e d f o r g g m l - t i n y \. b i n /
296+ } ,
297+ {
298+ name : 'model push failure' ,
299+ adbFailMatch : 'ggml-tiny.bin' ,
300+ expected : / a d b p u s h f a i l e d f o r g g m l - t i n y \. b i n /
301+ } ,
302+ {
303+ name : 'size sidecar push failure' ,
304+ adbFailMatch : 'ggml-tiny.bin.size' ,
305+ expected : / s i z e m e t a d a t a p u s h f a i l e d f o r g g m l - t i n y \. b i n /
306+ }
307+ ] ) {
308+ test ( `complete prestage script preserves Whisper fallback after ${ fallback . name } , ( ) => {
309+ const result = runCompleteScript ( {
310+ grep : 'runMobilePerfTinyCpuTest' ,
311+ curlFailMatch : fallback . curlFailMatch ,
312+ adbFailMatch : fallback . adbFailMatch
313+ } )
314+
315+ assert . equal ( result . status , 0 , result . stderr )
316+ assert . match ( result . stdout , fallback . expected )
317+ assert . match ( result . stdout , / d e v i c e w i l l u s e n e t w o r k f a l l b a c k / )
318+ } )
319+ }
0 commit comments