Skip to content

Commit 028ed4c

Browse files
committed
PR 7: Test Artifacts
1 parent 0ae5eb6 commit 028ed4c

8 files changed

Lines changed: 542 additions & 12 deletions

File tree

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
name: CI-VM scripts
2+
3+
# Static analysis for the CI-VM runner scripts (the scripts that run on the
4+
# throwaway test VMs). Runs only when those scripts change. shellcheck lints the
5+
# bash runners; PSScriptAnalyzer lints the PowerShell startup script.
6+
7+
on:
8+
pull_request:
9+
paths:
10+
- 'install/ci-vm/**'
11+
- '.github/workflows/ci-vm-scripts.yml'
12+
push:
13+
paths:
14+
- 'install/ci-vm/**'
15+
- '.github/workflows/ci-vm-scripts.yml'
16+
17+
jobs:
18+
shellcheck:
19+
runs-on: ubuntu-latest
20+
steps:
21+
- uses: actions/checkout@v4
22+
- name: Install shellcheck
23+
run: sudo apt-get update && sudo apt-get install -y shellcheck
24+
- name: shellcheck (bash runner scripts)
25+
run: shellcheck --severity=error install/ci-vm/ci-linux/ci/runCI install/ci-vm/ci-linux/startup-script.sh
26+
27+
psscriptanalyzer:
28+
runs-on: windows-latest
29+
steps:
30+
- uses: actions/checkout@v4
31+
- name: PSScriptAnalyzer (startup-script.ps1)
32+
shell: pwsh
33+
run: |
34+
Set-PSRepository PSGallery -InstallationPolicy Trusted
35+
Install-Module -Name PSScriptAnalyzer -Force -Scope CurrentUser
36+
$issues = Invoke-ScriptAnalyzer -Path install/ci-vm/ci-windows/startup-script.ps1 -Severity Error
37+
if ($issues) { $issues | Format-Table -AutoSize; exit 1 }

install/ci-vm/ci-linux/ci/runCI

Lines changed: 40 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -103,6 +103,27 @@ function executeCommand {
103103
fi
104104
}
105105

106+
# Send a build artifact to the server (best-effort: a failed upload must never
107+
# abort the run, so this always returns 0). Reuses the reportURL/curl pattern.
108+
function sendArtifact {
109+
local artifactType="$1"
110+
local filePath="$2"
111+
if [ ! -f "${filePath}" ]; then
112+
echo "Artifact ${artifactType}: no file at '${filePath}', skipping" >> "${logFile}"
113+
return 0
114+
fi
115+
echo "Uploading ${artifactType} artifact (${filePath})" >> "${logFile}"
116+
# --fail surfaces HTTP >= 400 (e.g. 413 on an oversized coredump) as a
117+
# curl error so failed uploads are visible in the log; still best-effort.
118+
curl -s --fail -A "${userAgent}" --form "type=artifactupload" --form "artifact_type=${artifactType}" \
119+
--form "file=@${filePath}" "${reportURL}" >> "${logFile}" 2>&1
120+
local curlStatus=$?
121+
if [ ${curlStatus} -ne 0 ]; then
122+
echo "Artifact ${artifactType}: upload failed (curl exit ${curlStatus})" >> "${logFile}"
123+
fi
124+
return 0
125+
}
126+
106127
# Source variables
107128
. "$DIR/variables"
108129

@@ -125,7 +146,25 @@ if [ -e "${dstDir}/ccextractor" ]; then
125146
echo "=== End Version Info ===" >> "${logFile}"
126147
postStatus "testing" "Running tests"
127148
executeCommand cd ${suiteDstDir}
128-
executeCommand ${tester} --debug --entries "${testFile}" --executable "ccextractor" --tempfolder "${tempFolder}" --timeout 600 --reportfolder "${reportFolder}" --resultfolder "${resultFolder}" --samplefolder "${sampleFolder}" --method Server --url "${reportURL}"
149+
150+
# Enable core dumps and capture the test run's combined stdout/stderr so
151+
# both can be uploaded as artifacts (the API serves them per run).
152+
ulimit -c unlimited 2>/dev/null
153+
echo "core.%p" | sudo tee /proc/sys/kernel/core_pattern >/dev/null 2>&1
154+
combinedLog="${reportFolder}/combined_stdout.log"
155+
${tester} --debug --entries "${testFile}" --executable "ccextractor" --tempfolder "${tempFolder}" --timeout 600 --reportfolder "${reportFolder}" --resultfolder "${resultFolder}" --samplefolder "${sampleFolder}" --method Server --url "${reportURL}" > "${combinedLog}" 2>&1
156+
testerStatus=$?
157+
cat "${combinedLog}" >> "${logFile}"
158+
159+
# Upload artifacts before any failure-halt so crash data survives.
160+
sendArtifact "binary" "${dstDir}/ccextractor"
161+
sendArtifact "combined_stdout" "${combinedLog}"
162+
sendArtifact "coredump" "$(ls -1t core.* 2>/dev/null | head -n1)"
163+
164+
if [ ${testerStatus} -ne 0 ]; then
165+
haltAndCatchFire ""
166+
fi
167+
129168
sendLogFile
130169
postStatus "completed" "Ran all tests"
131170

install/ci-vm/ci-linux/startup-script.sh

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ curl -L -O https://github.com/GoogleCloudPlatform/gcsfuse/releases/download/v3.2
44
dpkg --install gcsfuse_3.2.0_amd64.deb
55
rm gcsfuse_3.2.0_amd64.deb
66

7-
apt install gnupg ca-certificates
7+
apt install -y gnupg ca-certificates
88
gpg --homedir /tmp --no-default-keyring --keyring /usr/share/keyrings/mono-official-archive-keyring.gpg --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF
99
echo "deb [signed-by=/usr/share/keyrings/mono-official-archive-keyring.gpg] https://download.mono-project.com/repo/ubuntu stable-focal main" | sudo tee /etc/apt/sources.list.d/mono-official-stable.list
1010
sudo apt update
@@ -14,7 +14,8 @@ mkdir repository
1414
cd repository
1515

1616
# Use gcsfuse and import required files
17-
mkdir temp TestFiles TestResults vm_data reports
17+
# TempFiles is used by the tester (--tempfolder) and must exist
18+
mkdir temp TestFiles TestResults TempFiles vm_data reports
1819

1920
gcs_bucket=$(curl http://metadata/computeMetadata/v1/instance/attributes/bucket -H "Metadata-Flavor: Google")
2021

@@ -31,6 +32,9 @@ mount vm_data
3132
mount TestFiles
3233
mount TestResults
3334

35+
# Give gcsfuse mounts time to become ready
36+
sleep 10
37+
3438
cp temp/* ./
3539

3640
chmod +x bootstrap

install/ci-vm/ci-windows/ci/runCI.bat

Lines changed: 37 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,10 @@ for /F %%R in ('curl http://metadata/computeMetadata/v1/instance/attributes/repo
1515
SET userAgent="CCX/CI_BOT"
1616
SET logFile="%reportFolder%/log.html"
1717

18-
call :postStatus "preparation" "Loaded variables, created log file and checking for CCExtractor build artifact" >> "%logFile%"
18+
rem NB: no outer ">> %logFile%" here. postStatus already appends to %logFile%
19+
rem internally; an outer redirect to the same file self-locks on Windows
20+
rem (the inner append cannot open a file the outer redirect holds open).
21+
call :postStatus "preparation" "Loaded variables, created log file and checking for CCExtractor build artifact"
1922

2023
echo Checking for CCExtractor build artifact
2124
if EXIST "%dstDir%\ccextractorwinfull.exe" (
@@ -27,7 +30,21 @@ if EXIST "%dstDir%\ccextractorwinfull.exe" (
2730
echo === End Version Info === >> "%logFile%"
2831
call :postStatus "testing" "Running tests"
2932
call :executeCommand cd %suiteDstDir%
30-
call :executeCommand "%tester%" --debug True --entries "%testFile%" --executable "ccextractorwinfull.exe" --tempfolder "%tempFolder%" --timeout 600 --reportfolder "%reportFolder%" --resultfolder "%resultFolder%" --samplefolder "%sampleFolder%" --method Server --url "%reportURL%"
33+
34+
rem Capture the test run's combined stdout/stderr for upload as an artifact.
35+
rem NB: "||" instead of "if errorlevel 1" - it fires at runtime on ANY
36+
rem nonzero exit code, including the negative NTSTATUS codes a crash
37+
rem returns, e.g. 0xC0000005, and works inside this parenthesized block
38+
rem where %ERRORLEVEL% would expand too early.
39+
set "testerFailed="
40+
"%tester%" --debug True --entries "%testFile%" --executable "ccextractorwinfull.exe" --tempfolder "%tempFolder%" --timeout 600 --reportfolder "%reportFolder%" --resultfolder "%resultFolder%" --samplefolder "%sampleFolder%" --method Server --url "%reportURL%" > "%reportFolder%/combined_stdout.log" 2>&1 || set "testerFailed=1"
41+
type "%reportFolder%/combined_stdout.log" >> "%logFile%"
42+
43+
rem Upload artifacts (best-effort; never aborts the run). Windows skips coredump for v1.
44+
call :sendArtifact "binary" "%dstDir%\ccextractorwinfull.exe"
45+
call :sendArtifact "combined_stdout" "%reportFolder%/combined_stdout.log"
46+
47+
if defined testerFailed call :haltAndCatchFire ""
3148

3249
call :sendLogFile
3350

@@ -144,3 +161,21 @@ if !sl_attempt! LEQ %sl_max_retries% (
144161
echo ERROR: Failed to upload log after %sl_max_retries% attempts >> "%logFile%"
145162
endlocal
146163
EXIT /B 1
164+
165+
rem Send a build artifact to the server (best-effort; never aborts the run)
166+
:sendArtifact
167+
setlocal
168+
set "sa_type=%~1"
169+
set "sa_file=%~2"
170+
if NOT EXIST "%sa_file%" (
171+
echo Artifact %sa_type%: no file at "%sa_file%", skipping >> "%logFile%"
172+
endlocal
173+
EXIT /B 0
174+
)
175+
echo Uploading %sa_type% artifact (%sa_file%) >> "%logFile%"
176+
rem --fail makes HTTP >= 400 (e.g. 413 on an oversized file) a curl error so
177+
rem the failure is at least visible in the log; the upload stays best-effort.
178+
curl -s --fail -A "%userAgent%" --form "type=artifactupload" --form "artifact_type=%sa_type%" --form "file=@%sa_file%" "%reportURL%" >> "%logFile%" 2>&1
179+
if errorlevel 1 echo Artifact %sa_type%: upload failed with curl exit code %ERRORLEVEL% >> "%logFile%"
180+
endlocal
181+
EXIT /B 0

mod_api/routes/system.py

Lines changed: 148 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,24 +1,33 @@
11
"""
2-
System, health, and queue routes.
2+
System, health, queue, and artifact routes.
33
44
GET /system/health Health check (unauthenticated)
55
GET /system/queue Queue status — active + queued runs
6+
GET /runs/{id}/artifacts Run artifacts from GCS + local storage
67
"""
78

89
import os
910
from datetime import datetime, timezone
1011

1112
from flask import g, jsonify, request
1213
from sqlalchemy import text
14+
from sqlalchemy.orm import joinedload
1315

1416
from mod_api import mod_api
1517
from mod_api.middleware.auth import require_scope
1618
from mod_api.middleware.error_handler import make_error_response
17-
from mod_api.middleware.validation import validate_offset_pagination
19+
from mod_api.middleware.validation import (validate_offset_pagination,
20+
validate_path_id)
1821
from mod_api.schemas.common import DATETIME_FORMAT
19-
from mod_api.services.status import batch_get_run_data
20-
from mod_api.utils import paginated_response
21-
from mod_test.models import Test, TestPlatform, TestProgress, TestStatus
22+
from mod_api.services.status import batch_get_run_data, is_dummy_row
23+
from mod_api.services.storage import (get_log_file_path,
24+
get_test_results_base_path,
25+
resolve_artifact)
26+
from mod_api.utils import paginated_response, safe_resolve
27+
from mod_test.models import (Test, TestPlatform, TestProgress, TestResultFile,
28+
TestStatus)
29+
30+
OCTET_STREAM = 'application/octet-stream'
2231

2332

2433
@mod_api.route('/system/health', methods=['GET'])
@@ -201,3 +210,137 @@ def get_queue(limit=50, offset=0):
201210
'running_count': running_count,
202211
}
203212
)
213+
214+
215+
def _get_gcs_artifacts(run_id, platform):
216+
binary_name = (
217+
'ccextractor' if platform == TestPlatform.linux
218+
else 'ccextractorwinfull.exe'
219+
)
220+
gcs_artifacts = [
221+
('binary',
222+
f'test_artifacts/{run_id}/{binary_name}', binary_name, OCTET_STREAM),
223+
('coredump', f'test_artifacts/{run_id}/coredump',
224+
f'coredump-{run_id}', OCTET_STREAM),
225+
(
226+
'combined_stdout',
227+
f'test_artifacts/{run_id}/combined_stdout.log',
228+
f'combined_stdout-{run_id}.log',
229+
'text/plain',
230+
),
231+
]
232+
artifacts = []
233+
for artifact_type, gcs_path, filename, content_type in gcs_artifacts:
234+
download_url, storage_status = resolve_artifact(gcs_path)
235+
artifacts.append({
236+
'artifact_id': f'{artifact_type}_{run_id}',
237+
'run_id': run_id,
238+
'sample_id': None,
239+
'type': artifact_type,
240+
'filename': filename,
241+
'content_type': content_type,
242+
'size_bytes': None,
243+
'storage_status': storage_status,
244+
'download_url': download_url,
245+
})
246+
return artifacts
247+
248+
249+
def _get_output_artifacts(run_id):
250+
result_files = TestResultFile.query.options(
251+
joinedload(TestResultFile.regression_test_output),
252+
joinedload(TestResultFile.regression_test),
253+
).filter_by(test_id=run_id).all()
254+
for rf in result_files:
255+
if is_dummy_row(rf):
256+
continue
257+
258+
ext = rf.regression_test_output.correct_extension if rf.regression_test_output else ''
259+
sample_id = (rf.regression_test.sample_id
260+
if rf.regression_test else None)
261+
262+
expected_name = rf.expected + ext
263+
# NOTE: storage metadata (storage_status, download_url, size_bytes,
264+
# content_type) is resolved by list_artifacts for paged items only.
265+
266+
yield {
267+
'artifact_id': f'expected_{run_id}_{rf.regression_test_id}_{rf.regression_test_output_id}',
268+
'run_id': run_id,
269+
'sample_id': sample_id,
270+
'type': 'expected_output',
271+
'filename': expected_name,
272+
}
273+
274+
if rf.got is not None:
275+
actual_name = rf.got + ext
276+
yield {
277+
'artifact_id': f'actual_{run_id}_{rf.regression_test_id}_{rf.regression_test_output_id}',
278+
'run_id': run_id,
279+
'sample_id': sample_id,
280+
'type': 'actual_output',
281+
'filename': actual_name,
282+
}
283+
284+
285+
@mod_api.route('/runs/<run_id>/artifacts', methods=['GET'])
286+
@require_scope('results:read')
287+
@validate_path_id('run_id')
288+
@validate_offset_pagination()
289+
def list_artifacts(run_id, limit=50, offset=0):
290+
"""
291+
List all artifacts for a run.
292+
293+
Checks both GCS and local storage. Falls back to local when GCS
294+
is unavailable. Supports ?type filter.
295+
"""
296+
test = Test.query.filter(Test.id == run_id).first()
297+
if test is None:
298+
return make_error_response(
299+
'not_found',
300+
f'Run {run_id} not found.',
301+
http_status=404)
302+
303+
artifacts = _get_gcs_artifacts(run_id, test.platform)
304+
305+
# Build log — accessed via /runs/{id}/logs, no direct download link.
306+
log_path = get_log_file_path(run_id)
307+
artifacts.append({
308+
'artifact_id': f'buildlog_{run_id}',
309+
'run_id': run_id,
310+
'sample_id': None,
311+
'type': 'build_log',
312+
'filename': f'{run_id}.txt',
313+
'content_type': 'text/plain',
314+
'size_bytes': os.path.getsize(log_path) if log_path else None,
315+
'storage_status': 'ok' if log_path else 'missing',
316+
'download_url': None,
317+
})
318+
319+
artifacts.extend(list(_get_output_artifacts(run_id)))
320+
321+
# Apply optional ?type filter.
322+
type_filter = request.args.get('type')
323+
if type_filter:
324+
artifacts = [a for a in artifacts if a['type'] == type_filter]
325+
326+
total = len(artifacts)
327+
paged = artifacts[offset:offset + limit]
328+
329+
# Resolve heavy artifact metadata only for the returned page
330+
base_path = get_test_results_base_path()
331+
for a in paged:
332+
if 'storage_status' not in a:
333+
# It's an output artifact
334+
filename = a['filename']
335+
url, status = resolve_artifact(f'TestResults/{filename}')
336+
local = safe_resolve(base_path, filename)
337+
338+
a['content_type'] = OCTET_STREAM
339+
a['size_bytes'] = (
340+
os.path.getsize(local)
341+
if local and os.path.isfile(local) else None
342+
)
343+
a['storage_status'] = status
344+
a['download_url'] = url
345+
346+
return paginated_response(paged, total, limit, offset)

0 commit comments

Comments
 (0)