Skip to content
tests

Implement SMVE(Sparse Multi Vector Encoding) (#3372) #15081

Sign in to view logs

Implement SMVE(Sparse Multi Vector Encoding) (#3372)

Implement SMVE(Sparse Multi Vector Encoding) (#3372) #15081

Workflow file for this run

.github/workflows/tests.yml at ed8b010
name: tests
on:
push:
branches:
- 'main'
- '*.*.*'
paths-ignore:
- 'docs/**'
- '*.md'
# The only difference between pull_request and pull_request_target is the context in which the workflow runs:
# — pull_request_target workflows use the workflow files from the default branch, and secrets are available.
# — pull_request workflows use the workflow files from the pull request branch, and secrets are unavailable.
pull_request:
types: [ synchronize, ready_for_review ]
paths-ignore:
- 'docs/**'
- '*.md'
permissions:
contents: read
actions: read
checks: write
statuses: write
# https://docs.github.com/en/actions/using-jobs/using-concurrency
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
code_static_check:
name: code_static_check
if: ${{ github.event_name != 'pull_request' || (github.event.pull_request.draft == false && contains(github.event.pull_request.labels.*.name, 'ci')) }}
runs-on: ubuntu-latest
steps:
- name: Free disk space
run: |
sudo rm -rf /usr/share/dotnet /usr/local/lib/android/sdk /opt/ghc /opt/hostedtoolcache/CodeQL
df -h /
- name: Check out code
uses: actions/checkout@v6
with:
ref: ${{ (github.event_name == 'pull_request' || github.event_name == 'pull_request_target') && format('refs/pull/{0}/merge', github.event.pull_request.number) || github.sha }}
fetch-depth: 0
fetch-tags: false
submodules: recursive
- name: Static check with Ruff
uses: astral-sh/ruff-action@v3
with:
version: ">=0.14.x"
args: "check --exclude=**/remote_thrift/infinity_thrift_rpc/**"
- name: Start builder container
run: |
docker run --privileged --cap-add=NET_ADMIN -d --name infinity_builder -v ${PWD}:/infinity -v ${PWD}/resource:/usr/share/infinity/resource infiniflow/infinity_builder:ubuntu22_clang20
- name: Check format of changed C/C++ files
run: |
if [[ ${{ github.event_name }} == 'pull_request' ]]; then
CHANGED_FILES=$(git diff --name-only ${{ github.event.pull_request.base.sha }}...${{ github.event.pull_request.head.sha }} \
| grep -E '\.(cpp|h|hpp|cppm)$' \
| grep -v 'third_party/' \
| grep -v 'network/' \
| grep -v 'remote_thrift/infinity_thrift_rpc/' \
| grep -v 'parser/' || true)
if [ -n "$CHANGED_FILES" ]; then
echo "Check format of changed C/C++ files with clang-format-20"
readarray -t files <<< "$CHANGED_FILES"
HAS_ERROR=0
for file in "${files[@]}"; do
if [ -f "$file" ]; then
if docker exec infinity_builder bash -c "cd /infinity; clang-format-20 $file > ${file}.formatted; diff $file ${file}.formatted 2>/dev/null"; then
echo "✅ $file"
else
echo "❌ $file"
HAS_ERROR=1
fi
rm -f "${file}.formatted"
fi
done
if [ $HAS_ERROR -ne 0 ]; then
exit 1
fi
else
echo "No C++ files changed"
fi
fi
- name: Remove container
if: always()
run: docker rm -f infinity_builder || true
debug_amd64_build:
name: debug_amd64_build
runs-on: ubuntu-latest
needs: code_static_check
steps:
- name: Free disk space
run: |
sudo rm -rf /usr/share/dotnet /usr/local/lib/android/sdk /opt/ghc /opt/hostedtoolcache/CodeQL
df -h /
- name: Check out code
uses: actions/checkout@v6
with:
ref: ${{ (github.event_name == 'pull_request' || github.event_name == 'pull_request_target') && format('refs/pull/{0}/merge', github.event.pull_request.number) || github.sha }}
fetch-depth: 0
fetch-tags: false
submodules: recursive
- name: Start builder container
run: |
docker run --privileged -d --name infinity_builder -v ${PWD}:/infinity -v ${PWD}/resource:/usr/share/infinity/resource infiniflow/infinity_builder:ubuntu22_clang20
- name: Build debug version
run: docker exec infinity_builder bash -c "git config --global safe.directory \"*\" && cd /infinity && rm -rf cmake-build-debug && mkdir -p cmake-build-debug && cmake -G Ninja -DCMAKE_BUILD_TYPE=Debug -DCODE_COVERAGE=ON -DIS_CI_TEST_PIPELINE=ON -S /infinity -B /infinity/cmake-build-debug && cmake --build /infinity/cmake-build-debug -j --target infinity test_main"
- name: Upload artifacts
uses: actions/upload-artifact@v7
with:
name: debug-build-amd64
path: |
cmake-build-debug/src/infinity
cmake-build-debug/src/test_main
retention-days: 1
if-no-files-found: warn
- name: Remove build container
if: always()
run: docker rm -f infinity_builder || true
debug_amd64_unit_test:
name: debug_amd64_unit_test
runs-on: ubuntu-latest
needs: debug_amd64_build
steps:
- name: Free disk space
uses: jlumbroso/free-disk-space@main
with:
tool-cache: false
android: true
dotnet: true
haskell: true
large-packages: true
docker-images: true
swap-storage: true
- name: Check out code
uses: actions/checkout@v6
with:
ref: ${{ (github.event_name == 'pull_request' || github.event_name == 'pull_request_target') && format('refs/pull/{0}/merge', github.event.pull_request.number) || github.sha }}
fetch-depth: 0
submodules: recursive
- name: Start minio container
run: |
docker run -d --name minio --network host -e "MINIO_ROOT_PASSWORD=minioadmin" -e "MINIO_ROOT_USER=minioadmin" -v /tmp/minio-data:/data quay.io/minio/minio server /data --console-address ":9006" --address ":9005"
- name: Download debug unit test build artifacts
uses: actions/download-artifact@v8
with:
name: debug-build-amd64
- name: Prepare environment
run: |
chmod +x test_main
sudo mkdir -p /usr/share/infinity/resource
sudo cp -r resource/* /usr/share/infinity/resource/
sudo chown -R ${USER} /usr/share/infinity
sudo mkdir -p /var/infinity
sudo chown -R ${USER} /var/infinity
- name: Install Python SDK
uses: astral-sh/setup-uv@v7
# with:
# enable-cache: true
- run: uv sync --python 3.11 --extra test -i https://pypi.org/simple/
- name: Download nltk data
run: |
uv run python3 -c "
import os
import nltk
local_dir = os.path.abspath('nltk_data')
for data in ['wordnet', 'punkt', 'punkt_tab']:
print(f'Downloading nltk {data}...')
nltk.download(data, download_dir=local_dir)
"
- name: Install LLVM 20
run: |
wget https://apt.llvm.org/llvm.sh
chmod +x llvm.sh
sudo ./llvm.sh 20
echo "/usr/lib/llvm-20/bin" >> $GITHUB_PATH
- name: Wait for minio and run unit test
env:
NLTK_DATA: ${{ github.workspace }}/nltk_data
LLVM_PROFILE_FILE: "infinity-%p.profraw"
run: |
./scripts/wait_for_minio.sh
ASAN_OPTIONS=detect_leaks=0 ./test_main --gtest_filter=-*SLOW_* > debug_unit_test_stdout.log 2> debug_unit_test_stderr.log
- name: Collect infinity log
if: always()
run: |
echo "debug unit test log" && tail -n 1000 debug_unit_test*.log || true
- name: Generate LCOV report
if: always()
run: |
llvm-profdata merge -sparse infinity-*.profraw -o coverage.profdata
llvm-cov export ./test_main -instr-profile=coverage.profdata -format=lcov -ignore-filename-regex='.*unit_test.*|.*third_party.*|.*cmake-build-debug.*|.*cppm.*' > coverage.lcov
head -10 coverage.lcov
echo "LCOV file lines: $(wc -l < coverage.lcov)"
- name: Upload coverage artifact
if: always()
uses: actions/upload-artifact@v7
with:
name: coverage-debug-unit-test-amd64
path: coverage.lcov
retention-days: 1
if-no-files-found: warn
- name: Remove minio container
if: always()
run: docker rm -f minio || true
debug_amd64_sqllogical_test:
name: debug_amd64_sqllogical_test
runs-on: ubuntu-latest
needs: debug_amd64_build
steps:
- name: Free disk space
uses: jlumbroso/free-disk-space@main
with:
tool-cache: false
android: true
dotnet: true
haskell: true
large-packages: true
docker-images: true
swap-storage: true
- name: Check out code
uses: actions/checkout@v6
with:
ref: ${{ (github.event_name == 'pull_request' || github.event_name == 'pull_request_target') && format('refs/pull/{0}/merge', github.event.pull_request.number) || github.sha }}
fetch-depth: 0
submodules: recursive
- name: Download debug build artifacts
uses: actions/download-artifact@v8
with:
name: debug-build-amd64
- name: Prepare environment and start infinity
run: |
chmod +x infinity
sudo mkdir -p /usr/share/infinity/resource
sudo cp -r resource/* /usr/share/infinity/resource/
sudo chown -R ${USER} /usr/share/infinity
sudo mkdir -p /var/infinity
sudo chown -R ${USER} /var/infinity
export LLVM_PROFILE_FILE="infinity-%p.profraw"
nohup ./infinity --config=conf/pytest_parallel_infinity_conf.toml > infinity.log 2>&1 &
- name: Install sqllogictest
run: |
curl -sLO https://github.com/risinglightdb/sqllogictest-rs/releases/download/v0.28.4/sqllogictest-bin-v0.28.4-x86_64-unknown-linux-musl.tar.gz
tar -xzf sqllogictest-bin-v0.28.4-x86_64-unknown-linux-musl.tar.gz
sudo mv sqllogictest /usr/local/bin/
- name: Install Python SDK
uses: astral-sh/setup-uv@v7
# with:
# enable-cache: true
- run: uv sync --python 3.11 --extra test --index-url https://pypi.org/simple/
- name: Install LLVM 20
run: |
wget https://apt.llvm.org/llvm.sh
chmod +x llvm.sh
sudo ./llvm.sh 20
echo "/usr/lib/llvm-20/bin" >> $GITHUB_PATH
- name: Wait for infinity and run sqllogictest
run: |
./scripts/wait_for_infinity.sh
uv run python3 tools/sqllogictest.py
- name: Collect infinity log
if: always()
run: |
cat infinity.log || true
- name: Stop infinity server
if: always()
run: pkill -f infinity || true
- name: Generate LCOV report
if: always()
run: |
llvm-profdata merge -sparse infinity-*.profraw -o coverage.profdata
llvm-cov export ./infinity -instr-profile=coverage.profdata -format=lcov -ignore-filename-regex='.*unit_test.*|.*third_party.*|.*cmake-build-debug.*|.*cppm.*' > coverage.lcov
head -10 coverage.lcov
echo "LCOV file lines: $(wc -l < coverage.lcov)"
- name: Upload coverage artifact
if: always()
uses: actions/upload-artifact@v7
with:
name: coverage-debug-sqllogical-test-amd64
path: coverage.lcov
retention-days: 1
if-no-files-found: warn
debug_amd64_restart_test:
name: debug_amd64_restart_test
runs-on: ubuntu-latest
needs: debug_amd64_build
steps:
- name: Free disk space
uses: jlumbroso/free-disk-space@main
with:
tool-cache: false
android: true
dotnet: true
haskell: true
large-packages: true
docker-images: true
swap-storage: true
- name: Check out code
uses: actions/checkout@v6
with:
ref: ${{ (github.event_name == 'pull_request' || github.event_name == 'pull_request_target') && format('refs/pull/{0}/merge', github.event.pull_request.number) || github.sha }}
fetch-depth: 0
submodules: recursive
- name: Download debug build artifacts
uses: actions/download-artifact@v8
with:
name: debug-build-amd64
- name: Prepare environment
run: |
chmod +x infinity
sudo mkdir -p /usr/share/infinity/resource
sudo cp -r resource/* /usr/share/infinity/resource/
sudo chown -R ${USER} /usr/share/infinity
sudo mkdir -p /var/infinity
sudo chown -R ${USER} /var/infinity
- name: Install Python SDK
uses: astral-sh/setup-uv@v7
# with:
# enable-cache: true
- run: uv sync --python 3.11 --extra test --index-url https://pypi.org/simple/
- name: Run restart test
run: |
uv run python3 tools/run_restart_test.py --infinity_path=./infinity
- name: Collect infinity log
if: always()
run: |
echo "debug_restart_test log(last 1000 lines):" && tail -n 1000 restart_test*.log* || true
debug_amd64_pysdk_test:
name: debug_amd64_pysdk_test
runs-on: ubuntu-latest
needs: debug_amd64_build
steps:
- name: Free disk space
uses: jlumbroso/free-disk-space@main
with:
tool-cache: false
android: true
dotnet: true
haskell: true
large-packages: true
docker-images: true
swap-storage: true
- name: Check out code
uses: actions/checkout@v6
with:
ref: ${{ (github.event_name == 'pull_request' || github.event_name == 'pull_request_target') && format('refs/pull/{0}/merge', github.event.pull_request.number) || github.sha }}
fetch-depth: 0
submodules: recursive
- name: Download debug build artifacts
uses: actions/download-artifact@v8
with:
name: debug-build-amd64
- name: Prepare environment and start infinity
run: |
chmod +x infinity
sudo mkdir -p /usr/share/infinity/resource
sudo cp -r resource/* /usr/share/infinity/resource/
sudo chown -R ${USER} /usr/share/infinity
sudo mkdir -p /var/infinity
sudo chown -R ${USER} /var/infinity
export LLVM_PROFILE_FILE="infinity-%p.profraw"
nohup ./infinity --config=conf/pytest_parallel_infinity_conf.toml > infinity.log 2>&1 &
- name: Install Python SDK
uses: astral-sh/setup-uv@v7
# with:
# enable-cache: true
- run: uv sync --python 3.11 --extra test --index-url https://pypi.org/simple/
- name: Install LLVM 20
run: |
wget https://apt.llvm.org/llvm.sh
chmod +x llvm.sh
sudo ./llvm.sh 20
echo "/usr/lib/llvm-20/bin" >> $GITHUB_PATH
- name: Wait for infinity and run pysdk test
run: |
./scripts/wait_for_infinity.sh
uv run python3 tools/run_pysdk_remote_infinity_test.py
- name: Collect infinity log
if: always()
run: |
cat infinity.log || true
- name: Stop infinity server
if: always()
run: pkill -f infinity || true
- name: Generate LCOV report
if: always()
run: |
llvm-profdata merge -sparse infinity-*.profraw -o coverage.profdata
llvm-cov export ./infinity -instr-profile=coverage.profdata -format=lcov -ignore-filename-regex='.*unit_test.*|.*third_party.*|.*cmake-build-debug.*|.*cppm.*' > coverage.lcov
head -10 coverage.lcov
echo "LCOV file lines: $(wc -l < coverage.lcov)"
- name: Upload coverage artifact
if: always()
uses: actions/upload-artifact@v7
with:
name: coverage-debug-pysdk-test-amd64
path: coverage.lcov
retention-days: 1
if-no-files-found: warn
debug_amd64_go_test:
name: debug_amd64_go_test
runs-on: ubuntu-latest
needs: debug_amd64_build
steps:
- name: Free disk space
uses: jlumbroso/free-disk-space@main
with:
tool-cache: false
android: true
dotnet: true
haskell: true
large-packages: true
docker-images: true
swap-storage: true
- name: Check out code
uses: actions/checkout@v6
with:
ref: ${{ (github.event_name == 'pull_request' || github.event_name == 'pull_request_target') && format('refs/pull/{0}/merge', github.event.pull_request.number) || github.sha }}
fetch-depth: 0
submodules: recursive
- name: Download debug build artifacts
uses: actions/download-artifact@v8
with:
name: debug-build-amd64
- name: Prepare environment and start infinity
run: |
chmod +x infinity
sudo mkdir -p /usr/share/infinity/resource
sudo cp -r resource/* /usr/share/infinity/resource/
sudo chown -R ${USER} /usr/share/infinity
sudo mkdir -p /var/infinity
sudo chown -R ${USER} /var/infinity
export LLVM_PROFILE_FILE="infinity-%p.profraw"
nohup ./infinity --config=conf/pytest_parallel_infinity_conf.toml > infinity.log 2>&1 &
- name: Setup Go
uses: actions/setup-go@v6
with:
go-version: '1.23'
- name: Install sqllogictest
run: |
curl -sLO https://github.com/risinglightdb/sqllogictest-rs/releases/download/v0.28.4/sqllogictest-bin-v0.28.4-x86_64-unknown-linux-musl.tar.gz
tar -xzf sqllogictest-bin-v0.28.4-x86_64-unknown-linux-musl.tar.gz
sudo mv sqllogictest /usr/local/bin/
- name: Install Python SDK
uses: astral-sh/setup-uv@v7
# with:
# enable-cache: true
- run: uv sync --python 3.11 --extra test --index-url https://pypi.org/simple/
- name: Install LLVM 20
run: |
wget https://apt.llvm.org/llvm.sh
chmod +x llvm.sh
sudo ./llvm.sh 20
echo "/usr/lib/llvm-20/bin" >> $GITHUB_PATH
- name: Wait for infinity and run Go tests
run: |
./scripts/wait_for_infinity.sh
uv run python3 tools/sqllogictest.py -jc=1
cd go && go version && go test ./tests/... -v 2>&1 | tee ../debug_go_test.log; exit_code=${PIPESTATUS[0]}; if [ $exit_code -ne 0 ]; then echo "=== FAILED TESTS ==="; grep -E "^(FAIL|panic:|^--- FAIL)" ../debug_go_test.log || true; exit $exit_code; fi
- name: Collect infinity log
if: always()
run: |
cat infinity.log || true
cat debug_go_test.log || true
- name: Stop infinity server
if: always()
run: pkill -f infinity || true
- name: Generate LCOV report
if: always()
run: |
llvm-profdata merge -sparse infinity-*.profraw -o coverage.profdata
llvm-cov export ./infinity -instr-profile=coverage.profdata -format=lcov -ignore-filename-regex='.*unit_test.*|.*third_party.*|.*cmake-build-debug.*|.*cppm.*' > coverage.lcov
head -10 coverage.lcov
echo "LCOV file lines: $(wc -l < coverage.lcov)"
- name: Upload coverage artifact
if: always()
uses: actions/upload-artifact@v7
with:
name: coverage-debug-go-test-amd64
path: coverage.lcov
retention-days: 1
if-no-files-found: warn
debug_amd64_http_test:
name: debug_amd64_http_test
runs-on: ubuntu-latest
needs: debug_amd64_build
steps:
- name: Free disk space
uses: jlumbroso/free-disk-space@main
with:
tool-cache: false
android: true
dotnet: true
haskell: true
large-packages: true
docker-images: true
swap-storage: true
- name: Check out code
uses: actions/checkout@v6
with:
ref: ${{ (github.event_name == 'pull_request' || github.event_name == 'pull_request_target') && format('refs/pull/{0}/merge', github.event.pull_request.number) || github.sha }}
fetch-depth: 0
submodules: recursive
- name: Download debug build artifacts
uses: actions/download-artifact@v8
with:
name: debug-build-amd64
- name: Prepare environment and start infinity
run: |
chmod +x infinity
sudo mkdir -p /usr/share/infinity/resource
sudo cp -r resource/* /usr/share/infinity/resource/
sudo chown -R ${USER} /usr/share/infinity
sudo mkdir -p /var/infinity
sudo chown -R ${USER} /var/infinity
export LLVM_PROFILE_FILE="infinity-%p.profraw"
nohup ./infinity --config=conf/pytest_parallel_infinity_conf.toml > infinity.log 2>&1 &
- name: Install Python SDK
uses: astral-sh/setup-uv@v7
# with:
# enable-cache: true
- run: uv sync --python 3.11 --extra test --index-url https://pypi.org/simple/
- name: Install LLVM 20
run: |
wget https://apt.llvm.org/llvm.sh
chmod +x llvm.sh
sudo ./llvm.sh 20
echo "/usr/lib/llvm-20/bin" >> $GITHUB_PATH
- name: Wait for infinity and run http test
run: |
./scripts/wait_for_infinity.sh
uv run python3 tools/run_http_api.py
- name: Collect infinity log
if: always()
run: |
cat infinity.log || true
- name: Stop infinity server
if: always()
run: pkill -f infinity || true
- name: Generate LCOV report
if: always()
run: |
llvm-profdata merge -sparse infinity-*.profraw -o coverage.profdata
llvm-cov export ./infinity -instr-profile=coverage.profdata -format=lcov -ignore-filename-regex='.*unit_test.*|.*third_party.*|.*cmake-build-debug.*|.*cppm.*' > coverage.lcov
head -10 coverage.lcov
echo "LCOV file lines: $(wc -l < coverage.lcov)"
- name: Upload coverage artifact
if: always()
uses: actions/upload-artifact@v7
with:
name: coverage-debug-http-test-amd64
path: coverage.lcov
retention-days: 1
if-no-files-found: warn
debug_amd64_pysdk_parallel_test:
name: debug_amd64_pysdk_parallel_test
runs-on: ubuntu-latest
needs: debug_amd64_build
steps:
- name: Free disk space
uses: jlumbroso/free-disk-space@main
with:
tool-cache: false
android: true
dotnet: true
haskell: true
large-packages: true
docker-images: true
swap-storage: true
- name: Check out code
uses: actions/checkout@v6
with:
ref: ${{ (github.event_name == 'pull_request' || github.event_name == 'pull_request_target') && format('refs/pull/{0}/merge', github.event.pull_request.number) || github.sha }}
fetch-depth: 0
submodules: recursive
- name: Download debug build artifacts
uses: actions/download-artifact@v8
with:
name: debug-build-amd64
- name: Prepare environment and start infinity
run: |
chmod +x infinity
sudo mkdir -p /usr/share/infinity/resource
sudo cp -r resource/* /usr/share/infinity/resource/
sudo chown -R ${USER} /usr/share/infinity
sudo mkdir -p /var/infinity
sudo chown -R ${USER} /var/infinity
export LLVM_PROFILE_FILE="infinity-%p.profraw"
nohup ./infinity --config=conf/pytest_parallel_infinity_conf.toml > infinity.log 2>&1 &
- name: Install Python SDK
uses: astral-sh/setup-uv@v7
# with:
# enable-cache: true
- run: uv sync --python 3.11 --extra test --index-url https://pypi.org/simple/
- name: Install LLVM 20
run: |
wget https://apt.llvm.org/llvm.sh
chmod +x llvm.sh
sudo ./llvm.sh 20
echo "/usr/lib/llvm-20/bin" >> $GITHUB_PATH
- name: Wait for infinity and run pysdk test
run: |
./scripts/wait_for_infinity.sh
uv run python3 tools/run_parallel_test.py
- name: Collect infinity log
if: always()
run: |
cat infinity.log || true
- name: Stop infinity server
if: always()
run: pkill -f infinity || true
- name: Generate LCOV report
if: always()
run: |
llvm-profdata merge -sparse infinity-*.profraw -o coverage.profdata
llvm-cov export ./infinity -instr-profile=coverage.profdata -format=lcov -ignore-filename-regex='.*unit_test.*|.*third_party.*|.*cmake-build-debug.*|.*cppm.*' > coverage.lcov
head -10 coverage.lcov
echo "LCOV file lines: $(wc -l < coverage.lcov)"
- name: Upload coverage artifact
if: always()
uses: actions/upload-artifact@v7
with:
name: coverage-debug-pysdk-parallel-test-amd64
path: coverage.lcov
retention-days: 1
if-no-files-found: warn
debug_amd64_upload_codecov:
name: debug_amd64_upload_codecov
runs-on: ubuntu-latest
needs: [ debug_amd64_unit_test, debug_amd64_sqllogical_test, debug_amd64_pysdk_test, debug_amd64_go_test, debug_amd64_http_test, debug_amd64_pysdk_parallel_test ]
steps:
- name: Free disk space
uses: jlumbroso/free-disk-space@main
with:
tool-cache: false
android: true
dotnet: true
haskell: true
large-packages: true
docker-images: true
swap-storage: true
- name: Check out code
uses: actions/checkout@v6
with:
ref: ${{ (github.event_name == 'pull_request' || github.event_name == 'pull_request_target') && format('refs/pull/{0}/merge', github.event.pull_request.number) || github.sha }}
fetch-depth: 0
submodules: recursive
- name: Download unit test coverage
if: needs.debug_amd64_unit_test.result == 'success'
uses: actions/download-artifact@v8
with:
name: coverage-debug-unit-test-amd64
path: coverage/unit_test
- name: Download sqllogical test coverage
if: needs.debug_amd64_sqllogical_test.result == 'success'
uses: actions/download-artifact@v8
with:
name: coverage-debug-sqllogical-test-amd64
path: coverage/sqllogical_test
- name: Download pysdk test coverage
if: needs.debug_amd64_pysdk_test.result == 'success'
uses: actions/download-artifact@v8
with:
name: coverage-debug-pysdk-test-amd64
path: coverage/pysdk_test
- name: Download go test coverage
if: needs.debug_amd64_go_test.result == 'success'
uses: actions/download-artifact@v8
with:
name: coverage-debug-go-test-amd64
path: coverage/go_test
- name: Download http test coverage
if: needs.debug_amd64_http_test.result == 'success'
uses: actions/download-artifact@v8
with:
name: coverage-debug-http-test-amd64
path: coverage/http_test
- name: Download pysdk parallel test coverage
if: needs.debug_amd64_pysdk_parallel_test.result == 'success'
uses: actions/download-artifact@v8
with:
name: coverage-debug-pysdk-parallel-test-amd64
path: coverage/pysdk_parallel_test
- name: Merge coverage reports
run: |
mkdir -p merged_coverage
if [ -f coverage/unit_test/coverage.lcov ]; then
cat coverage/unit_test/coverage.lcov > merged_coverage/coverage.lcov
fi
for dir in coverage/sqllogical_test coverage/pysdk_test coverage/go_test coverage/http_test coverage/pysdk_parallel_test; do
if [ -f "$dir/coverage.lcov" ]; then
if [ -f merged_coverage/coverage.lcov ]; then
echo "" >> merged_coverage/coverage.lcov
grep -v "^TN:" "$dir/coverage.lcov" >> merged_coverage/coverage.lcov || true
else
cat "$dir/coverage.lcov" > merged_coverage/coverage.lcov
fi
fi
done
if [ -f merged_coverage/coverage.lcov ]; then
echo "Merged coverage file lines: $(wc -l < merged_coverage/coverage.lcov)"
else
echo "No coverage files found"
exit 1
fi
- name: Upload coverage to Codecov
run: |
curl -Os https://uploader.codecov.io/latest/linux/codecov
chmod +x codecov
./codecov -t ${{ secrets.CODECOV_TOKEN }} \
-f merged_coverage/coverage.lcov \
-F "debug_amd64" \
-C ${{ (github.event_name == 'pull_request' || github.event_name == 'pull_request_target') && github.event.pull_request.head.sha || github.sha }} \
-B ${{ github.ref_name }} \
-n "debug_amd64_coverage"
release_amd64_build:
name: release_amd64_build
runs-on: ubuntu-latest
needs: code_static_check
steps:
- name: Free disk space
run: |
sudo rm -rf /usr/share/dotnet /usr/local/lib/android/sdk /opt/ghc /opt/hostedtoolcache/CodeQL
df -h /
- name: Check out code
uses: actions/checkout@v6
with:
ref: ${{ (github.event_name == 'pull_request' || github.event_name == 'pull_request_target') && format('refs/pull/{0}/merge', github.event.pull_request.number) || github.sha }}
fetch-depth: 0
fetch-tags: false
submodules: recursive
- name: Start builder container
run: |
docker run --privileged -d --name infinity_builder -v ${PWD}:/infinity -v ${PWD}/resource:/usr/share/infinity/resource infiniflow/infinity_builder:ubuntu22_clang20
- name: Build release version
run: docker exec infinity_builder bash -c "git config --global safe.directory \"*\" && cd /infinity && rm -rf cmake-build-reldeb && mkdir -p cmake-build-reldeb && cmake -G Ninja -DCMAKE_BUILD_TYPE=RelWithDebInfo -S /infinity -B /infinity/cmake-build-reldeb && cmake --build /infinity/cmake-build-reldeb -j --target infinity test_main"
- name: Upload artifacts
uses: actions/upload-artifact@v7
with:
name: release-build-amd64
path: |
cmake-build-reldeb/src/infinity
cmake-build-reldeb/src/test_main
retention-days: 1
if-no-files-found: warn
- name: Remove build container
if: always()
run: docker rm -f infinity_builder || true
release_amd64_unit_test:
name: release_amd64_unit_test
runs-on: ubuntu-latest
needs: release_amd64_build
steps:
- name: Free disk space
uses: jlumbroso/free-disk-space@main
with:
tool-cache: false
android: true
dotnet: true
haskell: true
large-packages: true
docker-images: true
swap-storage: true
- name: Check out code
uses: actions/checkout@v6
with:
ref: ${{ (github.event_name == 'pull_request' || github.event_name == 'pull_request_target') && format('refs/pull/{0}/merge', github.event.pull_request.number) || github.sha }}
fetch-depth: 0
submodules: recursive
- name: Start minio container
run: |
docker run -d --name minio --network host -e "MINIO_ROOT_PASSWORD=minioadmin" -e "MINIO_ROOT_USER=minioadmin" -v /tmp/minio-data:/data quay.io/minio/minio server /data --console-address ":9006" --address ":9005"
- name: Download release build artifacts
uses: actions/download-artifact@v8
with:
name: release-build-amd64
- name: Install Python SDK
uses: astral-sh/setup-uv@v7
# with:
# enable-cache: true
- run: uv sync --python 3.11 --extra test --index-url https://pypi.org/simple/
- name: Download nltk data
run: |
uv run python -c "
import os
import nltk
local_dir = os.path.abspath('nltk_data')
for data in ['wordnet', 'punkt', 'punkt_tab']:
print(f'Downloading nltk {data}...')
nltk.download(data, download_dir=local_dir)
"
- name: Prepare environment
run: |
chmod +x ./test_main
sudo mkdir -p /usr/share/infinity/resource
sudo cp -r resource/* /usr/share/infinity/resource/
sudo chown -R ${USER} /usr/share/infinity
sudo mkdir -p /var/infinity
sudo chown -R ${USER} /var/infinity
- name: Wait for minio and run unit test
env:
NLTK_DATA: ${{ github.workspace }}/nltk_data
run: |
./scripts/wait_for_minio.sh
./test_main --gtest_filter=-*SLOW_* > release_amd64_unit_test_stdout.log 2> release_amd64_unit_test_stderr.log
- name: Collect infinity log
if: always()
run: |
echo "release amd64 unit test log" && tail -n 1000 release_amd64_unit_test*.log || true
- name: Remove minio container
if: always()
run: docker rm -f minio || true
release_amd64_sqllogical_test:
name: release_amd64_sqllogical_test
runs-on: ubuntu-latest
needs: release_amd64_build
steps:
- name: Free disk space
uses: jlumbroso/free-disk-space@main
with:
tool-cache: false
android: true
dotnet: true
haskell: true
large-packages: true
docker-images: true
swap-storage: true
- name: Check out code
uses: actions/checkout@v6
with:
ref: ${{ (github.event_name == 'pull_request' || github.event_name == 'pull_request_target') && format('refs/pull/{0}/merge', github.event.pull_request.number) || github.sha }}
fetch-depth: 0
submodules: recursive
- name: Download release build artifacts
uses: actions/download-artifact@v8
with:
name: release-build-amd64
- name: Prepare environment and start infinity
run: |
chmod +x ./infinity
sudo mkdir -p /usr/share/infinity/resource
sudo cp -r resource/* /usr/share/infinity/resource/
sudo chown -R ${USER} /usr/share/infinity
sudo mkdir -p /var/infinity
sudo chown -R ${USER} /var/infinity
nohup ./infinity --config=conf/pytest_parallel_infinity_conf.toml > infinity.log 2>&1 &
- name: Install sqllogictest
run: |
curl -sLO https://github.com/risinglightdb/sqllogictest-rs/releases/download/v0.28.4/sqllogictest-bin-v0.28.4-x86_64-unknown-linux-musl.tar.gz
tar -xzf sqllogictest-bin-v0.28.4-x86_64-unknown-linux-musl.tar.gz
sudo mv sqllogictest /usr/local/bin/
- name: Install Python SDK
uses: astral-sh/setup-uv@v7
# with:
# enable-cache: true
- run: uv sync --python 3.11 --extra test --index-url https://pypi.org/simple/
- name: Wait for infinity and run sqllogictest
run: |
./scripts/wait_for_infinity.sh
uv run python3 tools/sqllogictest.py
- name: Collect infinity log
if: always()
run: |
cat infinity.log || true
- name: Stop infinity server
if: always()
run: pkill -f infinity || true
release_amd64_restart_test:
name: release_amd64_restart_test
runs-on: ubuntu-latest
needs: release_amd64_build
steps:
- name: Free disk space
uses: jlumbroso/free-disk-space@main
with:
tool-cache: false
android: true
dotnet: true
haskell: true
large-packages: true
docker-images: true
swap-storage: true
- name: Check out code
uses: actions/checkout@v6
with:
ref: ${{ (github.event_name == 'pull_request' || github.event_name == 'pull_request_target') && format('refs/pull/{0}/merge', github.event.pull_request.number) || github.sha }}
fetch-depth: 0
submodules: recursive
- name: Download release build artifacts
uses: actions/download-artifact@v8
with:
name: release-build-amd64
- name: Prepare environment
run: |
chmod +x ./infinity
sudo mkdir -p /usr/share/infinity/resource
sudo cp -r resource/* /usr/share/infinity/resource/
sudo chown -R ${USER} /usr/share/infinity
sudo mkdir -p /var/infinity
sudo chown -R ${USER} /var/infinity
- name: Install Python SDK
uses: astral-sh/setup-uv@v7
# with:
# enable-cache: true
- run: uv sync --python 3.11 --extra test --index-url https://pypi.org/simple/
- name: Run restart test
run: |
uv run python3 tools/run_restart_test.py --infinity_path=./infinity
- name: Collect infinity log
if: always()
run: |
echo "release_amd64_restart_test log(last 1000 lines):" && tail -n 1000 restart_test*.log* || true
# release_amd64_pysdk_test:
# name: release_amd64_pysdk_test
# runs-on: ubuntu-latest
# needs: release_amd64_build
# steps:
# - name: Check out code
# uses: actions/checkout@v6
# with:
# ref: ${{ (github.event_name == 'pull_request' || github.event_name == 'pull_request_target') && format('refs/pull/{0}/merge', github.event.pull_request.number) || github.sha }}
# fetch-depth: 0
# submodules: recursive
#
# - name: Download release build artifacts
# uses: actions/download-artifact@v8
# with:
# name: release-build-amd64
#
# - name: Prepare environment
# run: |
# chmod +x ./infinity
# sudo mkdir -p /usr/share/infinity/resource
# sudo cp -r resource/* /usr/share/infinity/resource/
# sudo chown -R ${USER} /usr/share/infinity
# sudo mkdir -p /var/infinity
# sudo chown -R ${USER} /var/infinity
#
# - name: Install Python SDK
# uses: astral-sh/setup-uv@v7
# with:
# enable-cache: true
# - run: uv sync --python 3.11 --extra test --index-url https://pypi.org/simple/
#
# - name: Start infinity server
# run: |
# nohup ./infinity --config=conf/pytest_parallel_infinity_conf.toml > infinity.log 2>&1 &
# ./scripts/wait_for_infinity.sh
#
# - name: Run pysdk test
# run: uv run python3 tools/run_pysdk_remote_infinity_test.py
#
# - name: Collect infinity log
# if: always()
# run: |
# cat infinity.log || true
#
# - name: Stop infinity server
# if: always()
# run: pkill -f infinity || true
release_amd64_go_test:
name: release_amd64_go_test
runs-on: ubuntu-latest
needs: release_amd64_build
steps:
- name: Free disk space
uses: jlumbroso/free-disk-space@main
with:
tool-cache: false
android: true
dotnet: true
haskell: true
large-packages: true
docker-images: true
swap-storage: true
- name: Check out code
uses: actions/checkout@v6
with:
ref: ${{ (github.event_name == 'pull_request' || github.event_name == 'pull_request_target') && format('refs/pull/{0}/merge', github.event.pull_request.number) || github.sha }}
fetch-depth: 0
submodules: recursive
- name: Download release build artifacts
uses: actions/download-artifact@v8
with:
name: release-build-amd64
- name: Prepare environment and start infinity
run: |
chmod +x ./infinity
sudo mkdir -p /usr/share/infinity/resource
sudo cp -r resource/* /usr/share/infinity/resource/
sudo chown -R ${USER} /usr/share/infinity
sudo mkdir -p /var/infinity
sudo chown -R ${USER} /var/infinity
nohup ./infinity --config=conf/pytest_parallel_infinity_conf.toml > infinity.log 2>&1 &
- name: Setup Go
uses: actions/setup-go@v6
with:
go-version: '1.23'
- name: Install sqllogictest
run: |
curl -sLO https://github.com/risinglightdb/sqllogictest-rs/releases/download/v0.28.4/sqllogictest-bin-v0.28.4-x86_64-unknown-linux-musl.tar.gz
tar -xzf sqllogictest-bin-v0.28.4-x86_64-unknown-linux-musl.tar.gz
sudo mv sqllogictest /usr/local/bin/
- name: Install Python SDK
uses: astral-sh/setup-uv@v7
# with:
# enable-cache: true
- run: uv sync --python 3.11 --extra test --index-url https://pypi.org/simple/
- name: Wait for infinity and run Go tests
run: |
./scripts/wait_for_infinity.sh
uv run python3 tools/sqllogictest.py -jc=1
cd go && go version && go test ./tests/... -v 2>&1 | tee ../release_amd64_go_test.log; exit_code=${PIPESTATUS[0]}; if [ $exit_code -ne 0 ]; then echo "=== FAILED TESTS ==="; grep -E "^(FAIL|panic:|^--- FAIL)" ../release_amd64_go_test.log || true; exit $exit_code; fi
- name: Collect infinity log
if: always()
run: |
cat infinity.log || true
cat release_amd64_go_test.log || true
- name: Stop infinity server
if: always()
run: pkill -f infinity || true
release_amd64_http_test:
name: release_amd64_http_test
runs-on: ubuntu-latest
needs: release_amd64_build
steps:
- name: Free disk space
uses: jlumbroso/free-disk-space@main
with:
tool-cache: false
android: true
dotnet: true
haskell: true
large-packages: true
docker-images: true
swap-storage: true
- name: Check out code
uses: actions/checkout@v6
with:
ref: ${{ (github.event_name == 'pull_request' || github.event_name == 'pull_request_target') && format('refs/pull/{0}/merge', github.event.pull_request.number) || github.sha }}
fetch-depth: 0
submodules: recursive
- name: Download release build artifacts
uses: actions/download-artifact@v8
with:
name: release-build-amd64
- name: Prepare environment and start infinity
run: |
chmod +x ./infinity
sudo mkdir -p /usr/share/infinity/resource
sudo cp -r resource/* /usr/share/infinity/resource/
sudo chown -R ${USER} /usr/share/infinity
sudo mkdir -p /var/infinity
sudo chown -R ${USER} /var/infinity
nohup ./infinity --config=conf/pytest_parallel_infinity_conf.toml > infinity.log 2>&1 &
- name: Install Python SDK
uses: astral-sh/setup-uv@v7
# with:
# enable-cache: true
- run: uv sync --python 3.11 --extra test --index-url https://pypi.org/simple/
- name: Wait for infinity and run http test
run: |
./scripts/wait_for_infinity.sh
uv run python3 tools/run_http_api.py
- name: Collect infinity log
if: always()
run: |
cat infinity.log || true
- name: Stop infinity server
if: always()
run: pkill -f infinity || true
# release_amd64_pysdk_parallel_test:
# name: release_amd64_pysdk_parallel_test
# runs-on: ubuntu-latest
# needs: release_amd64_build
# steps:
# - name: Check out code
# uses: actions/checkout@v6
# with:
# ref: ${{ (github.event_name == 'pull_request' || github.event_name == 'pull_request_target') && format('refs/pull/{0}/merge', github.event.pull_request.number) || github.sha }}
# fetch-depth: 0
# submodules: recursive
#
# - name: Download release build artifacts
# uses: actions/download-artifact@v8
# with:
# name: release-build-amd64
#
# - name: Prepare environment
# run: |
# chmod +x ./infinity
# sudo mkdir -p /usr/share/infinity/resource
# sudo cp -r resource/* /usr/share/infinity/resource/
# sudo chown -R ${USER} /usr/share/infinity
# sudo mkdir -p /var/infinity
# sudo chown -R ${USER} /var/infinity
#
# - name: Install Python SDK
# uses: astral-sh/setup-uv@v7
# with:
# enable-cache: true
# - run: uv sync --python 3.11 --extra test --index-url https://pypi.org/simple/
#
# - name: Start infinity server
# run: |
# nohup ./infinity --config=conf/pytest_parallel_infinity_conf.toml > infinity.log 2>&1 &
# ./scripts/wait_for_infinity.sh
#
# - name: Run pysdk test
# run: uv run python3 tools/run_parallel_test.py
#
# - name: Collect infinity log
# if: always()
# run: |
# cat infinity.log || true
#
# - name: Stop infinity server
# if: always()
# run: pkill -f infinity || true
release_amd64_minio_pysdk_test:
name: release_amd64_minio_pysdk_test
runs-on: ubuntu-latest
needs: release_amd64_build
steps:
- name: Free disk space
run: |
sudo rm -rf /usr/share/dotnet /usr/local/lib/android/sdk /opt/ghc /opt/hostedtoolcache/CodeQL
df -h /
- name: Check out code
uses: actions/checkout@v6
with:
ref: ${{ (github.event_name == 'pull_request' || github.event_name == 'pull_request_target') && format('refs/pull/{0}/merge', github.event.pull_request.number) || github.sha }}
fetch-depth: 0
submodules: recursive
- name: Start minio container
run: |
docker run -d --name minio --network host -e "MINIO_ROOT_PASSWORD=minioadmin" -e "MINIO_ROOT_USER=minioadmin" -v /tmp/minio-data:/data quay.io/minio/minio server /data --console-address ":9006" --address ":9005"
- name: Download release build artifacts
uses: actions/download-artifact@v8
with:
name: release-build-amd64
- name: Prepare environment and start infinity
run: |
chmod +x infinity
sudo mkdir -p /usr/share/infinity/resource
sudo cp -r resource/* /usr/share/infinity/resource/
sudo chown -R ${USER} /usr/share/infinity
sudo mkdir -p /var/infinity
sudo chown -R ${USER} /var/infinity
nohup ./infinity --config=conf/pytest_parallel_infinity_minio.toml > infinity.log 2>&1 &
- name: Install Python SDK
uses: astral-sh/setup-uv@v7
# with:
# enable-cache: true
- run: uv sync --python 3.11 --extra test --index-url https://pypi.org/simple/
- name: Wait for services and run minio pysdk test
run: |
./scripts/wait_for_minio.sh
./scripts/wait_for_infinity.sh
uv run python3 tools/run_pysdk_remote_infinity_test.py
- name: Collect infinity log
if: always()
run: |
cat infinity.log || true
- name: Stop infinity server
if: always()
run: pkill -f infinity || true
- name: Remove minio container
if: always()
run: docker rm -f minio || true
release_amd64_minio_parallel_test:
name: release_amd64_minio_parallel_test
runs-on: ubuntu-latest
needs: release_amd64_build
steps:
- name: Free disk space
run: |
sudo rm -rf /usr/share/dotnet /usr/local/lib/android/sdk /opt/ghc /opt/hostedtoolcache/CodeQL
df -h /
- name: Check out code
uses: actions/checkout@v6
with:
ref: ${{ (github.event_name == 'pull_request' || github.event_name == 'pull_request_target') && format('refs/pull/{0}/merge', github.event.pull_request.number) || github.sha }}
fetch-depth: 0
submodules: recursive
- name: Start minio container
run: |
docker run -d --name minio --network host -e "MINIO_ROOT_PASSWORD=minioadmin" -e "MINIO_ROOT_USER=minioadmin" -v /tmp/minio-data:/data quay.io/minio/minio server /data --console-address ":9006" --address ":9005"
- name: Download release build artifacts
uses: actions/download-artifact@v8
with:
name: release-build-amd64
- name: Prepare environment and start infinity
run: |
chmod +x infinity
sudo mkdir -p /usr/share/infinity/resource
sudo cp -r resource/* /usr/share/infinity/resource/
sudo chown -R ${USER} /usr/share/infinity
sudo mkdir -p /var/infinity
sudo chown -R ${USER} /var/infinity
nohup ./infinity --config=conf/pytest_parallel_infinity_minio.toml > infinity.log 2>&1 &
- name: Install Python SDK
uses: astral-sh/setup-uv@v7
# with:
# enable-cache: true
- run: uv sync --python 3.11 --extra test --index-url https://pypi.org/simple/
- name: Wait for services and run minio parallel test
run: |
./scripts/wait_for_minio.sh
./scripts/wait_for_infinity.sh
uv run python3 tools/run_parallel_test.py
- name: Collect infinity log
if: always()
run: |
cat infinity.log || true
- name: Stop infinity server
if: always()
run: pkill -f infinity || true
- name: Remove minio container
if: always()
run: docker rm -f minio || true
debug_arm64_build:
name: debug_arm64_build
runs-on: ubuntu-24.04-arm
needs: code_static_check
steps:
- name: Free disk space
run: |
sudo rm -rf /usr/share/dotnet /usr/local/lib/android/sdk /opt/ghc /opt/hostedtoolcache/CodeQL
df -h /
- name: Check out code
uses: actions/checkout@v6
with:
ref: ${{ (github.event_name == 'pull_request' || github.event_name == 'pull_request_target') && format('refs/pull/{0}/merge', github.event.pull_request.number) || github.sha }}
fetch-depth: 0
fetch-tags: false
submodules: recursive
- name: Start builder container
run: |
docker run --privileged -d --name infinity_builder -v ${PWD}:/infinity -v ${PWD}/resource:/usr/share/infinity/resource infiniflow/infinity_builder:ubuntu22_clang20
- name: Build debug version
run: docker exec infinity_builder bash -c "git config --global safe.directory \"*\" && cd /infinity && rm -rf cmake-build-debug && mkdir -p cmake-build-debug && cmake -G Ninja -DCMAKE_BUILD_TYPE=Debug -S /infinity -B /infinity/cmake-build-debug && cmake --build /infinity/cmake-build-debug -j --target infinity test_main"
- name: Upload artifacts
uses: actions/upload-artifact@v7
with:
name: debug-build-arm64
path: |
cmake-build-debug/src/infinity
cmake-build-debug/src/test_main
retention-days: 1
if-no-files-found: warn
- name: Remove build container
if: always()
run: docker rm -f infinity_builder || true
debug_arm64_unit_test:
name: debug_arm64_unit_test
runs-on: ubuntu-24.04-arm
needs: debug_arm64_build
steps:
- name: Free disk space
uses: jlumbroso/free-disk-space@main
with:
tool-cache: false
android: true
dotnet: true
haskell: true
large-packages: true
docker-images: true
swap-storage: true
- name: Check out code
uses: actions/checkout@v6
with:
ref: ${{ (github.event_name == 'pull_request' || github.event_name == 'pull_request_target') && format('refs/pull/{0}/merge', github.event.pull_request.number) || github.sha }}
fetch-depth: 0
submodules: recursive
- name: Start minio container
run: |
docker run -d --name minio --network host -e "MINIO_ROOT_PASSWORD=minioadmin" -e "MINIO_ROOT_USER=minioadmin" -v /tmp/minio-data:/data quay.io/minio/minio server /data --console-address ":9006" --address ":9005"
- name: Download debug build artifacts
uses: actions/download-artifact@v8
with:
name: debug-build-arm64
- name: Install Python SDK
uses: astral-sh/setup-uv@v7
# with:
# enable-cache: true
- run: uv sync --python 3.11 --extra test --index-url https://pypi.org/simple/
- name: Download nltk data
run: |
uv run python -c "
import os
import nltk
local_dir = os.path.abspath('nltk_data')
for data in ['wordnet', 'punkt', 'punkt_tab']:
print(f'Downloading nltk {data}...')
nltk.download(data, download_dir=local_dir)
"
- name: Prepare environment
run: |
chmod +x test_main
sudo mkdir -p /usr/share/infinity/resource
sudo cp -r resource/* /usr/share/infinity/resource/
sudo chown -R ${USER} /usr/share/infinity
sudo mkdir -p /var/infinity
sudo chown -R ${USER} /var/infinity
- name: Wait for minio and run unit test
env:
NLTK_DATA: ${{ github.workspace }}/nltk_data
run: |
./scripts/wait_for_minio.sh
ASAN_OPTIONS=detect_leaks=0 ./test_main --gtest_filter=-*SLOW_* > debug_arm64_unit_test_stdout.log 2> debug_arm64_unit_test_stderr.log
- name: Collect infinity log
if: always()
run: |
echo "debug arm64 unit test log" && tail -n 1000 debug_arm64_unit_test*.log || true
- name: Remove minio container
if: always()
run: docker rm -f minio || true
debug_arm64_sqllogical_test:
name: debug_arm64_sqllogical_test
runs-on: ubuntu-24.04-arm
needs: debug_arm64_build
# if: false
steps:
- name: Check out code
uses: actions/checkout@v6
with:
ref: ${{ (github.event_name == 'pull_request' || github.event_name == 'pull_request_target') && format('refs/pull/{0}/merge', github.event.pull_request.number) || github.sha }}
fetch-depth: 0
submodules: recursive
- name: Download debug build artifacts
uses: actions/download-artifact@v8
with:
name: debug-build-arm64
- name: Prepare environment and start infinity
run: |
chmod +x infinity
sudo mkdir -p /usr/share/infinity/resource
sudo cp -r resource/* /usr/share/infinity/resource/
sudo chown -R ${USER} /usr/share/infinity
sudo mkdir -p /var/infinity
sudo chown -R ${USER} /var/infinity
nohup ./infinity --config=conf/pytest_parallel_infinity_conf.toml > infinity.log 2>&1 &
- name: Install sqllogictest
run: |
curl -sLO https://github.com/risinglightdb/sqllogictest-rs/releases/download/v0.28.4/sqllogictest-bin-v0.28.4-aarch64-unknown-linux-musl.tar.gz
tar -xzf sqllogictest-bin-v0.28.4-aarch64-unknown-linux-musl.tar.gz
sudo mv sqllogictest /usr/local/bin/
- name: Install Python SDK
uses: astral-sh/setup-uv@v7
# with:
# enable-cache: true
- run: uv sync --python 3.11 --extra test --index-url https://pypi.org/simple/
- name: Wait for infinity and run sqllogictest
run: |
./scripts/wait_for_infinity.sh
uv run python3 tools/sqllogictest.py
- name: Collect infinity log
if: always()
run: |
cat infinity.log || true
- name: Stop infinity server
if: always()
run: pkill -f infinity || true
debug_arm64_restart_test:
name: debug_arm64_restart_test
runs-on: ubuntu-24.04-arm
needs: debug_arm64_build
steps:
- name: Free disk space
uses: jlumbroso/free-disk-space@main
with:
tool-cache: false
android: true
dotnet: true
haskell: true
large-packages: true
docker-images: true
swap-storage: true
- name: Check out code
uses: actions/checkout@v6
with:
ref: ${{ (github.event_name == 'pull_request' || github.event_name == 'pull_request_target') && format('refs/pull/{0}/merge', github.event.pull_request.number) || github.sha }}
fetch-depth: 0
submodules: recursive
- name: Download debug build artifacts
uses: actions/download-artifact@v8
with:
name: debug-build-arm64
- name: Prepare environment
run: |
chmod +x infinity
sudo mkdir -p /usr/share/infinity/resource
sudo cp -r resource/* /usr/share/infinity/resource/
sudo chown -R ${USER} /usr/share/infinity
sudo mkdir -p /var/infinity
sudo chown -R ${USER} /var/infinity
- name: Install Python SDK
uses: astral-sh/setup-uv@v7
# with:
# enable-cache: true
- run: uv sync --python 3.11 --extra test --index-url https://pypi.org/simple/
- name: Run restart test
run: |
uv run python3 tools/run_restart_test.py --infinity_path=./infinity
- name: Collect infinity log
if: always()
run: |
echo "debug_arm64_restart_test log(last 1000 lines):" && tail -n 1000 restart_test*.log* || true
debug_arm64_pysdk_test:
name: debug_arm64_pysdk_test
runs-on: ubuntu-24.04-arm
needs: debug_arm64_build
steps:
- name: Free disk space
uses: jlumbroso/free-disk-space@main
with:
tool-cache: false
android: true
dotnet: true
haskell: true
large-packages: true
docker-images: true
swap-storage: true
- name: Check out code
uses: actions/checkout@v6
with:
ref: ${{ (github.event_name == 'pull_request' || github.event_name == 'pull_request_target') && format('refs/pull/{0}/merge', github.event.pull_request.number) || github.sha }}
fetch-depth: 0
submodules: recursive
- name: Download debug build artifacts
uses: actions/download-artifact@v8
with:
name: debug-build-arm64
- name: Prepare environment and start infinity
run: |
chmod +x infinity
sudo mkdir -p /usr/share/infinity/resource
sudo cp -r resource/* /usr/share/infinity/resource/
sudo chown -R ${USER} /usr/share/infinity
sudo mkdir -p /var/infinity
sudo chown -R ${USER} /var/infinity
nohup ./infinity --config=conf/pytest_parallel_infinity_conf.toml > infinity.log 2>&1 &
- name: Install Python SDK
uses: astral-sh/setup-uv@v7
# with:
# enable-cache: true
- run: uv sync --python 3.11 --extra test --index-url https://pypi.org/simple/
- name: Wait for infinity and run pysdk test
run: |
./scripts/wait_for_infinity.sh
uv run python3 tools/run_pysdk_remote_infinity_test.py
- name: Collect infinity log
if: always()
run: |
cat infinity.log || true
- name: Stop infinity server
if: always()
run: pkill -f infinity || true
debug_arm64_go_test:
name: debug_arm64_go_test
runs-on: ubuntu-24.04-arm
needs: debug_arm64_build
steps:
- name: Free disk space
uses: jlumbroso/free-disk-space@main
with:
tool-cache: false
android: true
dotnet: true
haskell: true
large-packages: true
docker-images: true
swap-storage: true
- name: Check out code
uses: actions/checkout@v6
with:
ref: ${{ (github.event_name == 'pull_request' || github.event_name == 'pull_request_target') && format('refs/pull/{0}/merge', github.event.pull_request.number) || github.sha }}
fetch-depth: 0
submodules: recursive
- name: Download debug build artifacts
uses: actions/download-artifact@v8
with:
name: debug-build-arm64
- name: Prepare environment and start infinity
run: |
chmod +x infinity
sudo mkdir -p /usr/share/infinity/resource
sudo cp -r resource/* /usr/share/infinity/resource/
sudo chown -R ${USER} /usr/share/infinity
sudo mkdir -p /var/infinity
sudo chown -R ${USER} /var/infinity
nohup ./infinity --config=conf/pytest_parallel_infinity_conf.toml > infinity.log 2>&1 &
- name: Setup Go
uses: actions/setup-go@v6
with:
go-version: '1.23'
- name: Install sqllogictest
run: |
curl -sLO https://github.com/risinglightdb/sqllogictest-rs/releases/download/v0.28.4/sqllogictest-bin-v0.28.4-aarch64-unknown-linux-musl.tar.gz
tar -xzf sqllogictest-bin-v0.28.4-aarch64-unknown-linux-musl.tar.gz
sudo mv sqllogictest /usr/local/bin/
- name: Install Python SDK
uses: astral-sh/setup-uv@v7
# with:
# enable-cache: true
- run: uv sync --python 3.11 --extra test --index-url https://pypi.org/simple/
- name: Wait for infinity and run Go tests
run: |
./scripts/wait_for_infinity.sh
uv run python3 tools/sqllogictest.py -jc=1
cd go && go version && go test ./tests/... -v 2>&1 | tee ../debug_arm64_go_test.log; exit_code=${PIPESTATUS[0]}; if [ $exit_code -ne 0 ]; then echo "=== FAILED TESTS ==="; grep -E "^(FAIL|panic:|^--- FAIL)" ../debug_arm64_go_test.log || true; exit $exit_code; fi
- name: Collect infinity log
if: always()
run: |
cat infinity.log || true
cat debug_arm64_go_test.log || true
- name: Stop infinity server
if: always()
run: pkill -f infinity || true
debug_arm64_http_test:
name: debug_arm64_http_test
runs-on: ubuntu-24.04-arm
needs: debug_arm64_build
steps:
- name: Free disk space
uses: jlumbroso/free-disk-space@main
with:
tool-cache: false
android: true
dotnet: true
haskell: true
large-packages: true
docker-images: true
swap-storage: true
- name: Check out code
uses: actions/checkout@v6
with:
ref: ${{ (github.event_name == 'pull_request' || github.event_name == 'pull_request_target') && format('refs/pull/{0}/merge', github.event.pull_request.number) || github.sha }}
fetch-depth: 0
submodules: recursive
- name: Download debug build artifacts
uses: actions/download-artifact@v8
with:
name: debug-build-arm64
- name: Prepare environment and start infinity
run: |
chmod +x infinity
sudo mkdir -p /usr/share/infinity/resource
sudo cp -r resource/* /usr/share/infinity/resource/
sudo chown -R ${USER} /usr/share/infinity
sudo mkdir -p /var/infinity
sudo chown -R ${USER} /var/infinity
nohup ./infinity --config=conf/pytest_parallel_infinity_conf.toml > infinity.log 2>&1 &
- name: Install Python SDK
uses: astral-sh/setup-uv@v7
# with:
# enable-cache: true
- run: uv sync --python 3.11 --extra test --index-url https://pypi.org/simple/
- name: Wait for infinity and run http test
run: |
./scripts/wait_for_infinity.sh
uv run python3 tools/run_http_api.py
- name: Collect infinity log
if: always()
run: |
cat infinity.log || true
- name: Stop infinity server
if: always()
run: pkill -f infinity || true
debug_arm64_pysdk_parallel_test:
name: debug_arm64_pysdk_parallel_test
runs-on: ubuntu-24.04-arm
needs: debug_arm64_build
steps:
- name: Free disk space
uses: jlumbroso/free-disk-space@main
with:
tool-cache: false
android: true
dotnet: true
haskell: true
large-packages: true
docker-images: true
swap-storage: true
- name: Check out code
uses: actions/checkout@v6
with:
ref: ${{ (github.event_name == 'pull_request' || github.event_name == 'pull_request_target') && format('refs/pull/{0}/merge', github.event.pull_request.number) || github.sha }}
fetch-depth: 0
submodules: recursive
- name: Download debug build artifacts
uses: actions/download-artifact@v8
with:
name: debug-build-arm64
- name: Prepare environment and start infinity
run: |
chmod +x infinity
sudo mkdir -p /usr/share/infinity/resource
sudo cp -r resource/* /usr/share/infinity/resource/
sudo chown -R ${USER} /usr/share/infinity
sudo mkdir -p /var/infinity
sudo chown -R ${USER} /var/infinity
nohup ./infinity --config=conf/pytest_parallel_infinity_conf.toml > infinity.log 2>&1 &
- name: Install Python SDK
uses: astral-sh/setup-uv@v7
# with:
# enable-cache: true
- run: uv sync --python 3.11 --extra test --index-url https://pypi.org/simple/
- name: Wait for infinity and run pysdk test
run: |
./scripts/wait_for_infinity.sh
uv run python3 tools/run_parallel_test.py
- name: Collect infinity log
if: always()
run: |
cat infinity.log || true
- name: Stop infinity server
if: always()
run: pkill -f infinity || true
release_arm64_build:
name: release_arm64_build
runs-on: ubuntu-24.04-arm
needs: code_static_check
steps:
- name: Free disk space
run: |
sudo rm -rf /usr/share/dotnet /usr/local/lib/android/sdk /opt/ghc /opt/hostedtoolcache/CodeQL
df -h /
- name: Check out code
uses: actions/checkout@v6
with:
ref: ${{ (github.event_name == 'pull_request' || github.event_name == 'pull_request_target') && format('refs/pull/{0}/merge', github.event.pull_request.number) || github.sha }}
fetch-depth: 0
fetch-tags: false
submodules: recursive
- name: Start builder container
run: |
docker run --privileged -d --name infinity_builder -v ${PWD}:/infinity -v ${PWD}/resource:/usr/share/infinity/resource infiniflow/infinity_builder:ubuntu22_clang20
- name: Build release version
run: docker exec infinity_builder bash -c "git config --global safe.directory \"*\" && cd /infinity && rm -rf cmake-build-reldeb && mkdir -p cmake-build-reldeb && cmake -G Ninja -DCMAKE_BUILD_TYPE=RelWithDebInfo -S /infinity -B /infinity/cmake-build-reldeb && cmake --build /infinity/cmake-build-reldeb -j --target infinity test_main"
- name: Upload artifacts
uses: actions/upload-artifact@v7
with:
name: release-build-arm64
path: |
cmake-build-reldeb/src/infinity
cmake-build-reldeb/src/test_main
retention-days: 1
if-no-files-found: warn
- name: Remove build container
if: always()
run: docker rm -f infinity_builder || true
release_arm64_unit_test:
name: release_arm64_unit_test
runs-on: ubuntu-24.04-arm
needs: release_arm64_build
steps:
- name: Free disk space
uses: jlumbroso/free-disk-space@main
with:
tool-cache: false
android: true
dotnet: true
haskell: true
large-packages: true
docker-images: true
swap-storage: true
- name: Check out code
uses: actions/checkout@v6
with:
ref: ${{ (github.event_name == 'pull_request' || github.event_name == 'pull_request_target') && format('refs/pull/{0}/merge', github.event.pull_request.number) || github.sha }}
fetch-depth: 0
submodules: recursive
- name: Start minio container
run: |
docker run -d --name minio --network host -e "MINIO_ROOT_PASSWORD=minioadmin" -e "MINIO_ROOT_USER=minioadmin" -v /tmp/minio-data:/data quay.io/minio/minio server /data --console-address ":9006" --address ":9005"
- name: Download release build artifacts
uses: actions/download-artifact@v8
with:
name: release-build-arm64
- name: Install Python SDK
uses: astral-sh/setup-uv@v7
# with:
# enable-cache: true
- run: uv sync --python 3.11 --extra test --index-url https://pypi.org/simple/
- name: Download nltk data
run: |
uv run python -c "
import os
import nltk
local_dir = os.path.abspath('nltk_data')
for data in ['wordnet', 'punkt', 'punkt_tab']:
print(f'Downloading nltk {data}...')
nltk.download(data, download_dir=local_dir)
"
- name: Prepare environment
run: |
chmod +x ./test_main
sudo mkdir -p /usr/share/infinity/resource
sudo cp -r resource/* /usr/share/infinity/resource/
sudo chown -R ${USER} /usr/share/infinity
sudo mkdir -p /var/infinity
sudo chown -R ${USER} /var/infinity
- name: Wait for minio and run unit test
env:
NLTK_DATA: ${{ github.workspace }}/nltk_data
run: |
./scripts/wait_for_minio.sh
./test_main --gtest_filter=-*SLOW_* > release_arm64_unit_test_stdout.log 2> release_arm64_unit_test_stderr.log
- name: Collect infinity log
if: always()
run: |
echo "release arm64 unit test log" && tail -n 1000 release_arm64_unit_test*.log || true
- name: Remove minio container
if: always()
run: docker rm -f minio || true
release_arm64_sqllogical_test:
name: release_arm64_sqllogical_test
runs-on: ubuntu-24.04-arm
needs: release_arm64_build
# if: false
steps:
- name: Check out code
uses: actions/checkout@v6
with:
ref: ${{ (github.event_name == 'pull_request' || github.event_name == 'pull_request_target') && format('refs/pull/{0}/merge', github.event.pull_request.number) || github.sha }}
fetch-depth: 0
submodules: recursive
- name: Download release build artifacts
uses: actions/download-artifact@v8
with:
name: release-build-arm64
- name: Prepare environment and start infinity
run: |
chmod +x ./infinity
sudo mkdir -p /usr/share/infinity/resource
sudo cp -r resource/* /usr/share/infinity/resource/
sudo chown -R ${USER} /usr/share/infinity
sudo mkdir -p /var/infinity
sudo chown -R ${USER} /var/infinity
nohup ./infinity --config=conf/pytest_parallel_infinity_conf.toml > infinity.log 2>&1 &
- name: Install sqllogictest
run: |
curl -sLO https://github.com/risinglightdb/sqllogictest-rs/releases/download/v0.28.4/sqllogictest-bin-v0.28.4-aarch64-unknown-linux-musl.tar.gz
tar -xzf sqllogictest-bin-v0.28.4-aarch64-unknown-linux-musl.tar.gz
sudo mv sqllogictest /usr/local/bin/
- name: Install Python SDK
uses: astral-sh/setup-uv@v7
# with:
# enable-cache: true
- run: uv sync --python 3.11 --extra test --index-url https://pypi.org/simple/
- name: Wait for infinity and run sqllogictest
run: |
./scripts/wait_for_infinity.sh
uv run python3 tools/sqllogictest.py
- name: Collect infinity log
if: always()
run: |
cat infinity.log || true
- name: Stop infinity server
if: always()
run: pkill -f infinity || true
release_arm64_restart_test:
name: release_arm64_restart_test
runs-on: ubuntu-24.04-arm
needs: release_arm64_build
steps:
- name: Free disk space
uses: jlumbroso/free-disk-space@main
with:
tool-cache: false
android: true
dotnet: true
haskell: true
large-packages: true
docker-images: true
swap-storage: true
- name: Check out code
uses: actions/checkout@v6
with:
ref: ${{ (github.event_name == 'pull_request' || github.event_name == 'pull_request_target') && format('refs/pull/{0}/merge', github.event.pull_request.number) || github.sha }}
fetch-depth: 0
submodules: recursive
- name: Download release build artifacts
uses: actions/download-artifact@v8
with:
name: release-build-arm64
- name: Prepare environment
run: |
chmod +x ./infinity
sudo mkdir -p /usr/share/infinity/resource
sudo cp -r resource/* /usr/share/infinity/resource/
sudo chown -R ${USER} /usr/share/infinity
sudo mkdir -p /var/infinity
sudo chown -R ${USER} /var/infinity
- name: Install Python SDK
uses: astral-sh/setup-uv@v7
# with:
# enable-cache: true
- run: uv sync --python 3.11 --extra test --index-url https://pypi.org/simple/
- name: Run restart test
run: |
uv run python3 tools/run_restart_test.py --infinity_path=./infinity
- name: Collect infinity log
if: always()
run: |
echo "release_arm64_restart_test log(last 1000 lines):" && tail -n 1000 restart_test*.log* || true
# release_arm64_pysdk_test:
# name: release_arm64_pysdk_test
# runs-on: ubuntu-24.04-arm
# needs: release_arm64_build
# steps:
# - name: Check out code
# uses: actions/checkout@v6
# with:
# ref: ${{ (github.event_name == 'pull_request' || github.event_name == 'pull_request_target') && format('refs/pull/{0}/merge', github.event.pull_request.number) || github.sha }}
# fetch-depth: 0
# submodules: recursive
#
# - name: Download release build artifacts
# uses: actions/download-artifact@v8
# with:
# name: release-build-arm64
#
# - name: Prepare environment
# run: |
# chmod +x ./infinity
# sudo mkdir -p /usr/share/infinity/resource
# sudo cp -r resource/* /usr/share/infinity/resource/
# sudo chown -R ${USER} /usr/share/infinity
# sudo mkdir -p /var/infinity
# sudo chown -R ${USER} /var/infinity
#
# - name: Install Python SDK
# uses: astral-sh/setup-uv@v7
# with:
# enable-cache: true
# - run: uv sync --python 3.11 --extra test --index-url https://pypi.org/simple/
#
# - name: Start infinity server
# run: |
# nohup ./infinity --config=conf/pytest_parallel_infinity_conf.toml > infinity.log 2>&1 &
# ./scripts/wait_for_infinity.sh
#
# - name: Run pysdk test
# run: uv run python3 tools/run_pysdk_remote_infinity_test.py
#
# - name: Collect infinity log
# if: always()
# run: |
# cat infinity.log || true
#
# - name: Stop infinity server
# if: always()
# run: pkill -f infinity || true
release_arm64_go_test:
name: release_arm64_go_test
runs-on: ubuntu-24.04-arm
needs: release_arm64_build
steps:
- name: Free disk space
uses: jlumbroso/free-disk-space@main
with:
tool-cache: false
android: true
dotnet: true
haskell: true
large-packages: true
docker-images: true
swap-storage: true
- name: Check out code
uses: actions/checkout@v6
with:
ref: ${{ (github.event_name == 'pull_request' || github.event_name == 'pull_request_target') && format('refs/pull/{0}/merge', github.event.pull_request.number) || github.sha }}
fetch-depth: 0
submodules: recursive
- name: Download release build artifacts
uses: actions/download-artifact@v8
with:
name: release-build-arm64
- name: Prepare environment and start infinity
run: |
chmod +x ./infinity
sudo mkdir -p /usr/share/infinity/resource
sudo cp -r resource/* /usr/share/infinity/resource/
sudo chown -R ${USER} /usr/share/infinity
sudo mkdir -p /var/infinity
sudo chown -R ${USER} /var/infinity
nohup ./infinity --config=conf/pytest_parallel_infinity_conf.toml > infinity.log 2>&1 &
- name: Setup Go
uses: actions/setup-go@v6
with:
go-version: '1.23'
- name: Install sqllogictest
run: |
curl -sLO https://github.com/risinglightdb/sqllogictest-rs/releases/download/v0.28.4/sqllogictest-bin-v0.28.4-aarch64-unknown-linux-musl.tar.gz
tar -xzf sqllogictest-bin-v0.28.4-aarch64-unknown-linux-musl.tar.gz
sudo mv sqllogictest /usr/local/bin/
- name: Install Python SDK
uses: astral-sh/setup-uv@v7
# with:
# enable-cache: true
- run: uv sync --python 3.11 --extra test --index-url https://pypi.org/simple/
- name: Wait for infinity and run Go tests
run: |
./scripts/wait_for_infinity.sh
uv run python3 tools/sqllogictest.py -jc=1
cd go && go version && go test ./tests/... -v 2>&1 | tee ../release_arm64_go_test.log; exit_code=${PIPESTATUS[0]}; if [ $exit_code -ne 0 ]; then echo "=== FAILED TESTS ==="; grep -E "^(FAIL|panic:|^--- FAIL)" ../release_arm64_go_test.log || true; exit $exit_code; fi
- name: Collect infinity log
if: always()
run: |
cat infinity.log || true
cat release_arm64_go_test.log || true
- name: Stop infinity server
if: always()
run: pkill -f infinity || true
release_arm64_http_test:
name: release_arm64_http_test
runs-on: ubuntu-24.04-arm
needs: release_arm64_build
steps:
- name: Free disk space
uses: jlumbroso/free-disk-space@main
with:
tool-cache: false
android: true
dotnet: true
haskell: true
large-packages: true
docker-images: true
swap-storage: true
- name: Check out code
uses: actions/checkout@v6
with:
ref: ${{ (github.event_name == 'pull_request' || github.event_name == 'pull_request_target') && format('refs/pull/{0}/merge', github.event.pull_request.number) || github.sha }}
fetch-depth: 0
submodules: recursive
- name: Download release build artifacts
uses: actions/download-artifact@v8
with:
name: release-build-arm64
- name: Prepare environment and start infinity
run: |
chmod +x ./infinity
sudo mkdir -p /usr/share/infinity/resource
sudo cp -r resource/* /usr/share/infinity/resource/
sudo chown -R ${USER} /usr/share/infinity
sudo mkdir -p /var/infinity
sudo chown -R ${USER} /var/infinity
nohup ./infinity --config=conf/pytest_parallel_infinity_conf.toml > infinity.log 2>&1 &
- name: Install Python SDK
uses: astral-sh/setup-uv@v7
# with:
# enable-cache: true
- run: uv sync --python 3.11 --extra test --index-url https://pypi.org/simple/
- name: Wait for infinity and run http test
run: |
./scripts/wait_for_infinity.sh
uv run python3 tools/run_http_api.py
- name: Collect infinity log
if: always()
run: |
cat infinity.log || true
- name: Stop infinity server
if: always()
run: pkill -f infinity || true
# release_arm64_pysdk_parallel_test:
# name: release_arm64_pysdk_parallel_test
# runs-on: ubuntu-24.04-arm
# needs: release_arm64_build
# steps:
# - name: Check out code
# uses: actions/checkout@v6
# with:
# ref: ${{ (github.event_name == 'pull_request' || github.event_name == 'pull_request_target') && format('refs/pull/{0}/merge', github.event.pull_request.number) || github.sha }}
# fetch-depth: 0
# submodules: recursive
#
# - name: Download release build artifacts
# uses: actions/download-artifact@v8
# with:
# name: release-build-arm64
#
# - name: Prepare environment
# run: |
# chmod +x ./infinity
# sudo mkdir -p /usr/share/infinity/resource
# sudo cp -r resource/* /usr/share/infinity/resource/
# sudo chown -R ${USER} /usr/share/infinity
# sudo mkdir -p /var/infinity
# sudo chown -R ${USER} /var/infinity
#
# - name: Install Python SDK
# uses: astral-sh/setup-uv@v7
# with:
# enable-cache: true
# - run: uv sync --python 3.11 --extra test --index-url https://pypi.org/simple/
#
# - name: Start infinity server
# run: |
# nohup ./infinity --config=conf/pytest_parallel_infinity_conf.toml > infinity.log 2>&1 &
# ./scripts/wait_for_infinity.sh
#
# - name: Run pysdk test
# run: uv run python3 tools/run_parallel_test.py
#
# - name: Collect infinity log
# if: always()
# run: |
# cat infinity.log || true
#
# - name: Stop infinity server
# if: always()
# run: pkill -f infinity || true
release_arm64_minio_pysdk_test:
name: release_arm64_minio_pysdk_test
runs-on: ubuntu-24.04-arm
needs: release_arm64_build
steps:
- name: Free disk space
run: |
sudo rm -rf /usr/share/dotnet /usr/local/lib/android/sdk /opt/ghc /opt/hostedtoolcache/CodeQL
df -h /
- name: Check out code
uses: actions/checkout@v6
with:
ref: ${{ (github.event_name == 'pull_request' || github.event_name == 'pull_request_target') && format('refs/pull/{0}/merge', github.event.pull_request.number) || github.sha }}
fetch-depth: 0
submodules: recursive
- name: Start minio container
run: |
docker run -d --name minio --network host -e "MINIO_ROOT_PASSWORD=minioadmin" -e "MINIO_ROOT_USER=minioadmin" -v /tmp/minio-data:/data quay.io/minio/minio server /data --console-address ":9006" --address ":9005"
- name: Download release build artifacts
uses: actions/download-artifact@v8
with:
name: release-build-arm64
- name: Prepare environment and start infinity
run: |
chmod +x infinity
sudo mkdir -p /usr/share/infinity/resource
sudo cp -r resource/* /usr/share/infinity/resource/
sudo chown -R ${USER} /usr/share/infinity
sudo mkdir -p /var/infinity
sudo chown -R ${USER} /var/infinity
nohup ./infinity --config=conf/pytest_parallel_infinity_minio.toml > infinity.log 2>&1 &
- name: Install Python SDK
uses: astral-sh/setup-uv@v7
# with:
# enable-cache: true
- run: uv sync --python 3.11 --extra test --index-url https://pypi.org/simple/
- name: Wait for services and run minio pysdk test
run: |
./scripts/wait_for_minio.sh
./scripts/wait_for_infinity.sh
uv run python3 tools/run_pysdk_remote_infinity_test.py
- name: Collect infinity log
if: always()
run: |
cat infinity.log || true
- name: Stop infinity server
if: always()
run: pkill -f infinity || true
- name: Remove minio container
if: always()
run: docker rm -f minio || true
release_arm64_minio_parallel_test:
name: release_arm64_minio_parallel_test
runs-on: ubuntu-24.04-arm
needs: release_arm64_build
steps:
- name: Free disk space
run: |
sudo rm -rf /usr/share/dotnet /usr/local/lib/android/sdk /opt/ghc /opt/hostedtoolcache/CodeQL
df -h /
- name: Check out code
uses: actions/checkout@v6
with:
ref: ${{ (github.event_name == 'pull_request' || github.event_name == 'pull_request_target') && format('refs/pull/{0}/merge', github.event.pull_request.number) || github.sha }}
fetch-depth: 0
submodules: recursive
- name: Start minio container
run: |
docker run -d --name minio --network host -e "MINIO_ROOT_PASSWORD=minioadmin" -e "MINIO_ROOT_USER=minioadmin" -v /tmp/minio-data:/data quay.io/minio/minio server /data --console-address ":9006" --address ":9005"
- name: Download release build artifacts
uses: actions/download-artifact@v8
with:
name: release-build-arm64
- name: Prepare environment and start infinity
run: |
chmod +x infinity
sudo mkdir -p /usr/share/infinity/resource
sudo cp -r resource/* /usr/share/infinity/resource/
sudo chown -R ${USER} /usr/share/infinity
sudo mkdir -p /var/infinity
sudo chown -R ${USER} /var/infinity
nohup ./infinity --config=conf/pytest_parallel_infinity_minio.toml > infinity.log 2>&1 &
- name: Install Python SDK
uses: astral-sh/setup-uv@v7
# with:
# enable-cache: true
- run: uv sync --python 3.11 --extra test --index-url https://pypi.org/simple/
- name: Wait for services and run minio parallel test
run: |
./scripts/wait_for_minio.sh
./scripts/wait_for_infinity.sh
uv run python3 tools/run_parallel_test.py
- name: Collect infinity log
if: always()
run: |
cat infinity.log || true
- name: Stop infinity server
if: always()
run: pkill -f infinity || true
- name: Remove minio container
if: always()
run: docker rm -f minio || true
# ============================================
# AMD64 Benchmark (self-hosted runner)
# ============================================
reldeb_amd64_benchmark_prepare:
name: reldeb_amd64_benchmark_prepare
runs-on: [ "self-hosted", "infinity-build" ]
needs: code_static_check
steps:
- name: Ensure workspace ownership
run: echo "chown -R ${USER} ${GITHUB_WORKSPACE}" && sudo chown -R ${USER} ${GITHUB_WORKSPACE}
- name: Check out code
uses: actions/checkout@v6
with:
ref: ${{ (github.event_name == 'pull_request' || github.event_name == 'pull_request_target') && format('refs/pull/{0}/merge', github.event.pull_request.number) || github.sha }}
fetch-depth: 0
submodules: recursive
- name: Prepare and upload source tarball
run: |
echo "chown -R ${USER} ${GITHUB_WORKSPACE}" && sudo chown -R ${USER} ${GITHUB_WORKSPACE}
tar --create --gzip --file=infinity.tgz .git $(git ls-files)
ARTIFACTS_DIR=${RUNNER_WORKSPACE_PREFIX}/artifacts/${GITHUB_REPOSITORY}/${GITHUB_RUN_ID}
echo "ARTIFACTS_DIR=${ARTIFACTS_DIR}" >> ${GITHUB_ENV}
mkdir -p ${ARTIFACTS_DIR} && cp infinity.tgz ${ARTIFACTS_DIR}/
reldeb_amd64_benchmark_build:
name: reldeb_benchmark_build
runs-on: [ "self-hosted", "infinity-build" ]
needs: reldeb_amd64_benchmark_prepare
continue-on-error: true
outputs:
build_status: ${{ steps.build.outcome }}
steps:
- name: Prepare code
if: ${{ !cancelled() && !failure() }}
run: |
echo "chown -R ${USER} ${GITHUB_WORKSPACE}" && sudo chown -R ${USER} ${GITHUB_WORKSPACE}
ARTIFACTS_DIR=${RUNNER_WORKSPACE_PREFIX}/artifacts/${GITHUB_REPOSITORY}/${GITHUB_RUN_ID}
echo "ARTIFACTS_DIR=${ARTIFACTS_DIR}" >> ${GITHUB_ENV}
tar xzf ${ARTIFACTS_DIR}/infinity.tgz && git reset --hard && git clean -ffdx
- name: Start builder container
if: ${{ !cancelled() && !failure() }}
run: |
BUILDER_CONTAINER=infinity_build_$(od -An -N4 -tx4 /dev/urandom | tr -d ' ')
echo "BUILDER_CONTAINER=${BUILDER_CONTAINER}" >> ${GITHUB_ENV}
TZ=${TZ:-$(readlink -f /etc/localtime | awk -F '/zoneinfo/' '{print $2}')}
sudo docker run --privileged -d --name ${BUILDER_CONTAINER} -e TZ=${TZ} -v ${PWD}:/infinity infiniflow/infinity_builder:ubuntu20_clang20
- name: Build reldeb version
id: build
if: ${{ !cancelled() && !failure() }}
run: sudo docker exec ${BUILDER_CONTAINER} bash -c "git config --global safe.directory \"*\" && cd /infinity && rm -rf cmake-build-reldeb && mkdir -p cmake-build-reldeb && cmake -G Ninja -DCMAKE_BUILD_TYPE=RelWithDebInfo -S /infinity -B /infinity/cmake-build-reldeb && cmake --build /infinity/cmake-build-reldeb -j --target infinity knn_import_benchmark knn_query_benchmark"
- name: Upload artifacts
if: ${{ !cancelled() && !failure() && steps.build.outcome == 'success' }}
run: |
tar -cf - cmake-build-reldeb/src/infinity cmake-build-reldeb/benchmark/local_infinity/{knn_import_benchmark,knn_query_benchmark} | tar -C ${ARTIFACTS_DIR} -xf -
- name: Remove build container
if: always()
run: |
if [[ -n "${BUILDER_CONTAINER}" ]]; then
sudo docker rm -f -v "${BUILDER_CONTAINER}"
fi
reldeb_amd64_benchmark:
name: reldeb_benchmark
runs-on: [ "self-hosted", "benchmark" ]
needs: reldeb_amd64_benchmark_build
if: needs.reldeb_amd64_benchmark_build.outputs.build_status == 'success'
steps:
- name: Prepare code and artifacts
run: |
echo "chown -R ${USER} ${GITHUB_WORKSPACE}" && sudo chown -R ${USER} ${GITHUB_WORKSPACE}
ARTIFACTS_DIR=${RUNNER_WORKSPACE_PREFIX}/artifacts/${GITHUB_REPOSITORY}/${GITHUB_RUN_ID}
echo "ARTIFACTS_DIR=${ARTIFACTS_DIR}" >> ${GITHUB_ENV}
tar xzf ${ARTIFACTS_DIR}/infinity.tgz && git reset --hard && git clean -ffdx
tar -C ${ARTIFACTS_DIR} -cf - cmake-build-reldeb/benchmark/local_infinity/{knn_import_benchmark,knn_query_benchmark} | tar -xf -
- name: Prepare sift dataset
run: |
RUNNER_WORKSPACE_PREFIX=${RUNNER_WORKSPACE_PREFIX:-${HOME}}
echo "RUNNER_WORKSPACE_PREFIX=${RUNNER_WORKSPACE_PREFIX}" >> ${GITHUB_ENV}
chmod +x ./tools/ci_tools/check_benchmark_result.py
rm -rf test/data/benchmark && ln -s ${RUNNER_WORKSPACE_PREFIX}/benchmark test/data/benchmark
- name: Golden benchmark sift
run: |
if [[ -f "${RUNNER_WORKSPACE_PREFIX}/benchmark/golden_benchmark_sift_1_thread.log" ]]; then
echo "golden benchmark result already exists!"
exit 0
fi
rm -rf ${PWD}/db_tmp && mkdir -p ${PWD}/db_tmp && cat ${RUNNER_WORKSPACE_PREFIX}/benchmark/infinity_conf.toml | sed -e "s|/var/infinity|${PWD}/db_tmp|g" > ${PWD}/db_tmp/infinity_conf.toml
${RUNNER_WORKSPACE_PREFIX}/benchmark/knn_import_benchmark --dataset=sift --dataset_dir=${PWD}/test/data --infinity_dir=${PWD}/db_tmp
echo "1 50" | ${RUNNER_WORKSPACE_PREFIX}/benchmark/knn_query_benchmark sift 200 false ${PWD}/test/data ${PWD}/db_tmp | sudo tee ${RUNNER_WORKSPACE_PREFIX}/benchmark/golden_benchmark_sift_1_thread.log
echo "8 50" | ${RUNNER_WORKSPACE_PREFIX}/benchmark/knn_query_benchmark sift 200 false ${PWD}/test/data ${PWD}/db_tmp | sudo tee ${RUNNER_WORKSPACE_PREFIX}/benchmark/golden_benchmark_sift_8_threads.log
- name: Latest benchmark sift
run: |
rm -rf ${PWD}/db_tmp && mkdir -p ${PWD}/db_tmp && cat conf/infinity_conf.toml | sed -e "s|/var/infinity|${PWD}/db_tmp|g" > ${PWD}/db_tmp/infinity_conf.toml
./cmake-build-reldeb/benchmark/local_infinity/knn_import_benchmark --dataset=sift --dataset_dir=${PWD}/test/data --infinity_dir=${PWD}/db_tmp --config_path=db_tmp/infinity_conf.toml
echo "1 50" | ./cmake-build-reldeb/benchmark/local_infinity/knn_query_benchmark sift 200 false ${PWD}/test/data ${PWD}/db_tmp db_tmp/infinity_conf.toml | tee benchmark_sift_1_thread.log
echo "8 50" | ./cmake-build-reldeb/benchmark/local_infinity/knn_query_benchmark sift 200 false ${PWD}/test/data ${PWD}/db_tmp db_tmp/infinity_conf.toml | tee benchmark_sift_8_threads.log
./tools/ci_tools/check_benchmark_result.py ${RUNNER_WORKSPACE_PREFIX}/benchmark/golden_benchmark_sift_1_thread.log benchmark_sift_1_thread.log
./tools/ci_tools/check_benchmark_result.py ${RUNNER_WORKSPACE_PREFIX}/benchmark/golden_benchmark_sift_8_threads.log benchmark_sift_8_threads.log
# ============================================
# Final cleanup job
# ============================================
workflow_cleanup:
name: workflow_cleanup
runs-on: ubuntu-slim
needs:
- code_static_check
- debug_amd64_build
- debug_amd64_unit_test
- debug_amd64_sqllogical_test
- debug_amd64_restart_test
- debug_amd64_pysdk_test
- debug_amd64_go_test
- debug_amd64_http_test
- debug_amd64_pysdk_parallel_test
- release_amd64_build
- release_amd64_unit_test
- release_amd64_sqllogical_test
- release_amd64_restart_test
- release_amd64_go_test
- release_amd64_http_test
- release_amd64_minio_pysdk_test
- release_amd64_minio_parallel_test
- debug_arm64_build
- debug_arm64_unit_test
- debug_arm64_sqllogical_test
- debug_arm64_restart_test
- debug_arm64_pysdk_test
- debug_arm64_go_test
- debug_arm64_http_test
- debug_arm64_pysdk_parallel_test
- release_arm64_build
- release_arm64_unit_test
- release_arm64_sqllogical_test
- release_arm64_restart_test
- release_arm64_go_test
- release_arm64_http_test
- release_arm64_minio_pysdk_test
- release_arm64_minio_parallel_test
if: always()
steps:
- name: Cleanup
run: echo "Workflow cleanup. Just a placeholder now."