Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
39 commits
Select commit Hold shift + click to select a range
3e5f9d1
feat: add vision subpackage with MediaPipe HeadTracker
alozowski Jun 1, 2026
c89faea
fix: add type annotations and make landmark helpers private
alozowski Jun 1, 2026
cc3a6c8
Merge remote-tracking branch 'origin/main' into feat/1175-move-mediapipe
alozowski Jun 25, 2026
02a7287
feat: add BlazeFace head tracker
alozowski Jun 25, 2026
c7c540a
feat: add daemon head tracking controls
alozowski Jun 25, 2026
0b46d50
feat: expose head tracking in SDKs
alozowski Jun 25, 2026
a50cdb1
test: cover daemon head tracking
alozowski Jun 25, 2026
767860f
Merge remote-tracking branch 'origin/main' into feat/1175-move-mediapipe
alozowski Jun 25, 2026
c86a701
fix: keep disabled tracking branch from blocking media startup
alozowski Jun 26, 2026
458e03c
fix: start head tracking without blocking sdk websocket
alozowski Jun 26, 2026
06e4e0e
fix: report daemon head tracking availability
alozowski Jun 26, 2026
4c774ff
fix: smooth daemon head tracking aim
alozowski Jun 26, 2026
e20e07a
fix: reduce head tracking daemon logs
alozowski Jun 26, 2026
69a78f4
test: cover SDK head tracking availability
alozowski Jun 26, 2026
a49229c
docs: update OpenAPI spec for head tracking
alozowski Jun 26, 2026
ff8a0bd
fix: reduce head tracking movement churn
alozowski Jun 26, 2026
a2b578f
fix: reduce IK churn during head tracking
alozowski Jun 26, 2026
c2b33f6
fix: stabilize daemon head tracking with aim gain and aspect-correct …
alozowski Jun 26, 2026
2833c67
fix: remove head-tracking upward bias and hold aim on transient face …
alozowski Jun 26, 2026
9865a95
feat: raise head-tracking rate via mediapipe VIDEO mode and smaller f…
alozowski Jun 26, 2026
6c7a911
feat: cap head-tracking at 15 fps to ease CM4 control-loop contention
alozowski Jun 27, 2026
f880042
refactor: drop in-process head-tracking detector and mediapipe extra,…
alozowski Jun 27, 2026
3aee278
feat: add YuNet face detector and bundle its ONNX model
alozowski Jun 27, 2026
ce98bde
feat: run head-tracking detector out-of-process with per-tick aim easing
alozowski Jun 27, 2026
b7235e1
fix: keep head-tracking detector cheap and low-priority, and track th…
alozowski Jun 28, 2026
5db905d
feat: feed the face tracker a dedicated downscaled camera branch, not…
alozowski Jun 28, 2026
a5217bd
fix: read tracker frames with the try-pull-sample action signal
alozowski Jun 28, 2026
26f7c6c
fix: forward sticky events on the tracking valve so unixfdsink negoti…
alozowski Jun 28, 2026
f1516d4
fix: disable tracking-branch unixfdsink preroll so the valve gate del…
alozowski Jun 28, 2026
9a5b52f
feat: steady head tracking, deadband jitter, hold then recenter on lo…
alozowski Jun 28, 2026
252ac94
fix: lower YuNet score threshold to 0.6 so tracking holds the face th…
alozowski Jun 29, 2026
55b2aa6
feat: ROI-crop face detection to cut worker CPU
alozowski Jun 29, 2026
8c8e3cb
feat: cache YuNet input size + fixed-size ROI crop to skip per-frame …
alozowski Jun 29, 2026
126104c
feat: lower tracking fps to 10 and ROI crop to 192 for CPU headroom u…
alozowski Jun 29, 2026
a3ade9a
feat: pause head tracking at weight 0 to free the head and CPU withou…
alozowski Jun 29, 2026
15cd4cb
Merge remote-tracking branch 'origin/main' into feat/1175-move-mediapipe
alozowski Jun 30, 2026
cb9d5bf
feat: shrink head-tracking ROI crop to 128 (half detection CPU, equal…
alozowski Jun 30, 2026
4e277fa
debug: log tracker fps/hit/locked every 2s (temp)
alozowski Jun 30, 2026
20960b1
chore: remove temp tracker debug logging
alozowski Jun 30, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
181 changes: 181 additions & 0 deletions docs/source/API/openapi.json
Original file line number Diff line number Diff line change
Expand Up @@ -1400,6 +1400,114 @@
}
}
},
"/api/media/tracking/enable": {
"post": {
"summary": "Enable Tracking",
"description": "Enable daemon-side visual head tracking.",
"operationId": "enable_tracking_api_media_tracking_enable_post",
"requestBody": {
"content": {
"application/json": {
"schema": {
"anyOf": [
{
"$ref": "#/components/schemas/EnableTrackingRequest"
},
{
"type": "null"
}
],
"title": "Body"
}
}
}
},
"responses": {
"200": {
"description": "Successful Response",
"content": {
"application/json": {
"schema": {
"additionalProperties": {
"anyOf": [
{
"type": "string"
},
{
"type": "boolean"
}
]
},
"type": "object",
"title": "Response Enable Tracking Api Media Tracking Enable Post"
}
}
}
},
"422": {
"description": "Validation Error",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/HTTPValidationError"
}
}
}
}
}
}
},
"/api/media/tracking/disable": {
"post": {
"summary": "Disable Tracking",
"description": "Disable daemon-side visual head tracking.",
"operationId": "disable_tracking_api_media_tracking_disable_post",
"responses": {
"200": {
"description": "Successful Response",
"content": {
"application/json": {
"schema": {
"additionalProperties": {
"anyOf": [
{
"type": "string"
},
{
"type": "boolean"
}
]
},
"type": "object",
"title": "Response Disable Tracking Api Media Tracking Disable Post"
}
}
}
}
}
}
},
"/api/media/tracking/face": {
"get": {
"summary": "Get Tracked Face",
"description": "Return the latest face observed by daemon-side head tracking.",
"operationId": "get_tracked_face_api_media_tracking_face_get",
"responses": {
"200": {
"description": "Successful Response",
"content": {
"application/json": {
"schema": {
"additionalProperties": true,
"type": "object",
"title": "Response Get Tracked Face Api Media Tracking Face Get"
}
}
}
}
}
}
},
"/api/media/sounds/upload": {
"post": {
"summary": "Upload Sound",
Expand Down Expand Up @@ -2686,6 +2794,9 @@
}
],
"title": "Hardware Id"
},
"face_target": {
"$ref": "#/components/schemas/FaceTarget"
}
},
"type": "object",
Expand Down Expand Up @@ -2720,6 +2831,76 @@
"title": "DoAInfo",
"description": "Direction of Arrival info from the microphone array."
},
"EnableTrackingRequest": {
"properties": {
"weight": {
"type": "number",
"maximum": 1.0,
"minimum": 0.0,
"title": "Weight",
"default": 1.0
}
},
"type": "object",
"title": "EnableTrackingRequest",
"description": "Request body for enabling daemon-side head tracking."
},
"FaceTarget": {
"properties": {
"detected": {
"type": "boolean",
"title": "Detected",
"default": false
},
"x": {
"anyOf": [
{
"type": "number"
},
{
"type": "null"
}
],
"title": "X"
},
"y": {
"anyOf": [
{
"type": "number"
},
{
"type": "null"
}
],
"title": "Y"
},
"roll": {
"anyOf": [
{
"type": "number"
},
{
"type": "null"
}
],
"title": "Roll"
},
"ts": {
"anyOf": [
{
"type": "number"
},
{
"type": "null"
}
],
"title": "Ts"
}
},
"type": "object",
"title": "FaceTarget",
"description": "Latest daemon-side face target used for head tracking."
},
"FullBodyTarget": {
"properties": {
"target_head_pose": {
Expand Down
3 changes: 3 additions & 0 deletions src/reachy_mini/assets/face_detection_yunet_2023mar.onnx
Git LFS file not shown
48 changes: 47 additions & 1 deletion src/reachy_mini/daemon/app/routers/media.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
from pathlib import Path

from fastapi import APIRouter, Depends, File, HTTPException, UploadFile
from pydantic import BaseModel
from pydantic import BaseModel, Field

from ....media.gstreamer_utils import is_valid_audio_file
from ...daemon import Daemon
Expand Down Expand Up @@ -68,6 +68,12 @@ class PlaySoundRequest(BaseModel):
file: str


class EnableTrackingRequest(BaseModel):
"""Request body for enabling daemon-side head tracking."""

weight: float = Field(default=1.0, ge=0.0, le=1.0)


@router.post("/play_sound")
async def play_sound(
body: PlaySoundRequest,
Expand Down Expand Up @@ -158,6 +164,46 @@ async def disable_wobbling(
return {"status": "ok"}


@router.post("/tracking/enable")
async def enable_tracking(
body: EnableTrackingRequest | None = None,
daemon: Daemon = Depends(get_daemon),
) -> dict[str, str | bool]:
"""Enable daemon-side visual head tracking."""
backend = daemon.backend
if backend is None or not backend.ready.is_set():
raise HTTPException(status_code=503, detail="Backend not running")

weight = body.weight if body is not None else 1.0
enabled = backend.enable_head_tracking(weight=weight)
return {"status": "ok" if enabled else "unavailable", "enabled": enabled}


@router.post("/tracking/disable")
async def disable_tracking(
daemon: Daemon = Depends(get_daemon),
) -> dict[str, str | bool]:
"""Disable daemon-side visual head tracking."""
backend = daemon.backend
if backend is None or not backend.ready.is_set():
raise HTTPException(status_code=503, detail="Backend not running")

backend.disable_head_tracking()
return {"status": "ok", "enabled": False}


@router.get("/tracking/face")
async def get_tracked_face(
daemon: Daemon = Depends(get_daemon),
) -> dict[str, object]:
"""Return the latest face observed by daemon-side head tracking."""
backend = daemon.backend
if backend is None or not backend.ready.is_set():
raise HTTPException(status_code=503, detail="Backend not running")

return {"status": "ok", "face_target": backend.get_tracked_face().model_dump()}


@router.post("/sounds/upload")
async def upload_sound(
file: UploadFile = File(...),
Expand Down
Loading
Loading