Skip to content

Miscellaneous performance optimizations #3624

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
ChihweiLHBird wants to merge 5 commits into
base: main
Choose a base branch
from
Open

Miscellaneous performance optimizations #3624

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
ca98739
Cache the OTel logs enablement check so it is not re-read from the en…
ChihweiLHBird Jul 18, 2026
f8a4c2a
Resolve the application name once at HTTP server construction instead…
ChihweiLHBird Jul 18, 2026
193b2ef
Check key existence in the SQLite key-value store with a lightweight …
ChihweiLHBird Jul 18, 2026
4815de8
Move embedding vectors out of the response instead of cloning them wh…
ChihweiLHBird Jul 18, 2026
07b919d
Clone only the eos_token_id field instead of the entire llama Config …
ChihweiLHBird Jul 18, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 9 additions & 1 deletion crates/key-value-spin/src/store.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -159,7 +159,15 @@ impl Store for SqliteStore {
}

async fn exists(&self, key: &str) -> Result<bool, Error> {
Ok(self.get(key, usize::MAX).await?.is_some())
task::block_in_place(|| {
self.connection
.lock()
.unwrap()
.prepare_cached("SELECT 1 FROM spin_key_value WHERE store=$1 AND key=$2 LIMIT 1")
.map_err(log_error)?
.exists([&self.name, key])
.map_err(log_error)
})
}

async fn get_keys(&self, max_result_bytes: usize) -> Result<Vec<String>, Error> {
Expand Down
2 changes: 1 addition & 1 deletion crates/llm-local/src/llama.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,7 @@ impl InferencingModel for LlamaModels {
let mut cache = self.cache.clone();
// Try to retrieve the End of Sentence (EOS) token ID from config or
// default to a single EOS token. EOS token is used to determine when to stop.
let eos_token_id = config.clone().eos_token_id.or_else(|| {
let eos_token_id = config.eos_token_id.clone().or_else(|| {
tokenizer
.token_to_id(EOS_TOKEN)
.map(llama::LlamaEosToks::Single)
Expand Down
11 changes: 1 addition & 10 deletions crates/llm-remote-http/src/open_ai/schemas.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,15 +63,6 @@ pub struct CreateEmbeddingResponse {
usage: EmbeddingUsage,
}

impl CreateEmbeddingResponse {
fn embeddings(&self) -> Vec<Vec<f32>> {
self.data
.iter()
.map(|embedding| embedding.embedding.clone())
.collect()
}
}

#[derive(Deserialize)]
struct EmbeddingUsage {
prompt_tokens: u32,
Expand All @@ -95,7 +86,7 @@ impl From<CreateChatCompletionResponse> for wasi_llm::InferencingResult {
impl From<CreateEmbeddingResponse> for wasi_llm::EmbeddingsResult {
fn from(value: CreateEmbeddingResponse) -> Self {
Self {
embeddings: value.embeddings(),
embeddings: value.data.into_iter().map(|e| e.embedding).collect(),
usage: wasi_llm::EmbeddingsUsage {
prompt_token_count: value.usage.prompt_tokens,
},
Expand Down
3 changes: 2 additions & 1 deletion crates/telemetry/src/logs.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,8 @@ pub fn handle_app_log(buf: &[u8], component_id: &str) {

/// Forward the app log to OTel.
fn app_log_to_otel(buf: &[u8], component_id: &str) {
if !otel_logs_enabled() {
static CELL: OnceLock<bool> = OnceLock::new();
if !*CELL.get_or_init(otel_logs_enabled) {
return;
}

Expand Down
16 changes: 9 additions & 7 deletions crates/trigger-http/src/server.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,8 @@ pub struct HttpServer<F: RuntimeFactors> {
router: Router,
/// The app being triggered.
trigger_app: Arc<TriggerApp<F>>,
/// The application name, resolved once for use as the `app_id` telemetry attribute.
app_id: String,
// Component ID -> component trigger config
component_trigger_configs: HashMap<spin_http::routes::TriggerLookupKey, HttpTriggerConfig>,
// Component ID -> handler type
Expand Down Expand Up @@ -158,6 +160,11 @@ impl<F: RuntimeFactors> HttpServer<F> {

let trigger_app = Arc::new(trigger_app);

let app_id = trigger_app
.app()
.get_metadata(APP_NAME_KEY)?
.unwrap_or_else(|| "<unnamed>".into());

let component_handler_types = component_trigger_configs
.iter()
.filter_map(|(key, trigger_config)| match key {
Expand All @@ -180,6 +187,7 @@ impl<F: RuntimeFactors> HttpServer<F> {
find_free_port,
router,
trigger_app,
app_id,
http1_max_buf_size,
component_trigger_configs,
component_handler_types,
Expand Down Expand Up @@ -350,18 +358,12 @@ impl<F: RuntimeFactors> HttpServer<F> {
client_addr: SocketAddr,
) -> anyhow::Result<Response<Body>> {
set_req_uri(&mut req, server_scheme)?;
let app_id = self
.trigger_app
.app()
.get_metadata(APP_NAME_KEY)?
.unwrap_or_else(|| "<unnamed>".into());

let lookup_key = route_match.lookup_key();

spin_telemetry::metrics::monotonic_counter!(
spin.request_count = 1,
trigger_type = "http",
app_id = app_id,
app_id = self.app_id.as_str(),
component_id = lookup_key.to_string()
);

Expand Down
Loading