fix: reinstall jobs not cleaning up if instance deleted (#6927)

Author: IMB11

packages/app-lib/.sqlx/query-15b4f72d367d329690f5daddafbdf0e51a285e35404053d62492e8df5fc7132f.json

@@ -77,6 +77,11 @@ pub async fn remove(instance_id: &str) -> crate::Result<()> {
     let instance =
         instance_rows::get_instance_display_info(instance_id, &state.pool)
             .await?;
+    crate::install::runner::cancel_jobs_for_instance_deletion(
+        instance_id,
+        &state,
+    )
+    .await?;
     crate::state::remove_instance(instance_id, &state).await?;
 
     if let Some(instance) = instance {

packages/app-lib/src/api/instance/lifecycle.rs

@@ -281,6 +281,38 @@ async fn recover_interrupted_job(
     if job.state.display.is_none() {
         job.state.display = display_from_request(&job.state);
     }
+
+    if let Some(instance_id) = target_instance_id(&job.state.target)
+        && instance_rows::get_instance_by_id(instance_id, &state.pool)
+            .await?
+            .is_none()
+    {
+        let canceled_phase = job.state.progress.phase;
+        job.state.error = Some(InstallErrorView::from_message(
+            "canceled",
+            canceled_phase,
+            "Install canceled because the instance was deleted",
+        ));
+        job.state.record_event(InstallJobEventKind::JobCanceled {
+            phase: canceled_phase,
+        });
+
+        if let Some(record) = store::finish_active(
+            job.id,
+            InstallJobStatus::Canceled,
+            &job.state,
+            state,
+        )
+        .await?
+        {
+            store::dismiss(job.id, state).await?;
+            clear_staging_dir(&job.state).await;
+            emit_install_job(&record.snapshot()).await?;
+        }
+
+        return Ok(());
+    }
+
     let interrupted_phase = job.state.progress.phase;
     job.state.record_event(InstallJobEventKind::Interrupted {
         reason: InstallInterruptReason::AppClosed,
@@ -341,6 +373,13 @@ async fn recover_interrupted_job(
     Ok(())
 }
 
+fn target_instance_id(target: &InstallTarget) -> Option<&str> {
+    match target {
+        InstallTarget::NewInstance { instance_id } => instance_id.as_deref(),
+        InstallTarget::ExistingInstance { instance_id } => Some(instance_id),
+    }
+}
+
 fn clear_deleted_new_instance_id(job_state: &mut InstallJobState) {
     if matches!(job_state.cleanup, InstallCleanup::DeleteNewInstance { .. }) {
         job_state.target = InstallTarget::NewInstance { instance_id: None };

packages/app-lib/src/install/recovery.rs

@@ -295,6 +295,39 @@ pub async fn cancel_job(job_id: Uuid) -> crate::Result<InstallJobSnapshot> {
     Ok(record.snapshot())
 }
 
+pub(crate) async fn cancel_jobs_for_instance_deletion(
+    instance_id: &str,
+    state: &State,
+) -> crate::Result<()> {
+    for mut job in store::list_active_for_instance(instance_id, state).await? {
+        let canceled_phase = job.state.progress.phase;
+        job.state.error = Some(InstallErrorView::from_message(
+            "canceled",
+            canceled_phase,
+            "Install canceled because the instance was deleted",
+        ));
+        job.state.record_event(InstallJobEventKind::JobCanceled {
+            phase: canceled_phase,
+        });
+
+        let Some(record) = store::finish_active(
+            job.id,
+            InstallJobStatus::Canceled,
+            &job.state,
+            state,
+        )
+        .await?
+        else {
+            continue;
+        };
+
+        store::dismiss(job.id, state).await?;
+        emit_install_job(&record.snapshot()).await?;
+    }
+
+    Ok(())
+}
+
 pub async fn dismiss_job(job_id: Uuid) -> crate::Result<()> {
     let state = State::get().await?;
     store::dismiss(job_id, &state).await
@@ -577,7 +610,11 @@ async fn run_job(job_id: Uuid) -> crate::Result<()> {
 
     let result = Box::pin(run_request(job_id, &mut job_state, &state)).await;
     if let Ok(record) = store::get_required(job_id, &state).await {
+        let status = record.status;
         job_state = record.state;
+        if status != InstallJobStatus::Running {
+            return Ok(());
+        }
     }
 
     let result = match result {

packages/app-lib/src/install/runner.rs

@@ -201,6 +201,17 @@ pub async fn list_interrupted_candidates(
     Ok(deserialize_rows(rows))
 }
 
+pub async fn list_active_for_instance(
+    instance_id: &str,
+    app_state: &State,
+) -> crate::Result<Vec<InstallJobRecord>> {
+    Ok(list_interrupted_candidates(app_state)
+        .await?
+        .into_iter()
+        .filter(|job| job.instance_id.as_deref() == Some(instance_id))
+        .collect())
+}
+
 pub async fn update_state(
     id: Uuid,
     state: &InstallJobState,
@@ -212,20 +223,30 @@ pub async fn update_state(
     let id_value = id.to_string();
     let modified = now.timestamp();
 
-    sqlx::query!(
+    let result = sqlx::query(
         "
 		UPDATE install_jobs
-		SET instance_id = ?, state = ?, modified = ?
-		WHERE id = ?
+		SET
+			instance_id = (SELECT id FROM instances WHERE id = ?),
+			state = ?,
+			modified = ?
+		WHERE id = ? AND status IN ('queued', 'running')
 		",
-        instance_id,
-        json,
-        modified,
-        id_value,
     )
+    .bind(instance_id)
+    .bind(json)
+    .bind(modified)
+    .bind(id_value)
     .execute(&app_state.pool)
     .await?;
 
+    if result.rows_affected() == 0 {
+        return Err(crate::ErrorKind::InputError(format!(
+            "Install job {id} is no longer active"
+        ))
+        .into());
+    }
+
     get_required(id, app_state).await
 }
 
@@ -319,7 +340,12 @@ pub async fn finish_active(
     let result = sqlx::query(
         "
 		UPDATE install_jobs
-		SET instance_id = ?, status = ?, state = ?, modified = ?, finished = ?
+		SET
+			instance_id = (SELECT id FROM instances WHERE id = ?),
+			status = ?,
+			state = ?,
+			modified = ?,
+			finished = ?
 		WHERE id = ? AND status IN ('queued', 'running')
 		",
     )
@@ -356,19 +382,24 @@ pub async fn complete_success(
     let mut transaction = app_state.pool.begin().await?;
 
     let job_result = sqlx::query(
-		"
+        "
 		UPDATE install_jobs
-		SET instance_id = ?, status = 'succeeded', state = ?, modified = ?, finished = ?
+		SET
+			instance_id = (SELECT id FROM instances WHERE id = ?),
+			status = 'succeeded',
+			state = ?,
+			modified = ?,
+			finished = ?
 		WHERE id = ? AND status = 'running'
 		",
-	)
-	.bind(&instance_id)
-	.bind(json)
-	.bind(now)
-	.bind(now)
-	.bind(id_value)
-	.execute(&mut *transaction)
-	.await?;
+    )
+    .bind(&instance_id)
+    .bind(json)
+    .bind(now)
+    .bind(now)
+    .bind(id_value)
+    .execute(&mut *transaction)
+    .await?;
 
     if job_result.rows_affected() == 0 {
         transaction.rollback().await?;