fix(installer): recover Windows setup when node-deps host exits abruptly (#81390)
* fix(installer): retry abrupt stage host exits * fix(installer): preserve cancellation across stage retries
This commit is contained in:
parent
d753957e8a
commit
d0bb377c96
|
|
@ -26,6 +26,8 @@ use crate::install_script::{self, Pin, ScriptKind, ScriptSource};
|
|||
use crate::powershell::{self, StreamSink};
|
||||
use crate::AppState;
|
||||
|
||||
const MAX_STAGE_ATTEMPTS: usize = 3;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Public Tauri commands
|
||||
// ---------------------------------------------------------------------------
|
||||
|
|
@ -518,12 +520,13 @@ async fn run_bootstrap(
|
|||
manifest_args_full.push("-IncludeDesktop".to_string());
|
||||
}
|
||||
|
||||
let mut manifest_cancel_rx = None;
|
||||
let manifest_result = run_install_script(
|
||||
&app,
|
||||
&script.path,
|
||||
&manifest_args_full,
|
||||
args.hermes_home.as_deref(),
|
||||
None,
|
||||
&mut manifest_cancel_rx,
|
||||
Some("__manifest__".to_string()),
|
||||
)
|
||||
.await?;
|
||||
|
|
@ -621,19 +624,58 @@ async fn run_bootstrap(
|
|||
stage_args.push("-IncludeDesktop".to_string());
|
||||
}
|
||||
|
||||
// Each stage gets its own cancel receiver because tokio::select!
|
||||
// in run_script consumes it. Take/return through the Arc<Mutex>.
|
||||
let local_cancel_rx = cancel_rx_holder.lock().await.take();
|
||||
// A Windows PowerShell host can occasionally terminate with raw status
|
||||
// 0xffffffff while a long-running native child (npm / Playwright) is
|
||||
// still active. That bypasses install.ps1's finally block, so there is
|
||||
// no JSON frame to distinguish success from failure. Stage workers are
|
||||
// required to be idempotent; retry only this exact abrupt-host shape,
|
||||
// and keep it bounded so ordinary script failures remain immediate.
|
||||
let mut attempt = 1;
|
||||
let mut local_cancel_rx = cancel_rx_holder.lock().await.take();
|
||||
let (stage_result, result_frame) = loop {
|
||||
let mut result = run_install_script(
|
||||
&app,
|
||||
&script.path,
|
||||
&stage_args,
|
||||
args.hermes_home.as_deref(),
|
||||
&mut local_cancel_rx,
|
||||
Some(stage.name.clone()),
|
||||
)
|
||||
.await?;
|
||||
let frame = powershell::parse_stage_result(&result.stdout);
|
||||
|
||||
let stage_result = run_install_script(
|
||||
&app,
|
||||
&script.path,
|
||||
&stage_args,
|
||||
args.hermes_home.as_deref(),
|
||||
local_cancel_rx,
|
||||
Some(stage.name.clone()),
|
||||
)
|
||||
.await?;
|
||||
if should_retry_missing_stage_frame(result.exit_code, result.killed, attempt)
|
||||
&& frame.is_none()
|
||||
{
|
||||
if retry_backoff_cancelled(local_cancel_rx.as_mut()).await {
|
||||
result.killed = true;
|
||||
break (result, frame);
|
||||
}
|
||||
attempt += 1;
|
||||
let line = format!(
|
||||
"[bootstrap] {} stage host exited unexpectedly before its JSON result; retrying ({attempt}/{MAX_STAGE_ATTEMPTS})",
|
||||
stage.name
|
||||
);
|
||||
tracing::warn!(
|
||||
stage = %stage.name,
|
||||
exit = ?result.exit_code,
|
||||
attempt,
|
||||
"stage host exited without a result frame; retrying"
|
||||
);
|
||||
emit_event(
|
||||
&app,
|
||||
BootstrapEvent::Log {
|
||||
stage: Some(stage.name.clone()),
|
||||
line,
|
||||
stream: LogStream::Stderr,
|
||||
},
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
break (result, frame);
|
||||
};
|
||||
*cancel_rx_holder.lock().await = local_cancel_rx;
|
||||
|
||||
let duration_ms = started.elapsed().as_millis() as u64;
|
||||
|
||||
|
|
@ -658,8 +700,6 @@ async fn run_bootstrap(
|
|||
return Err(anyhow!("cancelled by user"));
|
||||
}
|
||||
|
||||
let result_frame = powershell::parse_stage_result(&stage_result.stdout);
|
||||
|
||||
match result_frame {
|
||||
None => {
|
||||
let err = format!(
|
||||
|
|
@ -785,6 +825,31 @@ async fn run_bootstrap(
|
|||
Ok(install_root.to_string_lossy().into_owned())
|
||||
}
|
||||
|
||||
fn should_retry_missing_stage_frame(
|
||||
exit_code: Option<i32>,
|
||||
killed: bool,
|
||||
attempt: usize,
|
||||
) -> bool {
|
||||
!killed && exit_code == Some(-1) && attempt < MAX_STAGE_ATTEMPTS
|
||||
}
|
||||
|
||||
async fn retry_backoff_cancelled(cancel_rx: Option<&mut mpsc::Receiver<()>>) -> bool {
|
||||
let backoff = tokio::time::sleep(std::time::Duration::from_millis(500));
|
||||
tokio::pin!(backoff);
|
||||
|
||||
match cancel_rx {
|
||||
Some(rx) => tokio::select! {
|
||||
biased;
|
||||
signal = rx.recv() => signal.is_some(),
|
||||
_ = &mut backoff => false,
|
||||
},
|
||||
None => {
|
||||
backoff.await;
|
||||
false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn cancellation_signalled(holder: &Arc<Mutex<Option<mpsc::Receiver<()>>>>) -> bool {
|
||||
let mut guard = holder.lock().await;
|
||||
if let Some(rx) = guard.as_mut() {
|
||||
|
|
@ -799,7 +864,7 @@ async fn run_install_script(
|
|||
script_path: &std::path::Path,
|
||||
args: &[String],
|
||||
hermes_home_override: Option<&str>,
|
||||
cancel_rx: Option<mpsc::Receiver<()>>,
|
||||
cancel_rx: &mut Option<mpsc::Receiver<()>>,
|
||||
stage_name: Option<String>,
|
||||
) -> Result<powershell::ScriptResult> {
|
||||
let app_for_stdout = app.clone();
|
||||
|
|
@ -1117,4 +1182,30 @@ mod tests {
|
|||
);
|
||||
let _ = std::fs::remove_dir_all(&base);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn abrupt_windows_stage_exit_is_retried_but_never_forever() {
|
||||
assert!(should_retry_missing_stage_frame(Some(-1), false, 1));
|
||||
assert!(should_retry_missing_stage_frame(Some(-1), false, 2));
|
||||
assert!(
|
||||
!should_retry_missing_stage_frame(Some(-1), false, MAX_STAGE_ATTEMPTS),
|
||||
"the retry policy must stay bounded"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ordinary_failure_or_cancellation_is_not_retried_without_a_frame() {
|
||||
assert!(!should_retry_missing_stage_frame(Some(1), false, 1));
|
||||
assert!(!should_retry_missing_stage_frame(Some(0), false, 1));
|
||||
assert!(!should_retry_missing_stage_frame(None, false, 1));
|
||||
assert!(!should_retry_missing_stage_frame(Some(-1), true, 1));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cancellation_during_retry_backoff_stops_the_retry() {
|
||||
let (tx, mut rx) = mpsc::channel(1);
|
||||
tx.send(()).await.unwrap();
|
||||
|
||||
assert!(retry_backoff_cancelled(Some(&mut rx)).await);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -138,7 +138,7 @@ pub async fn run_script(
|
|||
args: &[String],
|
||||
sink: StreamSink,
|
||||
hermes_home_override: Option<&str>,
|
||||
mut cancel_rx: Option<CancelRx>,
|
||||
cancel_rx: &mut Option<CancelRx>,
|
||||
) -> Result<ScriptResult> {
|
||||
let mut cmd = build_command(script_path, args);
|
||||
|
||||
|
|
@ -221,7 +221,7 @@ pub async fn run_script(
|
|||
}
|
||||
}
|
||||
}
|
||||
_ = recv_cancel(&mut cancel_rx) => {
|
||||
_ = recv_cancel(cancel_rx) => {
|
||||
tracing::warn!("cancellation received — killing child");
|
||||
killed = true;
|
||||
// best-effort kill; don't propagate errors
|
||||
|
|
|
|||
Loading…
Reference in New Issue