cli: automatically remove servers that fail to start (#211500)

Fixes https://github.com/microsoft/vscode-remote-release/issues/9823

cli/src/tunnels/code_server.rs

@@ -394,6 +394,16 @@ impl<'a> ServerBuilder<'a> {
 		}
 	}
 
+	/// Removes a cached server.
+	pub async fn evict(&self) -> Result<(), WrappedError> {
+		let name = get_server_folder_name(
+			self.server_params.release.quality,
+			&self.server_params.release.commit,
+		);
+
+		self.launcher_paths.server_cache.delete(&name)
+	}
+
 	/// Ensures the server is set up in the configured directory.
 	pub async fn setup(&self) -> Result<(), AnyError> {
 		debug!(
@@ -487,16 +497,16 @@ impl<'a> ServerBuilder<'a> {
 			monitor_server::<PortMatcher, u16>(child, Some(log_file), plog, false);
 
 		let port = match timeout(Duration::from_secs(8), listen_rx).await {
-			Err(e) => {
+			Err(_) => {
 				origin.kill().await;
-				Err(wrap(e, "timed out looking for port"))
+				return Err(CodeError::ServerOriginTimeout.into());
 			}
-			Ok(Err(e)) => {
+			Ok(Err(s)) => {
 				origin.kill().await;
-				Err(wrap(e, "server exited without writing port"))
+				return Err(CodeError::ServerUnexpectedExit(format!("{}", s)).into());
 			}
-			Ok(Ok(p)) => Ok(p),
-		}?;
+			Ok(Ok(p)) => p,
+		};
 
 		info!(self.logger, "Server started");
 
@@ -561,16 +571,16 @@ impl<'a> ServerBuilder<'a> {
 			monitor_server::<SocketMatcher, PathBuf>(child, Some(log_file), plog, false);
 
 		let socket = match timeout(Duration::from_secs(30), listen_rx).await {
-			Err(e) => {
+			Err(_) => {
 				origin.kill().await;
-				Err(wrap(e, "timed out looking for socket"))
+				return Err(CodeError::ServerOriginTimeout.into());
 			}
-			Ok(Err(e)) => {
+			Ok(Err(s)) => {
 				origin.kill().await;
-				Err(wrap(e, "server exited without writing socket"))
+				return Err(CodeError::ServerUnexpectedExit(format!("{}", s)).into());
 			}
-			Ok(Ok(socket)) => Ok(socket),
-		}?;
+			Ok(Ok(socket)) => socket,
+		};
 
 		info!(self.logger, "Server started");
 
@@ -581,25 +591,6 @@ impl<'a> ServerBuilder<'a> {
 		})
 	}
 
-	/// Starts with a given opaque set of args. Does not set up any port or
-	/// socket, but does return one if present, in the form of a channel.
-	pub async fn start_opaque_with_args<M, R>(
-		&self,
-		args: &[String],
-	) -> Result<(CodeServerOrigin, Receiver<R>), AnyError>
-	where
-		M: ServerOutputMatcher<R>,
-		R: 'static + Send + std::fmt::Debug,
-	{
-		let mut cmd = self.get_base_command();
-		cmd.args(args);
-
-		let child = self.spawn_server_process(cmd).await?;
-		let plog = self.logger.prefixed(&log::new_code_server_prefix());
-
-		Ok(monitor_server::<M, R>(child, None, plog, true))
-	}
-
 	async fn spawn_server_process(&self, mut cmd: Command) -> Result<Child, AnyError> {
 		info!(self.logger, "Starting server...");
 
@@ -625,7 +616,7 @@ impl<'a> ServerBuilder<'a> {
 			.stderr(std::process::Stdio::piped())
 			.stdout(std::process::Stdio::piped())
 			.spawn()
-			.map_err(|e| wrap(e, "error spawning server"))?;
+			.map_err(|e| CodeError::ServerUnexpectedExit(format!("{}", e)))?;
 
 		self.server_paths
 			.write_pid(child.id().expect("expected server to have pid"))?;

cli/src/tunnels/control_server.rs

@@ -760,17 +760,18 @@ async fn handle_serve(
 			macro_rules! do_setup {
 				($sb:expr) => {
 					match $sb.get_running().await? {
-						Some(AnyCodeServer::Socket(s)) => s,
+						Some(AnyCodeServer::Socket(s)) => ($sb, Ok(s)),
 						Some(_) => return Err(AnyError::from(MismatchedLaunchModeError())),
 						None => {
 							$sb.setup().await?;
-							$sb.listen_on_default_socket().await?
+							let r = $sb.listen_on_default_socket().await;
+							($sb, r)
 						}
 					}
 				};
 			}
 
-			let server = if params.use_local_download {
+			let (sb, server) = if params.use_local_download {
 				let sb = ServerBuilder::new(
 					&install_log,
 					&resolved,
@@ -784,6 +785,24 @@ async fn handle_serve(
 				do_setup!(sb)
 			};
 
+			let server = match server {
+				Ok(s) => s,
+				Err(e) => {
+					// we don't loop to avoid doing so infinitely: allow the client to reconnect in this case.
+					if let AnyError::CodeError(CodeError::ServerUnexpectedExit(ref e)) = e {
+						warning!(
+							c.log,
+							"({}), removing server due to possible corruptions",
+							e
+						);
+						if let Err(e) = sb.evict().await {
+							warning!(c.log, "Failed to evict server: {}", e);
+						}
+					}
+					return Err(e);
+				}
+			};
+
 			server_ref.replace(server.clone());
 			server
 		}

cli/src/util/errors.rs

@@ -516,6 +516,10 @@ pub enum CodeError {
 	CouldNotCreateConnectionTokenFile(std::io::Error),
 	#[error("A tunnel with the name {0} exists and is in-use. Please pick a different name or stop the existing tunnel.")]
 	TunnelActiveAndInUse(String),
+	#[error("Timed out looking for port/socket")]
+	ServerOriginTimeout,
+	#[error("Server exited without writing port/socket: {0}")]
+	ServerUnexpectedExit(String),
 }
 
 makeAnyError!(