cli: implement 'server of server' for a local web server (#191014)

Closes https://github.com/microsoft/vscode/issues/168492

This implements @aeschli's 'server server' concept in a new
`code serve-web` command.

Command line args are similar to the standalone web server. The first
time a user hits that page, the latest version of the VS Code web server
will be downloaded and run. Thanks to Martin's previous PRs, all
resources the page requests are prefixed with `/<quality-<commit>`.

The latest release version is cached, but when the page is loaded again
and there's a new release, a the new server version will be downloaded
and started up.

Behind the scenes the servers all listen on named pipes/sockets and the
CLI acts as a proxy server to those sockets. Servers without connections
for an hour will be shut down automatically.

.vscode/shared.code-snippets

@@ -6,7 +6,7 @@
 	// Placeholders with the same ids are connected.
 	// Example:
 	"MSFT Copyright Header": {
-		"scope": "javascript,typescript,css",
+		"scope": "javascript,typescript,css,rust",
 		"prefix": [
 			"header",
 			"stub",

cli/Cargo.lock

@@ -983,9 +983,9 @@ dependencies = [
 
 [[package]]
 name = "http"
-version = "0.2.8"
+version = "0.2.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "75f43d41e26995c17e71ee126451dd3941010b0514a81a9d11f3b341debc2399"
+checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482"
 dependencies = [
  "bytes",
  "fnv",

cli/Cargo.toml

@@ -37,7 +37,7 @@ libc = "0.2.144"
 tunnels = { git = "https://github.com/microsoft/dev-tunnels", rev = "2621784a9ad72aa39500372391332a14bad581a3", default-features = false, features = ["connections"] }
 keyring = { version = "2.0.3", default-features = false, features = ["linux-secret-service-rt-tokio-crypto-openssl"] }
 dialoguer = "0.10.4"
-hyper = "0.14.26"
+hyper = { version = "0.14.26", features = ["server", "http1", "runtime"] }
 indicatif = "0.17.4"
 tempfile = "3.5.0"
 clap_lex = "0.5.0"

cli/src/bin/code/main.rs

@@ -8,7 +8,7 @@ use std::process::Command;
 
 use clap::Parser;
 use cli::{
-	commands::{args, tunnels, update, version, CommandContext},
+	commands::{args, serve_web, tunnels, update, version, CommandContext},
 	constants::get_default_user_agent,
 	desktop, log,
 	state::LauncherPaths,
@@ -99,6 +99,10 @@ async fn main() -> Result<(), std::convert::Infallible> {
 				tunnels::command_shell(context!(), cs_args).await
 			}
 
+			Some(args::Commands::ServeWeb(sw_args)) => {
+				serve_web::serve_web(context!(), sw_args).await
+			}
+
 			Some(args::Commands::Tunnel(tunnel_args)) => match tunnel_args.subcommand {
 				Some(args::TunnelSubcommand::Prune) => tunnels::prune(context!()).await,
 				Some(args::TunnelSubcommand::Unregister) => tunnels::unregister(context!()).await,

cli/src/commands.rs

@@ -9,4 +9,5 @@ pub mod args;
 pub mod tunnels;
 pub mod update;
 pub mod version;
+pub mod serve_web;
 pub use context::CommandContext;

cli/src/commands/args.rs

@@ -172,11 +172,42 @@ pub enum Commands {
 	/// Changes the version of the editor you're using.
 	Version(VersionArgs),
 
+	/// Runs a local web version of VS Code.
+	ServeWeb(ServeWebArgs),
+
 	/// Runs the control server on process stdin/stdout
 	#[clap(hide = true)]
 	CommandShell(CommandShellArgs),
 }
 
+#[derive(Args, Debug, Clone)]
+pub struct ServeWebArgs {
+	/// Host to listen on, defaults to 'localhost'
+	#[clap(long)]
+	pub host: Option<String>,
+	/// Port to listen on. If 0 is passed a random free port is picked.
+	#[clap(long, default_value_t = 8000)]
+	pub port: u16,
+	/// A secret that must be included with all requests.
+	#[clap(long)]
+	pub connection_token: Option<String>,
+	/// Run without a connection token. Only use this if the connection is secured by other means.
+	#[clap(long)]
+	pub without_connection_token: bool,
+	/// If set, the user accepts the server license terms and the server will be started without a user prompt.
+	#[clap(long)]
+	pub accept_server_license_terms: bool,
+	/// Specifies the directory that server data is kept in.
+	#[clap(long)]
+	pub server_data_dir: Option<String>,
+	/// Specifies the directory that user data is kept in. Can be used to open multiple distinct instances of Code.
+	#[clap(long)]
+	pub user_data_dir: Option<String>,
+	/// Set the root path for extensions.
+	#[clap(long)]
+	pub extensions_dir: Option<String>,
+}
+
 #[derive(Args, Debug, Clone)]
 pub struct CommandShellArgs {
 	/// Listen on a socket instead of stdin/stdout.

cli/src/commands/serve_web.rs

@@ -0,0 +1,617 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+use std::collections::HashMap;
+use std::convert::Infallible;
+use std::net::{IpAddr, Ipv4Addr, SocketAddr};
+use std::path::PathBuf;
+use std::sync::{Arc, Mutex};
+use std::time::{Duration, Instant};
+
+use hyper::service::{make_service_fn, service_fn};
+use hyper::{Body, Request, Response, Server};
+use tokio::io::{AsyncBufReadExt, BufReader};
+use tokio::pin;
+use tokio::process::Command;
+
+use crate::async_pipe::{get_socket_name, get_socket_rw_stream, AsyncPipe};
+use crate::constants::VSCODE_CLI_QUALITY;
+use crate::download_cache::DownloadCache;
+use crate::log;
+use crate::options::Quality;
+use crate::update_service::{
+	unzip_downloaded_release, Platform, Release, TargetKind, UpdateService,
+};
+use crate::util::errors::AnyError;
+use crate::util::http::{self, ReqwestSimpleHttp};
+use crate::util::io::SilentCopyProgress;
+use crate::util::sync::{new_barrier, Barrier, BarrierOpener};
+use crate::{
+	tunnels::legal,
+	util::{errors::CodeError, prereqs::PreReqChecker},
+};
+
+use super::{args::ServeWebArgs, CommandContext};
+
+/// Length of a commit hash, for validation
+const COMMIT_HASH_LEN: usize = 40;
+/// Number of seconds where, if there's no connections to a VS Code server,
+/// the server is shut down.
+const SERVER_IDLE_TIMEOUT_SECS: u64 = 60 * 60;
+/// Number of seconds in which the server times out when there is a connection
+/// (should be large enough to basically never happen)
+const SERVER_ACTIVE_TIMEOUT_SECS: u64 = SERVER_IDLE_TIMEOUT_SECS * 24 * 30 * 12;
+/// How long to cache the "latest" version we get from the update service.
+const RELEASE_CACHE_SECS: u64 = 60 * 60;
+
+/// Implements the vscode "server of servers". Clients who go to the URI get
+/// served the latest version of the VS Code server whenever they load the
+/// page. The VS Code server prefixes all assets and connections it loads with
+/// its version string, so existing clients can continue to get served even
+/// while new clients get new VS Code Server versions.
+pub async fn serve_web(ctx: CommandContext, mut args: ServeWebArgs) -> Result<i32, AnyError> {
+	legal::require_consent(&ctx.paths, args.accept_server_license_terms)?;
+	let mut addr: SocketAddr = match &args.host {
+		Some(h) => h.parse().map_err(CodeError::InvalidHostAddress)?,
+		None => SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 0),
+	};
+	addr.set_port(args.port);
+
+	let platform: crate::update_service::Platform = PreReqChecker::new().verify().await?;
+
+	if !args.without_connection_token {
+		// Ensure there's a defined connection token, since if multiple server versions
+		// are excuted, they will need to have a single shared token.
+		let connection_token = args
+			.connection_token
+			.clone()
+			.unwrap_or_else(|| uuid::Uuid::new_v4().to_string());
+		ctx.log.result(format!(
+			"Web UI available at http://{}?tkn={}",
+			addr, connection_token,
+		));
+		args.connection_token = Some(connection_token);
+	} else {
+		ctx.log
+			.result(format!("Web UI available at http://{}", addr));
+		args.connection_token = None;
+	}
+
+	let cm = ConnectionManager::new(&ctx, platform, args);
+	let make_svc = make_service_fn(move |_conn| {
+		let cm = cm.clone();
+		let log = ctx.log.clone();
+		let service = service_fn(move |req| handle(cm.clone(), log.clone(), req));
+		async move { Ok::<_, Infallible>(service) }
+	});
+
+	let server = Server::bind(&addr).serve(make_svc);
+
+	server.await.map_err(CodeError::CouldNotListenOnInterface)?;
+
+	Ok(0)
+}
+
+/// Handler function for an inbound request
+async fn handle(
+	cm: Arc<ConnectionManager>,
+	log: log::Logger,
+	req: Request<Body>,
+) -> Result<Response<Body>, Infallible> {
+	let release = if let Some((r, _)) = get_release_from_path(req.uri().path(), cm.platform) {
+		r
+	} else {
+		match cm.get_latest_release().await {
+			Ok(r) => r,
+			Err(e) => {
+				error!(log, "error getting latest version: {}", e);
+				return Ok(response::code_err(e));
+			}
+		}
+	};
+
+	Ok(match cm.get_connection(release).await {
+		Ok(rw) => {
+			if req.headers().contains_key(hyper::header::UPGRADE) {
+				forward_ws_req_to_server(cm.log.clone(), rw, req).await
+			} else {
+				forward_http_req_to_server(rw, req).await
+			}
+		}
+		Err(CodeError::ServerNotYetDownloaded) => response::wait_for_download(),
+		Err(e) => response::code_err(e),
+	})
+}
+
+/// Gets the release info from the VS Code path prefix, which is in the
+/// format `/<quality>-<commit>/...`
+fn get_release_from_path(path: &str, platform: Platform) -> Option<(Release, String)> {
+	if !path.starts_with('/') {
+		return None; // paths must start with '/'
+	}
+
+	let path = &path[1..];
+	let i = path.find('/').unwrap_or(path.len());
+	let quality_commit_sep = path.get(..i).and_then(|p| p.find('-'))?;
+
+	let (quality_commit, remaining) = path.split_at(i);
+	let (quality, commit) = quality_commit.split_at(quality_commit_sep);
+
+	if !is_commit_hash(commit) {
+		return None;
+	}
+
+	Some((
+		Release {
+			// remember to trim off the leading '/' which is now part of th quality
+			quality: Quality::try_from(quality).ok()?,
+			commit: commit.to_string(),
+			platform,
+			target: TargetKind::Web,
+			name: "".to_string(),
+		},
+		remaining.to_string(),
+	))
+}
+
+/// Proxies the standard HTTP request to the async pipe, returning the piped response
+async fn forward_http_req_to_server(
+	(rw, handle): (AsyncPipe, ConnectionHandle),
+	req: Request<Body>,
+) -> Response<Body> {
+	let (mut request_sender, connection) =
+		match hyper::client::conn::Builder::new().handshake(rw).await {
+			Ok(r) => r,
+			Err(e) => return response::connection_err(e),
+		};
+
+	tokio::spawn(connection);
+
+	let res = request_sender
+		.send_request(req)
+		.await
+		.unwrap_or_else(response::connection_err);
+
+	// technically, we should buffer the body into memory since it may not be
+	// read at this point, but because the keepalive time is very large
+	// there's not going to be responses that take hours to send and x
+	// cause us to kill the server before the response is sent
+	drop(handle);
+
+	res
+}
+
+/// Proxies the websocket request to the async pipe
+async fn forward_ws_req_to_server(
+	log: log::Logger,
+	(rw, handle): (AsyncPipe, ConnectionHandle),
+	mut req: Request<Body>,
+) -> Response<Body> {
+	// splicing of client and servers inspired by https://github.com/hyperium/hyper/blob/fece9f7f50431cf9533cfe7106b53a77b48db699/examples/upgrades.rs
+	let (mut request_sender, connection) =
+		match hyper::client::conn::Builder::new().handshake(rw).await {
+			Ok(r) => r,
+			Err(e) => return response::connection_err(e),
+		};
+
+	tokio::spawn(connection);
+
+	let mut proxied_req = Request::builder().uri(req.uri());
+	for (k, v) in req.headers() {
+		proxied_req = proxied_req.header(k, v);
+	}
+
+	let mut res = request_sender
+		.send_request(proxied_req.body(Body::empty()).unwrap())
+		.await
+		.unwrap_or_else(response::connection_err);
+
+	let mut proxied_res = Response::new(Body::empty());
+	*proxied_res.status_mut() = res.status();
+	for (k, v) in res.headers() {
+		proxied_res.headers_mut().insert(k, v.clone());
+	}
+
+	// only start upgrade at this point in case the server decides to deny socket
+	if res.status() == hyper::StatusCode::SWITCHING_PROTOCOLS {
+		tokio::spawn(async move {
+			let (s_req, s_res) =
+				tokio::join!(hyper::upgrade::on(&mut req), hyper::upgrade::on(&mut res));
+
+			match (s_req, s_res) {
+				(Err(e1), Err(e2)) => debug!(
+					log,
+					"client ({}) and server ({}) websocket upgrade failed", e1, e2
+				),
+				(Err(e1), _) => debug!(log, "client ({}) websocket upgrade failed", e1),
+				(_, Err(e2)) => debug!(log, "server ({}) websocket upgrade failed", e2),
+				(Ok(mut s_req), Ok(mut s_res)) => {
+					trace!(log, "websocket upgrade succeeded");
+					let r = tokio::io::copy_bidirectional(&mut s_req, &mut s_res).await;
+					trace!(log, "websocket closed (error: {:?})", r.err());
+				}
+			}
+
+			drop(handle);
+		});
+	}
+
+	proxied_res
+}
+
+/// Returns whether the string looks like a commit hash.
+fn is_commit_hash(s: &str) -> bool {
+	s.len() == COMMIT_HASH_LEN && s.chars().all(|c| c.is_ascii_hexdigit())
+}
+
+/// Module holding original responses the CLI's server makes.
+mod response {
+	use const_format::concatcp;
+
+	use crate::constants::QUALITYLESS_SERVER_NAME;
+
+	use super::*;
+
+	pub fn connection_err(err: hyper::Error) -> Response<Body> {
+		Response::builder()
+			.status(503)
+			.body(Body::from(format!("Error connecting to server: {:?}", err)))
+			.unwrap()
+	}
+
+	pub fn code_err(err: CodeError) -> Response<Body> {
+		Response::builder()
+			.status(500)
+			.body(Body::from(format!("Error serving request: {}", err)))
+			.unwrap()
+	}
+
+	pub fn wait_for_download() -> Response<Body> {
+		Response::builder()
+			.status(202)
+			.header("Content-Type", "text/html") // todo: get latest
+			.body(Body::from(concatcp!("The latest version of the ", QUALITYLESS_SERVER_NAME, " is downloading, please wait a moment...<script>setTimeout(()=>location.reload(),1500)</script>", )))
+			.unwrap()
+	}
+}
+
+/// Handle returned when getting a stream to the server, used to refcount
+/// connections to a server so it can be disposed when there are no more clients.
+struct ConnectionHandle {
+	client_counter: Arc<tokio::sync::watch::Sender<usize>>,
+}
+
+impl ConnectionHandle {
+	pub fn new(client_counter: Arc<tokio::sync::watch::Sender<usize>>) -> Self {
+		client_counter.send_modify(|v| {
+			*v += 1;
+		});
+		Self { client_counter }
+	}
+}
+
+impl Drop for ConnectionHandle {
+	fn drop(&mut self) {
+		self.client_counter.send_modify(|v| {
+			*v -= 1;
+		});
+	}
+}
+
+type StartData = (PathBuf, Arc<tokio::sync::watch::Sender<usize>>);
+
+/// State stored in the ConnectionManager for each server version.
+struct VersionState {
+	downloaded: bool,
+	socket_path: Barrier<Result<StartData, String>>,
+}
+
+type ConnectionStateMap = Arc<Mutex<HashMap<(Quality, String), VersionState>>>;
+
+/// Manages the connections to running web UI instances. Multiple web servers
+/// can run concurrently, with routing based on the URL path.
+struct ConnectionManager {
+	pub platform: Platform,
+	pub log: log::Logger,
+	args: ServeWebArgs,
+	/// Cache where servers are stored
+	cache: DownloadCache,
+	/// Mapping of (Quality, Commit) to the state each server is in
+	state: ConnectionStateMap,
+	/// Update service instance
+	update_service: UpdateService,
+	/// Cache of the latest released version, storing the time we checked as well
+	latest_version: tokio::sync::Mutex<Option<(Instant, Release)>>,
+}
+
+fn key_for_release(release: &Release) -> (Quality, String) {
+	(release.quality, release.commit.clone())
+}
+
+impl ConnectionManager {
+	pub fn new(ctx: &CommandContext, platform: Platform, args: ServeWebArgs) -> Arc<Self> {
+		Arc::new(Self {
+			platform,
+			args,
+			log: ctx.log.clone(),
+			cache: DownloadCache::new(ctx.paths.web_server_storage()),
+			update_service: UpdateService::new(
+				ctx.log.clone(),
+				Arc::new(ReqwestSimpleHttp::with_client(ctx.http.clone())),
+			),
+			state: ConnectionStateMap::default(),
+			latest_version: tokio::sync::Mutex::default(),
+		})
+	}
+
+	/// Gets a connection to a server version
+	pub async fn get_connection(
+		&self,
+		release: Release,
+	) -> Result<(AsyncPipe, ConnectionHandle), CodeError> {
+		// todo@connor4312: there is likely some performance benefit to
+		// implementing a 'keepalive' for these connections.
+		let (path, counter) = self.get_version_data(release).await?;
+		let handle = ConnectionHandle::new(counter);
+		let rw = get_socket_rw_stream(&path).await?;
+		Ok((rw, handle))
+	}
+
+	/// Gets the latest release for the CLI quality, caching its result for some
+	/// time to allow for fast loads.
+	pub async fn get_latest_release(&self) -> Result<Release, CodeError> {
+		let mut latest = self.latest_version.lock().await;
+		let now = Instant::now();
+		if let Some((checked_at, release)) = &*latest {
+			if checked_at.elapsed() < Duration::from_secs(RELEASE_CACHE_SECS) {
+				return Ok(release.clone());
+			}
+		}
+
+		let quality = VSCODE_CLI_QUALITY
+			.ok_or_else(|| CodeError::UpdatesNotConfigured("no configured quality"))
+			.and_then(|q| {
+				Quality::try_from(q).map_err(|_| CodeError::UpdatesNotConfigured("unknown quality"))
+			})?;
+
+		let release = self
+			.update_service
+			.get_latest_commit(self.platform, TargetKind::Web, quality)
+			.await
+			.map_err(|e| CodeError::UpdateCheckFailed(e.to_string()));
+
+		// If the update service is unavailable and we have stale data, use that
+		if let (Err(e), Some((_, previous))) = (&release, &*latest) {
+			warning!(self.log, "error getting latest release, using stale: {}", e);
+			return Ok(previous.clone());
+		}
+
+		let release = release?;
+		debug!(self.log, "refreshed latest release: {}", release);
+		*latest = Some((now, release.clone()));
+
+		Ok(release)
+	}
+
+	/// Gets the StartData for the a version of the VS Code server, triggering
+	/// download/start if necessary. It returns `CodeError::ServerNotYetDownloaded`
+	/// while the server is downloading, which is used to have a refresh loop on the page.
+	async fn get_version_data(&self, release: Release) -> Result<StartData, CodeError> {
+		self.get_version_data_inner(release)?
+			.wait()
+			.await
+			.unwrap()
+			.map_err(CodeError::ServerDownloadError)
+	}
+
+	fn get_version_data_inner(
+		&self,
+		release: Release,
+	) -> Result<Barrier<Result<StartData, String>>, CodeError> {
+		let mut state = self.state.lock().unwrap();
+		let key = key_for_release(&release);
+		if let Some(s) = state.get_mut(&key) {
+			if !s.downloaded {
+				if s.socket_path.is_open() {
+					s.downloaded = true;
+				} else {
+					return Err(CodeError::ServerNotYetDownloaded);
+				}
+			}
+
+			return Ok(s.socket_path.clone());
+		}
+
+		let (socket_path, opener) = new_barrier();
+		let state_map_dup = self.state.clone();
+		let args = StartArgs {
+			args: self.args.clone(),
+			log: self.log.clone(),
+			opener,
+			release,
+		};
+
+		if let Some(p) = self.cache.exists(&args.release.commit) {
+			state.insert(
+				key.clone(),
+				VersionState {
+					socket_path: socket_path.clone(),
+					downloaded: true,
+				},
+			);
+
+			tokio::spawn(async move {
+				Self::start_version(args, p).await;
+				state_map_dup.lock().unwrap().remove(&key);
+			});
+			Ok(socket_path)
+		} else {
+			state.insert(
+				key.clone(),
+				VersionState {
+					socket_path,
+					downloaded: false,
+				},
+			);
+			let update_service = self.update_service.clone();
+			let cache = self.cache.clone();
+			tokio::spawn(async move {
+				Self::download_version(args, update_service.clone(), cache.clone()).await;
+				state_map_dup.lock().unwrap().remove(&key);
+			});
+			Err(CodeError::ServerNotYetDownloaded)
+		}
+	}
+
+	/// Downloads a server version into the cache and starts it.
+	async fn download_version(
+		args: StartArgs,
+		update_service: UpdateService,
+		cache: DownloadCache,
+	) {
+		let release_for_fut = args.release.clone();
+		let log_for_fut = args.log.clone();
+		let dir_fut = cache.create(&args.release.commit, |target_dir| async move {
+			info!(log_for_fut, "Downloading server {}", release_for_fut.commit);
+			let tmpdir = tempfile::tempdir().unwrap();
+			let response = update_service.get_download_stream(&release_for_fut).await?;
+
+			let name = response.url_path_basename().unwrap();
+			let archive_path = tmpdir.path().join(name);
+			http::download_into_file(
+				&archive_path,
+				log_for_fut.get_download_logger("Downloading server:"),
+				response,
+			)
+			.await?;
+			unzip_downloaded_release(&archive_path, &target_dir, SilentCopyProgress())?;
+			Ok(())
+		});
+
+		match dir_fut.await {
+			Err(e) => args.opener.open(Err(e.to_string())),
+			Ok(dir) => Self::start_version(args, dir).await,
+		}
+	}
+
+	/// Starts a downloaded server that can be found in the given `path`.
+	async fn start_version(args: StartArgs, path: PathBuf) {
+		info!(args.log, "Starting server {}", args.release.commit);
+
+		let executable = path
+			.join("bin")
+			.join(args.release.quality.server_entrypoint());
+		let socket_path = get_socket_name();
+
+		#[cfg(not(windows))]
+		let mut cmd = Command::new(&executable);
+		#[cfg(windows)]
+		let mut cmd = {
+			let mut cmd = Command::new("cmd");
+			cmd.arg("/Q");
+			cmd.arg("/C");
+			cmd.arg(&executable);
+			cmd
+		};
+
+		cmd.stdin(std::process::Stdio::null());
+		cmd.stderr(std::process::Stdio::piped());
+		cmd.stdout(std::process::Stdio::piped());
+		cmd.arg("--socket-path");
+		cmd.arg(&socket_path);
+
+		// License agreement already checked by the `server_web` function.
+		cmd.args(["--accept-server-license-terms"]);
+
+		if let Some(a) = &args.args.server_data_dir {
+			cmd.arg("--server-data-dir");
+			cmd.arg(a);
+		}
+		if let Some(a) = &args.args.user_data_dir {
+			cmd.arg("--user-data-dir");
+			cmd.arg(a);
+		}
+		if let Some(a) = &args.args.extensions_dir {
+			cmd.arg("--extensions-dir");
+			cmd.arg(a);
+		}
+		if args.args.without_connection_token {
+			cmd.arg("--without-connection-token");
+		}
+		if let Some(ct) = &args.args.connection_token {
+			cmd.arg("--connection-token");
+			cmd.arg(ct);
+		}
+
+		// removed, otherwise the workbench will not be usable when running the CLI from sources.
+		cmd.env_remove("VSCODE_DEV");
+
+		let mut child = match cmd.spawn() {
+			Ok(c) => c,
+			Err(e) => {
+				args.opener.open(Err(e.to_string()));
+				return;
+			}
+		};
+
+		let (mut stdout, mut stderr) = (
+			BufReader::new(child.stdout.take().unwrap()).lines(),
+			BufReader::new(child.stderr.take().unwrap()).lines(),
+		);
+
+		// wrapped option to prove that we only use this once in the loop
+		let (counter_tx, mut counter_rx) = tokio::sync::watch::channel(0);
+		let mut opener = Some((args.opener, socket_path, Arc::new(counter_tx)));
+		let commit_prefix = &args.release.commit[..7];
+		let kill_timer = tokio::time::sleep(Duration::from_secs(SERVER_IDLE_TIMEOUT_SECS));
+		pin!(kill_timer);
+
+		loop {
+			tokio::select! {
+				Ok(Some(l)) = stdout.next_line() => {
+					info!(args.log, "[{} stdout]: {}", commit_prefix, l);
+
+					if l.contains("Server bound to") {
+						if let Some((opener, path, counter_tx)) = opener.take() {
+							opener.open(Ok((path, counter_tx)));
+						}
+					}
+				}
+				Ok(Some(l)) = stderr.next_line() => {
+					info!(args.log, "[{} stderr]: {}", commit_prefix, l);
+				},
+				n = counter_rx.changed() => {
+					kill_timer.as_mut().reset(match n {
+						// err means that the record was dropped
+						Err(_) => tokio::time::Instant::now(),
+						Ok(_) => {
+							if *counter_rx.borrow() == 0 {
+								tokio::time::Instant::now() + Duration::from_secs(SERVER_IDLE_TIMEOUT_SECS)
+							} else {
+								tokio::time::Instant::now() + Duration::from_secs(SERVER_ACTIVE_TIMEOUT_SECS)
+							}
+						}
+					});
+				}
+				_ = &mut kill_timer => {
+					info!(args.log, "[{} process]: idle timeout reached, ending", commit_prefix);
+					let _ = child.kill().await;
+					break;
+				}
+				e = child.wait() => {
+					info!(args.log, "[{} process]: exited: {:?}", commit_prefix, e);
+					break;
+				}
+			}
+		}
+	}
+}
+
+struct StartArgs {
+	log: log::Logger,
+	args: ServeWebArgs,
+	release: Release,
+	opener: BarrierOpener<Result<StartData, String>>,
+}

cli/src/self_update.rs

@@ -11,7 +11,7 @@ use crate::{
 	options::Quality,
 	update_service::{unzip_downloaded_release, Platform, Release, TargetKind, UpdateService},
 	util::{
-		errors::{wrap, AnyError, CorruptDownload, UpdatesNotConfigured},
+		errors::{wrap, AnyError, CodeError, CorruptDownload},
 		http,
 		io::{ReportCopyProgress, SilentCopyProgress},
 	},
@@ -27,14 +27,16 @@ pub struct SelfUpdate<'a> {
 impl<'a> SelfUpdate<'a> {
 	pub fn new(update_service: &'a UpdateService) -> Result<Self, AnyError> {
 		let commit = VSCODE_CLI_COMMIT
-			.ok_or_else(|| UpdatesNotConfigured("unknown build commit".to_string()))?;
+			.ok_or_else(|| CodeError::UpdatesNotConfigured("unknown build commit"))?;
 
 		let quality = VSCODE_CLI_QUALITY
-			.ok_or_else(|| UpdatesNotConfigured("no configured quality".to_string()))
-			.and_then(|q| Quality::try_from(q).map_err(UpdatesNotConfigured))?;
+			.ok_or_else(|| CodeError::UpdatesNotConfigured("no configured quality"))
+			.and_then(|q| {
+				Quality::try_from(q).map_err(|_| CodeError::UpdatesNotConfigured("unknown quality"))
+			})?;
 
 		let platform = Platform::env_default().ok_or_else(|| {
-			UpdatesNotConfigured("Unknown platform, please report this error".to_string())
+			CodeError::UpdatesNotConfigured("Unknown platform, please report this error")
 		})?;
 
 		Ok(Self {

cli/src/state.rs

@@ -212,4 +212,10 @@ impl LauncherPaths {
 			)
 		})
 	}
+
+	/// Suggested path for web server storage
+	pub fn web_server_storage(&self) -> PathBuf {
+		self.root.join("serve-web")
+	}
+
 }

cli/src/tunnels/legal.rs

@@ -2,9 +2,9 @@
  *  Copyright (c) Microsoft Corporation. All rights reserved.
  *  Licensed under the MIT License. See License.txt in the project root for license information.
  *--------------------------------------------------------------------------------------------*/
-use crate::constants::{IS_INTERACTIVE_CLI, PRODUCT_NAME_LONG};
+use crate::constants::IS_INTERACTIVE_CLI;
 use crate::state::{LauncherPaths, PersistedState};
-use crate::util::errors::{AnyError, MissingLegalConsent};
+use crate::util::errors::{AnyError, CodeError};
 use crate::util::input::prompt_yn;
 use lazy_static::lazy_static;
 use serde::{Deserialize, Serialize};
@@ -46,23 +46,14 @@ pub fn require_consent(
 	if accept_server_license_terms {
 		load.consented = Some(true);
 	} else if !*IS_INTERACTIVE_CLI {
-		return Err(MissingLegalConsent(
-			"Run this command again with --accept-server-license-terms to indicate your agreement."
-				.to_string(),
-		)
-		.into());
+		return Err(CodeError::NeedsInteractiveLegalConsent.into());
 	} else {
 		match prompt_yn(prompt) {
 			Ok(true) => {
 				load.consented = Some(true);
 			}
-			Ok(false) => {
-				return Err(AnyError::from(MissingLegalConsent(format!(
-					"Sorry you cannot use {} CLI without accepting the terms.",
-					PRODUCT_NAME_LONG
-				))))
-			}
-			Err(e) => return Err(AnyError::from(MissingLegalConsent(e.to_string()))),
+			Ok(false) => return Err(CodeError::DeniedLegalConset.into()),
+			Err(_) => return Err(CodeError::NeedsInteractiveLegalConsent.into()),
 		}
 	}
 

cli/src/update_service.rs

@@ -11,7 +11,7 @@ use crate::{
 	constants::VSCODE_CLI_UPDATE_ENDPOINT,
 	debug, log, options, spanf,
 	util::{
-		errors::{AnyError, CodeError, UpdatesNotConfigured, WrappedError},
+		errors::{AnyError, CodeError, WrappedError},
 		http::{BoxedHttp, SimpleResponse},
 		io::ReportCopyProgress,
 		tar, zipper,
@@ -19,6 +19,7 @@ use crate::{
 };
 
 /// Implementation of the VS Code Update service for use in the CLI.
+#[derive(Clone)]
 pub struct UpdateService {
 	client: BoxedHttp,
 	log: log::Logger,
@@ -54,6 +55,10 @@ fn quality_download_segment(quality: options::Quality) -> &'static str {
 	}
 }
 
+fn get_update_endpoint() -> Result<&'static str, CodeError> {
+	VSCODE_CLI_UPDATE_ENDPOINT.ok_or_else(|| CodeError::UpdatesNotConfigured("no service url"))
+}
+
 impl UpdateService {
 	pub fn new(log: log::Logger, http: BoxedHttp) -> Self {
 		UpdateService { client: http, log }
@@ -66,8 +71,7 @@ impl UpdateService {
 		quality: options::Quality,
 		version: &str,
 	) -> Result<Release, AnyError> {
-		let update_endpoint =
-			VSCODE_CLI_UPDATE_ENDPOINT.ok_or_else(UpdatesNotConfigured::no_url)?;
+		let update_endpoint = get_update_endpoint()?;
 		let download_segment = target
 			.download_segment(platform)
 			.ok_or_else(|| CodeError::UnsupportedPlatform(platform.to_string()))?;
@@ -108,8 +112,7 @@ impl UpdateService {
 		target: TargetKind,
 		quality: options::Quality,
 	) -> Result<Release, AnyError> {
-		let update_endpoint =
-			VSCODE_CLI_UPDATE_ENDPOINT.ok_or_else(UpdatesNotConfigured::no_url)?;
+		let update_endpoint = get_update_endpoint()?;
 		let download_segment = target
 			.download_segment(platform)
 			.ok_or_else(|| CodeError::UnsupportedPlatform(platform.to_string()))?;
@@ -144,8 +147,7 @@ impl UpdateService {
 
 	/// Gets the download stream for the release.
 	pub async fn get_download_stream(&self, release: &Release) -> Result<SimpleResponse, AnyError> {
-		let update_endpoint =
-			VSCODE_CLI_UPDATE_ENDPOINT.ok_or_else(UpdatesNotConfigured::no_url)?;
+		let update_endpoint = get_update_endpoint()?;
 		let download_segment = release
 			.target
 			.download_segment(release.platform)

cli/src/util/errors.rs

@@ -108,16 +108,6 @@ impl StatusError {
 	}
 }
 
-// When the user has not consented to the licensing terms in using the Launcher
-#[derive(Debug)]
-pub struct MissingLegalConsent(pub String);
-
-impl std::fmt::Display for MissingLegalConsent {
-	fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
-		write!(f, "{}", self.0)
-	}
-}
-
 // When the provided connection token doesn't match the one used to set up the original VS Code Server
 // This is most likely due to a new user joining.
 #[derive(Debug)]
@@ -313,20 +303,6 @@ impl std::fmt::Display for ServerHasClosed {
 	}
 }
 
-#[derive(Debug)]
-pub struct UpdatesNotConfigured(pub String);
-
-impl UpdatesNotConfigured {
-	pub fn no_url() -> Self {
-		UpdatesNotConfigured("no service url".to_owned())
-	}
-}
-
-impl std::fmt::Display for UpdatesNotConfigured {
-	fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
-		write!(f, "Update service is not configured: {}", self.0)
-	}
-}
 #[derive(Debug)]
 pub struct ServiceAlreadyRegistered();
 
@@ -517,10 +493,28 @@ pub enum CodeError {
 	KeyringTimeout,
 	#[error("no host is connected to the tunnel relay")]
 	NoTunnelEndpoint,
+	#[error("could not parse `host`: {0}")]
+	InvalidHostAddress(std::net::AddrParseError),
+	#[error("could not start server on the given host/port: {0}")]
+	CouldNotListenOnInterface(hyper::Error),
+	#[error(
+		"Run this command again with --accept-server-license-terms to indicate your agreement."
+	)]
+	NeedsInteractiveLegalConsent,
+	#[error("Sorry, you cannot use this CLI without accepting the terms.")]
+	DeniedLegalConset,
+	#[error("The server is not yet downloaded, try again shortly.")]
+	ServerNotYetDownloaded,
+	#[error("An error was encountered downloading the server, please retry: {0}")]
+	ServerDownloadError(String),
+	#[error("Updates are are not available: {0}")]
+	UpdatesNotConfigured(&'static str),
+	// todo: can be specialized when update service is moved to CodeErrors
+	#[error("Could not check for update: {0}")]
+	UpdateCheckFailed(String),
 }
 
 makeAnyError!(
-	MissingLegalConsent,
 	MismatchConnectionToken,
 	DevTunnelError,
 	StatusError,
@@ -543,7 +537,6 @@ makeAnyError!(
 	ServerHasClosed,
 	ServiceAlreadyRegistered,
 	WindowsNeedsElevation,
-	UpdatesNotConfigured,
 	CorruptDownload,
 	MissingHomeDirectory,
 	OAuthError,

cli/src/util/sync.rs

@@ -63,7 +63,7 @@ impl<T: Clone> BarrierOpener<T> {
 /// and is thereafter permanently closed. It can contain a value.
 pub fn new_barrier<T>() -> (Barrier<T>, BarrierOpener<T>)
 where
-	T: Copy,
+	T: Clone,
 {
 	let (closed_tx, closed_rx) = watch::channel(None);
 	(Barrier(closed_rx), BarrierOpener(Arc::new(closed_tx)))

src/vs/code/node/cli.ts

@@ -15,7 +15,7 @@ import { whenDeleted, writeFileSync } from 'vs/base/node/pfs';
 import { findFreePort } from 'vs/base/node/ports';
 import { watchFileContents } from 'vs/platform/files/node/watcher/nodejs/nodejsWatcherLib';
 import { NativeParsedArgs } from 'vs/platform/environment/common/argv';
-import { buildHelpMessage, buildVersionMessage, OPTIONS } from 'vs/platform/environment/node/argv';
+import { buildHelpMessage, buildVersionMessage, NATIVE_CLI_COMMANDS, OPTIONS } from 'vs/platform/environment/node/argv';
 import { addArg, parseCLIProcessArgv } from 'vs/platform/environment/node/argvHelper';
 import { getStdinFilePath, hasStdinWithoutTty, readFromStdin, stdinDataListener } from 'vs/platform/environment/node/stdin';
 import { createWaitMarkerFileSync } from 'vs/platform/environment/node/wait';
@@ -51,31 +51,33 @@ export async function main(argv: string[]): Promise<any> {
 		return;
 	}
 
-	if (args.tunnel) {
-		if (!product.tunnelApplicationName) {
-			console.error(`'tunnel' command not supported in ${product.applicationName}`);
-			return;
-		}
-		const tunnelArgs = argv.slice(argv.indexOf('tunnel') + 1); // all arguments behind `tunnel`
-		return new Promise((resolve, reject) => {
-			let tunnelProcess: ChildProcess;
-			const stdio: StdioOptions = ['ignore', 'pipe', 'pipe'];
-			if (process.env['VSCODE_DEV']) {
-				tunnelProcess = spawn('cargo', ['run', '--', 'tunnel', ...tunnelArgs], { cwd: join(getAppRoot(), 'cli'), stdio });
-			} else {
-				const appPath = process.platform === 'darwin'
-					// ./Contents/MacOS/Electron => ./Contents/Resources/app/bin/code-tunnel-insiders
-					? join(dirname(dirname(process.execPath)), 'Resources', 'app')
-					: dirname(process.execPath);
-				const tunnelCommand = join(appPath, 'bin', `${product.tunnelApplicationName}${isWindows ? '.exe' : ''}`);
-				tunnelProcess = spawn(tunnelCommand, ['tunnel', ...tunnelArgs], { cwd: cwd(), stdio });
+	for (const subcommand of NATIVE_CLI_COMMANDS) {
+		if (args[subcommand]) {
+			if (!product.tunnelApplicationName) {
+				console.error(`'${subcommand}' command not supported in ${product.applicationName}`);
+				return;
 			}
+			const tunnelArgs = argv.slice(argv.indexOf(subcommand) + 1); // all arguments behind `tunnel`
+			return new Promise((resolve, reject) => {
+				let tunnelProcess: ChildProcess;
+				const stdio: StdioOptions = ['ignore', 'pipe', 'pipe'];
+				if (process.env['VSCODE_DEV']) {
+					tunnelProcess = spawn('cargo', ['run', '--', subcommand, ...tunnelArgs], { cwd: join(getAppRoot(), 'cli'), stdio });
+				} else {
+					const appPath = process.platform === 'darwin'
+						// ./Contents/MacOS/Electron => ./Contents/Resources/app/bin/code-tunnel-insiders
+						? join(dirname(dirname(process.execPath)), 'Resources', 'app')
+						: dirname(process.execPath);
+					const tunnelCommand = join(appPath, 'bin', `${product.tunnelApplicationName}${isWindows ? '.exe' : ''}`);
+					tunnelProcess = spawn(tunnelCommand, [subcommand, ...tunnelArgs], { cwd: cwd(), stdio });
+				}
 
-			tunnelProcess.stdout!.pipe(process.stdout);
-			tunnelProcess.stderr!.pipe(process.stderr);
-			tunnelProcess.on('exit', resolve);
-			tunnelProcess.on('error', reject);
-		});
+				tunnelProcess.stdout!.pipe(process.stdout);
+				tunnelProcess.stderr!.pipe(process.stderr);
+				tunnelProcess.on('exit', resolve);
+				tunnelProcess.on('error', reject);
+			});
+		}
 	}
 
 	// Help

src/vs/platform/environment/common/argv.ts

@@ -3,15 +3,18 @@
  *  Licensed under the MIT License. See License.txt in the project root for license information.
  *--------------------------------------------------------------------------------------------*/
 
+export interface INativeCliOptions {
+	'cli-data-dir'?: string;
+	'disable-telemetry'?: boolean;
+	'telemetry-level'?: string;
+}
+
 /**
  * A list of command line arguments we support natively.
  */
 export interface NativeParsedArgs {
 	// subcommands
-	tunnel?: {
-		'cli-data-dir'?: string;
-		'disable-telemetry'?: boolean;
-		'telemetry-level'?: string;
+	tunnel?: INativeCliOptions & {
 		user: {
 			login: {
 				'access-token'?: string;
@@ -19,6 +22,7 @@ export interface NativeParsedArgs {
 			};
 		};
 	};
+	'serve-web'?: INativeCliOptions;
 	_: string[];
 	'folder-uri'?: string[]; // undefined or array of 1 or more
 	'file-uri'?: string[]; // undefined or array of 1 or more

src/vs/platform/environment/node/argv.ts

@@ -44,6 +44,8 @@ export type OptionDescriptions<T> = {
 	Subcommand<T[P]>
 };
 
+export const NATIVE_CLI_COMMANDS = ['tunnel', 'serve-web'] as const;
+
 export const OPTIONS: OptionDescriptions<Required<NativeParsedArgs>> = {
 	'tunnel': {
 		type: 'subcommand',
@@ -66,6 +68,15 @@ export const OPTIONS: OptionDescriptions<Required<NativeParsedArgs>> = {
 			}
 		}
 	},
+	'serve-web': {
+		type: 'subcommand',
+		description: 'Make the current machine accessible from vscode.dev or other machines through a secure tunnel',
+		options: {
+			'cli-data-dir': { type: 'string', args: 'dir', description: localize('cliDataDir', "Directory where CLI metadata should be stored.") },
+			'disable-telemetry': { type: 'boolean' },
+			'telemetry-level': { type: 'string' },
+		}
+	},
 
 	'diff': { type: 'boolean', cat: 'o', alias: 'd', args: ['file', 'file'], description: localize('diff', "Compare two files with each other.") },
 	'merge': { type: 'boolean', cat: 'o', alias: 'm', args: ['path1', 'path2', 'base', 'result'], description: localize('merge', "Perform a three-way merge by providing paths for two modified versions of a file, the common origin of both modified versions and the output file to save merge results.") },

src/vs/platform/environment/node/argvHelper.ts

@@ -7,7 +7,7 @@ import * as assert from 'assert';
 import { IProcessEnvironment } from 'vs/base/common/platform';
 import { localize } from 'vs/nls';
 import { NativeParsedArgs } from 'vs/platform/environment/common/argv';
-import { ErrorReporter, OPTIONS, parseArgs } from 'vs/platform/environment/node/argv';
+import { ErrorReporter, NATIVE_CLI_COMMANDS, OPTIONS, parseArgs } from 'vs/platform/environment/node/argv';
 
 function parseAndValidate(cmdLineArgs: string[], reportWarnings: boolean): NativeParsedArgs {
 	const onMultipleValues = (id: string, val: string) => {
@@ -21,14 +21,14 @@ function parseAndValidate(cmdLineArgs: string[], reportWarnings: boolean): Nativ
 	};
 	const getSubcommandReporter = (command: string) => ({
 		onUnknownOption: (id: string) => {
-			if (command !== 'tunnel') {
+			if (!(NATIVE_CLI_COMMANDS as readonly string[]).includes(command)) {
 				console.warn(localize('unknownSubCommandOption', "Warning: '{0}' is not in the list of known options for subcommand '{1}'", id, command));
 			}
 		},
 		onMultipleValues,
 		onEmptyValue,
 		onDeprecatedOption,
-		getSubcommandReporter: command !== 'tunnel' ? getSubcommandReporter : undefined
+		getSubcommandReporter: (NATIVE_CLI_COMMANDS as readonly string[]).includes(command) ? getSubcommandReporter : undefined
 	});
 	const errorReporter: ErrorReporter = {
 		onUnknownOption: (id) => {