cli: add stdio control server

* signing: implement signing service on the web

* wip

* cli: implement stdio service

This is used to implement the exec server for WSL. Guarded behind a signed handshake.

* update distro

* rm debug

* address pr comments

cli/src/msgpack_rpc.rs

@@ -4,8 +4,9 @@
  *--------------------------------------------------------------------------------------------*/
 
 use bytes::Buf;
+use serde::de::DeserializeOwned;
 use tokio::{
-	io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt, BufReader},
+	io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt},
 	pin,
 	sync::mpsc,
 };
@@ -18,7 +19,7 @@ use crate::{
 		sync::{Barrier, Receivable},
 	},
 };
-use std::io;
+use std::io::{self, Cursor, ErrorKind};
 
 #[derive(Copy, Clone)]
 pub struct MsgPackSerializer {}
@@ -35,21 +36,28 @@ impl Serialization for MsgPackSerializer {
 
 pub type MsgPackCaller = rpc::RpcCaller<MsgPackSerializer>;
 
-/// Creates a new RPC Builder that serializes to JSON.
+/// Creates a new RPC Builder that serializes to msgpack.
 pub fn new_msgpack_rpc() -> rpc::RpcBuilder<MsgPackSerializer> {
 	rpc::RpcBuilder::new(MsgPackSerializer {})
 }
 
-pub async fn start_msgpack_rpc<C: Send + Sync + 'static, S: Clone>(
-	dispatcher: rpc::RpcDispatcher<MsgPackSerializer, C>,
-	read: impl AsyncRead + Unpin,
-	mut write: impl AsyncWrite + Unpin,
+/// Starting processing msgpack rpc over the given i/o. It's recommended that
+/// the reader be passed in as a BufReader for efficiency.
+pub async fn start_msgpack_rpc<
+	C: Send + Sync + 'static,
+	X: Clone,
+	S: Send + Sync + Serialization,
+	Read: AsyncRead + Unpin,
+	Write: AsyncWrite + Unpin,
+>(
+	dispatcher: rpc::RpcDispatcher<S, C>,
+	mut read: Read,
+	mut write: Write,
 	mut msg_rx: impl Receivable<Vec<u8>>,
-	mut shutdown_rx: Barrier<S>,
-) -> io::Result<Option<S>> {
+	mut shutdown_rx: Barrier<X>,
+) -> io::Result<(Option<X>, Read, Write)> {
 	let (write_tx, mut write_rx) = mpsc::channel::<Vec<u8>>(8);
-	let mut read = BufReader::new(read);
-	let mut decoder = U32PrefixedCodec {};
+	let mut decoder = MsgPackCodec::new();
 	let mut decoder_buf = bytes::BytesMut::new();
 
 	let shutdown_fut = shutdown_rx.wait();
@@ -61,7 +69,7 @@ pub async fn start_msgpack_rpc<C: Send + Sync + 'static, S: Clone>(
 				r?;
 
 				while let Some(frame) = decoder.decode(&mut decoder_buf)? {
-					match dispatcher.dispatch(&frame) {
+					match dispatcher.dispatch_with_partial(&frame.vec, frame.obj) {
 						MaybeSync::Sync(Some(v)) => {
 							let _ = write_tx.send(v).await;
 						},
@@ -94,39 +102,94 @@ pub async fn start_msgpack_rpc<C: Send + Sync + 'static, S: Clone>(
 			Some(m) = msg_rx.recv_msg() => {
 				write.write_all(&m).await?;
 			},
-			r = &mut shutdown_fut => return Ok(r.ok()),
+			r = &mut shutdown_fut => return Ok((r.ok(), read, write)),
 		}
 
 		write.flush().await?;
 	}
 }
 
-/// Reader that reads length-prefixed msgpack messages in a cancellation-safe
-/// way using Tokio's codecs.
-pub struct U32PrefixedCodec {}
+/// Reader that reads msgpack object messages in a cancellation-safe way using Tokio's codecs.
+///
+/// rmp_serde does not support async reads, and does not plan to. But we know every
+/// type in protocol is some kind of object, so by asking to deserialize the
+/// requested object from a reader (repeatedly, if incomplete) we can
+/// accomplish streaming.
+pub struct MsgPackCodec<T> {
+	_marker: std::marker::PhantomData<T>,
+}
 
-const U32_SIZE: usize = 4;
+impl<T> MsgPackCodec<T> {
+	pub fn new() -> Self {
+		Self {
+			_marker: std::marker::PhantomData::default(),
+		}
+	}
+}
 
-impl tokio_util::codec::Decoder for U32PrefixedCodec {
-	type Item = Vec<u8>;
+pub struct MsgPackDecoded<T> {
+	pub obj: T,
+	pub vec: Vec<u8>,
+}
+
+impl<T: DeserializeOwned> tokio_util::codec::Decoder for MsgPackCodec<T> {
+	type Item = MsgPackDecoded<T>;
 	type Error = io::Error;
 
 	fn decode(&mut self, src: &mut bytes::BytesMut) -> Result<Option<Self::Item>, Self::Error> {
-		if src.len() < 4 {
-			src.reserve(U32_SIZE - src.len());
-			return Ok(None);
-		}
+		let bytes_ref = src.as_ref();
+		let mut cursor = Cursor::new(bytes_ref);
 
-		let mut be_bytes = [0; U32_SIZE];
-		be_bytes.copy_from_slice(&src[..U32_SIZE]);
-		let required_len = U32_SIZE + (u32::from_be_bytes(be_bytes) as usize);
-		if src.len() < required_len {
-			src.reserve(required_len - src.len());
-			return Ok(None);
+		match rmp_serde::decode::from_read::<_, T>(&mut cursor) {
+			Err(
+				rmp_serde::decode::Error::InvalidDataRead(e)
+				| rmp_serde::decode::Error::InvalidMarkerRead(e),
+			) if e.kind() == ErrorKind::UnexpectedEof => {
+				src.reserve(1024);
+				Ok(None)
+			}
+			Err(e) => Err(std::io::Error::new(
+				std::io::ErrorKind::InvalidData,
+				e.to_string(),
+			)),
+			Ok(obj) => {
+				let len = cursor.position() as usize;
+				let vec = src[..len].to_vec();
+				src.advance(len);
+				Ok(Some(MsgPackDecoded { obj, vec }))
+			}
 		}
-
-		let msg = src[U32_SIZE..required_len].to_vec();
-		src.advance(required_len);
-		Ok(Some(msg))
+	}
+}
+
+#[cfg(test)]
+mod tests {
+	use serde::{Deserialize, Serialize};
+
+	use super::*;
+
+	#[derive(Serialize, Deserialize, PartialEq, Eq, Debug)]
+	pub struct Msg {
+		pub x: i32,
+	}
+
+	#[test]
+	fn test_protocol() {
+		let mut c = MsgPackCodec::<Msg>::new();
+		let mut buf = bytes::BytesMut::new();
+
+		assert!(c.decode(&mut buf).unwrap().is_none());
+
+		buf.extend_from_slice(rmp_serde::to_vec_named(&Msg { x: 1 }).unwrap().as_slice());
+		buf.extend_from_slice(rmp_serde::to_vec_named(&Msg { x: 2 }).unwrap().as_slice());
+
+		assert_eq!(
+			c.decode(&mut buf).unwrap().expect("expected msg1").obj,
+			Msg { x: 1 }
+		);
+		assert_eq!(
+			c.decode(&mut buf).unwrap().expect("expected msg1").obj,
+			Msg { x: 2 }
+		);
 	}
 }