Working overlay

27 files changed, 209 insertions, 96 deletions

diff --git a/src/newton.rs b/src/common.rs
index 3599211..3599211 100644
--- a/src/newton.rs
+++ b/src/common.rs
diff --git a/src/newton/client.rs b/src/common/client.rs
index 69ac6b3..69ac6b3 100644
--- a/src/newton/client.rs
+++ b/src/common/client.rs
diff --git a/src/newton/client/assets.rs b/src/common/client/assets.rs
index 742f9ea..742f9ea 100644
--- a/src/newton/client/assets.rs
+++ b/src/common/client/assets.rs
diff --git a/src/newton/client/assets/audio/test.wav b/src/common/client/assets/audio/test.wav
index 0eabe85..0eabe85 100644
--- a/src/newton/client/assets/audio/test.wav
+++ b/src/common/client/assets/audio/test.wav
diff --git a/src/newton/client/assets/meshes/square.obj b/src/common/client/assets/meshes/square.obj
index 7328a6c..7328a6c 100644
--- a/src/newton/client/assets/meshes/square.obj
+++ b/src/common/client/assets/meshes/square.obj
diff --git a/src/newton/client/assets/shaders/flat/frag.glsl b/src/common/client/assets/shaders/flat/frag.glsl
index 7006a2b..7006a2b 100644
--- a/src/newton/client/assets/shaders/flat/frag.glsl
+++ b/src/common/client/assets/shaders/flat/frag.glsl
diff --git a/src/newton/client/assets/shaders/flat/vert.glsl b/src/common/client/assets/shaders/flat/vert.glsl
index e324f7e..e324f7e 100644
--- a/src/newton/client/assets/shaders/flat/vert.glsl
+++ b/src/common/client/assets/shaders/flat/vert.glsl
diff --git a/src/newton/client/assets/textures/test.png b/src/common/client/assets/textures/test.png
index 1f1edca..1f1edca 100644
--- a/src/newton/client/assets/textures/test.png
+++ b/src/common/client/assets/textures/test.png
diff --git a/src/common/overlay.rs b/src/common/overlay.rs
new file mode 100644
index 0000000..d7a0d87
--- /dev/null
+++ b/src/common/overlay.rs
@@ -0,0 +1,125 @@
+#![allow(dead_code, unused_variables)]
+mod assets;
+mod terminal;
+mod fig;
+
+use teleia::*;
+
+use std::{collections::HashMap, f32::consts::PI};
+use lexpr::sexp;
+
+pub struct Overlay {
+    assets: assets::Assets,
+    model: scene::Scene,
+    model_neck_base: glam::Mat4,
+    model_fb: framebuffer::Framebuffer,
+    terminal: terminal::Terminal,
+    fig: fig::Client,
+    tracking_eyes: (f32, f32),
+    tracking_neck: glam::Quat,
+}
+
+impl Overlay {
+    pub async fn new(ctx: &context::Context) -> Self {
+        let model = scene::Scene::from_gltf(ctx, include_bytes!("overlay/assets/scenes/lcolonq.vrm")); 
+        let model_neck_base = model.nodes_by_name.get("J_Bip_C_Neck")
+            .and_then(|i| model.nodes.get(*i))
+            .expect("failed to find neck joint")
+            .transform;
+        Self {
+            assets: assets::Assets::new(ctx),
+            model,
+            model_neck_base,
+            model_fb: framebuffer::Framebuffer::new(
+                ctx,
+                &glam::Vec2::new(terminal::WIDTH as _, terminal::HEIGHT as _),
+                &glam::Vec2::ZERO
+            ),
+            terminal: terminal::Terminal::new(ctx),
+            fig: fig::Client::new("shiro:32050", &[
+                sexp!((avatar toggle)),
+                sexp!((avatar text)),
+                sexp!((avatar frame)),
+                sexp!((avatar reset)),
+                sexp!((avatar tracking)),
+            ]),
+            tracking_eyes: (1.0, 1.0),
+            tracking_neck: glam::Quat::IDENTITY,
+        }
+    }
+    pub fn handle_tracking(&mut self, msg: fig::Message) -> Option<()> {
+        let eyes = msg.data.get(0)?;
+        let eye_left = eyes.get(0)?.as_str()?.parse::<f32>().ok()?;
+        let eye_right = eyes.get(1)?.as_str()?.parse::<f32>().ok()?;
+        let euler = msg.data.get(1)?;
+        let euler_x = euler.get(0)?.as_str()?.parse::<f32>().ok()?.to_radians();
+        let euler_y = PI - euler.get(1)?.as_str()?.parse::<f32>().ok()?.to_radians();
+        let euler_z = euler.get(2)?.as_str()?.parse::<f32>().ok()?.to_radians() + PI/2.0;
+        self.tracking_eyes = (eye_left, eye_right);
+        self.tracking_neck = glam::Quat::from_euler(glam::EulerRot::XYZ, euler_x, euler_y, euler_z);
+        Some(())
+    }
+    pub fn handle_text(&mut self, msg: fig::Message) -> Option<()> {
+        let s = msg.data.get(0)?.as_str()?;
+        self.terminal.fill_string(s);
+        Some(())
+    }
+}
+
+impl teleia::state::Game for Overlay {
+    fn initialize_audio(&self, ctx: &context::Context, st: &state::State, actx: &audio::Context) -> HashMap<String, audio::Audio> {
+        HashMap::new()
+    }
+    fn finish_title(&mut self, _st: &mut state::State) {}
+    fn mouse_press(&mut self, _ctx: &context::Context, _st: &mut state::State) {}
+    fn mouse_move(&mut self, _ctx: &context::Context, _st: &mut state::State, _x: i32, _y: i32) {}
+    fn update(&mut self, ctx: &context::Context, st: &mut state::State) -> Option<()> {
+        st.projection = glam::Mat4::perspective_lh(
+            PI / 4.0,
+            terminal::WIDTH as f32 / terminal::HEIGHT as f32,
+            0.1, 10.0
+        );
+        st.move_camera(
+            ctx,
+            &glam::Vec3::new(0.0, 0.0, 1.0),
+            &glam::Vec3::new(0.0, 0.0, -1.0),
+            &glam::Vec3::new(0.0, 1.0, 0.0),
+        );
+        while let Some(msg) = self.fig.pump() {
+            let malformed = format!("malformed {} data: {}", msg.event, msg.data);
+            if msg.event == sexp!((avatar tracking)) {
+                if self.handle_tracking(msg).is_none() { log::warn!("{}", malformed) }
+            } else if msg.event == sexp!((avatar text)) {
+                if self.handle_text(msg).is_none() { log::warn!("{}", malformed) }
+            } else {
+                log::info!("received unhandled event {} with data: {}", msg.event, msg.data);
+            }
+        }
+        if let Some(n) = self.model.nodes_by_name.get("J_Bip_C_Neck").and_then(|i| self.model.nodes.get_mut(*i)) {
+            n.transform = self.model_neck_base * glam::Mat4::from_quat(self.tracking_neck);
+        }
+        Some(())
+    }
+    fn render(&mut self, ctx: &context::Context, st: &mut state::State) -> Option<()> {
+        self.model_fb.bind(ctx);
+        ctx.clear_color(glam::Vec4::new(0.0, 0.0, 0.0, 1.0));
+        ctx.clear();
+        st.bind_3d(ctx, &self.assets.shader_scene);
+        self.assets.shader_scene.set_position_3d(
+            ctx,
+            &glam::Mat4::from_translation(
+                glam::Vec3::new(0.0, -1.6, 0.5),
+            ),
+        );
+        self.model.render(ctx, &self.assets.shader_scene);
+        st.render_framebuffer.bind(ctx);
+        self.terminal.update(ctx, &self.model_fb);
+        self.terminal.render(ctx, &glam::Vec2::new(400.0, 200.0));
+        // self.model_fb.blit(
+        //     ctx, &st.render_framebuffer,
+        //     &glam::Vec2::new(ctx.render_width / 2.0 - 512.0, ctx.render_height / 2.0 - 512.0),
+        //     &glam::Vec2::new(128.0, 128.0)
+        // );
+        Some(())
+    }
+}
diff --git a/src/newton/overlay/assets.rs b/src/common/overlay/assets.rs
index 9a0cefc..9a0cefc 100644
--- a/src/newton/overlay/assets.rs
+++ b/src/common/overlay/assets.rs
diff --git a/src/newton/overlay/assets/audio/test.wav b/src/common/overlay/assets/audio/test.wav
index 0eabe85..0eabe85 100644
--- a/src/newton/overlay/assets/audio/test.wav
+++ b/src/common/overlay/assets/audio/test.wav
diff --git a/src/newton/overlay/assets/fonts/iosevka-comfy-regular.ttf b/src/common/overlay/assets/fonts/iosevka-comfy-regular.ttf
index b474adc..b474adc 100644
--- a/src/newton/overlay/assets/fonts/iosevka-comfy-regular.ttf
+++ b/src/common/overlay/assets/fonts/iosevka-comfy-regular.ttf
diff --git a/src/newton/overlay/assets/fonts/terminus.png b/src/common/overlay/assets/fonts/terminus.png
index 287810b..287810b 100644
--- a/src/newton/overlay/assets/fonts/terminus.png
+++ b/src/common/overlay/assets/fonts/terminus.png
diff --git a/src/newton/overlay/assets/fonts/terminus.ttf b/src/common/overlay/assets/fonts/terminus.ttf
index d125e63..d125e63 100644
--- a/src/newton/overlay/assets/fonts/terminus.ttf
+++ b/src/common/overlay/assets/fonts/terminus.ttf
diff --git a/src/newton/overlay/assets/meshes/square.obj b/src/common/overlay/assets/meshes/square.obj
index 7328a6c..7328a6c 100644
--- a/src/newton/overlay/assets/meshes/square.obj
+++ b/src/common/overlay/assets/meshes/square.obj
diff --git a/src/newton/overlay/assets/scenes/lcolonq.vrm b/src/common/overlay/assets/scenes/lcolonq.vrm
index 49fa4a4..49fa4a4 100644
--- a/src/newton/overlay/assets/scenes/lcolonq.vrm
+++ b/src/common/overlay/assets/scenes/lcolonq.vrm
diff --git a/src/newton/overlay/assets/shaders/flat/frag.glsl b/src/common/overlay/assets/shaders/flat/frag.glsl
index 7006a2b..7006a2b 100644
--- a/src/newton/overlay/assets/shaders/flat/frag.glsl
+++ b/src/common/overlay/assets/shaders/flat/frag.glsl
diff --git a/src/newton/overlay/assets/shaders/flat/vert.glsl b/src/common/overlay/assets/shaders/flat/vert.glsl
index e324f7e..e324f7e 100644
--- a/src/newton/overlay/assets/shaders/flat/vert.glsl
+++ b/src/common/overlay/assets/shaders/flat/vert.glsl
diff --git a/src/newton/overlay/assets/shaders/scene/frag.glsl b/src/common/overlay/assets/shaders/scene/frag.glsl
index 81e08b9..81e08b9 100644
--- a/src/newton/overlay/assets/shaders/scene/frag.glsl
+++ b/src/common/overlay/assets/shaders/scene/frag.glsl
diff --git a/src/newton/overlay/assets/shaders/scene/vert.glsl b/src/common/overlay/assets/shaders/scene/vert.glsl
index 64f400c..64f400c 100644
--- a/src/newton/overlay/assets/shaders/scene/vert.glsl
+++ b/src/common/overlay/assets/shaders/scene/vert.glsl
diff --git a/src/newton/overlay/assets/textures/test.png b/src/common/overlay/assets/textures/test.png
index 1f1edca..1f1edca 100644
--- a/src/newton/overlay/assets/textures/test.png
+++ b/src/common/overlay/assets/textures/test.png
diff --git a/src/common/overlay/fig.rs b/src/common/overlay/fig.rs
new file mode 100644
index 0000000..b60f8e9
--- /dev/null
+++ b/src/common/overlay/fig.rs
@@ -0,0 +1,40 @@
+use std::io::{BufRead, Write};
+
+#[derive(Debug, Clone)]
+pub struct Message {
+    pub event: lexpr::Value,
+    pub data: lexpr::Value,
+}
+
+pub struct Client {
+    reader: std::io::BufReader<std::net::TcpStream>,
+}
+impl Client {
+    pub fn new(addr: &str, subs: &[lexpr::Value]) -> Self {
+        let mut socket = std::net::TcpStream::connect(addr).expect("failed to connect to message bus");
+        socket.set_nonblocking(true).expect("failed to set message bus socket nonblocking");
+        for s in subs {
+            write!(socket, "(sub {})\n", s).expect("failed to send subscribe message to bus");
+        }
+        let reader = std::io::BufReader::new(socket);
+        Self { reader, }
+    }
+    pub fn pump(&mut self) -> Option<Message> {
+        let mut buf = String::new();
+        match self.reader.read_line(&mut buf) {
+            Ok(l) => match lexpr::from_str(&buf) {
+                Ok(v) => {
+                    match v.as_cons() {
+                        Some(cs) => {
+                            Some(Message { event: cs.car().clone(), data: cs.cdr().clone() })
+                        },
+                        _ => { log::error!("malformed message bus input s-expression: {}", v); None },
+                    }
+                },
+                Err(e) => { log::error!("malformed message bus input line: {}", e); None },
+            },
+            Err(e) if e.kind() == std::io::ErrorKind::WouldBlock => None,
+            Err(e) => panic!("IO error on message bus: {}", e),
+        }
+    }
+}
diff --git a/src/newton/overlay/terminal.rs b/src/common/overlay/terminal.rs
index 3371789..a2382b1 100644
--- a/src/newton/overlay/terminal.rs
+++ b/src/common/overlay/terminal.rs
@@ -15,6 +15,20 @@ impl std::ops::Add for Pos {
     fn add(self, rhs: Self) -> Self { Self {x: self.x + rhs.x, y: self.y + rhs.y } }
 }
 
+#[derive(Debug, Clone)]
+pub struct CharPair {
+    pub first: char,
+    pub second: Option<char>,
+}
+impl Default for CharPair {
+    fn default() -> Self {
+        Self {
+            first: 'a',
+            second: Some('b'),
+        }
+    }
+}
+
 pub struct Layer<T> {
     pub data: [T; WIDTH * HEIGHT],
 }
@@ -26,9 +40,28 @@ impl<T> Layer<T> {
     }
     pub fn get(&self, p: Pos) -> Option<&T> {
         if p.x < 0 || p.x >= WIDTH as _ || p.y < 0 || p.y >= HEIGHT as _ { return None }
-        let idx = p.x  as usize + p.y as usize * WIDTH;
+        let idx = p.x as usize + p.y as usize * WIDTH;
         Some(&self.data[idx])
     }
+    pub fn set(&mut self, p: Pos, x: T) {
+        if p.x < 0 || p.x >= WIDTH as _ || p.y < 0 || p.y >= HEIGHT as _ { return }
+        let idx = p.x as usize + p.y as usize * WIDTH;
+        self.data[idx] = x;
+    }
+}
+impl Layer<CharPair> {
+    pub fn from_str(&mut self, s: &str) {
+        let chars: Vec<char> = s.chars().collect();
+        if chars.is_empty() { return }
+        let mut i: usize = 0;
+        for row in 0..64 {
+            for col in 0..64 {
+                let first = chars[i]; i += 1; i %= chars.len();
+                let second = Some(chars[i]); i += 1; i %= chars.len();
+                self.set(Pos::new(col, row), CharPair { first, second });
+            }
+        }
+    }
 }
 impl Layer<glam::Vec3> {
     pub fn from_framebuffer(&mut self, ctx: &context::Context, fb: &framebuffer::Framebuffer) {
@@ -36,20 +69,6 @@ impl Layer<glam::Vec3> {
     }
 }
 
-#[derive(Debug, Clone)]
-pub struct CharPair {
-    pub first: char,
-    pub second: Option<char>,
-}
-impl Default for CharPair {
-    fn default() -> Self {
-        Self {
-            first: 'a',
-            second: Some('b'),
-        }
-    }
-}
-
 pub struct Terminal {
     pub font: font::Bitmap,
     pub base_color: Layer<glam::Vec3>,
@@ -58,11 +77,13 @@ pub struct Terminal {
 }
 impl Terminal {
     pub fn new(ctx: &context::Context) -> Self {
+        let mut set_char = Layer::new();
+        set_char.from_str("lcolonq");
         Self {
             font: font::Bitmap::from_image(ctx, 6, 12, 96, 72, include_bytes!("assets/fonts/terminus.png")),
             base_color: Layer::new(),
             set_color: Layer::new(),
-            set_char: Layer::new(),
+            set_char,
         }
     }
     pub fn get_color(&self, pos: Pos) -> glam::Vec3 {
@@ -75,6 +96,9 @@ impl Terminal {
     pub fn update(&mut self, ctx: &context::Context, fb: &framebuffer::Framebuffer) {
         self.base_color.from_framebuffer(ctx, fb);
     }
+    pub fn fill_string(&mut self, s: &str) {
+        self.set_char.from_str(s);
+    }
     pub fn render(&self, ctx: &context::Context, pos: &glam::Vec2) {
         let mut s = String::new();
         let mut colors = Vec::new();
diff --git a/src/newton/server.rs b/src/common/server.rs
index e69de29..e69de29 100644
--- a/src/newton/server.rs
+++ b/src/common/server.rs
diff --git a/src/lib.rs b/src/lib.rs
index de8c271..3298f07 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -1,4 +1,4 @@
-mod newton;
+mod common;
 
 #[cfg(target_arch = "wasm32")]
 use wasm_bindgen::prelude::*;
@@ -6,5 +6,5 @@ use wasm_bindgen::prelude::*;
 #[cfg(target_arch = "wasm32")]
 #[wasm_bindgen]
 pub async fn main_js() {
-    teleia::run(240, 160, newton::client::Game::new).await;
+    teleia::run(240, 160, common::client::Game::new).await;
 }
diff --git a/src/main.rs b/src/main.rs
index 92a527b..8306cae 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -1,4 +1,4 @@
-mod newton;
+mod common;
 
 #[cfg(not(target_arch = "wasm32"))]
 use clap::{command, Command};
@@ -24,7 +24,7 @@ pub async fn main() {
         .get_matches();
     match matches.subcommand() {
         Some(("overlay", _cm)) => {
-            teleia::run("LCOLONQ", 1920, 1080, newton::overlay::Overlay::new).await;
+            teleia::run("LCOLONQ", 1920, 1080, true, common::overlay::Overlay::new).await;
         },
         Some(("server", _cm)) => {
             env_logger::Builder::new().filter(None, log::LevelFilter::Info).init();
diff --git a/src/newton/overlay.rs b/src/newton/overlay.rs
deleted file mode 100644
index 33b96cb..0000000
--- a/src/newton/overlay.rs
+++ /dev/null
@@ -1,76 +0,0 @@
-#![allow(dead_code, unused_variables)]
-mod assets;
-mod terminal;
-
-use std::collections::HashMap;
-use teleia::*;
-
-pub struct Overlay {
-    assets: assets::Assets,
-    model: scene::Scene,
-    model_fb: framebuffer::Framebuffer,
-    terminal: terminal::Terminal,
-}
-
-impl Overlay {
-    pub async fn new(ctx: &context::Context) -> Self {
-        Self {
-            assets: assets::Assets::new(ctx),
-            model: scene::Scene::from_gltf(ctx, include_bytes!("overlay/assets/scenes/lcolonq.vrm")),
-            model_fb: framebuffer::Framebuffer::new(
-                ctx,
-                &glam::Vec2::new(terminal::WIDTH as _, terminal::HEIGHT as _),
-                &glam::Vec2::ZERO
-            ),
-            terminal: terminal::Terminal::new(ctx),
-        }
-    }
-}
-
-impl teleia::state::Game for Overlay {
-    fn initialize_audio(&self, ctx: &context::Context, st: &state::State, actx: &audio::Context) -> HashMap<String, audio::Audio> {
-        HashMap::new()
-    }
-    fn finish_title(&mut self, _st: &mut state::State) {}
-    fn mouse_press(&mut self, _ctx: &context::Context, _st: &mut state::State) {}
-    fn mouse_move(&mut self, _ctx: &context::Context, _st: &mut state::State, _x: i32, _y: i32) {}
-    fn update(&mut self, ctx: &context::Context, st: &mut state::State) -> Option<()> {
-        st.projection = glam::Mat4::perspective_lh(
-            std::f32::consts::PI / 4.0,
-            terminal::WIDTH as f32 / terminal::HEIGHT as f32,
-            0.1, 10.0
-        );
-        st.move_camera(
-            ctx,
-            &glam::Vec3::new(0.0, 0.0, 1.0),
-            &glam::Vec3::new(0.0, 0.0, -1.0),
-            &glam::Vec3::new(0.0, 1.0, 0.0),
-        );
-        // if let Some(n) = self.model.nodes_by_name.get("J_Bip_C_Neck").and_then(|i| self.model.nodes.get_mut(*i)) {
-        //     n.transform *= glam::Mat4::from_rotation_y(0.05);
-        // }
-        Some(())
-    }
-    fn render(&mut self, ctx: &context::Context, st: &mut state::State) -> Option<()> {
-        self.model_fb.bind(ctx);
-        ctx.clear_color(glam::Vec4::new(0.0, 0.0, 0.0, 1.0));
-        ctx.clear();
-        st.bind_3d(ctx, &self.assets.shader_scene);
-        self.assets.shader_scene.set_position_3d(
-            ctx,
-            &glam::Mat4::from_translation(
-                glam::Vec3::new(0.0, -1.6, 0.5),
-            ),
-        );
-        self.model.render(ctx, &self.assets.shader_scene);
-        st.render_framebuffer.bind(ctx);
-        // self.model_fb.blit(
-        //     ctx, &st.render_framebuffer,
-        //     &glam::Vec2::new(ctx.render_width / 2.0 - 512.0, ctx.render_height / 2.0 - 512.0),
-        //     &glam::Vec2::new(1024.0, 1024.0)
-        // );
-        self.terminal.update(ctx, &self.model_fb);
-        self.terminal.render(ctx, &glam::Vec2::new(400.0, 200.0));
-        Some(())
-    }
-}