From ba1ff8e7e3ea915c6d673c05cdcdee10a9b5b9f0 Mon Sep 17 00:00:00 2001 From: LLLL Colonq Date: Fri, 25 Jul 2025 20:35:51 -0400 Subject: WIP --- crates/renderer/Cargo.toml | 3 +- crates/renderer/src/assets/shaders/white/frag.glsl | 11 ++ crates/renderer/src/assets/shaders/white/vert.glsl | 4 + crates/renderer/src/fig.rs | 133 ++++++++++++++++- crates/renderer/src/overlay/model.rs | 6 +- crates/renderer/src/overlay/shader.rs | 164 +++++++++++++++++++-- 6 files changed, 306 insertions(+), 15 deletions(-) create mode 100644 crates/renderer/src/assets/shaders/white/frag.glsl create mode 100644 crates/renderer/src/assets/shaders/white/vert.glsl (limited to 'crates/renderer') diff --git a/crates/renderer/Cargo.toml b/crates/renderer/Cargo.toml index 37a148f..8fa87fd 100644 --- a/crates/renderer/Cargo.toml +++ b/crates/renderer/Cargo.toml @@ -25,4 +25,5 @@ clap = { version = "*", features = ["cargo"] } # cli arg parsing base64 = "0.22.1" # base64 encoding/decoding polling = "*" # polling sockets termion = "*" # terminal escapes -mouse_position = "*" # get mouse position \ No newline at end of file +device_query = "*" # get pressed keys when unfocused +byteorder = "*" # read little-endian numbers \ No newline at end of file diff --git a/crates/renderer/src/assets/shaders/white/frag.glsl b/crates/renderer/src/assets/shaders/white/frag.glsl new file mode 100644 index 0000000..6f86139 --- /dev/null +++ b/crates/renderer/src/assets/shaders/white/frag.glsl @@ -0,0 +1,11 @@ +uniform sampler2D texture_data; + +void main() +{ + vec2 tcfull = vec2(vertex_texcoord.x, 1.0 - vertex_texcoord.y); + vec4 texel = texture(texture_data, tcfull); + if (texel.r == 0.0) { + discard; + } + frag_color = vec4(1.0, 1.0, 1.0, 1.0); +} diff --git a/crates/renderer/src/assets/shaders/white/vert.glsl b/crates/renderer/src/assets/shaders/white/vert.glsl new file mode 100644 index 0000000..e324f7e --- /dev/null +++ b/crates/renderer/src/assets/shaders/white/vert.glsl @@ -0,0 +1,4 @@ +void main() +{ + default_main(); +} \ No newline at end of file diff --git a/crates/renderer/src/fig.rs b/crates/renderer/src/fig.rs index aa972a4..8023f01 100644 --- a/crates/renderer/src/fig.rs +++ b/crates/renderer/src/fig.rs @@ -1,4 +1,5 @@ -use std::io::{BufRead, Write}; +use std::io::{BufRead, Read, Write}; +use byteorder::WriteBytesExt; #[derive(Debug, Clone)] pub struct Message { @@ -48,3 +49,133 @@ impl Client { } } } + +#[derive(Debug, Clone)] +pub struct BinaryMessage { + pub event: Vec, + pub data: Vec +} +#[derive(Debug, Clone)] +pub enum BinaryClientState { + PartialEventLength { buf_len: usize, buf: [u8; 4] }, + PartialEvent { len: usize, buf_len: usize, buf: Vec }, + PartialDataLength { event: Vec, buf_len: usize, buf: [u8; 4] }, + PartialData { event: Vec, len: usize, buf_len: usize, buf: Vec }, + Message { event: Vec, data: Vec }, +} +impl Default for BinaryClientState { + fn default() -> Self { + Self::PartialEventLength { buf_len: 0, buf: [0; 4] } + } +} +pub struct BinaryClient { + state: BinaryClientState, + reader: std::io::BufReader, +} +impl BinaryClient { + pub fn new(addr: &str, subs: &[&[u8]]) -> Self { + let mut socket = std::net::TcpStream::connect(addr).expect("failed to connect to message bus"); + socket.set_nonblocking(true).expect("failed to set message bus socket nonblocking"); + for s in subs { + write!(socket, "s").expect("failed to send subscribe message to bus"); + socket.write_u32::(s.len() as u32).expect("failed to send subscribe message length to bus"); + socket.write_all(s).expect("failed to send subscribe message to bus"); + } + socket.flush().expect("failed to flush bus connection"); + let reader = std::io::BufReader::new(socket); + Self { state: BinaryClientState::PartialEventLength { buf_len: 0, buf: [0; 4] }, reader } + } + fn read(reader: &mut std::io::BufReader, buf: &mut [u8]) -> Option { + match reader.read(buf) { + Ok(sz) => Some(sz), + Err(e) if e.kind() == std::io::ErrorKind::WouldBlock => { + None + }, + Err(e) => panic!("IO error on message bus: {}", e), + } + } + fn update_state( + reader: &mut std::io::BufReader, + mut state: BinaryClientState, + ) -> BinaryClientState { + loop { + let new_state = match state { + BinaryClientState::PartialEventLength { mut buf_len, mut buf } => { + buf_len += if let Some(x) = Self::read(reader, &mut buf[buf_len..]) { + x + } else { break; }; + if buf_len == 4 { + let len = u32::from_le_bytes(buf) as usize; + BinaryClientState::PartialEvent { + len, + buf_len: 0, + buf: vec![0; len], + } + } else { + BinaryClientState::PartialEventLength { buf_len, buf } + } + }, + BinaryClientState::PartialEvent { len, mut buf_len, mut buf } => { + buf_len += if let Some(x) = Self::read(reader, &mut buf[buf_len..]) { + x + } else { break; }; + if buf_len == len { + BinaryClientState::PartialDataLength { + event: buf.clone(), + buf_len: 0, + buf: [0; 4], + } + } else { + BinaryClientState::PartialEvent { len, buf_len, buf } + } + }, + BinaryClientState::PartialDataLength { event, mut buf_len, mut buf } => { + buf_len += if let Some(x) = Self::read(reader, &mut buf[buf_len..]) { + x + } else { break; }; + if buf_len == 4 { + let len = u32::from_le_bytes(buf) as usize; + BinaryClientState::PartialData { + event, + len, + buf_len: 0, + buf: vec![0; len], + } + } else { + BinaryClientState::PartialDataLength { event, buf_len, buf } + } + }, + BinaryClientState::PartialData { event, len, mut buf_len, mut buf } => { + buf_len += if let Some(x) = Self::read(reader, &mut buf[buf_len..]) { + x + } else { break; }; + if buf_len == len { + BinaryClientState::Message { + event, + data: buf.clone(), + } + } else { + BinaryClientState::PartialData { event, len, buf_len, buf } + } + }, + BinaryClientState::Message{..} => break, + }; + state = new_state; + }; + state + } + pub fn pump(&mut self) -> Option { + None + // loop { + // if let Some(new) = self.update_state(mem::take(&mut self.state)) { + // self.state = new; + // } else { break; } + // } + // if let BinaryClientState::Message { event, data } = self.state.clone() { + // self.state = BinaryClientState::PartialEventLength { buf_len: 0, buf: [0; 4] }; + // Some(BinaryMessage { event, data }) + // } else { + // None + // } + } +} diff --git a/crates/renderer/src/overlay/model.rs b/crates/renderer/src/overlay/model.rs index 654b905..779f36b 100644 --- a/crates/renderer/src/overlay/model.rs +++ b/crates/renderer/src/overlay/model.rs @@ -85,8 +85,10 @@ impl Overlay { Some(()) } pub fn handle_text(&mut self, msg: fig::Message) -> Option<()> { - let s = BASE64_STANDARD.decode(msg.data.get(0)?.as_str()?).ok()?; - self.terminal.fill_string(std::str::from_utf8(&s).ok()?); + let bs = BASE64_STANDARD.decode(msg.data.get(0)?.as_str()?).ok()?; + let s = std::str::from_utf8(&bs).ok()?; + log::info!("handle_text: {}", s); + self.terminal.fill_string(s); Some(()) } pub fn handle_frame(&mut self, msg: fig::Message) -> Option<()> { diff --git a/crates/renderer/src/overlay/shader.rs b/crates/renderer/src/overlay/shader.rs index cfaaee4..181ba07 100644 --- a/crates/renderer/src/overlay/shader.rs +++ b/crates/renderer/src/overlay/shader.rs @@ -3,18 +3,22 @@ use teleia::*; use std::{collections::HashMap, f32::consts::PI}; use lexpr::sexp; use base64::prelude::*; +use device_query::DeviceQuery; + +use glow::HasContext; use crate::{assets, fig, toggle}; pub struct Chat { + author: String, msg: String, time: f64, biblicality: f32, } - impl Chat { pub fn new() -> Self { Self { + author: format!(""), msg: format!(""), time: 0.0, biblicality: 0.0, @@ -22,11 +26,96 @@ impl Chat { } } +const DRAWING_WIDTH: usize = 1920 / 4; +const DRAWING_HEIGHT: usize = 1080 / 4; +pub enum DrawingCommand { + None, + Drawing, + EraseAll, +} +pub struct Drawing { + tex: texture::Texture, + pixels: [u8; DRAWING_WIDTH * DRAWING_HEIGHT], + last_point: Option<(i32, i32)>, + shader_white: shader::Shader, +} +impl Drawing { + pub fn new(ctx: &context::Context) -> Self { + Self { + tex: texture::Texture::new_empty(ctx), + pixels: [0; DRAWING_WIDTH * DRAWING_HEIGHT], + last_point: None, + shader_white: shader::Shader::new( + ctx, + include_str!("../assets/shaders/white/vert.glsl"), + include_str!("../assets/shaders/white/frag.glsl"), + ), + } + } + pub fn coord(&self, x: usize, y: usize) -> Option { + if x > DRAWING_WIDTH || y > DRAWING_HEIGHT { + None + } else { + Some(x + y * DRAWING_WIDTH) + } + } + pub fn set(&mut self, val: u8, x: i32, y: i32) { + self.coord(x as usize, y as usize).map(|idx| self.pixels[idx] = val); + } + pub fn point(&mut self, val: u8, x: i32, y: i32) { + self.set(val, x, y - 1); + self.set(val, x - 1, y); + self.set(val, x, y); + self.set(val, x + 1, y); + self.set(val, x, y + 1); + } + pub fn line(&mut self, val: u8, (mut x0, mut y0): (i32, i32), (x1, y1): (i32, i32)) { + let dx = (x1 - x0).abs(); + let sx = if x0 < x1 { 1 } else { -1 }; + let dy = -((y1 - y0).abs()); + let sy = if y0 < y1 { 1 } else { -1 }; + let mut error = dx + dy; + loop { + self.point(val, x0, y0); + let e2 = 2 * error; + if e2 >= dy { + if x0 == x1 { break; } + error += dy; + x0 += sx; + } + if e2 <= dx { + if y0 == y1 { break; } + error += dx; + y0 += sy; + } + } + } + pub fn upload(&self, ctx: &context::Context) { + unsafe { + let err = ctx.gl.get_error(); + self.tex.bind(ctx); + ctx.gl.tex_image_2d( + glow::TEXTURE_2D, + 0, + glow::R8 as i32, + DRAWING_WIDTH as i32, + DRAWING_HEIGHT as i32, + 0, + glow::RED, + glow::UNSIGNED_BYTE, + Some(&self.pixels), + ); + ctx.gl.generate_mipmap(glow::TEXTURE_2D); + } + } +} + pub struct Overlay { assets: assets::Assets, model: scene::Scene, model_neck_base: glam::Mat4, fig: fig::Client, + fig_binary: fig::BinaryClient, tracking_eyes: (f32, f32), tracking_mouth: f32, tracking_neck: glam::Quat, @@ -37,6 +126,8 @@ pub struct Overlay { muzak_author: Option, chat: Chat, toggles: toggle::Toggles, + drawing: Drawing, + device: device_query::DeviceState, } impl Overlay { @@ -66,6 +157,9 @@ impl Overlay { sexp!((avatar overlay cursor)), sexp!((avatar overlay emacs)), ]), + fig_binary: fig::BinaryClient::new("shiro:32051", &[ + b"test event" + ]), tracking_eyes: (1.0, 1.0), tracking_mouth: 0.0, tracking_neck: glam::Quat::IDENTITY, @@ -76,6 +170,21 @@ impl Overlay { muzak_author: None, chat: Chat::new(), toggles: toggle::Toggles::new(), + drawing: Drawing::new(ctx), + device: device_query::DeviceState::new(), + } + } + fn get_mouse(&self) -> (i32, i32) { + self.device.get_mouse().coords + } + fn get_drawing_command(&mut self) -> DrawingCommand { + let keys = self.device.get_keys(); + if keys.contains(&device_query::Keycode::LMeta) { + DrawingCommand::Drawing + } else if keys.contains(&device_query::Keycode::RMeta) { + DrawingCommand::EraseAll + } else { + DrawingCommand::None } } pub fn handle_reset(&mut self, ctx: &context::Context) { @@ -125,11 +234,13 @@ impl Overlay { Some(()) } pub fn handle_overlay_chat(&mut self, msg: fig::Message) -> Option<()> { - let bs = BASE64_STANDARD.decode(msg.data.get(0)?.as_str()?).ok()?; + let ba = BASE64_STANDARD.decode(msg.data.get(0)?.as_str()?).ok()?; + let a = String::from_utf8_lossy(&ba); + let bs = BASE64_STANDARD.decode(msg.data.get(1)?.as_str()?).ok()?; let s = String::from_utf8_lossy(&bs); - let time = msg.data.get(1)?.as_str()?.parse::().ok()?; - let biblicality = msg.data.get(2)?.as_str()?.parse::().ok()?; - // log::info!("received chat message: {} {} {}", s, time, biblicality); + let time = msg.data.get(2)?.as_str()?.parse::().ok()?; + let biblicality = msg.data.get(3)?.as_str()?.parse::().ok()?; + self.chat.author = a.to_string(); self.chat.msg = s.to_string(); self.chat.time = time; self.chat.biblicality = biblicality; @@ -145,6 +256,16 @@ impl Overlay { self.emacs_heartrate = msg.data.get(0)?.as_i64()? as i32; Some(()) } + pub fn render_drawing(&self, ctx: &context::Context, st: &mut state::State) { + st.bind_2d(ctx, &self.drawing.shader_white); + self.drawing.tex.bind(ctx); + self.drawing.shader_white.set_position_2d( + ctx, + &glam::Vec2::new(0.0, 0.0), + &glam::Vec2::new(1920.0, 1080.0) + ); + self.assets.mesh_square.render(ctx); + } } impl teleia::state::Game for Overlay { @@ -161,12 +282,8 @@ impl teleia::state::Game for Overlay { &glam::Vec3::new(0.0, 0.0, -1.0), &glam::Vec3::new(0.0, 1.0, 0.0), ); - match mouse_position::mouse_position::Mouse::get_mouse_position() { - mouse_position::mouse_position::Mouse::Position { x, y } => { - self.mouse_cursor = (x as f32, y as f32); - }, - _ => {}, - } + let (x, y) = self.get_mouse(); + self.mouse_cursor = (x as f32, y as f32); while let Some(msg) = self.fig.pump() { let malformed = format!("malformed {} data: {}", msg.event, msg.data); if msg.event == sexp!((avatar tracking)) { @@ -193,6 +310,9 @@ impl teleia::state::Game for Overlay { log::info!("received unhandled event {} with data: {}", msg.event, msg.data); } } + while let Some(msg) = self.fig_binary.pump() { + log::info!("binary message: {:?}", msg); + } if let Some(n) = self.model.nodes_by_name.get("J_Bip_C_Neck").and_then(|i| self.model.nodes.get_mut(*i)) { n.transform = self.model_neck_base * glam::Mat4::from_quat(self.tracking_neck); } @@ -247,6 +367,28 @@ impl teleia::state::Game for Overlay { } let astr: String = authors.join(", "); self.assets.font.render_text(ctx, &glam::Vec2::new(0.0, 0.0), &astr); + match self.get_drawing_command() { + DrawingCommand::Drawing => { + let (sx, sy) = self.get_mouse(); + let x = sx / 4; + let y = sy / 4; + if let Some(last) = self.drawing.last_point { + self.drawing.line(1, last, (x, y)); + } else { + self.drawing.point(1, x, y); + } + self.drawing.last_point = Some((x, y)); + }, + DrawingCommand::EraseAll => { + self.drawing.pixels.fill(0); + self.drawing.last_point = None; + }, + DrawingCommand::None => { + self.drawing.last_point = None; + }, + } + self.drawing.upload(ctx); + self.render_drawing(ctx, st); Ok(()) } } -- cgit v1.2.3