From ba1ff8e7e3ea915c6d673c05cdcdee10a9b5b9f0 Mon Sep 17 00:00:00 2001 From: LLLL Colonq Date: Fri, 25 Jul 2025 20:35:51 -0400 Subject: WIP --- Cargo.lock | 86 +++++++---- crates/renderer/Cargo.toml | 3 +- crates/renderer/src/assets/shaders/white/frag.glsl | 11 ++ crates/renderer/src/assets/shaders/white/vert.glsl | 4 + crates/renderer/src/fig.rs | 133 ++++++++++++++++- crates/renderer/src/overlay/model.rs | 6 +- crates/renderer/src/overlay/shader.rs | 164 +++++++++++++++++++-- flake.nix | 16 +- launch.sh | 2 + 9 files changed, 382 insertions(+), 43 deletions(-) create mode 100644 crates/renderer/src/assets/shaders/white/frag.glsl create mode 100644 crates/renderer/src/assets/shaders/white/vert.glsl create mode 100755 launch.sh diff --git a/Cargo.lock b/Cargo.lock index e4ccb0a..c6f8c39 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -519,19 +519,6 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" -[[package]] -name = "core-graphics" -version = "0.22.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2581bbab3b8ffc6fcbd550bf46c355135d16e9ff2a6ea032ad6b9bf1d7efe4fb" -dependencies = [ - "bitflags 1.3.2", - "core-foundation", - "core-graphics-types", - "foreign-types 0.3.2", - "libc", -] - [[package]] name = "core-graphics" version = "0.23.2" @@ -596,7 +583,7 @@ dependencies = [ "wasm-bindgen", "wasm-bindgen-futures", "web-sys", - "windows", + "windows 0.54.0", ] [[package]] @@ -679,6 +666,20 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c87e182de0887fd5361989c677c4e8f5000cd9491d6d563161a8f3a5519fc7f" +[[package]] +name = "device_query" +version = "4.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7331225604b9b097b41872d550134933d4be83465d70a2c4a132b929f99aaca" +dependencies = [ + "macos-accessibility-client", + "pkg-config", + "readkey", + "readmouse", + "windows 0.48.0", + "x11", +] + [[package]] name = "directories" version = "6.0.0" @@ -1619,6 +1620,16 @@ dependencies = [ "libc", ] +[[package]] +name = "macos-accessibility-client" +version = "0.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edf7710fbff50c24124331760978fb9086d6de6288dcdb38b25a97f8b1bdebbb" +dependencies = [ + "core-foundation", + "core-foundation-sys", +] + [[package]] name = "matrixmultiply" version = "0.3.9" @@ -1692,17 +1703,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "mouse_position" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "824feb0675ad2ffda7b1da534f394c5779fb39d123b5f376a221d50fad54b3c2" -dependencies = [ - "core-graphics 0.22.3", - "winapi", - "x11-dl", -] - [[package]] name = "nalgebra" version = "0.33.2" @@ -1806,14 +1806,15 @@ version = "0.1.0" dependencies = [ "base64 0.22.1", "bitflags 2.8.0", + "byteorder", "clap", + "device_query", "env_logger", "glam", "glow", "lazy_static", "lexpr", "log", - "mouse_position", "newton_throwshade", "polling", "rand", @@ -2312,6 +2313,18 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3" +[[package]] +name = "readkey" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a36870cefdfcff57edbc0fa62165f42dfd4e5a0d8965117c1ea84c5700e4450" + +[[package]] +name = "readmouse" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be105c72a1e6a5a1198acee3d5b506a15676b74a02ecd78060042a447f408d94" + [[package]] name = "redox_syscall" version = "0.3.5" @@ -3659,6 +3672,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e686886bc078bc1b0b600cac0147aadb815089b6e4da64016cbd754b6342700f" +dependencies = [ + "windows-targets 0.48.5", +] + [[package]] name = "windows" version = "0.54.0" @@ -3946,7 +3968,7 @@ dependencies = [ "calloop", "cfg_aliases", "core-foundation", - "core-graphics 0.23.2", + "core-graphics", "cursor-icon", "icrate", "js-sys", @@ -4011,6 +4033,16 @@ version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51" +[[package]] +name = "x11" +version = "2.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "502da5464ccd04011667b11c435cb992822c2c0dbde1770c988480d312a0db2e" +dependencies = [ + "libc", + "pkg-config", +] + [[package]] name = "x11-dl" version = "2.21.0" diff --git a/crates/renderer/Cargo.toml b/crates/renderer/Cargo.toml index 37a148f..8fa87fd 100644 --- a/crates/renderer/Cargo.toml +++ b/crates/renderer/Cargo.toml @@ -25,4 +25,5 @@ clap = { version = "*", features = ["cargo"] } # cli arg parsing base64 = "0.22.1" # base64 encoding/decoding polling = "*" # polling sockets termion = "*" # terminal escapes -mouse_position = "*" # get mouse position \ No newline at end of file +device_query = "*" # get pressed keys when unfocused +byteorder = "*" # read little-endian numbers \ No newline at end of file diff --git a/crates/renderer/src/assets/shaders/white/frag.glsl b/crates/renderer/src/assets/shaders/white/frag.glsl new file mode 100644 index 0000000..6f86139 --- /dev/null +++ b/crates/renderer/src/assets/shaders/white/frag.glsl @@ -0,0 +1,11 @@ +uniform sampler2D texture_data; + +void main() +{ + vec2 tcfull = vec2(vertex_texcoord.x, 1.0 - vertex_texcoord.y); + vec4 texel = texture(texture_data, tcfull); + if (texel.r == 0.0) { + discard; + } + frag_color = vec4(1.0, 1.0, 1.0, 1.0); +} diff --git a/crates/renderer/src/assets/shaders/white/vert.glsl b/crates/renderer/src/assets/shaders/white/vert.glsl new file mode 100644 index 0000000..e324f7e --- /dev/null +++ b/crates/renderer/src/assets/shaders/white/vert.glsl @@ -0,0 +1,4 @@ +void main() +{ + default_main(); +} \ No newline at end of file diff --git a/crates/renderer/src/fig.rs b/crates/renderer/src/fig.rs index aa972a4..8023f01 100644 --- a/crates/renderer/src/fig.rs +++ b/crates/renderer/src/fig.rs @@ -1,4 +1,5 @@ -use std::io::{BufRead, Write}; +use std::io::{BufRead, Read, Write}; +use byteorder::WriteBytesExt; #[derive(Debug, Clone)] pub struct Message { @@ -48,3 +49,133 @@ impl Client { } } } + +#[derive(Debug, Clone)] +pub struct BinaryMessage { + pub event: Vec, + pub data: Vec +} +#[derive(Debug, Clone)] +pub enum BinaryClientState { + PartialEventLength { buf_len: usize, buf: [u8; 4] }, + PartialEvent { len: usize, buf_len: usize, buf: Vec }, + PartialDataLength { event: Vec, buf_len: usize, buf: [u8; 4] }, + PartialData { event: Vec, len: usize, buf_len: usize, buf: Vec }, + Message { event: Vec, data: Vec }, +} +impl Default for BinaryClientState { + fn default() -> Self { + Self::PartialEventLength { buf_len: 0, buf: [0; 4] } + } +} +pub struct BinaryClient { + state: BinaryClientState, + reader: std::io::BufReader, +} +impl BinaryClient { + pub fn new(addr: &str, subs: &[&[u8]]) -> Self { + let mut socket = std::net::TcpStream::connect(addr).expect("failed to connect to message bus"); + socket.set_nonblocking(true).expect("failed to set message bus socket nonblocking"); + for s in subs { + write!(socket, "s").expect("failed to send subscribe message to bus"); + socket.write_u32::(s.len() as u32).expect("failed to send subscribe message length to bus"); + socket.write_all(s).expect("failed to send subscribe message to bus"); + } + socket.flush().expect("failed to flush bus connection"); + let reader = std::io::BufReader::new(socket); + Self { state: BinaryClientState::PartialEventLength { buf_len: 0, buf: [0; 4] }, reader } + } + fn read(reader: &mut std::io::BufReader, buf: &mut [u8]) -> Option { + match reader.read(buf) { + Ok(sz) => Some(sz), + Err(e) if e.kind() == std::io::ErrorKind::WouldBlock => { + None + }, + Err(e) => panic!("IO error on message bus: {}", e), + } + } + fn update_state( + reader: &mut std::io::BufReader, + mut state: BinaryClientState, + ) -> BinaryClientState { + loop { + let new_state = match state { + BinaryClientState::PartialEventLength { mut buf_len, mut buf } => { + buf_len += if let Some(x) = Self::read(reader, &mut buf[buf_len..]) { + x + } else { break; }; + if buf_len == 4 { + let len = u32::from_le_bytes(buf) as usize; + BinaryClientState::PartialEvent { + len, + buf_len: 0, + buf: vec![0; len], + } + } else { + BinaryClientState::PartialEventLength { buf_len, buf } + } + }, + BinaryClientState::PartialEvent { len, mut buf_len, mut buf } => { + buf_len += if let Some(x) = Self::read(reader, &mut buf[buf_len..]) { + x + } else { break; }; + if buf_len == len { + BinaryClientState::PartialDataLength { + event: buf.clone(), + buf_len: 0, + buf: [0; 4], + } + } else { + BinaryClientState::PartialEvent { len, buf_len, buf } + } + }, + BinaryClientState::PartialDataLength { event, mut buf_len, mut buf } => { + buf_len += if let Some(x) = Self::read(reader, &mut buf[buf_len..]) { + x + } else { break; }; + if buf_len == 4 { + let len = u32::from_le_bytes(buf) as usize; + BinaryClientState::PartialData { + event, + len, + buf_len: 0, + buf: vec![0; len], + } + } else { + BinaryClientState::PartialDataLength { event, buf_len, buf } + } + }, + BinaryClientState::PartialData { event, len, mut buf_len, mut buf } => { + buf_len += if let Some(x) = Self::read(reader, &mut buf[buf_len..]) { + x + } else { break; }; + if buf_len == len { + BinaryClientState::Message { + event, + data: buf.clone(), + } + } else { + BinaryClientState::PartialData { event, len, buf_len, buf } + } + }, + BinaryClientState::Message{..} => break, + }; + state = new_state; + }; + state + } + pub fn pump(&mut self) -> Option { + None + // loop { + // if let Some(new) = self.update_state(mem::take(&mut self.state)) { + // self.state = new; + // } else { break; } + // } + // if let BinaryClientState::Message { event, data } = self.state.clone() { + // self.state = BinaryClientState::PartialEventLength { buf_len: 0, buf: [0; 4] }; + // Some(BinaryMessage { event, data }) + // } else { + // None + // } + } +} diff --git a/crates/renderer/src/overlay/model.rs b/crates/renderer/src/overlay/model.rs index 654b905..779f36b 100644 --- a/crates/renderer/src/overlay/model.rs +++ b/crates/renderer/src/overlay/model.rs @@ -85,8 +85,10 @@ impl Overlay { Some(()) } pub fn handle_text(&mut self, msg: fig::Message) -> Option<()> { - let s = BASE64_STANDARD.decode(msg.data.get(0)?.as_str()?).ok()?; - self.terminal.fill_string(std::str::from_utf8(&s).ok()?); + let bs = BASE64_STANDARD.decode(msg.data.get(0)?.as_str()?).ok()?; + let s = std::str::from_utf8(&bs).ok()?; + log::info!("handle_text: {}", s); + self.terminal.fill_string(s); Some(()) } pub fn handle_frame(&mut self, msg: fig::Message) -> Option<()> { diff --git a/crates/renderer/src/overlay/shader.rs b/crates/renderer/src/overlay/shader.rs index cfaaee4..181ba07 100644 --- a/crates/renderer/src/overlay/shader.rs +++ b/crates/renderer/src/overlay/shader.rs @@ -3,18 +3,22 @@ use teleia::*; use std::{collections::HashMap, f32::consts::PI}; use lexpr::sexp; use base64::prelude::*; +use device_query::DeviceQuery; + +use glow::HasContext; use crate::{assets, fig, toggle}; pub struct Chat { + author: String, msg: String, time: f64, biblicality: f32, } - impl Chat { pub fn new() -> Self { Self { + author: format!(""), msg: format!(""), time: 0.0, biblicality: 0.0, @@ -22,11 +26,96 @@ impl Chat { } } +const DRAWING_WIDTH: usize = 1920 / 4; +const DRAWING_HEIGHT: usize = 1080 / 4; +pub enum DrawingCommand { + None, + Drawing, + EraseAll, +} +pub struct Drawing { + tex: texture::Texture, + pixels: [u8; DRAWING_WIDTH * DRAWING_HEIGHT], + last_point: Option<(i32, i32)>, + shader_white: shader::Shader, +} +impl Drawing { + pub fn new(ctx: &context::Context) -> Self { + Self { + tex: texture::Texture::new_empty(ctx), + pixels: [0; DRAWING_WIDTH * DRAWING_HEIGHT], + last_point: None, + shader_white: shader::Shader::new( + ctx, + include_str!("../assets/shaders/white/vert.glsl"), + include_str!("../assets/shaders/white/frag.glsl"), + ), + } + } + pub fn coord(&self, x: usize, y: usize) -> Option { + if x > DRAWING_WIDTH || y > DRAWING_HEIGHT { + None + } else { + Some(x + y * DRAWING_WIDTH) + } + } + pub fn set(&mut self, val: u8, x: i32, y: i32) { + self.coord(x as usize, y as usize).map(|idx| self.pixels[idx] = val); + } + pub fn point(&mut self, val: u8, x: i32, y: i32) { + self.set(val, x, y - 1); + self.set(val, x - 1, y); + self.set(val, x, y); + self.set(val, x + 1, y); + self.set(val, x, y + 1); + } + pub fn line(&mut self, val: u8, (mut x0, mut y0): (i32, i32), (x1, y1): (i32, i32)) { + let dx = (x1 - x0).abs(); + let sx = if x0 < x1 { 1 } else { -1 }; + let dy = -((y1 - y0).abs()); + let sy = if y0 < y1 { 1 } else { -1 }; + let mut error = dx + dy; + loop { + self.point(val, x0, y0); + let e2 = 2 * error; + if e2 >= dy { + if x0 == x1 { break; } + error += dy; + x0 += sx; + } + if e2 <= dx { + if y0 == y1 { break; } + error += dx; + y0 += sy; + } + } + } + pub fn upload(&self, ctx: &context::Context) { + unsafe { + let err = ctx.gl.get_error(); + self.tex.bind(ctx); + ctx.gl.tex_image_2d( + glow::TEXTURE_2D, + 0, + glow::R8 as i32, + DRAWING_WIDTH as i32, + DRAWING_HEIGHT as i32, + 0, + glow::RED, + glow::UNSIGNED_BYTE, + Some(&self.pixels), + ); + ctx.gl.generate_mipmap(glow::TEXTURE_2D); + } + } +} + pub struct Overlay { assets: assets::Assets, model: scene::Scene, model_neck_base: glam::Mat4, fig: fig::Client, + fig_binary: fig::BinaryClient, tracking_eyes: (f32, f32), tracking_mouth: f32, tracking_neck: glam::Quat, @@ -37,6 +126,8 @@ pub struct Overlay { muzak_author: Option, chat: Chat, toggles: toggle::Toggles, + drawing: Drawing, + device: device_query::DeviceState, } impl Overlay { @@ -66,6 +157,9 @@ impl Overlay { sexp!((avatar overlay cursor)), sexp!((avatar overlay emacs)), ]), + fig_binary: fig::BinaryClient::new("shiro:32051", &[ + b"test event" + ]), tracking_eyes: (1.0, 1.0), tracking_mouth: 0.0, tracking_neck: glam::Quat::IDENTITY, @@ -76,6 +170,21 @@ impl Overlay { muzak_author: None, chat: Chat::new(), toggles: toggle::Toggles::new(), + drawing: Drawing::new(ctx), + device: device_query::DeviceState::new(), + } + } + fn get_mouse(&self) -> (i32, i32) { + self.device.get_mouse().coords + } + fn get_drawing_command(&mut self) -> DrawingCommand { + let keys = self.device.get_keys(); + if keys.contains(&device_query::Keycode::LMeta) { + DrawingCommand::Drawing + } else if keys.contains(&device_query::Keycode::RMeta) { + DrawingCommand::EraseAll + } else { + DrawingCommand::None } } pub fn handle_reset(&mut self, ctx: &context::Context) { @@ -125,11 +234,13 @@ impl Overlay { Some(()) } pub fn handle_overlay_chat(&mut self, msg: fig::Message) -> Option<()> { - let bs = BASE64_STANDARD.decode(msg.data.get(0)?.as_str()?).ok()?; + let ba = BASE64_STANDARD.decode(msg.data.get(0)?.as_str()?).ok()?; + let a = String::from_utf8_lossy(&ba); + let bs = BASE64_STANDARD.decode(msg.data.get(1)?.as_str()?).ok()?; let s = String::from_utf8_lossy(&bs); - let time = msg.data.get(1)?.as_str()?.parse::().ok()?; - let biblicality = msg.data.get(2)?.as_str()?.parse::().ok()?; - // log::info!("received chat message: {} {} {}", s, time, biblicality); + let time = msg.data.get(2)?.as_str()?.parse::().ok()?; + let biblicality = msg.data.get(3)?.as_str()?.parse::().ok()?; + self.chat.author = a.to_string(); self.chat.msg = s.to_string(); self.chat.time = time; self.chat.biblicality = biblicality; @@ -145,6 +256,16 @@ impl Overlay { self.emacs_heartrate = msg.data.get(0)?.as_i64()? as i32; Some(()) } + pub fn render_drawing(&self, ctx: &context::Context, st: &mut state::State) { + st.bind_2d(ctx, &self.drawing.shader_white); + self.drawing.tex.bind(ctx); + self.drawing.shader_white.set_position_2d( + ctx, + &glam::Vec2::new(0.0, 0.0), + &glam::Vec2::new(1920.0, 1080.0) + ); + self.assets.mesh_square.render(ctx); + } } impl teleia::state::Game for Overlay { @@ -161,12 +282,8 @@ impl teleia::state::Game for Overlay { &glam::Vec3::new(0.0, 0.0, -1.0), &glam::Vec3::new(0.0, 1.0, 0.0), ); - match mouse_position::mouse_position::Mouse::get_mouse_position() { - mouse_position::mouse_position::Mouse::Position { x, y } => { - self.mouse_cursor = (x as f32, y as f32); - }, - _ => {}, - } + let (x, y) = self.get_mouse(); + self.mouse_cursor = (x as f32, y as f32); while let Some(msg) = self.fig.pump() { let malformed = format!("malformed {} data: {}", msg.event, msg.data); if msg.event == sexp!((avatar tracking)) { @@ -193,6 +310,9 @@ impl teleia::state::Game for Overlay { log::info!("received unhandled event {} with data: {}", msg.event, msg.data); } } + while let Some(msg) = self.fig_binary.pump() { + log::info!("binary message: {:?}", msg); + } if let Some(n) = self.model.nodes_by_name.get("J_Bip_C_Neck").and_then(|i| self.model.nodes.get_mut(*i)) { n.transform = self.model_neck_base * glam::Mat4::from_quat(self.tracking_neck); } @@ -247,6 +367,28 @@ impl teleia::state::Game for Overlay { } let astr: String = authors.join(", "); self.assets.font.render_text(ctx, &glam::Vec2::new(0.0, 0.0), &astr); + match self.get_drawing_command() { + DrawingCommand::Drawing => { + let (sx, sy) = self.get_mouse(); + let x = sx / 4; + let y = sy / 4; + if let Some(last) = self.drawing.last_point { + self.drawing.line(1, last, (x, y)); + } else { + self.drawing.point(1, x, y); + } + self.drawing.last_point = Some((x, y)); + }, + DrawingCommand::EraseAll => { + self.drawing.pixels.fill(0); + self.drawing.last_point = None; + }, + DrawingCommand::None => { + self.drawing.last_point = None; + }, + } + self.drawing.upload(ctx); + self.render_drawing(ctx, st); Ok(()) } } diff --git a/flake.nix b/flake.nix index d1d8c3a..f9d8a99 100644 --- a/flake.nix +++ b/flake.nix @@ -11,8 +11,22 @@ outputs = inputs@{ self, nixpkgs, ... }: let system = "x86_64-linux"; + pkgs = import nixpkgs { inherit system; }; native = { - renderer = inputs.teleia.native.build ./. "renderer"; + renderer = inputs.teleia.native.build ./. "newton_renderer"; + renderer-nonnix = pkgs.stdenv.mkDerivation { + name = "newton-renderer-nonnix"; + phases = [ "installPhase" ]; + installPhase = '' + mkdir $out + cp -rv ${native.renderer}/bin/newton_renderer $out + chmod +w $out/newton_renderer + patchelf --remove-rpath $out/newton_renderer + patchelf --set-interpreter /lib64/ld-linux-x86-64.so.2 $out/newton_renderer + strip $out/newton_renderer + chmod -w $out/newton_renderer + ''; + }; }; wasm = { throwshade = (inputs.teleia.wasm.build ./. "newton_throwshade").overrideAttrs (cur: prev: { diff --git a/launch.sh b/launch.sh new file mode 100755 index 0000000..7bf513f --- /dev/null +++ b/launch.sh @@ -0,0 +1,2 @@ +#!/bin/sh +cargo run -p newton_renderer -- shader-overlay -- cgit v1.2.3