diff options
| -rw-r--r-- | Cargo.lock | 86 | ||||
| -rw-r--r-- | crates/renderer/Cargo.toml | 3 | ||||
| -rw-r--r-- | crates/renderer/src/assets/shaders/white/frag.glsl | 11 | ||||
| -rw-r--r-- | crates/renderer/src/assets/shaders/white/vert.glsl | 4 | ||||
| -rw-r--r-- | crates/renderer/src/fig.rs | 133 | ||||
| -rw-r--r-- | crates/renderer/src/overlay/model.rs | 6 | ||||
| -rw-r--r-- | crates/renderer/src/overlay/shader.rs | 164 | ||||
| -rw-r--r-- | flake.nix | 16 | ||||
| -rwxr-xr-x | launch.sh | 2 |
9 files changed, 382 insertions, 43 deletions
@@ -521,19 +521,6 @@ checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] name = "core-graphics" -version = "0.22.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2581bbab3b8ffc6fcbd550bf46c355135d16e9ff2a6ea032ad6b9bf1d7efe4fb" -dependencies = [ - "bitflags 1.3.2", - "core-foundation", - "core-graphics-types", - "foreign-types 0.3.2", - "libc", -] - -[[package]] -name = "core-graphics" version = "0.23.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c07782be35f9e1140080c6b96f0d44b739e2278479f64e02fdab4e32dfd8b081" @@ -596,7 +583,7 @@ dependencies = [ "wasm-bindgen", "wasm-bindgen-futures", "web-sys", - "windows", + "windows 0.54.0", ] [[package]] @@ -680,6 +667,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c87e182de0887fd5361989c677c4e8f5000cd9491d6d563161a8f3a5519fc7f" [[package]] +name = "device_query" +version = "4.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7331225604b9b097b41872d550134933d4be83465d70a2c4a132b929f99aaca" +dependencies = [ + "macos-accessibility-client", + "pkg-config", + "readkey", + "readmouse", + "windows 0.48.0", + "x11", +] + +[[package]] name = "directories" version = "6.0.0" source = "git+https://github.com/lcolonq/directories-rs#164536daab2c07265167715934081cb1942387b0" @@ -1620,6 +1621,16 @@ dependencies = [ ] [[package]] +name = "macos-accessibility-client" +version = "0.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edf7710fbff50c24124331760978fb9086d6de6288dcdb38b25a97f8b1bdebbb" +dependencies = [ + "core-foundation", + "core-foundation-sys", +] + +[[package]] name = "matrixmultiply" version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1693,17 +1704,6 @@ dependencies = [ ] [[package]] -name = "mouse_position" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "824feb0675ad2ffda7b1da534f394c5779fb39d123b5f376a221d50fad54b3c2" -dependencies = [ - "core-graphics 0.22.3", - "winapi", - "x11-dl", -] - -[[package]] name = "nalgebra" version = "0.33.2" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1806,14 +1806,15 @@ version = "0.1.0" dependencies = [ "base64 0.22.1", "bitflags 2.8.0", + "byteorder", "clap", + "device_query", "env_logger", "glam", "glow", "lazy_static", "lexpr", "log", - "mouse_position", "newton_throwshade", "polling", "rand", @@ -2313,6 +2314,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3" [[package]] +name = "readkey" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a36870cefdfcff57edbc0fa62165f42dfd4e5a0d8965117c1ea84c5700e4450" + +[[package]] +name = "readmouse" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be105c72a1e6a5a1198acee3d5b506a15676b74a02ecd78060042a447f408d94" + +[[package]] name = "redox_syscall" version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -3661,6 +3674,15 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windows" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e686886bc078bc1b0b600cac0147aadb815089b6e4da64016cbd754b6342700f" +dependencies = [ + "windows-targets 0.48.5", +] + +[[package]] +name = "windows" version = "0.54.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9252e5725dbed82865af151df558e754e4a3c2c30818359eb17465f1346a1b49" @@ -3946,7 +3968,7 @@ dependencies = [ "calloop", "cfg_aliases", "core-foundation", - "core-graphics 0.23.2", + "core-graphics", "cursor-icon", "icrate", "js-sys", @@ -4012,6 +4034,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51" [[package]] +name = "x11" +version = "2.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "502da5464ccd04011667b11c435cb992822c2c0dbde1770c988480d312a0db2e" +dependencies = [ + "libc", + "pkg-config", +] + +[[package]] name = "x11-dl" version = "2.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" diff --git a/crates/renderer/Cargo.toml b/crates/renderer/Cargo.toml index 37a148f..8fa87fd 100644 --- a/crates/renderer/Cargo.toml +++ b/crates/renderer/Cargo.toml @@ -25,4 +25,5 @@ clap = { version = "*", features = ["cargo"] } # cli arg parsing base64 = "0.22.1" # base64 encoding/decoding polling = "*" # polling sockets termion = "*" # terminal escapes -mouse_position = "*" # get mouse position
\ No newline at end of file +device_query = "*" # get pressed keys when unfocused +byteorder = "*" # read little-endian numbers
\ No newline at end of file diff --git a/crates/renderer/src/assets/shaders/white/frag.glsl b/crates/renderer/src/assets/shaders/white/frag.glsl new file mode 100644 index 0000000..6f86139 --- /dev/null +++ b/crates/renderer/src/assets/shaders/white/frag.glsl @@ -0,0 +1,11 @@ +uniform sampler2D texture_data; + +void main() +{ + vec2 tcfull = vec2(vertex_texcoord.x, 1.0 - vertex_texcoord.y); + vec4 texel = texture(texture_data, tcfull); + if (texel.r == 0.0) { + discard; + } + frag_color = vec4(1.0, 1.0, 1.0, 1.0); +} diff --git a/crates/renderer/src/assets/shaders/white/vert.glsl b/crates/renderer/src/assets/shaders/white/vert.glsl new file mode 100644 index 0000000..e324f7e --- /dev/null +++ b/crates/renderer/src/assets/shaders/white/vert.glsl @@ -0,0 +1,4 @@ +void main() +{ + default_main(); +}
\ No newline at end of file diff --git a/crates/renderer/src/fig.rs b/crates/renderer/src/fig.rs index aa972a4..8023f01 100644 --- a/crates/renderer/src/fig.rs +++ b/crates/renderer/src/fig.rs @@ -1,4 +1,5 @@ -use std::io::{BufRead, Write}; +use std::io::{BufRead, Read, Write}; +use byteorder::WriteBytesExt; #[derive(Debug, Clone)] pub struct Message { @@ -48,3 +49,133 @@ impl Client { } } } + +#[derive(Debug, Clone)] +pub struct BinaryMessage { + pub event: Vec<u8>, + pub data: Vec<u8> +} +#[derive(Debug, Clone)] +pub enum BinaryClientState { + PartialEventLength { buf_len: usize, buf: [u8; 4] }, + PartialEvent { len: usize, buf_len: usize, buf: Vec<u8> }, + PartialDataLength { event: Vec<u8>, buf_len: usize, buf: [u8; 4] }, + PartialData { event: Vec<u8>, len: usize, buf_len: usize, buf: Vec<u8> }, + Message { event: Vec<u8>, data: Vec<u8> }, +} +impl Default for BinaryClientState { + fn default() -> Self { + Self::PartialEventLength { buf_len: 0, buf: [0; 4] } + } +} +pub struct BinaryClient { + state: BinaryClientState, + reader: std::io::BufReader<std::net::TcpStream>, +} +impl BinaryClient { + pub fn new(addr: &str, subs: &[&[u8]]) -> Self { + let mut socket = std::net::TcpStream::connect(addr).expect("failed to connect to message bus"); + socket.set_nonblocking(true).expect("failed to set message bus socket nonblocking"); + for s in subs { + write!(socket, "s").expect("failed to send subscribe message to bus"); + socket.write_u32::<byteorder::LE>(s.len() as u32).expect("failed to send subscribe message length to bus"); + socket.write_all(s).expect("failed to send subscribe message to bus"); + } + socket.flush().expect("failed to flush bus connection"); + let reader = std::io::BufReader::new(socket); + Self { state: BinaryClientState::PartialEventLength { buf_len: 0, buf: [0; 4] }, reader } + } + fn read(reader: &mut std::io::BufReader<std::net::TcpStream>, buf: &mut [u8]) -> Option<usize> { + match reader.read(buf) { + Ok(sz) => Some(sz), + Err(e) if e.kind() == std::io::ErrorKind::WouldBlock => { + None + }, + Err(e) => panic!("IO error on message bus: {}", e), + } + } + fn update_state( + reader: &mut std::io::BufReader<std::net::TcpStream>, + mut state: BinaryClientState, + ) -> BinaryClientState { + loop { + let new_state = match state { + BinaryClientState::PartialEventLength { mut buf_len, mut buf } => { + buf_len += if let Some(x) = Self::read(reader, &mut buf[buf_len..]) { + x + } else { break; }; + if buf_len == 4 { + let len = u32::from_le_bytes(buf) as usize; + BinaryClientState::PartialEvent { + len, + buf_len: 0, + buf: vec![0; len], + } + } else { + BinaryClientState::PartialEventLength { buf_len, buf } + } + }, + BinaryClientState::PartialEvent { len, mut buf_len, mut buf } => { + buf_len += if let Some(x) = Self::read(reader, &mut buf[buf_len..]) { + x + } else { break; }; + if buf_len == len { + BinaryClientState::PartialDataLength { + event: buf.clone(), + buf_len: 0, + buf: [0; 4], + } + } else { + BinaryClientState::PartialEvent { len, buf_len, buf } + } + }, + BinaryClientState::PartialDataLength { event, mut buf_len, mut buf } => { + buf_len += if let Some(x) = Self::read(reader, &mut buf[buf_len..]) { + x + } else { break; }; + if buf_len == 4 { + let len = u32::from_le_bytes(buf) as usize; + BinaryClientState::PartialData { + event, + len, + buf_len: 0, + buf: vec![0; len], + } + } else { + BinaryClientState::PartialDataLength { event, buf_len, buf } + } + }, + BinaryClientState::PartialData { event, len, mut buf_len, mut buf } => { + buf_len += if let Some(x) = Self::read(reader, &mut buf[buf_len..]) { + x + } else { break; }; + if buf_len == len { + BinaryClientState::Message { + event, + data: buf.clone(), + } + } else { + BinaryClientState::PartialData { event, len, buf_len, buf } + } + }, + BinaryClientState::Message{..} => break, + }; + state = new_state; + }; + state + } + pub fn pump(&mut self) -> Option<BinaryMessage> { + None + // loop { + // if let Some(new) = self.update_state(mem::take(&mut self.state)) { + // self.state = new; + // } else { break; } + // } + // if let BinaryClientState::Message { event, data } = self.state.clone() { + // self.state = BinaryClientState::PartialEventLength { buf_len: 0, buf: [0; 4] }; + // Some(BinaryMessage { event, data }) + // } else { + // None + // } + } +} diff --git a/crates/renderer/src/overlay/model.rs b/crates/renderer/src/overlay/model.rs index 654b905..779f36b 100644 --- a/crates/renderer/src/overlay/model.rs +++ b/crates/renderer/src/overlay/model.rs @@ -85,8 +85,10 @@ impl Overlay { Some(()) } pub fn handle_text(&mut self, msg: fig::Message) -> Option<()> { - let s = BASE64_STANDARD.decode(msg.data.get(0)?.as_str()?).ok()?; - self.terminal.fill_string(std::str::from_utf8(&s).ok()?); + let bs = BASE64_STANDARD.decode(msg.data.get(0)?.as_str()?).ok()?; + let s = std::str::from_utf8(&bs).ok()?; + log::info!("handle_text: {}", s); + self.terminal.fill_string(s); Some(()) } pub fn handle_frame(&mut self, msg: fig::Message) -> Option<()> { diff --git a/crates/renderer/src/overlay/shader.rs b/crates/renderer/src/overlay/shader.rs index cfaaee4..181ba07 100644 --- a/crates/renderer/src/overlay/shader.rs +++ b/crates/renderer/src/overlay/shader.rs @@ -3,18 +3,22 @@ use teleia::*; use std::{collections::HashMap, f32::consts::PI}; use lexpr::sexp; use base64::prelude::*; +use device_query::DeviceQuery; + +use glow::HasContext; use crate::{assets, fig, toggle}; pub struct Chat { + author: String, msg: String, time: f64, biblicality: f32, } - impl Chat { pub fn new() -> Self { Self { + author: format!(""), msg: format!(""), time: 0.0, biblicality: 0.0, @@ -22,11 +26,96 @@ impl Chat { } } +const DRAWING_WIDTH: usize = 1920 / 4; +const DRAWING_HEIGHT: usize = 1080 / 4; +pub enum DrawingCommand { + None, + Drawing, + EraseAll, +} +pub struct Drawing { + tex: texture::Texture, + pixels: [u8; DRAWING_WIDTH * DRAWING_HEIGHT], + last_point: Option<(i32, i32)>, + shader_white: shader::Shader, +} +impl Drawing { + pub fn new(ctx: &context::Context) -> Self { + Self { + tex: texture::Texture::new_empty(ctx), + pixels: [0; DRAWING_WIDTH * DRAWING_HEIGHT], + last_point: None, + shader_white: shader::Shader::new( + ctx, + include_str!("../assets/shaders/white/vert.glsl"), + include_str!("../assets/shaders/white/frag.glsl"), + ), + } + } + pub fn coord(&self, x: usize, y: usize) -> Option<usize> { + if x > DRAWING_WIDTH || y > DRAWING_HEIGHT { + None + } else { + Some(x + y * DRAWING_WIDTH) + } + } + pub fn set(&mut self, val: u8, x: i32, y: i32) { + self.coord(x as usize, y as usize).map(|idx| self.pixels[idx] = val); + } + pub fn point(&mut self, val: u8, x: i32, y: i32) { + self.set(val, x, y - 1); + self.set(val, x - 1, y); + self.set(val, x, y); + self.set(val, x + 1, y); + self.set(val, x, y + 1); + } + pub fn line(&mut self, val: u8, (mut x0, mut y0): (i32, i32), (x1, y1): (i32, i32)) { + let dx = (x1 - x0).abs(); + let sx = if x0 < x1 { 1 } else { -1 }; + let dy = -((y1 - y0).abs()); + let sy = if y0 < y1 { 1 } else { -1 }; + let mut error = dx + dy; + loop { + self.point(val, x0, y0); + let e2 = 2 * error; + if e2 >= dy { + if x0 == x1 { break; } + error += dy; + x0 += sx; + } + if e2 <= dx { + if y0 == y1 { break; } + error += dx; + y0 += sy; + } + } + } + pub fn upload(&self, ctx: &context::Context) { + unsafe { + let err = ctx.gl.get_error(); + self.tex.bind(ctx); + ctx.gl.tex_image_2d( + glow::TEXTURE_2D, + 0, + glow::R8 as i32, + DRAWING_WIDTH as i32, + DRAWING_HEIGHT as i32, + 0, + glow::RED, + glow::UNSIGNED_BYTE, + Some(&self.pixels), + ); + ctx.gl.generate_mipmap(glow::TEXTURE_2D); + } + } +} + pub struct Overlay { assets: assets::Assets, model: scene::Scene, model_neck_base: glam::Mat4, fig: fig::Client, + fig_binary: fig::BinaryClient, tracking_eyes: (f32, f32), tracking_mouth: f32, tracking_neck: glam::Quat, @@ -37,6 +126,8 @@ pub struct Overlay { muzak_author: Option<String>, chat: Chat, toggles: toggle::Toggles, + drawing: Drawing, + device: device_query::DeviceState, } impl Overlay { @@ -66,6 +157,9 @@ impl Overlay { sexp!((avatar overlay cursor)), sexp!((avatar overlay emacs)), ]), + fig_binary: fig::BinaryClient::new("shiro:32051", &[ + b"test event" + ]), tracking_eyes: (1.0, 1.0), tracking_mouth: 0.0, tracking_neck: glam::Quat::IDENTITY, @@ -76,6 +170,21 @@ impl Overlay { muzak_author: None, chat: Chat::new(), toggles: toggle::Toggles::new(), + drawing: Drawing::new(ctx), + device: device_query::DeviceState::new(), + } + } + fn get_mouse(&self) -> (i32, i32) { + self.device.get_mouse().coords + } + fn get_drawing_command(&mut self) -> DrawingCommand { + let keys = self.device.get_keys(); + if keys.contains(&device_query::Keycode::LMeta) { + DrawingCommand::Drawing + } else if keys.contains(&device_query::Keycode::RMeta) { + DrawingCommand::EraseAll + } else { + DrawingCommand::None } } pub fn handle_reset(&mut self, ctx: &context::Context) { @@ -125,11 +234,13 @@ impl Overlay { Some(()) } pub fn handle_overlay_chat(&mut self, msg: fig::Message) -> Option<()> { - let bs = BASE64_STANDARD.decode(msg.data.get(0)?.as_str()?).ok()?; + let ba = BASE64_STANDARD.decode(msg.data.get(0)?.as_str()?).ok()?; + let a = String::from_utf8_lossy(&ba); + let bs = BASE64_STANDARD.decode(msg.data.get(1)?.as_str()?).ok()?; let s = String::from_utf8_lossy(&bs); - let time = msg.data.get(1)?.as_str()?.parse::<f64>().ok()?; - let biblicality = msg.data.get(2)?.as_str()?.parse::<f32>().ok()?; - // log::info!("received chat message: {} {} {}", s, time, biblicality); + let time = msg.data.get(2)?.as_str()?.parse::<f64>().ok()?; + let biblicality = msg.data.get(3)?.as_str()?.parse::<f32>().ok()?; + self.chat.author = a.to_string(); self.chat.msg = s.to_string(); self.chat.time = time; self.chat.biblicality = biblicality; @@ -145,6 +256,16 @@ impl Overlay { self.emacs_heartrate = msg.data.get(0)?.as_i64()? as i32; Some(()) } + pub fn render_drawing(&self, ctx: &context::Context, st: &mut state::State) { + st.bind_2d(ctx, &self.drawing.shader_white); + self.drawing.tex.bind(ctx); + self.drawing.shader_white.set_position_2d( + ctx, + &glam::Vec2::new(0.0, 0.0), + &glam::Vec2::new(1920.0, 1080.0) + ); + self.assets.mesh_square.render(ctx); + } } impl teleia::state::Game for Overlay { @@ -161,12 +282,8 @@ impl teleia::state::Game for Overlay { &glam::Vec3::new(0.0, 0.0, -1.0), &glam::Vec3::new(0.0, 1.0, 0.0), ); - match mouse_position::mouse_position::Mouse::get_mouse_position() { - mouse_position::mouse_position::Mouse::Position { x, y } => { - self.mouse_cursor = (x as f32, y as f32); - }, - _ => {}, - } + let (x, y) = self.get_mouse(); + self.mouse_cursor = (x as f32, y as f32); while let Some(msg) = self.fig.pump() { let malformed = format!("malformed {} data: {}", msg.event, msg.data); if msg.event == sexp!((avatar tracking)) { @@ -193,6 +310,9 @@ impl teleia::state::Game for Overlay { log::info!("received unhandled event {} with data: {}", msg.event, msg.data); } } + while let Some(msg) = self.fig_binary.pump() { + log::info!("binary message: {:?}", msg); + } if let Some(n) = self.model.nodes_by_name.get("J_Bip_C_Neck").and_then(|i| self.model.nodes.get_mut(*i)) { n.transform = self.model_neck_base * glam::Mat4::from_quat(self.tracking_neck); } @@ -247,6 +367,28 @@ impl teleia::state::Game for Overlay { } let astr: String = authors.join(", "); self.assets.font.render_text(ctx, &glam::Vec2::new(0.0, 0.0), &astr); + match self.get_drawing_command() { + DrawingCommand::Drawing => { + let (sx, sy) = self.get_mouse(); + let x = sx / 4; + let y = sy / 4; + if let Some(last) = self.drawing.last_point { + self.drawing.line(1, last, (x, y)); + } else { + self.drawing.point(1, x, y); + } + self.drawing.last_point = Some((x, y)); + }, + DrawingCommand::EraseAll => { + self.drawing.pixels.fill(0); + self.drawing.last_point = None; + }, + DrawingCommand::None => { + self.drawing.last_point = None; + }, + } + self.drawing.upload(ctx); + self.render_drawing(ctx, st); Ok(()) } } @@ -11,8 +11,22 @@ outputs = inputs@{ self, nixpkgs, ... }: let system = "x86_64-linux"; + pkgs = import nixpkgs { inherit system; }; native = { - renderer = inputs.teleia.native.build ./. "renderer"; + renderer = inputs.teleia.native.build ./. "newton_renderer"; + renderer-nonnix = pkgs.stdenv.mkDerivation { + name = "newton-renderer-nonnix"; + phases = [ "installPhase" ]; + installPhase = '' + mkdir $out + cp -rv ${native.renderer}/bin/newton_renderer $out + chmod +w $out/newton_renderer + patchelf --remove-rpath $out/newton_renderer + patchelf --set-interpreter /lib64/ld-linux-x86-64.so.2 $out/newton_renderer + strip $out/newton_renderer + chmod -w $out/newton_renderer + ''; + }; }; wasm = { throwshade = (inputs.teleia.wasm.build ./. "newton_throwshade").overrideAttrs (cur: prev: { diff --git a/launch.sh b/launch.sh new file mode 100755 index 0000000..7bf513f --- /dev/null +++ b/launch.sh @@ -0,0 +1,2 @@ +#!/bin/sh +cargo run -p newton_renderer -- shader-overlay |