Automata rendering

15 files changed, 461 insertions, 367 deletions

diff --git a/crates/renderer/src/assets.rs b/crates/renderer/src/assets.rs
index 3c129c1..83b5a0d 100644
--- a/crates/renderer/src/assets.rs
+++ b/crates/renderer/src/assets.rs
@@ -4,7 +4,6 @@ pub struct Assets {
     pub font: font::Bitmap,
     pub shader_flat: shader::Shader,
     pub shader_scene: shader::Shader,
-    pub mesh_square: mesh::Mesh,
     pub texture_adblock: texture::Texture,
     pub texture_mod: texture::Texture,
     pub texture_operatop: texture::Texture,
@@ -25,7 +24,6 @@ impl Assets {
                 include_str!("assets/shaders/scene/vert.glsl"),
                 include_str!("assets/shaders/scene/frag.glsl")
             ),
-            mesh_square: mesh::Mesh::from_obj(ctx, include_bytes!("assets/meshes/square.obj")),
             texture_adblock: texture::Texture::new(ctx, include_bytes!("assets/textures/adblock.png")),
             texture_mod: texture::Texture::new(ctx, include_bytes!("assets/textures/mod.png")),
             texture_operatop: texture::Texture::new(ctx, include_bytes!("assets/textures/operatop.png")),
diff --git a/crates/renderer/src/assets/meshes/square.obj b/crates/renderer/src/assets/meshes/square.obj
deleted file mode 100644
index 7328a6c..0000000
--- a/crates/renderer/src/assets/meshes/square.obj
+++ /dev/null
@@ -1,15 +0,0 @@
-# Blender 3.6.2
-# www.blender.org
-o Cube
-v -1.000000 -1.000000 0.000000
-v 1.000000 -1.000000 0.000000
-v -1.000000 1.000000 0.000000
-v 1.000000 1.000000 0.000000
-vn -0.0000 -0.0000 -1.0000
-vt 0.0 1.0
-vt 1.0 0.0
-vt 0.0 0.0
-vt 1.0 1.0
-s 0
-f 3/1/1 2/2/1 1/3/1
-f 3/1/1 4/4/1 2/2/1
diff --git a/crates/renderer/src/assets/shaders/automata/frag.glsl b/crates/renderer/src/assets/shaders/automata/frag.glsl
new file mode 100644
index 0000000..7eda644
--- /dev/null
+++ b/crates/renderer/src/assets/shaders/automata/frag.glsl
@@ -0,0 +1,12 @@
+uniform sampler2D texture_data;
+uniform sampler2D background;
+
+void main()
+{
+    vec2 tcfull = vec2(vertex_texcoord.x, 1.0 - vertex_texcoord.y);
+    vec4 texel = texture(texture_data, tcfull);
+    if (texel.r == 0.0) {
+        discard;
+    }
+    frag_color = vec4(1.0, 1.0, 1.0, 1.0);
+}
diff --git a/crates/renderer/src/assets/shaders/automata/vert.glsl b/crates/renderer/src/assets/shaders/automata/vert.glsl
new file mode 100644
index 0000000..33e176c
--- /dev/null
+++ b/crates/renderer/src/assets/shaders/automata/vert.glsl
@@ -0,0 +1,4 @@
+void main()
+{
+    default_main();
+}
diff --git a/crates/renderer/src/background.rs b/crates/renderer/src/background.rs
index cf99fc9..4f483b4 100644
--- a/crates/renderer/src/background.rs
+++ b/crates/renderer/src/background.rs
@@ -13,19 +13,19 @@ pub struct Frame<'a> {
 impl<'a> Frame<'a> {
     fn read_length_prefixed(reader: &mut &'a [u8]) -> Option<&'a [u8]> {
         let len = reader.read_u32::<byteorder::LE>().ok()? as usize;
-        log::info!("len: {}", len);
+        // log::info!("len: {}", len);
         let (x, xs) = reader.split_at(len);
         *reader = xs;
         Some(x)
     }
     pub fn parse(reader: &mut &'a [u8]) -> Option<Self> {
-        log::info!("message: {:?}", reader);
+        // log::info!("message: {:?}", reader);
         let tag = Self::read_length_prefixed(reader)?;
-        log::info!("tag: {:?}", tag);
+        // log::info!("tag: {:?}", tag);
         let width = reader.read_u32::<byteorder::LE>().ok()?;
-        log::info!("width: {:?}", width);
+        // log::info!("width: {:?}", width);
         let height = reader.read_u32::<byteorder::LE>().ok()?;
-        log::info!("height: {:?}", height);
+        // log::info!("height: {:?}", height);
         let pixels = *reader;
         Some(Self { tag, width, height, pixels })
     }
diff --git a/crates/renderer/src/fig.rs b/crates/renderer/src/fig.rs
deleted file mode 100644
index a696a9a..0000000
--- a/crates/renderer/src/fig.rs
+++ /dev/null
@@ -1,173 +0,0 @@
-use std::io::{BufRead, Read, Write};
-use byteorder::WriteBytesExt;
-
-#[derive(Debug, Clone)]
-pub struct Message {
-    pub event: lexpr::Value,
-    pub data: lexpr::Value,
-}
-
-pub struct Client {
-    reader: std::io::BufReader<std::net::TcpStream>,
-    buf: String,
-}
-impl Client {
-    pub fn new(addr: &str, subs: &[lexpr::Value]) -> Self {
-        let mut socket = std::net::TcpStream::connect(addr).expect("failed to connect to message bus");
-        socket.set_nonblocking(true).expect("failed to set message bus socket nonblocking");
-        for s in subs {
-            write!(socket, "(sub {})\n", s).expect("failed to send subscribe message to bus");
-        }
-        let reader = std::io::BufReader::new(socket);
-        Self { reader, buf: String::new(), }
-    }
-    pub fn pump(&mut self) -> Option<Message> {
-        match self.reader.read_line(&mut self.buf) {
-            Ok(l) => {
-                // log::info!("read line: {}", self.buf);
-                let mv = lexpr::from_str(&self.buf);
-                self.buf.clear();
-                match mv {
-                    Ok(v) => {
-                        match v.as_cons() {
-                            Some(cs) => {
-                                Some(Message { event: cs.car().clone(), data: cs.cdr().clone() })
-                            },
-                            _ => { log::error!("malformed message bus input s-expression: {}", v); None },
-                        }
-                    },
-                    Err(e) => { log::error!("malformed message bus input line: {}", e); None },
-                }
-            },
-            Err(e) if e.kind() == std::io::ErrorKind::WouldBlock => {
-                // if self.buf.len() > 0 {
-                //     log::error!("error wouldblock: buf is {}", self.buf);
-                // }
-                None
-            },
-            Err(e) => panic!("IO error on message bus: {}", e),
-        }
-    }
-}
-
-#[derive(Debug, Clone)]
-pub struct BinaryMessage {
-    pub event: Vec<u8>,
-    pub data: Vec<u8>
-}
-#[derive(Debug, Clone)]
-pub enum BinaryClientState {
-    PartialEventLength { buf_len: usize, buf: [u8; 4] },
-    PartialEvent { len: usize, buf_len: usize, buf: Vec<u8> },
-    PartialDataLength { event: Vec<u8>, buf_len: usize, buf: [u8; 4] },
-    PartialData { event: Vec<u8>, len: usize, buf_len: usize, buf: Vec<u8> },
-    Message { event: Vec<u8>, data: Vec<u8> },
-}
-impl Default for BinaryClientState {
-    fn default() -> Self {
-        Self::PartialEventLength { buf_len: 0, buf: [0; 4] }
-    }
-}
-pub struct BinaryClient {
-    state: BinaryClientState,
-    reader: std::io::BufReader<std::net::TcpStream>,
-}
-impl BinaryClient {
-    pub fn new(addr: &str, subs: &[&[u8]]) -> Self {
-        let mut socket = std::net::TcpStream::connect(addr).expect("failed to connect to message bus");
-        socket.set_nonblocking(true).expect("failed to set message bus socket nonblocking");
-        for s in subs {
-            write!(socket, "s").expect("failed to send subscribe message to bus");
-            socket.write_u32::<byteorder::LE>(s.len() as u32).expect("failed to send subscribe message length to bus");
-            socket.write_all(s).expect("failed to send subscribe message to bus");
-        }
-        socket.flush().expect("failed to flush bus connection");
-        let reader = std::io::BufReader::new(socket);
-        Self { state: BinaryClientState::PartialEventLength { buf_len: 0, buf: [0; 4] }, reader }
-    }
-    fn read(reader: &mut std::io::BufReader<std::net::TcpStream>, buf: &mut [u8]) -> Option<usize> {
-        match reader.read(buf) {
-            Ok(sz) => Some(sz),
-            Err(e) if e.kind() == std::io::ErrorKind::WouldBlock => {
-                None
-            },
-            Err(e) => panic!("IO error on message bus: {}", e),
-        }
-    }
-    fn update_state(
-        reader: &mut std::io::BufReader<std::net::TcpStream>,
-        mut state: BinaryClientState,
-    ) -> BinaryClientState {
-        loop {
-            state = match state {
-                BinaryClientState::PartialEventLength { mut buf_len, mut buf } => {
-                    buf_len += if let Some(x) = Self::read(reader, &mut buf[buf_len..]) {
-                        x
-                    } else { break BinaryClientState::PartialEventLength { buf_len, buf }; };
-                    if buf_len == 4 {
-                        let len = u32::from_le_bytes(buf) as usize;
-                        BinaryClientState::PartialEvent {
-                            len,
-                            buf_len: 0,
-                            buf: vec![0; len],
-                        }
-                    } else { BinaryClientState::PartialEventLength { buf_len, buf } }
-                },
-                BinaryClientState::PartialEvent { len, mut buf_len, mut buf } => {
-                    buf_len += if let Some(x) = Self::read(reader, &mut buf[buf_len..]) {
-                        x
-                    } else { break BinaryClientState::PartialEvent { len, buf_len, buf }; };
-                    if buf_len == len {
-                        BinaryClientState::PartialDataLength {
-                            event: buf.clone(),
-                            buf_len: 0,
-                            buf: [0; 4],
-                        }
-                    } else { BinaryClientState::PartialEvent { len, buf_len, buf } }
-                },
-                BinaryClientState::PartialDataLength { event, mut buf_len, mut buf } => {
-                    buf_len += if let Some(x) = Self::read(reader, &mut buf[buf_len..]) {
-                        x
-                    } else { break BinaryClientState::PartialDataLength { event, buf_len, buf }; };
-                    if buf_len == 4 {
-                        let len = u32::from_le_bytes(buf) as usize;
-                        BinaryClientState::PartialData {
-                            event,
-                            len,
-                            buf_len: 0,
-                            buf: vec![0; len],
-                        }
-                    } else { BinaryClientState::PartialDataLength { event, buf_len, buf } }
-                },
-                BinaryClientState::PartialData { event, len, mut buf_len, mut buf } => {
-                    buf_len += if let Some(x) = Self::read(reader, &mut buf[buf_len..]) {
-                        x
-                    } else { break BinaryClientState::PartialData { event, len, buf_len, buf }; };
-                    if buf_len == len {
-                        BinaryClientState::Message {
-                            event,
-                            data: buf.clone(),
-                        }
-                    } else { BinaryClientState::PartialData { event, len, buf_len, buf } }
-                },
-                st@BinaryClientState::Message{..} => break st,
-            };
-        }
-    }
-    pub fn pump(&mut self) -> Option<BinaryMessage> {
-        self.state = Self::update_state(&mut self.reader, std::mem::take(&mut self.state));
-        match std::mem::take(&mut self.state) {
-            BinaryClientState::Message { event, data } => {
-                self.state = BinaryClientState::PartialEventLength { buf_len: 0, buf: [0; 4] };
-                Some(BinaryMessage {
-                    event: event,
-                    data: data,
-                })
-            },
-            st => {
-                self.state = st;
-                None
-            }
-        }
-    }
-}
diff --git a/crates/renderer/src/input.rs b/crates/renderer/src/input.rs
new file mode 100644
index 0000000..f0536e9
--- /dev/null
+++ b/crates/renderer/src/input.rs
@@ -0,0 +1,30 @@
+use device_query::DeviceQuery;
+
+pub enum Command {
+    None,
+    Drawing,
+    EraseAll,
+}
+pub struct Input {
+    pub device: device_query::DeviceState,
+}
+impl Input {
+    pub fn new() -> Self {
+        Self {
+            device: device_query::DeviceState::new(),
+        }
+    }
+    pub fn get_mouse(&self) -> (i32, i32) {
+        self.device.get_mouse().coords
+    }
+    pub fn get_command(&mut self) -> Command {
+        let keys = self.device.get_keys();
+        if keys.contains(&device_query::Keycode::LMeta) {
+            Command::Drawing
+        } else if keys.contains(&device_query::Keycode::RMeta) {
+            Command::EraseAll
+        } else {
+            Command::None
+        }
+    }
+}
diff --git a/crates/renderer/src/main.rs b/crates/renderer/src/main.rs
index 4e96e7f..4b3f6e3 100644
--- a/crates/renderer/src/main.rs
+++ b/crates/renderer/src/main.rs
@@ -1,10 +1,10 @@
 #![allow(dead_code, unused_variables)]
 mod assets;
 mod terminal;
-mod fig;
 mod background;
 mod toggle;
 mod overlay;
+mod input;
 
 use teleia::*;
 use clap::{command, Command};
@@ -30,10 +30,6 @@ pub fn main() -> Erm<()> {
             Command::new("model-multi-overlay")
                 .about("Run the LCOLONQ + Maude multi model renderer in a full-screen transparent overlay")
         )
-        .subcommand(
-            Command::new("server")
-                .about("Run the LCOLONQ online websocket server")
-        )
         .get_matches();
     match matches.subcommand() {
         Some(("shader-overlay", _cm)) => {
diff --git a/crates/renderer/src/overlay/automata.rs b/crates/renderer/src/overlay/automata.rs
new file mode 100644
index 0000000..869fd88
--- /dev/null
+++ b/crates/renderer/src/overlay/automata.rs
@@ -0,0 +1,242 @@
+use teleia::*;
+
+use glow::HasContext;
+
+const SCALE: usize = 15;
+const WIDTH: usize = 1920 / SCALE;
+const HEIGHT: usize = 1080 / SCALE;
+
+struct Pattern {
+    w: usize, h: usize,
+    cells: Vec<bool>,
+}
+impl Pattern {
+    pub fn from_rle(inp: &str) -> Option<Self> {
+        let s = inp.replace(";", "\n");
+        let mut data = String::new();
+        let mut w = 0;
+        let mut h = 0;
+        for line in s.split("\n") {
+            if let Some('#') = line.trim().chars().nth(0) {
+            } else if let Some('x') = line.trim().chars().nth(0) {
+                for assign in line.trim().split(",") {
+                    if let Some((svar, sval)) = assign.split_once(" = ") {
+                        if svar.trim() == "x" {
+                            w = sval.trim().parse().ok()?;
+                        } else if svar.trim() == "y" {
+                            h = sval.trim().parse().ok()?;
+                        }
+                    }
+                }
+            } else {
+                data.push_str(&line);
+            }
+        }
+        let mut ret = Self {
+            w, h,
+            cells: vec![false; w * h],
+        };
+        log::info!("dims: {w} {h} data: {data}");
+        ret.populate(&data);
+        Some(ret)
+    }
+    pub fn idx(&self, x: i32, y: i32) -> usize{
+        let ux = x.rem_euclid(self.w as i32) as usize;
+        let uy = y.rem_euclid(self.h as i32) as usize;
+        uy * self.w + ux
+    }
+    pub fn get(&self, x: i32, y: i32) -> bool {
+        let idx = self.idx(x, y);
+        self.cells[idx]
+    }
+    pub fn set(&mut self, x: i32, y: i32) {
+        let idx = self.idx(x, y);
+        self.cells[idx] = true;
+        log::info!("pattern: {x} {y}");
+    }
+    pub fn populate(&mut self, s: &str) {
+        let mut run = 0;
+        let mut x = 0;
+        let mut y = 0;
+        for c in s.chars() {
+            if let Some(d) = c.to_digit(10) {
+                run = run * 10 + d;
+            } else {
+                if run == 0 { run = 1 }
+                if c == '$' && x == 0 { run -= 1 }
+                while run > 0 {
+                    match c {
+                        'b' => {
+                            x = (x + 1) % (self.w as i32);
+                            if x == 0 { y += 1; }
+                        },
+                        'o' => {
+                            self.set(x, y);
+                            x = (x + 1) % (self.w as i32);
+                            if x == 0 { y += 1; }
+                        },
+                        '$' => {
+                            x = 0;
+                            y += 1;
+                        },
+                        _ => {},
+                    }
+                    run -= 1;
+                }
+            }
+        }
+    }
+}
+
+type Cell = u8;
+
+struct CellRule {
+    color: glam::Vec4,
+}
+
+struct CellBuffer {
+    buf: [Cell; WIDTH * HEIGHT],
+}
+impl CellBuffer {
+    pub fn new() -> Self {
+        Self {
+            buf: [0; WIDTH * HEIGHT],
+        }
+    }
+    fn idx(x: i32, y: i32) -> usize {
+        let ux = x.rem_euclid(WIDTH as i32) as usize;
+        let uy = y.rem_euclid(HEIGHT as i32) as usize;
+        uy * WIDTH + ux
+    }
+    pub fn get(&self, x: i32, y: i32) -> Cell {
+        self.buf[Self::idx(x, y)]
+    }
+    pub fn is_nonzero(&self, x: i32, y: i32) -> bool {
+        self.get(x, y) > 0
+    }
+    pub fn count_cell(&self, x: i32, y: i32) -> i32 {
+        self.get(x, y).min(1) as i32
+    }
+    pub fn count_neighbors(&self, x: i32, y: i32) -> i32 {
+        self.count_cell(x-1, y-1) + self.count_cell(x, y-1) + self.count_cell(x+1, y-1)
+            + self.count_cell(x-1, y) + self.count_cell(x+1, y)
+            + self.count_cell(x-1, y+1) + self.count_cell(x, y+1) + self.count_cell(x+1, y+1)
+    }
+    pub fn set(&mut self, x: i32, y: i32, v: Cell) {
+        self.buf[Self::idx(x, y)] = v;
+    }
+}
+
+pub struct Board {
+    shader: shader::Shader,
+    tex: texture::Texture,
+    active: bool,
+    buf0: CellBuffer,
+    buf1: CellBuffer,
+    rules: [CellRule; 256],
+}
+impl Board {
+    pub fn new(ctx: &context::Context) -> Self {
+        let rules = std::array::from_fn(|idx| match idx {
+            0 => CellRule { color: glam::Vec4::new(0.0, 0.0, 0.0, 0.0) },
+            _ => CellRule { color: glam::Vec4::new(1.0, 1.0, 1.0, 1.0) },
+        });
+        Self {
+            shader: shader::Shader::new(
+                ctx,
+                include_str!("../assets/shaders/automata/vert.glsl"),
+                include_str!("../assets/shaders/automata/frag.glsl"),
+            ),
+            tex: texture::Texture::new_empty(ctx),
+            active: false,
+            buf0: CellBuffer::new(),
+            buf1: CellBuffer::new(),
+            rules,
+        }
+    }
+    pub fn spawn(&mut self, x: i32, y: i32, c: Cell, pat: &Pattern) {
+        let cur = if self.active { &mut self.buf0 } else { &mut self.buf1 };
+        for uxoff in 0..pat.w {
+            for uyoff in 0..pat.h {
+                let xoff = uxoff as i32; let yoff = uyoff as i32;
+                cur.set(x + xoff, y + yoff, if pat.get(xoff, yoff) { c } else { 0 });
+            }
+        }
+    }
+    pub fn test_glider(&mut self) {
+        // let cur = if self.active { &mut self.buf0 } else { &mut self.buf1 };
+        // cur.set(1, 0, 1);
+        // cur.set(2, 1, 1);
+        // cur.set(0, 2, 1);
+        // cur.set(1, 2, 1);
+        // cur.set(2, 2, 1);
+        if let Some(pat) = Pattern::from_rle("
+#N Tanner's p46
+#O Tanner Jacobi
+#C A period 46 oscillator discovered by Tanner Jacobi in October 2017.
+#C https://conwaylife.com/wiki/Tanner%27s_p46
+x = 13, y = 26, rule = B3/S23
+2b2o9b$2bo10b$3bo9b$2b2o9b$13b$9b2o2b$9bo3b$10bo2b$9b2o2b$b2o10b$b2o6b
+2o2b$o7bobo2b$b2o6bo3b$b2o7b3o$12bo$13b$13b$13b$13b$13b$13b$b2o10b$b2o
+2b2o6b$5bobo5b$7bo5b$7b2o4b!
+") {
+            self.spawn(30, 10, 1, &pat);
+        }
+    }
+    pub fn step(&mut self) {
+        let (cur, next) = if self.active {
+            (&mut self.buf0, &mut self.buf1)
+        } else {
+            (&mut self.buf1, &mut self.buf0)
+        };
+        for ux in 0..WIDTH {
+            for uy in 0..HEIGHT {
+                let x = ux as _; let y = uy as _;
+                let n = cur.count_neighbors(x, y);
+                if cur.is_nonzero(x, y) && n != 2 && n != 3{
+                    next.set(x, y, 0)
+                } else if n == 3 {
+                    next.set(x, y, 1)
+                } else {
+                    next.set(x, y, cur.get(x, y))
+                }
+            }
+        }
+        self.active = !self.active;
+    }
+    pub fn upload(&self, ctx: &context::Context) {
+        let cur = if self.active { &self.buf0 } else { &self.buf1 };
+        unsafe {
+            let err = ctx.gl.get_error();
+            self.tex.bind(ctx);
+            ctx.gl.tex_image_2d(
+                glow::TEXTURE_2D,
+                0,
+                glow::R8 as i32,
+                WIDTH as i32,
+                HEIGHT as i32,
+                0,
+                glow::RED,
+                glow::UNSIGNED_BYTE,
+                Some(&cur.buf),
+            );
+            ctx.gl.generate_mipmap(glow::TEXTURE_2D);
+        }
+    }
+    pub fn update(&mut self, ctx: &context::Context, st: &mut state::State) {
+        if st.tick % 10 == 0 {
+            self.step();
+            self.upload(ctx);
+        }
+    }
+    pub fn render(&self, ctx: &context::Context, st: &mut state::State) {
+        st.bind_2d(ctx, &self.shader);
+        self.tex.bind(ctx);
+        self.shader.set_position_2d(
+            ctx,
+            &glam::Vec2::new(0.0, 0.0),
+            &glam::Vec2::new(1920.0, 1080.0)
+        );
+        st.mesh_square.render(ctx);
+    }
+}
diff --git a/crates/renderer/src/overlay/drawing.rs b/crates/renderer/src/overlay/drawing.rs
new file mode 100644
index 0000000..8088b5a
--- /dev/null
+++ b/crates/renderer/src/overlay/drawing.rs
@@ -0,0 +1,128 @@
+use teleia::*;
+
+use glow::HasContext;
+
+use crate::{background, input};
+
+pub const SCALE: usize = 4;
+pub const WIDTH: usize = 1920 / SCALE;
+pub const HEIGHT: usize = 1080 / SCALE;
+pub struct Drawing {
+    pub tex: texture::Texture,
+    pub pixels: [u8; WIDTH * HEIGHT],
+    pub last_point: Option<(i32, i32)>,
+    pub shader_white: shader::Shader,
+    pub shader_background: shader::Shader,
+}
+impl Drawing {
+    pub fn new(ctx: &context::Context) -> Self {
+        let shader_background = shader::Shader::new(
+            ctx,
+            include_str!("../assets/shaders/background/vert.glsl"),
+            include_str!("../assets/shaders/background/frag.glsl"),
+        );
+        shader_background.set_i32(ctx, "background", 1);
+        Self {
+            tex: texture::Texture::new_empty(ctx),
+            pixels: [0; WIDTH * HEIGHT],
+            last_point: None,
+            shader_white: shader::Shader::new(
+                ctx,
+                include_str!("../assets/shaders/white/vert.glsl"),
+                include_str!("../assets/shaders/white/frag.glsl"),
+            ),
+            shader_background,
+        }
+    }
+    pub fn coord(&self, x: usize, y: usize) -> Option<usize> {
+        if x >= WIDTH || y >= HEIGHT {
+            None
+        } else {
+            Some(x + y * WIDTH)
+        }
+    }
+    pub fn set(&mut self, val: u8, x: i32, y: i32) {
+        self.coord(x as usize, y as usize).map(|idx| self.pixels[idx] = val);
+    }
+    pub fn point(&mut self, val: u8, x: i32, y: i32) {
+        self.set(val, x, y - 1);
+        self.set(val, x - 1, y);
+        self.set(val, x, y);
+        self.set(val, x + 1, y);
+        self.set(val, x, y + 1);
+    }
+    pub fn line(&mut self, val: u8, (mut x0, mut y0): (i32, i32), (x1, y1): (i32, i32)) {
+        let dx = (x1 - x0).abs();
+        let sx = if x0 < x1 { 1 } else { -1 };
+        let dy = -((y1 - y0).abs());
+        let sy = if y0 < y1 { 1 } else { -1 };
+        let mut error = dx + dy;
+        loop {
+            self.point(val, x0, y0);
+            let e2 = 2 * error;
+            if e2 >= dy {
+                if x0 == x1 { break; }
+                error += dy;
+                x0 += sx;
+            }
+            if e2 <= dx {
+                if y0 == y1 { break; }
+                error += dx;
+                y0 += sy;
+            }
+        }
+    }
+    pub fn upload(&self, ctx: &context::Context) {
+        unsafe {
+            let err = ctx.gl.get_error();
+            self.tex.bind(ctx);
+            ctx.gl.tex_image_2d(
+                glow::TEXTURE_2D,
+                0,
+                glow::R8 as i32,
+                WIDTH as i32,
+                HEIGHT as i32,
+                0,
+                glow::RED,
+                glow::UNSIGNED_BYTE,
+                Some(&self.pixels),
+            );
+            ctx.gl.generate_mipmap(glow::TEXTURE_2D);
+        }
+    }
+    pub fn update(&mut self, ctx: &context::Context, st: &mut state::State, inp: &mut input::Input) {
+        match inp.get_command() {
+            input::Command::Drawing => {
+                let (sx, sy) = inp.get_mouse();
+                let x = sx / (SCALE as i32);
+                let y = sy / (SCALE as i32);
+                if let Some(last) = self.last_point {
+                    self.line(1, last, (x, y));
+                } else {
+                    self.point(1, x, y);
+                }
+                self.last_point = Some((x, y));
+            },
+            input::Command::EraseAll => {
+                self.pixels.fill(0);
+                self.last_point = None;
+            },
+            input::Command::None => {
+                self.last_point = None;
+            },
+            _ => {},
+        }
+        self.upload(ctx);
+    }
+    pub fn render(&self, ctx: &context::Context, st: &mut state::State, bg: &background::Backgrounds) {
+        st.bind_2d(ctx, &self.shader_background);
+        self.tex.bind(ctx);
+        bg.drawing.bind_index(ctx, 1);
+        self.shader_background.set_position_2d(
+            ctx,
+            &glam::Vec2::new(0.0, 0.0),
+            &glam::Vec2::new(1920.0, 1080.0)
+        );
+        st.mesh_square.render(ctx);
+    }
+}
diff --git a/crates/renderer/src/overlay/mod.rs b/crates/renderer/src/overlay/mod.rs
index 5fff8af..6bfa1ae 100644
--- a/crates/renderer/src/overlay/mod.rs
+++ b/crates/renderer/src/overlay/mod.rs
@@ -1,3 +1,5 @@
 pub mod model;
 pub mod shader;
 pub mod multi;
+pub mod drawing;
+pub mod automata;
diff --git a/crates/renderer/src/overlay/model.rs b/crates/renderer/src/overlay/model.rs
index 779f36b..14f6076 100644
--- a/crates/renderer/src/overlay/model.rs
+++ b/crates/renderer/src/overlay/model.rs
@@ -5,7 +5,7 @@ use std::{collections::HashMap, f32::consts::PI};
 use lexpr::sexp;
 use base64::prelude::*;
 
-use crate::{assets, fig, terminal, toggle};
+use crate::{assets, terminal, toggle};
 
 pub enum RenderMode {
     Overlay,
@@ -19,7 +19,7 @@ pub struct Overlay {
     model_neck_base: glam::Mat4,
     model_fb: framebuffer::Framebuffer,
     terminal: terminal::Terminal,
-    fig: fig::Client,
+    fig: fig::SexpClient,
     tracking_eyes: (f32, f32),
     tracking_mouth: f32,
     tracking_neck: glam::Quat,
@@ -44,7 +44,7 @@ impl Overlay {
                 &glam::Vec2::ZERO
             ),
             terminal: terminal::Terminal::new(ctx, 64, 64),
-            fig: fig::Client::new("shiro:32050", &[
+            fig: fig::SexpClient::new("shiro:32050", &[
                 sexp!((avatar toggle)),
                 sexp!((avatar toggle set)),
                 sexp!((avatar toggle unset)),
@@ -70,7 +70,7 @@ impl Overlay {
         // TODO also reset terminal
         self.toggles.reset();
     }
-    pub fn handle_tracking(&mut self, msg: fig::Message) -> Option<()> {
+    pub fn handle_tracking(&mut self, msg: fig::SexpMessage) -> Option<()> {
         let eyes = msg.data.get(0)?;
         let eye_left = eyes.get(0)?.as_str()?.parse::<f32>().ok()?;
         let eye_right = eyes.get(1)?.as_str()?.parse::<f32>().ok()?;
@@ -84,14 +84,14 @@ impl Overlay {
         self.tracking_neck = glam::Quat::from_euler(glam::EulerRot::XYZ, euler_x, euler_y, euler_z);
         Some(())
     }
-    pub fn handle_text(&mut self, msg: fig::Message) -> Option<()> {
+    pub fn handle_text(&mut self, msg: fig::SexpMessage) -> Option<()> {
         let bs = BASE64_STANDARD.decode(msg.data.get(0)?.as_str()?).ok()?;
         let s = std::str::from_utf8(&bs).ok()?;
         log::info!("handle_text: {}", s);
         self.terminal.fill_string(s);
         Some(())
     }
-    pub fn handle_frame(&mut self, msg: fig::Message) -> Option<()> {
+    pub fn handle_frame(&mut self, msg: fig::SexpMessage) -> Option<()> {
         let data = BASE64_STANDARD.decode(msg.data.get(0)?.as_str()?).ok()?;
         for (i, c) in data.chunks_exact(3).enumerate() {
             if let [r, g, b] = c {
diff --git a/crates/renderer/src/overlay/multi.rs b/crates/renderer/src/overlay/multi.rs
index 41ec12f..e3d0100 100644
--- a/crates/renderer/src/overlay/multi.rs
+++ b/crates/renderer/src/overlay/multi.rs
@@ -29,7 +29,7 @@ impl Model {
             tracking_neck: glam::Quat::IDENTITY,
         }
     }
-    pub fn handle_tracking(&mut self, msg: fig::Message) -> Option<()> {
+    pub fn handle_tracking(&mut self, msg: fig::SexpMessage) -> Option<()> {
         let eyes = msg.data.get(0)?;
         let eye_left = eyes.get(0)?.as_str()?.parse::<f32>().ok()?;
         let eye_right = eyes.get(1)?.as_str()?.parse::<f32>().ok()?;
@@ -66,7 +66,7 @@ pub struct Overlay {
     models: Vec<Model>,
     fb: framebuffer::Framebuffer,
     term: terminal::Terminal,
-    fig: fig::Client,
+    fig: fig::SexpClient,
 }
 impl Overlay {
     pub fn new(ctx: &context::Context) -> Self {
@@ -83,7 +83,7 @@ impl Overlay {
             ),
             term: terminal::Terminal::new(ctx, 128, 64),
             models,
-            fig: fig::Client::new("shiro:32050", &[
+            fig: fig::SexpClient::new("shiro:32050", &[
                 sexp!((avatar toggle)),
                 sexp!((avatar toggle set)),
                 sexp!((avatar toggle unset)),
@@ -124,7 +124,7 @@ impl Overlay {
             )
         );
         tex.bind(ctx);
-        self.assets.mesh_square.render(ctx);
+        st.mesh_square.render(ctx);
     }
 }
 impl teleia::state::Game for Overlay {
diff --git a/crates/renderer/src/overlay/shader.rs b/crates/renderer/src/overlay/shader.rs
index 97e4d92..81e29f4 100644
--- a/crates/renderer/src/overlay/shader.rs
+++ b/crates/renderer/src/overlay/shader.rs
@@ -3,11 +3,11 @@ use teleia::*;
 use std::{collections::HashMap, f32::consts::PI};
 use lexpr::sexp;
 use base64::prelude::*;
-use device_query::DeviceQuery;
 
 use glow::HasContext;
 
-use crate::{assets, fig, toggle, background};
+use crate::{assets, fig, toggle, input, background};
+use super::{drawing, automata};
 
 pub struct Chat {
     author: String,
@@ -26,103 +26,11 @@ impl Chat {
     }
 }
 
-const DRAWING_WIDTH: usize = 1920 / 4;
-const DRAWING_HEIGHT: usize = 1080 / 4;
-pub enum DrawingCommand {
-    None,
-    Drawing,
-    EraseAll,
-}
-pub struct Drawing {
-    tex: texture::Texture,
-    pixels: [u8; DRAWING_WIDTH * DRAWING_HEIGHT],
-    last_point: Option<(i32, i32)>,
-    shader_white: shader::Shader,
-    shader_background: shader::Shader,
-}
-impl Drawing {
-    pub fn new(ctx: &context::Context) -> Self {
-        let shader_background = shader::Shader::new(
-            ctx,
-            include_str!("../assets/shaders/background/vert.glsl"),
-            include_str!("../assets/shaders/background/frag.glsl"),
-        );
-        shader_background.set_i32(ctx, "background", 1);
-        Self {
-            tex: texture::Texture::new_empty(ctx),
-            pixels: [0; DRAWING_WIDTH * DRAWING_HEIGHT],
-            last_point: None,
-            shader_white: shader::Shader::new(
-                ctx,
-                include_str!("../assets/shaders/white/vert.glsl"),
-                include_str!("../assets/shaders/white/frag.glsl"),
-            ),
-            shader_background,
-        }
-    }
-    pub fn coord(&self, x: usize, y: usize) -> Option<usize> {
-        if x >= DRAWING_WIDTH || y >= DRAWING_HEIGHT {
-            None
-        } else {
-            Some(x + y * DRAWING_WIDTH)
-        }
-    }
-    pub fn set(&mut self, val: u8, x: i32, y: i32) {
-        self.coord(x as usize, y as usize).map(|idx| self.pixels[idx] = val);
-    }
-    pub fn point(&mut self, val: u8, x: i32, y: i32) {
-        self.set(val, x, y - 1);
-        self.set(val, x - 1, y);
-        self.set(val, x, y);
-        self.set(val, x + 1, y);
-        self.set(val, x, y + 1);
-    }
-    pub fn line(&mut self, val: u8, (mut x0, mut y0): (i32, i32), (x1, y1): (i32, i32)) {
-        let dx = (x1 - x0).abs();
-        let sx = if x0 < x1 { 1 } else { -1 };
-        let dy = -((y1 - y0).abs());
-        let sy = if y0 < y1 { 1 } else { -1 };
-        let mut error = dx + dy;
-        loop {
-            self.point(val, x0, y0);
-            let e2 = 2 * error;
-            if e2 >= dy {
-                if x0 == x1 { break; }
-                error += dy;
-                x0 += sx;
-            }
-            if e2 <= dx {
-                if y0 == y1 { break; }
-                error += dx;
-                y0 += sy;
-            }
-        }
-    }
-    pub fn upload(&self, ctx: &context::Context) {
-        unsafe {
-            let err = ctx.gl.get_error();
-            self.tex.bind(ctx);
-            ctx.gl.tex_image_2d(
-                glow::TEXTURE_2D,
-                0,
-                glow::R8 as i32,
-                DRAWING_WIDTH as i32,
-                DRAWING_HEIGHT as i32,
-                0,
-                glow::RED,
-                glow::UNSIGNED_BYTE,
-                Some(&self.pixels),
-            );
-            ctx.gl.generate_mipmap(glow::TEXTURE_2D);
-        }
-    }
-}
-
 pub struct Overlay {
     assets: assets::Assets,
     model: scene::Scene,
     model_neck_base: glam::Mat4,
-    fig: fig::Client,
+    fig: fig::SexpClient,
     fig_binary: fig::BinaryClient,
     tracking_eyes: (f32, f32),
     tracking_mouth: f32,
@@ -134,9 +42,10 @@ pub struct Overlay {
     muzak_author: Option<String>,
     chat: Chat,
     toggles: toggle::Toggles,
+    input: input::Input,
     backgrounds: background::Backgrounds,
-    drawing: Drawing,
-    device: device_query::DeviceState,
+    drawing: drawing::Drawing,
+    automata: automata::Board,
 }
 
 impl Overlay {
@@ -147,11 +56,13 @@ impl Overlay {
             .expect("failed to find neck joint")
             .transform;
         let throwshade = newton_throwshade::ThrowShade::new();
+        let mut automata = automata::Board::new(ctx);
+        automata.test_glider();
         Self {
             assets: assets::Assets::new(ctx),
             model,
             model_neck_base,
-            fig: fig::Client::new("shiro:32050", &[
+            fig: fig::SexpClient::new("shiro:32050", &[
                 sexp!((avatar toggle)),
                 sexp!((avatar toggle set)),
                 sexp!((avatar toggle unset)),
@@ -166,7 +77,7 @@ impl Overlay {
                 sexp!((avatar overlay cursor)),
                 sexp!((avatar overlay emacs)),
             ]),
-            fig_binary: fig::BinaryClient::new("shiro:32051", &[
+            fig_binary: fig::BinaryClient::new("shiro:32051", false, &[
                 b"background frame"
             ]),
             tracking_eyes: (1.0, 1.0),
@@ -180,21 +91,9 @@ impl Overlay {
             chat: Chat::new(),
             toggles: toggle::Toggles::new(),
             backgrounds: background::Backgrounds::new(ctx),
-            drawing: Drawing::new(ctx),
-            device: device_query::DeviceState::new(),
-        }
-    }
-    fn get_mouse(&self) -> (i32, i32) {
-        self.device.get_mouse().coords
-    }
-    fn get_drawing_command(&mut self) -> DrawingCommand {
-        let keys = self.device.get_keys();
-        if keys.contains(&device_query::Keycode::LMeta) {
-            DrawingCommand::Drawing
-        } else if keys.contains(&device_query::Keycode::RMeta) {
-            DrawingCommand::EraseAll
-        } else {
-            DrawingCommand::None
+            input: input::Input::new(),
+            drawing: drawing::Drawing::new(ctx),
+            automata,
         }
     }
     pub fn handle_reset(&mut self, ctx: &context::Context) {
@@ -203,7 +102,7 @@ impl Overlay {
         self.throwshade.shader = None;
         self.toggles.reset();
     }
-    pub fn handle_tracking(&mut self, msg: fig::Message) -> Option<()> {
+    pub fn handle_tracking(&mut self, msg: fig::SexpMessage) -> Option<()> {
         let eyes = msg.data.get(0)?;
         let eye_left = eyes.get(0)?.as_str()?.parse::<f32>().ok()?;
         let eye_right = eyes.get(1)?.as_str()?.parse::<f32>().ok()?;
@@ -220,7 +119,7 @@ impl Overlay {
     pub fn handle_overlay_shader(
         &mut self,
         ctx: &context::Context, st: &state::State,
-        msg: fig::Message
+        msg: fig::SexpMessage
     ) -> Option<()> {
         let ba = BASE64_STANDARD.decode(msg.data.get(0)?.as_str()?).ok()?;
         let author = String::from_utf8_lossy(&ba);
@@ -233,7 +132,7 @@ impl Overlay {
         }
         Some(())
     }
-    pub fn handle_overlay_muzak(&mut self, msg: fig::Message) -> Option<()> {
+    pub fn handle_overlay_muzak(&mut self, msg: fig::SexpMessage) -> Option<()> {
         let ba = BASE64_STANDARD.decode(msg.data.get(0)?.as_str()?).ok()?;
         let author = String::from_utf8_lossy(&ba);
         self.muzak_author = Some(author.to_string());
@@ -243,7 +142,7 @@ impl Overlay {
         self.muzak_author = None;
         Some(())
     }
-    pub fn handle_overlay_chat(&mut self, msg: fig::Message) -> Option<()> {
+    pub fn handle_overlay_chat(&mut self, msg: fig::SexpMessage) -> Option<()> {
         let ba = BASE64_STANDARD.decode(msg.data.get(0)?.as_str()?).ok()?;
         let a = String::from_utf8_lossy(&ba);
         let bs = BASE64_STANDARD.decode(msg.data.get(1)?.as_str()?).ok()?;
@@ -256,27 +155,16 @@ impl Overlay {
         self.chat.biblicality = biblicality;
         Some(())
     }
-    pub fn handle_overlay_cursor(&mut self, msg: fig::Message) -> Option<()> {
+    pub fn handle_overlay_cursor(&mut self, msg: fig::SexpMessage) -> Option<()> {
         let cursor_x = msg.data.get(0)?.as_i64()? as f32;
         let cursor_y = msg.data.get(1)?.as_i64()? as f32;
         self.emacs_cursor = (cursor_x, cursor_y);
         Some(())
     }
-    pub fn handle_overlay_emacs(&mut self, msg: fig::Message) -> Option<()> {
+    pub fn handle_overlay_emacs(&mut self, msg: fig::SexpMessage) -> Option<()> {
         self.emacs_heartrate = msg.data.get(0)?.as_i64()? as i32;
         Some(())
     }
-    pub fn render_drawing(&self, ctx: &context::Context, st: &mut state::State) {
-        st.bind_2d(ctx, &self.drawing.shader_background);
-        self.drawing.tex.bind(ctx);
-        self.backgrounds.drawing.bind_index(ctx, 1);
-        self.drawing.shader_background.set_position_2d(
-            ctx,
-            &glam::Vec2::new(0.0, 0.0),
-            &glam::Vec2::new(1920.0, 1080.0)
-        );
-        self.assets.mesh_square.render(ctx);
-    }
 }
 
 impl teleia::state::Game for Overlay {
@@ -293,7 +181,7 @@ impl teleia::state::Game for Overlay {
             &glam::Vec3::new(0.0, 0.0, -1.0),
             &glam::Vec3::new(0.0, 1.0, 0.0),
         );
-        let (x, y) = self.get_mouse();
+        let (x, y) = self.input.get_mouse();
         self.mouse_cursor = (x as f32, y as f32);
         while let Some(msg) = self.fig.pump() {
             let malformed = format!("malformed {} data: {}", msg.event, msg.data);
@@ -338,6 +226,8 @@ impl teleia::state::Game for Overlay {
         if let Some(n) = self.model.nodes_by_name.get("J_Bip_C_Neck").and_then(|i| self.model.nodes.get_mut(*i)) {
             n.transform = self.model_neck_base * glam::Mat4::from_quat(self.tracking_neck);
         }
+        self.drawing.update(ctx, st, &mut self.input);
+        self.automata.update(ctx, st);
         Ok(())
     }
     fn render(&mut self, ctx: &context::Context, st: &mut state::State) -> Erm<()> {
@@ -378,7 +268,7 @@ impl teleia::state::Game for Overlay {
                 &glam::Vec2::new(1100.0, 300.0),
                 &glam::Vec2::new(800.0, 600.0)
             );
-            self.assets.mesh_square.render(ctx);
+            st.mesh_square.render(ctx);
         }
         let mut authors = Vec::new();
         if let Some(_) = &self.throwshade.shader {
@@ -389,28 +279,8 @@ impl teleia::state::Game for Overlay {
         }
         let astr: String = authors.join(", ");
         self.assets.font.render_text(ctx, &glam::Vec2::new(0.0, 0.0), &astr);
-        match self.get_drawing_command() {
-            DrawingCommand::Drawing => {
-                let (sx, sy) = self.get_mouse();
-                let x = sx / 4;
-                let y = sy / 4;
-                if let Some(last) = self.drawing.last_point {
-                    self.drawing.line(1, last, (x, y));
-                } else {
-                    self.drawing.point(1, x, y);
-                }
-                self.drawing.last_point = Some((x, y));
-            },
-            DrawingCommand::EraseAll => {
-                self.drawing.pixels.fill(0);
-                self.drawing.last_point = None;
-            },
-            DrawingCommand::None => {
-                self.drawing.last_point = None;
-            },
-        }
-        self.drawing.upload(ctx);
-        self.render_drawing(ctx, st);
+        self.drawing.render(ctx, st, &self.backgrounds);
+        self.automata.render(ctx, st);
         Ok(())
     }
 }
diff --git a/crates/renderer/src/toggle.rs b/crates/renderer/src/toggle.rs
index 7bba735..f6d60a4 100644
--- a/crates/renderer/src/toggle.rs
+++ b/crates/renderer/src/toggle.rs
@@ -27,13 +27,13 @@ impl Toggles {
     pub fn reset(&mut self) {
         self.toggles.clear();
     }
-    pub fn handle(&mut self, ctx: &context::Context, st: &state::State, msg: fig::Message) -> Option<()> {
+    pub fn handle(&mut self, ctx: &context::Context, st: &state::State, msg: fig::SexpMessage) -> Option<()> {
         let nm = msg.data.get(0)?.as_str()?;
         let prev = self.get(ctx, st, nm).map(|t| t.val).unwrap_or(false);
         self.set(ctx, st, nm, !prev);
         Some(())
     }
-    pub fn handle_set(&mut self, ctx: &context::Context, st: &state::State, msg: fig::Message, val: bool) -> Option<()> {
+    pub fn handle_set(&mut self, ctx: &context::Context, st: &state::State, msg: fig::SexpMessage, val: bool) -> Option<()> {
         let nm = msg.data.get(0)?.as_str()?;
         self.set(ctx, st, nm, val);
         Some(())