Automata rendering

6 files changed, 407 insertions, 165 deletions

diff --git a/crates/renderer/src/overlay/automata.rs b/crates/renderer/src/overlay/automata.rs
new file mode 100644
index 0000000..869fd88
--- /dev/null
+++ b/crates/renderer/src/overlay/automata.rs
@@ -0,0 +1,242 @@
+use teleia::*;
+
+use glow::HasContext;
+
+const SCALE: usize = 15;
+const WIDTH: usize = 1920 / SCALE;
+const HEIGHT: usize = 1080 / SCALE;
+
+struct Pattern {
+    w: usize, h: usize,
+    cells: Vec<bool>,
+}
+impl Pattern {
+    pub fn from_rle(inp: &str) -> Option<Self> {
+        let s = inp.replace(";", "\n");
+        let mut data = String::new();
+        let mut w = 0;
+        let mut h = 0;
+        for line in s.split("\n") {
+            if let Some('#') = line.trim().chars().nth(0) {
+            } else if let Some('x') = line.trim().chars().nth(0) {
+                for assign in line.trim().split(",") {
+                    if let Some((svar, sval)) = assign.split_once(" = ") {
+                        if svar.trim() == "x" {
+                            w = sval.trim().parse().ok()?;
+                        } else if svar.trim() == "y" {
+                            h = sval.trim().parse().ok()?;
+                        }
+                    }
+                }
+            } else {
+                data.push_str(&line);
+            }
+        }
+        let mut ret = Self {
+            w, h,
+            cells: vec![false; w * h],
+        };
+        log::info!("dims: {w} {h} data: {data}");
+        ret.populate(&data);
+        Some(ret)
+    }
+    pub fn idx(&self, x: i32, y: i32) -> usize{
+        let ux = x.rem_euclid(self.w as i32) as usize;
+        let uy = y.rem_euclid(self.h as i32) as usize;
+        uy * self.w + ux
+    }
+    pub fn get(&self, x: i32, y: i32) -> bool {
+        let idx = self.idx(x, y);
+        self.cells[idx]
+    }
+    pub fn set(&mut self, x: i32, y: i32) {
+        let idx = self.idx(x, y);
+        self.cells[idx] = true;
+        log::info!("pattern: {x} {y}");
+    }
+    pub fn populate(&mut self, s: &str) {
+        let mut run = 0;
+        let mut x = 0;
+        let mut y = 0;
+        for c in s.chars() {
+            if let Some(d) = c.to_digit(10) {
+                run = run * 10 + d;
+            } else {
+                if run == 0 { run = 1 }
+                if c == '$' && x == 0 { run -= 1 }
+                while run > 0 {
+                    match c {
+                        'b' => {
+                            x = (x + 1) % (self.w as i32);
+                            if x == 0 { y += 1; }
+                        },
+                        'o' => {
+                            self.set(x, y);
+                            x = (x + 1) % (self.w as i32);
+                            if x == 0 { y += 1; }
+                        },
+                        '$' => {
+                            x = 0;
+                            y += 1;
+                        },
+                        _ => {},
+                    }
+                    run -= 1;
+                }
+            }
+        }
+    }
+}
+
+type Cell = u8;
+
+struct CellRule {
+    color: glam::Vec4,
+}
+
+struct CellBuffer {
+    buf: [Cell; WIDTH * HEIGHT],
+}
+impl CellBuffer {
+    pub fn new() -> Self {
+        Self {
+            buf: [0; WIDTH * HEIGHT],
+        }
+    }
+    fn idx(x: i32, y: i32) -> usize {
+        let ux = x.rem_euclid(WIDTH as i32) as usize;
+        let uy = y.rem_euclid(HEIGHT as i32) as usize;
+        uy * WIDTH + ux
+    }
+    pub fn get(&self, x: i32, y: i32) -> Cell {
+        self.buf[Self::idx(x, y)]
+    }
+    pub fn is_nonzero(&self, x: i32, y: i32) -> bool {
+        self.get(x, y) > 0
+    }
+    pub fn count_cell(&self, x: i32, y: i32) -> i32 {
+        self.get(x, y).min(1) as i32
+    }
+    pub fn count_neighbors(&self, x: i32, y: i32) -> i32 {
+        self.count_cell(x-1, y-1) + self.count_cell(x, y-1) + self.count_cell(x+1, y-1)
+            + self.count_cell(x-1, y) + self.count_cell(x+1, y)
+            + self.count_cell(x-1, y+1) + self.count_cell(x, y+1) + self.count_cell(x+1, y+1)
+    }
+    pub fn set(&mut self, x: i32, y: i32, v: Cell) {
+        self.buf[Self::idx(x, y)] = v;
+    }
+}
+
+pub struct Board {
+    shader: shader::Shader,
+    tex: texture::Texture,
+    active: bool,
+    buf0: CellBuffer,
+    buf1: CellBuffer,
+    rules: [CellRule; 256],
+}
+impl Board {
+    pub fn new(ctx: &context::Context) -> Self {
+        let rules = std::array::from_fn(|idx| match idx {
+            0 => CellRule { color: glam::Vec4::new(0.0, 0.0, 0.0, 0.0) },
+            _ => CellRule { color: glam::Vec4::new(1.0, 1.0, 1.0, 1.0) },
+        });
+        Self {
+            shader: shader::Shader::new(
+                ctx,
+                include_str!("../assets/shaders/automata/vert.glsl"),
+                include_str!("../assets/shaders/automata/frag.glsl"),
+            ),
+            tex: texture::Texture::new_empty(ctx),
+            active: false,
+            buf0: CellBuffer::new(),
+            buf1: CellBuffer::new(),
+            rules,
+        }
+    }
+    pub fn spawn(&mut self, x: i32, y: i32, c: Cell, pat: &Pattern) {
+        let cur = if self.active { &mut self.buf0 } else { &mut self.buf1 };
+        for uxoff in 0..pat.w {
+            for uyoff in 0..pat.h {
+                let xoff = uxoff as i32; let yoff = uyoff as i32;
+                cur.set(x + xoff, y + yoff, if pat.get(xoff, yoff) { c } else { 0 });
+            }
+        }
+    }
+    pub fn test_glider(&mut self) {
+        // let cur = if self.active { &mut self.buf0 } else { &mut self.buf1 };
+        // cur.set(1, 0, 1);
+        // cur.set(2, 1, 1);
+        // cur.set(0, 2, 1);
+        // cur.set(1, 2, 1);
+        // cur.set(2, 2, 1);
+        if let Some(pat) = Pattern::from_rle("
+#N Tanner's p46
+#O Tanner Jacobi
+#C A period 46 oscillator discovered by Tanner Jacobi in October 2017.
+#C https://conwaylife.com/wiki/Tanner%27s_p46
+x = 13, y = 26, rule = B3/S23
+2b2o9b$2bo10b$3bo9b$2b2o9b$13b$9b2o2b$9bo3b$10bo2b$9b2o2b$b2o10b$b2o6b
+2o2b$o7bobo2b$b2o6bo3b$b2o7b3o$12bo$13b$13b$13b$13b$13b$13b$b2o10b$b2o
+2b2o6b$5bobo5b$7bo5b$7b2o4b!
+") {
+            self.spawn(30, 10, 1, &pat);
+        }
+    }
+    pub fn step(&mut self) {
+        let (cur, next) = if self.active {
+            (&mut self.buf0, &mut self.buf1)
+        } else {
+            (&mut self.buf1, &mut self.buf0)
+        };
+        for ux in 0..WIDTH {
+            for uy in 0..HEIGHT {
+                let x = ux as _; let y = uy as _;
+                let n = cur.count_neighbors(x, y);
+                if cur.is_nonzero(x, y) && n != 2 && n != 3{
+                    next.set(x, y, 0)
+                } else if n == 3 {
+                    next.set(x, y, 1)
+                } else {
+                    next.set(x, y, cur.get(x, y))
+                }
+            }
+        }
+        self.active = !self.active;
+    }
+    pub fn upload(&self, ctx: &context::Context) {
+        let cur = if self.active { &self.buf0 } else { &self.buf1 };
+        unsafe {
+            let err = ctx.gl.get_error();
+            self.tex.bind(ctx);
+            ctx.gl.tex_image_2d(
+                glow::TEXTURE_2D,
+                0,
+                glow::R8 as i32,
+                WIDTH as i32,
+                HEIGHT as i32,
+                0,
+                glow::RED,
+                glow::UNSIGNED_BYTE,
+                Some(&cur.buf),
+            );
+            ctx.gl.generate_mipmap(glow::TEXTURE_2D);
+        }
+    }
+    pub fn update(&mut self, ctx: &context::Context, st: &mut state::State) {
+        if st.tick % 10 == 0 {
+            self.step();
+            self.upload(ctx);
+        }
+    }
+    pub fn render(&self, ctx: &context::Context, st: &mut state::State) {
+        st.bind_2d(ctx, &self.shader);
+        self.tex.bind(ctx);
+        self.shader.set_position_2d(
+            ctx,
+            &glam::Vec2::new(0.0, 0.0),
+            &glam::Vec2::new(1920.0, 1080.0)
+        );
+        st.mesh_square.render(ctx);
+    }
+}
diff --git a/crates/renderer/src/overlay/drawing.rs b/crates/renderer/src/overlay/drawing.rs
new file mode 100644
index 0000000..8088b5a
--- /dev/null
+++ b/crates/renderer/src/overlay/drawing.rs
@@ -0,0 +1,128 @@
+use teleia::*;
+
+use glow::HasContext;
+
+use crate::{background, input};
+
+pub const SCALE: usize = 4;
+pub const WIDTH: usize = 1920 / SCALE;
+pub const HEIGHT: usize = 1080 / SCALE;
+pub struct Drawing {
+    pub tex: texture::Texture,
+    pub pixels: [u8; WIDTH * HEIGHT],
+    pub last_point: Option<(i32, i32)>,
+    pub shader_white: shader::Shader,
+    pub shader_background: shader::Shader,
+}
+impl Drawing {
+    pub fn new(ctx: &context::Context) -> Self {
+        let shader_background = shader::Shader::new(
+            ctx,
+            include_str!("../assets/shaders/background/vert.glsl"),
+            include_str!("../assets/shaders/background/frag.glsl"),
+        );
+        shader_background.set_i32(ctx, "background", 1);
+        Self {
+            tex: texture::Texture::new_empty(ctx),
+            pixels: [0; WIDTH * HEIGHT],
+            last_point: None,
+            shader_white: shader::Shader::new(
+                ctx,
+                include_str!("../assets/shaders/white/vert.glsl"),
+                include_str!("../assets/shaders/white/frag.glsl"),
+            ),
+            shader_background,
+        }
+    }
+    pub fn coord(&self, x: usize, y: usize) -> Option<usize> {
+        if x >= WIDTH || y >= HEIGHT {
+            None
+        } else {
+            Some(x + y * WIDTH)
+        }
+    }
+    pub fn set(&mut self, val: u8, x: i32, y: i32) {
+        self.coord(x as usize, y as usize).map(|idx| self.pixels[idx] = val);
+    }
+    pub fn point(&mut self, val: u8, x: i32, y: i32) {
+        self.set(val, x, y - 1);
+        self.set(val, x - 1, y);
+        self.set(val, x, y);
+        self.set(val, x + 1, y);
+        self.set(val, x, y + 1);
+    }
+    pub fn line(&mut self, val: u8, (mut x0, mut y0): (i32, i32), (x1, y1): (i32, i32)) {
+        let dx = (x1 - x0).abs();
+        let sx = if x0 < x1 { 1 } else { -1 };
+        let dy = -((y1 - y0).abs());
+        let sy = if y0 < y1 { 1 } else { -1 };
+        let mut error = dx + dy;
+        loop {
+            self.point(val, x0, y0);
+            let e2 = 2 * error;
+            if e2 >= dy {
+                if x0 == x1 { break; }
+                error += dy;
+                x0 += sx;
+            }
+            if e2 <= dx {
+                if y0 == y1 { break; }
+                error += dx;
+                y0 += sy;
+            }
+        }
+    }
+    pub fn upload(&self, ctx: &context::Context) {
+        unsafe {
+            let err = ctx.gl.get_error();
+            self.tex.bind(ctx);
+            ctx.gl.tex_image_2d(
+                glow::TEXTURE_2D,
+                0,
+                glow::R8 as i32,
+                WIDTH as i32,
+                HEIGHT as i32,
+                0,
+                glow::RED,
+                glow::UNSIGNED_BYTE,
+                Some(&self.pixels),
+            );
+            ctx.gl.generate_mipmap(glow::TEXTURE_2D);
+        }
+    }
+    pub fn update(&mut self, ctx: &context::Context, st: &mut state::State, inp: &mut input::Input) {
+        match inp.get_command() {
+            input::Command::Drawing => {
+                let (sx, sy) = inp.get_mouse();
+                let x = sx / (SCALE as i32);
+                let y = sy / (SCALE as i32);
+                if let Some(last) = self.last_point {
+                    self.line(1, last, (x, y));
+                } else {
+                    self.point(1, x, y);
+                }
+                self.last_point = Some((x, y));
+            },
+            input::Command::EraseAll => {
+                self.pixels.fill(0);
+                self.last_point = None;
+            },
+            input::Command::None => {
+                self.last_point = None;
+            },
+            _ => {},
+        }
+        self.upload(ctx);
+    }
+    pub fn render(&self, ctx: &context::Context, st: &mut state::State, bg: &background::Backgrounds) {
+        st.bind_2d(ctx, &self.shader_background);
+        self.tex.bind(ctx);
+        bg.drawing.bind_index(ctx, 1);
+        self.shader_background.set_position_2d(
+            ctx,
+            &glam::Vec2::new(0.0, 0.0),
+            &glam::Vec2::new(1920.0, 1080.0)
+        );
+        st.mesh_square.render(ctx);
+    }
+}
diff --git a/crates/renderer/src/overlay/mod.rs b/crates/renderer/src/overlay/mod.rs
index 5fff8af..6bfa1ae 100644
--- a/crates/renderer/src/overlay/mod.rs
+++ b/crates/renderer/src/overlay/mod.rs
@@ -1,3 +1,5 @@
 pub mod model;
 pub mod shader;
 pub mod multi;
+pub mod drawing;
+pub mod automata;
diff --git a/crates/renderer/src/overlay/model.rs b/crates/renderer/src/overlay/model.rs
index 779f36b..14f6076 100644
--- a/crates/renderer/src/overlay/model.rs
+++ b/crates/renderer/src/overlay/model.rs
@@ -5,7 +5,7 @@ use std::{collections::HashMap, f32::consts::PI};
 use lexpr::sexp;
 use base64::prelude::*;
 
-use crate::{assets, fig, terminal, toggle};
+use crate::{assets, terminal, toggle};
 
 pub enum RenderMode {
     Overlay,
@@ -19,7 +19,7 @@ pub struct Overlay {
     model_neck_base: glam::Mat4,
     model_fb: framebuffer::Framebuffer,
     terminal: terminal::Terminal,
-    fig: fig::Client,
+    fig: fig::SexpClient,
     tracking_eyes: (f32, f32),
     tracking_mouth: f32,
     tracking_neck: glam::Quat,
@@ -44,7 +44,7 @@ impl Overlay {
                 &glam::Vec2::ZERO
             ),
             terminal: terminal::Terminal::new(ctx, 64, 64),
-            fig: fig::Client::new("shiro:32050", &[
+            fig: fig::SexpClient::new("shiro:32050", &[
                 sexp!((avatar toggle)),
                 sexp!((avatar toggle set)),
                 sexp!((avatar toggle unset)),
@@ -70,7 +70,7 @@ impl Overlay {
         // TODO also reset terminal
         self.toggles.reset();
     }
-    pub fn handle_tracking(&mut self, msg: fig::Message) -> Option<()> {
+    pub fn handle_tracking(&mut self, msg: fig::SexpMessage) -> Option<()> {
         let eyes = msg.data.get(0)?;
         let eye_left = eyes.get(0)?.as_str()?.parse::<f32>().ok()?;
         let eye_right = eyes.get(1)?.as_str()?.parse::<f32>().ok()?;
@@ -84,14 +84,14 @@ impl Overlay {
         self.tracking_neck = glam::Quat::from_euler(glam::EulerRot::XYZ, euler_x, euler_y, euler_z);
         Some(())
     }
-    pub fn handle_text(&mut self, msg: fig::Message) -> Option<()> {
+    pub fn handle_text(&mut self, msg: fig::SexpMessage) -> Option<()> {
         let bs = BASE64_STANDARD.decode(msg.data.get(0)?.as_str()?).ok()?;
         let s = std::str::from_utf8(&bs).ok()?;
         log::info!("handle_text: {}", s);
         self.terminal.fill_string(s);
         Some(())
     }
-    pub fn handle_frame(&mut self, msg: fig::Message) -> Option<()> {
+    pub fn handle_frame(&mut self, msg: fig::SexpMessage) -> Option<()> {
         let data = BASE64_STANDARD.decode(msg.data.get(0)?.as_str()?).ok()?;
         for (i, c) in data.chunks_exact(3).enumerate() {
             if let [r, g, b] = c {
diff --git a/crates/renderer/src/overlay/multi.rs b/crates/renderer/src/overlay/multi.rs
index 41ec12f..e3d0100 100644
--- a/crates/renderer/src/overlay/multi.rs
+++ b/crates/renderer/src/overlay/multi.rs
@@ -29,7 +29,7 @@ impl Model {
             tracking_neck: glam::Quat::IDENTITY,
         }
     }
-    pub fn handle_tracking(&mut self, msg: fig::Message) -> Option<()> {
+    pub fn handle_tracking(&mut self, msg: fig::SexpMessage) -> Option<()> {
         let eyes = msg.data.get(0)?;
         let eye_left = eyes.get(0)?.as_str()?.parse::<f32>().ok()?;
         let eye_right = eyes.get(1)?.as_str()?.parse::<f32>().ok()?;
@@ -66,7 +66,7 @@ pub struct Overlay {
     models: Vec<Model>,
     fb: framebuffer::Framebuffer,
     term: terminal::Terminal,
-    fig: fig::Client,
+    fig: fig::SexpClient,
 }
 impl Overlay {
     pub fn new(ctx: &context::Context) -> Self {
@@ -83,7 +83,7 @@ impl Overlay {
             ),
             term: terminal::Terminal::new(ctx, 128, 64),
             models,
-            fig: fig::Client::new("shiro:32050", &[
+            fig: fig::SexpClient::new("shiro:32050", &[
                 sexp!((avatar toggle)),
                 sexp!((avatar toggle set)),
                 sexp!((avatar toggle unset)),
@@ -124,7 +124,7 @@ impl Overlay {
             )
         );
         tex.bind(ctx);
-        self.assets.mesh_square.render(ctx);
+        st.mesh_square.render(ctx);
     }
 }
 impl teleia::state::Game for Overlay {
diff --git a/crates/renderer/src/overlay/shader.rs b/crates/renderer/src/overlay/shader.rs
index 97e4d92..81e29f4 100644
--- a/crates/renderer/src/overlay/shader.rs
+++ b/crates/renderer/src/overlay/shader.rs
@@ -3,11 +3,11 @@ use teleia::*;
 use std::{collections::HashMap, f32::consts::PI};
 use lexpr::sexp;
 use base64::prelude::*;
-use device_query::DeviceQuery;
 
 use glow::HasContext;
 
-use crate::{assets, fig, toggle, background};
+use crate::{assets, fig, toggle, input, background};
+use super::{drawing, automata};
 
 pub struct Chat {
     author: String,
@@ -26,103 +26,11 @@ impl Chat {
     }
 }
 
-const DRAWING_WIDTH: usize = 1920 / 4;
-const DRAWING_HEIGHT: usize = 1080 / 4;
-pub enum DrawingCommand {
-    None,
-    Drawing,
-    EraseAll,
-}
-pub struct Drawing {
-    tex: texture::Texture,
-    pixels: [u8; DRAWING_WIDTH * DRAWING_HEIGHT],
-    last_point: Option<(i32, i32)>,
-    shader_white: shader::Shader,
-    shader_background: shader::Shader,
-}
-impl Drawing {
-    pub fn new(ctx: &context::Context) -> Self {
-        let shader_background = shader::Shader::new(
-            ctx,
-            include_str!("../assets/shaders/background/vert.glsl"),
-            include_str!("../assets/shaders/background/frag.glsl"),
-        );
-        shader_background.set_i32(ctx, "background", 1);
-        Self {
-            tex: texture::Texture::new_empty(ctx),
-            pixels: [0; DRAWING_WIDTH * DRAWING_HEIGHT],
-            last_point: None,
-            shader_white: shader::Shader::new(
-                ctx,
-                include_str!("../assets/shaders/white/vert.glsl"),
-                include_str!("../assets/shaders/white/frag.glsl"),
-            ),
-            shader_background,
-        }
-    }
-    pub fn coord(&self, x: usize, y: usize) -> Option<usize> {
-        if x >= DRAWING_WIDTH || y >= DRAWING_HEIGHT {
-            None
-        } else {
-            Some(x + y * DRAWING_WIDTH)
-        }
-    }
-    pub fn set(&mut self, val: u8, x: i32, y: i32) {
-        self.coord(x as usize, y as usize).map(|idx| self.pixels[idx] = val);
-    }
-    pub fn point(&mut self, val: u8, x: i32, y: i32) {
-        self.set(val, x, y - 1);
-        self.set(val, x - 1, y);
-        self.set(val, x, y);
-        self.set(val, x + 1, y);
-        self.set(val, x, y + 1);
-    }
-    pub fn line(&mut self, val: u8, (mut x0, mut y0): (i32, i32), (x1, y1): (i32, i32)) {
-        let dx = (x1 - x0).abs();
-        let sx = if x0 < x1 { 1 } else { -1 };
-        let dy = -((y1 - y0).abs());
-        let sy = if y0 < y1 { 1 } else { -1 };
-        let mut error = dx + dy;
-        loop {
-            self.point(val, x0, y0);
-            let e2 = 2 * error;
-            if e2 >= dy {
-                if x0 == x1 { break; }
-                error += dy;
-                x0 += sx;
-            }
-            if e2 <= dx {
-                if y0 == y1 { break; }
-                error += dx;
-                y0 += sy;
-            }
-        }
-    }
-    pub fn upload(&self, ctx: &context::Context) {
-        unsafe {
-            let err = ctx.gl.get_error();
-            self.tex.bind(ctx);
-            ctx.gl.tex_image_2d(
-                glow::TEXTURE_2D,
-                0,
-                glow::R8 as i32,
-                DRAWING_WIDTH as i32,
-                DRAWING_HEIGHT as i32,
-                0,
-                glow::RED,
-                glow::UNSIGNED_BYTE,
-                Some(&self.pixels),
-            );
-            ctx.gl.generate_mipmap(glow::TEXTURE_2D);
-        }
-    }
-}
-
 pub struct Overlay {
     assets: assets::Assets,
     model: scene::Scene,
     model_neck_base: glam::Mat4,
-    fig: fig::Client,
+    fig: fig::SexpClient,
     fig_binary: fig::BinaryClient,
     tracking_eyes: (f32, f32),
     tracking_mouth: f32,
@@ -134,9 +42,10 @@ pub struct Overlay {
     muzak_author: Option<String>,
     chat: Chat,
     toggles: toggle::Toggles,
+    input: input::Input,
     backgrounds: background::Backgrounds,
-    drawing: Drawing,
-    device: device_query::DeviceState,
+    drawing: drawing::Drawing,
+    automata: automata::Board,
 }
 
 impl Overlay {
@@ -147,11 +56,13 @@ impl Overlay {
             .expect("failed to find neck joint")
             .transform;
         let throwshade = newton_throwshade::ThrowShade::new();
+        let mut automata = automata::Board::new(ctx);
+        automata.test_glider();
         Self {
             assets: assets::Assets::new(ctx),
             model,
             model_neck_base,
-            fig: fig::Client::new("shiro:32050", &[
+            fig: fig::SexpClient::new("shiro:32050", &[
                 sexp!((avatar toggle)),
                 sexp!((avatar toggle set)),
                 sexp!((avatar toggle unset)),
@@ -166,7 +77,7 @@ impl Overlay {
                 sexp!((avatar overlay cursor)),
                 sexp!((avatar overlay emacs)),
             ]),
-            fig_binary: fig::BinaryClient::new("shiro:32051", &[
+            fig_binary: fig::BinaryClient::new("shiro:32051", false, &[
                 b"background frame"
             ]),
             tracking_eyes: (1.0, 1.0),
@@ -180,21 +91,9 @@ impl Overlay {
             chat: Chat::new(),
             toggles: toggle::Toggles::new(),
             backgrounds: background::Backgrounds::new(ctx),
-            drawing: Drawing::new(ctx),
-            device: device_query::DeviceState::new(),
-        }
-    }
-    fn get_mouse(&self) -> (i32, i32) {
-        self.device.get_mouse().coords
-    }
-    fn get_drawing_command(&mut self) -> DrawingCommand {
-        let keys = self.device.get_keys();
-        if keys.contains(&device_query::Keycode::LMeta) {
-            DrawingCommand::Drawing
-        } else if keys.contains(&device_query::Keycode::RMeta) {
-            DrawingCommand::EraseAll
-        } else {
-            DrawingCommand::None
+            input: input::Input::new(),
+            drawing: drawing::Drawing::new(ctx),
+            automata,
         }
     }
     pub fn handle_reset(&mut self, ctx: &context::Context) {
@@ -203,7 +102,7 @@ impl Overlay {
         self.throwshade.shader = None;
         self.toggles.reset();
     }
-    pub fn handle_tracking(&mut self, msg: fig::Message) -> Option<()> {
+    pub fn handle_tracking(&mut self, msg: fig::SexpMessage) -> Option<()> {
         let eyes = msg.data.get(0)?;
         let eye_left = eyes.get(0)?.as_str()?.parse::<f32>().ok()?;
         let eye_right = eyes.get(1)?.as_str()?.parse::<f32>().ok()?;
@@ -220,7 +119,7 @@ impl Overlay {
     pub fn handle_overlay_shader(
         &mut self,
         ctx: &context::Context, st: &state::State,
-        msg: fig::Message
+        msg: fig::SexpMessage
     ) -> Option<()> {
         let ba = BASE64_STANDARD.decode(msg.data.get(0)?.as_str()?).ok()?;
         let author = String::from_utf8_lossy(&ba);
@@ -233,7 +132,7 @@ impl Overlay {
         }
         Some(())
     }
-    pub fn handle_overlay_muzak(&mut self, msg: fig::Message) -> Option<()> {
+    pub fn handle_overlay_muzak(&mut self, msg: fig::SexpMessage) -> Option<()> {
         let ba = BASE64_STANDARD.decode(msg.data.get(0)?.as_str()?).ok()?;
         let author = String::from_utf8_lossy(&ba);
         self.muzak_author = Some(author.to_string());
@@ -243,7 +142,7 @@ impl Overlay {
         self.muzak_author = None;
         Some(())
     }
-    pub fn handle_overlay_chat(&mut self, msg: fig::Message) -> Option<()> {
+    pub fn handle_overlay_chat(&mut self, msg: fig::SexpMessage) -> Option<()> {
         let ba = BASE64_STANDARD.decode(msg.data.get(0)?.as_str()?).ok()?;
         let a = String::from_utf8_lossy(&ba);
         let bs = BASE64_STANDARD.decode(msg.data.get(1)?.as_str()?).ok()?;
@@ -256,27 +155,16 @@ impl Overlay {
         self.chat.biblicality = biblicality;
         Some(())
     }
-    pub fn handle_overlay_cursor(&mut self, msg: fig::Message) -> Option<()> {
+    pub fn handle_overlay_cursor(&mut self, msg: fig::SexpMessage) -> Option<()> {
         let cursor_x = msg.data.get(0)?.as_i64()? as f32;
         let cursor_y = msg.data.get(1)?.as_i64()? as f32;
         self.emacs_cursor = (cursor_x, cursor_y);
         Some(())
     }
-    pub fn handle_overlay_emacs(&mut self, msg: fig::Message) -> Option<()> {
+    pub fn handle_overlay_emacs(&mut self, msg: fig::SexpMessage) -> Option<()> {
         self.emacs_heartrate = msg.data.get(0)?.as_i64()? as i32;
         Some(())
     }
-    pub fn render_drawing(&self, ctx: &context::Context, st: &mut state::State) {
-        st.bind_2d(ctx, &self.drawing.shader_background);
-        self.drawing.tex.bind(ctx);
-        self.backgrounds.drawing.bind_index(ctx, 1);
-        self.drawing.shader_background.set_position_2d(
-            ctx,
-            &glam::Vec2::new(0.0, 0.0),
-            &glam::Vec2::new(1920.0, 1080.0)
-        );
-        self.assets.mesh_square.render(ctx);
-    }
 }
 
 impl teleia::state::Game for Overlay {
@@ -293,7 +181,7 @@ impl teleia::state::Game for Overlay {
             &glam::Vec3::new(0.0, 0.0, -1.0),
             &glam::Vec3::new(0.0, 1.0, 0.0),
         );
-        let (x, y) = self.get_mouse();
+        let (x, y) = self.input.get_mouse();
         self.mouse_cursor = (x as f32, y as f32);
         while let Some(msg) = self.fig.pump() {
             let malformed = format!("malformed {} data: {}", msg.event, msg.data);
@@ -338,6 +226,8 @@ impl teleia::state::Game for Overlay {
         if let Some(n) = self.model.nodes_by_name.get("J_Bip_C_Neck").and_then(|i| self.model.nodes.get_mut(*i)) {
             n.transform = self.model_neck_base * glam::Mat4::from_quat(self.tracking_neck);
         }
+        self.drawing.update(ctx, st, &mut self.input);
+        self.automata.update(ctx, st);
         Ok(())
     }
     fn render(&mut self, ctx: &context::Context, st: &mut state::State) -> Erm<()> {
@@ -378,7 +268,7 @@ impl teleia::state::Game for Overlay {
                 &glam::Vec2::new(1100.0, 300.0),
                 &glam::Vec2::new(800.0, 600.0)
             );
-            self.assets.mesh_square.render(ctx);
+            st.mesh_square.render(ctx);
         }
         let mut authors = Vec::new();
         if let Some(_) = &self.throwshade.shader {
@@ -389,28 +279,8 @@ impl teleia::state::Game for Overlay {
         }
         let astr: String = authors.join(", ");
         self.assets.font.render_text(ctx, &glam::Vec2::new(0.0, 0.0), &astr);
-        match self.get_drawing_command() {
-            DrawingCommand::Drawing => {
-                let (sx, sy) = self.get_mouse();
-                let x = sx / 4;
-                let y = sy / 4;
-                if let Some(last) = self.drawing.last_point {
-                    self.drawing.line(1, last, (x, y));
-                } else {
-                    self.drawing.point(1, x, y);
-                }
-                self.drawing.last_point = Some((x, y));
-            },
-            DrawingCommand::EraseAll => {
-                self.drawing.pixels.fill(0);
-                self.drawing.last_point = None;
-            },
-            DrawingCommand::None => {
-                self.drawing.last_point = None;
-            },
-        }
-        self.drawing.upload(ctx);
-        self.render_drawing(ctx, st);
+        self.drawing.render(ctx, st, &self.backgrounds);
+        self.automata.render(ctx, st);
         Ok(())
     }
 }