From 314a11bcba99ee08e3599c6df6dc17ec4334eb2f Mon Sep 17 00:00:00 2001 From: Harivansh Rathi Date: Tue, 24 Mar 2026 21:33:30 -0400 Subject: [PATCH] Phase 4: mouse + keyboard input via enigo - Add enigo 0.6 dependency (x11rb/XTest backend) - Enigo field in X11Backend for input simulation - Click, double-click at absolute coords or @wN ref centers - Type text into focused window, press individual keys - Hotkey combinations (modifier press, key click, modifier release) - Mouse move, scroll (vertical/horizontal), drag operations - parse_key() mapping human-readable names to enigo Key values - Handler dispatchers with ref resolution and coord parsing --- Cargo.lock | 262 ++++++++++++++++++++++++++++++++++++++---- Cargo.toml | 1 + src/backend/x11.rs | 191 +++++++++++++++++++++++++----- src/daemon/handler.rs | 217 +++++++++++++++++++++++++++++++++- 4 files changed, 622 insertions(+), 49 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index efa560b..1cd7282 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -586,6 +586,46 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9885fa71e26b8ab7855e2ec7cae6e9b380edff76cd052e07c683a0319d51b3a2" +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "core-graphics" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "064badf302c3194842cf2c5d61f56cc88e54a759313879cdf03abdd27d0c3b97" +dependencies = [ + "bitflags 2.11.0", + "core-foundation", + "core-graphics-types", + "foreign-types", + "libc", +] + +[[package]] +name = "core-graphics-types" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d44a101f213f6c4cdc1853d4b78aef6db6bdfa3468798cc1d9912f4735013eb" +dependencies = [ + "bitflags 2.11.0", + "core-foundation", + "libc", +] + [[package]] name = "core2" version = "0.4.0" @@ -643,6 +683,7 @@ dependencies = [ "anyhow", "clap", "dirs", + "enigo", "image", "imageproc", "libc", @@ -764,6 +805,27 @@ version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "66b7e2430c6dff6a955451e2cfc438f09cea1965a9d6f87f7e3b90decc014099" +[[package]] +name = "enigo" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71c6c56e50f7acae2906a0dcbb34529ca647e40421119ad5d12e7f8ba6e50010" +dependencies = [ + "core-foundation", + "core-graphics", + "foreign-types-shared", + "libc", + "log", + "nom 8.0.0", + "objc2", + "objc2-app-kit", + "objc2-foundation", + "windows 0.61.3", + "x11rb", + "xkbcommon", + "xkeysym", +] + [[package]] name = "enumflags2" version = "0.7.12" @@ -914,6 +976,33 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" +[[package]] +name = "foreign-types" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d737d9aa519fb7b749cbc3b962edcf310a8dd1f4b67c91c4f83975dbdd17d965" +dependencies = [ + "foreign-types-macros", + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-macros" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a5c6c585bc94aaf2c7b51dd4c2ba22680844aba4c687be581871a6f518c5742" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "foreign-types-shared" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa9a19cbb55df58761df49b23516a86d432839add4af60fc256da840f66ed35b" + [[package]] name = "form_urlencoded" version = "1.2.2" @@ -972,6 +1061,16 @@ dependencies = [ "libc", ] +[[package]] +name = "gethostname" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bd49230192a3797a9a4d6abe9b3eed6f7fa4c8a8a4947977c6f80025f92cbd8" +dependencies = [ + "rustix 1.1.4", + "windows-link 0.2.1", +] + [[package]] name = "getrandom" version = "0.2.17" @@ -1483,7 +1582,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" dependencies = [ "cfg-if", - "windows-link", + "windows-link 0.2.1", ] [[package]] @@ -2163,7 +2262,7 @@ dependencies = [ "libc", "redox_syscall", "smallvec", - "windows-link", + "windows-link 0.2.1", ] [[package]] @@ -3341,16 +3440,38 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72069c3113ab32ab29e5584db3c6ec55d416895e60715417b5b883a357c3e471" +[[package]] +name = "windows" +version = "0.61.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9babd3a767a4c1aef6900409f85f5d53ce2544ccdfaa86dad48c91782c6d6893" +dependencies = [ + "windows-collections 0.2.0", + "windows-core 0.61.2", + "windows-future 0.2.1", + "windows-link 0.1.3", + "windows-numerics 0.2.0", +] + [[package]] name = "windows" version = "0.62.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "527fadee13e0c05939a6a05d5bd6eec6cd2e3dbd648b9f8e447c6518133d8580" dependencies = [ - "windows-collections", - "windows-core", - "windows-future", - "windows-numerics", + "windows-collections 0.3.2", + "windows-core 0.62.2", + "windows-future 0.3.2", + "windows-numerics 0.3.1", +] + +[[package]] +name = "windows-collections" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3beeceb5e5cfd9eb1d76b381630e82c4241ccd0d27f1a39ed41b2760b255c5e8" +dependencies = [ + "windows-core 0.61.2", ] [[package]] @@ -3359,7 +3480,20 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "23b2d95af1a8a14a3c7367e1ed4fc9c20e0a26e79551b1454d72583c97cc6610" dependencies = [ - "windows-core", + "windows-core 0.62.2", +] + +[[package]] +name = "windows-core" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0fdd3ddb90610c7638aa2b3a3ab2904fb9e5cdbecc643ddb3647212781c4ae3" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link 0.1.3", + "windows-result 0.3.4", + "windows-strings 0.4.2", ] [[package]] @@ -3370,9 +3504,20 @@ checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" dependencies = [ "windows-implement", "windows-interface", - "windows-link", - "windows-result", - "windows-strings", + "windows-link 0.2.1", + "windows-result 0.4.1", + "windows-strings 0.5.1", +] + +[[package]] +name = "windows-future" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc6a41e98427b19fe4b73c550f060b59fa592d7d686537eebf9385621bfbad8e" +dependencies = [ + "windows-core 0.61.2", + "windows-link 0.1.3", + "windows-threading 0.1.0", ] [[package]] @@ -3381,9 +3526,9 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e1d6f90251fe18a279739e78025bd6ddc52a7e22f921070ccdc67dde84c605cb" dependencies = [ - "windows-core", - "windows-link", - "windows-threading", + "windows-core 0.62.2", + "windows-link 0.2.1", + "windows-threading 0.2.1", ] [[package]] @@ -3408,20 +3553,45 @@ dependencies = [ "syn", ] +[[package]] +name = "windows-link" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e6ad25900d524eaabdbbb96d20b4311e1e7ae1699af4fb28c17ae66c80d798a" + [[package]] name = "windows-link" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +[[package]] +name = "windows-numerics" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9150af68066c4c5c07ddc0ce30421554771e528bde427614c61038bc2c92c2b1" +dependencies = [ + "windows-core 0.61.2", + "windows-link 0.1.3", +] + [[package]] name = "windows-numerics" version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e2e40844ac143cdb44aead537bbf727de9b044e107a0f1220392177d15b0f26" dependencies = [ - "windows-core", - "windows-link", + "windows-core 0.62.2", + "windows-link 0.2.1", +] + +[[package]] +name = "windows-result" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56f42bd332cc6c8eac5af113fc0c1fd6a8fd2aa08a0119358686e5160d0586c6" +dependencies = [ + "windows-link 0.1.3", ] [[package]] @@ -3430,7 +3600,16 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" dependencies = [ - "windows-link", + "windows-link 0.2.1", +] + +[[package]] +name = "windows-strings" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56e6c93f3a0c3b36176cb1327a4958a0353d5d166c2a35cb268ace15e91d3b57" +dependencies = [ + "windows-link 0.1.3", ] [[package]] @@ -3439,7 +3618,7 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" dependencies = [ - "windows-link", + "windows-link 0.2.1", ] [[package]] @@ -3457,7 +3636,7 @@ version = "0.61.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" dependencies = [ - "windows-link", + "windows-link 0.2.1", ] [[package]] @@ -3476,13 +3655,22 @@ dependencies = [ "windows_x86_64_msvc", ] +[[package]] +name = "windows-threading" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b66463ad2e0ea3bbf808b7f1d371311c80e115c0b71d60efc142cafbcfb057a6" +dependencies = [ + "windows-link 0.1.3", +] + [[package]] name = "windows-threading" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3949bd5b99cafdf1c7ca86b43ca564028dfe27d66958f2470940f73d86d75b37" dependencies = [ - "windows-link", + "windows-link 0.2.1", ] [[package]] @@ -3645,6 +3833,23 @@ version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9" +[[package]] +name = "x11rb" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9993aa5be5a26815fe2c3eacfc1fde061fc1a1f094bf1ad2a18bf9c495dd7414" +dependencies = [ + "gethostname", + "rustix 1.1.4", + "x11rb-protocol", +] + +[[package]] +name = "x11rb-protocol" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea6fc2961e4ef194dcbfe56bb845534d0dc8098940c7e5c012a258bfec6701bd" + [[package]] name = "xcap" version = "0.8.3" @@ -3672,7 +3877,7 @@ dependencies = [ "thiserror", "url", "widestring", - "windows", + "windows 0.62.2", "xcb", "zbus", ] @@ -3688,6 +3893,23 @@ dependencies = [ "quick-xml 0.30.0", ] +[[package]] +name = "xkbcommon" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7a974f48060a14e95705c01f24ad9c3345022f4d97441b8a36beb7ed5c4a02d" +dependencies = [ + "libc", + "memmap2", + "xkeysym", +] + +[[package]] +name = "xkeysym" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9cc00251562a284751c9973bace760d86c0276c471b4be569fe6b068ee97a56" + [[package]] name = "xml-rs" version = "0.8.28" diff --git a/Cargo.toml b/Cargo.toml index 4b36e64..3947a75 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,3 +19,4 @@ xcap = "0.8" image = { version = "0.25", features = ["png"] } imageproc = "0.26" ab_glyph = "0.2" +enigo = "0.6" diff --git a/src/backend/x11.rs b/src/backend/x11.rs index 8486f38..0ee3e04 100644 --- a/src/backend/x11.rs +++ b/src/backend/x11.rs @@ -1,31 +1,34 @@ use anyhow::{Context, Result}; +use enigo::{ + Axis, Button, Coordinate, Direction, Enigo, Key, Keyboard, Mouse, Settings, +}; -use crate::core::types::{Snapshot, WindowInfo}; use super::annotate::annotate_screenshot; +use crate::core::types::{Snapshot, WindowInfo}; pub struct X11Backend { - // enigo and x11rb connections added in later phases + enigo: Enigo, } impl X11Backend { pub fn new() -> Result { - Ok(Self {}) + let enigo = Enigo::new(&Settings::default()) + .map_err(|e| anyhow::anyhow!("Failed to initialize enigo: {e}"))?; + Ok(Self { enigo }) } } impl super::DesktopBackend for X11Backend { fn snapshot(&mut self, annotate: bool) -> Result { // Get z-ordered window list via xcap (topmost first internally) - let windows = xcap::Window::all() - .context("Failed to enumerate windows")?; + let windows = xcap::Window::all().context("Failed to enumerate windows")?; // Get primary monitor for screenshot - let monitors = xcap::Monitor::all() - .context("Failed to enumerate monitors")?; - let monitor = monitors.into_iter().next() - .context("No monitor found")?; + let monitors = xcap::Monitor::all().context("Failed to enumerate monitors")?; + let monitor = monitors.into_iter().next().context("No monitor found")?; - let mut image = monitor.capture_image() + let mut image = monitor + .capture_image() .context("Failed to capture screenshot")?; // Build window info list @@ -78,7 +81,8 @@ impl super::DesktopBackend for X11Backend { .unwrap_or_default() .as_millis(); let screenshot_path = format!("/tmp/desktop-ctl-{timestamp}.png"); - image.save(&screenshot_path) + image + .save(&screenshot_path) .context("Failed to save screenshot")?; Ok(Snapshot { @@ -87,7 +91,7 @@ impl super::DesktopBackend for X11Backend { }) } - // Stub implementations for methods added in later phases + // Phase 5: window management (stub) fn focus_window(&mut self, _xcb_id: u32) -> Result<()> { anyhow::bail!("Window management not yet implemented (Phase 5)") } @@ -104,38 +108,120 @@ impl super::DesktopBackend for X11Backend { anyhow::bail!("Window management not yet implemented (Phase 5)") } - fn click(&mut self, _x: i32, _y: i32) -> Result<()> { - anyhow::bail!("Input simulation not yet implemented (Phase 4)") + // Phase 4: input simulation via enigo + + fn click(&mut self, x: i32, y: i32) -> Result<()> { + self.enigo + .move_mouse(x, y, Coordinate::Abs) + .map_err(|e| anyhow::anyhow!("Mouse move failed: {e}"))?; + std::thread::sleep(std::time::Duration::from_millis(10)); + self.enigo + .button(Button::Left, Direction::Click) + .map_err(|e| anyhow::anyhow!("Click failed: {e}"))?; + Ok(()) } - fn dblclick(&mut self, _x: i32, _y: i32) -> Result<()> { - anyhow::bail!("Input simulation not yet implemented (Phase 4)") + fn dblclick(&mut self, x: i32, y: i32) -> Result<()> { + self.enigo + .move_mouse(x, y, Coordinate::Abs) + .map_err(|e| anyhow::anyhow!("Mouse move failed: {e}"))?; + std::thread::sleep(std::time::Duration::from_millis(10)); + self.enigo + .button(Button::Left, Direction::Click) + .map_err(|e| anyhow::anyhow!("First click failed: {e}"))?; + std::thread::sleep(std::time::Duration::from_millis(50)); + self.enigo + .button(Button::Left, Direction::Click) + .map_err(|e| anyhow::anyhow!("Second click failed: {e}"))?; + Ok(()) } - fn type_text(&mut self, _text: &str) -> Result<()> { - anyhow::bail!("Input simulation not yet implemented (Phase 4)") + fn type_text(&mut self, text: &str) -> Result<()> { + self.enigo + .text(text) + .map_err(|e| anyhow::anyhow!("Type failed: {e}"))?; + Ok(()) } - fn press_key(&mut self, _key: &str) -> Result<()> { - anyhow::bail!("Input simulation not yet implemented (Phase 4)") + fn press_key(&mut self, key: &str) -> Result<()> { + let k = parse_key(key)?; + self.enigo + .key(k, Direction::Click) + .map_err(|e| anyhow::anyhow!("Key press failed: {e}"))?; + Ok(()) } - fn hotkey(&mut self, _keys: &[String]) -> Result<()> { - anyhow::bail!("Input simulation not yet implemented (Phase 4)") + fn hotkey(&mut self, keys: &[String]) -> Result<()> { + // Press all modifier keys, click the last key, release modifiers in reverse + let parsed: Vec = keys + .iter() + .map(|k| parse_key(k)) + .collect::>>()?; + + if parsed.is_empty() { + anyhow::bail!("No keys specified for hotkey"); + } + + let (modifiers, tail) = parsed.split_at(parsed.len() - 1); + + for m in modifiers { + self.enigo + .key(*m, Direction::Press) + .map_err(|e| anyhow::anyhow!("Modifier press failed: {e}"))?; + } + + self.enigo + .key(tail[0], Direction::Click) + .map_err(|e| anyhow::anyhow!("Key click failed: {e}"))?; + + for m in modifiers.iter().rev() { + self.enigo + .key(*m, Direction::Release) + .map_err(|e| anyhow::anyhow!("Modifier release failed: {e}"))?; + } + + Ok(()) } - fn mouse_move(&mut self, _x: i32, _y: i32) -> Result<()> { - anyhow::bail!("Input simulation not yet implemented (Phase 4)") + fn mouse_move(&mut self, x: i32, y: i32) -> Result<()> { + self.enigo + .move_mouse(x, y, Coordinate::Abs) + .map_err(|e| anyhow::anyhow!("Mouse move failed: {e}"))?; + Ok(()) } - fn scroll(&mut self, _amount: i32, _axis: &str) -> Result<()> { - anyhow::bail!("Input simulation not yet implemented (Phase 4)") + fn scroll(&mut self, amount: i32, axis: &str) -> Result<()> { + let ax = match axis { + "horizontal" | "h" => Axis::Horizontal, + _ => Axis::Vertical, + }; + self.enigo + .scroll(amount, ax) + .map_err(|e| anyhow::anyhow!("Scroll failed: {e}"))?; + Ok(()) } - fn drag(&mut self, _x1: i32, _y1: i32, _x2: i32, _y2: i32) -> Result<()> { - anyhow::bail!("Input simulation not yet implemented (Phase 4)") + fn drag(&mut self, x1: i32, y1: i32, x2: i32, y2: i32) -> Result<()> { + self.enigo + .move_mouse(x1, y1, Coordinate::Abs) + .map_err(|e| anyhow::anyhow!("Mouse move failed: {e}"))?; + std::thread::sleep(std::time::Duration::from_millis(10)); + self.enigo + .button(Button::Left, Direction::Press) + .map_err(|e| anyhow::anyhow!("Button press failed: {e}"))?; + std::thread::sleep(std::time::Duration::from_millis(50)); + self.enigo + .move_mouse(x2, y2, Coordinate::Abs) + .map_err(|e| anyhow::anyhow!("Mouse move to target failed: {e}"))?; + std::thread::sleep(std::time::Duration::from_millis(10)); + self.enigo + .button(Button::Left, Direction::Release) + .map_err(|e| anyhow::anyhow!("Button release failed: {e}"))?; + Ok(()) } + // Phase 6: utility stubs + fn screen_size(&self) -> Result<(u32, u32)> { anyhow::bail!("Utility commands not yet implemented (Phase 6)") } @@ -152,3 +238,52 @@ impl super::DesktopBackend for X11Backend { anyhow::bail!("Launch not yet implemented (Phase 6)") } } + +fn parse_key(name: &str) -> Result { + match name.to_lowercase().as_str() { + // Modifiers + "ctrl" | "control" => Ok(Key::Control), + "alt" => Ok(Key::Alt), + "shift" => Ok(Key::Shift), + "super" | "meta" | "win" => Ok(Key::Meta), + + // Navigation / editing + "enter" | "return" => Ok(Key::Return), + "tab" => Ok(Key::Tab), + "escape" | "esc" => Ok(Key::Escape), + "backspace" => Ok(Key::Backspace), + "delete" | "del" => Ok(Key::Delete), + "space" => Ok(Key::Space), + + // Arrow keys + "up" => Ok(Key::UpArrow), + "down" => Ok(Key::DownArrow), + "left" => Ok(Key::LeftArrow), + "right" => Ok(Key::RightArrow), + + // Page navigation + "home" => Ok(Key::Home), + "end" => Ok(Key::End), + "pageup" => Ok(Key::PageUp), + "pagedown" => Ok(Key::PageDown), + + // Function keys + "f1" => Ok(Key::F1), + "f2" => Ok(Key::F2), + "f3" => Ok(Key::F3), + "f4" => Ok(Key::F4), + "f5" => Ok(Key::F5), + "f6" => Ok(Key::F6), + "f7" => Ok(Key::F7), + "f8" => Ok(Key::F8), + "f9" => Ok(Key::F9), + "f10" => Ok(Key::F10), + "f11" => Ok(Key::F11), + "f12" => Ok(Key::F12), + + // Single character - map to Unicode key + s if s.len() == 1 => Ok(Key::Unicode(s.chars().next().unwrap())), + + other => anyhow::bail!("Unknown key: {other}"), + } +} diff --git a/src/daemon/handler.rs b/src/daemon/handler.rs index 1d2aeb6..2749f6d 100644 --- a/src/daemon/handler.rs +++ b/src/daemon/handler.rs @@ -12,6 +12,14 @@ pub async fn handle_request( ) -> Response { match request.action.as_str() { "snapshot" => handle_snapshot(request, state).await, + "click" => handle_click(request, state).await, + "dblclick" => handle_dblclick(request, state).await, + "type" => handle_type(request, state).await, + "press" => handle_press(request, state).await, + "hotkey" => handle_hotkey(request, state).await, + "mouse-move" => handle_mouse_move(request, state).await, + "mouse-scroll" => handle_mouse_scroll(request, state).await, + "mouse-drag" => handle_mouse_drag(request, state).await, action => Response::err(format!("Unknown action: {action}")), } } @@ -20,7 +28,9 @@ async fn handle_snapshot( request: &Request, state: &Arc>, ) -> Response { - let annotate = request.extra.get("annotate") + let annotate = request + .extra + .get("annotate") .and_then(|v| v.as_bool()) .unwrap_or(false); @@ -50,3 +60,208 @@ async fn handle_snapshot( Err(e) => Response::err(format!("Snapshot failed: {e}")), } } + +async fn handle_click( + request: &Request, + state: &Arc>, +) -> Response { + let selector = match request.extra.get("selector").and_then(|v| v.as_str()) { + Some(s) => s.to_string(), + None => return Response::err("Missing 'selector' field"), + }; + + let mut state = state.lock().await; + + // Try to parse as coordinates "x,y" + if let Some((x, y)) = parse_coords(&selector) { + return match state.backend.click(x, y) { + Ok(()) => Response::ok(serde_json::json!({"clicked": {"x": x, "y": y}})), + Err(e) => Response::err(format!("Click failed: {e}")), + }; + } + + // Resolve as window ref + match state.ref_map.resolve_to_center(&selector) { + Some((x, y)) => match state.backend.click(x, y) { + Ok(()) => Response::ok( + serde_json::json!({"clicked": {"x": x, "y": y, "ref": selector}}), + ), + Err(e) => Response::err(format!("Click failed: {e}")), + }, + None => Response::err(format!("Could not resolve selector: {selector}")), + } +} + +async fn handle_dblclick( + request: &Request, + state: &Arc>, +) -> Response { + let selector = match request.extra.get("selector").and_then(|v| v.as_str()) { + Some(s) => s.to_string(), + None => return Response::err("Missing 'selector' field"), + }; + + let mut state = state.lock().await; + + if let Some((x, y)) = parse_coords(&selector) { + return match state.backend.dblclick(x, y) { + Ok(()) => Response::ok(serde_json::json!({"double_clicked": {"x": x, "y": y}})), + Err(e) => Response::err(format!("Double-click failed: {e}")), + }; + } + + match state.ref_map.resolve_to_center(&selector) { + Some((x, y)) => match state.backend.dblclick(x, y) { + Ok(()) => Response::ok( + serde_json::json!({"double_clicked": {"x": x, "y": y, "ref": selector}}), + ), + Err(e) => Response::err(format!("Double-click failed: {e}")), + }, + None => Response::err(format!("Could not resolve selector: {selector}")), + } +} + +async fn handle_type( + request: &Request, + state: &Arc>, +) -> Response { + let text = match request.extra.get("text").and_then(|v| v.as_str()) { + Some(t) => t.to_string(), + None => return Response::err("Missing 'text' field"), + }; + + let mut state = state.lock().await; + + match state.backend.type_text(&text) { + Ok(()) => Response::ok(serde_json::json!({"typed": text})), + Err(e) => Response::err(format!("Type failed: {e}")), + } +} + +async fn handle_press( + request: &Request, + state: &Arc>, +) -> Response { + let key = match request.extra.get("key").and_then(|v| v.as_str()) { + Some(k) => k.to_string(), + None => return Response::err("Missing 'key' field"), + }; + + let mut state = state.lock().await; + + match state.backend.press_key(&key) { + Ok(()) => Response::ok(serde_json::json!({"pressed": key})), + Err(e) => Response::err(format!("Key press failed: {e}")), + } +} + +async fn handle_hotkey( + request: &Request, + state: &Arc>, +) -> Response { + let keys: Vec = match request.extra.get("keys").and_then(|v| v.as_array()) { + Some(arr) => arr + .iter() + .filter_map(|v| v.as_str().map(|s| s.to_string())) + .collect(), + None => return Response::err("Missing 'keys' field"), + }; + + let mut state = state.lock().await; + + match state.backend.hotkey(&keys) { + Ok(()) => Response::ok(serde_json::json!({"hotkey": keys})), + Err(e) => Response::err(format!("Hotkey failed: {e}")), + } +} + +async fn handle_mouse_move( + request: &Request, + state: &Arc>, +) -> Response { + let x = match request.extra.get("x").and_then(|v| v.as_i64()) { + Some(v) => v as i32, + None => return Response::err("Missing 'x' field"), + }; + let y = match request.extra.get("y").and_then(|v| v.as_i64()) { + Some(v) => v as i32, + None => return Response::err("Missing 'y' field"), + }; + + let mut state = state.lock().await; + + match state.backend.mouse_move(x, y) { + Ok(()) => Response::ok(serde_json::json!({"moved": {"x": x, "y": y}})), + Err(e) => Response::err(format!("Mouse move failed: {e}")), + } +} + +async fn handle_mouse_scroll( + request: &Request, + state: &Arc>, +) -> Response { + let amount = match request.extra.get("amount").and_then(|v| v.as_i64()) { + Some(v) => v as i32, + None => return Response::err("Missing 'amount' field"), + }; + let axis = request + .extra + .get("axis") + .and_then(|v| v.as_str()) + .unwrap_or("vertical") + .to_string(); + + let mut state = state.lock().await; + + match state.backend.scroll(amount, &axis) { + Ok(()) => { + Response::ok(serde_json::json!({"scrolled": {"amount": amount, "axis": axis}})) + } + Err(e) => Response::err(format!("Scroll failed: {e}")), + } +} + +async fn handle_mouse_drag( + request: &Request, + state: &Arc>, +) -> Response { + let x1 = match request.extra.get("x1").and_then(|v| v.as_i64()) { + Some(v) => v as i32, + None => return Response::err("Missing 'x1' field"), + }; + let y1 = match request.extra.get("y1").and_then(|v| v.as_i64()) { + Some(v) => v as i32, + None => return Response::err("Missing 'y1' field"), + }; + let x2 = match request.extra.get("x2").and_then(|v| v.as_i64()) { + Some(v) => v as i32, + None => return Response::err("Missing 'x2' field"), + }; + let y2 = match request.extra.get("y2").and_then(|v| v.as_i64()) { + Some(v) => v as i32, + None => return Response::err("Missing 'y2' field"), + }; + + let mut state = state.lock().await; + + match state.backend.drag(x1, y1, x2, y2) { + Ok(()) => Response::ok(serde_json::json!({ + "dragged": { + "from": {"x": x1, "y": y1}, + "to": {"x": x2, "y": y2} + } + })), + Err(e) => Response::err(format!("Drag failed: {e}")), + } +} + +fn parse_coords(s: &str) -> Option<(i32, i32)> { + let parts: Vec<&str> = s.split(',').collect(); + if parts.len() == 2 { + let x = parts[0].trim().parse().ok()?; + let y = parts[1].trim().parse().ok()?; + Some((x, y)) + } else { + None + } +}