Phase 3: screenshot annotation with bounding boxes and @wN labels

- Add imageproc and ab_glyph dependencies
- Annotation module drawing colored bounding boxes per window
- White @wN labels on dark background at each window's top-left
- 8-color palette cycling for distinct window identification
- Back-to-front iteration so topmost labels are not occluded
- Embedded DejaVu Sans Mono font via include_bytes
- Wire --annotate flag into snapshot pipeline
This commit is contained in:
Harivansh Rathi 2026-03-24 21:28:10 -04:00
parent 79e6e0e25c
commit 0072a260b8
6 changed files with 1903 additions and 11 deletions

342
Cargo.lock generated
View file

@ -2,6 +2,22 @@
# It is not intended for manual editing.
version = 4
[[package]]
name = "ab_glyph"
version = "0.2.32"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "01c0457472c38ea5bd1c3b5ada5e368271cb550be7a4ca4a0b4634e9913f6cc2"
dependencies = [
"ab_glyph_rasterizer",
"owned_ttf_parser",
]
[[package]]
name = "ab_glyph_rasterizer"
version = "0.1.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "366ffbaa4442f4684d91e2cd7c5ea7c4ed8add41959a31447066e279e432b618"
[[package]]
name = "adler2"
version = "2.0.1"
@ -101,6 +117,15 @@ version = "1.0.102"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c"
[[package]]
name = "approx"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cab112f0a86d568ea0e627cc1d6be74a1e9cd55214684db5561995f6dad897c6"
dependencies = [
"num-traits",
]
[[package]]
name = "arbitrary"
version = "1.4.2"
@ -614,10 +639,12 @@ checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5"
name = "desktop-ctl"
version = "0.1.0"
dependencies = [
"ab_glyph",
"anyhow",
"clap",
"dirs",
"image",
"imageproc",
"libc",
"serde",
"serde_json",
@ -963,9 +990,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd"
dependencies = [
"cfg-if",
"js-sys",
"libc",
"r-efi 5.3.0",
"wasip2",
"wasm-bindgen",
]
[[package]]
@ -1011,6 +1040,102 @@ dependencies = [
"xml-rs",
]
[[package]]
name = "glam"
version = "0.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "333928d5eb103c5d4050533cec0384302db6be8ef7d3cebd30ec6a35350353da"
[[package]]
name = "glam"
version = "0.15.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3abb554f8ee44336b72d522e0a7fe86a29e09f839a36022fa869a7dfe941a54b"
[[package]]
name = "glam"
version = "0.16.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4126c0479ccf7e8664c36a2d719f5f2c140fbb4f9090008098d2c291fa5b3f16"
[[package]]
name = "glam"
version = "0.17.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e01732b97afd8508eee3333a541b9f7610f454bb818669e66e90f5f57c93a776"
[[package]]
name = "glam"
version = "0.18.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "525a3e490ba77b8e326fb67d4b44b4bd2f920f44d4cc73ccec50adc68e3bee34"
[[package]]
name = "glam"
version = "0.19.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2b8509e6791516e81c1a630d0bd7fbac36d2fa8712a9da8662e716b52d5051ca"
[[package]]
name = "glam"
version = "0.20.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f43e957e744be03f5801a55472f593d43fabdebf25a4585db250f04d86b1675f"
[[package]]
name = "glam"
version = "0.21.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "518faa5064866338b013ff9b2350dc318e14cc4fcd6cb8206d7e7c9886c98815"
[[package]]
name = "glam"
version = "0.22.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "12f597d56c1bd55a811a1be189459e8fad2bbc272616375602443bdfb37fa774"
[[package]]
name = "glam"
version = "0.23.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8e4afd9ad95555081e109fe1d21f2a30c691b5f0919c67dfa690a2e1eb6bd51c"
[[package]]
name = "glam"
version = "0.24.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b5418c17512bdf42730f9032c74e1ae39afc408745ebb2acf72fbc4691c17945"
[[package]]
name = "glam"
version = "0.25.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "151665d9be52f9bb40fc7966565d39666f2d1e69233571b71b87791c7e0528b3"
[[package]]
name = "glam"
version = "0.27.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9e05e7e6723e3455f4818c7b26e855439f7546cf617ef669d1adedb8669e5cb9"
[[package]]
name = "glam"
version = "0.28.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "779ae4bf7e8421cf91c0b3b64e7e8b40b862fba4d393f59150042de7c4965a94"
[[package]]
name = "glam"
version = "0.29.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8babf46d4c1c9d92deac9f7be466f76dfc4482b6452fc5024b5e8daf6ffeb3ee"
[[package]]
name = "glam"
version = "0.30.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "19fc433e8437a212d1b6f1e68c7824af3aed907da60afa994e7f542d18d12aa9"
[[package]]
name = "glob"
version = "0.3.3"
@ -1203,6 +1328,25 @@ dependencies = [
"quick-error",
]
[[package]]
name = "imageproc"
version = "0.26.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5a8046da590889acc65f5880004580ebb269bbef84d6c0f5c543ec2dece46638"
dependencies = [
"ab_glyph",
"approx",
"getrandom 0.3.4",
"image",
"itertools 0.14.0",
"nalgebra",
"num",
"rand",
"rand_distr",
"rayon",
"rustdct",
]
[[package]]
name = "imgref"
version = "1.12.0"
@ -1342,6 +1486,12 @@ dependencies = [
"windows-link",
]
[[package]]
name = "libm"
version = "0.2.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981"
[[package]]
name = "libredox"
version = "0.1.15"
@ -1448,6 +1598,16 @@ dependencies = [
"imgref",
]
[[package]]
name = "matrixmultiply"
version = "0.3.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a06de3016e9fae57a36fd14dba131fccf49f74b40b7fbdb472f96e361ec71a08"
dependencies = [
"autocfg",
"rawpointer",
]
[[package]]
name = "maybe-rayon"
version = "0.1.1"
@ -1519,6 +1679,37 @@ dependencies = [
"pxfm",
]
[[package]]
name = "nalgebra"
version = "0.34.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c4d5b3eff5cd580f93da45e64715e8c20a3996342f1e466599cf7a267a0c2f5f"
dependencies = [
"approx",
"glam 0.14.0",
"glam 0.15.2",
"glam 0.16.0",
"glam 0.17.3",
"glam 0.18.0",
"glam 0.19.0",
"glam 0.20.5",
"glam 0.21.3",
"glam 0.22.0",
"glam 0.23.0",
"glam 0.24.2",
"glam 0.25.0",
"glam 0.27.0",
"glam 0.28.0",
"glam 0.29.3",
"glam 0.30.10",
"matrixmultiply",
"num-complex",
"num-rational",
"num-traits",
"simba",
"typenum",
]
[[package]]
name = "new_debug_unreachable"
version = "1.0.6"
@ -1562,6 +1753,19 @@ version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0676bb32a98c1a483ce53e500a81ad9c3d5b3f7c920c28c24e9cb0980d0b5bc8"
[[package]]
name = "num"
version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23"
dependencies = [
"num-complex",
"num-integer",
"num-iter",
"num-rational",
"num-traits",
]
[[package]]
name = "num-bigint"
version = "0.4.6"
@ -1572,6 +1776,15 @@ dependencies = [
"num-traits",
]
[[package]]
name = "num-complex"
version = "0.4.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495"
dependencies = [
"num-traits",
]
[[package]]
name = "num-derive"
version = "0.4.2"
@ -1592,6 +1805,17 @@ dependencies = [
"num-traits",
]
[[package]]
name = "num-iter"
version = "0.1.45"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf"
dependencies = [
"autocfg",
"num-integer",
"num-traits",
]
[[package]]
name = "num-rational"
version = "0.4.2"
@ -1610,6 +1834,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841"
dependencies = [
"autocfg",
"libm",
]
[[package]]
@ -1903,6 +2128,15 @@ dependencies = [
"pin-project-lite",
]
[[package]]
name = "owned_ttf_parser"
version = "0.25.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "36820e9051aca1014ddc75770aab4d68bc1e9e632f0f5627c4086bc216fb583b"
dependencies = [
"ttf-parser",
]
[[package]]
name = "parking"
version = "2.2.1"
@ -2056,6 +2290,15 @@ dependencies = [
"syn",
]
[[package]]
name = "primal-check"
version = "0.3.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dc0d895b311e3af9902528fbb8f928688abbd95872819320517cc24ca6b2bd08"
dependencies = [
"num-integer",
]
[[package]]
name = "proc-macro-crate"
version = "3.5.0"
@ -2182,6 +2425,16 @@ dependencies = [
"getrandom 0.3.4",
]
[[package]]
name = "rand_distr"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6a8615d50dcf34fa31f7ab52692afec947c4dd0ab803cc87cb3b0b4570ff7463"
dependencies = [
"num-traits",
"rand",
]
[[package]]
name = "rav1e"
version = "0.8.1"
@ -2232,6 +2485,12 @@ dependencies = [
"rgb",
]
[[package]]
name = "rawpointer"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3"
[[package]]
name = "rayon"
version = "1.11.0"
@ -2313,6 +2572,29 @@ version = "2.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d"
[[package]]
name = "rustdct"
version = "0.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8b61555105d6a9bf98797c063c362a1d24ed8ab0431655e38f1cf51e52089551"
dependencies = [
"rustfft",
]
[[package]]
name = "rustfft"
version = "6.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "21db5f9893e91f41798c88680037dba611ca6674703c1a18601b01a72c8adb89"
dependencies = [
"num-complex",
"num-integer",
"num-traits",
"primal-check",
"strength_reduce",
"transpose",
]
[[package]]
name = "rustix"
version = "0.38.44"
@ -2345,6 +2627,15 @@ version = "1.0.22"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d"
[[package]]
name = "safe_arch"
version = "0.7.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "96b02de82ddbe1b636e6170c21be622223aea188ef2e139be0a5b219ec215323"
dependencies = [
"bytemuck",
]
[[package]]
name = "scoped-tls"
version = "1.0.1"
@ -2442,6 +2733,19 @@ dependencies = [
"libc",
]
[[package]]
name = "simba"
version = "0.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c99284beb21666094ba2b75bbceda012e610f5479dfcc2d6e2426f53197ffd95"
dependencies = [
"approx",
"num-complex",
"num-traits",
"paste",
"wide",
]
[[package]]
name = "simd-adler32"
version = "0.3.8"
@ -2485,6 +2789,12 @@ version = "1.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596"
[[package]]
name = "strength_reduce"
version = "0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fe895eb47f22e2ddd4dabc02bce419d2e643c8e3b585c78158b349195bc24d82"
[[package]]
name = "strsim"
version = "0.11.1"
@ -2708,6 +3018,28 @@ dependencies = [
"once_cell",
]
[[package]]
name = "transpose"
version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1ad61aed86bc3faea4300c7aee358b4c6d0c8d6ccc36524c96e4c92ccf26e77e"
dependencies = [
"num-integer",
"strength_reduce",
]
[[package]]
name = "ttf-parser"
version = "0.25.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d2df906b07856748fa3f6e0ad0cbaa047052d4a7dd609e231c4f72cee8c36f31"
[[package]]
name = "typenum"
version = "1.19.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb"
[[package]]
name = "uds_windows"
version = "1.2.1"
@ -2993,6 +3325,16 @@ version = "0.1.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a28ac98ddc8b9274cb41bb4d9d4d5c425b6020c50c46f25559911905610b4a88"
[[package]]
name = "wide"
version = "0.7.33"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0ce5da8ecb62bcd8ec8b7ea19f69a51275e91299be594ea5cc6ef7819e16cd03"
dependencies = [
"bytemuck",
"safe_arch",
]
[[package]]
name = "widestring"
version = "1.2.1"

View file

@ -17,3 +17,5 @@ libc = "0.2"
uuid = { version = "1", features = ["v4"] }
xcap = "0.8"
image = { version = "0.25", features = ["png"] }
imageproc = "0.26"
ab_glyph = "0.2"

1469
assets/DejaVuSansMono.ttf Normal file

File diff suppressed because one or more lines are too long

72
src/backend/annotate.rs Normal file
View file

@ -0,0 +1,72 @@
use ab_glyph::{FontArc, PxScale};
use image::{Rgba, RgbaImage};
use imageproc::drawing::{draw_filled_rect_mut, draw_hollow_rect_mut, draw_text_mut, text_size};
use imageproc::rect::Rect;
use crate::core::types::WindowInfo;
// Embedded font - DejaVu Sans Mono for guaranteed availability
const FONT_BYTES: &[u8] = include_bytes!("../../assets/DejaVuSansMono.ttf");
const COLORS: &[Rgba<u8>] = &[
Rgba([0, 255, 0, 255]), // green
Rgba([255, 100, 0, 255]), // orange
Rgba([0, 150, 255, 255]), // blue
Rgba([255, 0, 255, 255]), // magenta
Rgba([255, 255, 0, 255]), // yellow
Rgba([0, 255, 255, 255]), // cyan
Rgba([255, 50, 50, 255]), // red
Rgba([150, 100, 255, 255]), // purple
];
const LABEL_BG: Rgba<u8> = Rgba([0, 0, 0, 200]);
const LABEL_FG: Rgba<u8> = Rgba([255, 255, 255, 255]);
pub fn annotate_screenshot(image: &mut RgbaImage, windows: &[WindowInfo]) {
let font = match FontArc::try_from_slice(FONT_BYTES) {
Ok(f) => f,
Err(_) => return, // Silently skip annotation if font fails to load
};
let scale = PxScale { x: 18.0, y: 18.0 };
// Iterate back-to-front so topmost window labels are drawn last (not occluded)
for (i, win) in windows.iter().enumerate().rev() {
if win.minimized {
continue; // Don't annotate hidden windows
}
let color = COLORS[i % COLORS.len()];
let x = win.x.max(0);
let y = win.y.max(0);
let w = win.width;
let h = win.height;
// Draw bounding box (2px thick by drawing at offset 0 and 1)
if w > 0 && h > 0 {
draw_hollow_rect_mut(image, Rect::at(x, y).of_size(w, h), color);
if w > 2 && h > 2 {
draw_hollow_rect_mut(
image,
Rect::at(x + 1, y + 1).of_size(w - 2, h - 2),
color,
);
}
}
// Draw label
let label = format!("@{}", win.ref_id);
let (tw, th) = text_size(scale, &font, &label);
let label_x = x;
let label_y = (y - th as i32 - 4).max(0);
// Label background
draw_filled_rect_mut(
image,
Rect::at(label_x, label_y).of_size(tw + 6, th + 4),
LABEL_BG,
);
// Label text
draw_text_mut(image, LABEL_FG, label_x + 3, label_y + 2, scale, &font, &label);
}
}

View file

@ -1,3 +1,4 @@
pub mod annotate;
pub mod x11;
use anyhow::Result;

View file

@ -1,6 +1,7 @@
use anyhow::{Context, Result};
use crate::core::types::{Snapshot, WindowInfo};
use super::annotate::annotate_screenshot;
pub struct X11Backend {
// enigo and x11rb connections added in later phases
@ -13,7 +14,7 @@ impl X11Backend {
}
impl super::DesktopBackend for X11Backend {
fn snapshot(&mut self, _annotate: bool) -> Result<Snapshot> {
fn snapshot(&mut self, annotate: bool) -> Result<Snapshot> {
// Get z-ordered window list via xcap (topmost first internally)
let windows = xcap::Window::all()
.context("Failed to enumerate windows")?;
@ -24,18 +25,9 @@ impl super::DesktopBackend for X11Backend {
let monitor = monitors.into_iter().next()
.context("No monitor found")?;
let image = monitor.capture_image()
let mut image = monitor.capture_image()
.context("Failed to capture screenshot")?;
// Save screenshot
let timestamp = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap_or_default()
.as_millis();
let screenshot_path = format!("/tmp/desktop-ctl-{timestamp}.png");
image.save(&screenshot_path)
.context("Failed to save screenshot")?;
// Build window info list
let mut window_infos = Vec::new();
let mut ref_counter = 1usize;
@ -75,6 +67,20 @@ impl super::DesktopBackend for X11Backend {
});
}
// Annotate if requested - draw bounding boxes and @wN labels
if annotate {
annotate_screenshot(&mut image, &window_infos);
}
// Save screenshot
let timestamp = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap_or_default()
.as_millis();
let screenshot_path = format!("/tmp/desktop-ctl-{timestamp}.png");
image.save(&screenshot_path)
.context("Failed to save screenshot")?;
Ok(Snapshot {
screenshot: screenshot_path,
windows: window_infos,