Skip to main content

pixelflux/
computer_use.rs

1/*
2 * This Source Code Form is subject to the terms of the Mozilla Public
3 * License, v. 2.0. If a copy of the MPL was not distributed with this
4 * file, You can obtain one at https://mozilla.org/MPL/2.0/.
5 */
6
7//! HTTP server implementing the [Anthropic Computer Use](https://github.com/anthropics/claude-quickstarts/tree/main/computer-use-demo) specification.
8//!
9//! Enabled by setting the `PIXELFLUX_CU` environment variable to a bind: comma-separated entries,
10//! each a port, served on the loopback addresses only, or `host:port`, the address to serve on
11//! (`0.0.0.0:5000,[::]:5000` serves every interface). The server
12//! handles `POST /computer-use` requests for screenshots, mouse/keyboard injection, scrolling,
13//! and cursor position queries. Actions run against a [`CuBackend`], resolved per request: the
14//! Wayland compositor owned by this process when one is registered, otherwise the X server named
15//! by `DISPLAY` (XTEST injection on a private connection, no active capture required).
16//!
17//! A caller drives the session with its owner's full authority, and a loopback address is open to
18//! every account on the host, so the server does not start without a token (`PIXELFLUX_CU_TOKEN`,
19//! or the caller's own through [`start_cu_server`]) and every request has to carry it as
20//! `Authorization: Bearer <token>`.
21//!
22//! The same server also exposes the built-in MP4 recorder at `/record_start`, `/record_stop`,
23//! and `/record_status`, so a headless script can drive a session and record it over plain HTTP.
24//! A recording the caller names is a file name inside `PIXELFLUX_RECORD_DIR`, the directory the
25//! operator gives callers, and nowhere else.
26
27use std::collections::HashMap;
28use std::io::Cursor;
29use std::io::Read;
30use std::net::{Ipv4Addr, Ipv6Addr, SocketAddr, TcpListener, ToSocketAddrs};
31use std::sync::mpsc;
32use std::sync::{Mutex, OnceLock};
33use std::thread;
34use std::time::Duration;
35
36use smithay::input::keyboard::xkb;
37
38use base64::Engine;
39use base64::engine::general_purpose::STANDARD as BASE64;
40use image::{ImageBuffer, ImageFormat, Rgba};
41use serde::Deserialize;
42use tiny_http;
43
44use crate::ThreadCommand;
45use crate::wayland::keymap::keysym_for_char;
46
47fn clamp<T: PartialOrd>(v: T, lo: T, hi: T) -> T {
48    if v < lo {
49        lo
50    } else if v > hi {
51        hi
52    } else {
53        v
54    }
55}
56
57/// Turn a raw framebuffer into a PNG the Computer Use agent can actually look at.
58///
59/// The `screenshot` and `zoom` crop paths have to hand the agent an image, and the API carries it
60/// as base64 PNG, so this encodes a flat RGBA buffer (with its dimensions) through the `image`
61/// crate and returns the PNG bytes for that response payload.
62pub fn encode_png_rgba(data: &[u8], width: u32, height: u32) -> Result<Vec<u8>, String> {
63    let img = ImageBuffer::<Rgba<u8>, _>::from_raw(width, height, data.to_vec())
64        .ok_or("Failed to create image buffer")?;
65    let mut png = Vec::new();
66    img.write_to(&mut Cursor::new(&mut png), ImageFormat::Png)
67        .map_err(|e| format!("PNG encode error: {}", e))?;
68    Ok(png)
69}
70
71fn scancode_for_keyname(name: &str) -> Option<u32> {
72    Some(match name.to_lowercase().as_str() {
73        "return" | "enter" => 36,
74        "tab" => 23,
75        "escape" | "esc" => 9,
76        "backspace" => 22,
77        "delete" => 119,
78        "insert" => 118,
79        "home" => 110,
80        "end" => 115,
81        "pageup" => 112,
82        "pagedown" => 117,
83        "up" => 111,
84        "down" => 116,
85        "left" => 113,
86        "right" => 114,
87        "space" => 65,
88        "capslock" => 66,
89        "numlock" => 77,
90        "scrolllock" => 78,
91        "printscreen" | "sysrq" | "print" => 107,
92        "pause" | "break" => 127,
93        "f1" => 67,
94        "f2" => 68,
95        "f3" => 69,
96        "f4" => 70,
97        "f5" => 71,
98        "f6" => 72,
99        "f7" => 73,
100        "f8" => 74,
101        "f9" => 75,
102        "f10" => 76,
103        "f11" => 95,
104        "f12" => 96,
105        "f13" => 191,
106        "f14" => 192,
107        "f15" => 193,
108        "f16" => 194,
109        "f17" => 195,
110        "f18" => 196,
111        "f19" => 197,
112        "f20" => 198,
113        "f21" => 199,
114        "f22" => 200,
115        "f23" => 201,
116        "f24" => 202,
117        "ctrl" | "lctrl" | "leftctrl" => 37,
118        "rctrl" | "rightctrl" => 105,
119        "shift" | "lshift" | "leftshift" => 50,
120        "rshift" | "rightshift" => 62,
121        "alt" | "lalt" | "leftalt" => 64,
122        "ralt" | "rightalt" => 108,
123        "super" | "meta" | "lsuper" | "leftmeta" | "leftsuper" | "windows" | "leftwindows" => 133,
124        "rsuper" | "rightmeta" | "rightsuper" | "rightwindows" => 134,
125        "menu" | "compose" => 135,
126        "kp_0" | "kp0" => 90,
127        "kp_1" | "kp1" => 87,
128        "kp_2" | "kp2" => 88,
129        "kp_3" | "kp3" => 89,
130        "kp_4" | "kp4" => 83,
131        "kp_5" | "kp5" => 84,
132        "kp_6" | "kp6" => 85,
133        "kp_7" | "kp7" => 79,
134        "kp_8" | "kp8" => 80,
135        "kp_9" | "kp9" => 81,
136        "kp_decimal" | "kp_dot" => 91,
137        "kp_divide" | "kp_slash" => 106,
138        "kp_multiply" | "kp_asterisk" => 63,
139        "kp_subtract" | "kp_minus" => 82,
140        "kp_add" | "kp_plus" => 86,
141        "kp_enter" => 104,
142        _ => return None,
143    })
144}
145
146/// Keysym for one component of an xdotool-style key spec that is not a named key: a single
147/// literal character (`a`, `7`, `-`, `#`) via the Unicode mapping, or a keysym NAME
148/// (`minus`, `bracketleft`, `udiaeresis`, …) via xkb's name lookup (case-tolerant fallback
149/// so `Minus` still resolves). Returns 0 when the spec names no keysym.
150fn keysym_for_key_spec(name: &str) -> u32 {
151    let mut chars = name.chars();
152    if let (Some(c), None) = (chars.next(), chars.next()) {
153        return keysym_for_char(c);
154    }
155    let sym = xkb::keysym_from_name(name, xkb::KEYSYM_NO_FLAGS).raw();
156    if sym != 0 {
157        return sym;
158    }
159    xkb::keysym_from_name(name, xkb::KEYSYM_CASE_INSENSITIVE).raw()
160}
161
162fn is_modifier(scancode: u32) -> bool {
163    matches!(scancode, 37 | 105 | 50 | 62 | 64 | 108 | 133 | 134)
164}
165
166/// Semantic mouse-button vocabulary of the action layer; each backend maps it to its native
167/// codes (evdev `BTN_` on Wayland, X core buttons 1/2/3 on X11).
168#[derive(Clone, Copy)]
169pub enum CuButton {
170    Left,
171    Right,
172    Middle,
173}
174
175/// The desktop-side primitives one Computer Use action decomposes into. Keycodes are X/xkb
176/// keycodes (evdev + 8) — the numbering both the smithay seat and XTEST consume — and
177/// coordinates are framebuffer/root pixels, already clamped by the action layer.
178pub trait CuBackend {
179    fn name(&self) -> &'static str;
180    fn fb_size(&self) -> Result<(i32, i32), String>;
181    /// One display's framebuffer size; 0 = the primary. Unknown ids are an error. The
182    /// X11 backend serves only the root (display 0); Wayland resolves any live output id.
183    fn display_fb_size(&self, display: u32) -> Result<(i32, i32), String> {
184        if display == 0 {
185            self.fb_size()
186        } else {
187            Err(format!("Unknown display: {display}"))
188        }
189    }
190    fn key(&self, scancode: u32, pressed: bool);
191    fn mouse_move(&self, x: f64, y: f64);
192    fn button(&self, btn: CuButton, pressed: bool);
193    fn scroll(&self, dx: f64, dy: f64);
194    /// PNG of one display's framebuffer; 0 = the primary. Unknown ids are an error.
195    fn screenshot_png(&self, display: u32) -> Result<Vec<u8>, String>;
196    fn cursor_pos(&self) -> Result<(f64, f64), String>;
197    /// Run `seq` with every keysym in `keysyms` (which `resolve_keysyms` could not place)
198    /// made temporarily typeable, when the backend can arrange that; `seq` receives
199    /// keysym -> keycode for the transient bindings (empty when none could be arranged,
200    /// in which case those keysyms simply stay untypeable). Wayland's `resolve_keysyms`
201    /// already overlay-binds anything the base keymap lacks, so the default runs `seq`
202    /// with no bindings; the X11 backend overrides this with a grab-atomic spare-keycode
203    /// remap.
204    fn with_transient_keysyms(&self, keysyms: &[u32], seq: &mut dyn FnMut(&HashMap<u32, u32>)) {
205        let _ = keysyms;
206        seq(&HashMap::new());
207    }
208    /// Resolve keysyms against the backend's ACTIVE keymap: one `(keycode, shift level)`
209    /// per input keysym, `(0, 0)` where the keymap cannot produce it. Level bit 0 = Shift,
210    /// bit 1 = AltGr — the order xkb two/four-level key types use. Wayland resolves through
211    /// the compositor's keymap policy (overlay-binding what the base lacks); X11 through
212    /// the server's own `GetKeyboardMapping`.
213    fn resolve_keysyms(&self, keysyms: &[u32]) -> Vec<(u32, u32)>;
214    /// Keycode synthesized around AltGr-level hits (level bit 1). The default is the
215    /// pc105 right-Alt position the compositor's seat keymap binds to `ISO_Level3_Shift`;
216    /// the X11 backend overrides it with the level-3 modifier key found in the server's
217    /// keymap (0 = none, in which case `resolve_keysyms` reports no AltGr levels).
218    fn altgr_keycode(&self) -> u32 {
219        KC_ALTGR
220    }
221}
222
223const BTN_LEFT: u32 = 0x110;
224const BTN_RIGHT: u32 = 0x111;
225const BTN_MIDDLE: u32 = 0x112;
226
227/// Reply deadline for compositor round-trips (screenshot readback, cursor position, fb size).
228/// Bounded so a wedged render path turns into a JSON error instead of hanging the sequential
229/// HTTP loop — and every request behind it — forever.
230const REPLY_TIMEOUT: Duration = Duration::from_secs(5);
231
232/// Wayland implementation: every primitive is a `ThreadCommand` on the compositor's calloop
233/// channel, so injection and readback serialize naturally with rendering and encoding.
234pub struct CuWaylandBackend {
235    tx: smithay::reexports::calloop::channel::Sender<ThreadCommand>,
236}
237
238impl CuBackend for CuWaylandBackend {
239    fn name(&self) -> &'static str {
240        "wayland"
241    }
242
243    fn fb_size(&self) -> Result<(i32, i32), String> {
244        self.display_fb_size(0)
245    }
246
247    fn display_fb_size(&self, display: u32) -> Result<(i32, i32), String> {
248        let (resp_tx, resp_rx) = mpsc::channel();
249        self.tx
250            .send(ThreadCommand::CuGetInfo {
251                display_id: display,
252                resp: resp_tx,
253            })
254            .map_err(|_| "Failed to request compositor info".to_string())?;
255        let (w, h, _) = resp_rx
256            .recv_timeout(REPLY_TIMEOUT)
257            .map_err(|_| "Compositor info request failed".to_string())?;
258        // CuGetInfo reports zeros for output ids that don't exist (and for an output
259        // with no mode, which live outputs always have).
260        if w <= 0 || h <= 0 {
261            return Err(format!("Unknown display: {display}"));
262        }
263        Ok((w, h))
264    }
265
266    fn key(&self, scancode: u32, pressed: bool) {
267        let _ = self.tx.send(ThreadCommand::KeyboardKey {
268            scancode,
269            state: if pressed { 1 } else { 0 },
270        });
271    }
272
273    fn mouse_move(&self, x: f64, y: f64) {
274        let _ = self
275            .tx
276            .send(ThreadCommand::PointerMotion { x, y, motion: 0 });
277    }
278
279    fn button(&self, btn: CuButton, pressed: bool) {
280        let btn = match btn {
281            CuButton::Left => BTN_LEFT,
282            CuButton::Right => BTN_RIGHT,
283            CuButton::Middle => BTN_MIDDLE,
284        };
285        let _ = self.tx.send(ThreadCommand::PointerButton {
286            btn,
287            state: if pressed { 1 } else { 0 },
288        });
289    }
290
291    fn scroll(&self, dx: f64, dy: f64) {
292        let _ = self.tx.send(ThreadCommand::PointerAxis { x: dx, y: dy });
293    }
294
295    fn screenshot_png(&self, display: u32) -> Result<Vec<u8>, String> {
296        let (resp_tx, resp_rx) = mpsc::channel();
297        self.tx
298            .send(ThreadCommand::CuScreenshot {
299                display_id: display,
300                resp: resp_tx,
301            })
302            .map_err(|_| "Failed to request screenshot".to_string())?;
303        resp_rx
304            .recv_timeout(REPLY_TIMEOUT)
305            .map_err(|_| "Screenshot failed".to_string())?
306    }
307
308    fn cursor_pos(&self) -> Result<(f64, f64), String> {
309        let (resp_tx, resp_rx) = mpsc::channel();
310        self.tx
311            .send(ThreadCommand::CuCursorPosition { resp: resp_tx })
312            .map_err(|_| "Failed to request cursor position".to_string())?;
313        resp_rx
314            .recv_timeout(REPLY_TIMEOUT)
315            .map_err(|_| "Cursor position failed".to_string())
316    }
317
318    fn resolve_keysyms(&self, keysyms: &[u32]) -> Vec<(u32, u32)> {
319        let (resp_tx, resp_rx) = mpsc::channel();
320        if self
321            .tx
322            .send(ThreadCommand::BindKeysyms {
323                keysyms: keysyms.to_vec(),
324                reply: resp_tx,
325            })
326            .is_err()
327        {
328            return vec![(0, 0); keysyms.len()];
329        }
330        resp_rx
331            .recv_timeout(REPLY_TIMEOUT)
332            .unwrap_or_else(|_| vec![(0, 0); keysyms.len()])
333    }
334}
335
336/// Distinct characters a `type` request binds and types at a time: each run's binds then fit
337/// the seat's text spares (`keymap::TEXT_SPARE_KEYCODES`; 9 free on a Japanese layout, 11 on
338/// most), on which every client types, where a bind past them lands above 255 and Chromium
339/// drops it.
340const TYPE_RUN_KEYSYMS: usize = 8;
341
342/// `syms` cut into consecutive runs of at most `max` distinct keysyms.
343fn distinct_runs(syms: &[u32], max: usize) -> Vec<&[u32]> {
344    let mut runs = Vec::new();
345    let (mut start, mut seen) = (0, std::collections::HashSet::new());
346    for (i, sym) in syms.iter().enumerate() {
347        if !seen.contains(sym) && seen.len() == max {
348            runs.push(&syms[start..i]);
349            start = i;
350            seen.clear();
351        }
352        seen.insert(*sym);
353    }
354    if start < syms.len() {
355        runs.push(&syms[start..]);
356    }
357    runs
358}
359
360/// Per-request literal-key resolver: batches keysym lookups against the backend's active
361/// keymap and caches them for the request's burst of key events (a fresh backend — and thus
362/// a fresh cache — is resolved per HTTP request, so a runtime layout switch is picked up by
363/// the next request).
364struct KeyResolver<'a> {
365    backend: &'a dyn CuBackend,
366    cache: HashMap<u32, (u32, u32)>,
367}
368
369impl<'a> KeyResolver<'a> {
370    fn new(backend: &'a dyn CuBackend) -> Self {
371        Self {
372            backend,
373            cache: HashMap::new(),
374        }
375    }
376
377    /// Resolve a batch up front so a `type` action costs one backend round trip.
378    fn prefetch(&mut self, keysyms: &[u32]) {
379        let missing: Vec<u32> = {
380            let mut seen = std::collections::HashSet::new();
381            keysyms
382                .iter()
383                .copied()
384                .filter(|&s| s != 0 && !self.cache.contains_key(&s) && seen.insert(s))
385                .collect()
386        };
387        if missing.is_empty() {
388            return;
389        }
390        let out = self.backend.resolve_keysyms(&missing);
391        for (sym, hit) in missing.into_iter().zip(out) {
392            self.cache.insert(sym, hit);
393        }
394    }
395
396    /// `(keycode, shift level)` for `keysym`, or `None` when the active keymap cannot
397    /// produce it.
398    fn resolve(&mut self, keysym: u32) -> Option<(u32, u32)> {
399        if keysym == 0 {
400            return None;
401        }
402        self.prefetch(&[keysym]);
403        let hit = self.cache.get(&keysym).copied().unwrap_or((0, 0));
404        (hit.0 != 0).then_some(hit)
405    }
406}
407
408/// Shift and default AltGr (pc105 right Alt / ISO_Level3_Shift) keycodes, held to reach a
409/// resolved key's shift level: bit 0 = Shift, bit 1 = AltGr.
410const KC_SHIFT: u32 = 50;
411const KC_ALTGR: u32 = 108;
412
413/// The level modifiers `level` requires beyond whatever is already held (`held_mask` uses
414/// the same bit layout), in press order. `altgr` is the backend's AltGr keycode
415/// ([`CuBackend::altgr_keycode`]).
416fn level_modifiers(level: u32, held_mask: u32, altgr: u32) -> Vec<u32> {
417    let mut out = Vec::new();
418    if level & 1 != 0 && held_mask & 1 == 0 {
419        out.push(KC_SHIFT);
420    }
421    if level & 2 != 0 && held_mask & 2 == 0 && altgr != 0 {
422        out.push(altgr);
423    }
424    out
425}
426
427#[derive(Deserialize)]
428struct CuActionRequest {
429    action: String,
430    coordinate: Option<[f64; 2]>,
431    start_coordinate: Option<[f64; 2]>,
432    text: Option<String>,
433    key: Option<String>,
434    scroll_direction: Option<String>,
435    scroll_amount: Option<i32>,
436    duration: Option<f64>,
437    region: Option<[f64; 4]>,
438    /// Display id for `screenshot` / `zoom` (default 0, the primary; `record_start` uses
439    /// its own `display` field with the same meaning). Unknown ids get an error reply.
440    display: Option<u32>,
441}
442
443fn ok_json() -> String {
444    "{\"result\":\"ok\"}".to_string()
445}
446
447fn handle_action(req: CuActionRequest, backend: &dyn CuBackend) -> String {
448    let result = handle_action_inner(req, backend);
449    match result {
450        Ok(response) => response,
451        Err(e) => format!("{{\"error\":\"{}\"}}", e.replace('"', "\\\"")),
452    }
453}
454
455/// Turn one parsed Computer Use request into a real action on the captured desktop — the
456/// bridge that lets an external AI agent see and drive the session.
457///
458/// Every action ultimately becomes a backend input primitive or a framebuffer read, so this is
459/// where the API's vocabulary meets the running desktop. Coordinates are clamped to the current
460/// framebuffer so a mistaken agent click cannot address off-screen pixels; keyboard and pointer
461/// actions translate into input events; and actions that must look at the screen (`screenshot`,
462/// `zoom`) request a fresh capture first, so the agent reasons about the current frame rather than
463/// a stale one. Returns the JSON response, or an error string explaining why the action could not
464/// run.
465fn handle_action_inner(req: CuActionRequest, b: &dyn CuBackend) -> Result<String, String> {
466    let sleep_ms = |ms: u64| thread::sleep(Duration::from_millis(ms));
467
468    let (fb_w, fb_h) = b.fb_size()?;
469
470    let handle_coord = |coord: [f64; 2]| -> (f64, f64) {
471        (
472            clamp(coord[0], 0.0, (fb_w - 1) as f64),
473            clamp(coord[1], 0.0, (fb_h - 1) as f64),
474        )
475    };
476
477    let handle_modifier = |mod_name: &str| -> Option<u32> {
478        let sc = scancode_for_keyname(mod_name)?;
479        if is_modifier(sc) { Some(sc) } else { None }
480    };
481
482    match req.action.as_str() {
483        "screenshot" => {
484            let png = b.screenshot_png(req.display.unwrap_or(0))?;
485            let b64 = BASE64.encode(&png);
486            Ok(format!("{{\"data\":\"{}\"}}", b64))
487        }
488
489        "mouse_move" => {
490            let coord = req.coordinate.ok_or("Missing coordinate")?;
491            let (fx, fy) = handle_coord(coord);
492            b.mouse_move(fx, fy);
493            Ok(ok_json())
494        }
495
496        "left_click" | "right_click" | "middle_click" => {
497            let btn = match req.action.as_str() {
498                "left_click" => CuButton::Left,
499                "right_click" => CuButton::Right,
500                _ => CuButton::Middle,
501            };
502            if let Some(coord) = req.coordinate {
503                let (fx, fy) = handle_coord(coord);
504                b.mouse_move(fx, fy);
505                sleep_ms(30);
506            }
507            if let Some(ref mod_name) = req.text
508                && let Some(sc) = handle_modifier(mod_name)
509            {
510                b.key(sc, true);
511                sleep_ms(20);
512            }
513            b.button(btn, true);
514            sleep_ms(20);
515            b.button(btn, false);
516            if let Some(ref mod_name) = req.text
517                && let Some(sc) = handle_modifier(mod_name)
518            {
519                sleep_ms(10);
520                b.key(sc, false);
521            }
522            Ok(ok_json())
523        }
524
525        "double_click" | "triple_click" => {
526            let n = if req.action == "double_click" { 2 } else { 3 };
527            if let Some(coord) = req.coordinate {
528                let (fx, fy) = handle_coord(coord);
529                b.mouse_move(fx, fy);
530                sleep_ms(30);
531            }
532            if let Some(ref mod_name) = req.text
533                && let Some(sc) = handle_modifier(mod_name)
534            {
535                b.key(sc, true);
536                sleep_ms(20);
537            }
538            for _ in 0..n {
539                b.button(CuButton::Left, true);
540                sleep_ms(10);
541                b.button(CuButton::Left, false);
542                sleep_ms(10);
543            }
544            if let Some(ref mod_name) = req.text
545                && let Some(sc) = handle_modifier(mod_name)
546            {
547                sleep_ms(10);
548                b.key(sc, false);
549            }
550            Ok(ok_json())
551        }
552
553        "left_click_drag" => {
554            let start = req.start_coordinate.ok_or("Missing start_coordinate")?;
555            let end = req.coordinate.ok_or("Missing coordinate")?;
556            let (sx, sy) = handle_coord(start);
557            let (ex, ey) = handle_coord(end);
558            b.mouse_move(sx, sy);
559            sleep_ms(30);
560            b.button(CuButton::Left, true);
561            sleep_ms(30);
562            b.mouse_move(ex, ey);
563            sleep_ms(30);
564            b.button(CuButton::Left, false);
565            Ok(ok_json())
566        }
567
568        "left_mouse_down" => {
569            b.button(CuButton::Left, true);
570            Ok(ok_json())
571        }
572
573        "left_mouse_up" => {
574            b.button(CuButton::Left, false);
575            Ok(ok_json())
576        }
577
578        "type" => {
579            let text = req.text.as_deref().ok_or("Missing text")?;
580            // A nested app compositor owns the apps on its own socket; type there as
581            // a virtual-keyboard client, since keys on pixelflux's own seat carry an
582            // overlay keymap the inner compositor never sees. Falls through to the
583            // local seat if the app socket is unreachable.
584            if b.name() == "wayland"
585                && let Some(sock) = app_wayland_socket_path()
586            {
587                // Failures log once per socket value; every request still
588                // retries, so a compositor that comes back is used again
589                // immediately (and re-arms the logging).
590                static FAILED_SOCK: Mutex<Option<String>> = Mutex::new(None);
591                match crate::wayland::vkclient::type_text_to(&sock, text) {
592                    Ok(()) => {
593                        *FAILED_SOCK.lock().unwrap() = None;
594                        return Ok(ok_json());
595                    }
596                    Err(e) => {
597                        let mut last = FAILED_SOCK.lock().unwrap();
598                        if last.as_deref() != Some(sock.as_str()) {
599                            eprintln!(
600                                "[ComputerUse] app-compositor type via {sock} failed ({e}); using local seat until it is reachable"
601                            );
602                            *last = Some(sock);
603                        }
604                    }
605                }
606            }
607            let all: Vec<u32> = text.chars().map(keysym_for_char).collect();
608            for syms in distinct_runs(&all, TYPE_RUN_KEYSYMS) {
609                let mut resolver = KeyResolver::new(b);
610                resolver.prefetch(syms);
611                // Base+AltGr resolution stays the preferred path; only what the active keymap
612                // cannot reach at all goes through the backend's transient-bind fallback.
613                let mut unresolved: Vec<u32> = Vec::new();
614                for &sym in syms {
615                    if sym != 0 && resolver.resolve(sym).is_none() && !unresolved.contains(&sym) {
616                        unresolved.push(sym);
617                    }
618                }
619                b.with_transient_keysyms(&unresolved, &mut |bound| {
620                    for (i, &sym) in syms.iter().enumerate() {
621                        if i > 0 && i % 50 == 0 {
622                            sleep_ms(20);
623                        }
624                        if let Some((sc, level)) = resolver.resolve(sym) {
625                            let level_mods = level_modifiers(level, 0, b.altgr_keycode());
626                            for &m in &level_mods {
627                                b.key(m, true);
628                                sleep_ms(5);
629                            }
630                            b.key(sc, true);
631                            sleep_ms(10);
632                            b.key(sc, false);
633                            for &m in level_mods.iter().rev() {
634                                b.key(m, false);
635                            }
636                            sleep_ms(8);
637                        } else if let Some(&kc) = bound.get(&sym) {
638                            // Transient binds sit at the plain level: no modifiers needed.
639                            b.key(kc, true);
640                            sleep_ms(10);
641                            b.key(kc, false);
642                            sleep_ms(8);
643                        }
644                    }
645                });
646            }
647            Ok(ok_json())
648        }
649
650        "key" => {
651            let text = req.text.as_deref().ok_or("Missing text")?;
652            let mut resolver = KeyResolver::new(b);
653            let mut mods: Vec<u32> = Vec::new();
654            let mut main_key: Option<(u32, u32)> = None;
655            // Keysym of the last main-key spec the active keymap could not place (cleared
656            // when a later part resolves): typed via the transient-bind fallback.
657            let mut unresolved_sym: Option<u32> = None;
658            for part in text.split('+') {
659                let trimmed = part.trim();
660                if let Some(sc) = scancode_for_keyname(trimmed) {
661                    if is_modifier(sc) {
662                        mods.push(sc);
663                    } else {
664                        main_key = Some((sc, 0));
665                        unresolved_sym = None;
666                    }
667                } else {
668                    let sym = keysym_for_key_spec(trimmed);
669                    if let Some(hit) = resolver.resolve(sym) {
670                        main_key = Some(hit);
671                        unresolved_sym = None;
672                    } else if sym != 0 {
673                        main_key = None;
674                        unresolved_sym = Some(sym);
675                    }
676                }
677            }
678            let unresolved: Vec<u32> = unresolved_sym.into_iter().collect();
679            b.with_transient_keysyms(&unresolved, &mut |bound| {
680                let main_key = main_key.or_else(|| {
681                    // Transient binds sit at the plain level: no modifiers needed.
682                    unresolved_sym
683                        .and_then(|s| bound.get(&s))
684                        .map(|&kc| (kc, 0))
685                });
686                for &sc in &mods {
687                    b.key(sc, true);
688                    sleep_ms(10);
689                }
690                if let Some((sc, level)) = main_key {
691                    // Modifiers reaching the key's shift level are added unless the spec
692                    // already asked for them (Shift as 50/62, AltGr as 108 or the backend's
693                    // resolved level-3 keycode).
694                    let altgr = b.altgr_keycode();
695                    let held = mods.iter().fold(0u32, |m, &sc| match sc {
696                        50 | 62 => m | 1,
697                        sc if sc == KC_ALTGR || sc == altgr => m | 2,
698                        _ => m,
699                    });
700                    let level_mods = level_modifiers(level, held, altgr);
701                    for &m in &level_mods {
702                        b.key(m, true);
703                        sleep_ms(10);
704                    }
705                    b.key(sc, true);
706                    sleep_ms(30);
707                    b.key(sc, false);
708                    for &m in level_mods.iter().rev() {
709                        sleep_ms(10);
710                        b.key(m, false);
711                    }
712                } else if let Some(&last_mod) = mods.last() {
713                    b.key(last_mod, true);
714                    sleep_ms(30);
715                    b.key(last_mod, false);
716                }
717                for &sc in mods.iter().rev() {
718                    sleep_ms(10);
719                    b.key(sc, false);
720                }
721            });
722            Ok(ok_json())
723        }
724
725        "hold_key" => {
726            let text = req.text.as_deref().ok_or("Missing text")?;
727            let duration = req.duration.ok_or("Missing duration")?;
728            let duration = duration.min(100.0);
729            let trimmed = text.trim();
730            let mut resolver = KeyResolver::new(b);
731            let hit = match scancode_for_keyname(trimmed) {
732                Some(sc) => Some((sc, 0)),
733                None => resolver.resolve(keysym_for_key_spec(trimmed)),
734            };
735            if let Some((sc, level)) = hit {
736                let level_mods = level_modifiers(level, 0, b.altgr_keycode());
737                for &m in &level_mods {
738                    b.key(m, true);
739                    sleep_ms(10);
740                }
741                b.key(sc, true);
742                sleep_ms((duration * 1000.0) as u64);
743                b.key(sc, false);
744                for &m in level_mods.iter().rev() {
745                    sleep_ms(10);
746                    b.key(m, false);
747                }
748            }
749            Ok(ok_json())
750        }
751
752        "scroll" => {
753            let dir = req
754                .scroll_direction
755                .as_deref()
756                .ok_or("Missing scroll_direction")?;
757            let amount = req.scroll_amount.unwrap_or(1).max(0) as f64;
758            if let Some(coord) = req.coordinate {
759                let (fx, fy) = handle_coord(coord);
760                b.mouse_move(fx, fy);
761                sleep_ms(30);
762            }
763            if let Some(ref mod_name) = req.text
764                && let Some(sc) = handle_modifier(mod_name)
765            {
766                b.key(sc, true);
767                sleep_ms(20);
768            }
769            let (dx, dy) = match dir {
770                "up" => (0.0, -amount),
771                "down" => (0.0, amount),
772                "left" => (-amount, 0.0),
773                "right" => (amount, 0.0),
774                _ => return Err(format!("Invalid scroll_direction: {}", dir)),
775            };
776            b.scroll(dx, dy);
777            sleep_ms(30);
778            if let Some(ref mod_name) = req.text
779                && let Some(sc) = handle_modifier(mod_name)
780            {
781                sleep_ms(10);
782                b.key(sc, false);
783            }
784            Ok(ok_json())
785        }
786
787        "cursor_position" => {
788            let (x, y) = b.cursor_pos()?;
789            Ok(format!(
790                "{{\"text\":\"X={},Y={}\"}}",
791                x.round() as i32,
792                y.round() as i32
793            ))
794        }
795
796        "wait" => {
797            let duration = req.duration.unwrap_or(1.0).min(100.0);
798            sleep_ms((duration * 1000.0) as u64);
799            Ok(ok_json())
800        }
801
802        "zoom" => {
803            let region = req.region.ok_or("Missing region")?;
804            let display = req.display.unwrap_or(0);
805            // The region clamps against the TARGET display's own framebuffer, which need
806            // not match the primary's.
807            let (dw, dh) = b.display_fb_size(display)?;
808            let (x0, y0, x1, y1) = (region[0], region[1], region[2], region[3]);
809            let left = clamp(x0.round() as u32, 0, dw as u32 - 1);
810            let top = clamp(y0.round() as u32, 0, dh as u32 - 1);
811            let right = clamp(x1.round() as u32, left + 1, dw as u32);
812            let bottom = clamp(y1.round() as u32, top + 1, dh as u32);
813            let crop_w = right - left;
814            let crop_h = bottom - top;
815            let png = b.screenshot_png(display)?;
816            if crop_w > 0
817                && crop_h > 0
818                && let Ok(img) = image::load_from_memory(&png)
819            {
820                let cropped = img.crop_imm(left, top, crop_w, crop_h);
821                let mut out = Vec::new();
822                cropped
823                    .write_to(&mut Cursor::new(&mut out), ImageFormat::Png)
824                    .map_err(|e| format!("Crop/encode error: {}", e))?;
825                let b64 = BASE64.encode(&out);
826                return Ok(format!("{{\"data\":\"{}\"}}", b64));
827            }
828            let b64 = BASE64.encode(&png);
829            Ok(format!("{{\"data\":\"{}\"}}", b64))
830        }
831
832        _ => Err(format!("Unknown action: {}", req.action)),
833    }
834}
835
836static WAYLAND_TX: Mutex<Option<smithay::reexports::calloop::channel::Sender<ThreadCommand>>> =
837    Mutex::new(None);
838
839/// Body of `POST /record_start`. All fields are optional; unset ones fall back to the
840/// `PIXELFLUX_RECORD_*` environment variables and built-in defaults.
841#[derive(Deserialize, Default)]
842struct RecordStartRequest {
843    /// Output file name inside `PIXELFLUX_RECORD_DIR` (default: `$PIXELFLUX_RECORD`, else
844    /// `pixelflux-record-<unix_ts>.mp4` in that directory or `/tmp`).
845    path: Option<String>,
846    /// Wayland output id to record (default 0; ignored on X11).
847    display: Option<u32>,
848    /// Capture fps cap for a recorder-owned capture.
849    fps: Option<f64>,
850    /// Bitrate override in kbps for a recorder-owned capture.
851    bitrate_kbps: Option<i32>,
852    /// Unix socket serving an Ogg Opus stream to record as the audio track.
853    audio_socket: Option<String>,
854}
855
856/// The path of a recording the caller names `name`: a plain file name inside `dir`, so a request
857/// can neither leave the operator's directory nor name a file anywhere when none was given.
858fn named_record_path(dir: Option<&str>, name: &str) -> Result<String, String> {
859    let dir = dir.ok_or("a recording named by the caller needs PIXELFLUX_RECORD_DIR")?;
860    let mut parts = std::path::Path::new(name).components();
861    match (parts.next(), parts.next()) {
862        (Some(std::path::Component::Normal(file)), None) => Ok(std::path::Path::new(dir)
863            .join(file)
864            .to_string_lossy()
865            .into_owned()),
866        _ => Err(format!("'{name}' is not a file name")),
867    }
868}
869
870/// Handle the recorder REST endpoints sharing the CU server: `record_start`,
871/// `record_stop`, and `record_status`, all one JSON round-trip into the same recorder
872/// implementation the Python API and env vars use. Returns `None` for any other URL.
873fn handle_record_endpoint(url: &str, body: &str) -> Option<String> {
874    let reply = match url {
875        "/record_start" => {
876            let req: RecordStartRequest = if body.trim().is_empty() {
877                RecordStartRequest::default()
878            } else {
879                match serde_json::from_str(body) {
880                    Ok(r) => r,
881                    Err(e) => return Some(format!("{{\"error\":\"Invalid JSON: {}\"}}", e)),
882                }
883            };
884            let record_dir = std::env::var("PIXELFLUX_RECORD_DIR")
885                .ok()
886                .filter(|d| !d.is_empty());
887            let path = match req.path {
888                Some(name) => match named_record_path(record_dir.as_deref(), &name) {
889                    Ok(path) => path,
890                    Err(e) => return Some(serde_json::json!({ "error": e }).to_string()),
891                },
892                None => std::env::var("PIXELFLUX_RECORD")
893                    .ok()
894                    .filter(|p| !p.is_empty())
895                    .unwrap_or_else(|| {
896                        let ts = std::time::SystemTime::now()
897                            .duration_since(std::time::UNIX_EPOCH)
898                            .map(|d| d.as_secs())
899                            .unwrap_or(0);
900                        format!(
901                            "{}/pixelflux-record-{ts}.mp4",
902                            record_dir.as_deref().unwrap_or("/tmp")
903                        )
904                    }),
905            };
906            let mut opts = crate::recorder::RecordOptions::from_env(path);
907            if let Some(d) = req.display {
908                opts.display_id = d;
909            }
910            if let Some(f) = req.fps {
911                opts.fps = f;
912            }
913            if let Some(b) = req.bitrate_kbps {
914                opts.bitrate_kbps = b;
915            }
916            if let Some(a) = req.audio_socket {
917                opts.audio_socket = a;
918            }
919            match crate::recorder::start(opts) {
920                Ok(s) => crate::recorder::status_to_json(&s).to_string(),
921                Err(e) => serde_json::json!({ "error": e }).to_string(),
922            }
923        }
924        "/record_stop" => match crate::recorder::stop() {
925            Ok(s) => {
926                let mut v = crate::recorder::status_to_json(&s);
927                v["stopped"] = serde_json::Value::Bool(true);
928                v.to_string()
929            }
930            Err(e) => serde_json::json!({ "error": e }).to_string(),
931        },
932        "/record_status" => match crate::recorder::status() {
933            Some(s) => crate::recorder::status_to_json(&s).to_string(),
934            None => serde_json::json!({ "active": false }).to_string(),
935        },
936        _ => return None,
937    };
938    Some(reply)
939}
940
941/// Socket of the compositor apps run under when it is nested under pixelflux
942/// (labwc/kwin): keys injected into pixelflux's own seat carry an overlay keymap
943/// the inner compositor never sees, so CU text is typed as a client of this
944/// socket instead. Set by selkies over the ScreenCapture ABI.
945static CU_APP_WAYLAND_DISPLAY: Mutex<Option<String>> = Mutex::new(None);
946
947/// Set (or clear, with None/empty) the app compositor socket for CU text injection
948/// and for relative pointer motion into a nested KWin session.
949pub fn set_app_wayland_display(display: Option<String>) {
950    *CU_APP_WAYLAND_DISPLAY.lock().unwrap() = display.filter(|s| !s.is_empty());
951    crate::wayland::ficlient::arm(app_wayland_socket_path());
952}
953
954/// Resolve the app compositor socket PATH for CU typing, or None to type on the
955/// local seat. The ABI value selkies set wins; a standalone CU (no selkies) falls
956/// back to PIXELFLUX_APP_WAYLAND_DISPLAY. A value naming pixelflux's own
957/// compositor means nothing is nested.
958pub(crate) fn app_wayland_socket_path() -> Option<String> {
959    let name = CU_APP_WAYLAND_DISPLAY
960        .lock()
961        .unwrap()
962        .clone()
963        .or_else(|| std::env::var("PIXELFLUX_APP_WAYLAND_DISPLAY").ok())
964        .filter(|s| !s.is_empty())?;
965    if crate::wait_socket_name(Duration::from_millis(0)).as_deref() == Some(name.as_str()) {
966        return None;
967    }
968    crate::wayland::wlclient::socket_path(&name)
969}
970
971/// Make the Wayland compositor the preferred CU backend: once a live calloop sender is
972/// registered, every subsequent request routes to it instead of an X11 connection.
973pub fn register_wayland_backend(tx: smithay::reexports::calloop::channel::Sender<ThreadCommand>) {
974    *WAYLAND_TX.lock().unwrap() = Some(tx);
975}
976
977/// The registered compositor's command channel, if a Wayland compositor is running in this
978/// process. The recorder uses it to attach to (or start) a capture without any Python client.
979pub(crate) fn wayland_command_sender()
980-> Option<smithay::reexports::calloop::channel::Sender<ThreadCommand>> {
981    WAYLAND_TX.lock().unwrap().clone()
982}
983
984/// Start the CU server if `PIXELFLUX_CU` names a bind (the standalone fallback;
985/// a selkies-managed session passes the setting through [`start_cu_server`]).
986pub fn spawn_cu_from_env() {
987    if let Ok(bind) = std::env::var("PIXELFLUX_CU")
988        && let Err(e) = start_cu_server(&bind, None)
989    {
990        println!("[ComputerUse] Not started: {e}");
991    }
992}
993
994/// The token every request has to carry, fixed when the server starts.
995static CU_TOKEN: OnceLock<String> = OnceLock::new();
996
997/// Start the CU server on `bind`, comma-separated entries: a bare port listens on
998/// the loopback addresses only, `host:port` names the address to listen on
999/// (`0.0.0.0:port,[::]:port` accepts every interface), requiring `token`, else
1000/// `PIXELFLUX_CU_TOKEN`, of every request. Guarded so that exactly one server
1001/// binds per process no matter how many call sites (module import, Wayland
1002/// compositor init, the selkies setting) race to spawn it; backend selection stays
1003/// per-request, so a server bound at import serves a compositor that only starts
1004/// later.
1005pub fn start_cu_server(bind: &str, token: Option<&str>) -> Result<(), String> {
1006    static SPAWNED: OnceLock<()> = OnceLock::new();
1007    if SPAWNED.get().is_some() {
1008        return Ok(());
1009    }
1010    let token = token
1011        .map(str::to_string)
1012        .or_else(|| std::env::var("PIXELFLUX_CU_TOKEN").ok())
1013        .filter(|t| !t.is_empty())
1014        .ok_or("no token: set PIXELFLUX_CU_TOKEN, the bearer token every request has to carry")?;
1015    let listeners = cu_listeners(bind)?;
1016    if SPAWNED.set(()).is_err() {
1017        return Ok(());
1018    }
1019    let _ = CU_TOKEN.set(token);
1020    for listener in listeners {
1021        thread::spawn(move || run_cu_server(listener));
1022    }
1023    Ok(())
1024}
1025
1026/// Whether `request` carries the server's token, compared in constant time.
1027fn authorized(request: &tiny_http::Request) -> bool {
1028    let Some(token) = CU_TOKEN.get() else {
1029        return false;
1030    };
1031    request.headers().iter().any(|h| {
1032        h.field.equiv("Authorization")
1033            && h.value
1034                .as_str()
1035                .strip_prefix("Bearer ")
1036                .is_some_and(|given| {
1037                    given.len() == token.len()
1038                        && given
1039                            .bytes()
1040                            .zip(token.bytes())
1041                            .fold(0u8, |acc, (a, b)| acc | (a ^ b))
1042                            == 0
1043                })
1044    })
1045}
1046
1047/// Bound listeners for a CU bind: for each comma-separated entry, both loopback
1048/// addresses for a bare port and every address the host resolves to for
1049/// `host:port`. An address the host lacks (the IPv6 loopback with IPv6 disabled)
1050/// is skipped while another binds; any other failure, or nothing binding at all,
1051/// is an error.
1052fn cu_listeners(bind: &str) -> Result<Vec<TcpListener>, String> {
1053    let mut addrs: Vec<SocketAddr> = Vec::new();
1054    for entry in bind
1055        .split(',')
1056        .map(str::trim)
1057        .filter(|entry| !entry.is_empty())
1058    {
1059        let resolved: Vec<SocketAddr> = match entry.parse::<u16>() {
1060            Ok(port) => vec![
1061                SocketAddr::from((Ipv4Addr::LOCALHOST, port)),
1062                SocketAddr::from((Ipv6Addr::LOCALHOST, port)),
1063            ],
1064            Err(_) if entry.contains(':') => entry
1065                .to_socket_addrs()
1066                .map_err(|e| format!("cannot resolve '{entry}': {e}"))?
1067                .collect(),
1068            Err(_) => {
1069                return Err(format!(
1070                    "invalid bind '{entry}': expected a port or host:port"
1071                ));
1072            }
1073        };
1074        for addr in resolved {
1075            if !addrs.contains(&addr) {
1076                addrs.push(addr);
1077            }
1078        }
1079    }
1080    if addrs.is_empty() {
1081        return Err(format!(
1082            "invalid bind '{bind}': expected a port or host:port"
1083        ));
1084    }
1085    let mut listeners = Vec::new();
1086    let mut skipped = Vec::new();
1087    for addr in addrs {
1088        match bind_listener(addr) {
1089            Ok(listener) => listeners.push(listener),
1090            Err(e)
1091                if matches!(
1092                    e.raw_os_error(),
1093                    Some(libc::EADDRNOTAVAIL | libc::EAFNOSUPPORT)
1094                ) =>
1095            {
1096                skipped.push(format!("{addr} ({e})"));
1097            }
1098            Err(e) => return Err(format!("cannot bind {addr}: {e}")),
1099        }
1100    }
1101    if listeners.is_empty() {
1102        return Err(format!(
1103            "no address of '{bind}' is available: {}",
1104            skipped.join(", ")
1105        ));
1106    }
1107    for note in skipped {
1108        println!("[ComputerUse] Not listening on {note}");
1109    }
1110    Ok(listeners)
1111}
1112
1113/// A listening socket on `addr`, IPv6-only for an IPv6 address so `[::]` and
1114/// `0.0.0.0` can share a port: the standard library binds `[::]` dual-stack and
1115/// offers no way to change that before the bind.
1116fn bind_listener(addr: SocketAddr) -> std::io::Result<TcpListener> {
1117    use std::os::fd::{FromRawFd, IntoRawFd, OwnedFd};
1118
1119    let family = if addr.is_ipv4() {
1120        libc::AF_INET
1121    } else {
1122        libc::AF_INET6
1123    };
1124    let fd = unsafe { libc::socket(family, libc::SOCK_STREAM | libc::SOCK_CLOEXEC, 0) };
1125    if fd < 0 {
1126        return Err(std::io::Error::last_os_error());
1127    }
1128    let sock = unsafe { OwnedFd::from_raw_fd(fd) };
1129    let one: libc::c_int = 1;
1130    let enable = |level: libc::c_int, name: libc::c_int| -> std::io::Result<()> {
1131        let rc = unsafe {
1132            libc::setsockopt(
1133                fd,
1134                level,
1135                name,
1136                &one as *const libc::c_int as *const libc::c_void,
1137                std::mem::size_of::<libc::c_int>() as libc::socklen_t,
1138            )
1139        };
1140        if rc < 0 {
1141            Err(std::io::Error::last_os_error())
1142        } else {
1143            Ok(())
1144        }
1145    };
1146    enable(libc::SOL_SOCKET, libc::SO_REUSEADDR)?;
1147    if addr.is_ipv6() {
1148        enable(libc::IPPROTO_IPV6, libc::IPV6_V6ONLY)?;
1149    }
1150    let rc = match addr {
1151        SocketAddr::V4(v4) => {
1152            let mut sin: libc::sockaddr_in = unsafe { std::mem::zeroed() };
1153            sin.sin_family = libc::AF_INET as libc::sa_family_t;
1154            sin.sin_port = v4.port().to_be();
1155            sin.sin_addr = libc::in_addr {
1156                s_addr: u32::from_ne_bytes(v4.ip().octets()),
1157            };
1158            unsafe {
1159                libc::bind(
1160                    fd,
1161                    &sin as *const libc::sockaddr_in as *const libc::sockaddr,
1162                    std::mem::size_of::<libc::sockaddr_in>() as libc::socklen_t,
1163                )
1164            }
1165        }
1166        SocketAddr::V6(v6) => {
1167            let mut sin6: libc::sockaddr_in6 = unsafe { std::mem::zeroed() };
1168            sin6.sin6_family = libc::AF_INET6 as libc::sa_family_t;
1169            sin6.sin6_port = v6.port().to_be();
1170            sin6.sin6_flowinfo = v6.flowinfo();
1171            sin6.sin6_addr = libc::in6_addr {
1172                s6_addr: v6.ip().octets(),
1173            };
1174            sin6.sin6_scope_id = v6.scope_id();
1175            unsafe {
1176                libc::bind(
1177                    fd,
1178                    &sin6 as *const libc::sockaddr_in6 as *const libc::sockaddr,
1179                    std::mem::size_of::<libc::sockaddr_in6>() as libc::socklen_t,
1180                )
1181            }
1182        }
1183    };
1184    if rc < 0 {
1185        return Err(std::io::Error::last_os_error());
1186    }
1187    if unsafe { libc::listen(fd, 128) } < 0 {
1188        return Err(std::io::Error::last_os_error());
1189    }
1190    Ok(unsafe { TcpListener::from_raw_fd(sock.into_raw_fd()) })
1191}
1192
1193/// Pick the backend for one request: the registered Wayland compositor when present,
1194/// otherwise a fresh private connection to `DISPLAY`. The X11 connection is per-request so a
1195/// restarted X server never leaves the CU thread holding a dead connection.
1196pub(crate) fn resolve_backend() -> Result<Box<dyn CuBackend>, String> {
1197    if let Some(tx) = WAYLAND_TX.lock().unwrap().clone() {
1198        return Ok(Box::new(CuWaylandBackend { tx }));
1199    }
1200    crate::x11::computer_use::CuX11Backend::connect().map(|be| Box::new(be) as Box<dyn CuBackend>)
1201}
1202
1203/// Expose the captured desktop to an AI agent over HTTP, so a Computer Use client can drive
1204/// the session much as a human viewer would.
1205///
1206/// It runs on its own thread serving `listener` POST `/computer-use` JSON actions
1207/// (screenshot, mouse_move, click, key, scroll, …). The backend and the framebuffer dimensions
1208/// are re-resolved on every request rather than cached, because the compositor can start, the
1209/// stream can resize, or the X server can restart underneath the agent — a stale size would
1210/// misplace every coordinate. On Wayland a screenshot forces a one-frame GPU readback when the
1211/// pipeline is in zero-copy mode; on X11 it is a one-shot `GetImage` of the root window.
1212pub fn run_cu_server(listener: TcpListener) {
1213    let addr = listener
1214        .local_addr()
1215        .map(|a| a.to_string())
1216        .unwrap_or_default();
1217    let server = match tiny_http::Server::from_listener(listener, None) {
1218        Ok(s) => s,
1219        Err(e) => {
1220            eprintln!("[ComputerUse] Failed to start server on {addr}: {e}");
1221            return;
1222        }
1223    };
1224    println!("[ComputerUse] Server listening on {addr}");
1225
1226    let mut last_backend = "";
1227    for mut request in server.incoming_requests() {
1228        if !authorized(&request) {
1229            let _ = request.respond(
1230                tiny_http::Response::from_string("{\"error\":\"unauthorized\"}".to_string())
1231                    .with_status_code(401)
1232                    .with_header(
1233                        "WWW-Authenticate: Bearer"
1234                            .parse::<tiny_http::Header>()
1235                            .unwrap(),
1236                    )
1237                    .with_header(
1238                        "Content-Type: application/json"
1239                            .parse::<tiny_http::Header>()
1240                            .unwrap(),
1241                    ),
1242            );
1243            continue;
1244        }
1245        // A CU body is a single input command: cap its size so a hostile client
1246        // cannot exhaust memory with a giant POST.
1247        const MAX_CU_BODY: u64 = 4 * 1024 * 1024;
1248        let mut body = String::new();
1249        if let Err(e) = request
1250            .as_reader()
1251            .take(MAX_CU_BODY + 1)
1252            .read_to_string(&mut body)
1253        {
1254            let _ = request.respond(
1255                tiny_http::Response::from_string(format!("{{\"error\":\"{}\"}}", e))
1256                    .with_status_code(400)
1257                    .with_header(
1258                        "Content-Type: application/json"
1259                            .parse::<tiny_http::Header>()
1260                            .unwrap(),
1261                    ),
1262            );
1263            continue;
1264        }
1265
1266        if body.len() as u64 > MAX_CU_BODY {
1267            let resp = tiny_http::Response::from_string(
1268                "{\"error\":\"request body too large\"}".to_string(),
1269            )
1270            .with_status_code(413)
1271            .with_header(
1272                "Content-Type: application/json"
1273                    .parse::<tiny_http::Header>()
1274                    .unwrap(),
1275            );
1276            let _ = request.respond(resp);
1277            continue;
1278        }
1279
1280        if let Some(json) = handle_record_endpoint(request.url(), &body) {
1281            let _ = request.respond(
1282                tiny_http::Response::from_string(json)
1283                    .with_status_code(200)
1284                    .with_header(
1285                        "Content-Type: application/json"
1286                            .parse::<tiny_http::Header>()
1287                            .unwrap(),
1288                    ),
1289            );
1290            continue;
1291        }
1292
1293        let parsed: CuActionRequest = match serde_json::from_str(&body) {
1294            Ok(r) => r,
1295            Err(e) => {
1296                let _ = request.respond(
1297                    tiny_http::Response::from_string(format!(
1298                        "{{\"error\":\"Invalid JSON: {}\"}}",
1299                        e
1300                    ))
1301                    .with_status_code(400)
1302                    .with_header(
1303                        "Content-Type: application/json"
1304                            .parse::<tiny_http::Header>()
1305                            .unwrap(),
1306                    ),
1307                );
1308                continue;
1309            }
1310        };
1311
1312        let json_response = match resolve_backend() {
1313            Ok(backend) => {
1314                if backend.name() != last_backend {
1315                    last_backend = backend.name();
1316                    println!("[ComputerUse] Using {} backend", last_backend);
1317                }
1318                handle_action(parsed, backend.as_ref())
1319            }
1320            Err(e) => format!("{{\"error\":\"{}\"}}", e.replace('"', "\\\"")),
1321        };
1322        let _ = request.respond(
1323            tiny_http::Response::from_string(json_response)
1324                .with_status_code(200)
1325                .with_header(
1326                    "Content-Type: application/json"
1327                        .parse::<tiny_http::Header>()
1328                        .unwrap(),
1329                ),
1330        );
1331    }
1332}
1333
1334#[cfg(test)]
1335mod run_tests {
1336    use super::distinct_runs;
1337
1338    /// A text is cut where a run would name one distinct character too many, and a
1339    /// character repeated in a run does not count again.
1340    #[test]
1341    fn a_text_is_typed_in_runs_of_few_distinct_characters() {
1342        let syms = [1, 2, 1, 3, 4, 4, 5, 1, 6];
1343        assert_eq!(
1344            distinct_runs(&syms, 3),
1345            vec![&[1, 2, 1, 3][..], &[4, 4, 5, 1][..], &[6][..]]
1346        );
1347        assert_eq!(distinct_runs(&syms, 9), vec![&syms[..]]);
1348        assert!(distinct_runs(&[], 3).is_empty());
1349    }
1350}
1351
1352#[cfg(test)]
1353mod record_path_tests {
1354    use super::named_record_path;
1355
1356    #[test]
1357    fn a_named_recording_stays_in_the_operators_directory() {
1358        assert_eq!(
1359            named_record_path(Some("/rec"), "take.mp4").unwrap(),
1360            "/rec/take.mp4"
1361        );
1362        for name in ["/etc/passwd", "../x.mp4", "a/b.mp4", "..", ".", ""] {
1363            assert!(named_record_path(Some("/rec"), name).is_err(), "{name:?}");
1364        }
1365        assert!(
1366            named_record_path(None, "take.mp4").is_err(),
1367            "no directory, no named file"
1368        );
1369    }
1370}
1371
1372#[cfg(test)]
1373mod tests {
1374    use super::cu_listeners;
1375    use std::net::{IpAddr, Ipv4Addr, TcpListener};
1376
1377    #[test]
1378    fn bare_port_binds_loopback_only() {
1379        let listeners = cu_listeners("0").expect("loopback binds");
1380        assert!(!listeners.is_empty());
1381        for listener in &listeners {
1382            let addr = listener.local_addr().unwrap();
1383            assert!(addr.ip().is_loopback(), "{addr}");
1384            assert_ne!(addr.port(), 0);
1385        }
1386        assert!(
1387            listeners
1388                .iter()
1389                .any(|l| l.local_addr().unwrap().ip() == IpAddr::V4(Ipv4Addr::LOCALHOST))
1390        );
1391    }
1392
1393    #[test]
1394    fn host_port_names_the_address() {
1395        let scoped = cu_listeners("127.0.0.1:0").unwrap();
1396        assert_eq!(scoped.len(), 1);
1397        assert_eq!(
1398            scoped[0].local_addr().unwrap().ip(),
1399            IpAddr::V4(Ipv4Addr::LOCALHOST)
1400        );
1401        let open = cu_listeners("0.0.0.0:0").unwrap();
1402        assert_eq!(open.len(), 1);
1403        assert!(open[0].local_addr().unwrap().ip().is_unspecified());
1404    }
1405
1406    #[test]
1407    fn a_taken_port_is_an_error() {
1408        let taken = TcpListener::bind("127.0.0.1:0").unwrap();
1409        let port = taken.local_addr().unwrap().port();
1410        assert!(cu_listeners(&port.to_string()).is_err());
1411        assert!(cu_listeners(&format!("127.0.0.1:{port}")).is_err());
1412    }
1413
1414    #[test]
1415    fn malformed_binds_are_errors() {
1416        assert!(cu_listeners("abc").is_err());
1417        assert!(cu_listeners("70000").is_err());
1418        assert!(cu_listeners("").is_err());
1419        assert!(cu_listeners(",").is_err());
1420        assert!(cu_listeners("127.0.0.1:0,abc").is_err());
1421    }
1422
1423    fn free_port() -> u16 {
1424        TcpListener::bind("127.0.0.1:0")
1425            .unwrap()
1426            .local_addr()
1427            .unwrap()
1428            .port()
1429    }
1430
1431    fn v6only(listener: &TcpListener) -> libc::c_int {
1432        use std::os::fd::AsRawFd;
1433        let mut value: libc::c_int = -1;
1434        let mut len = std::mem::size_of::<libc::c_int>() as libc::socklen_t;
1435        let rc = unsafe {
1436            libc::getsockopt(
1437                listener.as_raw_fd(),
1438                libc::IPPROTO_IPV6,
1439                libc::IPV6_V6ONLY,
1440                &mut value as *mut libc::c_int as *mut libc::c_void,
1441                &mut len,
1442            )
1443        };
1444        assert_eq!(rc, 0);
1445        value
1446    }
1447
1448    #[test]
1449    fn both_wildcards_share_a_port() {
1450        let port = free_port();
1451        let listeners = cu_listeners(&format!("0.0.0.0:{port}, [::]:{port}")).unwrap();
1452        assert!(
1453            listeners
1454                .iter()
1455                .all(|l| l.local_addr().unwrap().port() == port)
1456        );
1457        assert!(
1458            listeners
1459                .iter()
1460                .all(|l| l.local_addr().unwrap().ip().is_unspecified())
1461        );
1462        if let Some(v6) = listeners.iter().find(|l| l.local_addr().unwrap().is_ipv6()) {
1463            assert_eq!(listeners.len(), 2);
1464            assert_eq!(v6only(v6), 1);
1465        }
1466    }
1467
1468    #[test]
1469    fn repeated_entries_bind_once() {
1470        let port = free_port();
1471        let listeners = cu_listeners(&format!("{port},127.0.0.1:{port},{port}")).unwrap();
1472        assert!(listeners.len() <= 2);
1473        assert!(
1474            listeners
1475                .iter()
1476                .all(|l| l.local_addr().unwrap().ip().is_loopback())
1477        );
1478    }
1479}