1use std::collections::HashMap;
28use std::io::Cursor;
29use std::io::Read;
30use std::net::{Ipv4Addr, Ipv6Addr, SocketAddr, TcpListener, ToSocketAddrs};
31use std::sync::mpsc;
32use std::sync::{Mutex, OnceLock};
33use std::thread;
34use std::time::Duration;
35
36use smithay::input::keyboard::xkb;
37
38use base64::Engine;
39use base64::engine::general_purpose::STANDARD as BASE64;
40use image::{ImageBuffer, ImageFormat, Rgba};
41use serde::Deserialize;
42use tiny_http;
43
44use crate::ThreadCommand;
45use crate::wayland::keymap::keysym_for_char;
46
47fn clamp<T: PartialOrd>(v: T, lo: T, hi: T) -> T {
48 if v < lo {
49 lo
50 } else if v > hi {
51 hi
52 } else {
53 v
54 }
55}
56
57pub fn encode_png_rgba(data: &[u8], width: u32, height: u32) -> Result<Vec<u8>, String> {
63 let img = ImageBuffer::<Rgba<u8>, _>::from_raw(width, height, data.to_vec())
64 .ok_or("Failed to create image buffer")?;
65 let mut png = Vec::new();
66 img.write_to(&mut Cursor::new(&mut png), ImageFormat::Png)
67 .map_err(|e| format!("PNG encode error: {}", e))?;
68 Ok(png)
69}
70
71fn scancode_for_keyname(name: &str) -> Option<u32> {
72 Some(match name.to_lowercase().as_str() {
73 "return" | "enter" => 36,
74 "tab" => 23,
75 "escape" | "esc" => 9,
76 "backspace" => 22,
77 "delete" => 119,
78 "insert" => 118,
79 "home" => 110,
80 "end" => 115,
81 "pageup" => 112,
82 "pagedown" => 117,
83 "up" => 111,
84 "down" => 116,
85 "left" => 113,
86 "right" => 114,
87 "space" => 65,
88 "capslock" => 66,
89 "numlock" => 77,
90 "scrolllock" => 78,
91 "printscreen" | "sysrq" | "print" => 107,
92 "pause" | "break" => 127,
93 "f1" => 67,
94 "f2" => 68,
95 "f3" => 69,
96 "f4" => 70,
97 "f5" => 71,
98 "f6" => 72,
99 "f7" => 73,
100 "f8" => 74,
101 "f9" => 75,
102 "f10" => 76,
103 "f11" => 95,
104 "f12" => 96,
105 "f13" => 191,
106 "f14" => 192,
107 "f15" => 193,
108 "f16" => 194,
109 "f17" => 195,
110 "f18" => 196,
111 "f19" => 197,
112 "f20" => 198,
113 "f21" => 199,
114 "f22" => 200,
115 "f23" => 201,
116 "f24" => 202,
117 "ctrl" | "lctrl" | "leftctrl" => 37,
118 "rctrl" | "rightctrl" => 105,
119 "shift" | "lshift" | "leftshift" => 50,
120 "rshift" | "rightshift" => 62,
121 "alt" | "lalt" | "leftalt" => 64,
122 "ralt" | "rightalt" => 108,
123 "super" | "meta" | "lsuper" | "leftmeta" | "leftsuper" | "windows" | "leftwindows" => 133,
124 "rsuper" | "rightmeta" | "rightsuper" | "rightwindows" => 134,
125 "menu" | "compose" => 135,
126 "kp_0" | "kp0" => 90,
127 "kp_1" | "kp1" => 87,
128 "kp_2" | "kp2" => 88,
129 "kp_3" | "kp3" => 89,
130 "kp_4" | "kp4" => 83,
131 "kp_5" | "kp5" => 84,
132 "kp_6" | "kp6" => 85,
133 "kp_7" | "kp7" => 79,
134 "kp_8" | "kp8" => 80,
135 "kp_9" | "kp9" => 81,
136 "kp_decimal" | "kp_dot" => 91,
137 "kp_divide" | "kp_slash" => 106,
138 "kp_multiply" | "kp_asterisk" => 63,
139 "kp_subtract" | "kp_minus" => 82,
140 "kp_add" | "kp_plus" => 86,
141 "kp_enter" => 104,
142 _ => return None,
143 })
144}
145
146fn keysym_for_key_spec(name: &str) -> u32 {
151 let mut chars = name.chars();
152 if let (Some(c), None) = (chars.next(), chars.next()) {
153 return keysym_for_char(c);
154 }
155 let sym = xkb::keysym_from_name(name, xkb::KEYSYM_NO_FLAGS).raw();
156 if sym != 0 {
157 return sym;
158 }
159 xkb::keysym_from_name(name, xkb::KEYSYM_CASE_INSENSITIVE).raw()
160}
161
162fn is_modifier(scancode: u32) -> bool {
163 matches!(scancode, 37 | 105 | 50 | 62 | 64 | 108 | 133 | 134)
164}
165
166#[derive(Clone, Copy)]
169pub enum CuButton {
170 Left,
171 Right,
172 Middle,
173}
174
175pub trait CuBackend {
179 fn name(&self) -> &'static str;
180 fn fb_size(&self) -> Result<(i32, i32), String>;
181 fn display_fb_size(&self, display: u32) -> Result<(i32, i32), String> {
184 if display == 0 {
185 self.fb_size()
186 } else {
187 Err(format!("Unknown display: {display}"))
188 }
189 }
190 fn key(&self, scancode: u32, pressed: bool);
191 fn mouse_move(&self, x: f64, y: f64);
192 fn button(&self, btn: CuButton, pressed: bool);
193 fn scroll(&self, dx: f64, dy: f64);
194 fn screenshot_png(&self, display: u32) -> Result<Vec<u8>, String>;
196 fn cursor_pos(&self) -> Result<(f64, f64), String>;
197 fn with_transient_keysyms(&self, keysyms: &[u32], seq: &mut dyn FnMut(&HashMap<u32, u32>)) {
205 let _ = keysyms;
206 seq(&HashMap::new());
207 }
208 fn resolve_keysyms(&self, keysyms: &[u32]) -> Vec<(u32, u32)>;
214 fn altgr_keycode(&self) -> u32 {
219 KC_ALTGR
220 }
221}
222
223const BTN_LEFT: u32 = 0x110;
224const BTN_RIGHT: u32 = 0x111;
225const BTN_MIDDLE: u32 = 0x112;
226
227const REPLY_TIMEOUT: Duration = Duration::from_secs(5);
231
232pub struct CuWaylandBackend {
235 tx: smithay::reexports::calloop::channel::Sender<ThreadCommand>,
236}
237
238impl CuBackend for CuWaylandBackend {
239 fn name(&self) -> &'static str {
240 "wayland"
241 }
242
243 fn fb_size(&self) -> Result<(i32, i32), String> {
244 self.display_fb_size(0)
245 }
246
247 fn display_fb_size(&self, display: u32) -> Result<(i32, i32), String> {
248 let (resp_tx, resp_rx) = mpsc::channel();
249 self.tx
250 .send(ThreadCommand::CuGetInfo {
251 display_id: display,
252 resp: resp_tx,
253 })
254 .map_err(|_| "Failed to request compositor info".to_string())?;
255 let (w, h, _) = resp_rx
256 .recv_timeout(REPLY_TIMEOUT)
257 .map_err(|_| "Compositor info request failed".to_string())?;
258 if w <= 0 || h <= 0 {
261 return Err(format!("Unknown display: {display}"));
262 }
263 Ok((w, h))
264 }
265
266 fn key(&self, scancode: u32, pressed: bool) {
267 let _ = self.tx.send(ThreadCommand::KeyboardKey {
268 scancode,
269 state: if pressed { 1 } else { 0 },
270 });
271 }
272
273 fn mouse_move(&self, x: f64, y: f64) {
274 let _ = self
275 .tx
276 .send(ThreadCommand::PointerMotion { x, y, motion: 0 });
277 }
278
279 fn button(&self, btn: CuButton, pressed: bool) {
280 let btn = match btn {
281 CuButton::Left => BTN_LEFT,
282 CuButton::Right => BTN_RIGHT,
283 CuButton::Middle => BTN_MIDDLE,
284 };
285 let _ = self.tx.send(ThreadCommand::PointerButton {
286 btn,
287 state: if pressed { 1 } else { 0 },
288 });
289 }
290
291 fn scroll(&self, dx: f64, dy: f64) {
292 let _ = self.tx.send(ThreadCommand::PointerAxis { x: dx, y: dy });
293 }
294
295 fn screenshot_png(&self, display: u32) -> Result<Vec<u8>, String> {
296 let (resp_tx, resp_rx) = mpsc::channel();
297 self.tx
298 .send(ThreadCommand::CuScreenshot {
299 display_id: display,
300 resp: resp_tx,
301 })
302 .map_err(|_| "Failed to request screenshot".to_string())?;
303 resp_rx
304 .recv_timeout(REPLY_TIMEOUT)
305 .map_err(|_| "Screenshot failed".to_string())?
306 }
307
308 fn cursor_pos(&self) -> Result<(f64, f64), String> {
309 let (resp_tx, resp_rx) = mpsc::channel();
310 self.tx
311 .send(ThreadCommand::CuCursorPosition { resp: resp_tx })
312 .map_err(|_| "Failed to request cursor position".to_string())?;
313 resp_rx
314 .recv_timeout(REPLY_TIMEOUT)
315 .map_err(|_| "Cursor position failed".to_string())
316 }
317
318 fn resolve_keysyms(&self, keysyms: &[u32]) -> Vec<(u32, u32)> {
319 let (resp_tx, resp_rx) = mpsc::channel();
320 if self
321 .tx
322 .send(ThreadCommand::BindKeysyms {
323 keysyms: keysyms.to_vec(),
324 reply: resp_tx,
325 })
326 .is_err()
327 {
328 return vec![(0, 0); keysyms.len()];
329 }
330 resp_rx
331 .recv_timeout(REPLY_TIMEOUT)
332 .unwrap_or_else(|_| vec![(0, 0); keysyms.len()])
333 }
334}
335
336const TYPE_RUN_KEYSYMS: usize = 8;
341
342fn distinct_runs(syms: &[u32], max: usize) -> Vec<&[u32]> {
344 let mut runs = Vec::new();
345 let (mut start, mut seen) = (0, std::collections::HashSet::new());
346 for (i, sym) in syms.iter().enumerate() {
347 if !seen.contains(sym) && seen.len() == max {
348 runs.push(&syms[start..i]);
349 start = i;
350 seen.clear();
351 }
352 seen.insert(*sym);
353 }
354 if start < syms.len() {
355 runs.push(&syms[start..]);
356 }
357 runs
358}
359
360struct KeyResolver<'a> {
365 backend: &'a dyn CuBackend,
366 cache: HashMap<u32, (u32, u32)>,
367}
368
369impl<'a> KeyResolver<'a> {
370 fn new(backend: &'a dyn CuBackend) -> Self {
371 Self {
372 backend,
373 cache: HashMap::new(),
374 }
375 }
376
377 fn prefetch(&mut self, keysyms: &[u32]) {
379 let missing: Vec<u32> = {
380 let mut seen = std::collections::HashSet::new();
381 keysyms
382 .iter()
383 .copied()
384 .filter(|&s| s != 0 && !self.cache.contains_key(&s) && seen.insert(s))
385 .collect()
386 };
387 if missing.is_empty() {
388 return;
389 }
390 let out = self.backend.resolve_keysyms(&missing);
391 for (sym, hit) in missing.into_iter().zip(out) {
392 self.cache.insert(sym, hit);
393 }
394 }
395
396 fn resolve(&mut self, keysym: u32) -> Option<(u32, u32)> {
399 if keysym == 0 {
400 return None;
401 }
402 self.prefetch(&[keysym]);
403 let hit = self.cache.get(&keysym).copied().unwrap_or((0, 0));
404 (hit.0 != 0).then_some(hit)
405 }
406}
407
408const KC_SHIFT: u32 = 50;
411const KC_ALTGR: u32 = 108;
412
413fn level_modifiers(level: u32, held_mask: u32, altgr: u32) -> Vec<u32> {
417 let mut out = Vec::new();
418 if level & 1 != 0 && held_mask & 1 == 0 {
419 out.push(KC_SHIFT);
420 }
421 if level & 2 != 0 && held_mask & 2 == 0 && altgr != 0 {
422 out.push(altgr);
423 }
424 out
425}
426
427#[derive(Deserialize)]
428struct CuActionRequest {
429 action: String,
430 coordinate: Option<[f64; 2]>,
431 start_coordinate: Option<[f64; 2]>,
432 text: Option<String>,
433 key: Option<String>,
434 scroll_direction: Option<String>,
435 scroll_amount: Option<i32>,
436 duration: Option<f64>,
437 region: Option<[f64; 4]>,
438 display: Option<u32>,
441}
442
443fn ok_json() -> String {
444 "{\"result\":\"ok\"}".to_string()
445}
446
447fn handle_action(req: CuActionRequest, backend: &dyn CuBackend) -> String {
448 let result = handle_action_inner(req, backend);
449 match result {
450 Ok(response) => response,
451 Err(e) => format!("{{\"error\":\"{}\"}}", e.replace('"', "\\\"")),
452 }
453}
454
455fn handle_action_inner(req: CuActionRequest, b: &dyn CuBackend) -> Result<String, String> {
466 let sleep_ms = |ms: u64| thread::sleep(Duration::from_millis(ms));
467
468 let (fb_w, fb_h) = b.fb_size()?;
469
470 let handle_coord = |coord: [f64; 2]| -> (f64, f64) {
471 (
472 clamp(coord[0], 0.0, (fb_w - 1) as f64),
473 clamp(coord[1], 0.0, (fb_h - 1) as f64),
474 )
475 };
476
477 let handle_modifier = |mod_name: &str| -> Option<u32> {
478 let sc = scancode_for_keyname(mod_name)?;
479 if is_modifier(sc) { Some(sc) } else { None }
480 };
481
482 match req.action.as_str() {
483 "screenshot" => {
484 let png = b.screenshot_png(req.display.unwrap_or(0))?;
485 let b64 = BASE64.encode(&png);
486 Ok(format!("{{\"data\":\"{}\"}}", b64))
487 }
488
489 "mouse_move" => {
490 let coord = req.coordinate.ok_or("Missing coordinate")?;
491 let (fx, fy) = handle_coord(coord);
492 b.mouse_move(fx, fy);
493 Ok(ok_json())
494 }
495
496 "left_click" | "right_click" | "middle_click" => {
497 let btn = match req.action.as_str() {
498 "left_click" => CuButton::Left,
499 "right_click" => CuButton::Right,
500 _ => CuButton::Middle,
501 };
502 if let Some(coord) = req.coordinate {
503 let (fx, fy) = handle_coord(coord);
504 b.mouse_move(fx, fy);
505 sleep_ms(30);
506 }
507 if let Some(ref mod_name) = req.text
508 && let Some(sc) = handle_modifier(mod_name)
509 {
510 b.key(sc, true);
511 sleep_ms(20);
512 }
513 b.button(btn, true);
514 sleep_ms(20);
515 b.button(btn, false);
516 if let Some(ref mod_name) = req.text
517 && let Some(sc) = handle_modifier(mod_name)
518 {
519 sleep_ms(10);
520 b.key(sc, false);
521 }
522 Ok(ok_json())
523 }
524
525 "double_click" | "triple_click" => {
526 let n = if req.action == "double_click" { 2 } else { 3 };
527 if let Some(coord) = req.coordinate {
528 let (fx, fy) = handle_coord(coord);
529 b.mouse_move(fx, fy);
530 sleep_ms(30);
531 }
532 if let Some(ref mod_name) = req.text
533 && let Some(sc) = handle_modifier(mod_name)
534 {
535 b.key(sc, true);
536 sleep_ms(20);
537 }
538 for _ in 0..n {
539 b.button(CuButton::Left, true);
540 sleep_ms(10);
541 b.button(CuButton::Left, false);
542 sleep_ms(10);
543 }
544 if let Some(ref mod_name) = req.text
545 && let Some(sc) = handle_modifier(mod_name)
546 {
547 sleep_ms(10);
548 b.key(sc, false);
549 }
550 Ok(ok_json())
551 }
552
553 "left_click_drag" => {
554 let start = req.start_coordinate.ok_or("Missing start_coordinate")?;
555 let end = req.coordinate.ok_or("Missing coordinate")?;
556 let (sx, sy) = handle_coord(start);
557 let (ex, ey) = handle_coord(end);
558 b.mouse_move(sx, sy);
559 sleep_ms(30);
560 b.button(CuButton::Left, true);
561 sleep_ms(30);
562 b.mouse_move(ex, ey);
563 sleep_ms(30);
564 b.button(CuButton::Left, false);
565 Ok(ok_json())
566 }
567
568 "left_mouse_down" => {
569 b.button(CuButton::Left, true);
570 Ok(ok_json())
571 }
572
573 "left_mouse_up" => {
574 b.button(CuButton::Left, false);
575 Ok(ok_json())
576 }
577
578 "type" => {
579 let text = req.text.as_deref().ok_or("Missing text")?;
580 if b.name() == "wayland"
585 && let Some(sock) = app_wayland_socket_path()
586 {
587 static FAILED_SOCK: Mutex<Option<String>> = Mutex::new(None);
591 match crate::wayland::vkclient::type_text_to(&sock, text) {
592 Ok(()) => {
593 *FAILED_SOCK.lock().unwrap() = None;
594 return Ok(ok_json());
595 }
596 Err(e) => {
597 let mut last = FAILED_SOCK.lock().unwrap();
598 if last.as_deref() != Some(sock.as_str()) {
599 eprintln!(
600 "[ComputerUse] app-compositor type via {sock} failed ({e}); using local seat until it is reachable"
601 );
602 *last = Some(sock);
603 }
604 }
605 }
606 }
607 let all: Vec<u32> = text.chars().map(keysym_for_char).collect();
608 for syms in distinct_runs(&all, TYPE_RUN_KEYSYMS) {
609 let mut resolver = KeyResolver::new(b);
610 resolver.prefetch(syms);
611 let mut unresolved: Vec<u32> = Vec::new();
614 for &sym in syms {
615 if sym != 0 && resolver.resolve(sym).is_none() && !unresolved.contains(&sym) {
616 unresolved.push(sym);
617 }
618 }
619 b.with_transient_keysyms(&unresolved, &mut |bound| {
620 for (i, &sym) in syms.iter().enumerate() {
621 if i > 0 && i % 50 == 0 {
622 sleep_ms(20);
623 }
624 if let Some((sc, level)) = resolver.resolve(sym) {
625 let level_mods = level_modifiers(level, 0, b.altgr_keycode());
626 for &m in &level_mods {
627 b.key(m, true);
628 sleep_ms(5);
629 }
630 b.key(sc, true);
631 sleep_ms(10);
632 b.key(sc, false);
633 for &m in level_mods.iter().rev() {
634 b.key(m, false);
635 }
636 sleep_ms(8);
637 } else if let Some(&kc) = bound.get(&sym) {
638 b.key(kc, true);
640 sleep_ms(10);
641 b.key(kc, false);
642 sleep_ms(8);
643 }
644 }
645 });
646 }
647 Ok(ok_json())
648 }
649
650 "key" => {
651 let text = req.text.as_deref().ok_or("Missing text")?;
652 let mut resolver = KeyResolver::new(b);
653 let mut mods: Vec<u32> = Vec::new();
654 let mut main_key: Option<(u32, u32)> = None;
655 let mut unresolved_sym: Option<u32> = None;
658 for part in text.split('+') {
659 let trimmed = part.trim();
660 if let Some(sc) = scancode_for_keyname(trimmed) {
661 if is_modifier(sc) {
662 mods.push(sc);
663 } else {
664 main_key = Some((sc, 0));
665 unresolved_sym = None;
666 }
667 } else {
668 let sym = keysym_for_key_spec(trimmed);
669 if let Some(hit) = resolver.resolve(sym) {
670 main_key = Some(hit);
671 unresolved_sym = None;
672 } else if sym != 0 {
673 main_key = None;
674 unresolved_sym = Some(sym);
675 }
676 }
677 }
678 let unresolved: Vec<u32> = unresolved_sym.into_iter().collect();
679 b.with_transient_keysyms(&unresolved, &mut |bound| {
680 let main_key = main_key.or_else(|| {
681 unresolved_sym
683 .and_then(|s| bound.get(&s))
684 .map(|&kc| (kc, 0))
685 });
686 for &sc in &mods {
687 b.key(sc, true);
688 sleep_ms(10);
689 }
690 if let Some((sc, level)) = main_key {
691 let altgr = b.altgr_keycode();
695 let held = mods.iter().fold(0u32, |m, &sc| match sc {
696 50 | 62 => m | 1,
697 sc if sc == KC_ALTGR || sc == altgr => m | 2,
698 _ => m,
699 });
700 let level_mods = level_modifiers(level, held, altgr);
701 for &m in &level_mods {
702 b.key(m, true);
703 sleep_ms(10);
704 }
705 b.key(sc, true);
706 sleep_ms(30);
707 b.key(sc, false);
708 for &m in level_mods.iter().rev() {
709 sleep_ms(10);
710 b.key(m, false);
711 }
712 } else if let Some(&last_mod) = mods.last() {
713 b.key(last_mod, true);
714 sleep_ms(30);
715 b.key(last_mod, false);
716 }
717 for &sc in mods.iter().rev() {
718 sleep_ms(10);
719 b.key(sc, false);
720 }
721 });
722 Ok(ok_json())
723 }
724
725 "hold_key" => {
726 let text = req.text.as_deref().ok_or("Missing text")?;
727 let duration = req.duration.ok_or("Missing duration")?;
728 let duration = duration.min(100.0);
729 let trimmed = text.trim();
730 let mut resolver = KeyResolver::new(b);
731 let hit = match scancode_for_keyname(trimmed) {
732 Some(sc) => Some((sc, 0)),
733 None => resolver.resolve(keysym_for_key_spec(trimmed)),
734 };
735 if let Some((sc, level)) = hit {
736 let level_mods = level_modifiers(level, 0, b.altgr_keycode());
737 for &m in &level_mods {
738 b.key(m, true);
739 sleep_ms(10);
740 }
741 b.key(sc, true);
742 sleep_ms((duration * 1000.0) as u64);
743 b.key(sc, false);
744 for &m in level_mods.iter().rev() {
745 sleep_ms(10);
746 b.key(m, false);
747 }
748 }
749 Ok(ok_json())
750 }
751
752 "scroll" => {
753 let dir = req
754 .scroll_direction
755 .as_deref()
756 .ok_or("Missing scroll_direction")?;
757 let amount = req.scroll_amount.unwrap_or(1).max(0) as f64;
758 if let Some(coord) = req.coordinate {
759 let (fx, fy) = handle_coord(coord);
760 b.mouse_move(fx, fy);
761 sleep_ms(30);
762 }
763 if let Some(ref mod_name) = req.text
764 && let Some(sc) = handle_modifier(mod_name)
765 {
766 b.key(sc, true);
767 sleep_ms(20);
768 }
769 let (dx, dy) = match dir {
770 "up" => (0.0, -amount),
771 "down" => (0.0, amount),
772 "left" => (-amount, 0.0),
773 "right" => (amount, 0.0),
774 _ => return Err(format!("Invalid scroll_direction: {}", dir)),
775 };
776 b.scroll(dx, dy);
777 sleep_ms(30);
778 if let Some(ref mod_name) = req.text
779 && let Some(sc) = handle_modifier(mod_name)
780 {
781 sleep_ms(10);
782 b.key(sc, false);
783 }
784 Ok(ok_json())
785 }
786
787 "cursor_position" => {
788 let (x, y) = b.cursor_pos()?;
789 Ok(format!(
790 "{{\"text\":\"X={},Y={}\"}}",
791 x.round() as i32,
792 y.round() as i32
793 ))
794 }
795
796 "wait" => {
797 let duration = req.duration.unwrap_or(1.0).min(100.0);
798 sleep_ms((duration * 1000.0) as u64);
799 Ok(ok_json())
800 }
801
802 "zoom" => {
803 let region = req.region.ok_or("Missing region")?;
804 let display = req.display.unwrap_or(0);
805 let (dw, dh) = b.display_fb_size(display)?;
808 let (x0, y0, x1, y1) = (region[0], region[1], region[2], region[3]);
809 let left = clamp(x0.round() as u32, 0, dw as u32 - 1);
810 let top = clamp(y0.round() as u32, 0, dh as u32 - 1);
811 let right = clamp(x1.round() as u32, left + 1, dw as u32);
812 let bottom = clamp(y1.round() as u32, top + 1, dh as u32);
813 let crop_w = right - left;
814 let crop_h = bottom - top;
815 let png = b.screenshot_png(display)?;
816 if crop_w > 0
817 && crop_h > 0
818 && let Ok(img) = image::load_from_memory(&png)
819 {
820 let cropped = img.crop_imm(left, top, crop_w, crop_h);
821 let mut out = Vec::new();
822 cropped
823 .write_to(&mut Cursor::new(&mut out), ImageFormat::Png)
824 .map_err(|e| format!("Crop/encode error: {}", e))?;
825 let b64 = BASE64.encode(&out);
826 return Ok(format!("{{\"data\":\"{}\"}}", b64));
827 }
828 let b64 = BASE64.encode(&png);
829 Ok(format!("{{\"data\":\"{}\"}}", b64))
830 }
831
832 _ => Err(format!("Unknown action: {}", req.action)),
833 }
834}
835
836static WAYLAND_TX: Mutex<Option<smithay::reexports::calloop::channel::Sender<ThreadCommand>>> =
837 Mutex::new(None);
838
839#[derive(Deserialize, Default)]
842struct RecordStartRequest {
843 path: Option<String>,
846 display: Option<u32>,
848 fps: Option<f64>,
850 bitrate_kbps: Option<i32>,
852 audio_socket: Option<String>,
854}
855
856fn named_record_path(dir: Option<&str>, name: &str) -> Result<String, String> {
859 let dir = dir.ok_or("a recording named by the caller needs PIXELFLUX_RECORD_DIR")?;
860 let mut parts = std::path::Path::new(name).components();
861 match (parts.next(), parts.next()) {
862 (Some(std::path::Component::Normal(file)), None) => Ok(std::path::Path::new(dir)
863 .join(file)
864 .to_string_lossy()
865 .into_owned()),
866 _ => Err(format!("'{name}' is not a file name")),
867 }
868}
869
870fn handle_record_endpoint(url: &str, body: &str) -> Option<String> {
874 let reply = match url {
875 "/record_start" => {
876 let req: RecordStartRequest = if body.trim().is_empty() {
877 RecordStartRequest::default()
878 } else {
879 match serde_json::from_str(body) {
880 Ok(r) => r,
881 Err(e) => return Some(format!("{{\"error\":\"Invalid JSON: {}\"}}", e)),
882 }
883 };
884 let record_dir = std::env::var("PIXELFLUX_RECORD_DIR")
885 .ok()
886 .filter(|d| !d.is_empty());
887 let path = match req.path {
888 Some(name) => match named_record_path(record_dir.as_deref(), &name) {
889 Ok(path) => path,
890 Err(e) => return Some(serde_json::json!({ "error": e }).to_string()),
891 },
892 None => std::env::var("PIXELFLUX_RECORD")
893 .ok()
894 .filter(|p| !p.is_empty())
895 .unwrap_or_else(|| {
896 let ts = std::time::SystemTime::now()
897 .duration_since(std::time::UNIX_EPOCH)
898 .map(|d| d.as_secs())
899 .unwrap_or(0);
900 format!(
901 "{}/pixelflux-record-{ts}.mp4",
902 record_dir.as_deref().unwrap_or("/tmp")
903 )
904 }),
905 };
906 let mut opts = crate::recorder::RecordOptions::from_env(path);
907 if let Some(d) = req.display {
908 opts.display_id = d;
909 }
910 if let Some(f) = req.fps {
911 opts.fps = f;
912 }
913 if let Some(b) = req.bitrate_kbps {
914 opts.bitrate_kbps = b;
915 }
916 if let Some(a) = req.audio_socket {
917 opts.audio_socket = a;
918 }
919 match crate::recorder::start(opts) {
920 Ok(s) => crate::recorder::status_to_json(&s).to_string(),
921 Err(e) => serde_json::json!({ "error": e }).to_string(),
922 }
923 }
924 "/record_stop" => match crate::recorder::stop() {
925 Ok(s) => {
926 let mut v = crate::recorder::status_to_json(&s);
927 v["stopped"] = serde_json::Value::Bool(true);
928 v.to_string()
929 }
930 Err(e) => serde_json::json!({ "error": e }).to_string(),
931 },
932 "/record_status" => match crate::recorder::status() {
933 Some(s) => crate::recorder::status_to_json(&s).to_string(),
934 None => serde_json::json!({ "active": false }).to_string(),
935 },
936 _ => return None,
937 };
938 Some(reply)
939}
940
941static CU_APP_WAYLAND_DISPLAY: Mutex<Option<String>> = Mutex::new(None);
946
947pub fn set_app_wayland_display(display: Option<String>) {
950 *CU_APP_WAYLAND_DISPLAY.lock().unwrap() = display.filter(|s| !s.is_empty());
951 crate::wayland::ficlient::arm(app_wayland_socket_path());
952}
953
954pub(crate) fn app_wayland_socket_path() -> Option<String> {
959 let name = CU_APP_WAYLAND_DISPLAY
960 .lock()
961 .unwrap()
962 .clone()
963 .or_else(|| std::env::var("PIXELFLUX_APP_WAYLAND_DISPLAY").ok())
964 .filter(|s| !s.is_empty())?;
965 if crate::wait_socket_name(Duration::from_millis(0)).as_deref() == Some(name.as_str()) {
966 return None;
967 }
968 crate::wayland::wlclient::socket_path(&name)
969}
970
971pub fn register_wayland_backend(tx: smithay::reexports::calloop::channel::Sender<ThreadCommand>) {
974 *WAYLAND_TX.lock().unwrap() = Some(tx);
975}
976
977pub(crate) fn wayland_command_sender()
980-> Option<smithay::reexports::calloop::channel::Sender<ThreadCommand>> {
981 WAYLAND_TX.lock().unwrap().clone()
982}
983
984pub fn spawn_cu_from_env() {
987 if let Ok(bind) = std::env::var("PIXELFLUX_CU")
988 && let Err(e) = start_cu_server(&bind, None)
989 {
990 println!("[ComputerUse] Not started: {e}");
991 }
992}
993
994static CU_TOKEN: OnceLock<String> = OnceLock::new();
996
997pub fn start_cu_server(bind: &str, token: Option<&str>) -> Result<(), String> {
1006 static SPAWNED: OnceLock<()> = OnceLock::new();
1007 if SPAWNED.get().is_some() {
1008 return Ok(());
1009 }
1010 let token = token
1011 .map(str::to_string)
1012 .or_else(|| std::env::var("PIXELFLUX_CU_TOKEN").ok())
1013 .filter(|t| !t.is_empty())
1014 .ok_or("no token: set PIXELFLUX_CU_TOKEN, the bearer token every request has to carry")?;
1015 let listeners = cu_listeners(bind)?;
1016 if SPAWNED.set(()).is_err() {
1017 return Ok(());
1018 }
1019 let _ = CU_TOKEN.set(token);
1020 for listener in listeners {
1021 thread::spawn(move || run_cu_server(listener));
1022 }
1023 Ok(())
1024}
1025
1026fn authorized(request: &tiny_http::Request) -> bool {
1028 let Some(token) = CU_TOKEN.get() else {
1029 return false;
1030 };
1031 request.headers().iter().any(|h| {
1032 h.field.equiv("Authorization")
1033 && h.value
1034 .as_str()
1035 .strip_prefix("Bearer ")
1036 .is_some_and(|given| {
1037 given.len() == token.len()
1038 && given
1039 .bytes()
1040 .zip(token.bytes())
1041 .fold(0u8, |acc, (a, b)| acc | (a ^ b))
1042 == 0
1043 })
1044 })
1045}
1046
1047fn cu_listeners(bind: &str) -> Result<Vec<TcpListener>, String> {
1053 let mut addrs: Vec<SocketAddr> = Vec::new();
1054 for entry in bind
1055 .split(',')
1056 .map(str::trim)
1057 .filter(|entry| !entry.is_empty())
1058 {
1059 let resolved: Vec<SocketAddr> = match entry.parse::<u16>() {
1060 Ok(port) => vec![
1061 SocketAddr::from((Ipv4Addr::LOCALHOST, port)),
1062 SocketAddr::from((Ipv6Addr::LOCALHOST, port)),
1063 ],
1064 Err(_) if entry.contains(':') => entry
1065 .to_socket_addrs()
1066 .map_err(|e| format!("cannot resolve '{entry}': {e}"))?
1067 .collect(),
1068 Err(_) => {
1069 return Err(format!(
1070 "invalid bind '{entry}': expected a port or host:port"
1071 ));
1072 }
1073 };
1074 for addr in resolved {
1075 if !addrs.contains(&addr) {
1076 addrs.push(addr);
1077 }
1078 }
1079 }
1080 if addrs.is_empty() {
1081 return Err(format!(
1082 "invalid bind '{bind}': expected a port or host:port"
1083 ));
1084 }
1085 let mut listeners = Vec::new();
1086 let mut skipped = Vec::new();
1087 for addr in addrs {
1088 match bind_listener(addr) {
1089 Ok(listener) => listeners.push(listener),
1090 Err(e)
1091 if matches!(
1092 e.raw_os_error(),
1093 Some(libc::EADDRNOTAVAIL | libc::EAFNOSUPPORT)
1094 ) =>
1095 {
1096 skipped.push(format!("{addr} ({e})"));
1097 }
1098 Err(e) => return Err(format!("cannot bind {addr}: {e}")),
1099 }
1100 }
1101 if listeners.is_empty() {
1102 return Err(format!(
1103 "no address of '{bind}' is available: {}",
1104 skipped.join(", ")
1105 ));
1106 }
1107 for note in skipped {
1108 println!("[ComputerUse] Not listening on {note}");
1109 }
1110 Ok(listeners)
1111}
1112
1113fn bind_listener(addr: SocketAddr) -> std::io::Result<TcpListener> {
1117 use std::os::fd::{FromRawFd, IntoRawFd, OwnedFd};
1118
1119 let family = if addr.is_ipv4() {
1120 libc::AF_INET
1121 } else {
1122 libc::AF_INET6
1123 };
1124 let fd = unsafe { libc::socket(family, libc::SOCK_STREAM | libc::SOCK_CLOEXEC, 0) };
1125 if fd < 0 {
1126 return Err(std::io::Error::last_os_error());
1127 }
1128 let sock = unsafe { OwnedFd::from_raw_fd(fd) };
1129 let one: libc::c_int = 1;
1130 let enable = |level: libc::c_int, name: libc::c_int| -> std::io::Result<()> {
1131 let rc = unsafe {
1132 libc::setsockopt(
1133 fd,
1134 level,
1135 name,
1136 &one as *const libc::c_int as *const libc::c_void,
1137 std::mem::size_of::<libc::c_int>() as libc::socklen_t,
1138 )
1139 };
1140 if rc < 0 {
1141 Err(std::io::Error::last_os_error())
1142 } else {
1143 Ok(())
1144 }
1145 };
1146 enable(libc::SOL_SOCKET, libc::SO_REUSEADDR)?;
1147 if addr.is_ipv6() {
1148 enable(libc::IPPROTO_IPV6, libc::IPV6_V6ONLY)?;
1149 }
1150 let rc = match addr {
1151 SocketAddr::V4(v4) => {
1152 let mut sin: libc::sockaddr_in = unsafe { std::mem::zeroed() };
1153 sin.sin_family = libc::AF_INET as libc::sa_family_t;
1154 sin.sin_port = v4.port().to_be();
1155 sin.sin_addr = libc::in_addr {
1156 s_addr: u32::from_ne_bytes(v4.ip().octets()),
1157 };
1158 unsafe {
1159 libc::bind(
1160 fd,
1161 &sin as *const libc::sockaddr_in as *const libc::sockaddr,
1162 std::mem::size_of::<libc::sockaddr_in>() as libc::socklen_t,
1163 )
1164 }
1165 }
1166 SocketAddr::V6(v6) => {
1167 let mut sin6: libc::sockaddr_in6 = unsafe { std::mem::zeroed() };
1168 sin6.sin6_family = libc::AF_INET6 as libc::sa_family_t;
1169 sin6.sin6_port = v6.port().to_be();
1170 sin6.sin6_flowinfo = v6.flowinfo();
1171 sin6.sin6_addr = libc::in6_addr {
1172 s6_addr: v6.ip().octets(),
1173 };
1174 sin6.sin6_scope_id = v6.scope_id();
1175 unsafe {
1176 libc::bind(
1177 fd,
1178 &sin6 as *const libc::sockaddr_in6 as *const libc::sockaddr,
1179 std::mem::size_of::<libc::sockaddr_in6>() as libc::socklen_t,
1180 )
1181 }
1182 }
1183 };
1184 if rc < 0 {
1185 return Err(std::io::Error::last_os_error());
1186 }
1187 if unsafe { libc::listen(fd, 128) } < 0 {
1188 return Err(std::io::Error::last_os_error());
1189 }
1190 Ok(unsafe { TcpListener::from_raw_fd(sock.into_raw_fd()) })
1191}
1192
1193pub(crate) fn resolve_backend() -> Result<Box<dyn CuBackend>, String> {
1197 if let Some(tx) = WAYLAND_TX.lock().unwrap().clone() {
1198 return Ok(Box::new(CuWaylandBackend { tx }));
1199 }
1200 crate::x11::computer_use::CuX11Backend::connect().map(|be| Box::new(be) as Box<dyn CuBackend>)
1201}
1202
1203pub fn run_cu_server(listener: TcpListener) {
1213 let addr = listener
1214 .local_addr()
1215 .map(|a| a.to_string())
1216 .unwrap_or_default();
1217 let server = match tiny_http::Server::from_listener(listener, None) {
1218 Ok(s) => s,
1219 Err(e) => {
1220 eprintln!("[ComputerUse] Failed to start server on {addr}: {e}");
1221 return;
1222 }
1223 };
1224 println!("[ComputerUse] Server listening on {addr}");
1225
1226 let mut last_backend = "";
1227 for mut request in server.incoming_requests() {
1228 if !authorized(&request) {
1229 let _ = request.respond(
1230 tiny_http::Response::from_string("{\"error\":\"unauthorized\"}".to_string())
1231 .with_status_code(401)
1232 .with_header(
1233 "WWW-Authenticate: Bearer"
1234 .parse::<tiny_http::Header>()
1235 .unwrap(),
1236 )
1237 .with_header(
1238 "Content-Type: application/json"
1239 .parse::<tiny_http::Header>()
1240 .unwrap(),
1241 ),
1242 );
1243 continue;
1244 }
1245 const MAX_CU_BODY: u64 = 4 * 1024 * 1024;
1248 let mut body = String::new();
1249 if let Err(e) = request
1250 .as_reader()
1251 .take(MAX_CU_BODY + 1)
1252 .read_to_string(&mut body)
1253 {
1254 let _ = request.respond(
1255 tiny_http::Response::from_string(format!("{{\"error\":\"{}\"}}", e))
1256 .with_status_code(400)
1257 .with_header(
1258 "Content-Type: application/json"
1259 .parse::<tiny_http::Header>()
1260 .unwrap(),
1261 ),
1262 );
1263 continue;
1264 }
1265
1266 if body.len() as u64 > MAX_CU_BODY {
1267 let resp = tiny_http::Response::from_string(
1268 "{\"error\":\"request body too large\"}".to_string(),
1269 )
1270 .with_status_code(413)
1271 .with_header(
1272 "Content-Type: application/json"
1273 .parse::<tiny_http::Header>()
1274 .unwrap(),
1275 );
1276 let _ = request.respond(resp);
1277 continue;
1278 }
1279
1280 if let Some(json) = handle_record_endpoint(request.url(), &body) {
1281 let _ = request.respond(
1282 tiny_http::Response::from_string(json)
1283 .with_status_code(200)
1284 .with_header(
1285 "Content-Type: application/json"
1286 .parse::<tiny_http::Header>()
1287 .unwrap(),
1288 ),
1289 );
1290 continue;
1291 }
1292
1293 let parsed: CuActionRequest = match serde_json::from_str(&body) {
1294 Ok(r) => r,
1295 Err(e) => {
1296 let _ = request.respond(
1297 tiny_http::Response::from_string(format!(
1298 "{{\"error\":\"Invalid JSON: {}\"}}",
1299 e
1300 ))
1301 .with_status_code(400)
1302 .with_header(
1303 "Content-Type: application/json"
1304 .parse::<tiny_http::Header>()
1305 .unwrap(),
1306 ),
1307 );
1308 continue;
1309 }
1310 };
1311
1312 let json_response = match resolve_backend() {
1313 Ok(backend) => {
1314 if backend.name() != last_backend {
1315 last_backend = backend.name();
1316 println!("[ComputerUse] Using {} backend", last_backend);
1317 }
1318 handle_action(parsed, backend.as_ref())
1319 }
1320 Err(e) => format!("{{\"error\":\"{}\"}}", e.replace('"', "\\\"")),
1321 };
1322 let _ = request.respond(
1323 tiny_http::Response::from_string(json_response)
1324 .with_status_code(200)
1325 .with_header(
1326 "Content-Type: application/json"
1327 .parse::<tiny_http::Header>()
1328 .unwrap(),
1329 ),
1330 );
1331 }
1332}
1333
1334#[cfg(test)]
1335mod run_tests {
1336 use super::distinct_runs;
1337
1338 #[test]
1341 fn a_text_is_typed_in_runs_of_few_distinct_characters() {
1342 let syms = [1, 2, 1, 3, 4, 4, 5, 1, 6];
1343 assert_eq!(
1344 distinct_runs(&syms, 3),
1345 vec![&[1, 2, 1, 3][..], &[4, 4, 5, 1][..], &[6][..]]
1346 );
1347 assert_eq!(distinct_runs(&syms, 9), vec![&syms[..]]);
1348 assert!(distinct_runs(&[], 3).is_empty());
1349 }
1350}
1351
1352#[cfg(test)]
1353mod record_path_tests {
1354 use super::named_record_path;
1355
1356 #[test]
1357 fn a_named_recording_stays_in_the_operators_directory() {
1358 assert_eq!(
1359 named_record_path(Some("/rec"), "take.mp4").unwrap(),
1360 "/rec/take.mp4"
1361 );
1362 for name in ["/etc/passwd", "../x.mp4", "a/b.mp4", "..", ".", ""] {
1363 assert!(named_record_path(Some("/rec"), name).is_err(), "{name:?}");
1364 }
1365 assert!(
1366 named_record_path(None, "take.mp4").is_err(),
1367 "no directory, no named file"
1368 );
1369 }
1370}
1371
1372#[cfg(test)]
1373mod tests {
1374 use super::cu_listeners;
1375 use std::net::{IpAddr, Ipv4Addr, TcpListener};
1376
1377 #[test]
1378 fn bare_port_binds_loopback_only() {
1379 let listeners = cu_listeners("0").expect("loopback binds");
1380 assert!(!listeners.is_empty());
1381 for listener in &listeners {
1382 let addr = listener.local_addr().unwrap();
1383 assert!(addr.ip().is_loopback(), "{addr}");
1384 assert_ne!(addr.port(), 0);
1385 }
1386 assert!(
1387 listeners
1388 .iter()
1389 .any(|l| l.local_addr().unwrap().ip() == IpAddr::V4(Ipv4Addr::LOCALHOST))
1390 );
1391 }
1392
1393 #[test]
1394 fn host_port_names_the_address() {
1395 let scoped = cu_listeners("127.0.0.1:0").unwrap();
1396 assert_eq!(scoped.len(), 1);
1397 assert_eq!(
1398 scoped[0].local_addr().unwrap().ip(),
1399 IpAddr::V4(Ipv4Addr::LOCALHOST)
1400 );
1401 let open = cu_listeners("0.0.0.0:0").unwrap();
1402 assert_eq!(open.len(), 1);
1403 assert!(open[0].local_addr().unwrap().ip().is_unspecified());
1404 }
1405
1406 #[test]
1407 fn a_taken_port_is_an_error() {
1408 let taken = TcpListener::bind("127.0.0.1:0").unwrap();
1409 let port = taken.local_addr().unwrap().port();
1410 assert!(cu_listeners(&port.to_string()).is_err());
1411 assert!(cu_listeners(&format!("127.0.0.1:{port}")).is_err());
1412 }
1413
1414 #[test]
1415 fn malformed_binds_are_errors() {
1416 assert!(cu_listeners("abc").is_err());
1417 assert!(cu_listeners("70000").is_err());
1418 assert!(cu_listeners("").is_err());
1419 assert!(cu_listeners(",").is_err());
1420 assert!(cu_listeners("127.0.0.1:0,abc").is_err());
1421 }
1422
1423 fn free_port() -> u16 {
1424 TcpListener::bind("127.0.0.1:0")
1425 .unwrap()
1426 .local_addr()
1427 .unwrap()
1428 .port()
1429 }
1430
1431 fn v6only(listener: &TcpListener) -> libc::c_int {
1432 use std::os::fd::AsRawFd;
1433 let mut value: libc::c_int = -1;
1434 let mut len = std::mem::size_of::<libc::c_int>() as libc::socklen_t;
1435 let rc = unsafe {
1436 libc::getsockopt(
1437 listener.as_raw_fd(),
1438 libc::IPPROTO_IPV6,
1439 libc::IPV6_V6ONLY,
1440 &mut value as *mut libc::c_int as *mut libc::c_void,
1441 &mut len,
1442 )
1443 };
1444 assert_eq!(rc, 0);
1445 value
1446 }
1447
1448 #[test]
1449 fn both_wildcards_share_a_port() {
1450 let port = free_port();
1451 let listeners = cu_listeners(&format!("0.0.0.0:{port}, [::]:{port}")).unwrap();
1452 assert!(
1453 listeners
1454 .iter()
1455 .all(|l| l.local_addr().unwrap().port() == port)
1456 );
1457 assert!(
1458 listeners
1459 .iter()
1460 .all(|l| l.local_addr().unwrap().ip().is_unspecified())
1461 );
1462 if let Some(v6) = listeners.iter().find(|l| l.local_addr().unwrap().is_ipv6()) {
1463 assert_eq!(listeners.len(), 2);
1464 assert_eq!(v6only(v6), 1);
1465 }
1466 }
1467
1468 #[test]
1469 fn repeated_entries_bind_once() {
1470 let port = free_port();
1471 let listeners = cu_listeners(&format!("{port},127.0.0.1:{port},{port}")).unwrap();
1472 assert!(listeners.len() <= 2);
1473 assert!(
1474 listeners
1475 .iter()
1476 .all(|l| l.local_addr().unwrap().ip().is_loopback())
1477 );
1478 }
1479}