From 63d4e644fafa694e9085cba81b41fe56bc78a761 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 21 Jun 2026 14:28:12 +0000 Subject: [PATCH] feat(os): Windows UI Automation control + state reads Port internal/adapter/driven/uia into hyperdeck-os behind cfg(windows), using the windows crate's typed COM interfaces instead of raw vtable dispatch: - uia::Engine owns an MTA COM apartment + IUIAutomation on a dedicated worker thread, serving requests serially over a channel (COM thread affinity), so it is Send + Sync for use as a shared adapter - implements PlayerController (Invoke the AutomationId-mapped control for control: uia profiles) and ElementNamer (read an element's Name for the UIA state probe); missing controls are an acked no-op / not-detected - ElementFromHandle + CreatePropertyCondition(AutomationId) + FindFirst (descendants) + GetCurrentPatternAs(Invoke) / CurrentName Adds windows features Win32_UI_Accessibility, Win32_System_Com, Win32_System_Variant. Verified with cargo check/clippy --target x86_64-pc-windows-gnu (real COM) + apple-darwin/linux; all clippy-clean. With this, every Go driven adapter is ported to Rust. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01R8X1NxFiaLGJzwUD5wgRTv --- crates/hyperdeck-os/Cargo.toml | 3 + crates/hyperdeck-os/src/lib.rs | 2 + crates/hyperdeck-os/src/uia.rs | 165 +++++++++++++++++++++++++++++++++ 3 files changed, 170 insertions(+) create mode 100644 crates/hyperdeck-os/src/uia.rs diff --git a/crates/hyperdeck-os/Cargo.toml b/crates/hyperdeck-os/Cargo.toml index f8d6960..112f5a3 100644 --- a/crates/hyperdeck-os/Cargo.toml +++ b/crates/hyperdeck-os/Cargo.toml @@ -19,7 +19,10 @@ windows = { version = "0.58", features = [ "Win32_Foundation", "Win32_UI_Input_KeyboardAndMouse", "Win32_UI_WindowsAndMessaging", + "Win32_UI_Accessibility", "Win32_System_Threading", + "Win32_System_Com", + "Win32_System_Variant", "Win32_System_Diagnostics_ToolHelp", ] } diff --git a/crates/hyperdeck-os/src/lib.rs b/crates/hyperdeck-os/src/lib.rs index 122a2a6..fec816b 100644 --- a/crates/hyperdeck-os/src/lib.rs +++ b/crates/hyperdeck-os/src/lib.rs @@ -6,4 +6,6 @@ //! keystroke injection, window enumeration, and Windows UI Automation. pub mod injector; +#[cfg(windows)] +pub mod uia; pub mod vlchttp; diff --git a/crates/hyperdeck-os/src/uia.rs b/crates/hyperdeck-os/src/uia.rs new file mode 100644 index 0000000..e101001 --- /dev/null +++ b/crates/hyperdeck-os/src/uia.rs @@ -0,0 +1,165 @@ +//! UI Automation control + state reads (port of `internal/adapter/driven/uia`). +//! +//! Backend for `control: uia` profiles (UWP/Store apps like Example Player whose +//! transport can't be driven by background keystrokes): every XAML control is a +//! UIA element invoked by AutomationId. The same engine also reads an element's +//! Name for the UIA state probe. +//! +//! All COM work runs on one dedicated thread that owns the MTA apartment and the +//! `IUIAutomation` instance (COM apartment rules require thread affinity); +//! requests are serviced serially over a channel. + +use std::ffi::c_void; +use std::sync::mpsc::{channel, Receiver, Sender}; +use std::thread; + +use windows::core::{BSTR, VARIANT}; +use windows::Win32::Foundation::HWND; +use windows::Win32::System::Com::{ + CoCreateInstance, CoInitializeEx, CLSCTX_INPROC_SERVER, COINIT_MULTITHREADED, +}; +use windows::Win32::UI::Accessibility::{ + CUIAutomation, IUIAutomation, IUIAutomationElement, IUIAutomationInvokePattern, + TreeScope_Descendants, UIA_AutomationIdPropertyId, UIA_InvokePatternId, +}; + +use hyperdeck_core::domain::{KeyName, Profile, Window}; +use hyperdeck_core::error::{DeckError, DeckResult}; +use hyperdeck_core::port::PlayerController; +use hyperdeck_core::stateprobe::ElementNamer; + +enum Op { + Invoke, + Name, +} + +struct Req { + op: Op, + hwnd: usize, + aid: String, + res: Sender>, +} + +/// Drives and reads a player's transport via UI Automation. Owns a COM apartment +/// on a single worker thread; construct with [`Engine::new`]. +pub struct Engine { + reqs: Sender, +} + +impl Engine { + /// Starts the COM worker thread and returns an engine. + pub fn new() -> Self { + let (tx, rx) = channel::(); + thread::spawn(move || worker(rx)); + Engine { reqs: tx } + } + + fn call(&self, op: Op, hwnd: usize, aid: &str) -> DeckResult { + let (rtx, rrx) = channel(); + self.reqs + .send(Req { + op, + hwnd, + aid: aid.to_string(), + res: rtx, + }) + .map_err(|_| DeckError::Other("uia: worker stopped".into()))?; + rrx.recv() + .map_err(|_| DeckError::Other("uia: no response".into()))? + } +} + +impl Default for Engine { + fn default() -> Self { + Self::new() + } +} + +fn worker(rx: Receiver) { + // SAFETY: standard COM init on this dedicated thread. + unsafe { + let _ = CoInitializeEx(None, COINIT_MULTITHREADED); + } + let automation: Option = + unsafe { CoCreateInstance(&CUIAutomation, None, CLSCTX_INPROC_SERVER).ok() }; + let Some(automation) = automation else { + for req in rx { + let _ = req + .res + .send(Err(DeckError::Other("uia: automation unavailable".into()))); + } + return; + }; + for req in rx { + let r = match req.op { + Op::Invoke => invoke(&automation, req.hwnd, &req.aid).map(|_| String::new()), + Op::Name => get_name(&automation, req.hwnd, &req.aid), + }; + let _ = req.res.send(r); + } +} + +/// Returns the first descendant of `hwnd`'s element with the given AutomationId, +/// or `None` when no such element exists right now (e.g. no clip open). +fn find_element( + automation: &IUIAutomation, + hwnd: usize, + aid: &str, +) -> DeckResult> { + if hwnd == 0 { + return Err(DeckError::Other("uia: nil window handle".into())); + } + let h = HWND(hwnd as *mut c_void); + let win_el = unsafe { automation.ElementFromHandle(h) } + .map_err(|e| DeckError::Other(format!("uia: ElementFromHandle: {e}")))?; + let cond = unsafe { + automation + .CreatePropertyCondition(UIA_AutomationIdPropertyId, &VARIANT::from(BSTR::from(aid))) + } + .map_err(|e| DeckError::Other(format!("uia: CreatePropertyCondition: {e}")))?; + // FindFirst returns an error / null when nothing matches; treat as not found. + match unsafe { win_el.FindFirst(TreeScope_Descendants, &cond) } { + Ok(el) => Ok(Some(el)), + Err(_) => Ok(None), + } +} + +fn invoke(automation: &IUIAutomation, hwnd: usize, aid: &str) -> DeckResult<()> { + let Some(el) = find_element(automation, hwnd, aid)? else { + return Ok(()); // control not present: acked no-op + }; + let pattern: IUIAutomationInvokePattern = + unsafe { el.GetCurrentPatternAs(UIA_InvokePatternId) } + .map_err(|e| DeckError::Other(format!("uia: {aid:?} has no Invoke pattern: {e}")))?; + unsafe { pattern.Invoke() } + .map_err(|e| DeckError::Other(format!("uia: Invoke({aid:?}): {e}")))?; + Ok(()) +} + +fn get_name(automation: &IUIAutomation, hwnd: usize, aid: &str) -> DeckResult { + let Some(el) = find_element(automation, hwnd, aid)? else { + return Ok(String::new()); // not present: not detectable + }; + match unsafe { el.CurrentName() } { + Ok(bstr) => Ok(bstr.to_string()), + Err(_) => Ok(String::new()), + } +} + +impl PlayerController for Engine { + fn control(&self, p: &Profile, w: &Window, key: KeyName) -> DeckResult<()> { + let Some(aid) = p.uia.get(&key) else { + return Ok(()); // no element mapped: acked no-op + }; + if aid.is_empty() { + return Ok(()); + } + self.call(Op::Invoke, w.handle, aid).map(|_| ()) + } +} + +impl ElementNamer for Engine { + fn name(&self, hwnd: usize, automation_id: &str) -> DeckResult { + self.call(Op::Name, hwnd, automation_id) + } +}