input_synthesis/
lib.rs

1// Copyright 2019 The Fuchsia Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5use crate::synthesizer::*;
6use anyhow::{format_err, Error};
7use fidl_fuchsia_io as fio;
8use fidl_fuchsia_ui_input::{KeyboardReport, Touch};
9use fuchsia_component::client::{new_protocol_connector, new_protocol_connector_in_dir};
10use keymaps::inverse_keymap::{InverseKeymap, Shift};
11use keymaps::usages::{self, Usages};
12use std::time::Duration;
13
14pub mod modern_backend;
15pub mod synthesizer;
16
17/// Simulates a media button event.
18pub async fn media_button_event_command(
19    volume_up: bool,
20    volume_down: bool,
21    mic_mute: bool,
22    reset: bool,
23    pause: bool,
24    camera_disable: bool,
25) -> Result<(), Error> {
26    let mut pressed_buttons = vec![];
27    if volume_up {
28        pressed_buttons.push(synthesizer::MediaButton::VolumeUp)
29    };
30    if volume_down {
31        pressed_buttons.push(synthesizer::MediaButton::VolumeDown)
32    };
33    if mic_mute {
34        pressed_buttons.push(synthesizer::MediaButton::MicMute)
35    };
36    if reset {
37        pressed_buttons.push(synthesizer::MediaButton::FactoryReset)
38    };
39    if pause {
40        pressed_buttons.push(synthesizer::MediaButton::Pause)
41    };
42    if camera_disable {
43        pressed_buttons.push(synthesizer::MediaButton::CameraDisable)
44    };
45    media_button_event(pressed_buttons, get_backend().await?.as_mut()).await
46}
47
48/// Simulates a key press of specified `usage`.
49///
50/// `key_event_duration` is the time spent between key-press and key-release events.
51///
52/// # Resolves to
53/// * `Ok(())` if the events were successfully injected.
54/// * `Err(Error)` otherwise.
55///
56/// # Corner case handling
57/// * `key_event_duration` of zero is permitted, and will result in events being generated as
58///    quickly as possible.
59///
60/// # Future directions
61/// Per https://fxbug.dev/42142047, this method will be replaced with a method that deals in
62/// `fuchsia.input.Key`s, instead of HID Usage IDs.
63pub async fn keyboard_event_command(usage: u32, key_event_duration: Duration) -> Result<(), Error> {
64    keyboard_event(usage, key_event_duration, get_backend().await?.as_mut()).await
65}
66
67/// Simulates `input` being typed on a keyboard, with `key_event_duration` between key events.
68///
69/// # Requirements
70/// * `input` must be non-empty
71/// * `input` must only contain characters representable using the current keyboard layout
72///    and locale. (At present, it is assumed that the current layout and locale are
73///   `US-QWERTY` and `en-US`, respectively.)
74///
75/// # Resolves to
76/// * `Ok(())` if the arguments met the requirements above, and the events were successfully
77///   injected.
78/// * `Err(Error)` otherwise.
79///
80/// # Corner case handling
81/// * `key_event_duration` of zero is permitted, and will result in events being generated as
82///    quickly as possible.
83pub async fn text_command(input: String, key_event_duration: Duration) -> Result<(), Error> {
84    text(input, key_event_duration, get_backend().await?.as_mut()).await
85}
86
87/// Simulates a sequence of key events (presses and releases) on a keyboard.
88///
89/// Dispatches the supplied `events` into a keyboard device, honoring the timing sequence that is
90/// requested in them, to the extent possible using the current scheduling system.
91///
92/// Since each individual key press is broken down into constituent pieces (presses, releases,
93/// pauses), it is possible to dispatch a key event sequence corresponding to multiple keys being
94/// pressed and released in an arbitrary sequence.  This sequence is usually understood as a timing
95/// diagram like this:
96///
97/// ```ignore
98///           v--- key press   v--- key release
99/// A: _______/^^^^^^^^^^^^^^^^\__________
100///    |<----->|   <-- duration from start for key press.
101///    |<--------------------->|   <-- duration from start for key release.
102///
103/// B: ____________/^^^^^^^^^^^^^^^^\_____
104///                ^--- key press   ^--- key release
105///    |<--------->|   <-- duration from start for key press.
106///    |<-------------------------->|   <-- duration for key release.
107/// ```
108///
109/// You would from there convert the desired timing diagram into a sequence of [TimedKeyEvent]s
110/// that you would pass into this function. Note that all durations are specified as durations
111/// from the start of the key event sequence.
112///
113/// Note that due to the way event timing works, it is in practice impossible to have two key
114/// events happen at exactly the same time even if you so specify.  Do not rely on simultaneous
115/// asynchronous event processing: it does not work in this code, and it is not how reality works
116/// either.  Instead, design your key event processing so that it is robust against the inherent
117/// non-determinism in key event delivery.
118pub async fn dispatch_key_events(events: &[TimedKeyEvent]) -> Result<(), Error> {
119    dispatch_key_events_async(events, get_backend().await?.as_mut()).await
120}
121
122/// Simulates `tap_event_count` taps at coordinates `(x, y)` for a touchscreen with horizontal
123/// resolution `width` and vertical resolution `height`. `(x, y)` _should_ be specified in absolute
124/// coordinations, with `x` normally in the range (0, `width`), `y` normally in the range
125/// (0, `height`).
126///
127/// `duration` is divided equally between touch-down and touch-up event pairs, while the
128/// transition between these pairs is immediate.
129pub async fn tap_event_command(
130    x: u32,
131    y: u32,
132    width: u32,
133    height: u32,
134    tap_event_count: usize,
135    duration: Duration,
136) -> Result<(), Error> {
137    tap_event(x, y, width, height, tap_event_count, duration, get_backend().await?.as_mut()).await
138}
139
140/// Simulates `tap_event_count` times to repeat the multi-finger-taps, for touchscreen with
141/// horizontal resolution `width` and vertical resolution `height`. Finger positions _should_
142/// be specified in absolute coordinations, with `x` values normally in the range (0, `width`),
143/// and `y` values normally in the range (0, `height`).
144///
145/// `duration` is divided equally between multi-touch-down and multi-touch-up
146/// pairs, while the transition between these is immediate.
147pub async fn multi_finger_tap_event_command(
148    fingers: Vec<Touch>,
149    width: u32,
150    height: u32,
151    tap_event_count: usize,
152    duration: Duration,
153) -> Result<(), Error> {
154    multi_finger_tap_event(
155        fingers,
156        width,
157        height,
158        tap_event_count,
159        duration,
160        get_backend().await?.as_mut(),
161    )
162    .await
163}
164
165/// Simulates swipe from coordinates `(x0, y0)` to `(x1, y1)` for a touchscreen with
166/// horizontal resolution `width` and vertical resolution `height`, with `move_event_count`
167/// touch-move events in between. Positions for move events are linearly interpolated.
168///
169/// Finger positions _should_ be specified in absolute coordinations, with `x` values normally in the
170/// range (0, `width`), and `y` values normally in the range (0, `height`).
171///
172/// `duration` is the total time from the touch-down event to the touch-up event, inclusive
173/// of all move events in between.
174pub async fn swipe_command(
175    x0: u32,
176    y0: u32,
177    x1: u32,
178    y1: u32,
179    width: u32,
180    height: u32,
181    move_event_count: usize,
182    duration: Duration,
183) -> Result<(), Error> {
184    swipe(x0, y0, x1, y1, width, height, move_event_count, duration, get_backend().await?.as_mut())
185        .await
186}
187
188/// Simulates swipe with fingers starting at `start_fingers`, and moving to `end_fingers`,
189/// for a touchscreen for a touchscreen with horizontal resolution `width` and vertical resolution
190/// `height`. Finger positions _should_ be specified in absolute coordinations, with `x` values
191/// normally in the range (0, `width`), and `y` values normally in the range (0, `height`).
192///
193/// Linearly interpolates `move_event_count` touch-move events between the start positions
194/// and end positions, over `duration` time. (`duration` is the total time from the touch-down
195/// event to the touch-up event, inclusive of all move events in between.)
196///
197/// # Requirements
198/// * `start_fingers` and `end_fingers` must have the same length
199/// * `start_fingers.len()` and `end_finger.len()` must be representable within a `u32`
200///
201/// # Resolves to
202/// * `Ok(())` if the arguments met the requirements above, and the events were successfully
203///   injected.
204/// * `Err(Error)` otherwise.
205///
206/// # Corner case handling
207/// * `move_event_count` of zero is permitted, and will result in just the DOWN and UP events
208///   being generated.
209/// * `duration.as_nanos() < move_event_count` is allowed, and will result in all events having
210///   the same timestamp.
211/// * `width` and `height` are permitted to be zero; such values are left to the interpretation
212///   of the system under test.
213/// * finger positions _may_ exceed the expected bounds; such values are left to the interpretation
214///   of the sytem under test.
215pub async fn multi_finger_swipe_command(
216    start_fingers: Vec<(u32, u32)>,
217    end_fingers: Vec<(u32, u32)>,
218    width: u32,
219    height: u32,
220    move_event_count: usize,
221    duration: Duration,
222) -> Result<(), Error> {
223    multi_finger_swipe(
224        start_fingers,
225        end_fingers,
226        width,
227        height,
228        move_event_count,
229        duration,
230        get_backend().await?.as_mut(),
231    )
232    .await
233}
234
235/// Add a synthesis mouse device.
236pub async fn add_mouse_device_command(
237    width: u32,
238    height: u32,
239) -> Result<Box<dyn InputDevice>, Error> {
240    add_mouse_device(width, height, get_backend().await?.as_mut()).await
241}
242
243/// Selects an injection protocol, and returns the corresponding implementation
244/// of `synthesizer::InputDeviceRegistry`.
245///
246/// # Returns
247/// * Ok(`modern_backend::InputDeviceRegistry`) if
248///   `fuchsia.input.injection.InputDeviceRegistry` is available.
249/// * Err otherwise. E.g.,
250///   * Protocol was not available.
251///   * Access to `/svc` was denied.
252async fn get_backend() -> Result<Box<dyn InputDeviceRegistry>, Error> {
253    let registry =
254        new_protocol_connector::<fidl_fuchsia_input_injection::InputDeviceRegistryMarker>()?;
255    if registry.exists().await? {
256        return Ok(Box::new(modern_backend::InputDeviceRegistry::new(registry.connect()?)));
257    }
258
259    Err(format_err!("fuchsia.input.injection.InputDeviceRegistry not available"))
260}
261
262/// Similar to [get_backend] but looks for the input injection backend inside the directory `dir`.
263///
264/// Useful for tests that prefer to connect to FIDL inside a specific test realm, rather than
265/// the default `/svc`.
266///
267/// # Returns
268///
269/// See [get_backend] for the discussion of the returned values.
270pub async fn get_modern_backend_at(
271    dir: &fio::DirectoryProxy,
272) -> Result<Box<dyn InputDeviceRegistry>, Error> {
273    let modern_registry = new_protocol_connector_in_dir::<
274        fidl_fuchsia_input_injection::InputDeviceRegistryMarker,
275    >(dir);
276    if modern_registry.exists().await? {
277        return Ok(Box::new(modern_backend::InputDeviceRegistry::new(modern_registry.connect()?)));
278    }
279
280    Err(format_err!("no available InputDeviceRegistry in the provided directory"))
281}
282
283/// Converts the `input` string into a key sequence under the `InverseKeymap` derived from `keymap`.
284///
285/// This is intended for end-to-end and input testing only; for production use cases and general
286/// testing, IME injection should be used instead.
287///
288/// A translation from `input` to a sequence of keystrokes is not guaranteed to exist. If a
289/// translation does not exist, `None` is returned.
290///
291/// The sequence does not contain pauses except between repeated keys or to clear a shift state,
292/// though the sequence does terminate with an empty report (no keys pressed). A shift key
293/// transition is sent in advance of each series of keys that needs it.
294///
295/// Note that there is currently no way to distinguish between particular key releases. As such,
296/// only one key release report is generated even in combinations, e.g. Shift + A.
297///
298/// # Example
299///
300/// ```
301/// let key_sequence = derive_key_sequence(&keymaps::US_QWERTY, "A").unwrap();
302///
303/// // [shift, A, clear]
304/// assert_eq!(key_sequence.len(), 3);
305/// ```
306fn derive_key_sequence(keymap: &keymaps::Keymap<'_>, input: &str) -> Option<Vec<KeyboardReport>> {
307    let inverse_keymap = InverseKeymap::new(keymap);
308    let mut reports = vec![];
309    let mut shift_pressed = false;
310    let mut last_usage = None;
311
312    for ch in input.chars() {
313        let key_stroke = inverse_keymap.get(&ch)?;
314
315        match key_stroke.shift {
316            Shift::Yes if !shift_pressed => {
317                shift_pressed = true;
318                last_usage = Some(0);
319            }
320            Shift::No if shift_pressed => {
321                shift_pressed = false;
322                last_usage = Some(0);
323            }
324            _ => {
325                if last_usage == Some(key_stroke.usage) {
326                    last_usage = Some(0);
327                }
328            }
329        }
330
331        if let Some(0) = last_usage {
332            reports.push(KeyboardReport {
333                pressed_keys: if shift_pressed {
334                    vec![Usages::HidUsageKeyLeftShift as u32]
335                } else {
336                    vec![]
337                },
338            });
339        }
340
341        last_usage = Some(key_stroke.usage);
342
343        reports.push(KeyboardReport {
344            pressed_keys: if shift_pressed {
345                vec![key_stroke.usage, Usages::HidUsageKeyLeftShift as u32]
346            } else {
347                vec![key_stroke.usage]
348            },
349        });
350    }
351
352    // TODO: In the future, we might want to distinguish between different key releases, instead
353    //       of sending one single release report even in the case of key combinations.
354    reports.push(KeyboardReport { pressed_keys: vec![] });
355
356    Some(reports)
357}
358
359#[cfg(test)]
360mod tests {
361    // Most of the functions in this file need to bind to FIDL services in
362    // this component's environment to do their work, but a component can't
363    // modify its own environment. Hence, we can't validate those functions.
364    //
365    // However, we can (and do) validate derive_key_sequence().
366
367    use super::{derive_key_sequence, KeyboardReport, Usages};
368    use pretty_assertions::assert_eq;
369
370    macro_rules! reports {
371        ( $( [ $( $usages:expr ),* ] ),* $( , )? ) => {
372            Some(vec![
373                $(
374                    KeyboardReport {
375                        pressed_keys: vec![$($usages as u32),*]
376                    }
377                ),*
378            ])
379        }
380    }
381
382    #[test]
383    fn lowercase() {
384        assert_eq!(
385            derive_key_sequence(&keymaps::US_QWERTY, "lowercase"),
386            reports![
387                [Usages::HidUsageKeyL],
388                [Usages::HidUsageKeyO],
389                [Usages::HidUsageKeyW],
390                [Usages::HidUsageKeyE],
391                [Usages::HidUsageKeyR],
392                [Usages::HidUsageKeyC],
393                [Usages::HidUsageKeyA],
394                [Usages::HidUsageKeyS],
395                [Usages::HidUsageKeyE],
396                [],
397            ]
398        );
399    }
400
401    #[test]
402    fn numerics() {
403        assert_eq!(
404            derive_key_sequence(&keymaps::US_QWERTY, "0123456789"),
405            reports![
406                [Usages::HidUsageKey0],
407                [Usages::HidUsageKey1],
408                [Usages::HidUsageKey2],
409                [Usages::HidUsageKey3],
410                [Usages::HidUsageKey4],
411                [Usages::HidUsageKey5],
412                [Usages::HidUsageKey6],
413                [Usages::HidUsageKey7],
414                [Usages::HidUsageKey8],
415                [Usages::HidUsageKey9],
416                [],
417            ]
418        );
419    }
420
421    #[test]
422    fn internet_text_entry() {
423        assert_eq!(
424            derive_key_sequence(&keymaps::US_QWERTY, "http://127.0.0.1:8080"),
425            reports![
426                [Usages::HidUsageKeyH],
427                [Usages::HidUsageKeyT],
428                [],
429                [Usages::HidUsageKeyT],
430                [Usages::HidUsageKeyP],
431                // ':'
432                // Shift is actuated first on its own, then together with
433                // the key.
434                [Usages::HidUsageKeyLeftShift],
435                [Usages::HidUsageKeySemicolon, Usages::HidUsageKeyLeftShift],
436                [],
437                [Usages::HidUsageKeySlash],
438                [],
439                [Usages::HidUsageKeySlash],
440                [Usages::HidUsageKey1],
441                [Usages::HidUsageKey2],
442                [Usages::HidUsageKey7],
443                [Usages::HidUsageKeyDot],
444                [Usages::HidUsageKey0],
445                [Usages::HidUsageKeyDot],
446                [Usages::HidUsageKey0],
447                [Usages::HidUsageKeyDot],
448                [Usages::HidUsageKey1],
449                [Usages::HidUsageKeyLeftShift],
450                [Usages::HidUsageKeySemicolon, Usages::HidUsageKeyLeftShift],
451                [],
452                [Usages::HidUsageKey8],
453                [Usages::HidUsageKey0],
454                [Usages::HidUsageKey8],
455                [Usages::HidUsageKey0],
456                [],
457            ]
458        );
459    }
460
461    #[test]
462    fn sentence() {
463        assert_eq!(
464            derive_key_sequence(&keymaps::US_QWERTY, "Hello, world!"),
465            reports![
466                [Usages::HidUsageKeyLeftShift],
467                [Usages::HidUsageKeyH, Usages::HidUsageKeyLeftShift],
468                [],
469                [Usages::HidUsageKeyE],
470                [Usages::HidUsageKeyL],
471                [],
472                [Usages::HidUsageKeyL],
473                [Usages::HidUsageKeyO],
474                [Usages::HidUsageKeyComma],
475                [Usages::HidUsageKeySpace],
476                [Usages::HidUsageKeyW],
477                [Usages::HidUsageKeyO],
478                [Usages::HidUsageKeyR],
479                [Usages::HidUsageKeyL],
480                [Usages::HidUsageKeyD],
481                [Usages::HidUsageKeyLeftShift],
482                [Usages::HidUsageKey1, Usages::HidUsageKeyLeftShift],
483                [],
484            ]
485        );
486    }
487
488    #[test]
489    fn hold_shift() {
490        assert_eq!(
491            derive_key_sequence(&keymaps::US_QWERTY, "ALL'S WELL!"),
492            reports![
493                [Usages::HidUsageKeyLeftShift],
494                [Usages::HidUsageKeyA, Usages::HidUsageKeyLeftShift],
495                [Usages::HidUsageKeyL, Usages::HidUsageKeyLeftShift],
496                [Usages::HidUsageKeyLeftShift],
497                [Usages::HidUsageKeyL, Usages::HidUsageKeyLeftShift],
498                [],
499                [Usages::HidUsageKeyApostrophe],
500                [Usages::HidUsageKeyLeftShift],
501                [Usages::HidUsageKeyS, Usages::HidUsageKeyLeftShift],
502                [Usages::HidUsageKeySpace, Usages::HidUsageKeyLeftShift],
503                [Usages::HidUsageKeyW, Usages::HidUsageKeyLeftShift],
504                [Usages::HidUsageKeyE, Usages::HidUsageKeyLeftShift],
505                [Usages::HidUsageKeyL, Usages::HidUsageKeyLeftShift],
506                [Usages::HidUsageKeyLeftShift],
507                [Usages::HidUsageKeyL, Usages::HidUsageKeyLeftShift],
508                [Usages::HidUsageKey1, Usages::HidUsageKeyLeftShift],
509                [],
510            ]
511        );
512    }
513
514    #[test]
515    fn tab_and_newline() {
516        assert_eq!(
517            derive_key_sequence(&keymaps::US_QWERTY, "\tHello\n"),
518            reports![
519                [Usages::HidUsageKeyTab],
520                [Usages::HidUsageKeyLeftShift],
521                [Usages::HidUsageKeyH, Usages::HidUsageKeyLeftShift],
522                [],
523                [Usages::HidUsageKeyE],
524                [Usages::HidUsageKeyL],
525                [],
526                [Usages::HidUsageKeyL],
527                [Usages::HidUsageKeyO],
528                [Usages::HidUsageKeyEnter],
529                [],
530            ]
531        );
532    }
533}
input_synthesis/lib.rs

input_synthesis/
lib.rs
