input_pipeline/
modifier_handler.rs

1// Copyright 2021 The Fuchsia Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5use crate::input_device::{Handled, InputDeviceEvent, InputEvent, UnhandledInputEvent};
6use crate::input_handler::{InputHandlerStatus, UnhandledInputHandler};
7use async_trait::async_trait;
8use fidl_fuchsia_ui_input3::{KeyMeaning, Modifiers, NonPrintableKey};
9use fuchsia_inspect::health::Reporter;
10use keymaps::{LockStateKeys, ModifierState};
11use std::cell::RefCell;
12use std::rc::Rc;
13
14/// Tracks modifier state and decorates passing events with the modifiers.
15///
16/// This handler should be installed as early as possible in the input pipeline,
17/// to ensure that all later stages have the modifiers and lock states available.
18/// This holds even for non-keyboard handlers, to allow handling `Ctrl+Click`
19/// events, for example.
20///
21/// One possible exception to this rule would be a hardware key rewriting handler for
22/// limited keyboards.
23#[derive(Debug)]
24pub struct ModifierHandler {
25    /// The tracked state of the modifiers.
26    modifier_state: RefCell<ModifierState>,
27
28    /// The tracked lock state.
29    lock_state: RefCell<LockStateKeys>,
30
31    /// The inventory of this handler's Inspect status.
32    pub inspect_status: InputHandlerStatus,
33}
34
35#[async_trait(?Send)]
36impl UnhandledInputHandler for ModifierHandler {
37    async fn handle_unhandled_input_event(
38        self: Rc<Self>,
39        unhandled_input_event: UnhandledInputEvent,
40    ) -> Vec<InputEvent> {
41        fuchsia_trace::duration!("input", "modifier_handler");
42        match unhandled_input_event {
43            UnhandledInputEvent {
44                device_event: InputDeviceEvent::Keyboard(mut event),
45                device_descriptor,
46                event_time,
47                trace_id,
48            } => {
49                fuchsia_trace::duration!("input", "modifier_handler[processing]");
50                if let Some(trace_id) = trace_id {
51                    fuchsia_trace::flow_step!(
52                        c"input",
53                        c"event_in_input_pipeline",
54                        trace_id.into()
55                    );
56                }
57
58                self.inspect_status.count_received_event(&event_time);
59                self.modifier_state.borrow_mut().update(event.get_event_type(), event.get_key());
60                self.lock_state.borrow_mut().update(event.get_event_type(), event.get_key());
61                event = event
62                    .into_with_lock_state(Some(self.lock_state.borrow().get_state()))
63                    .into_with_modifiers(Some(self.modifier_state.borrow().get_state()));
64                log::debug!("modifiers and lock state applied: {:?}", &event);
65                vec![InputEvent {
66                    device_event: InputDeviceEvent::Keyboard(event),
67                    device_descriptor,
68                    event_time,
69                    handled: Handled::No,
70                    trace_id,
71                }]
72            }
73            // Pass other events through.
74            _ => vec![InputEvent::from(unhandled_input_event)],
75        }
76    }
77
78    fn set_handler_healthy(self: std::rc::Rc<Self>) {
79        self.inspect_status.health_node.borrow_mut().set_ok();
80    }
81
82    fn set_handler_unhealthy(self: std::rc::Rc<Self>, msg: &str) {
83        self.inspect_status.health_node.borrow_mut().set_unhealthy(msg);
84    }
85
86    fn get_name(&self) -> &'static str {
87        "ModifierHandler"
88    }
89}
90
91impl ModifierHandler {
92    /// Creates a new [ModifierHandler].
93    pub fn new(input_handlers_node: &fuchsia_inspect::Node) -> Rc<Self> {
94        let inspect_status = InputHandlerStatus::new(
95            input_handlers_node,
96            "modifier_handler",
97            /* generates_events */ false,
98        );
99        Rc::new(Self {
100            modifier_state: RefCell::new(ModifierState::new()),
101            lock_state: RefCell::new(LockStateKeys::new()),
102            inspect_status,
103        })
104    }
105}
106
107/// Tracks the state of the modifiers that are tied to the key meaning (as opposed to hardware
108/// keys).
109#[derive(Debug)]
110pub struct ModifierMeaningHandler {
111    /// The tracked state of the modifiers.
112    modifier_state: RefCell<ModifierState>,
113
114    /// The inventory of this handler's Inspect status.
115    pub inspect_status: InputHandlerStatus,
116}
117
118impl ModifierMeaningHandler {
119    /// Creates a new [ModifierMeaningHandler].
120    pub fn new(input_handlers_node: &fuchsia_inspect::Node) -> Rc<Self> {
121        let inspect_status = InputHandlerStatus::new(
122            input_handlers_node,
123            "modifier_meaning_handler",
124            /* generates_events */ false,
125        );
126        Rc::new(Self { modifier_state: RefCell::new(ModifierState::new()), inspect_status })
127    }
128}
129
130#[async_trait(?Send)]
131impl UnhandledInputHandler for ModifierMeaningHandler {
132    async fn handle_unhandled_input_event(
133        self: Rc<Self>,
134        unhandled_input_event: UnhandledInputEvent,
135    ) -> Vec<InputEvent> {
136        fuchsia_trace::duration!("input", "modifier_meaning_handler");
137        match unhandled_input_event {
138            UnhandledInputEvent {
139                device_event: InputDeviceEvent::Keyboard(mut event),
140                device_descriptor,
141                event_time,
142                trace_id,
143            } if event.get_key_meaning()
144                == Some(KeyMeaning::NonPrintableKey(NonPrintableKey::AltGraph)) =>
145            {
146                fuchsia_trace::duration!("input", "modifier_meaning_handler[processing]");
147                if let Some(trace_id) = trace_id {
148                    fuchsia_trace::flow_step!(
149                        c"input",
150                        c"event_in_input_pipeline",
151                        trace_id.into()
152                    );
153                }
154                self.inspect_status.count_received_event(&event_time);
155                // The "obvious" rewrite of this if and the match guard above is
156                // unstable, so doing it this way.
157                if let Some(key_meaning) = event.get_key_meaning() {
158                    self.modifier_state
159                        .borrow_mut()
160                        .update_with_key_meaning(event.get_event_type(), key_meaning);
161                    let new_modifier = event.get_modifiers().unwrap_or(Modifiers::empty())
162                        | self.modifier_state.borrow().get_state();
163                    event = event.into_with_modifiers(Some(new_modifier));
164                    log::debug!("additinal modifiers and lock state applied: {:?}", &event);
165                }
166                vec![InputEvent {
167                    device_event: InputDeviceEvent::Keyboard(event),
168                    device_descriptor,
169                    event_time,
170                    handled: Handled::No,
171                    trace_id,
172                }]
173            }
174            // Pass other events through.
175            _ => vec![InputEvent::from(unhandled_input_event)],
176        }
177    }
178
179    fn set_handler_healthy(self: std::rc::Rc<Self>) {
180        self.inspect_status.health_node.borrow_mut().set_ok();
181    }
182
183    fn set_handler_unhealthy(self: std::rc::Rc<Self>, msg: &str) {
184        self.inspect_status.health_node.borrow_mut().set_unhealthy(msg);
185    }
186
187    fn get_name(&self) -> &'static str {
188        "ModifierHandler"
189    }
190}
191
192#[cfg(test)]
193mod tests {
194    use super::*;
195    use crate::input_device::InputDeviceDescriptor;
196    use crate::input_handler::InputHandler;
197    use crate::keyboard_binding::{self, KeyboardEvent};
198    use crate::testing_utilities;
199    use fidl_fuchsia_input::Key;
200    use fidl_fuchsia_ui_input3::{KeyEventType, LockState};
201    use fuchsia_async as fasync;
202    use pretty_assertions::assert_eq;
203
204    fn get_unhandled_input_event(event: KeyboardEvent) -> UnhandledInputEvent {
205        UnhandledInputEvent {
206            device_event: InputDeviceEvent::Keyboard(event),
207            event_time: zx::MonotonicInstant::from_nanos(42),
208            device_descriptor: InputDeviceDescriptor::Fake,
209            trace_id: None,
210        }
211    }
212
213    #[fasync::run_singlethreaded(test)]
214    async fn test_decoration() {
215        let inspector = fuchsia_inspect::Inspector::default();
216        let test_node = inspector.root().create_child("test_node");
217        let handler = ModifierHandler::new(&test_node);
218        let input_event =
219            get_unhandled_input_event(KeyboardEvent::new(Key::CapsLock, KeyEventType::Pressed));
220        let result = handler.handle_unhandled_input_event(input_event.clone()).await;
221
222        // This handler decorates, but does not handle the key. Hence,
223        // the key remains unhandled.
224        let expected = InputEvent::from(get_unhandled_input_event(
225            KeyboardEvent::new(Key::CapsLock, KeyEventType::Pressed)
226                .into_with_modifiers(Some(Modifiers::CAPS_LOCK))
227                .into_with_lock_state(Some(LockState::CAPS_LOCK)),
228        ));
229        assert_eq!(vec![expected], result);
230    }
231
232    #[fasync::run_singlethreaded(test)]
233    async fn test_key_meaning_decoration() {
234        let inspector = fuchsia_inspect::Inspector::default();
235        let test_node = inspector.root().create_child("test_node");
236        let handler = ModifierMeaningHandler::new(&test_node);
237        {
238            let input_event = get_unhandled_input_event(
239                KeyboardEvent::new(Key::RightAlt, KeyEventType::Pressed)
240                    .into_with_key_meaning(Some(KeyMeaning::NonPrintableKey(
241                        NonPrintableKey::AltGraph,
242                    )))
243                    .into_with_modifiers(Some(Modifiers::CAPS_LOCK)),
244            );
245            let result = handler.clone().handle_unhandled_input_event(input_event.clone()).await;
246            let expected = InputEvent::from(get_unhandled_input_event(
247                KeyboardEvent::new(Key::RightAlt, KeyEventType::Pressed)
248                    .into_with_key_meaning(Some(KeyMeaning::NonPrintableKey(
249                        NonPrintableKey::AltGraph,
250                    )))
251                    .into_with_modifiers(Some(Modifiers::ALT_GRAPH | Modifiers::CAPS_LOCK)),
252            ));
253            assert_eq!(vec![expected], result);
254        }
255        {
256            let input_event = get_unhandled_input_event(
257                KeyboardEvent::new(Key::RightAlt, KeyEventType::Released)
258                    .into_with_key_meaning(Some(KeyMeaning::NonPrintableKey(
259                        NonPrintableKey::AltGraph,
260                    )))
261                    .into_with_modifiers(Some(Modifiers::CAPS_LOCK)),
262            );
263            let handler = handler.clone();
264            let result = handler.handle_unhandled_input_event(input_event.clone()).await;
265            let expected = InputEvent::from(get_unhandled_input_event(
266                KeyboardEvent::new(Key::RightAlt, KeyEventType::Released)
267                    .into_with_key_meaning(Some(KeyMeaning::NonPrintableKey(
268                        NonPrintableKey::AltGraph,
269                    )))
270                    .into_with_modifiers(Some(Modifiers::CAPS_LOCK)),
271            ));
272            assert_eq!(vec![expected], result);
273        }
274    }
275
276    // CapsLock  """"""\______/""""""""""\_______/"""
277    // Modifiers ------CCCCCCCC----------CCCCCCCCC---
278    // LockState ------CCCCCCCCCCCCCCCCCCCCCCCCCCC---
279    //
280    // C == CapsLock
281    #[fasync::run_singlethreaded(test)]
282    async fn test_modifier_press_lock_release() {
283        let input_events = vec![
284            get_unhandled_input_event(KeyboardEvent::new(Key::CapsLock, KeyEventType::Pressed)),
285            get_unhandled_input_event(KeyboardEvent::new(Key::CapsLock, KeyEventType::Released)),
286            get_unhandled_input_event(KeyboardEvent::new(Key::CapsLock, KeyEventType::Pressed)),
287            get_unhandled_input_event(KeyboardEvent::new(Key::CapsLock, KeyEventType::Released)),
288        ];
289
290        let inspector = fuchsia_inspect::Inspector::default();
291        let test_node = inspector.root().create_child("test_node");
292        let handler = ModifierHandler::new(&test_node);
293        let clone_handler = move || handler.clone();
294        let result = futures::future::join_all(
295            input_events
296                .into_iter()
297                .map(|e| async { clone_handler().handle_unhandled_input_event(e).await }),
298        )
299        .await
300        .into_iter()
301        .flatten()
302        .collect::<Vec<InputEvent>>();
303
304        let expected = IntoIterator::into_iter([
305            get_unhandled_input_event(
306                KeyboardEvent::new(Key::CapsLock, KeyEventType::Pressed)
307                    .into_with_modifiers(Some(Modifiers::CAPS_LOCK))
308                    .into_with_lock_state(Some(LockState::CAPS_LOCK)),
309            ),
310            get_unhandled_input_event(
311                KeyboardEvent::new(Key::CapsLock, KeyEventType::Released)
312                    .into_with_modifiers(Some(Modifiers::from_bits_allow_unknown(0)))
313                    .into_with_lock_state(Some(LockState::CAPS_LOCK)),
314            ),
315            get_unhandled_input_event(
316                KeyboardEvent::new(Key::CapsLock, KeyEventType::Pressed)
317                    .into_with_modifiers(Some(Modifiers::CAPS_LOCK))
318                    .into_with_lock_state(Some(LockState::CAPS_LOCK)),
319            ),
320            get_unhandled_input_event(
321                KeyboardEvent::new(Key::CapsLock, KeyEventType::Released)
322                    .into_with_modifiers(Some(Modifiers::from_bits_allow_unknown(0)))
323                    .into_with_lock_state(Some(LockState::from_bits_allow_unknown(0))),
324            ),
325        ])
326        .map(InputEvent::from)
327        .collect::<Vec<_>>();
328
329        assert_eq!(expected, result);
330    }
331
332    // CapsLock  """"""\______/"""""""""""""""""""
333    // A         """""""""""""""""""\________/""""
334    // Modifiers ------CCCCCCCC-------------------
335    // LockState ------CCCCCCCCCCCCCCCCCCCCCCCCCCC
336    //
337    // C == CapsLock
338    #[fasync::run_singlethreaded(test)]
339    async fn repeated_modifier_key() {
340        let input_events = vec![
341            get_unhandled_input_event(KeyboardEvent::new(Key::CapsLock, KeyEventType::Pressed)),
342            get_unhandled_input_event(KeyboardEvent::new(Key::CapsLock, KeyEventType::Released)),
343            get_unhandled_input_event(KeyboardEvent::new(Key::A, KeyEventType::Pressed)),
344            get_unhandled_input_event(KeyboardEvent::new(Key::A, KeyEventType::Released)),
345        ];
346
347        let inspector = fuchsia_inspect::Inspector::default();
348        let test_node = inspector.root().create_child("test_node");
349        let handler = ModifierHandler::new(&test_node);
350        let clone_handler = move || handler.clone();
351        let result = futures::future::join_all(
352            input_events
353                .into_iter()
354                .map(|e| async { clone_handler().handle_unhandled_input_event(e).await }),
355        )
356        .await
357        .into_iter()
358        .flatten()
359        .collect::<Vec<InputEvent>>();
360
361        let expected = IntoIterator::into_iter([
362            get_unhandled_input_event(
363                KeyboardEvent::new(Key::CapsLock, KeyEventType::Pressed)
364                    .into_with_modifiers(Some(Modifiers::CAPS_LOCK))
365                    .into_with_lock_state(Some(LockState::CAPS_LOCK)),
366            ),
367            get_unhandled_input_event(
368                KeyboardEvent::new(Key::CapsLock, KeyEventType::Released)
369                    .into_with_modifiers(Some(Modifiers::from_bits_allow_unknown(0)))
370                    .into_with_lock_state(Some(LockState::CAPS_LOCK)),
371            ),
372            get_unhandled_input_event(
373                KeyboardEvent::new(Key::A, KeyEventType::Pressed)
374                    .into_with_modifiers(Some(Modifiers::from_bits_allow_unknown(0)))
375                    .into_with_lock_state(Some(LockState::CAPS_LOCK)),
376            ),
377            get_unhandled_input_event(
378                KeyboardEvent::new(Key::A, KeyEventType::Released)
379                    .into_with_modifiers(Some(Modifiers::from_bits_allow_unknown(0)))
380                    .into_with_lock_state(Some(LockState::CAPS_LOCK)),
381            ),
382        ])
383        .map(InputEvent::from)
384        .collect::<Vec<_>>();
385        assert_eq!(expected, result);
386    }
387
388    #[fuchsia::test]
389    async fn modifier_handlers_initialized_with_inspect_node() {
390        let inspector = fuchsia_inspect::Inspector::default();
391        let fake_handlers_node = inspector.root().create_child("input_handlers_node");
392        let _modifier_handler = ModifierHandler::new(&fake_handlers_node);
393        let _modifier_meaning_handler = ModifierMeaningHandler::new(&fake_handlers_node);
394        diagnostics_assertions::assert_data_tree!(inspector, root: {
395            input_handlers_node: {
396                modifier_handler: {
397                    events_received_count: 0u64,
398                    events_handled_count: 0u64,
399                    last_received_timestamp_ns: 0u64,
400                    "fuchsia.inspect.Health": {
401                        status: "STARTING_UP",
402                        // Timestamp value is unpredictable and not relevant in this context,
403                        // so we only assert that the property is present.
404                        start_timestamp_nanos: diagnostics_assertions::AnyProperty
405                    },
406                },
407                modifier_meaning_handler: {
408                    events_received_count: 0u64,
409                    events_handled_count: 0u64,
410                    last_received_timestamp_ns: 0u64,
411                    "fuchsia.inspect.Health": {
412                        status: "STARTING_UP",
413                        // Timestamp value is unpredictable and not relevant in this context,
414                        // so we only assert that the property is present.
415                        start_timestamp_nanos: diagnostics_assertions::AnyProperty
416                    },
417                }
418            }
419        });
420    }
421
422    #[fasync::run_singlethreaded(test)]
423    async fn modifier_handler_inspect_counts_events() {
424        let inspector = fuchsia_inspect::Inspector::default();
425        let fake_handlers_node = inspector.root().create_child("input_handlers_node");
426        let modifier_handler = ModifierHandler::new(&fake_handlers_node);
427        let modifier_meaning_handler = ModifierMeaningHandler::new(&fake_handlers_node);
428        let device_descriptor =
429            InputDeviceDescriptor::Keyboard(keyboard_binding::KeyboardDeviceDescriptor {
430                keys: vec![Key::A, Key::B, Key::RightAlt],
431                ..Default::default()
432            });
433        let (_, event_time_u64) = testing_utilities::event_times();
434        let input_events = vec![
435            testing_utilities::create_keyboard_event_with_time(
436                Key::A,
437                fidl_fuchsia_ui_input3::KeyEventType::Pressed,
438                None,
439                event_time_u64,
440                &device_descriptor,
441                /* keymap= */ None,
442            ),
443            // Should not count received events that have already been handled.
444            testing_utilities::create_keyboard_event_with_handled(
445                Key::B,
446                fidl_fuchsia_ui_input3::KeyEventType::Pressed,
447                None,
448                event_time_u64,
449                &device_descriptor,
450                /* keymap= */ None,
451                /* key_meaning= */ None,
452                Handled::Yes,
453            ),
454            testing_utilities::create_keyboard_event_with_time(
455                Key::A,
456                fidl_fuchsia_ui_input3::KeyEventType::Released,
457                None,
458                event_time_u64,
459                &device_descriptor,
460                /* keymap= */ None,
461            ),
462            // Should not count non-keyboard input events.
463            testing_utilities::create_fake_input_event(event_time_u64),
464            // Only event that should be counted by ModifierMeaningHandler.
465            testing_utilities::create_keyboard_event_with_key_meaning(
466                Key::RightAlt,
467                fidl_fuchsia_ui_input3::KeyEventType::Pressed,
468                None,
469                event_time_u64,
470                &device_descriptor,
471                /* keymap= */ None,
472                /* key_meaning= */
473                Some(KeyMeaning::NonPrintableKey(NonPrintableKey::AltGraph)),
474            ),
475        ];
476
477        for input_event in input_events {
478            let _ = modifier_handler.clone().handle_input_event(input_event.clone()).await;
479            let _ = modifier_meaning_handler.clone().handle_input_event(input_event).await;
480        }
481
482        let last_event_timestamp: u64 = event_time_u64.into_nanos().try_into().unwrap();
483
484        diagnostics_assertions::assert_data_tree!(inspector, root: {
485            input_handlers_node: {
486                modifier_handler: {
487                    events_received_count: 3u64,
488                    events_handled_count: 0u64,
489                    last_received_timestamp_ns: last_event_timestamp,
490                    "fuchsia.inspect.Health": {
491                        status: "STARTING_UP",
492                        // Timestamp value is unpredictable and not relevant in this context,
493                        // so we only assert that the property is present.
494                        start_timestamp_nanos: diagnostics_assertions::AnyProperty
495                    },
496                },
497                modifier_meaning_handler: {
498                    events_received_count: 1u64,
499                    events_handled_count: 0u64,
500                    last_received_timestamp_ns: last_event_timestamp,
501                    "fuchsia.inspect.Health": {
502                        status: "STARTING_UP",
503                        // Timestamp value is unpredictable and not relevant in this context,
504                        // so we only assert that the property is present.
505                        start_timestamp_nanos: diagnostics_assertions::AnyProperty
506                    },
507                }
508            }
509        });
510    }
511}
input_pipeline/modifier_handler.rs

input_pipeline/
modifier_handler.rs
