input_pipeline/
modifier_handler.rs

1// Copyright 2021 The Fuchsia Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5use crate::input_device::{
6    Handled, InputDeviceEvent, InputEvent, InputEventType, UnhandledInputEvent,
7};
8use crate::input_handler::{Handler, InputHandlerStatus, UnhandledInputHandler};
9use crate::metrics;
10use async_trait::async_trait;
11use fidl_fuchsia_ui_input3::{KeyMeaning, Modifiers, NonPrintableKey};
12use fuchsia_inspect::health::Reporter;
13use keymaps::{LockStateKeys, ModifierState};
14use metrics_registry::InputPipelineErrorMetricDimensionEvent;
15use std::cell::RefCell;
16use std::rc::Rc;
17
18/// Tracks modifier state and decorates passing events with the modifiers.
19///
20/// This handler should be installed as early as possible in the input pipeline,
21/// to ensure that all later stages have the modifiers and lock states available.
22/// This holds even for non-keyboard handlers, to allow handling `Ctrl+Click`
23/// events, for example.
24///
25/// One possible exception to this rule would be a hardware key rewriting handler for
26/// limited keyboards.
27#[derive(Debug)]
28pub struct ModifierHandler {
29    /// The tracked state of the modifiers.
30    modifier_state: RefCell<ModifierState>,
31
32    /// The tracked lock state.
33    lock_state: RefCell<LockStateKeys>,
34
35    /// The metrics logger.
36    metrics_logger: metrics::MetricsLogger,
37
38    /// The inventory of this handler's Inspect status.
39    pub inspect_status: InputHandlerStatus,
40}
41
42impl Handler for ModifierHandler {
43    fn set_handler_healthy(self: std::rc::Rc<Self>) {
44        self.inspect_status.health_node.borrow_mut().set_ok();
45    }
46
47    fn set_handler_unhealthy(self: std::rc::Rc<Self>, msg: &str) {
48        self.inspect_status.health_node.borrow_mut().set_unhealthy(msg);
49    }
50
51    fn get_name(&self) -> &'static str {
52        "ModifierHandler"
53    }
54
55    fn interest(&self) -> Vec<InputEventType> {
56        vec![InputEventType::Keyboard]
57    }
58}
59
60#[async_trait(?Send)]
61impl UnhandledInputHandler for ModifierHandler {
62    async fn handle_unhandled_input_event(
63        self: Rc<Self>,
64        unhandled_input_event: UnhandledInputEvent,
65    ) -> Vec<InputEvent> {
66        fuchsia_trace::duration!("input", "modifier_handler");
67        match unhandled_input_event {
68            UnhandledInputEvent {
69                device_event: InputDeviceEvent::Keyboard(mut event),
70                device_descriptor,
71                event_time,
72                trace_id,
73            } => {
74                fuchsia_trace::duration!("input", "modifier_handler[processing]");
75                if let Some(trace_id) = trace_id {
76                    fuchsia_trace::flow_step!(
77                        c"input",
78                        c"event_in_input_pipeline",
79                        trace_id.into()
80                    );
81                }
82
83                self.inspect_status.count_received_event(&event_time);
84                self.modifier_state.borrow_mut().update(event.get_event_type(), event.get_key());
85                self.lock_state.borrow_mut().update(event.get_event_type(), event.get_key());
86                event = event
87                    .into_with_lock_state(Some(self.lock_state.borrow().get_state()))
88                    .into_with_modifiers(Some(self.modifier_state.borrow().get_state()));
89                log::debug!("modifiers and lock state applied: {:?}", &event);
90                vec![InputEvent {
91                    device_event: InputDeviceEvent::Keyboard(event),
92                    device_descriptor,
93                    event_time,
94                    handled: Handled::No,
95                    trace_id,
96                }]
97            }
98            // Pass other events through.
99            _ => {
100                self.metrics_logger.log_error(
101                    InputPipelineErrorMetricDimensionEvent::HandlerReceivedUninterestedEvent,
102                    std::format!(
103                        "uninterested input event: {:?}",
104                        unhandled_input_event.get_event_type()
105                    ),
106                );
107                vec![InputEvent::from(unhandled_input_event)]
108            }
109        }
110    }
111}
112
113impl ModifierHandler {
114    pub fn new(
115        input_handlers_node: &fuchsia_inspect::Node,
116        metrics_logger: metrics::MetricsLogger,
117    ) -> Rc<Self> {
118        let inspect_status = InputHandlerStatus::new(
119            input_handlers_node,
120            "modifier_handler",
121            /* generates_events */ false,
122        );
123        Rc::new(Self {
124            modifier_state: RefCell::new(ModifierState::new()),
125            lock_state: RefCell::new(LockStateKeys::new()),
126            metrics_logger,
127            inspect_status,
128        })
129    }
130}
131
132/// Tracks the state of the modifiers that are tied to the key meaning (as opposed to hardware
133/// keys).
134#[derive(Debug)]
135pub struct ModifierMeaningHandler {
136    /// The tracked state of the modifiers.
137    modifier_state: RefCell<ModifierState>,
138
139    /// The metrics logger.
140    metrics_logger: metrics::MetricsLogger,
141
142    /// The inventory of this handler's Inspect status.
143    pub inspect_status: InputHandlerStatus,
144}
145
146impl ModifierMeaningHandler {
147    pub fn new(
148        input_handlers_node: &fuchsia_inspect::Node,
149        metrics_logger: metrics::MetricsLogger,
150    ) -> Rc<Self> {
151        let inspect_status = InputHandlerStatus::new(
152            input_handlers_node,
153            "modifier_meaning_handler",
154            /* generates_events */ false,
155        );
156        Rc::new(Self {
157            modifier_state: RefCell::new(ModifierState::new()),
158            metrics_logger,
159            inspect_status,
160        })
161    }
162}
163
164impl Handler for ModifierMeaningHandler {
165    fn set_handler_healthy(self: std::rc::Rc<Self>) {
166        self.inspect_status.health_node.borrow_mut().set_ok();
167    }
168
169    fn set_handler_unhealthy(self: std::rc::Rc<Self>, msg: &str) {
170        self.inspect_status.health_node.borrow_mut().set_unhealthy(msg);
171    }
172
173    fn get_name(&self) -> &'static str {
174        "ModifierHandler"
175    }
176
177    fn interest(&self) -> Vec<InputEventType> {
178        vec![InputEventType::Keyboard]
179    }
180}
181
182#[async_trait(?Send)]
183impl UnhandledInputHandler for ModifierMeaningHandler {
184    async fn handle_unhandled_input_event(
185        self: Rc<Self>,
186        unhandled_input_event: UnhandledInputEvent,
187    ) -> Vec<InputEvent> {
188        fuchsia_trace::duration!("input", "modifier_meaning_handler");
189        match unhandled_input_event {
190            UnhandledInputEvent {
191                device_event: InputDeviceEvent::Keyboard(mut event),
192                device_descriptor,
193                event_time,
194                trace_id,
195            } if event.get_key_meaning()
196                == Some(KeyMeaning::NonPrintableKey(NonPrintableKey::AltGraph)) =>
197            {
198                fuchsia_trace::duration!("input", "modifier_meaning_handler[processing]");
199                if let Some(trace_id) = trace_id {
200                    fuchsia_trace::flow_step!(
201                        c"input",
202                        c"event_in_input_pipeline",
203                        trace_id.into()
204                    );
205                }
206                self.inspect_status.count_received_event(&event_time);
207                // The "obvious" rewrite of this if and the match guard above is
208                // unstable, so doing it this way.
209                if let Some(key_meaning) = event.get_key_meaning() {
210                    self.modifier_state
211                        .borrow_mut()
212                        .update_with_key_meaning(event.get_event_type(), key_meaning);
213                    let new_modifier = event.get_modifiers().unwrap_or(Modifiers::empty())
214                        | self.modifier_state.borrow().get_state();
215                    event = event.into_with_modifiers(Some(new_modifier));
216                    log::debug!("additinal modifiers and lock state applied: {:?}", &event);
217                }
218                vec![InputEvent {
219                    device_event: InputDeviceEvent::Keyboard(event),
220                    device_descriptor,
221                    event_time,
222                    handled: Handled::No,
223                    trace_id,
224                }]
225            }
226            // Pass other events through.
227            _ => {
228                self.metrics_logger.log_error(
229                    InputPipelineErrorMetricDimensionEvent::HandlerReceivedUninterestedEvent,
230                    std::format!(
231                        "uninterested input event: {:?}",
232                        unhandled_input_event.get_event_type()
233                    ),
234                );
235                vec![InputEvent::from(unhandled_input_event)]
236            }
237        }
238    }
239}
240
241#[cfg(test)]
242mod tests {
243    use super::*;
244    use crate::input_device::InputDeviceDescriptor;
245    use crate::input_handler::InputHandler;
246    use crate::keyboard_binding::{self, KeyboardEvent};
247    use crate::testing_utilities;
248    use fidl_fuchsia_input::Key;
249    use fidl_fuchsia_ui_input3::{KeyEventType, LockState};
250    use fuchsia_async as fasync;
251    use pretty_assertions::assert_eq;
252
253    fn get_unhandled_input_event(event: KeyboardEvent) -> UnhandledInputEvent {
254        UnhandledInputEvent {
255            device_event: InputDeviceEvent::Keyboard(event),
256            event_time: zx::MonotonicInstant::from_nanos(42),
257            device_descriptor: InputDeviceDescriptor::Fake,
258            trace_id: None,
259        }
260    }
261
262    #[fasync::run_singlethreaded(test)]
263    async fn test_decoration() {
264        let inspector = fuchsia_inspect::Inspector::default();
265        let test_node = inspector.root().create_child("test_node");
266        let handler = ModifierHandler::new(&test_node, metrics::MetricsLogger::default());
267        let input_event =
268            get_unhandled_input_event(KeyboardEvent::new(Key::CapsLock, KeyEventType::Pressed));
269        let result = handler.handle_unhandled_input_event(input_event.clone()).await;
270
271        // This handler decorates, but does not handle the key. Hence,
272        // the key remains unhandled.
273        let expected = InputEvent::from(get_unhandled_input_event(
274            KeyboardEvent::new(Key::CapsLock, KeyEventType::Pressed)
275                .into_with_modifiers(Some(Modifiers::CAPS_LOCK))
276                .into_with_lock_state(Some(LockState::CAPS_LOCK)),
277        ));
278        assert_eq!(vec![expected], result);
279    }
280
281    #[fasync::run_singlethreaded(test)]
282    async fn test_key_meaning_decoration() {
283        let inspector = fuchsia_inspect::Inspector::default();
284        let test_node = inspector.root().create_child("test_node");
285        let handler = ModifierMeaningHandler::new(&test_node, metrics::MetricsLogger::default());
286        {
287            let input_event = get_unhandled_input_event(
288                KeyboardEvent::new(Key::RightAlt, KeyEventType::Pressed)
289                    .into_with_key_meaning(Some(KeyMeaning::NonPrintableKey(
290                        NonPrintableKey::AltGraph,
291                    )))
292                    .into_with_modifiers(Some(Modifiers::CAPS_LOCK)),
293            );
294            let result = handler.clone().handle_unhandled_input_event(input_event.clone()).await;
295            let expected = InputEvent::from(get_unhandled_input_event(
296                KeyboardEvent::new(Key::RightAlt, KeyEventType::Pressed)
297                    .into_with_key_meaning(Some(KeyMeaning::NonPrintableKey(
298                        NonPrintableKey::AltGraph,
299                    )))
300                    .into_with_modifiers(Some(Modifiers::ALT_GRAPH | Modifiers::CAPS_LOCK)),
301            ));
302            assert_eq!(vec![expected], result);
303        }
304        {
305            let input_event = get_unhandled_input_event(
306                KeyboardEvent::new(Key::RightAlt, KeyEventType::Released)
307                    .into_with_key_meaning(Some(KeyMeaning::NonPrintableKey(
308                        NonPrintableKey::AltGraph,
309                    )))
310                    .into_with_modifiers(Some(Modifiers::CAPS_LOCK)),
311            );
312            let handler = handler.clone();
313            let result = handler.handle_unhandled_input_event(input_event.clone()).await;
314            let expected = InputEvent::from(get_unhandled_input_event(
315                KeyboardEvent::new(Key::RightAlt, KeyEventType::Released)
316                    .into_with_key_meaning(Some(KeyMeaning::NonPrintableKey(
317                        NonPrintableKey::AltGraph,
318                    )))
319                    .into_with_modifiers(Some(Modifiers::CAPS_LOCK)),
320            ));
321            assert_eq!(vec![expected], result);
322        }
323    }
324
325    // CapsLock  """"""\______/""""""""""\_______/"""
326    // Modifiers ------CCCCCCCC----------CCCCCCCCC---
327    // LockState ------CCCCCCCCCCCCCCCCCCCCCCCCCCC---
328    //
329    // C == CapsLock
330    #[fasync::run_singlethreaded(test)]
331    async fn test_modifier_press_lock_release() {
332        let input_events = vec![
333            get_unhandled_input_event(KeyboardEvent::new(Key::CapsLock, KeyEventType::Pressed)),
334            get_unhandled_input_event(KeyboardEvent::new(Key::CapsLock, KeyEventType::Released)),
335            get_unhandled_input_event(KeyboardEvent::new(Key::CapsLock, KeyEventType::Pressed)),
336            get_unhandled_input_event(KeyboardEvent::new(Key::CapsLock, KeyEventType::Released)),
337        ];
338
339        let inspector = fuchsia_inspect::Inspector::default();
340        let test_node = inspector.root().create_child("test_node");
341        let handler = ModifierHandler::new(&test_node, metrics::MetricsLogger::default());
342        let clone_handler = move || handler.clone();
343        let result = futures::future::join_all(
344            input_events
345                .into_iter()
346                .map(|e| async { clone_handler().handle_unhandled_input_event(e).await }),
347        )
348        .await
349        .into_iter()
350        .flatten()
351        .collect::<Vec<InputEvent>>();
352
353        let expected = IntoIterator::into_iter([
354            get_unhandled_input_event(
355                KeyboardEvent::new(Key::CapsLock, KeyEventType::Pressed)
356                    .into_with_modifiers(Some(Modifiers::CAPS_LOCK))
357                    .into_with_lock_state(Some(LockState::CAPS_LOCK)),
358            ),
359            get_unhandled_input_event(
360                KeyboardEvent::new(Key::CapsLock, KeyEventType::Released)
361                    .into_with_modifiers(Some(Modifiers::from_bits_allow_unknown(0)))
362                    .into_with_lock_state(Some(LockState::CAPS_LOCK)),
363            ),
364            get_unhandled_input_event(
365                KeyboardEvent::new(Key::CapsLock, KeyEventType::Pressed)
366                    .into_with_modifiers(Some(Modifiers::CAPS_LOCK))
367                    .into_with_lock_state(Some(LockState::CAPS_LOCK)),
368            ),
369            get_unhandled_input_event(
370                KeyboardEvent::new(Key::CapsLock, KeyEventType::Released)
371                    .into_with_modifiers(Some(Modifiers::from_bits_allow_unknown(0)))
372                    .into_with_lock_state(Some(LockState::from_bits_allow_unknown(0))),
373            ),
374        ])
375        .map(InputEvent::from)
376        .collect::<Vec<_>>();
377
378        assert_eq!(expected, result);
379    }
380
381    // CapsLock  """"""\______/"""""""""""""""""""
382    // A         """""""""""""""""""\________/""""
383    // Modifiers ------CCCCCCCC-------------------
384    // LockState ------CCCCCCCCCCCCCCCCCCCCCCCCCCC
385    //
386    // C == CapsLock
387    #[fasync::run_singlethreaded(test)]
388    async fn repeated_modifier_key() {
389        let input_events = vec![
390            get_unhandled_input_event(KeyboardEvent::new(Key::CapsLock, KeyEventType::Pressed)),
391            get_unhandled_input_event(KeyboardEvent::new(Key::CapsLock, KeyEventType::Released)),
392            get_unhandled_input_event(KeyboardEvent::new(Key::A, KeyEventType::Pressed)),
393            get_unhandled_input_event(KeyboardEvent::new(Key::A, KeyEventType::Released)),
394        ];
395
396        let inspector = fuchsia_inspect::Inspector::default();
397        let test_node = inspector.root().create_child("test_node");
398        let handler = ModifierHandler::new(&test_node, metrics::MetricsLogger::default());
399        let clone_handler = move || handler.clone();
400        let result = futures::future::join_all(
401            input_events
402                .into_iter()
403                .map(|e| async { clone_handler().handle_unhandled_input_event(e).await }),
404        )
405        .await
406        .into_iter()
407        .flatten()
408        .collect::<Vec<InputEvent>>();
409
410        let expected = IntoIterator::into_iter([
411            get_unhandled_input_event(
412                KeyboardEvent::new(Key::CapsLock, KeyEventType::Pressed)
413                    .into_with_modifiers(Some(Modifiers::CAPS_LOCK))
414                    .into_with_lock_state(Some(LockState::CAPS_LOCK)),
415            ),
416            get_unhandled_input_event(
417                KeyboardEvent::new(Key::CapsLock, KeyEventType::Released)
418                    .into_with_modifiers(Some(Modifiers::from_bits_allow_unknown(0)))
419                    .into_with_lock_state(Some(LockState::CAPS_LOCK)),
420            ),
421            get_unhandled_input_event(
422                KeyboardEvent::new(Key::A, KeyEventType::Pressed)
423                    .into_with_modifiers(Some(Modifiers::from_bits_allow_unknown(0)))
424                    .into_with_lock_state(Some(LockState::CAPS_LOCK)),
425            ),
426            get_unhandled_input_event(
427                KeyboardEvent::new(Key::A, KeyEventType::Released)
428                    .into_with_modifiers(Some(Modifiers::from_bits_allow_unknown(0)))
429                    .into_with_lock_state(Some(LockState::CAPS_LOCK)),
430            ),
431        ])
432        .map(InputEvent::from)
433        .collect::<Vec<_>>();
434        assert_eq!(expected, result);
435    }
436
437    #[fuchsia::test]
438    async fn modifier_handlers_initialized_with_inspect_node() {
439        let inspector = fuchsia_inspect::Inspector::default();
440        let fake_handlers_node = inspector.root().create_child("input_handlers_node");
441        let _modifier_handler =
442            ModifierHandler::new(&fake_handlers_node, metrics::MetricsLogger::default());
443        let _modifier_meaning_handler =
444            ModifierMeaningHandler::new(&fake_handlers_node, metrics::MetricsLogger::default());
445        diagnostics_assertions::assert_data_tree!(inspector, root: {
446            input_handlers_node: {
447                modifier_handler: {
448                    events_received_count: 0u64,
449                    events_handled_count: 0u64,
450                    last_received_timestamp_ns: 0u64,
451                    "fuchsia.inspect.Health": {
452                        status: "STARTING_UP",
453                        // Timestamp value is unpredictable and not relevant in this context,
454                        // so we only assert that the property is present.
455                        start_timestamp_nanos: diagnostics_assertions::AnyProperty
456                    },
457                },
458                modifier_meaning_handler: {
459                    events_received_count: 0u64,
460                    events_handled_count: 0u64,
461                    last_received_timestamp_ns: 0u64,
462                    "fuchsia.inspect.Health": {
463                        status: "STARTING_UP",
464                        // Timestamp value is unpredictable and not relevant in this context,
465                        // so we only assert that the property is present.
466                        start_timestamp_nanos: diagnostics_assertions::AnyProperty
467                    },
468                }
469            }
470        });
471    }
472
473    #[fasync::run_singlethreaded(test)]
474    async fn modifier_handler_inspect_counts_events() {
475        let inspector = fuchsia_inspect::Inspector::default();
476        let fake_handlers_node = inspector.root().create_child("input_handlers_node");
477        let modifier_handler =
478            ModifierHandler::new(&fake_handlers_node, metrics::MetricsLogger::default());
479        let modifier_meaning_handler =
480            ModifierMeaningHandler::new(&fake_handlers_node, metrics::MetricsLogger::default());
481        let device_descriptor =
482            InputDeviceDescriptor::Keyboard(keyboard_binding::KeyboardDeviceDescriptor {
483                keys: vec![Key::A, Key::B, Key::RightAlt],
484                ..Default::default()
485            });
486        let (_, event_time_u64) = testing_utilities::event_times();
487        let input_events = vec![
488            testing_utilities::create_keyboard_event_with_time(
489                Key::A,
490                fidl_fuchsia_ui_input3::KeyEventType::Pressed,
491                None,
492                event_time_u64,
493                &device_descriptor,
494                /* keymap= */ None,
495            ),
496            // Should not count received events that have already been handled.
497            testing_utilities::create_keyboard_event_with_handled(
498                Key::B,
499                fidl_fuchsia_ui_input3::KeyEventType::Pressed,
500                None,
501                event_time_u64,
502                &device_descriptor,
503                /* keymap= */ None,
504                /* key_meaning= */ None,
505                Handled::Yes,
506            ),
507            testing_utilities::create_keyboard_event_with_time(
508                Key::A,
509                fidl_fuchsia_ui_input3::KeyEventType::Released,
510                None,
511                event_time_u64,
512                &device_descriptor,
513                /* keymap= */ None,
514            ),
515            // Should not count non-keyboard input events.
516            testing_utilities::create_fake_input_event(event_time_u64),
517            // Only event that should be counted by ModifierMeaningHandler.
518            testing_utilities::create_keyboard_event_with_key_meaning(
519                Key::RightAlt,
520                fidl_fuchsia_ui_input3::KeyEventType::Pressed,
521                None,
522                event_time_u64,
523                &device_descriptor,
524                /* keymap= */ None,
525                /* key_meaning= */
526                Some(KeyMeaning::NonPrintableKey(NonPrintableKey::AltGraph)),
527            ),
528        ];
529
530        for input_event in input_events {
531            let _ = modifier_handler.clone().handle_input_event(input_event.clone()).await;
532            let _ = modifier_meaning_handler.clone().handle_input_event(input_event).await;
533        }
534
535        let last_event_timestamp: u64 = event_time_u64.into_nanos().try_into().unwrap();
536
537        diagnostics_assertions::assert_data_tree!(inspector, root: {
538            input_handlers_node: {
539                modifier_handler: {
540                    events_received_count: 3u64,
541                    events_handled_count: 0u64,
542                    last_received_timestamp_ns: last_event_timestamp,
543                    "fuchsia.inspect.Health": {
544                        status: "STARTING_UP",
545                        // Timestamp value is unpredictable and not relevant in this context,
546                        // so we only assert that the property is present.
547                        start_timestamp_nanos: diagnostics_assertions::AnyProperty
548                    },
549                },
550                modifier_meaning_handler: {
551                    events_received_count: 1u64,
552                    events_handled_count: 0u64,
553                    last_received_timestamp_ns: last_event_timestamp,
554                    "fuchsia.inspect.Health": {
555                        status: "STARTING_UP",
556                        // Timestamp value is unpredictable and not relevant in this context,
557                        // so we only assert that the property is present.
558                        start_timestamp_nanos: diagnostics_assertions::AnyProperty
559                    },
560                }
561            }
562        });
563    }
564}
input_pipeline/modifier_handler.rs

input_pipeline/
modifier_handler.rs
