carnelian/view/strategies/
flatland.rs

1// Copyright 2021 The Fuchsia Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4use crate::app::{Config, MessageInternal};
5use crate::drawing::DisplayRotation;
6use crate::geometry::UintSize;
7use crate::input::flatland::{FlatlandMouseInputHandler, FlatlandTouchInputHandler};
8use crate::input::key3::KeyboardInputHandler;
9use crate::render::generic::{self, Backend};
10use crate::render::{self, ContextInner};
11use crate::view::strategies::base::{FlatlandParams, ViewStrategy, ViewStrategyPtr};
12use crate::view::{UserInputMessage, ViewAssistantContext, ViewAssistantPtr, ViewDetails, ViewKey};
13use crate::Size;
14use anyhow::{ensure, Context, Error, Result};
15use async_trait::async_trait;
16use async_utils::hanging_get::client::HangingGetStream;
17use display_utils::BufferCollectionId as DisplayBufferCollectionId;
18use euclid::size2;
19use fidl::endpoints::{create_endpoints, create_proxy, create_request_stream};
20use fidl_fuchsia_images2::PixelFormat;
21use fidl_fuchsia_ui_composition as flatland;
22use fidl_fuchsia_ui_views::ViewRef;
23use fuchsia_async::{self as fasync, OnSignals};
24use fuchsia_component::client::connect_to_protocol;
25use fuchsia_framebuffer::sysmem::BufferCollectionAllocator;
26use fuchsia_framebuffer::{FrameSet, FrameUsage, ImageId};
27use fuchsia_scenic::BufferCollectionTokenPair;
28use fuchsia_trace::{duration, flow_begin, instant};
29use futures::channel::mpsc::UnboundedSender;
30use futures::prelude::*;
31use futures::{StreamExt, TryStreamExt};
32use std::collections::{BTreeMap, BTreeSet, HashMap};
33use std::ffi::CStr;
34use zx::{self as zx, Event, HandleBased, MonotonicInstant, Signals};
35
36fn setup_handle_flatland_events(
37    event_stream: flatland::FlatlandEventStream,
38    view_key: ViewKey,
39    app_sender: UnboundedSender<MessageInternal>,
40) {
41    fasync::Task::local(
42        event_stream
43            .try_for_each(move |event| {
44                match event {
45                    flatland::FlatlandEvent::OnNextFrameBegin { values } => {
46                        app_sender
47                            .unbounded_send(MessageInternal::FlatlandOnNextFrameBegin(
48                                view_key, values,
49                            ))
50                            .expect("unbounded_send");
51                    }
52                    flatland::FlatlandEvent::OnFramePresented { frame_presented_info } => {
53                        app_sender
54                            .unbounded_send(MessageInternal::FlatlandOnFramePresented(
55                                view_key,
56                                frame_presented_info,
57                            ))
58                            .expect("unbounded_send");
59                    }
60                    flatland::FlatlandEvent::OnError { error } => {
61                        app_sender
62                            .unbounded_send(MessageInternal::FlatlandOnError(view_key, error))
63                            .expect("unbounded_send");
64                    }
65                };
66                future::ok(())
67            })
68            .unwrap_or_else(|e| eprintln!("error listening for Flatland Events: {:?}", e)),
69    )
70    .detach();
71}
72
73fn duplicate_import_token(
74    token: &fidl_fuchsia_ui_composition::BufferCollectionImportToken,
75) -> Result<fidl_fuchsia_ui_composition::BufferCollectionImportToken, Error> {
76    let value = token.value.duplicate_handle(zx::Rights::SAME_RIGHTS)?;
77    Ok(fidl_fuchsia_ui_composition::BufferCollectionImportToken { value })
78}
79
80struct Plumber {
81    pub size: UintSize,
82    pub collection_id: u32,
83    pub frame_set: FrameSet,
84    pub image_indexes: BTreeMap<ImageId, u32>,
85    pub context: render::Context,
86}
87
88impl Plumber {
89    async fn new(
90        flatland: &flatland::FlatlandProxy,
91        allocator: &flatland::AllocatorProxy,
92        size: UintSize,
93        pixel_format: PixelFormat,
94        buffer_count: usize,
95        collection_id: u32,
96        first_image_id: u64,
97    ) -> Result<Plumber, Error> {
98        let use_spinel = Config::get().use_spinel;
99
100        ensure!(use_spinel == false, "Spinel support is disabled");
101
102        let usage = if use_spinel { FrameUsage::Gpu } else { FrameUsage::Cpu };
103        let mut buffer_allocator = BufferCollectionAllocator::new(
104            size.width,
105            size.height,
106            pixel_format,
107            usage,
108            buffer_count,
109        )?;
110
111        buffer_allocator.set_name(100, "CarnelianSurface")?;
112
113        let context_token = buffer_allocator.duplicate_token().await?;
114        let mut context = render::Context {
115            inner: ContextInner::Forma(generic::Forma::new_context(
116                context_token,
117                size,
118                DisplayRotation::Deg0,
119            )),
120        };
121
122        let sysmem_buffer_collection_token = buffer_allocator.duplicate_token().await?;
123        let buffer_tokens = BufferCollectionTokenPair::new();
124        let args = flatland::RegisterBufferCollectionArgs {
125            export_token: Some(buffer_tokens.export_token),
126            // A sysmem token channel serves both sysmem(1) and sysmem2, so we can convert here
127            // until flatland has a field for a sysmem2 token.
128            buffer_collection_token2: Some(sysmem_buffer_collection_token),
129            ..Default::default()
130        };
131
132        allocator
133            .register_buffer_collection(args)
134            .await
135            .expect("fidl error")
136            .expect("error registering buffer collection");
137
138        let buffers = buffer_allocator.allocate_buffers(true).await.context("allocate_buffers")?;
139
140        let blend_mode = if Config::get().needs_blending {
141            flatland::BlendMode::SrcOver
142        } else {
143            flatland::BlendMode::Src
144        };
145        let mut image_ids = BTreeSet::new();
146        let mut image_indexes = BTreeMap::new();
147        let buffer_count = buffers.buffers.as_ref().unwrap().len();
148        for index in 0..buffer_count as usize {
149            let image_id = index + first_image_id as usize;
150            image_ids.insert(image_id as u64);
151            let uindex = index as u32;
152            image_indexes.insert(image_id as u64, uindex);
153            let image_props = flatland::ImageProperties {
154                size: Some(fidl_fuchsia_math::SizeU { width: size.width, height: size.height }),
155                ..Default::default()
156            };
157            let flatland_image_id = flatland::ContentId { value: image_id as u64 };
158            let import_token = duplicate_import_token(&buffer_tokens.import_token)?;
159            flatland.create_image(&flatland_image_id, import_token, uindex, &image_props)?;
160            flatland
161                .set_image_destination_size(
162                    &flatland_image_id,
163                    &fidl_fuchsia_math::SizeU { width: size.width, height: size.height },
164                )
165                .expect("fidl error");
166            flatland
167                .set_image_blending_function(&flatland_image_id, blend_mode)
168                .expect("fidl error");
169            // Get all the images at this point, since if we wait until we need them for
170            // rendering, Forma's private connection to sysmem has closed and it fails.
171            // https://bugs.fuchsia.dev/p/fuchsia/issues/detail?id=83543
172            context.get_image(index as u32);
173        }
174
175        let frame_set = FrameSet::new(DisplayBufferCollectionId(collection_id as u64), image_ids);
176        Ok(Plumber { size, collection_id, frame_set, image_indexes, context })
177    }
178
179    pub fn enter_retirement(&mut self, _flatland: &flatland::FlatlandProxy) {}
180}
181
182// If the first one changes, the second one should too.
183pub const FLATLAND_DEBUG_NAME: &str = "Carnelian View";
184pub static FLATLAND_DEBUG_FLOW_NAME: &CStr = c"Flatland::PerAppPresent[Carnelian View]";
185
186const RENDER_BUFFER_COUNT: usize = 3;
187const DEFAULT_PRESENT_INTERVAL: i64 = (1_000_000_000.0 / 60.0) as i64;
188const TRANSFORM_ID: flatland::TransformId = flatland::TransformId { value: 1 };
189
190#[derive(Clone, Copy)]
191struct PresentationTime {
192    presentation_time: i64,
193    latch_point: i64,
194}
195
196pub(crate) struct FlatlandViewStrategy {
197    flatland: flatland::FlatlandProxy,
198    allocator: flatland::AllocatorProxy,
199    view_key: ViewKey,
200    last_presentation_time: i64,
201    future_presentation_times: Vec<PresentationTime>,
202    custom_render_offset: Option<i64>,
203    present_interval: i64,
204    // The number of `Flatland.Present()` calls we're authorized to make.  Initial value is 1.
205    // Decremented whenever `Present()` is called, and incremented whenever Scenic authorizes us to.
206    num_presents_allowed: usize,
207    // Only used for tracing, does not affect behavior.
208    present_count: u64,
209    // Only used for tracing, does not affect behavior.
210    pending_present_count: usize,
211    render_timer_scheduled: bool,
212    missed_frame: bool,
213    app_sender: UnboundedSender<MessageInternal>,
214    next_buffer_collection: u32,
215    plumber: Option<Plumber>,
216    retiring_plumbers: Vec<Plumber>,
217    next_image_id: u64,
218    previous_present_release_event: Option<Event>,
219    current_present_release_event: Option<Event>,
220    flatland_mouse_input_handlers: HashMap<u32, FlatlandMouseInputHandler>,
221    flatland_touch_input_handler: FlatlandTouchInputHandler,
222    keyboard_handler: KeyboardInputHandler,
223}
224
225impl FlatlandViewStrategy {
226    pub(crate) async fn new(
227        key: ViewKey,
228        flatland_params: FlatlandParams,
229        app_sender: UnboundedSender<MessageInternal>,
230    ) -> Result<ViewStrategyPtr, Error> {
231        let flatland = connect_to_protocol::<flatland::FlatlandMarker>()?;
232        if let Some(debug_name) = flatland_params.debug_name {
233            flatland.set_debug_name(&debug_name)?;
234        }
235
236        // Add trace event with the Flatland channel's koid, so that it can be correlated with a
237        // Flatland session within Scenic.
238        {
239            use fidl::endpoints::Proxy;
240            use zx::AsHandleRef;
241
242            let koid = flatland.as_channel().get_koid().unwrap().raw_koid();
243            instant!(
244                c"gfx",
245                c"FlatlandViewStrategy::new",
246                fuchsia_trace::Scope::Process,
247                "flatland_koid" => koid
248            );
249        }
250
251        flatland.create_transform(&TRANSFORM_ID)?;
252        flatland.set_root_transform(&TRANSFORM_ID)?;
253        setup_handle_flatland_events(flatland.take_event_stream(), key, app_sender.clone());
254        let allocator = connect_to_protocol::<flatland::AllocatorMarker>()?;
255        Self::create_parent_viewport_watcher(
256            &flatland,
257            app_sender.clone(),
258            key,
259            flatland_params.args.view_creation_token.expect("view_creation_token"),
260        )?;
261
262        let strat = Self {
263            flatland,
264            allocator,
265            view_key: key,
266            app_sender: app_sender,
267            last_presentation_time: fasync::MonotonicInstant::now().into_nanos(),
268            future_presentation_times: Vec::new(),
269            custom_render_offset: None,
270            present_interval: DEFAULT_PRESENT_INTERVAL,
271            num_presents_allowed: 1,
272            present_count: 0,
273            pending_present_count: 0,
274            render_timer_scheduled: false,
275            missed_frame: false,
276            plumber: None,
277            retiring_plumbers: Vec::new(),
278            next_buffer_collection: 1,
279            next_image_id: 1,
280            previous_present_release_event: None,
281            current_present_release_event: None,
282            flatland_mouse_input_handlers: HashMap::new(),
283            flatland_touch_input_handler: FlatlandTouchInputHandler::default(),
284            keyboard_handler: KeyboardInputHandler::new(),
285        };
286
287        Ok(Box::new(strat))
288    }
289
290    fn create_parent_viewport_watcher(
291        flatland: &flatland::FlatlandProxy,
292        app_sender: UnboundedSender<MessageInternal>,
293        view_key: ViewKey,
294        view_creation_token: fidl_fuchsia_ui_views::ViewCreationToken,
295    ) -> Result<(), Error> {
296        let (parent_viewport_watcher, server_end) =
297            create_proxy::<flatland::ParentViewportWatcherMarker>();
298
299        let viewref_pair = fuchsia_scenic::ViewRefPair::new()?;
300        let view_ref = fuchsia_scenic::duplicate_view_ref(&viewref_pair.view_ref)?;
301
302        // Use CreateView2 if input events are expected.
303        let input = Config::get().input;
304        if input {
305            let view_identity = fidl_fuchsia_ui_views::ViewIdentityOnCreation::from(viewref_pair);
306            let mut view_bound_protocols = flatland::ViewBoundProtocols::default();
307
308            let (touch_client, touch_server) = create_endpoints();
309            let (mouse_client, mouse_server) = create_endpoints();
310            let (view_ref_focused_client, view_ref_focused_server) = create_endpoints();
311
312            view_bound_protocols.touch_source = Some(touch_server);
313            view_bound_protocols.mouse_source = Some(mouse_server);
314            view_bound_protocols.view_ref_focused = Some(view_ref_focused_server);
315
316            flatland.create_view2(
317                view_creation_token,
318                view_identity,
319                view_bound_protocols,
320                server_end,
321            )?;
322
323            let touch_proxy = touch_client.into_proxy();
324            let touch_sender = app_sender.clone();
325
326            fasync::Task::local(async move {
327                let mut events: Vec<fidl_fuchsia_ui_pointer::TouchResponse> = Vec::new();
328                loop {
329                    let result = touch_proxy.watch(&events).await;
330                    match result {
331                        Ok(returned_events) => {
332                            events = returned_events
333                                .iter()
334                                .map(|event| fidl_fuchsia_ui_pointer::TouchResponse {
335                                    trace_flow_id: event
336                                        .pointer_sample
337                                        .as_ref()
338                                        .and(event.trace_flow_id),
339                                    response_type: event.pointer_sample.as_ref().and_then(|_| {
340                                        Some(fidl_fuchsia_ui_pointer::TouchResponseType::Yes)
341                                    }),
342                                    ..Default::default()
343                                })
344                                .collect();
345                            touch_sender
346                                .unbounded_send(MessageInternal::UserInputMessage(
347                                    view_key,
348                                    UserInputMessage::FlatlandTouchEvents(returned_events),
349                                ))
350                                .expect("failed to send MessageInternal.");
351                        }
352                        Err(fidl::Error::ClientChannelClosed { .. }) => {
353                            println!("touch event connection closed.");
354                            return;
355                        }
356                        Err(fidl_error) => {
357                            println!("touch event connection closed error: {:?}", fidl_error);
358                            return;
359                        }
360                    }
361                }
362            })
363            .detach();
364
365            let mouse_proxy = mouse_client.into_proxy();
366            let mouse_sender = app_sender.clone();
367
368            fasync::Task::local(async move {
369                loop {
370                    let result = mouse_proxy.watch().await;
371                    match result {
372                        Ok(returned_events) => {
373                            mouse_sender
374                                .unbounded_send(MessageInternal::UserInputMessage(
375                                    view_key,
376                                    UserInputMessage::FlatlandMouseEvents(returned_events),
377                                ))
378                                .expect("failed to send MessageInternal.");
379                        }
380                        Err(fidl::Error::ClientChannelClosed { .. }) => {
381                            println!("mouse event connection closed.");
382                            return;
383                        }
384                        Err(fidl_error) => {
385                            println!("mouse event connection closed error: {:?}", fidl_error);
386                            return;
387                        }
388                    }
389                }
390            })
391            .detach();
392
393            let view_ref_focused_proxy = view_ref_focused_client.into_proxy();
394            let view_ref_focused_sender = app_sender.clone();
395
396            fasync::Task::local(async move {
397                loop {
398                    let result = view_ref_focused_proxy.watch().await;
399                    match result {
400                        Ok(msg) => {
401                            view_ref_focused_sender
402                                .unbounded_send(MessageInternal::Focus(
403                                    view_key,
404                                    msg.focused.unwrap_or(false),
405                                ))
406                                .expect("failed to send MessageInternal.");
407                        }
408                        Err(fidl::Error::ClientChannelClosed { .. }) => {
409                            println!("ViewRefFocused connection closed.");
410                            return;
411                        }
412                        Err(fidl_error) => {
413                            println!("ViewRefFocused connection closed error: {:?}", fidl_error);
414                            return;
415                        }
416                    }
417                }
418            })
419            .detach();
420
421            Self::listen_for_key_events(view_ref, &app_sender, view_key)?;
422        } else {
423            flatland.create_view(view_creation_token, server_end)?;
424        }
425
426        let sender = app_sender;
427        fasync::Task::local(async move {
428            let mut layout_info_stream = HangingGetStream::new(
429                parent_viewport_watcher,
430                flatland::ParentViewportWatcherProxy::get_layout,
431            );
432
433            while let Some(result) = layout_info_stream.next().await {
434                match result {
435                    Ok(layout_info) => {
436                        if let Some(fidl_fuchsia_math::SizeU { width, height }) =
437                            layout_info.logical_size
438                        {
439                            sender
440                                .unbounded_send(MessageInternal::SizeChanged(
441                                    view_key,
442                                    size2(width, height).to_f32(),
443                                ))
444                                .expect("failed to send MessageInternal.");
445                        }
446                        if let Some(fidl_fuchsia_math::VecF { x, y }) =
447                            layout_info.device_pixel_ratio
448                        {
449                            sender
450                                .unbounded_send(MessageInternal::MetricsChanged(
451                                    view_key,
452                                    size2(x, y),
453                                ))
454                                .expect("failed to send MessageInternal.");
455                        }
456                    }
457                    Err(fidl::Error::ClientChannelClosed { .. }) => {
458                        println!("graph link connection closed.");
459                        return; // from spawned task closure
460                    }
461                    Err(fidl_error) => {
462                        println!("graph link GetLayout() error: {:?}", fidl_error);
463                        return; // from spawned task closure
464                    }
465                }
466            }
467        })
468        .detach();
469        Ok(())
470    }
471
472    fn do_present(
473        flatland: &flatland::FlatlandProxy,
474        _app_sender: &UnboundedSender<MessageInternal>,
475        _key: ViewKey,
476        presentation_time: i64,
477        release_event: Option<Event>,
478        present_count: u64,
479    ) {
480        flow_begin!(c"gfx", FLATLAND_DEBUG_FLOW_NAME, fuchsia_trace::Id::from(present_count));
481
482        flatland
483            .present(flatland::PresentArgs {
484                requested_presentation_time: Some(presentation_time),
485                acquire_fences: None,
486                release_fences: release_event.and_then(|release_event| Some(vec![release_event])),
487                unsquashable: Some(true),
488                ..Default::default()
489            })
490            .expect("present error");
491    }
492
493    fn make_view_assistant_context_with_time(
494        view_details: &ViewDetails,
495        image_id: ImageId,
496        image_index: u32,
497        app_sender: UnboundedSender<MessageInternal>,
498        presentation_time: MonotonicInstant,
499    ) -> ViewAssistantContext {
500        ViewAssistantContext {
501            key: view_details.key,
502            size: view_details.logical_size,
503            metrics: view_details.metrics,
504            presentation_time,
505            buffer_count: None,
506            image_id,
507            image_index,
508            app_sender,
509            mouse_cursor_position: None,
510            display_info: None,
511        }
512    }
513
514    fn make_view_assistant_context(
515        view_details: &ViewDetails,
516        image_id: ImageId,
517        image_index: u32,
518        app_sender: UnboundedSender<MessageInternal>,
519    ) -> ViewAssistantContext {
520        Self::make_view_assistant_context_with_time(
521            view_details,
522            image_id,
523            image_index,
524            app_sender,
525            MonotonicInstant::get(),
526        )
527    }
528
529    async fn create_plumber(&mut self, size: UintSize) -> Result<(), Error> {
530        let buffer_collection_id = self.next_buffer_collection;
531        self.next_buffer_collection = self.next_buffer_collection.wrapping_add(1);
532        let next_image_id = self.next_image_id;
533        self.next_image_id = self.next_image_id.wrapping_add(RENDER_BUFFER_COUNT as u64);
534        self.plumber = Some(
535            Plumber::new(
536                &self.flatland,
537                &self.allocator,
538                size.to_u32(),
539                fidl_fuchsia_images2::PixelFormat::B8G8R8A8,
540                RENDER_BUFFER_COUNT,
541                buffer_collection_id,
542                next_image_id,
543            )
544            .await
545            .expect("VmoPlumber::new"),
546        );
547        Ok(())
548    }
549
550    fn next_presentation_time(&self) -> PresentationTime {
551        let now = fasync::MonotonicInstant::now().into_nanos();
552        let legal_next = PresentationTime {
553            presentation_time: self.last_presentation_time + self.present_interval,
554            latch_point: now,
555        };
556        // Find a future presentation time that is at least half an interval past
557        // the last presentation time and has a latch point that is in the future.
558        let earliest_presentation_time = self.last_presentation_time + self.present_interval / 2;
559        let next = self
560            .future_presentation_times
561            .iter()
562            .find(|t| t.presentation_time >= earliest_presentation_time && t.latch_point > now)
563            .unwrap_or(&legal_next);
564        *next
565    }
566
567    fn schedule_render_timer(&mut self) {
568        if !self.render_timer_scheduled && !self.missed_frame {
569            let presentation_time = self.next_presentation_time();
570            // Conservative render offset to prefer throughput over low-latency.
571            // TODO: allow applications that prefer low-latency to control this
572            // dynamically.
573            const DEFAULT_RENDER_OFFSET_DELTA: i64 = 1_000_000; // 1ms
574            let render_offset = self
575                .custom_render_offset
576                .unwrap_or_else(|| self.present_interval - DEFAULT_RENDER_OFFSET_DELTA);
577            let render_time = presentation_time.latch_point - render_offset;
578            let timer =
579                fasync::Timer::new(fuchsia_async::MonotonicInstant::from_nanos(render_time));
580            let timer_sender = self.app_sender.clone();
581            let key = self.view_key;
582            fasync::Task::local(async move {
583                timer.await;
584                timer_sender.unbounded_send(MessageInternal::Render(key)).expect("unbounded_send");
585            })
586            .detach();
587            self.render_timer_scheduled = true;
588        }
589    }
590
591    fn render_to_image_from_plumber(
592        &mut self,
593        view_details: &ViewDetails,
594        view_assistant: &mut ViewAssistantPtr,
595    ) -> bool {
596        let presentation_time = self.next_presentation_time();
597        let plumber = self.plumber.as_mut().expect("plumber");
598        if let Some(available) = plumber.frame_set.get_available_image() {
599            duration!(c"gfx", c"FlatlandViewStrategy::render.render_to_image");
600            let available_index = plumber.image_indexes.get(&available).expect("index for image");
601            let render_context = Self::make_view_assistant_context_with_time(
602                view_details,
603                available,
604                *available_index,
605                self.app_sender.clone(),
606                MonotonicInstant::from_nanos(presentation_time.presentation_time),
607            );
608            let buffer_ready_event = Event::create();
609            view_assistant
610                .render(&mut plumber.context, buffer_ready_event, &render_context)
611                .unwrap_or_else(|e| panic!("Update error: {:?}", e));
612            plumber.frame_set.mark_prepared(available);
613            let key = view_details.key;
614            let collection_id = plumber.collection_id;
615            let release_event = Event::create();
616            let local_release_event =
617                release_event.duplicate_handle(zx::Rights::SAME_RIGHTS).expect("duplicate_handle");
618            let app_sender = self.app_sender.clone();
619            fasync::Task::local(async move {
620                let signals = OnSignals::new(&local_release_event, Signals::EVENT_SIGNALED);
621                signals.await.expect("to wait");
622                app_sender
623                    .unbounded_send(MessageInternal::ImageFreed(key, available, collection_id))
624                    .expect("unbounded_send");
625            })
626            .detach();
627            self.current_present_release_event = Some(release_event);
628
629            let image_id = flatland::ContentId { value: available };
630            self.flatland.set_content(&TRANSFORM_ID, &image_id).expect("fidl error");
631
632            // Image is guaranteed to be presented at this point.
633            plumber.frame_set.mark_presented(available);
634            true
635        } else {
636            instant!(
637                c"gfx",
638                c"FlatlandViewStrategy::no_available_image",
639                fuchsia_trace::Scope::Process
640            );
641            self.missed_frame = true;
642            false
643        }
644    }
645
646    fn retry_missed_frame(&mut self) {
647        if self.missed_frame {
648            self.missed_frame = false;
649            self.render_requested();
650        }
651    }
652
653    fn listen_for_key_events(
654        view_ref: ViewRef,
655        app_sender: &UnboundedSender<MessageInternal>,
656        key: ViewKey,
657    ) -> Result<(), Error> {
658        let keyboard = connect_to_protocol::<fidl_fuchsia_ui_input3::KeyboardMarker>()
659            .context("Failed to connect to Keyboard service")?;
660
661        let (listener_client_end, mut listener_stream) =
662            create_request_stream::<fidl_fuchsia_ui_input3::KeyboardListenerMarker>();
663
664        let event_sender = app_sender.clone();
665
666        fasync::Task::local(async move {
667            keyboard.add_listener(view_ref, listener_client_end).await.expect("add_listener");
668
669            while let Some(event) =
670                listener_stream.try_next().await.expect("Failed to get next key event")
671            {
672                match event {
673                    fidl_fuchsia_ui_input3::KeyboardListenerRequest::OnKeyEvent {
674                        event,
675                        responder,
676                        ..
677                    } => {
678                        // Carnelian always considers key events handled. In the future, there
679                        // might be a use case that requires letting view assistants change
680                        // this behavior but none currently exists.
681                        responder
682                            .send(fidl_fuchsia_ui_input3::KeyEventStatus::Handled)
683                            .expect("send");
684                        event_sender
685                            .unbounded_send(MessageInternal::UserInputMessage(
686                                key,
687                                UserInputMessage::ScenicKeyEvent(event),
688                            ))
689                            .expect("unbounded_send");
690                    }
691                }
692            }
693        })
694        .detach();
695        Ok(())
696    }
697}
698
699#[async_trait(?Send)]
700impl ViewStrategy for FlatlandViewStrategy {
701    fn initial_metrics(&self) -> Size {
702        size2(1.0, 1.0)
703    }
704
705    fn create_view_assistant_context(&self, view_details: &ViewDetails) -> ViewAssistantContext {
706        ViewAssistantContext {
707            key: view_details.key,
708            size: view_details.logical_size,
709            metrics: view_details.metrics,
710            presentation_time: Default::default(),
711            buffer_count: None,
712            image_id: Default::default(),
713            image_index: Default::default(),
714            app_sender: self.app_sender.clone(),
715            mouse_cursor_position: None,
716            display_info: None,
717        }
718    }
719
720    fn setup(&mut self, view_details: &ViewDetails, view_assistant: &mut ViewAssistantPtr) {
721        duration!(c"gfx", c"FlatlandViewStrategy::setup");
722        let render_context =
723            Self::make_view_assistant_context(view_details, 0, 0, self.app_sender.clone());
724        view_assistant.setup(&render_context).unwrap_or_else(|e| panic!("Setup error: {:?}", e));
725        self.custom_render_offset = view_assistant.get_render_offset();
726        self.render_requested();
727    }
728
729    async fn render(
730        &mut self,
731        view_details: &ViewDetails,
732        view_assistant: &mut ViewAssistantPtr,
733    ) -> bool {
734        let size = view_details.physical_size.floor().to_u32();
735        duration!(c"gfx", c"FlatlandViewStrategy::render",
736            "width" => size.width,
737            "height" => size.height);
738
739        self.render_timer_scheduled = false;
740        if size.width > 0 && size.height > 0 {
741            if self.num_presents_allowed == 0 {
742                instant!(c"gfx", c"FlatlandViewStrategy::present_is_not_allowed",
743                    fuchsia_trace::Scope::Process,
744                    "counts" => format!("{} pending {} allowed",
745                        self.pending_present_count,
746                        self.num_presents_allowed).as_str());
747                self.missed_frame = true;
748                return false;
749            }
750
751            if self.plumber.is_none() {
752                duration!(c"gfx", c"FlatlandViewStrategy::render.create_plumber");
753                self.create_plumber(size).await.expect("create_plumber");
754            } else {
755                let current_size = self.plumber.as_ref().expect("plumber").size;
756                if current_size != size {
757                    duration!(c"gfx", c"FlatlandViewStrategy::render.create_plumber");
758                    let retired_plumber = self.plumber.take().expect("plumber");
759                    self.retiring_plumbers.push(retired_plumber);
760                    self.create_plumber(size).await.expect("create_plumber");
761                }
762            }
763            self.render_to_image_from_plumber(view_details, view_assistant)
764        } else {
765            true
766        }
767    }
768
769    fn present(&mut self, _view_details: &ViewDetails) {
770        duration!(c"gfx", c"FlatlandViewStrategy::present");
771        if !self.missed_frame {
772            assert!(self.num_presents_allowed > 0);
773            let release_event = self.previous_present_release_event.take();
774            let presentation_time = self.next_presentation_time();
775            Self::do_present(
776                &self.flatland,
777                &self.app_sender,
778                self.view_key,
779                presentation_time.presentation_time,
780                release_event,
781                self.present_count,
782            );
783            self.last_presentation_time = presentation_time.presentation_time;
784            self.present_count += 1;
785            self.pending_present_count += 1;
786            self.num_presents_allowed -= 1;
787            self.previous_present_release_event = self.current_present_release_event.take();
788        }
789    }
790
791    fn present_done(
792        &mut self,
793        _view_details: &ViewDetails,
794        _view_assistant: &mut ViewAssistantPtr,
795        info: fidl_fuchsia_scenic_scheduling::FramePresentedInfo,
796    ) {
797        let num_presents_handled = info.presentation_infos.len();
798        assert!(self.pending_present_count >= num_presents_handled);
799        self.pending_present_count -= num_presents_handled;
800        instant!(
801            c"gfx",
802            c"FlatlandViewStrategy::present_done",
803            fuchsia_trace::Scope::Process,
804            "presents" => format!("{} handled", num_presents_handled).as_str()
805        );
806    }
807
808    fn handle_focus(
809        &mut self,
810        view_details: &ViewDetails,
811        view_assistant: &mut ViewAssistantPtr,
812        focus: bool,
813    ) {
814        let mut render_context =
815            Self::make_view_assistant_context(view_details, 0, 0, self.app_sender.clone());
816        view_assistant
817            .handle_focus_event(&mut render_context, focus)
818            .unwrap_or_else(|e| panic!("handle_focus error: {:?}", e));
819    }
820
821    fn convert_user_input_message(
822        &mut self,
823        _view_details: &ViewDetails,
824        message: UserInputMessage,
825    ) -> Result<Vec<crate::input::Event>, Error> {
826        match message {
827            UserInputMessage::ScenicKeyEvent(key_event) => {
828                let converted_events = self.keyboard_handler.handle_key_event(&key_event);
829                Ok(converted_events)
830            }
831            UserInputMessage::FlatlandMouseEvents(mouse_events) => {
832                let mut events = Vec::new();
833                for mouse_event in &mouse_events {
834                    if let Some(pointer_sample) = mouse_event.pointer_sample.as_ref() {
835                        let device_id = pointer_sample.device_id.expect("device_id");
836                        let handler_entry = self
837                            .flatland_mouse_input_handlers
838                            .entry(device_id)
839                            .or_insert_with(|| FlatlandMouseInputHandler::new(device_id));
840                        events.extend(handler_entry.handle_mouse_events(&mouse_events));
841                    }
842                }
843                Ok(events)
844            }
845            UserInputMessage::FlatlandTouchEvents(touch_events) => {
846                Ok(self.flatland_touch_input_handler.handle_events(&touch_events))
847            }
848        }
849    }
850
851    fn image_freed(&mut self, image_id: u64, collection_id: u32) {
852        instant!(
853            c"gfx",
854            c"FlatlandViewStrategy::image_freed",
855            fuchsia_trace::Scope::Process,
856            "image_freed" => format!("{} in {}", image_id, collection_id).as_str()
857        );
858
859        self.retry_missed_frame();
860
861        if let Some(plumber) = self.plumber.as_mut() {
862            if plumber.collection_id == collection_id {
863                plumber.frame_set.mark_done_presenting(image_id);
864                return;
865            }
866        }
867
868        for retired_plumber in &mut self.retiring_plumbers {
869            if retired_plumber.collection_id == collection_id {
870                retired_plumber.frame_set.mark_done_presenting(image_id);
871                if retired_plumber.frame_set.no_images_in_use() {
872                    retired_plumber.enter_retirement(&self.flatland);
873                }
874            }
875        }
876
877        self.retiring_plumbers.retain(|plumber| !plumber.frame_set.no_images_in_use());
878    }
879
880    fn render_requested(&mut self) {
881        self.schedule_render_timer();
882    }
883
884    fn handle_on_next_frame_begin(
885        &mut self,
886        info: &fidl_fuchsia_ui_composition::OnNextFrameBeginValues,
887    ) {
888        self.num_presents_allowed += info.additional_present_credits.unwrap_or(0) as usize;
889        let future_presentation_infos =
890            info.future_presentation_infos.as_ref().expect("future_presentation_infos");
891        let present_intervals = future_presentation_infos.len();
892        instant!(
893            c"gfx",
894            c"FlatlandViewStrategy::handle_on_next_frame_begin",
895            fuchsia_trace::Scope::Process,
896            "counts" => format!("{} present_intervals", present_intervals).as_str()
897        );
898        if present_intervals > 0 {
899            let times: Vec<_> = future_presentation_infos
900                .iter()
901                .filter_map(|info| info.presentation_time)
902                .collect();
903            let average_interval: i64 =
904                times.as_slice().windows(2).map(|slice| slice[1] - slice[0]).sum::<i64>()
905                    / present_intervals as i64;
906            self.present_interval = average_interval;
907        } else {
908            self.present_interval = DEFAULT_PRESENT_INTERVAL;
909        }
910        self.future_presentation_times.splice(
911            ..,
912            future_presentation_infos.iter().map(|t| PresentationTime {
913                presentation_time: t.presentation_time.expect("presentation_time"),
914                latch_point: t.latch_point.expect("latch_point"),
915            }),
916        );
917
918        // Now that we have incremented `self.num_presents_allowed`, a missed frame might succeed.
919        self.retry_missed_frame();
920    }
921}