attribution_processing/
digest.rs

1// Copyright 2025 The Fuchsia Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5use crate::fplugin::Vmo;
6use crate::{AttributionDataProvider, ResourcesVisitor, ZXName};
7use regex::bytes::Regex;
8use serde::de::Error;
9use serde::{Deserialize, Deserializer};
10use std::collections::hash_map::Entry::Occupied;
11use std::collections::HashMap;
12use {fidl_fuchsia_kernel as fkernel, fidl_fuchsia_memory_attribution_plugin as fplugin};
13
14const UNDIGESTED: &str = "Undigested";
15const ORPHANED: &str = "Orphaned";
16const KERNEL: &str = "Kernel";
17const FREE: &str = "Free";
18const PAGER_TOTAL: &str = "[Addl]PagerTotal";
19const PAGER_NEWEST: &str = "[Addl]PagerNewest";
20const PAGER_OLDEST: &str = "[Addl]PagerOldest";
21const DISCARDABLE_LOCKED: &str = "[Addl]DiscardableLocked";
22const DISCARDABLE_UNLOCKED: &str = "[Addl]DiscardableUnlocked";
23const ZRAM_COMPRESSED_BYTES: &str = "[Addl]ZramCompressedBytes";
24
25/// Represents a specification for aggregating memory usage in meaningful groups.
26///
27/// `name` represents the meaningful name of the group; grouping is done based on process and VMO
28/// names.
29///
30// Note: This needs to mirror `//src/lib/assembly/memory_buckets/src/memory_buckets.rs`, but cannot
31// reuse it directly because it is an host-only library.
32#[derive(Clone, Debug, Deserialize)]
33pub struct BucketDefinition {
34    pub name: String,
35    #[serde(deserialize_with = "deserialize_regex")]
36    pub process: Option<Regex>,
37    #[serde(deserialize_with = "deserialize_regex")]
38    pub vmo: Option<Regex>,
39    pub event_code: u64,
40}
41
42impl BucketDefinition {
43    /// Tests whether a process matches this bucket's definition, based on its name.
44    fn process_match(&self, process: &ZXName) -> bool {
45        self.process.as_ref().map_or(true, |p| p.is_match(process.as_bstr()))
46    }
47
48    /// Tests whether a VMO matches this bucket's definition, based on its name.
49    fn vmo_match(&self, vmo: &ZXName) -> bool {
50        self.vmo.as_ref().map_or(true, |v| v.is_match(vmo.as_bstr()))
51    }
52}
53
54// Teach serde to deserialize an optional regex.
55fn deserialize_regex<'de, D>(d: D) -> Result<Option<Regex>, D::Error>
56where
57    D: Deserializer<'de>,
58{
59    // Deserialize as Option<&str>
60    Option::<String>::deserialize(d)
61        // If the parsing failed, return the error, otherwise transform the value
62        .and_then(|os| {
63            os
64                // If there is a value, try to parse it as a Regex.
65                .map(|s| {
66                    Regex::new(&s)
67                        // If the regex compilation failed, wrap the error in the error type expected
68                        // by serde.
69                        .map_err(D::Error::custom)
70                })
71                // If there was a value but it failed to compile, return an error, otherwise return
72                // the potentially parsed option.
73                .transpose()
74        })
75}
76
77/// Aggregates bytes in categories with human readable names.
78#[derive(Clone, Debug, PartialEq, Eq)]
79pub struct Bucket {
80    pub name: String,
81    pub size: u64,
82}
83/// Contains a view of the system's memory usage, aggregated in groups called buckets, which are
84/// configurable.
85#[derive(Debug, PartialEq, Eq)]
86pub struct Digest {
87    pub buckets: Vec<Bucket>,
88}
89
90/// Compute a bucket digest as the Jobs->Processes->VMOs tree is traversed.
91struct DigestComputer<'a> {
92    // Ordered pair with a bucket specification, and the current bucket result.
93    buckets: Vec<(&'a BucketDefinition, Bucket)>,
94    // Set of VMOs what didn't fell in any bucket.
95    undigested_vmos: HashMap<zx_types::zx_koid_t, (Vmo, ZXName)>,
96}
97
98impl<'a> DigestComputer<'a> {
99    fn new(bucket_definitions: &'a [BucketDefinition]) -> DigestComputer<'a> {
100        DigestComputer {
101            buckets: bucket_definitions
102                .iter()
103                .map(|def| (def, Bucket { name: def.name.clone(), size: 0 }))
104                .collect(),
105            undigested_vmos: Default::default(),
106        }
107    }
108}
109
110impl ResourcesVisitor for DigestComputer<'_> {
111    fn on_job(
112        &mut self,
113        _job_koid: zx_types::zx_koid_t,
114        _job_name: &ZXName,
115        _job: fplugin::Job,
116    ) -> Result<(), zx_status::Status> {
117        Ok(())
118    }
119
120    fn on_process(
121        &mut self,
122        _process_koid: zx_types::zx_koid_t,
123        process_name: &ZXName,
124        process: fplugin::Process,
125    ) -> Result<(), zx_status::Status> {
126        for (bucket_definition, bucket) in self.buckets.iter_mut() {
127            if bucket_definition.process_match(process_name) {
128                for koid in process.vmos.iter().flatten() {
129                    bucket.size += match self.undigested_vmos.entry(*koid) {
130                        Occupied(e) => {
131                            let (_vmo, name) = e.get();
132                            if bucket_definition.vmo_match(&name) {
133                                let (_, (vmo, _name)) = e.remove_entry();
134                                vmo.total_committed_bytes.unwrap_or_default()
135                            } else {
136                                0
137                            }
138                        }
139                        _ => 0,
140                    };
141                }
142            }
143        }
144        Ok(())
145    }
146
147    fn on_vmo(
148        &mut self,
149        vmo_koid: zx_types::zx_koid_t,
150        vmo_name: &ZXName,
151        vmo: fplugin::Vmo,
152    ) -> Result<(), zx_status::Status> {
153        self.undigested_vmos.insert(vmo_koid, (vmo, vmo_name.clone()));
154        Ok(())
155    }
156}
157
158impl Digest {
159    /// Given means to query the system for memory usage, and a specification, this function
160    /// aggregates the current memory usage into human displayable units we call buckets.
161    pub fn compute(
162        attribution_data_service: &impl AttributionDataProvider,
163        kmem_stats: &fkernel::MemoryStats,
164        kmem_stats_compression: &fkernel::MemoryStatsCompression,
165        bucket_definitions: &[BucketDefinition],
166    ) -> Result<Digest, anyhow::Error> {
167        let mut digest_visitor = DigestComputer::new(bucket_definitions);
168        attribution_data_service.for_each_resource(&mut digest_visitor)?;
169        let mut buckets: Vec<Bucket> =
170            digest_visitor.buckets.drain(..).map(|(_, bucket)| bucket).collect();
171
172        let vmo_size: u64 = buckets.iter().map(|Bucket { size, .. }| size).sum();
173        // Extend the configured aggregation with a number of additional, occasionally useful meta
174        // aggregations.
175        buckets.extend(vec![
176            // This bucket contains the total size of the VMOs that have not been covered by any
177            // other bucket.
178            Bucket {
179                name: UNDIGESTED.to_string(),
180                size: digest_visitor
181                    .undigested_vmos
182                    .values()
183                    .filter_map(|(vmo, _)| vmo.total_committed_bytes)
184                    .sum(),
185            },
186            // This bucket accounts for VMO bytes that have been allocated by the kernel, but not
187            // claimed by any VMO (anymore).
188            Bucket {
189                name: ORPHANED.to_string(),
190                size: kmem_stats.vmo_bytes.unwrap_or(0).saturating_sub(vmo_size),
191            },
192            // This bucket aggregates overall kernel memory usage.
193            Bucket {
194                name: KERNEL.to_string(),
195                size: (|| {
196                    Some(
197                        kmem_stats.wired_bytes?
198                            + kmem_stats.total_heap_bytes?
199                            + kmem_stats.mmu_overhead_bytes?
200                            + kmem_stats.ipc_bytes?
201                            + kmem_stats.other_bytes?,
202                    )
203                })()
204                .unwrap_or(0),
205            },
206            // This bucket contains this amount of free memory in the system.
207            Bucket { name: FREE.to_string(), size: kmem_stats.free_bytes.unwrap_or(0) },
208            // Those buckets contain pager related information.
209            Bucket {
210                name: PAGER_TOTAL.to_string(),
211                size: kmem_stats.vmo_reclaim_total_bytes.unwrap_or(0),
212            },
213            Bucket {
214                name: PAGER_NEWEST.to_string(),
215                size: kmem_stats.vmo_reclaim_newest_bytes.unwrap_or(0),
216            },
217            Bucket {
218                name: PAGER_OLDEST.to_string(),
219                size: kmem_stats.vmo_reclaim_oldest_bytes.unwrap_or(0),
220            },
221            // Those buckets account for discardable memory.
222            Bucket {
223                name: DISCARDABLE_LOCKED.to_string(),
224                size: kmem_stats.vmo_discardable_locked_bytes.unwrap_or(0),
225            },
226            Bucket {
227                name: DISCARDABLE_UNLOCKED.to_string(),
228                size: kmem_stats.vmo_discardable_unlocked_bytes.unwrap_or(0),
229            },
230            // This bucket accounts for compressed memory.
231            Bucket {
232                name: ZRAM_COMPRESSED_BYTES.to_string(),
233                size: kmem_stats_compression.compressed_storage_bytes.unwrap_or(0),
234            },
235        ]);
236        Ok(Digest { buckets })
237    }
238}
239
240#[cfg(test)]
241mod tests {
242    use super::*;
243    use crate::testing::FakeAttributionDataProvider;
244    use crate::{
245        Attribution, AttributionData, Principal, PrincipalDescription, PrincipalIdentifier,
246        PrincipalType, Resource, ResourceReference,
247    };
248    use fidl_fuchsia_memory_attribution_plugin as fplugin;
249
250    fn get_attribution_data_provider() -> FakeAttributionDataProvider {
251        let attribution_data = AttributionData {
252            principals_vec: vec![Principal {
253                identifier: PrincipalIdentifier(1),
254                description: PrincipalDescription::Component("principal".to_owned()),
255                principal_type: PrincipalType::Runnable,
256                parent: None,
257            }],
258            resources_vec: vec![
259                Resource {
260                    koid: 10,
261                    name_index: 0,
262                    resource_type: fplugin::ResourceType::Vmo(fplugin::Vmo {
263                        parent: None,
264                        private_committed_bytes: Some(1024),
265                        private_populated_bytes: Some(2048),
266                        scaled_committed_bytes: Some(1024),
267                        scaled_populated_bytes: Some(2048),
268                        total_committed_bytes: Some(1024),
269                        total_populated_bytes: Some(2048),
270                        ..Default::default()
271                    }),
272                },
273                Resource {
274                    koid: 20,
275                    name_index: 1,
276                    resource_type: fplugin::ResourceType::Vmo(fplugin::Vmo {
277                        parent: None,
278                        private_committed_bytes: Some(1024),
279                        private_populated_bytes: Some(2048),
280                        scaled_committed_bytes: Some(1024),
281                        scaled_populated_bytes: Some(2048),
282                        total_committed_bytes: Some(1024),
283                        total_populated_bytes: Some(2048),
284                        ..Default::default()
285                    }),
286                },
287                Resource {
288                    koid: 30,
289                    name_index: 1,
290                    resource_type: fplugin::ResourceType::Process(fplugin::Process {
291                        vmos: Some(vec![10, 20]),
292                        ..Default::default()
293                    }),
294                },
295            ],
296            resource_names: vec![
297                ZXName::try_from_bytes(b"resource").unwrap(),
298                ZXName::try_from_bytes(b"matched").unwrap(),
299            ],
300            attributions: vec![Attribution {
301                source: PrincipalIdentifier(1),
302                subject: PrincipalIdentifier(1),
303                resources: vec![ResourceReference::KernelObject(10)],
304            }],
305        };
306        FakeAttributionDataProvider { attribution_data }
307    }
308
309    fn get_kernel_stats() -> (fkernel::MemoryStats, fkernel::MemoryStatsCompression) {
310        (
311            fkernel::MemoryStats {
312                total_bytes: Some(1),
313                free_bytes: Some(2),
314                wired_bytes: Some(3),
315                total_heap_bytes: Some(4),
316                free_heap_bytes: Some(5),
317                vmo_bytes: Some(10000),
318                mmu_overhead_bytes: Some(7),
319                ipc_bytes: Some(8),
320                other_bytes: Some(9),
321                free_loaned_bytes: Some(10),
322                cache_bytes: Some(11),
323                slab_bytes: Some(12),
324                zram_bytes: Some(13),
325                vmo_reclaim_total_bytes: Some(14),
326                vmo_reclaim_newest_bytes: Some(15),
327                vmo_reclaim_oldest_bytes: Some(16),
328                vmo_reclaim_disabled_bytes: Some(17),
329                vmo_discardable_locked_bytes: Some(18),
330                vmo_discardable_unlocked_bytes: Some(19),
331                ..Default::default()
332            },
333            fkernel::MemoryStatsCompression {
334                uncompressed_storage_bytes: Some(20),
335                compressed_storage_bytes: Some(21),
336                compressed_fragmentation_bytes: Some(22),
337                compression_time: Some(23),
338                decompression_time: Some(24),
339                total_page_compression_attempts: Some(25),
340                failed_page_compression_attempts: Some(26),
341                total_page_decompressions: Some(27),
342                compressed_page_evictions: Some(28),
343                eager_page_compressions: Some(29),
344                memory_pressure_page_compressions: Some(30),
345                critical_memory_page_compressions: Some(31),
346                pages_decompressed_unit_ns: Some(32),
347                pages_decompressed_within_log_time: Some([40, 41, 42, 43, 44, 45, 46, 47]),
348                ..Default::default()
349            },
350        )
351    }
352
353    #[test]
354    fn test_digest_no_definitions() {
355        let (kernel_stats, kernel_stats_compression) = get_kernel_stats();
356        let digest = Digest::compute(
357            &get_attribution_data_provider(),
358            &kernel_stats,
359            &kernel_stats_compression,
360            &vec![],
361        )
362        .unwrap();
363        let expected_buckets = vec![
364            Bucket { name: UNDIGESTED.to_string(), size: 2048 }, // The two VMOs are unmatched, 1024 + 1024
365            Bucket { name: ORPHANED.to_string(), size: 10000 }, // No matched VMOs => kernel's VMO bytes
366            Bucket { name: KERNEL.to_string(), size: 31 }, // wired + heap + mmu + ipc + other => 3 + 4 + 7 + 8 + 9 = 31
367            Bucket { name: FREE.to_string(), size: 2 },
368            Bucket { name: PAGER_TOTAL.to_string(), size: 14 },
369            Bucket { name: PAGER_NEWEST.to_string(), size: 15 },
370            Bucket { name: PAGER_OLDEST.to_string(), size: 16 },
371            Bucket { name: DISCARDABLE_LOCKED.to_string(), size: 18 },
372            Bucket { name: DISCARDABLE_UNLOCKED.to_string(), size: 19 },
373            Bucket { name: ZRAM_COMPRESSED_BYTES.to_string(), size: 21 },
374        ];
375
376        assert_eq!(digest.buckets, expected_buckets);
377    }
378
379    #[test]
380    fn test_digest_with_matching_vmo() -> Result<(), anyhow::Error> {
381        let (kernel_stats, kernel_stats_compression) = get_kernel_stats();
382        let digest = Digest::compute(
383            &get_attribution_data_provider(),
384            &kernel_stats,
385            &kernel_stats_compression,
386            &vec![BucketDefinition {
387                name: "matched".to_string(),
388                process: None,
389                vmo: Some(Regex::new("matched")?),
390                event_code: Default::default(),
391            }],
392        )
393        .unwrap();
394        let expected_buckets = vec![
395            Bucket { name: "matched".to_string(), size: 1024 }, // One VMO is matched, the other is not
396            Bucket { name: UNDIGESTED.to_string(), size: 1024 }, // One unmatched VMO
397            Bucket { name: ORPHANED.to_string(), size: 8976 }, // One matched VMO => 10000 - 1024 = 8976
398            Bucket { name: KERNEL.to_string(), size: 31 }, // wired + heap + mmu + ipc + other => 3 + 4 + 7 + 8 + 9 = 31
399            Bucket { name: FREE.to_string(), size: 2 },
400            Bucket { name: PAGER_TOTAL.to_string(), size: 14 },
401            Bucket { name: PAGER_NEWEST.to_string(), size: 15 },
402            Bucket { name: PAGER_OLDEST.to_string(), size: 16 },
403            Bucket { name: DISCARDABLE_LOCKED.to_string(), size: 18 },
404            Bucket { name: DISCARDABLE_UNLOCKED.to_string(), size: 19 },
405            Bucket { name: ZRAM_COMPRESSED_BYTES.to_string(), size: 21 },
406        ];
407
408        assert_eq!(digest.buckets, expected_buckets);
409        Ok(())
410    }
411
412    #[test]
413    fn test_digest_with_matching_process() -> Result<(), anyhow::Error> {
414        let (kernel_stats, kernel_stats_compression) = get_kernel_stats();
415        let digest = Digest::compute(
416            &get_attribution_data_provider(),
417            &kernel_stats,
418            &kernel_stats_compression,
419            &vec![BucketDefinition {
420                name: "matched".to_string(),
421                process: Some(Regex::new("matched")?),
422                vmo: None,
423                event_code: Default::default(),
424            }],
425        )
426        .unwrap();
427        let expected_buckets = vec![
428            Bucket { name: "matched".to_string(), size: 2048 }, // Both VMOs are matched => 1024 + 1024 = 2048
429            Bucket { name: UNDIGESTED.to_string(), size: 0 },   // No unmatched VMO
430            Bucket { name: ORPHANED.to_string(), size: 7952 }, // Two matched VMO => 10000 - 1024 - 1024 = 7952
431            Bucket { name: KERNEL.to_string(), size: 31 }, // wired + heap + mmu + ipc + other => 3 + 4 + 7 + 8 + 9 = 31
432            Bucket { name: FREE.to_string(), size: 2 },
433            Bucket { name: PAGER_TOTAL.to_string(), size: 14 },
434            Bucket { name: PAGER_NEWEST.to_string(), size: 15 },
435            Bucket { name: PAGER_OLDEST.to_string(), size: 16 },
436            Bucket { name: DISCARDABLE_LOCKED.to_string(), size: 18 },
437            Bucket { name: DISCARDABLE_UNLOCKED.to_string(), size: 19 },
438            Bucket { name: ZRAM_COMPRESSED_BYTES.to_string(), size: 21 },
439        ];
440
441        assert_eq!(digest.buckets, expected_buckets);
442        Ok(())
443    }
444
445    #[test]
446    fn test_digest_with_matching_process_and_vmo() -> Result<(), anyhow::Error> {
447        let (kernel_stats, kernel_stats_compression) = get_kernel_stats();
448        let digest = Digest::compute(
449            &get_attribution_data_provider(),
450            &kernel_stats,
451            &kernel_stats_compression,
452            &vec![BucketDefinition {
453                name: "matched".to_string(),
454                process: Some(Regex::new("matched")?),
455                vmo: Some(Regex::new("matched")?),
456                event_code: Default::default(),
457            }],
458        )
459        .unwrap();
460        let expected_buckets = vec![
461            Bucket { name: "matched".to_string(), size: 1024 }, // One VMO is matched, the other is not
462            Bucket { name: UNDIGESTED.to_string(), size: 1024 }, // One unmatched VMO
463            Bucket { name: ORPHANED.to_string(), size: 8976 }, // One matched VMO => 10000 - 1024 = 8976
464            Bucket { name: KERNEL.to_string(), size: 31 }, // wired + heap + mmu + ipc + other => 3 + 4 + 7 + 8 + 9 = 31
465            Bucket { name: FREE.to_string(), size: 2 },
466            Bucket { name: PAGER_TOTAL.to_string(), size: 14 },
467            Bucket { name: PAGER_NEWEST.to_string(), size: 15 },
468            Bucket { name: PAGER_OLDEST.to_string(), size: 16 },
469            Bucket { name: DISCARDABLE_LOCKED.to_string(), size: 18 },
470            Bucket { name: DISCARDABLE_UNLOCKED.to_string(), size: 19 },
471            Bucket { name: ZRAM_COMPRESSED_BYTES.to_string(), size: 21 },
472        ];
473
474        assert_eq!(digest.buckets, expected_buckets);
475        Ok(())
476    }
477}
attribution_processing/digest.rs

attribution_processing/
digest.rs
