attribution_processing/
digest.rs

1// Copyright 2025 The Fuchsia Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5use crate::fplugin::Vmo;
6use crate::{AttributionDataProvider, ResourcesVisitor, ZXName};
7use regex::bytes::Regex;
8use serde::de::Error;
9use serde::{Deserialize, Deserializer};
10use std::collections::hash_map::Entry::Occupied;
11use std::collections::HashMap;
12use {fidl_fuchsia_kernel as fkernel, fidl_fuchsia_memory_attribution_plugin as fplugin};
13
14const UNDIGESTED: &str = "Undigested";
15const ORPHANED: &str = "Orphaned";
16const KERNEL: &str = "Kernel";
17const FREE: &str = "Free";
18const PAGER_TOTAL: &str = "[Addl]PagerTotal";
19const PAGER_NEWEST: &str = "[Addl]PagerNewest";
20const PAGER_OLDEST: &str = "[Addl]PagerOldest";
21const DISCARDABLE_LOCKED: &str = "[Addl]DiscardableLocked";
22const DISCARDABLE_UNLOCKED: &str = "[Addl]DiscardableUnlocked";
23const ZRAM_COMPRESSED_BYTES: &str = "[Addl]ZramCompressedBytes";
24
25/// Represents a specification for aggregating memory usage in meaningful groups.
26///
27/// `name` represents the meaningful name of the group; grouping is done based on process and VMO
28/// names.
29///
30// Note: This needs to mirror `//src/lib/assembly/memory_buckets/src/memory_buckets.rs`, but cannot
31// reuse it directly because it is an host-only library.
32#[derive(Clone, Debug, Deserialize)]
33pub struct BucketDefinition {
34    pub name: String,
35    #[serde(deserialize_with = "deserialize_regex")]
36    pub process: Option<Regex>,
37    #[serde(deserialize_with = "deserialize_regex")]
38    pub vmo: Option<Regex>,
39    pub event_code: u64,
40}
41
42impl BucketDefinition {
43    /// Tests whether a process matches this bucket's definition, based on its name.
44    fn process_match(&self, process: &ZXName) -> bool {
45        self.process.as_ref().map_or(true, |p| p.is_match(process.as_bstr()))
46    }
47
48    /// Tests whether a VMO matches this bucket's definition, based on its name.
49    fn vmo_match(&self, vmo: &ZXName) -> bool {
50        self.vmo.as_ref().map_or(true, |v| v.is_match(vmo.as_bstr()))
51    }
52}
53
54// Teach serde to deserialize an optional regex.
55fn deserialize_regex<'de, D>(d: D) -> Result<Option<Regex>, D::Error>
56where
57    D: Deserializer<'de>,
58{
59    // Deserialize as Option<&str>
60    Option::<String>::deserialize(d)
61        // If the parsing failed, return the error, otherwise transform the value
62        .and_then(|os| {
63            os
64                // If there is a value, try to parse it as a Regex.
65                .map(|s| {
66                    Regex::new(&s)
67                        // If the regex compilation failed, wrap the error in the error type expected
68                        // by serde.
69                        .map_err(D::Error::custom)
70                })
71                // If there was a value but it failed to compile, return an error, otherwise return
72                // the potentially parsed option.
73                .transpose()
74        })
75}
76
77/// Aggregates bytes in categories with human readable names.
78#[derive(Clone, Debug, PartialEq, Eq)]
79pub struct Bucket {
80    pub name: String,
81    pub size: u64,
82}
83/// Contains a view of the system's memory usage, aggregated in groups called buckets, which are
84/// configurable.
85#[derive(Debug, PartialEq, Eq)]
86pub struct Digest {
87    pub buckets: Vec<Bucket>,
88}
89
90/// Compute a bucket digest as the Jobs->Processes->VMOs tree is traversed.
91struct DigestComputer<'a> {
92    // Ordered pair with a bucket specification, and the current bucket result.
93    buckets: Vec<(&'a BucketDefinition, Bucket)>,
94    // Set of VMOs what didn't fell in any bucket.
95    undigested_vmos: HashMap<zx_types::zx_koid_t, (Vmo, ZXName)>,
96}
97
98impl<'a> DigestComputer<'a> {
99    fn new(bucket_definitions: &'a Vec<BucketDefinition>) -> DigestComputer<'a> {
100        DigestComputer {
101            buckets: bucket_definitions
102                .iter()
103                .map(|def| (def, Bucket { name: def.name.clone(), size: 0 }))
104                .collect(),
105            undigested_vmos: Default::default(),
106        }
107    }
108}
109
110impl ResourcesVisitor for DigestComputer<'_> {
111    fn on_job(
112        &mut self,
113        _job_koid: zx_types::zx_koid_t,
114        _job_name: &ZXName,
115        _job: fplugin::Job,
116    ) -> Result<(), zx_status::Status> {
117        Ok(())
118    }
119
120    fn on_process(
121        &mut self,
122        _process_koid: zx_types::zx_koid_t,
123        process_name: &ZXName,
124        process: fplugin::Process,
125    ) -> Result<(), zx_status::Status> {
126        for (bucket_definition, bucket) in self.buckets.iter_mut() {
127            if bucket_definition.process_match(process_name) {
128                for koid in process.vmos.iter().flatten() {
129                    bucket.size += match self.undigested_vmos.entry(*koid) {
130                        Occupied(e) => {
131                            let (_vmo, name) = e.get();
132                            if bucket_definition.vmo_match(&name) {
133                                let (_, (vmo, _name)) = e.remove_entry();
134                                vmo.total_committed_bytes.unwrap_or_default()
135                            } else {
136                                0
137                            }
138                        }
139                        _ => 0,
140                    };
141                }
142            }
143        }
144        Ok(())
145    }
146
147    fn on_vmo(
148        &mut self,
149        vmo_koid: zx_types::zx_koid_t,
150        vmo_name: &ZXName,
151        vmo: fplugin::Vmo,
152    ) -> Result<(), zx_status::Status> {
153        self.undigested_vmos.insert(vmo_koid, (vmo, vmo_name.clone()));
154        Ok(())
155    }
156}
157
158impl Digest {
159    /// Given means to query the system for memory usage, and a specification, this function
160    /// aggregates the current memory usage into human displayable units we call buckets.
161    pub fn compute(
162        attribution_data_service: &impl AttributionDataProvider,
163        kmem_stats: &fkernel::MemoryStats,
164        kmem_stats_compression: &fkernel::MemoryStatsCompression,
165        bucket_definitions: &Vec<BucketDefinition>,
166    ) -> Result<Digest, anyhow::Error> {
167        let mut digest_visitor = DigestComputer::new(bucket_definitions);
168        attribution_data_service.for_each_resource(&mut digest_visitor)?;
169        let mut buckets: Vec<Bucket> =
170            digest_visitor.buckets.drain(..).map(|(_, bucket)| bucket).collect();
171
172        let vmo_size: u64 = buckets.iter().map(|Bucket { size, .. }| size).sum();
173        // Extend the configured aggregation with a number of additional, occasionally useful meta
174        // aggregations.
175        buckets.extend(vec![
176            // This bucket contains the total size of the VMOs that have not been covered by any
177            // other bucket.
178            Bucket {
179                name: UNDIGESTED.to_string(),
180                size: digest_visitor
181                    .undigested_vmos
182                    .values()
183                    .filter_map(|(vmo, _)| vmo.total_committed_bytes)
184                    .sum(),
185            },
186            // This bucket accounts for VMO bytes that have been allocated by the kernel, but not
187            // claimed by any VMO (anymore).
188            Bucket {
189                name: ORPHANED.to_string(),
190                size: (kmem_stats.vmo_bytes.unwrap_or(0) - vmo_size)
191                    .clamp(0, kmem_stats.vmo_bytes.unwrap_or(0)),
192            },
193            // This bucket aggregates overall kernel memory usage.
194            Bucket {
195                name: KERNEL.to_string(),
196                size: (|| {
197                    Some(
198                        kmem_stats.wired_bytes?
199                            + kmem_stats.total_heap_bytes?
200                            + kmem_stats.mmu_overhead_bytes?
201                            + kmem_stats.ipc_bytes?
202                            + kmem_stats.other_bytes?,
203                    )
204                })()
205                .unwrap_or(0),
206            },
207            // This bucket contains this amount of free memory in the system.
208            Bucket { name: FREE.to_string(), size: kmem_stats.free_bytes.unwrap_or(0) },
209            // Those buckets contain pager related information.
210            Bucket {
211                name: PAGER_TOTAL.to_string(),
212                size: kmem_stats.vmo_reclaim_total_bytes.unwrap_or(0),
213            },
214            Bucket {
215                name: PAGER_NEWEST.to_string(),
216                size: kmem_stats.vmo_reclaim_newest_bytes.unwrap_or(0),
217            },
218            Bucket {
219                name: PAGER_OLDEST.to_string(),
220                size: kmem_stats.vmo_reclaim_oldest_bytes.unwrap_or(0),
221            },
222            // Those buckets account for discardable memory.
223            Bucket {
224                name: DISCARDABLE_LOCKED.to_string(),
225                size: kmem_stats.vmo_discardable_locked_bytes.unwrap_or(0),
226            },
227            Bucket {
228                name: DISCARDABLE_UNLOCKED.to_string(),
229                size: kmem_stats.vmo_discardable_unlocked_bytes.unwrap_or(0),
230            },
231            // This bucket accounts for compressed memory.
232            Bucket {
233                name: ZRAM_COMPRESSED_BYTES.to_string(),
234                size: kmem_stats_compression.compressed_storage_bytes.unwrap_or(0),
235            },
236        ]);
237        Ok(Digest { buckets })
238    }
239}
240
241#[cfg(test)]
242mod tests {
243    use super::*;
244    use crate::testing::FakeAttributionDataProvider;
245    use crate::{
246        Attribution, AttributionData, Principal, PrincipalDescription, PrincipalIdentifier,
247        PrincipalType, Resource, ResourceReference,
248    };
249    use fidl_fuchsia_memory_attribution_plugin as fplugin;
250
251    fn get_attribution_data_provider() -> FakeAttributionDataProvider {
252        let attribution_data = AttributionData {
253            principals_vec: vec![Principal {
254                identifier: PrincipalIdentifier(1),
255                description: PrincipalDescription::Component("principal".to_owned()),
256                principal_type: PrincipalType::Runnable,
257                parent: None,
258            }],
259            resources_vec: vec![
260                Resource {
261                    koid: 10,
262                    name_index: 0,
263                    resource_type: fplugin::ResourceType::Vmo(fplugin::Vmo {
264                        parent: None,
265                        private_committed_bytes: Some(1024),
266                        private_populated_bytes: Some(2048),
267                        scaled_committed_bytes: Some(1024),
268                        scaled_populated_bytes: Some(2048),
269                        total_committed_bytes: Some(1024),
270                        total_populated_bytes: Some(2048),
271                        ..Default::default()
272                    }),
273                },
274                Resource {
275                    koid: 20,
276                    name_index: 1,
277                    resource_type: fplugin::ResourceType::Vmo(fplugin::Vmo {
278                        parent: None,
279                        private_committed_bytes: Some(1024),
280                        private_populated_bytes: Some(2048),
281                        scaled_committed_bytes: Some(1024),
282                        scaled_populated_bytes: Some(2048),
283                        total_committed_bytes: Some(1024),
284                        total_populated_bytes: Some(2048),
285                        ..Default::default()
286                    }),
287                },
288                Resource {
289                    koid: 30,
290                    name_index: 1,
291                    resource_type: fplugin::ResourceType::Process(fplugin::Process {
292                        vmos: Some(vec![10, 20]),
293                        ..Default::default()
294                    }),
295                },
296            ],
297            resource_names: vec![
298                ZXName::try_from_bytes(b"resource").unwrap(),
299                ZXName::try_from_bytes(b"matched").unwrap(),
300            ],
301            attributions: vec![Attribution {
302                source: PrincipalIdentifier(1),
303                subject: PrincipalIdentifier(1),
304                resources: vec![ResourceReference::KernelObject(10)],
305            }],
306        };
307        FakeAttributionDataProvider { attribution_data }
308    }
309
310    fn get_kernel_stats() -> (fkernel::MemoryStats, fkernel::MemoryStatsCompression) {
311        (
312            fkernel::MemoryStats {
313                total_bytes: Some(1),
314                free_bytes: Some(2),
315                wired_bytes: Some(3),
316                total_heap_bytes: Some(4),
317                free_heap_bytes: Some(5),
318                vmo_bytes: Some(10000),
319                mmu_overhead_bytes: Some(7),
320                ipc_bytes: Some(8),
321                other_bytes: Some(9),
322                free_loaned_bytes: Some(10),
323                cache_bytes: Some(11),
324                slab_bytes: Some(12),
325                zram_bytes: Some(13),
326                vmo_reclaim_total_bytes: Some(14),
327                vmo_reclaim_newest_bytes: Some(15),
328                vmo_reclaim_oldest_bytes: Some(16),
329                vmo_reclaim_disabled_bytes: Some(17),
330                vmo_discardable_locked_bytes: Some(18),
331                vmo_discardable_unlocked_bytes: Some(19),
332                ..Default::default()
333            },
334            fkernel::MemoryStatsCompression {
335                uncompressed_storage_bytes: Some(20),
336                compressed_storage_bytes: Some(21),
337                compressed_fragmentation_bytes: Some(22),
338                compression_time: Some(23),
339                decompression_time: Some(24),
340                total_page_compression_attempts: Some(25),
341                failed_page_compression_attempts: Some(26),
342                total_page_decompressions: Some(27),
343                compressed_page_evictions: Some(28),
344                eager_page_compressions: Some(29),
345                memory_pressure_page_compressions: Some(30),
346                critical_memory_page_compressions: Some(31),
347                pages_decompressed_unit_ns: Some(32),
348                pages_decompressed_within_log_time: Some([40, 41, 42, 43, 44, 45, 46, 47]),
349                ..Default::default()
350            },
351        )
352    }
353
354    #[test]
355    fn test_digest_no_definitions() {
356        let (kernel_stats, kernel_stats_compression) = get_kernel_stats();
357        let digest = Digest::compute(
358            &get_attribution_data_provider(),
359            &kernel_stats,
360            &kernel_stats_compression,
361            &vec![],
362        )
363        .unwrap();
364        let expected_buckets = vec![
365            Bucket { name: UNDIGESTED.to_string(), size: 2048 }, // The two VMOs are unmatched, 1024 + 1024
366            Bucket { name: ORPHANED.to_string(), size: 10000 }, // No matched VMOs => kernel's VMO bytes
367            Bucket { name: KERNEL.to_string(), size: 31 }, // wired + heap + mmu + ipc + other => 3 + 4 + 7 + 8 + 9 = 31
368            Bucket { name: FREE.to_string(), size: 2 },
369            Bucket { name: PAGER_TOTAL.to_string(), size: 14 },
370            Bucket { name: PAGER_NEWEST.to_string(), size: 15 },
371            Bucket { name: PAGER_OLDEST.to_string(), size: 16 },
372            Bucket { name: DISCARDABLE_LOCKED.to_string(), size: 18 },
373            Bucket { name: DISCARDABLE_UNLOCKED.to_string(), size: 19 },
374            Bucket { name: ZRAM_COMPRESSED_BYTES.to_string(), size: 21 },
375        ];
376
377        assert_eq!(digest.buckets, expected_buckets);
378    }
379
380    #[test]
381    fn test_digest_with_matching_vmo() -> Result<(), anyhow::Error> {
382        let (kernel_stats, kernel_stats_compression) = get_kernel_stats();
383        let digest = Digest::compute(
384            &get_attribution_data_provider(),
385            &kernel_stats,
386            &kernel_stats_compression,
387            &vec![BucketDefinition {
388                name: "matched".to_string(),
389                process: None,
390                vmo: Some(Regex::new("matched")?),
391                event_code: Default::default(),
392            }],
393        )
394        .unwrap();
395        let expected_buckets = vec![
396            Bucket { name: "matched".to_string(), size: 1024 }, // One VMO is matched, the other is not
397            Bucket { name: UNDIGESTED.to_string(), size: 1024 }, // One unmatched VMO
398            Bucket { name: ORPHANED.to_string(), size: 8976 }, // One matched VMO => 10000 - 1024 = 8976
399            Bucket { name: KERNEL.to_string(), size: 31 }, // wired + heap + mmu + ipc + other => 3 + 4 + 7 + 8 + 9 = 31
400            Bucket { name: FREE.to_string(), size: 2 },
401            Bucket { name: PAGER_TOTAL.to_string(), size: 14 },
402            Bucket { name: PAGER_NEWEST.to_string(), size: 15 },
403            Bucket { name: PAGER_OLDEST.to_string(), size: 16 },
404            Bucket { name: DISCARDABLE_LOCKED.to_string(), size: 18 },
405            Bucket { name: DISCARDABLE_UNLOCKED.to_string(), size: 19 },
406            Bucket { name: ZRAM_COMPRESSED_BYTES.to_string(), size: 21 },
407        ];
408
409        assert_eq!(digest.buckets, expected_buckets);
410        Ok(())
411    }
412
413    #[test]
414    fn test_digest_with_matching_process() -> Result<(), anyhow::Error> {
415        let (kernel_stats, kernel_stats_compression) = get_kernel_stats();
416        let digest = Digest::compute(
417            &get_attribution_data_provider(),
418            &kernel_stats,
419            &kernel_stats_compression,
420            &vec![BucketDefinition {
421                name: "matched".to_string(),
422                process: Some(Regex::new("matched")?),
423                vmo: None,
424                event_code: Default::default(),
425            }],
426        )
427        .unwrap();
428        let expected_buckets = vec![
429            Bucket { name: "matched".to_string(), size: 2048 }, // Both VMOs are matched => 1024 + 1024 = 2048
430            Bucket { name: UNDIGESTED.to_string(), size: 0 },   // No unmatched VMO
431            Bucket { name: ORPHANED.to_string(), size: 7952 }, // Two matched VMO => 10000 - 1024 - 1024 = 7952
432            Bucket { name: KERNEL.to_string(), size: 31 }, // wired + heap + mmu + ipc + other => 3 + 4 + 7 + 8 + 9 = 31
433            Bucket { name: FREE.to_string(), size: 2 },
434            Bucket { name: PAGER_TOTAL.to_string(), size: 14 },
435            Bucket { name: PAGER_NEWEST.to_string(), size: 15 },
436            Bucket { name: PAGER_OLDEST.to_string(), size: 16 },
437            Bucket { name: DISCARDABLE_LOCKED.to_string(), size: 18 },
438            Bucket { name: DISCARDABLE_UNLOCKED.to_string(), size: 19 },
439            Bucket { name: ZRAM_COMPRESSED_BYTES.to_string(), size: 21 },
440        ];
441
442        assert_eq!(digest.buckets, expected_buckets);
443        Ok(())
444    }
445
446    #[test]
447    fn test_digest_with_matching_process_and_vmo() -> Result<(), anyhow::Error> {
448        let (kernel_stats, kernel_stats_compression) = get_kernel_stats();
449        let digest = Digest::compute(
450            &get_attribution_data_provider(),
451            &kernel_stats,
452            &kernel_stats_compression,
453            &vec![BucketDefinition {
454                name: "matched".to_string(),
455                process: Some(Regex::new("matched")?),
456                vmo: Some(Regex::new("matched")?),
457                event_code: Default::default(),
458            }],
459        )
460        .unwrap();
461        let expected_buckets = vec![
462            Bucket { name: "matched".to_string(), size: 1024 }, // One VMO is matched, the other is not
463            Bucket { name: UNDIGESTED.to_string(), size: 1024 }, // One unmatched VMO
464            Bucket { name: ORPHANED.to_string(), size: 8976 }, // One matched VMO => 10000 - 1024 = 8976
465            Bucket { name: KERNEL.to_string(), size: 31 }, // wired + heap + mmu + ipc + other => 3 + 4 + 7 + 8 + 9 = 31
466            Bucket { name: FREE.to_string(), size: 2 },
467            Bucket { name: PAGER_TOTAL.to_string(), size: 14 },
468            Bucket { name: PAGER_NEWEST.to_string(), size: 15 },
469            Bucket { name: PAGER_OLDEST.to_string(), size: 16 },
470            Bucket { name: DISCARDABLE_LOCKED.to_string(), size: 18 },
471            Bucket { name: DISCARDABLE_UNLOCKED.to_string(), size: 19 },
472            Bucket { name: ZRAM_COMPRESSED_BYTES.to_string(), size: 21 },
473        ];
474
475        assert_eq!(digest.buckets, expected_buckets);
476        Ok(())
477    }
478}
attribution_processing/digest.rs

attribution_processing/
digest.rs
