attribution_processing/
digest.rs

1// Copyright 2025 The Fuchsia Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5use crate::fplugin::Vmo;
6use crate::{ResourceEnumerator, ResourcesVisitor, ZXName};
7use anyhow::Result;
8use regex::bytes::Regex;
9use serde::de::Error;
10use serde::{Deserialize, Deserializer, Serialize};
11use std::collections::HashMap;
12use std::collections::hash_map::Entry::Occupied;
13use {fidl_fuchsia_kernel as fkernel, fidl_fuchsia_memory_attribution_plugin as fplugin};
14
15const UNDIGESTED: &str = "Undigested";
16const ORPHANED: &str = "Orphaned";
17const KERNEL: &str = "Kernel";
18const FREE: &str = "Free";
19const PAGER_TOTAL: &str = "[Addl]PagerTotal";
20const PAGER_NEWEST: &str = "[Addl]PagerNewest";
21const PAGER_OLDEST: &str = "[Addl]PagerOldest";
22const DISCARDABLE_LOCKED: &str = "[Addl]DiscardableLocked";
23const DISCARDABLE_UNLOCKED: &str = "[Addl]DiscardableUnlocked";
24const ZRAM_COMPRESSED_BYTES: &str = "[Addl]ZramCompressedBytes";
25
26/// Represents a specification for aggregating memory usage in meaningful groups.
27///
28/// `name` represents the meaningful name of the group; grouping is done based on process and VMO
29/// names.
30///
31// Note: This needs to mirror `//src/lib/assembly/memory_buckets/src/memory_buckets.rs`, but cannot
32// reuse it directly because it is an host-only library.
33#[derive(Clone, Debug, Deserialize)]
34pub struct BucketDefinition {
35    pub name: String,
36    #[serde(deserialize_with = "deserialize_regex")]
37    pub process: Option<Regex>,
38    #[serde(deserialize_with = "deserialize_regex")]
39    pub vmo: Option<Regex>,
40    pub event_code: u64,
41}
42
43impl BucketDefinition {
44    /// Tests whether a process matches this bucket's definition, based on its name.
45    fn process_match(&self, process: &ZXName) -> bool {
46        self.process.as_ref().map_or(true, |p| p.is_match(process.as_bstr()))
47    }
48
49    /// Tests whether a VMO matches this bucket's definition, based on its name.
50    fn vmo_match(&self, vmo: &ZXName) -> bool {
51        self.vmo.as_ref().map_or(true, |v| v.is_match(vmo.as_bstr()))
52    }
53}
54
55// Teach serde to deserialize an optional regex.
56fn deserialize_regex<'de, D>(d: D) -> Result<Option<Regex>, D::Error>
57where
58    D: Deserializer<'de>,
59{
60    // Deserialize as Option<&str>
61    Option::<String>::deserialize(d)
62        // If the parsing failed, return the error, otherwise transform the value
63        .and_then(|os| {
64            os
65                // If there is a value, try to parse it as a Regex.
66                .map(|s| {
67                    Regex::new(&s)
68                        // If the regex compilation failed, wrap the error in the error type expected
69                        // by serde.
70                        .map_err(D::Error::custom)
71                })
72                // If there was a value but it failed to compile, return an error, otherwise return
73                // the potentially parsed option.
74                .transpose()
75        })
76}
77
78/// Aggregates bytes in categories with human readable names.
79#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
80pub struct Bucket {
81    pub name: String,
82    pub size: u64,
83}
84/// Contains a view of the system's memory usage, aggregated in groups called buckets, which are
85/// configurable.
86#[derive(Debug, Default, PartialEq, Eq, Serialize)]
87pub struct Digest {
88    pub buckets: Vec<Bucket>,
89}
90
91/// Compute a bucket digest as the Jobs->Processes->VMOs tree is traversed.
92struct DigestComputer<'a> {
93    // Ordered pair with a bucket specification, and the current bucket result.
94    buckets: Vec<(&'a BucketDefinition, Bucket)>,
95    // Set of VMOs what didn't fell in any bucket.
96    undigested_vmos: HashMap<zx_types::zx_koid_t, (Vmo, ZXName)>,
97}
98
99impl<'a> DigestComputer<'a> {
100    fn new(bucket_definitions: &'a [BucketDefinition]) -> DigestComputer<'a> {
101        DigestComputer {
102            buckets: bucket_definitions
103                .iter()
104                .map(|def| (def, Bucket { name: def.name.clone(), size: 0 }))
105                .collect(),
106            undigested_vmos: Default::default(),
107        }
108    }
109}
110
111impl ResourcesVisitor for DigestComputer<'_> {
112    fn on_job(
113        &mut self,
114        _job_koid: zx_types::zx_koid_t,
115        _job_name: &ZXName,
116        _job: fplugin::Job,
117    ) -> Result<(), zx_status::Status> {
118        Ok(())
119    }
120
121    fn on_process(
122        &mut self,
123        _process_koid: zx_types::zx_koid_t,
124        process_name: &ZXName,
125        process: fplugin::Process,
126    ) -> Result<(), zx_status::Status> {
127        for (bucket_definition, bucket) in self.buckets.iter_mut() {
128            if bucket_definition.process_match(process_name) {
129                for koid in process.vmos.iter().flatten() {
130                    bucket.size += match self.undigested_vmos.entry(*koid) {
131                        Occupied(e) => {
132                            let (_vmo, name) = e.get();
133                            if bucket_definition.vmo_match(&name) {
134                                let (_, (vmo, _name)) = e.remove_entry();
135                                vmo.scaled_committed_bytes.unwrap_or_default()
136                            } else {
137                                0
138                            }
139                        }
140                        _ => 0,
141                    };
142                }
143            }
144        }
145        Ok(())
146    }
147
148    fn on_vmo(
149        &mut self,
150        vmo_koid: zx_types::zx_koid_t,
151        vmo_name: &ZXName,
152        vmo: fplugin::Vmo,
153    ) -> Result<(), zx_status::Status> {
154        self.undigested_vmos.insert(vmo_koid, (vmo, vmo_name.clone()));
155        Ok(())
156    }
157}
158
159impl Digest {
160    /// Given means to query the system for memory usage, and a specification, this function
161    /// aggregates the current memory usage into human displayable units we call buckets.
162    pub fn compute(
163        resource_enumerator: &impl ResourceEnumerator,
164        kmem_stats: &fkernel::MemoryStats,
165        kmem_stats_compression: &fkernel::MemoryStatsCompression,
166        bucket_definitions: &[BucketDefinition],
167    ) -> Result<Digest> {
168        let mut digest_visitor = DigestComputer::new(bucket_definitions);
169        resource_enumerator.for_each_resource(&mut digest_visitor)?;
170        let mut buckets: Vec<Bucket> =
171            digest_visitor.buckets.drain(..).map(|(_, bucket)| bucket).collect();
172
173        let total_vmo_size: u64 = buckets.iter().map(|Bucket { size, .. }| size).sum();
174
175        // Extend the configured aggregation with a number of additional, occasionally useful meta
176        // aggregations.
177        buckets.extend([
178            // This bucket contains the total size of the VMOs that have not been covered by any
179            // other bucket.
180            Bucket {
181                name: UNDIGESTED.to_string(),
182                size: digest_visitor
183                    .undigested_vmos
184                    .values()
185                    .filter_map(|vmo| vmo.0.scaled_committed_bytes)
186                    .sum(),
187            },
188            // This bucket accounts for VMO bytes that have been allocated by the kernel, but not
189            // claimed by any VMO (anymore).
190            Bucket {
191                name: ORPHANED.to_string(),
192                size: kmem_stats.vmo_bytes.unwrap_or(0).saturating_sub(total_vmo_size),
193            },
194            // This bucket aggregates overall kernel memory usage.
195            Bucket {
196                name: KERNEL.to_string(),
197                size: (|| {
198                    Some(
199                        kmem_stats.wired_bytes?
200                            + kmem_stats.total_heap_bytes?
201                            + kmem_stats.mmu_overhead_bytes?
202                            + kmem_stats.ipc_bytes?
203                            + kmem_stats.other_bytes?,
204                    )
205                })()
206                .unwrap_or(0),
207            },
208            // This bucket contains the amount of free memory in the system.
209            Bucket { name: FREE.to_string(), size: kmem_stats.free_bytes.unwrap_or(0) },
210            // Those buckets contain pager related information.
211            Bucket {
212                name: PAGER_TOTAL.to_string(),
213                size: kmem_stats.vmo_reclaim_total_bytes.unwrap_or(0),
214            },
215            Bucket {
216                name: PAGER_NEWEST.to_string(),
217                size: kmem_stats.vmo_reclaim_newest_bytes.unwrap_or(0),
218            },
219            Bucket {
220                name: PAGER_OLDEST.to_string(),
221                size: kmem_stats.vmo_reclaim_oldest_bytes.unwrap_or(0),
222            },
223            // Those buckets account for discardable memory.
224            Bucket {
225                name: DISCARDABLE_LOCKED.to_string(),
226                size: kmem_stats.vmo_discardable_locked_bytes.unwrap_or(0),
227            },
228            Bucket {
229                name: DISCARDABLE_UNLOCKED.to_string(),
230                size: kmem_stats.vmo_discardable_unlocked_bytes.unwrap_or(0),
231            },
232            // This bucket accounts for compressed memory.
233            Bucket {
234                name: ZRAM_COMPRESSED_BYTES.to_string(),
235                size: kmem_stats_compression.compressed_storage_bytes.unwrap_or(0),
236            },
237        ]);
238        Ok(Digest { buckets })
239    }
240}
241
242#[cfg(test)]
243mod tests {
244    use super::*;
245    use crate::{
246        Attribution, AttributionData, Principal, PrincipalDescription, PrincipalIdentifier,
247        PrincipalType, Resource, ResourceReference,
248    };
249    use fidl_fuchsia_memory_attribution_plugin as fplugin;
250
251    fn get_attribution_data() -> AttributionData {
252        let attribution_data = AttributionData {
253            principals_vec: vec![Principal {
254                identifier: PrincipalIdentifier(1),
255                description: Some(PrincipalDescription::Component("principal".to_owned())),
256                principal_type: PrincipalType::Runnable,
257                parent: None,
258            }],
259            resources_vec: vec![
260                Resource {
261                    koid: 10,
262                    name_index: 0,
263                    resource_type: fplugin::ResourceType::Vmo(fplugin::Vmo {
264                        parent: None,
265                        private_committed_bytes: Some(1024),
266                        private_populated_bytes: Some(2048),
267                        scaled_committed_bytes: Some(512),
268                        scaled_populated_bytes: Some(2048),
269                        total_committed_bytes: Some(1024),
270                        total_populated_bytes: Some(2048),
271                        ..Default::default()
272                    }),
273                },
274                Resource {
275                    koid: 20,
276                    name_index: 1,
277                    resource_type: fplugin::ResourceType::Vmo(fplugin::Vmo {
278                        parent: None,
279                        private_committed_bytes: Some(1024),
280                        private_populated_bytes: Some(2048),
281                        scaled_committed_bytes: Some(512),
282                        scaled_populated_bytes: Some(2048),
283                        total_committed_bytes: Some(1024),
284                        total_populated_bytes: Some(2048),
285                        ..Default::default()
286                    }),
287                },
288                Resource {
289                    koid: 30,
290                    name_index: 1,
291                    resource_type: fplugin::ResourceType::Process(fplugin::Process {
292                        vmos: Some(vec![10, 20]),
293                        ..Default::default()
294                    }),
295                },
296            ],
297            resource_names: vec![
298                ZXName::try_from_bytes(b"resource").unwrap(),
299                ZXName::try_from_bytes(b"matched").unwrap(),
300            ],
301            attributions: vec![Attribution {
302                source: PrincipalIdentifier(1),
303                subject: PrincipalIdentifier(1),
304                resources: vec![ResourceReference::KernelObject(10)],
305            }],
306        };
307        attribution_data
308    }
309
310    fn get_kernel_stats() -> (fkernel::MemoryStats, fkernel::MemoryStatsCompression) {
311        (
312            fkernel::MemoryStats {
313                total_bytes: Some(1),
314                free_bytes: Some(2),
315                wired_bytes: Some(3),
316                total_heap_bytes: Some(4),
317                free_heap_bytes: Some(5),
318                vmo_bytes: Some(10000),
319                mmu_overhead_bytes: Some(7),
320                ipc_bytes: Some(8),
321                other_bytes: Some(9),
322                free_loaned_bytes: Some(10),
323                cache_bytes: Some(11),
324                slab_bytes: Some(12),
325                zram_bytes: Some(13),
326                vmo_reclaim_total_bytes: Some(14),
327                vmo_reclaim_newest_bytes: Some(15),
328                vmo_reclaim_oldest_bytes: Some(16),
329                vmo_reclaim_disabled_bytes: Some(17),
330                vmo_discardable_locked_bytes: Some(18),
331                vmo_discardable_unlocked_bytes: Some(19),
332                ..Default::default()
333            },
334            fkernel::MemoryStatsCompression {
335                uncompressed_storage_bytes: Some(20),
336                compressed_storage_bytes: Some(21),
337                compressed_fragmentation_bytes: Some(22),
338                compression_time: Some(23),
339                decompression_time: Some(24),
340                total_page_compression_attempts: Some(25),
341                failed_page_compression_attempts: Some(26),
342                total_page_decompressions: Some(27),
343                compressed_page_evictions: Some(28),
344                eager_page_compressions: Some(29),
345                memory_pressure_page_compressions: Some(30),
346                critical_memory_page_compressions: Some(31),
347                pages_decompressed_unit_ns: Some(32),
348                pages_decompressed_within_log_time: Some([40, 41, 42, 43, 44, 45, 46, 47]),
349                ..Default::default()
350            },
351        )
352    }
353
354    #[test]
355    fn test_digest_no_definitions() {
356        let (kernel_stats, kernel_stats_compression) = get_kernel_stats();
357        let digest = Digest::compute(
358            &get_attribution_data(),
359            &kernel_stats,
360            &kernel_stats_compression,
361            &vec![],
362        )
363        .unwrap();
364        let expected_buckets = vec![
365            Bucket { name: UNDIGESTED.to_string(), size: 1024 }, // The two VMOs are unmatched, 512 + 512
366            Bucket { name: ORPHANED.to_string(), size: 10000 }, // No matched VMOs => kernel's VMO bytes
367            Bucket { name: KERNEL.to_string(), size: 31 }, // wired + heap + mmu + ipc + other => 3 + 4 + 7 + 8 + 9 = 31
368            Bucket { name: FREE.to_string(), size: 2 },
369            Bucket { name: PAGER_TOTAL.to_string(), size: 14 },
370            Bucket { name: PAGER_NEWEST.to_string(), size: 15 },
371            Bucket { name: PAGER_OLDEST.to_string(), size: 16 },
372            Bucket { name: DISCARDABLE_LOCKED.to_string(), size: 18 },
373            Bucket { name: DISCARDABLE_UNLOCKED.to_string(), size: 19 },
374            Bucket { name: ZRAM_COMPRESSED_BYTES.to_string(), size: 21 },
375        ];
376
377        assert_eq!(digest.buckets, expected_buckets);
378    }
379
380    #[test]
381    fn test_digest_with_matching_vmo() -> Result<(), anyhow::Error> {
382        let (kernel_stats, kernel_stats_compression) = get_kernel_stats();
383        let digest = Digest::compute(
384            &get_attribution_data(),
385            &kernel_stats,
386            &kernel_stats_compression,
387            &vec![BucketDefinition {
388                name: "matched".to_string(),
389                process: None,
390                vmo: Some(Regex::new("matched")?),
391                event_code: Default::default(),
392            }],
393        )
394        .unwrap();
395        let expected_buckets = vec![
396            Bucket { name: "matched".to_string(), size: 512 }, // One VMO is matched, the other is not
397            Bucket { name: UNDIGESTED.to_string(), size: 512 }, // One unmatched VMO
398            Bucket { name: ORPHANED.to_string(), size: 9488 }, // One matched VMO => 10000 - 512 = 9488
399            Bucket { name: KERNEL.to_string(), size: 31 }, // wired + heap + mmu + ipc + other => 3 + 4 + 7 + 8 + 9 = 31
400            Bucket { name: FREE.to_string(), size: 2 },
401            Bucket { name: PAGER_TOTAL.to_string(), size: 14 },
402            Bucket { name: PAGER_NEWEST.to_string(), size: 15 },
403            Bucket { name: PAGER_OLDEST.to_string(), size: 16 },
404            Bucket { name: DISCARDABLE_LOCKED.to_string(), size: 18 },
405            Bucket { name: DISCARDABLE_UNLOCKED.to_string(), size: 19 },
406            Bucket { name: ZRAM_COMPRESSED_BYTES.to_string(), size: 21 },
407        ];
408
409        assert_eq!(digest.buckets, expected_buckets);
410        Ok(())
411    }
412
413    #[test]
414    fn test_digest_with_matching_process() -> Result<(), anyhow::Error> {
415        let (kernel_stats, kernel_stats_compression) = get_kernel_stats();
416        let digest = Digest::compute(
417            &get_attribution_data(),
418            &kernel_stats,
419            &kernel_stats_compression,
420            &vec![BucketDefinition {
421                name: "matched".to_string(),
422                process: Some(Regex::new("matched")?),
423                vmo: None,
424                event_code: Default::default(),
425            }],
426        )
427        .unwrap();
428        let expected_buckets = vec![
429            Bucket { name: "matched".to_string(), size: 1024 }, // Both VMOs are matched => 512 + 512 = 1024
430            Bucket { name: UNDIGESTED.to_string(), size: 0 },   // No unmatched VMO
431            Bucket { name: ORPHANED.to_string(), size: 8976 }, // Two matched VMO => 10000 - 512 - 512 = 8976
432            Bucket { name: KERNEL.to_string(), size: 31 }, // wired + heap + mmu + ipc + other => 3 + 4 + 7 + 8 + 9 = 31
433            Bucket { name: FREE.to_string(), size: 2 },
434            Bucket { name: PAGER_TOTAL.to_string(), size: 14 },
435            Bucket { name: PAGER_NEWEST.to_string(), size: 15 },
436            Bucket { name: PAGER_OLDEST.to_string(), size: 16 },
437            Bucket { name: DISCARDABLE_LOCKED.to_string(), size: 18 },
438            Bucket { name: DISCARDABLE_UNLOCKED.to_string(), size: 19 },
439            Bucket { name: ZRAM_COMPRESSED_BYTES.to_string(), size: 21 },
440        ];
441
442        assert_eq!(digest.buckets, expected_buckets);
443        Ok(())
444    }
445
446    #[test]
447    fn test_digest_with_matching_process_and_vmo() -> Result<(), anyhow::Error> {
448        let (kernel_stats, kernel_stats_compression) = get_kernel_stats();
449        let digest = Digest::compute(
450            &get_attribution_data(),
451            &kernel_stats,
452            &kernel_stats_compression,
453            &vec![BucketDefinition {
454                name: "matched".to_string(),
455                process: Some(Regex::new("matched")?),
456                vmo: Some(Regex::new("matched")?),
457                event_code: Default::default(),
458            }],
459        )
460        .unwrap();
461        let expected_buckets = vec![
462            Bucket { name: "matched".to_string(), size: 512 }, // One VMO is matched, the other is not
463            Bucket { name: UNDIGESTED.to_string(), size: 512 }, // One unmatched VMO
464            Bucket { name: ORPHANED.to_string(), size: 9488 }, // One matched VMO => 10000 - 512 = 9488
465            Bucket { name: KERNEL.to_string(), size: 31 }, // wired + heap + mmu + ipc + other => 3 + 4 + 7 + 8 + 9 = 31
466            Bucket { name: FREE.to_string(), size: 2 },
467            Bucket { name: PAGER_TOTAL.to_string(), size: 14 },
468            Bucket { name: PAGER_NEWEST.to_string(), size: 15 },
469            Bucket { name: PAGER_OLDEST.to_string(), size: 16 },
470            Bucket { name: DISCARDABLE_LOCKED.to_string(), size: 18 },
471            Bucket { name: DISCARDABLE_UNLOCKED.to_string(), size: 19 },
472            Bucket { name: ZRAM_COMPRESSED_BYTES.to_string(), size: 21 },
473        ];
474
475        assert_eq!(digest.buckets, expected_buckets);
476        Ok(())
477    }
478}
attribution_processing/digest.rs

attribution_processing/
digest.rs
