attribution_processing/
digest.rs

1// Copyright 2025 The Fuchsia Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5use crate::fplugin::{ResourceType, Vmo};
6use crate::AttributionData;
7use fidl_fuchsia_kernel as fkernel;
8use regex::Regex;
9use serde::de::Error;
10use serde::{Deserialize, Deserializer};
11use std::collections::hash_map::Entry::Occupied;
12use std::collections::HashMap;
13
14const UNDIGESTED: &str = "Undigested";
15const ORPHANED: &str = "Orphaned";
16const KERNEL: &str = "Kernel";
17const FREE: &str = "Free";
18const PAGER_TOTAL: &str = "[Addl]PagerTotal";
19const PAGER_NEWEST: &str = "[Addl]PagerNewest";
20const PAGER_OLDEST: &str = "[Addl]PagerOldest";
21const DISCARDABLE_LOCKED: &str = "[Addl]DiscardableLocked";
22const DISCARDABLE_UNLOCKED: &str = "[Addl]DiscardableUnlocked";
23const ZRAM_COMPRESSED_BYTES: &str = "[Addl]ZramCompressedBytes";
24
25/// Represents a specification for aggregating memory usage in meaningful groups.
26///
27/// `name` represents the meaningful name of the group; grouping is done based on process and VMO
28/// names.
29#[derive(Deserialize)]
30pub struct BucketDefinition {
31    pub name: String,
32    #[serde(deserialize_with = "deserialize_regex")]
33    pub process: Option<Regex>,
34    #[serde(deserialize_with = "deserialize_regex")]
35    pub vmo: Option<Regex>,
36}
37
38impl BucketDefinition {
39    /// Tests whether a process matches this bucket's definition, based on its name.
40    fn process_match(&self, process: &str) -> bool {
41        self.process.as_ref().map_or(true, |p| p.is_match(process))
42    }
43
44    /// Tests whether a VMO matches this bucket's definition, based on its name.
45    fn vmo_match(&self, vmo: &str) -> bool {
46        self.vmo.as_ref().map_or(true, |v| v.is_match(vmo))
47    }
48}
49
50// Teach serde to deserialize an optional regex.
51fn deserialize_regex<'de, D>(d: D) -> Result<Option<Regex>, D::Error>
52where
53    D: Deserializer<'de>,
54{
55    // Deserialize as Option<&str>
56    Option::<&str>::deserialize(d)
57        // If the parsing failed, return the error, otherwise transform the value
58        .and_then(|os| {
59            os
60                // If there is a value, try to parse it as a Regex.
61                .map(|s| {
62                    Regex::new(s)
63                        // If the regex compilation failed, wrap the error in the error type expected
64                        // by serde.
65                        .map_err(D::Error::custom)
66                })
67                // If there was a value but it failed to compile, return an error, otherwise return
68                // the potentially parsed option.
69                .transpose()
70        })
71}
72
73/// Aggregates bytes in categories with human readable names.
74#[derive(Clone, Debug, PartialEq, Eq)]
75pub struct Bucket {
76    pub name: String,
77    pub size: u64,
78}
79
80/// Contains a view of the system's memory usage, aggregated in groups called buckets, which are
81/// configurable.
82#[derive(Debug, PartialEq, Eq)]
83pub struct Digest {
84    pub buckets: Vec<Bucket>,
85}
86
87impl Digest {
88    /// Given means to query the system for memory usage, and a specification, this function
89    /// aggregates the current memory usage into human displayable units we call buckets.
90    pub fn new(
91        attribution_data: &AttributionData,
92        kmem_stats: fkernel::MemoryStats,
93        kmem_stats_compression: fkernel::MemoryStatsCompression,
94        bucket_definitions: &Vec<BucketDefinition>,
95    ) -> Self {
96        // Contains all VMOs that have not yet been aggregated.
97        let mut undigested_vmos: HashMap<u64, (&Vmo, &String)> = attribution_data
98            .resources_vec
99            .iter()
100            .filter_map(|r| match &r.resource_type {
101                ResourceType::Vmo(vmo) => {
102                    Some((r.koid, (vmo, &attribution_data.resource_names[r.name_index])))
103                }
104                _ => None,
105            })
106            .collect();
107
108        // Collection of processes currently live in the system.
109        let processes =
110            attribution_data.resources_vec.iter().filter_map(|r| match &r.resource_type {
111                ResourceType::Process(process) => {
112                    Some((process, &attribution_data.resource_names[r.name_index]))
113                }
114                _ => None,
115            });
116        // For each bucket, filter matching processes, and among their VMOs, account for those that
117        // have not been previously grouped in another buckets.  This means that the order of bucket
118        // definitions defines a priority in this aggregation.
119        let mut buckets: Vec<Bucket> = bucket_definitions
120            .iter()
121            .map(|bucket_def| Bucket {
122                name: bucket_def.name.clone(),
123                size: processes
124                    .clone()
125                    .filter(|&(_, process_name)| bucket_def.process_match(process_name))
126                    .fold(0, |size, (process, _)| {
127                        size + process
128                            .vmos
129                            .iter()
130                            .flatten()
131                            .map(|vmo| match undigested_vmos.entry(*vmo) {
132                                Occupied(e) => {
133                                    let &(vmo, name) = e.get();
134                                    if bucket_def.vmo_match(name) {
135                                        e.remove_entry();
136                                        vmo.total_committed_bytes.unwrap_or(0)
137                                    } else {
138                                        0
139                                    }
140                                }
141                                _ => 0,
142                            })
143                            .sum::<u64>()
144                    }),
145            })
146            .collect();
147
148        let vmo_size: u64 = buckets.iter().map(|Bucket { size, .. }| size).sum();
149        // Extend the configured aggregation with a number of additional, occasionally useful meta
150        // aggregations.
151        buckets.extend(vec![
152            // This bucket contains the total size of the VMOs that have not been covered by any
153            // other bucket.
154            Bucket {
155                name: UNDIGESTED.to_string(),
156                size: undigested_vmos
157                    .iter()
158                    .filter_map(|(_, (vmo, _))| vmo.total_committed_bytes)
159                    .sum(),
160            },
161            // This bucket accounts for VMO bytes that have been allocated by the kernel, but not
162            // claimed by any VMO (anymore).
163            Bucket {
164                name: ORPHANED.to_string(),
165                size: (kmem_stats.vmo_bytes.unwrap_or(0) - vmo_size)
166                    .clamp(0, kmem_stats.vmo_bytes.unwrap_or(0)),
167            },
168            // This bucket aggregates overall kernel memory usage.
169            Bucket {
170                name: KERNEL.to_string(),
171                size: (|| {
172                    Some(
173                        kmem_stats.wired_bytes?
174                            + kmem_stats.total_heap_bytes?
175                            + kmem_stats.mmu_overhead_bytes?
176                            + kmem_stats.ipc_bytes?
177                            + kmem_stats.other_bytes?,
178                    )
179                })()
180                .unwrap_or(0),
181            },
182            // This bucket contains this amount of free memory in the system.
183            Bucket { name: FREE.to_string(), size: kmem_stats.free_bytes.unwrap_or(0) },
184            // Those buckets contain pager related information.
185            Bucket {
186                name: PAGER_TOTAL.to_string(),
187                size: kmem_stats.vmo_reclaim_total_bytes.unwrap_or(0),
188            },
189            Bucket {
190                name: PAGER_NEWEST.to_string(),
191                size: kmem_stats.vmo_reclaim_newest_bytes.unwrap_or(0),
192            },
193            Bucket {
194                name: PAGER_OLDEST.to_string(),
195                size: kmem_stats.vmo_reclaim_oldest_bytes.unwrap_or(0),
196            },
197            // Those buckets account for discardable memory.
198            Bucket {
199                name: DISCARDABLE_LOCKED.to_string(),
200                size: kmem_stats.vmo_discardable_locked_bytes.unwrap_or(0),
201            },
202            Bucket {
203                name: DISCARDABLE_UNLOCKED.to_string(),
204                size: kmem_stats.vmo_discardable_unlocked_bytes.unwrap_or(0),
205            },
206            // This bucket accounts for compressed memory.
207            Bucket {
208                name: ZRAM_COMPRESSED_BYTES.to_string(),
209                size: kmem_stats_compression.compressed_storage_bytes.unwrap_or(0),
210            },
211        ]);
212        Digest { buckets }
213    }
214}
215
216#[cfg(test)]
217mod tests {
218    use super::*;
219    use crate::{
220        Attribution, Principal, PrincipalDescription, PrincipalIdentifier, PrincipalType, Resource,
221        ResourceReference,
222    };
223
224    use fidl_fuchsia_memory_attribution_plugin as fplugin;
225
226    fn get_attribution_data() -> AttributionData {
227        AttributionData {
228            principals_vec: vec![Principal {
229                identifier: PrincipalIdentifier(1),
230                description: PrincipalDescription::Component("principal".to_owned()),
231                principal_type: PrincipalType::Runnable,
232                parent: None,
233            }],
234            resources_vec: vec![
235                Resource {
236                    koid: 10,
237                    name_index: 0,
238                    resource_type: fplugin::ResourceType::Vmo(fplugin::Vmo {
239                        parent: None,
240                        private_committed_bytes: Some(1024),
241                        private_populated_bytes: Some(2048),
242                        scaled_committed_bytes: Some(1024),
243                        scaled_populated_bytes: Some(2048),
244                        total_committed_bytes: Some(1024),
245                        total_populated_bytes: Some(2048),
246                        ..Default::default()
247                    }),
248                },
249                Resource {
250                    koid: 20,
251                    name_index: 1,
252                    resource_type: fplugin::ResourceType::Vmo(fplugin::Vmo {
253                        parent: None,
254                        private_committed_bytes: Some(1024),
255                        private_populated_bytes: Some(2048),
256                        scaled_committed_bytes: Some(1024),
257                        scaled_populated_bytes: Some(2048),
258                        total_committed_bytes: Some(1024),
259                        total_populated_bytes: Some(2048),
260                        ..Default::default()
261                    }),
262                },
263                Resource {
264                    koid: 30,
265                    name_index: 1,
266                    resource_type: fplugin::ResourceType::Process(fplugin::Process {
267                        vmos: Some(vec![10, 20]),
268                        ..Default::default()
269                    }),
270                },
271            ],
272            resource_names: vec!["resource".to_owned(), "matched".to_owned()],
273            attributions: vec![Attribution {
274                source: PrincipalIdentifier(1),
275                subject: PrincipalIdentifier(1),
276                resources: vec![ResourceReference::KernelObject(10)],
277            }],
278        }
279    }
280
281    fn get_kernel_stats() -> (fkernel::MemoryStats, fkernel::MemoryStatsCompression) {
282        (
283            fkernel::MemoryStats {
284                total_bytes: Some(1),
285                free_bytes: Some(2),
286                wired_bytes: Some(3),
287                total_heap_bytes: Some(4),
288                free_heap_bytes: Some(5),
289                vmo_bytes: Some(10000),
290                mmu_overhead_bytes: Some(7),
291                ipc_bytes: Some(8),
292                other_bytes: Some(9),
293                free_loaned_bytes: Some(10),
294                cache_bytes: Some(11),
295                slab_bytes: Some(12),
296                zram_bytes: Some(13),
297                vmo_reclaim_total_bytes: Some(14),
298                vmo_reclaim_newest_bytes: Some(15),
299                vmo_reclaim_oldest_bytes: Some(16),
300                vmo_reclaim_disabled_bytes: Some(17),
301                vmo_discardable_locked_bytes: Some(18),
302                vmo_discardable_unlocked_bytes: Some(19),
303                ..Default::default()
304            },
305            fkernel::MemoryStatsCompression {
306                uncompressed_storage_bytes: Some(20),
307                compressed_storage_bytes: Some(21),
308                compressed_fragmentation_bytes: Some(22),
309                compression_time: Some(23),
310                decompression_time: Some(24),
311                total_page_compression_attempts: Some(25),
312                failed_page_compression_attempts: Some(26),
313                total_page_decompressions: Some(27),
314                compressed_page_evictions: Some(28),
315                eager_page_compressions: Some(29),
316                memory_pressure_page_compressions: Some(30),
317                critical_memory_page_compressions: Some(31),
318                pages_decompressed_unit_ns: Some(32),
319                pages_decompressed_within_log_time: Some([40, 41, 42, 43, 44, 45, 46, 47]),
320                ..Default::default()
321            },
322        )
323    }
324
325    #[test]
326    fn test_digest_no_definitions() {
327        let (kernel_stats, kernel_stats_compression) = get_kernel_stats();
328        let digest =
329            Digest::new(&get_attribution_data(), kernel_stats, kernel_stats_compression, &vec![]);
330        let expected_buckets = vec![
331            Bucket { name: UNDIGESTED.to_string(), size: 2048 }, // The two VMOs are unmatched, 1024 + 1024
332            Bucket { name: ORPHANED.to_string(), size: 10000 }, // No matched VMOs => kernel's VMO bytes
333            Bucket { name: KERNEL.to_string(), size: 31 }, // wired + heap + mmu + ipc + other => 3 + 4 + 7 + 8 + 9 = 31
334            Bucket { name: FREE.to_string(), size: 2 },
335            Bucket { name: PAGER_TOTAL.to_string(), size: 14 },
336            Bucket { name: PAGER_NEWEST.to_string(), size: 15 },
337            Bucket { name: PAGER_OLDEST.to_string(), size: 16 },
338            Bucket { name: DISCARDABLE_LOCKED.to_string(), size: 18 },
339            Bucket { name: DISCARDABLE_UNLOCKED.to_string(), size: 19 },
340            Bucket { name: ZRAM_COMPRESSED_BYTES.to_string(), size: 21 },
341        ];
342
343        assert_eq!(digest.buckets, expected_buckets);
344    }
345
346    #[test]
347    fn test_digest_with_matching_vmo() -> Result<(), anyhow::Error> {
348        let (kernel_stats, kernel_stats_compression) = get_kernel_stats();
349        let digest = Digest::new(
350            &get_attribution_data(),
351            kernel_stats,
352            kernel_stats_compression,
353            &vec![BucketDefinition {
354                name: "matched".to_string(),
355                process: None,
356                vmo: Some(Regex::new("matched")?),
357            }],
358        );
359        let expected_buckets = vec![
360            Bucket { name: "matched".to_string(), size: 1024 }, // One VMO is matched, the other is not
361            Bucket { name: UNDIGESTED.to_string(), size: 1024 }, // One unmatched VMO
362            Bucket { name: ORPHANED.to_string(), size: 8976 }, // One matched VMO => 10000 - 1024 = 8976
363            Bucket { name: KERNEL.to_string(), size: 31 }, // wired + heap + mmu + ipc + other => 3 + 4 + 7 + 8 + 9 = 31
364            Bucket { name: FREE.to_string(), size: 2 },
365            Bucket { name: PAGER_TOTAL.to_string(), size: 14 },
366            Bucket { name: PAGER_NEWEST.to_string(), size: 15 },
367            Bucket { name: PAGER_OLDEST.to_string(), size: 16 },
368            Bucket { name: DISCARDABLE_LOCKED.to_string(), size: 18 },
369            Bucket { name: DISCARDABLE_UNLOCKED.to_string(), size: 19 },
370            Bucket { name: ZRAM_COMPRESSED_BYTES.to_string(), size: 21 },
371        ];
372
373        assert_eq!(digest.buckets, expected_buckets);
374        Ok(())
375    }
376
377    #[test]
378    fn test_digest_with_matching_process() -> Result<(), anyhow::Error> {
379        let (kernel_stats, kernel_stats_compression) = get_kernel_stats();
380        let digest = Digest::new(
381            &get_attribution_data(),
382            kernel_stats,
383            kernel_stats_compression,
384            &vec![BucketDefinition {
385                name: "matched".to_string(),
386                process: Some(Regex::new("matched")?),
387                vmo: None,
388            }],
389        );
390        let expected_buckets = vec![
391            Bucket { name: "matched".to_string(), size: 2048 }, // Both VMOs are matched => 1024 + 1024 = 2048
392            Bucket { name: UNDIGESTED.to_string(), size: 0 },   // No unmatched VMO
393            Bucket { name: ORPHANED.to_string(), size: 7952 }, // Two matched VMO => 10000 - 1024 - 1024 = 7952
394            Bucket { name: KERNEL.to_string(), size: 31 }, // wired + heap + mmu + ipc + other => 3 + 4 + 7 + 8 + 9 = 31
395            Bucket { name: FREE.to_string(), size: 2 },
396            Bucket { name: PAGER_TOTAL.to_string(), size: 14 },
397            Bucket { name: PAGER_NEWEST.to_string(), size: 15 },
398            Bucket { name: PAGER_OLDEST.to_string(), size: 16 },
399            Bucket { name: DISCARDABLE_LOCKED.to_string(), size: 18 },
400            Bucket { name: DISCARDABLE_UNLOCKED.to_string(), size: 19 },
401            Bucket { name: ZRAM_COMPRESSED_BYTES.to_string(), size: 21 },
402        ];
403
404        assert_eq!(digest.buckets, expected_buckets);
405        Ok(())
406    }
407
408    #[test]
409    fn test_digest_with_matching_process_and_vmo() -> Result<(), anyhow::Error> {
410        let (kernel_stats, kernel_stats_compression) = get_kernel_stats();
411        let digest = Digest::new(
412            &get_attribution_data(),
413            kernel_stats,
414            kernel_stats_compression,
415            &vec![BucketDefinition {
416                name: "matched".to_string(),
417                process: Some(Regex::new("matched")?),
418                vmo: Some(Regex::new("matched")?),
419            }],
420        );
421        let expected_buckets = vec![
422            Bucket { name: "matched".to_string(), size: 1024 }, // One VMO is matched, the other is not
423            Bucket { name: UNDIGESTED.to_string(), size: 1024 }, // One unmatched VMO
424            Bucket { name: ORPHANED.to_string(), size: 8976 }, // One matched VMO => 10000 - 1024 = 8976
425            Bucket { name: KERNEL.to_string(), size: 31 }, // wired + heap + mmu + ipc + other => 3 + 4 + 7 + 8 + 9 = 31
426            Bucket { name: FREE.to_string(), size: 2 },
427            Bucket { name: PAGER_TOTAL.to_string(), size: 14 },
428            Bucket { name: PAGER_NEWEST.to_string(), size: 15 },
429            Bucket { name: PAGER_OLDEST.to_string(), size: 16 },
430            Bucket { name: DISCARDABLE_LOCKED.to_string(), size: 18 },
431            Bucket { name: DISCARDABLE_UNLOCKED.to_string(), size: 19 },
432            Bucket { name: ZRAM_COMPRESSED_BYTES.to_string(), size: 21 },
433        ];
434
435        assert_eq!(digest.buckets, expected_buckets);
436        Ok(())
437    }
438}
attribution_processing/digest.rs

attribution_processing/
digest.rs
