export_ffs/
lib.rs

1// Copyright 2020 The Fuchsia Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5//! Export a fuchsia.io/Directory as a factory filesystem partition on a provided block device.
6
7#![deny(missing_docs)]
8
9use anyhow::{bail, Context, Error};
10use block_client::cache::Cache;
11use block_client::RemoteBlockClientSync;
12use byteorder::{LittleEndian, WriteBytesExt};
13use fidl_fuchsia_hardware_block::BlockMarker;
14use fidl_fuchsia_io as fio;
15use fuchsia_fs::directory::{readdir_recursive, DirEntry, DirentKind};
16use futures::StreamExt;
17use std::io::Write;
18
19const FACTORYFS_MAGIC: u64 = 0xa55d3ff91e694d21;
20const BLOCK_SIZE: u32 = 4096;
21const DIRENT_START_BLOCK: u32 = 1;
22/// Size of the actual data in the superblock in bytes. We are only writing, and only care about the
23/// latest version, so as long as this is updated whenever the superblock is changed we don't have
24/// to worry about backwards-compatibility.
25const SUPERBLOCK_DATA_SIZE: u32 = 52;
26const FACTORYFS_MAJOR_VERSION: u32 = 1;
27const FACTORYFS_MINOR_VERSION: u32 = 0;
28
29/// Rounds the given num up to the next multiple of multiple.
30fn round_up_to_align(x: u32, align: u32) -> u32 {
31    debug_assert_ne!(align, 0);
32    debug_assert_eq!(align & (align - 1), 0);
33    (x + align - 1) & !(align - 1)
34}
35
36/// Return the number of blocks needed to store the provided number of bytes. This function will
37/// overflow for values of [`bytes`] within about BLOCK_SIZE of u32::MAX, but it shouldn't be a
38/// problem because that's already getting dangerously close to the maximum file size, which is the
39/// same amount.
40fn num_blocks(bytes: u32) -> u32 {
41    // note: integer division in rust truncates the result
42    (bytes + BLOCK_SIZE - 1) / BLOCK_SIZE
43}
44
45/// Round the provided number of bytes up to the next block boundary. Similar to the C++
46/// `fbl::round_up(bytes, BLOCK_SIZE)`.
47fn round_up_to_block_size(bytes: u32) -> u32 {
48    num_blocks(bytes) * BLOCK_SIZE
49}
50
51/// Align the writer to the next block boundary by padding the rest of the current block with zeros.
52/// [`written_bytes`] is the number of bytes written since the last time this writer was block
53/// aligned. If the writer is already block aligned, no bytes are written.
54fn block_align<Writer>(writer: &mut Writer, written_bytes: u32) -> Result<(), Error>
55where
56    Writer: Write,
57{
58    let fill = round_up_to_block_size(written_bytes) - written_bytes;
59    for _ in 0..fill {
60        writer.write_u8(0)?;
61    }
62    Ok(())
63}
64
65/// in-memory representation of a factoryfs partition
66struct FactoryFS {
67    major_version: u32,
68    minor_version: u32,
69    flags: u32,
70    block_size: u32,
71    entries: Vec<DirectoryEntry>,
72}
73
74impl FactoryFS {
75    fn serialize_superblock<Writer>(&self, writer: &mut Writer) -> Result<(u32, u32), Error>
76    where
77        Writer: Write,
78    {
79        // on-disk format of the superblock for a factoryfs partition, in rust-ish notation
80        // #[repr(C, packed)]
81        // struct Superblock {
82        //     /// Must be FACTORYFS_MAGIC.
83        //     magic: u64,
84        //     major_version: u32,
85        //     minor_version: u32,
86        //     flags: u32,
87        //     /// Total number of data blocks.
88        //     data_blocks: u32,
89        //     /// Size in bytes of all the directory entries.
90        //     directory_size: u32,
91        //     /// Number of directory entries.
92        //     directory_entries: u32,
93        //     /// Time of creation of all files. We aren't going to write anything here though.
94        //     create_time: u64,
95        //     /// Filesystem block size.
96        //     block_size: u32,
97        //     /// Number of blocks for directory entries.
98        //     directory_ent_blocks: u32,
99        //     /// Start block for directory entries.
100        //     directory_ent_start_block: u32,
101        //     /// Reserved for future use. Written to disk as all zeros.
102        //     reserved: [u32; rest_of_the_block],
103        // }
104
105        writer.write_u64::<LittleEndian>(FACTORYFS_MAGIC).context("failed to write magic")?;
106        writer.write_u32::<LittleEndian>(self.major_version)?;
107        writer.write_u32::<LittleEndian>(self.minor_version)?;
108        writer.write_u32::<LittleEndian>(self.flags)?;
109
110        // calculate the number of blocks all the data will take
111        let data_blocks = self
112            .entries
113            .iter()
114            .fold(0, |blocks, entry| blocks + num_blocks(entry.data.len() as u32));
115        writer.write_u32::<LittleEndian>(data_blocks)?;
116
117        // calculate the size of all the directory entries
118        let entries_bytes = self.entries.iter().fold(0, |size, entry| size + entry.metadata_size());
119        let entries_blocks = num_blocks(entries_bytes);
120        writer.write_u32::<LittleEndian>(entries_bytes)?;
121        writer.write_u32::<LittleEndian>(self.entries.len() as u32)?;
122
123        // filesystem was created at the beginning of time
124        writer.write_u64::<LittleEndian>(0)?;
125
126        writer.write_u32::<LittleEndian>(self.block_size)?;
127
128        writer.write_u32::<LittleEndian>(entries_blocks)?;
129        writer.write_u32::<LittleEndian>(DIRENT_START_BLOCK)?;
130
131        Ok((entries_bytes, entries_blocks))
132    }
133
134    /// Write the bytes of a FactoryFS partition to a byte writer. We assume the writer is seeked to
135    /// the beginning. Serialization returns immediately after encountering a writing error for the
136    /// first time, and as such may be in the middle of writing the partition. On error, there is no
137    /// guarantee of a consistent partition.
138    ///
139    /// NOTE: if the superblock serialization is changed in any way, make sure SUPERBLOCK_DATA_SIZE
140    /// is still correct.
141    fn serialize<Writer>(&self, writer: &mut Writer) -> Result<(), Error>
142    where
143        Writer: Write,
144    {
145        let (entries_bytes, entries_blocks) =
146            self.serialize_superblock(writer).context("failed to serialize superblock")?;
147
148        // write out zeros for the rest of the first block
149        block_align(writer, SUPERBLOCK_DATA_SIZE)?;
150
151        // data starts after the superblock and all the blocks for the directory entries
152        let mut data_offset = DIRENT_START_BLOCK + entries_blocks;
153        // write out the directory entry metadata
154        for entry in &self.entries {
155            entry.serialize_metadata(writer, data_offset)?;
156            data_offset += num_blocks(entry.data.len() as u32);
157        }
158
159        block_align(writer, entries_bytes)?;
160
161        // write out the entry data
162        for entry in &self.entries {
163            entry.serialize_data(writer)?;
164        }
165
166        Ok(())
167    }
168}
169
170/// a directory entry (aka file). factoryfs is a flat filesystem - conceptually there is a single
171/// root directory that contains all the entries.
172#[derive(Debug, PartialEq, Eq)]
173struct DirectoryEntry {
174    name: Vec<u8>,
175    data: Vec<u8>,
176}
177
178impl DirectoryEntry {
179    /// Get the size of the serialized metadata for this directory entry in bytes.
180    fn metadata_size(&self) -> u32 {
181        let name_len = self.name.len() as u32;
182        let padding = round_up_to_align(name_len, 4) - name_len;
183
184        // size of name_len...
185        4
186        // ...plus the size of data_len...
187        + 4
188        // ...plus the size of data_offset...
189        + 4
190        // ...plus the number of bytes in the name...
191        + name_len
192        // ...plus some padding to align to name to a 4-byte boundary
193        + padding
194    }
195
196    /// Write the directory entry metadata to the provided byte writer. We assume that the calling
197    /// function has seeked to the expected metadata location. data_offset is the expected block at
198    /// which the data associated with this entry will be written in the future.
199    fn serialize_metadata<Writer>(&self, writer: &mut Writer, data_offset: u32) -> Result<(), Error>
200    where
201        Writer: Write,
202    {
203        // on-disk format of a directory entry, in rust-ish notation
204        // #[repr(C, packed)]
205        // struct DirectoryEntry {
206        //     /// Length of the name[] field at the end.
207        //     name_len: u32,
208
209        //     /// Length of the file in bytes. This is an exact size that is not rounded, though
210        //     /// the file is always padded with zeros up to a multiple of block size (aka
211        //     /// block-aligned).
212        //     data_len: u32,
213
214        //     /// Block offset where file data starts for this entry.
215        //     data_off: u32,
216
217        //     /// Pathname of the file, a UTF-8 string. It must not begin with a '/', but it may
218        //     /// contain '/' separators. The string is not null-terminated. The end of the struct
219        //     /// must be padded to align on a 4 byte boundary.
220        //     name: [u8; self.name_len]
221        // }
222
223        let name_len = self.name.len() as u32;
224        writer.write_u32::<LittleEndian>(name_len)?;
225        writer.write_u32::<LittleEndian>(self.data.len() as u32)?;
226        writer.write_u32::<LittleEndian>(data_offset)?;
227        writer.write_all(&self.name)?;
228
229        // align the directory entry to a 4-byte boundary
230        let padding = round_up_to_align(name_len, 4) - name_len;
231        for _ in 0..padding {
232            writer.write_u8(0)?;
233        }
234
235        Ok(())
236    }
237
238    /// Write the data associated with this directory entry to the provided byte writer. We assume
239    /// that the calling function has seeked to the expected block-aligned data location. This
240    /// function fills the rest of the block with zeros, leaving the cursor position at the
241    /// beginning of the next unused block.
242    fn serialize_data<Writer>(&self, writer: &mut Writer) -> Result<(), Error>
243    where
244        Writer: Write,
245    {
246        writer.write_all(&self.data)?;
247        block_align(writer, self.data.len() as u32)?;
248        Ok(())
249    }
250}
251
252async fn get_entries(dir: &fio::DirectoryProxy) -> Result<Vec<DirectoryEntry>, Error> {
253    let out: Vec<DirEntry> = readdir_recursive(dir, None).map(|x| x.unwrap()).collect().await;
254
255    let mut entries = vec![];
256    for ent in out {
257        if ent.kind != DirentKind::File {
258            // If we run into anything in this partition that isn't a file, there is some kind of
259            // problem with our environment. Surface that information so that it can get fixed.
260            bail!("Directory entry '{}' is not a file. FactoryFS can only contain files.", ent.name)
261        }
262
263        // NB: We are loading all the files we are going to serialize into memory first.
264        // if the partition is too big this will be a problem.
265        let (file_proxy, server_end) = fidl::endpoints::create_proxy::<fio::FileMarker>();
266        dir.open(
267            &ent.name,
268            fio::PERM_READABLE | fio::Flags::PROTOCOL_FILE,
269            &Default::default(),
270            server_end.into_channel(),
271        )
272        .with_context(|| format!("failed to open file {}", ent.name))?;
273
274        let result =
275            file_proxy.get_attributes(fio::NodeAttributesQuery::CONTENT_SIZE).await.with_context(
276                || format!("failed to get attributes of file {}: (fidl failure)", ent.name),
277            )?;
278        match result {
279            Err(_) => bail!("failed to get attributes of file {}", ent.name),
280            Ok((_, immutable_attributes)) => {
281                let data = file_proxy
282                    .read(immutable_attributes.content_size.unwrap_or_default())
283                    .await
284                    .with_context(|| {
285                        format!("failed to read contents of file {}: (fidl failure)", ent.name)
286                    })?
287                    .map_err(zx::Status::from_raw)
288                    .with_context(|| format!("failed to read contents of file {}", ent.name))?;
289
290                entries.push(DirectoryEntry { name: ent.name.as_bytes().to_vec(), data });
291            }
292        }
293    }
294
295    entries.sort_by(|a, b| a.name.cmp(&b.name));
296
297    Ok(entries)
298}
299
300async fn write_directory<W: Write>(dir: &fio::DirectoryProxy, device: &mut W) -> Result<(), Error> {
301    let entries = get_entries(dir).await.context("failed to get entries from directory")?;
302
303    let factoryfs = FactoryFS {
304        major_version: FACTORYFS_MAJOR_VERSION,
305        minor_version: FACTORYFS_MINOR_VERSION,
306        flags: 0,
307        block_size: BLOCK_SIZE,
308        entries,
309    };
310
311    factoryfs.serialize(device).context("failed to serialize factoryfs")?;
312
313    Ok(())
314}
315
316/// Export the contents of a fuchsia.io/Directory as a flat FactoryFS partition on the provided
317/// device. All files are extracted from the directory and placed in the FactoryFS partition, with a
318/// name that corresponds with the complete path of the file in the original directory, relative to
319/// that directory. It takes ownership of the channel to the device, which it assumes speaks
320/// fuchsia.hardware.Block, and closes it after all the writes are issued to the block device.
321pub async fn export_directory(
322    dir: &fio::DirectoryProxy,
323    client_end: fidl::endpoints::ClientEnd<BlockMarker>,
324) -> Result<(), Error> {
325    let device = RemoteBlockClientSync::new(client_end)
326        .context("failed to create remote block device client")?;
327    let mut device = Cache::new(device).context("failed to create cache layer for block device")?;
328
329    write_directory(dir, &mut device).await.context("failed to write out directory")?;
330
331    device.flush().context("failed to flush to device")?;
332
333    Ok(())
334}
335
336#[cfg(test)]
337mod tests {
338    use super::{
339        block_align, export_directory, get_entries, num_blocks, round_up_to_block_size,
340        DirectoryEntry, FactoryFS, BLOCK_SIZE, FACTORYFS_MAJOR_VERSION, FACTORYFS_MINOR_VERSION,
341        SUPERBLOCK_DATA_SIZE,
342    };
343
344    use assert_matches::assert_matches;
345    use fidl_fuchsia_io as fio;
346    use ramdevice_client::RamdiskClient;
347    use vfs::file::vmo::read_only;
348    use vfs::pseudo_directory;
349
350    #[test]
351    fn test_num_blocks() {
352        assert_eq!(num_blocks(0), 0);
353        assert_eq!(num_blocks(1), 1);
354        assert_eq!(num_blocks(10), 1);
355        assert_eq!(num_blocks(BLOCK_SIZE - 1), 1);
356        assert_eq!(num_blocks(BLOCK_SIZE), 1);
357        assert_eq!(num_blocks(BLOCK_SIZE + 1), 2);
358    }
359
360    #[test]
361    fn test_round_up() {
362        assert_eq!(round_up_to_block_size(0), 0);
363        assert_eq!(round_up_to_block_size(1), BLOCK_SIZE);
364        assert_eq!(round_up_to_block_size(BLOCK_SIZE - 1), BLOCK_SIZE);
365        assert_eq!(round_up_to_block_size(BLOCK_SIZE), BLOCK_SIZE);
366        assert_eq!(round_up_to_block_size(BLOCK_SIZE + 1), BLOCK_SIZE * 2);
367    }
368
369    #[test]
370    fn test_block_align() {
371        let mut cases = vec![
372            // (bytes already written, pad bytes to block align)
373            (0, 0),
374            (1, BLOCK_SIZE - 1),
375            (BLOCK_SIZE - 1, 1),
376            (BLOCK_SIZE, 0),
377            (BLOCK_SIZE + 1, BLOCK_SIZE - 1),
378        ];
379
380        for case in &mut cases {
381            let mut w = vec![];
382            assert_matches!(block_align(&mut w, case.0), Ok(()));
383            assert_eq!(w.len(), case.1 as usize);
384            assert!(w.into_iter().all(|v| v == 0));
385        }
386    }
387
388    #[test]
389    fn test_superblock_data() {
390        let name = "test_name".as_bytes();
391        let data = vec![1, 2, 3, 4, 5];
392
393        let entry = DirectoryEntry { name: name.to_owned(), data: data.clone() };
394
395        let metadata_size = entry.metadata_size();
396        let metadata_blocks = num_blocks(metadata_size);
397
398        let factoryfs = FactoryFS {
399            major_version: FACTORYFS_MAJOR_VERSION,
400            minor_version: FACTORYFS_MINOR_VERSION,
401            flags: 0,
402            block_size: BLOCK_SIZE,
403            entries: vec![entry],
404        };
405
406        let mut out = vec![];
407        assert_eq!(
408            factoryfs.serialize_superblock(&mut out).unwrap(),
409            (metadata_size, metadata_blocks),
410        );
411
412        assert_eq!(out.len() as u32, SUPERBLOCK_DATA_SIZE);
413    }
414
415    #[test]
416    fn test_dirent_metadata() {
417        let data_offset = 12;
418        let data = vec![1, 2, 3, 4, 5];
419
420        let mut out: Vec<u8> = vec![];
421        let name = "test_name".as_bytes();
422        let dirent = DirectoryEntry { name: name.to_owned(), data };
423
424        assert_matches!(dirent.serialize_metadata(&mut out, data_offset), Ok(()));
425        assert_eq!(dirent.metadata_size(), out.len() as u32);
426    }
427
428    #[fuchsia::test]
429    async fn test_export() {
430        let dir = pseudo_directory! {
431            "a" => read_only("a content"),
432            "b" => pseudo_directory! {
433                "c" => read_only("c content"),
434            },
435        };
436        let dir_proxy = vfs::directory::serve(dir, fio::PERM_READABLE | fio::PERM_WRITABLE);
437        let ramdisk = RamdiskClient::create(512, 1 << 16).await.unwrap();
438        let channel = ramdisk.open().unwrap();
439
440        assert_matches!(export_directory(&dir_proxy, channel).await, Ok(()));
441    }
442
443    #[fuchsia::test]
444    async fn test_get_entries() {
445        let dir = pseudo_directory! {
446            "a" => read_only("a content"),
447            "d" => read_only("d content"),
448            "b" => pseudo_directory! {
449                "c" => read_only("c content"),
450            },
451        };
452        let dir_proxy = vfs::directory::serve(dir, fio::PERM_READABLE | fio::PERM_WRITABLE);
453        let entries = get_entries(&dir_proxy).await.unwrap();
454
455        assert_eq!(
456            entries,
457            vec![
458                DirectoryEntry { name: b"a".to_vec(), data: b"a content".to_vec() },
459                DirectoryEntry { name: b"b/c".to_vec(), data: b"c content".to_vec() },
460                DirectoryEntry { name: b"d".to_vec(), data: b"d content".to_vec() },
461            ],
462        );
463    }
464}