wasmtime/runtime/vm/sys/unix/
vm.rs

1use crate::runtime::vm::sys::DecommitBehavior;
2use rustix::fd::AsRawFd;
3use rustix::mm::{MapFlags, MprotectFlags, ProtFlags, mmap_anonymous, mprotect};
4use std::fs::File;
5use std::io;
6#[cfg(feature = "std")]
7use std::sync::Arc;
8
9pub use super::pagemap::{PageMap, reset_with_pagemap};
10
11pub unsafe fn expose_existing_mapping(ptr: *mut u8, len: usize) -> io::Result<()> {
12    unsafe {
13        mprotect(ptr.cast(), len, MprotectFlags::READ | MprotectFlags::WRITE)?;
14    }
15    Ok(())
16}
17
18pub unsafe fn hide_existing_mapping(ptr: *mut u8, len: usize) -> io::Result<()> {
19    unsafe {
20        mprotect(ptr.cast(), len, MprotectFlags::empty())?;
21    }
22    Ok(())
23}
24
25pub unsafe fn erase_existing_mapping(ptr: *mut u8, len: usize) -> io::Result<()> {
26    let ret = unsafe {
27        mmap_anonymous(
28            ptr.cast(),
29            len,
30            ProtFlags::empty(),
31            MapFlags::PRIVATE | super::mmap::MMAP_NORESERVE_FLAG | MapFlags::FIXED,
32        )?
33    };
34    assert_eq!(ptr, ret.cast());
35    Ok(())
36}
37
38#[cfg(feature = "pooling-allocator")]
39pub unsafe fn commit_pages(_addr: *mut u8, _len: usize) -> io::Result<()> {
40    // Pages are always READ | WRITE so there's nothing that needs to be done
41    // here.
42    Ok(())
43}
44
45#[cfg(feature = "pooling-allocator")]
46pub unsafe fn decommit_pages(addr: *mut u8, len: usize) -> io::Result<()> {
47    if len == 0 {
48        return Ok(());
49    }
50
51    unsafe {
52        cfg_if::cfg_if! {
53            if #[cfg(target_os = "linux")] {
54                use rustix::mm::{madvise, Advice};
55
56                // On Linux, this is enough to cause the kernel to initialize
57                // the pages to 0 on next access
58                madvise(addr as _, len, Advice::LinuxDontNeed)?;
59            } else {
60                // By creating a new mapping at the same location, this will
61                // discard the mapping for the pages in the given range.
62                // The new mapping will be to the CoW zero page, so this
63                // effectively zeroes the pages.
64                mmap_anonymous(
65                    addr as _,
66                    len,
67                    ProtFlags::READ | ProtFlags::WRITE,
68                    MapFlags::PRIVATE | super::mmap::MMAP_NORESERVE_FLAG | MapFlags::FIXED,
69                )?;
70            }
71        }
72    }
73
74    Ok(())
75}
76
77// NB: this function is duplicated in `crates/fiber/src/unix.rs` so if this
78// changes that should probably get updated as well.
79pub fn get_page_size() -> usize {
80    unsafe { libc::sysconf(libc::_SC_PAGESIZE).try_into().unwrap() }
81}
82
83pub fn decommit_behavior() -> DecommitBehavior {
84    if cfg!(target_os = "linux") {
85        DecommitBehavior::RestoreOriginalMapping
86    } else {
87        DecommitBehavior::Zero
88    }
89}
90
91#[derive(Debug)]
92pub enum MemoryImageSource {
93    #[cfg(feature = "std")]
94    Mmap(Arc<File>),
95    #[cfg(target_os = "linux")]
96    Memfd(memfd::Memfd),
97}
98
99impl MemoryImageSource {
100    #[cfg(feature = "std")]
101    pub fn from_file(file: &Arc<File>) -> Option<MemoryImageSource> {
102        Some(MemoryImageSource::Mmap(file.clone()))
103    }
104
105    #[cfg(not(target_os = "linux"))]
106    pub fn from_data(_data: &[u8]) -> io::Result<Option<MemoryImageSource>> {
107        Ok(None)
108    }
109
110    #[cfg(target_os = "linux")]
111    pub fn from_data(data: &[u8]) -> anyhow::Result<Option<MemoryImageSource>> {
112        // On Linux `memfd_create` is used to create an anonymous
113        // in-memory file to represent the heap image. This anonymous
114        // file is then used as the basis for further mmaps.
115
116        use std::io::{ErrorKind, Write};
117
118        // Create the memfd. It needs a name, but the documentation for
119        // `memfd_create()` says that names can be duplicated with no issues.
120        let memfd = match memfd::MemfdOptions::new()
121            .allow_sealing(true)
122            .create("wasm-memory-image")
123        {
124            Ok(memfd) => memfd,
125            // If this kernel is old enough to not support memfd then attempt to
126            // gracefully handle that and fall back to skipping the memfd
127            // optimization.
128            Err(memfd::Error::Create(err)) if err.kind() == ErrorKind::Unsupported => {
129                return Ok(None);
130            }
131            Err(e) => return Err(e.into()),
132        };
133        memfd.as_file().write_all(data)?;
134
135        // Seal the memfd's data and length.
136        //
137        // This is a defense-in-depth security mitigation. The
138        // memfd will serve as the starting point for the heap of
139        // every instance of this module. If anything were to
140        // write to this, it could affect every execution. The
141        // memfd object itself is owned by the machinery here and
142        // not exposed elsewhere, but it is still an ambient open
143        // file descriptor at the syscall level, so some other
144        // vulnerability that allowed writes to arbitrary fds
145        // could modify it. Or we could have some issue with the
146        // way that we map it into each instance. To be
147        // extra-super-sure that it never changes, and because
148        // this costs very little, we use the kernel's "seal" API
149        // to make the memfd image permanently read-only.
150        memfd.add_seals(&[
151            memfd::FileSeal::SealGrow,
152            memfd::FileSeal::SealShrink,
153            memfd::FileSeal::SealWrite,
154            memfd::FileSeal::SealSeal,
155        ])?;
156
157        Ok(Some(MemoryImageSource::Memfd(memfd)))
158    }
159
160    pub(super) fn as_file(&self) -> &File {
161        match *self {
162            #[cfg(feature = "std")]
163            MemoryImageSource::Mmap(ref file) => file,
164            #[cfg(target_os = "linux")]
165            MemoryImageSource::Memfd(ref memfd) => memfd.as_file(),
166        }
167    }
168
169    pub unsafe fn remap_as_zeros_at(&self, base: *mut u8, len: usize) -> io::Result<()> {
170        let ptr = unsafe {
171            mmap_anonymous(
172                base.cast(),
173                len,
174                ProtFlags::READ | ProtFlags::WRITE,
175                MapFlags::PRIVATE | super::mmap::MMAP_NORESERVE_FLAG | MapFlags::FIXED,
176            )?
177        };
178        assert_eq!(base, ptr.cast());
179        Ok(())
180    }
181}
182
183impl PartialEq for MemoryImageSource {
184    fn eq(&self, other: &MemoryImageSource) -> bool {
185        self.as_file().as_raw_fd() == other.as_file().as_raw_fd()
186    }
187}
wasmtime/runtime/vm/sys/unix/vm.rs

wasmtime/runtime/vm/sys/unix/
vm.rs
