wasmtime/runtime/
vm.rs

1//! Runtime library support for Wasmtime.
2
3#![deny(missing_docs)]
4#![warn(clippy::cast_sign_loss)]
5
6use crate::prelude::*;
7use alloc::sync::Arc;
8use core::fmt;
9use core::mem;
10use core::ptr::NonNull;
11use core::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
12use wasmtime_environ::{
13    DefinedFuncIndex, DefinedMemoryIndex, HostPtr, ModuleInternedTypeIndex, VMOffsets,
14    VMSharedTypeIndex,
15};
16
17mod arch;
18mod async_yield;
19#[cfg(feature = "component-model")]
20pub mod component;
21mod const_expr;
22mod export;
23mod gc;
24mod imports;
25mod instance;
26mod memory;
27mod mmap;
28mod mmap_vec;
29mod send_sync_ptr;
30mod store_box;
31mod sys;
32mod table;
33mod traphandlers;
34mod vmcontext;
35
36mod threads;
37pub use self::threads::*;
38
39#[cfg(feature = "debug-builtins")]
40pub mod debug_builtins;
41pub mod libcalls;
42pub mod mpk;
43
44#[cfg(feature = "debug-builtins")]
45pub use wasmtime_jit_debug::gdb_jit_int::GdbJitImageRegistration;
46
47pub use crate::runtime::vm::arch::get_stack_pointer;
48pub use crate::runtime::vm::async_yield::*;
49pub use crate::runtime::vm::export::*;
50pub use crate::runtime::vm::gc::*;
51pub use crate::runtime::vm::imports::Imports;
52pub use crate::runtime::vm::instance::{
53    GcHeapAllocationIndex, Instance, InstanceAllocationRequest, InstanceAllocator,
54    InstanceAllocatorImpl, InstanceHandle, MemoryAllocationIndex, OnDemandInstanceAllocator,
55    StorePtr, TableAllocationIndex,
56};
57#[cfg(feature = "pooling-allocator")]
58pub use crate::runtime::vm::instance::{
59    InstanceLimits, PoolConcurrencyLimitError, PoolingInstanceAllocator,
60    PoolingInstanceAllocatorConfig,
61};
62pub use crate::runtime::vm::memory::{Memory, RuntimeLinearMemory, RuntimeMemoryCreator};
63pub use crate::runtime::vm::mmap::Mmap;
64pub use crate::runtime::vm::mmap_vec::MmapVec;
65pub use crate::runtime::vm::mpk::MpkEnabled;
66pub use crate::runtime::vm::store_box::*;
67pub use crate::runtime::vm::sys::unwind::UnwindRegistration;
68pub use crate::runtime::vm::table::{Table, TableElement};
69pub use crate::runtime::vm::traphandlers::*;
70pub use crate::runtime::vm::vmcontext::{
71    VMArrayCallFunction, VMArrayCallHostFuncContext, VMContext, VMFuncRef, VMFunctionBody,
72    VMFunctionImport, VMGlobalDefinition, VMGlobalImport, VMMemoryDefinition, VMMemoryImport,
73    VMOpaqueContext, VMRuntimeLimits, VMTableImport, VMWasmCallFunction, ValRaw,
74};
75pub use send_sync_ptr::SendSyncPtr;
76
77mod module_id;
78pub use module_id::CompiledModuleId;
79
80mod cow;
81pub use crate::runtime::vm::cow::{MemoryImage, MemoryImageSlot, ModuleMemoryImages};
82
83/// Dynamic runtime functionality needed by this crate throughout the execution
84/// of a wasm instance.
85///
86/// This trait is used to store a raw pointer trait object within each
87/// `VMContext`. This raw pointer trait object points back to the
88/// `wasmtime::Store` internally but is type-erased so this `wasmtime-runtime`
89/// crate doesn't need the entire `wasmtime` crate to build.
90///
91/// Note that this is an extra-unsafe trait because no heed is paid to the
92/// lifetime of this store or the Send/Sync-ness of this store. All of that must
93/// be respected by embedders (e.g. the `wasmtime::Store` structure). The theory
94/// is that `wasmtime::Store` handles all this correctly.
95pub unsafe trait Store {
96    /// Returns the raw pointer in memory where this store's shared
97    /// `VMRuntimeLimits` structure is located.
98    ///
99    /// Used to configure `VMContext` initialization and store the right pointer
100    /// in the `VMContext`.
101    fn vmruntime_limits(&self) -> *mut VMRuntimeLimits;
102
103    /// Returns a pointer to the global epoch counter.
104    ///
105    /// Used to configure the `VMContext` on initialization.
106    fn epoch_ptr(&self) -> *const AtomicU64;
107
108    /// Get this store's GC heap.
109    fn gc_store(&mut self) -> &mut GcStore {
110        self.maybe_gc_store()
111            .expect("attempt to access the GC store before it has been allocated")
112    }
113
114    /// Get this store's GC heap, if it has been allocated.
115    fn maybe_gc_store(&mut self) -> Option<&mut GcStore>;
116
117    /// Callback invoked to allow the store's resource limiter to reject a
118    /// memory grow operation.
119    fn memory_growing(
120        &mut self,
121        current: usize,
122        desired: usize,
123        maximum: Option<usize>,
124    ) -> Result<bool, Error>;
125
126    /// Callback invoked to notify the store's resource limiter that a memory
127    /// grow operation has failed.
128    ///
129    /// Note that this is not invoked if `memory_growing` returns an error.
130    fn memory_grow_failed(&mut self, error: Error) -> Result<()>;
131
132    /// Callback invoked to allow the store's resource limiter to reject a
133    /// table grow operation.
134    fn table_growing(
135        &mut self,
136        current: u32,
137        desired: u32,
138        maximum: Option<u32>,
139    ) -> Result<bool, Error>;
140
141    /// Callback invoked to notify the store's resource limiter that a table
142    /// grow operation has failed.
143    ///
144    /// Note that this is not invoked if `table_growing` returns an error.
145    fn table_grow_failed(&mut self, error: Error) -> Result<()>;
146
147    /// Callback invoked whenever fuel runs out by a wasm instance. If an error
148    /// is returned that's raised as a trap. Otherwise wasm execution will
149    /// continue as normal.
150    fn out_of_gas(&mut self) -> Result<(), Error>;
151
152    /// Callback invoked whenever an instance observes a new epoch
153    /// number. Cannot fail; cooperative epoch-based yielding is
154    /// completely semantically transparent. Returns the new deadline.
155    fn new_epoch(&mut self) -> Result<u64, Error>;
156
157    /// Callback invoked whenever an instance needs to trigger a GC.
158    ///
159    /// Optionally given a GC reference that is rooted for the collection, and
160    /// then whose updated GC reference is returned.
161    ///
162    /// Cooperative, async-yielding (if configured) is completely transparent.
163    ///
164    /// If the async GC was cancelled, returns an error. This should be raised
165    /// as a trap to clean up Wasm execution.
166    fn gc(&mut self, root: Option<VMGcRef>) -> Result<Option<VMGcRef>>;
167
168    /// Metadata required for resources for the component model.
169    #[cfg(feature = "component-model")]
170    fn component_calls(&mut self) -> &mut component::CallContexts;
171}
172
173/// Functionality required by this crate for a particular module. This
174/// is chiefly needed for lazy initialization of various bits of
175/// instance state.
176///
177/// When an instance is created, it holds an `Arc<dyn ModuleRuntimeInfo>`
178/// so that it can get to signatures, metadata on functions, memory and
179/// funcref-table images, etc. All of these things are ordinarily known
180/// by the higher-level layers of Wasmtime. Specifically, the main
181/// implementation of this trait is provided by
182/// `wasmtime::module::ModuleInner`.  Since the runtime crate sits at
183/// the bottom of the dependence DAG though, we don't know or care about
184/// that; we just need some implementor of this trait for each
185/// allocation request.
186#[derive(Clone)]
187pub enum ModuleRuntimeInfo {
188    Module(crate::Module),
189    Bare(Box<BareModuleInfo>),
190}
191
192/// A barebones implementation of ModuleRuntimeInfo that is useful for
193/// cases where a purpose-built environ::Module is used and a full
194/// CompiledModule does not exist (for example, for tests or for the
195/// default-callee instance).
196#[derive(Clone)]
197pub struct BareModuleInfo {
198    module: Arc<wasmtime_environ::Module>,
199    one_signature: Option<VMSharedTypeIndex>,
200    offsets: VMOffsets<HostPtr>,
201}
202
203impl ModuleRuntimeInfo {
204    pub(crate) fn bare(module: Arc<wasmtime_environ::Module>) -> Self {
205        ModuleRuntimeInfo::bare_maybe_imported_func(module, None)
206    }
207
208    pub(crate) fn bare_maybe_imported_func(
209        module: Arc<wasmtime_environ::Module>,
210        one_signature: Option<VMSharedTypeIndex>,
211    ) -> Self {
212        ModuleRuntimeInfo::Bare(Box::new(BareModuleInfo {
213            offsets: VMOffsets::new(HostPtr, &module),
214            module,
215            one_signature,
216        }))
217    }
218
219    /// The underlying Module.
220    pub(crate) fn module(&self) -> &Arc<wasmtime_environ::Module> {
221        match self {
222            ModuleRuntimeInfo::Module(m) => m.env_module(),
223            ModuleRuntimeInfo::Bare(b) => &b.module,
224        }
225    }
226
227    /// Translate a module-level interned type index into an engine-level
228    /// interned type index.
229    fn engine_type_index(&self, module_index: ModuleInternedTypeIndex) -> VMSharedTypeIndex {
230        match self {
231            ModuleRuntimeInfo::Module(m) => m
232                .code_object()
233                .signatures()
234                .shared_type(module_index)
235                .expect("bad module-level interned type index"),
236            ModuleRuntimeInfo::Bare(_) => unreachable!(),
237        }
238    }
239
240    /// Returns the address, in memory, that the function `index` resides at.
241    fn function(&self, index: DefinedFuncIndex) -> NonNull<VMWasmCallFunction> {
242        let module = match self {
243            ModuleRuntimeInfo::Module(m) => m,
244            ModuleRuntimeInfo::Bare(_) => unreachable!(),
245        };
246        let ptr = module
247            .compiled_module()
248            .finished_function(index)
249            .as_ptr()
250            .cast::<VMWasmCallFunction>()
251            .cast_mut();
252        NonNull::new(ptr).unwrap()
253    }
254
255    /// Returns the address, in memory, of the trampoline that allows the given
256    /// defined Wasm function to be called by the array calling convention.
257    ///
258    /// Returns `None` for Wasm functions which do not escape, and therefore are
259    /// not callable from outside the Wasm module itself.
260    fn array_to_wasm_trampoline(&self, index: DefinedFuncIndex) -> Option<VMArrayCallFunction> {
261        let m = match self {
262            ModuleRuntimeInfo::Module(m) => m,
263            ModuleRuntimeInfo::Bare(_) => unreachable!(),
264        };
265        let ptr = m
266            .compiled_module()
267            .array_to_wasm_trampoline(index)?
268            .as_ptr();
269        Some(unsafe { mem::transmute::<*const u8, VMArrayCallFunction>(ptr) })
270    }
271
272    /// Returns the `MemoryImage` structure used for copy-on-write
273    /// initialization of the memory, if it's applicable.
274    fn memory_image(
275        &self,
276        memory: DefinedMemoryIndex,
277    ) -> anyhow::Result<Option<&Arc<MemoryImage>>> {
278        match self {
279            ModuleRuntimeInfo::Module(m) => {
280                let images = m.memory_images()?;
281                Ok(images.and_then(|images| images.get_memory_image(memory)))
282            }
283            ModuleRuntimeInfo::Bare(_) => Ok(None),
284        }
285    }
286
287    /// A unique ID for this particular module. This can be used to
288    /// allow for fastpaths to optimize a "re-instantiate the same
289    /// module again" case.
290    fn unique_id(&self) -> Option<CompiledModuleId> {
291        match self {
292            ModuleRuntimeInfo::Module(m) => Some(m.id()),
293            ModuleRuntimeInfo::Bare(_) => None,
294        }
295    }
296
297    /// A slice pointing to all data that is referenced by this instance.
298    fn wasm_data(&self) -> &[u8] {
299        match self {
300            ModuleRuntimeInfo::Module(m) => m.compiled_module().code_memory().wasm_data(),
301            ModuleRuntimeInfo::Bare(_) => &[],
302        }
303    }
304
305    /// Returns an array, indexed by `ModuleInternedTypeIndex` of all
306    /// `VMSharedSignatureIndex` entries corresponding to the `SignatureIndex`.
307    fn type_ids(&self) -> &[VMSharedTypeIndex] {
308        match self {
309            ModuleRuntimeInfo::Module(m) => m
310                .code_object()
311                .signatures()
312                .as_module_map()
313                .values()
314                .as_slice(),
315            ModuleRuntimeInfo::Bare(b) => match &b.one_signature {
316                Some(s) => core::slice::from_ref(s),
317                None => &[],
318            },
319        }
320    }
321
322    /// Offset information for the current host.
323    pub(crate) fn offsets(&self) -> &VMOffsets<HostPtr> {
324        match self {
325            ModuleRuntimeInfo::Module(m) => m.offsets(),
326            ModuleRuntimeInfo::Bare(b) => &b.offsets,
327        }
328    }
329}
330
331/// Returns the host OS page size, in bytes.
332pub fn host_page_size() -> usize {
333    static PAGE_SIZE: AtomicUsize = AtomicUsize::new(0);
334
335    return match PAGE_SIZE.load(Ordering::Relaxed) {
336        0 => {
337            let size = sys::vm::get_page_size();
338            assert!(size != 0);
339            PAGE_SIZE.store(size, Ordering::Relaxed);
340            size
341        }
342        n => n,
343    };
344}
345
346/// Is `bytes` a multiple of the host page size?
347pub fn usize_is_multiple_of_host_page_size(bytes: usize) -> bool {
348    bytes % host_page_size() == 0
349}
350
351/// Round the given byte size up to a multiple of the host OS page size.
352///
353/// Returns an error if rounding up overflows.
354pub fn round_u64_up_to_host_pages(bytes: u64) -> Result<u64> {
355    let page_size = u64::try_from(crate::runtime::vm::host_page_size()).err2anyhow()?;
356    debug_assert!(page_size.is_power_of_two());
357    bytes
358        .checked_add(page_size - 1)
359        .ok_or_else(|| anyhow!(
360            "{bytes} is too large to be rounded up to a multiple of the host page size of {page_size}"
361        ))
362        .map(|val| val & !(page_size - 1))
363}
364
365/// Same as `round_u64_up_to_host_pages` but for `usize`s.
366pub fn round_usize_up_to_host_pages(bytes: usize) -> Result<usize> {
367    let bytes = u64::try_from(bytes).err2anyhow()?;
368    let rounded = round_u64_up_to_host_pages(bytes)?;
369    Ok(usize::try_from(rounded).err2anyhow()?)
370}
371
372/// Result of `Memory::atomic_wait32` and `Memory::atomic_wait64`
373#[derive(Copy, Clone, PartialEq, Eq, Debug)]
374pub enum WaitResult {
375    /// Indicates that a `wait` completed by being awoken by a different thread.
376    /// This means the thread went to sleep and didn't time out.
377    Ok = 0,
378    /// Indicates that `wait` did not complete and instead returned due to the
379    /// value in memory not matching the expected value.
380    Mismatch = 1,
381    /// Indicates that `wait` completed with a timeout, meaning that the
382    /// original value matched as expected but nothing ever called `notify`.
383    TimedOut = 2,
384}
385
386/// Description about a fault that occurred in WebAssembly.
387#[derive(Debug)]
388pub struct WasmFault {
389    /// The size of memory, in bytes, at the time of the fault.
390    pub memory_size: usize,
391    /// The WebAssembly address at which the fault occurred.
392    pub wasm_address: u64,
393}
394
395impl fmt::Display for WasmFault {
396    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
397        write!(
398            f,
399            "memory fault at wasm address 0x{:x} in linear memory of size 0x{:x}",
400            self.wasm_address, self.memory_size,
401        )
402    }
403}
wasmtime/runtime/vm.rs

wasmtime/runtime/
vm.rs
