wasmtime/runtime/vm/component/
libcalls.rs

1//! Implementation of string transcoding required by the component model.
2
3use crate::component::Instance;
4use crate::prelude::*;
5#[cfg(feature = "component-model-async")]
6use crate::runtime::component::concurrent::ResourcePair;
7use crate::runtime::vm::component::{ComponentInstance, VMComponentContext};
8use crate::runtime::vm::{HostResultHasUnwindSentinel, VMStore, VmSafe};
9use core::cell::Cell;
10use core::ptr::NonNull;
11use core::slice;
12use wasmtime_environ::component::*;
13
14const UTF16_TAG: usize = 1 << 31;
15
16macro_rules! signature {
17    (@ty size) => (usize);
18    (@ty ptr_u8) => (*mut u8);
19    (@ty ptr_u16) => (*mut u16);
20    (@ty ptr_size) => (*mut usize);
21    (@ty u8) => (u8);
22    (@ty u32) => (u32);
23    (@ty u64) => (u64);
24    (@ty bool) => (bool);
25    (@ty vmctx) => (NonNull<VMComponentContext>);
26}
27
28/// Defines a `VMComponentBuiltins` structure which contains any builtins such
29/// as resource-related intrinsics.
30macro_rules! define_builtins {
31    (
32        $(
33            $( #[$attr:meta] )*
34            $name:ident( $( $pname:ident: $param:ident ),* ) $( -> $result:ident )?;
35        )*
36    ) => {
37        /// An array that stores addresses of builtin functions. We translate code
38        /// to use indirect calls. This way, we don't have to patch the code.
39        #[repr(C)]
40        pub struct VMComponentBuiltins {
41            $(
42                $name: unsafe extern "C" fn(
43                    $(signature!(@ty $param),)*
44                ) $( -> signature!(@ty $result))?,
45            )*
46        }
47
48        // SAFETY: the above structure is repr(C) and only contains `VmSafe`
49        // fields.
50        unsafe impl VmSafe for VMComponentBuiltins {}
51
52        impl VMComponentBuiltins {
53            pub const INIT: VMComponentBuiltins = VMComponentBuiltins {
54                $($name: trampolines::$name,)*
55            };
56
57            /// Helper to call `expose_provenance()` on all contained pointers.
58            ///
59            /// This is required to be called at least once before entering wasm
60            /// to inform the compiler that these function pointers may all be
61            /// loaded/stored and used on the "other end" to reacquire
62            /// provenance in Pulley. Pulley models hostcalls with a host
63            /// pointer as the first parameter that's a function pointer under
64            /// the hood, and this call ensures that the use of the function
65            /// pointer is considered valid.
66            pub fn expose_provenance(&self) -> NonNull<Self>{
67                $(
68                    (self.$name as *mut u8).expose_provenance();
69                )*
70                NonNull::from(self)
71            }
72        }
73    };
74}
75
76wasmtime_environ::foreach_builtin_component_function!(define_builtins);
77
78/// Submodule with macro-generated constants which are the actual libcall
79/// transcoders that are invoked by Cranelift. These functions have a specific
80/// ABI defined by the macro itself and will defer to the actual bodies of each
81/// implementation following this submodule.
82mod trampolines {
83    use super::{ComponentInstance, VMComponentContext};
84    use core::ptr::NonNull;
85
86    macro_rules! shims {
87        (
88            $(
89                $( #[cfg($attr:meta)] )?
90                $name:ident( vmctx: vmctx $(, $pname:ident: $param:ident )* ) $( -> $result:ident )?;
91            )*
92        ) => (
93            $(
94                pub unsafe extern "C" fn $name(
95                    vmctx: NonNull<VMComponentContext>
96                    $(,$pname : signature!(@ty $param))*
97                ) $( -> signature!(@ty $result))? {
98                    $(#[cfg($attr)])?
99                    {
100                        $(shims!(@validate_param $pname $param);)*
101
102                        let ret = unsafe {
103                            ComponentInstance::enter_host_from_wasm(vmctx, |store, instance| {
104                                shims!(@invoke $name(store, instance,) $($pname)*)
105                            })
106                        };
107                        shims!(@convert_ret ret $($pname: $param)*)
108                    }
109                    $(
110                        #[cfg(not($attr))]
111                        {
112                            let _ = vmctx;
113                            unreachable!();
114                        }
115                    )?
116                }
117            )*
118        );
119
120        // Helper macro to convert a 2-tuple automatically when the last
121        // parameter is a `ptr_size` argument.
122        (@convert_ret $ret:ident) => ($ret);
123        (@convert_ret $ret:ident $retptr:ident: ptr_size) => ({
124            let (a, b) = $ret;
125            unsafe {
126                *$retptr = b;
127            }
128            a
129        });
130        (@convert_ret $ret:ident $name:ident: $ty:ident $($rest:tt)*) => (
131            shims!(@convert_ret $ret $($rest)*)
132        );
133
134        (@validate_param $arg:ident ptr_u16) => ({
135            // This should already be guaranteed by the canonical ABI and our
136            // adapter modules, but double-check here to be extra-sure. If this
137            // is a perf concern it can become a `debug_assert!`.
138            assert!(($arg as usize) % 2 == 0, "unaligned 16-bit pointer");
139        });
140        (@validate_param $arg:ident $ty:ident) => ();
141
142        // Helper macro to invoke `$m` with all of the tokens passed except for
143        // any argument named `ret2`
144        (@invoke $m:ident ($($args:tt)*)) => (super::$m($($args)*));
145
146        // ignore `ret2`-named arguments
147        (@invoke $m:ident ($($args:tt)*) ret2 $($rest:tt)*) => (
148            shims!(@invoke $m ($($args)*) $($rest)*)
149        );
150
151        // move all other arguments into the `$args` list
152        (@invoke $m:ident ($($args:tt)*) $param:ident $($rest:tt)*) => (
153            shims!(@invoke $m ($($args)* $param,) $($rest)*)
154        );
155    }
156
157    wasmtime_environ::foreach_builtin_component_function!(shims);
158}
159
160/// This property should already be guaranteed by construction in the component
161/// model but assert it here to be extra sure. Nothing below is sound if regions
162/// can overlap.
163fn assert_no_overlap<T, U>(a: &[T], b: &[U]) {
164    let a_start = a.as_ptr() as usize;
165    let a_end = a_start + (a.len() * core::mem::size_of::<T>());
166    let b_start = b.as_ptr() as usize;
167    let b_end = b_start + (b.len() * core::mem::size_of::<U>());
168
169    if a_start < b_start {
170        assert!(a_end < b_start);
171    } else {
172        assert!(b_end < a_start);
173    }
174}
175
176/// Converts a utf8 string to a utf8 string.
177///
178/// The length provided is length of both the source and the destination
179/// buffers. No value is returned other than whether an invalid string was
180/// found.
181unsafe fn utf8_to_utf8(
182    _: &mut dyn VMStore,
183    _: Instance,
184    src: *mut u8,
185    len: usize,
186    dst: *mut u8,
187) -> Result<()> {
188    let src = unsafe { slice::from_raw_parts(src, len) };
189    let dst = unsafe { slice::from_raw_parts_mut(dst, len) };
190    assert_no_overlap(src, dst);
191    log::trace!("utf8-to-utf8 {len}");
192    let src = core::str::from_utf8(src).map_err(|_| anyhow!("invalid utf8 encoding"))?;
193    dst.copy_from_slice(src.as_bytes());
194    Ok(())
195}
196
197/// Converts a utf16 string to a utf16 string.
198///
199/// The length provided is length of both the source and the destination
200/// buffers. No value is returned other than whether an invalid string was
201/// found.
202unsafe fn utf16_to_utf16(
203    _: &mut dyn VMStore,
204    _: Instance,
205    src: *mut u16,
206    len: usize,
207    dst: *mut u16,
208) -> Result<()> {
209    let src = unsafe { slice::from_raw_parts(src, len) };
210    let dst = unsafe { slice::from_raw_parts_mut(dst, len) };
211    assert_no_overlap(src, dst);
212    log::trace!("utf16-to-utf16 {len}");
213    run_utf16_to_utf16(src, dst)?;
214    Ok(())
215}
216
217/// Transcodes utf16 to itself, returning whether all code points were inside of
218/// the latin1 space.
219fn run_utf16_to_utf16(src: &[u16], mut dst: &mut [u16]) -> Result<bool> {
220    let mut all_latin1 = true;
221    for ch in core::char::decode_utf16(src.iter().map(|i| u16::from_le(*i))) {
222        let ch = ch.map_err(|_| anyhow!("invalid utf16 encoding"))?;
223        all_latin1 = all_latin1 && u8::try_from(u32::from(ch)).is_ok();
224        let result = ch.encode_utf16(dst);
225        let size = result.len();
226        for item in result {
227            *item = item.to_le();
228        }
229        dst = &mut dst[size..];
230    }
231    Ok(all_latin1)
232}
233
234/// Converts a latin1 string to a latin1 string.
235///
236/// Given that all byte sequences are valid latin1 strings this is simply a
237/// memory copy.
238unsafe fn latin1_to_latin1(
239    _: &mut dyn VMStore,
240    _: Instance,
241    src: *mut u8,
242    len: usize,
243    dst: *mut u8,
244) -> Result<()> {
245    let src = unsafe { slice::from_raw_parts(src, len) };
246    let dst = unsafe { slice::from_raw_parts_mut(dst, len) };
247    assert_no_overlap(src, dst);
248    log::trace!("latin1-to-latin1 {len}");
249    dst.copy_from_slice(src);
250    Ok(())
251}
252
253/// Converts a latin1 string to a utf16 string.
254///
255/// This simply inflates the latin1 characters to the u16 code points. The
256/// length provided is the same length of the source and destination buffers.
257unsafe fn latin1_to_utf16(
258    _: &mut dyn VMStore,
259    _: Instance,
260    src: *mut u8,
261    len: usize,
262    dst: *mut u16,
263) -> Result<()> {
264    let src = unsafe { slice::from_raw_parts(src, len) };
265    let dst = unsafe { slice::from_raw_parts_mut(dst, len) };
266    assert_no_overlap(src, dst);
267    for (src, dst) in src.iter().zip(dst) {
268        *dst = u16::from(*src).to_le();
269    }
270    log::trace!("latin1-to-utf16 {len}");
271    Ok(())
272}
273
274struct CopySizeReturn(usize);
275
276unsafe impl HostResultHasUnwindSentinel for CopySizeReturn {
277    type Abi = usize;
278    const SENTINEL: usize = usize::MAX;
279    fn into_abi(self) -> usize {
280        self.0
281    }
282}
283
284/// Converts utf8 to utf16.
285///
286/// The length provided is the same unit length of both buffers, and the
287/// returned value from this function is how many u16 units were written.
288unsafe fn utf8_to_utf16(
289    _: &mut dyn VMStore,
290    _: Instance,
291    src: *mut u8,
292    len: usize,
293    dst: *mut u16,
294) -> Result<CopySizeReturn> {
295    let src = unsafe { slice::from_raw_parts(src, len) };
296    let dst = unsafe { slice::from_raw_parts_mut(dst, len) };
297    assert_no_overlap(src, dst);
298
299    let result = run_utf8_to_utf16(src, dst)?;
300    log::trace!("utf8-to-utf16 {len} => {result}");
301    Ok(CopySizeReturn(result))
302}
303
304fn run_utf8_to_utf16(src: &[u8], dst: &mut [u16]) -> Result<usize> {
305    let src = core::str::from_utf8(src).map_err(|_| anyhow!("invalid utf8 encoding"))?;
306    let mut amt = 0;
307    for (i, dst) in src.encode_utf16().zip(dst) {
308        *dst = i.to_le();
309        amt += 1;
310    }
311    Ok(amt)
312}
313
314struct SizePair {
315    src_read: usize,
316    dst_written: usize,
317}
318
319unsafe impl HostResultHasUnwindSentinel for SizePair {
320    type Abi = (usize, usize);
321    const SENTINEL: (usize, usize) = (usize::MAX, 0);
322    fn into_abi(self) -> (usize, usize) {
323        (self.src_read, self.dst_written)
324    }
325}
326
327/// Converts utf16 to utf8.
328///
329/// Each buffer is specified independently here and the returned value is a pair
330/// of the number of code units read and code units written. This might perform
331/// a partial transcode if the destination buffer is not large enough to hold
332/// the entire contents.
333unsafe fn utf16_to_utf8(
334    _: &mut dyn VMStore,
335    _: Instance,
336    src: *mut u16,
337    src_len: usize,
338    dst: *mut u8,
339    dst_len: usize,
340) -> Result<SizePair> {
341    let src = unsafe { slice::from_raw_parts(src, src_len) };
342    let mut dst = unsafe { slice::from_raw_parts_mut(dst, dst_len) };
343    assert_no_overlap(src, dst);
344
345    // This iterator will convert to native endianness and additionally count
346    // how many items have been read from the iterator so far. This
347    // count is used to return how many of the source code units were read.
348    let src_iter_read = Cell::new(0);
349    let src_iter = src.iter().map(|i| {
350        src_iter_read.set(src_iter_read.get() + 1);
351        u16::from_le(*i)
352    });
353
354    let mut src_read = 0;
355    let mut dst_written = 0;
356
357    for ch in core::char::decode_utf16(src_iter) {
358        let ch = ch.map_err(|_| anyhow!("invalid utf16 encoding"))?;
359
360        // If the destination doesn't have enough space for this character
361        // then the loop is ended and this function will be called later with a
362        // larger destination buffer.
363        if dst.len() < 4 && dst.len() < ch.len_utf8() {
364            break;
365        }
366
367        // Record that characters were read and then convert the `char` to
368        // utf-8, advancing the destination buffer.
369        src_read = src_iter_read.get();
370        let len = ch.encode_utf8(dst).len();
371        dst_written += len;
372        dst = &mut dst[len..];
373    }
374
375    log::trace!("utf16-to-utf8 {src_len}/{dst_len} => {src_read}/{dst_written}");
376    Ok(SizePair {
377        src_read,
378        dst_written,
379    })
380}
381
382/// Converts latin1 to utf8.
383///
384/// Receives the independent size of both buffers and returns the number of code
385/// units read and code units written (both bytes in this case).
386///
387/// This may perform a partial encoding if the destination is not large enough.
388unsafe fn latin1_to_utf8(
389    _: &mut dyn VMStore,
390    _: Instance,
391    src: *mut u8,
392    src_len: usize,
393    dst: *mut u8,
394    dst_len: usize,
395) -> Result<SizePair> {
396    let src = unsafe { slice::from_raw_parts(src, src_len) };
397    let dst = unsafe { slice::from_raw_parts_mut(dst, dst_len) };
398    assert_no_overlap(src, dst);
399    let (read, written) = encoding_rs::mem::convert_latin1_to_utf8_partial(src, dst);
400    log::trace!("latin1-to-utf8 {src_len}/{dst_len} => ({read}, {written})");
401    Ok(SizePair {
402        src_read: read,
403        dst_written: written,
404    })
405}
406
407/// Converts utf16 to "latin1+utf16", probably using a utf16 encoding.
408///
409/// The length specified is the length of both the source and destination
410/// buffers. If the source string has any characters that don't fit in the
411/// latin1 code space (0xff and below) then a utf16-tagged length will be
412/// returned. Otherwise the string is "deflated" from a utf16 string to a latin1
413/// string and the latin1 length is returned.
414unsafe fn utf16_to_compact_probably_utf16(
415    _: &mut dyn VMStore,
416    _: Instance,
417    src: *mut u16,
418    len: usize,
419    dst: *mut u16,
420) -> Result<CopySizeReturn> {
421    let src = unsafe { slice::from_raw_parts(src, len) };
422    let dst = unsafe { slice::from_raw_parts_mut(dst, len) };
423    assert_no_overlap(src, dst);
424    let all_latin1 = run_utf16_to_utf16(src, dst)?;
425    if all_latin1 {
426        let (left, dst, right) = unsafe { dst.align_to_mut::<u8>() };
427        assert!(left.is_empty());
428        assert!(right.is_empty());
429        for i in 0..len {
430            dst[i] = dst[2 * i];
431        }
432        log::trace!("utf16-to-compact-probably-utf16 {len} => latin1 {len}");
433        Ok(CopySizeReturn(len))
434    } else {
435        log::trace!("utf16-to-compact-probably-utf16 {len} => utf16 {len}");
436        Ok(CopySizeReturn(len | UTF16_TAG))
437    }
438}
439
440/// Converts a utf8 string to latin1.
441///
442/// The length specified is the same length of both the input and the output
443/// buffers.
444///
445/// Returns the number of code units read from the source and the number of code
446/// units written to the destination.
447///
448/// Note that this may not convert the entire source into the destination if the
449/// original utf8 string has usvs not representable in latin1.
450unsafe fn utf8_to_latin1(
451    _: &mut dyn VMStore,
452    _: Instance,
453    src: *mut u8,
454    len: usize,
455    dst: *mut u8,
456) -> Result<SizePair> {
457    let src = unsafe { slice::from_raw_parts(src, len) };
458    let dst = unsafe { slice::from_raw_parts_mut(dst, len) };
459    assert_no_overlap(src, dst);
460    let read = encoding_rs::mem::utf8_latin1_up_to(src);
461    let written = encoding_rs::mem::convert_utf8_to_latin1_lossy(&src[..read], dst);
462    log::trace!("utf8-to-latin1 {len} => ({read}, {written})");
463    Ok(SizePair {
464        src_read: read,
465        dst_written: written,
466    })
467}
468
469/// Converts a utf16 string to latin1
470///
471/// This is the same as `utf8_to_latin1` in terms of parameters/results.
472unsafe fn utf16_to_latin1(
473    _: &mut dyn VMStore,
474    _: Instance,
475    src: *mut u16,
476    len: usize,
477    dst: *mut u8,
478) -> Result<SizePair> {
479    let src = unsafe { slice::from_raw_parts(src, len) };
480    let dst = unsafe { slice::from_raw_parts_mut(dst, len) };
481    assert_no_overlap(src, dst);
482
483    let mut size = 0;
484    for (src, dst) in src.iter().zip(dst) {
485        let src = u16::from_le(*src);
486        match u8::try_from(src) {
487            Ok(src) => *dst = src,
488            Err(_) => break,
489        }
490        size += 1;
491    }
492    log::trace!("utf16-to-latin1 {len} => {size}");
493    Ok(SizePair {
494        src_read: size,
495        dst_written: size,
496    })
497}
498
499/// Converts a utf8 string to a utf16 string which has been partially converted
500/// as latin1 prior.
501///
502/// The original string has already been partially transcoded with
503/// `utf8_to_latin1` and that was determined to not be able to transcode the
504/// entire string. The substring of the source that couldn't be encoded into
505/// latin1 is passed here via `src` and `src_len`.
506///
507/// The destination buffer is specified by `dst` and `dst_len`. The first
508/// `latin1_bytes_so_far` bytes (not code units) of the `dst` buffer have
509/// already been filled in with latin1 characters and need to be inflated
510/// in-place to their utf16 equivalents.
511///
512/// After the initial latin1 code units have been inflated the entirety of `src`
513/// is then transcoded into the remaining space within `dst`.
514unsafe fn utf8_to_compact_utf16(
515    _: &mut dyn VMStore,
516    _: Instance,
517    src: *mut u8,
518    src_len: usize,
519    dst: *mut u16,
520    dst_len: usize,
521    latin1_bytes_so_far: usize,
522) -> Result<CopySizeReturn> {
523    let src = unsafe { slice::from_raw_parts(src, src_len) };
524    let dst = unsafe { slice::from_raw_parts_mut(dst, dst_len) };
525    assert_no_overlap(src, dst);
526
527    let dst = inflate_latin1_bytes(dst, latin1_bytes_so_far);
528    let result = run_utf8_to_utf16(src, dst)?;
529    log::trace!("utf8-to-compact-utf16 {src_len}/{dst_len}/{latin1_bytes_so_far} => {result}");
530    Ok(CopySizeReturn(result + latin1_bytes_so_far))
531}
532
533/// Same as `utf8_to_compact_utf16` but for utf16 source strings.
534unsafe fn utf16_to_compact_utf16(
535    _: &mut dyn VMStore,
536    _: Instance,
537    src: *mut u16,
538    src_len: usize,
539    dst: *mut u16,
540    dst_len: usize,
541    latin1_bytes_so_far: usize,
542) -> Result<CopySizeReturn> {
543    let src = unsafe { slice::from_raw_parts(src, src_len) };
544    let dst = unsafe { slice::from_raw_parts_mut(dst, dst_len) };
545    assert_no_overlap(src, dst);
546
547    let dst = inflate_latin1_bytes(dst, latin1_bytes_so_far);
548    run_utf16_to_utf16(src, dst)?;
549    let result = src.len();
550    log::trace!("utf16-to-compact-utf16 {src_len}/{dst_len}/{latin1_bytes_so_far} => {result}");
551    Ok(CopySizeReturn(result + latin1_bytes_so_far))
552}
553
554/// Inflates the `latin1_bytes_so_far` number of bytes written to the beginning
555/// of `dst` into u16 codepoints.
556///
557/// Returns the remaining space in the destination that can be transcoded into,
558/// slicing off the prefix of the string that was inflated from the latin1
559/// bytes.
560fn inflate_latin1_bytes(dst: &mut [u16], latin1_bytes_so_far: usize) -> &mut [u16] {
561    // Note that `latin1_bytes_so_far` is a byte measure while `dst` is a region
562    // of u16 units. This `split_at_mut` uses the byte index as an index into
563    // the u16 unit because each of the latin1 bytes will become a whole code
564    // unit in the destination which is 2 bytes large.
565    let (to_inflate, rest) = dst.split_at_mut(latin1_bytes_so_far);
566
567    // Use a byte-oriented view to inflate the original latin1 bytes.
568    let (left, mid, right) = unsafe { to_inflate.align_to_mut::<u8>() };
569    assert!(left.is_empty());
570    assert!(right.is_empty());
571    for i in (0..latin1_bytes_so_far).rev() {
572        mid[2 * i] = mid[i];
573        mid[2 * i + 1] = 0;
574    }
575
576    return rest;
577}
578
579fn resource_new32(
580    store: &mut dyn VMStore,
581    instance: Instance,
582    caller_instance: u32,
583    resource: u32,
584    rep: u32,
585) -> Result<u32> {
586    let caller_instance = RuntimeComponentInstanceIndex::from_u32(caller_instance);
587    let resource = TypeResourceTableIndex::from_u32(resource);
588    instance.resource_new32(store, caller_instance, resource, rep)
589}
590
591fn resource_rep32(
592    store: &mut dyn VMStore,
593    instance: Instance,
594    caller_instance: u32,
595    resource: u32,
596    idx: u32,
597) -> Result<u32> {
598    let caller_instance = RuntimeComponentInstanceIndex::from_u32(caller_instance);
599    let resource = TypeResourceTableIndex::from_u32(resource);
600    instance.resource_rep32(store, caller_instance, resource, idx)
601}
602
603fn resource_drop(
604    store: &mut dyn VMStore,
605    instance: Instance,
606    caller_instance: u32,
607    resource: u32,
608    idx: u32,
609) -> Result<ResourceDropRet> {
610    let caller_instance = RuntimeComponentInstanceIndex::from_u32(caller_instance);
611    let resource = TypeResourceTableIndex::from_u32(resource);
612    Ok(ResourceDropRet(instance.resource_drop(
613        store,
614        caller_instance,
615        resource,
616        idx,
617    )?))
618}
619
620struct ResourceDropRet(Option<u32>);
621
622unsafe impl HostResultHasUnwindSentinel for ResourceDropRet {
623    type Abi = u64;
624    const SENTINEL: u64 = u64::MAX;
625    fn into_abi(self) -> u64 {
626        match self.0 {
627            Some(rep) => (u64::from(rep) << 1) | 1,
628            None => 0,
629        }
630    }
631}
632
633fn resource_transfer_own(
634    store: &mut dyn VMStore,
635    instance: Instance,
636    src_idx: u32,
637    src_table: u32,
638    dst_table: u32,
639) -> Result<u32> {
640    let src_table = TypeResourceTableIndex::from_u32(src_table);
641    let dst_table = TypeResourceTableIndex::from_u32(dst_table);
642    instance.resource_transfer_own(store, src_idx, src_table, dst_table)
643}
644
645fn resource_transfer_borrow(
646    store: &mut dyn VMStore,
647    instance: Instance,
648    src_idx: u32,
649    src_table: u32,
650    dst_table: u32,
651) -> Result<u32> {
652    let src_table = TypeResourceTableIndex::from_u32(src_table);
653    let dst_table = TypeResourceTableIndex::from_u32(dst_table);
654    instance.resource_transfer_borrow(store, src_idx, src_table, dst_table)
655}
656
657fn resource_enter_call(store: &mut dyn VMStore, instance: Instance) {
658    instance.resource_enter_call(store)
659}
660
661fn resource_exit_call(store: &mut dyn VMStore, instance: Instance) -> Result<()> {
662    instance.resource_exit_call(store)
663}
664
665fn trap(_store: &mut dyn VMStore, _instance: Instance, code: u8) -> Result<()> {
666    Err(wasmtime_environ::Trap::from_u8(code).unwrap().into())
667}
668
669#[cfg(feature = "component-model-async")]
670fn backpressure_set(
671    store: &mut dyn VMStore,
672    instance: Instance,
673    caller_instance: u32,
674    enabled: u32,
675) -> Result<()> {
676    instance.concurrent_state_mut(store).backpressure_modify(
677        RuntimeComponentInstanceIndex::from_u32(caller_instance),
678        |_| Some(if enabled != 0 { 1 } else { 0 }),
679    )
680}
681
682#[cfg(feature = "component-model-async")]
683fn backpressure_modify(
684    store: &mut dyn VMStore,
685    instance: Instance,
686    caller_instance: u32,
687    increment: u8,
688) -> Result<()> {
689    instance.concurrent_state_mut(store).backpressure_modify(
690        RuntimeComponentInstanceIndex::from_u32(caller_instance),
691        |old| {
692            if increment != 0 {
693                old.checked_add(1)
694            } else {
695                old.checked_sub(1)
696            }
697        },
698    )
699}
700
701#[cfg(feature = "component-model-async")]
702unsafe fn task_return(
703    store: &mut dyn VMStore,
704    instance: Instance,
705    caller_instance: u32,
706    ty: u32,
707    options: u32,
708    storage: *mut u8,
709    storage_len: usize,
710) -> Result<()> {
711    instance.task_return(
712        store,
713        RuntimeComponentInstanceIndex::from_u32(caller_instance),
714        TypeTupleIndex::from_u32(ty),
715        OptionsIndex::from_u32(options),
716        unsafe { core::slice::from_raw_parts(storage.cast(), storage_len) },
717    )
718}
719
720#[cfg(feature = "component-model-async")]
721fn task_cancel(store: &mut dyn VMStore, instance: Instance, caller_instance: u32) -> Result<()> {
722    instance.task_cancel(
723        store,
724        RuntimeComponentInstanceIndex::from_u32(caller_instance),
725    )
726}
727
728#[cfg(feature = "component-model-async")]
729fn waitable_set_new(
730    store: &mut dyn VMStore,
731    instance: Instance,
732    caller_instance: u32,
733) -> Result<u32> {
734    instance
735        .id()
736        .get_mut(store)
737        .waitable_set_new(RuntimeComponentInstanceIndex::from_u32(caller_instance))
738}
739
740#[cfg(feature = "component-model-async")]
741fn waitable_set_wait(
742    store: &mut dyn VMStore,
743    instance: Instance,
744    caller: u32,
745    options: u32,
746    set: u32,
747    payload: u32,
748) -> Result<u32> {
749    instance.waitable_set_wait(
750        store,
751        RuntimeComponentInstanceIndex::from_u32(caller),
752        OptionsIndex::from_u32(options),
753        set,
754        payload,
755    )
756}
757
758#[cfg(feature = "component-model-async")]
759fn waitable_set_poll(
760    store: &mut dyn VMStore,
761    instance: Instance,
762    caller: u32,
763    options: u32,
764    set: u32,
765    payload: u32,
766) -> Result<u32> {
767    instance.waitable_set_poll(
768        store,
769        RuntimeComponentInstanceIndex::from_u32(caller),
770        OptionsIndex::from_u32(options),
771        set,
772        payload,
773    )
774}
775
776#[cfg(feature = "component-model-async")]
777fn waitable_set_drop(
778    store: &mut dyn VMStore,
779    instance: Instance,
780    caller_instance: u32,
781    set: u32,
782) -> Result<()> {
783    instance.id().get_mut(store).waitable_set_drop(
784        RuntimeComponentInstanceIndex::from_u32(caller_instance),
785        set,
786    )
787}
788
789#[cfg(feature = "component-model-async")]
790fn waitable_join(
791    store: &mut dyn VMStore,
792    instance: Instance,
793    caller_instance: u32,
794    waitable: u32,
795    set: u32,
796) -> Result<()> {
797    instance.id().get_mut(store).waitable_join(
798        RuntimeComponentInstanceIndex::from_u32(caller_instance),
799        waitable,
800        set,
801    )
802}
803
804#[cfg(feature = "component-model-async")]
805fn thread_yield(
806    store: &mut dyn VMStore,
807    instance: Instance,
808    caller_instance: u32,
809    cancellable: u8,
810) -> Result<bool> {
811    instance.thread_yield(
812        store,
813        RuntimeComponentInstanceIndex::from_u32(caller_instance),
814        cancellable != 0,
815    )
816}
817
818#[cfg(feature = "component-model-async")]
819fn subtask_drop(
820    store: &mut dyn VMStore,
821    instance: Instance,
822    caller_instance: u32,
823    task_id: u32,
824) -> Result<()> {
825    instance.id().get_mut(store).subtask_drop(
826        RuntimeComponentInstanceIndex::from_u32(caller_instance),
827        task_id,
828    )
829}
830
831#[cfg(feature = "component-model-async")]
832fn subtask_cancel(
833    store: &mut dyn VMStore,
834    instance: Instance,
835    caller_instance: u32,
836    async_: u8,
837    task_id: u32,
838) -> Result<u32> {
839    instance.subtask_cancel(
840        store,
841        RuntimeComponentInstanceIndex::from_u32(caller_instance),
842        async_ != 0,
843        task_id,
844    )
845}
846
847#[cfg(feature = "component-model-async")]
848unsafe fn prepare_call(
849    store: &mut dyn VMStore,
850    instance: Instance,
851    memory: *mut u8,
852    start: *mut u8,
853    return_: *mut u8,
854    caller_instance: u32,
855    callee_instance: u32,
856    task_return_type: u32,
857    string_encoding: u32,
858    result_count_or_max_if_async: u32,
859    storage: *mut u8,
860    storage_len: usize,
861) -> Result<()> {
862    unsafe {
863        store.component_async_store().prepare_call(
864            instance,
865            memory.cast::<crate::vm::VMMemoryDefinition>(),
866            start.cast::<crate::vm::VMFuncRef>(),
867            return_.cast::<crate::vm::VMFuncRef>(),
868            RuntimeComponentInstanceIndex::from_u32(caller_instance),
869            RuntimeComponentInstanceIndex::from_u32(callee_instance),
870            TypeTupleIndex::from_u32(task_return_type),
871            u8::try_from(string_encoding).unwrap(),
872            result_count_or_max_if_async,
873            storage.cast::<crate::ValRaw>(),
874            storage_len,
875        )
876    }
877}
878
879#[cfg(feature = "component-model-async")]
880unsafe fn sync_start(
881    store: &mut dyn VMStore,
882    instance: Instance,
883    callback: *mut u8,
884    storage: *mut u8,
885    storage_len: usize,
886    callee: *mut u8,
887    param_count: u32,
888) -> Result<()> {
889    unsafe {
890        store.component_async_store().sync_start(
891            instance,
892            callback.cast::<crate::vm::VMFuncRef>(),
893            callee.cast::<crate::vm::VMFuncRef>(),
894            param_count,
895            storage.cast::<std::mem::MaybeUninit<crate::ValRaw>>(),
896            storage_len,
897        )
898    }
899}
900
901#[cfg(feature = "component-model-async")]
902unsafe fn async_start(
903    store: &mut dyn VMStore,
904    instance: Instance,
905    callback: *mut u8,
906    post_return: *mut u8,
907    callee: *mut u8,
908    param_count: u32,
909    result_count: u32,
910    flags: u32,
911) -> Result<u32> {
912    unsafe {
913        store.component_async_store().async_start(
914            instance,
915            callback.cast::<crate::vm::VMFuncRef>(),
916            post_return.cast::<crate::vm::VMFuncRef>(),
917            callee.cast::<crate::vm::VMFuncRef>(),
918            param_count,
919            result_count,
920            flags,
921        )
922    }
923}
924
925#[cfg(feature = "component-model-async")]
926fn future_transfer(
927    store: &mut dyn VMStore,
928    instance: Instance,
929    src_idx: u32,
930    src_table: u32,
931    dst_table: u32,
932) -> Result<u32> {
933    instance.id().get_mut(store).future_transfer(
934        src_idx,
935        TypeFutureTableIndex::from_u32(src_table),
936        TypeFutureTableIndex::from_u32(dst_table),
937    )
938}
939
940#[cfg(feature = "component-model-async")]
941fn stream_transfer(
942    store: &mut dyn VMStore,
943    instance: Instance,
944    src_idx: u32,
945    src_table: u32,
946    dst_table: u32,
947) -> Result<u32> {
948    instance.id().get_mut(store).stream_transfer(
949        src_idx,
950        TypeStreamTableIndex::from_u32(src_table),
951        TypeStreamTableIndex::from_u32(dst_table),
952    )
953}
954
955#[cfg(feature = "component-model-async")]
956fn error_context_transfer(
957    store: &mut dyn VMStore,
958    instance: Instance,
959    src_idx: u32,
960    src_table: u32,
961    dst_table: u32,
962) -> Result<u32> {
963    let src_table = TypeComponentLocalErrorContextTableIndex::from_u32(src_table);
964    let dst_table = TypeComponentLocalErrorContextTableIndex::from_u32(dst_table);
965    instance
966        .id()
967        .get_mut(store)
968        .error_context_transfer(src_idx, src_table, dst_table)
969}
970
971#[cfg(feature = "component-model-async")]
972unsafe impl HostResultHasUnwindSentinel for ResourcePair {
973    type Abi = u64;
974    const SENTINEL: u64 = u64::MAX;
975
976    fn into_abi(self) -> Self::Abi {
977        assert!(self.write & (1 << 31) == 0);
978        (u64::from(self.write) << 32) | u64::from(self.read)
979    }
980}
981
982#[cfg(feature = "component-model-async")]
983fn future_new(
984    store: &mut dyn VMStore,
985    instance: Instance,
986    caller_instance: u32,
987    ty: u32,
988) -> Result<ResourcePair> {
989    instance.id().get_mut(store).future_new(
990        RuntimeComponentInstanceIndex::from_u32(caller_instance),
991        TypeFutureTableIndex::from_u32(ty),
992    )
993}
994
995#[cfg(feature = "component-model-async")]
996fn future_write(
997    store: &mut dyn VMStore,
998    instance: Instance,
999    caller_instance: u32,
1000    ty: u32,
1001    options: u32,
1002    future: u32,
1003    address: u32,
1004) -> Result<u32> {
1005    store.component_async_store().future_write(
1006        instance,
1007        RuntimeComponentInstanceIndex::from_u32(caller_instance),
1008        TypeFutureTableIndex::from_u32(ty),
1009        OptionsIndex::from_u32(options),
1010        future,
1011        address,
1012    )
1013}
1014
1015#[cfg(feature = "component-model-async")]
1016fn future_read(
1017    store: &mut dyn VMStore,
1018    instance: Instance,
1019    caller_instance: u32,
1020    ty: u32,
1021    options: u32,
1022    future: u32,
1023    address: u32,
1024) -> Result<u32> {
1025    store.component_async_store().future_read(
1026        instance,
1027        RuntimeComponentInstanceIndex::from_u32(caller_instance),
1028        TypeFutureTableIndex::from_u32(ty),
1029        OptionsIndex::from_u32(options),
1030        future,
1031        address,
1032    )
1033}
1034
1035#[cfg(feature = "component-model-async")]
1036fn future_cancel_write(
1037    store: &mut dyn VMStore,
1038    instance: Instance,
1039    caller_instance: u32,
1040    ty: u32,
1041    async_: u8,
1042    writer: u32,
1043) -> Result<u32> {
1044    instance.future_cancel_write(
1045        store,
1046        RuntimeComponentInstanceIndex::from_u32(caller_instance),
1047        TypeFutureTableIndex::from_u32(ty),
1048        async_ != 0,
1049        writer,
1050    )
1051}
1052
1053#[cfg(feature = "component-model-async")]
1054fn future_cancel_read(
1055    store: &mut dyn VMStore,
1056    instance: Instance,
1057    caller_instance: u32,
1058    ty: u32,
1059    async_: u8,
1060    reader: u32,
1061) -> Result<u32> {
1062    instance.future_cancel_read(
1063        store,
1064        RuntimeComponentInstanceIndex::from_u32(caller_instance),
1065        TypeFutureTableIndex::from_u32(ty),
1066        async_ != 0,
1067        reader,
1068    )
1069}
1070
1071#[cfg(feature = "component-model-async")]
1072fn future_drop_writable(
1073    store: &mut dyn VMStore,
1074    instance: Instance,
1075    caller_instance: u32,
1076    ty: u32,
1077    writer: u32,
1078) -> Result<()> {
1079    store.component_async_store().future_drop_writable(
1080        instance,
1081        RuntimeComponentInstanceIndex::from_u32(caller_instance),
1082        TypeFutureTableIndex::from_u32(ty),
1083        writer,
1084    )
1085}
1086
1087#[cfg(feature = "component-model-async")]
1088fn future_drop_readable(
1089    store: &mut dyn VMStore,
1090    instance: Instance,
1091    caller_instance: u32,
1092    ty: u32,
1093    reader: u32,
1094) -> Result<()> {
1095    instance.future_drop_readable(
1096        store,
1097        RuntimeComponentInstanceIndex::from_u32(caller_instance),
1098        TypeFutureTableIndex::from_u32(ty),
1099        reader,
1100    )
1101}
1102
1103#[cfg(feature = "component-model-async")]
1104fn stream_new(
1105    store: &mut dyn VMStore,
1106    instance: Instance,
1107    caller_instance: u32,
1108    ty: u32,
1109) -> Result<ResourcePair> {
1110    instance.id().get_mut(store).stream_new(
1111        RuntimeComponentInstanceIndex::from_u32(caller_instance),
1112        TypeStreamTableIndex::from_u32(ty),
1113    )
1114}
1115
1116#[cfg(feature = "component-model-async")]
1117fn stream_write(
1118    store: &mut dyn VMStore,
1119    instance: Instance,
1120    caller_instance: u32,
1121    ty: u32,
1122    options: u32,
1123    stream: u32,
1124    address: u32,
1125    count: u32,
1126) -> Result<u32> {
1127    store.component_async_store().stream_write(
1128        instance,
1129        RuntimeComponentInstanceIndex::from_u32(caller_instance),
1130        TypeStreamTableIndex::from_u32(ty),
1131        OptionsIndex::from_u32(options),
1132        stream,
1133        address,
1134        count,
1135    )
1136}
1137
1138#[cfg(feature = "component-model-async")]
1139fn stream_read(
1140    store: &mut dyn VMStore,
1141    instance: Instance,
1142    caller_instance: u32,
1143    ty: u32,
1144    options: u32,
1145    stream: u32,
1146    address: u32,
1147    count: u32,
1148) -> Result<u32> {
1149    store.component_async_store().stream_read(
1150        instance,
1151        RuntimeComponentInstanceIndex::from_u32(caller_instance),
1152        TypeStreamTableIndex::from_u32(ty),
1153        OptionsIndex::from_u32(options),
1154        stream,
1155        address,
1156        count,
1157    )
1158}
1159
1160#[cfg(feature = "component-model-async")]
1161fn stream_cancel_write(
1162    store: &mut dyn VMStore,
1163    instance: Instance,
1164    caller_instance: u32,
1165    ty: u32,
1166    async_: u8,
1167    writer: u32,
1168) -> Result<u32> {
1169    instance.stream_cancel_write(
1170        store,
1171        RuntimeComponentInstanceIndex::from_u32(caller_instance),
1172        TypeStreamTableIndex::from_u32(ty),
1173        async_ != 0,
1174        writer,
1175    )
1176}
1177
1178#[cfg(feature = "component-model-async")]
1179fn stream_cancel_read(
1180    store: &mut dyn VMStore,
1181    instance: Instance,
1182    caller_instance: u32,
1183    ty: u32,
1184    async_: u8,
1185    reader: u32,
1186) -> Result<u32> {
1187    instance.stream_cancel_read(
1188        store,
1189        RuntimeComponentInstanceIndex::from_u32(caller_instance),
1190        TypeStreamTableIndex::from_u32(ty),
1191        async_ != 0,
1192        reader,
1193    )
1194}
1195
1196#[cfg(feature = "component-model-async")]
1197fn stream_drop_writable(
1198    store: &mut dyn VMStore,
1199    instance: Instance,
1200    caller_instance: u32,
1201    ty: u32,
1202    writer: u32,
1203) -> Result<()> {
1204    store.component_async_store().stream_drop_writable(
1205        instance,
1206        RuntimeComponentInstanceIndex::from_u32(caller_instance),
1207        TypeStreamTableIndex::from_u32(ty),
1208        writer,
1209    )
1210}
1211
1212#[cfg(feature = "component-model-async")]
1213fn stream_drop_readable(
1214    store: &mut dyn VMStore,
1215    instance: Instance,
1216    caller_instance: u32,
1217    ty: u32,
1218    reader: u32,
1219) -> Result<()> {
1220    instance.stream_drop_readable(
1221        store,
1222        RuntimeComponentInstanceIndex::from_u32(caller_instance),
1223        TypeStreamTableIndex::from_u32(ty),
1224        reader,
1225    )
1226}
1227
1228#[cfg(feature = "component-model-async")]
1229fn flat_stream_write(
1230    store: &mut dyn VMStore,
1231    instance: Instance,
1232    caller_instance: u32,
1233    ty: u32,
1234    options: u32,
1235    payload_size: u32,
1236    payload_align: u32,
1237    stream: u32,
1238    address: u32,
1239    count: u32,
1240) -> Result<u32> {
1241    store.component_async_store().flat_stream_write(
1242        instance,
1243        RuntimeComponentInstanceIndex::from_u32(caller_instance),
1244        TypeStreamTableIndex::from_u32(ty),
1245        OptionsIndex::from_u32(options),
1246        payload_size,
1247        payload_align,
1248        stream,
1249        address,
1250        count,
1251    )
1252}
1253
1254#[cfg(feature = "component-model-async")]
1255fn flat_stream_read(
1256    store: &mut dyn VMStore,
1257    instance: Instance,
1258    caller_instance: u32,
1259    ty: u32,
1260    options: u32,
1261    payload_size: u32,
1262    payload_align: u32,
1263    stream: u32,
1264    address: u32,
1265    count: u32,
1266) -> Result<u32> {
1267    store.component_async_store().flat_stream_read(
1268        instance,
1269        RuntimeComponentInstanceIndex::from_u32(caller_instance),
1270        TypeStreamTableIndex::from_u32(ty),
1271        OptionsIndex::from_u32(options),
1272        payload_size,
1273        payload_align,
1274        stream,
1275        address,
1276        count,
1277    )
1278}
1279
1280#[cfg(feature = "component-model-async")]
1281fn error_context_new(
1282    store: &mut dyn VMStore,
1283    instance: Instance,
1284    caller_instance: u32,
1285    ty: u32,
1286    options: u32,
1287    debug_msg_address: u32,
1288    debug_msg_len: u32,
1289) -> Result<u32> {
1290    instance.error_context_new(
1291        store.store_opaque_mut(),
1292        RuntimeComponentInstanceIndex::from_u32(caller_instance),
1293        TypeComponentLocalErrorContextTableIndex::from_u32(ty),
1294        OptionsIndex::from_u32(options),
1295        debug_msg_address,
1296        debug_msg_len,
1297    )
1298}
1299
1300#[cfg(feature = "component-model-async")]
1301fn error_context_debug_message(
1302    store: &mut dyn VMStore,
1303    instance: Instance,
1304    caller_instance: u32,
1305    ty: u32,
1306    options: u32,
1307    err_ctx_handle: u32,
1308    debug_msg_address: u32,
1309) -> Result<()> {
1310    store.component_async_store().error_context_debug_message(
1311        instance,
1312        RuntimeComponentInstanceIndex::from_u32(caller_instance),
1313        TypeComponentLocalErrorContextTableIndex::from_u32(ty),
1314        OptionsIndex::from_u32(options),
1315        err_ctx_handle,
1316        debug_msg_address,
1317    )
1318}
1319
1320#[cfg(feature = "component-model-async")]
1321fn error_context_drop(
1322    store: &mut dyn VMStore,
1323    instance: Instance,
1324    caller_instance: u32,
1325    ty: u32,
1326    err_ctx_handle: u32,
1327) -> Result<()> {
1328    instance.id().get_mut(store).error_context_drop(
1329        RuntimeComponentInstanceIndex::from_u32(caller_instance),
1330        TypeComponentLocalErrorContextTableIndex::from_u32(ty),
1331        err_ctx_handle,
1332    )
1333}
1334
1335#[cfg(feature = "component-model-async")]
1336fn context_get(
1337    store: &mut dyn VMStore,
1338    instance: Instance,
1339    caller_instance: u32,
1340    slot: u32,
1341) -> Result<u32> {
1342    instance.context_get(
1343        store,
1344        RuntimeComponentInstanceIndex::from_u32(caller_instance),
1345        slot,
1346    )
1347}
1348
1349#[cfg(feature = "component-model-async")]
1350fn context_set(
1351    store: &mut dyn VMStore,
1352    instance: Instance,
1353    caller_instance: u32,
1354    slot: u32,
1355    val: u32,
1356) -> Result<()> {
1357    instance.context_set(
1358        store,
1359        RuntimeComponentInstanceIndex::from_u32(caller_instance),
1360        slot,
1361        val,
1362    )
1363}
wasmtime/runtime/vm/component/libcalls.rs

wasmtime/runtime/vm/component/
libcalls.rs
