wasmtime_internal_cranelift/compiler/
component.rs

1//! Compilation support for the component model.
2
3use crate::{TRAP_ALWAYS, TRAP_CANNOT_ENTER, TRAP_INTERNAL_ASSERT, compiler::Compiler};
4use anyhow::Result;
5use cranelift_codegen::ir::condcodes::IntCC;
6use cranelift_codegen::ir::{self, InstBuilder, MemFlags, Value};
7use cranelift_codegen::isa::{CallConv, TargetIsa};
8use cranelift_frontend::FunctionBuilder;
9use wasmtime_environ::{
10    Abi, CompiledFunctionBody, EntityRef, FuncKey, HostCall, ModuleInternedTypeIndex, PtrSize,
11    TrapSentinel, Tunables, WasmFuncType, WasmValType, component::*,
12    fact::PREPARE_CALL_FIXED_PARAMS,
13};
14
15struct TrampolineCompiler<'a> {
16    compiler: &'a Compiler,
17    isa: &'a (dyn TargetIsa + 'static),
18    builder: FunctionBuilder<'a>,
19    component: &'a Component,
20    types: &'a ComponentTypesBuilder,
21    offsets: VMComponentOffsets<u8>,
22    abi: Abi,
23    block0: ir::Block,
24    signature: ModuleInternedTypeIndex,
25    tunables: &'a Tunables,
26}
27
28/// What host functions can be called, used in `translate_hostcall` below.
29enum HostCallee {
30    /// Call a host-lowered function specified by this index.
31    Lowering(LoweredIndex),
32    /// Call a host libcall, specified by this accessor.
33    Libcall(GetLibcallFn),
34}
35
36type GetLibcallFn =
37    fn(&dyn TargetIsa, &mut ir::Function) -> (ir::SigRef, ComponentBuiltinFunctionIndex);
38
39impl From<LoweredIndex> for HostCallee {
40    fn from(index: LoweredIndex) -> HostCallee {
41        HostCallee::Lowering(index)
42    }
43}
44
45impl From<GetLibcallFn> for HostCallee {
46    fn from(f: GetLibcallFn) -> HostCallee {
47        HostCallee::Libcall(f)
48    }
49}
50
51/// How to interpret the results of a host function.
52enum HostResult {
53    /// The host function has no results.
54    None,
55
56    /// The host function returns the sentinel specified which is interpreted
57    /// and translated to the real return value.
58    Sentinel(TrapSentinel),
59
60    /// The host function returns a `bool` indicating whether it succeeded or
61    /// not.
62    ///
63    /// After the return value is interpreted the host function also filled in
64    /// `ptr` and `len` with wasm return values which need to be returned.
65    ///
66    /// If `ptr` and `len` are not specified then this must be used with
67    /// `WasmArgs::ValRawList` and that ptr/len is used.
68    MultiValue {
69        /// The base pointer of the `ValRaw` list on the stack.
70        ptr: Option<ir::Value>,
71        /// The length of the `ValRaw` list on the stack.
72        len: Option<ir::Value>,
73    },
74}
75
76impl From<TrapSentinel> for HostResult {
77    fn from(sentinel: TrapSentinel) -> HostResult {
78        HostResult::Sentinel(sentinel)
79    }
80}
81
82/// Different means of passing WebAssembly arguments to host calls.
83#[derive(Debug, Copy, Clone)]
84enum WasmArgs {
85    /// All wasm arguments to the host are passed directly as values, typically
86    /// through registers.
87    InRegisters,
88
89    /// All wasm arguments to the host are passed indirectly by spilling them
90    /// to the stack as a sequence of contiguous `ValRaw`s.
91    ValRawList,
92
93    /// The first `n` arguments are passed in registers, but everything after
94    /// that is spilled to the stack.
95    InRegistersUpTo(usize),
96}
97
98impl<'a> TrampolineCompiler<'a> {
99    fn new(
100        compiler: &'a Compiler,
101        func_compiler: &'a mut super::FunctionCompiler<'_>,
102        component: &'a Component,
103        types: &'a ComponentTypesBuilder,
104        index: TrampolineIndex,
105        abi: Abi,
106        tunables: &'a Tunables,
107    ) -> TrampolineCompiler<'a> {
108        let isa = &*compiler.isa;
109        let signature = component.trampolines[index];
110        let ty = types[signature].unwrap_func();
111        let func = ir::Function::with_name_signature(
112            ir::UserFuncName::user(0, 0),
113            match abi {
114                Abi::Wasm => crate::wasm_call_signature(isa, ty, &compiler.tunables),
115                Abi::Array => crate::array_call_signature(isa),
116            },
117        );
118        let (builder, block0) = func_compiler.builder(func);
119        TrampolineCompiler {
120            compiler,
121            isa,
122            builder,
123            component,
124            types,
125            offsets: VMComponentOffsets::new(isa.pointer_bytes(), component),
126            abi,
127            block0,
128            signature,
129            tunables,
130        }
131    }
132
133    fn translate(&mut self, trampoline: &Trampoline) {
134        match trampoline {
135            Trampoline::Transcoder {
136                op,
137                from,
138                from64,
139                to,
140                to64,
141            } => {
142                match self.abi {
143                    Abi::Wasm => {
144                        self.translate_transcode(*op, *from, *from64, *to, *to64);
145                    }
146                    // Transcoders can only actually be called by Wasm, so let's assert
147                    // that here.
148                    Abi::Array => {
149                        self.builder.ins().trap(TRAP_INTERNAL_ASSERT);
150                    }
151                }
152            }
153            Trampoline::LowerImport {
154                index,
155                options,
156                lower_ty,
157            } => {
158                let pointer_type = self.isa.pointer_type();
159                self.translate_hostcall(
160                    HostCallee::Lowering(*index),
161                    HostResult::MultiValue {
162                        ptr: None,
163                        len: None,
164                    },
165                    WasmArgs::ValRawList,
166                    |me, params| {
167                        let vmctx = params[0];
168                        params.extend([
169                            me.builder.ins().load(
170                                pointer_type,
171                                MemFlags::trusted(),
172                                vmctx,
173                                i32::try_from(me.offsets.lowering_data(*index)).unwrap(),
174                            ),
175                            me.index_value(*lower_ty),
176                            me.index_value(*options),
177                        ]);
178                    },
179                );
180            }
181            Trampoline::AlwaysTrap => {
182                if self.tunables.signals_based_traps {
183                    self.builder.ins().trap(TRAP_ALWAYS);
184                    return;
185                }
186                self.translate_libcall(
187                    host::trap,
188                    TrapSentinel::Falsy,
189                    WasmArgs::InRegisters,
190                    |me, params| {
191                        let code = wasmtime_environ::Trap::AlwaysTrapAdapter as u8;
192                        params.push(me.builder.ins().iconst(ir::types::I8, i64::from(code)));
193                    },
194                );
195            }
196            Trampoline::ResourceNew { instance, ty } => {
197                // Currently this only supports resources represented by `i32`
198                assert_eq!(
199                    self.types[self.signature].unwrap_func().params()[0],
200                    WasmValType::I32
201                );
202                self.translate_libcall(
203                    host::resource_new32,
204                    TrapSentinel::NegativeOne,
205                    WasmArgs::InRegisters,
206                    |me, params| {
207                        params.push(me.index_value(*instance));
208                        params.push(me.index_value(*ty));
209                    },
210                );
211            }
212            Trampoline::ResourceRep { instance, ty } => {
213                // Currently this only supports resources represented by `i32`
214                assert_eq!(
215                    self.types[self.signature].unwrap_func().returns()[0],
216                    WasmValType::I32
217                );
218                self.translate_libcall(
219                    host::resource_rep32,
220                    TrapSentinel::NegativeOne,
221                    WasmArgs::InRegisters,
222                    |me, params| {
223                        params.push(me.index_value(*instance));
224                        params.push(me.index_value(*ty));
225                    },
226                );
227            }
228            Trampoline::ResourceDrop { instance, ty } => {
229                self.translate_resource_drop(*instance, *ty);
230            }
231            Trampoline::BackpressureSet { instance } => {
232                self.translate_libcall(
233                    host::backpressure_set,
234                    TrapSentinel::Falsy,
235                    WasmArgs::InRegisters,
236                    |me, params| {
237                        params.push(me.index_value(*instance));
238                    },
239                );
240            }
241            Trampoline::BackpressureInc { instance } => {
242                self.translate_libcall(
243                    host::backpressure_modify,
244                    TrapSentinel::Falsy,
245                    WasmArgs::InRegisters,
246                    |me, params| {
247                        params.push(me.index_value(*instance));
248                        params.push(me.builder.ins().iconst(ir::types::I8, 1));
249                    },
250                );
251            }
252            Trampoline::BackpressureDec { instance } => {
253                self.translate_libcall(
254                    host::backpressure_modify,
255                    TrapSentinel::Falsy,
256                    WasmArgs::InRegisters,
257                    |me, params| {
258                        params.push(me.index_value(*instance));
259                        params.push(me.builder.ins().iconst(ir::types::I8, 0));
260                    },
261                );
262            }
263            Trampoline::TaskReturn {
264                instance,
265                results,
266                options,
267            } => {
268                self.translate_libcall(
269                    host::task_return,
270                    TrapSentinel::Falsy,
271                    WasmArgs::ValRawList,
272                    |me, params| {
273                        params.push(me.index_value(*instance));
274                        params.push(me.index_value(*results));
275                        params.push(me.index_value(*options));
276                    },
277                );
278            }
279            Trampoline::TaskCancel { instance } => {
280                self.translate_libcall(
281                    host::task_cancel,
282                    TrapSentinel::Falsy,
283                    WasmArgs::InRegisters,
284                    |me, params| {
285                        params.push(me.index_value(*instance));
286                    },
287                );
288            }
289            Trampoline::WaitableSetNew { instance } => {
290                self.translate_libcall(
291                    host::waitable_set_new,
292                    TrapSentinel::NegativeOne,
293                    WasmArgs::InRegisters,
294                    |me, params| {
295                        params.push(me.index_value(*instance));
296                    },
297                );
298            }
299            Trampoline::WaitableSetWait { instance, options } => {
300                self.translate_libcall(
301                    host::waitable_set_wait,
302                    TrapSentinel::NegativeOne,
303                    WasmArgs::InRegisters,
304                    |me, params| {
305                        params.push(me.index_value(*instance));
306                        params.push(me.index_value(*options));
307                    },
308                );
309            }
310            Trampoline::WaitableSetPoll { instance, options } => {
311                self.translate_libcall(
312                    host::waitable_set_poll,
313                    TrapSentinel::NegativeOne,
314                    WasmArgs::InRegisters,
315                    |me, params| {
316                        params.push(me.index_value(*instance));
317                        params.push(me.index_value(*options));
318                    },
319                );
320            }
321            Trampoline::WaitableSetDrop { instance } => {
322                self.translate_libcall(
323                    host::waitable_set_drop,
324                    TrapSentinel::Falsy,
325                    WasmArgs::InRegisters,
326                    |me, params| {
327                        params.push(me.index_value(*instance));
328                    },
329                );
330            }
331            Trampoline::WaitableJoin { instance } => {
332                self.translate_libcall(
333                    host::waitable_join,
334                    TrapSentinel::Falsy,
335                    WasmArgs::InRegisters,
336                    |me, params| {
337                        params.push(me.index_value(*instance));
338                    },
339                );
340            }
341            Trampoline::ThreadYield {
342                instance,
343                cancellable,
344            } => {
345                self.translate_libcall(
346                    host::thread_yield,
347                    TrapSentinel::NegativeOne,
348                    WasmArgs::InRegisters,
349                    |me, params| {
350                        params.push(me.index_value(*instance));
351                        params.push(
352                            me.builder
353                                .ins()
354                                .iconst(ir::types::I8, i64::from(*cancellable)),
355                        );
356                    },
357                );
358            }
359            Trampoline::SubtaskDrop { instance } => {
360                self.translate_libcall(
361                    host::subtask_drop,
362                    TrapSentinel::Falsy,
363                    WasmArgs::InRegisters,
364                    |me, params| {
365                        params.push(me.index_value(*instance));
366                    },
367                );
368            }
369            Trampoline::SubtaskCancel { instance, async_ } => {
370                self.translate_libcall(
371                    host::subtask_cancel,
372                    TrapSentinel::NegativeOne,
373                    WasmArgs::InRegisters,
374                    |me, params| {
375                        params.push(me.index_value(*instance));
376                        params.push(me.builder.ins().iconst(ir::types::I8, i64::from(*async_)));
377                    },
378                );
379            }
380            Trampoline::StreamNew { instance, ty } => {
381                self.translate_libcall(
382                    host::stream_new,
383                    TrapSentinel::NegativeOne,
384                    WasmArgs::InRegisters,
385                    |me, params| {
386                        params.push(me.index_value(*instance));
387                        params.push(me.index_value(*ty));
388                    },
389                );
390            }
391            Trampoline::StreamRead {
392                instance,
393                ty,
394                options,
395            } => {
396                if let Some(info) = self.flat_stream_element_info(*ty).cloned() {
397                    self.translate_libcall(
398                        host::flat_stream_read,
399                        TrapSentinel::NegativeOne,
400                        WasmArgs::InRegisters,
401                        |me, params| {
402                            params.extend([
403                                me.index_value(*instance),
404                                me.index_value(*ty),
405                                me.index_value(*options),
406                                me.builder
407                                    .ins()
408                                    .iconst(ir::types::I32, i64::from(info.size32)),
409                                me.builder
410                                    .ins()
411                                    .iconst(ir::types::I32, i64::from(info.align32)),
412                            ]);
413                        },
414                    );
415                } else {
416                    self.translate_libcall(
417                        host::stream_read,
418                        TrapSentinel::NegativeOne,
419                        WasmArgs::InRegisters,
420                        |me, params| {
421                            params.push(me.index_value(*instance));
422                            params.push(me.index_value(*ty));
423                            params.push(me.index_value(*options));
424                        },
425                    );
426                }
427            }
428            Trampoline::StreamWrite {
429                instance,
430                ty,
431                options,
432            } => {
433                if let Some(info) = self.flat_stream_element_info(*ty).cloned() {
434                    self.translate_libcall(
435                        host::flat_stream_write,
436                        TrapSentinel::NegativeOne,
437                        WasmArgs::InRegisters,
438                        |me, params| {
439                            params.extend([
440                                me.index_value(*instance),
441                                me.index_value(*ty),
442                                me.index_value(*options),
443                                me.builder
444                                    .ins()
445                                    .iconst(ir::types::I32, i64::from(info.size32)),
446                                me.builder
447                                    .ins()
448                                    .iconst(ir::types::I32, i64::from(info.align32)),
449                            ]);
450                        },
451                    );
452                } else {
453                    self.translate_libcall(
454                        host::stream_write,
455                        TrapSentinel::NegativeOne,
456                        WasmArgs::InRegisters,
457                        |me, params| {
458                            params.push(me.index_value(*instance));
459                            params.push(me.index_value(*ty));
460                            params.push(me.index_value(*options));
461                        },
462                    );
463                }
464            }
465            Trampoline::StreamCancelRead {
466                instance,
467                ty,
468                async_,
469            } => {
470                self.translate_libcall(
471                    host::stream_cancel_read,
472                    TrapSentinel::NegativeOne,
473                    WasmArgs::InRegisters,
474                    |me, params| {
475                        params.push(me.index_value(*instance));
476                        params.push(me.index_value(*ty));
477                        params.push(me.builder.ins().iconst(ir::types::I8, i64::from(*async_)));
478                    },
479                );
480            }
481            Trampoline::StreamCancelWrite {
482                instance,
483                ty,
484                async_,
485            } => {
486                self.translate_libcall(
487                    host::stream_cancel_write,
488                    TrapSentinel::NegativeOne,
489                    WasmArgs::InRegisters,
490                    |me, params| {
491                        params.push(me.index_value(*instance));
492                        params.push(me.index_value(*ty));
493                        params.push(me.builder.ins().iconst(ir::types::I8, i64::from(*async_)));
494                    },
495                );
496            }
497            Trampoline::StreamDropReadable { instance, ty } => {
498                self.translate_libcall(
499                    host::stream_drop_readable,
500                    TrapSentinel::Falsy,
501                    WasmArgs::InRegisters,
502                    |me, params| {
503                        params.push(me.index_value(*instance));
504                        params.push(me.index_value(*ty));
505                    },
506                );
507            }
508            Trampoline::StreamDropWritable { instance, ty } => {
509                self.translate_libcall(
510                    host::stream_drop_writable,
511                    TrapSentinel::Falsy,
512                    WasmArgs::InRegisters,
513                    |me, params| {
514                        params.push(me.index_value(*instance));
515                        params.push(me.index_value(*ty));
516                    },
517                );
518            }
519            Trampoline::FutureNew { instance, ty } => {
520                self.translate_libcall(
521                    host::future_new,
522                    TrapSentinel::NegativeOne,
523                    WasmArgs::InRegisters,
524                    |me, params| {
525                        params.push(me.index_value(*instance));
526                        params.push(me.index_value(*ty));
527                    },
528                );
529            }
530            Trampoline::FutureRead {
531                instance,
532                ty,
533                options,
534            } => {
535                self.translate_libcall(
536                    host::future_read,
537                    TrapSentinel::NegativeOne,
538                    WasmArgs::InRegisters,
539                    |me, params| {
540                        params.push(me.index_value(*instance));
541                        params.push(me.index_value(*ty));
542                        params.push(me.index_value(*options));
543                    },
544                );
545            }
546            Trampoline::FutureWrite {
547                instance,
548                ty,
549                options,
550            } => {
551                self.translate_libcall(
552                    host::future_write,
553                    TrapSentinel::NegativeOne,
554                    WasmArgs::InRegisters,
555                    |me, params| {
556                        params.push(me.index_value(*instance));
557                        params.push(me.index_value(*ty));
558                        params.push(me.index_value(*options));
559                    },
560                );
561            }
562            Trampoline::FutureCancelRead {
563                instance,
564                ty,
565                async_,
566            } => {
567                self.translate_libcall(
568                    host::future_cancel_read,
569                    TrapSentinel::NegativeOne,
570                    WasmArgs::InRegisters,
571                    |me, params| {
572                        params.push(me.index_value(*instance));
573                        params.push(me.index_value(*ty));
574                        params.push(me.builder.ins().iconst(ir::types::I8, i64::from(*async_)));
575                    },
576                );
577            }
578            Trampoline::FutureCancelWrite {
579                instance,
580                ty,
581                async_,
582            } => {
583                self.translate_libcall(
584                    host::future_cancel_write,
585                    TrapSentinel::NegativeOne,
586                    WasmArgs::InRegisters,
587                    |me, params| {
588                        params.push(me.index_value(*instance));
589                        params.push(me.index_value(*ty));
590                        params.push(me.builder.ins().iconst(ir::types::I8, i64::from(*async_)));
591                    },
592                );
593            }
594            Trampoline::FutureDropReadable { instance, ty } => {
595                self.translate_libcall(
596                    host::future_drop_readable,
597                    TrapSentinel::Falsy,
598                    WasmArgs::InRegisters,
599                    |me, params| {
600                        params.push(me.index_value(*instance));
601                        params.push(me.index_value(*ty));
602                    },
603                );
604            }
605            Trampoline::FutureDropWritable { instance, ty } => {
606                self.translate_libcall(
607                    host::future_drop_writable,
608                    TrapSentinel::Falsy,
609                    WasmArgs::InRegisters,
610                    |me, params| {
611                        params.push(me.index_value(*instance));
612                        params.push(me.index_value(*ty));
613                    },
614                );
615            }
616            Trampoline::ErrorContextNew {
617                instance,
618                ty,
619                options,
620            } => {
621                self.translate_libcall(
622                    host::error_context_new,
623                    TrapSentinel::NegativeOne,
624                    WasmArgs::InRegisters,
625                    |me, params| {
626                        params.push(me.index_value(*instance));
627                        params.push(me.index_value(*ty));
628                        params.push(me.index_value(*options));
629                    },
630                );
631            }
632            Trampoline::ErrorContextDebugMessage {
633                instance,
634                ty,
635                options,
636            } => {
637                self.translate_libcall(
638                    host::error_context_debug_message,
639                    TrapSentinel::Falsy,
640                    WasmArgs::InRegisters,
641                    |me, params| {
642                        params.push(me.index_value(*instance));
643                        params.push(me.index_value(*ty));
644                        params.push(me.index_value(*options));
645                    },
646                );
647            }
648            Trampoline::ErrorContextDrop { instance, ty } => {
649                self.translate_libcall(
650                    host::error_context_drop,
651                    TrapSentinel::Falsy,
652                    WasmArgs::InRegisters,
653                    |me, params| {
654                        params.push(me.index_value(*instance));
655                        params.push(me.index_value(*ty));
656                    },
657                );
658            }
659            Trampoline::ResourceTransferOwn => {
660                self.translate_libcall(
661                    host::resource_transfer_own,
662                    TrapSentinel::NegativeOne,
663                    WasmArgs::InRegisters,
664                    |_, _| {},
665                );
666            }
667            Trampoline::ResourceTransferBorrow => {
668                self.translate_libcall(
669                    host::resource_transfer_borrow,
670                    TrapSentinel::NegativeOne,
671                    WasmArgs::InRegisters,
672                    |_, _| {},
673                );
674            }
675            Trampoline::ResourceEnterCall => {
676                self.translate_libcall(
677                    host::resource_enter_call,
678                    HostResult::None,
679                    WasmArgs::InRegisters,
680                    |_, _| {},
681                );
682            }
683            Trampoline::ResourceExitCall => {
684                self.translate_libcall(
685                    host::resource_exit_call,
686                    TrapSentinel::Falsy,
687                    WasmArgs::InRegisters,
688                    |_, _| {},
689                );
690            }
691            Trampoline::PrepareCall { memory } => {
692                self.translate_libcall(
693                    host::prepare_call,
694                    TrapSentinel::Falsy,
695                    WasmArgs::InRegistersUpTo(PREPARE_CALL_FIXED_PARAMS.len()),
696                    |me, params| {
697                        let vmctx = params[0];
698                        params.push(me.load_optional_memory(vmctx, *memory));
699                    },
700                );
701            }
702            Trampoline::SyncStartCall { callback } => {
703                let pointer_type = self.isa.pointer_type();
704                let wasm_func_ty = &self.types[self.signature].unwrap_func();
705                let (values_vec_ptr, len) = self.compiler.allocate_stack_array_and_spill_args(
706                    &WasmFuncType::new(
707                        Box::new([]),
708                        wasm_func_ty.returns().iter().copied().collect(),
709                    ),
710                    &mut self.builder,
711                    &[],
712                );
713                let values_vec_len = self.builder.ins().iconst(pointer_type, i64::from(len));
714                self.translate_libcall(
715                    host::sync_start,
716                    HostResult::MultiValue {
717                        ptr: Some(values_vec_ptr),
718                        len: Some(values_vec_len),
719                    },
720                    WasmArgs::InRegisters,
721                    |me, params| {
722                        let vmctx = params[0];
723                        params.push(me.load_callback(vmctx, *callback));
724                        params.push(values_vec_ptr);
725                        params.push(values_vec_len);
726                    },
727                );
728            }
729            Trampoline::AsyncStartCall {
730                callback,
731                post_return,
732            } => {
733                self.translate_libcall(
734                    host::async_start,
735                    TrapSentinel::NegativeOne,
736                    WasmArgs::InRegisters,
737                    |me, params| {
738                        let vmctx = params[0];
739                        params.extend([
740                            me.load_callback(vmctx, *callback),
741                            me.load_post_return(vmctx, *post_return),
742                        ]);
743                    },
744                );
745            }
746            Trampoline::FutureTransfer => {
747                self.translate_libcall(
748                    host::future_transfer,
749                    TrapSentinel::NegativeOne,
750                    WasmArgs::InRegisters,
751                    |_, _| {},
752                );
753            }
754            Trampoline::StreamTransfer => {
755                self.translate_libcall(
756                    host::stream_transfer,
757                    TrapSentinel::NegativeOne,
758                    WasmArgs::InRegisters,
759                    |_, _| {},
760                );
761            }
762            Trampoline::ErrorContextTransfer => {
763                self.translate_libcall(
764                    host::error_context_transfer,
765                    TrapSentinel::NegativeOne,
766                    WasmArgs::InRegisters,
767                    |_, _| {},
768                );
769            }
770            Trampoline::ContextGet { instance, slot } => {
771                self.translate_libcall(
772                    host::context_get,
773                    TrapSentinel::NegativeOne,
774                    WasmArgs::InRegisters,
775                    |me, params| {
776                        params.push(me.index_value(*instance));
777                        params.push(me.builder.ins().iconst(ir::types::I32, i64::from(*slot)));
778                    },
779                );
780            }
781            Trampoline::ContextSet { instance, slot } => {
782                self.translate_libcall(
783                    host::context_set,
784                    TrapSentinel::Falsy,
785                    WasmArgs::InRegisters,
786                    |me, params| {
787                        params.push(me.index_value(*instance));
788                        params.push(me.builder.ins().iconst(ir::types::I32, i64::from(*slot)));
789                    },
790                );
791            }
792        }
793    }
794
795    /// Determine whether the specified type can be optimized as a stream
796    /// payload by lifting and lowering with a simple `memcpy`.
797    ///
798    /// Any type containing only "flat", primitive data (i.e. no pointers or
799    /// handles) should qualify for this optimization, but it's also okay to
800    /// conservatively return `None` here; the fallback slow path will always
801    /// work -- it just won't be as efficient.
802    fn flat_stream_element_info(&self, ty: TypeStreamTableIndex) -> Option<&CanonicalAbiInfo> {
803        let payload = self.types[self.types[ty].ty].payload;
804        match payload {
805            None => Some(&CanonicalAbiInfo::ZERO),
806            Some(
807                payload @ (InterfaceType::Bool
808                | InterfaceType::S8
809                | InterfaceType::U8
810                | InterfaceType::S16
811                | InterfaceType::U16
812                | InterfaceType::S32
813                | InterfaceType::U32
814                | InterfaceType::S64
815                | InterfaceType::U64
816                | InterfaceType::Float32
817                | InterfaceType::Float64
818                | InterfaceType::Char),
819            ) => Some(self.types.canonical_abi(&payload)),
820            // TODO: Recursively check for other "flat" types (i.e. those without pointers or handles),
821            // e.g. `record`s, `variant`s, etc. which contain only flat types.
822            _ => None,
823        }
824    }
825
826    /// Helper function to spill the wasm arguments `args` to this function into
827    /// a stack-allocated array.
828    fn store_wasm_arguments(&mut self, args: &[Value]) -> (Value, Value) {
829        let pointer_type = self.isa.pointer_type();
830        let wasm_func_ty = &self.types[self.signature].unwrap_func();
831
832        match self.abi {
833            // For the wasm ABI a stack needs to be allocated and these
834            // arguments are stored onto the stack.
835            Abi::Wasm => {
836                let (ptr, len) = self.compiler.allocate_stack_array_and_spill_args(
837                    wasm_func_ty,
838                    &mut self.builder,
839                    args,
840                );
841                let len = self.builder.ins().iconst(pointer_type, i64::from(len));
842                (ptr, len)
843            }
844
845            // For the array ABI all arguments were already in a stack, so
846            // forward along that pointer/len.
847            Abi::Array => {
848                let params = self.builder.func.dfg.block_params(self.block0);
849                (params[2], params[3])
850            }
851        }
852    }
853
854    /// Convenience wrapper around `translate_hostcall` to enable type inference
855    /// on the `get_libcall` parameter here.
856    fn translate_libcall(
857        &mut self,
858        get_libcall: GetLibcallFn,
859        host_result: impl Into<HostResult>,
860        wasm_args: WasmArgs,
861        extra_host_args: impl FnOnce(&mut Self, &mut Vec<ir::Value>),
862    ) {
863        self.translate_hostcall(
864            HostCallee::Libcall(get_libcall),
865            host_result.into(),
866            wasm_args,
867            extra_host_args,
868        )
869    }
870
871    /// Translates an invocation of a host function and interpret the result.
872    ///
873    /// This is intended to be a relatively narrow waist which most intrinsics
874    /// go through. The configuration supported here is:
875    ///
876    /// * `host_callee` - what's being called, either a libcall or a lowered
877    ///   function
878    /// * `host_result` - how to interpret the return value to see if it's a
879    ///   trap
880    /// * `wasm_args` - how to pass wasm args to the host, either in registers
881    ///   or on the stack
882    /// * `extra_host_args` - a closure used to push extra arguments just before
883    ///   the wasm arguments are forwarded.
884    fn translate_hostcall(
885        &mut self,
886        host_callee: HostCallee,
887        host_result: impl Into<HostResult>,
888        wasm_args: WasmArgs,
889        extra_host_args: impl FnOnce(&mut Self, &mut Vec<ir::Value>),
890    ) {
891        let pointer_type = self.isa.pointer_type();
892        let wasm_func_ty = self.types[self.signature].unwrap_func();
893
894        // Load all parameters in an ABI-agnostic fashion, of which the
895        // `VMComponentContext` will be the first.
896        let params = self.abi_load_params();
897        let vmctx = params[0];
898        let wasm_params = &params[2..];
899
900        // Start building up arguments to the host. The first is always the
901        // vmctx. After is whatever `extra_host_args` appends, and then finally
902        // is what `WasmArgs` specifies.
903        let mut host_args = vec![vmctx];
904        extra_host_args(self, &mut host_args);
905        let mut val_raw_ptr = None;
906        let mut val_raw_len = None;
907        match wasm_args {
908            // Wasm params are passed through as values themselves.
909            WasmArgs::InRegisters => host_args.extend(wasm_params.iter().copied()),
910
911            // Wasm params are spilled and then the ptr/len is passed.
912            WasmArgs::ValRawList => {
913                let (ptr, len) = self.store_wasm_arguments(wasm_params);
914                val_raw_ptr = Some(ptr);
915                val_raw_len = Some(len);
916                host_args.push(ptr);
917                host_args.push(len);
918            }
919
920            // A mixture of the above two.
921            WasmArgs::InRegistersUpTo(n) => {
922                let (values_vec_ptr, len) = self.compiler.allocate_stack_array_and_spill_args(
923                    &WasmFuncType::new(
924                        wasm_func_ty.params().iter().skip(n).copied().collect(),
925                        Box::new([]),
926                    ),
927                    &mut self.builder,
928                    &wasm_params[n..],
929                );
930                let values_vec_len = self.builder.ins().iconst(pointer_type, i64::from(len));
931
932                host_args.extend(wasm_params[..n].iter().copied());
933                host_args.push(values_vec_ptr);
934                host_args.push(values_vec_len);
935            }
936        }
937
938        // Next perform the actual invocation of the host with `host_args`.
939        let call = match host_callee {
940            HostCallee::Libcall(get_libcall) => self.call_libcall(vmctx, get_libcall, &host_args),
941            HostCallee::Lowering(index) => {
942                // Load host function pointer from the vmcontext and then call that
943                // indirect function pointer with the list of arguments.
944                let host_fn = self.builder.ins().load(
945                    pointer_type,
946                    MemFlags::trusted(),
947                    vmctx,
948                    i32::try_from(self.offsets.lowering_callee(index)).unwrap(),
949                );
950                let host_sig = {
951                    let mut sig = ir::Signature::new(CallConv::triple_default(self.isa.triple()));
952                    for param in host_args.iter() {
953                        let ty = self.builder.func.dfg.value_type(*param);
954                        sig.params.push(ir::AbiParam::new(ty));
955                    }
956                    // return value is a bool whether a trap was raised or not
957                    sig.returns.push(ir::AbiParam::new(ir::types::I8));
958                    self.builder.import_signature(sig)
959                };
960                self.compiler.call_indirect_host(
961                    &mut self.builder,
962                    HostCall::ComponentLowerImport,
963                    host_sig,
964                    host_fn,
965                    &host_args,
966                )
967            }
968        };
969
970        // Acquire the result of this function (if any) and interpret it
971        // according to `host_result`.
972        //
973        // NOte that all match arms here end with `abi_store_results` which
974        // accounts for the ABI of this function when storing results.
975        let result = self.builder.func.dfg.inst_results(call).get(0).copied();
976        let result_ty = result.map(|v| self.builder.func.dfg.value_type(v));
977        let expected = wasm_func_ty.returns();
978        match host_result.into() {
979            HostResult::Sentinel(TrapSentinel::NegativeOne) => {
980                assert_eq!(expected.len(), 1);
981                let (result, result_ty) = (result.unwrap(), result_ty.unwrap());
982                let result = match (result_ty, expected[0]) {
983                    (ir::types::I64, WasmValType::I32) => {
984                        self.raise_if_negative_one_and_truncate(result)
985                    }
986                    (ir::types::I64, WasmValType::I64) | (ir::types::I32, WasmValType::I32) => {
987                        self.raise_if_negative_one(result)
988                    }
989                    other => panic!("unsupported NegativeOne combo {other:?}"),
990                };
991                self.abi_store_results(&[result]);
992            }
993            HostResult::Sentinel(TrapSentinel::Falsy) => {
994                assert_eq!(expected.len(), 0);
995                self.raise_if_host_trapped(result.unwrap());
996                self.abi_store_results(&[]);
997            }
998            HostResult::Sentinel(_) => todo!("support additional return types if/when necessary"),
999            HostResult::None => {
1000                assert!(result.is_none());
1001                self.abi_store_results(&[]);
1002            }
1003
1004            HostResult::MultiValue { ptr, len } => {
1005                let ptr = ptr.or(val_raw_ptr).unwrap();
1006                let len = len.or(val_raw_len).unwrap();
1007                self.raise_if_host_trapped(result.unwrap());
1008                let results = self.compiler.load_values_from_array(
1009                    wasm_func_ty.returns(),
1010                    &mut self.builder,
1011                    ptr,
1012                    len,
1013                );
1014                self.abi_store_results(&results);
1015            }
1016        }
1017    }
1018
1019    fn index_value(&mut self, index: impl EntityRef) -> ir::Value {
1020        self.builder
1021            .ins()
1022            .iconst(ir::types::I32, i64::try_from(index.index()).unwrap())
1023    }
1024
1025    fn translate_resource_drop(
1026        &mut self,
1027        instance: RuntimeComponentInstanceIndex,
1028        resource: TypeResourceTableIndex,
1029    ) {
1030        let args = self.abi_load_params();
1031        let vmctx = args[0];
1032        let caller_vmctx = args[1];
1033        let pointer_type = self.isa.pointer_type();
1034
1035        // The arguments this shim passes along to the libcall are:
1036        //
1037        //   * the vmctx
1038        //   * the calling component instance index
1039        //   * a constant value for this `ResourceDrop` intrinsic
1040        //   * the wasm handle index to drop
1041        let mut host_args = Vec::new();
1042        host_args.push(vmctx);
1043        host_args.push(
1044            self.builder
1045                .ins()
1046                .iconst(ir::types::I32, i64::from(instance.as_u32())),
1047        );
1048        host_args.push(
1049            self.builder
1050                .ins()
1051                .iconst(ir::types::I32, i64::from(resource.as_u32())),
1052        );
1053        host_args.push(args[2]);
1054
1055        let call = self.call_libcall(vmctx, host::resource_drop, &host_args);
1056
1057        // Immediately raise a trap if requested by the host
1058        let should_run_destructor =
1059            self.raise_if_negative_one(self.builder.func.dfg.inst_results(call)[0]);
1060
1061        let resource_ty = self.types[resource].ty;
1062        let resource_def = self
1063            .component
1064            .defined_resource_index(resource_ty)
1065            .map(|idx| {
1066                self.component
1067                    .initializers
1068                    .iter()
1069                    .filter_map(|i| match i {
1070                        GlobalInitializer::Resource(r) if r.index == idx => Some(r),
1071                        _ => None,
1072                    })
1073                    .next()
1074                    .unwrap()
1075            });
1076        let has_destructor = match resource_def {
1077            Some(def) => def.dtor.is_some(),
1078            None => true,
1079        };
1080        // Synthesize the following:
1081        //
1082        //      ...
1083        //      brif should_run_destructor, run_destructor_block, return_block
1084        //
1085        //    run_destructor_block:
1086        //      ;; test may_enter, but only if the component instances
1087        //      ;; differ
1088        //      flags = load.i32 vmctx+$offset
1089        //      masked = band flags, $FLAG_MAY_ENTER
1090        //      trapz masked, CANNOT_ENTER_CODE
1091        //
1092        //      ;; ============================================================
1093        //      ;; this is conditionally emitted based on whether the resource
1094        //      ;; has a destructor or not, and can be statically omitted
1095        //      ;; because that information is known at compile time here.
1096        //      rep = ushr.i64 rep, 1
1097        //      rep = ireduce.i32 rep
1098        //      dtor = load.ptr vmctx+$offset
1099        //      func_addr = load.ptr dtor+$offset
1100        //      callee_vmctx = load.ptr dtor+$offset
1101        //      call_indirect func_addr, callee_vmctx, vmctx, rep
1102        //      ;; ============================================================
1103        //
1104        //      jump return_block
1105        //
1106        //    return_block:
1107        //      return
1108        //
1109        // This will decode `should_run_destructor` and run the destructor
1110        // funcref if one is specified for this resource. Note that not all
1111        // resources have destructors, hence the null check.
1112        self.builder.ensure_inserted_block();
1113        let current_block = self.builder.current_block().unwrap();
1114        let run_destructor_block = self.builder.create_block();
1115        self.builder
1116            .insert_block_after(run_destructor_block, current_block);
1117        let return_block = self.builder.create_block();
1118        self.builder
1119            .insert_block_after(return_block, run_destructor_block);
1120
1121        self.builder.ins().brif(
1122            should_run_destructor,
1123            run_destructor_block,
1124            &[],
1125            return_block,
1126            &[],
1127        );
1128
1129        let trusted = ir::MemFlags::trusted().with_readonly();
1130
1131        self.builder.switch_to_block(run_destructor_block);
1132
1133        // If this is a defined resource within the component itself then a
1134        // check needs to be emitted for the `may_enter` flag. Note though
1135        // that this check can be elided if the resource table resides in
1136        // the same component instance that defined the resource as the
1137        // component is calling itself.
1138        if let Some(def) = resource_def {
1139            if self.types[resource].instance != def.instance {
1140                let flags = self.builder.ins().load(
1141                    ir::types::I32,
1142                    trusted,
1143                    vmctx,
1144                    i32::try_from(self.offsets.instance_flags(def.instance)).unwrap(),
1145                );
1146                let masked = self
1147                    .builder
1148                    .ins()
1149                    .band_imm(flags, i64::from(FLAG_MAY_ENTER));
1150                self.builder.ins().trapz(masked, TRAP_CANNOT_ENTER);
1151            }
1152        }
1153
1154        // Conditionally emit destructor-execution code based on whether we
1155        // statically know that a destructor exists or not.
1156        if has_destructor {
1157            let rep = self.builder.ins().ushr_imm(should_run_destructor, 1);
1158            let rep = self.builder.ins().ireduce(ir::types::I32, rep);
1159            let index = self.types[resource].ty;
1160            // NB: despite the vmcontext storing nullable funcrefs for function
1161            // pointers we know this is statically never null due to the
1162            // `has_destructor` check above.
1163            let dtor_func_ref = self.builder.ins().load(
1164                pointer_type,
1165                trusted,
1166                vmctx,
1167                i32::try_from(self.offsets.resource_destructor(index)).unwrap(),
1168            );
1169            if self.compiler.emit_debug_checks {
1170                self.builder
1171                    .ins()
1172                    .trapz(dtor_func_ref, TRAP_INTERNAL_ASSERT);
1173            }
1174            let func_addr = self.builder.ins().load(
1175                pointer_type,
1176                trusted,
1177                dtor_func_ref,
1178                i32::from(self.offsets.ptr.vm_func_ref_wasm_call()),
1179            );
1180            let callee_vmctx = self.builder.ins().load(
1181                pointer_type,
1182                trusted,
1183                dtor_func_ref,
1184                i32::from(self.offsets.ptr.vm_func_ref_vmctx()),
1185            );
1186
1187            let sig = crate::wasm_call_signature(
1188                self.isa,
1189                &self.types[self.signature].unwrap_func(),
1190                &self.compiler.tunables,
1191            );
1192            let sig_ref = self.builder.import_signature(sig);
1193
1194            // NB: note that the "caller" vmctx here is the caller of this
1195            // intrinsic itself, not the `VMComponentContext`. This effectively
1196            // takes ourselves out of the chain here but that's ok since the
1197            // caller is only used for store/limits and that same info is
1198            // stored, but elsewhere, in the component context.
1199            self.builder.ins().call_indirect(
1200                sig_ref,
1201                func_addr,
1202                &[callee_vmctx, caller_vmctx, rep],
1203            );
1204        }
1205        self.builder.ins().jump(return_block, &[]);
1206        self.builder.seal_block(run_destructor_block);
1207
1208        self.builder.switch_to_block(return_block);
1209        self.builder.seal_block(return_block);
1210        self.abi_store_results(&[]);
1211    }
1212
1213    fn load_optional_memory(
1214        &mut self,
1215        vmctx: ir::Value,
1216        memory: Option<RuntimeMemoryIndex>,
1217    ) -> ir::Value {
1218        match memory {
1219            Some(idx) => self.load_memory(vmctx, idx),
1220            None => self.builder.ins().iconst(self.isa.pointer_type(), 0),
1221        }
1222    }
1223
1224    fn load_memory(&mut self, vmctx: ir::Value, memory: RuntimeMemoryIndex) -> ir::Value {
1225        self.builder.ins().load(
1226            self.isa.pointer_type(),
1227            MemFlags::trusted(),
1228            vmctx,
1229            i32::try_from(self.offsets.runtime_memory(memory)).unwrap(),
1230        )
1231    }
1232
1233    fn load_callback(
1234        &mut self,
1235        vmctx: ir::Value,
1236        callback: Option<RuntimeCallbackIndex>,
1237    ) -> ir::Value {
1238        let pointer_type = self.isa.pointer_type();
1239        match callback {
1240            Some(idx) => self.builder.ins().load(
1241                pointer_type,
1242                MemFlags::trusted(),
1243                vmctx,
1244                i32::try_from(self.offsets.runtime_callback(idx)).unwrap(),
1245            ),
1246            None => self.builder.ins().iconst(pointer_type, 0),
1247        }
1248    }
1249
1250    fn load_post_return(
1251        &mut self,
1252        vmctx: ir::Value,
1253        post_return: Option<RuntimePostReturnIndex>,
1254    ) -> ir::Value {
1255        let pointer_type = self.isa.pointer_type();
1256        match post_return {
1257            Some(idx) => self.builder.ins().load(
1258                pointer_type,
1259                MemFlags::trusted(),
1260                vmctx,
1261                i32::try_from(self.offsets.runtime_post_return(idx)).unwrap(),
1262            ),
1263            None => self.builder.ins().iconst(pointer_type, 0),
1264        }
1265    }
1266
1267    /// Loads a host function pointer for a libcall stored at the `offset`
1268    /// provided in the libcalls array.
1269    ///
1270    /// The offset is calculated in the `host` module below.
1271    fn load_libcall(
1272        &mut self,
1273        vmctx: ir::Value,
1274        index: ComponentBuiltinFunctionIndex,
1275    ) -> ir::Value {
1276        let pointer_type = self.isa.pointer_type();
1277        // First load the pointer to the builtins structure which is static
1278        // per-process.
1279        let builtins_array = self.builder.ins().load(
1280            pointer_type,
1281            MemFlags::trusted().with_readonly(),
1282            vmctx,
1283            i32::try_from(self.offsets.builtins()).unwrap(),
1284        );
1285        // Next load the function pointer at `offset` and return that.
1286        self.builder.ins().load(
1287            pointer_type,
1288            MemFlags::trusted().with_readonly(),
1289            builtins_array,
1290            i32::try_from(index.index() * u32::from(self.offsets.ptr.size())).unwrap(),
1291        )
1292    }
1293
1294    fn abi_load_params(&mut self) -> Vec<ir::Value> {
1295        let mut block0_params = self.builder.func.dfg.block_params(self.block0).to_vec();
1296        match self.abi {
1297            // Wasm and native ABIs pass parameters as normal function
1298            // parameters.
1299            Abi::Wasm => block0_params,
1300
1301            // The array ABI passes a pointer/length as the 3rd/4th arguments
1302            // and those are used to load the actual wasm parameters.
1303            Abi::Array => {
1304                let results = self.compiler.load_values_from_array(
1305                    self.types[self.signature].unwrap_func().params(),
1306                    &mut self.builder,
1307                    block0_params[2],
1308                    block0_params[3],
1309                );
1310                block0_params.truncate(2);
1311                block0_params.extend(results);
1312                block0_params
1313            }
1314        }
1315    }
1316
1317    fn abi_store_results(&mut self, results: &[ir::Value]) {
1318        match self.abi {
1319            // Wasm/native ABIs return values as usual.
1320            Abi::Wasm => {
1321                self.builder.ins().return_(results);
1322            }
1323
1324            // The array ABI stores all results in the pointer/length passed
1325            // as arguments to this function, which contractually are required
1326            // to have enough space for the results.
1327            Abi::Array => {
1328                let block0_params = self.builder.func.dfg.block_params(self.block0);
1329                let (ptr, len) = (block0_params[2], block0_params[3]);
1330                self.compiler.store_values_to_array(
1331                    &mut self.builder,
1332                    self.types[self.signature].unwrap_func().returns(),
1333                    results,
1334                    ptr,
1335                    len,
1336                );
1337                let true_value = self.builder.ins().iconst(ir::types::I8, 1);
1338                self.builder.ins().return_(&[true_value]);
1339            }
1340        }
1341    }
1342
1343    fn raise_if_host_trapped(&mut self, succeeded: ir::Value) {
1344        let caller_vmctx = self.builder.func.dfg.block_params(self.block0)[1];
1345        self.compiler
1346            .raise_if_host_trapped(&mut self.builder, caller_vmctx, succeeded);
1347    }
1348
1349    fn raise_if_transcode_trapped(&mut self, amount_copied: ir::Value) {
1350        let pointer_type = self.isa.pointer_type();
1351        let minus_one = self.builder.ins().iconst(pointer_type, -1);
1352        let succeeded = self
1353            .builder
1354            .ins()
1355            .icmp(IntCC::NotEqual, amount_copied, minus_one);
1356        self.raise_if_host_trapped(succeeded);
1357    }
1358
1359    fn raise_if_negative_one_and_truncate(&mut self, ret: ir::Value) -> ir::Value {
1360        let ret = self.raise_if_negative_one(ret);
1361        self.builder.ins().ireduce(ir::types::I32, ret)
1362    }
1363
1364    fn raise_if_negative_one(&mut self, ret: ir::Value) -> ir::Value {
1365        let result_ty = self.builder.func.dfg.value_type(ret);
1366        let minus_one = self.builder.ins().iconst(result_ty, -1);
1367        let succeeded = self.builder.ins().icmp(IntCC::NotEqual, ret, minus_one);
1368        self.raise_if_host_trapped(succeeded);
1369        ret
1370    }
1371
1372    fn call_libcall(
1373        &mut self,
1374        vmctx: ir::Value,
1375        get_libcall: GetLibcallFn,
1376        args: &[ir::Value],
1377    ) -> ir::Inst {
1378        let (host_sig, index) = get_libcall(self.isa, &mut self.builder.func);
1379        let host_fn = self.load_libcall(vmctx, index);
1380        self.compiler
1381            .call_indirect_host(&mut self.builder, index, host_sig, host_fn, args)
1382    }
1383}
1384
1385impl ComponentCompiler for Compiler {
1386    fn compile_trampoline(
1387        &self,
1388        component: &ComponentTranslation,
1389        types: &ComponentTypesBuilder,
1390        key: FuncKey,
1391        abi: Abi,
1392        tunables: &Tunables,
1393        symbol: &str,
1394    ) -> Result<CompiledFunctionBody> {
1395        let (abi2, trampoline_index) = key.unwrap_component_trampoline();
1396        debug_assert_eq!(abi, abi2);
1397
1398        match abi {
1399            // Fall through to the trampoline compiler.
1400            Abi::Wasm => {}
1401
1402            // Implement the array-abi trampoline in terms of calling the
1403            // wasm-abi trampoline.
1404            Abi::Array => {
1405                let offsets =
1406                    VMComponentOffsets::new(self.isa.pointer_bytes(), &component.component);
1407                return Ok(self.array_to_wasm_trampoline(
1408                    key,
1409                    FuncKey::ComponentTrampoline(Abi::Wasm, trampoline_index),
1410                    types.module_types_builder(),
1411                    component.component.trampolines[trampoline_index],
1412                    symbol,
1413                    offsets.vm_store_context(),
1414                    wasmtime_environ::component::VMCOMPONENT_MAGIC,
1415                )?);
1416            }
1417        }
1418
1419        let mut compiler = self.function_compiler();
1420        let mut c = TrampolineCompiler::new(
1421            self,
1422            &mut compiler,
1423            &component.component,
1424            types,
1425            trampoline_index,
1426            abi,
1427            tunables,
1428        );
1429
1430        // If we are crossing the Wasm-to-native boundary, we need to save the
1431        // exit FP and return address for stack walking purposes. However, we
1432        // always debug assert that our vmctx is a component context, regardless
1433        // whether we are actually crossing that boundary because it should
1434        // always hold.
1435        let vmctx = c.builder.block_params(c.block0)[0];
1436        let pointer_type = self.isa.pointer_type();
1437        self.debug_assert_vmctx_kind(
1438            &mut c.builder,
1439            vmctx,
1440            wasmtime_environ::component::VMCOMPONENT_MAGIC,
1441        );
1442        if let Abi::Wasm = abi {
1443            let vm_store_context = c.builder.ins().load(
1444                pointer_type,
1445                MemFlags::trusted(),
1446                vmctx,
1447                i32::try_from(c.offsets.vm_store_context()).unwrap(),
1448            );
1449            super::save_last_wasm_exit_fp_and_pc(
1450                &mut c.builder,
1451                pointer_type,
1452                &c.offsets.ptr,
1453                vm_store_context,
1454            );
1455        }
1456
1457        c.translate(&component.trampolines[trampoline_index]);
1458        c.builder.finalize();
1459        compiler.cx.abi = Some(abi);
1460
1461        Ok(CompiledFunctionBody {
1462            code: super::box_dyn_any_compiler_context(Some(compiler.cx)),
1463            needs_gc_heap: false,
1464        })
1465    }
1466}
1467
1468impl TrampolineCompiler<'_> {
1469    fn translate_transcode(
1470        &mut self,
1471        op: Transcode,
1472        from: RuntimeMemoryIndex,
1473        from64: bool,
1474        to: RuntimeMemoryIndex,
1475        to64: bool,
1476    ) {
1477        let pointer_type = self.isa.pointer_type();
1478        let vmctx = self.builder.func.dfg.block_params(self.block0)[0];
1479
1480        // Determine the static signature of the host libcall for this transcode
1481        // operation and additionally calculate the static offset within the
1482        // transode libcalls array.
1483        let get_libcall = match op {
1484            Transcode::Copy(FixedEncoding::Utf8) => host::utf8_to_utf8,
1485            Transcode::Copy(FixedEncoding::Utf16) => host::utf16_to_utf16,
1486            Transcode::Copy(FixedEncoding::Latin1) => host::latin1_to_latin1,
1487            Transcode::Latin1ToUtf16 => host::latin1_to_utf16,
1488            Transcode::Latin1ToUtf8 => host::latin1_to_utf8,
1489            Transcode::Utf16ToCompactProbablyUtf16 => host::utf16_to_compact_probably_utf16,
1490            Transcode::Utf16ToCompactUtf16 => host::utf16_to_compact_utf16,
1491            Transcode::Utf16ToLatin1 => host::utf16_to_latin1,
1492            Transcode::Utf16ToUtf8 => host::utf16_to_utf8,
1493            Transcode::Utf8ToCompactUtf16 => host::utf8_to_compact_utf16,
1494            Transcode::Utf8ToLatin1 => host::utf8_to_latin1,
1495            Transcode::Utf8ToUtf16 => host::utf8_to_utf16,
1496        };
1497
1498        // Load the base pointers for the from/to linear memories.
1499        let from_base = self.load_runtime_memory_base(vmctx, from);
1500        let to_base = self.load_runtime_memory_base(vmctx, to);
1501
1502        let mut args = Vec::new();
1503        args.push(vmctx);
1504
1505        let uses_retptr = match op {
1506            Transcode::Utf16ToUtf8
1507            | Transcode::Latin1ToUtf8
1508            | Transcode::Utf8ToLatin1
1509            | Transcode::Utf16ToLatin1 => true,
1510            _ => false,
1511        };
1512
1513        // Most transcoders share roughly the same signature despite doing very
1514        // different things internally, so most libcalls are lumped together
1515        // here.
1516        match op {
1517            Transcode::Copy(_)
1518            | Transcode::Latin1ToUtf16
1519            | Transcode::Utf16ToCompactProbablyUtf16
1520            | Transcode::Utf8ToLatin1
1521            | Transcode::Utf16ToLatin1
1522            | Transcode::Utf8ToUtf16 => {
1523                args.push(self.ptr_param(0, from64, from_base));
1524                args.push(self.len_param(1, from64));
1525                args.push(self.ptr_param(2, to64, to_base));
1526            }
1527
1528            Transcode::Utf16ToUtf8 | Transcode::Latin1ToUtf8 => {
1529                args.push(self.ptr_param(0, from64, from_base));
1530                args.push(self.len_param(1, from64));
1531                args.push(self.ptr_param(2, to64, to_base));
1532                args.push(self.len_param(3, to64));
1533            }
1534
1535            Transcode::Utf8ToCompactUtf16 | Transcode::Utf16ToCompactUtf16 => {
1536                args.push(self.ptr_param(0, from64, from_base));
1537                args.push(self.len_param(1, from64));
1538                args.push(self.ptr_param(2, to64, to_base));
1539                args.push(self.len_param(3, to64));
1540                args.push(self.len_param(4, to64));
1541            }
1542        };
1543        if uses_retptr {
1544            let slot = self
1545                .builder
1546                .func
1547                .create_sized_stack_slot(ir::StackSlotData::new(
1548                    ir::StackSlotKind::ExplicitSlot,
1549                    pointer_type.bytes(),
1550                    0,
1551                ));
1552            args.push(self.builder.ins().stack_addr(pointer_type, slot, 0));
1553        }
1554        let call = self.call_libcall(vmctx, get_libcall, &args);
1555        let mut results = self.builder.func.dfg.inst_results(call).to_vec();
1556        if uses_retptr {
1557            results.push(self.builder.ins().load(
1558                pointer_type,
1559                ir::MemFlags::trusted(),
1560                *args.last().unwrap(),
1561                0,
1562            ));
1563        }
1564        let mut raw_results = Vec::new();
1565
1566        // Like the arguments the results are fairly similar across libcalls, so
1567        // they're lumped into various buckets here.
1568        match op {
1569            Transcode::Copy(_) | Transcode::Latin1ToUtf16 => {
1570                self.raise_if_host_trapped(results[0]);
1571            }
1572
1573            Transcode::Utf8ToUtf16
1574            | Transcode::Utf16ToCompactProbablyUtf16
1575            | Transcode::Utf8ToCompactUtf16
1576            | Transcode::Utf16ToCompactUtf16 => {
1577                self.raise_if_transcode_trapped(results[0]);
1578                raw_results.push(self.cast_from_pointer(results[0], to64));
1579            }
1580
1581            Transcode::Latin1ToUtf8
1582            | Transcode::Utf16ToUtf8
1583            | Transcode::Utf8ToLatin1
1584            | Transcode::Utf16ToLatin1 => {
1585                self.raise_if_transcode_trapped(results[0]);
1586                raw_results.push(self.cast_from_pointer(results[0], from64));
1587                raw_results.push(self.cast_from_pointer(results[1], to64));
1588            }
1589        };
1590
1591        self.builder.ins().return_(&raw_results);
1592    }
1593
1594    // Helper function to cast an input parameter to the host pointer type.
1595    fn len_param(&mut self, param: usize, is64: bool) -> ir::Value {
1596        let val = self.builder.func.dfg.block_params(self.block0)[2 + param];
1597        self.cast_to_pointer(val, is64)
1598    }
1599
1600    // Helper function to interpret an input parameter as a pointer into
1601    // linear memory. This will cast the input parameter to the host integer
1602    // type and then add that value to the base.
1603    //
1604    // Note that bounds-checking happens in adapter modules, and this
1605    // trampoline is simply calling the host libcall.
1606    fn ptr_param(&mut self, param: usize, is64: bool, base: ir::Value) -> ir::Value {
1607        let val = self.len_param(param, is64);
1608        self.builder.ins().iadd(base, val)
1609    }
1610
1611    // Helper function to cast a core wasm input to a host pointer type
1612    // which will go into the host libcall.
1613    fn cast_to_pointer(&mut self, val: ir::Value, is64: bool) -> ir::Value {
1614        let pointer_type = self.isa.pointer_type();
1615        let host64 = pointer_type == ir::types::I64;
1616        if is64 == host64 {
1617            val
1618        } else if !is64 {
1619            assert!(host64);
1620            self.builder.ins().uextend(pointer_type, val)
1621        } else {
1622            assert!(!host64);
1623            self.builder.ins().ireduce(pointer_type, val)
1624        }
1625    }
1626
1627    // Helper to cast a host pointer integer type to the destination type.
1628    fn cast_from_pointer(&mut self, val: ir::Value, is64: bool) -> ir::Value {
1629        let host64 = self.isa.pointer_type() == ir::types::I64;
1630        if is64 == host64 {
1631            val
1632        } else if !is64 {
1633            assert!(host64);
1634            self.builder.ins().ireduce(ir::types::I32, val)
1635        } else {
1636            assert!(!host64);
1637            self.builder.ins().uextend(ir::types::I64, val)
1638        }
1639    }
1640
1641    fn load_runtime_memory_base(&mut self, vmctx: ir::Value, mem: RuntimeMemoryIndex) -> ir::Value {
1642        let pointer_type = self.isa.pointer_type();
1643        let from_vmmemory_definition = self.load_memory(vmctx, mem);
1644        self.builder.ins().load(
1645            pointer_type,
1646            MemFlags::trusted(),
1647            from_vmmemory_definition,
1648            i32::from(self.offsets.ptr.vmmemory_definition_base()),
1649        )
1650    }
1651}
1652
1653/// Module with macro-generated contents that will return the signature and
1654/// offset for each of the host transcoder functions.
1655///
1656/// Note that a macro is used here to keep this in sync with the actual
1657/// transcoder functions themselves which are also defined via a macro.
1658mod host {
1659    use cranelift_codegen::ir::{self, AbiParam};
1660    use cranelift_codegen::isa::{CallConv, TargetIsa};
1661    use wasmtime_environ::component::ComponentBuiltinFunctionIndex;
1662
1663    macro_rules! define {
1664        (
1665            $(
1666                $( #[$attr:meta] )*
1667                $name:ident( $( $pname:ident: $param:ident ),* ) $( -> $result:ident )?;
1668            )*
1669        ) => {
1670            $(
1671                pub(super) fn $name(isa: &dyn TargetIsa, func: &mut ir::Function) -> (ir::SigRef, ComponentBuiltinFunctionIndex) {
1672                    let pointer_type = isa.pointer_type();
1673                    let sig = build_sig(
1674                        isa,
1675                        func,
1676                        &[$( define!(@ty pointer_type $param) ),*],
1677                        &[$( define!(@ty pointer_type $result) ),*],
1678                    );
1679
1680                    return (sig, ComponentBuiltinFunctionIndex::$name())
1681                }
1682            )*
1683        };
1684
1685        (@ty $ptr:ident size) => ($ptr);
1686        (@ty $ptr:ident ptr_u8) => ($ptr);
1687        (@ty $ptr:ident ptr_u16) => ($ptr);
1688        (@ty $ptr:ident ptr_size) => ($ptr);
1689        (@ty $ptr:ident bool) => (ir::types::I8);
1690        (@ty $ptr:ident u8) => (ir::types::I8);
1691        (@ty $ptr:ident u32) => (ir::types::I32);
1692        (@ty $ptr:ident u64) => (ir::types::I64);
1693        (@ty $ptr:ident vmctx) => ($ptr);
1694    }
1695
1696    wasmtime_environ::foreach_builtin_component_function!(define);
1697
1698    fn build_sig(
1699        isa: &dyn TargetIsa,
1700        func: &mut ir::Function,
1701        params: &[ir::Type],
1702        returns: &[ir::Type],
1703    ) -> ir::SigRef {
1704        let mut sig = ir::Signature {
1705            params: params.iter().map(|ty| AbiParam::new(*ty)).collect(),
1706            returns: returns.iter().map(|ty| AbiParam::new(*ty)).collect(),
1707            call_conv: CallConv::triple_default(isa.triple()),
1708        };
1709
1710        // Once we're declaring the signature of a host function we must respect
1711        // the default ABI of the platform which is where argument extension of
1712        // params/results may come into play.
1713        let extension = isa.default_argument_extension();
1714        for arg in sig.params.iter_mut().chain(sig.returns.iter_mut()) {
1715            if arg.value_type.is_int() {
1716                arg.extension = extension;
1717            }
1718        }
1719        func.import_signature(sig)
1720    }
1721}
wasmtime_internal_cranelift/compiler/component.rs

wasmtime_internal_cranelift/compiler/
component.rs
