cranelift_codegen/machinst/
isle.rs

1use crate::ir::{BlockCall, Value, ValueList};
2use alloc::boxed::Box;
3use alloc::vec::Vec;
4use smallvec::SmallVec;
5
6pub use super::MachLabel;
7use super::RetPair;
8pub use crate::ir::{condcodes::CondCode, *};
9pub use crate::isa::{TargetIsa, unwind::UnwindInst};
10pub use crate::machinst::{
11    ABIArg, ABIArgSlot, ABIMachineSpec, InputSourceInst, Lower, LowerBackend, RealReg, Reg,
12    RelocDistance, Sig, TryCallInfo, VCodeInst, Writable,
13};
14pub use crate::settings::{StackSwitchModel, TlsModel};
15
16pub type Unit = ();
17pub type ValueSlice = (ValueList, usize);
18pub type ValueArray2 = [Value; 2];
19pub type ValueArray3 = [Value; 3];
20pub type BlockArray2 = [BlockCall; 2];
21pub type WritableReg = Writable<Reg>;
22pub type VecRetPair = Vec<RetPair>;
23pub type VecMask = Vec<u8>;
24pub type ValueRegs = crate::machinst::ValueRegs<Reg>;
25pub type WritableValueRegs = crate::machinst::ValueRegs<WritableReg>;
26pub type ValueRegsVec = SmallVec<[ValueRegs; 2]>;
27pub type InstOutput = SmallVec<[ValueRegs; 2]>;
28pub type BoxExternalName = Box<ExternalName>;
29pub type MachLabelSlice = [MachLabel];
30pub type BoxVecMachLabel = Box<Vec<MachLabel>>;
31pub type OptionTryCallInfo = Option<TryCallInfo>;
32
33/// Helper macro to define methods in `prelude.isle` within `impl Context for
34/// ...` for each backend. These methods are shared amongst all backends.
35#[macro_export]
36#[doc(hidden)]
37macro_rules! isle_lower_prelude_methods {
38    () => {
39        crate::isle_lower_prelude_methods!(MInst);
40    };
41    ($inst:ty) => {
42        crate::isle_common_prelude_methods!();
43
44        #[inline]
45        fn value_type(&mut self, val: Value) -> Type {
46            self.lower_ctx.dfg().value_type(val)
47        }
48
49        #[inline]
50        fn value_reg(&mut self, reg: Reg) -> ValueRegs {
51            ValueRegs::one(reg)
52        }
53
54        #[inline]
55        fn value_regs(&mut self, r1: Reg, r2: Reg) -> ValueRegs {
56            ValueRegs::two(r1, r2)
57        }
58
59        #[inline]
60        fn writable_value_regs(&mut self, r1: WritableReg, r2: WritableReg) -> WritableValueRegs {
61            WritableValueRegs::two(r1, r2)
62        }
63
64        #[inline]
65        fn writable_value_reg(&mut self, r: WritableReg) -> WritableValueRegs {
66            WritableValueRegs::one(r)
67        }
68
69        #[inline]
70        fn value_regs_invalid(&mut self) -> ValueRegs {
71            ValueRegs::invalid()
72        }
73
74        #[inline]
75        fn output_none(&mut self) -> InstOutput {
76            smallvec::smallvec![]
77        }
78
79        #[inline]
80        fn output(&mut self, regs: ValueRegs) -> InstOutput {
81            smallvec::smallvec![regs]
82        }
83
84        #[inline]
85        fn output_pair(&mut self, r1: ValueRegs, r2: ValueRegs) -> InstOutput {
86            smallvec::smallvec![r1, r2]
87        }
88
89        #[inline]
90        fn output_vec(&mut self, output: &ValueRegsVec) -> InstOutput {
91            output.clone()
92        }
93
94        #[inline]
95        fn temp_writable_reg(&mut self, ty: Type) -> WritableReg {
96            let value_regs = self.lower_ctx.alloc_tmp(ty);
97            value_regs.only_reg().unwrap()
98        }
99
100        #[inline]
101        fn is_valid_reg(&mut self, reg: Reg) -> bool {
102            use crate::machinst::valueregs::InvalidSentinel;
103            !reg.is_invalid_sentinel()
104        }
105
106        #[inline]
107        fn invalid_reg(&mut self) -> Reg {
108            use crate::machinst::valueregs::InvalidSentinel;
109            Reg::invalid_sentinel()
110        }
111
112        #[inline]
113        fn mark_value_used(&mut self, val: Value) {
114            self.lower_ctx.increment_lowered_uses(val);
115        }
116
117        #[inline]
118        fn put_in_reg(&mut self, val: Value) -> Reg {
119            self.put_in_regs(val).only_reg().unwrap()
120        }
121
122        #[inline]
123        fn put_in_regs(&mut self, val: Value) -> ValueRegs {
124            self.lower_ctx.put_value_in_regs(val)
125        }
126
127        #[inline]
128        fn put_in_regs_vec(&mut self, (list, off): ValueSlice) -> ValueRegsVec {
129            (off..list.len(&self.lower_ctx.dfg().value_lists))
130                .map(|ix| {
131                    let val = list.get(ix, &self.lower_ctx.dfg().value_lists).unwrap();
132                    self.put_in_regs(val)
133                })
134                .collect()
135        }
136
137        #[inline]
138        fn ensure_in_vreg(&mut self, reg: Reg, ty: Type) -> Reg {
139            self.lower_ctx.ensure_in_vreg(reg, ty)
140        }
141
142        #[inline]
143        fn value_regs_get(&mut self, regs: ValueRegs, i: usize) -> Reg {
144            regs.regs()[i]
145        }
146
147        #[inline]
148        fn value_regs_len(&mut self, regs: ValueRegs) -> usize {
149            regs.regs().len()
150        }
151
152        #[inline]
153        fn value_list_slice(&mut self, list: ValueList) -> ValueSlice {
154            (list, 0)
155        }
156
157        #[inline]
158        fn value_slice_empty(&mut self, slice: ValueSlice) -> Option<()> {
159            let (list, off) = slice;
160            if off >= list.len(&self.lower_ctx.dfg().value_lists) {
161                Some(())
162            } else {
163                None
164            }
165        }
166
167        #[inline]
168        fn value_slice_unwrap(&mut self, slice: ValueSlice) -> Option<(Value, ValueSlice)> {
169            let (list, off) = slice;
170            if let Some(val) = list.get(off, &self.lower_ctx.dfg().value_lists) {
171                Some((val, (list, off + 1)))
172            } else {
173                None
174            }
175        }
176
177        #[inline]
178        fn value_slice_len(&mut self, slice: ValueSlice) -> usize {
179            let (list, off) = slice;
180            list.len(&self.lower_ctx.dfg().value_lists) - off
181        }
182
183        #[inline]
184        fn value_slice_get(&mut self, slice: ValueSlice, idx: usize) -> Value {
185            let (list, off) = slice;
186            list.get(off + idx, &self.lower_ctx.dfg().value_lists)
187                .unwrap()
188        }
189
190        #[inline]
191        fn writable_reg_to_reg(&mut self, r: WritableReg) -> Reg {
192            r.to_reg()
193        }
194
195        #[inline]
196        fn inst_results(&mut self, inst: Inst) -> ValueSlice {
197            (self.lower_ctx.dfg().inst_results_list(inst), 0)
198        }
199
200        #[inline]
201        fn first_result(&mut self, inst: Inst) -> Option<Value> {
202            self.lower_ctx.dfg().inst_results(inst).first().copied()
203        }
204
205        #[inline]
206        fn inst_data_value(&mut self, inst: Inst) -> InstructionData {
207            self.lower_ctx.dfg().insts[inst]
208        }
209
210        #[inline]
211        fn i64_from_iconst(&mut self, val: Value) -> Option<i64> {
212            let inst = self.def_inst(val)?;
213            let constant = match self.lower_ctx.data(inst) {
214                InstructionData::UnaryImm {
215                    opcode: Opcode::Iconst,
216                    imm,
217                } => imm.bits(),
218                _ => return None,
219            };
220            let ty = self.lower_ctx.output_ty(inst, 0);
221            let shift_amt = std::cmp::max(0, 64 - self.ty_bits(ty));
222            Some((constant << shift_amt) >> shift_amt)
223        }
224
225        fn zero_value(&mut self, value: Value) -> Option<Value> {
226            let insn = self.def_inst(value);
227            if insn.is_some() {
228                let insn = insn.unwrap();
229                let inst_data = self.lower_ctx.data(insn);
230                match inst_data {
231                    InstructionData::Unary {
232                        opcode: Opcode::Splat,
233                        arg,
234                    } => {
235                        let arg = arg.clone();
236                        return self.zero_value(arg);
237                    }
238                    InstructionData::UnaryConst {
239                        opcode: Opcode::Vconst | Opcode::F128const,
240                        constant_handle,
241                    } => {
242                        let constant_data =
243                            self.lower_ctx.get_constant_data(*constant_handle).clone();
244                        if constant_data.into_vec().iter().any(|&x| x != 0) {
245                            return None;
246                        } else {
247                            return Some(value);
248                        }
249                    }
250                    InstructionData::UnaryImm { imm, .. } => {
251                        if imm.bits() == 0 {
252                            return Some(value);
253                        } else {
254                            return None;
255                        }
256                    }
257                    InstructionData::UnaryIeee16 { imm, .. } => {
258                        if imm.bits() == 0 {
259                            return Some(value);
260                        } else {
261                            return None;
262                        }
263                    }
264                    InstructionData::UnaryIeee32 { imm, .. } => {
265                        if imm.bits() == 0 {
266                            return Some(value);
267                        } else {
268                            return None;
269                        }
270                    }
271                    InstructionData::UnaryIeee64 { imm, .. } => {
272                        if imm.bits() == 0 {
273                            return Some(value);
274                        } else {
275                            return None;
276                        }
277                    }
278                    _ => None,
279                }
280            } else {
281                None
282            }
283        }
284
285        #[inline]
286        fn tls_model(&mut self, _: Type) -> TlsModel {
287            self.backend.flags().tls_model()
288        }
289
290        #[inline]
291        fn tls_model_is_elf_gd(&mut self) -> Option<()> {
292            if self.backend.flags().tls_model() == TlsModel::ElfGd {
293                Some(())
294            } else {
295                None
296            }
297        }
298
299        #[inline]
300        fn tls_model_is_macho(&mut self) -> Option<()> {
301            if self.backend.flags().tls_model() == TlsModel::Macho {
302                Some(())
303            } else {
304                None
305            }
306        }
307
308        #[inline]
309        fn tls_model_is_coff(&mut self) -> Option<()> {
310            if self.backend.flags().tls_model() == TlsModel::Coff {
311                Some(())
312            } else {
313                None
314            }
315        }
316
317        #[inline]
318        fn preserve_frame_pointers(&mut self) -> Option<()> {
319            if self.backend.flags().preserve_frame_pointers() {
320                Some(())
321            } else {
322                None
323            }
324        }
325
326        #[inline]
327        fn stack_switch_model(&mut self) -> Option<StackSwitchModel> {
328            Some(self.backend.flags().stack_switch_model())
329        }
330
331        #[inline]
332        fn func_ref_data(&mut self, func_ref: FuncRef) -> (SigRef, ExternalName, RelocDistance) {
333            let funcdata = &self.lower_ctx.dfg().ext_funcs[func_ref];
334            let reloc_distance = if funcdata.colocated {
335                RelocDistance::Near
336            } else {
337                RelocDistance::Far
338            };
339            (funcdata.signature, funcdata.name.clone(), reloc_distance)
340        }
341
342        #[inline]
343        fn exception_sig(&mut self, et: ExceptionTable) -> SigRef {
344            self.lower_ctx.dfg().exception_tables[et].signature()
345        }
346
347        #[inline]
348        fn box_external_name(&mut self, extname: ExternalName) -> BoxExternalName {
349            Box::new(extname)
350        }
351
352        #[inline]
353        fn symbol_value_data(
354            &mut self,
355            global_value: GlobalValue,
356        ) -> Option<(ExternalName, RelocDistance, i64)> {
357            let (name, reloc, offset) = self.lower_ctx.symbol_value_data(global_value)?;
358            Some((name.clone(), reloc, offset))
359        }
360
361        #[inline]
362        fn u128_from_immediate(&mut self, imm: Immediate) -> Option<u128> {
363            let bytes = self.lower_ctx.get_immediate_data(imm).as_slice();
364            Some(u128::from_le_bytes(bytes.try_into().ok()?))
365        }
366
367        #[inline]
368        fn vconst_from_immediate(&mut self, imm: Immediate) -> Option<VCodeConstant> {
369            Some(self.lower_ctx.use_constant(VCodeConstantData::Generated(
370                self.lower_ctx.get_immediate_data(imm).clone(),
371            )))
372        }
373
374        #[inline]
375        fn vec_mask_from_immediate(&mut self, imm: Immediate) -> Option<VecMask> {
376            let data = self.lower_ctx.get_immediate_data(imm);
377            if data.len() == 16 {
378                Some(Vec::from(data.as_slice()))
379            } else {
380                None
381            }
382        }
383
384        #[inline]
385        fn u64_from_constant(&mut self, constant: Constant) -> Option<u64> {
386            let bytes = self.lower_ctx.get_constant_data(constant).as_slice();
387            Some(u64::from_le_bytes(bytes.try_into().ok()?))
388        }
389
390        #[inline]
391        fn u128_from_constant(&mut self, constant: Constant) -> Option<u128> {
392            let bytes = self.lower_ctx.get_constant_data(constant).as_slice();
393            Some(u128::from_le_bytes(bytes.try_into().ok()?))
394        }
395
396        #[inline]
397        fn emit_u64_le_const(&mut self, value: u64) -> VCodeConstant {
398            let data = VCodeConstantData::U64(value.to_le_bytes());
399            self.lower_ctx.use_constant(data)
400        }
401
402        #[inline]
403        fn emit_u64_be_const(&mut self, value: u64) -> VCodeConstant {
404            let data = VCodeConstantData::U64(value.to_be_bytes());
405            self.lower_ctx.use_constant(data)
406        }
407
408        #[inline]
409        fn emit_u128_le_const(&mut self, value: u128) -> VCodeConstant {
410            let data = VCodeConstantData::Generated(value.to_le_bytes().as_slice().into());
411            self.lower_ctx.use_constant(data)
412        }
413
414        #[inline]
415        fn emit_u128_be_const(&mut self, value: u128) -> VCodeConstant {
416            let data = VCodeConstantData::Generated(value.to_be_bytes().as_slice().into());
417            self.lower_ctx.use_constant(data)
418        }
419
420        #[inline]
421        fn const_to_vconst(&mut self, constant: Constant) -> VCodeConstant {
422            self.lower_ctx.use_constant(VCodeConstantData::Pool(
423                constant,
424                self.lower_ctx.get_constant_data(constant).clone(),
425            ))
426        }
427
428        fn only_writable_reg(&mut self, regs: WritableValueRegs) -> Option<WritableReg> {
429            regs.only_reg()
430        }
431
432        fn writable_regs_get(&mut self, regs: WritableValueRegs, idx: usize) -> WritableReg {
433            regs.regs()[idx]
434        }
435
436        fn abi_sig(&mut self, sig_ref: SigRef) -> Sig {
437            self.lower_ctx.sigs().abi_sig_for_sig_ref(sig_ref)
438        }
439
440        fn abi_num_args(&mut self, abi: Sig) -> usize {
441            self.lower_ctx.sigs().num_args(abi)
442        }
443
444        fn abi_get_arg(&mut self, abi: Sig, idx: usize) -> ABIArg {
445            self.lower_ctx.sigs().get_arg(abi, idx)
446        }
447
448        fn abi_num_rets(&mut self, abi: Sig) -> usize {
449            self.lower_ctx.sigs().num_rets(abi)
450        }
451
452        fn abi_get_ret(&mut self, abi: Sig, idx: usize) -> ABIArg {
453            self.lower_ctx.sigs().get_ret(abi, idx)
454        }
455
456        fn abi_ret_arg(&mut self, abi: Sig) -> Option<ABIArg> {
457            self.lower_ctx.sigs().get_ret_arg(abi)
458        }
459
460        fn abi_no_ret_arg(&mut self, abi: Sig) -> Option<()> {
461            if let Some(_) = self.lower_ctx.sigs().get_ret_arg(abi) {
462                None
463            } else {
464                Some(())
465            }
466        }
467
468        fn abi_arg_only_slot(&mut self, arg: &ABIArg) -> Option<ABIArgSlot> {
469            match arg {
470                &ABIArg::Slots { ref slots, .. } => {
471                    if slots.len() == 1 {
472                        Some(slots[0])
473                    } else {
474                        None
475                    }
476                }
477                _ => None,
478            }
479        }
480
481        fn abi_arg_implicit_pointer(&mut self, arg: &ABIArg) -> Option<(ABIArgSlot, i64, Type)> {
482            match arg {
483                &ABIArg::ImplicitPtrArg {
484                    pointer,
485                    offset,
486                    ty,
487                    ..
488                } => Some((pointer, offset, ty)),
489                _ => None,
490            }
491        }
492
493        fn abi_unwrap_ret_area_ptr(&mut self) -> Reg {
494            self.lower_ctx.abi().ret_area_ptr().unwrap()
495        }
496
497        fn abi_stackslot_addr(
498            &mut self,
499            dst: WritableReg,
500            stack_slot: StackSlot,
501            offset: Offset32,
502        ) -> MInst {
503            let offset = u32::try_from(i32::from(offset)).unwrap();
504            self.lower_ctx
505                .abi()
506                .sized_stackslot_addr(stack_slot, offset, dst)
507                .into()
508        }
509
510        fn abi_stackslot_offset_into_slot_region(
511            &mut self,
512            stack_slot: StackSlot,
513            offset1: Offset32,
514            offset2: Offset32,
515        ) -> i32 {
516            let offset1 = i32::from(offset1);
517            let offset2 = i32::from(offset2);
518            i32::try_from(self.lower_ctx.abi().sized_stackslot_offset(stack_slot))
519                .expect("Stack slot region cannot be larger than 2GiB")
520                .checked_add(offset1)
521                .expect("Stack slot region cannot be larger than 2GiB")
522                .checked_add(offset2)
523                .expect("Stack slot region cannot be larger than 2GiB")
524        }
525
526        fn abi_dynamic_stackslot_addr(
527            &mut self,
528            dst: WritableReg,
529            stack_slot: DynamicStackSlot,
530        ) -> MInst {
531            assert!(
532                self.lower_ctx
533                    .abi()
534                    .dynamic_stackslot_offsets()
535                    .is_valid(stack_slot)
536            );
537            self.lower_ctx
538                .abi()
539                .dynamic_stackslot_addr(stack_slot, dst)
540                .into()
541        }
542
543        fn real_reg_to_reg(&mut self, reg: RealReg) -> Reg {
544            Reg::from(reg)
545        }
546
547        fn real_reg_to_writable_reg(&mut self, reg: RealReg) -> WritableReg {
548            Writable::from_reg(Reg::from(reg))
549        }
550
551        fn is_sinkable_inst(&mut self, val: Value) -> Option<Inst> {
552            let input = self.lower_ctx.get_value_as_source_or_const(val);
553
554            if let InputSourceInst::UniqueUse(inst, _) = input.inst {
555                Some(inst)
556            } else {
557                None
558            }
559        }
560
561        #[inline]
562        fn sink_inst(&mut self, inst: Inst) {
563            self.lower_ctx.sink_inst(inst);
564        }
565
566        #[inline]
567        fn maybe_uextend(&mut self, value: Value) -> Option<Value> {
568            if let Some(def_inst) = self.def_inst(value) {
569                if let InstructionData::Unary {
570                    opcode: Opcode::Uextend,
571                    arg,
572                } = self.lower_ctx.data(def_inst)
573                {
574                    return Some(*arg);
575                }
576            }
577
578            Some(value)
579        }
580
581        #[inline]
582        fn uimm8(&mut self, x: Imm64) -> Option<u8> {
583            let x64: i64 = x.into();
584            let x8: u8 = x64.try_into().ok()?;
585            Some(x8)
586        }
587
588        #[inline]
589        fn preg_to_reg(&mut self, preg: PReg) -> Reg {
590            preg.into()
591        }
592
593        #[inline]
594        fn gen_move(&mut self, ty: Type, dst: WritableReg, src: Reg) -> MInst {
595            <$inst>::gen_move(dst, src, ty).into()
596        }
597
598        /// Generate the return instruction.
599        fn gen_return(&mut self, rets: &ValueRegsVec) {
600            self.lower_ctx.gen_return(rets);
601        }
602
603        fn gen_call_output(&mut self, sig_ref: SigRef) -> ValueRegsVec {
604            self.lower_ctx.gen_call_output_from_sig_ref(sig_ref)
605        }
606
607        fn gen_call_args(&mut self, sig: Sig, inputs: &ValueRegsVec) -> CallArgList {
608            self.lower_ctx.gen_call_args(sig, inputs)
609        }
610
611        fn gen_return_call_args(&mut self, sig: Sig, inputs: &ValueRegsVec) -> CallArgList {
612            self.lower_ctx.gen_return_call_args(sig, inputs)
613        }
614
615        fn gen_call_rets(&mut self, sig: Sig, outputs: &ValueRegsVec) -> CallRetList {
616            self.lower_ctx.gen_call_rets(sig, &outputs)
617        }
618
619        fn gen_try_call_rets(&mut self, sig: Sig) -> CallRetList {
620            self.lower_ctx.gen_try_call_rets(sig)
621        }
622
623        fn try_call_none(&mut self) -> OptionTryCallInfo {
624            None
625        }
626
627        fn try_call_info(
628            &mut self,
629            et: ExceptionTable,
630            labels: &MachLabelSlice,
631        ) -> OptionTryCallInfo {
632            let mut exception_handlers = vec![];
633            let mut labels = labels.iter().cloned();
634            for item in self.lower_ctx.dfg().exception_tables[et].clone().items() {
635                match item {
636                    crate::ir::ExceptionTableItem::Tag(tag, _) => {
637                        exception_handlers.push(crate::machinst::abi::TryCallHandler::Tag(
638                            tag,
639                            labels.next().unwrap(),
640                        ));
641                    }
642                    crate::ir::ExceptionTableItem::Default(_) => {
643                        exception_handlers.push(crate::machinst::abi::TryCallHandler::Default(
644                            labels.next().unwrap(),
645                        ));
646                    }
647                    crate::ir::ExceptionTableItem::Context(ctx) => {
648                        let reg = self.put_in_reg(ctx);
649                        exception_handlers.push(crate::machinst::abi::TryCallHandler::Context(reg));
650                    }
651                }
652            }
653
654            let continuation = labels.next().unwrap();
655            assert_eq!(labels.next(), None);
656
657            let exception_handlers = exception_handlers.into_boxed_slice();
658
659            Some(TryCallInfo {
660                continuation,
661                exception_handlers,
662            })
663        }
664
665        /// Same as `shuffle32_from_imm`, but for 64-bit lane shuffles.
666        fn shuffle64_from_imm(&mut self, imm: Immediate) -> Option<(u8, u8)> {
667            use crate::machinst::isle::shuffle_imm_as_le_lane_idx;
668
669            let bytes = self.lower_ctx.get_immediate_data(imm).as_slice();
670            Some((
671                shuffle_imm_as_le_lane_idx(8, &bytes[0..8])?,
672                shuffle_imm_as_le_lane_idx(8, &bytes[8..16])?,
673            ))
674        }
675
676        /// Attempts to interpret the shuffle immediate `imm` as a shuffle of
677        /// 32-bit lanes, returning four integers, each of which is less than 8,
678        /// which represents a permutation of 32-bit lanes as specified by
679        /// `imm`.
680        ///
681        /// For example the shuffle immediate
682        ///
683        /// `0 1 2 3 8 9 10 11 16 17 18 19 24 25 26 27`
684        ///
685        /// would return `Some((0, 2, 4, 6))`.
686        fn shuffle32_from_imm(&mut self, imm: Immediate) -> Option<(u8, u8, u8, u8)> {
687            use crate::machinst::isle::shuffle_imm_as_le_lane_idx;
688
689            let bytes = self.lower_ctx.get_immediate_data(imm).as_slice();
690            Some((
691                shuffle_imm_as_le_lane_idx(4, &bytes[0..4])?,
692                shuffle_imm_as_le_lane_idx(4, &bytes[4..8])?,
693                shuffle_imm_as_le_lane_idx(4, &bytes[8..12])?,
694                shuffle_imm_as_le_lane_idx(4, &bytes[12..16])?,
695            ))
696        }
697
698        /// Same as `shuffle32_from_imm`, but for 16-bit lane shuffles.
699        fn shuffle16_from_imm(
700            &mut self,
701            imm: Immediate,
702        ) -> Option<(u8, u8, u8, u8, u8, u8, u8, u8)> {
703            use crate::machinst::isle::shuffle_imm_as_le_lane_idx;
704            let bytes = self.lower_ctx.get_immediate_data(imm).as_slice();
705            Some((
706                shuffle_imm_as_le_lane_idx(2, &bytes[0..2])?,
707                shuffle_imm_as_le_lane_idx(2, &bytes[2..4])?,
708                shuffle_imm_as_le_lane_idx(2, &bytes[4..6])?,
709                shuffle_imm_as_le_lane_idx(2, &bytes[6..8])?,
710                shuffle_imm_as_le_lane_idx(2, &bytes[8..10])?,
711                shuffle_imm_as_le_lane_idx(2, &bytes[10..12])?,
712                shuffle_imm_as_le_lane_idx(2, &bytes[12..14])?,
713                shuffle_imm_as_le_lane_idx(2, &bytes[14..16])?,
714            ))
715        }
716
717        fn safe_divisor_from_imm64(&mut self, ty: Type, val: Imm64) -> Option<u64> {
718            let minus_one = if ty.bytes() == 8 {
719                -1
720            } else {
721                (1 << (ty.bytes() * 8)) - 1
722            };
723            let bits = val.bits() & minus_one;
724            if bits == 0 || bits == minus_one {
725                None
726            } else {
727                Some(bits as u64)
728            }
729        }
730
731        fn single_target(&mut self, targets: &MachLabelSlice) -> Option<MachLabel> {
732            if targets.len() == 1 {
733                Some(targets[0])
734            } else {
735                None
736            }
737        }
738
739        fn two_targets(&mut self, targets: &MachLabelSlice) -> Option<(MachLabel, MachLabel)> {
740            if targets.len() == 2 {
741                Some((targets[0], targets[1]))
742            } else {
743                None
744            }
745        }
746
747        fn jump_table_targets(
748            &mut self,
749            targets: &MachLabelSlice,
750        ) -> Option<(MachLabel, BoxVecMachLabel)> {
751            use std::boxed::Box;
752            if targets.is_empty() {
753                return None;
754            }
755
756            let default_label = targets[0];
757            let jt_targets = Box::new(targets[1..].to_vec());
758            Some((default_label, jt_targets))
759        }
760
761        fn jump_table_size(&mut self, targets: &BoxVecMachLabel) -> u32 {
762            targets.len() as u32
763        }
764
765        fn add_range_fact(&mut self, reg: Reg, bits: u16, min: u64, max: u64) -> Reg {
766            self.lower_ctx.add_range_fact(reg, bits, min, max);
767            reg
768        }
769
770        fn value_is_unused(&mut self, val: Value) -> bool {
771            self.lower_ctx.value_is_unused(val)
772        }
773
774        fn block_exn_successor_label(&mut self, block: &Block, exn_succ: u64) -> MachLabel {
775            // The first N successors are the exceptional edges, and
776            // the normal return is last; so the `exn_succ`'th
777            // exceptional edge is just the `exn_succ`'th edge overall.
778            let succ = usize::try_from(exn_succ).unwrap();
779            self.lower_ctx.block_successor_label(*block, succ)
780        }
781    };
782}
783
784/// Returns the `size`-byte lane referred to by the shuffle immediate specified
785/// in `bytes`.
786///
787/// This helper is used by `shuffleNN_from_imm` above and is used to interpret a
788/// byte-based shuffle as a higher-level shuffle of bigger lanes. This will see
789/// if the `bytes` specified, which must have `size` length, specifies a lane in
790/// vectors aligned to a `size`-byte boundary.
791///
792/// Returns `None` if `bytes` doesn't specify a `size`-byte lane aligned
793/// appropriately, or returns `Some(n)` where `n` is the index of the lane being
794/// shuffled.
795pub fn shuffle_imm_as_le_lane_idx(size: u8, bytes: &[u8]) -> Option<u8> {
796    assert_eq!(bytes.len(), usize::from(size));
797
798    // The first index in `bytes` must be aligned to a `size` boundary for the
799    // bytes to be a valid specifier for a lane of `size` bytes.
800    if bytes[0] % size != 0 {
801        return None;
802    }
803
804    // Afterwards the bytes must all be one larger than the prior to specify a
805    // contiguous sequence of bytes that's being shuffled. Basically `bytes`
806    // must refer to the entire `size`-byte lane, in little-endian order.
807    for i in 0..size - 1 {
808        let idx = usize::from(i);
809        if bytes[idx] + 1 != bytes[idx + 1] {
810            return None;
811        }
812    }
813
814    // All of the `bytes` are in-order, meaning that this is a valid shuffle
815    // immediate to specify a lane of `size` bytes. The index, when viewed as
816    // `size`-byte immediates, will be the first byte divided by the byte size.
817    Some(bytes[0] / size)
818}
819
820/// This structure is used to implement the ISLE-generated `Context` trait and
821/// internally has a temporary reference to a machinst `LowerCtx`.
822pub(crate) struct IsleContext<'a, 'b, I, B>
823where
824    I: VCodeInst,
825    B: LowerBackend,
826{
827    pub lower_ctx: &'a mut Lower<'b, I>,
828    pub backend: &'a B,
829}
830
831impl<I, B> IsleContext<'_, '_, I, B>
832where
833    I: VCodeInst,
834    B: LowerBackend,
835{
836    pub(crate) fn dfg(&self) -> &crate::ir::DataFlowGraph {
837        &self.lower_ctx.f.dfg
838    }
839}
cranelift_codegen/machinst/isle.rs

cranelift_codegen/machinst/
isle.rs
