cranelift_codegen/isa/x64/inst/
args.rs

1//! Instruction operand sub-components (aka "parts"): definitions and printing.
2
3use super::regs::{self};
4use super::EmitState;
5use crate::ir::condcodes::{FloatCC, IntCC};
6use crate::ir::types::*;
7use crate::ir::MemFlags;
8use crate::isa::x64::inst::regs::pretty_print_reg;
9use crate::isa::x64::inst::Inst;
10use crate::machinst::*;
11use smallvec::{smallvec, SmallVec};
12use std::fmt;
13use std::string::String;
14
15pub use crate::isa::x64::lower::isle::generated_code::DivSignedness;
16
17/// An extension trait for converting `Writable{Xmm,Gpr}` to `Writable<Reg>`.
18pub trait ToWritableReg {
19    /// Convert `Writable{Xmm,Gpr}` to `Writable<Reg>`.
20    fn to_writable_reg(&self) -> Writable<Reg>;
21}
22
23/// An extension trait for converting `Writable<Reg>` to `Writable{Xmm,Gpr}`.
24pub trait FromWritableReg: Sized {
25    /// Convert `Writable<Reg>` to `Writable{Xmm,Gpr}`.
26    fn from_writable_reg(w: Writable<Reg>) -> Option<Self>;
27}
28
29/// A macro for defining a newtype of `Reg` that enforces some invariant about
30/// the wrapped `Reg` (such as that it is of a particular register class).
31macro_rules! newtype_of_reg {
32    (
33        $newtype_reg:ident,
34        $newtype_writable_reg:ident,
35        $newtype_option_writable_reg:ident,
36        reg_mem: ($($newtype_reg_mem:ident $(aligned:$aligned:ident)?),*),
37        reg_mem_imm: ($($newtype_reg_mem_imm:ident $(aligned:$aligned_imm:ident)?),*),
38        $newtype_imm8_reg:ident,
39        |$check_reg:ident| $check:expr
40    ) => {
41        /// A newtype wrapper around `Reg`.
42        #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
43        pub struct $newtype_reg(Reg);
44
45        impl PartialEq<Reg> for $newtype_reg {
46            fn eq(&self, other: &Reg) -> bool {
47                self.0 == *other
48            }
49        }
50
51        impl From<$newtype_reg> for Reg {
52            fn from(r: $newtype_reg) -> Self {
53                r.0
54            }
55        }
56
57        impl $newtype_reg {
58            /// Create this newtype from the given register, or return `None` if the register
59            /// is not a valid instance of this newtype.
60            pub fn new($check_reg: Reg) -> Option<Self> {
61                if $check {
62                    Some(Self($check_reg))
63                } else {
64                    None
65                }
66            }
67
68            /// Like `Self::new(r).unwrap()` but with a better panic message on
69            /// failure.
70            pub fn unwrap_new($check_reg: Reg) -> Self {
71                if $check {
72                    Self($check_reg)
73                } else {
74                    panic!(
75                        "cannot construct {} from register {:?} with register class {:?}",
76                        stringify!($newtype_reg),
77                        $check_reg,
78                        $check_reg.class(),
79                    )
80                }
81            }
82
83            /// Get this newtype's underlying `Reg`.
84            pub fn to_reg(self) -> Reg {
85                self.0
86            }
87        }
88
89        // Convenience impl so that people working with this newtype can use it
90        // "just like" a plain `Reg`.
91        //
92        // NB: We cannot implement `DerefMut` because that would let people do
93        // nasty stuff like `*my_gpr.deref_mut() = some_xmm_reg`, breaking the
94        // invariants that `Gpr` provides.
95        impl std::ops::Deref for $newtype_reg {
96            type Target = Reg;
97
98            fn deref(&self) -> &Reg {
99                &self.0
100            }
101        }
102
103        /// If you know what you're doing, you can explicitly mutably borrow the
104        /// underlying `Reg`. Don't make it point to the wrong type of register
105        /// please.
106        impl AsMut<Reg> for $newtype_reg {
107            fn as_mut(&mut self) -> &mut Reg {
108                &mut self.0
109            }
110        }
111
112        /// Writable Gpr.
113        pub type $newtype_writable_reg = Writable<$newtype_reg>;
114
115        #[allow(dead_code)] // Used by some newtypes and not others.
116        /// Optional writable Gpr.
117        pub type $newtype_option_writable_reg = Option<Writable<$newtype_reg>>;
118
119        impl ToWritableReg for $newtype_writable_reg {
120            fn to_writable_reg(&self) -> Writable<Reg> {
121                Writable::from_reg(self.to_reg().to_reg())
122            }
123        }
124
125        impl FromWritableReg for $newtype_writable_reg {
126            fn from_writable_reg(w: Writable<Reg>) -> Option<Self> {
127                Some(Writable::from_reg($newtype_reg::new(w.to_reg())?))
128            }
129        }
130
131        $(
132            /// A newtype wrapper around `RegMem` for general-purpose registers.
133            #[derive(Clone, Debug)]
134            pub struct $newtype_reg_mem(RegMem);
135
136            impl From<$newtype_reg_mem> for RegMem {
137                fn from(rm: $newtype_reg_mem) -> Self {
138                    rm.0
139                }
140            }
141            impl<'a> From<&'a $newtype_reg_mem> for &'a RegMem {
142                fn from(rm: &'a $newtype_reg_mem) -> &'a RegMem {
143                    &rm.0
144                }
145            }
146
147            impl From<$newtype_reg> for $newtype_reg_mem {
148                fn from(r: $newtype_reg) -> Self {
149                    $newtype_reg_mem(RegMem::reg(r.into()))
150                }
151            }
152
153            impl $newtype_reg_mem {
154                /// Construct a `RegMem` newtype from the given `RegMem`, or return
155                /// `None` if the `RegMem` is not a valid instance of this `RegMem`
156                /// newtype.
157                pub fn new(rm: RegMem) -> Option<Self> {
158                    match rm {
159                        RegMem::Mem { addr } => {
160                            let mut _allow = true;
161                            $(
162                                if $aligned {
163                                    _allow = addr.aligned();
164                                }
165                            )?
166                            if _allow {
167                                Some(Self(RegMem::Mem { addr }))
168                            } else {
169                                None
170                            }
171                        }
172                        RegMem::Reg { reg } => Some($newtype_reg::new(reg)?.into()),
173                    }
174                }
175
176                /// Like `Self::new(rm).unwrap()` but with better panic messages
177                /// in case of failure.
178                pub fn unwrap_new(rm: RegMem) -> Self {
179                    match rm {
180                        RegMem::Mem { addr } => {
181                            $(
182                                if $aligned && !addr.aligned() {
183                                    panic!(
184                                        "cannot create {} from an unaligned memory address: {addr:?}",
185                                        stringify!($newtype_reg_mem),
186                                    );
187                                }
188                            )?
189                            Self(RegMem::Mem { addr })
190                        }
191                        RegMem::Reg { reg } => $newtype_reg::unwrap_new(reg).into(),
192                    }
193                }
194
195                /// Convert this newtype into its underlying `RegMem`.
196                pub fn to_reg_mem(self) -> RegMem {
197                    self.0
198                }
199
200                #[allow(dead_code)] // Used by some newtypes and not others.
201                pub(crate) fn get_operands(&mut self, collector: &mut impl OperandVisitor) {
202                    self.0.get_operands(collector);
203                }
204            }
205            impl PrettyPrint for $newtype_reg_mem {
206                fn pretty_print(&self, size: u8) -> String {
207                    self.0.pretty_print(size)
208                }
209            }
210        )*
211
212        $(
213            /// A newtype wrapper around `RegMemImm`.
214            #[derive(Clone, Debug)]
215            pub struct $newtype_reg_mem_imm(RegMemImm);
216
217            impl From<$newtype_reg_mem_imm> for RegMemImm {
218                fn from(rmi: $newtype_reg_mem_imm) -> RegMemImm {
219                    rmi.0
220                }
221            }
222            impl<'a> From<&'a $newtype_reg_mem_imm> for &'a RegMemImm {
223                fn from(rmi: &'a $newtype_reg_mem_imm) -> &'a RegMemImm {
224                    &rmi.0
225                }
226            }
227
228            impl From<$newtype_reg> for $newtype_reg_mem_imm {
229                fn from(r: $newtype_reg) -> Self {
230                    $newtype_reg_mem_imm(RegMemImm::reg(r.into()))
231                }
232            }
233
234            impl $newtype_reg_mem_imm {
235                /// Construct this newtype from the given `RegMemImm`, or return
236                /// `None` if the `RegMemImm` is not a valid instance of this
237                /// newtype.
238                pub fn new(rmi: RegMemImm) -> Option<Self> {
239                    match rmi {
240                        RegMemImm::Imm { .. } => Some(Self(rmi)),
241                        RegMemImm::Mem { addr } => {
242                            let mut _allow = true;
243                            $(
244                                if $aligned_imm {
245                                    _allow = addr.aligned();
246                                }
247                            )?
248                            if _allow {
249                                Some(Self(RegMemImm::Mem { addr }))
250                            } else {
251                                None
252                            }
253                        }
254                        RegMemImm::Reg { reg } => Some($newtype_reg::new(reg)?.into()),
255                    }
256                }
257
258                /// Like `Self::new(rmi).unwrap()` but with better panic
259                /// messages in case of failure.
260                pub fn unwrap_new(rmi: RegMemImm) -> Self {
261                    match rmi {
262                        RegMemImm::Imm { .. } => Self(rmi),
263                        RegMemImm::Mem { addr } => {
264                            $(
265                                if $aligned_imm && !addr.aligned() {
266                                    panic!(
267                                        "cannot construct {} from unaligned memory address: {:?}",
268                                        stringify!($newtype_reg_mem_imm),
269                                        addr,
270                                    );
271                                }
272                            )?
273                            Self(RegMemImm::Mem { addr })
274
275                        }
276                        RegMemImm::Reg { reg } => $newtype_reg::unwrap_new(reg).into(),
277                    }
278                }
279
280                /// Convert this newtype into its underlying `RegMemImm`.
281                #[allow(dead_code)] // Used by some newtypes and not others.
282                pub fn to_reg_mem_imm(self) -> RegMemImm {
283                    self.0
284                }
285
286                #[allow(dead_code)] // Used by some newtypes and not others.
287                pub(crate) fn get_operands(&mut self, collector: &mut impl OperandVisitor) {
288                    self.0.get_operands(collector);
289                }
290            }
291
292            impl PrettyPrint for $newtype_reg_mem_imm {
293                fn pretty_print(&self, size: u8) -> String {
294                    self.0.pretty_print(size)
295                }
296            }
297        )*
298
299        /// A newtype wrapper around `Imm8Reg`.
300        #[derive(Clone, Debug)]
301        #[allow(dead_code)] // Used by some newtypes and not others.
302        pub struct $newtype_imm8_reg(Imm8Reg);
303
304        impl From<$newtype_reg> for $newtype_imm8_reg {
305            fn from(r: $newtype_reg) -> Self {
306                Self(Imm8Reg::Reg { reg: r.to_reg() })
307            }
308        }
309
310        impl $newtype_imm8_reg {
311            /// Construct this newtype from the given `Imm8Reg`, or return
312            /// `None` if the `Imm8Reg` is not a valid instance of this newtype.
313            #[allow(dead_code)] // Used by some newtypes and not others.
314            pub fn new(imm8_reg: Imm8Reg) -> Option<Self> {
315                match imm8_reg {
316                    Imm8Reg::Imm8 { .. } => Some(Self(imm8_reg)),
317                    Imm8Reg::Reg { reg } => Some($newtype_reg::new(reg)?.into()),
318                }
319            }
320
321            /// Like `Self::new(imm8_reg).unwrap()` but with better panic
322            /// messages on failure.
323            pub fn unwrap_new(imm8_reg: Imm8Reg) -> Self {
324                match imm8_reg {
325                    Imm8Reg::Imm8 { .. } => Self(imm8_reg),
326                    Imm8Reg::Reg { reg } => $newtype_reg::unwrap_new(reg).into(),
327                }
328            }
329
330            /// Borrow this newtype as its underlying `Imm8Reg`.
331            #[allow(dead_code)] // Used by some newtypes and not others.
332            pub fn as_imm8_reg(&self) -> &Imm8Reg {
333                &self.0
334            }
335
336            /// Borrow this newtype as its underlying `Imm8Reg`.
337            #[allow(dead_code)] // Used by some newtypes and not others.
338            pub fn as_imm8_reg_mut(&mut self) -> &mut Imm8Reg {
339                &mut self.0
340            }
341        }
342    };
343}
344
345// Define a newtype of `Reg` for general-purpose registers.
346newtype_of_reg!(
347    Gpr,
348    WritableGpr,
349    OptionWritableGpr,
350    reg_mem: (GprMem),
351    reg_mem_imm: (GprMemImm),
352    Imm8Gpr,
353    |reg| reg.class() == RegClass::Int
354);
355
356// Define a newtype of `Reg` for XMM registers.
357newtype_of_reg!(
358    Xmm,
359    WritableXmm,
360    OptionWritableXmm,
361    reg_mem: (XmmMem, XmmMemAligned aligned:true),
362    reg_mem_imm: (XmmMemImm, XmmMemAlignedImm aligned:true),
363    Imm8Xmm,
364    |reg| reg.class() == RegClass::Float
365);
366
367// N.B.: `Amode` is defined in `inst.isle`. We add some convenience
368// constructors here.
369
370// Re-export the type from the ISLE generated code.
371pub use crate::isa::x64::lower::isle::generated_code::Amode;
372
373impl Amode {
374    /// Create an immediate sign-extended and register addressing mode.
375    pub fn imm_reg(simm32: i32, base: Reg) -> Self {
376        debug_assert!(base.class() == RegClass::Int);
377        Self::ImmReg {
378            simm32,
379            base,
380            flags: MemFlags::trusted(),
381        }
382    }
383
384    /// Create a sign-extended-32-to-64 with register and shift addressing mode.
385    pub fn imm_reg_reg_shift(simm32: i32, base: Gpr, index: Gpr, shift: u8) -> Self {
386        debug_assert!(base.class() == RegClass::Int);
387        debug_assert!(index.class() == RegClass::Int);
388        debug_assert!(shift <= 3);
389        Self::ImmRegRegShift {
390            simm32,
391            base,
392            index,
393            shift,
394            flags: MemFlags::trusted(),
395        }
396    }
397
398    pub(crate) fn rip_relative(target: MachLabel) -> Self {
399        Self::RipRelative { target }
400    }
401
402    /// Set the specified [MemFlags] to the [Amode].
403    pub fn with_flags(&self, flags: MemFlags) -> Self {
404        match self {
405            &Self::ImmReg { simm32, base, .. } => Self::ImmReg {
406                simm32,
407                base,
408                flags,
409            },
410            &Self::ImmRegRegShift {
411                simm32,
412                base,
413                index,
414                shift,
415                ..
416            } => Self::ImmRegRegShift {
417                simm32,
418                base,
419                index,
420                shift,
421                flags,
422            },
423            _ => panic!("Amode {self:?} cannot take memflags"),
424        }
425    }
426
427    /// Add the registers mentioned by `self` to `collector`.
428    pub(crate) fn get_operands(&mut self, collector: &mut impl OperandVisitor) {
429        match self {
430            Amode::ImmReg { base, .. } => {
431                if *base != regs::rbp() && *base != regs::rsp() {
432                    collector.reg_use(base);
433                }
434            }
435            Amode::ImmRegRegShift { base, index, .. } => {
436                debug_assert_ne!(base.to_reg(), regs::rbp());
437                debug_assert_ne!(base.to_reg(), regs::rsp());
438                collector.reg_use(base);
439                debug_assert_ne!(index.to_reg(), regs::rbp());
440                debug_assert_ne!(index.to_reg(), regs::rsp());
441                collector.reg_use(index);
442            }
443            Amode::RipRelative { .. } => {
444                // RIP isn't involved in regalloc.
445            }
446        }
447    }
448
449    /// Same as `get_operands`, but add the registers in the "late" phase.
450    pub(crate) fn get_operands_late(&mut self, collector: &mut impl OperandVisitor) {
451        match self {
452            Amode::ImmReg { base, .. } => {
453                collector.reg_late_use(base);
454            }
455            Amode::ImmRegRegShift { base, index, .. } => {
456                collector.reg_late_use(base);
457                collector.reg_late_use(index);
458            }
459            Amode::RipRelative { .. } => {
460                // RIP isn't involved in regalloc.
461            }
462        }
463    }
464
465    pub(crate) fn get_flags(&self) -> MemFlags {
466        match self {
467            Amode::ImmReg { flags, .. } | Amode::ImmRegRegShift { flags, .. } => *flags,
468            Amode::RipRelative { .. } => MemFlags::trusted(),
469        }
470    }
471
472    /// Offset the amode by a fixed offset.
473    pub(crate) fn offset(&self, offset: i32) -> Self {
474        let mut ret = self.clone();
475        match &mut ret {
476            &mut Amode::ImmReg { ref mut simm32, .. } => *simm32 += offset,
477            &mut Amode::ImmRegRegShift { ref mut simm32, .. } => *simm32 += offset,
478            _ => panic!("Cannot offset amode: {self:?}"),
479        }
480        ret
481    }
482
483    pub(crate) fn aligned(&self) -> bool {
484        self.get_flags().aligned()
485    }
486}
487
488impl PrettyPrint for Amode {
489    fn pretty_print(&self, _size: u8) -> String {
490        match self {
491            Amode::ImmReg { simm32, base, .. } => {
492                // Note: size is always 8; the address is 64 bits,
493                // even if the addressed operand is smaller.
494                format!("{}({})", *simm32, pretty_print_reg(*base, 8))
495            }
496            Amode::ImmRegRegShift {
497                simm32,
498                base,
499                index,
500                shift,
501                ..
502            } => format!(
503                "{}({},{},{})",
504                *simm32,
505                pretty_print_reg(base.to_reg(), 8),
506                pretty_print_reg(index.to_reg(), 8),
507                1 << shift
508            ),
509            Amode::RipRelative { ref target } => format!("label{}(%rip)", target.get()),
510        }
511    }
512}
513
514/// A Memory Address. These denote a 64-bit value only.
515/// Used for usual addressing modes as well as addressing modes used during compilation, when the
516/// moving SP offset is not known.
517#[derive(Clone, Debug)]
518pub enum SyntheticAmode {
519    /// A real amode.
520    Real(Amode),
521
522    /// A (virtual) offset into the incoming argument area.
523    IncomingArg {
524        /// The downward offset from the start of the incoming argument area.
525        offset: u32,
526    },
527
528    /// A (virtual) offset to the slot area of the function frame, which lies just above the
529    /// outgoing arguments.
530    SlotOffset {
531        /// The offset into the slot area.
532        simm32: i32,
533    },
534
535    /// A virtual offset to a constant that will be emitted in the constant section of the buffer.
536    ConstantOffset(VCodeConstant),
537}
538
539impl SyntheticAmode {
540    /// Create a real addressing mode.
541    pub fn real(amode: Amode) -> Self {
542        Self::Real(amode)
543    }
544
545    pub(crate) fn slot_offset(simm32: i32) -> Self {
546        SyntheticAmode::SlotOffset { simm32 }
547    }
548
549    /// Add the registers mentioned by `self` to `collector`.
550    pub(crate) fn get_operands(&mut self, collector: &mut impl OperandVisitor) {
551        match self {
552            SyntheticAmode::Real(addr) => addr.get_operands(collector),
553            SyntheticAmode::IncomingArg { .. } => {
554                // Nothing to do; the base is known and isn't involved in regalloc.
555            }
556            SyntheticAmode::SlotOffset { .. } => {
557                // Nothing to do; the base is SP and isn't involved in regalloc.
558            }
559            SyntheticAmode::ConstantOffset(_) => {}
560        }
561    }
562
563    /// Same as `get_operands`, but add the register in the "late" phase.
564    pub(crate) fn get_operands_late(&mut self, collector: &mut impl OperandVisitor) {
565        match self {
566            SyntheticAmode::Real(addr) => addr.get_operands_late(collector),
567            SyntheticAmode::IncomingArg { .. } => {
568                // Nothing to do; the base is known and isn't involved in regalloc.
569            }
570            SyntheticAmode::SlotOffset { .. } => {
571                // Nothing to do; the base is SP and isn't involved in regalloc.
572            }
573            SyntheticAmode::ConstantOffset(_) => {}
574        }
575    }
576
577    pub(crate) fn finalize(&self, state: &mut EmitState, buffer: &mut MachBuffer<Inst>) -> Amode {
578        match self {
579            SyntheticAmode::Real(addr) => addr.clone(),
580            SyntheticAmode::IncomingArg { offset } => {
581                // NOTE: this could be made relative to RSP by adding additional offsets from the
582                // frame_layout.
583                let args_max_fp_offset =
584                    state.frame_layout().tail_args_size + state.frame_layout().setup_area_size;
585                Amode::imm_reg(
586                    i32::try_from(args_max_fp_offset - offset).unwrap(),
587                    regs::rbp(),
588                )
589            }
590            SyntheticAmode::SlotOffset { simm32 } => {
591                let off = *simm32 as i64 + i64::from(state.frame_layout().outgoing_args_size);
592                Amode::imm_reg(off.try_into().expect("invalid sp offset"), regs::rsp())
593            }
594            SyntheticAmode::ConstantOffset(c) => {
595                Amode::rip_relative(buffer.get_label_for_constant(*c))
596            }
597        }
598    }
599
600    pub(crate) fn aligned(&self) -> bool {
601        match self {
602            SyntheticAmode::Real(addr) => addr.aligned(),
603            &SyntheticAmode::IncomingArg { .. }
604            | SyntheticAmode::SlotOffset { .. }
605            | SyntheticAmode::ConstantOffset { .. } => true,
606        }
607    }
608}
609
610impl Into<SyntheticAmode> for Amode {
611    fn into(self) -> SyntheticAmode {
612        SyntheticAmode::Real(self)
613    }
614}
615
616impl Into<SyntheticAmode> for VCodeConstant {
617    fn into(self) -> SyntheticAmode {
618        SyntheticAmode::ConstantOffset(self)
619    }
620}
621
622impl PrettyPrint for SyntheticAmode {
623    fn pretty_print(&self, _size: u8) -> String {
624        match self {
625            // See note in `Amode` regarding constant size of `8`.
626            SyntheticAmode::Real(addr) => addr.pretty_print(8),
627            &SyntheticAmode::IncomingArg { offset } => {
628                format!("rbp(stack args max - {offset})")
629            }
630            SyntheticAmode::SlotOffset { simm32 } => {
631                format!("rsp({} + virtual offset)", *simm32)
632            }
633            SyntheticAmode::ConstantOffset(c) => format!("const({})", c.as_u32()),
634        }
635    }
636}
637
638/// An operand which is either an integer Register, a value in Memory or an Immediate.  This can
639/// denote an 8, 16, 32 or 64 bit value.  For the Immediate form, in the 8- and 16-bit case, only
640/// the lower 8 or 16 bits of `simm32` is relevant.  In the 64-bit case, the value denoted by
641/// `simm32` is its sign-extension out to 64 bits.
642#[derive(Clone, Debug)]
643pub enum RegMemImm {
644    /// A register operand.
645    Reg {
646        /// The underlying register.
647        reg: Reg,
648    },
649    /// A memory operand.
650    Mem {
651        /// The memory address.
652        addr: SyntheticAmode,
653    },
654    /// An immediate operand.
655    Imm {
656        /// The immediate value.
657        simm32: u32,
658    },
659}
660
661impl RegMemImm {
662    /// Create a register operand.
663    pub fn reg(reg: Reg) -> Self {
664        debug_assert!(reg.class() == RegClass::Int || reg.class() == RegClass::Float);
665        Self::Reg { reg }
666    }
667
668    /// Create a memory operand.
669    pub fn mem(addr: impl Into<SyntheticAmode>) -> Self {
670        Self::Mem { addr: addr.into() }
671    }
672
673    /// Create an immediate operand.
674    pub fn imm(simm32: u32) -> Self {
675        Self::Imm { simm32 }
676    }
677
678    /// Asserts that in register mode, the reg class is the one that's expected.
679    pub(crate) fn assert_regclass_is(&self, expected_reg_class: RegClass) {
680        if let Self::Reg { reg } = self {
681            debug_assert_eq!(reg.class(), expected_reg_class);
682        }
683    }
684
685    /// Add the regs mentioned by `self` to `collector`.
686    pub(crate) fn get_operands(&mut self, collector: &mut impl OperandVisitor) {
687        match self {
688            Self::Reg { reg } => collector.reg_use(reg),
689            Self::Mem { addr } => addr.get_operands(collector),
690            Self::Imm { .. } => {}
691        }
692    }
693}
694
695impl From<RegMem> for RegMemImm {
696    fn from(rm: RegMem) -> RegMemImm {
697        match rm {
698            RegMem::Reg { reg } => RegMemImm::Reg { reg },
699            RegMem::Mem { addr } => RegMemImm::Mem { addr },
700        }
701    }
702}
703
704impl From<Reg> for RegMemImm {
705    fn from(reg: Reg) -> Self {
706        RegMemImm::Reg { reg }
707    }
708}
709
710impl PrettyPrint for RegMemImm {
711    fn pretty_print(&self, size: u8) -> String {
712        match self {
713            Self::Reg { reg } => pretty_print_reg(*reg, size),
714            Self::Mem { addr } => addr.pretty_print(size),
715            Self::Imm { simm32 } => format!("${}", *simm32 as i32),
716        }
717    }
718}
719
720/// An operand which is either an 8-bit integer immediate or a register.
721#[derive(Clone, Debug)]
722pub enum Imm8Reg {
723    /// 8-bit immediate operand.
724    Imm8 {
725        /// The 8-bit immediate value.
726        imm: u8,
727    },
728    /// A register operand.
729    Reg {
730        /// The underlying register.
731        reg: Reg,
732    },
733}
734
735impl From<u8> for Imm8Reg {
736    fn from(imm: u8) -> Self {
737        Self::Imm8 { imm }
738    }
739}
740
741impl From<Reg> for Imm8Reg {
742    fn from(reg: Reg) -> Self {
743        Self::Reg { reg }
744    }
745}
746
747/// An operand which is either an integer Register or a value in Memory.  This can denote an 8, 16,
748/// 32, 64, or 128 bit value.
749#[derive(Clone, Debug)]
750pub enum RegMem {
751    /// A register operand.
752    Reg {
753        /// The underlying register.
754        reg: Reg,
755    },
756    /// A memory operand.
757    Mem {
758        /// The memory address.
759        addr: SyntheticAmode,
760    },
761}
762
763impl RegMem {
764    /// Create a register operand.
765    pub fn reg(reg: Reg) -> Self {
766        debug_assert!(reg.class() == RegClass::Int || reg.class() == RegClass::Float);
767        Self::Reg { reg }
768    }
769
770    /// Create a memory operand.
771    pub fn mem(addr: impl Into<SyntheticAmode>) -> Self {
772        Self::Mem { addr: addr.into() }
773    }
774    /// Asserts that in register mode, the reg class is the one that's expected.
775    pub(crate) fn assert_regclass_is(&self, expected_reg_class: RegClass) {
776        if let Self::Reg { reg } = self {
777            debug_assert_eq!(reg.class(), expected_reg_class);
778        }
779    }
780    /// Add the regs mentioned by `self` to `collector`.
781    pub(crate) fn get_operands(&mut self, collector: &mut impl OperandVisitor) {
782        match self {
783            RegMem::Reg { reg } => collector.reg_use(reg),
784            RegMem::Mem { addr, .. } => addr.get_operands(collector),
785        }
786    }
787}
788
789impl From<Reg> for RegMem {
790    fn from(reg: Reg) -> RegMem {
791        RegMem::Reg { reg }
792    }
793}
794
795impl From<Writable<Reg>> for RegMem {
796    fn from(r: Writable<Reg>) -> Self {
797        RegMem::reg(r.to_reg())
798    }
799}
800
801impl PrettyPrint for RegMem {
802    fn pretty_print(&self, size: u8) -> String {
803        match self {
804            RegMem::Reg { reg } => pretty_print_reg(*reg, size),
805            RegMem::Mem { addr, .. } => addr.pretty_print(size),
806        }
807    }
808}
809
810/// Some basic ALU operations.
811#[derive(Copy, Clone, PartialEq)]
812pub enum AluRmiROpcode {
813    /// Add operation.
814    Add,
815    /// Add with carry.
816    Adc,
817    /// Integer subtraction.
818    Sub,
819    /// Integer subtraction with borrow.
820    Sbb,
821    /// Bitwise AND operation.
822    And,
823    /// Bitwise inclusive OR.
824    Or,
825    /// Bitwise exclusive OR.
826    Xor,
827}
828
829impl fmt::Debug for AluRmiROpcode {
830    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
831        let name = match self {
832            AluRmiROpcode::Add => "add",
833            AluRmiROpcode::Adc => "adc",
834            AluRmiROpcode::Sub => "sub",
835            AluRmiROpcode::Sbb => "sbb",
836            AluRmiROpcode::And => "and",
837            AluRmiROpcode::Or => "or",
838            AluRmiROpcode::Xor => "xor",
839        };
840        write!(fmt, "{name}")
841    }
842}
843
844impl fmt::Display for AluRmiROpcode {
845    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
846        fmt::Debug::fmt(self, f)
847    }
848}
849
850pub use crate::isa::x64::lower::isle::generated_code::AluRmROpcode;
851
852impl AluRmROpcode {
853    pub(crate) fn available_from(&self) -> SmallVec<[InstructionSet; 2]> {
854        match self {
855            AluRmROpcode::Andn => smallvec![InstructionSet::BMI1],
856            AluRmROpcode::Sarx | AluRmROpcode::Shrx | AluRmROpcode::Shlx | AluRmROpcode::Bzhi => {
857                smallvec![InstructionSet::BMI2]
858            }
859        }
860    }
861}
862
863impl fmt::Display for AluRmROpcode {
864    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
865        f.write_str(&format!("{self:?}").to_lowercase())
866    }
867}
868
869#[derive(Clone, PartialEq)]
870/// Unary operations requiring register or memory and register operands.
871pub enum UnaryRmROpcode {
872    /// Bit-scan reverse.
873    Bsr,
874    /// Bit-scan forward.
875    Bsf,
876    /// Counts leading zeroes (Leading Zero CouNT).
877    Lzcnt,
878    /// Counts trailing zeroes (Trailing Zero CouNT).
879    Tzcnt,
880    /// Counts the number of ones (POPulation CouNT).
881    Popcnt,
882}
883
884impl UnaryRmROpcode {
885    pub(crate) fn available_from(&self) -> SmallVec<[InstructionSet; 2]> {
886        match self {
887            UnaryRmROpcode::Bsr | UnaryRmROpcode::Bsf => smallvec![],
888            UnaryRmROpcode::Lzcnt => smallvec![InstructionSet::Lzcnt],
889            UnaryRmROpcode::Tzcnt => smallvec![InstructionSet::BMI1],
890            UnaryRmROpcode::Popcnt => smallvec![InstructionSet::Popcnt],
891        }
892    }
893}
894
895impl fmt::Debug for UnaryRmROpcode {
896    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
897        match self {
898            UnaryRmROpcode::Bsr => write!(fmt, "bsr"),
899            UnaryRmROpcode::Bsf => write!(fmt, "bsf"),
900            UnaryRmROpcode::Lzcnt => write!(fmt, "lzcnt"),
901            UnaryRmROpcode::Tzcnt => write!(fmt, "tzcnt"),
902            UnaryRmROpcode::Popcnt => write!(fmt, "popcnt"),
903        }
904    }
905}
906
907impl fmt::Display for UnaryRmROpcode {
908    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
909        fmt::Debug::fmt(self, f)
910    }
911}
912
913pub use crate::isa::x64::lower::isle::generated_code::UnaryRmRVexOpcode;
914
915impl UnaryRmRVexOpcode {
916    pub(crate) fn available_from(&self) -> SmallVec<[InstructionSet; 2]> {
917        match self {
918            UnaryRmRVexOpcode::Blsi | UnaryRmRVexOpcode::Blsmsk | UnaryRmRVexOpcode::Blsr => {
919                smallvec![InstructionSet::BMI1]
920            }
921        }
922    }
923}
924
925impl fmt::Display for UnaryRmRVexOpcode {
926    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
927        f.write_str(&format!("{self:?}").to_lowercase())
928    }
929}
930
931pub use crate::isa::x64::lower::isle::generated_code::UnaryRmRImmVexOpcode;
932
933impl UnaryRmRImmVexOpcode {
934    pub(crate) fn available_from(&self) -> SmallVec<[InstructionSet; 2]> {
935        match self {
936            UnaryRmRImmVexOpcode::Rorx => {
937                smallvec![InstructionSet::BMI2]
938            }
939        }
940    }
941}
942
943impl fmt::Display for UnaryRmRImmVexOpcode {
944    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
945        f.write_str(&format!("{self:?}").to_lowercase())
946    }
947}
948
949#[derive(Clone, Copy, PartialEq)]
950/// Comparison operations.
951pub enum CmpOpcode {
952    /// CMP instruction: compute `a - b` and set flags from result.
953    Cmp,
954    /// TEST instruction: compute `a & b` and set flags from result.
955    Test,
956}
957
958#[derive(Debug)]
959pub(crate) enum InstructionSet {
960    SSE,
961    SSE2,
962    SSSE3,
963    SSE41,
964    SSE42,
965    Popcnt,
966    Lzcnt,
967    BMI1,
968    #[allow(dead_code)] // never constructed (yet).
969    BMI2,
970    FMA,
971    AVX,
972    AVX2,
973    AVX512BITALG,
974    AVX512DQ,
975    AVX512F,
976    AVX512VBMI,
977    AVX512VL,
978}
979
980/// Some SSE operations requiring 2 operands r/m and r.
981#[derive(Clone, Copy, PartialEq)]
982#[allow(dead_code)] // some variants here aren't used just yet
983#[allow(missing_docs)]
984pub enum SseOpcode {
985    Addps,
986    Addpd,
987    Addss,
988    Addsd,
989    Andps,
990    Andpd,
991    Andnps,
992    Andnpd,
993    Blendvpd,
994    Blendvps,
995    Comiss,
996    Comisd,
997    Cmpps,
998    Cmppd,
999    Cmpss,
1000    Cmpsd,
1001    Cvtdq2ps,
1002    Cvtdq2pd,
1003    Cvtpd2ps,
1004    Cvtps2pd,
1005    Cvtsd2ss,
1006    Cvtsd2si,
1007    Cvtsi2ss,
1008    Cvtsi2sd,
1009    Cvtss2si,
1010    Cvtss2sd,
1011    Cvttpd2dq,
1012    Cvttps2dq,
1013    Cvttss2si,
1014    Cvttsd2si,
1015    Divps,
1016    Divpd,
1017    Divss,
1018    Divsd,
1019    Insertps,
1020    Maxps,
1021    Maxpd,
1022    Maxss,
1023    Maxsd,
1024    Minps,
1025    Minpd,
1026    Minss,
1027    Minsd,
1028    Movaps,
1029    Movapd,
1030    Movd,
1031    Movdqa,
1032    Movdqu,
1033    Movlhps,
1034    Movmskps,
1035    Movmskpd,
1036    Movq,
1037    Movss,
1038    Movsd,
1039    Movups,
1040    Movupd,
1041    Mulps,
1042    Mulpd,
1043    Mulss,
1044    Mulsd,
1045    Orps,
1046    Orpd,
1047    Pabsb,
1048    Pabsw,
1049    Pabsd,
1050    Packssdw,
1051    Packsswb,
1052    Packusdw,
1053    Packuswb,
1054    Paddb,
1055    Paddd,
1056    Paddq,
1057    Paddw,
1058    Paddsb,
1059    Paddsw,
1060    Paddusb,
1061    Paddusw,
1062    Palignr,
1063    Pand,
1064    Pandn,
1065    Pavgb,
1066    Pavgw,
1067    Pblendvb,
1068    Pcmpeqb,
1069    Pcmpeqw,
1070    Pcmpeqd,
1071    Pcmpeqq,
1072    Pcmpgtb,
1073    Pcmpgtw,
1074    Pcmpgtd,
1075    Pcmpgtq,
1076    Pextrb,
1077    Pextrw,
1078    Pextrd,
1079    Pextrq,
1080    Pinsrb,
1081    Pinsrw,
1082    Pinsrd,
1083    Pmaddubsw,
1084    Pmaddwd,
1085    Pmaxsb,
1086    Pmaxsw,
1087    Pmaxsd,
1088    Pmaxub,
1089    Pmaxuw,
1090    Pmaxud,
1091    Pminsb,
1092    Pminsw,
1093    Pminsd,
1094    Pminub,
1095    Pminuw,
1096    Pminud,
1097    Pmovmskb,
1098    Pmovsxbd,
1099    Pmovsxbw,
1100    Pmovsxbq,
1101    Pmovsxwd,
1102    Pmovsxwq,
1103    Pmovsxdq,
1104    Pmovzxbd,
1105    Pmovzxbw,
1106    Pmovzxbq,
1107    Pmovzxwd,
1108    Pmovzxwq,
1109    Pmovzxdq,
1110    Pmuldq,
1111    Pmulhw,
1112    Pmulhuw,
1113    Pmulhrsw,
1114    Pmulld,
1115    Pmullw,
1116    Pmuludq,
1117    Por,
1118    Pshufb,
1119    Pshufd,
1120    Psllw,
1121    Pslld,
1122    Psllq,
1123    Psraw,
1124    Psrad,
1125    Psrlw,
1126    Psrld,
1127    Psrlq,
1128    Psubb,
1129    Psubd,
1130    Psubq,
1131    Psubw,
1132    Psubsb,
1133    Psubsw,
1134    Psubusb,
1135    Psubusw,
1136    Ptest,
1137    Punpckhbw,
1138    Punpckhwd,
1139    Punpcklbw,
1140    Punpcklwd,
1141    Pxor,
1142    Rcpss,
1143    Roundps,
1144    Roundpd,
1145    Roundss,
1146    Roundsd,
1147    Rsqrtss,
1148    Shufps,
1149    Sqrtps,
1150    Sqrtpd,
1151    Sqrtss,
1152    Sqrtsd,
1153    Subps,
1154    Subpd,
1155    Subss,
1156    Subsd,
1157    Ucomiss,
1158    Ucomisd,
1159    Unpcklps,
1160    Unpcklpd,
1161    Unpckhps,
1162    Xorps,
1163    Xorpd,
1164    Phaddw,
1165    Phaddd,
1166    Punpckhdq,
1167    Punpckldq,
1168    Punpckhqdq,
1169    Punpcklqdq,
1170    Pshuflw,
1171    Pshufhw,
1172    Pblendw,
1173    Movddup,
1174}
1175
1176impl SseOpcode {
1177    /// Which `InstructionSet` is the first supporting this opcode?
1178    pub(crate) fn available_from(&self) -> InstructionSet {
1179        use InstructionSet::*;
1180        match self {
1181            SseOpcode::Addps
1182            | SseOpcode::Addss
1183            | SseOpcode::Andps
1184            | SseOpcode::Andnps
1185            | SseOpcode::Comiss
1186            | SseOpcode::Cmpps
1187            | SseOpcode::Cmpss
1188            | SseOpcode::Cvtsi2ss
1189            | SseOpcode::Cvtss2si
1190            | SseOpcode::Cvttss2si
1191            | SseOpcode::Divps
1192            | SseOpcode::Divss
1193            | SseOpcode::Maxps
1194            | SseOpcode::Maxss
1195            | SseOpcode::Minps
1196            | SseOpcode::Minss
1197            | SseOpcode::Movaps
1198            | SseOpcode::Movlhps
1199            | SseOpcode::Movmskps
1200            | SseOpcode::Movss
1201            | SseOpcode::Movups
1202            | SseOpcode::Mulps
1203            | SseOpcode::Mulss
1204            | SseOpcode::Orps
1205            | SseOpcode::Rcpss
1206            | SseOpcode::Rsqrtss
1207            | SseOpcode::Shufps
1208            | SseOpcode::Sqrtps
1209            | SseOpcode::Sqrtss
1210            | SseOpcode::Subps
1211            | SseOpcode::Subss
1212            | SseOpcode::Ucomiss
1213            | SseOpcode::Unpcklps
1214            | SseOpcode::Unpckhps
1215            | SseOpcode::Xorps => SSE,
1216
1217            SseOpcode::Addpd
1218            | SseOpcode::Addsd
1219            | SseOpcode::Andpd
1220            | SseOpcode::Andnpd
1221            | SseOpcode::Cmppd
1222            | SseOpcode::Cmpsd
1223            | SseOpcode::Comisd
1224            | SseOpcode::Cvtdq2ps
1225            | SseOpcode::Cvtdq2pd
1226            | SseOpcode::Cvtpd2ps
1227            | SseOpcode::Cvtps2pd
1228            | SseOpcode::Cvtsd2ss
1229            | SseOpcode::Cvtsd2si
1230            | SseOpcode::Cvtsi2sd
1231            | SseOpcode::Cvtss2sd
1232            | SseOpcode::Cvttpd2dq
1233            | SseOpcode::Cvttps2dq
1234            | SseOpcode::Cvttsd2si
1235            | SseOpcode::Divpd
1236            | SseOpcode::Divsd
1237            | SseOpcode::Maxpd
1238            | SseOpcode::Maxsd
1239            | SseOpcode::Minpd
1240            | SseOpcode::Minsd
1241            | SseOpcode::Movapd
1242            | SseOpcode::Movd
1243            | SseOpcode::Movmskpd
1244            | SseOpcode::Movq
1245            | SseOpcode::Movsd
1246            | SseOpcode::Movupd
1247            | SseOpcode::Movdqa
1248            | SseOpcode::Movdqu
1249            | SseOpcode::Mulpd
1250            | SseOpcode::Mulsd
1251            | SseOpcode::Orpd
1252            | SseOpcode::Packssdw
1253            | SseOpcode::Packsswb
1254            | SseOpcode::Packuswb
1255            | SseOpcode::Paddb
1256            | SseOpcode::Paddd
1257            | SseOpcode::Paddq
1258            | SseOpcode::Paddw
1259            | SseOpcode::Paddsb
1260            | SseOpcode::Paddsw
1261            | SseOpcode::Paddusb
1262            | SseOpcode::Paddusw
1263            | SseOpcode::Pand
1264            | SseOpcode::Pandn
1265            | SseOpcode::Pavgb
1266            | SseOpcode::Pavgw
1267            | SseOpcode::Pcmpeqb
1268            | SseOpcode::Pcmpeqw
1269            | SseOpcode::Pcmpeqd
1270            | SseOpcode::Pcmpgtb
1271            | SseOpcode::Pcmpgtw
1272            | SseOpcode::Pcmpgtd
1273            | SseOpcode::Pextrw
1274            | SseOpcode::Pinsrw
1275            | SseOpcode::Pmaddwd
1276            | SseOpcode::Pmaxsw
1277            | SseOpcode::Pmaxub
1278            | SseOpcode::Pminsw
1279            | SseOpcode::Pminub
1280            | SseOpcode::Pmovmskb
1281            | SseOpcode::Pmulhw
1282            | SseOpcode::Pmulhuw
1283            | SseOpcode::Pmullw
1284            | SseOpcode::Pmuludq
1285            | SseOpcode::Por
1286            | SseOpcode::Pshufd
1287            | SseOpcode::Psllw
1288            | SseOpcode::Pslld
1289            | SseOpcode::Psllq
1290            | SseOpcode::Psraw
1291            | SseOpcode::Psrad
1292            | SseOpcode::Psrlw
1293            | SseOpcode::Psrld
1294            | SseOpcode::Psrlq
1295            | SseOpcode::Psubb
1296            | SseOpcode::Psubd
1297            | SseOpcode::Psubq
1298            | SseOpcode::Psubw
1299            | SseOpcode::Psubsb
1300            | SseOpcode::Psubsw
1301            | SseOpcode::Psubusb
1302            | SseOpcode::Psubusw
1303            | SseOpcode::Punpckhbw
1304            | SseOpcode::Punpckhwd
1305            | SseOpcode::Punpcklbw
1306            | SseOpcode::Punpcklwd
1307            | SseOpcode::Pxor
1308            | SseOpcode::Sqrtpd
1309            | SseOpcode::Sqrtsd
1310            | SseOpcode::Subpd
1311            | SseOpcode::Subsd
1312            | SseOpcode::Ucomisd
1313            | SseOpcode::Xorpd
1314            | SseOpcode::Punpckldq
1315            | SseOpcode::Punpckhdq
1316            | SseOpcode::Punpcklqdq
1317            | SseOpcode::Punpckhqdq
1318            | SseOpcode::Pshuflw
1319            | SseOpcode::Pshufhw
1320            | SseOpcode::Unpcklpd => SSE2,
1321
1322            SseOpcode::Pabsb
1323            | SseOpcode::Pabsw
1324            | SseOpcode::Pabsd
1325            | SseOpcode::Palignr
1326            | SseOpcode::Pmulhrsw
1327            | SseOpcode::Pshufb
1328            | SseOpcode::Phaddw
1329            | SseOpcode::Phaddd
1330            | SseOpcode::Pmaddubsw
1331            | SseOpcode::Movddup => SSSE3,
1332
1333            SseOpcode::Blendvpd
1334            | SseOpcode::Blendvps
1335            | SseOpcode::Insertps
1336            | SseOpcode::Packusdw
1337            | SseOpcode::Pblendvb
1338            | SseOpcode::Pcmpeqq
1339            | SseOpcode::Pextrb
1340            | SseOpcode::Pextrd
1341            | SseOpcode::Pextrq
1342            | SseOpcode::Pinsrb
1343            | SseOpcode::Pinsrd
1344            | SseOpcode::Pmaxsb
1345            | SseOpcode::Pmaxsd
1346            | SseOpcode::Pmaxuw
1347            | SseOpcode::Pmaxud
1348            | SseOpcode::Pminsb
1349            | SseOpcode::Pminsd
1350            | SseOpcode::Pminuw
1351            | SseOpcode::Pminud
1352            | SseOpcode::Pmovsxbd
1353            | SseOpcode::Pmovsxbw
1354            | SseOpcode::Pmovsxbq
1355            | SseOpcode::Pmovsxwd
1356            | SseOpcode::Pmovsxwq
1357            | SseOpcode::Pmovsxdq
1358            | SseOpcode::Pmovzxbd
1359            | SseOpcode::Pmovzxbw
1360            | SseOpcode::Pmovzxbq
1361            | SseOpcode::Pmovzxwd
1362            | SseOpcode::Pmovzxwq
1363            | SseOpcode::Pmovzxdq
1364            | SseOpcode::Pmuldq
1365            | SseOpcode::Pmulld
1366            | SseOpcode::Ptest
1367            | SseOpcode::Roundps
1368            | SseOpcode::Roundpd
1369            | SseOpcode::Roundss
1370            | SseOpcode::Roundsd
1371            | SseOpcode::Pblendw => SSE41,
1372
1373            SseOpcode::Pcmpgtq => SSE42,
1374        }
1375    }
1376
1377    /// Returns the src operand size for an instruction.
1378    pub(crate) fn src_size(&self) -> u8 {
1379        match self {
1380            SseOpcode::Movd => 4,
1381            _ => 8,
1382        }
1383    }
1384
1385    /// Is `src2` with this opcode a scalar, as for lane insertions?
1386    pub(crate) fn has_scalar_src2(self) -> bool {
1387        match self {
1388            SseOpcode::Pinsrb | SseOpcode::Pinsrw | SseOpcode::Pinsrd => true,
1389            SseOpcode::Pmovsxbw
1390            | SseOpcode::Pmovsxbd
1391            | SseOpcode::Pmovsxbq
1392            | SseOpcode::Pmovsxwd
1393            | SseOpcode::Pmovsxwq
1394            | SseOpcode::Pmovsxdq => true,
1395            SseOpcode::Pmovzxbw
1396            | SseOpcode::Pmovzxbd
1397            | SseOpcode::Pmovzxbq
1398            | SseOpcode::Pmovzxwd
1399            | SseOpcode::Pmovzxwq
1400            | SseOpcode::Pmovzxdq => true,
1401            _ => false,
1402        }
1403    }
1404}
1405
1406impl fmt::Debug for SseOpcode {
1407    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
1408        let name = match self {
1409            SseOpcode::Addps => "addps",
1410            SseOpcode::Addpd => "addpd",
1411            SseOpcode::Addss => "addss",
1412            SseOpcode::Addsd => "addsd",
1413            SseOpcode::Andpd => "andpd",
1414            SseOpcode::Andps => "andps",
1415            SseOpcode::Andnps => "andnps",
1416            SseOpcode::Andnpd => "andnpd",
1417            SseOpcode::Blendvpd => "blendvpd",
1418            SseOpcode::Blendvps => "blendvps",
1419            SseOpcode::Cmpps => "cmpps",
1420            SseOpcode::Cmppd => "cmppd",
1421            SseOpcode::Cmpss => "cmpss",
1422            SseOpcode::Cmpsd => "cmpsd",
1423            SseOpcode::Comiss => "comiss",
1424            SseOpcode::Comisd => "comisd",
1425            SseOpcode::Cvtdq2ps => "cvtdq2ps",
1426            SseOpcode::Cvtdq2pd => "cvtdq2pd",
1427            SseOpcode::Cvtpd2ps => "cvtpd2ps",
1428            SseOpcode::Cvtps2pd => "cvtps2pd",
1429            SseOpcode::Cvtsd2ss => "cvtsd2ss",
1430            SseOpcode::Cvtsd2si => "cvtsd2si",
1431            SseOpcode::Cvtsi2ss => "cvtsi2ss",
1432            SseOpcode::Cvtsi2sd => "cvtsi2sd",
1433            SseOpcode::Cvtss2si => "cvtss2si",
1434            SseOpcode::Cvtss2sd => "cvtss2sd",
1435            SseOpcode::Cvttpd2dq => "cvttpd2dq",
1436            SseOpcode::Cvttps2dq => "cvttps2dq",
1437            SseOpcode::Cvttss2si => "cvttss2si",
1438            SseOpcode::Cvttsd2si => "cvttsd2si",
1439            SseOpcode::Divps => "divps",
1440            SseOpcode::Divpd => "divpd",
1441            SseOpcode::Divss => "divss",
1442            SseOpcode::Divsd => "divsd",
1443            SseOpcode::Insertps => "insertps",
1444            SseOpcode::Maxps => "maxps",
1445            SseOpcode::Maxpd => "maxpd",
1446            SseOpcode::Maxss => "maxss",
1447            SseOpcode::Maxsd => "maxsd",
1448            SseOpcode::Minps => "minps",
1449            SseOpcode::Minpd => "minpd",
1450            SseOpcode::Minss => "minss",
1451            SseOpcode::Minsd => "minsd",
1452            SseOpcode::Movaps => "movaps",
1453            SseOpcode::Movapd => "movapd",
1454            SseOpcode::Movd => "movd",
1455            SseOpcode::Movdqa => "movdqa",
1456            SseOpcode::Movdqu => "movdqu",
1457            SseOpcode::Movlhps => "movlhps",
1458            SseOpcode::Movmskps => "movmskps",
1459            SseOpcode::Movmskpd => "movmskpd",
1460            SseOpcode::Movq => "movq",
1461            SseOpcode::Movss => "movss",
1462            SseOpcode::Movsd => "movsd",
1463            SseOpcode::Movups => "movups",
1464            SseOpcode::Movupd => "movupd",
1465            SseOpcode::Mulps => "mulps",
1466            SseOpcode::Mulpd => "mulpd",
1467            SseOpcode::Mulss => "mulss",
1468            SseOpcode::Mulsd => "mulsd",
1469            SseOpcode::Orpd => "orpd",
1470            SseOpcode::Orps => "orps",
1471            SseOpcode::Pabsb => "pabsb",
1472            SseOpcode::Pabsw => "pabsw",
1473            SseOpcode::Pabsd => "pabsd",
1474            SseOpcode::Packssdw => "packssdw",
1475            SseOpcode::Packsswb => "packsswb",
1476            SseOpcode::Packusdw => "packusdw",
1477            SseOpcode::Packuswb => "packuswb",
1478            SseOpcode::Paddb => "paddb",
1479            SseOpcode::Paddd => "paddd",
1480            SseOpcode::Paddq => "paddq",
1481            SseOpcode::Paddw => "paddw",
1482            SseOpcode::Paddsb => "paddsb",
1483            SseOpcode::Paddsw => "paddsw",
1484            SseOpcode::Paddusb => "paddusb",
1485            SseOpcode::Paddusw => "paddusw",
1486            SseOpcode::Palignr => "palignr",
1487            SseOpcode::Pand => "pand",
1488            SseOpcode::Pandn => "pandn",
1489            SseOpcode::Pavgb => "pavgb",
1490            SseOpcode::Pavgw => "pavgw",
1491            SseOpcode::Pblendvb => "pblendvb",
1492            SseOpcode::Pcmpeqb => "pcmpeqb",
1493            SseOpcode::Pcmpeqw => "pcmpeqw",
1494            SseOpcode::Pcmpeqd => "pcmpeqd",
1495            SseOpcode::Pcmpeqq => "pcmpeqq",
1496            SseOpcode::Pcmpgtb => "pcmpgtb",
1497            SseOpcode::Pcmpgtw => "pcmpgtw",
1498            SseOpcode::Pcmpgtd => "pcmpgtd",
1499            SseOpcode::Pcmpgtq => "pcmpgtq",
1500            SseOpcode::Pextrb => "pextrb",
1501            SseOpcode::Pextrw => "pextrw",
1502            SseOpcode::Pextrd => "pextrd",
1503            SseOpcode::Pextrq => "pextrq",
1504            SseOpcode::Pinsrb => "pinsrb",
1505            SseOpcode::Pinsrw => "pinsrw",
1506            SseOpcode::Pinsrd => "pinsrd",
1507            SseOpcode::Pmaddubsw => "pmaddubsw",
1508            SseOpcode::Pmaddwd => "pmaddwd",
1509            SseOpcode::Pmaxsb => "pmaxsb",
1510            SseOpcode::Pmaxsw => "pmaxsw",
1511            SseOpcode::Pmaxsd => "pmaxsd",
1512            SseOpcode::Pmaxub => "pmaxub",
1513            SseOpcode::Pmaxuw => "pmaxuw",
1514            SseOpcode::Pmaxud => "pmaxud",
1515            SseOpcode::Pminsb => "pminsb",
1516            SseOpcode::Pminsw => "pminsw",
1517            SseOpcode::Pminsd => "pminsd",
1518            SseOpcode::Pminub => "pminub",
1519            SseOpcode::Pminuw => "pminuw",
1520            SseOpcode::Pminud => "pminud",
1521            SseOpcode::Pmovmskb => "pmovmskb",
1522            SseOpcode::Pmovsxbd => "pmovsxbd",
1523            SseOpcode::Pmovsxbw => "pmovsxbw",
1524            SseOpcode::Pmovsxbq => "pmovsxbq",
1525            SseOpcode::Pmovsxwd => "pmovsxwd",
1526            SseOpcode::Pmovsxwq => "pmovsxwq",
1527            SseOpcode::Pmovsxdq => "pmovsxdq",
1528            SseOpcode::Pmovzxbd => "pmovzxbd",
1529            SseOpcode::Pmovzxbw => "pmovzxbw",
1530            SseOpcode::Pmovzxbq => "pmovzxbq",
1531            SseOpcode::Pmovzxwd => "pmovzxwd",
1532            SseOpcode::Pmovzxwq => "pmovzxwq",
1533            SseOpcode::Pmovzxdq => "pmovzxdq",
1534            SseOpcode::Pmuldq => "pmuldq",
1535            SseOpcode::Pmulhw => "pmulhw",
1536            SseOpcode::Pmulhuw => "pmulhuw",
1537            SseOpcode::Pmulhrsw => "pmulhrsw",
1538            SseOpcode::Pmulld => "pmulld",
1539            SseOpcode::Pmullw => "pmullw",
1540            SseOpcode::Pmuludq => "pmuludq",
1541            SseOpcode::Por => "por",
1542            SseOpcode::Pshufb => "pshufb",
1543            SseOpcode::Pshufd => "pshufd",
1544            SseOpcode::Psllw => "psllw",
1545            SseOpcode::Pslld => "pslld",
1546            SseOpcode::Psllq => "psllq",
1547            SseOpcode::Psraw => "psraw",
1548            SseOpcode::Psrad => "psrad",
1549            SseOpcode::Psrlw => "psrlw",
1550            SseOpcode::Psrld => "psrld",
1551            SseOpcode::Psrlq => "psrlq",
1552            SseOpcode::Psubb => "psubb",
1553            SseOpcode::Psubd => "psubd",
1554            SseOpcode::Psubq => "psubq",
1555            SseOpcode::Psubw => "psubw",
1556            SseOpcode::Psubsb => "psubsb",
1557            SseOpcode::Psubsw => "psubsw",
1558            SseOpcode::Psubusb => "psubusb",
1559            SseOpcode::Psubusw => "psubusw",
1560            SseOpcode::Ptest => "ptest",
1561            SseOpcode::Punpckhbw => "punpckhbw",
1562            SseOpcode::Punpckhwd => "punpckhwd",
1563            SseOpcode::Punpcklbw => "punpcklbw",
1564            SseOpcode::Punpcklwd => "punpcklwd",
1565            SseOpcode::Pxor => "pxor",
1566            SseOpcode::Rcpss => "rcpss",
1567            SseOpcode::Roundps => "roundps",
1568            SseOpcode::Roundpd => "roundpd",
1569            SseOpcode::Roundss => "roundss",
1570            SseOpcode::Roundsd => "roundsd",
1571            SseOpcode::Rsqrtss => "rsqrtss",
1572            SseOpcode::Shufps => "shufps",
1573            SseOpcode::Sqrtps => "sqrtps",
1574            SseOpcode::Sqrtpd => "sqrtpd",
1575            SseOpcode::Sqrtss => "sqrtss",
1576            SseOpcode::Sqrtsd => "sqrtsd",
1577            SseOpcode::Subps => "subps",
1578            SseOpcode::Subpd => "subpd",
1579            SseOpcode::Subss => "subss",
1580            SseOpcode::Subsd => "subsd",
1581            SseOpcode::Ucomiss => "ucomiss",
1582            SseOpcode::Ucomisd => "ucomisd",
1583            SseOpcode::Unpcklps => "unpcklps",
1584            SseOpcode::Unpckhps => "unpckhps",
1585            SseOpcode::Xorps => "xorps",
1586            SseOpcode::Xorpd => "xorpd",
1587            SseOpcode::Phaddw => "phaddw",
1588            SseOpcode::Phaddd => "phaddd",
1589            SseOpcode::Punpckldq => "punpckldq",
1590            SseOpcode::Punpckhdq => "punpckhdq",
1591            SseOpcode::Punpcklqdq => "punpcklqdq",
1592            SseOpcode::Punpckhqdq => "punpckhqdq",
1593            SseOpcode::Pshuflw => "pshuflw",
1594            SseOpcode::Pshufhw => "pshufhw",
1595            SseOpcode::Pblendw => "pblendw",
1596            SseOpcode::Movddup => "movddup",
1597            SseOpcode::Unpcklpd => "unpcklpd",
1598        };
1599        write!(fmt, "{name}")
1600    }
1601}
1602
1603impl fmt::Display for SseOpcode {
1604    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
1605        fmt::Debug::fmt(self, f)
1606    }
1607}
1608
1609pub use crate::isa::x64::lower::isle::generated_code::AvxOpcode;
1610
1611impl AvxOpcode {
1612    /// Which `InstructionSet`s support the opcode?
1613    pub(crate) fn available_from(&self) -> SmallVec<[InstructionSet; 2]> {
1614        match self {
1615            AvxOpcode::Vfmadd213ss
1616            | AvxOpcode::Vfmadd213sd
1617            | AvxOpcode::Vfmadd213ps
1618            | AvxOpcode::Vfmadd213pd
1619            | AvxOpcode::Vfmadd132ss
1620            | AvxOpcode::Vfmadd132sd
1621            | AvxOpcode::Vfmadd132ps
1622            | AvxOpcode::Vfmadd132pd
1623            | AvxOpcode::Vfnmadd213ss
1624            | AvxOpcode::Vfnmadd213sd
1625            | AvxOpcode::Vfnmadd213ps
1626            | AvxOpcode::Vfnmadd213pd
1627            | AvxOpcode::Vfnmadd132ss
1628            | AvxOpcode::Vfnmadd132sd
1629            | AvxOpcode::Vfnmadd132ps
1630            | AvxOpcode::Vfnmadd132pd
1631            | AvxOpcode::Vfmsub213ss
1632            | AvxOpcode::Vfmsub213sd
1633            | AvxOpcode::Vfmsub213ps
1634            | AvxOpcode::Vfmsub213pd
1635            | AvxOpcode::Vfmsub132ss
1636            | AvxOpcode::Vfmsub132sd
1637            | AvxOpcode::Vfmsub132ps
1638            | AvxOpcode::Vfmsub132pd
1639            | AvxOpcode::Vfnmsub213ss
1640            | AvxOpcode::Vfnmsub213sd
1641            | AvxOpcode::Vfnmsub213ps
1642            | AvxOpcode::Vfnmsub213pd
1643            | AvxOpcode::Vfnmsub132ss
1644            | AvxOpcode::Vfnmsub132sd
1645            | AvxOpcode::Vfnmsub132ps
1646            | AvxOpcode::Vfnmsub132pd => smallvec![InstructionSet::FMA],
1647            AvxOpcode::Vminps
1648            | AvxOpcode::Vminpd
1649            | AvxOpcode::Vmaxps
1650            | AvxOpcode::Vmaxpd
1651            | AvxOpcode::Vandnps
1652            | AvxOpcode::Vandnpd
1653            | AvxOpcode::Vpandn
1654            | AvxOpcode::Vcmpps
1655            | AvxOpcode::Vcmppd
1656            | AvxOpcode::Vpsrlw
1657            | AvxOpcode::Vpsrld
1658            | AvxOpcode::Vpsrlq
1659            | AvxOpcode::Vpaddb
1660            | AvxOpcode::Vpaddw
1661            | AvxOpcode::Vpaddd
1662            | AvxOpcode::Vpaddq
1663            | AvxOpcode::Vpaddsb
1664            | AvxOpcode::Vpaddsw
1665            | AvxOpcode::Vpaddusb
1666            | AvxOpcode::Vpaddusw
1667            | AvxOpcode::Vpsubb
1668            | AvxOpcode::Vpsubw
1669            | AvxOpcode::Vpsubd
1670            | AvxOpcode::Vpsubq
1671            | AvxOpcode::Vpsubsb
1672            | AvxOpcode::Vpsubsw
1673            | AvxOpcode::Vpsubusb
1674            | AvxOpcode::Vpsubusw
1675            | AvxOpcode::Vpavgb
1676            | AvxOpcode::Vpavgw
1677            | AvxOpcode::Vpand
1678            | AvxOpcode::Vandps
1679            | AvxOpcode::Vandpd
1680            | AvxOpcode::Vpor
1681            | AvxOpcode::Vorps
1682            | AvxOpcode::Vorpd
1683            | AvxOpcode::Vpxor
1684            | AvxOpcode::Vxorps
1685            | AvxOpcode::Vxorpd
1686            | AvxOpcode::Vpmullw
1687            | AvxOpcode::Vpmulld
1688            | AvxOpcode::Vpmulhw
1689            | AvxOpcode::Vpmulhd
1690            | AvxOpcode::Vpmulhrsw
1691            | AvxOpcode::Vpmulhuw
1692            | AvxOpcode::Vpmuldq
1693            | AvxOpcode::Vpmuludq
1694            | AvxOpcode::Vpunpckhwd
1695            | AvxOpcode::Vpunpcklwd
1696            | AvxOpcode::Vunpcklps
1697            | AvxOpcode::Vunpckhps
1698            | AvxOpcode::Vaddps
1699            | AvxOpcode::Vaddpd
1700            | AvxOpcode::Vsubps
1701            | AvxOpcode::Vsubpd
1702            | AvxOpcode::Vmulps
1703            | AvxOpcode::Vmulpd
1704            | AvxOpcode::Vdivps
1705            | AvxOpcode::Vdivpd
1706            | AvxOpcode::Vpcmpeqb
1707            | AvxOpcode::Vpcmpeqw
1708            | AvxOpcode::Vpcmpeqd
1709            | AvxOpcode::Vpcmpeqq
1710            | AvxOpcode::Vpcmpgtb
1711            | AvxOpcode::Vpcmpgtw
1712            | AvxOpcode::Vpcmpgtd
1713            | AvxOpcode::Vpcmpgtq
1714            | AvxOpcode::Vblendvps
1715            | AvxOpcode::Vblendvpd
1716            | AvxOpcode::Vpblendvb
1717            | AvxOpcode::Vmovlhps
1718            | AvxOpcode::Vpminsb
1719            | AvxOpcode::Vpminsw
1720            | AvxOpcode::Vpminsd
1721            | AvxOpcode::Vpminub
1722            | AvxOpcode::Vpminuw
1723            | AvxOpcode::Vpminud
1724            | AvxOpcode::Vpmaxsb
1725            | AvxOpcode::Vpmaxsw
1726            | AvxOpcode::Vpmaxsd
1727            | AvxOpcode::Vpmaxub
1728            | AvxOpcode::Vpmaxuw
1729            | AvxOpcode::Vpmaxud
1730            | AvxOpcode::Vpunpcklbw
1731            | AvxOpcode::Vpunpckhbw
1732            | AvxOpcode::Vpacksswb
1733            | AvxOpcode::Vpackssdw
1734            | AvxOpcode::Vpackuswb
1735            | AvxOpcode::Vpackusdw
1736            | AvxOpcode::Vpalignr
1737            | AvxOpcode::Vpinsrb
1738            | AvxOpcode::Vpinsrw
1739            | AvxOpcode::Vpinsrd
1740            | AvxOpcode::Vpinsrq
1741            | AvxOpcode::Vpmaddwd
1742            | AvxOpcode::Vpmaddubsw
1743            | AvxOpcode::Vinsertps
1744            | AvxOpcode::Vpshufb
1745            | AvxOpcode::Vshufps
1746            | AvxOpcode::Vpsllw
1747            | AvxOpcode::Vpslld
1748            | AvxOpcode::Vpsllq
1749            | AvxOpcode::Vpsraw
1750            | AvxOpcode::Vpsrad
1751            | AvxOpcode::Vpmovsxbw
1752            | AvxOpcode::Vpmovzxbw
1753            | AvxOpcode::Vpmovsxwd
1754            | AvxOpcode::Vpmovzxwd
1755            | AvxOpcode::Vpmovsxdq
1756            | AvxOpcode::Vpmovzxdq
1757            | AvxOpcode::Vaddss
1758            | AvxOpcode::Vaddsd
1759            | AvxOpcode::Vmulss
1760            | AvxOpcode::Vmulsd
1761            | AvxOpcode::Vsubss
1762            | AvxOpcode::Vsubsd
1763            | AvxOpcode::Vdivss
1764            | AvxOpcode::Vdivsd
1765            | AvxOpcode::Vpabsb
1766            | AvxOpcode::Vpabsw
1767            | AvxOpcode::Vpabsd
1768            | AvxOpcode::Vminss
1769            | AvxOpcode::Vminsd
1770            | AvxOpcode::Vmaxss
1771            | AvxOpcode::Vmaxsd
1772            | AvxOpcode::Vsqrtps
1773            | AvxOpcode::Vsqrtpd
1774            | AvxOpcode::Vroundpd
1775            | AvxOpcode::Vroundps
1776            | AvxOpcode::Vcvtdq2pd
1777            | AvxOpcode::Vcvtdq2ps
1778            | AvxOpcode::Vcvtpd2ps
1779            | AvxOpcode::Vcvtps2pd
1780            | AvxOpcode::Vcvttpd2dq
1781            | AvxOpcode::Vcvttps2dq
1782            | AvxOpcode::Vphaddw
1783            | AvxOpcode::Vphaddd
1784            | AvxOpcode::Vpunpckldq
1785            | AvxOpcode::Vpunpckhdq
1786            | AvxOpcode::Vpunpcklqdq
1787            | AvxOpcode::Vpunpckhqdq
1788            | AvxOpcode::Vpshuflw
1789            | AvxOpcode::Vpshufhw
1790            | AvxOpcode::Vpshufd
1791            | AvxOpcode::Vmovss
1792            | AvxOpcode::Vmovsd
1793            | AvxOpcode::Vmovups
1794            | AvxOpcode::Vmovupd
1795            | AvxOpcode::Vmovdqu
1796            | AvxOpcode::Vpextrb
1797            | AvxOpcode::Vpextrw
1798            | AvxOpcode::Vpextrd
1799            | AvxOpcode::Vpextrq
1800            | AvxOpcode::Vpblendw
1801            | AvxOpcode::Vmovddup
1802            | AvxOpcode::Vbroadcastss
1803            | AvxOpcode::Vmovd
1804            | AvxOpcode::Vmovq
1805            | AvxOpcode::Vmovmskps
1806            | AvxOpcode::Vmovmskpd
1807            | AvxOpcode::Vpmovmskb
1808            | AvxOpcode::Vcvtsi2ss
1809            | AvxOpcode::Vcvtsi2sd
1810            | AvxOpcode::Vcvtss2sd
1811            | AvxOpcode::Vcvtsd2ss
1812            | AvxOpcode::Vsqrtss
1813            | AvxOpcode::Vsqrtsd
1814            | AvxOpcode::Vroundss
1815            | AvxOpcode::Vroundsd
1816            | AvxOpcode::Vunpcklpd
1817            | AvxOpcode::Vptest
1818            | AvxOpcode::Vucomiss
1819            | AvxOpcode::Vucomisd => {
1820                smallvec![InstructionSet::AVX]
1821            }
1822
1823            AvxOpcode::Vpbroadcastb | AvxOpcode::Vpbroadcastw | AvxOpcode::Vpbroadcastd => {
1824                smallvec![InstructionSet::AVX2]
1825            }
1826        }
1827    }
1828
1829    /// Is the opcode known to be commutative?
1830    ///
1831    /// Note that this method is not exhaustive, and there may be commutative
1832    /// opcodes that we don't recognize as commutative.
1833    pub(crate) fn is_commutative(&self) -> bool {
1834        match *self {
1835            AvxOpcode::Vpaddb
1836            | AvxOpcode::Vpaddw
1837            | AvxOpcode::Vpaddd
1838            | AvxOpcode::Vpaddq
1839            | AvxOpcode::Vpaddsb
1840            | AvxOpcode::Vpaddsw
1841            | AvxOpcode::Vpaddusb
1842            | AvxOpcode::Vpaddusw
1843            | AvxOpcode::Vpand
1844            | AvxOpcode::Vandps
1845            | AvxOpcode::Vandpd
1846            | AvxOpcode::Vpor
1847            | AvxOpcode::Vorps
1848            | AvxOpcode::Vorpd
1849            | AvxOpcode::Vpxor
1850            | AvxOpcode::Vxorps
1851            | AvxOpcode::Vxorpd
1852            | AvxOpcode::Vpmuldq
1853            | AvxOpcode::Vpmuludq
1854            | AvxOpcode::Vaddps
1855            | AvxOpcode::Vaddpd
1856            | AvxOpcode::Vmulps
1857            | AvxOpcode::Vmulpd
1858            | AvxOpcode::Vpcmpeqb
1859            | AvxOpcode::Vpcmpeqw
1860            | AvxOpcode::Vpcmpeqd
1861            | AvxOpcode::Vpcmpeqq
1862            | AvxOpcode::Vaddss
1863            | AvxOpcode::Vaddsd
1864            | AvxOpcode::Vmulss
1865            | AvxOpcode::Vmulsd => true,
1866            _ => false,
1867        }
1868    }
1869}
1870
1871impl fmt::Display for AvxOpcode {
1872    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
1873        format!("{self:?}").to_lowercase().fmt(f)
1874    }
1875}
1876
1877#[derive(Copy, Clone, PartialEq)]
1878#[allow(missing_docs)]
1879pub enum Avx512TupleType {
1880    Full,
1881    FullMem,
1882    Mem128,
1883}
1884
1885pub use crate::isa::x64::lower::isle::generated_code::Avx512Opcode;
1886
1887impl Avx512Opcode {
1888    /// Which `InstructionSet`s support the opcode?
1889    pub(crate) fn available_from(&self) -> SmallVec<[InstructionSet; 2]> {
1890        match self {
1891            Avx512Opcode::Vcvtudq2ps
1892            | Avx512Opcode::Vpabsq
1893            | Avx512Opcode::Vpsraq
1894            | Avx512Opcode::VpsraqImm => {
1895                smallvec![InstructionSet::AVX512F, InstructionSet::AVX512VL]
1896            }
1897            Avx512Opcode::Vpermi2b => {
1898                smallvec![InstructionSet::AVX512VL, InstructionSet::AVX512VBMI]
1899            }
1900            Avx512Opcode::Vpmullq => smallvec![InstructionSet::AVX512VL, InstructionSet::AVX512DQ],
1901            Avx512Opcode::Vpopcntb => {
1902                smallvec![InstructionSet::AVX512VL, InstructionSet::AVX512BITALG]
1903            }
1904        }
1905    }
1906
1907    /// What is the "TupleType" of this opcode, which affects the scaling factor
1908    /// for 8-bit displacements when this instruction uses memory operands.
1909    ///
1910    /// This can be found in the encoding table for each instruction and is
1911    /// interpreted according to Table 2-34 and 2-35 in the Intel instruction
1912    /// manual.
1913    pub fn tuple_type(&self) -> Avx512TupleType {
1914        use Avx512Opcode::*;
1915        use Avx512TupleType::*;
1916
1917        match self {
1918            Vcvtudq2ps | Vpabsq | Vpmullq | VpsraqImm => Full,
1919            Vpermi2b | Vpopcntb => FullMem,
1920            Vpsraq => Mem128,
1921        }
1922    }
1923}
1924
1925impl fmt::Display for Avx512Opcode {
1926    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
1927        let s = format!("{self:?}");
1928        f.write_str(&s.to_lowercase())
1929    }
1930}
1931
1932/// This defines the ways a value can be extended: either signed- or zero-extension, or none for
1933/// types that are not extended. Contrast with [ExtMode], which defines the widths from and to which
1934/// values can be extended.
1935#[allow(dead_code)]
1936#[derive(Clone, PartialEq)]
1937pub enum ExtKind {
1938    /// No extension.
1939    None,
1940    /// Sign-extend.
1941    SignExtend,
1942    /// Zero-extend.
1943    ZeroExtend,
1944}
1945
1946/// These indicate ways of extending (widening) a value, using the Intel
1947/// naming: B(yte) = u8, W(ord) = u16, L(ong)word = u32, Q(uad)word = u64
1948#[derive(Clone, PartialEq)]
1949pub enum ExtMode {
1950    /// Byte -> Longword.
1951    BL,
1952    /// Byte -> Quadword.
1953    BQ,
1954    /// Word -> Longword.
1955    WL,
1956    /// Word -> Quadword.
1957    WQ,
1958    /// Longword -> Quadword.
1959    LQ,
1960}
1961
1962impl ExtMode {
1963    /// Calculate the `ExtMode` from passed bit lengths of the from/to types.
1964    pub(crate) fn new(from_bits: u16, to_bits: u16) -> Option<ExtMode> {
1965        match (from_bits, to_bits) {
1966            (1, 8) | (1, 16) | (1, 32) | (8, 16) | (8, 32) => Some(ExtMode::BL),
1967            (1, 64) | (8, 64) => Some(ExtMode::BQ),
1968            (16, 32) => Some(ExtMode::WL),
1969            (16, 64) => Some(ExtMode::WQ),
1970            (32, 64) => Some(ExtMode::LQ),
1971            _ => None,
1972        }
1973    }
1974
1975    /// Return the source register size in bytes.
1976    pub(crate) fn src_size(&self) -> u8 {
1977        match self {
1978            ExtMode::BL | ExtMode::BQ => 1,
1979            ExtMode::WL | ExtMode::WQ => 2,
1980            ExtMode::LQ => 4,
1981        }
1982    }
1983
1984    /// Return the destination register size in bytes.
1985    pub(crate) fn dst_size(&self) -> u8 {
1986        match self {
1987            ExtMode::BL | ExtMode::WL => 4,
1988            ExtMode::BQ | ExtMode::WQ | ExtMode::LQ => 8,
1989        }
1990    }
1991
1992    /// Source size, as an integer type.
1993    pub(crate) fn src_type(&self) -> Type {
1994        match self {
1995            ExtMode::BL | ExtMode::BQ => I8,
1996            ExtMode::WL | ExtMode::WQ => I16,
1997            ExtMode::LQ => I32,
1998        }
1999    }
2000}
2001
2002impl fmt::Debug for ExtMode {
2003    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
2004        let name = match self {
2005            ExtMode::BL => "bl",
2006            ExtMode::BQ => "bq",
2007            ExtMode::WL => "wl",
2008            ExtMode::WQ => "wq",
2009            ExtMode::LQ => "lq",
2010        };
2011        write!(fmt, "{name}")
2012    }
2013}
2014
2015impl fmt::Display for ExtMode {
2016    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
2017        fmt::Debug::fmt(self, f)
2018    }
2019}
2020
2021/// These indicate the form of a scalar shift/rotate: left, signed right, unsigned right.
2022#[derive(Clone, Copy)]
2023pub enum ShiftKind {
2024    /// Left shift.
2025    ShiftLeft,
2026    /// Inserts zeros in the most significant bits.
2027    ShiftRightLogical,
2028    /// Replicates the sign bit in the most significant bits.
2029    ShiftRightArithmetic,
2030    /// Left rotation.
2031    RotateLeft,
2032    /// Right rotation.
2033    RotateRight,
2034}
2035
2036impl fmt::Debug for ShiftKind {
2037    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
2038        let name = match self {
2039            ShiftKind::ShiftLeft => "shl",
2040            ShiftKind::ShiftRightLogical => "shr",
2041            ShiftKind::ShiftRightArithmetic => "sar",
2042            ShiftKind::RotateLeft => "rol",
2043            ShiftKind::RotateRight => "ror",
2044        };
2045        write!(fmt, "{name}")
2046    }
2047}
2048
2049impl fmt::Display for ShiftKind {
2050    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
2051        fmt::Debug::fmt(self, f)
2052    }
2053}
2054
2055/// These indicate condition code tests.  Not all are represented since not all are useful in
2056/// compiler-generated code.
2057#[derive(Copy, Clone, PartialEq, Eq)]
2058#[repr(u8)]
2059pub enum CC {
2060    ///  overflow
2061    O = 0,
2062    /// no overflow
2063    NO = 1,
2064
2065    /// < unsigned
2066    B = 2,
2067    /// >= unsigned
2068    NB = 3,
2069
2070    /// zero
2071    Z = 4,
2072    /// not-zero
2073    NZ = 5,
2074
2075    /// <= unsigned
2076    BE = 6,
2077    /// > unsigned
2078    NBE = 7,
2079
2080    /// negative
2081    S = 8,
2082    /// not-negative
2083    NS = 9,
2084
2085    /// < signed
2086    L = 12,
2087    /// >= signed
2088    NL = 13,
2089
2090    /// <= signed
2091    LE = 14,
2092    /// > signed
2093    NLE = 15,
2094
2095    /// parity
2096    P = 10,
2097
2098    /// not parity
2099    NP = 11,
2100}
2101
2102impl CC {
2103    pub(crate) fn from_intcc(intcc: IntCC) -> Self {
2104        match intcc {
2105            IntCC::Equal => CC::Z,
2106            IntCC::NotEqual => CC::NZ,
2107            IntCC::SignedGreaterThanOrEqual => CC::NL,
2108            IntCC::SignedGreaterThan => CC::NLE,
2109            IntCC::SignedLessThanOrEqual => CC::LE,
2110            IntCC::SignedLessThan => CC::L,
2111            IntCC::UnsignedGreaterThanOrEqual => CC::NB,
2112            IntCC::UnsignedGreaterThan => CC::NBE,
2113            IntCC::UnsignedLessThanOrEqual => CC::BE,
2114            IntCC::UnsignedLessThan => CC::B,
2115        }
2116    }
2117
2118    pub(crate) fn invert(&self) -> Self {
2119        match self {
2120            CC::O => CC::NO,
2121            CC::NO => CC::O,
2122
2123            CC::B => CC::NB,
2124            CC::NB => CC::B,
2125
2126            CC::Z => CC::NZ,
2127            CC::NZ => CC::Z,
2128
2129            CC::BE => CC::NBE,
2130            CC::NBE => CC::BE,
2131
2132            CC::S => CC::NS,
2133            CC::NS => CC::S,
2134
2135            CC::L => CC::NL,
2136            CC::NL => CC::L,
2137
2138            CC::LE => CC::NLE,
2139            CC::NLE => CC::LE,
2140
2141            CC::P => CC::NP,
2142            CC::NP => CC::P,
2143        }
2144    }
2145
2146    pub(crate) fn get_enc(self) -> u8 {
2147        self as u8
2148    }
2149}
2150
2151impl fmt::Debug for CC {
2152    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
2153        let name = match self {
2154            CC::O => "o",
2155            CC::NO => "no",
2156            CC::B => "b",
2157            CC::NB => "nb",
2158            CC::Z => "z",
2159            CC::NZ => "nz",
2160            CC::BE => "be",
2161            CC::NBE => "nbe",
2162            CC::S => "s",
2163            CC::NS => "ns",
2164            CC::L => "l",
2165            CC::NL => "nl",
2166            CC::LE => "le",
2167            CC::NLE => "nle",
2168            CC::P => "p",
2169            CC::NP => "np",
2170        };
2171        write!(fmt, "{name}")
2172    }
2173}
2174
2175impl fmt::Display for CC {
2176    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
2177        fmt::Debug::fmt(self, f)
2178    }
2179}
2180
2181/// Encode the ways that floats can be compared. This is used in float comparisons such as `cmpps`,
2182/// e.g.; it is distinguished from other float comparisons (e.g. `ucomiss`) in that those use EFLAGS
2183/// whereas [FcmpImm] is used as an immediate.
2184#[derive(Clone, Copy)]
2185pub enum FcmpImm {
2186    /// Equal comparison.
2187    Equal = 0x00,
2188    /// Less than comparison.
2189    LessThan = 0x01,
2190    /// Less than or equal comparison.
2191    LessThanOrEqual = 0x02,
2192    /// Unordered.
2193    Unordered = 0x03,
2194    /// Not equal comparison.
2195    NotEqual = 0x04,
2196    /// Unordered of greater than or equal comparison.
2197    UnorderedOrGreaterThanOrEqual = 0x05,
2198    /// Unordered or greater than comparison.
2199    UnorderedOrGreaterThan = 0x06,
2200    /// Ordered.
2201    Ordered = 0x07,
2202}
2203
2204impl FcmpImm {
2205    pub(crate) fn encode(self) -> u8 {
2206        self as u8
2207    }
2208}
2209
2210impl From<FloatCC> for FcmpImm {
2211    fn from(cond: FloatCC) -> Self {
2212        match cond {
2213            FloatCC::Equal => FcmpImm::Equal,
2214            FloatCC::LessThan => FcmpImm::LessThan,
2215            FloatCC::LessThanOrEqual => FcmpImm::LessThanOrEqual,
2216            FloatCC::Unordered => FcmpImm::Unordered,
2217            FloatCC::NotEqual => FcmpImm::NotEqual,
2218            FloatCC::UnorderedOrGreaterThanOrEqual => FcmpImm::UnorderedOrGreaterThanOrEqual,
2219            FloatCC::UnorderedOrGreaterThan => FcmpImm::UnorderedOrGreaterThan,
2220            FloatCC::Ordered => FcmpImm::Ordered,
2221            _ => panic!("unable to create comparison predicate for {cond}"),
2222        }
2223    }
2224}
2225
2226/// Encode the rounding modes used as part of the Rounding Control field.
2227/// Note, these rounding immediates only consider the rounding control field
2228/// (i.e. the rounding mode) which only take up the first two bits when encoded.
2229/// However the rounding immediate which this field helps make up, also includes
2230/// bits 3 and 4 which define the rounding select and precision mask respectively.
2231/// These two bits are not defined here and are implicitly set to zero when encoded.
2232#[derive(Clone, Copy)]
2233pub enum RoundImm {
2234    /// Round to nearest mode.
2235    RoundNearest = 0x00,
2236    /// Round down mode.
2237    RoundDown = 0x01,
2238    /// Round up mode.
2239    RoundUp = 0x02,
2240    /// Round to zero mode.
2241    RoundZero = 0x03,
2242}
2243
2244impl RoundImm {
2245    pub(crate) fn encode(self) -> u8 {
2246        self as u8
2247    }
2248}
2249
2250/// An operand's size in bits.
2251#[derive(Clone, Copy, PartialEq)]
2252pub enum OperandSize {
2253    /// 8-bit.
2254    Size8,
2255    /// 16-bit.
2256    Size16,
2257    /// 32-bit.
2258    Size32,
2259    /// 64-bit.
2260    Size64,
2261}
2262
2263impl OperandSize {
2264    pub(crate) fn from_bytes(num_bytes: u32) -> Self {
2265        match num_bytes {
2266            1 => OperandSize::Size8,
2267            2 => OperandSize::Size16,
2268            4 => OperandSize::Size32,
2269            8 => OperandSize::Size64,
2270            _ => unreachable!("Invalid OperandSize: {}", num_bytes),
2271        }
2272    }
2273
2274    // Computes the OperandSize for a given type.
2275    // For vectors, the OperandSize of the lanes is returned.
2276    pub(crate) fn from_ty(ty: Type) -> Self {
2277        Self::from_bytes(ty.lane_type().bytes())
2278    }
2279
2280    // Check that the value of self is one of the allowed sizes.
2281    pub(crate) fn is_one_of(&self, sizes: &[Self]) -> bool {
2282        sizes.iter().any(|val| *self == *val)
2283    }
2284
2285    pub(crate) fn to_bytes(&self) -> u8 {
2286        match self {
2287            Self::Size8 => 1,
2288            Self::Size16 => 2,
2289            Self::Size32 => 4,
2290            Self::Size64 => 8,
2291        }
2292    }
2293
2294    pub(crate) fn to_bits(&self) -> u8 {
2295        self.to_bytes() * 8
2296    }
2297
2298    pub(crate) fn to_type(&self) -> Type {
2299        match self {
2300            Self::Size8 => I8,
2301            Self::Size16 => I16,
2302            Self::Size32 => I32,
2303            Self::Size64 => I64,
2304        }
2305    }
2306}
2307
2308/// An x64 memory fence kind.
2309#[derive(Clone)]
2310#[allow(dead_code)]
2311pub enum FenceKind {
2312    /// `mfence` instruction ("Memory Fence")
2313    MFence,
2314    /// `lfence` instruction ("Load Fence")
2315    LFence,
2316    /// `sfence` instruction ("Store Fence")
2317    SFence,
2318}
cranelift_codegen/isa/x64/inst/args.rs

cranelift_codegen/isa/x64/inst/
args.rs
