1pub use emit_state::EmitState;
4
5use crate::binemit::{Addend, CodeOffset, Reloc};
6use crate::ir::{types, ExternalName, LibCall, TrapCode, Type};
7use crate::isa::x64::abi::X64ABIMachineSpec;
8use crate::isa::x64::inst::regs::{pretty_print_reg, show_ireg_sized};
9use crate::isa::x64::settings as x64_settings;
10use crate::isa::{CallConv, FunctionAlignment};
11use crate::{machinst::*, trace};
12use crate::{settings, CodegenError, CodegenResult};
13use alloc::boxed::Box;
14use smallvec::{smallvec, SmallVec};
15use std::fmt::{self, Write};
16use std::string::{String, ToString};
17
18pub mod args;
19mod emit;
20mod emit_state;
21#[cfg(test)]
22mod emit_tests;
23pub mod regs;
24mod stack_switch;
25pub mod unwind;
26
27use args::*;
28
29pub use super::lower::isle::generated_code::MInst as Inst;
34
35#[derive(Clone, Debug)]
37pub struct ReturnCallInfo<T> {
38 pub dest: T,
40
41 pub new_stack_arg_size: u32,
44
45 pub uses: CallArgList,
47
48 pub tmp: WritableGpr,
50}
51
52#[test]
53#[cfg(target_pointer_width = "64")]
54fn inst_size_test() {
55 assert_eq!(40, std::mem::size_of::<Inst>());
58}
59
60pub(crate) fn low32_will_sign_extend_to_64(x: u64) -> bool {
61 let xs = x as i64;
62 xs == ((xs << 32) >> 32)
63}
64
65impl Inst {
66 fn available_in_any_isa(&self) -> SmallVec<[InstructionSet; 2]> {
71 match self {
72 Inst::AluRmiR { .. }
75 | Inst::AluRM { .. }
76 | Inst::AtomicRmwSeq { .. }
77 | Inst::Bswap { .. }
78 | Inst::CallKnown { .. }
79 | Inst::CallUnknown { .. }
80 | Inst::ReturnCallKnown { .. }
81 | Inst::ReturnCallUnknown { .. }
82 | Inst::CheckedSRemSeq { .. }
83 | Inst::CheckedSRemSeq8 { .. }
84 | Inst::Cmove { .. }
85 | Inst::CmpRmiR { .. }
86 | Inst::CvtFloatToSintSeq { .. }
87 | Inst::CvtFloatToUintSeq { .. }
88 | Inst::CvtUint64ToFloatSeq { .. }
89 | Inst::Div { .. }
90 | Inst::Div8 { .. }
91 | Inst::Fence { .. }
92 | Inst::Hlt
93 | Inst::Imm { .. }
94 | Inst::JmpCond { .. }
95 | Inst::JmpIf { .. }
96 | Inst::JmpKnown { .. }
97 | Inst::JmpTableSeq { .. }
98 | Inst::JmpUnknown { .. }
99 | Inst::LoadEffectiveAddress { .. }
100 | Inst::LoadExtName { .. }
101 | Inst::LockCmpxchg { .. }
102 | Inst::Mov64MR { .. }
103 | Inst::MovImmM { .. }
104 | Inst::MovRM { .. }
105 | Inst::MovRR { .. }
106 | Inst::MovFromPReg { .. }
107 | Inst::MovToPReg { .. }
108 | Inst::MovsxRmR { .. }
109 | Inst::MovzxRmR { .. }
110 | Inst::Mul { .. }
111 | Inst::Mul8 { .. }
112 | Inst::IMul { .. }
113 | Inst::IMulImm { .. }
114 | Inst::Neg { .. }
115 | Inst::Not { .. }
116 | Inst::Nop { .. }
117 | Inst::Pop64 { .. }
118 | Inst::Push64 { .. }
119 | Inst::StackProbeLoop { .. }
120 | Inst::Args { .. }
121 | Inst::Rets { .. }
122 | Inst::Ret { .. }
123 | Inst::Setcc { .. }
124 | Inst::ShiftR { .. }
125 | Inst::SignExtendData { .. }
126 | Inst::StackSwitchBasic { .. }
127 | Inst::TrapIf { .. }
128 | Inst::TrapIfAnd { .. }
129 | Inst::TrapIfOr { .. }
130 | Inst::Ud2 { .. }
131 | Inst::XmmCmove { .. }
132 | Inst::XmmCmpRmR { .. }
133 | Inst::XmmMinMaxSeq { .. }
134 | Inst::XmmUninitializedValue { .. }
135 | Inst::ElfTlsGetAddr { .. }
136 | Inst::MachOTlsGetAddr { .. }
137 | Inst::CoffTlsGetAddr { .. }
138 | Inst::Unwind { .. }
139 | Inst::DummyUse { .. }
140 | Inst::AluConstOp { .. } => smallvec![],
141
142 Inst::AluRmRVex { op, .. } => op.available_from(),
143 Inst::UnaryRmR { op, .. } => op.available_from(),
144 Inst::UnaryRmRVex { op, .. } => op.available_from(),
145 Inst::UnaryRmRImmVex { op, .. } => op.available_from(),
146
147 Inst::GprToXmm { op, .. }
149 | Inst::XmmMovRM { op, .. }
150 | Inst::XmmMovRMImm { op, .. }
151 | Inst::XmmRmiReg { opcode: op, .. }
152 | Inst::XmmRmR { op, .. }
153 | Inst::XmmRmRUnaligned { op, .. }
154 | Inst::XmmRmRBlend { op, .. }
155 | Inst::XmmRmRImm { op, .. }
156 | Inst::XmmToGpr { op, .. }
157 | Inst::XmmToGprImm { op, .. }
158 | Inst::XmmUnaryRmRImm { op, .. }
159 | Inst::XmmUnaryRmRUnaligned { op, .. }
160 | Inst::XmmUnaryRmR { op, .. }
161 | Inst::CvtIntToFloat { op, .. } => smallvec![op.available_from()],
162
163 Inst::XmmUnaryRmREvex { op, .. }
164 | Inst::XmmRmREvex { op, .. }
165 | Inst::XmmRmREvex3 { op, .. }
166 | Inst::XmmUnaryRmRImmEvex { op, .. } => op.available_from(),
167
168 Inst::XmmRmiRVex { op, .. }
169 | Inst::XmmRmRVex3 { op, .. }
170 | Inst::XmmRmRImmVex { op, .. }
171 | Inst::XmmRmRBlendVex { op, .. }
172 | Inst::XmmVexPinsr { op, .. }
173 | Inst::XmmUnaryRmRVex { op, .. }
174 | Inst::XmmUnaryRmRImmVex { op, .. }
175 | Inst::XmmMovRMVex { op, .. }
176 | Inst::XmmMovRMImmVex { op, .. }
177 | Inst::XmmToGprImmVex { op, .. }
178 | Inst::XmmToGprVex { op, .. }
179 | Inst::GprToXmmVex { op, .. }
180 | Inst::CvtIntToFloatVex { op, .. }
181 | Inst::XmmCmpRmRVex { op, .. } => op.available_from(),
182 }
183 }
184}
185
186impl Inst {
189 pub(crate) fn nop(len: u8) -> Self {
190 debug_assert!(len <= 15);
191 Self::Nop { len }
192 }
193
194 pub(crate) fn alu_rmi_r(
195 size: OperandSize,
196 op: AluRmiROpcode,
197 src: RegMemImm,
198 dst: Writable<Reg>,
199 ) -> Self {
200 src.assert_regclass_is(RegClass::Int);
201 debug_assert!(dst.to_reg().class() == RegClass::Int);
202 Self::AluRmiR {
203 size,
204 op,
205 src1: Gpr::unwrap_new(dst.to_reg()),
206 src2: GprMemImm::unwrap_new(src),
207 dst: WritableGpr::from_writable_reg(dst).unwrap(),
208 }
209 }
210
211 #[allow(dead_code)]
212 pub(crate) fn unary_rm_r(
213 size: OperandSize,
214 op: UnaryRmROpcode,
215 src: RegMem,
216 dst: Writable<Reg>,
217 ) -> Self {
218 src.assert_regclass_is(RegClass::Int);
219 debug_assert!(dst.to_reg().class() == RegClass::Int);
220 debug_assert!(size.is_one_of(&[
221 OperandSize::Size16,
222 OperandSize::Size32,
223 OperandSize::Size64
224 ]));
225 Self::UnaryRmR {
226 size,
227 op,
228 src: GprMem::unwrap_new(src),
229 dst: WritableGpr::from_writable_reg(dst).unwrap(),
230 }
231 }
232
233 pub(crate) fn not(size: OperandSize, src: Writable<Reg>) -> Inst {
234 debug_assert_eq!(src.to_reg().class(), RegClass::Int);
235 Inst::Not {
236 size,
237 src: Gpr::unwrap_new(src.to_reg()),
238 dst: WritableGpr::from_writable_reg(src).unwrap(),
239 }
240 }
241
242 pub(crate) fn div(
243 size: OperandSize,
244 sign: DivSignedness,
245 trap: TrapCode,
246 divisor: RegMem,
247 dividend_lo: Gpr,
248 dividend_hi: Gpr,
249 dst_quotient: WritableGpr,
250 dst_remainder: WritableGpr,
251 ) -> Inst {
252 divisor.assert_regclass_is(RegClass::Int);
253 Inst::Div {
254 size,
255 sign,
256 trap,
257 divisor: GprMem::unwrap_new(divisor),
258 dividend_lo,
259 dividend_hi,
260 dst_quotient,
261 dst_remainder,
262 }
263 }
264
265 pub(crate) fn div8(
266 sign: DivSignedness,
267 trap: TrapCode,
268 divisor: RegMem,
269 dividend: Gpr,
270 dst: WritableGpr,
271 ) -> Inst {
272 divisor.assert_regclass_is(RegClass::Int);
273 Inst::Div8 {
274 sign,
275 trap,
276 divisor: GprMem::unwrap_new(divisor),
277 dividend,
278 dst,
279 }
280 }
281
282 pub(crate) fn imm(dst_size: OperandSize, simm64: u64, dst: Writable<Reg>) -> Inst {
283 debug_assert!(dst_size.is_one_of(&[OperandSize::Size32, OperandSize::Size64]));
284 debug_assert!(dst.to_reg().class() == RegClass::Int);
285 let dst_size = match dst_size {
288 OperandSize::Size64 if simm64 > u32::max_value() as u64 => OperandSize::Size64,
289 _ => OperandSize::Size32,
290 };
291 Inst::Imm {
292 dst_size,
293 simm64,
294 dst: WritableGpr::from_writable_reg(dst).unwrap(),
295 }
296 }
297
298 pub(crate) fn mov_r_r(size: OperandSize, src: Reg, dst: Writable<Reg>) -> Inst {
299 debug_assert!(size.is_one_of(&[OperandSize::Size32, OperandSize::Size64]));
300 debug_assert!(src.class() == RegClass::Int);
301 debug_assert!(dst.to_reg().class() == RegClass::Int);
302 let src = Gpr::unwrap_new(src);
303 let dst = WritableGpr::from_writable_reg(dst).unwrap();
304 Inst::MovRR { size, src, dst }
305 }
306
307 pub(crate) fn xmm_unary_rm_r(op: SseOpcode, src: RegMem, dst: Writable<Reg>) -> Inst {
309 src.assert_regclass_is(RegClass::Float);
310 debug_assert!(dst.to_reg().class() == RegClass::Float);
311 Inst::XmmUnaryRmR {
312 op,
313 src: XmmMemAligned::unwrap_new(src),
314 dst: WritableXmm::from_writable_reg(dst).unwrap(),
315 }
316 }
317
318 pub(crate) fn xmm_rm_r(op: SseOpcode, src: RegMem, dst: Writable<Reg>) -> Self {
319 src.assert_regclass_is(RegClass::Float);
320 debug_assert!(dst.to_reg().class() == RegClass::Float);
321 Inst::XmmRmR {
322 op,
323 src1: Xmm::unwrap_new(dst.to_reg()),
324 src2: XmmMemAligned::unwrap_new(src),
325 dst: WritableXmm::from_writable_reg(dst).unwrap(),
326 }
327 }
328
329 #[cfg(test)]
330 pub(crate) fn xmm_rmr_vex3(op: AvxOpcode, src3: RegMem, src2: Reg, dst: Writable<Reg>) -> Self {
331 src3.assert_regclass_is(RegClass::Float);
332 debug_assert!(src2.class() == RegClass::Float);
333 debug_assert!(dst.to_reg().class() == RegClass::Float);
334 Inst::XmmRmRVex3 {
335 op,
336 src3: XmmMem::unwrap_new(src3),
337 src2: Xmm::unwrap_new(src2),
338 src1: Xmm::unwrap_new(dst.to_reg()),
339 dst: WritableXmm::from_writable_reg(dst).unwrap(),
340 }
341 }
342
343 pub(crate) fn xmm_mov_r_m(op: SseOpcode, src: Reg, dst: impl Into<SyntheticAmode>) -> Inst {
344 debug_assert!(src.class() == RegClass::Float);
345 Inst::XmmMovRM {
346 op,
347 src: Xmm::unwrap_new(src),
348 dst: dst.into(),
349 }
350 }
351
352 pub(crate) fn xmm_to_gpr(
353 op: SseOpcode,
354 src: Reg,
355 dst: Writable<Reg>,
356 dst_size: OperandSize,
357 ) -> Inst {
358 debug_assert!(src.class() == RegClass::Float);
359 debug_assert!(dst.to_reg().class() == RegClass::Int);
360 debug_assert!(dst_size.is_one_of(&[OperandSize::Size32, OperandSize::Size64]));
361 Inst::XmmToGpr {
362 op,
363 src: Xmm::unwrap_new(src),
364 dst: WritableGpr::from_writable_reg(dst).unwrap(),
365 dst_size,
366 }
367 }
368
369 pub(crate) fn gpr_to_xmm(
370 op: SseOpcode,
371 src: RegMem,
372 src_size: OperandSize,
373 dst: Writable<Reg>,
374 ) -> Inst {
375 src.assert_regclass_is(RegClass::Int);
376 debug_assert!(src_size.is_one_of(&[OperandSize::Size32, OperandSize::Size64]));
377 debug_assert!(dst.to_reg().class() == RegClass::Float);
378 Inst::GprToXmm {
379 op,
380 src: GprMem::unwrap_new(src),
381 dst: WritableXmm::from_writable_reg(dst).unwrap(),
382 src_size,
383 }
384 }
385
386 pub(crate) fn xmm_cmp_rm_r(op: SseOpcode, src1: Reg, src2: RegMem) -> Inst {
387 src2.assert_regclass_is(RegClass::Float);
388 debug_assert!(src1.class() == RegClass::Float);
389 let src2 = XmmMemAligned::unwrap_new(src2);
390 let src1 = Xmm::unwrap_new(src1);
391 Inst::XmmCmpRmR { op, src1, src2 }
392 }
393
394 #[allow(dead_code)]
395 pub(crate) fn xmm_min_max_seq(
396 size: OperandSize,
397 is_min: bool,
398 lhs: Reg,
399 rhs: Reg,
400 dst: Writable<Reg>,
401 ) -> Inst {
402 debug_assert!(size.is_one_of(&[OperandSize::Size32, OperandSize::Size64]));
403 debug_assert_eq!(lhs.class(), RegClass::Float);
404 debug_assert_eq!(rhs.class(), RegClass::Float);
405 debug_assert_eq!(dst.to_reg().class(), RegClass::Float);
406 Inst::XmmMinMaxSeq {
407 size,
408 is_min,
409 lhs: Xmm::unwrap_new(lhs),
410 rhs: Xmm::unwrap_new(rhs),
411 dst: WritableXmm::from_writable_reg(dst).unwrap(),
412 }
413 }
414
415 pub(crate) fn movzx_rm_r(ext_mode: ExtMode, src: RegMem, dst: Writable<Reg>) -> Inst {
416 src.assert_regclass_is(RegClass::Int);
417 debug_assert!(dst.to_reg().class() == RegClass::Int);
418 let src = GprMem::unwrap_new(src);
419 let dst = WritableGpr::from_writable_reg(dst).unwrap();
420 Inst::MovzxRmR { ext_mode, src, dst }
421 }
422
423 pub(crate) fn movsx_rm_r(ext_mode: ExtMode, src: RegMem, dst: Writable<Reg>) -> Inst {
424 src.assert_regclass_is(RegClass::Int);
425 debug_assert!(dst.to_reg().class() == RegClass::Int);
426 let src = GprMem::unwrap_new(src);
427 let dst = WritableGpr::from_writable_reg(dst).unwrap();
428 Inst::MovsxRmR { ext_mode, src, dst }
429 }
430
431 pub(crate) fn mov64_m_r(src: impl Into<SyntheticAmode>, dst: Writable<Reg>) -> Inst {
432 debug_assert!(dst.to_reg().class() == RegClass::Int);
433 Inst::Mov64MR {
434 src: src.into(),
435 dst: WritableGpr::from_writable_reg(dst).unwrap(),
436 }
437 }
438
439 pub(crate) fn mov_r_m(size: OperandSize, src: Reg, dst: impl Into<SyntheticAmode>) -> Inst {
440 debug_assert!(src.class() == RegClass::Int);
441 Inst::MovRM {
442 size,
443 src: Gpr::unwrap_new(src),
444 dst: dst.into(),
445 }
446 }
447
448 pub(crate) fn lea(addr: impl Into<SyntheticAmode>, dst: Writable<Reg>) -> Inst {
449 debug_assert!(dst.to_reg().class() == RegClass::Int);
450 Inst::LoadEffectiveAddress {
451 addr: addr.into(),
452 dst: WritableGpr::from_writable_reg(dst).unwrap(),
453 size: OperandSize::Size64,
454 }
455 }
456
457 pub(crate) fn shift_r(
458 size: OperandSize,
459 kind: ShiftKind,
460 num_bits: Imm8Gpr,
461 src: Reg,
462 dst: Writable<Reg>,
463 ) -> Inst {
464 if let &Imm8Reg::Imm8 { imm: num_bits } = num_bits.as_imm8_reg() {
465 debug_assert!(num_bits < size.to_bits());
466 }
467 debug_assert!(dst.to_reg().class() == RegClass::Int);
468 Inst::ShiftR {
469 size,
470 kind,
471 src: Gpr::unwrap_new(src),
472 num_bits,
473 dst: WritableGpr::from_writable_reg(dst).unwrap(),
474 }
475 }
476
477 pub(crate) fn cmp_rmi_r(size: OperandSize, src1: Reg, src2: RegMemImm) -> Inst {
480 src2.assert_regclass_is(RegClass::Int);
481 debug_assert_eq!(src1.class(), RegClass::Int);
482 Inst::CmpRmiR {
483 size,
484 src1: Gpr::unwrap_new(src1),
485 src2: GprMemImm::unwrap_new(src2),
486 opcode: CmpOpcode::Cmp,
487 }
488 }
489
490 pub(crate) fn trap(trap_code: TrapCode) -> Inst {
491 Inst::Ud2 { trap_code }
492 }
493
494 pub(crate) fn trap_if(cc: CC, trap_code: TrapCode) -> Inst {
495 Inst::TrapIf { cc, trap_code }
496 }
497
498 pub(crate) fn cmove(size: OperandSize, cc: CC, src: RegMem, dst: Writable<Reg>) -> Inst {
499 debug_assert!(size.is_one_of(&[
500 OperandSize::Size16,
501 OperandSize::Size32,
502 OperandSize::Size64
503 ]));
504 debug_assert!(dst.to_reg().class() == RegClass::Int);
505 Inst::Cmove {
506 size,
507 cc,
508 consequent: GprMem::unwrap_new(src),
509 alternative: Gpr::unwrap_new(dst.to_reg()),
510 dst: WritableGpr::from_writable_reg(dst).unwrap(),
511 }
512 }
513
514 pub(crate) fn push64(src: RegMemImm) -> Inst {
515 src.assert_regclass_is(RegClass::Int);
516 let src = GprMemImm::unwrap_new(src);
517 Inst::Push64 { src }
518 }
519
520 pub(crate) fn pop64(dst: Writable<Reg>) -> Inst {
521 debug_assert!(dst.to_reg().class() == RegClass::Int);
522 let dst = WritableGpr::from_writable_reg(dst).unwrap();
523 Inst::Pop64 { dst }
524 }
525
526 pub(crate) fn call_known(info: Box<CallInfo<ExternalName>>) -> Inst {
527 Inst::CallKnown { info }
528 }
529
530 pub(crate) fn call_unknown(info: Box<CallInfo<RegMem>>) -> Inst {
531 info.dest.assert_regclass_is(RegClass::Int);
532 Inst::CallUnknown { info }
533 }
534
535 pub(crate) fn ret(stack_bytes_to_pop: u32) -> Inst {
536 Inst::Ret { stack_bytes_to_pop }
537 }
538
539 pub(crate) fn jmp_known(dst: MachLabel) -> Inst {
540 Inst::JmpKnown { dst }
541 }
542
543 pub(crate) fn jmp_unknown(target: RegMem) -> Inst {
544 target.assert_regclass_is(RegClass::Int);
545 Inst::JmpUnknown { target }
546 }
547
548 pub(crate) fn load(
552 ty: Type,
553 from_addr: impl Into<SyntheticAmode>,
554 to_reg: Writable<Reg>,
555 ext_kind: ExtKind,
556 ) -> Inst {
557 let rc = to_reg.to_reg().class();
558 match rc {
559 RegClass::Int => {
560 let ext_mode = match ty.bytes() {
561 1 => Some(ExtMode::BQ),
562 2 => Some(ExtMode::WQ),
563 4 => Some(ExtMode::LQ),
564 8 => None,
565 _ => unreachable!("the type should never use a scalar load: {}", ty),
566 };
567 if let Some(ext_mode) = ext_mode {
568 match ext_kind {
570 ExtKind::SignExtend => {
571 Inst::movsx_rm_r(ext_mode, RegMem::mem(from_addr), to_reg)
572 }
573 ExtKind::ZeroExtend => {
574 Inst::movzx_rm_r(ext_mode, RegMem::mem(from_addr), to_reg)
575 }
576 ExtKind::None => {
577 panic!("expected an extension kind for extension mode: {ext_mode:?}")
578 }
579 }
580 } else {
581 Inst::mov64_m_r(from_addr, to_reg)
583 }
584 }
585 RegClass::Float => {
586 let opcode = match ty {
587 types::F16 => panic!("loading a f16 requires multiple instructions"),
588 types::F32 => SseOpcode::Movss,
589 types::F64 => SseOpcode::Movsd,
590 types::F32X4 => SseOpcode::Movups,
591 types::F64X2 => SseOpcode::Movupd,
592 _ if (ty.is_float() || ty.is_vector()) && ty.bits() == 128 => SseOpcode::Movdqu,
593 _ => unimplemented!("unable to load type: {}", ty),
594 };
595 Inst::xmm_unary_rm_r(opcode, RegMem::mem(from_addr), to_reg)
596 }
597 RegClass::Vector => unreachable!(),
598 }
599 }
600
601 pub(crate) fn store(ty: Type, from_reg: Reg, to_addr: impl Into<SyntheticAmode>) -> Inst {
603 let rc = from_reg.class();
604 match rc {
605 RegClass::Int => Inst::mov_r_m(OperandSize::from_ty(ty), from_reg, to_addr),
606 RegClass::Float => {
607 let opcode = match ty {
608 types::F16 => panic!("storing a f16 requires multiple instructions"),
609 types::F32 => SseOpcode::Movss,
610 types::F64 => SseOpcode::Movsd,
611 types::F32X4 => SseOpcode::Movups,
612 types::F64X2 => SseOpcode::Movupd,
613 _ if (ty.is_float() || ty.is_vector()) && ty.bits() == 128 => SseOpcode::Movdqu,
614 _ => unimplemented!("unable to store type: {}", ty),
615 };
616 Inst::xmm_mov_r_m(opcode, from_reg, to_addr)
617 }
618 RegClass::Vector => unreachable!(),
619 }
620 }
621}
622
623impl PrettyPrint for Inst {
627 fn pretty_print(&self, _size: u8) -> String {
628 fn ljustify(s: String) -> String {
629 let w = 7;
630 if s.len() >= w {
631 s
632 } else {
633 let need = usize::min(w, w - s.len());
634 s + &format!("{nil: <width$}", nil = "", width = need)
635 }
636 }
637
638 fn ljustify2(s1: String, s2: String) -> String {
639 ljustify(s1 + &s2)
640 }
641
642 fn suffix_lq(size: OperandSize) -> String {
643 match size {
644 OperandSize::Size32 => "l",
645 OperandSize::Size64 => "q",
646 _ => unreachable!(),
647 }
648 .to_string()
649 }
650
651 #[allow(dead_code)]
652 fn suffix_lqb(size: OperandSize) -> String {
653 match size {
654 OperandSize::Size32 => "l",
655 OperandSize::Size64 => "q",
656 _ => unreachable!(),
657 }
658 .to_string()
659 }
660
661 fn suffix_bwlq(size: OperandSize) -> String {
662 match size {
663 OperandSize::Size8 => "b".to_string(),
664 OperandSize::Size16 => "w".to_string(),
665 OperandSize::Size32 => "l".to_string(),
666 OperandSize::Size64 => "q".to_string(),
667 }
668 }
669
670 match self {
671 Inst::Nop { len } => format!("{} len={}", ljustify("nop".to_string()), len),
672
673 Inst::AluRmiR {
674 size,
675 op,
676 src1,
677 src2,
678 dst,
679 } => {
680 let size_bytes = size.to_bytes();
681 let src1 = pretty_print_reg(src1.to_reg(), size_bytes);
682 let dst = pretty_print_reg(dst.to_reg().to_reg(), size_bytes);
683 let src2 = src2.pretty_print(size_bytes);
684 let op = ljustify2(op.to_string(), suffix_bwlq(*size));
685 format!("{op} {src1}, {src2}, {dst}")
686 }
687 Inst::AluConstOp { op, dst, size } => {
688 let size_bytes = size.to_bytes();
689 let dst = pretty_print_reg(dst.to_reg().to_reg(), size_bytes);
690 let op = ljustify2(op.to_string(), suffix_lqb(*size));
691 format!("{op} {dst}, {dst}, {dst}")
692 }
693 Inst::AluRM {
694 size,
695 op,
696 src1_dst,
697 src2,
698 } => {
699 let size_bytes = size.to_bytes();
700 let src2 = pretty_print_reg(src2.to_reg(), size_bytes);
701 let src1_dst = src1_dst.pretty_print(size_bytes);
702 let op = ljustify2(op.to_string(), suffix_bwlq(*size));
703 format!("{op} {src2}, {src1_dst}")
704 }
705 Inst::AluRmRVex {
706 size,
707 op,
708 src1,
709 src2,
710 dst,
711 } => {
712 let size_bytes = size.to_bytes();
713 let dst = pretty_print_reg(dst.to_reg().to_reg(), size.to_bytes());
714 let src1 = pretty_print_reg(src1.to_reg(), size_bytes);
715 let src2 = src2.pretty_print(size_bytes);
716 let op = ljustify2(op.to_string(), String::new());
717 format!("{op} {src2}, {src1}, {dst}")
718 }
719 Inst::UnaryRmR { src, dst, op, size } => {
720 let dst = pretty_print_reg(dst.to_reg().to_reg(), size.to_bytes());
721 let src = src.pretty_print(size.to_bytes());
722 let op = ljustify2(op.to_string(), suffix_bwlq(*size));
723 format!("{op} {src}, {dst}")
724 }
725
726 Inst::UnaryRmRVex { src, dst, op, size } => {
727 let dst = pretty_print_reg(dst.to_reg().to_reg(), size.to_bytes());
728 let src = src.pretty_print(size.to_bytes());
729 let op = ljustify2(op.to_string(), suffix_bwlq(*size));
730 format!("{op} {src}, {dst}")
731 }
732
733 Inst::UnaryRmRImmVex {
734 src,
735 dst,
736 op,
737 size,
738 imm,
739 } => {
740 let dst = pretty_print_reg(dst.to_reg().to_reg(), size.to_bytes());
741 let src = src.pretty_print(size.to_bytes());
742 format!(
743 "{} ${imm}, {src}, {dst}",
744 ljustify2(op.to_string(), suffix_bwlq(*size))
745 )
746 }
747
748 Inst::Not { size, src, dst } => {
749 let src = pretty_print_reg(src.to_reg(), size.to_bytes());
750 let dst = pretty_print_reg(dst.to_reg().to_reg(), size.to_bytes());
751 let op = ljustify2("not".to_string(), suffix_bwlq(*size));
752 format!("{op} {src}, {dst}")
753 }
754
755 Inst::Neg { size, src, dst } => {
756 let src = pretty_print_reg(src.to_reg(), size.to_bytes());
757 let dst = pretty_print_reg(dst.to_reg().to_reg(), size.to_bytes());
758 let op = ljustify2("neg".to_string(), suffix_bwlq(*size));
759 format!("{op} {src}, {dst}")
760 }
761
762 Inst::Div {
763 size,
764 sign,
765 trap,
766 divisor,
767 dividend_lo,
768 dividend_hi,
769 dst_quotient,
770 dst_remainder,
771 } => {
772 let divisor = divisor.pretty_print(size.to_bytes());
773 let dividend_lo = pretty_print_reg(dividend_lo.to_reg(), size.to_bytes());
774 let dividend_hi = pretty_print_reg(dividend_hi.to_reg(), size.to_bytes());
775 let dst_quotient =
776 pretty_print_reg(dst_quotient.to_reg().to_reg(), size.to_bytes());
777 let dst_remainder =
778 pretty_print_reg(dst_remainder.to_reg().to_reg(), size.to_bytes());
779 let op = ljustify(match sign {
780 DivSignedness::Signed => "idiv".to_string(),
781 DivSignedness::Unsigned => "div".to_string(),
782 });
783 format!(
784 "{op} {dividend_lo}, {dividend_hi}, {divisor}, {dst_quotient}, {dst_remainder} ; trap={trap}"
785 )
786 }
787
788 Inst::Div8 {
789 sign,
790 trap,
791 divisor,
792 dividend,
793 dst,
794 } => {
795 let divisor = divisor.pretty_print(1);
796 let dividend = pretty_print_reg(dividend.to_reg(), 1);
797 let dst = pretty_print_reg(dst.to_reg().to_reg(), 1);
798 let op = ljustify(match sign {
799 DivSignedness::Signed => "idiv".to_string(),
800 DivSignedness::Unsigned => "div".to_string(),
801 });
802 format!("{op} {dividend}, {divisor}, {dst} ; trap={trap}")
803 }
804
805 Inst::Mul {
806 size,
807 signed,
808 src1,
809 src2,
810 dst_lo,
811 dst_hi,
812 } => {
813 let src1 = pretty_print_reg(src1.to_reg(), size.to_bytes());
814 let dst_lo = pretty_print_reg(dst_lo.to_reg().to_reg(), size.to_bytes());
815 let dst_hi = pretty_print_reg(dst_hi.to_reg().to_reg(), size.to_bytes());
816 let src2 = src2.pretty_print(size.to_bytes());
817 let suffix = suffix_bwlq(*size);
818 let op = ljustify(if *signed {
819 format!("imul{suffix}")
820 } else {
821 format!("mul{suffix}")
822 });
823 format!("{op} {src1}, {src2}, {dst_lo}, {dst_hi}")
824 }
825
826 Inst::Mul8 {
827 signed,
828 src1,
829 src2,
830 dst,
831 } => {
832 let src1 = pretty_print_reg(src1.to_reg(), 1);
833 let dst = pretty_print_reg(dst.to_reg().to_reg(), 1);
834 let src2 = src2.pretty_print(1);
835 let op = ljustify(if *signed {
836 "imulb".to_string()
837 } else {
838 "mulb".to_string()
839 });
840 format!("{op} {src1}, {src2}, {dst}")
841 }
842
843 Inst::IMul {
844 size,
845 src1,
846 src2,
847 dst,
848 } => {
849 let src1 = pretty_print_reg(src1.to_reg(), size.to_bytes());
850 let dst = pretty_print_reg(dst.to_reg().to_reg(), size.to_bytes());
851 let src2 = src2.pretty_print(size.to_bytes());
852 let suffix = suffix_bwlq(*size);
853 let op = ljustify(format!("imul{suffix}"));
854 format!("{op} {src1}, {src2}, {dst}")
855 }
856
857 Inst::IMulImm {
858 size,
859 src1,
860 src2,
861 dst,
862 } => {
863 let dst = pretty_print_reg(dst.to_reg().to_reg(), size.to_bytes());
864 let src1 = src1.pretty_print(size.to_bytes());
865 let suffix = suffix_bwlq(*size);
866 let op = ljustify(format!("imul{suffix}"));
867 format!("{op} {src1}, {src2:#x}, {dst}")
868 }
869
870 Inst::CheckedSRemSeq {
871 size,
872 divisor,
873 dividend_lo,
874 dividend_hi,
875 dst_quotient,
876 dst_remainder,
877 } => {
878 let divisor = pretty_print_reg(divisor.to_reg(), size.to_bytes());
879 let dividend_lo = pretty_print_reg(dividend_lo.to_reg(), size.to_bytes());
880 let dividend_hi = pretty_print_reg(dividend_hi.to_reg(), size.to_bytes());
881 let dst_quotient =
882 pretty_print_reg(dst_quotient.to_reg().to_reg(), size.to_bytes());
883 let dst_remainder =
884 pretty_print_reg(dst_remainder.to_reg().to_reg(), size.to_bytes());
885 format!(
886 "checked_srem_seq {dividend_lo}, {dividend_hi}, \
887 {divisor}, {dst_quotient}, {dst_remainder}",
888 )
889 }
890
891 Inst::CheckedSRemSeq8 {
892 divisor,
893 dividend,
894 dst,
895 } => {
896 let divisor = pretty_print_reg(divisor.to_reg(), 1);
897 let dividend = pretty_print_reg(dividend.to_reg(), 1);
898 let dst = pretty_print_reg(dst.to_reg().to_reg(), 1);
899 format!("checked_srem_seq {dividend}, {divisor}, {dst}")
900 }
901
902 Inst::SignExtendData { size, src, dst } => {
903 let src = pretty_print_reg(src.to_reg(), size.to_bytes());
904 let dst = pretty_print_reg(dst.to_reg().to_reg(), size.to_bytes());
905 let op = match size {
906 OperandSize::Size8 => "cbw",
907 OperandSize::Size16 => "cwd",
908 OperandSize::Size32 => "cdq",
909 OperandSize::Size64 => "cqo",
910 };
911 format!("{op} {src}, {dst}")
912 }
913
914 Inst::XmmUnaryRmR { op, src, dst, .. } => {
915 let dst = pretty_print_reg(dst.to_reg().to_reg(), op.src_size());
916 let src = src.pretty_print(op.src_size());
917 let op = ljustify(op.to_string());
918 format!("{op} {src}, {dst}")
919 }
920
921 Inst::XmmUnaryRmRUnaligned { op, src, dst, .. } => {
922 let dst = pretty_print_reg(dst.to_reg().to_reg(), op.src_size());
923 let src = src.pretty_print(op.src_size());
924 let op = ljustify(op.to_string());
925 format!("{op} {src}, {dst}")
926 }
927
928 Inst::XmmUnaryRmRImm {
929 op, src, dst, imm, ..
930 } => {
931 let dst = pretty_print_reg(dst.to_reg().to_reg(), op.src_size());
932 let src = src.pretty_print(op.src_size());
933 let op = ljustify(op.to_string());
934 format!("{op} ${imm}, {src}, {dst}")
935 }
936
937 Inst::XmmUnaryRmRVex { op, src, dst, .. } => {
938 let dst = pretty_print_reg(dst.to_reg().to_reg(), 8);
939 let src = src.pretty_print(8);
940 let op = ljustify(op.to_string());
941 format!("{op} {src}, {dst}")
942 }
943
944 Inst::XmmUnaryRmRImmVex {
945 op, src, dst, imm, ..
946 } => {
947 let dst = pretty_print_reg(dst.to_reg().to_reg(), 8);
948 let src = src.pretty_print(8);
949 let op = ljustify(op.to_string());
950 format!("{op} ${imm}, {src}, {dst}")
951 }
952
953 Inst::XmmUnaryRmREvex { op, src, dst, .. } => {
954 let dst = pretty_print_reg(dst.to_reg().to_reg(), 8);
955 let src = src.pretty_print(8);
956 let op = ljustify(op.to_string());
957 format!("{op} {src}, {dst}")
958 }
959
960 Inst::XmmUnaryRmRImmEvex {
961 op, src, dst, imm, ..
962 } => {
963 let dst = pretty_print_reg(dst.to_reg().to_reg(), 8);
964 let src = src.pretty_print(8);
965 let op = ljustify(op.to_string());
966 format!("{op} ${imm}, {src}, {dst}")
967 }
968
969 Inst::XmmMovRM { op, src, dst, .. } => {
970 let src = pretty_print_reg(src.to_reg(), 8);
971 let dst = dst.pretty_print(8);
972 let op = ljustify(op.to_string());
973 format!("{op} {src}, {dst}")
974 }
975
976 Inst::XmmMovRMVex { op, src, dst, .. } => {
977 let src = pretty_print_reg(src.to_reg(), 8);
978 let dst = dst.pretty_print(8);
979 let op = ljustify(op.to_string());
980 format!("{op} {src}, {dst}")
981 }
982
983 Inst::XmmMovRMImm {
984 op, src, dst, imm, ..
985 } => {
986 let src = pretty_print_reg(src.to_reg(), 8);
987 let dst = dst.pretty_print(8);
988 let op = ljustify(op.to_string());
989 format!("{op} ${imm}, {src}, {dst}")
990 }
991
992 Inst::XmmMovRMImmVex {
993 op, src, dst, imm, ..
994 } => {
995 let src = pretty_print_reg(src.to_reg(), 8);
996 let dst = dst.pretty_print(8);
997 let op = ljustify(op.to_string());
998 format!("{op} ${imm}, {src}, {dst}")
999 }
1000
1001 Inst::XmmRmR {
1002 op,
1003 src1,
1004 src2,
1005 dst,
1006 ..
1007 } => {
1008 let src1 = pretty_print_reg(src1.to_reg(), 8);
1009 let dst = pretty_print_reg(dst.to_reg().to_reg(), 8);
1010 let src2 = src2.pretty_print(8);
1011 let op = ljustify(op.to_string());
1012 format!("{op} {src1}, {src2}, {dst}")
1013 }
1014
1015 Inst::XmmRmRUnaligned {
1016 op,
1017 src1,
1018 src2,
1019 dst,
1020 ..
1021 } => {
1022 let src1 = pretty_print_reg(src1.to_reg(), 8);
1023 let dst = pretty_print_reg(dst.to_reg().to_reg(), 8);
1024 let src2 = src2.pretty_print(8);
1025 let op = ljustify(op.to_string());
1026 format!("{op} {src1}, {src2}, {dst}")
1027 }
1028
1029 Inst::XmmRmRBlend {
1030 op,
1031 src1,
1032 src2,
1033 mask,
1034 dst,
1035 } => {
1036 let src1 = pretty_print_reg(src1.to_reg(), 8);
1037 let mask = mask.to_reg();
1038 let mask = if mask.is_virtual() {
1039 format!(" <{}>", show_ireg_sized(mask, 8))
1040 } else {
1041 debug_assert_eq!(mask, regs::xmm0());
1042 String::new()
1043 };
1044 let dst = pretty_print_reg(dst.to_reg().to_reg(), 8);
1045 let src2 = src2.pretty_print(8);
1046 let op = ljustify(op.to_string());
1047 format!("{op} {src1}, {src2}, {dst}{mask}")
1048 }
1049
1050 Inst::XmmRmiRVex {
1051 op,
1052 src1,
1053 src2,
1054 dst,
1055 ..
1056 } => {
1057 let dst = pretty_print_reg(dst.to_reg().to_reg(), 8);
1058 let src1 = pretty_print_reg(src1.to_reg(), 8);
1059 let src2 = src2.pretty_print(8);
1060 let op = ljustify(op.to_string());
1061 format!("{op} {src1}, {src2}, {dst}")
1062 }
1063
1064 Inst::XmmRmRImmVex {
1065 op,
1066 src1,
1067 src2,
1068 dst,
1069 imm,
1070 ..
1071 } => {
1072 let dst = pretty_print_reg(dst.to_reg().to_reg(), 8);
1073 let src1 = pretty_print_reg(src1.to_reg(), 8);
1074 let src2 = src2.pretty_print(8);
1075 let op = ljustify(op.to_string());
1076 format!("{op} ${imm}, {src1}, {src2}, {dst}")
1077 }
1078
1079 Inst::XmmVexPinsr {
1080 op,
1081 src1,
1082 src2,
1083 dst,
1084 imm,
1085 ..
1086 } => {
1087 let dst = pretty_print_reg(dst.to_reg().to_reg(), 8);
1088 let src1 = pretty_print_reg(src1.to_reg(), 8);
1089 let src2 = src2.pretty_print(8);
1090 let op = ljustify(op.to_string());
1091 format!("{op} ${imm}, {src1}, {src2}, {dst}")
1092 }
1093
1094 Inst::XmmRmRVex3 {
1095 op,
1096 src1,
1097 src2,
1098 src3,
1099 dst,
1100 ..
1101 } => {
1102 let src1 = pretty_print_reg(src1.to_reg(), 8);
1103 let dst = pretty_print_reg(dst.to_reg().to_reg(), 8);
1104 let src2 = pretty_print_reg(src2.to_reg(), 8);
1105 let src3 = src3.pretty_print(8);
1106 let op = ljustify(op.to_string());
1107 format!("{op} {src1}, {src2}, {src3}, {dst}")
1108 }
1109
1110 Inst::XmmRmRBlendVex {
1111 op,
1112 src1,
1113 src2,
1114 mask,
1115 dst,
1116 ..
1117 } => {
1118 let src1 = pretty_print_reg(src1.to_reg(), 8);
1119 let dst = pretty_print_reg(dst.to_reg().to_reg(), 8);
1120 let src2 = src2.pretty_print(8);
1121 let mask = pretty_print_reg(mask.to_reg(), 8);
1122 let op = ljustify(op.to_string());
1123 format!("{op} {src1}, {src2}, {mask}, {dst}")
1124 }
1125
1126 Inst::XmmRmREvex {
1127 op,
1128 src1,
1129 src2,
1130 dst,
1131 ..
1132 } => {
1133 let src1 = pretty_print_reg(src1.to_reg(), 8);
1134 let src2 = src2.pretty_print(8);
1135 let dst = pretty_print_reg(dst.to_reg().to_reg(), 8);
1136 let op = ljustify(op.to_string());
1137 format!("{op} {src2}, {src1}, {dst}")
1138 }
1139
1140 Inst::XmmRmREvex3 {
1141 op,
1142 src1,
1143 src2,
1144 src3,
1145 dst,
1146 ..
1147 } => {
1148 let src1 = pretty_print_reg(src1.to_reg(), 8);
1149 let src2 = pretty_print_reg(src2.to_reg(), 8);
1150 let src3 = src3.pretty_print(8);
1151 let dst = pretty_print_reg(dst.to_reg().to_reg(), 8);
1152 let op = ljustify(op.to_string());
1153 format!("{op} {src3}, {src2}, {src1}, {dst}")
1154 }
1155
1156 Inst::XmmMinMaxSeq {
1157 lhs,
1158 rhs,
1159 dst,
1160 is_min,
1161 size,
1162 } => {
1163 let rhs = pretty_print_reg(rhs.to_reg(), 8);
1164 let lhs = pretty_print_reg(lhs.to_reg(), 8);
1165 let dst = pretty_print_reg(dst.to_reg().to_reg(), 8);
1166 let op = ljustify2(
1167 if *is_min {
1168 "xmm min seq ".to_string()
1169 } else {
1170 "xmm max seq ".to_string()
1171 },
1172 format!("f{}", size.to_bits()),
1173 );
1174 format!("{op} {lhs}, {rhs}, {dst}")
1175 }
1176
1177 Inst::XmmRmRImm {
1178 op,
1179 src1,
1180 src2,
1181 dst,
1182 imm,
1183 size,
1184 ..
1185 } => {
1186 let src1 = pretty_print_reg(*src1, 8);
1187 let dst = pretty_print_reg(dst.to_reg(), 8);
1188 let src2 = src2.pretty_print(8);
1189 let op = ljustify(format!(
1190 "{}{}",
1191 op.to_string(),
1192 if *size == OperandSize::Size64 {
1193 ".w"
1194 } else {
1195 ""
1196 }
1197 ));
1198 format!("{op} ${imm}, {src1}, {src2}, {dst}")
1199 }
1200
1201 Inst::XmmUninitializedValue { dst } => {
1202 let dst = pretty_print_reg(dst.to_reg().to_reg(), 8);
1203 let op = ljustify("uninit".into());
1204 format!("{op} {dst}")
1205 }
1206
1207 Inst::XmmToGpr {
1208 op,
1209 src,
1210 dst,
1211 dst_size,
1212 } => {
1213 let dst_size = dst_size.to_bytes();
1214 let src = pretty_print_reg(src.to_reg(), 8);
1215 let dst = pretty_print_reg(dst.to_reg().to_reg(), dst_size);
1216 let op = ljustify(op.to_string());
1217 format!("{op} {src}, {dst}")
1218 }
1219
1220 Inst::XmmToGprVex {
1221 op,
1222 src,
1223 dst,
1224 dst_size,
1225 } => {
1226 let dst_size = dst_size.to_bytes();
1227 let src = pretty_print_reg(src.to_reg(), 8);
1228 let dst = pretty_print_reg(dst.to_reg().to_reg(), dst_size);
1229 let op = ljustify(op.to_string());
1230 format!("{op} {src}, {dst}")
1231 }
1232
1233 Inst::XmmToGprImm { op, src, dst, imm } => {
1234 let src = pretty_print_reg(src.to_reg(), 8);
1235 let dst = pretty_print_reg(dst.to_reg().to_reg(), 8);
1236 let op = ljustify(op.to_string());
1237 format!("{op} ${imm}, {src}, {dst}")
1238 }
1239
1240 Inst::XmmToGprImmVex { op, src, dst, imm } => {
1241 let src = pretty_print_reg(src.to_reg(), 8);
1242 let dst = pretty_print_reg(dst.to_reg().to_reg(), 8);
1243 let op = ljustify(op.to_string());
1244 format!("{op} ${imm}, {src}, {dst}")
1245 }
1246
1247 Inst::GprToXmm {
1248 op,
1249 src,
1250 src_size,
1251 dst,
1252 } => {
1253 let dst = pretty_print_reg(dst.to_reg().to_reg(), 8);
1254 let src = src.pretty_print(src_size.to_bytes());
1255 let op = ljustify(op.to_string());
1256 format!("{op} {src}, {dst}")
1257 }
1258
1259 Inst::GprToXmmVex {
1260 op,
1261 src,
1262 src_size,
1263 dst,
1264 } => {
1265 let dst = pretty_print_reg(dst.to_reg().to_reg(), 8);
1266 let src = src.pretty_print(src_size.to_bytes());
1267 let op = ljustify(op.to_string());
1268 format!("{op} {src}, {dst}")
1269 }
1270
1271 Inst::XmmCmpRmR { op, src1, src2 } => {
1272 let src1 = pretty_print_reg(src1.to_reg(), 8);
1273 let src2 = src2.pretty_print(8);
1274 let op = ljustify(op.to_string());
1275 format!("{op} {src2}, {src1}")
1276 }
1277
1278 Inst::CvtIntToFloat {
1279 op,
1280 src1,
1281 src2,
1282 dst,
1283 src2_size,
1284 } => {
1285 let src1 = pretty_print_reg(src1.to_reg(), 8);
1286 let dst = pretty_print_reg(*dst.to_reg(), 8);
1287 let src2 = src2.pretty_print(src2_size.to_bytes());
1288 let op = ljustify(op.to_string());
1289 format!("{op} {src1}, {src2}, {dst}")
1290 }
1291
1292 Inst::CvtIntToFloatVex {
1293 op,
1294 src1,
1295 src2,
1296 dst,
1297 src2_size,
1298 } => {
1299 let dst = pretty_print_reg(*dst.to_reg(), 8);
1300 let src1 = pretty_print_reg(src1.to_reg(), 8);
1301 let src2 = src2.pretty_print(src2_size.to_bytes());
1302 let op = ljustify(op.to_string());
1303 format!("{op} {src1}, {src2}, {dst}")
1304 }
1305
1306 Inst::XmmCmpRmRVex { op, src1, src2 } => {
1307 let src1 = pretty_print_reg(src1.to_reg(), 8);
1308 let src2 = src2.pretty_print(8);
1309 format!("{} {src2}, {src1}", ljustify(op.to_string()))
1310 }
1311
1312 Inst::CvtUint64ToFloatSeq {
1313 src,
1314 dst,
1315 dst_size,
1316 tmp_gpr1,
1317 tmp_gpr2,
1318 ..
1319 } => {
1320 let src = pretty_print_reg(src.to_reg(), 8);
1321 let dst = pretty_print_reg(dst.to_reg().to_reg(), dst_size.to_bytes());
1322 let tmp_gpr1 = pretty_print_reg(tmp_gpr1.to_reg().to_reg(), 8);
1323 let tmp_gpr2 = pretty_print_reg(tmp_gpr2.to_reg().to_reg(), 8);
1324 let op = ljustify(format!(
1325 "u64_to_{}_seq",
1326 if *dst_size == OperandSize::Size64 {
1327 "f64"
1328 } else {
1329 "f32"
1330 }
1331 ));
1332 format!("{op} {src}, {dst}, {tmp_gpr1}, {tmp_gpr2}")
1333 }
1334
1335 Inst::CvtFloatToSintSeq {
1336 src,
1337 dst,
1338 src_size,
1339 dst_size,
1340 tmp_xmm,
1341 tmp_gpr,
1342 is_saturating,
1343 } => {
1344 let src = pretty_print_reg(src.to_reg(), src_size.to_bytes());
1345 let dst = pretty_print_reg(dst.to_reg().to_reg(), dst_size.to_bytes());
1346 let tmp_gpr = pretty_print_reg(tmp_gpr.to_reg().to_reg(), 8);
1347 let tmp_xmm = pretty_print_reg(tmp_xmm.to_reg().to_reg(), 8);
1348 let op = ljustify(format!(
1349 "cvt_float{}_to_sint{}{}_seq",
1350 src_size.to_bits(),
1351 dst_size.to_bits(),
1352 if *is_saturating { "_sat" } else { "" },
1353 ));
1354 format!("{op} {src}, {dst}, {tmp_gpr}, {tmp_xmm}")
1355 }
1356
1357 Inst::CvtFloatToUintSeq {
1358 src,
1359 dst,
1360 src_size,
1361 dst_size,
1362 tmp_gpr,
1363 tmp_xmm,
1364 tmp_xmm2,
1365 is_saturating,
1366 } => {
1367 let src = pretty_print_reg(src.to_reg(), src_size.to_bytes());
1368 let dst = pretty_print_reg(dst.to_reg().to_reg(), dst_size.to_bytes());
1369 let tmp_gpr = pretty_print_reg(tmp_gpr.to_reg().to_reg(), 8);
1370 let tmp_xmm = pretty_print_reg(tmp_xmm.to_reg().to_reg(), 8);
1371 let tmp_xmm2 = pretty_print_reg(tmp_xmm2.to_reg().to_reg(), 8);
1372 let op = ljustify(format!(
1373 "cvt_float{}_to_uint{}{}_seq",
1374 src_size.to_bits(),
1375 dst_size.to_bits(),
1376 if *is_saturating { "_sat" } else { "" },
1377 ));
1378 format!("{op} {src}, {dst}, {tmp_gpr}, {tmp_xmm}, {tmp_xmm2}")
1379 }
1380
1381 Inst::Imm {
1382 dst_size,
1383 simm64,
1384 dst,
1385 } => {
1386 let dst = pretty_print_reg(dst.to_reg().to_reg(), dst_size.to_bytes());
1387 if *dst_size == OperandSize::Size64 {
1388 let op = ljustify("movabsq".to_string());
1389 let imm = *simm64 as i64;
1390 format!("{op} ${imm}, {dst}")
1391 } else {
1392 let op = ljustify("movl".to_string());
1393 let imm = (*simm64 as u32) as i32;
1394 format!("{op} ${imm}, {dst}")
1395 }
1396 }
1397
1398 Inst::MovImmM { size, simm32, dst } => {
1399 let dst = dst.pretty_print(size.to_bytes());
1400 let suffix = suffix_bwlq(*size);
1401 let imm = match *size {
1402 OperandSize::Size8 => ((*simm32 as u8) as i8).to_string(),
1403 OperandSize::Size16 => ((*simm32 as u16) as i16).to_string(),
1404 OperandSize::Size32 => simm32.to_string(),
1405 OperandSize::Size64 => (*simm32 as i64).to_string(),
1406 };
1407 let op = ljustify2("mov".to_string(), suffix);
1408 format!("{op} ${imm}, {dst}")
1409 }
1410
1411 Inst::MovRR { size, src, dst } => {
1412 let src = pretty_print_reg(src.to_reg(), size.to_bytes());
1413 let dst = pretty_print_reg(dst.to_reg().to_reg(), size.to_bytes());
1414 let op = ljustify2("mov".to_string(), suffix_lq(*size));
1415 format!("{op} {src}, {dst}")
1416 }
1417
1418 Inst::MovFromPReg { src, dst } => {
1419 let src: Reg = (*src).into();
1420 let src = regs::show_ireg_sized(src, 8);
1421 let dst = pretty_print_reg(dst.to_reg().to_reg(), 8);
1422 let op = ljustify("movq".to_string());
1423 format!("{op} {src}, {dst}")
1424 }
1425
1426 Inst::MovToPReg { src, dst } => {
1427 let src = pretty_print_reg(src.to_reg(), 8);
1428 let dst: Reg = (*dst).into();
1429 let dst = regs::show_ireg_sized(dst, 8);
1430 let op = ljustify("movq".to_string());
1431 format!("{op} {src}, {dst}")
1432 }
1433
1434 Inst::MovzxRmR {
1435 ext_mode, src, dst, ..
1436 } => {
1437 let dst_size = if *ext_mode == ExtMode::LQ {
1438 4
1439 } else {
1440 ext_mode.dst_size()
1441 };
1442 let dst = pretty_print_reg(dst.to_reg().to_reg(), dst_size);
1443 let src = src.pretty_print(ext_mode.src_size());
1444
1445 if *ext_mode == ExtMode::LQ {
1446 let op = ljustify("movl".to_string());
1447 format!("{op} {src}, {dst}")
1448 } else {
1449 let op = ljustify2("movz".to_string(), ext_mode.to_string());
1450 format!("{op} {src}, {dst}")
1451 }
1452 }
1453
1454 Inst::Mov64MR { src, dst, .. } => {
1455 let dst = pretty_print_reg(dst.to_reg().to_reg(), 8);
1456 let src = src.pretty_print(8);
1457 let op = ljustify("movq".to_string());
1458 format!("{op} {src}, {dst}")
1459 }
1460
1461 Inst::LoadEffectiveAddress { addr, dst, size } => {
1462 let dst = pretty_print_reg(dst.to_reg().to_reg(), size.to_bytes());
1463 let addr = addr.pretty_print(8);
1464 let op = ljustify("lea".to_string());
1465 format!("{op} {addr}, {dst}")
1466 }
1467
1468 Inst::MovsxRmR {
1469 ext_mode, src, dst, ..
1470 } => {
1471 let dst = pretty_print_reg(dst.to_reg().to_reg(), ext_mode.dst_size());
1472 let src = src.pretty_print(ext_mode.src_size());
1473 let op = ljustify2("movs".to_string(), ext_mode.to_string());
1474 format!("{op} {src}, {dst}")
1475 }
1476
1477 Inst::MovRM { size, src, dst, .. } => {
1478 let src = pretty_print_reg(src.to_reg(), size.to_bytes());
1479 let dst = dst.pretty_print(size.to_bytes());
1480 let op = ljustify2("mov".to_string(), suffix_bwlq(*size));
1481 format!("{op} {src}, {dst}")
1482 }
1483
1484 Inst::ShiftR {
1485 size,
1486 kind,
1487 num_bits,
1488 src,
1489 dst,
1490 ..
1491 } => {
1492 let src = pretty_print_reg(src.to_reg(), size.to_bytes());
1493 let dst = pretty_print_reg(dst.to_reg().to_reg(), size.to_bytes());
1494 match num_bits.as_imm8_reg() {
1495 &Imm8Reg::Reg { reg } => {
1496 let reg = pretty_print_reg(reg, 1);
1497 let op = ljustify2(kind.to_string(), suffix_bwlq(*size));
1498 format!("{op} {reg}, {src}, {dst}")
1499 }
1500
1501 &Imm8Reg::Imm8 { imm: num_bits } => {
1502 let op = ljustify2(kind.to_string(), suffix_bwlq(*size));
1503 format!("{op} ${num_bits}, {src}, {dst}")
1504 }
1505 }
1506 }
1507
1508 Inst::XmmRmiReg {
1509 opcode,
1510 src1,
1511 src2,
1512 dst,
1513 ..
1514 } => {
1515 let src1 = pretty_print_reg(src1.to_reg(), 8);
1516 let dst = pretty_print_reg(dst.to_reg().to_reg(), 8);
1517 let src2 = src2.pretty_print(8);
1518 let op = ljustify(opcode.to_string());
1519 format!("{op} {src1}, {src2}, {dst}")
1520 }
1521
1522 Inst::CmpRmiR {
1523 size,
1524 src1,
1525 src2,
1526 opcode,
1527 } => {
1528 let src1 = pretty_print_reg(src1.to_reg(), size.to_bytes());
1529 let src2 = src2.pretty_print(size.to_bytes());
1530 let op = match opcode {
1531 CmpOpcode::Cmp => "cmp",
1532 CmpOpcode::Test => "test",
1533 };
1534 let op = ljustify2(op.to_string(), suffix_bwlq(*size));
1535 format!("{op} {src2}, {src1}")
1536 }
1537
1538 Inst::Setcc { cc, dst } => {
1539 let dst = pretty_print_reg(dst.to_reg().to_reg(), 1);
1540 let op = ljustify2("set".to_string(), cc.to_string());
1541 format!("{op} {dst}")
1542 }
1543
1544 Inst::Bswap { size, src, dst } => {
1545 let src = pretty_print_reg(src.to_reg(), size.to_bytes());
1546 let dst = pretty_print_reg(dst.to_reg().to_reg(), size.to_bytes());
1547 let op = ljustify2("bswap".to_string(), suffix_bwlq(*size));
1548 format!("{op} {src}, {dst}")
1549 }
1550
1551 Inst::Cmove {
1552 size,
1553 cc,
1554 consequent,
1555 alternative,
1556 dst,
1557 } => {
1558 let alternative = pretty_print_reg(alternative.to_reg(), size.to_bytes());
1559 let dst = pretty_print_reg(dst.to_reg().to_reg(), size.to_bytes());
1560 let consequent = consequent.pretty_print(size.to_bytes());
1561 let op = ljustify(format!("cmov{}{}", cc.to_string(), suffix_bwlq(*size)));
1562 format!("{op} {consequent}, {alternative}, {dst}")
1563 }
1564
1565 Inst::XmmCmove {
1566 ty,
1567 cc,
1568 consequent,
1569 alternative,
1570 dst,
1571 ..
1572 } => {
1573 let size = u8::try_from(ty.bytes()).unwrap();
1574 let alternative = pretty_print_reg(alternative.to_reg(), size);
1575 let dst = pretty_print_reg(dst.to_reg().to_reg(), size);
1576 let consequent = pretty_print_reg(consequent.to_reg(), size);
1577 let suffix = match *ty {
1578 types::F64 => "sd",
1579 types::F32 => "ss",
1580 types::F16 => "ss",
1581 types::F32X4 => "aps",
1582 types::F64X2 => "apd",
1583 _ => "dqa",
1584 };
1585 let cc = cc.invert();
1586 format!(
1587 "mov{suffix} {alternative}, {dst}; \
1588 j{cc} $next; \
1589 mov{suffix} {consequent}, {dst}; \
1590 $next:"
1591 )
1592 }
1593
1594 Inst::Push64 { src } => {
1595 let src = src.pretty_print(8);
1596 let op = ljustify("pushq".to_string());
1597 format!("{op} {src}")
1598 }
1599
1600 Inst::StackProbeLoop {
1601 tmp,
1602 frame_size,
1603 guard_size,
1604 } => {
1605 let tmp = pretty_print_reg(tmp.to_reg(), 8);
1606 let op = ljustify("stack_probe_loop".to_string());
1607 format!("{op} {tmp}, frame_size={frame_size}, guard_size={guard_size}")
1608 }
1609
1610 Inst::Pop64 { dst } => {
1611 let dst = pretty_print_reg(dst.to_reg().to_reg(), 8);
1612 let op = ljustify("popq".to_string());
1613 format!("{op} {dst}")
1614 }
1615
1616 Inst::CallKnown { info } => {
1617 let op = ljustify("call".to_string());
1618 format!("{op} {:?}", info.dest)
1619 }
1620
1621 Inst::CallUnknown { info } => {
1622 let dest = info.dest.pretty_print(8);
1623 let op = ljustify("call".to_string());
1624 format!("{op} *{dest}")
1625 }
1626
1627 Inst::ReturnCallKnown { info } => {
1628 let ReturnCallInfo {
1629 uses,
1630 new_stack_arg_size,
1631 tmp,
1632 dest,
1633 } = &**info;
1634 let tmp = pretty_print_reg(tmp.to_reg().to_reg(), 8);
1635 let mut s = format!("return_call_known {dest:?} ({new_stack_arg_size}) tmp={tmp}");
1636 for ret in uses {
1637 let preg = regs::show_reg(ret.preg);
1638 let vreg = pretty_print_reg(ret.vreg, 8);
1639 write!(&mut s, " {vreg}={preg}").unwrap();
1640 }
1641 s
1642 }
1643
1644 Inst::ReturnCallUnknown { info } => {
1645 let ReturnCallInfo {
1646 uses,
1647 new_stack_arg_size,
1648 tmp,
1649 dest,
1650 } = &**info;
1651 let callee = pretty_print_reg(*dest, 8);
1652 let tmp = pretty_print_reg(tmp.to_reg().to_reg(), 8);
1653 let mut s =
1654 format!("return_call_unknown {callee} ({new_stack_arg_size}) tmp={tmp}");
1655 for ret in uses {
1656 let preg = regs::show_reg(ret.preg);
1657 let vreg = pretty_print_reg(ret.vreg, 8);
1658 write!(&mut s, " {vreg}={preg}").unwrap();
1659 }
1660 s
1661 }
1662
1663 Inst::Args { args } => {
1664 let mut s = "args".to_string();
1665 for arg in args {
1666 let preg = regs::show_reg(arg.preg);
1667 let def = pretty_print_reg(arg.vreg.to_reg(), 8);
1668 write!(&mut s, " {def}={preg}").unwrap();
1669 }
1670 s
1671 }
1672
1673 Inst::Rets { rets } => {
1674 let mut s = "rets".to_string();
1675 for ret in rets {
1676 let preg = regs::show_reg(ret.preg);
1677 let vreg = pretty_print_reg(ret.vreg, 8);
1678 write!(&mut s, " {vreg}={preg}").unwrap();
1679 }
1680 s
1681 }
1682
1683 Inst::Ret { stack_bytes_to_pop } => {
1684 let mut s = "ret".to_string();
1685 if *stack_bytes_to_pop != 0 {
1686 write!(&mut s, " {stack_bytes_to_pop}").unwrap();
1687 }
1688 s
1689 }
1690
1691 Inst::StackSwitchBasic {
1692 store_context_ptr,
1693 load_context_ptr,
1694 in_payload0,
1695 out_payload0,
1696 } => {
1697 let store_context_ptr = pretty_print_reg(**store_context_ptr, 8);
1698 let load_context_ptr = pretty_print_reg(**load_context_ptr, 8);
1699 let in_payload0 = pretty_print_reg(**in_payload0, 8);
1700 let out_payload0 = pretty_print_reg(*out_payload0.to_reg(), 8);
1701 format!("{out_payload0} = stack_switch_basic {store_context_ptr}, {load_context_ptr}, {in_payload0}")
1702 }
1703
1704 Inst::JmpKnown { dst } => {
1705 let op = ljustify("jmp".to_string());
1706 let dst = dst.to_string();
1707 format!("{op} {dst}")
1708 }
1709
1710 Inst::JmpIf { cc, taken } => {
1711 let taken = taken.to_string();
1712 let op = ljustify2("j".to_string(), cc.to_string());
1713 format!("{op} {taken}")
1714 }
1715
1716 Inst::JmpCond {
1717 cc,
1718 taken,
1719 not_taken,
1720 } => {
1721 let taken = taken.to_string();
1722 let not_taken = not_taken.to_string();
1723 let op = ljustify2("j".to_string(), cc.to_string());
1724 format!("{op} {taken}; j {not_taken}")
1725 }
1726
1727 Inst::JmpTableSeq {
1728 idx, tmp1, tmp2, ..
1729 } => {
1730 let idx = pretty_print_reg(*idx, 8);
1731 let tmp1 = pretty_print_reg(tmp1.to_reg(), 8);
1732 let tmp2 = pretty_print_reg(tmp2.to_reg(), 8);
1733 let op = ljustify("br_table".into());
1734 format!("{op} {idx}, {tmp1}, {tmp2}")
1735 }
1736
1737 Inst::JmpUnknown { target } => {
1738 let target = target.pretty_print(8);
1739 let op = ljustify("jmp".to_string());
1740 format!("{op} *{target}")
1741 }
1742
1743 Inst::TrapIf { cc, trap_code, .. } => {
1744 format!("j{cc} #trap={trap_code}")
1745 }
1746
1747 Inst::TrapIfAnd {
1748 cc1,
1749 cc2,
1750 trap_code,
1751 ..
1752 } => {
1753 let cc1 = cc1.invert();
1754 let cc2 = cc2.invert();
1755 format!("trap_if_and {cc1}, {cc2}, {trap_code}")
1756 }
1757
1758 Inst::TrapIfOr {
1759 cc1,
1760 cc2,
1761 trap_code,
1762 ..
1763 } => {
1764 let cc2 = cc2.invert();
1765 format!("trap_if_or {cc1}, {cc2}, {trap_code}")
1766 }
1767
1768 Inst::LoadExtName {
1769 dst, name, offset, ..
1770 } => {
1771 let dst = pretty_print_reg(dst.to_reg(), 8);
1772 let name = name.display(None);
1773 let op = ljustify("load_ext_name".into());
1774 format!("{op} {name}+{offset}, {dst}")
1775 }
1776
1777 Inst::LockCmpxchg {
1778 ty,
1779 replacement,
1780 expected,
1781 mem,
1782 dst_old,
1783 ..
1784 } => {
1785 let size = ty.bytes() as u8;
1786 let replacement = pretty_print_reg(*replacement, size);
1787 let expected = pretty_print_reg(*expected, size);
1788 let dst_old = pretty_print_reg(dst_old.to_reg(), size);
1789 let mem = mem.pretty_print(size);
1790 let suffix = suffix_bwlq(OperandSize::from_bytes(size as u32));
1791 format!(
1792 "lock cmpxchg{suffix} {replacement}, {mem}, expected={expected}, dst_old={dst_old}"
1793 )
1794 }
1795
1796 Inst::AtomicRmwSeq { ty, op, .. } => {
1797 let ty = ty.bits();
1798 format!(
1799 "atomically {{ {ty}_bits_at_[%r9]) {op:?}= %r10; %rax = old_value_at_[%r9]; %r11, %rflags = trash }}"
1800 )
1801 }
1802
1803 Inst::Fence { kind } => match kind {
1804 FenceKind::MFence => "mfence".to_string(),
1805 FenceKind::LFence => "lfence".to_string(),
1806 FenceKind::SFence => "sfence".to_string(),
1807 },
1808
1809 Inst::Hlt => "hlt".into(),
1810
1811 Inst::Ud2 { trap_code } => format!("ud2 {trap_code}"),
1812
1813 Inst::ElfTlsGetAddr { ref symbol, dst } => {
1814 let dst = pretty_print_reg(dst.to_reg().to_reg(), 8);
1815 format!("{dst} = elf_tls_get_addr {symbol:?}")
1816 }
1817
1818 Inst::MachOTlsGetAddr { ref symbol, dst } => {
1819 let dst = pretty_print_reg(dst.to_reg().to_reg(), 8);
1820 format!("{dst} = macho_tls_get_addr {symbol:?}")
1821 }
1822
1823 Inst::CoffTlsGetAddr {
1824 ref symbol,
1825 dst,
1826 tmp,
1827 } => {
1828 let dst = pretty_print_reg(dst.to_reg().to_reg(), 8);
1829 let tmp = tmp.to_reg().to_reg();
1830
1831 let mut s = format!("{dst} = coff_tls_get_addr {symbol:?}");
1832 if tmp.is_virtual() {
1833 let tmp = show_ireg_sized(tmp, 8);
1834 write!(&mut s, ", {tmp}").unwrap();
1835 };
1836
1837 s
1838 }
1839
1840 Inst::Unwind { inst } => format!("unwind {inst:?}"),
1841
1842 Inst::DummyUse { reg } => {
1843 let reg = pretty_print_reg(*reg, 8);
1844 format!("dummy_use {reg}")
1845 }
1846 }
1847 }
1848}
1849
1850impl fmt::Debug for Inst {
1851 fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
1852 write!(fmt, "{}", self.pretty_print_inst(&mut Default::default()))
1853 }
1854}
1855
1856fn x64_get_operands(inst: &mut Inst, collector: &mut impl OperandVisitor) {
1857 match inst {
1867 Inst::AluRmiR {
1868 src1, src2, dst, ..
1869 } => {
1870 collector.reg_use(src1);
1871 collector.reg_reuse_def(dst, 0);
1872 src2.get_operands(collector);
1873 }
1874 Inst::AluConstOp { dst, .. } => collector.reg_def(dst),
1875 Inst::AluRM { src1_dst, src2, .. } => {
1876 collector.reg_use(src2);
1877 src1_dst.get_operands(collector);
1878 }
1879 Inst::AluRmRVex {
1880 src1, src2, dst, ..
1881 } => {
1882 collector.reg_def(dst);
1883 collector.reg_use(src1);
1884 src2.get_operands(collector);
1885 }
1886 Inst::Not { src, dst, .. } => {
1887 collector.reg_use(src);
1888 collector.reg_reuse_def(dst, 0);
1889 }
1890 Inst::Neg { src, dst, .. } => {
1891 collector.reg_use(src);
1892 collector.reg_reuse_def(dst, 0);
1893 }
1894 Inst::Div {
1895 divisor,
1896 dividend_lo,
1897 dividend_hi,
1898 dst_quotient,
1899 dst_remainder,
1900 ..
1901 } => {
1902 divisor.get_operands(collector);
1903 collector.reg_fixed_use(dividend_lo, regs::rax());
1904 collector.reg_fixed_use(dividend_hi, regs::rdx());
1905 collector.reg_fixed_def(dst_quotient, regs::rax());
1906 collector.reg_fixed_def(dst_remainder, regs::rdx());
1907 }
1908 Inst::CheckedSRemSeq {
1909 divisor,
1910 dividend_lo,
1911 dividend_hi,
1912 dst_quotient,
1913 dst_remainder,
1914 ..
1915 } => {
1916 collector.reg_use(divisor);
1917 collector.reg_fixed_use(dividend_lo, regs::rax());
1918 collector.reg_fixed_use(dividend_hi, regs::rdx());
1919 collector.reg_fixed_def(dst_quotient, regs::rax());
1920 collector.reg_fixed_def(dst_remainder, regs::rdx());
1921 }
1922 Inst::Div8 {
1923 divisor,
1924 dividend,
1925 dst,
1926 ..
1927 } => {
1928 divisor.get_operands(collector);
1929 collector.reg_fixed_use(dividend, regs::rax());
1930 collector.reg_fixed_def(dst, regs::rax());
1931 }
1932 Inst::CheckedSRemSeq8 {
1933 divisor,
1934 dividend,
1935 dst,
1936 ..
1937 } => {
1938 collector.reg_use(divisor);
1939 collector.reg_fixed_use(dividend, regs::rax());
1940 collector.reg_fixed_def(dst, regs::rax());
1941 }
1942 Inst::Mul {
1943 src1,
1944 src2,
1945 dst_lo,
1946 dst_hi,
1947 ..
1948 } => {
1949 collector.reg_fixed_use(src1, regs::rax());
1950 collector.reg_fixed_def(dst_lo, regs::rax());
1951 collector.reg_fixed_def(dst_hi, regs::rdx());
1952 src2.get_operands(collector);
1953 }
1954 Inst::Mul8 {
1955 src1, src2, dst, ..
1956 } => {
1957 collector.reg_fixed_use(src1, regs::rax());
1958 collector.reg_fixed_def(dst, regs::rax());
1959 src2.get_operands(collector);
1960 }
1961 Inst::IMul {
1962 src1, src2, dst, ..
1963 } => {
1964 collector.reg_use(src1);
1965 collector.reg_reuse_def(dst, 0);
1966 src2.get_operands(collector);
1967 }
1968 Inst::IMulImm { src1, dst, .. } => {
1969 collector.reg_def(dst);
1970 src1.get_operands(collector);
1971 }
1972 Inst::SignExtendData { size, src, dst } => {
1973 match size {
1974 OperandSize::Size8 => {
1975 collector.reg_fixed_use(src, regs::rax());
1978 collector.reg_fixed_def(dst, regs::rax());
1979 }
1980 _ => {
1981 collector.reg_fixed_use(src, regs::rax());
1984 collector.reg_fixed_def(dst, regs::rdx());
1985 }
1986 }
1987 }
1988 Inst::UnaryRmR { src, dst, .. }
1989 | Inst::UnaryRmRVex { src, dst, .. }
1990 | Inst::UnaryRmRImmVex { src, dst, .. } => {
1991 collector.reg_def(dst);
1992 src.get_operands(collector);
1993 }
1994 Inst::XmmUnaryRmR { src, dst, .. } | Inst::XmmUnaryRmRImm { src, dst, .. } => {
1995 collector.reg_def(dst);
1996 src.get_operands(collector);
1997 }
1998 Inst::XmmUnaryRmREvex { src, dst, .. }
1999 | Inst::XmmUnaryRmRImmEvex { src, dst, .. }
2000 | Inst::XmmUnaryRmRUnaligned { src, dst, .. }
2001 | Inst::XmmUnaryRmRVex { src, dst, .. }
2002 | Inst::XmmUnaryRmRImmVex { src, dst, .. } => {
2003 collector.reg_def(dst);
2004 src.get_operands(collector);
2005 }
2006 Inst::XmmRmR {
2007 src1, src2, dst, ..
2008 } => {
2009 collector.reg_use(src1);
2010 collector.reg_reuse_def(dst, 0);
2011 src2.get_operands(collector);
2012 }
2013 Inst::XmmRmRUnaligned {
2014 src1, src2, dst, ..
2015 } => {
2016 collector.reg_use(src1);
2017 collector.reg_reuse_def(dst, 0);
2018 src2.get_operands(collector);
2019 }
2020 Inst::XmmRmRBlend {
2021 src1,
2022 src2,
2023 mask,
2024 dst,
2025 op,
2026 } => {
2027 assert!(matches!(
2028 op,
2029 SseOpcode::Blendvpd | SseOpcode::Blendvps | SseOpcode::Pblendvb
2030 ));
2031 collector.reg_use(src1);
2032 collector.reg_fixed_use(mask, regs::xmm0());
2033 collector.reg_reuse_def(dst, 0);
2034 src2.get_operands(collector);
2035 }
2036 Inst::XmmRmiRVex {
2037 src1, src2, dst, ..
2038 } => {
2039 collector.reg_def(dst);
2040 collector.reg_use(src1);
2041 src2.get_operands(collector);
2042 }
2043 Inst::XmmRmRImmVex {
2044 src1, src2, dst, ..
2045 } => {
2046 collector.reg_def(dst);
2047 collector.reg_use(src1);
2048 src2.get_operands(collector);
2049 }
2050 Inst::XmmVexPinsr {
2051 src1, src2, dst, ..
2052 } => {
2053 collector.reg_def(dst);
2054 collector.reg_use(src1);
2055 src2.get_operands(collector);
2056 }
2057 Inst::XmmRmRVex3 {
2058 src1,
2059 src2,
2060 src3,
2061 dst,
2062 ..
2063 } => {
2064 collector.reg_use(src1);
2065 collector.reg_reuse_def(dst, 0);
2066 collector.reg_use(src2);
2067 src3.get_operands(collector);
2068 }
2069 Inst::XmmRmRBlendVex {
2070 src1,
2071 src2,
2072 mask,
2073 dst,
2074 ..
2075 } => {
2076 collector.reg_def(dst);
2077 collector.reg_use(src1);
2078 src2.get_operands(collector);
2079 collector.reg_use(mask);
2080 }
2081 Inst::XmmRmREvex {
2082 op,
2083 src1,
2084 src2,
2085 dst,
2086 ..
2087 } => {
2088 assert_ne!(*op, Avx512Opcode::Vpermi2b);
2089 collector.reg_use(src1);
2090 src2.get_operands(collector);
2091 collector.reg_def(dst);
2092 }
2093 Inst::XmmRmREvex3 {
2094 op,
2095 src1,
2096 src2,
2097 src3,
2098 dst,
2099 ..
2100 } => {
2101 assert_eq!(*op, Avx512Opcode::Vpermi2b);
2102 collector.reg_use(src1);
2103 collector.reg_use(src2);
2104 src3.get_operands(collector);
2105 collector.reg_reuse_def(dst, 0); }
2107 Inst::XmmRmRImm {
2108 src1, src2, dst, ..
2109 } => {
2110 collector.reg_use(src1);
2111 collector.reg_reuse_def(dst, 0);
2112 src2.get_operands(collector);
2113 }
2114 Inst::XmmUninitializedValue { dst } => collector.reg_def(dst),
2115 Inst::XmmMinMaxSeq { lhs, rhs, dst, .. } => {
2116 collector.reg_use(rhs);
2117 collector.reg_use(lhs);
2118 collector.reg_reuse_def(dst, 0); }
2120 Inst::XmmRmiReg {
2121 src1, src2, dst, ..
2122 } => {
2123 collector.reg_use(src1);
2124 collector.reg_reuse_def(dst, 0); src2.get_operands(collector);
2126 }
2127 Inst::XmmMovRM { src, dst, .. }
2128 | Inst::XmmMovRMVex { src, dst, .. }
2129 | Inst::XmmMovRMImm { src, dst, .. }
2130 | Inst::XmmMovRMImmVex { src, dst, .. } => {
2131 collector.reg_use(src);
2132 dst.get_operands(collector);
2133 }
2134 Inst::XmmCmpRmR { src1, src2, .. } => {
2135 collector.reg_use(src1);
2136 src2.get_operands(collector);
2137 }
2138 Inst::XmmCmpRmRVex { src1, src2, .. } => {
2139 collector.reg_use(src1);
2140 src2.get_operands(collector);
2141 }
2142 Inst::Imm { dst, .. } => {
2143 collector.reg_def(dst);
2144 }
2145 Inst::MovRR { src, dst, .. } => {
2146 collector.reg_use(src);
2147 collector.reg_def(dst);
2148 }
2149 Inst::MovFromPReg { dst, src } => {
2150 debug_assert!(dst.to_reg().to_reg().is_virtual());
2151 collector.reg_fixed_nonallocatable(*src);
2152 collector.reg_def(dst);
2153 }
2154 Inst::MovToPReg { dst, src } => {
2155 debug_assert!(src.to_reg().is_virtual());
2156 collector.reg_use(src);
2157 collector.reg_fixed_nonallocatable(*dst);
2158 }
2159 Inst::XmmToGpr { src, dst, .. }
2160 | Inst::XmmToGprVex { src, dst, .. }
2161 | Inst::XmmToGprImm { src, dst, .. }
2162 | Inst::XmmToGprImmVex { src, dst, .. } => {
2163 collector.reg_use(src);
2164 collector.reg_def(dst);
2165 }
2166 Inst::GprToXmm { src, dst, .. } | Inst::GprToXmmVex { src, dst, .. } => {
2167 collector.reg_def(dst);
2168 src.get_operands(collector);
2169 }
2170 Inst::CvtIntToFloat {
2171 src1, src2, dst, ..
2172 } => {
2173 collector.reg_use(src1);
2174 collector.reg_reuse_def(dst, 0);
2175 src2.get_operands(collector);
2176 }
2177 Inst::CvtIntToFloatVex {
2178 src1, src2, dst, ..
2179 } => {
2180 collector.reg_def(dst);
2181 collector.reg_use(src1);
2182 src2.get_operands(collector);
2183 }
2184 Inst::CvtUint64ToFloatSeq {
2185 src,
2186 dst,
2187 tmp_gpr1,
2188 tmp_gpr2,
2189 ..
2190 } => {
2191 collector.reg_use(src);
2192 collector.reg_early_def(dst);
2193 collector.reg_early_def(tmp_gpr1);
2194 collector.reg_early_def(tmp_gpr2);
2195 }
2196 Inst::CvtFloatToSintSeq {
2197 src,
2198 dst,
2199 tmp_xmm,
2200 tmp_gpr,
2201 ..
2202 } => {
2203 collector.reg_use(src);
2204 collector.reg_early_def(dst);
2205 collector.reg_early_def(tmp_gpr);
2206 collector.reg_early_def(tmp_xmm);
2207 }
2208 Inst::CvtFloatToUintSeq {
2209 src,
2210 dst,
2211 tmp_gpr,
2212 tmp_xmm,
2213 tmp_xmm2,
2214 ..
2215 } => {
2216 collector.reg_use(src);
2217 collector.reg_early_def(dst);
2218 collector.reg_early_def(tmp_gpr);
2219 collector.reg_early_def(tmp_xmm);
2220 collector.reg_early_def(tmp_xmm2);
2221 }
2222
2223 Inst::MovImmM { dst, .. } => {
2224 dst.get_operands(collector);
2225 }
2226
2227 Inst::MovzxRmR { src, dst, .. } => {
2228 collector.reg_def(dst);
2229 src.get_operands(collector);
2230 }
2231 Inst::Mov64MR { src, dst, .. } => {
2232 collector.reg_def(dst);
2233 src.get_operands(collector);
2234 }
2235 Inst::LoadEffectiveAddress { addr: src, dst, .. } => {
2236 collector.reg_def(dst);
2237 src.get_operands(collector);
2238 }
2239 Inst::MovsxRmR { src, dst, .. } => {
2240 collector.reg_def(dst);
2241 src.get_operands(collector);
2242 }
2243 Inst::MovRM { src, dst, .. } => {
2244 collector.reg_use(src);
2245 dst.get_operands(collector);
2246 }
2247 Inst::ShiftR {
2248 num_bits, src, dst, ..
2249 } => {
2250 collector.reg_use(src);
2251 collector.reg_reuse_def(dst, 0);
2252 if let Imm8Reg::Reg { reg } = num_bits.as_imm8_reg_mut() {
2253 collector.reg_fixed_use(reg, regs::rcx());
2254 }
2255 }
2256 Inst::CmpRmiR { src1, src2, .. } => {
2257 collector.reg_use(src1);
2258 src2.get_operands(collector);
2259 }
2260 Inst::Setcc { dst, .. } => {
2261 collector.reg_def(dst);
2262 }
2263 Inst::Bswap { src, dst, .. } => {
2264 collector.reg_use(src);
2265 collector.reg_reuse_def(dst, 0);
2266 }
2267 Inst::Cmove {
2268 consequent,
2269 alternative,
2270 dst,
2271 ..
2272 } => {
2273 collector.reg_use(alternative);
2274 collector.reg_reuse_def(dst, 0);
2275 consequent.get_operands(collector);
2276 }
2277 Inst::XmmCmove {
2278 consequent,
2279 alternative,
2280 dst,
2281 ..
2282 } => {
2283 collector.reg_use(alternative);
2284 collector.reg_reuse_def(dst, 0);
2285 collector.reg_use(consequent);
2286 }
2287 Inst::Push64 { src } => {
2288 src.get_operands(collector);
2289 }
2290 Inst::Pop64 { dst } => {
2291 collector.reg_def(dst);
2292 }
2293 Inst::StackProbeLoop { tmp, .. } => {
2294 collector.reg_early_def(tmp);
2295 }
2296
2297 Inst::CallKnown { info } => {
2298 let CallInfo {
2303 uses,
2304 defs,
2305 clobbers,
2306 dest,
2307 ..
2308 } = &mut **info;
2309 debug_assert_ne!(*dest, ExternalName::LibCall(LibCall::Probestack));
2310 for CallArgPair { vreg, preg } in uses {
2311 collector.reg_fixed_use(vreg, *preg);
2312 }
2313 for CallRetPair { vreg, preg } in defs {
2314 collector.reg_fixed_def(vreg, *preg);
2315 }
2316 collector.reg_clobbers(*clobbers);
2317 }
2318
2319 Inst::CallUnknown { info } => {
2320 let CallInfo {
2321 uses,
2322 defs,
2323 clobbers,
2324 callee_conv,
2325 dest,
2326 ..
2327 } = &mut **info;
2328 match dest {
2329 RegMem::Reg { reg } if *callee_conv == CallConv::Winch => {
2330 collector.reg_fixed_use(reg, regs::r10())
2334 }
2335 _ => dest.get_operands(collector),
2336 }
2337 for CallArgPair { vreg, preg } in uses {
2338 collector.reg_fixed_use(vreg, *preg);
2339 }
2340 for CallRetPair { vreg, preg } in defs {
2341 collector.reg_fixed_def(vreg, *preg);
2342 }
2343 collector.reg_clobbers(*clobbers);
2344 }
2345 Inst::StackSwitchBasic {
2346 store_context_ptr,
2347 load_context_ptr,
2348 in_payload0,
2349 out_payload0,
2350 } => {
2351 collector.reg_use(load_context_ptr);
2352 collector.reg_use(store_context_ptr);
2353 collector.reg_fixed_use(in_payload0, stack_switch::payload_register());
2354 collector.reg_fixed_def(out_payload0, stack_switch::payload_register());
2355
2356 let mut clobbers = crate::isa::x64::abi::ALL_CLOBBERS;
2357 clobbers.remove(
2359 stack_switch::payload_register()
2360 .to_real_reg()
2361 .unwrap()
2362 .into(),
2363 );
2364 collector.reg_clobbers(clobbers);
2365 }
2366
2367 Inst::ReturnCallKnown { info } => {
2368 let ReturnCallInfo {
2369 dest, uses, tmp, ..
2370 } = &mut **info;
2371 collector.reg_fixed_def(tmp, regs::r11());
2372 debug_assert_ne!(*dest, ExternalName::LibCall(LibCall::Probestack));
2374 for CallArgPair { vreg, preg } in uses {
2375 collector.reg_fixed_use(vreg, *preg);
2376 }
2377 }
2378
2379 Inst::ReturnCallUnknown { info } => {
2380 let ReturnCallInfo {
2381 dest, uses, tmp, ..
2382 } = &mut **info;
2383
2384 collector.reg_fixed_use(dest, regs::r10());
2390
2391 collector.reg_fixed_def(tmp, regs::r11());
2392 for CallArgPair { vreg, preg } in uses {
2393 collector.reg_fixed_use(vreg, *preg);
2394 }
2395 }
2396
2397 Inst::JmpTableSeq {
2398 idx, tmp1, tmp2, ..
2399 } => {
2400 collector.reg_use(idx);
2401 collector.reg_early_def(tmp1);
2402 collector.reg_def(tmp2);
2406 }
2407
2408 Inst::JmpUnknown { target } => {
2409 target.get_operands(collector);
2410 }
2411
2412 Inst::LoadExtName { dst, .. } => {
2413 collector.reg_def(dst);
2414 }
2415
2416 Inst::LockCmpxchg {
2417 replacement,
2418 expected,
2419 mem,
2420 dst_old,
2421 ..
2422 } => {
2423 collector.reg_use(replacement);
2424 collector.reg_fixed_use(expected, regs::rax());
2425 collector.reg_fixed_def(dst_old, regs::rax());
2426 mem.get_operands(collector);
2427 }
2428
2429 Inst::AtomicRmwSeq {
2430 operand,
2431 temp,
2432 dst_old,
2433 mem,
2434 ..
2435 } => {
2436 collector.reg_late_use(operand);
2437 collector.reg_early_def(temp);
2438 collector.reg_fixed_def(dst_old, regs::rax());
2441 mem.get_operands_late(collector)
2442 }
2443
2444 Inst::Args { args } => {
2445 for ArgPair { vreg, preg } in args {
2446 collector.reg_fixed_def(vreg, *preg);
2447 }
2448 }
2449
2450 Inst::Rets { rets } => {
2451 for RetPair { vreg, preg } in rets {
2454 collector.reg_fixed_use(vreg, *preg);
2455 }
2456 }
2457
2458 Inst::JmpKnown { .. }
2459 | Inst::JmpIf { .. }
2460 | Inst::JmpCond { .. }
2461 | Inst::Ret { .. }
2462 | Inst::Nop { .. }
2463 | Inst::TrapIf { .. }
2464 | Inst::TrapIfAnd { .. }
2465 | Inst::TrapIfOr { .. }
2466 | Inst::Hlt
2467 | Inst::Ud2 { .. }
2468 | Inst::Fence { .. } => {
2469 }
2471
2472 Inst::ElfTlsGetAddr { dst, .. } | Inst::MachOTlsGetAddr { dst, .. } => {
2473 collector.reg_fixed_def(dst, regs::rax());
2474 let mut clobbers = X64ABIMachineSpec::get_regs_clobbered_by_call(CallConv::SystemV);
2481 clobbers.remove(regs::gpr_preg(regs::ENC_RAX));
2482 collector.reg_clobbers(clobbers);
2483 }
2484
2485 Inst::CoffTlsGetAddr { dst, tmp, .. } => {
2486 collector.reg_fixed_def(dst, regs::rax());
2491
2492 collector.reg_fixed_def(tmp, regs::rcx());
2494 }
2495
2496 Inst::Unwind { .. } => {}
2497
2498 Inst::DummyUse { reg } => {
2499 collector.reg_use(reg);
2500 }
2501 }
2502}
2503
2504impl MachInst for Inst {
2508 type ABIMachineSpec = X64ABIMachineSpec;
2509
2510 fn get_operands(&mut self, collector: &mut impl OperandVisitor) {
2511 x64_get_operands(self, collector)
2512 }
2513
2514 fn is_move(&self) -> Option<(Writable<Reg>, Reg)> {
2515 match self {
2516 Self::MovRR { size, src, dst, .. } if *size == OperandSize::Size64 => {
2521 Some((dst.to_writable_reg(), src.to_reg()))
2522 }
2523 Self::XmmUnaryRmR { op, src, dst, .. }
2528 if *op == SseOpcode::Movss
2529 || *op == SseOpcode::Movsd
2530 || *op == SseOpcode::Movaps
2531 || *op == SseOpcode::Movapd
2532 || *op == SseOpcode::Movups
2533 || *op == SseOpcode::Movupd
2534 || *op == SseOpcode::Movdqa
2535 || *op == SseOpcode::Movdqu =>
2536 {
2537 if let RegMem::Reg { reg } = src.clone().to_reg_mem() {
2538 Some((dst.to_writable_reg(), reg))
2539 } else {
2540 None
2541 }
2542 }
2543 _ => None,
2544 }
2545 }
2546
2547 fn is_included_in_clobbers(&self) -> bool {
2548 match self {
2549 &Inst::Args { .. } => false,
2550 _ => true,
2551 }
2552 }
2553
2554 fn is_trap(&self) -> bool {
2555 match self {
2556 Self::Ud2 { .. } => true,
2557 _ => false,
2558 }
2559 }
2560
2561 fn is_args(&self) -> bool {
2562 match self {
2563 Self::Args { .. } => true,
2564 _ => false,
2565 }
2566 }
2567
2568 fn is_term(&self) -> MachTerminator {
2569 match self {
2570 &Self::Rets { .. } => MachTerminator::Ret,
2572 &Self::ReturnCallKnown { .. } | &Self::ReturnCallUnknown { .. } => {
2573 MachTerminator::RetCall
2574 }
2575 &Self::JmpKnown { .. } => MachTerminator::Uncond,
2576 &Self::JmpCond { .. } => MachTerminator::Cond,
2577 &Self::JmpTableSeq { .. } => MachTerminator::Indirect,
2578 _ => MachTerminator::None,
2580 }
2581 }
2582
2583 fn is_mem_access(&self) -> bool {
2584 panic!("TODO FILL ME OUT")
2585 }
2586
2587 fn gen_move(dst_reg: Writable<Reg>, src_reg: Reg, ty: Type) -> Inst {
2588 trace!(
2589 "Inst::gen_move {:?} -> {:?} (type: {:?})",
2590 src_reg,
2591 dst_reg.to_reg(),
2592 ty
2593 );
2594 let rc_dst = dst_reg.to_reg().class();
2595 let rc_src = src_reg.class();
2596 debug_assert!(rc_dst == rc_src);
2598 match rc_dst {
2599 RegClass::Int => Inst::mov_r_r(OperandSize::Size64, src_reg, dst_reg),
2600 RegClass::Float => {
2601 let opcode = match ty {
2606 types::F16 | types::F32 | types::F64 | types::F32X4 => SseOpcode::Movaps,
2607 types::F64X2 => SseOpcode::Movapd,
2608 _ if (ty.is_float() || ty.is_vector()) && ty.bits() == 128 => SseOpcode::Movdqa,
2609 _ => unimplemented!("unable to move type: {}", ty),
2610 };
2611 Inst::xmm_unary_rm_r(opcode, RegMem::reg(src_reg), dst_reg)
2612 }
2613 RegClass::Vector => unreachable!(),
2614 }
2615 }
2616
2617 fn gen_nop(preferred_size: usize) -> Inst {
2618 Inst::nop(std::cmp::min(preferred_size, 15) as u8)
2619 }
2620
2621 fn rc_for_type(ty: Type) -> CodegenResult<(&'static [RegClass], &'static [Type])> {
2622 match ty {
2623 types::I8 => Ok((&[RegClass::Int], &[types::I8])),
2624 types::I16 => Ok((&[RegClass::Int], &[types::I16])),
2625 types::I32 => Ok((&[RegClass::Int], &[types::I32])),
2626 types::I64 => Ok((&[RegClass::Int], &[types::I64])),
2627 types::F16 => Ok((&[RegClass::Float], &[types::F16])),
2628 types::F32 => Ok((&[RegClass::Float], &[types::F32])),
2629 types::F64 => Ok((&[RegClass::Float], &[types::F64])),
2630 types::F128 => Ok((&[RegClass::Float], &[types::F128])),
2631 types::I128 => Ok((&[RegClass::Int, RegClass::Int], &[types::I64, types::I64])),
2632 _ if ty.is_vector() => {
2633 assert!(ty.bits() <= 128);
2634 Ok((&[RegClass::Float], &[types::I8X16]))
2635 }
2636 _ => Err(CodegenError::Unsupported(format!(
2637 "Unexpected SSA-value type: {ty}"
2638 ))),
2639 }
2640 }
2641
2642 fn canonical_type_for_rc(rc: RegClass) -> Type {
2643 match rc {
2644 RegClass::Float => types::I8X16,
2645 RegClass::Int => types::I64,
2646 RegClass::Vector => unreachable!(),
2647 }
2648 }
2649
2650 fn gen_jump(label: MachLabel) -> Inst {
2651 Inst::jmp_known(label)
2652 }
2653
2654 fn gen_imm_u64(value: u64, dst: Writable<Reg>) -> Option<Self> {
2655 Some(Inst::imm(OperandSize::Size64, value, dst))
2656 }
2657
2658 fn gen_imm_f64(value: f64, tmp: Writable<Reg>, dst: Writable<Reg>) -> SmallVec<[Self; 2]> {
2659 let imm_to_gpr = Inst::imm(OperandSize::Size64, value.to_bits(), tmp);
2660 let gpr_to_xmm = Self::gpr_to_xmm(
2661 SseOpcode::Movd,
2662 tmp.to_reg().into(),
2663 OperandSize::Size64,
2664 dst,
2665 );
2666 smallvec![imm_to_gpr, gpr_to_xmm]
2667 }
2668
2669 fn gen_dummy_use(reg: Reg) -> Self {
2670 Inst::DummyUse { reg }
2671 }
2672
2673 fn worst_case_size() -> CodeOffset {
2674 15
2675 }
2676
2677 fn ref_type_regclass(_: &settings::Flags) -> RegClass {
2678 RegClass::Int
2679 }
2680
2681 fn is_safepoint(&self) -> bool {
2682 match self {
2683 Inst::CallKnown { .. } | Inst::CallUnknown { .. } => true,
2684 _ => false,
2685 }
2686 }
2687
2688 fn function_alignment() -> FunctionAlignment {
2689 FunctionAlignment {
2690 minimum: 1,
2691 preferred: 32,
2694 }
2695 }
2696
2697 type LabelUse = LabelUse;
2698
2699 const TRAP_OPCODE: &'static [u8] = &[0x0f, 0x0b];
2700}
2701
2702pub struct EmitInfo {
2704 pub(super) flags: settings::Flags,
2705 isa_flags: x64_settings::Flags,
2706}
2707
2708impl EmitInfo {
2709 pub fn new(flags: settings::Flags, isa_flags: x64_settings::Flags) -> Self {
2711 Self { flags, isa_flags }
2712 }
2713}
2714
2715impl MachInstEmit for Inst {
2716 type State = EmitState;
2717 type Info = EmitInfo;
2718
2719 fn emit(&self, sink: &mut MachBuffer<Inst>, info: &Self::Info, state: &mut Self::State) {
2720 emit::emit(self, sink, info, state);
2721 }
2722
2723 fn pretty_print_inst(&self, _: &mut Self::State) -> String {
2724 PrettyPrint::pretty_print(self, 0)
2725 }
2726}
2727
2728#[derive(Clone, Copy, Debug, PartialEq, Eq)]
2730pub enum LabelUse {
2731 JmpRel32,
2735
2736 PCRel32,
2739}
2740
2741impl MachInstLabelUse for LabelUse {
2742 const ALIGN: CodeOffset = 1;
2743
2744 fn max_pos_range(self) -> CodeOffset {
2745 match self {
2746 LabelUse::JmpRel32 | LabelUse::PCRel32 => 0x7fff_ffff,
2747 }
2748 }
2749
2750 fn max_neg_range(self) -> CodeOffset {
2751 match self {
2752 LabelUse::JmpRel32 | LabelUse::PCRel32 => 0x8000_0000,
2753 }
2754 }
2755
2756 fn patch_size(self) -> CodeOffset {
2757 match self {
2758 LabelUse::JmpRel32 | LabelUse::PCRel32 => 4,
2759 }
2760 }
2761
2762 fn patch(self, buffer: &mut [u8], use_offset: CodeOffset, label_offset: CodeOffset) {
2763 let pc_rel = (label_offset as i64) - (use_offset as i64);
2764 debug_assert!(pc_rel <= self.max_pos_range() as i64);
2765 debug_assert!(pc_rel >= -(self.max_neg_range() as i64));
2766 let pc_rel = pc_rel as u32;
2767 match self {
2768 LabelUse::JmpRel32 => {
2769 let addend = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2770 let value = pc_rel.wrapping_add(addend).wrapping_sub(4);
2771 buffer.copy_from_slice(&value.to_le_bytes()[..]);
2772 }
2773 LabelUse::PCRel32 => {
2774 let addend = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2775 let value = pc_rel.wrapping_add(addend);
2776 buffer.copy_from_slice(&value.to_le_bytes()[..]);
2777 }
2778 }
2779 }
2780
2781 fn supports_veneer(self) -> bool {
2782 match self {
2783 LabelUse::JmpRel32 | LabelUse::PCRel32 => false,
2784 }
2785 }
2786
2787 fn veneer_size(self) -> CodeOffset {
2788 match self {
2789 LabelUse::JmpRel32 | LabelUse::PCRel32 => 0,
2790 }
2791 }
2792
2793 fn worst_case_veneer_size() -> CodeOffset {
2794 0
2795 }
2796
2797 fn generate_veneer(self, _: &mut [u8], _: CodeOffset) -> (CodeOffset, LabelUse) {
2798 match self {
2799 LabelUse::JmpRel32 | LabelUse::PCRel32 => {
2800 panic!("Veneer not supported for JumpRel32 label-use.");
2801 }
2802 }
2803 }
2804
2805 fn from_reloc(reloc: Reloc, addend: Addend) -> Option<Self> {
2806 match (reloc, addend) {
2807 (Reloc::X86CallPCRel4, -4) => Some(LabelUse::JmpRel32),
2808 _ => None,
2809 }
2810 }
2811}