github
/
cranelift
mirror of https://github.com/bytecodealliance/wasmtime.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

Add methods to construct RexFlags from OperandSizes. 

This unifies the logic around Rex prefix emission and hopefully makes REX prefix errors less likely.
There are still several instructions that use other sources to determine the flags, so set_w and clear_w are left as is.

Additional cleanups:
  * Change always_emit_if_8bit_needed to take a Reg instead of a u8 for type safety.
  * Deduplicated emission code in MovRM.
pull/2654/head
Kasey Carrothers 4 years ago
parent
8d90ea0390
commit
9c3edee9d0
1 changed files with 129 additions and 193 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 322
      cranelift/codegen/src/isa/x64/inst/emit.rs

322
cranelift/codegen/src/isa/x64/inst/emit.rs
View File

@ -85,8 +85,9 @@ impl RexFlags {
} }
#[inline(always)] #[inline(always)]
fn always_emit_if_8bit_needed(&mut self, reg: u8) -> &mut Self { fn always_emit_if_8bit_needed(&mut self, reg: Reg) -> &mut Self {
if reg >= 4 && reg <= 7 { let enc_reg = int_reg_enc(reg);
if enc_reg >= 4 && enc_reg <= 7 {
self.always_emit(); self.always_emit();
} }
self self
@ -126,6 +127,26 @@ impl RexFlags {
} }
} }
/// Generate the proper Rex flags for the given operand size.
impl From<OperandSize> for RexFlags {
fn from(size: OperandSize) -> Self {
match size {
OperandSize::Size64 => RexFlags::set_w(),
_ => RexFlags::clear_w(),
}
}
}
/// Generate Rex flags for an OperandSize/register tuple.
impl From<(OperandSize, Reg)> for RexFlags {
fn from((size, reg): (OperandSize, Reg)) -> Self {
let mut rex = RexFlags::from(size);
if size == OperandSize::Size8 {
rex.always_emit_if_8bit_needed(reg);
}
rex
}
}
/// We may need to include one or more legacy prefix bytes before the REX prefix. This enum /// We may need to include one or more legacy prefix bytes before the REX prefix. This enum
/// covers only the small set of possibilities that we actually need. /// covers only the small set of possibilities that we actually need.
enum LegacyPrefixes { enum LegacyPrefixes {
@ -546,12 +567,7 @@ pub(crate) fn emit(
src, src,
dst: reg_g, dst: reg_g,
} => { } => {
let mut rex = if *size == OperandSize::Size64 { let mut rex = RexFlags::from(*size);
RexFlags::set_w()
} else {
RexFlags::clear_w()
};
if *op == AluRmiROpcode::Mul { if *op == AluRmiROpcode::Mul {
// We kinda freeloaded Mul into RMI_R_Op, but it doesn't fit the usual pattern, so // We kinda freeloaded Mul into RMI_R_Op, but it doesn't fit the usual pattern, so
// we have to special-case it. // we have to special-case it.
@ -617,8 +633,8 @@ pub(crate) fn emit(
match src { match src {
RegMemImm::Reg { reg: reg_e } => { RegMemImm::Reg { reg: reg_e } => {
if is_8bit { if is_8bit {
rex.always_emit_if_8bit_needed(int_reg_enc(*reg_e)); rex.always_emit_if_8bit_needed(*reg_e);
rex.always_emit_if_8bit_needed(int_reg_enc(reg_g.to_reg())); rex.always_emit_if_8bit_needed(reg_g.to_reg());
} }
// GCC/llvm use the swapped operand encoding (viz., the R/RM vs RM/R // GCC/llvm use the swapped operand encoding (viz., the R/RM vs RM/R
// duality). Do this too, so as to be able to compare generated machine // duality). Do this too, so as to be able to compare generated machine
@ -636,7 +652,7 @@ pub(crate) fn emit(
RegMemImm::Mem { addr } => { RegMemImm::Mem { addr } => {
if is_8bit { if is_8bit {
rex.always_emit_if_8bit_needed(int_reg_enc(reg_g.to_reg())); rex.always_emit_if_8bit_needed(reg_g.to_reg());
} }
// Here we revert to the "normal" G-E ordering. // Here we revert to the "normal" G-E ordering.
let amode = addr.finalize(state, sink); let amode = addr.finalize(state, sink);
@ -675,12 +691,7 @@ pub(crate) fn emit(
} }
Inst::UnaryRmR { size, op, src, dst } => { Inst::UnaryRmR { size, op, src, dst } => {
let rex_flags = match size { let rex_flags = RexFlags::from(*size);
OperandSize::Size16 | OperandSize::Size32 => RexFlags::clear_w(),
OperandSize::Size64 => RexFlags::set_w(),
_ => unreachable!(),
};
use UnaryRmROpcode::*; use UnaryRmROpcode::*;
let prefix = match size { let prefix = match size {
OperandSize::Size16 => match op { OperandSize::Size16 => match op {
@ -730,37 +741,31 @@ pub(crate) fn emit(
} }
Inst::Not { size, src } => { Inst::Not { size, src } => {
let src = int_reg_enc(src.to_reg()); let rex_flags = RexFlags::from((*size, src.to_reg()));
let (opcode, prefix, rex_flags) = match size { let (opcode, prefix) = match size {
OperandSize::Size8 => ( OperandSize::Size8 => (0xF6, LegacyPrefixes::None),
0xF6, OperandSize::Size16 => (0xF7, LegacyPrefixes::_66),
LegacyPrefixes::None, OperandSize::Size32 => (0xF7, LegacyPrefixes::None),
*RexFlags::clear_w().always_emit_if_8bit_needed(src), OperandSize::Size64 => (0xF7, LegacyPrefixes::None),
),
OperandSize::Size16 => (0xF7, LegacyPrefixes::_66, RexFlags::clear_w()),
OperandSize::Size32 => (0xF7, LegacyPrefixes::None, RexFlags::clear_w()),
OperandSize::Size64 => (0xF7, LegacyPrefixes::None, RexFlags::set_w()),
}; };
let subopcode = 2; let subopcode = 2;
emit_std_enc_enc(sink, prefix, opcode, 1, subopcode, src, rex_flags) let enc_src = int_reg_enc(src.to_reg());
emit_std_enc_enc(sink, prefix, opcode, 1, subopcode, enc_src, rex_flags)
} }
Inst::Neg { size, src } => { Inst::Neg { size, src } => {
let src = int_reg_enc(src.to_reg()); let rex_flags = RexFlags::from((*size, src.to_reg()));
let (opcode, prefix, rex_flags) = match size { let (opcode, prefix) = match size {
OperandSize::Size8 => ( OperandSize::Size8 => (0xF6, LegacyPrefixes::None),
0xF6, OperandSize::Size16 => (0xF7, LegacyPrefixes::_66),
LegacyPrefixes::None, OperandSize::Size32 => (0xF7, LegacyPrefixes::None),
*RexFlags::clear_w().always_emit_if_8bit_needed(src), OperandSize::Size64 => (0xF7, LegacyPrefixes::None),
),
OperandSize::Size16 => (0xF7, LegacyPrefixes::_66, RexFlags::clear_w()),
OperandSize::Size32 => (0xF7, LegacyPrefixes::None, RexFlags::clear_w()),
OperandSize::Size64 => (0xF7, LegacyPrefixes::None, RexFlags::set_w()),
}; };
let subopcode = 3; let subopcode = 3;
emit_std_enc_enc(sink, prefix, opcode, 1, subopcode, src, rex_flags) let enc_src = int_reg_enc(src.to_reg());
emit_std_enc_enc(sink, prefix, opcode, 1, subopcode, enc_src, rex_flags)
} }
Inst::Div { Inst::Div {
@ -768,11 +773,11 @@ pub(crate) fn emit(
signed, signed,
divisor, divisor,
} => { } => {
let (opcode, prefix, mut rex_flags) = match size { let (opcode, prefix) = match size {
OperandSize::Size8 => (0xF6, LegacyPrefixes::None, RexFlags::clear_w()), OperandSize::Size8 => (0xF6, LegacyPrefixes::None),
OperandSize::Size16 => (0xF7, LegacyPrefixes::_66, RexFlags::clear_w()), OperandSize::Size16 => (0xF7, LegacyPrefixes::_66),
OperandSize::Size32 => (0xF7, LegacyPrefixes::None, RexFlags::clear_w()), OperandSize::Size32 => (0xF7, LegacyPrefixes::None),
OperandSize::Size64 => (0xF7, LegacyPrefixes::None, RexFlags::set_w()), OperandSize::Size64 => (0xF7, LegacyPrefixes::None),
}; };
let loc = state.cur_srcloc(); let loc = state.cur_srcloc();
@ -782,25 +787,39 @@ pub(crate) fn emit(
match divisor { match divisor {
RegMem::Reg { reg } => { RegMem::Reg { reg } => {
let src = int_reg_enc(*reg); let src = int_reg_enc(*reg);
if *size == OperandSize::Size8 { emit_std_enc_enc(
rex_flags.always_emit_if_8bit_needed(src); sink,
} prefix,
emit_std_enc_enc(sink, prefix, opcode, 1, subopcode, src, rex_flags) opcode,
1,
subopcode,
src,
RexFlags::from((*size, *reg)),
)
} }
RegMem::Mem { addr: src } => { RegMem::Mem { addr: src } => {
let amode = src.finalize(state, sink); let amode = src.finalize(state, sink);
emit_std_enc_mem( emit_std_enc_mem(
sink, state, info, prefix, opcode, 1, subopcode, &amode, rex_flags, sink,
state,
info,
prefix,
opcode,
1,
subopcode,
&amode,
RexFlags::from(*size),
); );
} }
} }
} }
Inst::MulHi { size, signed, rhs } => { Inst::MulHi { size, signed, rhs } => {
let (prefix, rex_flags) = match size { let rex_flags = RexFlags::from(*size);
OperandSize::Size16 => (LegacyPrefixes::_66, RexFlags::clear_w()), let prefix = match size {
OperandSize::Size32 => (LegacyPrefixes::None, RexFlags::clear_w()), OperandSize::Size16 => LegacyPrefixes::_66,
OperandSize::Size64 => (LegacyPrefixes::None, RexFlags::set_w()), OperandSize::Size32 => LegacyPrefixes::None,
OperandSize::Size64 => LegacyPrefixes::None,
_ => unreachable!(), _ => unreachable!(),
}; };
@ -993,12 +1012,15 @@ pub(crate) fn emit(
} }
Inst::MovRR { size, src, dst } => { Inst::MovRR { size, src, dst } => {
let rex = if *size == OperandSize::Size64 { emit_std_reg_reg(
RexFlags::set_w() sink,
} else { LegacyPrefixes::None,
RexFlags::clear_w() 0x89,
}; 1,
emit_std_reg_reg(sink, LegacyPrefixes::None, 0x89, 1, *src, dst.to_reg(), rex); *src,
dst.to_reg(),
RexFlags::from(*size),
);
} }
Inst::MovzxRmR { ext_mode, src, dst } => { Inst::MovzxRmR { ext_mode, src, dst } => {
@ -1038,8 +1060,7 @@ pub(crate) fn emit(
match ext_mode { match ext_mode {
ExtMode::BL | ExtMode::BQ => { ExtMode::BL | ExtMode::BQ => {
// A redundant REX prefix must be emitted for certain register inputs. // A redundant REX prefix must be emitted for certain register inputs.
let enc_src = int_reg_enc(*src); rex_flags.always_emit_if_8bit_needed(*src);
rex_flags.always_emit_if_8bit_needed(enc_src);
} }
_ => {} _ => {}
} }
@ -1133,8 +1154,7 @@ pub(crate) fn emit(
match ext_mode { match ext_mode {
ExtMode::BL | ExtMode::BQ => { ExtMode::BL | ExtMode::BQ => {
// A redundant REX prefix must be emitted for certain register inputs. // A redundant REX prefix must be emitted for certain register inputs.
let enc_src = int_reg_enc(*src); rex_flags.always_emit_if_8bit_needed(*src);
rex_flags.always_emit_if_8bit_needed(enc_src);
} }
_ => {} _ => {}
} }
@ -1170,75 +1190,26 @@ pub(crate) fn emit(
Inst::MovRM { size, src, dst } => { Inst::MovRM { size, src, dst } => {
let dst = &dst.finalize(state, sink); let dst = &dst.finalize(state, sink);
match size { let prefix = match size {
OperandSize::Size8 => { OperandSize::Size16 => LegacyPrefixes::_66,
// This is one of the few places where the presence of a _ => LegacyPrefixes::None,
// redundant REX prefix changes the meaning of the };
// instruction.
let mut rex = RexFlags::clear_w();
let enc_src = int_reg_enc(*src);
rex.always_emit_if_8bit_needed(enc_src);
// MOV r8, r/m8 is (REX.W==0) 88 /r
emit_std_reg_mem(
sink,
state,
info,
LegacyPrefixes::None,
0x88,
1,
*src,
dst,
rex,
)
}
OperandSize::Size16 => { let opcode = match size {
// MOV r16, r/m16 is 66 (REX.W==0) 89 /r OperandSize::Size8 => 0x88,
emit_std_reg_mem( _ => 0x89,
sink, };
state,
info,
LegacyPrefixes::_66,
0x89,
1,
*src,
dst,
RexFlags::clear_w(),
)
}
OperandSize::Size32 => { // This is one of the few places where the presence of a
// MOV r32, r/m32 is (REX.W==0) 89 /r // redundant REX prefix changes the meaning of the
emit_std_reg_mem( // instruction.
sink, let rex = RexFlags::from((*size, *src));
state,
info,
LegacyPrefixes::None,
0x89,
1,
*src,
dst,
RexFlags::clear_w(),
)
}
OperandSize::Size64 => { // 8-bit: MOV r8, r/m8 is (REX.W==0) 88 /r
// MOV r64, r/m64 is (REX.W==1) 89 /r // 16-bit: MOV r16, r/m16 is 66 (REX.W==0) 89 /r
emit_std_reg_mem( // 32-bit: MOV r32, r/m32 is (REX.W==0) 89 /r
sink, // 64-bit: MOV r64, r/m64 is (REX.W==1) 89 /r
state, emit_std_reg_mem(sink, state, info, prefix, opcode, 1, *src, dst, rex);
info,
LegacyPrefixes::None,
0x89,
1,
*src,
dst,
RexFlags::set_w(),
)
}
}
} }
Inst::ShiftR { Inst::ShiftR {
@ -1247,7 +1218,6 @@ pub(crate) fn emit(
num_bits, num_bits,
dst, dst,
} => { } => {
let enc_dst = int_reg_enc(dst.to_reg());
let subopcode = match kind { let subopcode = match kind {
ShiftKind::RotateLeft => 0, ShiftKind::RotateLeft => 0,
ShiftKind::RotateRight => 1, ShiftKind::RotateRight => 1,
@ -1255,18 +1225,15 @@ pub(crate) fn emit(
ShiftKind::ShiftRightLogical => 5, ShiftKind::ShiftRightLogical => 5,
ShiftKind::ShiftRightArithmetic => 7, ShiftKind::ShiftRightArithmetic => 7,
}; };
let enc_dst = int_reg_enc(dst.to_reg());
let rex_flags = RexFlags::from((*size, dst.to_reg()));
match num_bits { match num_bits {
None => { None => {
let (opcode, prefix, rex_flags) = match size { let (opcode, prefix) = match size {
OperandSize::Size8 => ( OperandSize::Size8 => (0xD2, LegacyPrefixes::None),
0xD2, OperandSize::Size16 => (0xD3, LegacyPrefixes::_66),
LegacyPrefixes::None, OperandSize::Size32 => (0xD3, LegacyPrefixes::None),
*RexFlags::clear_w().always_emit_if_8bit_needed(enc_dst), OperandSize::Size64 => (0xD3, LegacyPrefixes::None),
),
OperandSize::Size16 => (0xD3, LegacyPrefixes::_66, RexFlags::clear_w()),
OperandSize::Size32 => (0xD3, LegacyPrefixes::None, RexFlags::clear_w()),
OperandSize::Size64 => (0xD3, LegacyPrefixes::None, RexFlags::set_w()),
}; };
// SHL/SHR/SAR %cl, reg8 is (REX.W==0) D2 /subopcode // SHL/SHR/SAR %cl, reg8 is (REX.W==0) D2 /subopcode
@ -1277,15 +1244,11 @@ pub(crate) fn emit(
} }
Some(num_bits) => { Some(num_bits) => {
let (opcode, prefix, rex_flags) = match size { let (opcode, prefix) = match size {
OperandSize::Size8 => ( OperandSize::Size8 => (0xC0, LegacyPrefixes::None),
0xC0, OperandSize::Size16 => (0xC1, LegacyPrefixes::_66),
LegacyPrefixes::None, OperandSize::Size32 => (0xC1, LegacyPrefixes::None),
*RexFlags::clear_w().always_emit_if_8bit_needed(enc_dst), OperandSize::Size64 => (0xC1, LegacyPrefixes::None),
),
OperandSize::Size16 => (0xC1, LegacyPrefixes::_66, RexFlags::clear_w()),
OperandSize::Size32 => (0xC1, LegacyPrefixes::None, RexFlags::clear_w()),
OperandSize::Size64 => (0xC1, LegacyPrefixes::None, RexFlags::set_w()),
}; };
// SHL/SHR/SAR $ib, reg8 is (REX.W==0) C0 /subopcode // SHL/SHR/SAR $ib, reg8 is (REX.W==0) C0 /subopcode
@ -1372,25 +1335,14 @@ pub(crate) fn emit(
if *size == OperandSize::Size16 { if *size == OperandSize::Size16 {
prefix = LegacyPrefixes::_66; prefix = LegacyPrefixes::_66;
} }
// A redundant REX prefix can change the meaning of this instruction.
let mut rex = match size { let mut rex = RexFlags::from((*size, *reg_g));
OperandSize::Size64 => RexFlags::set_w(),
OperandSize::Size16 | OperandSize::Size32 => RexFlags::clear_w(),
OperandSize::Size8 => {
let mut rex = RexFlags::clear_w();
// Here, a redundant REX prefix changes the meaning of the instruction.
let enc_g = int_reg_enc(*reg_g);
rex.always_emit_if_8bit_needed(enc_g);
rex
}
};
match src_e { match src_e {
RegMemImm::Reg { reg: reg_e } => { RegMemImm::Reg { reg: reg_e } => {
if *size == OperandSize::Size8 { if *size == OperandSize::Size8 {
// Check whether the E register forces the use of a redundant REX. // Check whether the E register forces the use of a redundant REX.
let enc_e = int_reg_enc(*reg_e); rex.always_emit_if_8bit_needed(*reg_e);
rex.always_emit_if_8bit_needed(enc_e);
} }
// Use the swapped operands encoding for CMP, to stay consistent with the output of // Use the swapped operands encoding for CMP, to stay consistent with the output of
@ -1467,10 +1419,11 @@ pub(crate) fn emit(
src, src,
dst: reg_g, dst: reg_g,
} => { } => {
let (prefix, rex_flags) = match size { let rex_flags = RexFlags::from(*size);
OperandSize::Size16 => (LegacyPrefixes::_66, RexFlags::clear_w()), let prefix = match size {
OperandSize::Size32 => (LegacyPrefixes::None, RexFlags::clear_w()), OperandSize::Size16 => LegacyPrefixes::_66,
OperandSize::Size64 => (LegacyPrefixes::None, RexFlags::set_w()), OperandSize::Size32 => LegacyPrefixes::None,
OperandSize::Size64 => LegacyPrefixes::None,
_ => unreachable!("invalid size spec for cmove"), _ => unreachable!("invalid size spec for cmove"),
}; };
let opcode = 0x0F40 + cc.get_enc() as u32; let opcode = 0x0F40 + cc.get_enc() as u32;
@ -2111,11 +2064,7 @@ pub(crate) fn emit(
SseOpcode::Roundsd => (LegacyPrefixes::_66, 0x0F3A0B, 3), SseOpcode::Roundsd => (LegacyPrefixes::_66, 0x0F3A0B, 3),
_ => unimplemented!("Opcode {:?} not implemented", op), _ => unimplemented!("Opcode {:?} not implemented", op),
}; };
let rex = if *size == OperandSize::Size64 { let rex = RexFlags::from(*size);
RexFlags::set_w()
} else {
RexFlags::clear_w()
};
let regs_swapped = match *op { let regs_swapped = match *op {
// These opcodes (and not the SSE2 version of PEXTRW) flip the operand // These opcodes (and not the SSE2 version of PEXTRW) flip the operand
// encoding: `dst` in ModRM's r/m, `src` in ModRM's reg field. // encoding: `dst` in ModRM's r/m, `src` in ModRM's reg field.
@ -2208,12 +2157,7 @@ pub(crate) fn emit(
SseOpcode::Pmovmskb => (LegacyPrefixes::_66, 0x0FD7, true), SseOpcode::Pmovmskb => (LegacyPrefixes::_66, 0x0FD7, true),
_ => panic!("unexpected opcode {:?}", op), _ => panic!("unexpected opcode {:?}", op),
}; };
let rex = match dst_size { let rex = RexFlags::from(*dst_size);
OperandSize::Size32 => RexFlags::clear_w(),
OperandSize::Size64 => RexFlags::set_w(),
_ => unreachable!(),
};
let (src, dst) = if dst_first { let (src, dst) = if dst_first {
(dst.to_reg(), *src) (dst.to_reg(), *src)
} else { } else {
@ -2237,11 +2181,7 @@ pub(crate) fn emit(
SseOpcode::Cvtsi2sd => (LegacyPrefixes::_F2, 0x0F2A), SseOpcode::Cvtsi2sd => (LegacyPrefixes::_F2, 0x0F2A),
_ => panic!("unexpected opcode {:?}", op), _ => panic!("unexpected opcode {:?}", op),
}; };
let rex = match *src_size { let rex = RexFlags::from(*src_size);
OperandSize::Size32 => RexFlags::clear_w(),
OperandSize::Size64 => RexFlags::set_w(),
_ => unreachable!(),
};
match src_e { match src_e {
RegMem::Reg { reg: reg_e } => { RegMem::Reg { reg: reg_e } => {
emit_std_reg_reg(sink, prefix, opcode, 2, reg_g.to_reg(), *reg_e, rex); emit_std_reg_reg(sink, prefix, opcode, 2, reg_g.to_reg(), *reg_e, rex);
@ -2827,18 +2767,14 @@ pub(crate) fn emit(
Inst::LockCmpxchg { ty, src, dst } => { Inst::LockCmpxchg { ty, src, dst } => {
// lock cmpxchg{b,w,l,q} %src, (dst) // lock cmpxchg{b,w,l,q} %src, (dst)
// Note that 0xF0 is the Lock prefix. // Note that 0xF0 is the Lock prefix.
let (prefix, rex, opcodes) = match *ty { let (prefix, opcodes) = match *ty {
types::I8 => { types::I8 => (LegacyPrefixes::_F0, 0x0FB0),
let mut rex_flags = RexFlags::clear_w(); types::I16 => (LegacyPrefixes::_66F0, 0x0FB1),
let enc_src = int_reg_enc(*src); types::I32 => (LegacyPrefixes::_F0, 0x0FB1),
rex_flags.always_emit_if_8bit_needed(enc_src); types::I64 => (LegacyPrefixes::_F0, 0x0FB1),
(LegacyPrefixes::_F0, rex_flags, 0x0FB0)
}
types::I16 => (LegacyPrefixes::_66F0, RexFlags::clear_w(), 0x0FB1),
types::I32 => (LegacyPrefixes::_F0, RexFlags::clear_w(), 0x0FB1),
types::I64 => (LegacyPrefixes::_F0, RexFlags::set_w(), 0x0FB1),
_ => unreachable!(), _ => unreachable!(),
}; };
let rex = RexFlags::from((OperandSize::from_ty(*ty), *src));
let amode = dst.finalize(state, sink); let amode = dst.finalize(state, sink);
emit_std_reg_mem(sink, state, info, prefix, opcodes, 2, *src, &amode, rex); emit_std_reg_mem(sink, state, info, prefix, opcodes, 2, *src, &amode, rex);
} }

Write Preview
Loading…
Cancel
Save