diff --git a/cranelift/codegen/src/isa/s390x/lower.rs b/cranelift/codegen/src/isa/s390x/lower.rs index fd33f8e1f3..fe402f83ad 100644 --- a/cranelift/codegen/src/isa/s390x/lower.rs +++ b/cranelift/codegen/src/isa/s390x/lower.rs @@ -1,329 +1,15 @@ //! Lowering rules for S390x. -use crate::ir::condcodes::IntCC; use crate::ir::Inst as IRInst; -use crate::ir::{MemFlags, Opcode}; -use crate::isa::s390x::abi::*; -use crate::isa::s390x::inst::*; -use crate::isa::s390x::settings as s390x_settings; +use crate::ir::Opcode; +use crate::isa::s390x::inst::Inst; use crate::isa::s390x::S390xBackend; -use crate::machinst::lower::*; -use crate::machinst::*; -use crate::settings::Flags; +use crate::machinst::{InsnOutput, LowerBackend, LowerCtx, MachLabel}; use crate::CodegenResult; use smallvec::SmallVec; pub mod isle; -//============================================================================ -// Lowering: force instruction input into a register - -/// Sign-extend the low `from_bits` bits of `value` to a full u64. -fn sign_extend_to_u64(value: u64, from_bits: u8) -> u64 { - assert!(from_bits <= 64); - if from_bits >= 64 { - value - } else { - (((value << (64 - from_bits)) as i64) >> (64 - from_bits)) as u64 - } -} - -//============================================================================= -// Lowering: comparisons - -/// Determines whether this condcode interprets inputs as signed or -/// unsigned. See the documentation for the `icmp` instruction in -/// cranelift-codegen/meta/src/shared/instructions.rs for further insights -/// into this. -pub fn condcode_is_signed(cc: IntCC) -> bool { - match cc { - IntCC::Equal => false, - IntCC::NotEqual => false, - IntCC::SignedGreaterThanOrEqual => true, - IntCC::SignedGreaterThan => true, - IntCC::SignedLessThanOrEqual => true, - IntCC::SignedLessThan => true, - IntCC::UnsignedGreaterThanOrEqual => false, - IntCC::UnsignedGreaterThan => false, - IntCC::UnsignedLessThanOrEqual => false, - IntCC::UnsignedLessThan => false, - IntCC::Overflow => true, - IntCC::NotOverflow => true, - } -} - -//============================================================================ -// Lowering: main entry point for lowering a instruction - -fn lower_insn_to_regs>( - ctx: &mut C, - insn: IRInst, - flags: &Flags, - isa_flags: &s390x_settings::Flags, -) -> CodegenResult<()> { - let op = ctx.data(insn).opcode(); - let outputs: SmallVec<[InsnOutput; 2]> = (0..ctx.num_outputs(insn)) - .map(|i| InsnOutput { insn, output: i }) - .collect(); - let ty = if outputs.len() > 0 { - Some(ctx.output_ty(insn, 0)) - } else { - None - }; - - if let Ok(()) = super::lower::isle::lower(ctx, flags, isa_flags, &outputs, insn) { - return Ok(()); - } - - let implemented_in_isle = || { - unreachable!( - "implemented in ISLE: inst = `{}`, type = `{:?}`", - ctx.dfg().display_inst(insn), - ty - ); - }; - - match op { - Opcode::Nop - | Opcode::Copy - | Opcode::Iconst - | Opcode::Bconst - | Opcode::F32const - | Opcode::F64const - | Opcode::Null - | Opcode::Iadd - | Opcode::IaddIfcout - | Opcode::Isub - | Opcode::Iabs - | Opcode::Ineg - | Opcode::Imul - | Opcode::Umulhi - | Opcode::Smulhi - | Opcode::Udiv - | Opcode::Urem - | Opcode::Sdiv - | Opcode::Srem - | Opcode::Ishl - | Opcode::Ushr - | Opcode::Sshr - | Opcode::Rotr - | Opcode::Rotl - | Opcode::Ireduce - | Opcode::Uextend - | Opcode::Sextend - | Opcode::Bnot - | Opcode::Band - | Opcode::Bor - | Opcode::Bxor - | Opcode::BandNot - | Opcode::BorNot - | Opcode::BxorNot - | Opcode::Bitselect - | Opcode::Breduce - | Opcode::Bextend - | Opcode::Bmask - | Opcode::Bint - | Opcode::Clz - | Opcode::Cls - | Opcode::Ctz - | Opcode::Popcnt - | Opcode::Fadd - | Opcode::Fsub - | Opcode::Fmul - | Opcode::Fdiv - | Opcode::Fmin - | Opcode::Fmax - | Opcode::Sqrt - | Opcode::Fneg - | Opcode::Fabs - | Opcode::Fpromote - | Opcode::Fdemote - | Opcode::Ceil - | Opcode::Floor - | Opcode::Trunc - | Opcode::Nearest - | Opcode::Fma - | Opcode::Fcopysign - | Opcode::FcvtFromUint - | Opcode::FcvtFromSint - | Opcode::FcvtToUint - | Opcode::FcvtToSint - | Opcode::FcvtToUintSat - | Opcode::FcvtToSintSat - | Opcode::Bitcast - | Opcode::Load - | Opcode::Uload8 - | Opcode::Sload8 - | Opcode::Uload16 - | Opcode::Sload16 - | Opcode::Uload32 - | Opcode::Sload32 - | Opcode::Store - | Opcode::Istore8 - | Opcode::Istore16 - | Opcode::Istore32 - | Opcode::AtomicRmw - | Opcode::AtomicCas - | Opcode::AtomicLoad - | Opcode::AtomicStore - | Opcode::Fence - | Opcode::Icmp - | Opcode::Fcmp - | Opcode::IsNull - | Opcode::IsInvalid - | Opcode::Select - | Opcode::SelectifSpectreGuard - | Opcode::Trap - | Opcode::ResumableTrap - | Opcode::Trapz - | Opcode::Trapnz - | Opcode::ResumableTrapnz - | Opcode::Trapif - | Opcode::Debugtrap - | Opcode::Call - | Opcode::CallIndirect - | Opcode::FallthroughReturn - | Opcode::Return - | Opcode::StackAddr - | Opcode::FuncAddr - | Opcode::SymbolValue => implemented_in_isle(), - - Opcode::UaddSat | Opcode::SaddSat => unimplemented!(), - Opcode::UsubSat | Opcode::SsubSat => unimplemented!(), - - Opcode::Bitrev => unimplemented!(), - - Opcode::FcvtLowFromSint => unimplemented!("FcvtLowFromSint"), - - Opcode::StackLoad | Opcode::StackStore => { - panic!("Direct stack memory access not supported; should not be used by Wasm"); - } - - Opcode::ConstAddr => unimplemented!(), - - Opcode::HeapAddr => { - panic!("heap_addr should have been removed by legalization!"); - } - - Opcode::TableAddr => { - panic!("table_addr should have been removed by legalization!"); - } - - Opcode::GlobalValue => { - panic!("global_value should have been removed by legalization!"); - } - - Opcode::TlsValue => { - unimplemented!("Thread-local storage support not implemented!"); - } - - Opcode::GetPinnedReg | Opcode::SetPinnedReg => { - unimplemented!("Pinned register support not implemented!"); - } - - Opcode::RawBitcast - | Opcode::Splat - | Opcode::Swizzle - | Opcode::Insertlane - | Opcode::Extractlane - | Opcode::Imin - | Opcode::Umin - | Opcode::Imax - | Opcode::Umax - | Opcode::AvgRound - | Opcode::FminPseudo - | Opcode::FmaxPseudo - | Opcode::Uload8x8 - | Opcode::Sload8x8 - | Opcode::Uload16x4 - | Opcode::Sload16x4 - | Opcode::Uload32x2 - | Opcode::Sload32x2 - | Opcode::Vconst - | Opcode::Shuffle - | Opcode::Vsplit - | Opcode::Vconcat - | Opcode::Vselect - | Opcode::VanyTrue - | Opcode::VallTrue - | Opcode::VhighBits - | Opcode::ScalarToVector - | Opcode::Snarrow - | Opcode::Unarrow - | Opcode::Uunarrow - | Opcode::SwidenLow - | Opcode::SwidenHigh - | Opcode::UwidenLow - | Opcode::UwidenHigh - | Opcode::WideningPairwiseDotProductS - | Opcode::SqmulRoundSat - | Opcode::FvpromoteLow - | Opcode::Fvdemote - | Opcode::IaddPairwise => { - // TODO - unimplemented!("Vector ops not implemented."); - } - - Opcode::Isplit | Opcode::Iconcat => unimplemented!("Wide integer ops not implemented."), - - Opcode::IfcmpSp => { - panic!("Unused opcode should not be encountered."); - } - - Opcode::Ifcmp - | Opcode::Ffcmp - | Opcode::Trapff - | Opcode::Trueif - | Opcode::Trueff - | Opcode::Selectif => { - panic!("Flags opcode should not be encountered."); - } - - Opcode::Jump - | Opcode::Brz - | Opcode::Brnz - | Opcode::BrIcmp - | Opcode::Brif - | Opcode::Brff - | Opcode::BrTable => { - panic!("Branch opcode reached non-branch lowering logic!"); - } - - Opcode::IaddImm - | Opcode::ImulImm - | Opcode::UdivImm - | Opcode::SdivImm - | Opcode::UremImm - | Opcode::SremImm - | Opcode::IrsubImm - | Opcode::IaddCin - | Opcode::IaddIfcin - | Opcode::IaddCout - | Opcode::IaddCarry - | Opcode::IaddIfcarry - | Opcode::IsubBin - | Opcode::IsubIfbin - | Opcode::IsubBout - | Opcode::IsubIfbout - | Opcode::IsubBorrow - | Opcode::IsubIfborrow - | Opcode::BandImm - | Opcode::BorImm - | Opcode::BxorImm - | Opcode::RotlImm - | Opcode::RotrImm - | Opcode::IshlImm - | Opcode::UshrImm - | Opcode::SshrImm - | Opcode::IcmpImm - | Opcode::IfcmpImm => { - panic!("ALU+imm and ALU+carry ops should not appear here!"); - } - } - - Ok(()) -} - //============================================================================= // Lowering-backend trait implementation. @@ -331,7 +17,255 @@ impl LowerBackend for S390xBackend { type MInst = Inst; fn lower>(&self, ctx: &mut C, ir_inst: IRInst) -> CodegenResult<()> { - lower_insn_to_regs(ctx, ir_inst, &self.flags, &self.isa_flags) + let op = ctx.data(ir_inst).opcode(); + let outputs: SmallVec<[InsnOutput; 2]> = (0..ctx.num_outputs(ir_inst)) + .map(|i| InsnOutput { + insn: ir_inst, + output: i, + }) + .collect(); + let ty = if outputs.len() > 0 { + Some(ctx.output_ty(ir_inst, 0)) + } else { + None + }; + + if let Ok(()) = + super::lower::isle::lower(ctx, &self.flags, &self.isa_flags, &outputs, ir_inst) + { + return Ok(()); + } + + match op { + Opcode::Nop + | Opcode::Copy + | Opcode::Iconst + | Opcode::Bconst + | Opcode::F32const + | Opcode::F64const + | Opcode::Null + | Opcode::Iadd + | Opcode::IaddIfcout + | Opcode::Isub + | Opcode::Iabs + | Opcode::Ineg + | Opcode::Imul + | Opcode::Umulhi + | Opcode::Smulhi + | Opcode::Udiv + | Opcode::Urem + | Opcode::Sdiv + | Opcode::Srem + | Opcode::Ishl + | Opcode::Ushr + | Opcode::Sshr + | Opcode::Rotr + | Opcode::Rotl + | Opcode::Ireduce + | Opcode::Uextend + | Opcode::Sextend + | Opcode::Bnot + | Opcode::Band + | Opcode::Bor + | Opcode::Bxor + | Opcode::BandNot + | Opcode::BorNot + | Opcode::BxorNot + | Opcode::Bitselect + | Opcode::Breduce + | Opcode::Bextend + | Opcode::Bmask + | Opcode::Bint + | Opcode::Clz + | Opcode::Cls + | Opcode::Ctz + | Opcode::Popcnt + | Opcode::Fadd + | Opcode::Fsub + | Opcode::Fmul + | Opcode::Fdiv + | Opcode::Fmin + | Opcode::Fmax + | Opcode::Sqrt + | Opcode::Fneg + | Opcode::Fabs + | Opcode::Fpromote + | Opcode::Fdemote + | Opcode::Ceil + | Opcode::Floor + | Opcode::Trunc + | Opcode::Nearest + | Opcode::Fma + | Opcode::Fcopysign + | Opcode::FcvtFromUint + | Opcode::FcvtFromSint + | Opcode::FcvtToUint + | Opcode::FcvtToSint + | Opcode::FcvtToUintSat + | Opcode::FcvtToSintSat + | Opcode::Bitcast + | Opcode::Load + | Opcode::Uload8 + | Opcode::Sload8 + | Opcode::Uload16 + | Opcode::Sload16 + | Opcode::Uload32 + | Opcode::Sload32 + | Opcode::Store + | Opcode::Istore8 + | Opcode::Istore16 + | Opcode::Istore32 + | Opcode::AtomicRmw + | Opcode::AtomicCas + | Opcode::AtomicLoad + | Opcode::AtomicStore + | Opcode::Fence + | Opcode::Icmp + | Opcode::Fcmp + | Opcode::IsNull + | Opcode::IsInvalid + | Opcode::Select + | Opcode::SelectifSpectreGuard + | Opcode::Trap + | Opcode::ResumableTrap + | Opcode::Trapz + | Opcode::Trapnz + | Opcode::ResumableTrapnz + | Opcode::Trapif + | Opcode::Debugtrap + | Opcode::Call + | Opcode::CallIndirect + | Opcode::FallthroughReturn + | Opcode::Return + | Opcode::StackAddr + | Opcode::FuncAddr + | Opcode::SymbolValue => { + unreachable!( + "implemented in ISLE: inst = `{}`, type = `{:?}`", + ctx.dfg().display_inst(ir_inst), + ty + ) + } + + Opcode::UaddSat + | Opcode::SaddSat + | Opcode::UsubSat + | Opcode::SsubSat + | Opcode::Bitrev + | Opcode::FcvtLowFromSint + | Opcode::ConstAddr + | Opcode::TlsValue + | Opcode::GetPinnedReg + | Opcode::SetPinnedReg + | Opcode::Isplit + | Opcode::Iconcat + | Opcode::RawBitcast + | Opcode::Splat + | Opcode::Swizzle + | Opcode::Insertlane + | Opcode::Extractlane + | Opcode::Imin + | Opcode::Umin + | Opcode::Imax + | Opcode::Umax + | Opcode::AvgRound + | Opcode::FminPseudo + | Opcode::FmaxPseudo + | Opcode::Uload8x8 + | Opcode::Sload8x8 + | Opcode::Uload16x4 + | Opcode::Sload16x4 + | Opcode::Uload32x2 + | Opcode::Sload32x2 + | Opcode::Vconst + | Opcode::Shuffle + | Opcode::Vsplit + | Opcode::Vconcat + | Opcode::Vselect + | Opcode::VanyTrue + | Opcode::VallTrue + | Opcode::VhighBits + | Opcode::ScalarToVector + | Opcode::Snarrow + | Opcode::Unarrow + | Opcode::Uunarrow + | Opcode::SwidenLow + | Opcode::SwidenHigh + | Opcode::UwidenLow + | Opcode::UwidenHigh + | Opcode::WideningPairwiseDotProductS + | Opcode::SqmulRoundSat + | Opcode::FvpromoteLow + | Opcode::Fvdemote + | Opcode::IaddPairwise => { + unreachable!( + "TODO: not yet implemented in ISLE: inst = `{}`, type = `{:?}`", + ctx.dfg().display_inst(ir_inst), + ty + ) + } + + Opcode::StackLoad | Opcode::StackStore => { + panic!("Direct stack memory access not supported; should not be used by Wasm"); + } + Opcode::HeapAddr => { + panic!("heap_addr should have been removed by legalization!"); + } + Opcode::TableAddr => { + panic!("table_addr should have been removed by legalization!"); + } + Opcode::GlobalValue => { + panic!("global_value should have been removed by legalization!"); + } + Opcode::Ifcmp + | Opcode::IfcmpSp + | Opcode::Ffcmp + | Opcode::Trapff + | Opcode::Trueif + | Opcode::Trueff + | Opcode::Selectif => { + panic!("Flags opcode should not be encountered."); + } + Opcode::Jump + | Opcode::Brz + | Opcode::Brnz + | Opcode::BrIcmp + | Opcode::Brif + | Opcode::Brff + | Opcode::BrTable => { + panic!("Branch opcode reached non-branch lowering logic!"); + } + Opcode::IaddImm + | Opcode::ImulImm + | Opcode::UdivImm + | Opcode::SdivImm + | Opcode::UremImm + | Opcode::SremImm + | Opcode::IrsubImm + | Opcode::IaddCin + | Opcode::IaddIfcin + | Opcode::IaddCout + | Opcode::IaddCarry + | Opcode::IaddIfcarry + | Opcode::IsubBin + | Opcode::IsubIfbin + | Opcode::IsubBout + | Opcode::IsubIfbout + | Opcode::IsubBorrow + | Opcode::IsubIfborrow + | Opcode::BandImm + | Opcode::BorImm + | Opcode::BxorImm + | Opcode::RotlImm + | Opcode::RotrImm + | Opcode::IshlImm + | Opcode::UshrImm + | Opcode::SshrImm + | Opcode::IcmpImm + | Opcode::IfcmpImm => { + panic!("ALU+imm and ALU+carry ops should not appear here!"); + } + } } fn lower_branch_group>( diff --git a/cranelift/codegen/src/isa/s390x/lower/isle.rs b/cranelift/codegen/src/isa/s390x/lower/isle.rs index c7132c6ed0..aad9c8ac7e 100644 --- a/cranelift/codegen/src/isa/s390x/lower/isle.rs +++ b/cranelift/codegen/src/isa/s390x/lower/isle.rs @@ -4,17 +4,19 @@ pub mod generated_code; // Types that the generated ISLE code uses via `use super::*`. -use super::{ - CallIndInfo, CallInfo, Cond, Inst as MInst, MachLabel, MemArg, MemFlags, Opcode, Reg, - S390xMachineDeps, UImm16Shifted, UImm32Shifted, +use crate::isa::s390x::abi::S390xMachineDeps; +use crate::isa::s390x::inst::{ + stack_reg, writable_gpr, zero_reg, CallIndInfo, CallInfo, Cond, Inst as MInst, MemArg, + UImm16Shifted, UImm32Shifted, }; use crate::isa::s390x::settings::Flags as IsaFlags; use crate::machinst::isle::*; +use crate::machinst::{MachLabel, Reg}; use crate::settings::Flags; use crate::{ ir::{ condcodes::*, immediates::*, types::*, AtomicRmwOp, Endianness, Inst, InstructionData, - StackSlot, TrapCode, Value, ValueList, + MemFlags, Opcode, StackSlot, TrapCode, Value, ValueList, }, isa::unwind::UnwindInst, machinst::{InsnOutput, LowerCtx, VCodeConstant, VCodeConstantData}, @@ -150,12 +152,12 @@ where #[inline] fn writable_gpr(&mut self, regno: u8) -> WritableReg { - super::writable_gpr(regno) + writable_gpr(regno) } #[inline] fn zero_reg(&mut self) -> Reg { - super::zero_reg() + zero_reg() } #[inline] @@ -282,10 +284,7 @@ where let inst = self.lower_ctx.dfg().value_def(val).inst()?; let constant = self.lower_ctx.get_constant(inst)?; let ty = self.lower_ctx.output_ty(inst, 0); - Some(super::sign_extend_to_u64( - constant, - self.ty_bits(ty).unwrap(), - )) + Some(sign_extend_to_u64(constant, self.ty_bits(ty).unwrap())) } #[inline] @@ -390,7 +389,7 @@ where #[inline] fn signed(&mut self, cc: &IntCC) -> Option<()> { - if super::condcode_is_signed(*cc) { + if condcode_is_signed(*cc) { Some(()) } else { None @@ -399,7 +398,7 @@ where #[inline] fn unsigned(&mut self, cc: &IntCC) -> Option<()> { - if !super::condcode_is_signed(*cc) { + if !condcode_is_signed(*cc) { Some(()) } else { None @@ -468,7 +467,7 @@ where #[inline] fn memarg_stack_off(&mut self, base: i64, off: i64) -> MemArg { - MemArg::reg_plus_off(super::stack_reg(), base + off, MemFlags::trusted()) + MemArg::reg_plus_off(stack_reg(), base + off, MemFlags::trusted()) } #[inline] @@ -555,3 +554,36 @@ where self.lower_ctx.emit(inst.clone()); } } + +/// Sign-extend the low `from_bits` bits of `value` to a full u64. +#[inline] +fn sign_extend_to_u64(value: u64, from_bits: u8) -> u64 { + assert!(from_bits <= 64); + if from_bits >= 64 { + value + } else { + (((value << (64 - from_bits)) as i64) >> (64 - from_bits)) as u64 + } +} + +/// Determines whether this condcode interprets inputs as signed or +/// unsigned. See the documentation for the `icmp` instruction in +/// cranelift-codegen/meta/src/shared/instructions.rs for further insights +/// into this. +#[inline] +fn condcode_is_signed(cc: IntCC) -> bool { + match cc { + IntCC::Equal => false, + IntCC::NotEqual => false, + IntCC::SignedGreaterThanOrEqual => true, + IntCC::SignedGreaterThan => true, + IntCC::SignedLessThanOrEqual => true, + IntCC::SignedLessThan => true, + IntCC::UnsignedGreaterThanOrEqual => false, + IntCC::UnsignedGreaterThan => false, + IntCC::UnsignedLessThanOrEqual => false, + IntCC::UnsignedLessThan => false, + IntCC::Overflow => true, + IntCC::NotOverflow => true, + } +}