@ -119,6 +119,7 @@ use std::collections::HashMap;
use std ::convert ::TryFrom ;
use std ::marker ::PhantomData ;
use std ::mem ;
use std ::ops ::Range ;
/// A small vector of instructions (with some reasonable size); appropriate for
/// a small fixed sequence implementing one operation.
@ -304,6 +305,40 @@ pub trait IsaFlags: Clone {
}
}
/// Used as an out-parameter to accumulate a sequence of `ABIArg`s in
/// `ABIMachineSpec::compute_arg_locs`. Wraps the shared allocation for all
/// `ABIArg`s in `SigSet` and exposes just the args for the current
/// `compute_arg_locs` call.
pub struct ArgsAccumulator < 'a > {
sig_set_abi_args : & 'a mut Vec < ABIArg > ,
start : usize ,
}
impl < 'a > ArgsAccumulator < 'a > {
fn new ( sig_set_abi_args : & 'a mut Vec < ABIArg > ) -> Self {
let start = sig_set_abi_args . len ( ) ;
ArgsAccumulator {
sig_set_abi_args ,
start ,
}
}
#[ inline ]
pub fn push ( & mut self , arg : ABIArg ) {
self . sig_set_abi_args . push ( arg )
}
#[ inline ]
pub fn args ( & self ) -> & [ ABIArg ] {
& self . sig_set_abi_args [ self . start . . ]
}
#[ inline ]
pub fn args_mut ( & mut self ) -> & mut [ ABIArg ] {
& mut self . sig_set_abi_args [ self . start . . ]
}
}
/// Trait implemented by machine-specific backend to provide information about
/// register assignments and to allow generating the specific instructions for
/// stack loads/saves, prologues/epilogues, etc.
@ -342,16 +377,20 @@ pub trait ABIMachineSpec {
/// Process a list of parameters or return values and allocate them to registers
/// and stack slots.
///
/// Returns the list of argument locations, the stack-space used (rounded up
/// to as alignment requires), and if `add_ret_area_ptr` was passed, the
/// index of the extra synthetic arg that was added.
/// The argument locations should be pushed onto the given `ArgsAccumulator`
/// in order.
///
/// Returns the stack-space used (rounded up to as alignment requires), and
/// if `add_ret_area_ptr` was passed, the index of the extra synthetic arg
/// that was added.
fn compute_arg_locs < 'a , I > (
call_conv : isa ::CallConv ,
flags : & settings ::Flags ,
params : I ,
args_or_rets : ArgsOrRets ,
add_ret_area_ptr : bool ,
) -> CodegenResult < ( ABIArgVec , i64 , Option < usize > ) >
args : ArgsAccumulator < '_ > ,
) -> CodegenResult < ( i64 , Option < usize > ) >
where
I : IntoIterator < Item = & 'a ir ::AbiParam > ;
@ -554,9 +593,6 @@ pub trait ABIMachineSpec {
) -> ir ::ArgumentExtension ;
}
// A vector of `ABIArg`s with inline capacity, since they are typically small.
pub type ABIArgVec = SmallVec < [ ABIArg ; 6 ] > ;
/// The id of an ABI signature within the `SigSet`.
#[ derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord) ]
pub struct Sig ( u32 ) ;
@ -567,22 +603,32 @@ cranelift_entity::entity_impl!(Sig);
pub struct SigData {
/// Argument locations (regs or stack slots). Stack offsets are relative to
/// SP on entry to function.
args : ABIArgVec ,
///
/// These are indices into the `SigSet::abi_args`.
arg_indices : Range < u32 > ,
/// Return-value locations. Stack offsets are relative to the return-area
/// pointer.
rets : ABIArgVec ,
///
/// These are indices into the `SigSet::abi_args`.
ret_indices : Range < u32 > ,
/// Space on stack used to store arguments.
sized_stack_arg_space : i64 ,
/// Space on stack used to store return values.
sized_stack_ret_space : i64 ,
/// Index in `args` of the stack-return-value-area argument.
stack_ret_arg : Option < usize > ,
/// Calling convention used.
call_conv : isa ::CallConv ,
}
impl SigData {
pub fn from_func_sig < M : ABIMachineSpec > (
sigs : & mut SigSet ,
sig : & ir ::Signature ,
flags : & settings ::Flags ,
) -> CodegenResult < SigData > {
@ -591,35 +637,45 @@ impl SigData {
// Compute args and retvals from signature. Handle retvals first,
// because we may need to add a return-area arg to the args.
let ( rets , sized_stack_ret_space , _ ) = M ::compute_arg_locs (
let rets_start = u32 ::try_from ( sigs . abi_args . len ( ) ) . unwrap ( ) ;
let ( sized_stack_ret_space , _ ) = M ::compute_arg_locs (
sig . call_conv ,
flags ,
returns ,
ArgsOrRets ::Rets ,
/* extra ret-area ptr = */ false ,
ArgsAccumulator ::new ( & mut sigs . abi_args ) ,
) ? ;
let rets_end = u32 ::try_from ( sigs . abi_args . len ( ) ) . unwrap ( ) ;
let need_stack_return_area = sized_stack_ret_space > 0 ;
let ( args , sized_stack_arg_space , stack_ret_arg ) = M ::compute_arg_locs (
let args_start = u32 ::try_from ( sigs . abi_args . len ( ) ) . unwrap ( ) ;
let ( sized_stack_arg_space , stack_ret_arg ) = M ::compute_arg_locs (
sig . call_conv ,
flags ,
& sig . params ,
ArgsOrRets ::Args ,
need_stack_return_area ,
ArgsAccumulator ::new ( & mut sigs . abi_args ) ,
) ? ;
let args_end = u32 ::try_from ( sigs . abi_args . len ( ) ) . unwrap ( ) ;
let arg_indices = args_start . . args_end ;
let ret_indices = rets_start . . rets_end ;
trace ! (
"ABISig: sig {:?} => args = {:?} rets = {:?} arg stack = {} ret stack = {} stack_ret_arg = {:?}" ,
sig ,
args ,
rets ,
arg_indice s ,
ret_indice s ,
sized_stack_arg_space ,
sized_stack_ret_space ,
stack_ret_arg ,
) ;
Ok ( SigData {
args ,
rets ,
arg_indice s ,
ret_indice s ,
sized_stack_arg_space ,
sized_stack_ret_space ,
stack_ret_arg ,
@ -627,8 +683,22 @@ impl SigData {
} )
}
/// Get this signature's ABI arguments.
pub fn args < 'a > ( & self , sigs : & 'a SigSet ) -> & 'a [ ABIArg ] {
let start = usize ::try_from ( self . arg_indices . start ) . unwrap ( ) ;
let end = usize ::try_from ( self . arg_indices . end ) . unwrap ( ) ;
& sigs . abi_args [ start . . end ]
}
/// Get this signature's ABI returns.
pub fn rets < 'a > ( & self , sigs : & 'a SigSet ) -> & 'a [ ABIArg ] {
let start = usize ::try_from ( self . ret_indices . start ) . unwrap ( ) ;
let end = usize ::try_from ( self . ret_indices . end ) . unwrap ( ) ;
& sigs . abi_args [ start . . end ]
}
/// Return all clobbers for the callsite.
pub fn call_clobbers < M : ABIMachineSpec > ( & self ) -> PRegSet {
pub fn call_clobbers < M : ABIMachineSpec > ( & self , sigs : & SigSet ) -> PRegSet {
// Get clobbers: all caller-saves. These may include return value
// regs, which we will remove from the clobber set below.
let mut clobbers = M ::get_regs_clobbered_by_call ( self . call_conv ) ;
@ -636,7 +706,7 @@ impl SigData {
// Remove retval regs from clobbers. Skip StructRets: these
// are not, semantically, returns at the CLIF level, so we
// treat such a value as a clobber instead.
for ret in & self . rets {
for ret in self . rets ( sigs ) {
if let & ABIArg ::Slots {
ref slots , purpose , . .
} = ret
@ -661,16 +731,18 @@ impl SigData {
/// Get the number of arguments expected.
pub fn num_args ( & self ) -> usize {
let len = self . arg_indices . end - self . arg_indices . start ;
let len = usize ::try_from ( len ) . unwrap ( ) ;
if self . stack_ret_arg . is_some ( ) {
self . args . len ( ) - 1
len - 1
} else {
self . args . len ( )
len
}
}
/// Get information specifying how to pass one argument.
pub fn get_arg ( & self , idx : usize ) -> ABIArg {
self . args [ idx ] . clone ( )
pub fn get_arg ( & self , sigs : & SigSet , idx : usize ) -> ABIArg {
self . args ( sigs ) [ idx ] . clone ( )
}
/// Get total stack space required for arguments.
@ -680,12 +752,13 @@ impl SigData {
/// Get the number of return values expected.
pub fn num_rets ( & self ) -> usize {
self . rets . len ( )
let len = self . ret_indices . end - self . ret_indices . start ;
usize ::try_from ( len ) . unwrap ( )
}
/// Get information specifying how to pass one return value.
pub fn get_ret ( & self , idx : usize ) -> ABIArg {
self . rets [ idx ] . clone ( )
pub fn get_ret ( & self , sigs : & SigSet , idx : usize ) -> ABIArg {
self . rets ( sigs ) [ idx ] . clone ( )
}
/// Get total stack space required for return values.
@ -695,9 +768,9 @@ impl SigData {
/// Get information specifying how to pass the implicit pointer
/// to the return-value area on the stack, if required.
pub fn get_ret_arg ( & self ) -> Option < ABIArg > {
pub fn get_ret_arg ( & self , sigs : & SigSet ) -> Option < ABIArg > {
let ret_arg = self . stack_ret_arg ? ;
Some ( self . args [ ret_arg ] . clone ( ) )
Some ( self . args ( sigs ) [ ret_arg ] . clone ( ) )
}
/// Get calling convention used.
@ -728,6 +801,11 @@ pub struct SigSet {
/// Interned `ir::SigRef`s that we already have an ABI signature for.
ir_sig_ref_to_abi_sig : SecondaryMap < ir ::SigRef , Option < Sig > > ,
/// A single, shared allocation for all `ABIArg`s used by all
/// `SigData`s. Each `SigData` references its args/rets via indices into
/// this allocation.
abi_args : Vec < ABIArg > ,
/// The actual ABI signatures, keyed by `Sig`.
sigs : PrimaryMap < Sig , SigData > ,
}
@ -739,9 +817,12 @@ impl SigSet {
where
M : ABIMachineSpec ,
{
let arg_estimate = func . dfg . signatures . len ( ) * 6 ;
let mut sigs = SigSet {
ir_signature_to_abi_sig : FxHashMap ::default ( ) ,
ir_sig_ref_to_abi_sig : SecondaryMap ::with_capacity ( func . dfg . signatures . len ( ) ) ,
abi_args : Vec ::with_capacity ( arg_estimate ) ,
sigs : PrimaryMap ::with_capacity ( 1 + func . dfg . signatures . len ( ) ) ,
} ;
@ -776,7 +857,7 @@ impl SigSet {
// `ir::Signature`.
debug_assert ! ( ! self . have_abi_sig_for_signature ( & signature ) ) ;
let sig_data = SigData ::from_func_sig ::< M > ( & signature , flags ) ? ;
let sig_data = SigData ::from_func_sig ::< M > ( self , & signature , flags ) ? ;
let sig = self . sigs . push ( sig_data ) ;
self . ir_signature_to_abi_sig . insert ( signature , sig ) ;
Ok ( sig )
@ -795,7 +876,7 @@ impl SigSet {
return Ok ( sig ) ;
}
let signature = & dfg . signatures [ sig_ref ] ;
let sig_data = SigData ::from_func_sig ::< M > ( signature , flags ) ? ;
let sig_data = SigData ::from_func_sig ::< M > ( self , signature , flags ) ? ;
let sig = self . sigs . push ( sig_data ) ;
self . ir_sig_ref_to_abi_sig [ sig_ref ] = Some ( sig ) ;
Ok ( sig )
@ -896,11 +977,12 @@ pub struct Callee<M: ABIMachineSpec> {
fn get_special_purpose_param_register (
f : & ir ::Function ,
sigs : & SigSet ,
abi : & SigData ,
purpose : ir ::ArgumentPurpose ,
) -> Option < Reg > {
let idx = f . signature . special_param_index ( purpose ) ? ;
match & abi . args [ idx ] {
match & abi . args ( sigs ) [ idx ] {
& ABIArg ::Slots { ref slots , . . } = > match & slots [ 0 ] {
& ABIArgSlot ::Reg { reg , . . } = > Some ( reg . into ( ) ) ,
_ = > None ,
@ -977,13 +1059,17 @@ impl<M: ABIMachineSpec> Callee<M> {
// stack limit. This can either be specified as a special-purpose
// argument or as a global value which often calculates the stack limit
// from the arguments.
let stack_limit =
get_special_purpose_param_register ( f , & sigs [ sig ] , ir ::ArgumentPurpose ::StackLimit )
. map ( | reg | ( reg , smallvec ! [ ] ) )
. or_else ( | | {
f . stack_limit
. map ( | gv | gen_stack_limit ::< M > ( f , & sigs [ sig ] , gv ) )
} ) ;
let stack_limit = get_special_purpose_param_register (
f ,
sigs ,
& sigs [ sig ] ,
ir ::ArgumentPurpose ::StackLimit ,
)
. map ( | reg | ( reg , smallvec ! [ ] ) )
. or_else ( | | {
f . stack_limit
. map ( | gv | gen_stack_limit ::< M > ( f , sigs , & sigs [ sig ] , gv ) )
} ) ;
// Determine whether a probestack call is required for large enough
// frames (and the minimum frame size if so).
@ -1102,16 +1188,18 @@ impl<M: ABIMachineSpec> Callee<M> {
/// it's used, because we're not participating in register allocation anyway!
fn gen_stack_limit < M : ABIMachineSpec > (
f : & ir ::Function ,
sigs : & SigSet ,
abi : & SigData ,
gv : ir ::GlobalValue ,
) -> ( Reg , SmallInstVec < M ::I > ) {
let mut insts = smallvec ! [ ] ;
let reg = generate_gv ::< M > ( f , abi , gv , & mut insts ) ;
let reg = generate_gv ::< M > ( f , sigs , abi , gv , & mut insts ) ;
return ( reg , insts ) ;
}
fn generate_gv < M : ABIMachineSpec > (
f : & ir ::Function ,
sigs : & SigSet ,
abi : & SigData ,
gv : ir ::GlobalValue ,
insts : & mut SmallInstVec < M ::I > ,
@ -1119,7 +1207,7 @@ fn generate_gv<M: ABIMachineSpec>(
match f . global_values [ gv ] {
// Return the direct register the vmcontext is in
ir ::GlobalValueData ::VMContext = > {
get_special_purpose_param_register ( f , abi , ir ::ArgumentPurpose ::VMContext )
get_special_purpose_param_register ( f , sigs , abi , ir ::ArgumentPurpose ::VMContext )
. expect ( "no vmcontext parameter found" )
}
// Load our base value into a register, then load from that register
@ -1130,7 +1218,7 @@ fn generate_gv<M: ABIMachineSpec>(
global_type : _ ,
readonly : _ ,
} = > {
let base = generate_gv ::< M > ( f , abi , base , insts ) ;
let base = generate_gv ::< M > ( f , sigs , abi , base , insts ) ;
let into_reg = Writable ::from_reg ( M ::get_stacklimit_reg ( ) ) ;
insts . push ( M ::gen_load_base_offset (
into_reg ,
@ -1212,7 +1300,7 @@ impl<M: ABIMachineSpec> Callee<M> {
/// They will be provided to `init()` as the `temps` arg if so.
pub fn temps_needed ( & self , sigs : & SigSet ) -> Vec < Type > {
let mut temp_tys = vec ! [ ] ;
for arg in & sigs [ self . sig ] . args {
for arg in sigs [ self . sig ] . args ( sigs ) {
match arg {
& ABIArg ::ImplicitPtrArg { pointer , . . } = > match & pointer {
& ABIArgSlot ::Reg { . . } = > { }
@ -1234,7 +1322,7 @@ impl<M: ABIMachineSpec> Callee<M> {
/// once the lowering context exists.
pub fn init ( & mut self , sigs : & SigSet , temps : Vec < Writable < Reg > > ) {
let mut temps_iter = temps . into_iter ( ) ;
for arg in & sigs [ self . sig ] . args {
for arg in sigs [ self . sig ] . args ( sigs ) {
let temp = match arg {
& ABIArg ::ImplicitPtrArg { pointer , . . } = > match & pointer {
& ABIArgSlot ::Reg { . . } = > None ,
@ -1331,7 +1419,7 @@ impl<M: ABIMachineSpec> Callee<M> {
}
} ;
match & sigs [ self . sig ] . args [ idx ] {
match & sigs [ self . sig ] . args ( sigs ) [ idx ] {
& ABIArg ::Slots { ref slots , . . } = > {
assert_eq ! ( into_regs . len ( ) , slots . len ( ) ) ;
for ( slot , into_reg ) in slots . iter ( ) . zip ( into_regs . regs ( ) . iter ( ) ) {
@ -1399,7 +1487,7 @@ impl<M: ABIMachineSpec> Callee<M> {
) -> SmallInstVec < M ::I > {
let mut ret = smallvec ! [ ] ;
let word_bits = M ::word_bits ( ) as u8 ;
match & sigs [ self . sig ] . rets [ idx ] {
match & sigs [ self . sig ] . rets ( sigs ) [ idx ] {
& ABIArg ::Slots { ref slots , . . } = > {
assert_eq ! ( from_regs . len ( ) , slots . len ( ) ) ;
for ( slot , & from_reg ) in slots . iter ( ) . zip ( from_regs . regs ( ) . iter ( ) ) {
@ -1508,7 +1596,7 @@ impl<M: ABIMachineSpec> Callee<M> {
/// Generate a return instruction.
pub fn gen_ret ( & self , sigs : & SigSet ) -> M ::I {
let mut rets = vec ! [ ] ;
for ret in & sigs [ self . sig ] . rets {
for ret in sigs [ self . sig ] . rets ( sigs ) {
match ret {
ABIArg ::Slots { slots , . . } = > {
for slot in slots {
@ -1895,7 +1983,7 @@ impl<M: ABIMachineSpec> Caller<M> {
flags : settings ::Flags ,
) -> CodegenResult < Caller < M > > {
let sig = sigs . abi_sig_for_sig_ref ( sig_ref ) ;
let clobbers = sigs [ sig ] . call_clobbers ::< M > ( ) ;
let clobbers = sigs [ sig ] . call_clobbers ::< M > ( sigs ) ;
Ok ( Caller {
sig ,
uses : smallvec ! [ ] ,
@ -1920,7 +2008,7 @@ impl<M: ABIMachineSpec> Caller<M> {
flags : settings ::Flags ,
) -> CodegenResult < Caller < M > > {
let sig = sigs . abi_sig_for_signature ( sig ) ;
let clobbers = sigs [ sig ] . call_clobbers ::< M > ( ) ;
let clobbers = sigs [ sig ] . call_clobbers ::< M > ( sigs ) ;
Ok ( Caller {
sig ,
uses : smallvec ! [ ] ,
@ -1945,7 +2033,7 @@ impl<M: ABIMachineSpec> Caller<M> {
flags : settings ::Flags ,
) -> CodegenResult < Caller < M > > {
let sig = sigs . abi_sig_for_sig_ref ( sig_ref ) ;
let clobbers = sigs [ sig ] . call_clobbers ::< M > ( ) ;
let clobbers = sigs [ sig ] . call_clobbers ::< M > ( sigs ) ;
Ok ( Caller {
sig ,
uses : smallvec ! [ ] ,
@ -1975,10 +2063,11 @@ impl<M: ABIMachineSpec> Caller<M> {
/// Get the number of arguments expected.
pub fn num_args ( & self , sigs : & SigSet ) -> usize {
let data = & sigs [ self . sig ] ;
let len = data . args ( sigs ) . len ( ) ;
if data . stack_ret_arg . is_some ( ) {
data . args . len ( ) - 1
len - 1
} else {
data . args . len ( )
len
}
}
@ -2007,7 +2096,7 @@ impl<M: ABIMachineSpec> Caller<M> {
idx : usize ,
from_regs : ValueRegs < Reg > ,
) {
match & ctx . sigs ( ) [ self . sig ] . args [ idx ] {
match & ctx . sigs ( ) [ self . sig ] . args ( ctx . sigs ( ) ) [ idx ] {
& ABIArg ::Slots { . . } = > { }
& ABIArg ::StructArg { offset , size , . . } = > {
let src_ptr = from_regs . only_reg ( ) . unwrap ( ) ;
@ -2059,7 +2148,7 @@ impl<M: ABIMachineSpec> Caller<M> {
// How many temps do we need for extends? Allocate them ahead
// of time, since we can't do it while we're iterating over
// the sig and immutably borrowing `ctx`.
let needed_tmps = match & ctx . sigs ( ) [ self . sig ] . args [ idx ] {
let needed_tmps = match & ctx . sigs ( ) [ self . sig ] . args ( ctx . sigs ( ) ) [ idx ] {
& ABIArg ::Slots { ref slots , . . } = > slots
. iter ( )
. map ( | slot | match slot {
@ -2084,7 +2173,7 @@ impl<M: ABIMachineSpec> Caller<M> {
. map ( | _ | ctx . alloc_tmp ( M ::word_type ( ) ) . only_reg ( ) . unwrap ( ) )
. collect ( ) ;
match & ctx . sigs ( ) [ self . sig ] . args [ idx ] {
match & ctx . sigs ( ) [ self . sig ] . args ( ctx . sigs ( ) ) [ idx ] {
& ABIArg ::Slots { ref slots , . . } = > {
assert_eq ! ( from_regs . len ( ) , slots . len ( ) ) ;
for ( slot , from_reg ) in slots . iter ( ) . zip ( from_regs . regs ( ) . iter ( ) ) {
@ -2186,7 +2275,7 @@ impl<M: ABIMachineSpec> Caller<M> {
into_regs : ValueRegs < Writable < Reg > > ,
) -> SmallInstVec < M ::I > {
let mut insts = smallvec ! [ ] ;
match & ctx . sigs ( ) [ self . sig ] . rets [ idx ] {
match & ctx . sigs ( ) [ self . sig ] . rets ( ctx . sigs ( ) ) [ idx ] {
& ABIArg ::Slots { ref slots , . . } = > {
assert_eq ! ( into_regs . len ( ) , slots . len ( ) ) ;
for ( slot , into_reg ) in slots . iter ( ) . zip ( into_regs . regs ( ) . iter ( ) ) {
Nick Fitzgerald
2 years ago
GitHub