|
|
|
@ -5,86 +5,218 @@ |
|
|
|
*/ |
|
|
|
|
|
|
|
#include <assert.h> |
|
|
|
#include <cdefs.h> |
|
|
|
#include <stdbool.h> |
|
|
|
|
|
|
|
#include "../amu_private.h" |
|
|
|
#include <arch.h> |
|
|
|
#include <arch_features.h> |
|
|
|
#include <arch_helpers.h> |
|
|
|
|
|
|
|
#include <common/debug.h> |
|
|
|
#include <lib/el3_runtime/pubsub_events.h> |
|
|
|
#include <lib/extensions/amu.h> |
|
|
|
#include <lib/extensions/amu_private.h> |
|
|
|
|
|
|
|
#include <plat/common/platform.h> |
|
|
|
|
|
|
|
static struct amu_ctx amu_ctxs[PLATFORM_CORE_COUNT]; |
|
|
|
#if ENABLE_AMU_FCONF |
|
|
|
# include <lib/fconf/fconf.h> |
|
|
|
# include <lib/fconf/fconf_amu_getter.h> |
|
|
|
#endif |
|
|
|
|
|
|
|
/*
|
|
|
|
* Get AMU version value from aa64pfr0. |
|
|
|
* Return values |
|
|
|
* ID_AA64PFR0_AMU_V1: FEAT_AMUv1 supported (introduced in ARM v8.4) |
|
|
|
* ID_AA64PFR0_AMU_V1P1: FEAT_AMUv1p1 supported (introduced in ARM v8.6) |
|
|
|
* ID_AA64PFR0_AMU_NOT_SUPPORTED: not supported |
|
|
|
*/ |
|
|
|
unsigned int amu_get_version(void) |
|
|
|
#if ENABLE_MPMM |
|
|
|
# include <lib/mpmm/mpmm.h> |
|
|
|
#endif |
|
|
|
|
|
|
|
struct amu_ctx { |
|
|
|
uint64_t group0_cnts[AMU_GROUP0_MAX_COUNTERS]; |
|
|
|
#if ENABLE_AMU_AUXILIARY_COUNTERS |
|
|
|
uint64_t group1_cnts[AMU_GROUP1_MAX_COUNTERS]; |
|
|
|
#endif |
|
|
|
|
|
|
|
/* Architected event counter 1 does not have an offset register */ |
|
|
|
uint64_t group0_voffsets[AMU_GROUP0_MAX_COUNTERS - 1U]; |
|
|
|
#if ENABLE_AMU_AUXILIARY_COUNTERS |
|
|
|
uint64_t group1_voffsets[AMU_GROUP1_MAX_COUNTERS]; |
|
|
|
#endif |
|
|
|
|
|
|
|
uint16_t group0_enable; |
|
|
|
#if ENABLE_AMU_AUXILIARY_COUNTERS |
|
|
|
uint16_t group1_enable; |
|
|
|
#endif |
|
|
|
}; |
|
|
|
|
|
|
|
static struct amu_ctx amu_ctxs_[PLATFORM_CORE_COUNT]; |
|
|
|
|
|
|
|
CASSERT((sizeof(amu_ctxs_[0].group0_enable) * CHAR_BIT) <= AMU_GROUP0_MAX_COUNTERS, |
|
|
|
amu_ctx_group0_enable_cannot_represent_all_group0_counters); |
|
|
|
|
|
|
|
#if ENABLE_AMU_AUXILIARY_COUNTERS |
|
|
|
CASSERT((sizeof(amu_ctxs_[0].group1_enable) * CHAR_BIT) <= AMU_GROUP1_MAX_COUNTERS, |
|
|
|
amu_ctx_group1_enable_cannot_represent_all_group1_counters); |
|
|
|
#endif |
|
|
|
|
|
|
|
static inline __unused uint64_t read_id_aa64pfr0_el1_amu(void) |
|
|
|
{ |
|
|
|
return (unsigned int)(read_id_aa64pfr0_el1() >> ID_AA64PFR0_AMU_SHIFT) & |
|
|
|
return (read_id_aa64pfr0_el1() >> ID_AA64PFR0_AMU_SHIFT) & |
|
|
|
ID_AA64PFR0_AMU_MASK; |
|
|
|
} |
|
|
|
|
|
|
|
#if AMU_GROUP1_NR_COUNTERS |
|
|
|
/* Check if group 1 counters is implemented */ |
|
|
|
bool amu_group1_supported(void) |
|
|
|
static inline __unused uint64_t read_hcr_el2_amvoffen(void) |
|
|
|
{ |
|
|
|
return (read_hcr_el2() & HCR_AMVOFFEN_BIT) >> |
|
|
|
HCR_AMVOFFEN_SHIFT; |
|
|
|
} |
|
|
|
|
|
|
|
static inline __unused void write_cptr_el2_tam(uint64_t value) |
|
|
|
{ |
|
|
|
uint64_t features = read_amcfgr_el0() >> AMCFGR_EL0_NCG_SHIFT; |
|
|
|
write_cptr_el2((read_cptr_el2() & ~CPTR_EL2_TAM_BIT) | |
|
|
|
((value << CPTR_EL2_TAM_SHIFT) & CPTR_EL2_TAM_BIT)); |
|
|
|
} |
|
|
|
|
|
|
|
static inline __unused void write_cptr_el3_tam(cpu_context_t *ctx, uint64_t tam) |
|
|
|
{ |
|
|
|
uint64_t value = read_ctx_reg(get_el3state_ctx(ctx), CTX_CPTR_EL3); |
|
|
|
|
|
|
|
return (features & AMCFGR_EL0_NCG_MASK) == 1U; |
|
|
|
value &= ~TAM_BIT; |
|
|
|
value |= (tam << TAM_SHIFT) & TAM_BIT; |
|
|
|
|
|
|
|
write_ctx_reg(get_el3state_ctx(ctx), CTX_CPTR_EL3, value); |
|
|
|
} |
|
|
|
|
|
|
|
static inline __unused void write_hcr_el2_amvoffen(uint64_t value) |
|
|
|
{ |
|
|
|
write_hcr_el2((read_hcr_el2() & ~HCR_AMVOFFEN_BIT) | |
|
|
|
((value << HCR_AMVOFFEN_SHIFT) & HCR_AMVOFFEN_BIT)); |
|
|
|
} |
|
|
|
|
|
|
|
static inline __unused void write_amcr_el0_cg1rz(uint64_t value) |
|
|
|
{ |
|
|
|
write_amcr_el0((read_amcr_el0() & ~AMCR_CG1RZ_BIT) | |
|
|
|
((value << AMCR_CG1RZ_SHIFT) & AMCR_CG1RZ_BIT)); |
|
|
|
} |
|
|
|
|
|
|
|
static inline __unused uint64_t read_amcfgr_el0_ncg(void) |
|
|
|
{ |
|
|
|
return (read_amcfgr_el0() >> AMCFGR_EL0_NCG_SHIFT) & |
|
|
|
AMCFGR_EL0_NCG_MASK; |
|
|
|
} |
|
|
|
|
|
|
|
static inline __unused uint64_t read_amcgcr_el0_cg0nc(void) |
|
|
|
{ |
|
|
|
return (read_amcgcr_el0() >> AMCGCR_EL0_CG0NC_SHIFT) & |
|
|
|
AMCGCR_EL0_CG0NC_MASK; |
|
|
|
} |
|
|
|
|
|
|
|
static inline __unused uint64_t read_amcg1idr_el0_voff(void) |
|
|
|
{ |
|
|
|
return (read_amcg1idr_el0() >> AMCG1IDR_VOFF_SHIFT) & |
|
|
|
AMCG1IDR_VOFF_MASK; |
|
|
|
} |
|
|
|
|
|
|
|
static inline __unused uint64_t read_amcgcr_el0_cg1nc(void) |
|
|
|
{ |
|
|
|
return (read_amcgcr_el0() >> AMCGCR_EL0_CG1NC_SHIFT) & |
|
|
|
AMCGCR_EL0_CG1NC_MASK; |
|
|
|
} |
|
|
|
|
|
|
|
static inline __unused uint64_t read_amcntenset0_el0_px(void) |
|
|
|
{ |
|
|
|
return (read_amcntenset0_el0() >> AMCNTENSET0_EL0_Pn_SHIFT) & |
|
|
|
AMCNTENSET0_EL0_Pn_MASK; |
|
|
|
} |
|
|
|
|
|
|
|
static inline __unused uint64_t read_amcntenset1_el0_px(void) |
|
|
|
{ |
|
|
|
return (read_amcntenset1_el0() >> AMCNTENSET1_EL0_Pn_SHIFT) & |
|
|
|
AMCNTENSET1_EL0_Pn_MASK; |
|
|
|
} |
|
|
|
|
|
|
|
static inline __unused void write_amcntenset0_el0_px(uint64_t px) |
|
|
|
{ |
|
|
|
uint64_t value = read_amcntenset0_el0(); |
|
|
|
|
|
|
|
value &= ~AMCNTENSET0_EL0_Pn_MASK; |
|
|
|
value |= (px << AMCNTENSET0_EL0_Pn_SHIFT) & AMCNTENSET0_EL0_Pn_MASK; |
|
|
|
|
|
|
|
write_amcntenset0_el0(value); |
|
|
|
} |
|
|
|
|
|
|
|
static inline __unused void write_amcntenset1_el0_px(uint64_t px) |
|
|
|
{ |
|
|
|
uint64_t value = read_amcntenset1_el0(); |
|
|
|
|
|
|
|
value &= ~AMCNTENSET1_EL0_Pn_MASK; |
|
|
|
value |= (px << AMCNTENSET1_EL0_Pn_SHIFT) & AMCNTENSET1_EL0_Pn_MASK; |
|
|
|
|
|
|
|
write_amcntenset1_el0(value); |
|
|
|
} |
|
|
|
|
|
|
|
static inline __unused void write_amcntenclr0_el0_px(uint64_t px) |
|
|
|
{ |
|
|
|
uint64_t value = read_amcntenclr0_el0(); |
|
|
|
|
|
|
|
value &= ~AMCNTENCLR0_EL0_Pn_MASK; |
|
|
|
value |= (px << AMCNTENCLR0_EL0_Pn_SHIFT) & AMCNTENCLR0_EL0_Pn_MASK; |
|
|
|
|
|
|
|
write_amcntenclr0_el0(value); |
|
|
|
} |
|
|
|
|
|
|
|
static inline __unused void write_amcntenclr1_el0_px(uint64_t px) |
|
|
|
{ |
|
|
|
uint64_t value = read_amcntenclr1_el0(); |
|
|
|
|
|
|
|
value &= ~AMCNTENCLR1_EL0_Pn_MASK; |
|
|
|
value |= (px << AMCNTENCLR1_EL0_Pn_SHIFT) & AMCNTENCLR1_EL0_Pn_MASK; |
|
|
|
|
|
|
|
write_amcntenclr1_el0(value); |
|
|
|
} |
|
|
|
|
|
|
|
static __unused bool amu_supported(void) |
|
|
|
{ |
|
|
|
return read_id_aa64pfr0_el1_amu() >= ID_AA64PFR0_AMU_V1; |
|
|
|
} |
|
|
|
|
|
|
|
static __unused bool amu_v1p1_supported(void) |
|
|
|
{ |
|
|
|
return read_id_aa64pfr0_el1_amu() >= ID_AA64PFR0_AMU_V1P1; |
|
|
|
} |
|
|
|
|
|
|
|
#if ENABLE_AMU_AUXILIARY_COUNTERS |
|
|
|
static __unused bool amu_group1_supported(void) |
|
|
|
{ |
|
|
|
return read_amcfgr_el0_ncg() > 0U; |
|
|
|
} |
|
|
|
#endif |
|
|
|
|
|
|
|
/*
|
|
|
|
* Enable counters. This function is meant to be invoked |
|
|
|
* by the context management library before exiting from EL3. |
|
|
|
* Enable counters. This function is meant to be invoked by the context |
|
|
|
* management library before exiting from EL3. |
|
|
|
*/ |
|
|
|
void amu_enable(bool el2_unused, cpu_context_t *ctx) |
|
|
|
{ |
|
|
|
uint64_t v; |
|
|
|
unsigned int amu_version = amu_get_version(); |
|
|
|
uint64_t id_aa64pfr0_el1_amu; /* AMU version */ |
|
|
|
|
|
|
|
if (amu_version == ID_AA64PFR0_AMU_NOT_SUPPORTED) { |
|
|
|
return; |
|
|
|
} |
|
|
|
uint64_t amcfgr_el0_ncg; /* Number of counter groups */ |
|
|
|
uint64_t amcgcr_el0_cg0nc; /* Number of group 0 counters */ |
|
|
|
|
|
|
|
#if AMU_GROUP1_NR_COUNTERS |
|
|
|
/* Check and set presence of group 1 counters */ |
|
|
|
if (!amu_group1_supported()) { |
|
|
|
ERROR("AMU Counter Group 1 is not implemented\n"); |
|
|
|
panic(); |
|
|
|
} |
|
|
|
uint64_t amcntenset0_el0_px = 0x0; /* Group 0 enable mask */ |
|
|
|
uint64_t amcntenset1_el0_px = 0x0; /* Group 1 enable mask */ |
|
|
|
|
|
|
|
/* Check number of group 1 counters */ |
|
|
|
uint64_t cnt_num = (read_amcgcr_el0() >> AMCGCR_EL0_CG1NC_SHIFT) & |
|
|
|
AMCGCR_EL0_CG1NC_MASK; |
|
|
|
VERBOSE("%s%llu. %s%u\n", |
|
|
|
"Number of AMU Group 1 Counters ", cnt_num, |
|
|
|
"Requested number ", AMU_GROUP1_NR_COUNTERS); |
|
|
|
|
|
|
|
if (cnt_num < AMU_GROUP1_NR_COUNTERS) { |
|
|
|
ERROR("%s%llu is less than %s%u\n", |
|
|
|
"Number of AMU Group 1 Counters ", cnt_num, |
|
|
|
"Requested number ", AMU_GROUP1_NR_COUNTERS); |
|
|
|
panic(); |
|
|
|
id_aa64pfr0_el1_amu = read_id_aa64pfr0_el1_amu(); |
|
|
|
if (id_aa64pfr0_el1_amu == ID_AA64PFR0_AMU_NOT_SUPPORTED) { |
|
|
|
/*
|
|
|
|
* If the AMU is unsupported, nothing needs to be done. |
|
|
|
*/ |
|
|
|
|
|
|
|
return; |
|
|
|
} |
|
|
|
#endif |
|
|
|
|
|
|
|
if (el2_unused) { |
|
|
|
/*
|
|
|
|
* CPTR_EL2.TAM: Set to zero so any accesses to |
|
|
|
* the Activity Monitor registers do not trap to EL2. |
|
|
|
* CPTR_EL2.TAM: Set to zero so any accesses to the Activity |
|
|
|
* Monitor registers do not trap to EL2. |
|
|
|
*/ |
|
|
|
v = read_cptr_el2(); |
|
|
|
v &= ~CPTR_EL2_TAM_BIT; |
|
|
|
write_cptr_el2(v); |
|
|
|
write_cptr_el2_tam(0U); |
|
|
|
} |
|
|
|
|
|
|
|
/*
|
|
|
|
@ -92,72 +224,141 @@ void amu_enable(bool el2_unused, cpu_context_t *ctx) |
|
|
|
* in 'ctx'. Set CPTR_EL3.TAM to zero so that any accesses to |
|
|
|
* the Activity Monitor registers do not trap to EL3. |
|
|
|
*/ |
|
|
|
v = read_ctx_reg(get_el3state_ctx(ctx), CTX_CPTR_EL3); |
|
|
|
v &= ~TAM_BIT; |
|
|
|
write_ctx_reg(get_el3state_ctx(ctx), CTX_CPTR_EL3, v); |
|
|
|
write_cptr_el3_tam(ctx, 0U); |
|
|
|
|
|
|
|
/* Enable group 0 counters */ |
|
|
|
write_amcntenset0_el0(AMU_GROUP0_COUNTERS_MASK); |
|
|
|
/*
|
|
|
|
* Retrieve the number of architected counters. All of these counters |
|
|
|
* are enabled by default. |
|
|
|
*/ |
|
|
|
|
|
|
|
#if AMU_GROUP1_NR_COUNTERS |
|
|
|
/* Enable group 1 counters */ |
|
|
|
write_amcntenset1_el0(AMU_GROUP1_COUNTERS_MASK); |
|
|
|
#endif |
|
|
|
amcgcr_el0_cg0nc = read_amcgcr_el0_cg0nc(); |
|
|
|
amcntenset0_el0_px = (UINT64_C(1) << (amcgcr_el0_cg0nc)) - 1U; |
|
|
|
|
|
|
|
/* Initialize FEAT_AMUv1p1 features if present. */ |
|
|
|
if (amu_version < ID_AA64PFR0_AMU_V1P1) { |
|
|
|
return; |
|
|
|
} |
|
|
|
assert(amcgcr_el0_cg0nc <= AMU_AMCGCR_CG0NC_MAX); |
|
|
|
|
|
|
|
if (el2_unused) { |
|
|
|
/* Make sure virtual offsets are disabled if EL2 not used. */ |
|
|
|
write_hcr_el2(read_hcr_el2() & ~HCR_AMVOFFEN_BIT); |
|
|
|
/*
|
|
|
|
* The platform may opt to enable specific auxiliary counters. This can |
|
|
|
* be done via the common FCONF getter, or via the platform-implemented |
|
|
|
* function. |
|
|
|
*/ |
|
|
|
|
|
|
|
#if ENABLE_AMU_AUXILIARY_COUNTERS |
|
|
|
const struct amu_topology *topology; |
|
|
|
|
|
|
|
#if ENABLE_AMU_FCONF |
|
|
|
topology = FCONF_GET_PROPERTY(amu, config, topology); |
|
|
|
#else |
|
|
|
topology = plat_amu_topology(); |
|
|
|
#endif /* ENABLE_AMU_FCONF */ |
|
|
|
|
|
|
|
if (topology != NULL) { |
|
|
|
unsigned int core_pos = plat_my_core_pos(); |
|
|
|
|
|
|
|
amcntenset1_el0_px = topology->cores[core_pos].enable; |
|
|
|
} else { |
|
|
|
ERROR("AMU: failed to generate AMU topology\n"); |
|
|
|
} |
|
|
|
#endif /* ENABLE_AMU_AUXILIARY_COUNTERS */ |
|
|
|
|
|
|
|
#if AMU_RESTRICT_COUNTERS |
|
|
|
/*
|
|
|
|
* FEAT_AMUv1p1 adds a register field to restrict access to group 1 |
|
|
|
* counters at all but the highest implemented EL. This is controlled |
|
|
|
* with the AMU_RESTRICT_COUNTERS compile time flag, when set, system |
|
|
|
* register reads at lower ELs return zero. Reads from the memory |
|
|
|
* mapped view are unaffected. |
|
|
|
* Enable the requested counters. |
|
|
|
*/ |
|
|
|
VERBOSE("AMU group 1 counter access restricted.\n"); |
|
|
|
write_amcr_el0(read_amcr_el0() | AMCR_CG1RZ_BIT); |
|
|
|
|
|
|
|
write_amcntenset0_el0_px(amcntenset0_el0_px); |
|
|
|
|
|
|
|
amcfgr_el0_ncg = read_amcfgr_el0_ncg(); |
|
|
|
if (amcfgr_el0_ncg > 0U) { |
|
|
|
write_amcntenset1_el0_px(amcntenset1_el0_px); |
|
|
|
|
|
|
|
#if !ENABLE_AMU_AUXILIARY_COUNTERS |
|
|
|
VERBOSE("AMU: auxiliary counters detected but support is disabled\n"); |
|
|
|
#endif |
|
|
|
} |
|
|
|
|
|
|
|
/* Initialize FEAT_AMUv1p1 features if present. */ |
|
|
|
if (id_aa64pfr0_el1_amu >= ID_AA64PFR0_AMU_V1P1) { |
|
|
|
if (el2_unused) { |
|
|
|
/*
|
|
|
|
* Make sure virtual offsets are disabled if EL2 not |
|
|
|
* used. |
|
|
|
*/ |
|
|
|
write_hcr_el2_amvoffen(0U); |
|
|
|
} |
|
|
|
|
|
|
|
#if AMU_RESTRICT_COUNTERS |
|
|
|
/*
|
|
|
|
* FEAT_AMUv1p1 adds a register field to restrict access to |
|
|
|
* group 1 counters at all but the highest implemented EL. This |
|
|
|
* is controlled with the `AMU_RESTRICT_COUNTERS` compile time |
|
|
|
* flag, when set, system register reads at lower ELs return |
|
|
|
* zero. Reads from the memory mapped view are unaffected. |
|
|
|
*/ |
|
|
|
VERBOSE("AMU group 1 counter access restricted.\n"); |
|
|
|
write_amcr_el0_cg1rz(1U); |
|
|
|
#else |
|
|
|
write_amcr_el0(read_amcr_el0() & ~AMCR_CG1RZ_BIT); |
|
|
|
write_amcr_el0_cg1rz(0U); |
|
|
|
#endif |
|
|
|
} |
|
|
|
|
|
|
|
#if ENABLE_MPMM |
|
|
|
mpmm_enable(); |
|
|
|
#endif |
|
|
|
} |
|
|
|
|
|
|
|
/* Read the group 0 counter identified by the given `idx`. */ |
|
|
|
uint64_t amu_group0_cnt_read(unsigned int idx) |
|
|
|
static uint64_t amu_group0_cnt_read(unsigned int idx) |
|
|
|
{ |
|
|
|
assert(amu_get_version() != ID_AA64PFR0_AMU_NOT_SUPPORTED); |
|
|
|
assert(idx < AMU_GROUP0_NR_COUNTERS); |
|
|
|
assert(amu_supported()); |
|
|
|
assert(idx < read_amcgcr_el0_cg0nc()); |
|
|
|
|
|
|
|
return amu_group0_cnt_read_internal(idx); |
|
|
|
} |
|
|
|
|
|
|
|
/* Write the group 0 counter identified by the given `idx` with `val` */ |
|
|
|
void amu_group0_cnt_write(unsigned int idx, uint64_t val) |
|
|
|
static void amu_group0_cnt_write(unsigned int idx, uint64_t val) |
|
|
|
{ |
|
|
|
assert(amu_get_version() != ID_AA64PFR0_AMU_NOT_SUPPORTED); |
|
|
|
assert(idx < AMU_GROUP0_NR_COUNTERS); |
|
|
|
assert(amu_supported()); |
|
|
|
assert(idx < read_amcgcr_el0_cg0nc()); |
|
|
|
|
|
|
|
amu_group0_cnt_write_internal(idx, val); |
|
|
|
isb(); |
|
|
|
} |
|
|
|
|
|
|
|
/*
|
|
|
|
* Unlike with auxiliary counters, we cannot detect at runtime whether an |
|
|
|
* architected counter supports a virtual offset. These are instead fixed |
|
|
|
* according to FEAT_AMUv1p1, but this switch will need to be updated if later |
|
|
|
* revisions of FEAT_AMU add additional architected counters. |
|
|
|
*/ |
|
|
|
static bool amu_group0_voffset_supported(uint64_t idx) |
|
|
|
{ |
|
|
|
switch (idx) { |
|
|
|
case 0U: |
|
|
|
case 2U: |
|
|
|
case 3U: |
|
|
|
return true; |
|
|
|
|
|
|
|
case 1U: |
|
|
|
return false; |
|
|
|
|
|
|
|
default: |
|
|
|
ERROR("AMU: can't set up virtual offset for unknown " |
|
|
|
"architected counter %llu!\n", idx); |
|
|
|
|
|
|
|
panic(); |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
/*
|
|
|
|
* Read the group 0 offset register for a given index. Index must be 0, 2, |
|
|
|
* or 3, the register for 1 does not exist. |
|
|
|
* |
|
|
|
* Using this function requires FEAT_AMUv1p1 support. |
|
|
|
*/ |
|
|
|
uint64_t amu_group0_voffset_read(unsigned int idx) |
|
|
|
static uint64_t amu_group0_voffset_read(unsigned int idx) |
|
|
|
{ |
|
|
|
assert(amu_get_version() >= ID_AA64PFR0_AMU_V1P1); |
|
|
|
assert(idx < AMU_GROUP0_NR_COUNTERS); |
|
|
|
assert(amu_v1p1_supported()); |
|
|
|
assert(idx < read_amcgcr_el0_cg0nc()); |
|
|
|
assert(idx != 1U); |
|
|
|
|
|
|
|
return amu_group0_voffset_read_internal(idx); |
|
|
|
@ -169,33 +370,33 @@ uint64_t amu_group0_voffset_read(unsigned int idx) |
|
|
|
* |
|
|
|
* Using this function requires FEAT_AMUv1p1 support. |
|
|
|
*/ |
|
|
|
void amu_group0_voffset_write(unsigned int idx, uint64_t val) |
|
|
|
static void amu_group0_voffset_write(unsigned int idx, uint64_t val) |
|
|
|
{ |
|
|
|
assert(amu_get_version() >= ID_AA64PFR0_AMU_V1P1); |
|
|
|
assert(idx < AMU_GROUP0_NR_COUNTERS); |
|
|
|
assert(amu_v1p1_supported()); |
|
|
|
assert(idx < read_amcgcr_el0_cg0nc()); |
|
|
|
assert(idx != 1U); |
|
|
|
|
|
|
|
amu_group0_voffset_write_internal(idx, val); |
|
|
|
isb(); |
|
|
|
} |
|
|
|
|
|
|
|
#if AMU_GROUP1_NR_COUNTERS |
|
|
|
#if ENABLE_AMU_AUXILIARY_COUNTERS |
|
|
|
/* Read the group 1 counter identified by the given `idx` */ |
|
|
|
uint64_t amu_group1_cnt_read(unsigned int idx) |
|
|
|
static uint64_t amu_group1_cnt_read(unsigned int idx) |
|
|
|
{ |
|
|
|
assert(amu_get_version() != ID_AA64PFR0_AMU_NOT_SUPPORTED); |
|
|
|
assert(amu_supported()); |
|
|
|
assert(amu_group1_supported()); |
|
|
|
assert(idx < AMU_GROUP1_NR_COUNTERS); |
|
|
|
assert(idx < read_amcgcr_el0_cg1nc()); |
|
|
|
|
|
|
|
return amu_group1_cnt_read_internal(idx); |
|
|
|
} |
|
|
|
|
|
|
|
/* Write the group 1 counter identified by the given `idx` with `val` */ |
|
|
|
void amu_group1_cnt_write(unsigned int idx, uint64_t val) |
|
|
|
static void amu_group1_cnt_write(unsigned int idx, uint64_t val) |
|
|
|
{ |
|
|
|
assert(amu_get_version() != ID_AA64PFR0_AMU_NOT_SUPPORTED); |
|
|
|
assert(amu_supported()); |
|
|
|
assert(amu_group1_supported()); |
|
|
|
assert(idx < AMU_GROUP1_NR_COUNTERS); |
|
|
|
assert(idx < read_amcgcr_el0_cg1nc()); |
|
|
|
|
|
|
|
amu_group1_cnt_write_internal(idx, val); |
|
|
|
isb(); |
|
|
|
@ -206,13 +407,12 @@ void amu_group1_cnt_write(unsigned int idx, uint64_t val) |
|
|
|
* |
|
|
|
* Using this function requires FEAT_AMUv1p1 support. |
|
|
|
*/ |
|
|
|
uint64_t amu_group1_voffset_read(unsigned int idx) |
|
|
|
static uint64_t amu_group1_voffset_read(unsigned int idx) |
|
|
|
{ |
|
|
|
assert(amu_get_version() >= ID_AA64PFR0_AMU_V1P1); |
|
|
|
assert(amu_v1p1_supported()); |
|
|
|
assert(amu_group1_supported()); |
|
|
|
assert(idx < AMU_GROUP1_NR_COUNTERS); |
|
|
|
assert(((read_amcg1idr_el0() >> AMCG1IDR_VOFF_SHIFT) & |
|
|
|
(1ULL << idx)) != 0ULL); |
|
|
|
assert(idx < read_amcgcr_el0_cg1nc()); |
|
|
|
assert((read_amcg1idr_el0_voff() & (UINT64_C(1) << idx)) != 0U); |
|
|
|
|
|
|
|
return amu_group1_voffset_read_internal(idx); |
|
|
|
} |
|
|
|
@ -222,167 +422,211 @@ uint64_t amu_group1_voffset_read(unsigned int idx) |
|
|
|
* |
|
|
|
* Using this function requires FEAT_AMUv1p1 support. |
|
|
|
*/ |
|
|
|
void amu_group1_voffset_write(unsigned int idx, uint64_t val) |
|
|
|
static void amu_group1_voffset_write(unsigned int idx, uint64_t val) |
|
|
|
{ |
|
|
|
assert(amu_get_version() >= ID_AA64PFR0_AMU_V1P1); |
|
|
|
assert(amu_v1p1_supported()); |
|
|
|
assert(amu_group1_supported()); |
|
|
|
assert(idx < AMU_GROUP1_NR_COUNTERS); |
|
|
|
assert(((read_amcg1idr_el0() >> AMCG1IDR_VOFF_SHIFT) & |
|
|
|
(1ULL << idx)) != 0ULL); |
|
|
|
assert(idx < read_amcgcr_el0_cg1nc()); |
|
|
|
assert((read_amcg1idr_el0_voff() & (UINT64_C(1) << idx)) != 0U); |
|
|
|
|
|
|
|
amu_group1_voffset_write_internal(idx, val); |
|
|
|
isb(); |
|
|
|
} |
|
|
|
#endif |
|
|
|
|
|
|
|
/*
|
|
|
|
* Program the event type register for the given `idx` with |
|
|
|
* the event number `val` |
|
|
|
*/ |
|
|
|
void amu_group1_set_evtype(unsigned int idx, unsigned int val) |
|
|
|
static void *amu_context_save(const void *arg) |
|
|
|
{ |
|
|
|
assert(amu_get_version() != ID_AA64PFR0_AMU_NOT_SUPPORTED); |
|
|
|
assert(amu_group1_supported()); |
|
|
|
assert(idx < AMU_GROUP1_NR_COUNTERS); |
|
|
|
uint64_t i, j; |
|
|
|
|
|
|
|
amu_group1_set_evtype_internal(idx, val); |
|
|
|
isb(); |
|
|
|
} |
|
|
|
#endif /* AMU_GROUP1_NR_COUNTERS */ |
|
|
|
unsigned int core_pos; |
|
|
|
struct amu_ctx *ctx; |
|
|
|
|
|
|
|
static void *amu_context_save(const void *arg) |
|
|
|
{ |
|
|
|
struct amu_ctx *ctx = &amu_ctxs[plat_my_core_pos()]; |
|
|
|
unsigned int i; |
|
|
|
uint64_t id_aa64pfr0_el1_amu; /* AMU version */ |
|
|
|
uint64_t hcr_el2_amvoffen; /* AMU virtual offsets enabled */ |
|
|
|
uint64_t amcgcr_el0_cg0nc; /* Number of group 0 counters */ |
|
|
|
|
|
|
|
if (amu_get_version() == ID_AA64PFR0_AMU_NOT_SUPPORTED) { |
|
|
|
return (void *)-1; |
|
|
|
} |
|
|
|
#if ENABLE_AMU_AUXILIARY_COUNTERS |
|
|
|
uint64_t amcg1idr_el0_voff; /* Auxiliary counters with virtual offsets */ |
|
|
|
uint64_t amcfgr_el0_ncg; /* Number of counter groups */ |
|
|
|
uint64_t amcgcr_el0_cg1nc; /* Number of group 1 counters */ |
|
|
|
#endif |
|
|
|
|
|
|
|
#if AMU_GROUP1_NR_COUNTERS |
|
|
|
if (!amu_group1_supported()) { |
|
|
|
return (void *)-1; |
|
|
|
id_aa64pfr0_el1_amu = read_id_aa64pfr0_el1_amu(); |
|
|
|
if (id_aa64pfr0_el1_amu == ID_AA64PFR0_AMU_NOT_SUPPORTED) { |
|
|
|
return (void *)0; |
|
|
|
} |
|
|
|
#endif |
|
|
|
/* Assert that group 0/1 counter configuration is what we expect */ |
|
|
|
assert(read_amcntenset0_el0() == AMU_GROUP0_COUNTERS_MASK); |
|
|
|
|
|
|
|
#if AMU_GROUP1_NR_COUNTERS |
|
|
|
assert(read_amcntenset1_el0() == AMU_GROUP1_COUNTERS_MASK); |
|
|
|
core_pos = plat_my_core_pos(); |
|
|
|
ctx = &amu_ctxs_[core_pos]; |
|
|
|
|
|
|
|
amcgcr_el0_cg0nc = read_amcgcr_el0_cg0nc(); |
|
|
|
hcr_el2_amvoffen = (id_aa64pfr0_el1_amu >= ID_AA64PFR0_AMU_V1P1) ? |
|
|
|
read_hcr_el2_amvoffen() : 0U; |
|
|
|
|
|
|
|
#if ENABLE_AMU_AUXILIARY_COUNTERS |
|
|
|
amcfgr_el0_ncg = read_amcfgr_el0_ncg(); |
|
|
|
amcgcr_el0_cg1nc = (amcfgr_el0_ncg > 0U) ? read_amcgcr_el0_cg1nc() : 0U; |
|
|
|
amcg1idr_el0_voff = (hcr_el2_amvoffen != 0U) ? read_amcg1idr_el0_voff() : 0U; |
|
|
|
#endif |
|
|
|
|
|
|
|
/*
|
|
|
|
* Disable group 0/1 counters to avoid other observers like SCP sampling |
|
|
|
* counter values from the future via the memory mapped view. |
|
|
|
* Disable all AMU counters. |
|
|
|
*/ |
|
|
|
write_amcntenclr0_el0(AMU_GROUP0_COUNTERS_MASK); |
|
|
|
|
|
|
|
#if AMU_GROUP1_NR_COUNTERS |
|
|
|
write_amcntenclr1_el0(AMU_GROUP1_COUNTERS_MASK); |
|
|
|
ctx->group0_enable = read_amcntenset0_el0_px(); |
|
|
|
write_amcntenclr0_el0_px(ctx->group0_enable); |
|
|
|
|
|
|
|
#if ENABLE_AMU_AUXILIARY_COUNTERS |
|
|
|
if (amcfgr_el0_ncg > 0U) { |
|
|
|
ctx->group1_enable = read_amcntenset1_el0_px(); |
|
|
|
write_amcntenclr1_el0_px(ctx->group1_enable); |
|
|
|
} |
|
|
|
#endif |
|
|
|
isb(); |
|
|
|
|
|
|
|
/* Save all group 0 counters */ |
|
|
|
for (i = 0U; i < AMU_GROUP0_NR_COUNTERS; i++) { |
|
|
|
/*
|
|
|
|
* Save the counters to the local context. |
|
|
|
*/ |
|
|
|
|
|
|
|
isb(); /* Ensure counters have been stopped */ |
|
|
|
|
|
|
|
for (i = 0U; i < amcgcr_el0_cg0nc; i++) { |
|
|
|
ctx->group0_cnts[i] = amu_group0_cnt_read(i); |
|
|
|
} |
|
|
|
|
|
|
|
/* Save group 0 virtual offsets if supported and enabled. */ |
|
|
|
if ((amu_get_version() >= ID_AA64PFR0_AMU_V1P1) && |
|
|
|
((read_hcr_el2() & HCR_AMVOFFEN_BIT) != 0ULL)) { |
|
|
|
/* Not using a loop because count is fixed and index 1 DNE. */ |
|
|
|
ctx->group0_voffsets[0U] = amu_group0_voffset_read(0U); |
|
|
|
ctx->group0_voffsets[1U] = amu_group0_voffset_read(2U); |
|
|
|
ctx->group0_voffsets[2U] = amu_group0_voffset_read(3U); |
|
|
|
#if ENABLE_AMU_AUXILIARY_COUNTERS |
|
|
|
for (i = 0U; i < amcgcr_el0_cg1nc; i++) { |
|
|
|
ctx->group1_cnts[i] = amu_group1_cnt_read(i); |
|
|
|
} |
|
|
|
#endif |
|
|
|
|
|
|
|
/*
|
|
|
|
* Save virtual offsets for counters that offer them. |
|
|
|
*/ |
|
|
|
|
|
|
|
if (hcr_el2_amvoffen != 0U) { |
|
|
|
for (i = 0U, j = 0U; i < amcgcr_el0_cg0nc; i++) { |
|
|
|
if (!amu_group0_voffset_supported(i)) { |
|
|
|
continue; /* No virtual offset */ |
|
|
|
} |
|
|
|
|
|
|
|
#if AMU_GROUP1_NR_COUNTERS |
|
|
|
/* Save group 1 counters */ |
|
|
|
for (i = 0U; i < AMU_GROUP1_NR_COUNTERS; i++) { |
|
|
|
if ((AMU_GROUP1_COUNTERS_MASK & (1UL << i)) != 0U) { |
|
|
|
ctx->group1_cnts[i] = amu_group1_cnt_read(i); |
|
|
|
ctx->group0_voffsets[j++] = amu_group0_voffset_read(i); |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
/* Save group 1 virtual offsets if supported and enabled. */ |
|
|
|
if ((amu_get_version() >= ID_AA64PFR0_AMU_V1P1) && |
|
|
|
((read_hcr_el2() & HCR_AMVOFFEN_BIT) != 0ULL)) { |
|
|
|
u_register_t amcg1idr = read_amcg1idr_el0() >> |
|
|
|
AMCG1IDR_VOFF_SHIFT; |
|
|
|
amcg1idr = amcg1idr & AMU_GROUP1_COUNTERS_MASK; |
|
|
|
|
|
|
|
for (i = 0U; i < AMU_GROUP1_NR_COUNTERS; i++) { |
|
|
|
if (((amcg1idr >> i) & 1ULL) != 0ULL) { |
|
|
|
ctx->group1_voffsets[i] = |
|
|
|
amu_group1_voffset_read(i); |
|
|
|
#if ENABLE_AMU_AUXILIARY_COUNTERS |
|
|
|
for (i = 0U, j = 0U; i < amcgcr_el0_cg1nc; i++) { |
|
|
|
if ((amcg1idr_el0_voff >> i) & 1U) { |
|
|
|
continue; /* No virtual offset */ |
|
|
|
} |
|
|
|
|
|
|
|
ctx->group1_voffsets[j++] = amu_group1_voffset_read(i); |
|
|
|
} |
|
|
|
} |
|
|
|
#endif |
|
|
|
} |
|
|
|
|
|
|
|
return (void *)0; |
|
|
|
} |
|
|
|
|
|
|
|
static void *amu_context_restore(const void *arg) |
|
|
|
{ |
|
|
|
struct amu_ctx *ctx = &amu_ctxs[plat_my_core_pos()]; |
|
|
|
unsigned int i; |
|
|
|
uint64_t i, j; |
|
|
|
|
|
|
|
if (amu_get_version() == ID_AA64PFR0_AMU_NOT_SUPPORTED) { |
|
|
|
return (void *)-1; |
|
|
|
} |
|
|
|
unsigned int core_pos; |
|
|
|
struct amu_ctx *ctx; |
|
|
|
|
|
|
|
#if AMU_GROUP1_NR_COUNTERS |
|
|
|
if (!amu_group1_supported()) { |
|
|
|
return (void *)-1; |
|
|
|
} |
|
|
|
uint64_t id_aa64pfr0_el1_amu; /* AMU version */ |
|
|
|
|
|
|
|
uint64_t hcr_el2_amvoffen; /* AMU virtual offsets enabled */ |
|
|
|
|
|
|
|
uint64_t amcfgr_el0_ncg; /* Number of counter groups */ |
|
|
|
uint64_t amcgcr_el0_cg0nc; /* Number of group 0 counters */ |
|
|
|
|
|
|
|
#if ENABLE_AMU_AUXILIARY_COUNTERS |
|
|
|
uint64_t amcgcr_el0_cg1nc; /* Number of group 1 counters */ |
|
|
|
uint64_t amcg1idr_el0_voff; /* Auxiliary counters with virtual offsets */ |
|
|
|
#endif |
|
|
|
/* Counters were disabled in `amu_context_save()` */ |
|
|
|
assert(read_amcntenset0_el0() == 0U); |
|
|
|
|
|
|
|
#if AMU_GROUP1_NR_COUNTERS |
|
|
|
assert(read_amcntenset1_el0() == 0U); |
|
|
|
id_aa64pfr0_el1_amu = read_id_aa64pfr0_el1_amu(); |
|
|
|
if (id_aa64pfr0_el1_amu == ID_AA64PFR0_AMU_NOT_SUPPORTED) { |
|
|
|
return (void *)0; |
|
|
|
} |
|
|
|
|
|
|
|
core_pos = plat_my_core_pos(); |
|
|
|
ctx = &amu_ctxs_[core_pos]; |
|
|
|
|
|
|
|
amcfgr_el0_ncg = read_amcfgr_el0_ncg(); |
|
|
|
amcgcr_el0_cg0nc = read_amcgcr_el0_cg0nc(); |
|
|
|
|
|
|
|
hcr_el2_amvoffen = (id_aa64pfr0_el1_amu >= ID_AA64PFR0_AMU_V1P1) ? |
|
|
|
read_hcr_el2_amvoffen() : 0U; |
|
|
|
|
|
|
|
#if ENABLE_AMU_AUXILIARY_COUNTERS |
|
|
|
amcgcr_el0_cg1nc = (amcfgr_el0_ncg > 0U) ? read_amcgcr_el0_cg1nc() : 0U; |
|
|
|
amcg1idr_el0_voff = (hcr_el2_amvoffen != 0U) ? read_amcg1idr_el0_voff() : 0U; |
|
|
|
#endif |
|
|
|
|
|
|
|
/* Restore all group 0 counters */ |
|
|
|
for (i = 0U; i < AMU_GROUP0_NR_COUNTERS; i++) { |
|
|
|
/*
|
|
|
|
* Sanity check that all counters were disabled when the context was |
|
|
|
* previously saved. |
|
|
|
*/ |
|
|
|
|
|
|
|
assert(read_amcntenset0_el0_px() == 0U); |
|
|
|
|
|
|
|
if (amcfgr_el0_ncg > 0U) { |
|
|
|
assert(read_amcntenset1_el0_px() == 0U); |
|
|
|
} |
|
|
|
|
|
|
|
/*
|
|
|
|
* Restore the counter values from the local context. |
|
|
|
*/ |
|
|
|
|
|
|
|
for (i = 0U; i < amcgcr_el0_cg0nc; i++) { |
|
|
|
amu_group0_cnt_write(i, ctx->group0_cnts[i]); |
|
|
|
} |
|
|
|
|
|
|
|
/* Restore group 0 virtual offsets if supported and enabled. */ |
|
|
|
if ((amu_get_version() >= ID_AA64PFR0_AMU_V1P1) && |
|
|
|
((read_hcr_el2() & HCR_AMVOFFEN_BIT) != 0ULL)) { |
|
|
|
/* Not using a loop because count is fixed and index 1 DNE. */ |
|
|
|
amu_group0_voffset_write(0U, ctx->group0_voffsets[0U]); |
|
|
|
amu_group0_voffset_write(2U, ctx->group0_voffsets[1U]); |
|
|
|
amu_group0_voffset_write(3U, ctx->group0_voffsets[2U]); |
|
|
|
#if ENABLE_AMU_AUXILIARY_COUNTERS |
|
|
|
for (i = 0U; i < amcgcr_el0_cg1nc; i++) { |
|
|
|
amu_group1_cnt_write(i, ctx->group1_cnts[i]); |
|
|
|
} |
|
|
|
#endif |
|
|
|
|
|
|
|
/*
|
|
|
|
* Restore virtual offsets for counters that offer them. |
|
|
|
*/ |
|
|
|
|
|
|
|
/* Restore group 0 counter configuration */ |
|
|
|
write_amcntenset0_el0(AMU_GROUP0_COUNTERS_MASK); |
|
|
|
if (hcr_el2_amvoffen != 0U) { |
|
|
|
for (i = 0U, j = 0U; i < amcgcr_el0_cg0nc; i++) { |
|
|
|
if (!amu_group0_voffset_supported(i)) { |
|
|
|
continue; /* No virtual offset */ |
|
|
|
} |
|
|
|
|
|
|
|
#if AMU_GROUP1_NR_COUNTERS |
|
|
|
/* Restore group 1 counters */ |
|
|
|
for (i = 0U; i < AMU_GROUP1_NR_COUNTERS; i++) { |
|
|
|
if ((AMU_GROUP1_COUNTERS_MASK & (1UL << i)) != 0U) { |
|
|
|
amu_group1_cnt_write(i, ctx->group1_cnts[i]); |
|
|
|
amu_group0_voffset_write(i, ctx->group0_voffsets[j++]); |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
/* Restore group 1 virtual offsets if supported and enabled. */ |
|
|
|
if ((amu_get_version() >= ID_AA64PFR0_AMU_V1P1) && |
|
|
|
((read_hcr_el2() & HCR_AMVOFFEN_BIT) != 0ULL)) { |
|
|
|
u_register_t amcg1idr = read_amcg1idr_el0() >> |
|
|
|
AMCG1IDR_VOFF_SHIFT; |
|
|
|
amcg1idr = amcg1idr & AMU_GROUP1_COUNTERS_MASK; |
|
|
|
|
|
|
|
for (i = 0U; i < AMU_GROUP1_NR_COUNTERS; i++) { |
|
|
|
if (((amcg1idr >> i) & 1ULL) != 0ULL) { |
|
|
|
amu_group1_voffset_write(i, |
|
|
|
ctx->group1_voffsets[i]); |
|
|
|
#if ENABLE_AMU_AUXILIARY_COUNTERS |
|
|
|
for (i = 0U, j = 0U; i < amcgcr_el0_cg1nc; i++) { |
|
|
|
if ((amcg1idr_el0_voff >> i) & 1U) { |
|
|
|
continue; /* No virtual offset */ |
|
|
|
} |
|
|
|
|
|
|
|
amu_group1_voffset_write(i, ctx->group1_voffsets[j++]); |
|
|
|
} |
|
|
|
#endif |
|
|
|
} |
|
|
|
|
|
|
|
/*
|
|
|
|
* Re-enable counters that were disabled during context save. |
|
|
|
*/ |
|
|
|
|
|
|
|
write_amcntenset0_el0_px(ctx->group0_enable); |
|
|
|
|
|
|
|
#if ENABLE_AMU_AUXILIARY_COUNTERS |
|
|
|
if (amcfgr_el0_ncg > 0) { |
|
|
|
write_amcntenset1_el0_px(ctx->group1_enable); |
|
|
|
} |
|
|
|
#endif |
|
|
|
|
|
|
|
/* Restore group 1 counter configuration */ |
|
|
|
write_amcntenset1_el0(AMU_GROUP1_COUNTERS_MASK); |
|
|
|
#if ENABLE_MPMM |
|
|
|
mpmm_enable(); |
|
|
|
#endif |
|
|
|
|
|
|
|
return (void *)0; |
|
|
|
|
Manish Pandey
3 years ago
TrustedFirmware Code Review