From 7593252cee8745bbf1b05deb2f4a5f742d36c412 Mon Sep 17 00:00:00 2001 From: Dimitris Papastamos Date: Tue, 28 Nov 2017 15:16:00 +0000 Subject: [PATCH 1/5] Add PubSub events for CPU powerdown/powerup The suspend hook is published at the start of a CPU powerdown operation. The resume hook is published at the end of a CPU powerup operation. Change-Id: I50c05e2dde0d33834095ac41b4fcea4c161bb434 Signed-off-by: Dimitris Papastamos --- include/lib/el3_runtime/pubsub_events.h | 7 +++++++ lib/psci/psci_suspend.c | 5 +++++ 2 files changed, 12 insertions(+) diff --git a/include/lib/el3_runtime/pubsub_events.h b/include/lib/el3_runtime/pubsub_events.h index 9cfedb4de7..64b3f630c9 100644 --- a/include/lib/el3_runtime/pubsub_events.h +++ b/include/lib/el3_runtime/pubsub_events.h @@ -17,6 +17,13 @@ */ REGISTER_PUBSUB_EVENT(psci_cpu_on_finish); +/* + * These events are published before/after a CPU has been powered down/up + * via the PSCI CPU SUSPEND API. + */ +REGISTER_PUBSUB_EVENT(psci_suspend_pwrdown_start); +REGISTER_PUBSUB_EVENT(psci_suspend_pwrdown_finish); + #ifdef AARCH64 /* * These events are published by the AArch64 context management framework diff --git a/lib/psci/psci_suspend.c b/lib/psci/psci_suspend.c index d9490672eb..a77972d38c 100644 --- a/lib/psci/psci_suspend.c +++ b/lib/psci/psci_suspend.c @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include "psci_private.h" @@ -68,6 +69,8 @@ static void psci_suspend_to_pwrdown_start(unsigned int end_pwrlvl, { unsigned int max_off_lvl = psci_find_max_off_lvl(state_info); + PUBLISH_EVENT(psci_suspend_pwrdown_start); + /* Save PSCI target power level for the suspend finisher handler */ psci_set_suspend_pwrlvl(end_pwrlvl); @@ -308,6 +311,8 @@ void psci_cpu_suspend_finish(unsigned int cpu_idx, /* Invalidate the suspend level for the cpu */ psci_set_suspend_pwrlvl(PSCI_INVALID_PWR_LVL); + PUBLISH_EVENT(psci_suspend_pwrdown_finish); + /* * Generic management: Now we just need to retrieve the * information that we had stashed away during the suspend From 59902b7c4ca2413c2ee783da1bc66789e35868e5 Mon Sep 17 00:00:00 2001 From: Dimitris Papastamos Date: Wed, 13 Dec 2017 10:54:37 +0000 Subject: [PATCH 2/5] AMU: Add plat interface to select which group 1 counters to enable A new platform macro `PLAT_AMU_GROUP1_COUNTERS_MASK` controls which group 1 counters should be enabled. The maximum number of group 1 counters supported by AMUv1 is 16 so the mask can be at most 0xffff. If the platform does not define this mask, no group 1 counters are enabled. A related platform macro `PLAT_AMU_GROUP1_NR_COUNTERS` is used by generic code to allocate an array to save and restore the counters on CPU suspend. Change-Id: I6d135badf4846292de931a43bb563077f42bb47b Signed-off-by: Dimitris Papastamos --- docs/porting-guide.rst | 16 ++++++++++++++++ include/lib/extensions/amu.h | 21 ++++++++++++++++++++- lib/extensions/amu/aarch64/amu.c | 2 ++ 3 files changed, 38 insertions(+), 1 deletion(-) diff --git a/docs/porting-guide.rst b/docs/porting-guide.rst index f020ec97d0..51d2e64a4d 100644 --- a/docs/porting-guide.rst +++ b/docs/porting-guide.rst @@ -549,6 +549,22 @@ behaviour of the ``assert()`` function (for example, to save memory). doesn't print anything to the console. If ``PLAT_LOG_LEVEL_ASSERT`` isn't defined, it defaults to ``LOG_LEVEL``. +If the platform port uses the Activity Monitor Unit, the following constants +may be defined: + +- **PLAT\_AMU\_GROUP1\_COUNTERS\_MASK** + This mask reflects the set of group counters that should be enabled. The + maximum number of group 1 counters supported by AMUv1 is 16 so the mask + can be at most 0xffff. If the platform does not define this mask, no group 1 + counters are enabled. If the platform defines this mask, the following + constant needs to also be defined. + +- **PLAT\_AMU\_GROUP1\_NR\_COUNTERS** + This value is used to allocate an array to save and restore the counters + specified by ``PLAT_AMU_GROUP1_COUNTERS_MASK`` on CPU suspend. + This value should be equal to the highest bit position set in the + mask, plus 1. The maximum number of group 1 counters in AMUv1 is 16. + File : plat\_macros.S [mandatory] ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/include/lib/extensions/amu.h b/include/lib/extensions/amu.h index bbefe8ff66..2ecbea5682 100644 --- a/include/lib/extensions/amu.h +++ b/include/lib/extensions/amu.h @@ -7,9 +7,28 @@ #ifndef __AMU_H__ #define __AMU_H__ -/* Enable all group 0 counters */ +#include /* for CASSERT() */ +#include +#include + +/* All group 0 counters */ #define AMU_GROUP0_COUNTERS_MASK 0xf +#ifdef PLAT_AMU_GROUP1_COUNTERS_MASK +#define AMU_GROUP1_COUNTERS_MASK PLAT_AMU_GROUP1_COUNTERS_MASK +#else +#define AMU_GROUP1_COUNTERS_MASK 0 +#endif + +#ifdef PLAT_AMU_GROUP1_NR_COUNTERS +#define AMU_GROUP1_NR_COUNTERS PLAT_AMU_GROUP1_NR_COUNTERS +#else +#define AMU_GROUP1_NR_COUNTERS 0 +#endif + +CASSERT(AMU_GROUP1_COUNTERS_MASK <= 0xffff, invalid_amu_group1_counters_mask); +CASSERT(AMU_GROUP1_NR_COUNTERS <= 16, invalid_amu_group1_nr_counters); + void amu_enable(int el2_unused); #endif /* __AMU_H__ */ diff --git a/lib/extensions/amu/aarch64/amu.c b/lib/extensions/amu/aarch64/amu.c index 007b3494fe..c00aa5a17e 100644 --- a/lib/extensions/amu/aarch64/amu.c +++ b/lib/extensions/amu/aarch64/amu.c @@ -36,5 +36,7 @@ void amu_enable(int el2_unused) /* Enable group 0 counters */ write_amcntenset0_el0(AMU_GROUP0_COUNTERS_MASK); + /* Enable group 1 counters */ + write_amcntenset1_el0(AMU_GROUP1_COUNTERS_MASK); } } From 0767d50e699d3d9cb827172dd7742618d37aabe0 Mon Sep 17 00:00:00 2001 From: Dimitris Papastamos Date: Mon, 13 Nov 2017 09:49:45 +0000 Subject: [PATCH 3/5] AMU: Add configuration helpers for aarch64 Add some AMU helper functions to allow configuring, reading and writing of the Group 0 and Group 1 counters. Documentation for these helpers will come in a separate patch. Change-Id: I656e070d2dae830c22414f694aa655341d4e2c40 Signed-off-by: Dimitris Papastamos --- bl31/bl31.mk | 3 +- include/lib/aarch64/arch.h | 41 ++++ include/lib/aarch64/arch_helpers.h | 1 + include/lib/extensions/amu.h | 11 + include/lib/extensions/amu_private.h | 19 ++ lib/extensions/amu/aarch32/amu.c | 34 +-- lib/extensions/amu/aarch64/amu.c | 122 +++++++--- lib/extensions/amu/aarch64/amu_helpers.S | 281 +++++++++++++++++++++++ 8 files changed, 469 insertions(+), 43 deletions(-) create mode 100644 include/lib/extensions/amu_private.h create mode 100644 lib/extensions/amu/aarch64/amu_helpers.S diff --git a/bl31/bl31.mk b/bl31/bl31.mk index fdcc93139a..6820983b59 100644 --- a/bl31/bl31.mk +++ b/bl31/bl31.mk @@ -51,7 +51,8 @@ BL31_SOURCES += lib/extensions/spe/spe.c endif ifeq (${ENABLE_AMU},1) -BL31_SOURCES += lib/extensions/amu/aarch64/amu.c +BL31_SOURCES += lib/extensions/amu/aarch64/amu.c \ + lib/extensions/amu/aarch64/amu_helpers.S endif ifeq (${ENABLE_SVE_FOR_NS},1) diff --git a/include/lib/aarch64/arch.h b/include/lib/aarch64/arch.h index 96e2d5fe22..1d86205e1c 100644 --- a/include/lib/aarch64/arch.h +++ b/include/lib/aarch64/arch.h @@ -656,4 +656,45 @@ #define AMEVTYPER02_EL0 S3_3_C13_C6_2 #define AMEVTYPER03_EL0 S3_3_C13_C6_3 +/* Activity Monitor Group 1 Event Counter Registers */ +#define AMEVCNTR10_EL0 S3_3_C13_C12_0 +#define AMEVCNTR11_EL0 S3_3_C13_C12_1 +#define AMEVCNTR12_EL0 S3_3_C13_C12_2 +#define AMEVCNTR13_EL0 S3_3_C13_C12_3 +#define AMEVCNTR14_EL0 S3_3_C13_C12_4 +#define AMEVCNTR15_EL0 S3_3_C13_C12_5 +#define AMEVCNTR16_EL0 S3_3_C13_C12_6 +#define AMEVCNTR17_EL0 S3_3_C13_C12_7 +#define AMEVCNTR18_EL0 S3_3_C13_C13_0 +#define AMEVCNTR19_EL0 S3_3_C13_C13_1 +#define AMEVCNTR1A_EL0 S3_3_C13_C13_2 +#define AMEVCNTR1B_EL0 S3_3_C13_C13_3 +#define AMEVCNTR1C_EL0 S3_3_C13_C13_4 +#define AMEVCNTR1D_EL0 S3_3_C13_C13_5 +#define AMEVCNTR1E_EL0 S3_3_C13_C13_6 +#define AMEVCNTR1F_EL0 S3_3_C13_C13_7 + +/* Activity Monitor Group 1 Event Type Registers */ +#define AMEVTYPER10_EL0 S3_3_C13_C14_0 +#define AMEVTYPER11_EL0 S3_3_C13_C14_1 +#define AMEVTYPER12_EL0 S3_3_C13_C14_2 +#define AMEVTYPER13_EL0 S3_3_C13_C14_3 +#define AMEVTYPER14_EL0 S3_3_C13_C14_4 +#define AMEVTYPER15_EL0 S3_3_C13_C14_5 +#define AMEVTYPER16_EL0 S3_3_C13_C14_6 +#define AMEVTYPER17_EL0 S3_3_C13_C14_7 +#define AMEVTYPER18_EL0 S3_3_C13_C15_0 +#define AMEVTYPER19_EL0 S3_3_C13_C15_1 +#define AMEVTYPER1A_EL0 S3_3_C13_C15_2 +#define AMEVTYPER1B_EL0 S3_3_C13_C15_3 +#define AMEVTYPER1C_EL0 S3_3_C13_C15_4 +#define AMEVTYPER1D_EL0 S3_3_C13_C15_5 +#define AMEVTYPER1E_EL0 S3_3_C13_C15_6 +#define AMEVTYPER1F_EL0 S3_3_C13_C15_7 + +/* AMCGCR_EL0 definitions */ +#define AMCGCR_EL0_CG1NC_SHIFT U(8) +#define AMCGCR_EL0_CG1NC_LENGTH U(8) +#define AMCGCR_EL0_CG1NC_MASK U(0xff) + #endif /* __ARCH_H__ */ diff --git a/include/lib/aarch64/arch_helpers.h b/include/lib/aarch64/arch_helpers.h index 831dfb067f..485ed43253 100644 --- a/include/lib/aarch64/arch_helpers.h +++ b/include/lib/aarch64/arch_helpers.h @@ -322,6 +322,7 @@ DEFINE_RENAME_SYSREG_WRITE_FUNC(icc_eoir0_el1, ICC_EOIR0_EL1) DEFINE_RENAME_SYSREG_WRITE_FUNC(icc_eoir1_el1, ICC_EOIR1_EL1) DEFINE_RENAME_SYSREG_WRITE_FUNC(icc_sgi0r_el1, ICC_SGI0R_EL1) +DEFINE_RENAME_SYSREG_RW_FUNCS(amcgcr_el0, AMCGCR_EL0) DEFINE_RENAME_SYSREG_RW_FUNCS(amcntenclr0_el0, AMCNTENCLR0_EL0) DEFINE_RENAME_SYSREG_RW_FUNCS(amcntenset0_el0, AMCNTENSET0_EL0) DEFINE_RENAME_SYSREG_RW_FUNCS(amcntenclr1_el0, AMCNTENCLR1_EL0) diff --git a/include/lib/extensions/amu.h b/include/lib/extensions/amu.h index 2ecbea5682..faa0ee125f 100644 --- a/include/lib/extensions/amu.h +++ b/include/lib/extensions/amu.h @@ -10,6 +10,7 @@ #include /* for CASSERT() */ #include #include +#include /* All group 0 counters */ #define AMU_GROUP0_COUNTERS_MASK 0xf @@ -29,6 +30,16 @@ CASSERT(AMU_GROUP1_COUNTERS_MASK <= 0xffff, invalid_amu_group1_counters_mask); CASSERT(AMU_GROUP1_NR_COUNTERS <= 16, invalid_amu_group1_nr_counters); +int amu_supported(void); void amu_enable(int el2_unused); +/* Group 0 configuration helpers */ +uint64_t amu_group0_cnt_read(int idx); +void amu_group0_cnt_write(int idx, uint64_t val); + +/* Group 1 configuration helpers */ +uint64_t amu_group1_cnt_read(int idx); +void amu_group1_cnt_write(int idx, uint64_t val); +void amu_group1_set_evtype(int idx, unsigned int val); + #endif /* __AMU_H__ */ diff --git a/include/lib/extensions/amu_private.h b/include/lib/extensions/amu_private.h new file mode 100644 index 0000000000..0c660bb836 --- /dev/null +++ b/include/lib/extensions/amu_private.h @@ -0,0 +1,19 @@ +/* + * Copyright (c) 2017, ARM Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef __AMU_PRIVATE_H__ +#define __AMU_PRIVATE_H__ + +#include + +uint64_t amu_group0_cnt_read_internal(int idx); +void amu_group0_cnt_write_internal(int idx, uint64_t); + +uint64_t amu_group1_cnt_read_internal(int idx); +void amu_group1_cnt_write_internal(int idx, uint64_t); +void amu_group1_set_evtype_internal(int idx, unsigned int val); + +#endif /* __AMU_PRIVATE_H__ */ diff --git a/lib/extensions/amu/aarch32/amu.c b/lib/extensions/amu/aarch32/amu.c index d450bd6920..be37006636 100644 --- a/lib/extensions/amu/aarch32/amu.c +++ b/lib/extensions/amu/aarch32/amu.c @@ -7,26 +7,30 @@ #include #include #include +#include void amu_enable(int el2_unused) { uint64_t features; features = read_id_pfr0() >> ID_PFR0_AMU_SHIFT; - if ((features & ID_PFR0_AMU_MASK) == 1) { - if (el2_unused) { - uint64_t v; - - /* - * Non-secure access from EL0 or EL1 to the Activity Monitor - * registers do not trap to EL2. - */ - v = read_hcptr(); - v &= ~TAM_BIT; - write_hcptr(v); - } - - /* Enable group 0 counters */ - write_amcntenset0(AMU_GROUP0_COUNTERS_MASK); + if ((features & ID_PFR0_AMU_MASK) != 1) { + WARN("Cannot enable AMU - not supported\n"); + return; } + + if (el2_unused) { + uint64_t v; + + /* + * Non-secure access from EL0 or EL1 to the Activity Monitor + * registers do not trap to EL2. + */ + v = read_hcptr(); + v &= ~TAM_BIT; + write_hcptr(v); + } + + /* Enable group 0 counters */ + write_amcntenset0(AMU_GROUP0_COUNTERS_MASK); } diff --git a/lib/extensions/amu/aarch64/amu.c b/lib/extensions/amu/aarch64/amu.c index c00aa5a17e..27936a2e17 100644 --- a/lib/extensions/amu/aarch64/amu.c +++ b/lib/extensions/amu/aarch64/amu.c @@ -5,38 +5,106 @@ */ #include +#include #include #include +#include +#include -void amu_enable(int el2_unused) +#define AMU_GROUP0_NR_COUNTERS 4 + +int amu_supported(void) { uint64_t features; features = read_id_aa64pfr0_el1() >> ID_AA64PFR0_AMU_SHIFT; - if ((features & ID_AA64PFR0_AMU_MASK) == 1) { - uint64_t v; - - if (el2_unused) { - /* - * CPTR_EL2.TAM: Set to zero so any accesses to - * the Activity Monitor registers do not trap to EL2. - */ - v = read_cptr_el2(); - v &= ~CPTR_EL2_TAM_BIT; - write_cptr_el2(v); - } - - /* - * CPTR_EL3.TAM: Set to zero so that any accesses to - * the Activity Monitor registers do not trap to EL3. - */ - v = read_cptr_el3(); - v &= ~TAM_BIT; - write_cptr_el3(v); - - /* Enable group 0 counters */ - write_amcntenset0_el0(AMU_GROUP0_COUNTERS_MASK); - /* Enable group 1 counters */ - write_amcntenset1_el0(AMU_GROUP1_COUNTERS_MASK); - } + return (features & ID_AA64PFR0_AMU_MASK) == 1; +} + +/* + * Enable counters. This function is meant to be invoked + * by the context management library before exiting from EL3. + */ +void amu_enable(int el2_unused) +{ + uint64_t v; + + if (!amu_supported()) { + WARN("Cannot enable AMU - not supported\n"); + return; + } + + if (el2_unused) { + /* + * CPTR_EL2.TAM: Set to zero so any accesses to + * the Activity Monitor registers do not trap to EL2. + */ + v = read_cptr_el2(); + v &= ~CPTR_EL2_TAM_BIT; + write_cptr_el2(v); + } + + /* + * CPTR_EL3.TAM: Set to zero so that any accesses to + * the Activity Monitor registers do not trap to EL3. + */ + v = read_cptr_el3(); + v &= ~TAM_BIT; + write_cptr_el3(v); + + /* Enable group 0 counters */ + write_amcntenset0_el0(AMU_GROUP0_COUNTERS_MASK); + /* Enable group 1 counters */ + write_amcntenset1_el0(AMU_GROUP1_COUNTERS_MASK); +} + +/* Read the group 0 counter identified by the given `idx`. */ +uint64_t amu_group0_cnt_read(int idx) +{ + assert(amu_supported()); + assert(idx >= 0 && idx < AMU_GROUP0_NR_COUNTERS); + + return amu_group0_cnt_read_internal(idx); +} + +/* Write the group 0 counter identified by the given `idx` with `val`. */ +void amu_group0_cnt_write(int idx, uint64_t val) +{ + assert(amu_supported()); + assert(idx >= 0 && idx < AMU_GROUP0_NR_COUNTERS); + + amu_group0_cnt_write_internal(idx, val); + isb(); +} + +/* Read the group 1 counter identified by the given `idx`. */ +uint64_t amu_group1_cnt_read(int idx) +{ + assert(amu_supported()); + assert(idx >= 0 && idx < AMU_GROUP1_NR_COUNTERS); + + return amu_group1_cnt_read_internal(idx); +} + +/* Write the group 1 counter identified by the given `idx` with `val`. */ +void amu_group1_cnt_write(int idx, uint64_t val) +{ + assert(amu_supported()); + assert(idx >= 0 && idx < AMU_GROUP1_NR_COUNTERS); + + amu_group1_cnt_write_internal(idx, val); + isb(); +} + +/* + * Program the event type register for the given `idx` with + * the event number `val`. + */ +void amu_group1_set_evtype(int idx, unsigned int val) +{ + assert(amu_supported()); + assert (idx >= 0 && idx < AMU_GROUP1_NR_COUNTERS); + + amu_group1_set_evtype_internal(idx, val); + isb(); } diff --git a/lib/extensions/amu/aarch64/amu_helpers.S b/lib/extensions/amu/aarch64/amu_helpers.S new file mode 100644 index 0000000000..e0b1f56414 --- /dev/null +++ b/lib/extensions/amu/aarch64/amu_helpers.S @@ -0,0 +1,281 @@ +/* + * Copyright (c) 2017, ARM Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include +#include +#include + + .globl amu_group0_cnt_read_internal + .globl amu_group0_cnt_write_internal + .globl amu_group1_cnt_read_internal + .globl amu_group1_cnt_write_internal + .globl amu_group1_set_evtype_internal + +/* + * uint64_t amu_group0_cnt_read_internal(int idx); + * + * Given `idx`, read the corresponding AMU counter + * and return it in `x0`. + */ +func amu_group0_cnt_read_internal +#if ENABLE_ASSERTIONS + /* + * It can be dangerous to call this function with an + * out of bounds index. Ensure `idx` is valid. + */ + mov x1, x0 + lsr x1, x1, #2 + cmp x1, #0 + ASM_ASSERT(eq) +#endif + + /* + * Given `idx` calculate address of mrs/ret instruction pair + * in the table below. + */ + adr x1, 1f + lsl x0, x0, #3 /* each mrs/ret sequence is 8 bytes */ + add x1, x1, x0 + br x1 + +1: + mrs x0, AMEVCNTR00_EL0 /* index 0 */ + ret + mrs x0, AMEVCNTR01_EL0 /* index 1 */ + ret + mrs x0, AMEVCNTR02_EL0 /* index 2 */ + ret + mrs x0, AMEVCNTR03_EL0 /* index 3 */ + ret +endfunc amu_group0_cnt_read_internal + +/* + * void amu_group0_cnt_write_internal(int idx, uint64_t val); + * + * Given `idx`, write `val` to the corresponding AMU counter. + */ +func amu_group0_cnt_write_internal +#if ENABLE_ASSERTIONS + /* + * It can be dangerous to call this function with an + * out of bounds index. Ensure `idx` is valid. + */ + mov x2, x0 + lsr x2, x2, #2 + cmp x2, #0 + ASM_ASSERT(eq) +#endif + + /* + * Given `idx` calculate address of mrs/ret instruction pair + * in the table below. + */ + adr x2, 1f + lsl x0, x0, #3 /* each msr/ret sequence is 8 bytes */ + add x2, x2, x0 + br x2 + +1: + msr AMEVCNTR00_EL0, x1 /* index 0 */ + ret + msr AMEVCNTR01_EL0, x1 /* index 1 */ + ret + msr AMEVCNTR02_EL0, x1 /* index 2 */ + ret + msr AMEVCNTR03_EL0, x1 /* index 3 */ + ret +endfunc amu_group0_cnt_write_internal + +/* + * uint64_t amu_group1_cnt_read_internal(int idx); + * + * Given `idx`, read the corresponding AMU counter + * and return it in `x0`. + */ +func amu_group1_cnt_read_internal +#if ENABLE_ASSERTIONS + /* + * It can be dangerous to call this function with an + * out of bounds index. Ensure `idx` is valid. + */ + mov x1, x0 + lsr x1, x1, #4 + cmp x1, #0 + ASM_ASSERT(eq) +#endif + + /* + * Given `idx` calculate address of mrs/ret instruction pair + * in the table below. + */ + adr x1, 1f + lsl x0, x0, #3 /* each mrs/ret sequence is 8 bytes */ + add x1, x1, x0 + br x1 + +1: + mrs x0, AMEVCNTR10_EL0 /* index 0 */ + ret + mrs x0, AMEVCNTR11_EL0 /* index 1 */ + ret + mrs x0, AMEVCNTR12_EL0 /* index 2 */ + ret + mrs x0, AMEVCNTR13_EL0 /* index 3 */ + ret + mrs x0, AMEVCNTR14_EL0 /* index 4 */ + ret + mrs x0, AMEVCNTR15_EL0 /* index 5 */ + ret + mrs x0, AMEVCNTR16_EL0 /* index 6 */ + ret + mrs x0, AMEVCNTR17_EL0 /* index 7 */ + ret + mrs x0, AMEVCNTR18_EL0 /* index 8 */ + ret + mrs x0, AMEVCNTR19_EL0 /* index 9 */ + ret + mrs x0, AMEVCNTR1A_EL0 /* index 10 */ + ret + mrs x0, AMEVCNTR1B_EL0 /* index 11 */ + ret + mrs x0, AMEVCNTR1C_EL0 /* index 12 */ + ret + mrs x0, AMEVCNTR1D_EL0 /* index 13 */ + ret + mrs x0, AMEVCNTR1E_EL0 /* index 14 */ + ret + mrs x0, AMEVCNTR1F_EL0 /* index 15 */ + ret +endfunc amu_group1_cnt_read_internal + +/* + * void amu_group1_cnt_write_internal(int idx, uint64_t val); + * + * Given `idx`, write `val` to the corresponding AMU counter. + */ +func amu_group1_cnt_write_internal +#if ENABLE_ASSERTIONS + /* + * It can be dangerous to call this function with an + * out of bounds index. Ensure `idx` is valid. + */ + mov x2, x0 + lsr x2, x2, #4 + cmp x2, #0 + ASM_ASSERT(eq) +#endif + + /* + * Given `idx` calculate address of mrs/ret instruction pair + * in the table below. + */ + adr x2, 1f + lsl x0, x0, #3 /* each msr/ret sequence is 8 bytes */ + add x2, x2, x0 + br x2 + +1: + msr AMEVCNTR10_EL0, x1 /* index 0 */ + ret + msr AMEVCNTR11_EL0, x1 /* index 1 */ + ret + msr AMEVCNTR12_EL0, x1 /* index 2 */ + ret + msr AMEVCNTR13_EL0, x1 /* index 3 */ + ret + msr AMEVCNTR14_EL0, x1 /* index 4 */ + ret + msr AMEVCNTR15_EL0, x1 /* index 5 */ + ret + msr AMEVCNTR16_EL0, x1 /* index 6 */ + ret + msr AMEVCNTR17_EL0, x1 /* index 7 */ + ret + msr AMEVCNTR18_EL0, x1 /* index 8 */ + ret + msr AMEVCNTR19_EL0, x1 /* index 9 */ + ret + msr AMEVCNTR1A_EL0, x1 /* index 10 */ + ret + msr AMEVCNTR1B_EL0, x1 /* index 11 */ + ret + msr AMEVCNTR1C_EL0, x1 /* index 12 */ + ret + msr AMEVCNTR1D_EL0, x1 /* index 13 */ + ret + msr AMEVCNTR1E_EL0, x1 /* index 14 */ + ret + msr AMEVCNTR1F_EL0, x1 /* index 15 */ + ret +endfunc amu_group1_cnt_write_internal + +/* + * void amu_group1_set_evtype_internal(int idx, unsigned int val); + * + * Program the AMU event type register indexed by `idx` + * with the value `val`. + */ +func amu_group1_set_evtype_internal +#if ENABLE_ASSERTIONS + /* + * It can be dangerous to call this function with an + * out of bounds index. Ensure `idx` is valid. + */ + mov x2, x0 + lsr x2, x2, #4 + cmp x2, #0 + ASM_ASSERT(eq) + + /* val should be between [0, 65535] */ + mov x2, x1 + lsr x2, x2, #16 + cmp x2, #0 + ASM_ASSERT(eq) +#endif + + /* + * Given `idx` calculate address of msr/ret instruction pair + * in the table below. + */ + adr x2, 1f + lsl x0, x0, #3 /* each msr/ret sequence is 8 bytes */ + add x2, x2, x0 + br x2 + +1: + msr AMEVTYPER10_EL0, x1 /* index 0 */ + ret + msr AMEVTYPER11_EL0, x1 /* index 1 */ + ret + msr AMEVTYPER12_EL0, x1 /* index 2 */ + ret + msr AMEVTYPER13_EL0, x1 /* index 3 */ + ret + msr AMEVTYPER14_EL0, x1 /* index 4 */ + ret + msr AMEVTYPER15_EL0, x1 /* index 5 */ + ret + msr AMEVTYPER16_EL0, x1 /* index 6 */ + ret + msr AMEVTYPER17_EL0, x1 /* index 7 */ + ret + msr AMEVTYPER18_EL0, x1 /* index 8 */ + ret + msr AMEVTYPER19_EL0, x1 /* index 9 */ + ret + msr AMEVTYPER1A_EL0, x1 /* index 10 */ + ret + msr AMEVTYPER1B_EL0, x1 /* index 11 */ + ret + msr AMEVTYPER1C_EL0, x1 /* index 12 */ + ret + msr AMEVTYPER1D_EL0, x1 /* index 13 */ + ret + msr AMEVTYPER1E_EL0, x1 /* index 14 */ + ret + msr AMEVTYPER1F_EL0, x1 /* index 15 */ + ret +endfunc amu_group1_set_evtype_internal From b6eb39327c5b009bc0bbecdb6867d8bf7c05f2fd Mon Sep 17 00:00:00 2001 From: Dimitris Papastamos Date: Tue, 28 Nov 2017 13:47:06 +0000 Subject: [PATCH 4/5] AMU: Add hooks to save/restore AMU context On some systems, the AMU counters might reset to 0 when a CPU powerdown happens. This behaviour conflicts with the intended use-case of AMU as lower ELs are only expected to see non-decreasing counter values. Change-Id: If25519965d4e6e47e09225d0e732947986cbb5ec Signed-off-by: Dimitris Papastamos --- include/lib/aarch32/arch_helpers.h | 5 ++ lib/extensions/amu/aarch32/amu.c | 73 ++++++++++++++++++++++++++++ lib/extensions/amu/aarch64/amu.c | 78 ++++++++++++++++++++++++++++++ 3 files changed, 156 insertions(+) diff --git a/include/lib/aarch32/arch_helpers.h b/include/lib/aarch32/arch_helpers.h index 0230195a04..beae5d0658 100644 --- a/include/lib/aarch32/arch_helpers.h +++ b/include/lib/aarch32/arch_helpers.h @@ -287,6 +287,11 @@ DEFINE_COPROCR_RW_FUNCS(amcntenset1, AMCNTENSET1) DEFINE_COPROCR_RW_FUNCS(amcntenclr0, AMCNTENCLR0) DEFINE_COPROCR_RW_FUNCS(amcntenclr1, AMCNTENCLR1) +DEFINE_COPROCR_RW_FUNCS_64(amevcntr00, AMEVCNTR00) +DEFINE_COPROCR_RW_FUNCS_64(amevcntr01, AMEVCNTR01) +DEFINE_COPROCR_RW_FUNCS_64(amevcntr02, AMEVCNTR02) +DEFINE_COPROCR_RW_FUNCS_64(amevcntr03, AMEVCNTR03) + /* * TLBI operation prototypes */ diff --git a/lib/extensions/amu/aarch32/amu.c b/lib/extensions/amu/aarch32/amu.c index be37006636..55462cbf21 100644 --- a/lib/extensions/amu/aarch32/amu.c +++ b/lib/extensions/amu/aarch32/amu.c @@ -8,6 +8,16 @@ #include #include #include +#include +#include + +#define AMU_GROUP0_NR_COUNTERS 4 + +struct amu_ctx { + uint64_t group0_cnts[AMU_GROUP0_NR_COUNTERS]; +}; + +static struct amu_ctx amu_ctxs[PLATFORM_CORE_COUNT]; void amu_enable(int el2_unused) { @@ -34,3 +44,66 @@ void amu_enable(int el2_unused) /* Enable group 0 counters */ write_amcntenset0(AMU_GROUP0_COUNTERS_MASK); } + +static void *amu_context_save(const void *arg) +{ + struct amu_ctx *ctx; + uint64_t features; + + features = read_id_pfr0() >> ID_PFR0_AMU_SHIFT; + if ((features & ID_PFR0_AMU_MASK) != 1) + return (void *)-1; + + ctx = &amu_ctxs[plat_my_core_pos()]; + + /* Assert that group 0 counter configuration is what we expect */ + assert(read_amcntenset0() == AMU_GROUP0_COUNTERS_MASK); + + /* + * Disable group 0 counters to avoid other observers like SCP sampling + * counter values from the future via the memory mapped view. + */ + write_amcntenclr0(AMU_GROUP0_COUNTERS_MASK); + isb(); + + ctx->group0_cnts[0] = read64_amevcntr00(); + ctx->group0_cnts[1] = read64_amevcntr01(); + ctx->group0_cnts[2] = read64_amevcntr02(); + ctx->group0_cnts[3] = read64_amevcntr03(); + + return 0; +} + +static void *amu_context_restore(const void *arg) +{ + struct amu_ctx *ctx; + uint64_t features; + + features = read_id_pfr0() >> ID_PFR0_AMU_SHIFT; + if ((features & ID_PFR0_AMU_MASK) != 1) + return (void *)-1; + + ctx = &amu_ctxs[plat_my_core_pos()]; + + /* Counters were disabled in `amu_context_save()` */ + assert(read_amcntenset0() == 0); + + /* Restore group 0 counters */ + if (AMU_GROUP0_COUNTERS_MASK & (1U << 0)) + write64_amevcntr00(ctx->group0_cnts[0]); + if (AMU_GROUP0_COUNTERS_MASK & (1U << 1)) + write64_amevcntr01(ctx->group0_cnts[1]); + if (AMU_GROUP0_COUNTERS_MASK & (1U << 2)) + write64_amevcntr02(ctx->group0_cnts[2]); + if (AMU_GROUP0_COUNTERS_MASK & (1U << 3)) + write64_amevcntr03(ctx->group0_cnts[3]); + isb(); + + /* Enable group 0 counters */ + write_amcntenset0(AMU_GROUP0_COUNTERS_MASK); + + return 0; +} + +SUBSCRIBE_TO_EVENT(psci_suspend_pwrdown_start, amu_context_save); +SUBSCRIBE_TO_EVENT(psci_suspend_pwrdown_finish, amu_context_restore); diff --git a/lib/extensions/amu/aarch64/amu.c b/lib/extensions/amu/aarch64/amu.c index 27936a2e17..f743e209f9 100644 --- a/lib/extensions/amu/aarch64/amu.c +++ b/lib/extensions/amu/aarch64/amu.c @@ -10,9 +10,18 @@ #include #include #include +#include +#include #define AMU_GROUP0_NR_COUNTERS 4 +struct amu_ctx { + uint64_t group0_cnts[AMU_GROUP0_NR_COUNTERS]; + uint64_t group1_cnts[AMU_GROUP1_NR_COUNTERS]; +}; + +static struct amu_ctx amu_ctxs[PLATFORM_CORE_COUNT]; + int amu_supported(void) { uint64_t features; @@ -108,3 +117,72 @@ void amu_group1_set_evtype(int idx, unsigned int val) amu_group1_set_evtype_internal(idx, val); isb(); } + +static void *amu_context_save(const void *arg) +{ + struct amu_ctx *ctx = &amu_ctxs[plat_my_core_pos()]; + int i; + + if (!amu_supported()) + return (void *)-1; + + /* Assert that group 0/1 counter configuration is what we expect */ + assert(read_amcntenset0_el0() == AMU_GROUP0_COUNTERS_MASK && + read_amcntenset1_el0() == AMU_GROUP1_COUNTERS_MASK); + + assert((sizeof(int) * 8) - __builtin_clz(AMU_GROUP1_COUNTERS_MASK) + <= AMU_GROUP1_NR_COUNTERS); + + /* + * Disable group 0/1 counters to avoid other observers like SCP sampling + * counter values from the future via the memory mapped view. + */ + write_amcntenclr0_el0(AMU_GROUP0_COUNTERS_MASK); + write_amcntenclr1_el0(AMU_GROUP1_COUNTERS_MASK); + isb(); + + /* Save group 0 counters */ + for (i = 0; i < AMU_GROUP0_NR_COUNTERS; i++) + ctx->group0_cnts[i] = amu_group0_cnt_read(i); + + /* Save group 1 counters */ + for (i = 0; i < AMU_GROUP1_NR_COUNTERS; i++) + ctx->group1_cnts[i] = amu_group1_cnt_read(i); + + return 0; +} + +static void *amu_context_restore(const void *arg) +{ + struct amu_ctx *ctx = &amu_ctxs[plat_my_core_pos()]; + int i; + + if (!amu_supported()) + return (void *)-1; + + /* Counters were disabled in `amu_context_save()` */ + assert(read_amcntenset0_el0() == 0 && read_amcntenset1_el0() == 0); + + assert((sizeof(int) * 8) - __builtin_clz(AMU_GROUP1_COUNTERS_MASK) + <= AMU_GROUP1_NR_COUNTERS); + + /* Restore group 0 counters */ + for (i = 0; i < AMU_GROUP0_NR_COUNTERS; i++) + if (AMU_GROUP0_COUNTERS_MASK & (1U << i)) + amu_group0_cnt_write(i, ctx->group0_cnts[i]); + + /* Restore group 1 counters */ + for (i = 0; i < AMU_GROUP1_NR_COUNTERS; i++) + if (AMU_GROUP1_COUNTERS_MASK & (1U << i)) + amu_group1_cnt_write(i, ctx->group1_cnts[i]); + isb(); + + /* Restore group 0/1 counter configuration */ + write_amcntenset0_el0(AMU_GROUP0_COUNTERS_MASK); + write_amcntenset1_el0(AMU_GROUP1_COUNTERS_MASK); + + return 0; +} + +SUBSCRIBE_TO_EVENT(psci_suspend_pwrdown_start, amu_context_save); +SUBSCRIBE_TO_EVENT(psci_suspend_pwrdown_finish, amu_context_restore); From 53bfb94ececbed0fd6eb3550c79254a928a13067 Mon Sep 17 00:00:00 2001 From: Dimitris Papastamos Date: Mon, 11 Dec 2017 11:45:35 +0000 Subject: [PATCH 5/5] Add hooks to save/restore AMU context for Cortex A75 Change-Id: I504d3f65ca5829bc1f4ebadb764931f8379ee81f Signed-off-by: Dimitris Papastamos --- include/lib/cpus/aarch64/cortex_a75.h | 12 ++++ lib/cpus/aarch64/cortex_a75.S | 98 +++++++++++++++++++++++++++ lib/cpus/aarch64/cortex_a75_pubsub.c | 75 ++++++++++++++++++++ plat/arm/board/fvp/platform.mk | 4 ++ 4 files changed, 189 insertions(+) create mode 100644 lib/cpus/aarch64/cortex_a75_pubsub.c diff --git a/include/lib/cpus/aarch64/cortex_a75.h b/include/lib/cpus/aarch64/cortex_a75.h index d68c957213..940125dafd 100644 --- a/include/lib/cpus/aarch64/cortex_a75.h +++ b/include/lib/cpus/aarch64/cortex_a75.h @@ -50,7 +50,19 @@ * CPUAMEVTYPER register and are disabled by default. Platforms may * enable this with suitable programming. */ +#define CORTEX_A75_AMU_NR_COUNTERS 5 #define CORTEX_A75_AMU_GROUP0_MASK 0x7 #define CORTEX_A75_AMU_GROUP1_MASK (0 << 3) +#ifndef __ASSEMBLY__ +#include + +uint64_t cortex_a75_amu_cnt_read(int idx); +void cortex_a75_amu_cnt_write(int idx, uint64_t val); +unsigned int cortex_a75_amu_read_cpuamcntenset_el0(void); +unsigned int cortex_a75_amu_read_cpuamcntenclr_el0(void); +void cortex_a75_amu_write_cpuamcntenset_el0(unsigned int mask); +void cortex_a75_amu_write_cpuamcntenclr_el0(unsigned int mask); +#endif /* __ASSEMBLY__ */ + #endif /* __CORTEX_A75_H__ */ diff --git a/lib/cpus/aarch64/cortex_a75.S b/lib/cpus/aarch64/cortex_a75.S index 4cab9e4fcc..7bd2020231 100644 --- a/lib/cpus/aarch64/cortex_a75.S +++ b/lib/cpus/aarch64/cortex_a75.S @@ -11,6 +11,104 @@ #include #include + .globl cortex_a75_amu_cnt_read + .globl cortex_a75_amu_cnt_write + .globl cortex_a75_amu_read_cpuamcntenset_el0 + .globl cortex_a75_amu_read_cpuamcntenclr_el0 + .globl cortex_a75_amu_write_cpuamcntenset_el0 + .globl cortex_a75_amu_write_cpuamcntenclr_el0 + +/* + * uint64_t cortex_a75_amu_cnt_read(int idx); + * + * Given `idx`, read the corresponding AMU counter + * and return it in `x0`. + */ +func cortex_a75_amu_cnt_read + adr x1, 1f + lsl x0, x0, #3 + add x1, x1, x0 + br x1 + +1: + mrs x0, CPUAMEVCNTR0_EL0 + ret + mrs x0, CPUAMEVCNTR1_EL0 + ret + mrs x0, CPUAMEVCNTR2_EL0 + ret + mrs x0, CPUAMEVCNTR3_EL0 + ret + mrs x0, CPUAMEVCNTR4_EL0 + ret +endfunc cortex_a75_amu_cnt_read + +/* + * void cortex_a75_amu_cnt_write(int idx, uint64_t val); + * + * Given `idx`, write `val` to the corresponding AMU counter. + */ +func cortex_a75_amu_cnt_write + adr x2, 1f + lsl x0, x0, #3 + add x2, x2, x0 + br x2 + +1: + msr CPUAMEVCNTR0_EL0, x0 + ret + msr CPUAMEVCNTR1_EL0, x0 + ret + msr CPUAMEVCNTR2_EL0, x0 + ret + msr CPUAMEVCNTR3_EL0, x0 + ret + msr CPUAMEVCNTR4_EL0, x0 + ret +endfunc cortex_a75_amu_cnt_write + +/* + * unsigned int cortex_a75_amu_read_cpuamcntenset_el0(void); + * + * Read the `CPUAMCNTENSET_EL0` CPU register and return + * it in `x0`. + */ +func cortex_a75_amu_read_cpuamcntenset_el0 + mrs x0, CPUAMCNTENSET_EL0 + ret +endfunc cortex_a75_amu_read_cpuamcntenset_el0 + +/* + * unsigned int cortex_a75_amu_read_cpuamcntenclr_el0(void); + * + * Read the `CPUAMCNTENCLR_EL0` CPU register and return + * it in `x0`. + */ +func cortex_a75_amu_read_cpuamcntenclr_el0 + mrs x0, CPUAMCNTENCLR_EL0 + ret +endfunc cortex_a75_amu_read_cpuamcntenclr_el0 + +/* + * void cortex_a75_amu_write_cpuamcntenset_el0(unsigned int mask); + * + * Write `mask` to the `CPUAMCNTENSET_EL0` CPU register. + */ +func cortex_a75_amu_write_cpuamcntenset_el0 + msr CPUAMCNTENSET_EL0, x0 + ret +endfunc cortex_a75_amu_write_cpuamcntenset_el0 + +/* + * void cortex_a75_amu_write_cpuamcntenclr_el0(unsigned int mask); + * + * Write `mask` to the `CPUAMCNTENCLR_EL0` CPU register. + */ +func cortex_a75_amu_write_cpuamcntenclr_el0 + mrs x0, CPUAMCNTENCLR_EL0 + ret +endfunc cortex_a75_amu_write_cpuamcntenclr_el0 + func cortex_a75_reset_func #if ENABLE_AMU /* Make sure accesses from EL0/EL1 and EL2 are not trapped to EL3 */ diff --git a/lib/cpus/aarch64/cortex_a75_pubsub.c b/lib/cpus/aarch64/cortex_a75_pubsub.c new file mode 100644 index 0000000000..c1089a6071 --- /dev/null +++ b/lib/cpus/aarch64/cortex_a75_pubsub.c @@ -0,0 +1,75 @@ +/* + * Copyright (c) 2017, ARM Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include +#include +#include + +struct amu_ctx { + uint64_t cnts[CORTEX_A75_AMU_NR_COUNTERS]; + uint16_t mask; +}; + +static struct amu_ctx amu_ctxs[PLATFORM_CORE_COUNT]; + +static void *cortex_a75_context_save(const void *arg) +{ + struct amu_ctx *ctx = &amu_ctxs[plat_my_core_pos()]; + unsigned int midr; + unsigned int midr_mask; + int i; + + midr = read_midr(); + midr_mask = (MIDR_IMPL_MASK << MIDR_IMPL_SHIFT) | + (MIDR_PN_MASK << MIDR_PN_SHIFT); + if ((midr & midr_mask) != (CORTEX_A75_MIDR & midr_mask)) + return 0; + + /* Save counter configuration */ + ctx->mask = cortex_a75_amu_read_cpuamcntenset_el0(); + + /* Ensure counters are disabled */ + cortex_a75_amu_write_cpuamcntenclr_el0(ctx->mask); + isb(); + + /* Save counters */ + for (i = 0; i < CORTEX_A75_AMU_NR_COUNTERS; i++) + ctx->cnts[i] = cortex_a75_amu_cnt_read(i); + + return 0; +} + +static void *cortex_a75_context_restore(const void *arg) +{ + struct amu_ctx *ctx = &amu_ctxs[plat_my_core_pos()]; + unsigned int midr; + unsigned int midr_mask; + int i; + + midr = read_midr(); + midr_mask = (MIDR_IMPL_MASK << MIDR_IMPL_SHIFT) | + (MIDR_PN_MASK << MIDR_PN_SHIFT); + if ((midr & midr_mask) != (CORTEX_A75_MIDR & midr_mask)) + return 0; + + ctx = &amu_ctxs[plat_my_core_pos()]; + + /* Counters were disabled in `cortex_a75_context_save()` */ + assert(cortex_a75_amu_read_cpuamcntenset_el0() == 0); + + /* Restore counters */ + for (i = 0; i < CORTEX_A75_AMU_NR_COUNTERS; i++) + cortex_a75_amu_cnt_write(i, ctx->cnts[i]); + isb(); + + /* Restore counter configuration */ + cortex_a75_amu_write_cpuamcntenset_el0(ctx->mask); + + return 0; +} + +SUBSCRIBE_TO_EVENT(psci_suspend_pwrdown_start, cortex_a75_context_save); +SUBSCRIBE_TO_EVENT(psci_suspend_pwrdown_finish, cortex_a75_context_restore); diff --git a/plat/arm/board/fvp/platform.mk b/plat/arm/board/fvp/platform.mk index 7edbd3dfa1..9d3c5f6b7e 100644 --- a/plat/arm/board/fvp/platform.mk +++ b/plat/arm/board/fvp/platform.mk @@ -150,6 +150,10 @@ ENABLE_PLAT_COMPAT := 0 # Enable Activity Monitor Unit extensions by default ENABLE_AMU := 1 +ifeq (${ENABLE_AMU},1) +BL31_SOURCES += lib/cpus/aarch64/cortex_a75_pubsub.c +endif + ifneq (${ENABLE_STACK_PROTECTOR},0) PLAT_BL_COMMON_SOURCES += plat/arm/board/fvp/fvp_stack_protector.c endif