cortex-m: add D-cache support

Add support to enable the architectural D-cache on ARMv7-M CPU
supporting it.
Update the MPU code in order to be able to declare an 'uncached' RAM
region (e.g. to store the DMA buffer).

Signed-off-by: Vincent Palatin <vpalatin@chromium.org>

BRANCH=poppy
BUG=b:78535052, b:75068419
TEST=with the following CL, on ZerbleBarn, boot and capture a finger
image.

Change-Id: I275445e7c0b558cedc3e7d6fc6840ff9b4b76285
Reviewed-on: https://chromium-review.googlesource.com/1032776
Commit-Ready: Vincent Palatin <vpalatin@chromium.org>
Tested-by: Vincent Palatin <vpalatin@chromium.org>
Reviewed-by: Nicolas Boichat <drinkcat@chromium.org>

chip/stm32/system.c

@@ -205,10 +205,6 @@ void chip_pre_init(void)
 	uint32_t apb1fz_reg = 0;
 	uint32_t apb2fz_reg = 0;
 
-#ifdef CONFIG_ARMV7M_CACHE
-	cpu_enable_icache();
-#endif
-
 #if defined(CHIP_FAMILY_STM32F0)
 	apb1fz_reg =
 		STM32_RCC_PB1_TIM2 | STM32_RCC_PB1_TIM3 | STM32_RCC_PB1_TIM6 |

core/cortex-m/build.mk

@@ -23,6 +23,7 @@ LDFLAGS_EXTRA+=-flto
 endif
 
 core-y=cpu.o init.o ldivmod.o llsr.o uldivmod.o
+core-$(CONFIG_ARMV7M_CACHE)+=cache.o
 core-$(CONFIG_COMMON_PANIC_OUTPUT)+=panic.o
 core-$(CONFIG_COMMON_RUNTIME)+=switch.o task.o
 core-$(CONFIG_WATCHDOG)+=watchdog.o

core/cortex-m/cache.S

@@ -0,0 +1,76 @@
+/* Copyright 2018 The Chromium OS Authors. All rights reserved.
+ * Use of this source code is governed by a BSD-style license that can be
+ * found in the LICENSE file.
+ *
+ * ARMv7-M architectural caches maintenance operations.
+ */
+
+.syntax unified
+.text
+.thumb
+
+/* System Control Block: cache registers */
+#define SCB_CCSIDR 0xe000ed80
+#define SCB_CCSELR 0xe000ed84
+#define SCB_DCISW  0xe000ef60
+#define SCB_DCCISW 0xe000ef74
+
+.macro dcache_set_way_op name register
+@
+@ Perform an operation on all D-cache sets/ways.
+@
+@ Note: implemented in assembly to guarantee that we are not touching the
+@ D-cache in the middle of the loop.
+@
+.thumb_func
+.section .text.\name
+.global \name
+\name:
+	/* Select Level-1 Data cache (for operations on CCSIDR). */
+	ldr  r1, =SCB_CCSELR
+	movs r0, #0
+	ldr  r2, =SCB_CCSIDR
+	str  r0, [r1]  /* set CCSELR = 0 */
+
+	/* Ensure the CCSELR write is effective before reading CCSIDR. */
+	dsb
+	/* CCSIDR contains the cache geometry. */
+	ldr  r3, [r2]  /* [27:13] Number of sets -1  [12:3] Number of ways -1 */
+
+	/* register used to do the set/way cache operation. */
+	ldr  r0, =\register
+	/* r2 is the number of cache 'sets' - 1 */
+	ubfx r2, r3, #13, #15
+	/* r12 is the number of cache 'ways' - 1 */
+	ubfx r12, r3, #3, #10
+
+1:
+	mov  r1, r12    /* reset way index */
+2:
+	/*
+	 * Build address Set/Way operation e.g DC(C)ISW
+	 * [31:30] way index [13:5] set index
+	 */
+	lsls r3, r2, #5 /* set index */
+	/* TODO(crbug.com/848704) remove cache geometry assumptions */
+	orr  r3, r3, r1, lsl #30 /* way index */
+	/* Perform operation (e.g invalidate) on a D-cache line */
+	str  r3, [r0]
+	/* go to previous way */
+	subs r1, #1
+	bcs  2b
+	/* go to previous set */
+	subs r2, #1
+	bcs  1b
+
+	/* Ensure everything has propagated and return. */
+	dsb
+	isb
+	bx  lr
+.endm
+
+/* D-cache Invalidate by set-way */
+dcache_set_way_op cpu_invalidate_dcache SCB_DCISW
+
+/* D-cache Clean and Invalidate by set-way, to Point of Coherency */
+dcache_set_way_op cpu_clean_invalidate_dcache SCB_DCCISW

core/cortex-m/cpu.c

@@ -30,7 +30,7 @@ static void cpu_invalidate_icache(void)
 	asm volatile("dsb; isb");
 }
 
-void cpu_enable_icache(void)
+void cpu_enable_caches(void)
 {
 	/* Check whether the I-cache is already enabled */
 	if (!(CPU_NVIC_CCR & CPU_NVIC_CCR_ICACHE)) {
@@ -40,16 +40,25 @@ void cpu_enable_icache(void)
 		CPU_NVIC_CCR |= CPU_NVIC_CCR_ICACHE;
 		asm volatile("dsb; isb");
 	}
+	/* Check whether the D-cache is already enabled */
+	if (!(CPU_NVIC_CCR & CPU_NVIC_CCR_DCACHE)) {
+		/* Invalidate the D-cache first */
+		cpu_invalidate_dcache();
+		/* Turn on the caching */
+		CPU_NVIC_CCR |= CPU_NVIC_CCR_DCACHE;
+		asm volatile("dsb; isb");
+	}
 }
 
 static void cpu_sysjump_cache(void)
 {
 	/*
-	 * Disable the I-cache
-	 * so we will invalidate it after the sysjump if needed
+	 * Disable the I-cache and the D-cache
+	 * The I-cache will be invalidated after the sysjump if needed
 	 * (e.g after a flash update).
 	 */
-	CPU_NVIC_CCR &= ~CPU_NVIC_CCR_ICACHE;
+	cpu_clean_invalidate_dcache();
+	CPU_NVIC_CCR &= ~(CPU_NVIC_CCR_ICACHE | CPU_NVIC_CCR_DCACHE);
 	asm volatile("dsb; isb");
 }
 DECLARE_HOOK(HOOK_SYSJUMP, cpu_sysjump_cache, HOOK_PRIO_LAST);

core/cortex-m/cpu.h

@@ -60,10 +60,15 @@ enum {
 #define CPU_SCB_CCSELR         CPUREG(0xe000ed84)
 #define CPU_SCB_ICIALLU        CPUREG(0xe000ef50)
 #define CPU_SCB_DCISW          CPUREG(0xe000ef60)
+#define CPU_SCB_DCCISW         CPUREG(0xe000ef74)
 
 /* Set up the cpu to detect faults */
 void cpu_init(void);
-/* Enable the CPU instruction cache if it is not already enabled */
-void cpu_enable_icache(void);
+/* Enable the CPU I-cache and D-cache if they are not already enabled */
+void cpu_enable_caches(void);
+/* Invalidate the D-cache */
+void cpu_invalidate_dcache(void);
+/* Clean and Invalidate the D-cache to the Point of Coherency */
+void cpu_clean_invalidate_dcache(void);
 
 #endif /* __CROS_EC_CPU_H */

core/cortex-m/include/mpu.h

@@ -9,6 +9,7 @@
 #define __CROS_EC_MPU_H
 
 #include "common.h"
+#include "config.h" /* chips might override MPU attribute settings */
 
 /*
  * Region assignment. 7 as the highest, a higher index has a higher priority.
@@ -28,6 +29,8 @@ enum mpu_region {
 	REGION_STORAGE2 = 5,		/* Second region for unaligned size */
 	REGION_DATA_RAM_TEXT = 6,	/* Exempt region of data RAM */
 	REGION_CHIP_RESERVED = 7,	/* Reserved for use in chip/ */
+	/* only for chips with MPU supporting 16 regions */
+	REGION_UNCACHED_RAM = 8,        /* For uncached data RAM */
 };
 
 #define MPU_TYPE		REG32(0xe000ed90)
@@ -37,6 +40,9 @@ enum mpu_region {
 #define MPU_SIZE		REG16(0xe000eda0)
 #define MPU_ATTR		REG16(0xe000eda2)
 
+#define MPU_TYPE_UNIFIED_MASK	0x00FF0001
+#define MPU_TYPE_REG_COUNT(t)	(((t) >> 8) & 0xFF)
+
 #define MPU_CTRL_PRIVDEFEN	(1 << 2)
 #define MPU_CTRL_HFNMIENA	(1 << 1)
 #define MPU_CTRL_ENABLE		(1 << 0)
@@ -55,8 +61,12 @@ enum mpu_region {
 
 /* Suggested value for TEX S/C/B bit. See table 3-6 of Stellaris LM4F232H5QC
  * datasheet and table 38 of STM32F10xxx Cortex-M3 programming manual. */
+#ifndef MPU_ATTR_INTERNAL_SRAM
 #define MPU_ATTR_INTERNAL_SRAM  6  /* for Internal SRAM */
+#endif
+#ifndef MPU_ATTR_FLASH_MEMORY
 #define MPU_ATTR_FLASH_MEMORY   2  /* for flash memory */
+#endif
 
 /**
  * Enable MPU

core/cortex-m/mpu.c

@@ -7,6 +7,7 @@
 
 #include "mpu.h"
 #include "console.h"
+#include "cpu.h"
 #include "registers.h"
 #include "task.h"
 #include "util.h"
@@ -32,8 +33,16 @@ static void mpu_update_region(uint8_t region, uint32_t addr, uint8_t size_bit,
 	MPU_SIZE &= ~1;					/* Disable */
 	if (enable) {
 		MPU_BASE = addr;
-		MPU_ATTR = attr;
-		MPU_SIZE = (srd << 8) | ((size_bit - 1) << 1) | 1; /* Enable */
+		/*
+		 * MPU_ATTR = attr;
+		 * MPU_SIZE = (srd << 8) | ((size_bit - 1) << 1) | 1;
+		 *
+		 * WORKAROUND: the 2 half-word accesses above should work
+		 * according to the doc, but they don't ..., do a single 32-bit
+		 * one.
+		 */
+		REG32(&MPU_SIZE) = ((uint32_t)attr << 16)
+				 | (srd << 8) | ((size_bit - 1) << 1) | 1;
 	}
 
 	asm volatile("isb; dsb;");
@@ -202,16 +211,42 @@ int mpu_lock_rw_flash(void)
 }
 #endif /* !CONFIG_EXTERNAL_STORAGE */
 
+#ifdef CONFIG_CHIP_UNCACHED_REGION
+/* Store temporarily the regions ranges to use them for the MPU configuration */
+#define REGION(_name, _flag, _start, _size) \
+	static const uint32_t CONCAT2(_region_start_, _name) \
+		__attribute__((unused, section(".unused"))) = _start; \
+	static const uint32_t CONCAT2(_region_size_, _name) \
+		__attribute__((unused, section(".unused"))) = _size;
+#include "memory_regions.inc"
+#undef REGION
+#endif /* CONFIG_CHIP_UNCACHED_REGION */
+
 int mpu_pre_init(void)
 {
 	int i;
+	uint32_t mpu_type = mpu_get_type();
 
-	if (mpu_get_type() != 0x00000800)
+	/* Supports MPU with 8 or 16 unified regions */
+	if ((mpu_type & MPU_TYPE_UNIFIED_MASK) ||
+	    (MPU_TYPE_REG_COUNT(mpu_type) != 8 &&
+	     MPU_TYPE_REG_COUNT(mpu_type) != 16))
 		return EC_ERROR_UNIMPLEMENTED;
 
 	mpu_disable();
-	for (i = 0; i < 8; ++i)
+	for (i = 0; i < MPU_TYPE_REG_COUNT(mpu_type); ++i)
 		mpu_config_region(i, CONFIG_RAM_BASE, CONFIG_RAM_SIZE, 0, 0);
 
+#ifdef CONFIG_ARMV7M_CACHE
+#ifdef CONFIG_CHIP_UNCACHED_REGION
+	mpu_config_region(REGION_UNCACHED_RAM,
+			  CONCAT2(_region_start_, CONFIG_CHIP_UNCACHED_REGION),
+			  CONCAT2(_region_size_, CONFIG_CHIP_UNCACHED_REGION),
+			  MPU_ATTR_XN | MPU_ATTR_RW_RW, 1);
+	mpu_enable();
+#endif /* CONFIG_CHIP_UNCACHED_REGION */
+	cpu_enable_caches();
+#endif /* CONFIG_ARMV7M_CACHE */
+
 	return EC_SUCCESS;
 }

include/config.h

@@ -742,6 +742,14 @@
  */
 #undef CONFIG_CHIP_PRE_INIT
 
+/*
+ * Set the caching attributes of one of the RAM regions to uncached.
+ *
+ * When defined, CONFIG_CHIP_UNCACHED_REGION must be equal to the name of one
+ * of the regions defined in memory_regions.inc for CONFIG_CHIP_MEMORY_REGIONS.
+ */
+#undef CONFIG_CHIP_UNCACHED_REGION
+
 /*****************************************************************************/
 /* Chipset config */
 

include/link_defs.h

@@ -110,5 +110,10 @@ extern const void *__data_end;
 #else
 #define __SECTION(name)
 #endif /* CONFIG_MEMORY_REGIONS */
+#ifdef CONFIG_CHIP_UNCACHED_REGION
+#define __uncached __SECTION(CONFIG_CHIP_UNCACHED_REGION)
+#else
+#define __uncached
+#endif
 
 #endif /* __CROS_EC_LINK_DEFS_H */