diff options
| -rw-r--r-- | payloads/libpayload/arch/armv7/cache.c | 141 | ||||
| -rw-r--r-- | payloads/libpayload/include/armv7/arch/cache.h | 102 |
2 files changed, 203 insertions, 40 deletions
diff --git a/payloads/libpayload/arch/armv7/cache.c b/payloads/libpayload/arch/armv7/cache.c index 60d5f742b9..9fac442f17 100644 --- a/payloads/libpayload/arch/armv7/cache.c +++ b/payloads/libpayload/arch/armv7/cache.c @@ -26,7 +26,9 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * cache.c: Low-level cache operations for ARMv7 + * cache.c: Cache maintenance routines for ARMv7-A and ARMv7-R + * + * Reference: ARM Architecture Reference Manual, ARMv7-A and ARMv7-R edition */ #include <inttypes.h> @@ -52,8 +54,8 @@ void tlb_invalidate_all(void) { /* * FIXME: ARMv7 Architecture Ref. Manual claims that the distinction - * instruction vs. data TLBs is deprecated in ARMv7. But that doesn't - * really seem true for Cortex-A15? + * instruction vs. data TLBs is deprecated in ARMv7, however this does + * not seem to be the case as of Cortex-A15. */ tlbiall(); dtlbiall(); @@ -64,7 +66,8 @@ void tlb_invalidate_all(void) void icache_invalidate_all(void) { - /* icache can be entirely invalidated with one operation. + /* + * icache can be entirely invalidated with one operation. * Note: If branch predictors are architecturally-visible, ICIALLU * also performs a BPIALL operation (B2-1283 in arch manual) */ @@ -74,10 +77,17 @@ void icache_invalidate_all(void) enum dcache_op { OP_DCCISW, - OP_DCISW + OP_DCISW, + OP_DCCIMVAC, + OP_DCCMVAC, }; -/* do a dcache operation on entire cache by set/way */ +/* + * Do a dcache operation on entire cache by set/way. This is done for + * portability because mapping of memory address to cache location is + * implementation defined (See note on "Requirements for operations by + * set/way" in arch ref. manual). + */ static void dcache_op_set_way(enum dcache_op op) { uint32_t ccsidr; @@ -101,6 +111,8 @@ static void dcache_op_set_way(enum dcache_op op) /* FIXME: do we need to use CTR.DminLine here? */ linesize_bytes = (1 << ((ccsidr & 0x7) + 2)) * 4; + dsb(); + /* * Set/way operations require an interesting bit packing. See section * B4-35 in the ARMv7 Architecture Reference Manual: @@ -134,8 +146,7 @@ static void dcache_op_set_way(enum dcache_op op) } } } - - dsb(); + isb(); } void dcache_clean_invalidate_all(void) @@ -161,18 +172,116 @@ static unsigned int line_bytes(void) return size; } +/* + * Do a dcache operation by modified virtual address. This is useful for + * maintaining coherency in drivers which do DMA transfers and only need to + * perform cache maintenance on a particular memory range rather than the + * entire cache. + */ +static void dcache_op_mva(unsigned long addr, + unsigned long len, enum dcache_op op) +{ + unsigned long line, linesize; + + linesize = line_bytes(); + line = addr & ~(linesize - 1); + + dsb(); + while (line < addr + len) { + switch(op) { + case OP_DCCIMVAC: + dccimvac(line); + break; + default: + break; + } + line += linesize; + } + isb(); +} + +void dcache_clean_by_mva(unsigned long addr, unsigned long len) +{ + dcache_op_mva(addr, len, OP_DCCMVAC); +} + void dcache_clean_invalidate_by_mva(unsigned long addr, unsigned long len) { - unsigned long line, i; + dcache_op_mva(addr, len, OP_DCCIMVAC); +} + +void dcache_mmu_disable(void) +{ + uint32_t sctlr, csselr; + + /* ensure L1 data/unified cache is selected */ + csselr = read_csselr(); + csselr &= ~0xf; + write_csselr(csselr); + + dcache_clean_invalidate_all(); + + sctlr = read_sctlr(); + sctlr &= ~(SCTLR_C | SCTLR_M); + write_sctlr(sctlr); +} + + +void dcache_mmu_enable(void) +{ + uint32_t sctlr; - line = line_bytes(); - for (i = addr & ~(line - 1); i < addr + len - 1; i += line) - dccimvac(addr); + sctlr = read_sctlr(); + dcache_clean_invalidate_all(); + sctlr |= SCTLR_C | SCTLR_M; + write_sctlr(sctlr); } -/* FIXME: wrapper around imported mmu_setup() for now */ -extern void mmu_setup(unsigned long start, unsigned long size); -void mmu_setup_by_mva(unsigned long start, unsigned long size) +void armv7_invalidate_caches(void) { - mmu_setup(start, size); + uint32_t clidr; + int level; + + /* Invalidate branch predictor */ + bpiall(); + + /* Iterate thru each cache identified in CLIDR and invalidate */ + clidr = read_clidr(); + for (level = 0; level < 7; level++) { + unsigned int ctype = (clidr >> (level * 3)) & 0x7; + uint32_t csselr; + + switch(ctype) { + case 0x0: + /* no cache */ + break; + case 0x1: + /* icache only */ + csselr = (level << 1) | 1; + write_csselr(csselr); + icache_invalidate_all(); + break; + case 0x2: + case 0x4: + /* dcache only or unified cache */ + dcache_invalidate_all(); + break; + case 0x3: + /* separate icache and dcache */ + csselr = (level << 1) | 1; + write_csselr(csselr); + icache_invalidate_all(); + + csselr = level < 1; + write_csselr(csselr); + dcache_invalidate_all(); + break; + default: + /* reserved */ + break; + } + } + + /* Invalidate TLB */ + tlb_invalidate_all(); } diff --git a/payloads/libpayload/include/armv7/arch/cache.h b/payloads/libpayload/include/armv7/arch/cache.h index 5125b8c449..f074a3b158 100644 --- a/payloads/libpayload/include/armv7/arch/cache.h +++ b/payloads/libpayload/include/armv7/arch/cache.h @@ -25,6 +25,8 @@ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. + * + * cache.h: Cache maintenance API for ARMv7 */ #ifndef ARMV7_CACHE_H @@ -91,7 +93,7 @@ static inline void isb(void) /* invalidate entire data TLB */ static inline void dtlbiall(void) { - asm volatile ("mcr p15, 0, %0, c8, c6, 0" : : "r" (0)); + asm volatile ("mcr p15, 0, %0, c8, c6, 0" : : "r" (0) : "memory"); } /* invalidate entire instruction TLB */ @@ -103,7 +105,33 @@ static inline void itlbiall(void) /* invalidate entire unified TLB */ static inline void tlbiall(void) { - asm volatile ("mcr p15, 0, %0, c8, c7, 0" : : "r" (0)); + asm volatile ("mcr p15, 0, %0, c8, c7, 0" : : "r" (0) : "memory"); +} + +/* write data access control register (DACR) */ +static inline void write_dacr(uint32_t val) +{ + asm volatile ("mcr p15, 0, %0, c3, c0, 0" : : "r" (val)); +} + +/* write translation table base register 0 (TTBR0) */ +static inline void write_ttbr0(uint32_t val) +{ + asm volatile ("mcr p15, 0, %0, c2, c0, 0" : : "r" (val) : "memory"); +} + +/* read translation table base control register (TTBCR) */ +static inline uint32_t read_ttbcr(void) +{ + uint32_t val = 0; + asm volatile ("mrc p15, 0, %0, c2, c0, 2" : "=r" (val)); + return val; +} + +/* write translation table base control register (TTBCR) */ +static inline void write_ttbcr(uint32_t val) +{ + asm volatile ("mcr p15, 0, %0, c2, c0, 2" : : "r" (val) : "memory"); } /* @@ -119,31 +147,31 @@ static inline void bpiall(void) /* data cache clean and invalidate by MVA to PoC */ static inline void dccimvac(unsigned long mva) { - asm volatile ("mcr p15, 0, %0, c7, c14, 1" : : "r" (mva)); + asm volatile ("mcr p15, 0, %0, c7, c14, 1" : : "r" (mva) : "memory"); } /* data cache invalidate by set/way */ static inline void dccisw(uint32_t val) { - asm volatile ("mcr p15, 0, %0, c7, c14, 2" : : "r" (val)); -} - -/* data cache invalidate by set/way */ -static inline void dcisw(uint32_t val) -{ - asm volatile ("mcr p15, 0, %0, c7, c6, 2" : : "r" (val)); + asm volatile ("mcr p15, 0, %0, c7, c14, 2" : : "r" (val) : "memory"); } /* data cache clean by MVA to PoC */ static inline void dccmvac(unsigned long mva) { - asm volatile ("mcr p15, 0, %0, c7, c10, 1" : : "r" (mva)); + asm volatile ("mcr p15, 0, %0, c7, c10, 1" : : "r" (mva) : "memory"); } /* data cache invalidate by MVA to PoC */ static inline void dcimvac(unsigned long mva) { - asm volatile ("mcr p15, 0, %0, c7, c6, 1" : : "r" (mva)); + asm volatile ("mcr p15, 0, %0, c7, c6, 1" : : "r" (mva) : "memory"); +} + +/* data cache invalidate by set/way */ +static inline void dcisw(uint32_t val) +{ + asm volatile ("mcr p15, 0, %0, c7, c6, 2" : : "r" (val) : "memory"); } /* instruction cache invalidate all by PoU */ @@ -195,12 +223,12 @@ static inline void write_csselr(uint32_t val) static inline unsigned int read_sctlr(void) { unsigned int val; - asm volatile ("mrc p15, 0, %0, c1, c0, 0" : "=r" (val) : : "cc"); + asm volatile ("mrc p15, 0, %0, c1, c0, 0" : "=r" (val)); return val; } /* write system control register (SCTLR) */ -static inline void write_sctlr(unsigned int val) +static inline void write_sctlr(uint32_t val) { asm volatile ("mcr p15, 0, %0, c1, c0, 0" : : "r" (val) : "cc"); isb(); @@ -210,22 +238,48 @@ static inline void write_sctlr(unsigned int val) * Cache maintenance API */ -/* invalidate all TLBs */ -void tlb_invalidate_all(void); - -/* clean and invalidate entire dcache on current level (given by CCSELR) */ +/* dcache clean and invalidate all (on current level given by CCSELR) */ void dcache_clean_invalidate_all(void); -/* invalidate entire dcache on current level (given by CCSELR) */ -void dcache_invalidate_all(void); +/* dcache clean by modified virtual address to PoC */ +void dcache_clean_by_mva(unsigned long addr, unsigned long len); -/* invalidate and clean dcache by machine virtual address to PoC */ +/* dcache clean and invalidate by modified virtual address to PoC */ void dcache_clean_invalidate_by_mva(unsigned long addr, unsigned long len); -/* invalidate entire icache on current level (given by CSSELR) */ +/* dcache invalidate all (on current level given by CCSELR) */ +void dcache_invalidate_all(void); + +/* dcache and MMU disable */ +void dcache_mmu_disable(void); + +/* dcache and MMU enable */ +void dcache_mmu_enable(void); + +/* icache invalidate all (on current level given by CSSELR) */ void icache_invalidate_all(void); -/* MMU setup by machine virtual address */ -void mmu_setup_by_mva(unsigned long start, unsigned long size); +/* tlb invalidate all */ +void tlb_invalidate_all(void); + +/* + * Generalized setup/init functions + */ + +/* invalidate all caches on ARMv7 */ +void armv7_invalidate_caches(void); + +/* mmu initialization (set page table address, set permissions, etc) */ +void mmu_init(void); + +enum dcache_policy { + DCACHE_OFF, + DCACHE_WRITEBACK, + DCACHE_WRITETHROUGH, +}; + +/* mmu range configuration (set dcache policy) */ +void mmu_config_range(unsigned long start_mb, unsigned long size_mb, + enum dcache_policy policy); #endif /* ARMV7_CACHE_H */ |