diff options
| -rw-r--r-- | payloads/libpayload/arch/x86/Makefile.inc | 2 | ||||
| -rw-r--r-- | payloads/libpayload/arch/x86/cache.c | 134 | ||||
| -rw-r--r-- | payloads/libpayload/drivers/udc/chipidea.c | 6 | ||||
| -rw-r--r-- | payloads/libpayload/include/x86/arch/cache.h | 18 | ||||
| -rw-r--r-- | payloads/libpayload/include/x86/arch/cpuid.h | 19 |
5 files changed, 169 insertions, 10 deletions
diff --git a/payloads/libpayload/arch/x86/Makefile.inc b/payloads/libpayload/arch/x86/Makefile.inc index e010329eba..911f9dc91e 100644 --- a/payloads/libpayload/arch/x86/Makefile.inc +++ b/payloads/libpayload/arch/x86/Makefile.inc @@ -32,7 +32,7 @@ head.o-y += head.S libc-y += main.c sysinfo.c libc-y += timer.c coreboot.c util.S libc-y += exec.S virtual.c -libc-y += selfboot.c +libc-y += selfboot.c cache.c libc-y += exception_asm.S exception.c libc-y += delay.c diff --git a/payloads/libpayload/arch/x86/cache.c b/payloads/libpayload/arch/x86/cache.c new file mode 100644 index 0000000000..9fb33bc2b8 --- /dev/null +++ b/payloads/libpayload/arch/x86/cache.c @@ -0,0 +1,134 @@ +/* + * + * Copyright 2022 Google LLC + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include <stdint.h> + +#include <arch/cache.h> +#include <arch/cpuid.h> + +unsigned int dcache_line_bytes(void) +{ + /* + * The value returned in EBX[15:8] is in 8-byte increments. + * Cache line size is EBX[15:8] * 8 + */ + return (cpuid_ebx(1) & 0xff00) >> 5; +} + +static inline int cpu_supports_wbnoinvd(void) +{ + return (cpuid_ebx(0x80000008) >> 9) & 1; +} + +static inline int cpu_supports_clwb(void) +{ + return (cpuid_ebx(7) >> 24) & 1; +} + +static inline int cpu_supports_clflushopt(void) +{ + return (cpuid_sub_leaf_ebx(7, 0) >> 23) & 1; +} + +static inline int cpu_supports_clflush(void) +{ + return (cpuid_ebx(1) >> 19) & 1; +} + +inline void dcache_invalidate_all(void) +{ + asm volatile("invd" ::: "memory"); +} + +inline void dcache_clean_invalidate_all(void) +{ + asm volatile("wbinvd" ::: "memory"); +} + +inline void dcache_clean_all(void) +{ + if (cpu_supports_wbnoinvd()) { + asm volatile( + "sfence\n\t" + "wbnoinvd\n\t" + ::: "memory"); + } else { + dcache_clean_invalidate_all(); + } +} + +void dcache_clean_by_mva(void const *addr, size_t len) +{ + unsigned long line, linesize; + + linesize = dcache_line_bytes(); + line = (uintptr_t)addr & ~(linesize - 1); + + if (cpu_supports_clwb()) { + asm volatile("sfence"); + while (line < (uintptr_t)addr + len) { + asm volatile("clwb (%0)" : : "r"(line) : "memory"); + line += linesize; + } + } else { + dcache_clean_invalidate_by_mva(addr, len); + } +} + +void dcache_invalidate_by_mva(void const *addr, size_t len) +{ + /* + * x86 doesn't have a "invalidate without clean" for a cache line, fall + * back to both. + */ + dcache_clean_invalidate_by_mva(addr, len); +} + +void dcache_clean_invalidate_by_mva(void const *addr, size_t len) +{ + unsigned long line, linesize; + + linesize = dcache_line_bytes(); + line = (uintptr_t)addr & ~(linesize - 1); + + if (cpu_supports_clflushopt()) { + asm volatile("sfence"); + while (line < (uintptr_t)addr + len) { + asm volatile("clflushopt (%0)" ::"r"(line) : "memory"); + line += linesize; + } + } else if (cpu_supports_clflush()) { + asm volatile("sfence"); + while (line < (uintptr_t)addr + len) { + asm volatile("clflush (%0)" : : "r"(line) : "memory"); + line += linesize; + } + } else { + dcache_clean_invalidate_all(); + } +} diff --git a/payloads/libpayload/drivers/udc/chipidea.c b/payloads/libpayload/drivers/udc/chipidea.c index 3df7d2409c..54c0ca9f2e 100644 --- a/payloads/libpayload/drivers/udc/chipidea.c +++ b/payloads/libpayload/drivers/udc/chipidea.c @@ -221,10 +221,10 @@ static void advance_endpoint(struct chipidea_pdata *p, int endpoint, int in_dir) job->tds = tds; job->td_count = td_count; - dcache_clean_by_mva(tds, sizeof(struct td) * td_count); - dcache_clean_by_mva(job->data, job->length); dcache_clean_by_mva(qh, sizeof(*qh)); + if (!dma_coherent(job->data)) + dcache_clean_by_mva(job->data, job->length); debug("priming EP %d-%d with %zx bytes starting at %x (%p)\n", endpoint, in_dir, job->length, tds[0].page0, job->data); @@ -240,7 +240,7 @@ static void handle_endpoint(struct usbdev_ctrl *this, int endpoint, int in_dir) struct job *job = SIMPLEQ_FIRST(&p->job_queue[endpoint][in_dir]); SIMPLEQ_REMOVE_HEAD(&p->job_queue[endpoint][in_dir], queue); - if (in_dir) + if (in_dir && !dma_coherent(job->data)) dcache_invalidate_by_mva(job->data, job->length); int length = job->length; diff --git a/payloads/libpayload/include/x86/arch/cache.h b/payloads/libpayload/include/x86/arch/cache.h index 22e5940962..8c915c34fb 100644 --- a/payloads/libpayload/include/x86/arch/cache.h +++ b/payloads/libpayload/include/x86/arch/cache.h @@ -31,15 +31,21 @@ #ifndef __ARCH_CACHE_H__ #define __ARCH_CACHE_H__ +#include <stddef.h> +#include <stdint.h> + +/* returns number of bytes per cache line */ +unsigned int dcache_line_bytes(void); + +void dcache_invalidate_all(void); +void dcache_clean_invalidate_all(void); +void dcache_clean_all(void); +void dcache_clean_by_mva(void const *addr, size_t len); +void dcache_invalidate_by_mva(void const *addr, size_t len); +void dcache_clean_invalidate_by_mva(void const *addr, size_t len); /* NOOPs mirroring ARM's cache API, since x86 devices usually cache snoop */ #define dmb() #define dsb() -#define dcache_clean_all() -#define dcache_clean_by_mva(addr, len) -#define dcache_invalidate_all() -#define dcache_invalidate_by_mva(addr, len) -#define dcache_clean_invalidate_all() -#define dcache_clean_invalidate_by_mva(addr, len) #define cache_sync_instructions() #endif diff --git a/payloads/libpayload/include/x86/arch/cpuid.h b/payloads/libpayload/include/x86/arch/cpuid.h index ddd606a072..b01646dac0 100644 --- a/payloads/libpayload/include/x86/arch/cpuid.h +++ b/payloads/libpayload/include/x86/arch/cpuid.h @@ -47,6 +47,25 @@ _declare_cpuid(edx) #undef _declare_cpuid +#define cpuid_sub_leaf(fn, sub_leaf, eax, ebx, ecx, edx) \ + asm("cpuid" : "=a"(eax), "=b"(ebx), "=c"(ecx), "=d"(edx) : "0"(fn), "1"(sub_leaf)) + +#define _declare_cpuid_sub_leaf(reg) \ + static inline unsigned int cpuid_sub_leaf_##reg( \ + unsigned int fn, unsigned int sub_leaf) \ + { \ + unsigned int eax, ebx, ecx, edx; \ + cpuid_sub_leaf(fn, sub_leaf, eax, ebx, ecx, edx); \ + return reg; \ + } + +_declare_cpuid_sub_leaf(eax) +_declare_cpuid_sub_leaf(ebx) +_declare_cpuid_sub_leaf(ecx) +_declare_cpuid_sub_leaf(edx) + +#undef _declare_cpuid_sub_leaf + static inline unsigned int cpuid_max(void) { return cpuid_eax(0); |