diff options
Diffstat (limited to 'payloads/libpayload')
| -rw-r--r-- | payloads/libpayload/Kconfig | 19 | ||||
| -rw-r--r-- | payloads/libpayload/Makefile | 3 | ||||
| -rw-r--r-- | payloads/libpayload/Makefile.mk | 8 | ||||
| -rw-r--r-- | payloads/libpayload/arch/x86/Makefile.mk | 16 | ||||
| -rw-r--r-- | payloads/libpayload/arch/x86/exception.c | 58 | ||||
| -rw-r--r-- | payloads/libpayload/arch/x86/exception_asm_64.S | 221 | ||||
| -rw-r--r-- | payloads/libpayload/arch/x86/exec.c (renamed from payloads/libpayload/libc/exec.c) | 6 | ||||
| -rw-r--r-- | payloads/libpayload/arch/x86/gdb.c | 18 | ||||
| -rw-r--r-- | payloads/libpayload/arch/x86/head_64.S | 141 | ||||
| -rw-r--r-- | payloads/libpayload/arch/x86/libpayload.ldscript | 5 | ||||
| -rw-r--r-- | payloads/libpayload/arch/x86/pt.S | 149 | ||||
| -rw-r--r-- | payloads/libpayload/arch/x86/string.c | 11 | ||||
| -rwxr-xr-x | payloads/libpayload/bin/lpgcc | 11 | ||||
| -rw-r--r-- | payloads/libpayload/drivers/storage/ahci_common.c | 4 | ||||
| -rw-r--r-- | payloads/libpayload/drivers/usb/uhci.c | 2 | ||||
| -rw-r--r-- | payloads/libpayload/include/x86/arch/exception.h | 67 | ||||
| -rw-r--r-- | payloads/libpayload/libc/Makefile.mk | 2 | ||||
| -rw-r--r-- | payloads/libpayload/vboot/Makefile.mk | 3 |
18 files changed, 686 insertions, 58 deletions
diff --git a/payloads/libpayload/Kconfig b/payloads/libpayload/Kconfig index 39d316b1ab..0c07ecd998 100644 --- a/payloads/libpayload/Kconfig +++ b/payloads/libpayload/Kconfig @@ -106,17 +106,22 @@ menu "Architecture Options" choice prompt "Target Architecture" - default ARCH_X86 + default ARCH_X86_32 config ARCH_ARM bool "ARM" help Support the ARM architecture -config ARCH_X86 - bool "x86" +config ARCH_X86_32 + bool "x86_32" + help + Support the x86_32 architecture + +config ARCH_X86_64 + bool "x86_64" help - Support the x86 architecture + Support the x86_64 architecture config ARCH_ARM64 bool "ARM64" @@ -133,6 +138,12 @@ config ARCH_MOCK endchoice +config ARCH_X86 + bool + default y if ARCH_X86_32 || ARCH_X86_64 + help + Support the x86 architecture + config MULTIBOOT bool "Multiboot header support" depends on ARCH_X86 diff --git a/payloads/libpayload/Makefile b/payloads/libpayload/Makefile index 71d60bce9b..afe38dc6e7 100644 --- a/payloads/libpayload/Makefile +++ b/payloads/libpayload/Makefile @@ -118,7 +118,8 @@ ARCH-y := $(ARCHDIR-y) # override here. ARCH-$(CONFIG_LP_ARCH_ARM) := arm ARCH-$(CONFIG_LP_ARCH_ARM64) := arm64 -ARCH-$(CONFIG_LP_ARCH_X86) := x86_32 +ARCH-$(CONFIG_LP_ARCH_X86_32) := x86_32 +ARCH-$(CONFIG_LP_ARCH_X86_64) := x86_64 ARCH-$(CONFIG_LP_ARCH_MOCK) := mock # Five cases where we don't need fully populated $(obj) lists: diff --git a/payloads/libpayload/Makefile.mk b/payloads/libpayload/Makefile.mk index d654995889..d5a9a817fc 100644 --- a/payloads/libpayload/Makefile.mk +++ b/payloads/libpayload/Makefile.mk @@ -96,7 +96,7 @@ $(obj)/libpayload-config.h: $(KCONFIG_AUTOHEADER) $(obj)/libpayload.config cmp $@ $< 2>/dev/null || cp $< $@ library-targets = $(addsuffix .a,$(addprefix $(obj)/,$(libraries))) $(obj)/libpayload.a -lib: $$(library-targets) +lib: $$(library-targets) $(obj)/libpayload.ldscript extract_nth=$(word $(1), $(subst |, ,$(2))) @@ -115,11 +115,15 @@ $(obj)/%.a: $$(%-objs) printf " AR $(subst $(CURDIR)/,,$(@))\n" printf "create $@\n$(foreach objc,$(filter-out %.a,$^),addmod $(objc)\n)$(foreach lib,$(filter %.a,$^),addlib $(lib)\n)save\nend\n" | $(AR) -M +$(obj)/libpayload.ldscript: arch/$(ARCHDIR-y)/libpayload.ldscript + @printf " LDSCRIPT $@\n" + $(CC) $(CFLAGS) $(EXTRA_CFLAGS) -E -P -x assembler-with-cpp -undef -o $@ $< + install: real-target printf " INSTALL $(DESTDIR)/libpayload/lib\n" install -m 755 -d $(DESTDIR)/libpayload/lib install -m 644 $(library-targets) $(DESTDIR)/libpayload/lib/ - install -m 644 arch/$(ARCHDIR-y)/libpayload.ldscript $(DESTDIR)/libpayload/lib/ + install -m 644 $(obj)/libpayload.ldscript $(DESTDIR)/libpayload/lib/ install -m 755 -d $(DESTDIR)/libpayload/lib/$(ARCHDIR-y) printf " INSTALL $(DESTDIR)/libpayload/include\n" install -m 755 -d $(DESTDIR)/libpayload/include diff --git a/payloads/libpayload/arch/x86/Makefile.mk b/payloads/libpayload/arch/x86/Makefile.mk index d2c33b4fb8..35e69cbe4d 100644 --- a/payloads/libpayload/arch/x86/Makefile.mk +++ b/payloads/libpayload/arch/x86/Makefile.mk @@ -27,16 +27,26 @@ ## ifneq ($(CONFIG_LP_COMPILER_LLVM_CLANG),y) +ifeq ($(CONFIG_LP_ARCH_X86_64),y) +CFLAGS += -mpreferred-stack-boundary=4 +else CFLAGS += -mpreferred-stack-boundary=2 endif +endif -libc-y += head.S +libc-$(CONFIG_LP_ARCH_X86_32) += head.S +libc-$(CONFIG_LP_ARCH_X86_64) += head_64.S +libc-$(CONFIG_LP_ARCH_X86_64) += pt.S libc-y += main.c sysinfo.c libc-y += timer.c coreboot.c util.S -libc-y += exec.S virtual.c +libc-y += virtual.c libc-y += selfboot.c cache.c -libc-y += exception_asm.S exception.c +libc-y += exception.c libc-y += delay.c +libc-$(CONFIG_LP_ARCH_X86_32) += exec.c +libc-$(CONFIG_LP_ARCH_X86_32) += exec.S +libc-$(CONFIG_LP_ARCH_X86_32) += exception_asm.S +libc-$(CONFIG_LP_ARCH_X86_64) += exception_asm_64.S # Will fall back to default_memXXX() in libc/memory.c if GPL not allowed. libc-$(CONFIG_LP_GPL) += string.c diff --git a/payloads/libpayload/arch/x86/exception.c b/payloads/libpayload/arch/x86/exception.c index 23d1b05b9a..a43a993220 100644 --- a/payloads/libpayload/arch/x86/exception.c +++ b/payloads/libpayload/arch/x86/exception.c @@ -34,7 +34,13 @@ #define IF_FLAG (1 << 9) -u32 exception_stack[0x400] __attribute__((aligned(8))); +#if CONFIG(LP_ARCH_X86_64) +#define REGISTER_FMT "0x%016zx" +#else +#define REGISTER_FMT "0x%08zx" +#endif + +u8 exception_stack[0x400] __aligned(16); static interrupt_handler handlers[256]; @@ -143,17 +149,27 @@ static void dump_exception_state(void) break; } printf("\n"); - printf("EIP: 0x%08x\n", exception_state->regs.eip); + printf("REG_IP: " REGISTER_FMT "\n", exception_state->regs.reg_ip); + printf("REG_FLAGS: " REGISTER_FMT "\n", exception_state->regs.reg_flags); + printf("REG_AX: " REGISTER_FMT "\n", exception_state->regs.reg_ax); + printf("REG_BX: " REGISTER_FMT "\n", exception_state->regs.reg_bx); + printf("REG_CX: " REGISTER_FMT "\n", exception_state->regs.reg_cx); + printf("REG_DX: " REGISTER_FMT "\n", exception_state->regs.reg_dx); + printf("REG_SP: " REGISTER_FMT "\n", exception_state->regs.reg_sp); + printf("REG_BP: " REGISTER_FMT "\n", exception_state->regs.reg_bp); + printf("REG_SI: " REGISTER_FMT "\n", exception_state->regs.reg_si); + printf("REG_DI: " REGISTER_FMT "\n", exception_state->regs.reg_di); +#if CONFIG(LP_ARCH_X86_64) + printf("REG_R8: 0x%016zx\n", exception_state->regs.reg_r8); + printf("REG_R9: 0x%016zx\n", exception_state->regs.reg_r9); + printf("REG_R10: 0x%016zx\n", exception_state->regs.reg_r10); + printf("REG_R11: 0x%016zx\n", exception_state->regs.reg_r11); + printf("REG_R12: 0x%016zx\n", exception_state->regs.reg_r12); + printf("REG_R13: 0x%016zx\n", exception_state->regs.reg_r13); + printf("REG_R14: 0x%016zx\n", exception_state->regs.reg_r14); + printf("REG_R15: 0x%016zx\n", exception_state->regs.reg_r15); +#endif printf("CS: 0x%04x\n", exception_state->regs.cs); - printf("EFLAGS: 0x%08x\n", exception_state->regs.eflags); - printf("EAX: 0x%08x\n", exception_state->regs.eax); - printf("ECX: 0x%08x\n", exception_state->regs.ecx); - printf("EDX: 0x%08x\n", exception_state->regs.edx); - printf("EBX: 0x%08x\n", exception_state->regs.ebx); - printf("ESP: 0x%08x\n", exception_state->regs.esp); - printf("EBP: 0x%08x\n", exception_state->regs.ebp); - printf("ESI: 0x%08x\n", exception_state->regs.esi); - printf("EDI: 0x%08x\n", exception_state->regs.edi); printf("DS: 0x%04x\n", exception_state->regs.ds); printf("ES: 0x%04x\n", exception_state->regs.es); printf("SS: 0x%04x\n", exception_state->regs.ss); @@ -164,7 +180,7 @@ static void dump_exception_state(void) void exception_dispatch(void) { die_if(exception_state->vector >= ARRAY_SIZE(handlers), - "Invalid vector %u\n", exception_state->vector); + "Invalid vector %zu\n", exception_state->vector); u8 vec = exception_state->vector; @@ -184,7 +200,7 @@ void exception_dispatch(void) vec); dump_exception_state(); - dump_stack(exception_state->regs.esp, 512); + dump_stack(exception_state->regs.reg_sp, 512); /* We don't call apic_eoi because we don't want to ack the interrupt and allow another interrupt to wake the processor. */ halt(); @@ -197,6 +213,10 @@ success: void exception_init(void) { + /* TODO: Add exception init code for x64, currently only supporting 32-bit code */ + if (CONFIG(LP_ARCH_X86_64)) + return; + exception_stack_end = exception_stack + ARRAY_SIZE(exception_stack); exception_init_asm(); } @@ -206,6 +226,17 @@ void set_interrupt_handler(u8 vector, interrupt_handler handler) handlers[vector] = handler; } +#if CONFIG(LP_ARCH_X86_64) +static uint64_t eflags(void) +{ + uint64_t eflags; + asm volatile( + "pushfq\n\t" + "popq %0\n\t" + : "=rm" (eflags)); + return eflags; +} +#else static uint32_t eflags(void) { uint32_t eflags; @@ -215,6 +246,7 @@ static uint32_t eflags(void) : "=rm" (eflags)); return eflags; } +#endif void enable_interrupts(void) { diff --git a/payloads/libpayload/arch/x86/exception_asm_64.S b/payloads/libpayload/arch/x86/exception_asm_64.S new file mode 100644 index 0000000000..ee4e9e5b76 --- /dev/null +++ b/payloads/libpayload/arch/x86/exception_asm_64.S @@ -0,0 +1,221 @@ +/* + * + * Copyright 2024 Google Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + + .align 16 + .global exception_stack_end +exception_stack_end: + .quad 0 + .global exception_state +exception_state: + .quad 0 + +/* Some temporary variables which are used while saving exception state. */ +vector: + .quad 0 +error_code: + .quad 0 +old_rsp: + .quad 0 +old_rax: + .quad 0 + + .align 16 + +/* + * Each exception vector has a small stub associated with it which sets aside + * the error code, if any, records which vector we entered from, and calls + * the common exception entry point. Some exceptions have error codes and some + * don't, so we have a macro for each type. + */ + + .macro stub num +exception_stub_\num: + movq $0, error_code + movq $\num, vector + jmp exception_common + .endm + + .macro stub_err num +exception_stub_\num: + pop error_code + movq $\num, vector + jmp exception_common + .endm + + .altmacro + .macro user_defined_stubs from, to + stub \from + .if \to-\from + user_defined_stubs %(from+1),\to + .endif + .endm + + stub 0 + stub 1 + stub 2 + stub 3 + stub 4 + stub 5 + stub 6 + stub 7 + stub_err 8 + stub 9 + stub_err 10 + stub_err 11 + stub_err 12 + stub_err 13 + stub_err 14 + stub 15 + stub 16 + stub_err 17 + stub 18 + stub 19 + stub 20 + stub 21 + stub 22 + stub 23 + stub 24 + stub 25 + stub 26 + stub 27 + stub 28 + stub 29 + stub_err 30 + stub 31 + /* Split the macro so we avoid a stack overflow. */ + user_defined_stubs 32, 63 + user_defined_stubs 64, 127 + user_defined_stubs 128, 191 + user_defined_stubs 192, 255 + +exception_common: + + /* Return from the exception. */ + iretl + +/* + * We need segment selectors for the IDT, so we need to know where things are + * in the GDT. We set one up here which is pretty standard and largely copied + * from coreboot. + */ + .align 16 +gdt: + /* selgdt 0, unused */ + .word 0x0000, 0x0000 + .byte 0x00, 0x00, 0x00, 0x00 + + /* selgdt 8, unused */ + .word 0x0000, 0x0000 + .byte 0x00, 0x00, 0x00, 0x00 + + /* selgdt 0x10, flat 4GB code segment */ + .word 0xffff, 0x0000 + .byte 0x00, 0x9b, 0xcf, 0x00 + + /* selgdt 0x18, flat 4GB data segment */ + .word 0xffff, 0x0000 + .byte 0x00, 0x92, 0xcf, 0x00 + + /* selgdt 0x20, flat x64 code segment */ + .word 0xffff, 0x0000 + .byte 0x00, 0x9b, 0xaf, 0x00 +gdt_end: + +/* GDT pointer for use with lgdt */ +.global gdt_ptr +gdt_ptr: + .word gdt_end - gdt - 1 + .quad gdt + + /* + * Record the target and construct the actual entry at init time. This + * is necessary because the linker doesn't want to construct the entry + * for us. + */ + .macro interrupt_gate target + .quad \target + .quad \target + .endm + + .altmacro + .macro user_defined_gates from, to + interrupt_gate exception_stub_\from + .if \to-\from + user_defined_gates %(from+1),\to + .endif + .endm + + .align 16 + .global idt +idt: + interrupt_gate exception_stub_0 + interrupt_gate exception_stub_1 + interrupt_gate exception_stub_2 + interrupt_gate exception_stub_3 + interrupt_gate exception_stub_4 + interrupt_gate exception_stub_5 + interrupt_gate exception_stub_6 + interrupt_gate exception_stub_7 + interrupt_gate exception_stub_8 + interrupt_gate exception_stub_9 + interrupt_gate exception_stub_10 + interrupt_gate exception_stub_11 + interrupt_gate exception_stub_12 + interrupt_gate exception_stub_13 + interrupt_gate exception_stub_14 + interrupt_gate exception_stub_15 + interrupt_gate exception_stub_16 + interrupt_gate exception_stub_17 + interrupt_gate exception_stub_18 + interrupt_gate exception_stub_19 + interrupt_gate exception_stub_20 + interrupt_gate exception_stub_21 + interrupt_gate exception_stub_22 + interrupt_gate exception_stub_23 + interrupt_gate exception_stub_24 + interrupt_gate exception_stub_25 + interrupt_gate exception_stub_26 + interrupt_gate exception_stub_27 + interrupt_gate exception_stub_28 + interrupt_gate exception_stub_29 + interrupt_gate exception_stub_30 + interrupt_gate exception_stub_31 + user_defined_gates 32, 63 + user_defined_gates 64, 127 + user_defined_gates 128, 191 + user_defined_gates 192, 255 +idt_end: + +/* IDT pointer for use with lidt */ +idt_ptr: + .word idt_end - idt - 1 + .quad idt + + .global exception_init_asm +exception_init_asm: + ret diff --git a/payloads/libpayload/libc/exec.c b/payloads/libpayload/arch/x86/exec.c index ae75a3e510..06a5ed2bcd 100644 --- a/payloads/libpayload/libc/exec.c +++ b/payloads/libpayload/arch/x86/exec.c @@ -29,8 +29,10 @@ #include <libpayload-config.h> #include <libpayload.h> -#if CONFIG(LP_ARCH_X86) +#if CONFIG(LP_ARCH_X86_32) extern void i386_do_exec(long addr, int argc, char **argv, int *ret); +#else +#error "exec does not currently support x86_64." #endif /** @@ -45,7 +47,7 @@ int exec(long addr, int argc, char **argv) { int val = -1; -#if CONFIG(LP_ARCH_X86) +#if CONFIG(LP_ARCH_X86_32) i386_do_exec(addr, argc, argv, &val); #endif return val; diff --git a/payloads/libpayload/arch/x86/gdb.c b/payloads/libpayload/arch/x86/gdb.c index 7d2951282e..ad249b82da 100644 --- a/payloads/libpayload/arch/x86/gdb.c +++ b/payloads/libpayload/arch/x86/gdb.c @@ -15,6 +15,7 @@ #include <exception.h> #include <gdb.h> #include <libpayload.h> +#include <stddef.h> static const u8 type_to_signal[] = { [EXC_DE] = GDB_SIGFPE, @@ -53,12 +54,15 @@ void gdb_arch_init(void) void gdb_arch_enter(void) { - u32 *esp; - - asm volatile ("mov %%esp, %0" : "=r"(esp) ); + u8 *stack_pointer; +#if CONFIG(LP_ARCH_X86_64) + asm volatile ("movq %%rsp, %0" : "=r"(stack_pointer)); +#else + asm volatile ("mov %%esp, %0" : "=r"(stack_pointer)); +#endif /* Avoid reentrant exceptions, just call the hook if in one already. */ - if (esp >= exception_stack && esp <= exception_stack_end) + if (stack_pointer >= exception_stack && stack_pointer <= exception_stack_end) gdb_exception_hook(EXC_BP); else asm volatile ("int3"); @@ -66,12 +70,12 @@ void gdb_arch_enter(void) int gdb_arch_set_single_step(int on) { - const u32 tf_bit = 1 << 8; + const size_t tf_bit = 1 << 8; if (on) - exception_state->regs.eflags |= tf_bit; + exception_state->regs.reg_flags |= tf_bit; else - exception_state->regs.eflags &= ~tf_bit; + exception_state->regs.reg_flags &= ~tf_bit; return 0; } diff --git a/payloads/libpayload/arch/x86/head_64.S b/payloads/libpayload/arch/x86/head_64.S new file mode 100644 index 0000000000..65242749fa --- /dev/null +++ b/payloads/libpayload/arch/x86/head_64.S @@ -0,0 +1,141 @@ +/* + * + * Copyright (C) 2024 Google Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#define IA32_EFER 0xC0000080 +#define EFER_LME (1 << 8) + + .code32 + .global _entry + .section .text._entry + .align 4 + +/* + * WARNING: Critical Code Section - 32/64-bit Compatibility + * This code between `_entry` and `jnz _init64` is executed during system initialization. + * It MUST function correctly regardless of whether the system is booting in: + * - 32-bit protected mode + * - 64-bit long mode + * To achieve this, ONLY use instructions that produce identical binary output in both modes. + * Thoroughly test ALL modifications to this section in BOTH 32-bit and 64-bit boot + * environments. + */ + +_entry: + + /* Add multiboot header and jump around it when building with multiboot support. */ +#if CONFIG(LP_MULTIBOOT) + #include "multiboot_header.inc" +#endif + /* No interrupts, please. */ + cli + + movl $IA32_EFER, %ecx + rdmsr + testl $EFER_LME, %eax + jnz _init64 + + lgdt %cs:gdt_ptr + + /* save pointer to coreboot tables */ + movl 4(%esp), %eax + /* + * NOTE: coreboot tables has passed over the top of the stack + * while calling in protected mode. + */ + movl %eax, cb_header_ptr + + call init_page_table + movl $pm4le, %eax + + /* load identity mapped page tables */ + movl %eax, %cr3 + + /* enable PAE */ + movl %cr4, %eax + btsl $5, %eax + movl %eax, %cr4 + + /* enable long mode */ + movl $(IA32_EFER), %ecx + rdmsr + btsl $8, %eax + wrmsr + + /* enable paging */ + movl %cr0, %eax + btsl $31, %eax + movl %eax, %cr0 + + /* Jump to selgdt 0x20, flat x64 code segment */ + ljmp $0x20, $_entry64 + +.code64 +.align 16 +_init64: + movabs $gdt_ptr, %rax + lgdt (%rax) + + /* + * Note: The `cb_header_ptr` has passed as the first argument + * to the x86-64 calling convention. + */ + movq %rdi, cb_header_ptr + + call init_page_table + movq $pm4le, %rax + + /* load identity mapped page tables */ + movq %rax, %cr3 + +_entry64: + /* Store current stack pointer and set up new stack. */ + movq %rsp, %rax + movabs $_estack, %rsp + + push %rax + + fninit + movq %cr0, %rax + andq $0xFFFFFFFFFFFFFFFB, %rax /* clear EM */ + orq $0x00000022, %rax /* set MP, NE */ + movq %rax, %cr0 + + movq %cr4, %rax + orq $0x00000600, %rax /* set OSFXSR, OSXMMEXCPT */ + movq %rax, %cr4 + + /* Let's rock. */ + call start_main + + /* %rax has the return value - pass it on unmolested */ +_leave: + /* Restore old stack. */ + pop %rsp + + /* Return to the original context. */ + ret diff --git a/payloads/libpayload/arch/x86/libpayload.ldscript b/payloads/libpayload/arch/x86/libpayload.ldscript index 0f27ed9233..49c7e37f55 100644 --- a/payloads/libpayload/arch/x86/libpayload.ldscript +++ b/payloads/libpayload/arch/x86/libpayload.ldscript @@ -26,8 +26,13 @@ * SUCH DAMAGE. */ +#if CONFIG(LP_ARCH_X86_64) +OUTPUT_FORMAT(elf64-x86-64) +OUTPUT_ARCH(x86_64) +#else OUTPUT_FORMAT(elf32-i386) OUTPUT_ARCH(i386) +#endif ENTRY(_entry) diff --git a/payloads/libpayload/arch/x86/pt.S b/payloads/libpayload/arch/x86/pt.S new file mode 100644 index 0000000000..9b085e53ff --- /dev/null +++ b/payloads/libpayload/arch/x86/pt.S @@ -0,0 +1,149 @@ +/* + * + * Copyright 2024 Google Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +/* + * For reference see "AMD64 Architecture Programmer's Manual Volume 2", + * Document 24593-Rev. 3.31-July 2019 Chapter 5.3.4 + * + * Page table attributes: WB, User+Supervisor, Present, Writeable, Accessed, Dirty + */ + +.section .bss +#define _PRES (1ULL << 0) +#define _RW (1ULL << 1) +#define _US (1ULL << 2) +#define _A (1ULL << 5) +#define _D (1ULL << 6) +#define _PS (1ULL << 7) + +.section .bss.pm4le +.global pm4le +.align 4096 +pm4le: +.skip 8 + +.section .bss.main_page_table +.global main_page_table +.align 4096 +main_page_table: +.skip 8192 + +.section .bss.extra_page_table +.global extra_page_table +.align 4096 +extra_page_table: +.skip 32 + +/* + * WARNING: 32-bit/64-bit Mode Compatibility for Page Table Initialization + * This `init_page_table` function is designed to work in both 32-bit protected + * mode AND 64-bit long mode. + * + * Key Considerations: + * - Assembly Instructions: Use ONLY instructions that have the SAME binary representation + * in both 32-bit and 64-bit modes. + * - `.code64` Directive: We're compiling with `.code64` to ensure the assembler uses + * the correct 64-bit version of instructions (e.g., `inc`). + * - Register Notation: + * - Use 64-bit register names (like `%rsi`) for register-indirect addressing to avoid + * incorrect address size prefixes. + * - It's safe to use `%esi` with `mov` instructions, as the high 32 bits are zeroed + * in 64-bit mode. + * + * IMPORTANT: + * Thoroughly test ANY changes to this function in BOTH 32-bit and 64-bit boot environments. + */ + +.code64 +.section .text.init_page_table +.globl init_page_table +.type init_page_table, @function + +init_page_table: + mov $0x80000001, %eax + cpuid + test $(1 << 26), %edx + jnz setup_1gb + +setup_2mb: + mov $2048, %edi + mov $(_PRES + _RW + _US + _PS + _A + _D), %eax + mov $0, %ecx + mov $main_page_table, %esi + +loop_2mb: + mov %eax, (%rsi, %rcx, 8) + mov $0, 4(%rsi, %rcx, 8) + add $0x200000, %eax + inc %ecx + cmp %edi, %ecx + jb loop_2mb + + mov $4, %edi + mov $main_page_table, %eax + add $(_PRES + _RW + _US + _A), %eax + mov $0, %ecx + mov $extra_page_table, %esi + +fill_extra_page_table: + mov %eax, (%rsi, %rcx, 8) + mov $0, 4(%rsi, %rcx, 8) + add $4096, %eax + inc %ecx + cmp %edi, %ecx + jb fill_extra_page_table + + mov $extra_page_table, %eax + jmp leave + +setup_1gb: + mov $512, %edi + mov $(_PRES + _RW + _US + _PS + _A + _D), %eax + mov $0, %ebx + mov $0, %ecx + mov $main_page_table, %esi + +loop_1gb: + mov %eax, (%rsi, %rcx, 8) + mov %ebx, 4(%rsi, %rcx, 8) + add $0x40000000, %eax + cmp $0x40000000, %eax + ja no_overflow_1gb + inc %ebx +no_overflow_1gb: + inc %ecx + cmp %edi, %ecx + jb loop_1gb + + mov $main_page_table, %eax + +leave: + or $(_PRES + _RW + _US + _A), %eax + mov %eax, pm4le + + ret diff --git a/payloads/libpayload/arch/x86/string.c b/payloads/libpayload/arch/x86/string.c index 836d049831..11bcae7acb 100644 --- a/payloads/libpayload/arch/x86/string.c +++ b/payloads/libpayload/arch/x86/string.c @@ -81,6 +81,16 @@ void *memcpy(void *dest, const void *src, size_t n) { unsigned long d0, d1, d2; +#if CONFIG(LP_ARCH_X86_64) + asm volatile( + "rep ; movsq\n\t" + "mov %4,%%rcx\n\t" + "rep ; movsb\n\t" + : "=&c" (d0), "=&D" (d1), "=&S" (d2) + : "0" (n >> 3), "g" (n & 7), "1" (dest), "2" (src) + : "memory" + ); +#else asm volatile( "rep ; movsl\n\t" "movl %4,%%ecx\n\t" @@ -89,6 +99,7 @@ void *memcpy(void *dest, const void *src, size_t n) : "0" (n >> 2), "g" (n & 3), "1" (dest), "2" (src) : "memory" ); +#endif return dest; } diff --git a/payloads/libpayload/bin/lpgcc b/payloads/libpayload/bin/lpgcc index a954d4ff5b..08fcc297a5 100755 --- a/payloads/libpayload/bin/lpgcc +++ b/payloads/libpayload/bin/lpgcc @@ -84,7 +84,11 @@ if [ "$CONFIG_LP_ARCH_ARM64" = "y" ]; then fi if [ "$CONFIG_LP_ARCH_X86" = "y" ]; then _ARCHDIR=x86 - _ARCHEXTRA="-m32 " + if [ "$CONFIG_LP_ARCH_X86_32" = "y" ]; then + _ARCHEXTRA="-m32 " + else + _ARCHEXTRA="-m64 " + fi fi if [ "$CONFIG_LP_ARCH_MOCK" = "y" ]; then _ARCHDIR=mock @@ -122,13 +126,10 @@ CMDLINE= while [ $# -gt 0 ]; do case $1 in - -m32|-fno-stack-protector) + -m32|-fno-stack-protector|-m64) shift continue ;; - -m64) - error "Invalid option --64 - only 32 bit architectures are supported" - ;; -c) DOLINK=0 ;; diff --git a/payloads/libpayload/drivers/storage/ahci_common.c b/payloads/libpayload/drivers/storage/ahci_common.c index f3abc5f7fb..abc67c3656 100644 --- a/payloads/libpayload/drivers/storage/ahci_common.c +++ b/payloads/libpayload/drivers/storage/ahci_common.c @@ -66,7 +66,7 @@ static u8 *ahci_prdbuf_init(ahci_dev_t *const dev, u8 *const user_buf, const size_t len, const int out) { - if ((u32)user_buf & 1) { + if ((uintptr_t)user_buf & 1) { printf("ahci: Odd buffer pointer (%p).\n", user_buf); if (dev->buf) /* orphaned buffer */ free(dev->buf - *(dev->buf - 1)); @@ -76,7 +76,7 @@ static u8 *ahci_prdbuf_init(ahci_dev_t *const dev, dev->user_buf = user_buf; dev->write_back = !out; dev->buflen = len; - if ((u32)dev->buf & 1) { + if ((uintptr_t)dev->buf & 1) { dev->buf[0] = 1; dev->buf += 1; } else { diff --git a/payloads/libpayload/drivers/usb/uhci.c b/payloads/libpayload/drivers/usb/uhci.c index eb252cd8ad..7590ab3c85 100644 --- a/payloads/libpayload/drivers/usb/uhci.c +++ b/payloads/libpayload/drivers/usb/uhci.c @@ -274,7 +274,7 @@ uhci_stop(hci_t *controller) #define UHCI_SLEEP_TIME_US 30 #define UHCI_TIMEOUT (USB_MAX_PROCESSING_TIME_US / UHCI_SLEEP_TIME_US) -#define GET_TD(x) ((void*)(((unsigned int)(x))&~0xf)) +#define GET_TD(x) ((void *)(((unsigned long)(x))&~0xf)) static td_t * wait_for_completed_qh(hci_t *controller, qh_t *qh) diff --git a/payloads/libpayload/include/x86/arch/exception.h b/payloads/libpayload/include/x86/arch/exception.h index d88029b39e..76099f2b71 100644 --- a/payloads/libpayload/include/x86/arch/exception.h +++ b/payloads/libpayload/include/x86/arch/exception.h @@ -29,6 +29,7 @@ #ifndef _ARCH_EXCEPTION_H #define _ARCH_EXCEPTION_H +#include <stddef.h> #include <stdint.h> void exception_init_asm(void); @@ -38,20 +39,28 @@ void disable_interrupts(void); /** Returns 1 if interrupts are enabled. */ int interrupts_enabled(void); -struct exception_state -{ +#if CONFIG(LP_ARCH_X86_64) +struct exception_state { /* Careful: x86/gdb.c currently relies on the size and order of regs. */ struct { - u32 eax; - u32 ecx; - u32 edx; - u32 ebx; - u32 esp; - u32 ebp; - u32 esi; - u32 edi; - u32 eip; - u32 eflags; + size_t reg_ax; + size_t reg_bx; + size_t reg_cx; + size_t reg_dx; + size_t reg_si; + size_t reg_di; + size_t reg_bp; + size_t reg_sp; + size_t reg_r8; + size_t reg_r9; + size_t reg_r10; + size_t reg_r11; + size_t reg_r12; + size_t reg_r13; + size_t reg_r14; + size_t reg_r15; + size_t reg_ip; + size_t reg_flags; u32 cs; u32 ss; u32 ds; @@ -59,13 +68,39 @@ struct exception_state u32 fs; u32 gs; } regs; - u32 error_code; - u32 vector; + size_t error_code; + size_t vector; } __packed; +#else +struct exception_state { + /* Careful: x86/gdb.c currently relies on the size and order of regs. */ + struct { + size_t reg_ax; + size_t reg_cx; + size_t reg_dx; + size_t reg_bx; + size_t reg_sp; + size_t reg_bp; + size_t reg_si; + size_t reg_di; + size_t reg_ip; + size_t reg_flags; + u32 cs; + u32 ss; + u32 ds; + u32 es; + u32 fs; + u32 gs; + } regs; + size_t error_code; + size_t vector; +} __packed; +#endif + extern struct exception_state *exception_state; -extern u32 exception_stack[]; -extern u32 *exception_stack_end; +extern u8 exception_stack[]; +extern u8 *exception_stack_end; enum { EXC_DE = 0, /* Divide by zero */ diff --git a/payloads/libpayload/libc/Makefile.mk b/payloads/libpayload/libc/Makefile.mk index 2840476619..c9fc17a7b5 100644 --- a/payloads/libpayload/libc/Makefile.mk +++ b/payloads/libpayload/libc/Makefile.mk @@ -29,7 +29,7 @@ libc-$(CONFIG_LP_LIBC) += malloc.c printf.c console.c string.c libc-$(CONFIG_LP_LIBC) += memory.c ctype.c lib.c libgcc.c -libc-$(CONFIG_LP_LIBC) += rand.c time.c exec.c +libc-$(CONFIG_LP_LIBC) += rand.c time.c libc-$(CONFIG_LP_LIBC) += readline.c getopt_long.c sysinfo.c libc-$(CONFIG_LP_LIBC) += args.c libc-$(CONFIG_LP_LIBC) += strlcpy.c diff --git a/payloads/libpayload/vboot/Makefile.mk b/payloads/libpayload/vboot/Makefile.mk index 554fec864f..15af9807fe 100644 --- a/payloads/libpayload/vboot/Makefile.mk +++ b/payloads/libpayload/vboot/Makefile.mk @@ -15,7 +15,8 @@ vboot-fixup-includes = $(filter -I$(coreboottop)/%, $(1)) \ $(filter-out -I$(coreboottop)/%,$(1))))) VBOOT_FIRMWARE_ARCH-$(CONFIG_LP_ARCH_ARM) := arm -VBOOT_FIRMWARE_ARCH-$(CONFIG_LP_ARCH_X86) := x86 +VBOOT_FIRMWARE_ARCH-$(CONFIG_LP_ARCH_X86_32) := x86 +VBOOT_FIRMWARE_ARCH-$(CONFIG_LP_ARCH_X86_64) := x86_64 VBOOT_FIRMWARE_ARCH-$(CONFIG_LP_ARCH_ARM64) := arm64 ifneq ($(CONFIG_LP_ARCH_MOCK),) |