From 42c5c6de9531c017b679d75ab56c9f7160cac23e Mon Sep 17 00:00:00 2001 From: Shuo Liu Date: Tue, 16 Jul 2024 18:41:28 +0800 Subject: soc/intel/xeon_sp: Reserve MMIO high range Xeon-SP supports MMIO high range, a.k.a. MMIO range above 4G. FSP will assign domain MMIO high windows from this range. However, there will be unassigned parts among these high windows for non-domain device usage (e.g. misc devices belonging to an IIO stack but not belonged to any PCIe domains under that stack). This will cause segmentation in MTRR UC coverage. For example, in SPR-XCC where only CPM0/HQM0 are supported and instantiated to PCIe domains, MMIO ranges are still reserved for CPM1/HQM1. See more at src/soc/intel/xeon_sp/spr/ioat.c. Reserve MMIO high range as a whole under domain0/00:0.0. During MTRR calculation, this reservation will connect the discontinued domain MMIO high windows together to form one continuous range, and save MTRR register usage from inadequacy. This change is initially raised for SPR but could be effective for GNR as well. TESTED = Build and boot in intel/archercity CRB, MTRR register usage decreases from 7 to 3 in 2S system. TESTED = Only setting MTRR for below 4GB ranges test fails with LinuxBoot on SPR (through x86_setup_mtrrs_with_detect_no_above_4gb) tsc: Detected 2000.000 MHz processor last_pfn = 0x2080000 max_arch_pfn = 0x10000000000 x86/PAT: Configuration [0-7]: WB WC UC- UC WB WP UC- WT WARNING: BIOS bug: CPU MTRRs don't cover all of memory, losing 129024MB of RAM. ------------[ cut here ]------------ WARNING: CPU: 0 PID: 0 at arch/x86/kernel/cpu/mtrr/cleanup.c:978 mtrr_trim_uncached_memory+0x2b9/0x2f9 ... Call Trace: ? 0xffffffff8f600000 ? setup_arch+0x4bb/0xaed ? printk+0x53/0x6a ? start_kernel+0x55/0x507 ? load_ucode_intel_bsp+0x1c/0x4d ? secondary_startup_64_no_verify+0xc2/0xcb random: get_random_bytes called from init_oops_id+0x1d/0x2c with crng_init=0 ---[ end trace 0e56686fd458f0c5 ]--- update e820 for mtrr modified physical RAM map: modified: [mem 0x0000000000000000-0x0000000000000fff] reserved ... modified: [mem 0x00000000ff000000-0x000000207fffffff] reserved last_pfn = 0x6354e max_arch_pfn = 0x10000000000 Memory KASLR using RDRAND RDTSC... x2apic: enabled by BIOS, switching to x2apic ops Using GB pages for direct mapping ... Initmem setup node 0 [mem 0x0000000000001000-0x000000006354dfff] DMA zone: 28769 pages in unavailable ranges DMA32 zone: 19122 pages in unavailable ranges BUG: unable to handle page fault for address: ff24b56eba60cff8 BAD Oops: 0000 [#1] SMP NOPTI CPU: 0 PID: 0 Comm: swapper Tainted: G W 5.10.50 #2 ... Call Trace: ? set_pte_vaddr_p4d+0x24/0x35 ? __native_set_fixmap+0x21/0x28 ? map_vsyscall+0x35/0x56 ? setup_arch+0xa00/0xaed ? printk+0x53/0x6a ? start_kernel+0x55/0x507 ? load_ucode_intel_bsp+0x1c/0x4d ? secondary_startup_64_no_verify+0xc2/0xcb CR2: ff24b56eba60cff8 ---[ end trace 0e56686fd458f0c6 ]--- RIP: 0010:fill_pud+0xa/0x62 ... Kernel panic - not syncing: Attempted to kill the idle task! ---[ end Kernel panic - not syncing: Attempted to kill the idle task! ]--- Change-Id: Ib2a0e1f1f13e797c1fab6aca589d060c4d3fa15b Signed-off-by: Shuo Liu Reviewed-on: https://review.coreboot.org/c/coreboot/+/83538 Reviewed-by: Lean Sheng Tan Tested-by: build bot (Jenkins) Reviewed-by: Nico Huber --- src/soc/intel/xeon_sp/cpx/soc_util.c | 5 +++++ src/soc/intel/xeon_sp/gnr/soc_util.c | 5 +++++ src/soc/intel/xeon_sp/include/soc/util.h | 2 ++ src/soc/intel/xeon_sp/skx/soc_util.c | 5 +++++ src/soc/intel/xeon_sp/spr/soc_util.c | 9 +++++++++ src/soc/intel/xeon_sp/uncore.c | 27 +++++++++++++++++++++++++-- 6 files changed, 51 insertions(+), 2 deletions(-) (limited to 'src/soc/intel/xeon_sp') diff --git a/src/soc/intel/xeon_sp/cpx/soc_util.c b/src/soc/intel/xeon_sp/cpx/soc_util.c index 3472d589a8..aac5589073 100644 --- a/src/soc/intel/xeon_sp/cpx/soc_util.c +++ b/src/soc/intel/xeon_sp/cpx/soc_util.c @@ -141,3 +141,8 @@ uint8_t get_cxl_node_count(void) { return 0; } + +bool get_mmio_high_base_size(resource_t *base, resource_t *size) +{ + return false; +} diff --git a/src/soc/intel/xeon_sp/gnr/soc_util.c b/src/soc/intel/xeon_sp/gnr/soc_util.c index a65e5fcc52..7ea65c213c 100644 --- a/src/soc/intel/xeon_sp/gnr/soc_util.c +++ b/src/soc/intel/xeon_sp/gnr/soc_util.c @@ -148,3 +148,8 @@ bool is_memtype_processor_attached(uint16_t mem_type) { return true; } + +bool get_mmio_high_base_size(resource_t *base, resource_t *size) +{ + return false; +} diff --git a/src/soc/intel/xeon_sp/include/soc/util.h b/src/soc/intel/xeon_sp/include/soc/util.h index e694af3e3c..a7b98f3ae6 100644 --- a/src/soc/intel/xeon_sp/include/soc/util.h +++ b/src/soc/intel/xeon_sp/include/soc/util.h @@ -31,4 +31,6 @@ void bios_done_msr(void *unused); union p2sb_bdf soc_get_hpet_bdf(void); union p2sb_bdf soc_get_ioapic_bdf(void); +bool get_mmio_high_base_size(resource_t *base, resource_t *size); + #endif diff --git a/src/soc/intel/xeon_sp/skx/soc_util.c b/src/soc/intel/xeon_sp/skx/soc_util.c index 0ccacfb73c..0938260b0a 100644 --- a/src/soc/intel/xeon_sp/skx/soc_util.c +++ b/src/soc/intel/xeon_sp/skx/soc_util.c @@ -214,3 +214,8 @@ uint8_t get_cxl_node_count(void) { return 0; } + +bool get_mmio_high_base_size(resource_t *base, resource_t *size) +{ + return false; +} diff --git a/src/soc/intel/xeon_sp/spr/soc_util.c b/src/soc/intel/xeon_sp/spr/soc_util.c index 584532785e..6dd06acd50 100644 --- a/src/soc/intel/xeon_sp/spr/soc_util.c +++ b/src/soc/intel/xeon_sp/spr/soc_util.c @@ -188,3 +188,12 @@ bool is_memtype_processor_attached(uint16_t mem_type) */ return (mem_type < MemTypeCxlAccVolatileMem); } + +bool get_mmio_high_base_size(resource_t *base, resource_t *size) +{ + const IIO_UDS *hob = get_iio_uds(); + *base = hob->PlatformData.PlatGlobalMmio64Base; + *size = hob->PlatformData.PlatGlobalMmio64Limit - (*base) + 1; + + return true; +} diff --git a/src/soc/intel/xeon_sp/uncore.c b/src/soc/intel/xeon_sp/uncore.c index 2e5023d04e..be5b38f9c3 100644 --- a/src/soc/intel/xeon_sp/uncore.c +++ b/src/soc/intel/xeon_sp/uncore.c @@ -151,6 +151,9 @@ static void configure_dpr(struct device *dev) pci_write_config32(dev, VTD_LTDPR, dpr.raw); } +#define MC_DRAM_RESOURCE_MMIO_HIGH 0x1000 +#define MC_DRAM_RESOURCE_ANON_START 0x1001 + /* * Host Memory Map: * @@ -350,7 +353,7 @@ static void mc_add_dram_resources(struct device *dev, int *res_count) static void mmapvtd_read_resources(struct device *dev) { - int index = 0; + int index = MC_DRAM_RESOURCE_ANON_START; /* Read standard PCI resources. */ pci_dev_read_resources(dev); @@ -362,13 +365,33 @@ static void mmapvtd_read_resources(struct device *dev) mc_add_dram_resources(dev, &index); } +static void mmapvtd_set_resources(struct device *dev) +{ + /* + * The MMIO high window has to be added in set_resources() instead of + * read_resources(). Because adding in read_resources() would cause the + * whole window to be reserved, and it couldn't be used for resource + * allocation. + */ + if (is_domain0(dev->upstream->dev)) { + resource_t mmio64_base, mmio64_size; + if (get_mmio_high_base_size(&mmio64_base, &mmio64_size)) { + assert(!probe_resource(dev, MC_DRAM_RESOURCE_MMIO_HIGH)); + fixed_mem_range_flags(dev, MC_DRAM_RESOURCE_MMIO_HIGH, + mmio64_base, mmio64_size, IORESOURCE_STORED); + } + } + + pci_dev_set_resources(dev); +} + static void mmapvtd_init(struct device *dev) { } static struct device_operations mmapvtd_ops = { .read_resources = mmapvtd_read_resources, - .set_resources = pci_dev_set_resources, + .set_resources = mmapvtd_set_resources, .enable_resources = pci_dev_enable_resources, .init = mmapvtd_init, .ops_pci = &soc_pci_ops, -- cgit v1.2.3