soc/intel/xeon_sp: Reserve MMIO high range

Xeon-SP supports MMIO high range, a.k.a. MMIO range above 4G. FSP will
assign domain MMIO high windows from this range.

However, there will be unassigned parts among these high windows for
non-domain device usage (e.g. misc devices belonging to an IIO stack
but not belonged to any PCIe domains under that stack). This will cause
segmentation in MTRR UC coverage.

For example, in SPR-XCC where only CPM0/HQM0 are supported and
instantiated to PCIe domains, MMIO ranges are still reserved for
CPM1/HQM1. See more at src/soc/intel/xeon_sp/spr/ioat.c.

Reserve MMIO high range as a whole under domain0/00:0.0. During MTRR
calculation, this reservation will connect the discontinued domain MMIO
high windows together to form one continuous range, and save MTRR
register usage from inadequacy.

This change is initially raised for SPR but could be effective for GNR
as well.

TESTED = Build and boot in intel/archercity CRB, MTRR register usage
decreases from 7 to 3 in 2S system.

TESTED = Only setting MTRR for below 4GB ranges test fails with
LinuxBoot on SPR (through x86_setup_mtrrs_with_detect_no_above_4gb)

tsc: Detected 2000.000 MHz processor
last_pfn = 0x2080000 max_arch_pfn = 0x10000000000
x86/PAT: Configuration [0-7]: WB  WC  UC- UC  WB  WP  UC- WT
WARNING: BIOS bug: CPU MTRRs don't cover all of memory, losing 129024MB of RAM.
------------[ cut here ]------------
WARNING: CPU: 0 PID: 0 at arch/x86/kernel/cpu/mtrr/cleanup.c:978 mtrr_trim_uncached_memory+0x2b9/0x2f9
...
Call Trace:
 ? 0xffffffff8f600000
 ? setup_arch+0x4bb/0xaed
 ? printk+0x53/0x6a
 ? start_kernel+0x55/0x507
 ? load_ucode_intel_bsp+0x1c/0x4d
 ? secondary_startup_64_no_verify+0xc2/0xcb
random: get_random_bytes called from init_oops_id+0x1d/0x2c with crng_init=0
---[ end trace 0e56686fd458f0c5 ]---
update e820 for mtrr
modified physical RAM map:
modified: [mem 0x0000000000000000-0x0000000000000fff] reserved
...
modified: [mem 0x00000000ff000000-0x000000207fffffff] reserved
last_pfn = 0x6354e max_arch_pfn = 0x10000000000
Memory KASLR using RDRAND RDTSC...
x2apic: enabled by BIOS, switching to x2apic ops
Using GB pages for direct mapping
...
Initmem setup node 0 [mem 0x0000000000001000-0x000000006354dfff]
  DMA zone: 28769 pages in unavailable ranges
  DMA32 zone: 19122 pages in unavailable ranges
BUG: unable to handle page fault for address: ff24b56eba60cff8
BAD
Oops: 0000 [#1] SMP NOPTI
CPU: 0 PID: 0 Comm: swapper Tainted: G        W         5.10.50 #2
...
Call Trace:
 ? set_pte_vaddr_p4d+0x24/0x35
 ? __native_set_fixmap+0x21/0x28
 ? map_vsyscall+0x35/0x56
 ? setup_arch+0xa00/0xaed
 ? printk+0x53/0x6a
 ? start_kernel+0x55/0x507
 ? load_ucode_intel_bsp+0x1c/0x4d
 ? secondary_startup_64_no_verify+0xc2/0xcb
CR2: ff24b56eba60cff8
---[ end trace 0e56686fd458f0c6 ]---
RIP: 0010:fill_pud+0xa/0x62
...
Kernel panic - not syncing: Attempted to kill the idle task!
---[ end Kernel panic - not syncing: Attempted to kill the idle task! ]---

Change-Id: Ib2a0e1f1f13e797c1fab6aca589d060c4d3fa15b
Signed-off-by: Shuo Liu <shuo.liu@intel.com>
Reviewed-on: https://review.coreboot.org/c/coreboot/+/83538
Reviewed-by: Lean Sheng Tan <sheng.tan@9elements.com>
Tested-by: build bot (Jenkins) <no-reply@coreboot.org>
Reviewed-by: Nico Huber <nico.h@gmx.de>

6 files changed, 51 insertions, 2 deletions

diff --git a/src/soc/intel/xeon_sp/cpx/soc_util.c b/src/soc/intel/xeon_sp/cpx/soc_util.c
index 3472d589a8..aac5589073 100644
--- a/src/soc/intel/xeon_sp/cpx/soc_util.c
+++ b/src/soc/intel/xeon_sp/cpx/soc_util.c
@@ -141,3 +141,8 @@ uint8_t get_cxl_node_count(void)
 {
 	return 0;
 }
+
+bool get_mmio_high_base_size(resource_t *base, resource_t *size)
+{
+	return false;
+}
diff --git a/src/soc/intel/xeon_sp/gnr/soc_util.c b/src/soc/intel/xeon_sp/gnr/soc_util.c
index a65e5fcc52..7ea65c213c 100644
--- a/src/soc/intel/xeon_sp/gnr/soc_util.c
+++ b/src/soc/intel/xeon_sp/gnr/soc_util.c
@@ -148,3 +148,8 @@ bool is_memtype_processor_attached(uint16_t mem_type)
 {
 	return true;
 }
+
+bool get_mmio_high_base_size(resource_t *base, resource_t *size)
+{
+	return false;
+}
diff --git a/src/soc/intel/xeon_sp/include/soc/util.h b/src/soc/intel/xeon_sp/include/soc/util.h
index e694af3e3c..a7b98f3ae6 100644
--- a/src/soc/intel/xeon_sp/include/soc/util.h
+++ b/src/soc/intel/xeon_sp/include/soc/util.h
@@ -31,4 +31,6 @@ void bios_done_msr(void *unused);
 union p2sb_bdf soc_get_hpet_bdf(void);
 union p2sb_bdf soc_get_ioapic_bdf(void);
 
+bool get_mmio_high_base_size(resource_t *base, resource_t *size);
+
 #endif
diff --git a/src/soc/intel/xeon_sp/skx/soc_util.c b/src/soc/intel/xeon_sp/skx/soc_util.c
index 0ccacfb73c..0938260b0a 100644
--- a/src/soc/intel/xeon_sp/skx/soc_util.c
+++ b/src/soc/intel/xeon_sp/skx/soc_util.c
@@ -214,3 +214,8 @@ uint8_t get_cxl_node_count(void)
 {
 	return 0;
 }
+
+bool get_mmio_high_base_size(resource_t *base, resource_t *size)
+{
+	return false;
+}
diff --git a/src/soc/intel/xeon_sp/spr/soc_util.c b/src/soc/intel/xeon_sp/spr/soc_util.c
index 584532785e..6dd06acd50 100644
--- a/src/soc/intel/xeon_sp/spr/soc_util.c
+++ b/src/soc/intel/xeon_sp/spr/soc_util.c
@@ -188,3 +188,12 @@ bool is_memtype_processor_attached(uint16_t mem_type)
 	 */
 	return (mem_type < MemTypeCxlAccVolatileMem);
 }
+
+bool get_mmio_high_base_size(resource_t *base, resource_t *size)
+{
+	const IIO_UDS *hob = get_iio_uds();
+	*base = hob->PlatformData.PlatGlobalMmio64Base;
+	*size = hob->PlatformData.PlatGlobalMmio64Limit - (*base) + 1;
+
+	return true;
+}
diff --git a/src/soc/intel/xeon_sp/uncore.c b/src/soc/intel/xeon_sp/uncore.c
index 2e5023d04e..be5b38f9c3 100644
--- a/src/soc/intel/xeon_sp/uncore.c
+++ b/src/soc/intel/xeon_sp/uncore.c
@@ -151,6 +151,9 @@ static void configure_dpr(struct device *dev)
 	pci_write_config32(dev, VTD_LTDPR, dpr.raw);
 }
 
+#define MC_DRAM_RESOURCE_MMIO_HIGH	0x1000
+#define MC_DRAM_RESOURCE_ANON_START	0x1001
+
 /*
  * Host Memory Map:
  *
@@ -350,7 +353,7 @@ static void mc_add_dram_resources(struct device *dev, int *res_count)
 
 static void mmapvtd_read_resources(struct device *dev)
 {
-	int index = 0;
+	int index = MC_DRAM_RESOURCE_ANON_START;
 
 	/* Read standard PCI resources. */
 	pci_dev_read_resources(dev);
@@ -362,13 +365,33 @@ static void mmapvtd_read_resources(struct device *dev)
 	mc_add_dram_resources(dev, &index);
 }
 
+static void mmapvtd_set_resources(struct device *dev)
+{
+	/*
+	 * The MMIO high window has to be added in set_resources() instead of
+	 * read_resources(). Because adding in read_resources() would cause the
+	 * whole window to be reserved, and it couldn't be used for resource
+	 * allocation.
+	 */
+	if (is_domain0(dev->upstream->dev)) {
+		resource_t mmio64_base, mmio64_size;
+		if (get_mmio_high_base_size(&mmio64_base, &mmio64_size)) {
+			assert(!probe_resource(dev, MC_DRAM_RESOURCE_MMIO_HIGH));
+			fixed_mem_range_flags(dev, MC_DRAM_RESOURCE_MMIO_HIGH,
+					      mmio64_base, mmio64_size, IORESOURCE_STORED);
+		}
+	}
+
+	pci_dev_set_resources(dev);
+}
+
 static void mmapvtd_init(struct device *dev)
 {
 }
 
 static struct device_operations mmapvtd_ops = {
 	.read_resources    = mmapvtd_read_resources,
-	.set_resources     = pci_dev_set_resources,
+	.set_resources     = mmapvtd_set_resources,
 	.enable_resources  = pci_dev_enable_resources,
 	.init              = mmapvtd_init,
 	.ops_pci           = &soc_pci_ops,