device: Add support for PCIe Resizable BARs

Section 7.8.6 of the PCIe spec (rev 4) indicates that some devices can
indicates support for "Resizable BARs" via a PCIe extended capability.

When support this capability is indicated by the device, the size of
each BAR is determined in a different way than the normal "moving
bits" method. Instead, a pair of capability and control registers is
allocated in config space for each BAR, which can be used to both
indicate the different sizes the device is capable of supporting for
the BAR (powers-of-2 number of bits from 20 [1 MiB] to 63 [8 EiB]), and
to also inform the device of the size that the allocator actually
reserved for the MMIO range.

This patch adds a Kconfig for a mainboard to select if it knows that it
will have a device that requires this support during PCI enumeration.
If so, there is a corresponding Kconfig to indicate the maximum number
of bits of address space to hand out to devices this way (again, limited
by what devices can support and each individual system may want to
support, but just like above, this number can range from 20 to 63) If
the device can support more bits than this Kconfig, the resource request
is truncated to the number indicated by this Kconfig.

BUG=b:214443809
TEST=compile (device with this capability not available yet),
also verify that no changes are seen in resource allocation for
google/brya0 before and after this change.

Signed-off-by: Tim Wawrzynczak <twawrzynczak@chromium.org>
Change-Id: I14fcbe0ef09fdc7f6061bcf7439d1160d3bc4abf
Reviewed-on: https://review.coreboot.org/c/coreboot/+/61215
Tested-by: build bot (Jenkins) <no-reply@coreboot.org>
Reviewed-by: Subrata Banik <subratabanik@google.com>

4 files changed, 183 insertions, 9 deletions

diff --git a/src/device/Kconfig b/src/device/Kconfig
index cd8d8e9873..8471d5d00e 100644
--- a/src/device/Kconfig
+++ b/src/device/Kconfig
@@ -632,6 +632,35 @@ config PCIEXP_L1_SUB_STATE
 	help
 	  Detect and enable ASPM on PCIe links.
 
+config PCIEXP_SUPPORT_RESIZABLE_BARS
+	prompt "Support PCIe Resizable BARs"
+	bool
+	depends on (ECAM_MMCONF_SUPPORT || PCI_IO_CFG_EXT)
+	default n
+	help
+	  When enabled, this will check PCIe devices for Resizable BAR support,
+	  and if found, will use this to discover the preferred BAR sizes of
+	  the device in preference over the traditional moving bits method. The
+	  amount of address space given out to devices in this manner (since
+	  it can range up to 8 EB) can be limited with the
+	  PCIEXP_DEFAULT_MAX_RESIZABLE_BAR_BITS Kconfig setting below.
+
+if PCIEXP_SUPPORT_RESIZABLE_BARS
+
+config PCIEXP_DEFAULT_MAX_RESIZABLE_BAR_BITS
+	int "Bits of address space to give to Resizable BARs"
+	range 20 63	# 1 MiB - 8 EiB
+	default 29	# 512 MiB
+	help
+	  This is the maximum number of bits of address space to allocate for
+	  PCIe devices with resizable BARs. For instance, if a device requests
+	  30 bits of address space (1 GiB), but this field is set to 29, then
+	  the device will only be allocated 29 bits worth of address space (512
+	  MiB). Valid values range from 20 (1 MiB) to 63 (8 EiB); these come
+	  from the Resizable BAR portion of the PCIe spec (7.8.6).
+
+endif # PCIEXP_SUPPORT_RESIZABLE_BARS
+
 config PCIEXP_HOTPLUG
 	prompt "Enable PCIe Hotplug Support"
 	bool
diff --git a/src/device/pci_device.c b/src/device/pci_device.c
index 5938f0de83..2acc517804 100644
--- a/src/device/pci_device.c
+++ b/src/device/pci_device.c
@@ -19,6 +19,7 @@
 #include <device/pci_ids.h>
 #include <device/pcix.h>
 #include <device/pciexp.h>
+#include <lib.h>
 #include <pc80/i8259.h>
 #include <security/vboot/vbnv.h>
 #include <timestamp.h>
@@ -309,6 +310,127 @@ struct msix_entry *pci_msix_get_table(struct device *dev)
 	return (struct msix_entry *)((uintptr_t)res->base + offset);
 }
 
+static unsigned int get_rebar_offset(const struct device *dev, unsigned long index)
+{
+	uint32_t offset = pciexp_find_extended_cap(dev, PCIE_EXT_CAP_RESIZABLE_BAR);
+	if (!offset)
+		return 0;
+
+	/* Convert PCI_BASE_ADDRESS_0, ..._1, ..._2 into 0, 1, 2... */
+	const unsigned int find_bar_idx = (index - PCI_BASE_ADDRESS_0) /
+		sizeof(uint32_t);
+
+	/* Although all of the Resizable BAR Control Registers contain an
+	   "NBARs" field, it is only valid in the Control Register for BAR 0 */
+	const uint32_t rebar_ctrl0 = pci_read_config32(dev, offset + PCI_REBAR_CTRL_OFFSET);
+	const unsigned int nbars = (rebar_ctrl0 & PCI_REBAR_CTRL_NBARS_MASK) >>
+		PCI_REBAR_CTRL_NBARS_SHIFT;
+
+	for (unsigned int i = 0; i < nbars; i++, offset += sizeof(uint64_t)) {
+		const uint32_t rebar_ctrl = pci_read_config32(
+			dev, offset + PCI_REBAR_CTRL_OFFSET);
+		const uint32_t bar_idx = rebar_ctrl & PCI_REBAR_CTRL_IDX_MASK;
+		if (bar_idx == find_bar_idx)
+			return offset;
+	}
+
+	return 0;
+}
+
+/* Bit 20 = 1 MiB, bit 21 = 2 MiB, bit 22 = 4 MiB, ... bit 63 = 8 EiB */
+static uint64_t get_rebar_sizes_mask(const struct device *dev,
+				     unsigned long index)
+{
+	uint64_t size_mask = 0ULL;
+	const uint32_t offset = get_rebar_offset(dev, index);
+	if (!offset)
+		return 0;
+
+	/* Get 1 MB - 128 TB support from CAP register */
+	const uint32_t cap = pci_read_config32(dev, offset + PCI_REBAR_CAP_OFFSET);
+	/* Shift the bits from 4-31 to 0-27 (i.e., down by 4 bits) */
+	size_mask |= ((cap & PCI_REBAR_CAP_SIZE_MASK) >> 4);
+
+	/* Get 256 TB - 8 EB support from CTRL register and store it in bits 28-43 */
+	const uint64_t ctrl = pci_read_config32(dev, offset + PCI_REBAR_CTRL_OFFSET);
+	/* Shift ctrl mask from bit 16 to bit 28, so that the two
+	   masks (fom cap and ctrl) form a contiguous bitmask when
+	   concatenated (i.e., up by 12 bits). */
+	size_mask |= ((ctrl & PCI_REBAR_CTRL_SIZE_MASK) << 12);
+
+	/* Now that the mask occupies bits 0-43, shift it up to 20-63, so they
+	   represent the actual powers of 2. */
+	return size_mask << 20;
+}
+
+static void pci_store_rebar_size(const struct device *dev,
+				 const struct resource *resource)
+{
+	const unsigned int num_bits = __fls64(resource->size);
+	const uint32_t offset = get_rebar_offset(dev, resource->index);
+	if (!offset)
+		return;
+
+	pci_update_config32(dev, offset + PCI_REBAR_CTRL_OFFSET,
+			    ~PCI_REBAR_CTRL_SIZE_MASK,
+			    num_bits << PCI_REBAR_CTRL_SIZE_SHIFT);
+}
+
+static void configure_adjustable_base(const struct device *dev,
+				      unsigned long index,
+				      struct resource *res)
+{
+	/*
+	 * Excerpt from an implementation note from the PCIe spec:
+	 *
+	 * System software uses this capability in place of the above mentioned
+	 * method of determining the resource size[0], and prior to assigning
+	 * the base address to the BAR. Potential usable resource sizes are
+	 * reported by the Function via its Resizable BAR Capability and Control
+	 * registers. It is intended that the software allocate the largest of
+	 * the reported sizes that it can, since allocating less address space
+	 * than the largest reported size can result in lower
+	 * performance. Software then writes the size to the Resizable BAR
+	 * Control register for the appropriate BAR for the Function. Following
+	 * this, the base address is written to the BAR.
+	 *
+	 * [0] Referring to using the moving bits in the BAR to determine the
+	 *     requested size of the MMIO region
+	 */
+	const uint64_t size_mask = get_rebar_sizes_mask(dev, index);
+	if (!size_mask)
+		return;
+
+	int max_requested_bits = __fls64(size_mask);
+	if (max_requested_bits > CONFIG_PCIEXP_DEFAULT_MAX_RESIZABLE_BAR_BITS) {
+		printk(BIOS_WARNING, "WARNING: Device %s requests a BAR with"
+		       "%u bits of address space, which coreboot is not"
+		       "configured to hand out, truncating to %u bits\n",
+		       dev_path(dev), max_requested_bits,
+		       CONFIG_PCIEXP_DEFAULT_MAX_RESIZABLE_BAR_BITS);
+		max_requested_bits = CONFIG_PCIEXP_DEFAULT_MAX_RESIZABLE_BAR_BITS;
+	}
+
+	if (!(res->flags & IORESOURCE_PCI64) && max_requested_bits > 32) {
+		printk(BIOS_ERR, "ERROR: Resizable BAR requested"
+		       "above 32 bits, but PCI function reported a"
+		       "32-bit BAR.");
+		return;
+	}
+
+	/* Configure the resource parameters for the adjustable BAR */
+	res->size = 1ULL << max_requested_bits;
+	res->align = max_requested_bits;
+	res->gran = max_requested_bits;
+	res->limit = (res->flags & IORESOURCE_PCI64) ? UINT64_MAX : UINT32_MAX;
+	res->flags |= IORESOURCE_PCIE_RESIZABLE_BAR;
+
+	printk(BIOS_INFO, "%s: Adjusting resource index %lu: base: %llx size: %llx "
+	       "align: %d gran: %d limit: %llx\n",
+	       dev_path(dev), res->index, res->base, res->size,
+	       res->align, res->gran, res->limit);
+}
+
 /**
  * Read the base address registers for a given device.
  *
@@ -323,6 +445,11 @@ static void pci_read_bases(struct device *dev, unsigned int howmany)
 	     (index < PCI_BASE_ADDRESS_0 + (howmany << 2));) {
 		struct resource *resource;
 		resource = pci_get_resource(dev, index);
+
+		const bool is_pcie = pci_find_capability(dev, PCI_CAP_ID_PCIE) != 0;
+		if (CONFIG(PCIEXP_SUPPORT_RESIZABLE_BARS) && is_pcie)
+			configure_adjustable_base(dev, index, resource);
+
 		index += (resource->flags & IORESOURCE_PCI64) ? 8 : 4;
 	}
 
@@ -547,11 +674,17 @@ static void pci_set_resource(struct device *dev, struct resource *resource)
 	/* Now store the resource. */
 	resource->flags |= IORESOURCE_STORED;
 
-	if (resource->flags & IORESOURCE_PCI_BRIDGE)
-		pci_store_bridge_resource(dev, resource);
-	else
+	if (!(resource->flags & IORESOURCE_PCI_BRIDGE)) {
+		if (CONFIG(PCIEXP_SUPPORT_RESIZABLE_BARS) &&
+		    (resource->flags & IORESOURCE_PCIE_RESIZABLE_BAR))
+			pci_store_rebar_size(dev, resource);
+
 		pci_store_resource(dev, resource);
 
+	} else {
+		pci_store_bridge_resource(dev, resource);
+	}
+
 	report_resource_stored(dev, resource, "");
 }
 
diff --git a/src/include/device/pci_def.h b/src/include/device/pci_def.h
index 06114363f6..1c4a3e76ef 100644
--- a/src/include/device/pci_def.h
+++ b/src/include/device/pci_def.h
@@ -460,10 +460,11 @@
 #define PCI_EXT_CAP_ID_PWR	4
 
 /* Extended Capability lists*/
-#define PCIE_EXT_CAP_OFFSET	0x100
-#define  PCIE_EXT_CAP_AER_ID	0x0001
-#define  PCIE_EXT_CAP_L1SS_ID	0x001E
-#define  PCIE_EXT_CAP_LTR_ID	0x0018
+#define PCIE_EXT_CAP_OFFSET		0x100
+#define  PCIE_EXT_CAP_AER_ID		 0x0001
+#define  PCIE_EXT_CAP_L1SS_ID		 0x001E
+#define  PCIE_EXT_CAP_LTR_ID		 0x0018
+#define  PCIE_EXT_CAP_RESIZABLE_BAR	 0x0015
 
 /* Advanced Error Reporting */
 #define PCI_ERR_UNCOR_STATUS	4	/* Uncorrectable Error Status */
@@ -527,6 +528,16 @@
 #define PCI_LTR_MAX_SNOOP	4
 #define PCI_LTR_MAX_NOSNOOP	6
 
+/* PCIe Resizable BARs */
+#define PCI_REBAR_CAP_OFFSET		0x4
+#define  PCI_REBAR_CAP_SIZE_MASK	 0xfffffff0
+#define PCI_REBAR_CTRL_OFFSET		0x8
+#define  PCI_REBAR_CTRL_NBARS_MASK	 0xe0
+#define  PCI_REBAR_CTRL_NBARS_SHIFT	 5
+#define  PCI_REBAR_CTRL_IDX_MASK	 0x07
+#define  PCI_REBAR_CTRL_SIZE_MASK	 0xffff0000
+#define  PCI_REBAR_CTRL_SIZE_SHIFT	 16
+
 /*
  * The PCI interface treats multi-function devices as independent
  * devices.  The slot/function address of each device is encoded
diff --git a/src/include/device/resource.h b/src/include/device/resource.h
index 098d0b653a..fb1f691190 100644
--- a/src/include/device/resource.h
+++ b/src/include/device/resource.h
@@ -36,8 +36,9 @@
 #define IORESOURCE_FIXED	0x80000000
 
 /* PCI specific resource bits (IORESOURCE_BITS) */
-#define IORESOURCE_PCI64	(1<<0)	/* 64bit long pci resource */
-#define IORESOURCE_PCI_BRIDGE	(1<<1)  /* A bridge pci resource */
+#define IORESOURCE_PCI64		(1<<0)	/* 64bit long pci resource */
+#define IORESOURCE_PCI_BRIDGE		(1<<1)  /* A bridge pci resource */
+#define IORESOURCE_PCIE_RESIZABLE_BAR	(1<<2)  /* A Resizable BAR */
 
 typedef u64 resource_t;
 struct resource {