20 files changed, 138 insertions, 34 deletions
diff --git a/src/arch/x86/Makefile.inc b/src/arch/x86/Makefile.inc
index e0e8a3b7b9..accb022a81 100644
--- a/src/arch/x86/Makefile.inc
+++ b/src/arch/x86/Makefile.inc
@@ -64,11 +64,6 @@ $(1)-S-ccopts += -I.
 $$(objcbfs)/$(1).debug: $$$$($(1)-libs) $$$$($(1)-objs)
 	@printf "    LINK       $$(subst $$(obj)/,,$$(@))\n"
 	$$(LD_$(1)) $$(LDFLAGS_$(1)) -o $$@ -L$$(obj) $$(COMPILER_RT_FLAGS_$(1)) --whole-archive --start-group $$(filter-out %.ld,$$($(1)-objs)) $$($(1)-libs) --no-whole-archive $$(COMPILER_RT_$(1)) --end-group -T $(call src-to-obj,$(1),$(CONFIG_MEMLAYOUT_LD_FILE)) --oformat $(2)
-	-LANG=C LC_ALL= $$(OBJCOPY_$(1)) --only-section .illegal_globals $$(@) $$(objcbfs)/$(1)_null.offenders >/dev/null 2>&1
-	if [ -z "$$$$($$(NM_$(1)) $$(objcbfs)/$(1)_null.offenders 2>&1 | grep 'no symbols')" ];then \
-		echo "Forbidden global variables in $(1):"; \
-		$$(NM_$(1)) $$(objcbfs)/$(1)_null.offenders; false; \
-	fi
 endef
 
 ###############################################################################
diff --git a/src/arch/x86/assembly_entry.S b/src/arch/x86/assembly_entry.S
index 869acc84e2..9a9a0465dc 100644
--- a/src/arch/x86/assembly_entry.S
+++ b/src/arch/x86/assembly_entry.S
@@ -33,8 +33,8 @@ _start:
 	/* reset stack pointer to CAR/EARLYRAM stack */
 	mov	$_STACK_TOP, %esp
 
+#if ENV_SEPARATE_DATA_AND_BSS
 	/* clear .bss section as it is not shared */
-#if ENV_SEPARATE_BSS
 	cld
 	xor	%eax, %eax
 	movl	$(_ebss), %ecx
@@ -42,6 +42,14 @@ _start:
 	sub	%edi, %ecx
 	shrl	$2, %ecx
 	rep	stosl
+
+	/* Copy .data section content to Cache-As-Ram */
+	movl	$(_edata), %ecx
+	movl	$(_data), %edi
+	sub	%edi, %ecx
+	shrl	$2, %ecx
+	movl	$(_data_load),%esi
+	rep	movsl
 #endif
 
 #if ((ENV_SEPARATE_VERSTAGE && CONFIG(VERSTAGE_DEBUG_SPINLOOP)) \
diff --git a/src/arch/x86/car.ld b/src/arch/x86/car.ld
index ab6c3b0df8..a1a782ffe0 100644
--- a/src/arch/x86/car.ld
+++ b/src/arch/x86/car.ld
@@ -59,7 +59,7 @@
 	 * cbmem console. This is useful for clearing this area on a per-stage
 	 * basis when more than one stage uses cache-as-ram. */
 
-#if ENV_SEPARATE_BSS
+#if ENV_SEPARATE_DATA_AND_BSS
 	. = ALIGN(ARCH_POINTER_ALIGN_SIZE);
 	_bss = .;
 	/* Allow global uninitialized variables for stages without CAR teardown. */
@@ -89,11 +89,32 @@
 	_shadow_size = (_ebss - _car_region_start) >> 3;
 	REGION(asan_shadow, ., _shadow_size, ARCH_POINTER_ALIGN_SIZE)
 #endif
-	_car_unallocated_start = .;
-	_car_region_end = . + CONFIG_DCACHE_RAM_SIZE - (. - _car_region_start)
-			  - CONFIG_FSP_T_RESERVED_SIZE;
 }
 
+#if ENV_SEPARATE_DATA_AND_BSS
+/* This symbol defines the load address of the Cache-As-RAM .data
+ * section. It should be right at the end of the .text section (_etext)
+ * and ARCH_POINTER_ALIGN_SIZE aligned. */
+_data_load = _etext;
+
+_bogus = ASSERT(_etext == ALIGN(_etext, ARCH_POINTER_ALIGN_SIZE), "Cache-As-RAM load address is improperly defined.");
+
+.data ALIGN(ARCH_POINTER_ALIGN_SIZE) : AT (_data_load) {
+	_data = .;
+	*(.data);
+	*(.data.*);
+	*(.sdata);
+	*(.sdata.*);
+	. = ALIGN(ARCH_POINTER_ALIGN_SIZE);
+	_edata = .;
+	RECORD_SIZE(data)
+} : data_segment
+#endif
+
+_car_unallocated_start = .;
+_car_region_end = . + CONFIG_DCACHE_RAM_SIZE - (. - _car_region_start)
+		  - CONFIG_FSP_T_RESERVED_SIZE;
+
 . = _car_region_start;
 .car.fspm_rc_heap . (NOLOAD) : {
 . += CONFIG_FSP_M_RC_HEAP_SIZE;
@@ -124,18 +145,11 @@ _rom_mtrr_mask = ~(CACHE_ROM_SIZE - 1);
 _rom_mtrr_base = _rom_mtrr_mask;
 #endif
 
-/* Global variables are not allowed in romstage
- * This section is checked during stage creation to ensure
- * that there are no global variables present
- */
-
-. = 0xffffff00;
-.illegal_globals . : {
-	*(.data)
-	*(.data.*)
-}
-
+#if ENV_SEPARATE_DATA_AND_BSS
+_bogus = ASSERT((CONFIG_DCACHE_RAM_SIZE == 0) || (SIZEOF(.car.data) + SIZEOF(.data) <= CONFIG_DCACHE_RAM_SIZE), "Cache as RAM area is too full");
+#else
 _bogus = ASSERT((CONFIG_DCACHE_RAM_SIZE == 0) || (SIZEOF(.car.data) <= CONFIG_DCACHE_RAM_SIZE), "Cache as RAM area is too full");
+#endif
 #if CONFIG(PAGING_IN_CACHE_AS_RAM)
 _bogus2 = ASSERT(_pagetables == ALIGN(_pagetables, 4096), "_pagetables aren't 4KiB aligned");
 #endif
diff --git a/src/arch/x86/include/arch/header.ld b/src/arch/x86/include/arch/header.ld
index 5b380faad5..2ee021226c 100644
--- a/src/arch/x86/include/arch/header.ld
+++ b/src/arch/x86/include/arch/header.ld
@@ -3,6 +3,9 @@
 PHDRS
 {
 	to_load PT_LOAD;
+#if ENV_SEPARATE_DATA_AND_BSS
+	data_segment PT_LOAD;
+#endif
 }
 
 ENTRY(_start)
diff --git a/src/arch/x86/memlayout.ld b/src/arch/x86/memlayout.ld
index 549c2f9041..f448bf89de 100644
--- a/src/arch/x86/memlayout.ld
+++ b/src/arch/x86/memlayout.ld
@@ -3,6 +3,16 @@
 #include <memlayout.h>
 #include <arch/header.ld>
 
+/*
+ * The bootblock linker script should be included before the Cache-As-RAM linker
+ * script. Indeed, if it is included after and Cache-As-RAM .data section
+ * support is enabled, the definition order of the sections makes the linker
+ * create an image with an almost 4 GB hole.
+ */
+#if ENV_BOOTBLOCK
+INCLUDE "bootblock/arch/x86/bootblock.ld"
+#endif /* ENV_BOOTBLOCK */
+
 SECTIONS
 {
 	/*
@@ -36,7 +46,3 @@ SECTIONS
 	POSTCAR(32M, 1M)
 #endif
 }
-
-#if ENV_BOOTBLOCK
-	INCLUDE "bootblock/arch/x86/bootblock.ld"
-#endif  /* ENV_BOOTBLOCK */
diff --git a/src/cpu/intel/car/core2/cache_as_ram.S b/src/cpu/intel/car/core2/cache_as_ram.S
index 9c60308b28..e134717b40 100644
--- a/src/cpu/intel/car/core2/cache_as_ram.S
+++ b/src/cpu/intel/car/core2/cache_as_ram.S
@@ -180,6 +180,9 @@ addrsize_set_high:
 	pushl	%eax	/* tsc[31:0] */
 #endif
 
+	/* Copy .data section content to Cache-As-Ram */
+#include <cpu/x86/copy_data_section.inc>
+
 before_c_entry:
 	post_code(POSTCODE_BOOTBLOCK_BEFORE_C_ENTRY)
 	call	bootblock_c_entry_bist
diff --git a/src/cpu/intel/car/non-evict/cache_as_ram.S b/src/cpu/intel/car/non-evict/cache_as_ram.S
index 18ac07036e..76986ff68e 100644
--- a/src/cpu/intel/car/non-evict/cache_as_ram.S
+++ b/src/cpu/intel/car/non-evict/cache_as_ram.S
@@ -233,6 +233,9 @@ end_microcode_update:
 	pushl	%eax	/* tsc[31:0] */
 #endif
 
+	/* Copy .data section content to Cache-As-Ram */
+#include <cpu/x86/copy_data_section.inc>
+
 before_c_entry:
 	post_code(POSTCODE_BOOTBLOCK_BEFORE_C_ENTRY)
 	call	bootblock_c_entry_bist
diff --git a/src/cpu/intel/car/p3/cache_as_ram.S b/src/cpu/intel/car/p3/cache_as_ram.S
index 779dbcca8a..623cf41e73 100644
--- a/src/cpu/intel/car/p3/cache_as_ram.S
+++ b/src/cpu/intel/car/p3/cache_as_ram.S
@@ -155,6 +155,9 @@ addrsize_set_high:
 	movd	%mm1, %eax
 	pushl	%eax	/* tsc[31:0] */
 
+	/* Copy .data section content to Cache-As-Ram */
+#include <cpu/x86/copy_data_section.inc>
+
 before_c_entry:
 	post_code(POSTCODE_BOOTBLOCK_BEFORE_C_ENTRY)
 	call	bootblock_c_entry_bist
diff --git a/src/cpu/intel/car/p4-netburst/cache_as_ram.S b/src/cpu/intel/car/p4-netburst/cache_as_ram.S
index 9f514ef592..f7c023b402 100644
--- a/src/cpu/intel/car/p4-netburst/cache_as_ram.S
+++ b/src/cpu/intel/car/p4-netburst/cache_as_ram.S
@@ -380,6 +380,9 @@ fill_cache:
 	pushl	%eax	/* tsc[31:0] */
 #endif
 
+	/* Copy .data section content to Cache-As-Ram */
+#include <cpu/x86/copy_data_section.inc>
+
 before_c_entry:
 	post_code(POSTCODE_BOOTBLOCK_BEFORE_C_ENTRY)
 	call	bootblock_c_entry_bist
diff --git a/src/cpu/qemu-x86/cache_as_ram_bootblock.S b/src/cpu/qemu-x86/cache_as_ram_bootblock.S
index fe872debea..0943e356da 100644
--- a/src/cpu/qemu-x86/cache_as_ram_bootblock.S
+++ b/src/cpu/qemu-x86/cache_as_ram_bootblock.S
@@ -100,6 +100,9 @@ pages_done:
 	pushl	%eax
 #endif
 
+	/* Copy .data section content to Cache-As-Ram */
+#include <cpu/x86/copy_data_section.inc>
+
 before_c_entry:
 	call	bootblock_c_entry_bist
 	/* Never returns */
diff --git a/src/cpu/x86/copy_data_section.inc b/src/cpu/x86/copy_data_section.inc
new file mode 100644
index 0000000000..dccb8d3975
--- /dev/null
+++ b/src/cpu/x86/copy_data_section.inc
@@ -0,0 +1,38 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+
+#if ENV_SEPARATE_DATA_AND_BSS
+
+/*
+ * Copy .data section content to Cache-As-Ram.
+ * This code can be included from 32 bits or 64 bits code. It also preserves
+ * registers.
+ */
+copy_data_section:
+#if ENV_X86_64
+	push	%rcx
+	push	%rdi
+	push	%rsi
+#else
+	pushl	%ecx
+	pushl	%edi
+	pushl	%esi
+#endif
+
+	movl	$(_edata), %ecx
+	movl	$(_data), %edi
+	sub	%edi, %ecx
+	movl	$(_data_load),%esi
+	shrl	$2, %ecx
+	rep	movsl
+
+#if ENV_X86_64
+	pop	%rsi
+	pop	%rdi
+	pop	%rcx
+#else
+	popl	%esi
+	popl	%edi
+	popl	%ecx
+#endif
+
+#endif	/* ENV_SEPARATE_DATA_AND_BSS */
diff --git a/src/drivers/amd/agesa/cache_as_ram.S b/src/drivers/amd/agesa/cache_as_ram.S
index 5e77263c97..c10c369643 100644
--- a/src/drivers/amd/agesa/cache_as_ram.S
+++ b/src/drivers/amd/agesa/cache_as_ram.S
@@ -57,6 +57,9 @@ bootblock_pre_c_entry:
 	movd	%mm1, %eax
 	pushl	%eax		/* tsc[31:0] */
 
+	/* Copy .data section content to Cache-As-Ram */
+#include <cpu/x86/copy_data_section.inc>
+
 	post_code(POSTCODE_BOOTBLOCK_PRE_C_DONE)
 
 	call	bootblock_c_entry
diff --git a/src/drivers/intel/fsp1_1/cache_as_ram.S b/src/drivers/intel/fsp1_1/cache_as_ram.S
index 17e0a69cd8..c8eae7fdc1 100644
--- a/src/drivers/intel/fsp1_1/cache_as_ram.S
+++ b/src/drivers/intel/fsp1_1/cache_as_ram.S
@@ -180,6 +180,9 @@ CAR_init_done:
 	movd	%mm0, %eax
 	pushl	%eax	/* tsc[31:0] */
 
+	/* Copy .data section content to Cache-As-Ram */
+#include <cpu/x86/copy_data_section.inc>
+
 before_romstage:
 	/* Call bootblock_c_entry(uint64_t base_timestamp) */
 	call	bootblock_c_entry
diff --git a/src/include/rules.h b/src/include/rules.h
index ae8118ebb4..02c36f2b25 100644
--- a/src/include/rules.h
+++ b/src/include/rules.h
@@ -274,16 +274,16 @@
 #if ENV_X86
 /* Indicates memory layout is determined with arch/x86/car.ld. */
 #define ENV_CACHE_AS_RAM		(ENV_ROMSTAGE_OR_BEFORE && !CONFIG(RESET_VECTOR_IN_RAM))
-/* No .data sections with execute-in-place from ROM.  */
-#define ENV_HAS_DATA_SECTION	!ENV_CACHE_AS_RAM
 #else
-/* Both .data and .bss, sometimes SRAM not DRAM. */
-#define ENV_HAS_DATA_SECTION	1
 #define ENV_CACHE_AS_RAM		0
 #endif
 
-/* Indicates if the stage uses the _bss region defined in arch/x86/car.ld */
-#define ENV_SEPARATE_BSS	(ENV_CACHE_AS_RAM && (ENV_BOOTBLOCK || !CONFIG(NO_XIP_EARLY_STAGES)))
+/* Indicates .data section support. */
+#define ENV_HAS_DATA_SECTION		1
+
+/* Indicates if the stage uses the _data and _bss regions defined in
+ * arch/x86/car.ld */
+#define ENV_SEPARATE_DATA_AND_BSS	(ENV_CACHE_AS_RAM && (ENV_BOOTBLOCK || !CONFIG(NO_XIP_EARLY_STAGES)))
 
 /* Currently rmodules, ramstage and smm have heap. */
 #define ENV_HAS_HEAP_SECTION	(ENV_RMODULE || ENV_RAMSTAGE || ENV_SMM)
diff --git a/src/include/symbols.h b/src/include/symbols.h
index a03af08463..5410798f06 100644
--- a/src/include/symbols.h
+++ b/src/include/symbols.h
@@ -50,6 +50,11 @@ DECLARE_REGION(heap)
 DECLARE_REGION(asan_shadow)
 #endif
 
+#if ENV_SEPARATE_DATA_AND_BSS
+DECLARE_REGION(data)
+DECLARE_REGION(data_load)
+#endif
+
 /* Regions for execution units. */
 
 DECLARE_REGION(payload)
diff --git a/src/lib/program.ld b/src/lib/program.ld
index 67f685fa08..f406f9f3b4 100644
--- a/src/lib/program.ld
+++ b/src/lib/program.ld
@@ -72,7 +72,7 @@
 #endif
 
 /* Include data, bss, and heap in that order. Not defined for all stages. */
-#if ENV_HAS_DATA_SECTION
+#if !ENV_SEPARATE_DATA_AND_BSS
 .data . : {
 	. = ALIGN(ARCH_CACHELINE_ALIGN_SIZE);
 	_data = .;
@@ -116,7 +116,7 @@
 }
 #endif
 
-#if !ENV_SEPARATE_BSS
+#if !ENV_SEPARATE_DATA_AND_BSS
 .bss . : {
 	. = ALIGN(ARCH_POINTER_ALIGN_SIZE);
 	_bss = .;
diff --git a/src/security/vboot/Makefile.inc b/src/security/vboot/Makefile.inc
index 62a566019f..f152444044 100644
--- a/src/security/vboot/Makefile.inc
+++ b/src/security/vboot/Makefile.inc
@@ -137,7 +137,12 @@ $(CONFIG_CBFS_PREFIX)/verstage-compression := $(CBFS_PRERAM_COMPRESS_FLAG)
 endif	# CONFIG_VBOOT_STARTS_BEFORE_BOOTBLOCK
 
 ifeq ($(CONFIG_ARCH_VERSTAGE_X86_32)$(CONFIG_ARCH_VERSTAGE_X86_64),y)
-$(CONFIG_CBFS_PREFIX)/verstage-options := -a 64 -S ".car.data"
+$(CONFIG_CBFS_PREFIX)/verstage-options := -a 64
+ifeq ($(CONFIG_NO_XIP_EARLY_STAGES),y)
+$(CONFIG_CBFS_PREFIX)/verstage-options += -S ".car.data"
+else
+$(CONFIG_CBFS_PREFIX)/verstage-options += -S ".car.data,.data"
+endif
 
 # If CAR does not support execution of code, verstage on x86 is expected to be
 # xip.
diff --git a/src/soc/amd/common/block/cpu/car/cache_as_ram.S b/src/soc/amd/common/block/cpu/car/cache_as_ram.S
index 2bd3f5061e..8d41826bfd 100644
--- a/src/soc/amd/common/block/cpu/car/cache_as_ram.S
+++ b/src/soc/amd/common/block/cpu/car/cache_as_ram.S
@@ -41,6 +41,9 @@ bootblock_pre_c_entry:
 	movd	%mm1, %eax
 	pushl	%eax		/* tsc[31:0] */
 
+	/* Copy .data section content to Cache-As-Ram */
+#include <cpu/x86/copy_data_section.inc>
+
 before_carstage:
 	post_code(POSTCODE_BOOTBLOCK_PRE_C_DONE)
 
diff --git a/src/soc/intel/common/block/cpu/car/cache_as_ram.S b/src/soc/intel/common/block/cpu/car/cache_as_ram.S
index 3c8dc2e686..61cbe307ff 100644
--- a/src/soc/intel/common/block/cpu/car/cache_as_ram.S
+++ b/src/soc/intel/common/block/cpu/car/cache_as_ram.S
@@ -295,6 +295,9 @@ car_init_done:
 	pushl	%eax	/* tsc[31:0] */
 #endif
 
+	/* Copy .data section content to Cache-As-Ram */
+#include <cpu/x86/copy_data_section.inc>
+
 before_carstage:
 	post_code(POSTCODE_SOC_BEFORE_CARSTAGE)
 
diff --git a/src/soc/intel/common/block/cpu/car/cache_as_ram_fsp.S b/src/soc/intel/common/block/cpu/car/cache_as_ram_fsp.S
index 7532c7d707..238a57e99f 100644
--- a/src/soc/intel/common/block/cpu/car/cache_as_ram_fsp.S
+++ b/src/soc/intel/common/block/cpu/car/cache_as_ram_fsp.S
@@ -93,6 +93,9 @@ CAR_init_done:
 	movd	%mm1, %eax
 	push	%eax
 
+	/* Copy .data section content to Cache-As-Ram */
+#include <cpu/x86/copy_data_section.inc>
+
 	/* We can call into C functions now */
 	call	bootblock_c_entry