x86/mtrr: Enable Rd/WrDram mod in AMD fixed MTRRs

AMD's fixed MTRRs have RdDram and WrDram bits that route memory
accesses to DRAM vs. MMIO.  These are typically hidden for normal
operation by clearing SYS_CFG[19] (MtrrFixDramModEn).  According to
BKDGs and AMD64 Programmer's Manual vol 2, this bit is clear at
reset, should be set for configuration during POST, then cleared for
normal operation.

Attempting to modify the RdDram and WrDram settings without unhiding
them causes a General Protection Fault.  Add functions to enable and
disable MtrrFixDramModEn.  Unhide/hide as necessary when copying or
writing the fixed MTRRs.

Finally, modify sipi_vector.S to enable the bits prior to writing
the fixed MTRRs and disable when complete.

This functionality is compiled out on non-AMD platforms.

BUG=b:68019051
TEST=Boot Kahlee, check steps with HDT

Change-Id: Ie195131ff752400eb886dfccc39b314b4fa6b3f3
Signed-off-by: Marshall Dawson <marshalldawson3rd@gmail.com>
Reviewed-on: https://review.coreboot.org/23722
Tested-by: build bot (Jenkins) <no-reply@coreboot.org>
Reviewed-by: Aaron Durbin <adurbin@chromium.org>

4 files changed, 54 insertions, 1 deletions

diff --git a/src/cpu/x86/mp_init.c b/src/cpu/x86/mp_init.c
index baa6aec998..92eb722eb4 100644
--- a/src/cpu/x86/mp_init.c
+++ b/src/cpu/x86/mp_init.c
@@ -299,6 +299,8 @@ static int save_bsp_msrs(char *start, int size)
 		return -1;
 	}
 
+	fixed_mtrrs_expose_amd_rwdram();
+
 	msr_entry = (void *)start;
 	for (i = 0; i < NUM_FIXED_MTRRS; i++)
 		msr_entry = save_msr(fixed_mtrrs[i], msr_entry);
@@ -310,6 +312,8 @@ static int save_bsp_msrs(char *start, int size)
 
 	msr_entry = save_msr(MTRR_DEF_TYPE_MSR, msr_entry);
 
+	fixed_mtrrs_hide_amd_rwdram();
+
 	return msr_count;
 }
 
diff --git a/src/cpu/x86/mtrr/mtrr.c b/src/cpu/x86/mtrr/mtrr.c
index c2c629cbe4..08312fbebf 100644
--- a/src/cpu/x86/mtrr/mtrr.c
+++ b/src/cpu/x86/mtrr/mtrr.c
@@ -36,8 +36,8 @@
 #include <arch/cpu.h>
 #include <arch/acpi.h>
 #include <memrange.h>
-#if IS_ENABLED(CONFIG_X86_AMD_FIXED_MTRRS)
 #include <cpu/amd/mtrr.h>
+#if IS_ENABLED(CONFIG_X86_AMD_FIXED_MTRRS)
 #define MTRR_FIXED_WRBACK_BITS (MTRR_READ_MEM | MTRR_WRITE_MEM)
 #else
 #define MTRR_FIXED_WRBACK_BITS 0
@@ -83,6 +83,30 @@ void enable_fixed_mtrr(void)
 	wrmsr(MTRR_DEF_TYPE_MSR, msr);
 }
 
+void fixed_mtrrs_expose_amd_rwdram(void)
+{
+	msr_t syscfg;
+
+	if (!IS_ENABLED(CONFIG_X86_AMD_FIXED_MTRRS))
+		return;
+
+	syscfg = rdmsr(SYSCFG_MSR);
+	syscfg.lo |= SYSCFG_MSR_MtrrFixDramModEn;
+	wrmsr(SYSCFG_MSR, syscfg);
+}
+
+void fixed_mtrrs_hide_amd_rwdram(void)
+{
+	msr_t syscfg;
+
+	if (!IS_ENABLED(CONFIG_X86_AMD_FIXED_MTRRS))
+		return;
+
+	syscfg = rdmsr(SYSCFG_MSR);
+	syscfg.lo &= ~SYSCFG_MSR_MtrrFixDramModEn;
+	wrmsr(SYSCFG_MSR, syscfg);
+}
+
 static void enable_var_mtrr(unsigned char deftype)
 {
 	msr_t msr;
@@ -310,6 +334,8 @@ static void commit_fixed_mtrrs(void)
 	msr_t fixed_msrs[NUM_FIXED_MTRRS];
 	unsigned long msr_index[NUM_FIXED_MTRRS];
 
+	fixed_mtrrs_expose_amd_rwdram();
+
 	memset(&fixed_msrs, 0, sizeof(fixed_msrs));
 
 	msr_num = 0;
@@ -351,6 +377,8 @@ static void commit_fixed_mtrrs(void)
 	for (i = 0; i < ARRAY_SIZE(fixed_msrs); i++)
 		wrmsr(msr_index[i], fixed_msrs[i]);
 	enable_cache();
+	fixed_mtrrs_hide_amd_rwdram();
+
 }
 
 void x86_setup_fixed_mtrrs_no_enable(void)
diff --git a/src/cpu/x86/sipi_vector.S b/src/cpu/x86/sipi_vector.S
index bd60c65a88..83606bd920 100644
--- a/src/cpu/x86/sipi_vector.S
+++ b/src/cpu/x86/sipi_vector.S
@@ -15,6 +15,7 @@
  */
 
 #include <cpu/x86/cr.h>
+#include <cpu/amd/mtrr.h>
 
 /* The SIPI vector is responsible for initializing the APs in the sytem. It
  * loads microcode, sets up MSRs, and enables caching before calling into
@@ -172,6 +173,15 @@ microcode_done:
 	mov	msr_count, %ebx
 	test	%ebx, %ebx
 	jz	1f
+
+#if IS_ENABLED(CONFIG_X86_AMD_FIXED_MTRRS)
+	/* Allow modification of RdDram and WrDram bits */
+	mov	$SYSCFG_MSR, %ecx
+	rdmsr
+	or	$SYSCFG_MSR_MtrrFixDramModEn, %eax
+	wrmsr
+#endif
+
 load_msr:
 	mov	(%edi), %ecx
 	mov	4(%edi), %eax
@@ -181,6 +191,13 @@ load_msr:
 	dec	%ebx
 	jnz	load_msr
 
+#if IS_ENABLED(CONFIG_X86_AMD_FIXED_MTRRS)
+	mov	$SYSCFG_MSR, %ecx
+	rdmsr
+	and	$~SYSCFG_MSR_MtrrFixDramModEn, %eax
+	wrmsr
+#endif
+
 1:
 	/* Enable caching. */
 	mov	%cr0, %eax
diff --git a/src/include/cpu/x86/mtrr.h b/src/include/cpu/x86/mtrr.h
index a72d602ccd..0d64be517e 100644
--- a/src/include/cpu/x86/mtrr.h
+++ b/src/include/cpu/x86/mtrr.h
@@ -76,6 +76,10 @@ void x86_setup_mtrrs_with_detect(void);
  */
 void x86_setup_var_mtrrs(unsigned int address_bits, unsigned int above4gb);
 void enable_fixed_mtrr(void);
+/* Unhide Rd/WrDram bits and allow modification for AMD. */
+void fixed_mtrrs_expose_amd_rwdram(void);
+/* Hide Rd/WrDram bits and allow modification for AMD. */
+void fixed_mtrrs_hide_amd_rwdram(void);
 void x86_setup_fixed_mtrrs(void);
 /* Set up fixed MTRRs but do not enable them. */
 void x86_setup_fixed_mtrrs_no_enable(void);