sandybridge ivybridge: Treat native init as first class citizen

This is a sad story. We have three different code paths for
sandybridge and ivybridge: proper native path, google MRC path, and,
everyone's favorite: Intel FSP path. For the purpose of this patch,
the FSP path lives in its own little world, and doesn't concern us.

Since MRC was first, when native files and variables were added, they
were suffixed with "_native" to separate them from the existing code.
This can cause confusion, as the suffix might make the native files
seem parasitical.

This has been bothering me for many months. MRC should be the
parasitical path, especially since we fully support native init, and
it works more reliably, on a wider range of hardware. There have been
a few board ports that never made it to coreboot.org because MRC would
hang.

gigabyte/ga-b75m-d3h is a prime example: it did not work with MRC, so
the effort was abandoned at first. Once the native path became
available, the effort was restarted and the board is now supported.

In honor of the hackers and pioneers who made the native code
possible, rename things so that their effort is the first class
citizen.

Change-Id: Ic86cee5e00bf7f598716d3d15d1ea81ca673932f
Signed-off-by: Alexandru Gagniuc <mr.nuke.me@gmail.com>
Reviewed-on: http://review.coreboot.org/11788
Tested-by: build bot (Jenkins)
Reviewed-by: Philipp Deppenwiese <zaolin@das-labor.org>

37 files changed, 4931 insertions, 4931 deletions

diff --git a/src/Kconfig b/src/Kconfig
index 2c75750ecc..902855ec8d 100644
--- a/src/Kconfig
+++ b/src/Kconfig
@@ -246,8 +246,8 @@ config CACHE_RELOCATED_RAMSTAGE_OUTSIDE_CBMEM
 
 config FLASHMAP_OFFSET
 	hex "Flash Map Offset"
-	default 0x00670000 if NORTHBRIDGE_INTEL_SANDYBRIDGE
-	default 0x00610000 if NORTHBRIDGE_INTEL_IVYBRIDGE
+	default 0x00670000 if NORTHBRIDGE_INTEL_SANDYBRIDGE_MRC
+	default 0x00610000 if NORTHBRIDGE_INTEL_IVYBRIDGE_MRC
 	default CBFS_SIZE if !ARCH_X86
 	default 0
 	help
@@ -337,9 +337,9 @@ source "src/mainboard/Kconfig"
 config CBFS_SIZE
 	hex "Size of CBFS filesystem in ROM"
 	default 0x100000 if HAVE_INTEL_FIRMWARE || \
-	  NORTHBRIDGE_INTEL_GM45 || NORTHBRIDGE_INTEL_SANDYBRIDGE || \
-	  NORTHBRIDGE_INTEL_IVYBRIDGE || NORTHBRIDGE_INTEL_IVYBRIDGE_NATIVE || \
-	  NORTHBRIDGE_INTEL_SANDYBRIDGE_NATIVE || \
+	  NORTHBRIDGE_INTEL_GM45 || NORTHBRIDGE_INTEL_SANDYBRIDGE_MRC || \
+	  NORTHBRIDGE_INTEL_IVYBRIDGE_MRC || NORTHBRIDGE_INTEL_IVYBRIDGE || \
+	  NORTHBRIDGE_INTEL_SANDYBRIDGE || \
 	  NORTHBRIDGE_INTEL_NEHALEM || SOC_INTEL_BRASWELL || \
 	  SOC_INTEL_BROADWELL
 	default 0x200000 if SOC_INTEL_SKYLAKE
diff --git a/src/cpu/intel/Makefile.inc b/src/cpu/intel/Makefile.inc
index 51451e90d0..859b73beab 100644
--- a/src/cpu/intel/Makefile.inc
+++ b/src/cpu/intel/Makefile.inc
@@ -17,10 +17,10 @@ subdirs-$(CONFIG_CPU_INTEL_SOCKET_PGA370) += socket_PGA370
 subdirs-$(CONFIG_CPU_INTEL_SOCKET_RPGA988B) += socket_rPGA988B
 subdirs-$(CONFIG_CPU_INTEL_SOCKET_RPGA989) += socket_rPGA989
 subdirs-$(CONFIG_NORTHBRIDGE_INTEL_NEHALEM) += model_2065x
+subdirs-$(CONFIG_NORTHBRIDGE_INTEL_SANDYBRIDGE_MRC) += model_206ax
 subdirs-$(CONFIG_NORTHBRIDGE_INTEL_SANDYBRIDGE) += model_206ax
-subdirs-$(CONFIG_NORTHBRIDGE_INTEL_SANDYBRIDGE_NATIVE) += model_206ax
+subdirs-$(CONFIG_NORTHBRIDGE_INTEL_IVYBRIDGE_MRC) += model_206ax
 subdirs-$(CONFIG_NORTHBRIDGE_INTEL_IVYBRIDGE) += model_206ax
-subdirs-$(CONFIG_NORTHBRIDGE_INTEL_IVYBRIDGE_NATIVE) += model_206ax
 subdirs-$(CONFIG_NORTHBRIDGE_INTEL_HASWELL) += haswell
 subdirs-$(CONFIG_NORTHBRIDGE_INTEL_FSP_SANDYBRIDGE) += fsp_model_206ax
 subdirs-$(CONFIG_NORTHBRIDGE_INTEL_FSP_IVYBRIDGE) += fsp_model_206ax
diff --git a/src/mainboard/gigabyte/ga-b75m-d3h/Kconfig b/src/mainboard/gigabyte/ga-b75m-d3h/Kconfig
index a98e490992..8a53bef91a 100644
--- a/src/mainboard/gigabyte/ga-b75m-d3h/Kconfig
+++ b/src/mainboard/gigabyte/ga-b75m-d3h/Kconfig
@@ -4,7 +4,7 @@ config BOARD_SPECIFIC_OPTIONS
 	def_bool y
 	select ARCH_X86
 	select CPU_INTEL_SOCKET_LGA1155
-	select NORTHBRIDGE_INTEL_IVYBRIDGE_NATIVE
+	select NORTHBRIDGE_INTEL_IVYBRIDGE
 	select SOUTHBRIDGE_INTEL_C216
 	select SUPERIO_ITE_IT8728F
 	select BOARD_ROMSIZE_KB_8192
diff --git a/src/mainboard/gigabyte/ga-b75m-d3v/Kconfig b/src/mainboard/gigabyte/ga-b75m-d3v/Kconfig
index bf2f28079a..511d57eb79 100644
--- a/src/mainboard/gigabyte/ga-b75m-d3v/Kconfig
+++ b/src/mainboard/gigabyte/ga-b75m-d3v/Kconfig
@@ -4,7 +4,7 @@ config BOARD_SPECIFIC_OPTIONS
 	def_bool y
 	select ARCH_X86
 	select CPU_INTEL_SOCKET_LGA1155
-	select NORTHBRIDGE_INTEL_SANDYBRIDGE_NATIVE
+	select NORTHBRIDGE_INTEL_SANDYBRIDGE
 	select SOUTHBRIDGE_INTEL_C216
 	select CPU_MICROCODE_CBFS_NONE
 	select SUPERIO_ITE_IT8728F
diff --git a/src/mainboard/google/butterfly/Kconfig b/src/mainboard/google/butterfly/Kconfig
index 0c37565145..320981a4de 100644
--- a/src/mainboard/google/butterfly/Kconfig
+++ b/src/mainboard/google/butterfly/Kconfig
@@ -4,7 +4,7 @@ config BOARD_SPECIFIC_OPTIONS # dummy
 	def_bool y
 	select SYSTEM_TYPE_LAPTOP
 	select CPU_INTEL_SOCKET_RPGA989
-	select NORTHBRIDGE_INTEL_IVYBRIDGE_NATIVE
+	select NORTHBRIDGE_INTEL_IVYBRIDGE
 	select SOUTHBRIDGE_INTEL_C216
 	select EC_QUANTA_ENE_KB3940Q
 	select BOARD_ROMSIZE_KB_8192
diff --git a/src/mainboard/google/link/Kconfig b/src/mainboard/google/link/Kconfig
index ac06a6226b..2d9a9de0de 100644
--- a/src/mainboard/google/link/Kconfig
+++ b/src/mainboard/google/link/Kconfig
@@ -4,7 +4,7 @@ config BOARD_SPECIFIC_OPTIONS # dummy
 	def_bool y
 	select SYSTEM_TYPE_LAPTOP
 	select CPU_INTEL_SOCKET_RPGA989
-	select NORTHBRIDGE_INTEL_IVYBRIDGE
+	select NORTHBRIDGE_INTEL_IVYBRIDGE_MRC
 	select SOUTHBRIDGE_INTEL_C216
 	select BOARD_ROMSIZE_KB_8192
 	select EC_GOOGLE_CHROMEEC
diff --git a/src/mainboard/google/parrot/Kconfig b/src/mainboard/google/parrot/Kconfig
index 56ebf86fb9..e9b55a4ae8 100644
--- a/src/mainboard/google/parrot/Kconfig
+++ b/src/mainboard/google/parrot/Kconfig
@@ -4,7 +4,7 @@ config BOARD_SPECIFIC_OPTIONS # dummy
 	def_bool y
 	select SYSTEM_TYPE_LAPTOP
 	select CPU_INTEL_SOCKET_RPGA989
-	select NORTHBRIDGE_INTEL_IVYBRIDGE
+	select NORTHBRIDGE_INTEL_IVYBRIDGE_MRC
 	select SOUTHBRIDGE_INTEL_C216
 	select EC_COMPAL_ENE932
 	select BOARD_ROMSIZE_KB_8192
diff --git a/src/mainboard/google/stout/Kconfig b/src/mainboard/google/stout/Kconfig
index 3f73295b22..666c1ae3d4 100644
--- a/src/mainboard/google/stout/Kconfig
+++ b/src/mainboard/google/stout/Kconfig
@@ -4,7 +4,7 @@ config BOARD_SPECIFIC_OPTIONS # dummy
 	def_bool y
 	select SYSTEM_TYPE_LAPTOP
 	select CPU_INTEL_SOCKET_RPGA989
-	select NORTHBRIDGE_INTEL_IVYBRIDGE
+	select NORTHBRIDGE_INTEL_IVYBRIDGE_MRC
 	select SOUTHBRIDGE_INTEL_C216
 	select EC_QUANTA_IT8518
 	select BOARD_ROMSIZE_KB_8192
diff --git a/src/mainboard/intel/emeraldlake2/Kconfig b/src/mainboard/intel/emeraldlake2/Kconfig
index 90b40ac4bf..1d63e76a55 100644
--- a/src/mainboard/intel/emeraldlake2/Kconfig
+++ b/src/mainboard/intel/emeraldlake2/Kconfig
@@ -3,7 +3,7 @@ if BOARD_INTEL_EMERALDLAKE2
 config BOARD_SPECIFIC_OPTIONS # dummy
 	def_bool y
 	select CPU_INTEL_SOCKET_RPGA989
-	select NORTHBRIDGE_INTEL_IVYBRIDGE
+	select NORTHBRIDGE_INTEL_IVYBRIDGE_MRC
 	select SOUTHBRIDGE_INTEL_C216
 	select SUPERIO_SMSC_SIO1007
 	select BOARD_ROMSIZE_KB_8192
diff --git a/src/mainboard/kontron/ktqm77/Kconfig b/src/mainboard/kontron/ktqm77/Kconfig
index 485978c487..9dc75cc707 100644
--- a/src/mainboard/kontron/ktqm77/Kconfig
+++ b/src/mainboard/kontron/ktqm77/Kconfig
@@ -3,7 +3,7 @@ if BOARD_KONTRON_KTQM77
 config BOARD_SPECIFIC_OPTIONS # dummy
 	def_bool y
 	select CPU_INTEL_SOCKET_RPGA989
-	select NORTHBRIDGE_INTEL_IVYBRIDGE
+	select NORTHBRIDGE_INTEL_IVYBRIDGE_MRC
 	select SOUTHBRIDGE_INTEL_C216
 	select SUPERIO_WINBOND_W83627DHG
 	select EC_KONTRON_IT8516E
diff --git a/src/mainboard/lenovo/t420s/Kconfig b/src/mainboard/lenovo/t420s/Kconfig
index 47e70d60e7..65d37a75b1 100644
--- a/src/mainboard/lenovo/t420s/Kconfig
+++ b/src/mainboard/lenovo/t420s/Kconfig
@@ -4,7 +4,7 @@ config BOARD_SPECIFIC_OPTIONS # dummy
 	def_bool y
 	select SYSTEM_TYPE_LAPTOP
 	select CPU_INTEL_SOCKET_RPGA988B
-	select NORTHBRIDGE_INTEL_SANDYBRIDGE_NATIVE
+	select NORTHBRIDGE_INTEL_SANDYBRIDGE
 	select SOUTHBRIDGE_INTEL_BD82X6X
 	select EC_LENOVO_PMH7
 	select EC_LENOVO_H8
diff --git a/src/mainboard/lenovo/t430s/Kconfig b/src/mainboard/lenovo/t430s/Kconfig
index 0769466467..6e257ddbb1 100644
--- a/src/mainboard/lenovo/t430s/Kconfig
+++ b/src/mainboard/lenovo/t430s/Kconfig
@@ -4,7 +4,7 @@ config BOARD_SPECIFIC_OPTIONS # dummy
 	def_bool y
 	select SYSTEM_TYPE_LAPTOP
 	select CPU_INTEL_SOCKET_RPGA989
-	select NORTHBRIDGE_INTEL_IVYBRIDGE_NATIVE
+	select NORTHBRIDGE_INTEL_IVYBRIDGE
 	select SOUTHBRIDGE_INTEL_C216
 	select EC_LENOVO_PMH7
 	select EC_LENOVO_H8
diff --git a/src/mainboard/lenovo/t520/Kconfig b/src/mainboard/lenovo/t520/Kconfig
index 56fa395b28..df7c2db602 100644
--- a/src/mainboard/lenovo/t520/Kconfig
+++ b/src/mainboard/lenovo/t520/Kconfig
@@ -4,7 +4,7 @@ config BOARD_SPECIFIC_OPTIONS # dummy
 	def_bool y
 	select SYSTEM_TYPE_LAPTOP
 	select CPU_INTEL_SOCKET_RPGA988B
-	select NORTHBRIDGE_INTEL_SANDYBRIDGE_NATIVE
+	select NORTHBRIDGE_INTEL_SANDYBRIDGE
 	select SOUTHBRIDGE_INTEL_BD82X6X
 	select EC_LENOVO_PMH7
 	select EC_LENOVO_H8
diff --git a/src/mainboard/lenovo/t530/Kconfig b/src/mainboard/lenovo/t530/Kconfig
index f144afb77b..7b4ca7a6e6 100644
--- a/src/mainboard/lenovo/t530/Kconfig
+++ b/src/mainboard/lenovo/t530/Kconfig
@@ -4,7 +4,7 @@ config BOARD_SPECIFIC_OPTIONS # dummy
 	def_bool y
 	select SYSTEM_TYPE_LAPTOP
 	select CPU_INTEL_SOCKET_RPGA989
-	select NORTHBRIDGE_INTEL_IVYBRIDGE_NATIVE
+	select NORTHBRIDGE_INTEL_IVYBRIDGE
 	select SOUTHBRIDGE_INTEL_C216
 	select EC_LENOVO_PMH7
 	select EC_LENOVO_H8
diff --git a/src/mainboard/lenovo/x220/Kconfig b/src/mainboard/lenovo/x220/Kconfig
index 9fa95b917b..c13b644ba3 100644
--- a/src/mainboard/lenovo/x220/Kconfig
+++ b/src/mainboard/lenovo/x220/Kconfig
@@ -4,7 +4,7 @@ config BOARD_SPECIFIC_OPTIONS # dummy
 	def_bool y
 	select SYSTEM_TYPE_LAPTOP
 	select CPU_INTEL_SOCKET_RPGA989
-	select NORTHBRIDGE_INTEL_SANDYBRIDGE_NATIVE
+	select NORTHBRIDGE_INTEL_SANDYBRIDGE
 	select SOUTHBRIDGE_INTEL_C216
 	select EC_LENOVO_PMH7
 	select EC_LENOVO_H8
diff --git a/src/mainboard/lenovo/x230/Kconfig b/src/mainboard/lenovo/x230/Kconfig
index 1551e27f69..1d336ebf88 100644
--- a/src/mainboard/lenovo/x230/Kconfig
+++ b/src/mainboard/lenovo/x230/Kconfig
@@ -4,7 +4,7 @@ config BOARD_SPECIFIC_OPTIONS # dummy
 	def_bool y
 	select SYSTEM_TYPE_LAPTOP
 	select CPU_INTEL_SOCKET_RPGA989
-	select NORTHBRIDGE_INTEL_IVYBRIDGE_NATIVE
+	select NORTHBRIDGE_INTEL_IVYBRIDGE
 	select SOUTHBRIDGE_INTEL_C216
 	select EC_LENOVO_PMH7
 	select EC_LENOVO_H8
diff --git a/src/mainboard/samsung/lumpy/Kconfig b/src/mainboard/samsung/lumpy/Kconfig
index e14454592a..ac19be57b4 100644
--- a/src/mainboard/samsung/lumpy/Kconfig
+++ b/src/mainboard/samsung/lumpy/Kconfig
@@ -11,7 +11,7 @@ config BOARD_SPECIFIC_OPTIONS # dummy
 	select HAVE_ACPI_RESUME
 	select HAVE_ACPI_TABLES
 	select HAVE_OPTION_TABLE
-	select NORTHBRIDGE_INTEL_SANDYBRIDGE
+	select NORTHBRIDGE_INTEL_SANDYBRIDGE_MRC
 	select SOUTHBRIDGE_INTEL_BD82X6X
 	select SUPERIO_SMSC_MEC1308
 # LPC47N207 selected for external LPC card
diff --git a/src/mainboard/samsung/stumpy/Kconfig b/src/mainboard/samsung/stumpy/Kconfig
index 874dd6c83f..d4b8cc23b6 100644
--- a/src/mainboard/samsung/stumpy/Kconfig
+++ b/src/mainboard/samsung/stumpy/Kconfig
@@ -9,7 +9,7 @@ config BOARD_SPECIFIC_OPTIONS # dummy
 	select HAVE_ACPI_RESUME
 	select HAVE_ACPI_TABLES
 	select HAVE_OPTION_TABLE
-	select NORTHBRIDGE_INTEL_SANDYBRIDGE
+	select NORTHBRIDGE_INTEL_SANDYBRIDGE_MRC
 	select SOUTHBRIDGE_INTEL_BD82X6X
 	select SUPERIO_ITE_IT8772F
 # LPC47N207 selected for external LPC card
diff --git a/src/northbridge/intel/sandybridge/Kconfig b/src/northbridge/intel/sandybridge/Kconfig
index 093224f841..cbca042659 100644
--- a/src/northbridge/intel/sandybridge/Kconfig
+++ b/src/northbridge/intel/sandybridge/Kconfig
@@ -17,14 +17,14 @@
 ## Foundation, Inc.
 ##
 
-config NORTHBRIDGE_INTEL_SANDYBRIDGE
+config NORTHBRIDGE_INTEL_SANDYBRIDGE_MRC
 	bool
 	select MMCONF_SUPPORT
 	select MMCONF_SUPPORT_DEFAULT
 	select CPU_INTEL_MODEL_206AX
 	select INTEL_GMA_ACPI
 
-config NORTHBRIDGE_INTEL_SANDYBRIDGE_NATIVE
+config NORTHBRIDGE_INTEL_SANDYBRIDGE
 	bool
 	select MMCONF_SUPPORT
 	select MMCONF_SUPPORT_DEFAULT
@@ -32,14 +32,14 @@ config NORTHBRIDGE_INTEL_SANDYBRIDGE_NATIVE
 	select HAVE_DEBUG_RAM_SETUP
 	select INTEL_GMA_ACPI
 
-config NORTHBRIDGE_INTEL_IVYBRIDGE
+config NORTHBRIDGE_INTEL_IVYBRIDGE_MRC
 	bool
 	select MMCONF_SUPPORT
 	select MMCONF_SUPPORT_DEFAULT
 	select CPU_INTEL_MODEL_306AX
 	select INTEL_GMA_ACPI
 
-config NORTHBRIDGE_INTEL_IVYBRIDGE_NATIVE
+config NORTHBRIDGE_INTEL_IVYBRIDGE
 	bool
 	select MMCONF_SUPPORT
 	select MMCONF_SUPPORT_DEFAULT
@@ -47,7 +47,7 @@ config NORTHBRIDGE_INTEL_IVYBRIDGE_NATIVE
 	select HAVE_DEBUG_RAM_SETUP
 	select INTEL_GMA_ACPI
 
-if NORTHBRIDGE_INTEL_SANDYBRIDGE || NORTHBRIDGE_INTEL_IVYBRIDGE || NORTHBRIDGE_INTEL_IVYBRIDGE_NATIVE || NORTHBRIDGE_INTEL_SANDYBRIDGE_NATIVE
+if NORTHBRIDGE_INTEL_SANDYBRIDGE_MRC || NORTHBRIDGE_INTEL_IVYBRIDGE_MRC || NORTHBRIDGE_INTEL_IVYBRIDGE || NORTHBRIDGE_INTEL_SANDYBRIDGE
 
 config VGA_BIOS_ID
 	string
@@ -72,10 +72,10 @@ config MRC_CACHE_SIZE
 
 config DCACHE_RAM_BASE
 	hex
-	default 0xff7e0000 if NORTHBRIDGE_INTEL_IVYBRIDGE
-	default 0xff7e0000 if NORTHBRIDGE_INTEL_SANDYBRIDGE
-	default 0xfefe0000 if NORTHBRIDGE_INTEL_IVYBRIDGE_NATIVE
-	default 0xfefe0000 if NORTHBRIDGE_INTEL_SANDYBRIDGE_NATIVE
+	default 0xff7e0000 if NORTHBRIDGE_INTEL_IVYBRIDGE_MRC
+	default 0xff7e0000 if NORTHBRIDGE_INTEL_SANDYBRIDGE_MRC
+	default 0xfefe0000 if NORTHBRIDGE_INTEL_IVYBRIDGE
+	default 0xfefe0000 if NORTHBRIDGE_INTEL_SANDYBRIDGE
 
 config DCACHE_RAM_SIZE
 	hex
@@ -91,7 +91,7 @@ config DCACHE_RAM_MRC_VAR_SIZE
 
 config HAVE_MRC
 	bool "Add a System Agent binary"
-	depends on !NORTHBRIDGE_INTEL_IVYBRIDGE_NATIVE && !NORTHBRIDGE_INTEL_SANDYBRIDGE_NATIVE
+	depends on !NORTHBRIDGE_INTEL_IVYBRIDGE && !NORTHBRIDGE_INTEL_SANDYBRIDGE
 	help
 	  Select this option to add a System Agent binary to
 	  the resulting coreboot image.
diff --git a/src/northbridge/intel/sandybridge/Makefile.inc b/src/northbridge/intel/sandybridge/Makefile.inc
index 52fe23c513..49c0712a6f 100644
--- a/src/northbridge/intel/sandybridge/Makefile.inc
+++ b/src/northbridge/intel/sandybridge/Makefile.inc
@@ -17,7 +17,7 @@
 # Foundation, Inc.
 #
 
-ifeq ($(CONFIG_NORTHBRIDGE_INTEL_SANDYBRIDGE_NATIVE)$(CONFIG_NORTHBRIDGE_INTEL_SANDYBRIDGE)$(CONFIG_NORTHBRIDGE_INTEL_IVYBRIDGE_NATIVE)$(CONFIG_NORTHBRIDGE_INTEL_IVYBRIDGE),y)
+ifeq ($(CONFIG_NORTHBRIDGE_INTEL_SANDYBRIDGE)$(CONFIG_NORTHBRIDGE_INTEL_SANDYBRIDGE_MRC)$(CONFIG_NORTHBRIDGE_INTEL_IVYBRIDGE)$(CONFIG_NORTHBRIDGE_INTEL_IVYBRIDGE_MRC),y)
 
 ramstage-y += ram_calc.c
 ramstage-y += northbridge.c
@@ -29,14 +29,14 @@ ramstage-y += acpi.c
 ramstage-y += mrccache.c
 
 romstage-y += ram_calc.c
+romstage-$(CONFIG_NORTHBRIDGE_INTEL_IVYBRIDGE_MRC) += raminit_mrc.c
+romstage-$(CONFIG_NORTHBRIDGE_INTEL_SANDYBRIDGE_MRC) += raminit_mrc.c
 romstage-$(CONFIG_NORTHBRIDGE_INTEL_IVYBRIDGE) += raminit.c
+romstage-$(CONFIG_NORTHBRIDGE_INTEL_IVYBRIDGE) += romstage.c
+romstage-$(CONFIG_NORTHBRIDGE_INTEL_IVYBRIDGE) += ../../../device/dram/ddr3.c
 romstage-$(CONFIG_NORTHBRIDGE_INTEL_SANDYBRIDGE) += raminit.c
-romstage-$(CONFIG_NORTHBRIDGE_INTEL_IVYBRIDGE_NATIVE) += raminit_native.c
-romstage-$(CONFIG_NORTHBRIDGE_INTEL_IVYBRIDGE_NATIVE) += romstage_native.c
-romstage-$(CONFIG_NORTHBRIDGE_INTEL_IVYBRIDGE_NATIVE) += ../../../device/dram/ddr3.c
-romstage-$(CONFIG_NORTHBRIDGE_INTEL_SANDYBRIDGE_NATIVE) += raminit_native.c
-romstage-$(CONFIG_NORTHBRIDGE_INTEL_SANDYBRIDGE_NATIVE) += romstage_native.c
-romstage-$(CONFIG_NORTHBRIDGE_INTEL_SANDYBRIDGE_NATIVE) += ../../../device/dram/ddr3.c
+romstage-$(CONFIG_NORTHBRIDGE_INTEL_SANDYBRIDGE) += romstage.c
+romstage-$(CONFIG_NORTHBRIDGE_INTEL_SANDYBRIDGE) += ../../../device/dram/ddr3.c
 romstage-y += mrccache.c
 romstage-y += early_init.c
 romstage-y += report_platform.c
diff --git a/src/northbridge/intel/sandybridge/raminit.c b/src/northbridge/intel/sandybridge/raminit.c
index 053a4873cd..790f47c46c 100644
--- a/src/northbridge/intel/sandybridge/raminit.c
+++ b/src/northbridge/intel/sandybridge/raminit.c
@@ -1,7 +1,8 @@
 /*
  * This file is part of the coreboot project.
  *
- * Copyright (C) 2011 Google Inc.
+ * Copyright (C) 2014 Damien Zammit <damien@zamaudio.com>
+ * Copyright (C) 2014 Vladimir Serbinenko <phcoder@gmail.com>
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -25,144 +26,179 @@
 #include <cbmem.h>
 #include <arch/cbfs.h>
 #include <cbfs.h>
+#include <halt.h>
 #include <ip_checksum.h>
 #include <pc80/mc146818rtc.h>
 #include <device/pci_def.h>
-#include <halt.h>
-#include "raminit.h"
-#include "pei_data.h"
+#include "raminit_native.h"
 #include "sandybridge.h"
+#include <delay.h>
+#include <lib.h>
 
 /* Management Engine is in the southbridge */
 #include "southbridge/intel/bd82x6x/me.h"
+/* For SPD.  */
+#include "southbridge/intel/bd82x6x/smbus.h"
+#include "arch/cpu.h"
+#include "cpu/x86/msr.h"
 
-/*
- * MRC scrambler seed offsets should be reserved in
- * mainboard cmos.layout and not covered by checksum.
- */
-#if CONFIG_USE_OPTION_TABLE
-#include "option_table.h"
-#define CMOS_OFFSET_MRC_SEED     (CMOS_VSTART_mrc_scrambler_seed >> 3)
-#define CMOS_OFFSET_MRC_SEED_S3  (CMOS_VSTART_mrc_scrambler_seed_s3 >> 3)
-#define CMOS_OFFSET_MRC_SEED_CHK (CMOS_VSTART_mrc_scrambler_seed_chk >> 3)
-#else
-#define CMOS_OFFSET_MRC_SEED     152
-#define CMOS_OFFSET_MRC_SEED_S3  156
-#define CMOS_OFFSET_MRC_SEED_CHK 160
-#endif
-
-void save_mrc_data(struct pei_data *pei_data)
-{
-	u16 c1, c2, checksum;
-	struct mrc_data_container *mrcdata;
-	int output_len = ALIGN(pei_data->mrc_output_len, 16);
+/* FIXME: no ECC support.  */
+/* FIXME: no support for 3-channel chipsets.  */
 
-	/* Save the MRC S3 restore data to cbmem */
-	mrcdata = cbmem_add
-		(CBMEM_ID_MRCDATA,
-		 output_len + sizeof(struct mrc_data_container));
+#define BASEFREQ 133
+#define tDLLK 512
 
-	if (mrcdata != NULL) {
-		printk(BIOS_DEBUG, "Relocate MRC DATA from %p to %p (%u bytes)\n",
-			pei_data->mrc_output, mrcdata, output_len);
+#define IS_SANDY_CPU(x) ((x & 0xffff0) == 0x206a0)
+#define IS_SANDY_CPU_C(x) ((x & 0xf) == 4)
+#define IS_SANDY_CPU_D0(x) ((x & 0xf) == 5)
+#define IS_SANDY_CPU_D1(x) ((x & 0xf) == 6)
+#define IS_SANDY_CPU_D2(x) ((x & 0xf) == 7)
 
-		mrcdata->mrc_signature = MRC_DATA_SIGNATURE;
-		mrcdata->mrc_data_size = output_len;
-		mrcdata->reserved = 0;
-		memcpy(mrcdata->mrc_data, pei_data->mrc_output,
-			pei_data->mrc_output_len);
+#define IS_IVY_CPU(x) ((x & 0xffff0) == 0x306a0)
+#define IS_IVY_CPU_C(x) ((x & 0xf) == 4)
+#define IS_IVY_CPU_K(x) ((x & 0xf) == 5)
+#define IS_IVY_CPU_D(x) ((x & 0xf) == 6)
+#define IS_IVY_CPU_E(x) ((x & 0xf) >= 8)
 
-		/* Zero the unused space in aligned buffer. */
-		if (output_len > pei_data->mrc_output_len)
-			memset(mrcdata->mrc_data+pei_data->mrc_output_len, 0,
-			output_len - pei_data->mrc_output_len);
+#define NUM_CHANNELS 2
+#define NUM_SLOTRANKS 4
+#define NUM_SLOTS 2
+#define NUM_LANES 8
 
-		mrcdata->mrc_checksum = compute_ip_checksum(mrcdata->mrc_data,
-						mrcdata->mrc_data_size);
-	}
+/* FIXME: Vendor BIOS uses 64 but our algorithms are less
+   performant and even 1 seems to be enough in practice.  */
+#define NUM_PATTERNS 4
 
-	/* Save the MRC seed values to CMOS */
-	cmos_write32(CMOS_OFFSET_MRC_SEED, pei_data->scrambler_seed);
-	printk(BIOS_DEBUG, "Save scrambler seed    0x%08x to CMOS 0x%02x\n",
-	       pei_data->scrambler_seed, CMOS_OFFSET_MRC_SEED);
+typedef struct odtmap_st {
+	u16 rttwr;
+	u16 rttnom;
+} odtmap;
 
-	cmos_write32(CMOS_OFFSET_MRC_SEED_S3, pei_data->scrambler_seed_s3);
-	printk(BIOS_DEBUG, "Save s3 scrambler seed 0x%08x to CMOS 0x%02x\n",
-	       pei_data->scrambler_seed_s3, CMOS_OFFSET_MRC_SEED_S3);
+typedef struct dimm_info_st {
+	dimm_attr dimm[NUM_CHANNELS][NUM_SLOTS];
+} dimm_info;
 
-	/* Save a simple checksum of the seed values */
-	c1 = compute_ip_checksum((u8*)&pei_data->scrambler_seed,
-				 sizeof(u32));
-	c2 = compute_ip_checksum((u8*)&pei_data->scrambler_seed_s3,
-				 sizeof(u32));
-	checksum = add_ip_checksums(sizeof(u32), c1, c2);
+struct ram_rank_timings {
+	/* Register 4024. One byte per slotrank.  */
+	u8 val_4024;
+	/* Register 4028. One nibble per slotrank.  */
+	u8 val_4028;
 
-	cmos_write(checksum & 0xff, CMOS_OFFSET_MRC_SEED_CHK);
-	cmos_write((checksum >> 8) & 0xff, CMOS_OFFSET_MRC_SEED_CHK+1);
-}
+	int val_320c;
 
-static void prepare_mrc_cache(struct pei_data *pei_data)
-{
-	struct mrc_data_container *mrc_cache;
-	u16 c1, c2, checksum, seed_checksum;
+	struct ram_lane_timings {
+		/* lane register offset 0x10.  */
+		u16 timA;	/* bits 0 - 5, bits 16 - 18 */
+		u8 rising;	/* bits 8 - 14 */
+		u8 falling;	/* bits 20 - 26.  */
 
-	// preset just in case there is an error
-	pei_data->mrc_input = NULL;
-	pei_data->mrc_input_len = 0;
+		/* lane register offset 0x20.  */
+		int timC;	/* bit 0 - 5, 19.  */
+		u16 timB;	/* bits 8 - 13, 15 - 17.  */
+	} lanes[NUM_LANES];
+};
 
-	/* Read scrambler seeds from CMOS */
-	pei_data->scrambler_seed = cmos_read32(CMOS_OFFSET_MRC_SEED);
-	printk(BIOS_DEBUG, "Read scrambler seed    0x%08x from CMOS 0x%02x\n",
-	       pei_data->scrambler_seed, CMOS_OFFSET_MRC_SEED);
+struct ramctr_timing_st;
 
-	pei_data->scrambler_seed_s3 = cmos_read32(CMOS_OFFSET_MRC_SEED_S3);
-	printk(BIOS_DEBUG, "Read S3 scrambler seed 0x%08x from CMOS 0x%02x\n",
-	       pei_data->scrambler_seed_s3, CMOS_OFFSET_MRC_SEED_S3);
+typedef struct ramctr_timing_st {
+	int mobile;
 
-	/* Compute seed checksum and compare */
-	c1 = compute_ip_checksum((u8*)&pei_data->scrambler_seed,
-				 sizeof(u32));
-	c2 = compute_ip_checksum((u8*)&pei_data->scrambler_seed_s3,
-				 sizeof(u32));
-	checksum = add_ip_checksums(sizeof(u32), c1, c2);
+	u16 cas_supported;
+	/* tLatencies are in units of ns, scaled by x256 */
+	u32 tCK;
+	u32 tAA;
+	u32 tWR;
+	u32 tRCD;
+	u32 tRRD;
+	u32 tRP;
+	u32 tRAS;
+	u32 tRFC;
+	u32 tWTR;
+	u32 tRTP;
+	u32 tFAW;
+	/* Latencies in terms of clock cycles
+	 * They are saved separately as they are needed for DRAM MRS commands*/
+	u8 CAS;			/* CAS read latency */
+	u8 CWL;			/* CAS write latency */
 
-	seed_checksum = cmos_read(CMOS_OFFSET_MRC_SEED_CHK);
-	seed_checksum |= cmos_read(CMOS_OFFSET_MRC_SEED_CHK+1) << 8;
+	u32 tREFI;
+	u32 tMOD;
+	u32 tXSOffset;
+	u32 tWLO;
+	u32 tCKE;
+	u32 tXPDLL;
+	u32 tXP;
+	u32 tAONPD;
 
-	if (checksum != seed_checksum) {
-		printk(BIOS_ERR, "%s: invalid seed checksum\n", __func__);
-		pei_data->scrambler_seed = 0;
-		pei_data->scrambler_seed_s3 = 0;
-		return;
-	}
+	u16 reg_5064b0; /* bits 0-11. */
 
-	if ((mrc_cache = find_current_mrc_cache()) == NULL) {
-		/* error message printed in find_current_mrc_cache */
-		return;
-	}
+	u8 rankmap[NUM_CHANNELS];
+	int ref_card_offset[NUM_CHANNELS];
+	u32 mad_dimm[NUM_CHANNELS];
+	int channel_size_mb[NUM_CHANNELS];
+	u32 cmd_stretch[NUM_CHANNELS];
 
-	pei_data->mrc_input = mrc_cache->mrc_data;
-	pei_data->mrc_input_len = mrc_cache->mrc_data_size;
+	int reg_c14_offset;
+	int reg_320c_range_threshold;
 
-	printk(BIOS_DEBUG, "%s: at %p, size %x checksum %04x\n",
-	       __func__, pei_data->mrc_input,
-	       pei_data->mrc_input_len, mrc_cache->mrc_checksum);
-}
+	int edge_offset[3];
+	int timC_offset[3];
+
+	int extended_temperature_range;
+	int auto_self_refresh;
+
+	int rank_mirror[NUM_CHANNELS][NUM_SLOTRANKS];
 
-static const char* ecc_decoder[] = {
+	struct ram_rank_timings timings[NUM_CHANNELS][NUM_SLOTRANKS];
+} ramctr_timing;
+
+#define SOUTHBRIDGE PCI_DEV(0, 0x1f, 0)
+#define NORTHBRIDGE PCI_DEV(0, 0x0, 0)
+#define FOR_ALL_LANES for (lane = 0; lane < NUM_LANES; lane++)
+#define FOR_ALL_CHANNELS for (channel = 0; channel < NUM_CHANNELS; channel++)
+#define FOR_ALL_POPULATED_RANKS for (slotrank = 0; slotrank < NUM_SLOTRANKS; slotrank++) if (ctrl->rankmap[channel] & (1 << slotrank))
+#define FOR_ALL_POPULATED_CHANNELS for (channel = 0; channel < NUM_CHANNELS; channel++) if (ctrl->rankmap[channel])
+#define MAX_EDGE_TIMING 71
+#define MAX_TIMC 127
+#define MAX_TIMB 511
+#define MAX_TIMA 127
+
+static void program_timings(ramctr_timing * ctrl, int channel);
+
+static const char *ecc_decoder[] = {
 	"inactive",
 	"active on IO",
 	"disabled on IO",
 	"active"
 };
 
+static void wait_txt_clear(void)
+{
+	struct cpuid_result cp;
+
+	cp = cpuid_ext(0x1, 0x0);
+	/* Check if TXT is supported?  */
+	if (!(cp.ecx & 0x40))
+		return;
+	/* Some TXT public bit.  */
+	if (!(read32((void *)0xfed30010) & 1))
+		return;
+	/* Wait for TXT clear.  */
+	while (!(read8((void *)0xfed40000) & (1 << 7))) ;
+}
+
+static void sfence(void)
+{
+	asm volatile ("sfence");
+}
+
 /*
  * Dump in the log memory controller configuration as read from the memory
  * controller registers.
  */
 static void report_memory_config(void)
 {
-	u32 addr_decoder_common, addr_decode_ch[2];
+	u32 addr_decoder_common, addr_decode_ch[NUM_CHANNELS];
 	int i;
 
 	addr_decoder_common = MCHBAR32(0x5000);
@@ -170,16 +206,15 @@ static void report_memory_config(void)
 	addr_decode_ch[1] = MCHBAR32(0x5008);
 
 	printk(BIOS_DEBUG, "memcfg DDR3 clock %d MHz\n",
-	       (MCHBAR32(0x5e04) * 13333 * 2 + 50)/100);
+	       (MCHBAR32(0x5e04) * 13333 * 2 + 50) / 100);
 	printk(BIOS_DEBUG, "memcfg channel assignment: A: %d, B % d, C % d\n",
-	       addr_decoder_common & 3,
-	       (addr_decoder_common >> 2) & 3,
+	       addr_decoder_common & 3, (addr_decoder_common >> 2) & 3,
 	       (addr_decoder_common >> 4) & 3);
 
 	for (i = 0; i < ARRAY_SIZE(addr_decode_ch); i++) {
 		u32 ch_conf = addr_decode_ch[i];
-		printk(BIOS_DEBUG, "memcfg channel[%d] config (%8.8x):\n",
-		       i, ch_conf);
+		printk(BIOS_DEBUG, "memcfg channel[%d] config (%8.8x):\n", i,
+		       ch_conf);
 		printk(BIOS_DEBUG, "   ECC %s\n",
 		       ecc_decoder[(ch_conf >> 24) & 3]);
 		printk(BIOS_DEBUG, "   enhanced interleave mode %s\n",
@@ -199,95 +234,3669 @@ static void report_memory_config(void)
 	}
 }
 
-static void post_system_agent_init(struct pei_data *pei_data)
+static void post_system_agent_init(void)
 {
 	/* If PCIe init is skipped, set the PEG clock gating */
-	if (!pei_data->pcie_init)
-		MCHBAR32(0x7010) = MCHBAR32(0x7010) | 0x01;
+	MCHBAR32(0x7010) = MCHBAR32(0x7010) | 0x01;
 }
 
-/**
- * Find PEI executable in coreboot filesystem and execute it.
- *
- * @param pei_data: configuration data for UEFI PEI reference code
- */
-void sdram_initialize(struct pei_data *pei_data)
+void read_spd(spd_raw_data * spd, u8 addr)
 {
-	struct sys_info sysinfo;
-	int (*entry) (struct pei_data *pei_data) __attribute__ ((regparm(1)));
+	int j;
+	for (j = 0; j < 256; j++)
+		(*spd)[j] = do_smbus_read_byte(SMBUS_IO_BASE, addr, j);
+}
 
-	report_platform_info();
+static void dram_find_spds_ddr3(spd_raw_data * spd, dimm_info * dimm,
+				ramctr_timing * ctrl)
+{
+	int dimms = 0;
+	int channel, slot, spd_slot;
 
-	/* Wait for ME to be ready */
-	intel_early_me_init();
-	intel_early_me_uma_size();
+	memset (ctrl->rankmap, 0, sizeof (ctrl->rankmap));
+
+	ctrl->extended_temperature_range = 1;
+	ctrl->auto_self_refresh = 1;
+
+	FOR_ALL_CHANNELS {
+		ctrl->channel_size_mb[channel] = 0;
+
+		for (slot = 0; slot < NUM_SLOTS; slot++) {
+			spd_slot = 2 * channel + slot;
+			spd_decode_ddr3(&dimm->dimm[channel][slot], spd[spd_slot]);
+			if (dimm->dimm[channel][slot].dram_type != SPD_MEMORY_TYPE_SDRAM_DDR3) {
+				// set dimm invalid
+				dimm->dimm[channel][slot].ranks = 0;
+				dimm->dimm[channel][slot].size_mb = 0;
+				continue;
+			}
+
+			dram_print_spd_ddr3(&dimm->dimm[channel][slot]);
+			dimms++;
+			ctrl->rank_mirror[channel][slot * 2] = 0;
+			ctrl->rank_mirror[channel][slot * 2 + 1] = dimm->dimm[channel][slot].flags.pins_mirrored;
+			ctrl->channel_size_mb[channel] += dimm->dimm[channel][slot].size_mb;
+
+			ctrl->auto_self_refresh &= dimm->dimm[channel][slot].flags.asr;
+			ctrl->extended_temperature_range &= dimm->dimm[channel][slot].flags.ext_temp_refresh;
+
+			ctrl->rankmap[channel] |= ((1 << dimm->dimm[channel][slot].ranks) - 1) << (2 * slot);
+			printk(BIOS_DEBUG, "rankmap[%d] = 0x%x\n", channel, ctrl->rankmap[channel]);
+		}
+		if ((ctrl->rankmap[channel] & 3) && (ctrl->rankmap[channel] & 0xc)
+			&& dimm->dimm[channel][0].reference_card <= 5 && dimm->dimm[channel][1].reference_card <= 5) {
+			const int ref_card_offset_table[6][6] = {
+				{ 0, 0, 0, 0, 2, 2, },
+				{ 0, 0, 0, 0, 2, 2, },
+				{ 0, 0, 0, 0, 2, 2, },
+				{ 0, 0, 0, 0, 1, 1, },
+				{ 2, 2, 2, 1, 0, 0, },
+				{ 2, 2, 2, 1, 0, 0, },
+			};
+			ctrl->ref_card_offset[channel] = ref_card_offset_table[dimm->dimm[channel][0].reference_card]
+				[dimm->dimm[channel][1].reference_card];
+		} else
+			ctrl->ref_card_offset[channel] = 0;
+	}
 
-	printk(BIOS_DEBUG, "Starting UEFI PEI System Agent\n");
+	if (!dimms)
+		die("No DIMMs were found");
+}
+
+static void dram_find_common_params(const dimm_info * dimms,
+				    ramctr_timing * ctrl)
+{
+	size_t valid_dimms;
+	int channel, slot;
+	ctrl->cas_supported = 0xff;
+	valid_dimms = 0;
+	FOR_ALL_CHANNELS for (slot = 0; slot < 2; slot++) {
+		const dimm_attr *dimm = &dimms->dimm[channel][slot];
+		if (dimm->dram_type != SPD_MEMORY_TYPE_SDRAM_DDR3)
+			continue;
+		valid_dimms++;
+
+		/* Find all possible CAS combinations */
+		ctrl->cas_supported &= dimm->cas_supported;
 
-	memset(&sysinfo, 0, sizeof(sysinfo));
+		/* Find the smallest common latencies supported by all DIMMs */
+		ctrl->tCK = MAX(ctrl->tCK, dimm->tCK);
+		ctrl->tAA = MAX(ctrl->tAA, dimm->tAA);
+		ctrl->tWR = MAX(ctrl->tWR, dimm->tWR);
+		ctrl->tRCD = MAX(ctrl->tRCD, dimm->tRCD);
+		ctrl->tRRD = MAX(ctrl->tRRD, dimm->tRRD);
+		ctrl->tRP = MAX(ctrl->tRP, dimm->tRP);
+		ctrl->tRAS = MAX(ctrl->tRAS, dimm->tRAS);
+		ctrl->tRFC = MAX(ctrl->tRFC, dimm->tRFC);
+		ctrl->tWTR = MAX(ctrl->tWTR, dimm->tWTR);
+		ctrl->tRTP = MAX(ctrl->tRTP, dimm->tRTP);
+		ctrl->tFAW = MAX(ctrl->tFAW, dimm->tFAW);
+	}
 
-	sysinfo.boot_path = pei_data->boot_mode;
+	if (!ctrl->cas_supported)
+		die("Unsupported DIMM combination. "
+		    "DIMMS do not support common CAS latency");
+	if (!valid_dimms)
+		die("No valid DIMMs found");
+}
 
-	/*
-	 * Do not pass MRC data in for recovery mode boot,
-	 * Always pass it in for S3 resume.
+static u8 get_CWL(u8 CAS)
+{
+	/* Get CWL based on CAS using the following rule:
+	 *       _________________________________________
+	 * CAS: | 4T | 5T | 6T | 7T | 8T | 9T | 10T | 11T |
+	 * CWL: | 5T | 5T | 5T | 6T | 6T | 7T |  7T |  8T |
 	 */
-	if (!recovery_mode_enabled() || pei_data->boot_mode == 2)
-		prepare_mrc_cache(pei_data);
+	static const u8 cas_cwl_map[] = { 5, 5, 5, 6, 6, 7, 7, 8 };
+	if (CAS > 11)
+		return 8;
+	return cas_cwl_map[CAS - 4];
+}
 
-	/* If MRC data is not found we cannot continue S3 resume. */
-	if (pei_data->boot_mode == 2 && !pei_data->mrc_input) {
-		printk(BIOS_DEBUG, "Giving up in sdram_initialize: No MRC data\n");
-		outb(0x6, 0xcf9);
-		halt();
+/* Frequency multiplier.  */
+static u32 get_FRQ(u32 tCK)
+{
+	u32 FRQ;
+	FRQ = 256000 / (tCK * BASEFREQ);
+	if (FRQ > 8)
+		return 8;
+	if (FRQ < 3)
+		return 3;
+	return FRQ;
+}
+
+static u32 get_REFI(u32 tCK)
+{
+	/* Get REFI based on MCU frequency using the following rule:
+	 *        _________________________________________
+	 * FRQ : | 3    | 4    | 5    | 6    | 7    | 8    |
+	 * REFI: | 3120 | 4160 | 5200 | 6240 | 7280 | 8320 |
+	 */
+	static const u32 frq_refi_map[] =
+	    { 3120, 4160, 5200, 6240, 7280, 8320 };
+	return frq_refi_map[get_FRQ(tCK) - 3];
+}
+
+static u8 get_XSOffset(u32 tCK)
+{
+	/* Get XSOffset based on MCU frequency using the following rule:
+	 *             _________________________
+	 * FRQ      : | 3 | 4 | 5 | 6 | 7  | 8  |
+	 * XSOffset : | 4 | 6 | 7 | 8 | 10 | 11 |
+	 */
+	static const u8 frq_xs_map[] = { 4, 6, 7, 8, 10, 11 };
+	return frq_xs_map[get_FRQ(tCK) - 3];
+}
+
+static u8 get_MOD(u32 tCK)
+{
+	/* Get MOD based on MCU frequency using the following rule:
+	 *        _____________________________
+	 * FRQ : | 3  | 4  | 5  | 6  | 7  | 8  |
+	 * MOD : | 12 | 12 | 12 | 12 | 15 | 16 |
+	 */
+	static const u8 frq_mod_map[] = { 12, 12, 12, 12, 15, 16 };
+	return frq_mod_map[get_FRQ(tCK) - 3];
+}
+
+static u8 get_WLO(u32 tCK)
+{
+	/* Get WLO based on MCU frequency using the following rule:
+	 *        _______________________
+	 * FRQ : | 3 | 4 | 5 | 6 | 7 | 8 |
+	 * WLO : | 4 | 5 | 6 | 6 | 8 | 8 |
+	 */
+	static const u8 frq_wlo_map[] = { 4, 5, 6, 6, 8, 8 };
+	return frq_wlo_map[get_FRQ(tCK) - 3];
+}
+
+static u8 get_CKE(u32 tCK)
+{
+	/* Get CKE based on MCU frequency using the following rule:
+	 *        _______________________
+	 * FRQ : | 3 | 4 | 5 | 6 | 7 | 8 |
+	 * CKE : | 3 | 3 | 4 | 4 | 5 | 6 |
+	 */
+	static const u8 frq_cke_map[] = { 3, 3, 4, 4, 5, 6 };
+	return frq_cke_map[get_FRQ(tCK) - 3];
+}
+
+static u8 get_XPDLL(u32 tCK)
+{
+	/* Get XPDLL based on MCU frequency using the following rule:
+	 *          _____________________________
+	 * FRQ   : | 3  | 4  | 5  | 6  | 7  | 8  |
+	 * XPDLL : | 10 | 13 | 16 | 20 | 23 | 26 |
+	 */
+	static const u8 frq_xpdll_map[] = { 10, 13, 16, 20, 23, 26 };
+	return frq_xpdll_map[get_FRQ(tCK) - 3];
+}
+
+static u8 get_XP(u32 tCK)
+{
+	/* Get XP based on MCU frequency using the following rule:
+	 *        _______________________
+	 * FRQ : | 3 | 4 | 5 | 6 | 7 | 8 |
+	 * XP  : | 3 | 4 | 4 | 5 | 6 | 7 |
+	 */
+	static const u8 frq_xp_map[] = { 3, 4, 4, 5, 6, 7 };
+	return frq_xp_map[get_FRQ(tCK) - 3];
+}
+
+static u8 get_AONPD(u32 tCK)
+{
+	/* Get AONPD based on MCU frequency using the following rule:
+	 *          ________________________
+	 * FRQ   : | 3 | 4 | 5 | 6 | 7 | 8  |
+	 * AONPD : | 4 | 5 | 6 | 8 | 8 | 10 |
+	 */
+	static const u8 frq_aonpd_map[] = { 4, 5, 6, 8, 8, 10 };
+	return frq_aonpd_map[get_FRQ(tCK) - 3];
+}
+
+static u32 get_COMP2(u32 tCK)
+{
+	/* Get COMP2 based on MCU frequency using the following rule:
+	 *         ___________________________________________________________
+	 * FRQ  : | 3       | 4       | 5       | 6       | 7       | 8       |
+	 * COMP : | D6BEDCC | CE7C34C | CA57A4C | C6369CC | C42514C | C21410C |
+	 */
+	static const u32 frq_comp2_map[] = { 0xD6BEDCC, 0xCE7C34C, 0xCA57A4C,
+		0xC6369CC, 0xC42514C, 0xC21410C
+	};
+	return frq_comp2_map[get_FRQ(tCK) - 3];
+}
+
+static void dram_timing(ramctr_timing * ctrl)
+{
+	u8 val;
+	u32 val32;
+
+	/* Maximum supported DDR3 frequency is 1066MHz (DDR3 2133) so make sure
+	 * we cap it if we have faster DIMMs.
+	 * Then, align it to the closest JEDEC standard frequency */
+	if (ctrl->tCK <= TCK_1066MHZ) {
+		ctrl->tCK = TCK_1066MHZ;
+		ctrl->edge_offset[0] = 16;
+		ctrl->edge_offset[1] = 7;
+		ctrl->edge_offset[2] = 7;
+		ctrl->timC_offset[0] = 18;
+		ctrl->timC_offset[1] = 7;
+		ctrl->timC_offset[2] = 7;
+		ctrl->reg_c14_offset = 16;
+		ctrl->reg_5064b0 = 0x218;
+		ctrl->reg_320c_range_threshold = 13;
+	} else if (ctrl->tCK <= TCK_933MHZ) {
+		ctrl->tCK = TCK_933MHZ;
+		ctrl->edge_offset[0] = 14;
+		ctrl->edge_offset[1] = 6;
+		ctrl->edge_offset[2] = 6;
+		ctrl->timC_offset[0] = 15;
+		ctrl->timC_offset[1] = 6;
+		ctrl->timC_offset[2] = 6;
+		ctrl->reg_c14_offset = 14;
+		ctrl->reg_5064b0 = 0x1d5;
+		ctrl->reg_320c_range_threshold = 15;
+	} else if (ctrl->tCK <= TCK_800MHZ) {
+		ctrl->tCK = TCK_800MHZ;
+		ctrl->edge_offset[0] = 13;
+		ctrl->edge_offset[1] = 5;
+		ctrl->edge_offset[2] = 5;
+		ctrl->timC_offset[0] = 14;
+		ctrl->timC_offset[1] = 5;
+		ctrl->timC_offset[2] = 5;
+		ctrl->reg_c14_offset = 12;
+		ctrl->reg_5064b0 = 0x193;
+		ctrl->reg_320c_range_threshold = 15;
+	} else if (ctrl->tCK <= TCK_666MHZ) {
+		ctrl->tCK = TCK_666MHZ;
+		ctrl->edge_offset[0] = 10;
+		ctrl->edge_offset[1] = 4;
+		ctrl->edge_offset[2] = 4;
+		ctrl->timC_offset[0] = 11;
+		ctrl->timC_offset[1] = 4;
+		ctrl->timC_offset[2] = 4;
+		ctrl->reg_c14_offset = 10;
+		ctrl->reg_5064b0 = 0x150;
+		ctrl->reg_320c_range_threshold = 16;
+	} else if (ctrl->tCK <= TCK_533MHZ) {
+		ctrl->tCK = TCK_533MHZ;
+		ctrl->edge_offset[0] = 8;
+		ctrl->edge_offset[1] = 3;
+		ctrl->edge_offset[2] = 3;
+		ctrl->timC_offset[0] = 9;
+		ctrl->timC_offset[1] = 3;
+		ctrl->timC_offset[2] = 3;
+		ctrl->reg_c14_offset = 8;
+		ctrl->reg_5064b0 = 0x10d;
+		ctrl->reg_320c_range_threshold = 17;
+	} else  {
+		ctrl->tCK = TCK_400MHZ;
+		ctrl->edge_offset[0] = 6;
+		ctrl->edge_offset[1] = 2;
+		ctrl->edge_offset[2] = 2;
+		ctrl->timC_offset[0] = 6;
+		ctrl->timC_offset[1] = 2;
+		ctrl->timC_offset[2] = 2;
+		ctrl->reg_c14_offset = 8;
+		ctrl->reg_5064b0 = 0xcd;
+		ctrl->reg_320c_range_threshold = 17;
+	}
+
+	val32 = (1000 << 8) / ctrl->tCK;
+	printk(BIOS_DEBUG, "Selected DRAM frequency: %u MHz\n", val32);
+
+	/* Find CAS and CWL latencies */
+	val = (ctrl->tAA + ctrl->tCK - 1) / ctrl->tCK;
+	printk(BIOS_DEBUG, "Minimum  CAS latency   : %uT\n", val);
+	/* Find lowest supported CAS latency that satisfies the minimum value */
+	while (!((ctrl->cas_supported >> (val - 4)) & 1)
+	       && (ctrl->cas_supported >> (val - 4))) {
+		val++;
+	}
+	/* Is CAS supported */
+	if (!(ctrl->cas_supported & (1 << (val - 4))))
+		printk(BIOS_DEBUG, "CAS not supported\n");
+	printk(BIOS_DEBUG, "Selected CAS latency   : %uT\n", val);
+	ctrl->CAS = val;
+	ctrl->CWL = get_CWL(ctrl->CAS);
+	printk(BIOS_DEBUG, "Selected CWL latency   : %uT\n", ctrl->CWL);
+
+	/* Find tRCD */
+	ctrl->tRCD = (ctrl->tRCD + ctrl->tCK - 1) / ctrl->tCK;
+	printk(BIOS_DEBUG, "Selected tRCD          : %uT\n", ctrl->tRCD);
+
+	ctrl->tRP = (ctrl->tRP + ctrl->tCK - 1) / ctrl->tCK;
+	printk(BIOS_DEBUG, "Selected tRP           : %uT\n", ctrl->tRP);
+
+	/* Find tRAS */
+	ctrl->tRAS = (ctrl->tRAS + ctrl->tCK - 1) / ctrl->tCK;
+	printk(BIOS_DEBUG, "Selected tRAS          : %uT\n", ctrl->tRAS);
+
+	/* Find tWR */
+	ctrl->tWR = (ctrl->tWR + ctrl->tCK - 1) / ctrl->tCK;
+	printk(BIOS_DEBUG, "Selected tWR           : %uT\n", ctrl->tWR);
+
+	/* Find tFAW */
+	ctrl->tFAW = (ctrl->tFAW + ctrl->tCK - 1) / ctrl->tCK;
+	printk(BIOS_DEBUG, "Selected tFAW          : %uT\n", ctrl->tFAW);
+
+	/* Find tRRD */
+	ctrl->tRRD = (ctrl->tRRD + ctrl->tCK - 1) / ctrl->tCK;
+	printk(BIOS_DEBUG, "Selected tRRD          : %uT\n", ctrl->tRRD);
+
+	/* Find tRTP */
+	ctrl->tRTP = (ctrl->tRTP + ctrl->tCK - 1) / ctrl->tCK;
+	printk(BIOS_DEBUG, "Selected tRTP          : %uT\n", ctrl->tRTP);
+
+	/* Find tWTR */
+	ctrl->tWTR = (ctrl->tWTR + ctrl->tCK - 1) / ctrl->tCK;
+	printk(BIOS_DEBUG, "Selected tWTR          : %uT\n", ctrl->tWTR);
+
+	/* Refresh-to-Active or Refresh-to-Refresh (tRFC) */
+	ctrl->tRFC = (ctrl->tRFC + ctrl->tCK - 1) / ctrl->tCK;
+	printk(BIOS_DEBUG, "Selected tRFC          : %uT\n", ctrl->tRFC);
+
+	ctrl->tREFI = get_REFI(ctrl->tCK);
+	ctrl->tMOD = get_MOD(ctrl->tCK);
+	ctrl->tXSOffset = get_XSOffset(ctrl->tCK);
+	ctrl->tWLO = get_WLO(ctrl->tCK);
+	ctrl->tCKE = get_CKE(ctrl->tCK);
+	ctrl->tXPDLL = get_XPDLL(ctrl->tCK);
+	ctrl->tXP = get_XP(ctrl->tCK);
+	ctrl->tAONPD = get_AONPD(ctrl->tCK);
+}
+
+static void dram_freq(ramctr_timing * ctrl)
+{
+	if (ctrl->tCK > TCK_400MHZ) {
+		printk (BIOS_ERR, "DRAM frequency is under lowest supported frequency (400 MHz). Increasing to 400 MHz as last resort");
+		ctrl->tCK = TCK_400MHZ;
+	}
+	while (1) {
+		u8 val2;
+		u32 reg1 = 0;
+
+		/* Step 1 - Set target PCU frequency */
+
+		if (ctrl->tCK <= TCK_1066MHZ) {
+			ctrl->tCK = TCK_1066MHZ;
+		} else if (ctrl->tCK <= TCK_933MHZ) {
+			ctrl->tCK = TCK_933MHZ;
+		} else if (ctrl->tCK <= TCK_800MHZ) {
+			ctrl->tCK = TCK_800MHZ;
+		} else if (ctrl->tCK <= TCK_666MHZ) {
+			ctrl->tCK = TCK_666MHZ;
+		} else if (ctrl->tCK <= TCK_533MHZ) {
+			ctrl->tCK = TCK_533MHZ;
+		} else if (ctrl->tCK <= TCK_400MHZ) {
+			ctrl->tCK = TCK_400MHZ;
+		} else {
+			die ("No lock frequency found");
+		}
+
+		/* Frequency mulitplier.  */
+		u32 FRQ = get_FRQ(ctrl->tCK);
+
+		/* Step 2 - Select frequency in the MCU */
+		reg1 = FRQ;
+		reg1 |= 0x80000000;	// set running bit
+		MCHBAR32(0x5e00) = reg1;
+		while (reg1 & 0x80000000) {
+			printk(BIOS_DEBUG, " PLL busy...");
+			reg1 = MCHBAR32(0x5e00);
+		}
+		printk(BIOS_DEBUG, "done\n");
+
+		/* Step 3 - Verify lock frequency */
+		reg1 = MCHBAR32(0x5e04);
+		val2 = (u8) reg1;
+		if (val2 >= FRQ) {
+			printk(BIOS_DEBUG, "MCU frequency is set at : %d MHz\n",
+			       (1000 << 8) / ctrl->tCK);
+			return;
+		}
+		printk(BIOS_DEBUG, "PLL didn't lock. Retrying at lower frequency\n");
+		ctrl->tCK++;
+	}
+}
+
+static void dram_xover(ramctr_timing * ctrl)
+{
+	u32 reg;
+	int channel;
+
+	FOR_ALL_CHANNELS {
+		// enable xover clk
+		printk(BIOS_DEBUG, "[%x] = %x\n", channel * 0x100 + 0xc14,
+		       (ctrl->rankmap[channel] << 24));
+		MCHBAR32(channel * 0x100 + 0xc14) = (ctrl->rankmap[channel] << 24);
+
+		// enable xover ctl
+		reg = 0;
+		if (ctrl->rankmap[channel] & 0x5) {
+			reg |= 0x20000;
+		}
+		if (ctrl->rankmap[channel] & 0xa) {
+			reg |= 0x4000000;
+		}
+		// enable xover cmd
+		reg |= 0x4000;
+		printk(BIOS_DEBUG, "[%x] = %x\n", 0x100 * channel + 0x320c,
+		       reg);
+		MCHBAR32(0x100 * channel + 0x320c) = reg;
+	}
+}
+
+static void dram_timing_regs(ramctr_timing * ctrl)
+{
+	u32 reg, addr, val32, cpu, stretch;
+	struct cpuid_result cpures;
+	int channel;
+
+	FOR_ALL_CHANNELS {
+		// DBP
+		reg = 0;
+		reg |= ctrl->tRCD;
+		reg |= (ctrl->tRP << 4);
+		reg |= (ctrl->CAS << 8);
+		reg |= (ctrl->CWL << 12);
+		reg |= (ctrl->tRAS << 16);
+		printk(BIOS_DEBUG, "[%x] = %x\n", 0x400 * channel + 0x4000,
+		       reg);
+		MCHBAR32(0x400 * channel + 0x4000) = reg;
+
+		// RAP
+		reg = 0;
+		reg |= ctrl->tRRD;
+		reg |= (ctrl->tRTP << 4);
+		reg |= (ctrl->tCKE << 8);
+		reg |= (ctrl->tWTR << 12);
+		reg |= (ctrl->tFAW << 16);
+		reg |= (ctrl->tWR << 24);
+		reg |= (3 << 30);
+		printk(BIOS_DEBUG, "[%x] = %x\n", 0x400 * channel + 0x4004,
+		       reg);
+		MCHBAR32(0x400 * channel + 0x4004) = reg;
+
+		// OTHP
+		addr = 0x400 * channel + 0x400c;
+		reg = 0;
+		reg |= ctrl->tXPDLL;
+		reg |= (ctrl->tXP << 5);
+		reg |= (ctrl->tAONPD << 8);
+		reg |= 0xa0000;
+		printk(BIOS_DEBUG, "[%x] = %x\n", addr, reg);
+		MCHBAR32(addr) = reg;
+
+		MCHBAR32(0x400 * channel + 0x4014) = 0;
+
+		MCHBAR32(addr) |= 0x00020000;
+
+		// ODT stretch
+		reg = 0;
+
+		cpures = cpuid(0);
+		cpu = cpures.eax;
+		if (IS_IVY_CPU(cpu)
+		    || (IS_SANDY_CPU(cpu) && IS_SANDY_CPU_D2(cpu))) {
+			stretch = 2;
+			addr = 0x400 * channel + 0x400c;
+			printk(BIOS_DEBUG, "[%x] = %x\n",
+			       0x400 * channel + 0x400c, reg);
+			reg = MCHBAR32(addr);
+
+			if (((ctrl->rankmap[channel] & 3) == 0)
+			    || (ctrl->rankmap[channel] & 0xc) == 0) {
+
+				// Rank 0 - operate on rank 2
+				reg = (reg & ~0xc0000) | (stretch << 18);
+
+				// Rank 2 - operate on rank 0
+				reg = (reg & ~0x30000) | (stretch << 16);
+
+				printk(BIOS_DEBUG, "[%x] = %x\n", addr, reg);
+				MCHBAR32(addr) = reg;
+			}
+
+		} else if (IS_SANDY_CPU(cpu) && IS_SANDY_CPU_C(cpu)) {
+			stretch = 3;
+			addr = 0x400 * channel + 0x401c;
+			reg = MCHBAR32(addr);
+
+			if (((ctrl->rankmap[channel] & 3) == 0)
+			    || (ctrl->rankmap[channel] & 0xc) == 0) {
+
+				// Rank 0 - operate on rank 2
+				reg = (reg & ~0x3000) | (stretch << 12);
+
+				// Rank 2 - operate on rank 0
+				reg = (reg & ~0xc00) | (stretch << 10);
+
+				printk(BIOS_DEBUG, "[%x] = %x\n", addr, reg);
+				MCHBAR32(addr) = reg;
+			}
+		} else {
+			stretch = 0;
+		}
+
+		// REFI
+		reg = 0;
+		val32 = ctrl->tREFI;
+		reg = (reg & ~0xffff) | val32;
+		val32 = ctrl->tRFC;
+		reg = (reg & ~0x1ff0000) | (val32 << 16);
+		val32 = (u32) (ctrl->tREFI * 9) / 1024;
+		reg = (reg & ~0xfe000000) | (val32 << 25);
+		printk(BIOS_DEBUG, "[%x] = %x\n", 0x400 * channel + 0x4298,
+		       reg);
+		MCHBAR32(0x400 * channel + 0x4298) = reg;
+
+		MCHBAR32(0x400 * channel + 0x4294) |= 0xff;
+
+		// SRFTP
+		reg = 0;
+		val32 = tDLLK;
+		reg = (reg & ~0xfff) | val32;
+		val32 = ctrl->tXSOffset;
+		reg = (reg & ~0xf000) | (val32 << 12);
+		val32 = tDLLK - ctrl->tXSOffset;
+		reg = (reg & ~0x3ff0000) | (val32 << 16);
+		val32 = ctrl->tMOD - 8;
+		reg = (reg & ~0xf0000000) | (val32 << 28);
+		printk(BIOS_DEBUG, "[%x] = %x\n", 0x400 * channel + 0x42a4,
+		       reg);
+		MCHBAR32(0x400 * channel + 0x42a4) = reg;
 	}
+}
+
+static void dram_dimm_mapping(dimm_info * info, ramctr_timing * ctrl)
+{
+	u32 reg, val32;
+	int channel;
+
+	FOR_ALL_CHANNELS {
+		dimm_attr *dimmA = 0;
+		dimm_attr *dimmB = 0;
+		reg = 0;
+		val32 = 0;
+		if (info->dimm[channel][0].size_mb >=
+		    info->dimm[channel][1].size_mb) {
+			// dimm 0 is bigger, set it to dimmA
+			dimmA = &info->dimm[channel][0];
+			dimmB = &info->dimm[channel][1];
+			reg |= (0 << 16);
+		} else {
+			// dimm 1 is bigger, set it to dimmA
+			dimmA = &info->dimm[channel][1];
+			dimmB = &info->dimm[channel][0];
+			reg |= (1 << 16);
+		}
+		// dimmA
+		if (dimmA && (dimmA->ranks > 0)) {
+			val32 = dimmA->size_mb / 256;
+			reg = (reg & ~0xff) | val32;
+			val32 = dimmA->ranks - 1;
+			reg = (reg & ~0x20000) | (val32 << 17);
+			val32 = (dimmA->width / 8) - 1;
+			reg = (reg & ~0x80000) | (val32 << 19);
+		}
+		// dimmB
+		if (dimmB && (dimmB->ranks > 0)) {
+			val32 = dimmB->size_mb / 256;
+			reg = (reg & ~0xff00) | (val32 << 8);
+			val32 = dimmB->ranks - 1;
+			reg = (reg & ~0x40000) | (val32 << 18);
+			val32 = (dimmB->width / 8) - 1;
+			reg = (reg & ~0x100000) | (val32 << 20);
+		}
+		reg = (reg & ~0x200000) | (1 << 21);	// rank interleave
+		reg = (reg & ~0x400000) | (1 << 22);	// enhanced interleave
+
+		// Save MAD-DIMM register
+		if ((dimmA && (dimmA->ranks > 0))
+		    || (dimmB && (dimmB->ranks > 0))) {
+			ctrl->mad_dimm[channel] = reg;
+		} else {
+			ctrl->mad_dimm[channel] = 0;
+		}
+	}
+}
+
+static void dram_dimm_set_mapping(ramctr_timing * ctrl)
+{
+	int channel;
+	FOR_ALL_CHANNELS {
+		MCHBAR32(0x5004 + channel * 4) = ctrl->mad_dimm[channel];
+	}
+}
+
+static void dram_zones(ramctr_timing * ctrl, int training)
+{
+	u32 reg, ch0size, ch1size;
+	u8 val;
+	reg = 0;
+	val = 0;
+	if (training) {
+		ch0size = ctrl->channel_size_mb[0] ? 256 : 0;
+		ch1size = ctrl->channel_size_mb[1] ? 256 : 0;
+	} else {
+		ch0size = ctrl->channel_size_mb[0];
+		ch1size = ctrl->channel_size_mb[1];
+	}
+
+	if (ch0size >= ch1size) {
+		reg = MCHBAR32(0x5014);
+		val = ch1size / 256;
+		reg = (reg & ~0xff000000) | val << 24;
+		reg = (reg & ~0xff0000) | (2 * val) << 16;
+		MCHBAR32(0x5014) = reg;
+		MCHBAR32(0x5000) = 0x24;
+	} else {
+		reg = MCHBAR32(0x5014);
+		val = ch0size / 256;
+		reg = (reg & ~0xff000000) | val << 24;
+		reg = (reg & ~0xff0000) | (2 * val) << 16;
+		MCHBAR32(0x5014) = reg;
+		MCHBAR32(0x5000) = 0x21;
+	}
+}
+
+static void dram_memorymap(ramctr_timing * ctrl, int me_uma_size)
+{
+	u32 reg, val, reclaim;
+	u32 tom, gfxstolen, gttsize;
+	size_t tsegsize, mmiosize, toludbase, touudbase, gfxstolenbase, gttbase,
+	    tsegbase, mestolenbase;
+	size_t tsegbasedelta, remapbase, remaplimit;
+	uint16_t ggc;
+
+	mmiosize = 0x400;
+
+	ggc = pci_read_config16(NORTHBRIDGE, GGC);
+	if (!(ggc & 2)) {
+		gfxstolen = ((ggc >> 3) & 0x1f) * 32;
+		gttsize = ((ggc >> 8) & 0x3);
+	} else {
+		gfxstolen = 0;
+		gttsize = 0;
+	}
+
+	tsegsize = CONFIG_SMM_TSEG_SIZE >> 20;
+
+	tom = ctrl->channel_size_mb[0] + ctrl->channel_size_mb[1];
+
+	mestolenbase = tom - me_uma_size;
+
+	toludbase = MIN(4096 - mmiosize + gfxstolen + gttsize + tsegsize,
+			tom - me_uma_size);
+	gfxstolenbase = toludbase - gfxstolen;
+	gttbase = gfxstolenbase - gttsize;
+
+	tsegbase = gttbase - tsegsize;
+
+	// Round tsegbase down to nearest address aligned to tsegsize
+	tsegbasedelta = tsegbase & (tsegsize - 1);
+	tsegbase &= ~(tsegsize - 1);
+
+	gttbase -= tsegbasedelta;
+	gfxstolenbase -= tsegbasedelta;
+	toludbase -= tsegbasedelta;
+
+	// Test if it is possible to reclaim a hole in the ram addressing
+	if (tom - me_uma_size > toludbase) {
+		// Reclaim is possible
+		reclaim = 1;
+		remapbase = MAX(4096, tom - me_uma_size);
+		remaplimit =
+		    remapbase + MIN(4096, tom - me_uma_size) - toludbase - 1;
+		touudbase = remaplimit + 1;
+	} else {
+		// Reclaim not possible
+		reclaim = 0;
+		touudbase = tom - me_uma_size;
+	}
+
+	// Update memory map in pci-e configuration space
+
+	// TOM (top of memory)
+	reg = pcie_read_config32(PCI_DEV(0, 0, 0), 0xa0);
+	val = tom & 0xfff;
+	reg = (reg & ~0xfff00000) | (val << 20);
+	printk(BIOS_DEBUG, "PCI:[%x] = %x\n", 0xa0, reg);
+	pcie_write_config32(PCI_DEV(0, 0, 0), 0xa0, reg);
+
+	reg = pcie_read_config32(PCI_DEV(0, 0, 0), 0xa4);
+	val = tom & 0xfffff000;
+	reg = (reg & ~0x000fffff) | (val >> 12);
+	printk(BIOS_DEBUG, "PCI:[%x] = %x\n", 0xa4, reg);
+	pcie_write_config32(PCI_DEV(0, 0, 0), 0xa4, reg);
+
+	// TOLUD (top of low used dram)
+	reg = pcie_read_config32(PCI_DEV(0, 0, 0), 0xbc);
+	val = toludbase & 0xfff;
+	reg = (reg & ~0xfff00000) | (val << 20);
+	printk(BIOS_DEBUG, "PCI:[%x] = %x\n", 0xbc, reg);
+	pcie_write_config32(PCI_DEV(0, 0, 0), 0xbc, reg);
+
+	// TOUUD LSB (top of upper usable dram)
+	reg = pcie_read_config32(PCI_DEV(0, 0, 0), 0xa8);
+	val = touudbase & 0xfff;
+	reg = (reg & ~0xfff00000) | (val << 20);
+	printk(BIOS_DEBUG, "PCI:[%x] = %x\n", 0xa8, reg);
+	pcie_write_config32(PCI_DEV(0, 0, 0), 0xa8, reg);
+
+	// TOUUD MSB
+	reg = pcie_read_config32(PCI_DEV(0, 0, 0), 0xac);
+	val = touudbase & 0xfffff000;
+	reg = (reg & ~0x000fffff) | (val >> 12);
+	printk(BIOS_DEBUG, "PCI:[%x] = %x\n", 0xac, reg);
+	pcie_write_config32(PCI_DEV(0, 0, 0), 0xac, reg);
+
+	if (reclaim) {
+		// REMAP BASE
+		pcie_write_config32(PCI_DEV(0, 0, 0), 0x90, remapbase << 20);
+		pcie_write_config32(PCI_DEV(0, 0, 0), 0x94, remapbase >> 12);
+
+		// REMAP LIMIT
+		pcie_write_config32(PCI_DEV(0, 0, 0), 0x98, remaplimit << 20);
+		pcie_write_config32(PCI_DEV(0, 0, 0), 0x9c, remaplimit >> 12);
+	}
+	// TSEG
+	reg = pcie_read_config32(PCI_DEV(0, 0, 0), 0xb8);
+	val = tsegbase & 0xfff;
+	reg = (reg & ~0xfff00000) | (val << 20);
+	printk(BIOS_DEBUG, "PCI:[%x] = %x\n", 0xb8, reg);
+	pcie_write_config32(PCI_DEV(0, 0, 0), 0xb8, reg);
+
+	// GFX stolen memory
+	reg = pcie_read_config32(PCI_DEV(0, 0, 0), 0xb0);
+	val = gfxstolenbase & 0xfff;
+	reg = (reg & ~0xfff00000) | (val << 20);
+	printk(BIOS_DEBUG, "PCI:[%x] = %x\n", 0xb0, reg);
+	pcie_write_config32(PCI_DEV(0, 0, 0), 0xb0, reg);
+
+	// GTT stolen memory
+	reg = pcie_read_config32(PCI_DEV(0, 0, 0), 0xb4);
+	val = gttbase & 0xfff;
+	reg = (reg & ~0xfff00000) | (val << 20);
+	printk(BIOS_DEBUG, "PCI:[%x] = %x\n", 0xb4, reg);
+	pcie_write_config32(PCI_DEV(0, 0, 0), 0xb4, reg);
+
+	if (me_uma_size) {
+		reg = pcie_read_config32(PCI_DEV(0, 0, 0), 0x7c);
+		val = (0x80000 - me_uma_size) & 0xfffff000;
+		reg = (reg & ~0x000fffff) | (val >> 12);
+		printk(BIOS_DEBUG, "PCI:[%x] = %x\n", 0x7c, reg);
+		pcie_write_config32(PCI_DEV(0, 0, 0), 0x7c, reg);
+
+		// ME base
+		reg = pcie_read_config32(PCI_DEV(0, 0, 0), 0x70);
+		val = mestolenbase & 0xfff;
+		reg = (reg & ~0xfff00000) | (val << 20);
+		printk(BIOS_DEBUG, "PCI:[%x] = %x\n", 0x70, reg);
+		pcie_write_config32(PCI_DEV(0, 0, 0), 0x70, reg);
+
+		reg = pcie_read_config32(PCI_DEV(0, 0, 0), 0x74);
+		val = mestolenbase & 0xfffff000;
+		reg = (reg & ~0x000fffff) | (val >> 12);
+		printk(BIOS_DEBUG, "PCI:[%x] = %x\n", 0x74, reg);
+		pcie_write_config32(PCI_DEV(0, 0, 0), 0x74, reg);
+
+		// ME mask
+		reg = pcie_read_config32(PCI_DEV(0, 0, 0), 0x78);
+		val = (0x80000 - me_uma_size) & 0xfff;
+		reg = (reg & ~0xfff00000) | (val << 20);
+		reg = (reg & ~0x400) | (1 << 10);	// set lockbit on ME mem
+
+		reg = (reg & ~0x800) | (1 << 11);	// set ME memory enable
+		printk(BIOS_DEBUG, "PCI:[%x] = %x\n", 0x78, reg);
+		pcie_write_config32(PCI_DEV(0, 0, 0), 0x78, reg);
+	}
+}
+
+static void dram_ioregs(ramctr_timing * ctrl)
+{
+	u32 reg, comp2;
+
+	int channel;
+
+	// IO clock
+	FOR_ALL_CHANNELS {
+		MCHBAR32(0xc00 + 0x100 * channel) = ctrl->rankmap[channel];
+	}
+
+	// IO command
+	FOR_ALL_CHANNELS {
+		MCHBAR32(0x3200 + 0x100 * channel) = ctrl->rankmap[channel];
+	}
+
+	// IO control
+	FOR_ALL_POPULATED_CHANNELS {
+		program_timings(ctrl, channel);
+	}
+
+	// Rcomp
+	printk(BIOS_DEBUG, "RCOMP...");
+	reg = 0;
+	while (reg == 0) {
+		reg = MCHBAR32(0x5084) & 0x10000;
+	}
+	printk(BIOS_DEBUG, "done\n");
+
+	// Set comp2
+	comp2 = get_COMP2(ctrl->tCK);
+	MCHBAR32(0x3714) = comp2;
+	printk(BIOS_DEBUG, "COMP2 done\n");
+
+	// Set comp1
+	FOR_ALL_POPULATED_CHANNELS {
+		reg = MCHBAR32(0x1810 + channel * 0x100);	//ch0
+		reg = (reg & ~0xe00) | (1 << 9);	//odt
+		reg = (reg & ~0xe00000) | (1 << 21);	//clk drive up
+		reg = (reg & ~0x38000000) | (1 << 27);	//ctl drive up
+		MCHBAR32(0x1810 + channel * 0x100) = reg;
+	}
+	printk(BIOS_DEBUG, "COMP1 done\n");
+
+	printk(BIOS_DEBUG, "FORCE RCOMP and wait 20us...");
+	MCHBAR32(0x5f08) |= 0x100;
+	udelay(20);
+	printk(BIOS_DEBUG, "done\n");
+}
+
+static void wait_428c(int channel)
+{
+	while (1) {
+		if (read32(DEFAULT_MCHBAR + 0x428c + (channel << 10)) & 0x50)
+			return;
+	}
+}
+
+static void write_reset(ramctr_timing * ctrl)
+{
+	int channel, slotrank;
+
+	/* choose a populated channel.  */
+	channel = (ctrl->rankmap[0]) ? 0 : 1;
+
+	wait_428c(channel);
+
+	/* choose a populated rank.  */
+	slotrank = (ctrl->rankmap[channel] & 1) ? 0 : 2;
+
+	write32(DEFAULT_MCHBAR + 0x4220 + 0x400 * channel, 0x0f003);
+	write32(DEFAULT_MCHBAR + 0x4230 + 0x400 * channel, 0x80c01);
+
+	write32(DEFAULT_MCHBAR + 0x4200 + 0x400 * channel,
+		(slotrank << 24) | 0x60000);
+
+	write32(DEFAULT_MCHBAR + 0x4210 + 0x400 * channel, 0);
+
+	write32(DEFAULT_MCHBAR + 0x4284 + 0x400 * channel, 0x400001);
+	wait_428c(channel);
+}
+
+static void dram_jedecreset(ramctr_timing * ctrl)
+{
+	u32 reg, addr;
+	int channel;
+
+	while (!(MCHBAR32(0x5084) & 0x10000)) ;
+	do {
+		reg = MCHBAR32(0x428c);
+	} while ((reg & 0x14) == 0);
+
+	// Set state of memory controller
+	reg = 0x112;
+	MCHBAR32(0x5030) = reg;
+	MCHBAR32(0x4ea0) = 0;
+	reg |= 2;		//ddr reset
+	MCHBAR32(0x5030) = reg;
+
+	// Assert dimm reset signal
+	reg = MCHBAR32(0x5030);
+	reg &= ~0x2;
+	MCHBAR32(0x5030) = reg;
+
+	// Wait 200us
+	udelay(200);
+
+	// Deassert dimm reset signal
+	MCHBAR32(0x5030) |= 2;
+
+	// Wait 500us
+	udelay(500);
 
-	/* Pass console handler in pei_data */
-	pei_data->tx_byte = do_putchar;
+	// Enable DCLK
+	MCHBAR32(0x5030) |= 4;
 
-	/* Locate and call UEFI System Agent binary. */
-	entry = cbfs_boot_map_with_leak("mrc.bin", CBFS_TYPE_MRC, NULL);
-	if (entry) {
-		int rv;
-		rv = entry (pei_data);
-		if (rv) {
-			switch (rv) {
-			case -1:
-				printk(BIOS_ERR, "PEI version mismatch.\n");
+	// XXX Wait 20ns
+	udelay(1);
+
+	FOR_ALL_CHANNELS {
+		// Set valid rank CKE
+		reg = 0;
+		reg = (reg & ~0xf) | ctrl->rankmap[channel];
+		addr = 0x400 * channel + 0x42a0;
+		MCHBAR32(addr) = reg;
+
+		// Wait 10ns for ranks to settle
+		//udelay(0.01);
+
+		reg = (reg & ~0xf0) | (ctrl->rankmap[channel] << 4);
+		MCHBAR32(addr) = reg;
+
+		// Write reset using a NOP
+		write_reset(ctrl);
+	}
+}
+
+static odtmap get_ODT(ramctr_timing * ctrl, u8 rank)
+{
+	/* Get ODT based on rankmap: */
+	int dimms_per_ch = 0;
+	int channel;
+
+	FOR_ALL_CHANNELS {
+		dimms_per_ch = max ((ctrl->rankmap[channel] & 1)
+				    + ((ctrl->rankmap[channel] >> 2) & 1),
+				    dimms_per_ch);
+	}
+
+	if (dimms_per_ch == 1) {
+		return (const odtmap){60, 60};
+	} else if (dimms_per_ch == 2) {
+		return (const odtmap){120, 30};
+	} else {
+		printk(BIOS_DEBUG,
+		       "Huh, no dimms? m0 = %d m1 = %d dpc = %d\n",
+		       ctrl->rankmap[0],
+		       ctrl->rankmap[1], dimms_per_ch);
+		die("");
+	}
+}
+
+static void write_mrreg(ramctr_timing * ctrl, int channel, int slotrank,
+			int reg, u32 val)
+{
+	wait_428c(channel);
+
+	printram("MRd: %x <= %x\n", reg, val);
+
+	if (ctrl->rank_mirror[channel][slotrank]) {
+		/* DDR3 Rank1 Address mirror
+		 * swap the following pins:
+		 * A3<->A4, A5<->A6, A7<->A8, BA0<->BA1 */
+		reg = ((reg >> 1) & 1) | ((reg << 1) & 2);
+		val = (val & ~0x1f8) | ((val >> 1) & 0xa8)
+		    | ((val & 0xa8) << 1);
+	}
+
+	printram("MRd: %x <= %x\n", reg, val);
+
+	write32(DEFAULT_MCHBAR + 0x4220 + 0x400 * channel, 0x0f000);
+	write32(DEFAULT_MCHBAR + 0x4230 + 0x400 * channel, 0x41001);
+	write32(DEFAULT_MCHBAR + 0x4200 + 0x400 * channel,
+		(slotrank << 24) | (reg << 20) | val | 0x60000);
+	write32(DEFAULT_MCHBAR + 0x4210 + 0x400 * channel, 0);
+
+	write32(DEFAULT_MCHBAR + 0x4224 + 0x400 * channel, 0x1f000);
+	write32(DEFAULT_MCHBAR + 0x4234 + 0x400 * channel, 0x41001);
+	write32(DEFAULT_MCHBAR + 0x4204 + 0x400 * channel,
+		(slotrank << 24) | (reg << 20) | val | 0x60000);
+	write32(DEFAULT_MCHBAR + 0x4214 + 0x400 * channel, 0);
+
+	write32(DEFAULT_MCHBAR + 0x4228 + 0x400 * channel, 0x0f000);
+	write32(DEFAULT_MCHBAR + 0x4238 + 0x400 * channel,
+		0x1001 | (ctrl->tMOD << 16));
+	write32(DEFAULT_MCHBAR + 0x4208 + 0x400 * channel,
+		(slotrank << 24) | (reg << 20) | val | 0x60000);
+	write32(DEFAULT_MCHBAR + 0x4218 + 0x400 * channel, 0);
+	write32(DEFAULT_MCHBAR + 0x4284 + 0x400 * channel, 0x80001);
+}
+
+static u32 make_mr0(ramctr_timing * ctrl, u8 rank)
+{
+	u16 mr0reg, mch_cas, mch_wr;
+	static const u8 mch_wr_t[12] = { 1, 2, 3, 4, 0, 5, 0, 6, 0, 7, 0, 0 };
+	mr0reg = 0x100;
+
+	// Convert CAS to MCH register friendly
+	if (ctrl->CAS < 12) {
+		mch_cas = (u16) ((ctrl->CAS - 4) << 1);
+	} else {
+		mch_cas = (u16) (ctrl->CAS - 12);
+		mch_cas = ((mch_cas << 1) | 0x1);
+	}
+
+	// Convert tWR to MCH register friendly
+	mch_wr = mch_wr_t[ctrl->tWR - 5];
+
+	mr0reg = (mr0reg & ~0x4) | (mch_cas & 0x1);
+	mr0reg = (mr0reg & ~0x70) | ((mch_cas & 0xe) << 3);
+	mr0reg = (mr0reg & ~0xe00) | (mch_wr << 9);
+	// Fast (desktop) 0x1 or slow (mobile) 0x0
+	mr0reg = (mr0reg & ~0x1000) | (!ctrl->mobile << 12);
+	return mr0reg;
+}
+
+static void dram_mr0(ramctr_timing * ctrl, u8 rank)
+{
+	int channel;
+
+	FOR_ALL_POPULATED_CHANNELS write_mrreg(ctrl, channel, rank, 0,
+					       make_mr0(ctrl, rank));
+}
+
+static u32 encode_odt(u32 odt)
+{
+	switch (odt) {
+	case 30:
+		return (1 << 9) | (1 << 2);	// RZQ/8, RZQ/4
+	case 60:
+		return (1 << 2);	// RZQ/4
+	case 120:
+		return (1 << 6);	// RZQ/2
+	default:
+	case 0:
+		return 0;
+	}
+}
+
+static u32 make_mr1(ramctr_timing * ctrl, u8 rank)
+{
+	odtmap odt;
+	u32 mr1reg;
+
+	odt = get_ODT(ctrl, rank);
+	mr1reg = 0x2;
+
+	mr1reg |= encode_odt(odt.rttnom);
+
+	return mr1reg;
+}
+
+static void dram_mr1(ramctr_timing * ctrl, u8 rank)
+{
+	u16 mr1reg;
+	int channel;
+
+	mr1reg = make_mr1(ctrl, rank);
+
+	FOR_ALL_CHANNELS {
+		write_mrreg(ctrl, channel, rank, 1, mr1reg);
+	}
+}
+
+static void dram_mr2(ramctr_timing * ctrl, u8 rank)
+{
+	u16 pasr, cwl, mr2reg;
+	odtmap odt;
+	int channel;
+	int srt;
+
+	pasr = 0;
+	cwl = ctrl->CWL - 5;
+	odt = get_ODT(ctrl, rank);
+
+	srt = ctrl->extended_temperature_range && !ctrl->auto_self_refresh;
+
+	mr2reg = 0;
+	mr2reg = (mr2reg & ~0x7) | pasr;
+	mr2reg = (mr2reg & ~0x38) | (cwl << 3);
+	mr2reg = (mr2reg & ~0x40) | (ctrl->auto_self_refresh << 6);
+	mr2reg = (mr2reg & ~0x80) | (srt << 7);
+	mr2reg |= (odt.rttwr / 60) << 9;
+
+	FOR_ALL_CHANNELS {
+		write_mrreg(ctrl, channel, rank, 2, mr2reg);
+	}
+}
+
+static void dram_mr3(ramctr_timing * ctrl, u8 rank)
+{
+	int channel;
+
+	FOR_ALL_CHANNELS {
+		write_mrreg(ctrl, channel, rank, 3, 0);
+	}
+}
+
+static void dram_mrscommands(ramctr_timing * ctrl)
+{
+	u8 rank;
+	u32 reg, addr;
+	int channel;
+
+	for (rank = 0; rank < 4; rank++) {
+		// MR2
+		printram("MR2 rank %d...", rank);
+		dram_mr2(ctrl, rank);
+		printram("done\n");
+
+		// MR3
+		printram("MR3 rank %d...", rank);
+		dram_mr3(ctrl, rank);
+		printram("done\n");
+
+		// MR1
+		printram("MR1 rank %d...", rank);
+		dram_mr1(ctrl, rank);
+		printram("done\n");
+
+		// MR0
+		printram("MR0 rank %d...", rank);
+		dram_mr0(ctrl, rank);
+		printram("done\n");
+	}
+
+	write32(DEFAULT_MCHBAR + 0x4e20, 0x7);
+	write32(DEFAULT_MCHBAR + 0x4e30, 0xf1001);
+	write32(DEFAULT_MCHBAR + 0x4e00, 0x60002);
+	write32(DEFAULT_MCHBAR + 0x4e10, 0);
+	write32(DEFAULT_MCHBAR + 0x4e24, 0x1f003);
+	write32(DEFAULT_MCHBAR + 0x4e34, 0x1901001);
+	write32(DEFAULT_MCHBAR + 0x4e04, 0x60400);
+	write32(DEFAULT_MCHBAR + 0x4e14, 0x288);
+	write32(DEFAULT_MCHBAR + 0x4e84, 0x40004);
+
+	// Drain
+	FOR_ALL_CHANNELS {
+		// Wait for ref drained
+		wait_428c(channel);
+	}
+
+	// Refresh enable
+	MCHBAR32(0x5030) |= 8;
+
+	FOR_ALL_POPULATED_CHANNELS {
+		addr = 0x400 * channel + 0x4020;
+		reg = MCHBAR32(addr);
+		reg &= ~0x200000;
+		MCHBAR32(addr) = reg;
+
+		wait_428c(channel);
+
+		rank = (ctrl->rankmap[channel] & 1) ? 0 : 2;
+
+		// Drain
+		wait_428c(channel);
+
+		write32(DEFAULT_MCHBAR + 0x4220 + 0x400 * channel, 0x0f003);
+		write32(DEFAULT_MCHBAR + 0x4230 + 0x400 * channel, 0x659001);
+		write32(DEFAULT_MCHBAR + 0x4200 + 0x400 * channel,
+			(rank << 24) | 0x60000);
+		write32(DEFAULT_MCHBAR + 0x4210 + 0x400 * channel, 0x3e0);
+		write32(DEFAULT_MCHBAR + 0x4284 + 0x400 * channel, 0x1);
+
+		// Drain
+		wait_428c(channel);
+	}
+}
+
+const u32 lane_registers[] = {
+	0x0000, 0x0200, 0x0400, 0x0600,
+	0x1000, 0x1200, 0x1400, 0x1600,
+	0x0800
+};
+
+static void program_timings(ramctr_timing * ctrl, int channel)
+{
+	u32 reg32, reg_4024, reg_c14, reg_c18, reg_4028;
+	int lane;
+	int slotrank, slot;
+	int full_shift = 0;
+	u16 slot320c[NUM_SLOTS];
+
+	FOR_ALL_POPULATED_RANKS {
+		if (full_shift < -ctrl->timings[channel][slotrank].val_320c)
+			full_shift = -ctrl->timings[channel][slotrank].val_320c;
+	}
+
+	for (slot = 0; slot < NUM_SLOTS; slot++)
+		switch ((ctrl->rankmap[channel] >> (2 * slot)) & 3) {
+		case 0:
+		default:
+			slot320c[slot] = 0x7f;
+			break;
+		case 1:
+			slot320c[slot] =
+			    ctrl->timings[channel][2 * slot + 0].val_320c +
+			    full_shift;
+			break;
+		case 2:
+			slot320c[slot] =
+			    ctrl->timings[channel][2 * slot + 1].val_320c +
+			    full_shift;
+			break;
+		case 3:
+			slot320c[slot] =
+			    (ctrl->timings[channel][2 * slot].val_320c +
+			     ctrl->timings[channel][2 * slot +
+						    1].val_320c) / 2 +
+			    full_shift;
+			break;
+		}
+
+	reg32 = (1 << 17) | (1 << 14);
+	reg32 |= ((slot320c[0] & 0x3f) << 6) | ((slot320c[0] & 0x40) << 9);
+	reg32 |= (slot320c[1] & 0x7f) << 18;
+	reg32 |= (full_shift & 0x3f) | ((full_shift & 0x40) << 6);
+
+	MCHBAR32(0x320c + 0x100 * channel) = reg32;
+
+	reg_c14 = ctrl->rankmap[channel] << 24;
+	reg_c18 = 0;
+
+	FOR_ALL_POPULATED_RANKS {
+		int shift =
+		    ctrl->timings[channel][slotrank].val_320c + full_shift;
+		int offset_val_c14;
+		if (shift < 0)
+			shift = 0;
+		offset_val_c14 = ctrl->reg_c14_offset + shift;
+		reg_c14 |= (offset_val_c14 & 0x3f) << (6 * slotrank);
+		reg_c18 |= ((offset_val_c14 >> 6) & 1) << slotrank;
+	}
+
+	MCHBAR32(0xc14 + channel * 0x100) = reg_c14;
+	MCHBAR32(0xc18 + channel * 0x100) = reg_c18;
+
+	reg_4028 = MCHBAR32(0x4028 + 0x400 * channel);
+	reg_4028 &= 0xffff0000;
+
+	reg_4024 = 0;
+
+	FOR_ALL_POPULATED_RANKS {
+		int post_timA_min_high = 7, post_timA_max_high = 0;
+		int pre_timA_min_high = 7, pre_timA_max_high = 0;
+		int shift_402x = 0;
+		int shift =
+		    ctrl->timings[channel][slotrank].val_320c + full_shift;
+
+		if (shift < 0)
+			shift = 0;
+
+		FOR_ALL_LANES {
+			if (post_timA_min_high >
+			    ((ctrl->timings[channel][slotrank].lanes[lane].
+			      timA + shift) >> 6))
+				post_timA_min_high =
+				    ((ctrl->timings[channel][slotrank].
+				      lanes[lane].timA + shift) >> 6);
+			if (pre_timA_min_high >
+			    (ctrl->timings[channel][slotrank].lanes[lane].
+			     timA >> 6))
+				pre_timA_min_high =
+				    (ctrl->timings[channel][slotrank].
+				     lanes[lane].timA >> 6);
+			if (post_timA_max_high <
+			    ((ctrl->timings[channel][slotrank].lanes[lane].
+			      timA + shift) >> 6))
+				post_timA_max_high =
+				    ((ctrl->timings[channel][slotrank].
+				      lanes[lane].timA + shift) >> 6);
+			if (pre_timA_max_high <
+			    (ctrl->timings[channel][slotrank].lanes[lane].
+			     timA >> 6))
+				pre_timA_max_high =
+				    (ctrl->timings[channel][slotrank].
+				     lanes[lane].timA >> 6);
+		}
+
+		if (pre_timA_max_high - pre_timA_min_high <
+		    post_timA_max_high - post_timA_min_high)
+			shift_402x = +1;
+		else if (pre_timA_max_high - pre_timA_min_high >
+			 post_timA_max_high - post_timA_min_high)
+			shift_402x = -1;
+
+		reg_4028 |=
+		    (ctrl->timings[channel][slotrank].val_4028 + shift_402x -
+		     post_timA_min_high) << (4 * slotrank);
+		reg_4024 |=
+		    (ctrl->timings[channel][slotrank].val_4024 +
+		     shift_402x) << (8 * slotrank);
+
+		FOR_ALL_LANES {
+			MCHBAR32(lane_registers[lane] + 0x10 + 0x100 * channel +
+				 4 * slotrank)
+			    =
+			    (((ctrl->timings[channel][slotrank].lanes[lane].
+			       timA + shift) & 0x3f)
+			     |
+			     ((ctrl->timings[channel][slotrank].lanes[lane].
+			       rising + shift) << 8)
+			     |
+			     (((ctrl->timings[channel][slotrank].lanes[lane].
+				timA + shift -
+				(post_timA_min_high << 6)) & 0x1c0) << 10)
+			     | (ctrl->timings[channel][slotrank].lanes[lane].
+				falling << 20));
+
+			MCHBAR32(lane_registers[lane] + 0x20 + 0x100 * channel +
+				 4 * slotrank)
+			    =
+			    (((ctrl->timings[channel][slotrank].lanes[lane].
+			       timC + shift) & 0x3f)
+			     |
+			     (((ctrl->timings[channel][slotrank].lanes[lane].
+				timB + shift) & 0x3f) << 8)
+			     |
+			     (((ctrl->timings[channel][slotrank].lanes[lane].
+				timB + shift) & 0x1c0) << 9)
+			     |
+			     (((ctrl->timings[channel][slotrank].lanes[lane].
+				timC + shift) & 0x40) << 13));
+		}
+	}
+	MCHBAR32(0x4024 + 0x400 * channel) = reg_4024;
+	MCHBAR32(0x4028 + 0x400 * channel) = reg_4028;
+}
+
+static void test_timA(ramctr_timing * ctrl, int channel, int slotrank)
+{
+	wait_428c(channel);
+
+	write32(DEFAULT_MCHBAR + 0x4220 + 0x400 * channel, 0x1f000);
+	write32(DEFAULT_MCHBAR + 0x4230 + 0x400 * channel,
+		(0xc01 | (ctrl->tMOD << 16)));
+	write32(DEFAULT_MCHBAR + 0x4200 + 0x400 * channel,
+		(slotrank << 24) | 0x360004);
+	write32(DEFAULT_MCHBAR + 0x4210 + 0x400 * channel, 0);
+
+	write32(DEFAULT_MCHBAR + 0x4224 + 0x400 * channel, 0x1f105);
+	write32(DEFAULT_MCHBAR + 0x4234 + 0x400 * channel, 0x4040c01);
+	write32(DEFAULT_MCHBAR + 0x4204 + 0x400 * channel, (slotrank << 24));
+	write32(DEFAULT_MCHBAR + 0x4214 + 0x400 * channel, 0);
+
+	write32(DEFAULT_MCHBAR + 0x4228 + 0x400 * channel, 0x1f105);
+	write32(DEFAULT_MCHBAR + 0x4238 + 0x400 * channel,
+		0x100f | ((ctrl->CAS + 36) << 16));
+	write32(DEFAULT_MCHBAR + 0x4208 + 0x400 * channel,
+		(slotrank << 24) | 0x60000);
+	write32(DEFAULT_MCHBAR + 0x4218 + 0x400 * channel, 0);
+
+	write32(DEFAULT_MCHBAR + 0x422c + 0x400 * channel, 0x1f000);
+	write32(DEFAULT_MCHBAR + 0x423c + 0x400 * channel,
+		(0xc01 | (ctrl->tMOD << 16)));
+	write32(DEFAULT_MCHBAR + 0x420c + 0x400 * channel,
+		(slotrank << 24) | 0x360000);
+	write32(DEFAULT_MCHBAR + 0x421c + 0x400 * channel, 0);
+
+	write32(DEFAULT_MCHBAR + 0x4284 + 0x400 * channel, 0xc0001);
+
+	wait_428c(channel);
+}
+
+static int does_lane_work(ramctr_timing * ctrl, int channel, int slotrank,
+			  int lane)
+{
+	u32 timA = ctrl->timings[channel][slotrank].lanes[lane].timA;
+	return ((read32
+		 (DEFAULT_MCHBAR + lane_registers[lane] + channel * 0x100 + 4 +
+		  ((timA / 32) & 1) * 4)
+		 >> (timA % 32)) & 1);
+}
+
+struct run {
+	int middle;
+	int end;
+	int start;
+	int all;
+	int length;
+};
+
+static struct run get_longest_zero_run(int *seq, int sz)
+{
+	int i, ls;
+	int bl = 0, bs = 0;
+	struct run ret;
+
+	ls = 0;
+	for (i = 0; i < 2 * sz; i++)
+		if (seq[i % sz]) {
+			if (i - ls > bl) {
+				bl = i - ls;
+				bs = ls;
+			}
+			ls = i + 1;
+		}
+	if (bl == 0) {
+		ret.middle = sz / 2;
+		ret.start = 0;
+		ret.end = sz;
+		ret.all = 1;
+		return ret;
+	}
+
+	ret.start = bs % sz;
+	ret.end = (bs + bl - 1) % sz;
+	ret.middle = (bs + (bl - 1) / 2) % sz;
+	ret.length = bl;
+	ret.all = 0;
+
+	return ret;
+}
+
+static void discover_timA_coarse(ramctr_timing * ctrl, int channel,
+				 int slotrank, int *upperA)
+{
+	int timA;
+	int statistics[NUM_LANES][128];
+	int lane;
+
+	for (timA = 0; timA < 128; timA++) {
+		FOR_ALL_LANES {
+			ctrl->timings[channel][slotrank].lanes[lane].timA = timA;
+		}
+		program_timings(ctrl, channel);
+
+		test_timA(ctrl, channel, slotrank);
+
+		FOR_ALL_LANES {
+			statistics[lane][timA] =
+			    !does_lane_work(ctrl, channel, slotrank, lane);
+			printram("Astat: %d, %d, %d, %x, %x\n",
+			       channel, slotrank, lane, timA,
+			       statistics[lane][timA]);
+		}
+	}
+	FOR_ALL_LANES {
+		struct run rn = get_longest_zero_run(statistics[lane], 128);
+		ctrl->timings[channel][slotrank].lanes[lane].timA = rn.middle;
+		upperA[lane] = rn.end;
+		if (upperA[lane] < rn.middle)
+			upperA[lane] += 128;
+		printram("Aval: %d, %d, %d, %x\n", channel, slotrank,
+		       lane, ctrl->timings[channel][slotrank].lanes[lane].timA);
+		printram("Aend: %d, %d, %d, %x\n", channel, slotrank,
+		       lane, upperA[lane]);
+	}
+}
+
+static void discover_timA_fine(ramctr_timing * ctrl, int channel, int slotrank,
+			       int *upperA)
+{
+	int timA_delta;
+	int statistics[NUM_LANES][51];
+	int lane, i;
+
+	memset(statistics, 0, sizeof(statistics));
+
+	for (timA_delta = -25; timA_delta <= 25; timA_delta++) {
+		FOR_ALL_LANES ctrl->timings[channel][slotrank].lanes[lane].
+		    timA = upperA[lane] + timA_delta + 0x40;
+		program_timings(ctrl, channel);
+
+		for (i = 0; i < 100; i++) {
+			test_timA(ctrl, channel, slotrank);
+			FOR_ALL_LANES {
+				statistics[lane][timA_delta + 25] +=
+				    does_lane_work(ctrl, channel, slotrank,
+						   lane);
+			}
+		}
+	}
+	FOR_ALL_LANES {
+		int last_zero, first_all;
+
+		for (last_zero = -25; last_zero <= 25; last_zero++)
+			if (statistics[lane][last_zero + 25])
 				break;
-			case -2:
-				printk(BIOS_ERR, "Invalid memory frequency.\n");
+		last_zero--;
+		for (first_all = -25; first_all <= 25; first_all++)
+			if (statistics[lane][first_all + 25] == 100)
 				break;
-			default:
-				printk(BIOS_ERR, "MRC returned %x.\n", rv);
+
+		printram("lane %d: %d, %d\n", lane, last_zero,
+		       first_all);
+
+		ctrl->timings[channel][slotrank].lanes[lane].timA =
+		    (last_zero + first_all) / 2 + upperA[lane];
+		printram("Aval: %d, %d, %d, %x\n", channel, slotrank,
+		       lane, ctrl->timings[channel][slotrank].lanes[lane].timA);
+	}
+}
+
+static void discover_402x(ramctr_timing * ctrl, int channel, int slotrank,
+			  int *upperA)
+{
+	int works[NUM_LANES];
+	int lane;
+	while (1) {
+		int all_works = 1, some_works = 0;
+		program_timings(ctrl, channel);
+		test_timA(ctrl, channel, slotrank);
+		FOR_ALL_LANES {
+			works[lane] =
+			    !does_lane_work(ctrl, channel, slotrank, lane);
+			if (works[lane])
+				some_works = 1;
+			else
+				all_works = 0;
+		}
+		if (all_works)
+			return;
+		if (!some_works) {
+			if (ctrl->timings[channel][slotrank].val_4024 < 2)
+				die("402x discovery failed");
+			ctrl->timings[channel][slotrank].val_4024 -= 2;
+			printram("4024 -= 2;\n");
+			continue;
+		}
+		ctrl->timings[channel][slotrank].val_4028 += 2;
+		printram("4028 += 2;\n");
+		if (ctrl->timings[channel][slotrank].val_4028 >= 0x10)
+			die("402x discovery failed");
+		FOR_ALL_LANES if (works[lane]) {
+			ctrl->timings[channel][slotrank].lanes[lane].timA +=
+			    128;
+			upperA[lane] += 128;
+			printram("increment %d, %d, %d\n", channel,
+			       slotrank, lane);
+		}
+	}
+}
+
+struct timA_minmax {
+	int timA_min_high, timA_max_high;
+};
+
+static void pre_timA_change(ramctr_timing * ctrl, int channel, int slotrank,
+			    struct timA_minmax *mnmx)
+{
+	int lane;
+	mnmx->timA_min_high = 7;
+	mnmx->timA_max_high = 0;
+
+	FOR_ALL_LANES {
+		if (mnmx->timA_min_high >
+		    (ctrl->timings[channel][slotrank].lanes[lane].timA >> 6))
+			mnmx->timA_min_high =
+			    (ctrl->timings[channel][slotrank].lanes[lane].
+			     timA >> 6);
+		if (mnmx->timA_max_high <
+		    (ctrl->timings[channel][slotrank].lanes[lane].timA >> 6))
+			mnmx->timA_max_high =
+			    (ctrl->timings[channel][slotrank].lanes[lane].
+			     timA >> 6);
+	}
+}
+
+static void post_timA_change(ramctr_timing * ctrl, int channel, int slotrank,
+			     struct timA_minmax *mnmx)
+{
+	struct timA_minmax post;
+	int shift_402x = 0;
+
+	/* Get changed maxima.  */
+	pre_timA_change(ctrl, channel, slotrank, &post);
+
+	if (mnmx->timA_max_high - mnmx->timA_min_high <
+	    post.timA_max_high - post.timA_min_high)
+		shift_402x = +1;
+	else if (mnmx->timA_max_high - mnmx->timA_min_high >
+		 post.timA_max_high - post.timA_min_high)
+		shift_402x = -1;
+	else
+		shift_402x = 0;
+
+	ctrl->timings[channel][slotrank].val_4028 += shift_402x;
+	ctrl->timings[channel][slotrank].val_4024 += shift_402x;
+	printram("4024 += %d;\n", shift_402x);
+	printram("4028 += %d;\n", shift_402x);
+}
+
+static void read_training(ramctr_timing * ctrl)
+{
+	int channel, slotrank, lane;
+
+	FOR_ALL_CHANNELS FOR_ALL_POPULATED_RANKS {
+		u32 r32;
+		int all_high, some_high;
+		int upperA[NUM_LANES];
+		struct timA_minmax mnmx;
+
+		 wait_428c(channel);
+		 write32(DEFAULT_MCHBAR + 0x4220 + 0x400 * channel, 0x1f002);
+		 write32(DEFAULT_MCHBAR + 0x4230 + 0x400 * channel,
+			 0xc01 | (ctrl->tRP << 16));
+		 write32(DEFAULT_MCHBAR + 0x4200 + 0x400 * channel,
+			 (slotrank << 24) | 0x60400);
+		 write32(DEFAULT_MCHBAR + 0x4210 + 0x400 * channel, 0);
+		 write32(DEFAULT_MCHBAR + 0x4284 + 0x400 * channel, 1);
+
+		 write32(DEFAULT_MCHBAR + 0x3400, (slotrank << 2) | 0x8001);
+
+		 ctrl->timings[channel][slotrank].val_4028 = 4;
+		 ctrl->timings[channel][slotrank].val_4024 = 55;
+		 program_timings(ctrl, channel);
+
+		 discover_timA_coarse(ctrl, channel, slotrank, upperA);
+
+		 all_high = 1;
+		 some_high = 0;
+		 FOR_ALL_LANES {
+			 if (ctrl->timings[channel][slotrank].lanes[lane].
+			     timA >= 0x40)
+				 some_high = 1;
+			 else
+				 all_high = 0;
+		 }
+
+		if (all_high) {
+			ctrl->timings[channel][slotrank].val_4028--;
+			printram("4028--;\n");
+			FOR_ALL_LANES {
+				ctrl->timings[channel][slotrank].lanes[lane].
+				    timA -= 0x40;
+				upperA[lane] -= 0x40;
+
+			}
+		} else if (some_high) {
+			ctrl->timings[channel][slotrank].val_4024++;
+			ctrl->timings[channel][slotrank].val_4028++;
+			printram("4024++;\n");
+			printram("4028++;\n");
+		}
+
+		program_timings(ctrl, channel);
+
+		pre_timA_change(ctrl, channel, slotrank, &mnmx);
+
+		discover_402x(ctrl, channel, slotrank, upperA);
+
+		post_timA_change(ctrl, channel, slotrank, &mnmx);
+		pre_timA_change(ctrl, channel, slotrank, &mnmx);
+
+		discover_timA_fine(ctrl, channel, slotrank, upperA);
+
+		post_timA_change(ctrl, channel, slotrank, &mnmx);
+		pre_timA_change(ctrl, channel, slotrank, &mnmx);
+
+		FOR_ALL_LANES {
+			ctrl->timings[channel][slotrank].lanes[lane].timA -= mnmx.timA_min_high * 0x40;
+		}
+		ctrl->timings[channel][slotrank].val_4028 -= mnmx.timA_min_high;
+		printram("4028 -= %d;\n", mnmx.timA_min_high);
+
+		post_timA_change(ctrl, channel, slotrank, &mnmx);
+
+		printram("4/8: %d, %d, %x, %x\n", channel, slotrank,
+		       ctrl->timings[channel][slotrank].val_4024,
+		       ctrl->timings[channel][slotrank].val_4028);
+
+		FOR_ALL_LANES
+		    printram("%d, %d, %d, %x\n", channel, slotrank,
+			   lane,
+			   ctrl->timings[channel][slotrank].lanes[lane].timA);
+
+		write32(DEFAULT_MCHBAR + 0x3400, 0);
+
+		r32 = read32(DEFAULT_MCHBAR + 0x5030);
+		write32(DEFAULT_MCHBAR + 0x5030, r32 | 0x20);
+		udelay(1);
+
+		write32(DEFAULT_MCHBAR + 0x5030, r32 & ~0x20);
+
+		udelay(1);
+	}
+
+	FOR_ALL_POPULATED_CHANNELS {
+		program_timings(ctrl, channel);
+	}
+	FOR_ALL_CHANNELS FOR_ALL_POPULATED_RANKS FOR_ALL_LANES {
+		write32(DEFAULT_MCHBAR + 0x4080 + 0x400 * channel
+			+ 4 * lane, 0);
+	}
+}
+
+static void test_timC(ramctr_timing * ctrl, int channel, int slotrank)
+{
+	int lane;
+
+	FOR_ALL_LANES {
+		write32(DEFAULT_MCHBAR + 0x4340 + 0x400 * channel + 4 * lane, 0);
+		read32(DEFAULT_MCHBAR + 0x4140 + 0x400 * channel + 4 * lane);
+	}
+
+	wait_428c(channel);
+
+	write32(DEFAULT_MCHBAR + 0x4220 + 0x400 * channel, 0x1f006);
+	write32(DEFAULT_MCHBAR + 0x4230 + 0x400 * channel,
+		(max((ctrl->tFAW >> 2) + 1, ctrl->tRRD) << 10)
+		| 4 | (ctrl->tRCD << 16));
+
+	write32(DEFAULT_MCHBAR + 0x4200 + 0x400 * channel,
+		(slotrank << 24) | (6 << 16));
+
+	write32(DEFAULT_MCHBAR + 0x4210 + 0x400 * channel, 0x244);
+
+	write32(DEFAULT_MCHBAR + 0x4224 + 0x400 * channel, 0x1f207);
+	write32(DEFAULT_MCHBAR + 0x4234 + 0x400 * channel, 0x8041001);
+	write32(DEFAULT_MCHBAR + 0x4204 + 0x400 * channel,
+		(slotrank << 24) | 8);
+	write32(DEFAULT_MCHBAR + 0x4214 + 0x400 * channel, 0x3e0);
+
+	write32(DEFAULT_MCHBAR + 0x4228 + 0x400 * channel, 0x1f201);
+	write32(DEFAULT_MCHBAR + 0x4238 + 0x400 * channel, 0x80411f4);
+	write32(DEFAULT_MCHBAR + 0x4208 + 0x400 * channel, (slotrank << 24));
+	write32(DEFAULT_MCHBAR + 0x4218 + 0x400 * channel, 0x242);
+
+	write32(DEFAULT_MCHBAR + 0x422c + 0x400 * channel, 0x1f207);
+	write32(DEFAULT_MCHBAR + 0x423c + 0x400 * channel,
+		0x8000c01 | ((ctrl->CWL + ctrl->tWTR + 5) << 16));
+	write32(DEFAULT_MCHBAR + 0x420c + 0x400 * channel,
+		(slotrank << 24) | 8);
+	write32(DEFAULT_MCHBAR + 0x421c + 0x400 * channel, 0x3e0);
+
+	write32(DEFAULT_MCHBAR + 0x4284 + 0x400 * channel, 0xc0001);
+
+	wait_428c(channel);
+
+	write32(DEFAULT_MCHBAR + 0x4220 + 0x400 * channel, 0x1f002);
+	write32(DEFAULT_MCHBAR + 0x4230 + 0x400 * channel,
+		0xc01 | (ctrl->tRP << 16));
+	write32(DEFAULT_MCHBAR + 0x4200 + 0x400 * channel,
+		(slotrank << 24) | 0x60400);
+	write32(DEFAULT_MCHBAR + 0x4210 + 0x400 * channel, 0x240);
+
+	write32(DEFAULT_MCHBAR + 0x4224 + 0x400 * channel, 0x1f006);
+	write32(DEFAULT_MCHBAR + 0x4234 + 0x400 * channel,
+		(max(ctrl->tRRD, (ctrl->tFAW >> 2) + 1) << 10)
+		| 8 | (ctrl->CAS << 16));
+
+	write32(DEFAULT_MCHBAR + 0x4204 + 0x400 * channel,
+		(slotrank << 24) | 0x60000);
+
+	write32(DEFAULT_MCHBAR + 0x4214 + 0x400 * channel, 0x244);
+
+	write32(DEFAULT_MCHBAR + 0x4228 + 0x400 * channel, 0x1f105);
+	write32(DEFAULT_MCHBAR + 0x4238 + 0x400 * channel,
+		0x40011f4 | (max(ctrl->tRTP, 8) << 16));
+	write32(DEFAULT_MCHBAR + 0x4208 + 0x400 * channel, (slotrank << 24));
+	write32(DEFAULT_MCHBAR + 0x4218 + 0x400 * channel, 0x242);
+
+	write32(DEFAULT_MCHBAR + 0x422c + 0x400 * channel, 0x1f002);
+	write32(DEFAULT_MCHBAR + 0x423c + 0x400 * channel,
+		0xc01 | (ctrl->tRP << 16));
+	write32(DEFAULT_MCHBAR + 0x420c + 0x400 * channel,
+		(slotrank << 24) | 0x60400);
+	write32(DEFAULT_MCHBAR + 0x421c + 0x400 * channel, 0x240);
+	write32(DEFAULT_MCHBAR + 0x4284 + 0x400 * channel, 0xc0001);
+	wait_428c(channel);
+}
+
+static void discover_timC(ramctr_timing * ctrl, int channel, int slotrank)
+{
+	int timC;
+	int statistics[NUM_LANES][MAX_TIMC + 1];
+	int lane;
+
+	wait_428c(channel);
+
+	write32(DEFAULT_MCHBAR + 0x4220 + 0x400 * channel, 0x1f002);
+	write32(DEFAULT_MCHBAR + 0x4230 + 0x400 * channel,
+		0xc01 | (ctrl->tRP << 16));
+	write32(DEFAULT_MCHBAR + 0x4200 + 0x400 * channel,
+		(slotrank << 24) | 0x60400);
+	write32(DEFAULT_MCHBAR + 0x4210 + 0x400 * channel, 0x240);
+	write32(DEFAULT_MCHBAR + 0x4284 + 0x400 * channel, 1);
+
+	for (timC = 0; timC <= MAX_TIMC; timC++) {
+		FOR_ALL_LANES ctrl->timings[channel][slotrank].lanes[lane].
+		    timC = timC;
+		program_timings(ctrl, channel);
+
+		test_timC(ctrl, channel, slotrank);
+
+		FOR_ALL_LANES {
+			statistics[lane][timC] =
+			    read32(DEFAULT_MCHBAR + 0x4340 + 4 * lane +
+				   0x400 * channel);
+			printram("Cstat: %d, %d, %d, %x, %x\n",
+			       channel, slotrank, lane, timC,
+			       statistics[lane][timC]);
+		}
+	}
+	FOR_ALL_LANES {
+		struct run rn =
+		    get_longest_zero_run(statistics[lane], MAX_TIMC + 1);
+		ctrl->timings[channel][slotrank].lanes[lane].timC = rn.middle;
+		if (rn.all)
+			printk(BIOS_CRIT, "timC discovery failed");
+		printram("Cval: %d, %d, %d, %x\n", channel, slotrank,
+		       lane, ctrl->timings[channel][slotrank].lanes[lane].timC);
+	}
+}
+
+static int get_precedening_channels(ramctr_timing * ctrl, int target_channel)
+{
+	int channel, ret = 0;
+	FOR_ALL_POPULATED_CHANNELS if (channel < target_channel)
+		 ret++;
+	return ret;
+}
+
+static void fill_pattern0(ramctr_timing * ctrl, int channel, u32 a, u32 b)
+{
+	unsigned j;
+	unsigned channel_offset =
+	    get_precedening_channels(ctrl, channel) * 0x40;
+	printram("channel_offset=%x\n", channel_offset);
+	for (j = 0; j < 16; j++)
+		write32((void *)(0x04000000 + channel_offset + 4 * j), j & 2 ? b : a);
+	sfence();
+}
+
+static int num_of_channels(const ramctr_timing * ctrl)
+{
+	int ret = 0;
+	int channel;
+	FOR_ALL_POPULATED_CHANNELS ret++;
+	return ret;
+}
+
+static void fill_pattern1(ramctr_timing * ctrl, int channel)
+{
+	unsigned j;
+	unsigned channel_offset =
+	    get_precedening_channels(ctrl, channel) * 0x40;
+	unsigned channel_step = 0x40 * num_of_channels(ctrl);
+	for (j = 0; j < 16; j++)
+		write32((void *)(0x04000000 + channel_offset + j * 4), 0xffffffff);
+	for (j = 0; j < 16; j++)
+		write32((void *)(0x04000000 + channel_offset + channel_step + j * 4), 0);
+	sfence();
+}
+
+static void precharge(ramctr_timing * ctrl)
+{
+	int channel, slotrank, lane;
+
+	FOR_ALL_POPULATED_CHANNELS {
+		FOR_ALL_POPULATED_RANKS FOR_ALL_LANES {
+			ctrl->timings[channel][slotrank].lanes[lane].falling =
+			    16;
+			ctrl->timings[channel][slotrank].lanes[lane].rising =
+			    16;
+		} program_timings(ctrl, channel);
+
+		FOR_ALL_POPULATED_RANKS {
+			wait_428c(channel);
+
+			write32(DEFAULT_MCHBAR + 0x4220 + 0x400 * channel,
+				0x1f000);
+			write32(DEFAULT_MCHBAR + 0x4230 + 0x400 * channel,
+				0xc01 | (ctrl->tMOD << 16));
+			write32(DEFAULT_MCHBAR + 0x4200 + 0x400 * channel,
+				(slotrank << 24) | 0x360004);
+			write32(DEFAULT_MCHBAR + 0x4210 + 0x400 * channel, 0);
+
+			write32(DEFAULT_MCHBAR + 0x4224 + 0x400 * channel,
+				0x1f105);
+			write32(DEFAULT_MCHBAR + 0x4234 + 0x400 * channel,
+				0x4041003);
+			write32(DEFAULT_MCHBAR + 0x4204 + 0x400 * channel,
+				(slotrank << 24) | 0);
+			write32(DEFAULT_MCHBAR + 0x4214 + 0x400 * channel, 0);
+
+			write32(DEFAULT_MCHBAR + 0x4228 + 0x400 * channel,
+				0x1f105);
+			write32(DEFAULT_MCHBAR + 0x4238 + 0x400 * channel,
+				0x1001 | ((ctrl->CAS + 8) << 16));
+			write32(DEFAULT_MCHBAR + 0x4208 + 0x400 * channel,
+				(slotrank << 24) | 0x60000);
+			write32(DEFAULT_MCHBAR + 0x4218 + 0x400 * channel, 0);
+
+			write32(DEFAULT_MCHBAR + 0x422c + 0x400 * channel,
+				0x1f000);
+			write32(DEFAULT_MCHBAR + 0x423c + 0x400 * channel,
+				0xc01 | (ctrl->tMOD << 16));
+			write32(DEFAULT_MCHBAR + 0x420c + 0x400 * channel,
+				(slotrank << 24) | 0x360000);
+			write32(DEFAULT_MCHBAR + 0x421c + 0x400 * channel, 0);
+			write32(DEFAULT_MCHBAR + 0x4284 + 0x400 * channel,
+				0xc0001);
+
+			wait_428c(channel);
+		}
+
+		FOR_ALL_POPULATED_RANKS FOR_ALL_LANES {
+			ctrl->timings[channel][slotrank].lanes[lane].falling =
+			    48;
+			ctrl->timings[channel][slotrank].lanes[lane].rising =
+			    48;
+		}
+
+		program_timings(ctrl, channel);
+
+		FOR_ALL_POPULATED_RANKS {
+			wait_428c(channel);
+
+			write32(DEFAULT_MCHBAR + 0x4220 + 0x400 * channel,
+				0x1f000);
+			write32(DEFAULT_MCHBAR + 0x4230 + 0x400 * channel,
+				0xc01 | (ctrl->tMOD << 16));
+			write32(DEFAULT_MCHBAR + 0x4200 + 0x400 * channel,
+				(slotrank << 24) | 0x360004);
+			write32(DEFAULT_MCHBAR + 0x4210 + 0x400 * channel, 0);
+
+			write32(DEFAULT_MCHBAR + 0x4224 + 0x400 * channel,
+				0x1f105);
+			write32(DEFAULT_MCHBAR + 0x4234 + 0x400 * channel,
+				0x4041003);
+			write32(DEFAULT_MCHBAR + 0x4204 + 0x400 * channel,
+				(slotrank << 24) | 0);
+			write32(DEFAULT_MCHBAR + 0x4214 + 0x400 * channel, 0);
+
+			write32(DEFAULT_MCHBAR + 0x4228 + 0x400 * channel,
+				0x1f105);
+			write32(DEFAULT_MCHBAR + 0x4238 + 0x400 * channel,
+				0x1001 | ((ctrl->CAS + 8) << 16));
+			write32(DEFAULT_MCHBAR + 0x4208 + 0x400 * channel,
+				(slotrank << 24) | 0x60000);
+			write32(DEFAULT_MCHBAR + 0x4218 + 0x400 * channel, 0);
+
+			write32(DEFAULT_MCHBAR + 0x422c + 0x400 * channel,
+				0x1f000);
+			write32(DEFAULT_MCHBAR + 0x423c + 0x400 * channel,
+				0xc01 | (ctrl->tMOD << 16));
+
+			write32(DEFAULT_MCHBAR + 0x420c + 0x400 * channel,
+				(slotrank << 24) | 0x360000);
+			write32(DEFAULT_MCHBAR + 0x421c + 0x400 * channel, 0);
+
+			write32(DEFAULT_MCHBAR + 0x4284 + 0x400 * channel,
+				0xc0001);
+			wait_428c(channel);
+		}
+	}
+}
+
+static void test_timB(ramctr_timing * ctrl, int channel, int slotrank)
+{
+	write_mrreg(ctrl, channel, slotrank, 1,
+		    0x80 | make_mr1(ctrl, slotrank));
+
+	wait_428c(channel);
+	write32(DEFAULT_MCHBAR + 0x4220 + 0x400 * channel, 0x1f207);
+	write32(DEFAULT_MCHBAR + 0x4230 + 0x400 * channel,
+		0x8000c01 | ((ctrl->CWL + ctrl->tWLO) << 16));
+	write32(DEFAULT_MCHBAR + 0x4200 + 0x400 * channel,
+		8 | (slotrank << 24));
+	write32(DEFAULT_MCHBAR + 0x4210 + 0x400 * channel, 0);
+
+	write32(DEFAULT_MCHBAR + 0x4224 + 0x400 * channel, 0x1f107);
+	write32(DEFAULT_MCHBAR + 0x4234 + 0x400 * channel,
+		0x4000c01 | ((ctrl->CAS + 38) << 16));
+	write32(DEFAULT_MCHBAR + 0x4204 + 0x400 * channel,
+		(slotrank << 24) | 4);
+	write32(DEFAULT_MCHBAR + 0x4214 + 0x400 * channel, 0);
+
+	write32(DEFAULT_MCHBAR + 0x400 * channel + 0x4284, 0x40001);
+	wait_428c(channel);
+
+	write_mrreg(ctrl, channel, slotrank, 1,
+		    0x1080 | make_mr1(ctrl, slotrank));
+}
+
+static void discover_timB(ramctr_timing * ctrl, int channel, int slotrank)
+{
+	int timB;
+	int statistics[NUM_LANES][128];
+	int lane;
+
+	write32(DEFAULT_MCHBAR + 0x3400, 0x108052 | (slotrank << 2));
+
+	for (timB = 0; timB < 128; timB++) {
+		FOR_ALL_LANES {
+			ctrl->timings[channel][slotrank].lanes[lane].timB = timB;
+		}
+		program_timings(ctrl, channel);
+
+		test_timB(ctrl, channel, slotrank);
+
+		FOR_ALL_LANES {
+			statistics[lane][timB] =
+			    !((read32
+			       (DEFAULT_MCHBAR + lane_registers[lane] +
+				channel * 0x100 + 4 + ((timB / 32) & 1) * 4)
+			       >> (timB % 32)) & 1);
+			printram("Bstat: %d, %d, %d, %x, %x\n",
+			       channel, slotrank, lane, timB,
+			       statistics[lane][timB]);
+		}
+	}
+	FOR_ALL_LANES {
+		struct run rn = get_longest_zero_run(statistics[lane], 128);
+		ctrl->timings[channel][slotrank].lanes[lane].timB = rn.start;
+		if (rn.all)
+			die("timB discovery failed");
+		printram("Bval: %d, %d, %d, %x\n", channel, slotrank,
+		       lane, ctrl->timings[channel][slotrank].lanes[lane].timB);
+	}
+}
+
+static int get_timB_high_adjust(u64 val)
+{
+	int i;
+
+	/* good */
+	if (val == 0xffffffffffffffffLL)
+		return 0;
+
+	if (val >= 0xf000000000000000LL) {
+		/* needs negative adjustment */
+		for (i = 0; i < 8; i++)
+			if (val << (8 * (7 - i) + 4))
+				return -i;
+	} else {
+		/* needs positive adjustment */
+		for (i = 0; i < 8; i++)
+			if (val >> (8 * (7 - i) + 4))
+				return i;
+	}
+	return 8;
+}
+
+static void adjust_high_timB(ramctr_timing * ctrl)
+{
+	int channel, slotrank, lane, old;
+	write32(DEFAULT_MCHBAR + 0x3400, 0x200);
+	FOR_ALL_POPULATED_CHANNELS {
+		fill_pattern1(ctrl, channel);
+		write32(DEFAULT_MCHBAR + 0x4288 + (channel << 10), 1);
+	}
+	FOR_ALL_POPULATED_CHANNELS FOR_ALL_POPULATED_RANKS {
+
+		write32(DEFAULT_MCHBAR + 0x4288 + 0x400 * channel, 0x10001);
+
+		wait_428c(channel);
+
+		write32(DEFAULT_MCHBAR + 0x4220 + 0x400 * channel, 0x1f006);
+		write32(DEFAULT_MCHBAR + 0x4230 + 0x400 * channel,
+			0xc01 | (ctrl->tRCD << 16));
+		write32(DEFAULT_MCHBAR + 0x4200 + 0x400 * channel,
+			(slotrank << 24) | 0x60000);
+		write32(DEFAULT_MCHBAR + 0x4210 + 0x400 * channel, 0);
+
+		write32(DEFAULT_MCHBAR + 0x4224 + 0x400 * channel, 0x1f207);
+		write32(DEFAULT_MCHBAR + 0x4234 + 0x400 * channel, 0x8040c01);
+		write32(DEFAULT_MCHBAR + 0x4204 + 0x400 * channel,
+			(slotrank << 24) | 0x8);
+		write32(DEFAULT_MCHBAR + 0x4214 + 0x400 * channel, 0x3e0);
+
+		write32(DEFAULT_MCHBAR + 0x4228 + 0x400 * channel, 0x1f201);
+		write32(DEFAULT_MCHBAR + 0x4238 + 0x400 * channel, 0x8041003);
+		write32(DEFAULT_MCHBAR + 0x4208 + 0x400 * channel,
+			(slotrank << 24));
+		write32(DEFAULT_MCHBAR + 0x4218 + 0x400 * channel, 0x3e2);
+
+		write32(DEFAULT_MCHBAR + 0x422c + 0x400 * channel, 0x1f207);
+		write32(DEFAULT_MCHBAR + 0x423c + 0x400 * channel,
+			0x8000c01 | ((ctrl->CWL + ctrl->tWTR + 5) << 16));
+		write32(DEFAULT_MCHBAR + 0x420c + 0x400 * channel,
+			(slotrank << 24) | 0x8);
+		write32(DEFAULT_MCHBAR + 0x421c + 0x400 * channel, 0x3e0);
+
+		write32(DEFAULT_MCHBAR + 0x4284 + 0x400 * channel, 0xc0001);
+
+		wait_428c(channel);
+
+		write32(DEFAULT_MCHBAR + 0x4220 + 0x400 * channel, 0x1f002);
+		write32(DEFAULT_MCHBAR + 0x4230 + 0x400 * channel,
+			0xc01 | ((ctrl->tRP) << 16));
+		write32(DEFAULT_MCHBAR + 0x4200 + 0x400 * channel,
+			(slotrank << 24) | 0x60400);
+		write32(DEFAULT_MCHBAR + 0x4210 + 0x400 * channel, 0x240);
+
+		write32(DEFAULT_MCHBAR + 0x4224 + 0x400 * channel, 0x1f006);
+		write32(DEFAULT_MCHBAR + 0x4234 + 0x400 * channel,
+			0xc01 | ((ctrl->tRCD) << 16));
+		write32(DEFAULT_MCHBAR + 0x4204 + 0x400 * channel,
+			(slotrank << 24) | 0x60000);
+		write32(DEFAULT_MCHBAR + 0x4214 + 0x400 * channel, 0);
+
+		write32(DEFAULT_MCHBAR + 0x4228 + 0x400 * channel, 0x3f105);
+		write32(DEFAULT_MCHBAR + 0x4238 + 0x400 * channel,
+			0x4000c01 |
+			((ctrl->tRP +
+			  ctrl->timings[channel][slotrank].val_4024 +
+			  ctrl->timings[channel][slotrank].val_4028) << 16));
+		write32(DEFAULT_MCHBAR + 0x4208 + 0x400 * channel,
+			(slotrank << 24) | 0x60008);
+		write32(DEFAULT_MCHBAR + 0x4218 + 0x400 * channel, 0);
+
+		write32(DEFAULT_MCHBAR + 0x4284 + 0x400 * channel, 0x80001);
+		wait_428c(channel);
+		FOR_ALL_LANES {
+			u64 res =
+				read32(DEFAULT_MCHBAR + lane_registers[lane] +
+					0x100 * channel + 4);
+			res |=
+				((u64) read32(DEFAULT_MCHBAR + lane_registers[lane] +
+					0x100 * channel + 8)) << 32;
+			old = ctrl->timings[channel][slotrank].lanes[lane].timB;
+			ctrl->timings[channel][slotrank].lanes[lane].timB +=
+				get_timB_high_adjust(res) * 64;
+
+			printk(BIOS_DEBUG, "High adjust %d:%016llx\n", lane, res);
+			printram("Bval+: %d, %d, %d, %x -> %x\n", channel,
+				slotrank, lane, old,
+				ctrl->timings[channel][slotrank].lanes[lane].
+				timB);
+		}
+	}
+	write32(DEFAULT_MCHBAR + 0x3400, 0);
+}
+
+static void write_op(ramctr_timing * ctrl, int channel)
+{
+	int slotrank;
+
+	wait_428c(channel);
+
+	/* choose an existing rank.  */
+	slotrank = !(ctrl->rankmap[channel] & 1) ? 2 : 0;
+
+	write32(DEFAULT_MCHBAR + 0x4220 + 0x400 * channel, 0x0f003);
+	write32(DEFAULT_MCHBAR + 0x4230 + 0x400 * channel, 0x41001);
+
+	write32(DEFAULT_MCHBAR + 0x4200 + 0x400 * channel,
+		(slotrank << 24) | 0x60000);
+
+	write32(DEFAULT_MCHBAR + 0x4210 + 0x400 * channel, 0x3e0);
+
+	write32(DEFAULT_MCHBAR + 0x4284 + 0x400 * channel, 1);
+	wait_428c(channel);
+}
+
+static void write_training(ramctr_timing * ctrl)
+{
+	int channel, slotrank, lane;
+	u32 r32;
+
+	FOR_ALL_POPULATED_CHANNELS
+	    write32(DEFAULT_MCHBAR + 0x4008 + 0x400 * channel,
+		    read32(DEFAULT_MCHBAR + 0x4008 +
+			   0x400 * channel) | 0x8000000);
+
+	FOR_ALL_POPULATED_CHANNELS {
+		write_op(ctrl, channel);
+		write32(DEFAULT_MCHBAR + 0x4020 + 0x400 * channel,
+			read32(DEFAULT_MCHBAR + 0x4020 +
+			       0x400 * channel) | 0x200000);
+	}
+	write32(DEFAULT_MCHBAR + 0x5030, read32(DEFAULT_MCHBAR + 0x5030) & ~8);
+	FOR_ALL_POPULATED_CHANNELS {
+		write_op(ctrl, channel);
+	}
+
+	FOR_ALL_CHANNELS
+	    FOR_ALL_POPULATED_RANKS
+		write_mrreg(ctrl, channel, slotrank, 1,
+			    make_mr1(ctrl, slotrank) | 0x1080);
+
+	write32(DEFAULT_MCHBAR + 0x3400, 0x108052);
+
+	r32 = read32(DEFAULT_MCHBAR + 0x5030);
+	write32(DEFAULT_MCHBAR + 0x5030, r32 | 0x20);
+	udelay(1);
+
+	write32(DEFAULT_MCHBAR + 0x5030, r32 & ~0x20);
+
+	udelay(1);
+
+	FOR_ALL_CHANNELS FOR_ALL_POPULATED_RANKS
+		discover_timB(ctrl, channel, slotrank);
+
+	FOR_ALL_CHANNELS FOR_ALL_POPULATED_RANKS
+		write_mrreg(ctrl, channel,
+			    slotrank, 1, make_mr1(ctrl, slotrank));
+
+	write32(DEFAULT_MCHBAR + 0x3400, 0);
+
+	FOR_ALL_POPULATED_CHANNELS
+		wait_428c(channel);
+
+	write32(DEFAULT_MCHBAR + 0x5030, read32(DEFAULT_MCHBAR + 0x5030) | 8);
+
+	FOR_ALL_POPULATED_CHANNELS {
+		write32(DEFAULT_MCHBAR + 0x4020 + 0x400 * channel,
+			~0x00200000 & read32(DEFAULT_MCHBAR + 0x4020 +
+					     0x400 * channel));
+		read32(DEFAULT_MCHBAR + 0x428c + 0x400 * channel);
+		wait_428c(channel);
+
+		write32(DEFAULT_MCHBAR + 0x4220 + 0x400 * channel, 0x0f003);
+		write32(DEFAULT_MCHBAR + 0x4230 + 0x400 * channel, 0x659001);
+		write32(DEFAULT_MCHBAR + 0x4200 + 0x400 * channel, 0x60000);
+		write32(DEFAULT_MCHBAR + 0x4210 + 0x400 * channel, 0x3e0);
+
+		write32(DEFAULT_MCHBAR + 0x4284 + 0x400 * channel, 1);
+		wait_428c(channel);
+	}
+
+	r32 = read32(DEFAULT_MCHBAR + 0x5030);
+	write32(DEFAULT_MCHBAR + 0x5030, r32 | 0x20);
+	udelay(1);
+
+	write32(DEFAULT_MCHBAR + 0x5030, r32 & ~0x20);
+
+	udelay(1);
+
+	printram("CPE\n");
+	precharge(ctrl);
+	printram("CPF\n");
+
+	FOR_ALL_CHANNELS FOR_ALL_POPULATED_RANKS FOR_ALL_LANES {
+		read32(DEFAULT_MCHBAR + 0x4080 + 0x400 * channel + 4 * lane);
+		write32(DEFAULT_MCHBAR + 0x4080 + 0x400 * channel + 4 * lane,
+			0);
+	}
+
+	FOR_ALL_POPULATED_CHANNELS {
+		fill_pattern0(ctrl, channel, 0xaaaaaaaa, 0x55555555);
+		write32(DEFAULT_MCHBAR + 0x4288 + (channel << 10), 0);
+	}
+
+	FOR_ALL_CHANNELS FOR_ALL_POPULATED_RANKS
+		discover_timC(ctrl, channel, slotrank);
+
+	FOR_ALL_POPULATED_CHANNELS
+		program_timings(ctrl, channel);
+
+	adjust_high_timB(ctrl);
+
+	FOR_ALL_POPULATED_CHANNELS
+		program_timings(ctrl, channel);
+
+	FOR_ALL_CHANNELS FOR_ALL_POPULATED_RANKS FOR_ALL_LANES {
+		read32(DEFAULT_MCHBAR + 0x4080 + 0x400 * channel + 4 * lane);
+		write32(DEFAULT_MCHBAR + 0x4080 + 0x400 * channel + 4 * lane,
+			0);
+	}
+}
+
+static int test_320c(ramctr_timing * ctrl, int channel, int slotrank)
+{
+	struct ram_rank_timings saved_rt = ctrl->timings[channel][slotrank];
+	int timC_delta;
+	int lanes_ok = 0;
+	int ctr = 0;
+	int lane;
+
+	for (timC_delta = -5; timC_delta <= 5; timC_delta++) {
+		FOR_ALL_LANES {
+			ctrl->timings[channel][slotrank].lanes[lane].timC =
+			    saved_rt.lanes[lane].timC + timC_delta;
+		}
+		program_timings(ctrl, channel);
+		FOR_ALL_LANES {
+			write32(DEFAULT_MCHBAR + 4 * lane + 0x4f40, 0);
+		}
+
+		write32(DEFAULT_MCHBAR + 0x4288 + 0x400 * channel, 0x1f);
+
+		wait_428c(channel);
+
+		write32(DEFAULT_MCHBAR + 0x4220 + 0x400 * channel, 0x1f006);
+		write32(DEFAULT_MCHBAR + 0x4230 + 0x400 * channel,
+			((max(ctrl->tRRD, (ctrl->tFAW >> 2) + 1)) << 10)
+			| 8 | (ctrl->tRCD << 16));
+
+		write32(DEFAULT_MCHBAR + 0x4200 + 0x400 * channel,
+			(slotrank << 24) | ctr | 0x60000);
+
+		write32(DEFAULT_MCHBAR + 0x4210 + 0x400 * channel, 0x244);
+
+		write32(DEFAULT_MCHBAR + 0x4224 + 0x400 * channel, 0x1f201);
+		write32(DEFAULT_MCHBAR + 0x4234 + 0x400 * channel,
+			0x8001020 | ((ctrl->CWL + ctrl->tWTR + 8) << 16));
+		write32(DEFAULT_MCHBAR + 0x4204 + 0x400 * channel,
+			(slotrank << 24));
+		write32(DEFAULT_MCHBAR + 0x4244 + 0x400 * channel, 0x389abcd);
+		write32(DEFAULT_MCHBAR + 0x4214 + 0x400 * channel, 0x20e42);
+
+		write32(DEFAULT_MCHBAR + 0x4228 + 0x400 * channel, 0x1f105);
+		write32(DEFAULT_MCHBAR + 0x4238 + 0x400 * channel,
+			0x4001020 | (max(ctrl->tRTP, 8) << 16));
+		write32(DEFAULT_MCHBAR + 0x4208 + 0x400 * channel,
+			(slotrank << 24));
+		write32(DEFAULT_MCHBAR + 0x4248 + 0x400 * channel, 0x389abcd);
+		write32(DEFAULT_MCHBAR + 0x4218 + 0x400 * channel, 0x20e42);
+
+		write32(DEFAULT_MCHBAR + 0x422c + 0x400 * channel, 0x1f002);
+		write32(DEFAULT_MCHBAR + 0x423c + 0x400 * channel, 0xf1001);
+		write32(DEFAULT_MCHBAR + 0x420c + 0x400 * channel,
+			(slotrank << 24) | 0x60400);
+		write32(DEFAULT_MCHBAR + 0x421c + 0x400 * channel, 0x240);
+
+		write32(DEFAULT_MCHBAR + 0x4284 + 0x400 * channel, 0xc0001);
+		wait_428c(channel);
+		FOR_ALL_LANES {
+			u32 r32 =
+			    read32(DEFAULT_MCHBAR + 0x4340 + 4 * lane +
+				   0x400 * channel);
+
+			if (r32 == 0)
+				lanes_ok |= 1 << lane;
+		}
+		ctr++;
+		if (lanes_ok == ((1 << NUM_LANES) - 1))
+			break;
+	}
+
+	ctrl->timings[channel][slotrank] = saved_rt;
+
+	printram("3lanes: %x\n", lanes_ok);
+	return lanes_ok != ((1 << NUM_LANES) - 1);
+}
+
+#include "raminit_patterns.h"
+
+static void fill_pattern5(ramctr_timing * ctrl, int channel, int patno)
+{
+	unsigned i, j;
+	unsigned channel_offset =
+	    get_precedening_channels(ctrl, channel) * 0x40;
+	unsigned channel_step = 0x40 * num_of_channels(ctrl);
+
+	if (patno) {
+		u8 base8 = 0x80 >> ((patno - 1) % 8);
+		u32 base = base8 | (base8 << 8) | (base8 << 16) | (base8 << 24);
+		for (i = 0; i < 32; i++) {
+			for (j = 0; j < 16; j++) {
+				u32 val = use_base[patno - 1][i] & (1 << (j / 2)) ? base : 0;
+				if (invert[patno - 1][i] & (1 << (j / 2)))
+					val = ~val;
+				write32((void *)(0x04000000 + channel_offset + i * channel_step +
+						 j * 4), val);
 			}
-			die("Nonzero MRC return value.\n");
 		}
+
 	} else {
-		die("UEFI PEI System Agent not found.\n");
+		for (i = 0; i < sizeof(pattern) / sizeof(pattern[0]); i++) {
+			for (j = 0; j < 16; j++)
+				write32((void *)(0x04000000 + channel_offset + i * channel_step +
+						 j * 4), pattern[i][j]);
+		}
+		sfence();
 	}
+}
 
-#if CONFIG_USBDEBUG_IN_ROMSTAGE
-	/* mrc.bin reconfigures USB, so reinit it to have debug */
-	usbdebug_init();
-#endif
+static void reprogram_320c(ramctr_timing * ctrl)
+{
+	int channel, slotrank;
+	u32 r32;
 
-	/* For reference print the System Agent version
-	 * after executing the UEFI PEI stage.
-	 */
-	u32 version = MCHBAR32(0x5034);
-	printk(BIOS_DEBUG, "System Agent Version %d.%d.%d Build %d\n",
-		version >> 24 , (version >> 16) & 0xff,
-		(version >> 8) & 0xff, version & 0xff);
-
-	/* Send ME init done for SandyBridge here.  This is done
-	 * inside the SystemAgent binary on IvyBridge. */
-	if (BASE_REV_SNB ==
-	    (pci_read_config16(PCI_CPU_DEVICE, PCI_DEVICE_ID) & BASE_REV_MASK))
-		intel_early_me_init_done(ME_INIT_STATUS_SUCCESS);
+	FOR_ALL_POPULATED_CHANNELS {
+		wait_428c(channel);
+
+		/* choose an existing rank.  */
+		slotrank = !(ctrl->rankmap[channel] & 1) ? 2 : 0;
+
+		write32(DEFAULT_MCHBAR + 0x4220 + 0x400 * channel, 0x0f003);
+		write32(DEFAULT_MCHBAR + 0x4230 + 0x400 * channel, 0x41001);
+
+		write32(DEFAULT_MCHBAR + 0x4200 + 0x400 * channel,
+			(slotrank << 24) | 0x60000);
+
+		write32(DEFAULT_MCHBAR + 0x4210 + 0x400 * channel, 0x3e0);
+
+		write32(DEFAULT_MCHBAR + 0x4284 + 0x400 * channel, 1);
+		wait_428c(channel);
+		write32(DEFAULT_MCHBAR + 0x4020 + 0x400 * channel,
+			read32(DEFAULT_MCHBAR + 0x4020 +
+			       0x400 * channel) | 0x200000);
+	}
+	write32(DEFAULT_MCHBAR + 0x5030, read32(DEFAULT_MCHBAR + 0x5030) & ~8);
+	FOR_ALL_POPULATED_CHANNELS {
+		wait_428c(channel);
+
+		/* choose an existing rank.  */
+		slotrank = !(ctrl->rankmap[channel] & 1) ? 2 : 0;
+
+		write32(DEFAULT_MCHBAR + 0x4220 + 0x400 * channel, 0x0f003);
+		write32(DEFAULT_MCHBAR + 0x4230 + 0x400 * channel, 0x41001);
+
+		write32(DEFAULT_MCHBAR + 0x4200 + 0x400 * channel,
+			(slotrank << 24) | 0x60000);
+
+		write32(DEFAULT_MCHBAR + 0x4210 + 0x400 * channel, 0x3e0);
+
+		write32(DEFAULT_MCHBAR + 0x4284 + 0x400 * channel, 1);
+		wait_428c(channel);
+	}
+
+	/* jedec reset */
+	dram_jedecreset(ctrl);
+	/* mrs commands. */
+	dram_mrscommands(ctrl);
+
+	r32 = read32(DEFAULT_MCHBAR + 0x5030);
+	write32(DEFAULT_MCHBAR + 0x5030, r32 | 0x20);
+	udelay(1);
+
+	write32(DEFAULT_MCHBAR + 0x5030, r32 & ~0x20);
+
+	udelay(1);
+}
+
+#define MIN_C320C_LEN 13
+
+static int try_cmd_stretch(ramctr_timing * ctrl, int cmd_stretch)
+{
+	struct ram_rank_timings saved_timings[NUM_CHANNELS][NUM_SLOTRANKS];
+	int channel, slotrank;
+	int c320c;
+	int stat[NUM_SLOTRANKS][256];
+
+	FOR_ALL_CHANNELS FOR_ALL_POPULATED_RANKS {
+		saved_timings[channel][slotrank] = ctrl->timings[channel][slotrank];
+	}
+
+	FOR_ALL_POPULATED_CHANNELS {
+		ctrl->cmd_stretch[channel] = cmd_stretch;
+	}
+
+	FOR_ALL_POPULATED_CHANNELS
+	    MCHBAR32(0x4004 + 0x400 * channel) =
+		ctrl->tRRD
+		| (ctrl->tRTP << 4)
+		| (ctrl->tCKE << 8)
+		| (ctrl->tWTR << 12)
+		| (ctrl->tFAW << 16)
+		| (ctrl->tWR << 24)
+		| (ctrl->cmd_stretch[channel] << 30);
+
+
+	FOR_ALL_CHANNELS {
+		int delta = 0;
+		if (ctrl->cmd_stretch[channel] == 2)
+			delta = 2;
+		else if (ctrl->cmd_stretch[channel] == 0)
+			delta = 4;
+
+		FOR_ALL_POPULATED_RANKS {
+			ctrl->timings[channel][slotrank].val_4024 -= delta;
+		}
+	}
+
+	FOR_ALL_POPULATED_CHANNELS {
+		for (c320c = -127; c320c <= 127; c320c++) {
+			FOR_ALL_POPULATED_RANKS {
+				ctrl->timings[channel][slotrank].val_320c = c320c;
+			}
+			program_timings(ctrl, channel);
+			reprogram_320c(ctrl);
+			FOR_ALL_POPULATED_RANKS {
+				stat[slotrank][c320c + 127] =
+				    test_320c(ctrl, channel, slotrank);
+				printram("3stat: %d, %d, %d: %d\n",
+				       channel, slotrank, c320c,
+				       stat[slotrank][c320c + 127]);
+			}
+		}
+		FOR_ALL_POPULATED_RANKS {
+			struct run rn =
+			    get_longest_zero_run(stat[slotrank], 255);
+			ctrl->timings[channel][slotrank].val_320c =
+			    rn.middle - 127;
+			printram("3val: %d, %d: %d\n", channel,
+			       slotrank,
+			       ctrl->timings[channel][slotrank].val_320c);
+			if (rn.all || rn.length < MIN_C320C_LEN) {
+				FOR_ALL_CHANNELS FOR_ALL_POPULATED_RANKS {
+					ctrl->timings[channel][slotrank] = saved_timings[channel][slotrank];
+				}
+				return 0;
+			}
+		}
+	}
+	return 1;
+}
+
+static void command_training(ramctr_timing * ctrl)
+{
+	int channel;
+
+	FOR_ALL_POPULATED_CHANNELS {
+		fill_pattern5(ctrl, channel, 0);
+		write32(DEFAULT_MCHBAR + 0x4288 + 0x400 * channel, 0x1f);
+	}
+
+	/* try command rate 1T and 2T */
+	if (!try_cmd_stretch(ctrl, 0) && !try_cmd_stretch(ctrl, 2))
+		die("c320c discovery failed");
+
+	FOR_ALL_POPULATED_CHANNELS {
+		program_timings(ctrl, channel);
+	}
+
+	reprogram_320c(ctrl);
+}
+
+static void discover_edges_real(ramctr_timing * ctrl, int channel, int slotrank,
+				int *edges)
+{
+	int edge;
+	int statistics[NUM_LANES][MAX_EDGE_TIMING + 1];
+	int lane;
+
+	for (edge = 0; edge <= MAX_EDGE_TIMING; edge++) {
+		FOR_ALL_LANES {
+			ctrl->timings[channel][slotrank].lanes[lane].rising =
+			    edge;
+			ctrl->timings[channel][slotrank].lanes[lane].falling =
+			    edge;
+		}
+		printram("edge %02x\n", edge);
+		program_timings(ctrl, channel);
+
+		FOR_ALL_LANES {
+			write32(DEFAULT_MCHBAR + 0x4340 + 0x400 * channel +
+				4 * lane, 0);
+			read32(DEFAULT_MCHBAR + 0x400 * channel + 4 * lane +
+			       0x4140);
+		}
+
+		wait_428c(channel);
+
+		write32(DEFAULT_MCHBAR + 0x4220 + 0x400 * channel, 0x1f000);
+		write32(DEFAULT_MCHBAR + 0x4230 + 0x400 * channel,
+			(0xc01 | (ctrl->tMOD << 16)));
+		write32(DEFAULT_MCHBAR + 0x4200 + 0x400 * channel,
+			(slotrank << 24) | 0x360004);
+		write32(DEFAULT_MCHBAR + 0x4210 + 0x400 * channel, 0);
+
+		write32(DEFAULT_MCHBAR + 0x4224 + 0x400 * channel, 0x1f105);
+		write32(DEFAULT_MCHBAR + 0x4234 + 0x400 * channel, 0x40411f4);
+		write32(DEFAULT_MCHBAR + 0x4204 + 0x400 * channel,
+			(slotrank << 24));
+		write32(DEFAULT_MCHBAR + 0x4214 + 0x400 * channel, 0);
+
+		write32(DEFAULT_MCHBAR + 0x4228 + 0x400 * channel, 0x1f105);
+		write32(DEFAULT_MCHBAR + 0x4238 + 0x400 * channel,
+			0x1001 | ((ctrl->CAS + 8) << 16));
+		write32(DEFAULT_MCHBAR + 0x4208 + 0x400 * channel,
+			(slotrank << 24) | 0x60000);
+		write32(DEFAULT_MCHBAR + 0x4218 + 0x400 * channel, 0);
+
+		write32(DEFAULT_MCHBAR + 0x422c + 0x400 * channel, 0x1f000);
+		write32(DEFAULT_MCHBAR + 0x423c + 0x400 * channel,
+			(0xc01 | (ctrl->tMOD << 16)));
+		write32(DEFAULT_MCHBAR + 0x420c + 0x400 * channel,
+			(slotrank << 24) | 0x360000);
+		write32(DEFAULT_MCHBAR + 0x421c + 0x400 * channel, 0);
+
+		write32(DEFAULT_MCHBAR + 0x4284 + 0x400 * channel, 0xc0001);
+
+		wait_428c(channel);
+
+		FOR_ALL_LANES {
+			statistics[lane][edge] =
+			    read32(DEFAULT_MCHBAR + 0x4340 + 0x400 * channel +
+				   lane * 4);
+		}
+	}
+	FOR_ALL_LANES {
+		struct run rn =
+		    get_longest_zero_run(statistics[lane], MAX_EDGE_TIMING + 1);
+		edges[lane] = rn.middle;
+		if (rn.all)
+			die("edge discovery failed");
+		printram("eval %d, %d, %d, %02x\n", channel, slotrank,
+		       lane, edges[lane]);
+	}
+}
+
+static void discover_edges(ramctr_timing * ctrl)
+{
+	int falling_edges[NUM_CHANNELS][NUM_SLOTRANKS][NUM_LANES];
+	int rising_edges[NUM_CHANNELS][NUM_SLOTRANKS][NUM_LANES];
+	int channel, slotrank, lane;
+	u32 r32;
+
+	write32(DEFAULT_MCHBAR + 0x3400, 0);
+
+	r32 = read32(DEFAULT_MCHBAR + 0x5030);
+	write32(DEFAULT_MCHBAR + 0x5030, r32 | 0x20);
+	udelay(1);
+
+	write32(DEFAULT_MCHBAR + 0x5030, r32 & ~0x20);
+
+	udelay(1);
+
+	FOR_ALL_POPULATED_CHANNELS FOR_ALL_LANES {
+		write32(DEFAULT_MCHBAR + 4 * lane +
+			0x400 * channel + 0x4080, 0);
+	}
+
+	FOR_ALL_POPULATED_CHANNELS {
+		fill_pattern0(ctrl, channel, 0, 0);
+		write32(DEFAULT_MCHBAR + 0x4288 + (channel << 10), 0);
+		FOR_ALL_LANES {
+			read32(DEFAULT_MCHBAR + 0x400 * channel +
+			       lane * 4 + 0x4140);
+		}
+
+		FOR_ALL_POPULATED_RANKS FOR_ALL_LANES {
+			ctrl->timings[channel][slotrank].lanes[lane].falling =
+			    16;
+			ctrl->timings[channel][slotrank].lanes[lane].rising =
+			    16;
+		}
+
+		program_timings(ctrl, channel);
+
+		FOR_ALL_POPULATED_RANKS {
+			wait_428c(channel);
+
+			write32(DEFAULT_MCHBAR + 0x4220 + 0x400 * channel,
+				0x1f000);
+			write32(DEFAULT_MCHBAR + 0x4230 + 0x400 * channel,
+				0xc01 | (ctrl->tMOD << 16));
+			write32(DEFAULT_MCHBAR + 0x4200 + 0x400 * channel,
+				(slotrank << 24) | 0x360004);
+			write32(DEFAULT_MCHBAR + 0x4210 + 0x400 * channel, 0);
+
+			write32(DEFAULT_MCHBAR + 0x4224 + 0x400 * channel,
+				0x1f105);
+			write32(DEFAULT_MCHBAR + 0x4234 + 0x400 * channel,
+				0x4041003);
+			write32(DEFAULT_MCHBAR + 0x4204 + 0x400 * channel,
+				(slotrank << 24) | 0);
+			write32(DEFAULT_MCHBAR + 0x4214 + 0x400 * channel, 0);
+
+			write32(DEFAULT_MCHBAR + 0x4228 + 0x400 * channel,
+				0x1f105);
+			write32(DEFAULT_MCHBAR + 0x4238 + 0x400 * channel,
+				0x1001 | ((ctrl->CAS + 8) << 16));
+			write32(DEFAULT_MCHBAR + 0x4208 + 0x400 * channel,
+				(slotrank << 24) | 0x60000);
+			write32(DEFAULT_MCHBAR + 0x4218 + 0x400 * channel, 0);
+
+			write32(DEFAULT_MCHBAR + 0x422c + 0x400 * channel,
+				0x1f000);
+			write32(DEFAULT_MCHBAR + 0x423c + 0x400 * channel,
+				0xc01 | (ctrl->tMOD << 16));
+			write32(DEFAULT_MCHBAR + 0x420c + 0x400 * channel,
+				(slotrank << 24) | 0x360000);
+			write32(DEFAULT_MCHBAR + 0x421c + 0x400 * channel, 0);
+			write32(DEFAULT_MCHBAR + 0x4284 + 0x400 * channel,
+				0xc0001);
+
+			wait_428c(channel);
+		}
+
+		FOR_ALL_POPULATED_RANKS FOR_ALL_LANES {
+			ctrl->timings[channel][slotrank].lanes[lane].falling =
+			    48;
+			ctrl->timings[channel][slotrank].lanes[lane].rising =
+			    48;
+		}
+
+		program_timings(ctrl, channel);
+
+		FOR_ALL_POPULATED_RANKS {
+			wait_428c(channel);
+
+			write32(DEFAULT_MCHBAR + 0x4220 + 0x400 * channel,
+				0x1f000);
+			write32(DEFAULT_MCHBAR + 0x4230 + 0x400 * channel,
+				0xc01 | (ctrl->tMOD << 16));
+			write32(DEFAULT_MCHBAR + 0x4200 + 0x400 * channel,
+				(slotrank << 24) | 0x360004);
+			write32(DEFAULT_MCHBAR + 0x4210 + 0x400 * channel, 0);
+
+			write32(DEFAULT_MCHBAR + 0x4224 + 0x400 * channel,
+				0x1f105);
+			write32(DEFAULT_MCHBAR + 0x4234 + 0x400 * channel,
+				0x4041003);
+			write32(DEFAULT_MCHBAR + 0x4204 + 0x400 * channel,
+				(slotrank << 24) | 0);
+			write32(DEFAULT_MCHBAR + 0x4214 + 0x400 * channel, 0);
+
+			write32(DEFAULT_MCHBAR + 0x4228 + 0x400 * channel,
+				0x1f105);
+			write32(DEFAULT_MCHBAR + 0x4238 + 0x400 * channel,
+				0x1001 | ((ctrl->CAS + 8) << 16));
+			write32(DEFAULT_MCHBAR + 0x4208 + 0x400 * channel,
+				(slotrank << 24) | 0x60000);
+			write32(DEFAULT_MCHBAR + 0x4218 + 0x400 * channel, 0);
+
+			write32(DEFAULT_MCHBAR + 0x422c + 0x400 * channel,
+				0x1f000);
+			write32(DEFAULT_MCHBAR + 0x423c + 0x400 * channel,
+				0xc01 | (ctrl->tMOD << 16));
+			write32(DEFAULT_MCHBAR + 0x420c + 0x400 * channel,
+				(slotrank << 24) | 0x360000);
+			write32(DEFAULT_MCHBAR + 0x421c + 0x400 * channel, 0);
+
+			write32(DEFAULT_MCHBAR + 0x4284 + 0x400 * channel,
+				0xc0001);
+			wait_428c(channel);
+		}
+
+		FOR_ALL_LANES {
+			write32(DEFAULT_MCHBAR + 0x4080 + 0x400 * channel +
+				lane * 4,
+				~read32(DEFAULT_MCHBAR + 0x4040 +
+					0x400 * channel + lane * 4) & 0xff);
+		}
+
+		fill_pattern0(ctrl, channel, 0, 0xffffffff);
+		write32(DEFAULT_MCHBAR + 0x4288 + (channel << 10), 0);
+	}
+
+	/* FIXME: under some conditions (older chipsets?) vendor BIOS sets both edges to the same value.  */
+	write32(DEFAULT_MCHBAR + 0x4eb0, 0x300);
+
+	FOR_ALL_CHANNELS FOR_ALL_POPULATED_RANKS {
+		discover_edges_real(ctrl, channel, slotrank,
+				    falling_edges[channel][slotrank]);
+	}
+
+	write32(DEFAULT_MCHBAR + 0x4eb0, 0x200);
+
+	FOR_ALL_CHANNELS FOR_ALL_POPULATED_RANKS {
+		discover_edges_real(ctrl, channel, slotrank,
+				    rising_edges[channel][slotrank]);
+	}
+
+	write32(DEFAULT_MCHBAR + 0x4eb0, 0);
+
+	FOR_ALL_CHANNELS FOR_ALL_POPULATED_RANKS FOR_ALL_LANES {
+		ctrl->timings[channel][slotrank].lanes[lane].falling =
+		    falling_edges[channel][slotrank][lane];
+		ctrl->timings[channel][slotrank].lanes[lane].rising =
+		    rising_edges[channel][slotrank][lane];
+	}
+
+	FOR_ALL_POPULATED_CHANNELS {
+		program_timings(ctrl, channel);
+	}
+
+	FOR_ALL_CHANNELS FOR_ALL_POPULATED_RANKS FOR_ALL_LANES {
+		write32(DEFAULT_MCHBAR + 0x4080 + 0x400 * channel + 4 * lane,
+			0);
+	}
+}
+
+static void discover_edges_write_real(ramctr_timing * ctrl, int channel,
+				      int slotrank, int *edges)
+{
+	int edge;
+	u32 raw_statistics[MAX_EDGE_TIMING + 1];
+	int statistics[MAX_EDGE_TIMING + 1];
+	const int reg3000b24[] = { 0, 0xc, 0x2c };
+	int lane, i;
+	int lower[NUM_LANES];
+	int upper[NUM_LANES];
+	int pat;
+
+	FOR_ALL_LANES {
+		lower[lane] = 0;
+		upper[lane] = MAX_EDGE_TIMING;
+	}
+
+	for (i = 0; i < 3; i++) {
+		write32(DEFAULT_MCHBAR + 0x3000 + 0x100 * channel,
+			reg3000b24[i] << 24);
+		for (pat = 0; pat < NUM_PATTERNS; pat++) {
+			fill_pattern5(ctrl, channel, pat);
+			write32(DEFAULT_MCHBAR + 0x4288 + 0x400 * channel, 0x1f);
+			printram("patterned\n");
+			printram("[%x] = 0x%08x\n(%d, %d)\n",
+			       0x3000 + 0x100 * channel, reg3000b24[i] << 24, channel,
+			       slotrank);
+			for (edge = 0; edge <= MAX_EDGE_TIMING; edge++) {
+				FOR_ALL_LANES {
+					ctrl->timings[channel][slotrank].lanes[lane].
+						rising = edge;
+					ctrl->timings[channel][slotrank].lanes[lane].
+						falling = edge;
+				}
+				program_timings(ctrl, channel);
+
+				FOR_ALL_LANES {
+					write32(DEFAULT_MCHBAR + 0x4340 +
+						0x400 * channel + 4 * lane, 0);
+					read32(DEFAULT_MCHBAR + 0x400 * channel +
+					       4 * lane + 0x4140);
+				}
+				wait_428c(channel);
+
+				write32(DEFAULT_MCHBAR + 0x4220 + 0x400 * channel,
+					0x1f006);
+				write32(DEFAULT_MCHBAR + 0x4230 + 0x400 * channel,
+					0x4 | (ctrl->tRCD << 16)
+					| (max(ctrl->tRRD, (ctrl->tFAW >> 2) + 1) <<
+					   10));
+				write32(DEFAULT_MCHBAR + 0x4200 + 0x400 * channel,
+					(slotrank << 24) | 0x60000);
+				write32(DEFAULT_MCHBAR + 0x4210 + 0x400 * channel,
+					0x240);
+
+				write32(DEFAULT_MCHBAR + 0x4224 + 0x400 * channel,
+					0x1f201);
+				write32(DEFAULT_MCHBAR + 0x4234 + 0x400 * channel,
+					0x8005020 | ((ctrl->tWTR + ctrl->CWL + 8) <<
+						     16));
+				write32(DEFAULT_MCHBAR + 0x4204 + 0x400 * channel,
+					(slotrank << 24));
+				write32(DEFAULT_MCHBAR + 0x4214 + 0x400 * channel,
+					0x242);
+
+				write32(DEFAULT_MCHBAR + 0x4228 + 0x400 * channel,
+					0x1f105);
+				write32(DEFAULT_MCHBAR + 0x4238 + 0x400 * channel,
+					0x4005020 | (max(ctrl->tRTP, 8) << 16));
+				write32(DEFAULT_MCHBAR + 0x4208 + 0x400 * channel,
+					(slotrank << 24));
+				write32(DEFAULT_MCHBAR + 0x4218 + 0x400 * channel,
+					0x242);
+
+				write32(DEFAULT_MCHBAR + 0x422c + 0x400 * channel,
+					0x1f002);
+				write32(DEFAULT_MCHBAR + 0x423c + 0x400 * channel,
+					0xc01 | (ctrl->tRP << 16));
+				write32(DEFAULT_MCHBAR + 0x420c + 0x400 * channel,
+					(slotrank << 24) | 0x60400);
+				write32(DEFAULT_MCHBAR + 0x421c + 0x400 * channel, 0);
+
+				write32(DEFAULT_MCHBAR + 0x4284 + 0x400 * channel,
+					0xc0001);
+				wait_428c(channel);
+				FOR_ALL_LANES {
+					read32(DEFAULT_MCHBAR + 0x4340 +
+					       0x400 * channel + lane * 4);
+				}
+
+				raw_statistics[edge] =
+					MCHBAR32(0x436c + 0x400 * channel);
+			}
+			FOR_ALL_LANES {
+				struct run rn;
+				for (edge = 0; edge <= MAX_EDGE_TIMING; edge++)
+					statistics[edge] =
+						! !(raw_statistics[edge] & (1 << lane));
+				rn = get_longest_zero_run(statistics,
+							  MAX_EDGE_TIMING + 1);
+				printram("edges: %d, %d, %d: 0x%x-0x%x-0x%x, 0x%x-0x%x\n",
+					 channel, slotrank, i, rn.start, rn.middle,
+					 rn.end, rn.start + ctrl->edge_offset[i],
+					 rn.end - ctrl->edge_offset[i]);
+				lower[lane] =
+					max(rn.start + ctrl->edge_offset[i], lower[lane]);
+				upper[lane] =
+					min(rn.end - ctrl->edge_offset[i], upper[lane]);
+				edges[lane] = (lower[lane] + upper[lane]) / 2;
+
+			}
+		}
+	}
+
+	write32(DEFAULT_MCHBAR + 0x3000, 0);
+	printram("CPA\n");
+}
+
+static void discover_edges_write(ramctr_timing * ctrl)
+{
+	int falling_edges[NUM_CHANNELS][NUM_SLOTRANKS][NUM_LANES];
+	int rising_edges[NUM_CHANNELS][NUM_SLOTRANKS][NUM_LANES];
+	int channel, slotrank, lane;
+
+	/* FIXME: under some conditions (older chipsets?) vendor BIOS sets both edges to the same value.  */
+	write32(DEFAULT_MCHBAR + 0x4eb0, 0x300);
+
+	FOR_ALL_CHANNELS FOR_ALL_POPULATED_RANKS {
+		discover_edges_write_real(ctrl, channel, slotrank,
+					  falling_edges[channel][slotrank]);
+	}
+
+	write32(DEFAULT_MCHBAR + 0x4eb0, 0x200);
+
+	FOR_ALL_CHANNELS FOR_ALL_POPULATED_RANKS {
+		discover_edges_write_real(ctrl, channel, slotrank,
+					  rising_edges[channel][slotrank]);
+	}
+
+	write32(DEFAULT_MCHBAR + 0x4eb0, 0);
+
+	FOR_ALL_CHANNELS FOR_ALL_POPULATED_RANKS FOR_ALL_LANES {
+		ctrl->timings[channel][slotrank].lanes[lane].falling =
+		    falling_edges[channel][slotrank][lane];
+		ctrl->timings[channel][slotrank].lanes[lane].rising =
+		    rising_edges[channel][slotrank][lane];
+	}
+
+	FOR_ALL_POPULATED_CHANNELS
+		program_timings(ctrl, channel);
+
+	FOR_ALL_CHANNELS FOR_ALL_POPULATED_RANKS FOR_ALL_LANES {
+		write32(DEFAULT_MCHBAR + 0x4080 + 0x400 * channel + 4 * lane,
+			0);
+	}
+}
+
+static void test_timC_write(ramctr_timing *ctrl, int channel, int slotrank)
+{
+	wait_428c(channel);
+	write32(DEFAULT_MCHBAR + 0x4220 + 0x400 * channel, 0x1f006);
+	write32(DEFAULT_MCHBAR + 0x4230 + 0x400 * channel,
+		(max((ctrl->tFAW >> 2) + 1, ctrl->tRRD)
+		 << 10) | (ctrl->tRCD << 16) | 4);
+	write32(DEFAULT_MCHBAR + 0x4200 + 0x400 * channel,
+		(slotrank << 24) | 0x60000);
+	write32(DEFAULT_MCHBAR + 0x4210 + 0x400 * channel, 0x244);
+
+	write32(DEFAULT_MCHBAR + 0x4224 + 0x400 * channel, 0x1f201);
+	write32(DEFAULT_MCHBAR + 0x4234 + 0x400 * channel,
+		0x80011e0 |
+		((ctrl->tWTR + ctrl->CWL + 8) << 16));
+	write32(DEFAULT_MCHBAR + 0x4204 +
+		0x400 * channel, (slotrank << 24));
+	write32(DEFAULT_MCHBAR + 0x4214 +
+		0x400 * channel, 0x242);
+
+	write32(DEFAULT_MCHBAR + 0x4228 +
+		0x400 * channel, 0x1f105);
+	write32(DEFAULT_MCHBAR + 0x4238 +
+		0x400 * channel,
+		0x40011e0 | (max(ctrl->tRTP, 8) << 16));
+	write32(DEFAULT_MCHBAR + 0x4208 +
+		0x400 * channel, (slotrank << 24));
+	write32(DEFAULT_MCHBAR + 0x4218 +
+		0x400 * channel, 0x242);
+
+	write32(DEFAULT_MCHBAR + 0x422c +
+		0x400 * channel, 0x1f002);
+	write32(DEFAULT_MCHBAR + 0x423c +
+		0x400 * channel,
+		0x1001 | (ctrl->tRP << 16));
+	write32(DEFAULT_MCHBAR + 0x420c +
+		0x400 * channel,
+		(slotrank << 24) | 0x60400);
+	write32(DEFAULT_MCHBAR + 0x421c +
+		0x400 * channel, 0);
+
+	write32(DEFAULT_MCHBAR + 0x4284 +
+		0x400 * channel, 0xc0001);
+	wait_428c(channel);
+}
+
+static void discover_timC_write(ramctr_timing * ctrl)
+{
+	const u8 rege3c_b24[3] = { 0, 0xf, 0x2f };
+	int i, pat;
+
+	int lower[NUM_CHANNELS][NUM_SLOTRANKS][NUM_LANES];
+	int upper[NUM_CHANNELS][NUM_SLOTRANKS][NUM_LANES];
+	int channel, slotrank, lane;
+
+	FOR_ALL_CHANNELS FOR_ALL_POPULATED_RANKS FOR_ALL_LANES {
+		lower[channel][slotrank][lane] = 0;
+		upper[channel][slotrank][lane] = MAX_TIMC;
+	}
+
+	write32(DEFAULT_MCHBAR + 0x4ea8, 1);
+
+	for (i = 0; i < 3; i++)
+		FOR_ALL_POPULATED_CHANNELS {
+			write32(DEFAULT_MCHBAR + 0xe3c + (channel * 0x100),
+				(rege3c_b24[i] << 24)
+				| (read32(DEFAULT_MCHBAR + 0xe3c + (channel * 0x100))
+				   & ~0x3f000000));
+			udelay(2);
+			for (pat = 0; pat < NUM_PATTERNS; pat++) {
+				FOR_ALL_POPULATED_RANKS {
+					int timC;
+					u32 raw_statistics[MAX_TIMC + 1];
+					int statistics[MAX_TIMC + 1];
+
+					fill_pattern5(ctrl, channel, pat);
+					write32(DEFAULT_MCHBAR + 0x4288 + 0x400 * channel, 0x1f);
+					for (timC = 0; timC < MAX_TIMC + 1; timC++) {
+						FOR_ALL_LANES
+							ctrl->timings[channel][slotrank].lanes[lane].timC = timC;
+						program_timings(ctrl, channel);
+
+						test_timC_write (ctrl, channel, slotrank);
+
+						raw_statistics[timC] =
+							MCHBAR32(0x436c + 0x400 * channel);
+					}
+					FOR_ALL_LANES {
+						struct run rn;
+						for (timC = 0; timC <= MAX_TIMC; timC++)
+							statistics[timC] =
+								!!(raw_statistics[timC] &
+								   (1 << lane));
+						rn = get_longest_zero_run(statistics,
+									  MAX_TIMC + 1);
+						if (rn.all)
+							die("timC write discovery failed");
+						printram("timC: %d, %d, %d: 0x%x-0x%x-0x%x, 0x%x-0x%x\n",
+							 channel, slotrank, i, rn.start,
+							 rn.middle, rn.end,
+							 rn.start + ctrl->timC_offset[i],
+							 rn.end - ctrl->timC_offset[i]);
+						lower[channel][slotrank][lane] =
+							max(rn.start + ctrl->timC_offset[i],
+							    lower[channel][slotrank][lane]);
+						upper[channel][slotrank][lane] =
+							min(rn.end - ctrl->timC_offset[i],
+							    upper[channel][slotrank][lane]);
+
+					}
+				}
+			}
+		}
+
+	FOR_ALL_CHANNELS {
+		write32(DEFAULT_MCHBAR + (channel * 0x100) + 0xe3c,
+			0 | (read32(DEFAULT_MCHBAR + (channel * 0x100) + 0xe3c) &
+			     ~0x3f000000));
+		udelay(2);
+	}
+
+	write32(DEFAULT_MCHBAR + 0x4ea8, 0);
+
+	printram("CPB\n");
+
+	FOR_ALL_CHANNELS FOR_ALL_POPULATED_RANKS FOR_ALL_LANES {
+		printram("timC [%d, %d, %d] = 0x%x\n", channel,
+		       slotrank, lane,
+		       (lower[channel][slotrank][lane] +
+			upper[channel][slotrank][lane]) / 2);
+		ctrl->timings[channel][slotrank].lanes[lane].timC =
+		    (lower[channel][slotrank][lane] +
+		     upper[channel][slotrank][lane]) / 2;
+	}
+	FOR_ALL_POPULATED_CHANNELS {
+		program_timings(ctrl, channel);
+	}
+}
+
+static void normalize_training(ramctr_timing * ctrl)
+{
+	int channel, slotrank, lane;
+	int mat = 0;
+
+	FOR_ALL_CHANNELS FOR_ALL_POPULATED_RANKS {
+		int delta;
+		FOR_ALL_LANES mat =
+		    max(ctrl->timings[channel][slotrank].lanes[lane].timA, mat);
+		 delta = (mat >> 6) - ctrl->timings[channel][slotrank].val_4028;
+		 ctrl->timings[channel][slotrank].val_4024 += delta;
+		 ctrl->timings[channel][slotrank].val_4028 += delta;
+	}
+
+	FOR_ALL_POPULATED_CHANNELS {
+		program_timings(ctrl, channel);
+	}
+}
+
+static void write_controller_mr(ramctr_timing * ctrl)
+{
+	int channel, slotrank;
+
+	FOR_ALL_CHANNELS FOR_ALL_POPULATED_RANKS {
+		write32(DEFAULT_MCHBAR + 0x0004 + (channel << 8) +
+			lane_registers[slotrank], make_mr0(ctrl, slotrank));
+		write32(DEFAULT_MCHBAR + 0x0008 + (channel << 8) +
+			lane_registers[slotrank], make_mr1(ctrl, slotrank));
+	}
+}
+
+static void channel_test(ramctr_timing * ctrl)
+{
+	int channel, slotrank, lane;
+
+	FOR_ALL_POPULATED_CHANNELS
+	    if (read32(DEFAULT_MCHBAR + 0x42a0 + (channel << 10)) & 0xa000)
+		 die("Mini channel test failed (1)\n");
+	FOR_ALL_POPULATED_CHANNELS {
+		fill_pattern0(ctrl, channel, 0x12345678, 0x98765432);
+
+		write32(DEFAULT_MCHBAR + 0x4288 + (channel << 10), 0);
+	}
+
+	for (slotrank = 0; slotrank < 4; slotrank++)
+		FOR_ALL_CHANNELS
+			if (ctrl->rankmap[channel] & (1 << slotrank)) {
+		FOR_ALL_LANES {
+			write32(DEFAULT_MCHBAR + (0x4f40 + 4 * lane), 0);
+			write32(DEFAULT_MCHBAR + (0x4d40 + 4 * lane), 0);
+		}
+		wait_428c(channel);
+		write32(DEFAULT_MCHBAR + 0x4220 + (channel << 10), 0x0001f006);
+		write32(DEFAULT_MCHBAR + 0x4230 + (channel << 10), 0x0028a004);
+		write32(DEFAULT_MCHBAR + 0x4200 + (channel << 10),
+			0x00060000 | (slotrank << 24));
+		write32(DEFAULT_MCHBAR + 0x4210 + (channel << 10), 0x00000244);
+		write32(DEFAULT_MCHBAR + 0x4224 + (channel << 10), 0x0001f201);
+		write32(DEFAULT_MCHBAR + 0x4234 + (channel << 10), 0x08281064);
+		write32(DEFAULT_MCHBAR + 0x4204 + (channel << 10),
+			0x00000000 | (slotrank << 24));
+		write32(DEFAULT_MCHBAR + 0x4214 + (channel << 10), 0x00000242);
+		write32(DEFAULT_MCHBAR + 0x4228 + (channel << 10), 0x0001f105);
+		write32(DEFAULT_MCHBAR + 0x4238 + (channel << 10), 0x04281064);
+		write32(DEFAULT_MCHBAR + 0x4208 + (channel << 10),
+			0x00000000 | (slotrank << 24));
+		write32(DEFAULT_MCHBAR + 0x4218 + (channel << 10), 0x00000242);
+		write32(DEFAULT_MCHBAR + 0x422c + (channel << 10), 0x0001f002);
+		write32(DEFAULT_MCHBAR + 0x423c + (channel << 10), 0x00280c01);
+		write32(DEFAULT_MCHBAR + 0x420c + (channel << 10),
+			0x00060400 | (slotrank << 24));
+		write32(DEFAULT_MCHBAR + 0x421c + (channel << 10), 0x00000240);
+		write32(DEFAULT_MCHBAR + 0x4284 + (channel << 10), 0x000c0001);
+		wait_428c(channel);
+		FOR_ALL_LANES
+		    if (read32(DEFAULT_MCHBAR + 0x4340 + (channel << 10) + 4 * lane))
+			 die("Mini channel test failed (2)\n");
+	}
+}
+
+static void set_scrambling_seed(ramctr_timing * ctrl)
+{
+	int channel;
+
+	/* FIXME: we hardcode seeds. Do we need to use some PRNG for them?
+	   I don't think so.  */
+	static u32 seeds[NUM_CHANNELS][3] = {
+		{0x00009a36, 0xbafcfdcf, 0x46d1ab68},
+		{0x00028bfa, 0x53fe4b49, 0x19ed5483}
+	};
+	FOR_ALL_POPULATED_CHANNELS {
+		MCHBAR32(0x4020 + 0x400 * channel) &= ~0x10000000;
+		write32(DEFAULT_MCHBAR + 0x4034, seeds[channel][0]);
+		write32(DEFAULT_MCHBAR + 0x403c, seeds[channel][1]);
+		write32(DEFAULT_MCHBAR + 0x4038, seeds[channel][2]);
+	}
+}
+
+static void set_4f8c(void)
+{
+	struct cpuid_result cpures;
+	u32 cpu;
+
+	cpures = cpuid(0);
+	cpu = (cpures.eax);
+	if (IS_SANDY_CPU(cpu) && (IS_SANDY_CPU_D0(cpu) || IS_SANDY_CPU_D1(cpu))) {
+		MCHBAR32(0x4f8c) = 0x141D1519;
+	} else {
+		MCHBAR32(0x4f8c) = 0x551D1519;
+	}
+}
+
+static void prepare_training(ramctr_timing * ctrl)
+{
+	int channel;
+
+	FOR_ALL_POPULATED_CHANNELS {
+		// Always drive command bus
+		MCHBAR32(0x4004 + 0x400 * channel) |= 0x20000000;
+	}
+
+	udelay(1);
+
+	FOR_ALL_POPULATED_CHANNELS {
+		wait_428c(channel);
+	}
+}
+
+static void set_4008c(ramctr_timing * ctrl)
+{
+	int channel, slotrank;
+	u32 reg;
+	FOR_ALL_POPULATED_CHANNELS {
+		u32 b20, b4_8_12;
+		int min_320c = 10000;
+		int max_320c = -10000;
+
+		FOR_ALL_POPULATED_RANKS {
+			max_320c = max(ctrl->timings[channel][slotrank].val_320c, max_320c);
+			min_320c = min(ctrl->timings[channel][slotrank].val_320c, min_320c);
+		}
+
+		if (max_320c - min_320c > 51)
+			b20 = 0;
+		else
+			b20 = ctrl->ref_card_offset[channel];
+
+		if (ctrl->reg_320c_range_threshold < max_320c - min_320c)
+			b4_8_12 = 0x3330;
+		else
+			b4_8_12 = 0x2220;
+
+		reg = read32(DEFAULT_MCHBAR + 0x400c + (channel << 10));
+		write32(DEFAULT_MCHBAR + 0x400c + (channel << 10),
+			(reg & 0xFFF0FFFF)
+			| (ctrl->ref_card_offset[channel] << 16)
+			| (ctrl->ref_card_offset[channel] << 18));
+		write32(DEFAULT_MCHBAR + 0x4008 + (channel << 10),
+			0x0a000000
+			| (b20 << 20)
+			| ((ctrl->ref_card_offset[channel] + 2) << 16)
+			| b4_8_12);
+	}
+}
+
+static void set_42a0(ramctr_timing * ctrl)
+{
+	int channel;
+	FOR_ALL_POPULATED_CHANNELS {
+		write32(DEFAULT_MCHBAR + (0x42a0 + 0x400 * channel),
+			0x00001000 | ctrl->rankmap[channel]);
+		MCHBAR32(0x4004 + 0x400 * channel) &= ~0x20000000;	// OK
+	}
+}
+
+static int encode_5d10(int ns)
+{
+  return (ns + 499) / 500;
+}
+
+/* FIXME: values in this function should be hardware revision-dependent.  */
+static void final_registers(ramctr_timing * ctrl)
+{
+	int channel;
+	int t1_cycles = 0, t1_ns = 0, t2_ns;
+	int t3_ns;
+	u32 r32;
+
+	write32(DEFAULT_MCHBAR + 0x4cd4, 0x00000046);
+
+	write32(DEFAULT_MCHBAR + 0x400c, (read32(DEFAULT_MCHBAR + 0x400c) & 0xFFFFCFFF) | 0x1000);	// OK
+	write32(DEFAULT_MCHBAR + 0x440c, (read32(DEFAULT_MCHBAR + 0x440c) & 0xFFFFCFFF) | 0x1000);	// OK
+	write32(DEFAULT_MCHBAR + 0x4cb0, 0x00000740);
+	write32(DEFAULT_MCHBAR + 0x4380, 0x00000aaa);	// OK
+	write32(DEFAULT_MCHBAR + 0x4780, 0x00000aaa);	// OK
+	write32(DEFAULT_MCHBAR + 0x4f88, 0x5f7003ff);	// OK
+	write32(DEFAULT_MCHBAR + 0x5064, 0x00073000 | ctrl->reg_5064b0); // OK
+
+	FOR_ALL_CHANNELS {
+		switch (ctrl->rankmap[channel]) {
+			/* Unpopulated channel.  */
+		case 0:
+			write32(DEFAULT_MCHBAR + 0x4384 + channel * 0x400, 0);
+			break;
+			/* Only single-ranked dimms.  */
+		case 1:
+		case 4:
+		case 5:
+			write32(DEFAULT_MCHBAR + 0x4384 + channel * 0x400, 0x373131);
+			break;
+			/* Dual-ranked dimms present.  */
+		default:
+			write32(DEFAULT_MCHBAR + 0x4384 + channel * 0x400, 0x9b6ea1);
+			break;
+		}
+	}
+
+	write32 (DEFAULT_MCHBAR + 0x5880, 0xca9171e5);
+	write32 (DEFAULT_MCHBAR + 0x5888,
+		 (read32 (DEFAULT_MCHBAR + 0x5888) & ~0xffffff) | 0xe4d5d0);
+	write32 (DEFAULT_MCHBAR + 0x58a8, read32 (DEFAULT_MCHBAR + 0x58a8) & ~0x1f);
+	write32 (DEFAULT_MCHBAR + 0x4294,
+		 (read32 (DEFAULT_MCHBAR + 0x4294) & ~0x30000)
+		 | (1 << 16));
+	write32 (DEFAULT_MCHBAR + 0x4694,
+		 (read32 (DEFAULT_MCHBAR + 0x4694) & ~0x30000)
+		 | (1 << 16));
+
+	MCHBAR32(0x5030) |= 1;	// OK
+	MCHBAR32(0x5030) |= 0x80;	// OK
+	MCHBAR32(0x5f18) = 0xfa;	// OK
+
+	/* Find a populated channel.  */
+	FOR_ALL_POPULATED_CHANNELS
+		break;
+
+	t1_cycles = ((read32(DEFAULT_MCHBAR + 0x4290 + channel * 0x400) >> 8) & 0xff);
+	r32 = read32(DEFAULT_MCHBAR + 0x5064);
+	if (r32 & 0x20000)
+		t1_cycles += (r32 & 0xfff);
+	t1_cycles += (read32(DEFAULT_MCHBAR + channel * 0x400 + 0x42a4) & 0xfff);
+	t1_ns = t1_cycles * ctrl->tCK / 256 + 544;
+	if (!(r32 & 0x20000))
+		t1_ns += 500;
+
+	t2_ns = 10 * ((read32(DEFAULT_MCHBAR + 0x5f10) >> 8) & 0xfff);
+	if ( read32(DEFAULT_MCHBAR + 0x5f00) & 8 )
+	{
+		t3_ns = 10 * ((read32(DEFAULT_MCHBAR + 0x5f20) >> 8) & 0xfff);
+		t3_ns += 10 * (read32(DEFAULT_MCHBAR + 0x5f18) & 0xff);
+	}
 	else
-		intel_early_me_status();
+	{
+		t3_ns = 500;
+	}
+	printk(BIOS_DEBUG, "t123: %d, %d, %d\n",
+	       t1_ns, t2_ns, t3_ns);
+	write32 (DEFAULT_MCHBAR + 0x5d10,
+		 ((encode_5d10(t1_ns) + encode_5d10(t2_ns)) << 16)
+		 | (encode_5d10(t1_ns) << 8)
+		 | ((encode_5d10(t3_ns) + encode_5d10(t2_ns) + encode_5d10(t1_ns)) << 24)
+		 | (read32(DEFAULT_MCHBAR + 0x5d10) & 0xC0C0C0C0)
+		 | 0xc);
+}
+
+static void save_timings(ramctr_timing * ctrl)
+{
+	struct mrc_data_container *mrcdata;
+	int output_len = ALIGN(sizeof (*ctrl), 16);
+
+	/* Save the MRC S3 restore data to cbmem */
+	mrcdata = cbmem_add
+		(CBMEM_ID_MRCDATA,
+		 output_len + sizeof(struct mrc_data_container));
+
+	printk(BIOS_DEBUG, "Relocate MRC DATA from %p to %p (%u bytes)\n",
+	       ctrl, mrcdata, output_len);
+
+	mrcdata->mrc_signature = MRC_DATA_SIGNATURE;
+	mrcdata->mrc_data_size = output_len;
+	mrcdata->reserved = 0;
+	memcpy(mrcdata->mrc_data, ctrl, sizeof (*ctrl));
+
+	/* Zero the unused space in aligned buffer. */
+	if (output_len > sizeof (*ctrl))
+		memset(mrcdata->mrc_data+sizeof (*ctrl), 0,
+		       output_len - sizeof (*ctrl));
+
+	mrcdata->mrc_checksum = compute_ip_checksum(mrcdata->mrc_data,
+						    mrcdata->mrc_data_size);
+}
+
+static void restore_timings(ramctr_timing * ctrl)
+{
+	int channel, slotrank, lane;
+
+	FOR_ALL_POPULATED_CHANNELS
+	    MCHBAR32(0x4004 + 0x400 * channel) =
+		ctrl->tRRD
+		| (ctrl->tRTP << 4)
+		| (ctrl->tCKE << 8)
+		| (ctrl->tWTR << 12)
+		| (ctrl->tFAW << 16)
+		| (ctrl->tWR << 24)
+		| (ctrl->cmd_stretch[channel] << 30);
+
+	udelay(1);
+
+	FOR_ALL_POPULATED_CHANNELS {
+		wait_428c(channel);
+	}
+
+	FOR_ALL_CHANNELS FOR_ALL_POPULATED_RANKS FOR_ALL_LANES {
+		write32(DEFAULT_MCHBAR + 0x4080 + 0x400 * channel
+			+ 4 * lane, 0);
+	}
+
+	FOR_ALL_POPULATED_CHANNELS
+	    write32(DEFAULT_MCHBAR + 0x4008 + 0x400 * channel,
+		    read32(DEFAULT_MCHBAR + 0x4008 +
+			   0x400 * channel) | 0x8000000);
+
+	FOR_ALL_POPULATED_CHANNELS {
+		udelay (1);
+		write32(DEFAULT_MCHBAR + 0x4020 + 0x400 * channel,
+			read32(DEFAULT_MCHBAR + 0x4020 +
+			       0x400 * channel) | 0x200000);
+	}
+
+	printram("CPE\n");
+
+	write32(DEFAULT_MCHBAR + 0x3400, 0);
+	write32(DEFAULT_MCHBAR + 0x4eb0, 0);
+
+	printram("CP5b\n");
+
+	FOR_ALL_POPULATED_CHANNELS {
+		program_timings(ctrl, channel);
+	}
+
+	u32 reg, addr;
+
+	while (!(MCHBAR32(0x5084) & 0x10000)) ;
+	do {
+		reg = MCHBAR32(0x428c);
+	} while ((reg & 0x14) == 0);
+
+	// Set state of memory controller
+	MCHBAR32(0x5030) = 0x116;
+	MCHBAR32(0x4ea0) = 0;
+
+	// Wait 500us
+	udelay(500);
+
+	FOR_ALL_CHANNELS {
+		// Set valid rank CKE
+		reg = 0;
+		reg = (reg & ~0xf) | ctrl->rankmap[channel];
+		addr = 0x400 * channel + 0x42a0;
+		MCHBAR32(addr) = reg;
+
+		// Wait 10ns for ranks to settle
+		//udelay(0.01);
+
+		reg = (reg & ~0xf0) | (ctrl->rankmap[channel] << 4);
+		MCHBAR32(addr) = reg;
+
+		// Write reset using a NOP
+		write_reset(ctrl);
+	}
+
+	/* mrs commands. */
+	dram_mrscommands(ctrl);
+
+	printram("CP5c\n");
+
+	write32(DEFAULT_MCHBAR + 0x3000, 0);
+
+	FOR_ALL_CHANNELS {
+		write32(DEFAULT_MCHBAR + (channel * 0x100) + 0xe3c,
+			0 | (read32(DEFAULT_MCHBAR + (channel * 0x100) + 0xe3c) &
+			     ~0x3f000000));
+		udelay(2);
+	}
+
+	write32(DEFAULT_MCHBAR + 0x4ea8, 0);
+}
+
+void init_dram_ddr3(spd_raw_data * spds, int mobile, int min_tck,
+	int s3resume)
+{
+	int me_uma_size;
+	int cbmem_was_inited;
+
+	MCHBAR32(0x5f00) |= 1;
+
+	report_platform_info();
+
+	/* Wait for ME to be ready */
+	intel_early_me_init();
+	me_uma_size = intel_early_me_uma_size();
+
+	printk(BIOS_DEBUG, "Starting native Platform init\n");
+
+	u32 reg_5d10;
+
+	wait_txt_clear();
+
+	wrmsr(0x000002e6, (msr_t) { .lo = 0, .hi = 0 });
 
-	post_system_agent_init(pei_data);
+	reg_5d10 = read32(DEFAULT_MCHBAR + 0x5d10);	// !!! = 0x00000000
+	if ((pcie_read_config16(SOUTHBRIDGE, 0xa2) & 0xa0) == 0x20	/* 0x0004 */
+	    && reg_5d10 && !s3resume) {
+		write32(DEFAULT_MCHBAR + 0x5d10, 0);
+		/* Need reset.  */
+		outb(0x6, 0xcf9);
+
+		halt();
+	}
+
+	ramctr_timing ctrl;
+
+	memset(&ctrl, 0, sizeof (ctrl));
+
+	early_pch_init_native();
+	early_thermal_init();
+
+	ctrl.mobile = mobile;
+	ctrl.tCK = min_tck;
+
+	/* FIXME: for non-S3 we should be able to use timing caching with
+	   proper verification. Right now we use timings only for S3 case.
+	 */
+	if (s3resume) {
+		struct mrc_data_container *mrc_cache;
+
+		mrc_cache = find_current_mrc_cache();
+		if (!mrc_cache || mrc_cache->mrc_data_size < sizeof (ctrl)) {
+			/* Failed S3 resume, reset to come up cleanly */
+			outb(0x6, 0xcf9);
+			halt();
+		}
+		memcpy(&ctrl, mrc_cache->mrc_data, sizeof (ctrl));
+	}
+
+	if (!s3resume) {
+		dimm_info info;
+
+		/* Get DDR3 SPD data */
+		dram_find_spds_ddr3(spds, &info, &ctrl);
+
+		/* Find fastest common supported parameters */
+		dram_find_common_params(&info, &ctrl);
+
+		dram_dimm_mapping(&info, &ctrl);
+	}
+
+	/* Set MCU frequency */
+	dram_freq(&ctrl);
+
+	if (!s3resume) {
+		/* Calculate timings */
+		dram_timing(&ctrl);
+	}
+
+	/* Set version register */
+	MCHBAR32(0x5034) = 0xC04EB002;
+
+	/* Enable crossover */
+	dram_xover(&ctrl);
+
+	/* Set timing and refresh registers */
+	dram_timing_regs(&ctrl);
+
+	/* Power mode preset */
+	MCHBAR32(0x4e80) = 0x5500;
+
+	/* Set scheduler parameters */
+	MCHBAR32(0x4c20) = 0x10100005;
+
+	/* Set cpu specific register */
+	set_4f8c();
+
+	/* Clear IO reset bit */
+	MCHBAR32(0x5030) &= ~0x20;
+
+	/* Set MAD-DIMM registers */
+	dram_dimm_set_mapping(&ctrl);
+	printk(BIOS_DEBUG, "Done dimm mapping\n");
+
+	/* Zone config */
+	dram_zones(&ctrl, 1);
+
+	/* Set memory map */
+	dram_memorymap(&ctrl, me_uma_size);
+	printk(BIOS_DEBUG, "Done memory map\n");
+
+	/* Set IO registers */
+	dram_ioregs(&ctrl);
+	printk(BIOS_DEBUG, "Done io registers\n");
+
+	udelay(1);
+
+	if (s3resume) {
+		restore_timings(&ctrl);
+	} else {
+		/* Do jedec ddr3 reset sequence */
+		dram_jedecreset(&ctrl);
+		printk(BIOS_DEBUG, "Done jedec reset\n");
+
+		/* MRS commands */
+		dram_mrscommands(&ctrl);
+		printk(BIOS_DEBUG, "Done MRS commands\n");
+		dram_mrscommands(&ctrl);
+
+		/* Prepare for memory training */
+		prepare_training(&ctrl);
+
+		read_training(&ctrl);
+		write_training(&ctrl);
+
+		printram("CP5a\n");
+
+		discover_edges(&ctrl);
+
+		printram("CP5b\n");
+
+		command_training(&ctrl);
+
+		printram("CP5c\n");
+
+		discover_edges_write(&ctrl);
+
+		discover_timC_write(&ctrl);
+
+		normalize_training(&ctrl);
+	}
+
+	set_4008c(&ctrl);
+
+	write_controller_mr(&ctrl);
+
+	if (!s3resume) {
+		channel_test(&ctrl);
+	}
+
+	/* FIXME: should be hardware revision-dependent.  */
+	write32(DEFAULT_MCHBAR + 0x5024, 0x00a030ce);
+
+	set_scrambling_seed(&ctrl);
+
+	set_42a0(&ctrl);
+
+	final_registers(&ctrl);
+
+	/* Zone config */
+	dram_zones(&ctrl, 0);
+
+	if (!s3resume)
+		quick_ram_check();
+
+	intel_early_me_status();
+	intel_early_me_init_done(ME_INIT_STATUS_SUCCESS);
+	intel_early_me_status();
+
+	post_system_agent_init();
 	report_memory_config();
+
+	cbmem_was_inited = !cbmem_recovery(s3resume);
+	if (!s3resume)
+		save_timings(&ctrl);
+	if (s3resume && !cbmem_was_inited) {
+		/* Failed S3 resume, reset to come up cleanly */
+		outb(0x6, 0xcf9);
+		halt();
+	}
 }
diff --git a/src/northbridge/intel/sandybridge/raminit_mrc.c b/src/northbridge/intel/sandybridge/raminit_mrc.c
new file mode 100644
index 0000000000..053a4873cd
--- /dev/null
+++ b/src/northbridge/intel/sandybridge/raminit_mrc.c
@@ -0,0 +1,293 @@
+/*
+ * This file is part of the coreboot project.
+ *
+ * Copyright (C) 2011 Google Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; version 2 of the License.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc.
+ */
+
+#include <console/console.h>
+#include <console/usb.h>
+#include <bootmode.h>
+#include <string.h>
+#include <arch/io.h>
+#include <cbmem.h>
+#include <arch/cbfs.h>
+#include <cbfs.h>
+#include <ip_checksum.h>
+#include <pc80/mc146818rtc.h>
+#include <device/pci_def.h>
+#include <halt.h>
+#include "raminit.h"
+#include "pei_data.h"
+#include "sandybridge.h"
+
+/* Management Engine is in the southbridge */
+#include "southbridge/intel/bd82x6x/me.h"
+
+/*
+ * MRC scrambler seed offsets should be reserved in
+ * mainboard cmos.layout and not covered by checksum.
+ */
+#if CONFIG_USE_OPTION_TABLE
+#include "option_table.h"
+#define CMOS_OFFSET_MRC_SEED     (CMOS_VSTART_mrc_scrambler_seed >> 3)
+#define CMOS_OFFSET_MRC_SEED_S3  (CMOS_VSTART_mrc_scrambler_seed_s3 >> 3)
+#define CMOS_OFFSET_MRC_SEED_CHK (CMOS_VSTART_mrc_scrambler_seed_chk >> 3)
+#else
+#define CMOS_OFFSET_MRC_SEED     152
+#define CMOS_OFFSET_MRC_SEED_S3  156
+#define CMOS_OFFSET_MRC_SEED_CHK 160
+#endif
+
+void save_mrc_data(struct pei_data *pei_data)
+{
+	u16 c1, c2, checksum;
+	struct mrc_data_container *mrcdata;
+	int output_len = ALIGN(pei_data->mrc_output_len, 16);
+
+	/* Save the MRC S3 restore data to cbmem */
+	mrcdata = cbmem_add
+		(CBMEM_ID_MRCDATA,
+		 output_len + sizeof(struct mrc_data_container));
+
+	if (mrcdata != NULL) {
+		printk(BIOS_DEBUG, "Relocate MRC DATA from %p to %p (%u bytes)\n",
+			pei_data->mrc_output, mrcdata, output_len);
+
+		mrcdata->mrc_signature = MRC_DATA_SIGNATURE;
+		mrcdata->mrc_data_size = output_len;
+		mrcdata->reserved = 0;
+		memcpy(mrcdata->mrc_data, pei_data->mrc_output,
+			pei_data->mrc_output_len);
+
+		/* Zero the unused space in aligned buffer. */
+		if (output_len > pei_data->mrc_output_len)
+			memset(mrcdata->mrc_data+pei_data->mrc_output_len, 0,
+			output_len - pei_data->mrc_output_len);
+
+		mrcdata->mrc_checksum = compute_ip_checksum(mrcdata->mrc_data,
+						mrcdata->mrc_data_size);
+	}
+
+	/* Save the MRC seed values to CMOS */
+	cmos_write32(CMOS_OFFSET_MRC_SEED, pei_data->scrambler_seed);
+	printk(BIOS_DEBUG, "Save scrambler seed    0x%08x to CMOS 0x%02x\n",
+	       pei_data->scrambler_seed, CMOS_OFFSET_MRC_SEED);
+
+	cmos_write32(CMOS_OFFSET_MRC_SEED_S3, pei_data->scrambler_seed_s3);
+	printk(BIOS_DEBUG, "Save s3 scrambler seed 0x%08x to CMOS 0x%02x\n",
+	       pei_data->scrambler_seed_s3, CMOS_OFFSET_MRC_SEED_S3);
+
+	/* Save a simple checksum of the seed values */
+	c1 = compute_ip_checksum((u8*)&pei_data->scrambler_seed,
+				 sizeof(u32));
+	c2 = compute_ip_checksum((u8*)&pei_data->scrambler_seed_s3,
+				 sizeof(u32));
+	checksum = add_ip_checksums(sizeof(u32), c1, c2);
+
+	cmos_write(checksum & 0xff, CMOS_OFFSET_MRC_SEED_CHK);
+	cmos_write((checksum >> 8) & 0xff, CMOS_OFFSET_MRC_SEED_CHK+1);
+}
+
+static void prepare_mrc_cache(struct pei_data *pei_data)
+{
+	struct mrc_data_container *mrc_cache;
+	u16 c1, c2, checksum, seed_checksum;
+
+	// preset just in case there is an error
+	pei_data->mrc_input = NULL;
+	pei_data->mrc_input_len = 0;
+
+	/* Read scrambler seeds from CMOS */
+	pei_data->scrambler_seed = cmos_read32(CMOS_OFFSET_MRC_SEED);
+	printk(BIOS_DEBUG, "Read scrambler seed    0x%08x from CMOS 0x%02x\n",
+	       pei_data->scrambler_seed, CMOS_OFFSET_MRC_SEED);
+
+	pei_data->scrambler_seed_s3 = cmos_read32(CMOS_OFFSET_MRC_SEED_S3);
+	printk(BIOS_DEBUG, "Read S3 scrambler seed 0x%08x from CMOS 0x%02x\n",
+	       pei_data->scrambler_seed_s3, CMOS_OFFSET_MRC_SEED_S3);
+
+	/* Compute seed checksum and compare */
+	c1 = compute_ip_checksum((u8*)&pei_data->scrambler_seed,
+				 sizeof(u32));
+	c2 = compute_ip_checksum((u8*)&pei_data->scrambler_seed_s3,
+				 sizeof(u32));
+	checksum = add_ip_checksums(sizeof(u32), c1, c2);
+
+	seed_checksum = cmos_read(CMOS_OFFSET_MRC_SEED_CHK);
+	seed_checksum |= cmos_read(CMOS_OFFSET_MRC_SEED_CHK+1) << 8;
+
+	if (checksum != seed_checksum) {
+		printk(BIOS_ERR, "%s: invalid seed checksum\n", __func__);
+		pei_data->scrambler_seed = 0;
+		pei_data->scrambler_seed_s3 = 0;
+		return;
+	}
+
+	if ((mrc_cache = find_current_mrc_cache()) == NULL) {
+		/* error message printed in find_current_mrc_cache */
+		return;
+	}
+
+	pei_data->mrc_input = mrc_cache->mrc_data;
+	pei_data->mrc_input_len = mrc_cache->mrc_data_size;
+
+	printk(BIOS_DEBUG, "%s: at %p, size %x checksum %04x\n",
+	       __func__, pei_data->mrc_input,
+	       pei_data->mrc_input_len, mrc_cache->mrc_checksum);
+}
+
+static const char* ecc_decoder[] = {
+	"inactive",
+	"active on IO",
+	"disabled on IO",
+	"active"
+};
+
+/*
+ * Dump in the log memory controller configuration as read from the memory
+ * controller registers.
+ */
+static void report_memory_config(void)
+{
+	u32 addr_decoder_common, addr_decode_ch[2];
+	int i;
+
+	addr_decoder_common = MCHBAR32(0x5000);
+	addr_decode_ch[0] = MCHBAR32(0x5004);
+	addr_decode_ch[1] = MCHBAR32(0x5008);
+
+	printk(BIOS_DEBUG, "memcfg DDR3 clock %d MHz\n",
+	       (MCHBAR32(0x5e04) * 13333 * 2 + 50)/100);
+	printk(BIOS_DEBUG, "memcfg channel assignment: A: %d, B % d, C % d\n",
+	       addr_decoder_common & 3,
+	       (addr_decoder_common >> 2) & 3,
+	       (addr_decoder_common >> 4) & 3);
+
+	for (i = 0; i < ARRAY_SIZE(addr_decode_ch); i++) {
+		u32 ch_conf = addr_decode_ch[i];
+		printk(BIOS_DEBUG, "memcfg channel[%d] config (%8.8x):\n",
+		       i, ch_conf);
+		printk(BIOS_DEBUG, "   ECC %s\n",
+		       ecc_decoder[(ch_conf >> 24) & 3]);
+		printk(BIOS_DEBUG, "   enhanced interleave mode %s\n",
+		       ((ch_conf >> 22) & 1) ? "on" : "off");
+		printk(BIOS_DEBUG, "   rank interleave %s\n",
+		       ((ch_conf >> 21) & 1) ? "on" : "off");
+		printk(BIOS_DEBUG, "   DIMMA %d MB width x%d %s rank%s\n",
+		       ((ch_conf >> 0) & 0xff) * 256,
+		       ((ch_conf >> 19) & 1) ? 16 : 8,
+		       ((ch_conf >> 17) & 1) ? "dual" : "single",
+		       ((ch_conf >> 16) & 1) ? "" : ", selected");
+		printk(BIOS_DEBUG, "   DIMMB %d MB width x%d %s rank%s\n",
+		       ((ch_conf >> 8) & 0xff) * 256,
+		       ((ch_conf >> 20) & 1) ? 16 : 8,
+		       ((ch_conf >> 18) & 1) ? "dual" : "single",
+		       ((ch_conf >> 16) & 1) ? ", selected" : "");
+	}
+}
+
+static void post_system_agent_init(struct pei_data *pei_data)
+{
+	/* If PCIe init is skipped, set the PEG clock gating */
+	if (!pei_data->pcie_init)
+		MCHBAR32(0x7010) = MCHBAR32(0x7010) | 0x01;
+}
+
+/**
+ * Find PEI executable in coreboot filesystem and execute it.
+ *
+ * @param pei_data: configuration data for UEFI PEI reference code
+ */
+void sdram_initialize(struct pei_data *pei_data)
+{
+	struct sys_info sysinfo;
+	int (*entry) (struct pei_data *pei_data) __attribute__ ((regparm(1)));
+
+	report_platform_info();
+
+	/* Wait for ME to be ready */
+	intel_early_me_init();
+	intel_early_me_uma_size();
+
+	printk(BIOS_DEBUG, "Starting UEFI PEI System Agent\n");
+
+	memset(&sysinfo, 0, sizeof(sysinfo));
+
+	sysinfo.boot_path = pei_data->boot_mode;
+
+	/*
+	 * Do not pass MRC data in for recovery mode boot,
+	 * Always pass it in for S3 resume.
+	 */
+	if (!recovery_mode_enabled() || pei_data->boot_mode == 2)
+		prepare_mrc_cache(pei_data);
+
+	/* If MRC data is not found we cannot continue S3 resume. */
+	if (pei_data->boot_mode == 2 && !pei_data->mrc_input) {
+		printk(BIOS_DEBUG, "Giving up in sdram_initialize: No MRC data\n");
+		outb(0x6, 0xcf9);
+		halt();
+	}
+
+	/* Pass console handler in pei_data */
+	pei_data->tx_byte = do_putchar;
+
+	/* Locate and call UEFI System Agent binary. */
+	entry = cbfs_boot_map_with_leak("mrc.bin", CBFS_TYPE_MRC, NULL);
+	if (entry) {
+		int rv;
+		rv = entry (pei_data);
+		if (rv) {
+			switch (rv) {
+			case -1:
+				printk(BIOS_ERR, "PEI version mismatch.\n");
+				break;
+			case -2:
+				printk(BIOS_ERR, "Invalid memory frequency.\n");
+				break;
+			default:
+				printk(BIOS_ERR, "MRC returned %x.\n", rv);
+			}
+			die("Nonzero MRC return value.\n");
+		}
+	} else {
+		die("UEFI PEI System Agent not found.\n");
+	}
+
+#if CONFIG_USBDEBUG_IN_ROMSTAGE
+	/* mrc.bin reconfigures USB, so reinit it to have debug */
+	usbdebug_init();
+#endif
+
+	/* For reference print the System Agent version
+	 * after executing the UEFI PEI stage.
+	 */
+	u32 version = MCHBAR32(0x5034);
+	printk(BIOS_DEBUG, "System Agent Version %d.%d.%d Build %d\n",
+		version >> 24 , (version >> 16) & 0xff,
+		(version >> 8) & 0xff, version & 0xff);
+
+	/* Send ME init done for SandyBridge here.  This is done
+	 * inside the SystemAgent binary on IvyBridge. */
+	if (BASE_REV_SNB ==
+	    (pci_read_config16(PCI_CPU_DEVICE, PCI_DEVICE_ID) & BASE_REV_MASK))
+		intel_early_me_init_done(ME_INIT_STATUS_SUCCESS);
+	else
+		intel_early_me_status();
+
+	post_system_agent_init(pei_data);
+	report_memory_config();
+}
diff --git a/src/northbridge/intel/sandybridge/raminit_native.c b/src/northbridge/intel/sandybridge/raminit_native.c
deleted file mode 100644
index 790f47c46c..0000000000
--- a/src/northbridge/intel/sandybridge/raminit_native.c
+++ /dev/null
@@ -1,3902 +0,0 @@
-/*
- * This file is part of the coreboot project.
- *
- * Copyright (C) 2014 Damien Zammit <damien@zamaudio.com>
- * Copyright (C) 2014 Vladimir Serbinenko <phcoder@gmail.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; version 2 of the License.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc.
- */
-
-#include <console/console.h>
-#include <console/usb.h>
-#include <bootmode.h>
-#include <string.h>
-#include <arch/io.h>
-#include <cbmem.h>
-#include <arch/cbfs.h>
-#include <cbfs.h>
-#include <halt.h>
-#include <ip_checksum.h>
-#include <pc80/mc146818rtc.h>
-#include <device/pci_def.h>
-#include "raminit_native.h"
-#include "sandybridge.h"
-#include <delay.h>
-#include <lib.h>
-
-/* Management Engine is in the southbridge */
-#include "southbridge/intel/bd82x6x/me.h"
-/* For SPD.  */
-#include "southbridge/intel/bd82x6x/smbus.h"
-#include "arch/cpu.h"
-#include "cpu/x86/msr.h"
-
-/* FIXME: no ECC support.  */
-/* FIXME: no support for 3-channel chipsets.  */
-
-#define BASEFREQ 133
-#define tDLLK 512
-
-#define IS_SANDY_CPU(x) ((x & 0xffff0) == 0x206a0)
-#define IS_SANDY_CPU_C(x) ((x & 0xf) == 4)
-#define IS_SANDY_CPU_D0(x) ((x & 0xf) == 5)
-#define IS_SANDY_CPU_D1(x) ((x & 0xf) == 6)
-#define IS_SANDY_CPU_D2(x) ((x & 0xf) == 7)
-
-#define IS_IVY_CPU(x) ((x & 0xffff0) == 0x306a0)
-#define IS_IVY_CPU_C(x) ((x & 0xf) == 4)
-#define IS_IVY_CPU_K(x) ((x & 0xf) == 5)
-#define IS_IVY_CPU_D(x) ((x & 0xf) == 6)
-#define IS_IVY_CPU_E(x) ((x & 0xf) >= 8)
-
-#define NUM_CHANNELS 2
-#define NUM_SLOTRANKS 4
-#define NUM_SLOTS 2
-#define NUM_LANES 8
-
-/* FIXME: Vendor BIOS uses 64 but our algorithms are less
-   performant and even 1 seems to be enough in practice.  */
-#define NUM_PATTERNS 4
-
-typedef struct odtmap_st {
-	u16 rttwr;
-	u16 rttnom;
-} odtmap;
-
-typedef struct dimm_info_st {
-	dimm_attr dimm[NUM_CHANNELS][NUM_SLOTS];
-} dimm_info;
-
-struct ram_rank_timings {
-	/* Register 4024. One byte per slotrank.  */
-	u8 val_4024;
-	/* Register 4028. One nibble per slotrank.  */
-	u8 val_4028;
-
-	int val_320c;
-
-	struct ram_lane_timings {
-		/* lane register offset 0x10.  */
-		u16 timA;	/* bits 0 - 5, bits 16 - 18 */
-		u8 rising;	/* bits 8 - 14 */
-		u8 falling;	/* bits 20 - 26.  */
-
-		/* lane register offset 0x20.  */
-		int timC;	/* bit 0 - 5, 19.  */
-		u16 timB;	/* bits 8 - 13, 15 - 17.  */
-	} lanes[NUM_LANES];
-};
-
-struct ramctr_timing_st;
-
-typedef struct ramctr_timing_st {
-	int mobile;
-
-	u16 cas_supported;
-	/* tLatencies are in units of ns, scaled by x256 */
-	u32 tCK;
-	u32 tAA;
-	u32 tWR;
-	u32 tRCD;
-	u32 tRRD;
-	u32 tRP;
-	u32 tRAS;
-	u32 tRFC;
-	u32 tWTR;
-	u32 tRTP;
-	u32 tFAW;
-	/* Latencies in terms of clock cycles
-	 * They are saved separately as they are needed for DRAM MRS commands*/
-	u8 CAS;			/* CAS read latency */
-	u8 CWL;			/* CAS write latency */
-
-	u32 tREFI;
-	u32 tMOD;
-	u32 tXSOffset;
-	u32 tWLO;
-	u32 tCKE;
-	u32 tXPDLL;
-	u32 tXP;
-	u32 tAONPD;
-
-	u16 reg_5064b0; /* bits 0-11. */
-
-	u8 rankmap[NUM_CHANNELS];
-	int ref_card_offset[NUM_CHANNELS];
-	u32 mad_dimm[NUM_CHANNELS];
-	int channel_size_mb[NUM_CHANNELS];
-	u32 cmd_stretch[NUM_CHANNELS];
-
-	int reg_c14_offset;
-	int reg_320c_range_threshold;
-
-	int edge_offset[3];
-	int timC_offset[3];
-
-	int extended_temperature_range;
-	int auto_self_refresh;
-
-	int rank_mirror[NUM_CHANNELS][NUM_SLOTRANKS];
-
-	struct ram_rank_timings timings[NUM_CHANNELS][NUM_SLOTRANKS];
-} ramctr_timing;
-
-#define SOUTHBRIDGE PCI_DEV(0, 0x1f, 0)
-#define NORTHBRIDGE PCI_DEV(0, 0x0, 0)
-#define FOR_ALL_LANES for (lane = 0; lane < NUM_LANES; lane++)
-#define FOR_ALL_CHANNELS for (channel = 0; channel < NUM_CHANNELS; channel++)
-#define FOR_ALL_POPULATED_RANKS for (slotrank = 0; slotrank < NUM_SLOTRANKS; slotrank++) if (ctrl->rankmap[channel] & (1 << slotrank))
-#define FOR_ALL_POPULATED_CHANNELS for (channel = 0; channel < NUM_CHANNELS; channel++) if (ctrl->rankmap[channel])
-#define MAX_EDGE_TIMING 71
-#define MAX_TIMC 127
-#define MAX_TIMB 511
-#define MAX_TIMA 127
-
-static void program_timings(ramctr_timing * ctrl, int channel);
-
-static const char *ecc_decoder[] = {
-	"inactive",
-	"active on IO",
-	"disabled on IO",
-	"active"
-};
-
-static void wait_txt_clear(void)
-{
-	struct cpuid_result cp;
-
-	cp = cpuid_ext(0x1, 0x0);
-	/* Check if TXT is supported?  */
-	if (!(cp.ecx & 0x40))
-		return;
-	/* Some TXT public bit.  */
-	if (!(read32((void *)0xfed30010) & 1))
-		return;
-	/* Wait for TXT clear.  */
-	while (!(read8((void *)0xfed40000) & (1 << 7))) ;
-}
-
-static void sfence(void)
-{
-	asm volatile ("sfence");
-}
-
-/*
- * Dump in the log memory controller configuration as read from the memory
- * controller registers.
- */
-static void report_memory_config(void)
-{
-	u32 addr_decoder_common, addr_decode_ch[NUM_CHANNELS];
-	int i;
-
-	addr_decoder_common = MCHBAR32(0x5000);
-	addr_decode_ch[0] = MCHBAR32(0x5004);
-	addr_decode_ch[1] = MCHBAR32(0x5008);
-
-	printk(BIOS_DEBUG, "memcfg DDR3 clock %d MHz\n",
-	       (MCHBAR32(0x5e04) * 13333 * 2 + 50) / 100);
-	printk(BIOS_DEBUG, "memcfg channel assignment: A: %d, B % d, C % d\n",
-	       addr_decoder_common & 3, (addr_decoder_common >> 2) & 3,
-	       (addr_decoder_common >> 4) & 3);
-
-	for (i = 0; i < ARRAY_SIZE(addr_decode_ch); i++) {
-		u32 ch_conf = addr_decode_ch[i];
-		printk(BIOS_DEBUG, "memcfg channel[%d] config (%8.8x):\n", i,
-		       ch_conf);
-		printk(BIOS_DEBUG, "   ECC %s\n",
-		       ecc_decoder[(ch_conf >> 24) & 3]);
-		printk(BIOS_DEBUG, "   enhanced interleave mode %s\n",
-		       ((ch_conf >> 22) & 1) ? "on" : "off");
-		printk(BIOS_DEBUG, "   rank interleave %s\n",
-		       ((ch_conf >> 21) & 1) ? "on" : "off");
-		printk(BIOS_DEBUG, "   DIMMA %d MB width x%d %s rank%s\n",
-		       ((ch_conf >> 0) & 0xff) * 256,
-		       ((ch_conf >> 19) & 1) ? 16 : 8,
-		       ((ch_conf >> 17) & 1) ? "dual" : "single",
-		       ((ch_conf >> 16) & 1) ? "" : ", selected");
-		printk(BIOS_DEBUG, "   DIMMB %d MB width x%d %s rank%s\n",
-		       ((ch_conf >> 8) & 0xff) * 256,
-		       ((ch_conf >> 20) & 1) ? 16 : 8,
-		       ((ch_conf >> 18) & 1) ? "dual" : "single",
-		       ((ch_conf >> 16) & 1) ? ", selected" : "");
-	}
-}
-
-static void post_system_agent_init(void)
-{
-	/* If PCIe init is skipped, set the PEG clock gating */
-	MCHBAR32(0x7010) = MCHBAR32(0x7010) | 0x01;
-}
-
-void read_spd(spd_raw_data * spd, u8 addr)
-{
-	int j;
-	for (j = 0; j < 256; j++)
-		(*spd)[j] = do_smbus_read_byte(SMBUS_IO_BASE, addr, j);
-}
-
-static void dram_find_spds_ddr3(spd_raw_data * spd, dimm_info * dimm,
-				ramctr_timing * ctrl)
-{
-	int dimms = 0;
-	int channel, slot, spd_slot;
-
-	memset (ctrl->rankmap, 0, sizeof (ctrl->rankmap));
-
-	ctrl->extended_temperature_range = 1;
-	ctrl->auto_self_refresh = 1;
-
-	FOR_ALL_CHANNELS {
-		ctrl->channel_size_mb[channel] = 0;
-
-		for (slot = 0; slot < NUM_SLOTS; slot++) {
-			spd_slot = 2 * channel + slot;
-			spd_decode_ddr3(&dimm->dimm[channel][slot], spd[spd_slot]);
-			if (dimm->dimm[channel][slot].dram_type != SPD_MEMORY_TYPE_SDRAM_DDR3) {
-				// set dimm invalid
-				dimm->dimm[channel][slot].ranks = 0;
-				dimm->dimm[channel][slot].size_mb = 0;
-				continue;
-			}
-
-			dram_print_spd_ddr3(&dimm->dimm[channel][slot]);
-			dimms++;
-			ctrl->rank_mirror[channel][slot * 2] = 0;
-			ctrl->rank_mirror[channel][slot * 2 + 1] = dimm->dimm[channel][slot].flags.pins_mirrored;
-			ctrl->channel_size_mb[channel] += dimm->dimm[channel][slot].size_mb;
-
-			ctrl->auto_self_refresh &= dimm->dimm[channel][slot].flags.asr;
-			ctrl->extended_temperature_range &= dimm->dimm[channel][slot].flags.ext_temp_refresh;
-
-			ctrl->rankmap[channel] |= ((1 << dimm->dimm[channel][slot].ranks) - 1) << (2 * slot);
-			printk(BIOS_DEBUG, "rankmap[%d] = 0x%x\n", channel, ctrl->rankmap[channel]);
-		}
-		if ((ctrl->rankmap[channel] & 3) && (ctrl->rankmap[channel] & 0xc)
-			&& dimm->dimm[channel][0].reference_card <= 5 && dimm->dimm[channel][1].reference_card <= 5) {
-			const int ref_card_offset_table[6][6] = {
-				{ 0, 0, 0, 0, 2, 2, },
-				{ 0, 0, 0, 0, 2, 2, },
-				{ 0, 0, 0, 0, 2, 2, },
-				{ 0, 0, 0, 0, 1, 1, },
-				{ 2, 2, 2, 1, 0, 0, },
-				{ 2, 2, 2, 1, 0, 0, },
-			};
-			ctrl->ref_card_offset[channel] = ref_card_offset_table[dimm->dimm[channel][0].reference_card]
-				[dimm->dimm[channel][1].reference_card];
-		} else
-			ctrl->ref_card_offset[channel] = 0;
-	}
-
-	if (!dimms)
-		die("No DIMMs were found");
-}
-
-static void dram_find_common_params(const dimm_info * dimms,
-				    ramctr_timing * ctrl)
-{
-	size_t valid_dimms;
-	int channel, slot;
-	ctrl->cas_supported = 0xff;
-	valid_dimms = 0;
-	FOR_ALL_CHANNELS for (slot = 0; slot < 2; slot++) {
-		const dimm_attr *dimm = &dimms->dimm[channel][slot];
-		if (dimm->dram_type != SPD_MEMORY_TYPE_SDRAM_DDR3)
-			continue;
-		valid_dimms++;
-
-		/* Find all possible CAS combinations */
-		ctrl->cas_supported &= dimm->cas_supported;
-
-		/* Find the smallest common latencies supported by all DIMMs */
-		ctrl->tCK = MAX(ctrl->tCK, dimm->tCK);
-		ctrl->tAA = MAX(ctrl->tAA, dimm->tAA);
-		ctrl->tWR = MAX(ctrl->tWR, dimm->tWR);
-		ctrl->tRCD = MAX(ctrl->tRCD, dimm->tRCD);
-		ctrl->tRRD = MAX(ctrl->tRRD, dimm->tRRD);
-		ctrl->tRP = MAX(ctrl->tRP, dimm->tRP);
-		ctrl->tRAS = MAX(ctrl->tRAS, dimm->tRAS);
-		ctrl->tRFC = MAX(ctrl->tRFC, dimm->tRFC);
-		ctrl->tWTR = MAX(ctrl->tWTR, dimm->tWTR);
-		ctrl->tRTP = MAX(ctrl->tRTP, dimm->tRTP);
-		ctrl->tFAW = MAX(ctrl->tFAW, dimm->tFAW);
-	}
-
-	if (!ctrl->cas_supported)
-		die("Unsupported DIMM combination. "
-		    "DIMMS do not support common CAS latency");
-	if (!valid_dimms)
-		die("No valid DIMMs found");
-}
-
-static u8 get_CWL(u8 CAS)
-{
-	/* Get CWL based on CAS using the following rule:
-	 *       _________________________________________
-	 * CAS: | 4T | 5T | 6T | 7T | 8T | 9T | 10T | 11T |
-	 * CWL: | 5T | 5T | 5T | 6T | 6T | 7T |  7T |  8T |
-	 */
-	static const u8 cas_cwl_map[] = { 5, 5, 5, 6, 6, 7, 7, 8 };
-	if (CAS > 11)
-		return 8;
-	return cas_cwl_map[CAS - 4];
-}
-
-/* Frequency multiplier.  */
-static u32 get_FRQ(u32 tCK)
-{
-	u32 FRQ;
-	FRQ = 256000 / (tCK * BASEFREQ);
-	if (FRQ > 8)
-		return 8;
-	if (FRQ < 3)
-		return 3;
-	return FRQ;
-}
-
-static u32 get_REFI(u32 tCK)
-{
-	/* Get REFI based on MCU frequency using the following rule:
-	 *        _________________________________________
-	 * FRQ : | 3    | 4    | 5    | 6    | 7    | 8    |
-	 * REFI: | 3120 | 4160 | 5200 | 6240 | 7280 | 8320 |
-	 */
-	static const u32 frq_refi_map[] =
-	    { 3120, 4160, 5200, 6240, 7280, 8320 };
-	return frq_refi_map[get_FRQ(tCK) - 3];
-}
-
-static u8 get_XSOffset(u32 tCK)
-{
-	/* Get XSOffset based on MCU frequency using the following rule:
-	 *             _________________________
-	 * FRQ      : | 3 | 4 | 5 | 6 | 7  | 8  |
-	 * XSOffset : | 4 | 6 | 7 | 8 | 10 | 11 |
-	 */
-	static const u8 frq_xs_map[] = { 4, 6, 7, 8, 10, 11 };
-	return frq_xs_map[get_FRQ(tCK) - 3];
-}
-
-static u8 get_MOD(u32 tCK)
-{
-	/* Get MOD based on MCU frequency using the following rule:
-	 *        _____________________________
-	 * FRQ : | 3  | 4  | 5  | 6  | 7  | 8  |
-	 * MOD : | 12 | 12 | 12 | 12 | 15 | 16 |
-	 */
-	static const u8 frq_mod_map[] = { 12, 12, 12, 12, 15, 16 };
-	return frq_mod_map[get_FRQ(tCK) - 3];
-}
-
-static u8 get_WLO(u32 tCK)
-{
-	/* Get WLO based on MCU frequency using the following rule:
-	 *        _______________________
-	 * FRQ : | 3 | 4 | 5 | 6 | 7 | 8 |
-	 * WLO : | 4 | 5 | 6 | 6 | 8 | 8 |
-	 */
-	static const u8 frq_wlo_map[] = { 4, 5, 6, 6, 8, 8 };
-	return frq_wlo_map[get_FRQ(tCK) - 3];
-}
-
-static u8 get_CKE(u32 tCK)
-{
-	/* Get CKE based on MCU frequency using the following rule:
-	 *        _______________________
-	 * FRQ : | 3 | 4 | 5 | 6 | 7 | 8 |
-	 * CKE : | 3 | 3 | 4 | 4 | 5 | 6 |
-	 */
-	static const u8 frq_cke_map[] = { 3, 3, 4, 4, 5, 6 };
-	return frq_cke_map[get_FRQ(tCK) - 3];
-}
-
-static u8 get_XPDLL(u32 tCK)
-{
-	/* Get XPDLL based on MCU frequency using the following rule:
-	 *          _____________________________
-	 * FRQ   : | 3  | 4  | 5  | 6  | 7  | 8  |
-	 * XPDLL : | 10 | 13 | 16 | 20 | 23 | 26 |
-	 */
-	static const u8 frq_xpdll_map[] = { 10, 13, 16, 20, 23, 26 };
-	return frq_xpdll_map[get_FRQ(tCK) - 3];
-}
-
-static u8 get_XP(u32 tCK)
-{
-	/* Get XP based on MCU frequency using the following rule:
-	 *        _______________________
-	 * FRQ : | 3 | 4 | 5 | 6 | 7 | 8 |
-	 * XP  : | 3 | 4 | 4 | 5 | 6 | 7 |
-	 */
-	static const u8 frq_xp_map[] = { 3, 4, 4, 5, 6, 7 };
-	return frq_xp_map[get_FRQ(tCK) - 3];
-}
-
-static u8 get_AONPD(u32 tCK)
-{
-	/* Get AONPD based on MCU frequency using the following rule:
-	 *          ________________________
-	 * FRQ   : | 3 | 4 | 5 | 6 | 7 | 8  |
-	 * AONPD : | 4 | 5 | 6 | 8 | 8 | 10 |
-	 */
-	static const u8 frq_aonpd_map[] = { 4, 5, 6, 8, 8, 10 };
-	return frq_aonpd_map[get_FRQ(tCK) - 3];
-}
-
-static u32 get_COMP2(u32 tCK)
-{
-	/* Get COMP2 based on MCU frequency using the following rule:
-	 *         ___________________________________________________________
-	 * FRQ  : | 3       | 4       | 5       | 6       | 7       | 8       |
-	 * COMP : | D6BEDCC | CE7C34C | CA57A4C | C6369CC | C42514C | C21410C |
-	 */
-	static const u32 frq_comp2_map[] = { 0xD6BEDCC, 0xCE7C34C, 0xCA57A4C,
-		0xC6369CC, 0xC42514C, 0xC21410C
-	};
-	return frq_comp2_map[get_FRQ(tCK) - 3];
-}
-
-static void dram_timing(ramctr_timing * ctrl)
-{
-	u8 val;
-	u32 val32;
-
-	/* Maximum supported DDR3 frequency is 1066MHz (DDR3 2133) so make sure
-	 * we cap it if we have faster DIMMs.
-	 * Then, align it to the closest JEDEC standard frequency */
-	if (ctrl->tCK <= TCK_1066MHZ) {
-		ctrl->tCK = TCK_1066MHZ;
-		ctrl->edge_offset[0] = 16;
-		ctrl->edge_offset[1] = 7;
-		ctrl->edge_offset[2] = 7;
-		ctrl->timC_offset[0] = 18;
-		ctrl->timC_offset[1] = 7;
-		ctrl->timC_offset[2] = 7;
-		ctrl->reg_c14_offset = 16;
-		ctrl->reg_5064b0 = 0x218;
-		ctrl->reg_320c_range_threshold = 13;
-	} else if (ctrl->tCK <= TCK_933MHZ) {
-		ctrl->tCK = TCK_933MHZ;
-		ctrl->edge_offset[0] = 14;
-		ctrl->edge_offset[1] = 6;
-		ctrl->edge_offset[2] = 6;
-		ctrl->timC_offset[0] = 15;
-		ctrl->timC_offset[1] = 6;
-		ctrl->timC_offset[2] = 6;
-		ctrl->reg_c14_offset = 14;
-		ctrl->reg_5064b0 = 0x1d5;
-		ctrl->reg_320c_range_threshold = 15;
-	} else if (ctrl->tCK <= TCK_800MHZ) {
-		ctrl->tCK = TCK_800MHZ;
-		ctrl->edge_offset[0] = 13;
-		ctrl->edge_offset[1] = 5;
-		ctrl->edge_offset[2] = 5;
-		ctrl->timC_offset[0] = 14;
-		ctrl->timC_offset[1] = 5;
-		ctrl->timC_offset[2] = 5;
-		ctrl->reg_c14_offset = 12;
-		ctrl->reg_5064b0 = 0x193;
-		ctrl->reg_320c_range_threshold = 15;
-	} else if (ctrl->tCK <= TCK_666MHZ) {
-		ctrl->tCK = TCK_666MHZ;
-		ctrl->edge_offset[0] = 10;
-		ctrl->edge_offset[1] = 4;
-		ctrl->edge_offset[2] = 4;
-		ctrl->timC_offset[0] = 11;
-		ctrl->timC_offset[1] = 4;
-		ctrl->timC_offset[2] = 4;
-		ctrl->reg_c14_offset = 10;
-		ctrl->reg_5064b0 = 0x150;
-		ctrl->reg_320c_range_threshold = 16;
-	} else if (ctrl->tCK <= TCK_533MHZ) {
-		ctrl->tCK = TCK_533MHZ;
-		ctrl->edge_offset[0] = 8;
-		ctrl->edge_offset[1] = 3;
-		ctrl->edge_offset[2] = 3;
-		ctrl->timC_offset[0] = 9;
-		ctrl->timC_offset[1] = 3;
-		ctrl->timC_offset[2] = 3;
-		ctrl->reg_c14_offset = 8;
-		ctrl->reg_5064b0 = 0x10d;
-		ctrl->reg_320c_range_threshold = 17;
-	} else  {
-		ctrl->tCK = TCK_400MHZ;
-		ctrl->edge_offset[0] = 6;
-		ctrl->edge_offset[1] = 2;
-		ctrl->edge_offset[2] = 2;
-		ctrl->timC_offset[0] = 6;
-		ctrl->timC_offset[1] = 2;
-		ctrl->timC_offset[2] = 2;
-		ctrl->reg_c14_offset = 8;
-		ctrl->reg_5064b0 = 0xcd;
-		ctrl->reg_320c_range_threshold = 17;
-	}
-
-	val32 = (1000 << 8) / ctrl->tCK;
-	printk(BIOS_DEBUG, "Selected DRAM frequency: %u MHz\n", val32);
-
-	/* Find CAS and CWL latencies */
-	val = (ctrl->tAA + ctrl->tCK - 1) / ctrl->tCK;
-	printk(BIOS_DEBUG, "Minimum  CAS latency   : %uT\n", val);
-	/* Find lowest supported CAS latency that satisfies the minimum value */
-	while (!((ctrl->cas_supported >> (val - 4)) & 1)
-	       && (ctrl->cas_supported >> (val - 4))) {
-		val++;
-	}
-	/* Is CAS supported */
-	if (!(ctrl->cas_supported & (1 << (val - 4))))
-		printk(BIOS_DEBUG, "CAS not supported\n");
-	printk(BIOS_DEBUG, "Selected CAS latency   : %uT\n", val);
-	ctrl->CAS = val;
-	ctrl->CWL = get_CWL(ctrl->CAS);
-	printk(BIOS_DEBUG, "Selected CWL latency   : %uT\n", ctrl->CWL);
-
-	/* Find tRCD */
-	ctrl->tRCD = (ctrl->tRCD + ctrl->tCK - 1) / ctrl->tCK;
-	printk(BIOS_DEBUG, "Selected tRCD          : %uT\n", ctrl->tRCD);
-
-	ctrl->tRP = (ctrl->tRP + ctrl->tCK - 1) / ctrl->tCK;
-	printk(BIOS_DEBUG, "Selected tRP           : %uT\n", ctrl->tRP);
-
-	/* Find tRAS */
-	ctrl->tRAS = (ctrl->tRAS + ctrl->tCK - 1) / ctrl->tCK;
-	printk(BIOS_DEBUG, "Selected tRAS          : %uT\n", ctrl->tRAS);
-
-	/* Find tWR */
-	ctrl->tWR = (ctrl->tWR + ctrl->tCK - 1) / ctrl->tCK;
-	printk(BIOS_DEBUG, "Selected tWR           : %uT\n", ctrl->tWR);
-
-	/* Find tFAW */
-	ctrl->tFAW = (ctrl->tFAW + ctrl->tCK - 1) / ctrl->tCK;
-	printk(BIOS_DEBUG, "Selected tFAW          : %uT\n", ctrl->tFAW);
-
-	/* Find tRRD */
-	ctrl->tRRD = (ctrl->tRRD + ctrl->tCK - 1) / ctrl->tCK;
-	printk(BIOS_DEBUG, "Selected tRRD          : %uT\n", ctrl->tRRD);
-
-	/* Find tRTP */
-	ctrl->tRTP = (ctrl->tRTP + ctrl->tCK - 1) / ctrl->tCK;
-	printk(BIOS_DEBUG, "Selected tRTP          : %uT\n", ctrl->tRTP);
-
-	/* Find tWTR */
-	ctrl->tWTR = (ctrl->tWTR + ctrl->tCK - 1) / ctrl->tCK;
-	printk(BIOS_DEBUG, "Selected tWTR          : %uT\n", ctrl->tWTR);
-
-	/* Refresh-to-Active or Refresh-to-Refresh (tRFC) */
-	ctrl->tRFC = (ctrl->tRFC + ctrl->tCK - 1) / ctrl->tCK;
-	printk(BIOS_DEBUG, "Selected tRFC          : %uT\n", ctrl->tRFC);
-
-	ctrl->tREFI = get_REFI(ctrl->tCK);
-	ctrl->tMOD = get_MOD(ctrl->tCK);
-	ctrl->tXSOffset = get_XSOffset(ctrl->tCK);
-	ctrl->tWLO = get_WLO(ctrl->tCK);
-	ctrl->tCKE = get_CKE(ctrl->tCK);
-	ctrl->tXPDLL = get_XPDLL(ctrl->tCK);
-	ctrl->tXP = get_XP(ctrl->tCK);
-	ctrl->tAONPD = get_AONPD(ctrl->tCK);
-}
-
-static void dram_freq(ramctr_timing * ctrl)
-{
-	if (ctrl->tCK > TCK_400MHZ) {
-		printk (BIOS_ERR, "DRAM frequency is under lowest supported frequency (400 MHz). Increasing to 400 MHz as last resort");
-		ctrl->tCK = TCK_400MHZ;
-	}
-	while (1) {
-		u8 val2;
-		u32 reg1 = 0;
-
-		/* Step 1 - Set target PCU frequency */
-
-		if (ctrl->tCK <= TCK_1066MHZ) {
-			ctrl->tCK = TCK_1066MHZ;
-		} else if (ctrl->tCK <= TCK_933MHZ) {
-			ctrl->tCK = TCK_933MHZ;
-		} else if (ctrl->tCK <= TCK_800MHZ) {
-			ctrl->tCK = TCK_800MHZ;
-		} else if (ctrl->tCK <= TCK_666MHZ) {
-			ctrl->tCK = TCK_666MHZ;
-		} else if (ctrl->tCK <= TCK_533MHZ) {
-			ctrl->tCK = TCK_533MHZ;
-		} else if (ctrl->tCK <= TCK_400MHZ) {
-			ctrl->tCK = TCK_400MHZ;
-		} else {
-			die ("No lock frequency found");
-		}
-
-		/* Frequency mulitplier.  */
-		u32 FRQ = get_FRQ(ctrl->tCK);
-
-		/* Step 2 - Select frequency in the MCU */
-		reg1 = FRQ;
-		reg1 |= 0x80000000;	// set running bit
-		MCHBAR32(0x5e00) = reg1;
-		while (reg1 & 0x80000000) {
-			printk(BIOS_DEBUG, " PLL busy...");
-			reg1 = MCHBAR32(0x5e00);
-		}
-		printk(BIOS_DEBUG, "done\n");
-
-		/* Step 3 - Verify lock frequency */
-		reg1 = MCHBAR32(0x5e04);
-		val2 = (u8) reg1;
-		if (val2 >= FRQ) {
-			printk(BIOS_DEBUG, "MCU frequency is set at : %d MHz\n",
-			       (1000 << 8) / ctrl->tCK);
-			return;
-		}
-		printk(BIOS_DEBUG, "PLL didn't lock. Retrying at lower frequency\n");
-		ctrl->tCK++;
-	}
-}
-
-static void dram_xover(ramctr_timing * ctrl)
-{
-	u32 reg;
-	int channel;
-
-	FOR_ALL_CHANNELS {
-		// enable xover clk
-		printk(BIOS_DEBUG, "[%x] = %x\n", channel * 0x100 + 0xc14,
-		       (ctrl->rankmap[channel] << 24));
-		MCHBAR32(channel * 0x100 + 0xc14) = (ctrl->rankmap[channel] << 24);
-
-		// enable xover ctl
-		reg = 0;
-		if (ctrl->rankmap[channel] & 0x5) {
-			reg |= 0x20000;
-		}
-		if (ctrl->rankmap[channel] & 0xa) {
-			reg |= 0x4000000;
-		}
-		// enable xover cmd
-		reg |= 0x4000;
-		printk(BIOS_DEBUG, "[%x] = %x\n", 0x100 * channel + 0x320c,
-		       reg);
-		MCHBAR32(0x100 * channel + 0x320c) = reg;
-	}
-}
-
-static void dram_timing_regs(ramctr_timing * ctrl)
-{
-	u32 reg, addr, val32, cpu, stretch;
-	struct cpuid_result cpures;
-	int channel;
-
-	FOR_ALL_CHANNELS {
-		// DBP
-		reg = 0;
-		reg |= ctrl->tRCD;
-		reg |= (ctrl->tRP << 4);
-		reg |= (ctrl->CAS << 8);
-		reg |= (ctrl->CWL << 12);
-		reg |= (ctrl->tRAS << 16);
-		printk(BIOS_DEBUG, "[%x] = %x\n", 0x400 * channel + 0x4000,
-		       reg);
-		MCHBAR32(0x400 * channel + 0x4000) = reg;
-
-		// RAP
-		reg = 0;
-		reg |= ctrl->tRRD;
-		reg |= (ctrl->tRTP << 4);
-		reg |= (ctrl->tCKE << 8);
-		reg |= (ctrl->tWTR << 12);
-		reg |= (ctrl->tFAW << 16);
-		reg |= (ctrl->tWR << 24);
-		reg |= (3 << 30);
-		printk(BIOS_DEBUG, "[%x] = %x\n", 0x400 * channel + 0x4004,
-		       reg);
-		MCHBAR32(0x400 * channel + 0x4004) = reg;
-
-		// OTHP
-		addr = 0x400 * channel + 0x400c;
-		reg = 0;
-		reg |= ctrl->tXPDLL;
-		reg |= (ctrl->tXP << 5);
-		reg |= (ctrl->tAONPD << 8);
-		reg |= 0xa0000;
-		printk(BIOS_DEBUG, "[%x] = %x\n", addr, reg);
-		MCHBAR32(addr) = reg;
-
-		MCHBAR32(0x400 * channel + 0x4014) = 0;
-
-		MCHBAR32(addr) |= 0x00020000;
-
-		// ODT stretch
-		reg = 0;
-
-		cpures = cpuid(0);
-		cpu = cpures.eax;
-		if (IS_IVY_CPU(cpu)
-		    || (IS_SANDY_CPU(cpu) && IS_SANDY_CPU_D2(cpu))) {
-			stretch = 2;
-			addr = 0x400 * channel + 0x400c;
-			printk(BIOS_DEBUG, "[%x] = %x\n",
-			       0x400 * channel + 0x400c, reg);
-			reg = MCHBAR32(addr);
-
-			if (((ctrl->rankmap[channel] & 3) == 0)
-			    || (ctrl->rankmap[channel] & 0xc) == 0) {
-
-				// Rank 0 - operate on rank 2
-				reg = (reg & ~0xc0000) | (stretch << 18);
-
-				// Rank 2 - operate on rank 0
-				reg = (reg & ~0x30000) | (stretch << 16);
-
-				printk(BIOS_DEBUG, "[%x] = %x\n", addr, reg);
-				MCHBAR32(addr) = reg;
-			}
-
-		} else if (IS_SANDY_CPU(cpu) && IS_SANDY_CPU_C(cpu)) {
-			stretch = 3;
-			addr = 0x400 * channel + 0x401c;
-			reg = MCHBAR32(addr);
-
-			if (((ctrl->rankmap[channel] & 3) == 0)
-			    || (ctrl->rankmap[channel] & 0xc) == 0) {
-
-				// Rank 0 - operate on rank 2
-				reg = (reg & ~0x3000) | (stretch << 12);
-
-				// Rank 2 - operate on rank 0
-				reg = (reg & ~0xc00) | (stretch << 10);
-
-				printk(BIOS_DEBUG, "[%x] = %x\n", addr, reg);
-				MCHBAR32(addr) = reg;
-			}
-		} else {
-			stretch = 0;
-		}
-
-		// REFI
-		reg = 0;
-		val32 = ctrl->tREFI;
-		reg = (reg & ~0xffff) | val32;
-		val32 = ctrl->tRFC;
-		reg = (reg & ~0x1ff0000) | (val32 << 16);
-		val32 = (u32) (ctrl->tREFI * 9) / 1024;
-		reg = (reg & ~0xfe000000) | (val32 << 25);
-		printk(BIOS_DEBUG, "[%x] = %x\n", 0x400 * channel + 0x4298,
-		       reg);
-		MCHBAR32(0x400 * channel + 0x4298) = reg;
-
-		MCHBAR32(0x400 * channel + 0x4294) |= 0xff;
-
-		// SRFTP
-		reg = 0;
-		val32 = tDLLK;
-		reg = (reg & ~0xfff) | val32;
-		val32 = ctrl->tXSOffset;
-		reg = (reg & ~0xf000) | (val32 << 12);
-		val32 = tDLLK - ctrl->tXSOffset;
-		reg = (reg & ~0x3ff0000) | (val32 << 16);
-		val32 = ctrl->tMOD - 8;
-		reg = (reg & ~0xf0000000) | (val32 << 28);
-		printk(BIOS_DEBUG, "[%x] = %x\n", 0x400 * channel + 0x42a4,
-		       reg);
-		MCHBAR32(0x400 * channel + 0x42a4) = reg;
-	}
-}
-
-static void dram_dimm_mapping(dimm_info * info, ramctr_timing * ctrl)
-{
-	u32 reg, val32;
-	int channel;
-
-	FOR_ALL_CHANNELS {
-		dimm_attr *dimmA = 0;
-		dimm_attr *dimmB = 0;
-		reg = 0;
-		val32 = 0;
-		if (info->dimm[channel][0].size_mb >=
-		    info->dimm[channel][1].size_mb) {
-			// dimm 0 is bigger, set it to dimmA
-			dimmA = &info->dimm[channel][0];
-			dimmB = &info->dimm[channel][1];
-			reg |= (0 << 16);
-		} else {
-			// dimm 1 is bigger, set it to dimmA
-			dimmA = &info->dimm[channel][1];
-			dimmB = &info->dimm[channel][0];
-			reg |= (1 << 16);
-		}
-		// dimmA
-		if (dimmA && (dimmA->ranks > 0)) {
-			val32 = dimmA->size_mb / 256;
-			reg = (reg & ~0xff) | val32;
-			val32 = dimmA->ranks - 1;
-			reg = (reg & ~0x20000) | (val32 << 17);
-			val32 = (dimmA->width / 8) - 1;
-			reg = (reg & ~0x80000) | (val32 << 19);
-		}
-		// dimmB
-		if (dimmB && (dimmB->ranks > 0)) {
-			val32 = dimmB->size_mb / 256;
-			reg = (reg & ~0xff00) | (val32 << 8);
-			val32 = dimmB->ranks - 1;
-			reg = (reg & ~0x40000) | (val32 << 18);
-			val32 = (dimmB->width / 8) - 1;
-			reg = (reg & ~0x100000) | (val32 << 20);
-		}
-		reg = (reg & ~0x200000) | (1 << 21);	// rank interleave
-		reg = (reg & ~0x400000) | (1 << 22);	// enhanced interleave
-
-		// Save MAD-DIMM register
-		if ((dimmA && (dimmA->ranks > 0))
-		    || (dimmB && (dimmB->ranks > 0))) {
-			ctrl->mad_dimm[channel] = reg;
-		} else {
-			ctrl->mad_dimm[channel] = 0;
-		}
-	}
-}
-
-static void dram_dimm_set_mapping(ramctr_timing * ctrl)
-{
-	int channel;
-	FOR_ALL_CHANNELS {
-		MCHBAR32(0x5004 + channel * 4) = ctrl->mad_dimm[channel];
-	}
-}
-
-static void dram_zones(ramctr_timing * ctrl, int training)
-{
-	u32 reg, ch0size, ch1size;
-	u8 val;
-	reg = 0;
-	val = 0;
-	if (training) {
-		ch0size = ctrl->channel_size_mb[0] ? 256 : 0;
-		ch1size = ctrl->channel_size_mb[1] ? 256 : 0;
-	} else {
-		ch0size = ctrl->channel_size_mb[0];
-		ch1size = ctrl->channel_size_mb[1];
-	}
-
-	if (ch0size >= ch1size) {
-		reg = MCHBAR32(0x5014);
-		val = ch1size / 256;
-		reg = (reg & ~0xff000000) | val << 24;
-		reg = (reg & ~0xff0000) | (2 * val) << 16;
-		MCHBAR32(0x5014) = reg;
-		MCHBAR32(0x5000) = 0x24;
-	} else {
-		reg = MCHBAR32(0x5014);
-		val = ch0size / 256;
-		reg = (reg & ~0xff000000) | val << 24;
-		reg = (reg & ~0xff0000) | (2 * val) << 16;
-		MCHBAR32(0x5014) = reg;
-		MCHBAR32(0x5000) = 0x21;
-	}
-}
-
-static void dram_memorymap(ramctr_timing * ctrl, int me_uma_size)
-{
-	u32 reg, val, reclaim;
-	u32 tom, gfxstolen, gttsize;
-	size_t tsegsize, mmiosize, toludbase, touudbase, gfxstolenbase, gttbase,
-	    tsegbase, mestolenbase;
-	size_t tsegbasedelta, remapbase, remaplimit;
-	uint16_t ggc;
-
-	mmiosize = 0x400;
-
-	ggc = pci_read_config16(NORTHBRIDGE, GGC);
-	if (!(ggc & 2)) {
-		gfxstolen = ((ggc >> 3) & 0x1f) * 32;
-		gttsize = ((ggc >> 8) & 0x3);
-	} else {
-		gfxstolen = 0;
-		gttsize = 0;
-	}
-
-	tsegsize = CONFIG_SMM_TSEG_SIZE >> 20;
-
-	tom = ctrl->channel_size_mb[0] + ctrl->channel_size_mb[1];
-
-	mestolenbase = tom - me_uma_size;
-
-	toludbase = MIN(4096 - mmiosize + gfxstolen + gttsize + tsegsize,
-			tom - me_uma_size);
-	gfxstolenbase = toludbase - gfxstolen;
-	gttbase = gfxstolenbase - gttsize;
-
-	tsegbase = gttbase - tsegsize;
-
-	// Round tsegbase down to nearest address aligned to tsegsize
-	tsegbasedelta = tsegbase & (tsegsize - 1);
-	tsegbase &= ~(tsegsize - 1);
-
-	gttbase -= tsegbasedelta;
-	gfxstolenbase -= tsegbasedelta;
-	toludbase -= tsegbasedelta;
-
-	// Test if it is possible to reclaim a hole in the ram addressing
-	if (tom - me_uma_size > toludbase) {
-		// Reclaim is possible
-		reclaim = 1;
-		remapbase = MAX(4096, tom - me_uma_size);
-		remaplimit =
-		    remapbase + MIN(4096, tom - me_uma_size) - toludbase - 1;
-		touudbase = remaplimit + 1;
-	} else {
-		// Reclaim not possible
-		reclaim = 0;
-		touudbase = tom - me_uma_size;
-	}
-
-	// Update memory map in pci-e configuration space
-
-	// TOM (top of memory)
-	reg = pcie_read_config32(PCI_DEV(0, 0, 0), 0xa0);
-	val = tom & 0xfff;
-	reg = (reg & ~0xfff00000) | (val << 20);
-	printk(BIOS_DEBUG, "PCI:[%x] = %x\n", 0xa0, reg);
-	pcie_write_config32(PCI_DEV(0, 0, 0), 0xa0, reg);
-
-	reg = pcie_read_config32(PCI_DEV(0, 0, 0), 0xa4);
-	val = tom & 0xfffff000;
-	reg = (reg & ~0x000fffff) | (val >> 12);
-	printk(BIOS_DEBUG, "PCI:[%x] = %x\n", 0xa4, reg);
-	pcie_write_config32(PCI_DEV(0, 0, 0), 0xa4, reg);
-
-	// TOLUD (top of low used dram)
-	reg = pcie_read_config32(PCI_DEV(0, 0, 0), 0xbc);
-	val = toludbase & 0xfff;
-	reg = (reg & ~0xfff00000) | (val << 20);
-	printk(BIOS_DEBUG, "PCI:[%x] = %x\n", 0xbc, reg);
-	pcie_write_config32(PCI_DEV(0, 0, 0), 0xbc, reg);
-
-	// TOUUD LSB (top of upper usable dram)
-	reg = pcie_read_config32(PCI_DEV(0, 0, 0), 0xa8);
-	val = touudbase & 0xfff;
-	reg = (reg & ~0xfff00000) | (val << 20);
-	printk(BIOS_DEBUG, "PCI:[%x] = %x\n", 0xa8, reg);
-	pcie_write_config32(PCI_DEV(0, 0, 0), 0xa8, reg);
-
-	// TOUUD MSB
-	reg = pcie_read_config32(PCI_DEV(0, 0, 0), 0xac);
-	val = touudbase & 0xfffff000;
-	reg = (reg & ~0x000fffff) | (val >> 12);
-	printk(BIOS_DEBUG, "PCI:[%x] = %x\n", 0xac, reg);
-	pcie_write_config32(PCI_DEV(0, 0, 0), 0xac, reg);
-
-	if (reclaim) {
-		// REMAP BASE
-		pcie_write_config32(PCI_DEV(0, 0, 0), 0x90, remapbase << 20);
-		pcie_write_config32(PCI_DEV(0, 0, 0), 0x94, remapbase >> 12);
-
-		// REMAP LIMIT
-		pcie_write_config32(PCI_DEV(0, 0, 0), 0x98, remaplimit << 20);
-		pcie_write_config32(PCI_DEV(0, 0, 0), 0x9c, remaplimit >> 12);
-	}
-	// TSEG
-	reg = pcie_read_config32(PCI_DEV(0, 0, 0), 0xb8);
-	val = tsegbase & 0xfff;
-	reg = (reg & ~0xfff00000) | (val << 20);
-	printk(BIOS_DEBUG, "PCI:[%x] = %x\n", 0xb8, reg);
-	pcie_write_config32(PCI_DEV(0, 0, 0), 0xb8, reg);
-
-	// GFX stolen memory
-	reg = pcie_read_config32(PCI_DEV(0, 0, 0), 0xb0);
-	val = gfxstolenbase & 0xfff;
-	reg = (reg & ~0xfff00000) | (val << 20);
-	printk(BIOS_DEBUG, "PCI:[%x] = %x\n", 0xb0, reg);
-	pcie_write_config32(PCI_DEV(0, 0, 0), 0xb0, reg);
-
-	// GTT stolen memory
-	reg = pcie_read_config32(PCI_DEV(0, 0, 0), 0xb4);
-	val = gttbase & 0xfff;
-	reg = (reg & ~0xfff00000) | (val << 20);
-	printk(BIOS_DEBUG, "PCI:[%x] = %x\n", 0xb4, reg);
-	pcie_write_config32(PCI_DEV(0, 0, 0), 0xb4, reg);
-
-	if (me_uma_size) {
-		reg = pcie_read_config32(PCI_DEV(0, 0, 0), 0x7c);
-		val = (0x80000 - me_uma_size) & 0xfffff000;
-		reg = (reg & ~0x000fffff) | (val >> 12);
-		printk(BIOS_DEBUG, "PCI:[%x] = %x\n", 0x7c, reg);
-		pcie_write_config32(PCI_DEV(0, 0, 0), 0x7c, reg);
-
-		// ME base
-		reg = pcie_read_config32(PCI_DEV(0, 0, 0), 0x70);
-		val = mestolenbase & 0xfff;
-		reg = (reg & ~0xfff00000) | (val << 20);
-		printk(BIOS_DEBUG, "PCI:[%x] = %x\n", 0x70, reg);
-		pcie_write_config32(PCI_DEV(0, 0, 0), 0x70, reg);
-
-		reg = pcie_read_config32(PCI_DEV(0, 0, 0), 0x74);
-		val = mestolenbase & 0xfffff000;
-		reg = (reg & ~0x000fffff) | (val >> 12);
-		printk(BIOS_DEBUG, "PCI:[%x] = %x\n", 0x74, reg);
-		pcie_write_config32(PCI_DEV(0, 0, 0), 0x74, reg);
-
-		// ME mask
-		reg = pcie_read_config32(PCI_DEV(0, 0, 0), 0x78);
-		val = (0x80000 - me_uma_size) & 0xfff;
-		reg = (reg & ~0xfff00000) | (val << 20);
-		reg = (reg & ~0x400) | (1 << 10);	// set lockbit on ME mem
-
-		reg = (reg & ~0x800) | (1 << 11);	// set ME memory enable
-		printk(BIOS_DEBUG, "PCI:[%x] = %x\n", 0x78, reg);
-		pcie_write_config32(PCI_DEV(0, 0, 0), 0x78, reg);
-	}
-}
-
-static void dram_ioregs(ramctr_timing * ctrl)
-{
-	u32 reg, comp2;
-
-	int channel;
-
-	// IO clock
-	FOR_ALL_CHANNELS {
-		MCHBAR32(0xc00 + 0x100 * channel) = ctrl->rankmap[channel];
-	}
-
-	// IO command
-	FOR_ALL_CHANNELS {
-		MCHBAR32(0x3200 + 0x100 * channel) = ctrl->rankmap[channel];
-	}
-
-	// IO control
-	FOR_ALL_POPULATED_CHANNELS {
-		program_timings(ctrl, channel);
-	}
-
-	// Rcomp
-	printk(BIOS_DEBUG, "RCOMP...");
-	reg = 0;
-	while (reg == 0) {
-		reg = MCHBAR32(0x5084) & 0x10000;
-	}
-	printk(BIOS_DEBUG, "done\n");
-
-	// Set comp2
-	comp2 = get_COMP2(ctrl->tCK);
-	MCHBAR32(0x3714) = comp2;
-	printk(BIOS_DEBUG, "COMP2 done\n");
-
-	// Set comp1
-	FOR_ALL_POPULATED_CHANNELS {
-		reg = MCHBAR32(0x1810 + channel * 0x100);	//ch0
-		reg = (reg & ~0xe00) | (1 << 9);	//odt
-		reg = (reg & ~0xe00000) | (1 << 21);	//clk drive up
-		reg = (reg & ~0x38000000) | (1 << 27);	//ctl drive up
-		MCHBAR32(0x1810 + channel * 0x100) = reg;
-	}
-	printk(BIOS_DEBUG, "COMP1 done\n");
-
-	printk(BIOS_DEBUG, "FORCE RCOMP and wait 20us...");
-	MCHBAR32(0x5f08) |= 0x100;
-	udelay(20);
-	printk(BIOS_DEBUG, "done\n");
-}
-
-static void wait_428c(int channel)
-{
-	while (1) {
-		if (read32(DEFAULT_MCHBAR + 0x428c + (channel << 10)) & 0x50)
-			return;
-	}
-}
-
-static void write_reset(ramctr_timing * ctrl)
-{
-	int channel, slotrank;
-
-	/* choose a populated channel.  */
-	channel = (ctrl->rankmap[0]) ? 0 : 1;
-
-	wait_428c(channel);
-
-	/* choose a populated rank.  */
-	slotrank = (ctrl->rankmap[channel] & 1) ? 0 : 2;
-
-	write32(DEFAULT_MCHBAR + 0x4220 + 0x400 * channel, 0x0f003);
-	write32(DEFAULT_MCHBAR + 0x4230 + 0x400 * channel, 0x80c01);
-
-	write32(DEFAULT_MCHBAR + 0x4200 + 0x400 * channel,
-		(slotrank << 24) | 0x60000);
-
-	write32(DEFAULT_MCHBAR + 0x4210 + 0x400 * channel, 0);
-
-	write32(DEFAULT_MCHBAR + 0x4284 + 0x400 * channel, 0x400001);
-	wait_428c(channel);
-}
-
-static void dram_jedecreset(ramctr_timing * ctrl)
-{
-	u32 reg, addr;
-	int channel;
-
-	while (!(MCHBAR32(0x5084) & 0x10000)) ;
-	do {
-		reg = MCHBAR32(0x428c);
-	} while ((reg & 0x14) == 0);
-
-	// Set state of memory controller
-	reg = 0x112;
-	MCHBAR32(0x5030) = reg;
-	MCHBAR32(0x4ea0) = 0;
-	reg |= 2;		//ddr reset
-	MCHBAR32(0x5030) = reg;
-
-	// Assert dimm reset signal
-	reg = MCHBAR32(0x5030);
-	reg &= ~0x2;
-	MCHBAR32(0x5030) = reg;
-
-	// Wait 200us
-	udelay(200);
-
-	// Deassert dimm reset signal
-	MCHBAR32(0x5030) |= 2;
-
-	// Wait 500us
-	udelay(500);
-
-	// Enable DCLK
-	MCHBAR32(0x5030) |= 4;
-
-	// XXX Wait 20ns
-	udelay(1);
-
-	FOR_ALL_CHANNELS {
-		// Set valid rank CKE
-		reg = 0;
-		reg = (reg & ~0xf) | ctrl->rankmap[channel];
-		addr = 0x400 * channel + 0x42a0;
-		MCHBAR32(addr) = reg;
-
-		// Wait 10ns for ranks to settle
-		//udelay(0.01);
-
-		reg = (reg & ~0xf0) | (ctrl->rankmap[channel] << 4);
-		MCHBAR32(addr) = reg;
-
-		// Write reset using a NOP
-		write_reset(ctrl);
-	}
-}
-
-static odtmap get_ODT(ramctr_timing * ctrl, u8 rank)
-{
-	/* Get ODT based on rankmap: */
-	int dimms_per_ch = 0;
-	int channel;
-
-	FOR_ALL_CHANNELS {
-		dimms_per_ch = max ((ctrl->rankmap[channel] & 1)
-				    + ((ctrl->rankmap[channel] >> 2) & 1),
-				    dimms_per_ch);
-	}
-
-	if (dimms_per_ch == 1) {
-		return (const odtmap){60, 60};
-	} else if (dimms_per_ch == 2) {
-		return (const odtmap){120, 30};
-	} else {
-		printk(BIOS_DEBUG,
-		       "Huh, no dimms? m0 = %d m1 = %d dpc = %d\n",
-		       ctrl->rankmap[0],
-		       ctrl->rankmap[1], dimms_per_ch);
-		die("");
-	}
-}
-
-static void write_mrreg(ramctr_timing * ctrl, int channel, int slotrank,
-			int reg, u32 val)
-{
-	wait_428c(channel);
-
-	printram("MRd: %x <= %x\n", reg, val);
-
-	if (ctrl->rank_mirror[channel][slotrank]) {
-		/* DDR3 Rank1 Address mirror
-		 * swap the following pins:
-		 * A3<->A4, A5<->A6, A7<->A8, BA0<->BA1 */
-		reg = ((reg >> 1) & 1) | ((reg << 1) & 2);
-		val = (val & ~0x1f8) | ((val >> 1) & 0xa8)
-		    | ((val & 0xa8) << 1);
-	}
-
-	printram("MRd: %x <= %x\n", reg, val);
-
-	write32(DEFAULT_MCHBAR + 0x4220 + 0x400 * channel, 0x0f000);
-	write32(DEFAULT_MCHBAR + 0x4230 + 0x400 * channel, 0x41001);
-	write32(DEFAULT_MCHBAR + 0x4200 + 0x400 * channel,
-		(slotrank << 24) | (reg << 20) | val | 0x60000);
-	write32(DEFAULT_MCHBAR + 0x4210 + 0x400 * channel, 0);
-
-	write32(DEFAULT_MCHBAR + 0x4224 + 0x400 * channel, 0x1f000);
-	write32(DEFAULT_MCHBAR + 0x4234 + 0x400 * channel, 0x41001);
-	write32(DEFAULT_MCHBAR + 0x4204 + 0x400 * channel,
-		(slotrank << 24) | (reg << 20) | val | 0x60000);
-	write32(DEFAULT_MCHBAR + 0x4214 + 0x400 * channel, 0);
-
-	write32(DEFAULT_MCHBAR + 0x4228 + 0x400 * channel, 0x0f000);
-	write32(DEFAULT_MCHBAR + 0x4238 + 0x400 * channel,
-		0x1001 | (ctrl->tMOD << 16));
-	write32(DEFAULT_MCHBAR + 0x4208 + 0x400 * channel,
-		(slotrank << 24) | (reg << 20) | val | 0x60000);
-	write32(DEFAULT_MCHBAR + 0x4218 + 0x400 * channel, 0);
-	write32(DEFAULT_MCHBAR + 0x4284 + 0x400 * channel, 0x80001);
-}
-
-static u32 make_mr0(ramctr_timing * ctrl, u8 rank)
-{
-	u16 mr0reg, mch_cas, mch_wr;
-	static const u8 mch_wr_t[12] = { 1, 2, 3, 4, 0, 5, 0, 6, 0, 7, 0, 0 };
-	mr0reg = 0x100;
-
-	// Convert CAS to MCH register friendly
-	if (ctrl->CAS < 12) {
-		mch_cas = (u16) ((ctrl->CAS - 4) << 1);
-	} else {
-		mch_cas = (u16) (ctrl->CAS - 12);
-		mch_cas = ((mch_cas << 1) | 0x1);
-	}
-
-	// Convert tWR to MCH register friendly
-	mch_wr = mch_wr_t[ctrl->tWR - 5];
-
-	mr0reg = (mr0reg & ~0x4) | (mch_cas & 0x1);
-	mr0reg = (mr0reg & ~0x70) | ((mch_cas & 0xe) << 3);
-	mr0reg = (mr0reg & ~0xe00) | (mch_wr << 9);
-	// Fast (desktop) 0x1 or slow (mobile) 0x0
-	mr0reg = (mr0reg & ~0x1000) | (!ctrl->mobile << 12);
-	return mr0reg;
-}
-
-static void dram_mr0(ramctr_timing * ctrl, u8 rank)
-{
-	int channel;
-
-	FOR_ALL_POPULATED_CHANNELS write_mrreg(ctrl, channel, rank, 0,
-					       make_mr0(ctrl, rank));
-}
-
-static u32 encode_odt(u32 odt)
-{
-	switch (odt) {
-	case 30:
-		return (1 << 9) | (1 << 2);	// RZQ/8, RZQ/4
-	case 60:
-		return (1 << 2);	// RZQ/4
-	case 120:
-		return (1 << 6);	// RZQ/2
-	default:
-	case 0:
-		return 0;
-	}
-}
-
-static u32 make_mr1(ramctr_timing * ctrl, u8 rank)
-{
-	odtmap odt;
-	u32 mr1reg;
-
-	odt = get_ODT(ctrl, rank);
-	mr1reg = 0x2;
-
-	mr1reg |= encode_odt(odt.rttnom);
-
-	return mr1reg;
-}
-
-static void dram_mr1(ramctr_timing * ctrl, u8 rank)
-{
-	u16 mr1reg;
-	int channel;
-
-	mr1reg = make_mr1(ctrl, rank);
-
-	FOR_ALL_CHANNELS {
-		write_mrreg(ctrl, channel, rank, 1, mr1reg);
-	}
-}
-
-static void dram_mr2(ramctr_timing * ctrl, u8 rank)
-{
-	u16 pasr, cwl, mr2reg;
-	odtmap odt;
-	int channel;
-	int srt;
-
-	pasr = 0;
-	cwl = ctrl->CWL - 5;
-	odt = get_ODT(ctrl, rank);
-
-	srt = ctrl->extended_temperature_range && !ctrl->auto_self_refresh;
-
-	mr2reg = 0;
-	mr2reg = (mr2reg & ~0x7) | pasr;
-	mr2reg = (mr2reg & ~0x38) | (cwl << 3);
-	mr2reg = (mr2reg & ~0x40) | (ctrl->auto_self_refresh << 6);
-	mr2reg = (mr2reg & ~0x80) | (srt << 7);
-	mr2reg |= (odt.rttwr / 60) << 9;
-
-	FOR_ALL_CHANNELS {
-		write_mrreg(ctrl, channel, rank, 2, mr2reg);
-	}
-}
-
-static void dram_mr3(ramctr_timing * ctrl, u8 rank)
-{
-	int channel;
-
-	FOR_ALL_CHANNELS {
-		write_mrreg(ctrl, channel, rank, 3, 0);
-	}
-}
-
-static void dram_mrscommands(ramctr_timing * ctrl)
-{
-	u8 rank;
-	u32 reg, addr;
-	int channel;
-
-	for (rank = 0; rank < 4; rank++) {
-		// MR2
-		printram("MR2 rank %d...", rank);
-		dram_mr2(ctrl, rank);
-		printram("done\n");
-
-		// MR3
-		printram("MR3 rank %d...", rank);
-		dram_mr3(ctrl, rank);
-		printram("done\n");
-
-		// MR1
-		printram("MR1 rank %d...", rank);
-		dram_mr1(ctrl, rank);
-		printram("done\n");
-
-		// MR0
-		printram("MR0 rank %d...", rank);
-		dram_mr0(ctrl, rank);
-		printram("done\n");
-	}
-
-	write32(DEFAULT_MCHBAR + 0x4e20, 0x7);
-	write32(DEFAULT_MCHBAR + 0x4e30, 0xf1001);
-	write32(DEFAULT_MCHBAR + 0x4e00, 0x60002);
-	write32(DEFAULT_MCHBAR + 0x4e10, 0);
-	write32(DEFAULT_MCHBAR + 0x4e24, 0x1f003);
-	write32(DEFAULT_MCHBAR + 0x4e34, 0x1901001);
-	write32(DEFAULT_MCHBAR + 0x4e04, 0x60400);
-	write32(DEFAULT_MCHBAR + 0x4e14, 0x288);
-	write32(DEFAULT_MCHBAR + 0x4e84, 0x40004);
-
-	// Drain
-	FOR_ALL_CHANNELS {
-		// Wait for ref drained
-		wait_428c(channel);
-	}
-
-	// Refresh enable
-	MCHBAR32(0x5030) |= 8;
-
-	FOR_ALL_POPULATED_CHANNELS {
-		addr = 0x400 * channel + 0x4020;
-		reg = MCHBAR32(addr);
-		reg &= ~0x200000;
-		MCHBAR32(addr) = reg;
-
-		wait_428c(channel);
-
-		rank = (ctrl->rankmap[channel] & 1) ? 0 : 2;
-
-		// Drain
-		wait_428c(channel);
-
-		write32(DEFAULT_MCHBAR + 0x4220 + 0x400 * channel, 0x0f003);
-		write32(DEFAULT_MCHBAR + 0x4230 + 0x400 * channel, 0x659001);
-		write32(DEFAULT_MCHBAR + 0x4200 + 0x400 * channel,
-			(rank << 24) | 0x60000);
-		write32(DEFAULT_MCHBAR + 0x4210 + 0x400 * channel, 0x3e0);
-		write32(DEFAULT_MCHBAR + 0x4284 + 0x400 * channel, 0x1);
-
-		// Drain
-		wait_428c(channel);
-	}
-}
-
-const u32 lane_registers[] = {
-	0x0000, 0x0200, 0x0400, 0x0600,
-	0x1000, 0x1200, 0x1400, 0x1600,
-	0x0800
-};
-
-static void program_timings(ramctr_timing * ctrl, int channel)
-{
-	u32 reg32, reg_4024, reg_c14, reg_c18, reg_4028;
-	int lane;
-	int slotrank, slot;
-	int full_shift = 0;
-	u16 slot320c[NUM_SLOTS];
-
-	FOR_ALL_POPULATED_RANKS {
-		if (full_shift < -ctrl->timings[channel][slotrank].val_320c)
-			full_shift = -ctrl->timings[channel][slotrank].val_320c;
-	}
-
-	for (slot = 0; slot < NUM_SLOTS; slot++)
-		switch ((ctrl->rankmap[channel] >> (2 * slot)) & 3) {
-		case 0:
-		default:
-			slot320c[slot] = 0x7f;
-			break;
-		case 1:
-			slot320c[slot] =
-			    ctrl->timings[channel][2 * slot + 0].val_320c +
-			    full_shift;
-			break;
-		case 2:
-			slot320c[slot] =
-			    ctrl->timings[channel][2 * slot + 1].val_320c +
-			    full_shift;
-			break;
-		case 3:
-			slot320c[slot] =
-			    (ctrl->timings[channel][2 * slot].val_320c +
-			     ctrl->timings[channel][2 * slot +
-						    1].val_320c) / 2 +
-			    full_shift;
-			break;
-		}
-
-	reg32 = (1 << 17) | (1 << 14);
-	reg32 |= ((slot320c[0] & 0x3f) << 6) | ((slot320c[0] & 0x40) << 9);
-	reg32 |= (slot320c[1] & 0x7f) << 18;
-	reg32 |= (full_shift & 0x3f) | ((full_shift & 0x40) << 6);
-
-	MCHBAR32(0x320c + 0x100 * channel) = reg32;
-
-	reg_c14 = ctrl->rankmap[channel] << 24;
-	reg_c18 = 0;
-
-	FOR_ALL_POPULATED_RANKS {
-		int shift =
-		    ctrl->timings[channel][slotrank].val_320c + full_shift;
-		int offset_val_c14;
-		if (shift < 0)
-			shift = 0;
-		offset_val_c14 = ctrl->reg_c14_offset + shift;
-		reg_c14 |= (offset_val_c14 & 0x3f) << (6 * slotrank);
-		reg_c18 |= ((offset_val_c14 >> 6) & 1) << slotrank;
-	}
-
-	MCHBAR32(0xc14 + channel * 0x100) = reg_c14;
-	MCHBAR32(0xc18 + channel * 0x100) = reg_c18;
-
-	reg_4028 = MCHBAR32(0x4028 + 0x400 * channel);
-	reg_4028 &= 0xffff0000;
-
-	reg_4024 = 0;
-
-	FOR_ALL_POPULATED_RANKS {
-		int post_timA_min_high = 7, post_timA_max_high = 0;
-		int pre_timA_min_high = 7, pre_timA_max_high = 0;
-		int shift_402x = 0;
-		int shift =
-		    ctrl->timings[channel][slotrank].val_320c + full_shift;
-
-		if (shift < 0)
-			shift = 0;
-
-		FOR_ALL_LANES {
-			if (post_timA_min_high >
-			    ((ctrl->timings[channel][slotrank].lanes[lane].
-			      timA + shift) >> 6))
-				post_timA_min_high =
-				    ((ctrl->timings[channel][slotrank].
-				      lanes[lane].timA + shift) >> 6);
-			if (pre_timA_min_high >
-			    (ctrl->timings[channel][slotrank].lanes[lane].
-			     timA >> 6))
-				pre_timA_min_high =
-				    (ctrl->timings[channel][slotrank].
-				     lanes[lane].timA >> 6);
-			if (post_timA_max_high <
-			    ((ctrl->timings[channel][slotrank].lanes[lane].
-			      timA + shift) >> 6))
-				post_timA_max_high =
-				    ((ctrl->timings[channel][slotrank].
-				      lanes[lane].timA + shift) >> 6);
-			if (pre_timA_max_high <
-			    (ctrl->timings[channel][slotrank].lanes[lane].
-			     timA >> 6))
-				pre_timA_max_high =
-				    (ctrl->timings[channel][slotrank].
-				     lanes[lane].timA >> 6);
-		}
-
-		if (pre_timA_max_high - pre_timA_min_high <
-		    post_timA_max_high - post_timA_min_high)
-			shift_402x = +1;
-		else if (pre_timA_max_high - pre_timA_min_high >
-			 post_timA_max_high - post_timA_min_high)
-			shift_402x = -1;
-
-		reg_4028 |=
-		    (ctrl->timings[channel][slotrank].val_4028 + shift_402x -
-		     post_timA_min_high) << (4 * slotrank);
-		reg_4024 |=
-		    (ctrl->timings[channel][slotrank].val_4024 +
-		     shift_402x) << (8 * slotrank);
-
-		FOR_ALL_LANES {
-			MCHBAR32(lane_registers[lane] + 0x10 + 0x100 * channel +
-				 4 * slotrank)
-			    =
-			    (((ctrl->timings[channel][slotrank].lanes[lane].
-			       timA + shift) & 0x3f)
-			     |
-			     ((ctrl->timings[channel][slotrank].lanes[lane].
-			       rising + shift) << 8)
-			     |
-			     (((ctrl->timings[channel][slotrank].lanes[lane].
-				timA + shift -
-				(post_timA_min_high << 6)) & 0x1c0) << 10)
-			     | (ctrl->timings[channel][slotrank].lanes[lane].
-				falling << 20));
-
-			MCHBAR32(lane_registers[lane] + 0x20 + 0x100 * channel +
-				 4 * slotrank)
-			    =
-			    (((ctrl->timings[channel][slotrank].lanes[lane].
-			       timC + shift) & 0x3f)
-			     |
-			     (((ctrl->timings[channel][slotrank].lanes[lane].
-				timB + shift) & 0x3f) << 8)
-			     |
-			     (((ctrl->timings[channel][slotrank].lanes[lane].
-				timB + shift) & 0x1c0) << 9)
-			     |
-			     (((ctrl->timings[channel][slotrank].lanes[lane].
-				timC + shift) & 0x40) << 13));
-		}
-	}
-	MCHBAR32(0x4024 + 0x400 * channel) = reg_4024;
-	MCHBAR32(0x4028 + 0x400 * channel) = reg_4028;
-}
-
-static void test_timA(ramctr_timing * ctrl, int channel, int slotrank)
-{
-	wait_428c(channel);
-
-	write32(DEFAULT_MCHBAR + 0x4220 + 0x400 * channel, 0x1f000);
-	write32(DEFAULT_MCHBAR + 0x4230 + 0x400 * channel,
-		(0xc01 | (ctrl->tMOD << 16)));
-	write32(DEFAULT_MCHBAR + 0x4200 + 0x400 * channel,
-		(slotrank << 24) | 0x360004);
-	write32(DEFAULT_MCHBAR + 0x4210 + 0x400 * channel, 0);
-
-	write32(DEFAULT_MCHBAR + 0x4224 + 0x400 * channel, 0x1f105);
-	write32(DEFAULT_MCHBAR + 0x4234 + 0x400 * channel, 0x4040c01);
-	write32(DEFAULT_MCHBAR + 0x4204 + 0x400 * channel, (slotrank << 24));
-	write32(DEFAULT_MCHBAR + 0x4214 + 0x400 * channel, 0);
-
-	write32(DEFAULT_MCHBAR + 0x4228 + 0x400 * channel, 0x1f105);
-	write32(DEFAULT_MCHBAR + 0x4238 + 0x400 * channel,
-		0x100f | ((ctrl->CAS + 36) << 16));
-	write32(DEFAULT_MCHBAR + 0x4208 + 0x400 * channel,
-		(slotrank << 24) | 0x60000);
-	write32(DEFAULT_MCHBAR + 0x4218 + 0x400 * channel, 0);
-
-	write32(DEFAULT_MCHBAR + 0x422c + 0x400 * channel, 0x1f000);
-	write32(DEFAULT_MCHBAR + 0x423c + 0x400 * channel,
-		(0xc01 | (ctrl->tMOD << 16)));
-	write32(DEFAULT_MCHBAR + 0x420c + 0x400 * channel,
-		(slotrank << 24) | 0x360000);
-	write32(DEFAULT_MCHBAR + 0x421c + 0x400 * channel, 0);
-
-	write32(DEFAULT_MCHBAR + 0x4284 + 0x400 * channel, 0xc0001);
-
-	wait_428c(channel);
-}
-
-static int does_lane_work(ramctr_timing * ctrl, int channel, int slotrank,
-			  int lane)
-{
-	u32 timA = ctrl->timings[channel][slotrank].lanes[lane].timA;
-	return ((read32
-		 (DEFAULT_MCHBAR + lane_registers[lane] + channel * 0x100 + 4 +
-		  ((timA / 32) & 1) * 4)
-		 >> (timA % 32)) & 1);
-}
-
-struct run {
-	int middle;
-	int end;
-	int start;
-	int all;
-	int length;
-};
-
-static struct run get_longest_zero_run(int *seq, int sz)
-{
-	int i, ls;
-	int bl = 0, bs = 0;
-	struct run ret;
-
-	ls = 0;
-	for (i = 0; i < 2 * sz; i++)
-		if (seq[i % sz]) {
-			if (i - ls > bl) {
-				bl = i - ls;
-				bs = ls;
-			}
-			ls = i + 1;
-		}
-	if (bl == 0) {
-		ret.middle = sz / 2;
-		ret.start = 0;
-		ret.end = sz;
-		ret.all = 1;
-		return ret;
-	}
-
-	ret.start = bs % sz;
-	ret.end = (bs + bl - 1) % sz;
-	ret.middle = (bs + (bl - 1) / 2) % sz;
-	ret.length = bl;
-	ret.all = 0;
-
-	return ret;
-}
-
-static void discover_timA_coarse(ramctr_timing * ctrl, int channel,
-				 int slotrank, int *upperA)
-{
-	int timA;
-	int statistics[NUM_LANES][128];
-	int lane;
-
-	for (timA = 0; timA < 128; timA++) {
-		FOR_ALL_LANES {
-			ctrl->timings[channel][slotrank].lanes[lane].timA = timA;
-		}
-		program_timings(ctrl, channel);
-
-		test_timA(ctrl, channel, slotrank);
-
-		FOR_ALL_LANES {
-			statistics[lane][timA] =
-			    !does_lane_work(ctrl, channel, slotrank, lane);
-			printram("Astat: %d, %d, %d, %x, %x\n",
-			       channel, slotrank, lane, timA,
-			       statistics[lane][timA]);
-		}
-	}
-	FOR_ALL_LANES {
-		struct run rn = get_longest_zero_run(statistics[lane], 128);
-		ctrl->timings[channel][slotrank].lanes[lane].timA = rn.middle;
-		upperA[lane] = rn.end;
-		if (upperA[lane] < rn.middle)
-			upperA[lane] += 128;
-		printram("Aval: %d, %d, %d, %x\n", channel, slotrank,
-		       lane, ctrl->timings[channel][slotrank].lanes[lane].timA);
-		printram("Aend: %d, %d, %d, %x\n", channel, slotrank,
-		       lane, upperA[lane]);
-	}
-}
-
-static void discover_timA_fine(ramctr_timing * ctrl, int channel, int slotrank,
-			       int *upperA)
-{
-	int timA_delta;
-	int statistics[NUM_LANES][51];
-	int lane, i;
-
-	memset(statistics, 0, sizeof(statistics));
-
-	for (timA_delta = -25; timA_delta <= 25; timA_delta++) {
-		FOR_ALL_LANES ctrl->timings[channel][slotrank].lanes[lane].
-		    timA = upperA[lane] + timA_delta + 0x40;
-		program_timings(ctrl, channel);
-
-		for (i = 0; i < 100; i++) {
-			test_timA(ctrl, channel, slotrank);
-			FOR_ALL_LANES {
-				statistics[lane][timA_delta + 25] +=
-				    does_lane_work(ctrl, channel, slotrank,
-						   lane);
-			}
-		}
-	}
-	FOR_ALL_LANES {
-		int last_zero, first_all;
-
-		for (last_zero = -25; last_zero <= 25; last_zero++)
-			if (statistics[lane][last_zero + 25])
-				break;
-		last_zero--;
-		for (first_all = -25; first_all <= 25; first_all++)
-			if (statistics[lane][first_all + 25] == 100)
-				break;
-
-		printram("lane %d: %d, %d\n", lane, last_zero,
-		       first_all);
-
-		ctrl->timings[channel][slotrank].lanes[lane].timA =
-		    (last_zero + first_all) / 2 + upperA[lane];
-		printram("Aval: %d, %d, %d, %x\n", channel, slotrank,
-		       lane, ctrl->timings[channel][slotrank].lanes[lane].timA);
-	}
-}
-
-static void discover_402x(ramctr_timing * ctrl, int channel, int slotrank,
-			  int *upperA)
-{
-	int works[NUM_LANES];
-	int lane;
-	while (1) {
-		int all_works = 1, some_works = 0;
-		program_timings(ctrl, channel);
-		test_timA(ctrl, channel, slotrank);
-		FOR_ALL_LANES {
-			works[lane] =
-			    !does_lane_work(ctrl, channel, slotrank, lane);
-			if (works[lane])
-				some_works = 1;
-			else
-				all_works = 0;
-		}
-		if (all_works)
-			return;
-		if (!some_works) {
-			if (ctrl->timings[channel][slotrank].val_4024 < 2)
-				die("402x discovery failed");
-			ctrl->timings[channel][slotrank].val_4024 -= 2;
-			printram("4024 -= 2;\n");
-			continue;
-		}
-		ctrl->timings[channel][slotrank].val_4028 += 2;
-		printram("4028 += 2;\n");
-		if (ctrl->timings[channel][slotrank].val_4028 >= 0x10)
-			die("402x discovery failed");
-		FOR_ALL_LANES if (works[lane]) {
-			ctrl->timings[channel][slotrank].lanes[lane].timA +=
-			    128;
-			upperA[lane] += 128;
-			printram("increment %d, %d, %d\n", channel,
-			       slotrank, lane);
-		}
-	}
-}
-
-struct timA_minmax {
-	int timA_min_high, timA_max_high;
-};
-
-static void pre_timA_change(ramctr_timing * ctrl, int channel, int slotrank,
-			    struct timA_minmax *mnmx)
-{
-	int lane;
-	mnmx->timA_min_high = 7;
-	mnmx->timA_max_high = 0;
-
-	FOR_ALL_LANES {
-		if (mnmx->timA_min_high >
-		    (ctrl->timings[channel][slotrank].lanes[lane].timA >> 6))
-			mnmx->timA_min_high =
-			    (ctrl->timings[channel][slotrank].lanes[lane].
-			     timA >> 6);
-		if (mnmx->timA_max_high <
-		    (ctrl->timings[channel][slotrank].lanes[lane].timA >> 6))
-			mnmx->timA_max_high =
-			    (ctrl->timings[channel][slotrank].lanes[lane].
-			     timA >> 6);
-	}
-}
-
-static void post_timA_change(ramctr_timing * ctrl, int channel, int slotrank,
-			     struct timA_minmax *mnmx)
-{
-	struct timA_minmax post;
-	int shift_402x = 0;
-
-	/* Get changed maxima.  */
-	pre_timA_change(ctrl, channel, slotrank, &post);
-
-	if (mnmx->timA_max_high - mnmx->timA_min_high <
-	    post.timA_max_high - post.timA_min_high)
-		shift_402x = +1;
-	else if (mnmx->timA_max_high - mnmx->timA_min_high >
-		 post.timA_max_high - post.timA_min_high)
-		shift_402x = -1;
-	else
-		shift_402x = 0;
-
-	ctrl->timings[channel][slotrank].val_4028 += shift_402x;
-	ctrl->timings[channel][slotrank].val_4024 += shift_402x;
-	printram("4024 += %d;\n", shift_402x);
-	printram("4028 += %d;\n", shift_402x);
-}
-
-static void read_training(ramctr_timing * ctrl)
-{
-	int channel, slotrank, lane;
-
-	FOR_ALL_CHANNELS FOR_ALL_POPULATED_RANKS {
-		u32 r32;
-		int all_high, some_high;
-		int upperA[NUM_LANES];
-		struct timA_minmax mnmx;
-
-		 wait_428c(channel);
-		 write32(DEFAULT_MCHBAR + 0x4220 + 0x400 * channel, 0x1f002);
-		 write32(DEFAULT_MCHBAR + 0x4230 + 0x400 * channel,
-			 0xc01 | (ctrl->tRP << 16));
-		 write32(DEFAULT_MCHBAR + 0x4200 + 0x400 * channel,
-			 (slotrank << 24) | 0x60400);
-		 write32(DEFAULT_MCHBAR + 0x4210 + 0x400 * channel, 0);
-		 write32(DEFAULT_MCHBAR + 0x4284 + 0x400 * channel, 1);
-
-		 write32(DEFAULT_MCHBAR + 0x3400, (slotrank << 2) | 0x8001);
-
-		 ctrl->timings[channel][slotrank].val_4028 = 4;
-		 ctrl->timings[channel][slotrank].val_4024 = 55;
-		 program_timings(ctrl, channel);
-
-		 discover_timA_coarse(ctrl, channel, slotrank, upperA);
-
-		 all_high = 1;
-		 some_high = 0;
-		 FOR_ALL_LANES {
-			 if (ctrl->timings[channel][slotrank].lanes[lane].
-			     timA >= 0x40)
-				 some_high = 1;
-			 else
-				 all_high = 0;
-		 }
-
-		if (all_high) {
-			ctrl->timings[channel][slotrank].val_4028--;
-			printram("4028--;\n");
-			FOR_ALL_LANES {
-				ctrl->timings[channel][slotrank].lanes[lane].
-				    timA -= 0x40;
-				upperA[lane] -= 0x40;
-
-			}
-		} else if (some_high) {
-			ctrl->timings[channel][slotrank].val_4024++;
-			ctrl->timings[channel][slotrank].val_4028++;
-			printram("4024++;\n");
-			printram("4028++;\n");
-		}
-
-		program_timings(ctrl, channel);
-
-		pre_timA_change(ctrl, channel, slotrank, &mnmx);
-
-		discover_402x(ctrl, channel, slotrank, upperA);
-
-		post_timA_change(ctrl, channel, slotrank, &mnmx);
-		pre_timA_change(ctrl, channel, slotrank, &mnmx);
-
-		discover_timA_fine(ctrl, channel, slotrank, upperA);
-
-		post_timA_change(ctrl, channel, slotrank, &mnmx);
-		pre_timA_change(ctrl, channel, slotrank, &mnmx);
-
-		FOR_ALL_LANES {
-			ctrl->timings[channel][slotrank].lanes[lane].timA -= mnmx.timA_min_high * 0x40;
-		}
-		ctrl->timings[channel][slotrank].val_4028 -= mnmx.timA_min_high;
-		printram("4028 -= %d;\n", mnmx.timA_min_high);
-
-		post_timA_change(ctrl, channel, slotrank, &mnmx);
-
-		printram("4/8: %d, %d, %x, %x\n", channel, slotrank,
-		       ctrl->timings[channel][slotrank].val_4024,
-		       ctrl->timings[channel][slotrank].val_4028);
-
-		FOR_ALL_LANES
-		    printram("%d, %d, %d, %x\n", channel, slotrank,
-			   lane,
-			   ctrl->timings[channel][slotrank].lanes[lane].timA);
-
-		write32(DEFAULT_MCHBAR + 0x3400, 0);
-
-		r32 = read32(DEFAULT_MCHBAR + 0x5030);
-		write32(DEFAULT_MCHBAR + 0x5030, r32 | 0x20);
-		udelay(1);
-
-		write32(DEFAULT_MCHBAR + 0x5030, r32 & ~0x20);
-
-		udelay(1);
-	}
-
-	FOR_ALL_POPULATED_CHANNELS {
-		program_timings(ctrl, channel);
-	}
-	FOR_ALL_CHANNELS FOR_ALL_POPULATED_RANKS FOR_ALL_LANES {
-		write32(DEFAULT_MCHBAR + 0x4080 + 0x400 * channel
-			+ 4 * lane, 0);
-	}
-}
-
-static void test_timC(ramctr_timing * ctrl, int channel, int slotrank)
-{
-	int lane;
-
-	FOR_ALL_LANES {
-		write32(DEFAULT_MCHBAR + 0x4340 + 0x400 * channel + 4 * lane, 0);
-		read32(DEFAULT_MCHBAR + 0x4140 + 0x400 * channel + 4 * lane);
-	}
-
-	wait_428c(channel);
-
-	write32(DEFAULT_MCHBAR + 0x4220 + 0x400 * channel, 0x1f006);
-	write32(DEFAULT_MCHBAR + 0x4230 + 0x400 * channel,
-		(max((ctrl->tFAW >> 2) + 1, ctrl->tRRD) << 10)
-		| 4 | (ctrl->tRCD << 16));
-
-	write32(DEFAULT_MCHBAR + 0x4200 + 0x400 * channel,
-		(slotrank << 24) | (6 << 16));
-
-	write32(DEFAULT_MCHBAR + 0x4210 + 0x400 * channel, 0x244);
-
-	write32(DEFAULT_MCHBAR + 0x4224 + 0x400 * channel, 0x1f207);
-	write32(DEFAULT_MCHBAR + 0x4234 + 0x400 * channel, 0x8041001);
-	write32(DEFAULT_MCHBAR + 0x4204 + 0x400 * channel,
-		(slotrank << 24) | 8);
-	write32(DEFAULT_MCHBAR + 0x4214 + 0x400 * channel, 0x3e0);
-
-	write32(DEFAULT_MCHBAR + 0x4228 + 0x400 * channel, 0x1f201);
-	write32(DEFAULT_MCHBAR + 0x4238 + 0x400 * channel, 0x80411f4);
-	write32(DEFAULT_MCHBAR + 0x4208 + 0x400 * channel, (slotrank << 24));
-	write32(DEFAULT_MCHBAR + 0x4218 + 0x400 * channel, 0x242);
-
-	write32(DEFAULT_MCHBAR + 0x422c + 0x400 * channel, 0x1f207);
-	write32(DEFAULT_MCHBAR + 0x423c + 0x400 * channel,
-		0x8000c01 | ((ctrl->CWL + ctrl->tWTR + 5) << 16));
-	write32(DEFAULT_MCHBAR + 0x420c + 0x400 * channel,
-		(slotrank << 24) | 8);
-	write32(DEFAULT_MCHBAR + 0x421c + 0x400 * channel, 0x3e0);
-
-	write32(DEFAULT_MCHBAR + 0x4284 + 0x400 * channel, 0xc0001);
-
-	wait_428c(channel);
-
-	write32(DEFAULT_MCHBAR + 0x4220 + 0x400 * channel, 0x1f002);
-	write32(DEFAULT_MCHBAR + 0x4230 + 0x400 * channel,
-		0xc01 | (ctrl->tRP << 16));
-	write32(DEFAULT_MCHBAR + 0x4200 + 0x400 * channel,
-		(slotrank << 24) | 0x60400);
-	write32(DEFAULT_MCHBAR + 0x4210 + 0x400 * channel, 0x240);
-
-	write32(DEFAULT_MCHBAR + 0x4224 + 0x400 * channel, 0x1f006);
-	write32(DEFAULT_MCHBAR + 0x4234 + 0x400 * channel,
-		(max(ctrl->tRRD, (ctrl->tFAW >> 2) + 1) << 10)
-		| 8 | (ctrl->CAS << 16));
-
-	write32(DEFAULT_MCHBAR + 0x4204 + 0x400 * channel,
-		(slotrank << 24) | 0x60000);
-
-	write32(DEFAULT_MCHBAR + 0x4214 + 0x400 * channel, 0x244);
-
-	write32(DEFAULT_MCHBAR + 0x4228 + 0x400 * channel, 0x1f105);
-	write32(DEFAULT_MCHBAR + 0x4238 + 0x400 * channel,
-		0x40011f4 | (max(ctrl->tRTP, 8) << 16));
-	write32(DEFAULT_MCHBAR + 0x4208 + 0x400 * channel, (slotrank << 24));
-	write32(DEFAULT_MCHBAR + 0x4218 + 0x400 * channel, 0x242);
-
-	write32(DEFAULT_MCHBAR + 0x422c + 0x400 * channel, 0x1f002);
-	write32(DEFAULT_MCHBAR + 0x423c + 0x400 * channel,
-		0xc01 | (ctrl->tRP << 16));
-	write32(DEFAULT_MCHBAR + 0x420c + 0x400 * channel,
-		(slotrank << 24) | 0x60400);
-	write32(DEFAULT_MCHBAR + 0x421c + 0x400 * channel, 0x240);
-	write32(DEFAULT_MCHBAR + 0x4284 + 0x400 * channel, 0xc0001);
-	wait_428c(channel);
-}
-
-static void discover_timC(ramctr_timing * ctrl, int channel, int slotrank)
-{
-	int timC;
-	int statistics[NUM_LANES][MAX_TIMC + 1];
-	int lane;
-
-	wait_428c(channel);
-
-	write32(DEFAULT_MCHBAR + 0x4220 + 0x400 * channel, 0x1f002);
-	write32(DEFAULT_MCHBAR + 0x4230 + 0x400 * channel,
-		0xc01 | (ctrl->tRP << 16));
-	write32(DEFAULT_MCHBAR + 0x4200 + 0x400 * channel,
-		(slotrank << 24) | 0x60400);
-	write32(DEFAULT_MCHBAR + 0x4210 + 0x400 * channel, 0x240);
-	write32(DEFAULT_MCHBAR + 0x4284 + 0x400 * channel, 1);
-
-	for (timC = 0; timC <= MAX_TIMC; timC++) {
-		FOR_ALL_LANES ctrl->timings[channel][slotrank].lanes[lane].
-		    timC = timC;
-		program_timings(ctrl, channel);
-
-		test_timC(ctrl, channel, slotrank);
-
-		FOR_ALL_LANES {
-			statistics[lane][timC] =
-			    read32(DEFAULT_MCHBAR + 0x4340 + 4 * lane +
-				   0x400 * channel);
-			printram("Cstat: %d, %d, %d, %x, %x\n",
-			       channel, slotrank, lane, timC,
-			       statistics[lane][timC]);
-		}
-	}
-	FOR_ALL_LANES {
-		struct run rn =
-		    get_longest_zero_run(statistics[lane], MAX_TIMC + 1);
-		ctrl->timings[channel][slotrank].lanes[lane].timC = rn.middle;
-		if (rn.all)
-			printk(BIOS_CRIT, "timC discovery failed");
-		printram("Cval: %d, %d, %d, %x\n", channel, slotrank,
-		       lane, ctrl->timings[channel][slotrank].lanes[lane].timC);
-	}
-}
-
-static int get_precedening_channels(ramctr_timing * ctrl, int target_channel)
-{
-	int channel, ret = 0;
-	FOR_ALL_POPULATED_CHANNELS if (channel < target_channel)
-		 ret++;
-	return ret;
-}
-
-static void fill_pattern0(ramctr_timing * ctrl, int channel, u32 a, u32 b)
-{
-	unsigned j;
-	unsigned channel_offset =
-	    get_precedening_channels(ctrl, channel) * 0x40;
-	printram("channel_offset=%x\n", channel_offset);
-	for (j = 0; j < 16; j++)
-		write32((void *)(0x04000000 + channel_offset + 4 * j), j & 2 ? b : a);
-	sfence();
-}
-
-static int num_of_channels(const ramctr_timing * ctrl)
-{
-	int ret = 0;
-	int channel;
-	FOR_ALL_POPULATED_CHANNELS ret++;
-	return ret;
-}
-
-static void fill_pattern1(ramctr_timing * ctrl, int channel)
-{
-	unsigned j;
-	unsigned channel_offset =
-	    get_precedening_channels(ctrl, channel) * 0x40;
-	unsigned channel_step = 0x40 * num_of_channels(ctrl);
-	for (j = 0; j < 16; j++)
-		write32((void *)(0x04000000 + channel_offset + j * 4), 0xffffffff);
-	for (j = 0; j < 16; j++)
-		write32((void *)(0x04000000 + channel_offset + channel_step + j * 4), 0);
-	sfence();
-}
-
-static void precharge(ramctr_timing * ctrl)
-{
-	int channel, slotrank, lane;
-
-	FOR_ALL_POPULATED_CHANNELS {
-		FOR_ALL_POPULATED_RANKS FOR_ALL_LANES {
-			ctrl->timings[channel][slotrank].lanes[lane].falling =
-			    16;
-			ctrl->timings[channel][slotrank].lanes[lane].rising =
-			    16;
-		} program_timings(ctrl, channel);
-
-		FOR_ALL_POPULATED_RANKS {
-			wait_428c(channel);
-
-			write32(DEFAULT_MCHBAR + 0x4220 + 0x400 * channel,
-				0x1f000);
-			write32(DEFAULT_MCHBAR + 0x4230 + 0x400 * channel,
-				0xc01 | (ctrl->tMOD << 16));
-			write32(DEFAULT_MCHBAR + 0x4200 + 0x400 * channel,
-				(slotrank << 24) | 0x360004);
-			write32(DEFAULT_MCHBAR + 0x4210 + 0x400 * channel, 0);
-
-			write32(DEFAULT_MCHBAR + 0x4224 + 0x400 * channel,
-				0x1f105);
-			write32(DEFAULT_MCHBAR + 0x4234 + 0x400 * channel,
-				0x4041003);
-			write32(DEFAULT_MCHBAR + 0x4204 + 0x400 * channel,
-				(slotrank << 24) | 0);
-			write32(DEFAULT_MCHBAR + 0x4214 + 0x400 * channel, 0);
-
-			write32(DEFAULT_MCHBAR + 0x4228 + 0x400 * channel,
-				0x1f105);
-			write32(DEFAULT_MCHBAR + 0x4238 + 0x400 * channel,
-				0x1001 | ((ctrl->CAS + 8) << 16));
-			write32(DEFAULT_MCHBAR + 0x4208 + 0x400 * channel,
-				(slotrank << 24) | 0x60000);
-			write32(DEFAULT_MCHBAR + 0x4218 + 0x400 * channel, 0);
-
-			write32(DEFAULT_MCHBAR + 0x422c + 0x400 * channel,
-				0x1f000);
-			write32(DEFAULT_MCHBAR + 0x423c + 0x400 * channel,
-				0xc01 | (ctrl->tMOD << 16));
-			write32(DEFAULT_MCHBAR + 0x420c + 0x400 * channel,
-				(slotrank << 24) | 0x360000);
-			write32(DEFAULT_MCHBAR + 0x421c + 0x400 * channel, 0);
-			write32(DEFAULT_MCHBAR + 0x4284 + 0x400 * channel,
-				0xc0001);
-
-			wait_428c(channel);
-		}
-
-		FOR_ALL_POPULATED_RANKS FOR_ALL_LANES {
-			ctrl->timings[channel][slotrank].lanes[lane].falling =
-			    48;
-			ctrl->timings[channel][slotrank].lanes[lane].rising =
-			    48;
-		}
-
-		program_timings(ctrl, channel);
-
-		FOR_ALL_POPULATED_RANKS {
-			wait_428c(channel);
-
-			write32(DEFAULT_MCHBAR + 0x4220 + 0x400 * channel,
-				0x1f000);
-			write32(DEFAULT_MCHBAR + 0x4230 + 0x400 * channel,
-				0xc01 | (ctrl->tMOD << 16));
-			write32(DEFAULT_MCHBAR + 0x4200 + 0x400 * channel,
-				(slotrank << 24) | 0x360004);
-			write32(DEFAULT_MCHBAR + 0x4210 + 0x400 * channel, 0);
-
-			write32(DEFAULT_MCHBAR + 0x4224 + 0x400 * channel,
-				0x1f105);
-			write32(DEFAULT_MCHBAR + 0x4234 + 0x400 * channel,
-				0x4041003);
-			write32(DEFAULT_MCHBAR + 0x4204 + 0x400 * channel,
-				(slotrank << 24) | 0);
-			write32(DEFAULT_MCHBAR + 0x4214 + 0x400 * channel, 0);
-
-			write32(DEFAULT_MCHBAR + 0x4228 + 0x400 * channel,
-				0x1f105);
-			write32(DEFAULT_MCHBAR + 0x4238 + 0x400 * channel,
-				0x1001 | ((ctrl->CAS + 8) << 16));
-			write32(DEFAULT_MCHBAR + 0x4208 + 0x400 * channel,
-				(slotrank << 24) | 0x60000);
-			write32(DEFAULT_MCHBAR + 0x4218 + 0x400 * channel, 0);
-
-			write32(DEFAULT_MCHBAR + 0x422c + 0x400 * channel,
-				0x1f000);
-			write32(DEFAULT_MCHBAR + 0x423c + 0x400 * channel,
-				0xc01 | (ctrl->tMOD << 16));
-
-			write32(DEFAULT_MCHBAR + 0x420c + 0x400 * channel,
-				(slotrank << 24) | 0x360000);
-			write32(DEFAULT_MCHBAR + 0x421c + 0x400 * channel, 0);
-
-			write32(DEFAULT_MCHBAR + 0x4284 + 0x400 * channel,
-				0xc0001);
-			wait_428c(channel);
-		}
-	}
-}
-
-static void test_timB(ramctr_timing * ctrl, int channel, int slotrank)
-{
-	write_mrreg(ctrl, channel, slotrank, 1,
-		    0x80 | make_mr1(ctrl, slotrank));
-
-	wait_428c(channel);
-	write32(DEFAULT_MCHBAR + 0x4220 + 0x400 * channel, 0x1f207);
-	write32(DEFAULT_MCHBAR + 0x4230 + 0x400 * channel,
-		0x8000c01 | ((ctrl->CWL + ctrl->tWLO) << 16));
-	write32(DEFAULT_MCHBAR + 0x4200 + 0x400 * channel,
-		8 | (slotrank << 24));
-	write32(DEFAULT_MCHBAR + 0x4210 + 0x400 * channel, 0);
-
-	write32(DEFAULT_MCHBAR + 0x4224 + 0x400 * channel, 0x1f107);
-	write32(DEFAULT_MCHBAR + 0x4234 + 0x400 * channel,
-		0x4000c01 | ((ctrl->CAS + 38) << 16));
-	write32(DEFAULT_MCHBAR + 0x4204 + 0x400 * channel,
-		(slotrank << 24) | 4);
-	write32(DEFAULT_MCHBAR + 0x4214 + 0x400 * channel, 0);
-
-	write32(DEFAULT_MCHBAR + 0x400 * channel + 0x4284, 0x40001);
-	wait_428c(channel);
-
-	write_mrreg(ctrl, channel, slotrank, 1,
-		    0x1080 | make_mr1(ctrl, slotrank));
-}
-
-static void discover_timB(ramctr_timing * ctrl, int channel, int slotrank)
-{
-	int timB;
-	int statistics[NUM_LANES][128];
-	int lane;
-
-	write32(DEFAULT_MCHBAR + 0x3400, 0x108052 | (slotrank << 2));
-
-	for (timB = 0; timB < 128; timB++) {
-		FOR_ALL_LANES {
-			ctrl->timings[channel][slotrank].lanes[lane].timB = timB;
-		}
-		program_timings(ctrl, channel);
-
-		test_timB(ctrl, channel, slotrank);
-
-		FOR_ALL_LANES {
-			statistics[lane][timB] =
-			    !((read32
-			       (DEFAULT_MCHBAR + lane_registers[lane] +
-				channel * 0x100 + 4 + ((timB / 32) & 1) * 4)
-			       >> (timB % 32)) & 1);
-			printram("Bstat: %d, %d, %d, %x, %x\n",
-			       channel, slotrank, lane, timB,
-			       statistics[lane][timB]);
-		}
-	}
-	FOR_ALL_LANES {
-		struct run rn = get_longest_zero_run(statistics[lane], 128);
-		ctrl->timings[channel][slotrank].lanes[lane].timB = rn.start;
-		if (rn.all)
-			die("timB discovery failed");
-		printram("Bval: %d, %d, %d, %x\n", channel, slotrank,
-		       lane, ctrl->timings[channel][slotrank].lanes[lane].timB);
-	}
-}
-
-static int get_timB_high_adjust(u64 val)
-{
-	int i;
-
-	/* good */
-	if (val == 0xffffffffffffffffLL)
-		return 0;
-
-	if (val >= 0xf000000000000000LL) {
-		/* needs negative adjustment */
-		for (i = 0; i < 8; i++)
-			if (val << (8 * (7 - i) + 4))
-				return -i;
-	} else {
-		/* needs positive adjustment */
-		for (i = 0; i < 8; i++)
-			if (val >> (8 * (7 - i) + 4))
-				return i;
-	}
-	return 8;
-}
-
-static void adjust_high_timB(ramctr_timing * ctrl)
-{
-	int channel, slotrank, lane, old;
-	write32(DEFAULT_MCHBAR + 0x3400, 0x200);
-	FOR_ALL_POPULATED_CHANNELS {
-		fill_pattern1(ctrl, channel);
-		write32(DEFAULT_MCHBAR + 0x4288 + (channel << 10), 1);
-	}
-	FOR_ALL_POPULATED_CHANNELS FOR_ALL_POPULATED_RANKS {
-
-		write32(DEFAULT_MCHBAR + 0x4288 + 0x400 * channel, 0x10001);
-
-		wait_428c(channel);
-
-		write32(DEFAULT_MCHBAR + 0x4220 + 0x400 * channel, 0x1f006);
-		write32(DEFAULT_MCHBAR + 0x4230 + 0x400 * channel,
-			0xc01 | (ctrl->tRCD << 16));
-		write32(DEFAULT_MCHBAR + 0x4200 + 0x400 * channel,
-			(slotrank << 24) | 0x60000);
-		write32(DEFAULT_MCHBAR + 0x4210 + 0x400 * channel, 0);
-
-		write32(DEFAULT_MCHBAR + 0x4224 + 0x400 * channel, 0x1f207);
-		write32(DEFAULT_MCHBAR + 0x4234 + 0x400 * channel, 0x8040c01);
-		write32(DEFAULT_MCHBAR + 0x4204 + 0x400 * channel,
-			(slotrank << 24) | 0x8);
-		write32(DEFAULT_MCHBAR + 0x4214 + 0x400 * channel, 0x3e0);
-
-		write32(DEFAULT_MCHBAR + 0x4228 + 0x400 * channel, 0x1f201);
-		write32(DEFAULT_MCHBAR + 0x4238 + 0x400 * channel, 0x8041003);
-		write32(DEFAULT_MCHBAR + 0x4208 + 0x400 * channel,
-			(slotrank << 24));
-		write32(DEFAULT_MCHBAR + 0x4218 + 0x400 * channel, 0x3e2);
-
-		write32(DEFAULT_MCHBAR + 0x422c + 0x400 * channel, 0x1f207);
-		write32(DEFAULT_MCHBAR + 0x423c + 0x400 * channel,
-			0x8000c01 | ((ctrl->CWL + ctrl->tWTR + 5) << 16));
-		write32(DEFAULT_MCHBAR + 0x420c + 0x400 * channel,
-			(slotrank << 24) | 0x8);
-		write32(DEFAULT_MCHBAR + 0x421c + 0x400 * channel, 0x3e0);
-
-		write32(DEFAULT_MCHBAR + 0x4284 + 0x400 * channel, 0xc0001);
-
-		wait_428c(channel);
-
-		write32(DEFAULT_MCHBAR + 0x4220 + 0x400 * channel, 0x1f002);
-		write32(DEFAULT_MCHBAR + 0x4230 + 0x400 * channel,
-			0xc01 | ((ctrl->tRP) << 16));
-		write32(DEFAULT_MCHBAR + 0x4200 + 0x400 * channel,
-			(slotrank << 24) | 0x60400);
-		write32(DEFAULT_MCHBAR + 0x4210 + 0x400 * channel, 0x240);
-
-		write32(DEFAULT_MCHBAR + 0x4224 + 0x400 * channel, 0x1f006);
-		write32(DEFAULT_MCHBAR + 0x4234 + 0x400 * channel,
-			0xc01 | ((ctrl->tRCD) << 16));
-		write32(DEFAULT_MCHBAR + 0x4204 + 0x400 * channel,
-			(slotrank << 24) | 0x60000);
-		write32(DEFAULT_MCHBAR + 0x4214 + 0x400 * channel, 0);
-
-		write32(DEFAULT_MCHBAR + 0x4228 + 0x400 * channel, 0x3f105);
-		write32(DEFAULT_MCHBAR + 0x4238 + 0x400 * channel,
-			0x4000c01 |
-			((ctrl->tRP +
-			  ctrl->timings[channel][slotrank].val_4024 +
-			  ctrl->timings[channel][slotrank].val_4028) << 16));
-		write32(DEFAULT_MCHBAR + 0x4208 + 0x400 * channel,
-			(slotrank << 24) | 0x60008);
-		write32(DEFAULT_MCHBAR + 0x4218 + 0x400 * channel, 0);
-
-		write32(DEFAULT_MCHBAR + 0x4284 + 0x400 * channel, 0x80001);
-		wait_428c(channel);
-		FOR_ALL_LANES {
-			u64 res =
-				read32(DEFAULT_MCHBAR + lane_registers[lane] +
-					0x100 * channel + 4);
-			res |=
-				((u64) read32(DEFAULT_MCHBAR + lane_registers[lane] +
-					0x100 * channel + 8)) << 32;
-			old = ctrl->timings[channel][slotrank].lanes[lane].timB;
-			ctrl->timings[channel][slotrank].lanes[lane].timB +=
-				get_timB_high_adjust(res) * 64;
-
-			printk(BIOS_DEBUG, "High adjust %d:%016llx\n", lane, res);
-			printram("Bval+: %d, %d, %d, %x -> %x\n", channel,
-				slotrank, lane, old,
-				ctrl->timings[channel][slotrank].lanes[lane].
-				timB);
-		}
-	}
-	write32(DEFAULT_MCHBAR + 0x3400, 0);
-}
-
-static void write_op(ramctr_timing * ctrl, int channel)
-{
-	int slotrank;
-
-	wait_428c(channel);
-
-	/* choose an existing rank.  */
-	slotrank = !(ctrl->rankmap[channel] & 1) ? 2 : 0;
-
-	write32(DEFAULT_MCHBAR + 0x4220 + 0x400 * channel, 0x0f003);
-	write32(DEFAULT_MCHBAR + 0x4230 + 0x400 * channel, 0x41001);
-
-	write32(DEFAULT_MCHBAR + 0x4200 + 0x400 * channel,
-		(slotrank << 24) | 0x60000);
-
-	write32(DEFAULT_MCHBAR + 0x4210 + 0x400 * channel, 0x3e0);
-
-	write32(DEFAULT_MCHBAR + 0x4284 + 0x400 * channel, 1);
-	wait_428c(channel);
-}
-
-static void write_training(ramctr_timing * ctrl)
-{
-	int channel, slotrank, lane;
-	u32 r32;
-
-	FOR_ALL_POPULATED_CHANNELS
-	    write32(DEFAULT_MCHBAR + 0x4008 + 0x400 * channel,
-		    read32(DEFAULT_MCHBAR + 0x4008 +
-			   0x400 * channel) | 0x8000000);
-
-	FOR_ALL_POPULATED_CHANNELS {
-		write_op(ctrl, channel);
-		write32(DEFAULT_MCHBAR + 0x4020 + 0x400 * channel,
-			read32(DEFAULT_MCHBAR + 0x4020 +
-			       0x400 * channel) | 0x200000);
-	}
-	write32(DEFAULT_MCHBAR + 0x5030, read32(DEFAULT_MCHBAR + 0x5030) & ~8);
-	FOR_ALL_POPULATED_CHANNELS {
-		write_op(ctrl, channel);
-	}
-
-	FOR_ALL_CHANNELS
-	    FOR_ALL_POPULATED_RANKS
-		write_mrreg(ctrl, channel, slotrank, 1,
-			    make_mr1(ctrl, slotrank) | 0x1080);
-
-	write32(DEFAULT_MCHBAR + 0x3400, 0x108052);
-
-	r32 = read32(DEFAULT_MCHBAR + 0x5030);
-	write32(DEFAULT_MCHBAR + 0x5030, r32 | 0x20);
-	udelay(1);
-
-	write32(DEFAULT_MCHBAR + 0x5030, r32 & ~0x20);
-
-	udelay(1);
-
-	FOR_ALL_CHANNELS FOR_ALL_POPULATED_RANKS
-		discover_timB(ctrl, channel, slotrank);
-
-	FOR_ALL_CHANNELS FOR_ALL_POPULATED_RANKS
-		write_mrreg(ctrl, channel,
-			    slotrank, 1, make_mr1(ctrl, slotrank));
-
-	write32(DEFAULT_MCHBAR + 0x3400, 0);
-
-	FOR_ALL_POPULATED_CHANNELS
-		wait_428c(channel);
-
-	write32(DEFAULT_MCHBAR + 0x5030, read32(DEFAULT_MCHBAR + 0x5030) | 8);
-
-	FOR_ALL_POPULATED_CHANNELS {
-		write32(DEFAULT_MCHBAR + 0x4020 + 0x400 * channel,
-			~0x00200000 & read32(DEFAULT_MCHBAR + 0x4020 +
-					     0x400 * channel));
-		read32(DEFAULT_MCHBAR + 0x428c + 0x400 * channel);
-		wait_428c(channel);
-
-		write32(DEFAULT_MCHBAR + 0x4220 + 0x400 * channel, 0x0f003);
-		write32(DEFAULT_MCHBAR + 0x4230 + 0x400 * channel, 0x659001);
-		write32(DEFAULT_MCHBAR + 0x4200 + 0x400 * channel, 0x60000);
-		write32(DEFAULT_MCHBAR + 0x4210 + 0x400 * channel, 0x3e0);
-
-		write32(DEFAULT_MCHBAR + 0x4284 + 0x400 * channel, 1);
-		wait_428c(channel);
-	}
-
-	r32 = read32(DEFAULT_MCHBAR + 0x5030);
-	write32(DEFAULT_MCHBAR + 0x5030, r32 | 0x20);
-	udelay(1);
-
-	write32(DEFAULT_MCHBAR + 0x5030, r32 & ~0x20);
-
-	udelay(1);
-
-	printram("CPE\n");
-	precharge(ctrl);
-	printram("CPF\n");
-
-	FOR_ALL_CHANNELS FOR_ALL_POPULATED_RANKS FOR_ALL_LANES {
-		read32(DEFAULT_MCHBAR + 0x4080 + 0x400 * channel + 4 * lane);
-		write32(DEFAULT_MCHBAR + 0x4080 + 0x400 * channel + 4 * lane,
-			0);
-	}
-
-	FOR_ALL_POPULATED_CHANNELS {
-		fill_pattern0(ctrl, channel, 0xaaaaaaaa, 0x55555555);
-		write32(DEFAULT_MCHBAR + 0x4288 + (channel << 10), 0);
-	}
-
-	FOR_ALL_CHANNELS FOR_ALL_POPULATED_RANKS
-		discover_timC(ctrl, channel, slotrank);
-
-	FOR_ALL_POPULATED_CHANNELS
-		program_timings(ctrl, channel);
-
-	adjust_high_timB(ctrl);
-
-	FOR_ALL_POPULATED_CHANNELS
-		program_timings(ctrl, channel);
-
-	FOR_ALL_CHANNELS FOR_ALL_POPULATED_RANKS FOR_ALL_LANES {
-		read32(DEFAULT_MCHBAR + 0x4080 + 0x400 * channel + 4 * lane);
-		write32(DEFAULT_MCHBAR + 0x4080 + 0x400 * channel + 4 * lane,
-			0);
-	}
-}
-
-static int test_320c(ramctr_timing * ctrl, int channel, int slotrank)
-{
-	struct ram_rank_timings saved_rt = ctrl->timings[channel][slotrank];
-	int timC_delta;
-	int lanes_ok = 0;
-	int ctr = 0;
-	int lane;
-
-	for (timC_delta = -5; timC_delta <= 5; timC_delta++) {
-		FOR_ALL_LANES {
-			ctrl->timings[channel][slotrank].lanes[lane].timC =
-			    saved_rt.lanes[lane].timC + timC_delta;
-		}
-		program_timings(ctrl, channel);
-		FOR_ALL_LANES {
-			write32(DEFAULT_MCHBAR + 4 * lane + 0x4f40, 0);
-		}
-
-		write32(DEFAULT_MCHBAR + 0x4288 + 0x400 * channel, 0x1f);
-
-		wait_428c(channel);
-
-		write32(DEFAULT_MCHBAR + 0x4220 + 0x400 * channel, 0x1f006);
-		write32(DEFAULT_MCHBAR + 0x4230 + 0x400 * channel,
-			((max(ctrl->tRRD, (ctrl->tFAW >> 2) + 1)) << 10)
-			| 8 | (ctrl->tRCD << 16));
-
-		write32(DEFAULT_MCHBAR + 0x4200 + 0x400 * channel,
-			(slotrank << 24) | ctr | 0x60000);
-
-		write32(DEFAULT_MCHBAR + 0x4210 + 0x400 * channel, 0x244);
-
-		write32(DEFAULT_MCHBAR + 0x4224 + 0x400 * channel, 0x1f201);
-		write32(DEFAULT_MCHBAR + 0x4234 + 0x400 * channel,
-			0x8001020 | ((ctrl->CWL + ctrl->tWTR + 8) << 16));
-		write32(DEFAULT_MCHBAR + 0x4204 + 0x400 * channel,
-			(slotrank << 24));
-		write32(DEFAULT_MCHBAR + 0x4244 + 0x400 * channel, 0x389abcd);
-		write32(DEFAULT_MCHBAR + 0x4214 + 0x400 * channel, 0x20e42);
-
-		write32(DEFAULT_MCHBAR + 0x4228 + 0x400 * channel, 0x1f105);
-		write32(DEFAULT_MCHBAR + 0x4238 + 0x400 * channel,
-			0x4001020 | (max(ctrl->tRTP, 8) << 16));
-		write32(DEFAULT_MCHBAR + 0x4208 + 0x400 * channel,
-			(slotrank << 24));
-		write32(DEFAULT_MCHBAR + 0x4248 + 0x400 * channel, 0x389abcd);
-		write32(DEFAULT_MCHBAR + 0x4218 + 0x400 * channel, 0x20e42);
-
-		write32(DEFAULT_MCHBAR + 0x422c + 0x400 * channel, 0x1f002);
-		write32(DEFAULT_MCHBAR + 0x423c + 0x400 * channel, 0xf1001);
-		write32(DEFAULT_MCHBAR + 0x420c + 0x400 * channel,
-			(slotrank << 24) | 0x60400);
-		write32(DEFAULT_MCHBAR + 0x421c + 0x400 * channel, 0x240);
-
-		write32(DEFAULT_MCHBAR + 0x4284 + 0x400 * channel, 0xc0001);
-		wait_428c(channel);
-		FOR_ALL_LANES {
-			u32 r32 =
-			    read32(DEFAULT_MCHBAR + 0x4340 + 4 * lane +
-				   0x400 * channel);
-
-			if (r32 == 0)
-				lanes_ok |= 1 << lane;
-		}
-		ctr++;
-		if (lanes_ok == ((1 << NUM_LANES) - 1))
-			break;
-	}
-
-	ctrl->timings[channel][slotrank] = saved_rt;
-
-	printram("3lanes: %x\n", lanes_ok);
-	return lanes_ok != ((1 << NUM_LANES) - 1);
-}
-
-#include "raminit_patterns.h"
-
-static void fill_pattern5(ramctr_timing * ctrl, int channel, int patno)
-{
-	unsigned i, j;
-	unsigned channel_offset =
-	    get_precedening_channels(ctrl, channel) * 0x40;
-	unsigned channel_step = 0x40 * num_of_channels(ctrl);
-
-	if (patno) {
-		u8 base8 = 0x80 >> ((patno - 1) % 8);
-		u32 base = base8 | (base8 << 8) | (base8 << 16) | (base8 << 24);
-		for (i = 0; i < 32; i++) {
-			for (j = 0; j < 16; j++) {
-				u32 val = use_base[patno - 1][i] & (1 << (j / 2)) ? base : 0;
-				if (invert[patno - 1][i] & (1 << (j / 2)))
-					val = ~val;
-				write32((void *)(0x04000000 + channel_offset + i * channel_step +
-						 j * 4), val);
-			}
-		}
-
-	} else {
-		for (i = 0; i < sizeof(pattern) / sizeof(pattern[0]); i++) {
-			for (j = 0; j < 16; j++)
-				write32((void *)(0x04000000 + channel_offset + i * channel_step +
-						 j * 4), pattern[i][j]);
-		}
-		sfence();
-	}
-}
-
-static void reprogram_320c(ramctr_timing * ctrl)
-{
-	int channel, slotrank;
-	u32 r32;
-
-	FOR_ALL_POPULATED_CHANNELS {
-		wait_428c(channel);
-
-		/* choose an existing rank.  */
-		slotrank = !(ctrl->rankmap[channel] & 1) ? 2 : 0;
-
-		write32(DEFAULT_MCHBAR + 0x4220 + 0x400 * channel, 0x0f003);
-		write32(DEFAULT_MCHBAR + 0x4230 + 0x400 * channel, 0x41001);
-
-		write32(DEFAULT_MCHBAR + 0x4200 + 0x400 * channel,
-			(slotrank << 24) | 0x60000);
-
-		write32(DEFAULT_MCHBAR + 0x4210 + 0x400 * channel, 0x3e0);
-
-		write32(DEFAULT_MCHBAR + 0x4284 + 0x400 * channel, 1);
-		wait_428c(channel);
-		write32(DEFAULT_MCHBAR + 0x4020 + 0x400 * channel,
-			read32(DEFAULT_MCHBAR + 0x4020 +
-			       0x400 * channel) | 0x200000);
-	}
-	write32(DEFAULT_MCHBAR + 0x5030, read32(DEFAULT_MCHBAR + 0x5030) & ~8);
-	FOR_ALL_POPULATED_CHANNELS {
-		wait_428c(channel);
-
-		/* choose an existing rank.  */
-		slotrank = !(ctrl->rankmap[channel] & 1) ? 2 : 0;
-
-		write32(DEFAULT_MCHBAR + 0x4220 + 0x400 * channel, 0x0f003);
-		write32(DEFAULT_MCHBAR + 0x4230 + 0x400 * channel, 0x41001);
-
-		write32(DEFAULT_MCHBAR + 0x4200 + 0x400 * channel,
-			(slotrank << 24) | 0x60000);
-
-		write32(DEFAULT_MCHBAR + 0x4210 + 0x400 * channel, 0x3e0);
-
-		write32(DEFAULT_MCHBAR + 0x4284 + 0x400 * channel, 1);
-		wait_428c(channel);
-	}
-
-	/* jedec reset */
-	dram_jedecreset(ctrl);
-	/* mrs commands. */
-	dram_mrscommands(ctrl);
-
-	r32 = read32(DEFAULT_MCHBAR + 0x5030);
-	write32(DEFAULT_MCHBAR + 0x5030, r32 | 0x20);
-	udelay(1);
-
-	write32(DEFAULT_MCHBAR + 0x5030, r32 & ~0x20);
-
-	udelay(1);
-}
-
-#define MIN_C320C_LEN 13
-
-static int try_cmd_stretch(ramctr_timing * ctrl, int cmd_stretch)
-{
-	struct ram_rank_timings saved_timings[NUM_CHANNELS][NUM_SLOTRANKS];
-	int channel, slotrank;
-	int c320c;
-	int stat[NUM_SLOTRANKS][256];
-
-	FOR_ALL_CHANNELS FOR_ALL_POPULATED_RANKS {
-		saved_timings[channel][slotrank] = ctrl->timings[channel][slotrank];
-	}
-
-	FOR_ALL_POPULATED_CHANNELS {
-		ctrl->cmd_stretch[channel] = cmd_stretch;
-	}
-
-	FOR_ALL_POPULATED_CHANNELS
-	    MCHBAR32(0x4004 + 0x400 * channel) =
-		ctrl->tRRD
-		| (ctrl->tRTP << 4)
-		| (ctrl->tCKE << 8)
-		| (ctrl->tWTR << 12)
-		| (ctrl->tFAW << 16)
-		| (ctrl->tWR << 24)
-		| (ctrl->cmd_stretch[channel] << 30);
-
-
-	FOR_ALL_CHANNELS {
-		int delta = 0;
-		if (ctrl->cmd_stretch[channel] == 2)
-			delta = 2;
-		else if (ctrl->cmd_stretch[channel] == 0)
-			delta = 4;
-
-		FOR_ALL_POPULATED_RANKS {
-			ctrl->timings[channel][slotrank].val_4024 -= delta;
-		}
-	}
-
-	FOR_ALL_POPULATED_CHANNELS {
-		for (c320c = -127; c320c <= 127; c320c++) {
-			FOR_ALL_POPULATED_RANKS {
-				ctrl->timings[channel][slotrank].val_320c = c320c;
-			}
-			program_timings(ctrl, channel);
-			reprogram_320c(ctrl);
-			FOR_ALL_POPULATED_RANKS {
-				stat[slotrank][c320c + 127] =
-				    test_320c(ctrl, channel, slotrank);
-				printram("3stat: %d, %d, %d: %d\n",
-				       channel, slotrank, c320c,
-				       stat[slotrank][c320c + 127]);
-			}
-		}
-		FOR_ALL_POPULATED_RANKS {
-			struct run rn =
-			    get_longest_zero_run(stat[slotrank], 255);
-			ctrl->timings[channel][slotrank].val_320c =
-			    rn.middle - 127;
-			printram("3val: %d, %d: %d\n", channel,
-			       slotrank,
-			       ctrl->timings[channel][slotrank].val_320c);
-			if (rn.all || rn.length < MIN_C320C_LEN) {
-				FOR_ALL_CHANNELS FOR_ALL_POPULATED_RANKS {
-					ctrl->timings[channel][slotrank] = saved_timings[channel][slotrank];
-				}
-				return 0;
-			}
-		}
-	}
-	return 1;
-}
-
-static void command_training(ramctr_timing * ctrl)
-{
-	int channel;
-
-	FOR_ALL_POPULATED_CHANNELS {
-		fill_pattern5(ctrl, channel, 0);
-		write32(DEFAULT_MCHBAR + 0x4288 + 0x400 * channel, 0x1f);
-	}
-
-	/* try command rate 1T and 2T */
-	if (!try_cmd_stretch(ctrl, 0) && !try_cmd_stretch(ctrl, 2))
-		die("c320c discovery failed");
-
-	FOR_ALL_POPULATED_CHANNELS {
-		program_timings(ctrl, channel);
-	}
-
-	reprogram_320c(ctrl);
-}
-
-static void discover_edges_real(ramctr_timing * ctrl, int channel, int slotrank,
-				int *edges)
-{
-	int edge;
-	int statistics[NUM_LANES][MAX_EDGE_TIMING + 1];
-	int lane;
-
-	for (edge = 0; edge <= MAX_EDGE_TIMING; edge++) {
-		FOR_ALL_LANES {
-			ctrl->timings[channel][slotrank].lanes[lane].rising =
-			    edge;
-			ctrl->timings[channel][slotrank].lanes[lane].falling =
-			    edge;
-		}
-		printram("edge %02x\n", edge);
-		program_timings(ctrl, channel);
-
-		FOR_ALL_LANES {
-			write32(DEFAULT_MCHBAR + 0x4340 + 0x400 * channel +
-				4 * lane, 0);
-			read32(DEFAULT_MCHBAR + 0x400 * channel + 4 * lane +
-			       0x4140);
-		}
-
-		wait_428c(channel);
-
-		write32(DEFAULT_MCHBAR + 0x4220 + 0x400 * channel, 0x1f000);
-		write32(DEFAULT_MCHBAR + 0x4230 + 0x400 * channel,
-			(0xc01 | (ctrl->tMOD << 16)));
-		write32(DEFAULT_MCHBAR + 0x4200 + 0x400 * channel,
-			(slotrank << 24) | 0x360004);
-		write32(DEFAULT_MCHBAR + 0x4210 + 0x400 * channel, 0);
-
-		write32(DEFAULT_MCHBAR + 0x4224 + 0x400 * channel, 0x1f105);
-		write32(DEFAULT_MCHBAR + 0x4234 + 0x400 * channel, 0x40411f4);
-		write32(DEFAULT_MCHBAR + 0x4204 + 0x400 * channel,
-			(slotrank << 24));
-		write32(DEFAULT_MCHBAR + 0x4214 + 0x400 * channel, 0);
-
-		write32(DEFAULT_MCHBAR + 0x4228 + 0x400 * channel, 0x1f105);
-		write32(DEFAULT_MCHBAR + 0x4238 + 0x400 * channel,
-			0x1001 | ((ctrl->CAS + 8) << 16));
-		write32(DEFAULT_MCHBAR + 0x4208 + 0x400 * channel,
-			(slotrank << 24) | 0x60000);
-		write32(DEFAULT_MCHBAR + 0x4218 + 0x400 * channel, 0);
-
-		write32(DEFAULT_MCHBAR + 0x422c + 0x400 * channel, 0x1f000);
-		write32(DEFAULT_MCHBAR + 0x423c + 0x400 * channel,
-			(0xc01 | (ctrl->tMOD << 16)));
-		write32(DEFAULT_MCHBAR + 0x420c + 0x400 * channel,
-			(slotrank << 24) | 0x360000);
-		write32(DEFAULT_MCHBAR + 0x421c + 0x400 * channel, 0);
-
-		write32(DEFAULT_MCHBAR + 0x4284 + 0x400 * channel, 0xc0001);
-
-		wait_428c(channel);
-
-		FOR_ALL_LANES {
-			statistics[lane][edge] =
-			    read32(DEFAULT_MCHBAR + 0x4340 + 0x400 * channel +
-				   lane * 4);
-		}
-	}
-	FOR_ALL_LANES {
-		struct run rn =
-		    get_longest_zero_run(statistics[lane], MAX_EDGE_TIMING + 1);
-		edges[lane] = rn.middle;
-		if (rn.all)
-			die("edge discovery failed");
-		printram("eval %d, %d, %d, %02x\n", channel, slotrank,
-		       lane, edges[lane]);
-	}
-}
-
-static void discover_edges(ramctr_timing * ctrl)
-{
-	int falling_edges[NUM_CHANNELS][NUM_SLOTRANKS][NUM_LANES];
-	int rising_edges[NUM_CHANNELS][NUM_SLOTRANKS][NUM_LANES];
-	int channel, slotrank, lane;
-	u32 r32;
-
-	write32(DEFAULT_MCHBAR + 0x3400, 0);
-
-	r32 = read32(DEFAULT_MCHBAR + 0x5030);
-	write32(DEFAULT_MCHBAR + 0x5030, r32 | 0x20);
-	udelay(1);
-
-	write32(DEFAULT_MCHBAR + 0x5030, r32 & ~0x20);
-
-	udelay(1);
-
-	FOR_ALL_POPULATED_CHANNELS FOR_ALL_LANES {
-		write32(DEFAULT_MCHBAR + 4 * lane +
-			0x400 * channel + 0x4080, 0);
-	}
-
-	FOR_ALL_POPULATED_CHANNELS {
-		fill_pattern0(ctrl, channel, 0, 0);
-		write32(DEFAULT_MCHBAR + 0x4288 + (channel << 10), 0);
-		FOR_ALL_LANES {
-			read32(DEFAULT_MCHBAR + 0x400 * channel +
-			       lane * 4 + 0x4140);
-		}
-
-		FOR_ALL_POPULATED_RANKS FOR_ALL_LANES {
-			ctrl->timings[channel][slotrank].lanes[lane].falling =
-			    16;
-			ctrl->timings[channel][slotrank].lanes[lane].rising =
-			    16;
-		}
-
-		program_timings(ctrl, channel);
-
-		FOR_ALL_POPULATED_RANKS {
-			wait_428c(channel);
-
-			write32(DEFAULT_MCHBAR + 0x4220 + 0x400 * channel,
-				0x1f000);
-			write32(DEFAULT_MCHBAR + 0x4230 + 0x400 * channel,
-				0xc01 | (ctrl->tMOD << 16));
-			write32(DEFAULT_MCHBAR + 0x4200 + 0x400 * channel,
-				(slotrank << 24) | 0x360004);
-			write32(DEFAULT_MCHBAR + 0x4210 + 0x400 * channel, 0);
-
-			write32(DEFAULT_MCHBAR + 0x4224 + 0x400 * channel,
-				0x1f105);
-			write32(DEFAULT_MCHBAR + 0x4234 + 0x400 * channel,
-				0x4041003);
-			write32(DEFAULT_MCHBAR + 0x4204 + 0x400 * channel,
-				(slotrank << 24) | 0);
-			write32(DEFAULT_MCHBAR + 0x4214 + 0x400 * channel, 0);
-
-			write32(DEFAULT_MCHBAR + 0x4228 + 0x400 * channel,
-				0x1f105);
-			write32(DEFAULT_MCHBAR + 0x4238 + 0x400 * channel,
-				0x1001 | ((ctrl->CAS + 8) << 16));
-			write32(DEFAULT_MCHBAR + 0x4208 + 0x400 * channel,
-				(slotrank << 24) | 0x60000);
-			write32(DEFAULT_MCHBAR + 0x4218 + 0x400 * channel, 0);
-
-			write32(DEFAULT_MCHBAR + 0x422c + 0x400 * channel,
-				0x1f000);
-			write32(DEFAULT_MCHBAR + 0x423c + 0x400 * channel,
-				0xc01 | (ctrl->tMOD << 16));
-			write32(DEFAULT_MCHBAR + 0x420c + 0x400 * channel,
-				(slotrank << 24) | 0x360000);
-			write32(DEFAULT_MCHBAR + 0x421c + 0x400 * channel, 0);
-			write32(DEFAULT_MCHBAR + 0x4284 + 0x400 * channel,
-				0xc0001);
-
-			wait_428c(channel);
-		}
-
-		FOR_ALL_POPULATED_RANKS FOR_ALL_LANES {
-			ctrl->timings[channel][slotrank].lanes[lane].falling =
-			    48;
-			ctrl->timings[channel][slotrank].lanes[lane].rising =
-			    48;
-		}
-
-		program_timings(ctrl, channel);
-
-		FOR_ALL_POPULATED_RANKS {
-			wait_428c(channel);
-
-			write32(DEFAULT_MCHBAR + 0x4220 + 0x400 * channel,
-				0x1f000);
-			write32(DEFAULT_MCHBAR + 0x4230 + 0x400 * channel,
-				0xc01 | (ctrl->tMOD << 16));
-			write32(DEFAULT_MCHBAR + 0x4200 + 0x400 * channel,
-				(slotrank << 24) | 0x360004);
-			write32(DEFAULT_MCHBAR + 0x4210 + 0x400 * channel, 0);
-
-			write32(DEFAULT_MCHBAR + 0x4224 + 0x400 * channel,
-				0x1f105);
-			write32(DEFAULT_MCHBAR + 0x4234 + 0x400 * channel,
-				0x4041003);
-			write32(DEFAULT_MCHBAR + 0x4204 + 0x400 * channel,
-				(slotrank << 24) | 0);
-			write32(DEFAULT_MCHBAR + 0x4214 + 0x400 * channel, 0);
-
-			write32(DEFAULT_MCHBAR + 0x4228 + 0x400 * channel,
-				0x1f105);
-			write32(DEFAULT_MCHBAR + 0x4238 + 0x400 * channel,
-				0x1001 | ((ctrl->CAS + 8) << 16));
-			write32(DEFAULT_MCHBAR + 0x4208 + 0x400 * channel,
-				(slotrank << 24) | 0x60000);
-			write32(DEFAULT_MCHBAR + 0x4218 + 0x400 * channel, 0);
-
-			write32(DEFAULT_MCHBAR + 0x422c + 0x400 * channel,
-				0x1f000);
-			write32(DEFAULT_MCHBAR + 0x423c + 0x400 * channel,
-				0xc01 | (ctrl->tMOD << 16));
-			write32(DEFAULT_MCHBAR + 0x420c + 0x400 * channel,
-				(slotrank << 24) | 0x360000);
-			write32(DEFAULT_MCHBAR + 0x421c + 0x400 * channel, 0);
-
-			write32(DEFAULT_MCHBAR + 0x4284 + 0x400 * channel,
-				0xc0001);
-			wait_428c(channel);
-		}
-
-		FOR_ALL_LANES {
-			write32(DEFAULT_MCHBAR + 0x4080 + 0x400 * channel +
-				lane * 4,
-				~read32(DEFAULT_MCHBAR + 0x4040 +
-					0x400 * channel + lane * 4) & 0xff);
-		}
-
-		fill_pattern0(ctrl, channel, 0, 0xffffffff);
-		write32(DEFAULT_MCHBAR + 0x4288 + (channel << 10), 0);
-	}
-
-	/* FIXME: under some conditions (older chipsets?) vendor BIOS sets both edges to the same value.  */
-	write32(DEFAULT_MCHBAR + 0x4eb0, 0x300);
-
-	FOR_ALL_CHANNELS FOR_ALL_POPULATED_RANKS {
-		discover_edges_real(ctrl, channel, slotrank,
-				    falling_edges[channel][slotrank]);
-	}
-
-	write32(DEFAULT_MCHBAR + 0x4eb0, 0x200);
-
-	FOR_ALL_CHANNELS FOR_ALL_POPULATED_RANKS {
-		discover_edges_real(ctrl, channel, slotrank,
-				    rising_edges[channel][slotrank]);
-	}
-
-	write32(DEFAULT_MCHBAR + 0x4eb0, 0);
-
-	FOR_ALL_CHANNELS FOR_ALL_POPULATED_RANKS FOR_ALL_LANES {
-		ctrl->timings[channel][slotrank].lanes[lane].falling =
-		    falling_edges[channel][slotrank][lane];
-		ctrl->timings[channel][slotrank].lanes[lane].rising =
-		    rising_edges[channel][slotrank][lane];
-	}
-
-	FOR_ALL_POPULATED_CHANNELS {
-		program_timings(ctrl, channel);
-	}
-
-	FOR_ALL_CHANNELS FOR_ALL_POPULATED_RANKS FOR_ALL_LANES {
-		write32(DEFAULT_MCHBAR + 0x4080 + 0x400 * channel + 4 * lane,
-			0);
-	}
-}
-
-static void discover_edges_write_real(ramctr_timing * ctrl, int channel,
-				      int slotrank, int *edges)
-{
-	int edge;
-	u32 raw_statistics[MAX_EDGE_TIMING + 1];
-	int statistics[MAX_EDGE_TIMING + 1];
-	const int reg3000b24[] = { 0, 0xc, 0x2c };
-	int lane, i;
-	int lower[NUM_LANES];
-	int upper[NUM_LANES];
-	int pat;
-
-	FOR_ALL_LANES {
-		lower[lane] = 0;
-		upper[lane] = MAX_EDGE_TIMING;
-	}
-
-	for (i = 0; i < 3; i++) {
-		write32(DEFAULT_MCHBAR + 0x3000 + 0x100 * channel,
-			reg3000b24[i] << 24);
-		for (pat = 0; pat < NUM_PATTERNS; pat++) {
-			fill_pattern5(ctrl, channel, pat);
-			write32(DEFAULT_MCHBAR + 0x4288 + 0x400 * channel, 0x1f);
-			printram("patterned\n");
-			printram("[%x] = 0x%08x\n(%d, %d)\n",
-			       0x3000 + 0x100 * channel, reg3000b24[i] << 24, channel,
-			       slotrank);
-			for (edge = 0; edge <= MAX_EDGE_TIMING; edge++) {
-				FOR_ALL_LANES {
-					ctrl->timings[channel][slotrank].lanes[lane].
-						rising = edge;
-					ctrl->timings[channel][slotrank].lanes[lane].
-						falling = edge;
-				}
-				program_timings(ctrl, channel);
-
-				FOR_ALL_LANES {
-					write32(DEFAULT_MCHBAR + 0x4340 +
-						0x400 * channel + 4 * lane, 0);
-					read32(DEFAULT_MCHBAR + 0x400 * channel +
-					       4 * lane + 0x4140);
-				}
-				wait_428c(channel);
-
-				write32(DEFAULT_MCHBAR + 0x4220 + 0x400 * channel,
-					0x1f006);
-				write32(DEFAULT_MCHBAR + 0x4230 + 0x400 * channel,
-					0x4 | (ctrl->tRCD << 16)
-					| (max(ctrl->tRRD, (ctrl->tFAW >> 2) + 1) <<
-					   10));
-				write32(DEFAULT_MCHBAR + 0x4200 + 0x400 * channel,
-					(slotrank << 24) | 0x60000);
-				write32(DEFAULT_MCHBAR + 0x4210 + 0x400 * channel,
-					0x240);
-
-				write32(DEFAULT_MCHBAR + 0x4224 + 0x400 * channel,
-					0x1f201);
-				write32(DEFAULT_MCHBAR + 0x4234 + 0x400 * channel,
-					0x8005020 | ((ctrl->tWTR + ctrl->CWL + 8) <<
-						     16));
-				write32(DEFAULT_MCHBAR + 0x4204 + 0x400 * channel,
-					(slotrank << 24));
-				write32(DEFAULT_MCHBAR + 0x4214 + 0x400 * channel,
-					0x242);
-
-				write32(DEFAULT_MCHBAR + 0x4228 + 0x400 * channel,
-					0x1f105);
-				write32(DEFAULT_MCHBAR + 0x4238 + 0x400 * channel,
-					0x4005020 | (max(ctrl->tRTP, 8) << 16));
-				write32(DEFAULT_MCHBAR + 0x4208 + 0x400 * channel,
-					(slotrank << 24));
-				write32(DEFAULT_MCHBAR + 0x4218 + 0x400 * channel,
-					0x242);
-
-				write32(DEFAULT_MCHBAR + 0x422c + 0x400 * channel,
-					0x1f002);
-				write32(DEFAULT_MCHBAR + 0x423c + 0x400 * channel,
-					0xc01 | (ctrl->tRP << 16));
-				write32(DEFAULT_MCHBAR + 0x420c + 0x400 * channel,
-					(slotrank << 24) | 0x60400);
-				write32(DEFAULT_MCHBAR + 0x421c + 0x400 * channel, 0);
-
-				write32(DEFAULT_MCHBAR + 0x4284 + 0x400 * channel,
-					0xc0001);
-				wait_428c(channel);
-				FOR_ALL_LANES {
-					read32(DEFAULT_MCHBAR + 0x4340 +
-					       0x400 * channel + lane * 4);
-				}
-
-				raw_statistics[edge] =
-					MCHBAR32(0x436c + 0x400 * channel);
-			}
-			FOR_ALL_LANES {
-				struct run rn;
-				for (edge = 0; edge <= MAX_EDGE_TIMING; edge++)
-					statistics[edge] =
-						! !(raw_statistics[edge] & (1 << lane));
-				rn = get_longest_zero_run(statistics,
-							  MAX_EDGE_TIMING + 1);
-				printram("edges: %d, %d, %d: 0x%x-0x%x-0x%x, 0x%x-0x%x\n",
-					 channel, slotrank, i, rn.start, rn.middle,
-					 rn.end, rn.start + ctrl->edge_offset[i],
-					 rn.end - ctrl->edge_offset[i]);
-				lower[lane] =
-					max(rn.start + ctrl->edge_offset[i], lower[lane]);
-				upper[lane] =
-					min(rn.end - ctrl->edge_offset[i], upper[lane]);
-				edges[lane] = (lower[lane] + upper[lane]) / 2;
-
-			}
-		}
-	}
-
-	write32(DEFAULT_MCHBAR + 0x3000, 0);
-	printram("CPA\n");
-}
-
-static void discover_edges_write(ramctr_timing * ctrl)
-{
-	int falling_edges[NUM_CHANNELS][NUM_SLOTRANKS][NUM_LANES];
-	int rising_edges[NUM_CHANNELS][NUM_SLOTRANKS][NUM_LANES];
-	int channel, slotrank, lane;
-
-	/* FIXME: under some conditions (older chipsets?) vendor BIOS sets both edges to the same value.  */
-	write32(DEFAULT_MCHBAR + 0x4eb0, 0x300);
-
-	FOR_ALL_CHANNELS FOR_ALL_POPULATED_RANKS {
-		discover_edges_write_real(ctrl, channel, slotrank,
-					  falling_edges[channel][slotrank]);
-	}
-
-	write32(DEFAULT_MCHBAR + 0x4eb0, 0x200);
-
-	FOR_ALL_CHANNELS FOR_ALL_POPULATED_RANKS {
-		discover_edges_write_real(ctrl, channel, slotrank,
-					  rising_edges[channel][slotrank]);
-	}
-
-	write32(DEFAULT_MCHBAR + 0x4eb0, 0);
-
-	FOR_ALL_CHANNELS FOR_ALL_POPULATED_RANKS FOR_ALL_LANES {
-		ctrl->timings[channel][slotrank].lanes[lane].falling =
-		    falling_edges[channel][slotrank][lane];
-		ctrl->timings[channel][slotrank].lanes[lane].rising =
-		    rising_edges[channel][slotrank][lane];
-	}
-
-	FOR_ALL_POPULATED_CHANNELS
-		program_timings(ctrl, channel);
-
-	FOR_ALL_CHANNELS FOR_ALL_POPULATED_RANKS FOR_ALL_LANES {
-		write32(DEFAULT_MCHBAR + 0x4080 + 0x400 * channel + 4 * lane,
-			0);
-	}
-}
-
-static void test_timC_write(ramctr_timing *ctrl, int channel, int slotrank)
-{
-	wait_428c(channel);
-	write32(DEFAULT_MCHBAR + 0x4220 + 0x400 * channel, 0x1f006);
-	write32(DEFAULT_MCHBAR + 0x4230 + 0x400 * channel,
-		(max((ctrl->tFAW >> 2) + 1, ctrl->tRRD)
-		 << 10) | (ctrl->tRCD << 16) | 4);
-	write32(DEFAULT_MCHBAR + 0x4200 + 0x400 * channel,
-		(slotrank << 24) | 0x60000);
-	write32(DEFAULT_MCHBAR + 0x4210 + 0x400 * channel, 0x244);
-
-	write32(DEFAULT_MCHBAR + 0x4224 + 0x400 * channel, 0x1f201);
-	write32(DEFAULT_MCHBAR + 0x4234 + 0x400 * channel,
-		0x80011e0 |
-		((ctrl->tWTR + ctrl->CWL + 8) << 16));
-	write32(DEFAULT_MCHBAR + 0x4204 +
-		0x400 * channel, (slotrank << 24));
-	write32(DEFAULT_MCHBAR + 0x4214 +
-		0x400 * channel, 0x242);
-
-	write32(DEFAULT_MCHBAR + 0x4228 +
-		0x400 * channel, 0x1f105);
-	write32(DEFAULT_MCHBAR + 0x4238 +
-		0x400 * channel,
-		0x40011e0 | (max(ctrl->tRTP, 8) << 16));
-	write32(DEFAULT_MCHBAR + 0x4208 +
-		0x400 * channel, (slotrank << 24));
-	write32(DEFAULT_MCHBAR + 0x4218 +
-		0x400 * channel, 0x242);
-
-	write32(DEFAULT_MCHBAR + 0x422c +
-		0x400 * channel, 0x1f002);
-	write32(DEFAULT_MCHBAR + 0x423c +
-		0x400 * channel,
-		0x1001 | (ctrl->tRP << 16));
-	write32(DEFAULT_MCHBAR + 0x420c +
-		0x400 * channel,
-		(slotrank << 24) | 0x60400);
-	write32(DEFAULT_MCHBAR + 0x421c +
-		0x400 * channel, 0);
-
-	write32(DEFAULT_MCHBAR + 0x4284 +
-		0x400 * channel, 0xc0001);
-	wait_428c(channel);
-}
-
-static void discover_timC_write(ramctr_timing * ctrl)
-{
-	const u8 rege3c_b24[3] = { 0, 0xf, 0x2f };
-	int i, pat;
-
-	int lower[NUM_CHANNELS][NUM_SLOTRANKS][NUM_LANES];
-	int upper[NUM_CHANNELS][NUM_SLOTRANKS][NUM_LANES];
-	int channel, slotrank, lane;
-
-	FOR_ALL_CHANNELS FOR_ALL_POPULATED_RANKS FOR_ALL_LANES {
-		lower[channel][slotrank][lane] = 0;
-		upper[channel][slotrank][lane] = MAX_TIMC;
-	}
-
-	write32(DEFAULT_MCHBAR + 0x4ea8, 1);
-
-	for (i = 0; i < 3; i++)
-		FOR_ALL_POPULATED_CHANNELS {
-			write32(DEFAULT_MCHBAR + 0xe3c + (channel * 0x100),
-				(rege3c_b24[i] << 24)
-				| (read32(DEFAULT_MCHBAR + 0xe3c + (channel * 0x100))
-				   & ~0x3f000000));
-			udelay(2);
-			for (pat = 0; pat < NUM_PATTERNS; pat++) {
-				FOR_ALL_POPULATED_RANKS {
-					int timC;
-					u32 raw_statistics[MAX_TIMC + 1];
-					int statistics[MAX_TIMC + 1];
-
-					fill_pattern5(ctrl, channel, pat);
-					write32(DEFAULT_MCHBAR + 0x4288 + 0x400 * channel, 0x1f);
-					for (timC = 0; timC < MAX_TIMC + 1; timC++) {
-						FOR_ALL_LANES
-							ctrl->timings[channel][slotrank].lanes[lane].timC = timC;
-						program_timings(ctrl, channel);
-
-						test_timC_write (ctrl, channel, slotrank);
-
-						raw_statistics[timC] =
-							MCHBAR32(0x436c + 0x400 * channel);
-					}
-					FOR_ALL_LANES {
-						struct run rn;
-						for (timC = 0; timC <= MAX_TIMC; timC++)
-							statistics[timC] =
-								!!(raw_statistics[timC] &
-								   (1 << lane));
-						rn = get_longest_zero_run(statistics,
-									  MAX_TIMC + 1);
-						if (rn.all)
-							die("timC write discovery failed");
-						printram("timC: %d, %d, %d: 0x%x-0x%x-0x%x, 0x%x-0x%x\n",
-							 channel, slotrank, i, rn.start,
-							 rn.middle, rn.end,
-							 rn.start + ctrl->timC_offset[i],
-							 rn.end - ctrl->timC_offset[i]);
-						lower[channel][slotrank][lane] =
-							max(rn.start + ctrl->timC_offset[i],
-							    lower[channel][slotrank][lane]);
-						upper[channel][slotrank][lane] =
-							min(rn.end - ctrl->timC_offset[i],
-							    upper[channel][slotrank][lane]);
-
-					}
-				}
-			}
-		}
-
-	FOR_ALL_CHANNELS {
-		write32(DEFAULT_MCHBAR + (channel * 0x100) + 0xe3c,
-			0 | (read32(DEFAULT_MCHBAR + (channel * 0x100) + 0xe3c) &
-			     ~0x3f000000));
-		udelay(2);
-	}
-
-	write32(DEFAULT_MCHBAR + 0x4ea8, 0);
-
-	printram("CPB\n");
-
-	FOR_ALL_CHANNELS FOR_ALL_POPULATED_RANKS FOR_ALL_LANES {
-		printram("timC [%d, %d, %d] = 0x%x\n", channel,
-		       slotrank, lane,
-		       (lower[channel][slotrank][lane] +
-			upper[channel][slotrank][lane]) / 2);
-		ctrl->timings[channel][slotrank].lanes[lane].timC =
-		    (lower[channel][slotrank][lane] +
-		     upper[channel][slotrank][lane]) / 2;
-	}
-	FOR_ALL_POPULATED_CHANNELS {
-		program_timings(ctrl, channel);
-	}
-}
-
-static void normalize_training(ramctr_timing * ctrl)
-{
-	int channel, slotrank, lane;
-	int mat = 0;
-
-	FOR_ALL_CHANNELS FOR_ALL_POPULATED_RANKS {
-		int delta;
-		FOR_ALL_LANES mat =
-		    max(ctrl->timings[channel][slotrank].lanes[lane].timA, mat);
-		 delta = (mat >> 6) - ctrl->timings[channel][slotrank].val_4028;
-		 ctrl->timings[channel][slotrank].val_4024 += delta;
-		 ctrl->timings[channel][slotrank].val_4028 += delta;
-	}
-
-	FOR_ALL_POPULATED_CHANNELS {
-		program_timings(ctrl, channel);
-	}
-}
-
-static void write_controller_mr(ramctr_timing * ctrl)
-{
-	int channel, slotrank;
-
-	FOR_ALL_CHANNELS FOR_ALL_POPULATED_RANKS {
-		write32(DEFAULT_MCHBAR + 0x0004 + (channel << 8) +
-			lane_registers[slotrank], make_mr0(ctrl, slotrank));
-		write32(DEFAULT_MCHBAR + 0x0008 + (channel << 8) +
-			lane_registers[slotrank], make_mr1(ctrl, slotrank));
-	}
-}
-
-static void channel_test(ramctr_timing * ctrl)
-{
-	int channel, slotrank, lane;
-
-	FOR_ALL_POPULATED_CHANNELS
-	    if (read32(DEFAULT_MCHBAR + 0x42a0 + (channel << 10)) & 0xa000)
-		 die("Mini channel test failed (1)\n");
-	FOR_ALL_POPULATED_CHANNELS {
-		fill_pattern0(ctrl, channel, 0x12345678, 0x98765432);
-
-		write32(DEFAULT_MCHBAR + 0x4288 + (channel << 10), 0);
-	}
-
-	for (slotrank = 0; slotrank < 4; slotrank++)
-		FOR_ALL_CHANNELS
-			if (ctrl->rankmap[channel] & (1 << slotrank)) {
-		FOR_ALL_LANES {
-			write32(DEFAULT_MCHBAR + (0x4f40 + 4 * lane), 0);
-			write32(DEFAULT_MCHBAR + (0x4d40 + 4 * lane), 0);
-		}
-		wait_428c(channel);
-		write32(DEFAULT_MCHBAR + 0x4220 + (channel << 10), 0x0001f006);
-		write32(DEFAULT_MCHBAR + 0x4230 + (channel << 10), 0x0028a004);
-		write32(DEFAULT_MCHBAR + 0x4200 + (channel << 10),
-			0x00060000 | (slotrank << 24));
-		write32(DEFAULT_MCHBAR + 0x4210 + (channel << 10), 0x00000244);
-		write32(DEFAULT_MCHBAR + 0x4224 + (channel << 10), 0x0001f201);
-		write32(DEFAULT_MCHBAR + 0x4234 + (channel << 10), 0x08281064);
-		write32(DEFAULT_MCHBAR + 0x4204 + (channel << 10),
-			0x00000000 | (slotrank << 24));
-		write32(DEFAULT_MCHBAR + 0x4214 + (channel << 10), 0x00000242);
-		write32(DEFAULT_MCHBAR + 0x4228 + (channel << 10), 0x0001f105);
-		write32(DEFAULT_MCHBAR + 0x4238 + (channel << 10), 0x04281064);
-		write32(DEFAULT_MCHBAR + 0x4208 + (channel << 10),
-			0x00000000 | (slotrank << 24));
-		write32(DEFAULT_MCHBAR + 0x4218 + (channel << 10), 0x00000242);
-		write32(DEFAULT_MCHBAR + 0x422c + (channel << 10), 0x0001f002);
-		write32(DEFAULT_MCHBAR + 0x423c + (channel << 10), 0x00280c01);
-		write32(DEFAULT_MCHBAR + 0x420c + (channel << 10),
-			0x00060400 | (slotrank << 24));
-		write32(DEFAULT_MCHBAR + 0x421c + (channel << 10), 0x00000240);
-		write32(DEFAULT_MCHBAR + 0x4284 + (channel << 10), 0x000c0001);
-		wait_428c(channel);
-		FOR_ALL_LANES
-		    if (read32(DEFAULT_MCHBAR + 0x4340 + (channel << 10) + 4 * lane))
-			 die("Mini channel test failed (2)\n");
-	}
-}
-
-static void set_scrambling_seed(ramctr_timing * ctrl)
-{
-	int channel;
-
-	/* FIXME: we hardcode seeds. Do we need to use some PRNG for them?
-	   I don't think so.  */
-	static u32 seeds[NUM_CHANNELS][3] = {
-		{0x00009a36, 0xbafcfdcf, 0x46d1ab68},
-		{0x00028bfa, 0x53fe4b49, 0x19ed5483}
-	};
-	FOR_ALL_POPULATED_CHANNELS {
-		MCHBAR32(0x4020 + 0x400 * channel) &= ~0x10000000;
-		write32(DEFAULT_MCHBAR + 0x4034, seeds[channel][0]);
-		write32(DEFAULT_MCHBAR + 0x403c, seeds[channel][1]);
-		write32(DEFAULT_MCHBAR + 0x4038, seeds[channel][2]);
-	}
-}
-
-static void set_4f8c(void)
-{
-	struct cpuid_result cpures;
-	u32 cpu;
-
-	cpures = cpuid(0);
-	cpu = (cpures.eax);
-	if (IS_SANDY_CPU(cpu) && (IS_SANDY_CPU_D0(cpu) || IS_SANDY_CPU_D1(cpu))) {
-		MCHBAR32(0x4f8c) = 0x141D1519;
-	} else {
-		MCHBAR32(0x4f8c) = 0x551D1519;
-	}
-}
-
-static void prepare_training(ramctr_timing * ctrl)
-{
-	int channel;
-
-	FOR_ALL_POPULATED_CHANNELS {
-		// Always drive command bus
-		MCHBAR32(0x4004 + 0x400 * channel) |= 0x20000000;
-	}
-
-	udelay(1);
-
-	FOR_ALL_POPULATED_CHANNELS {
-		wait_428c(channel);
-	}
-}
-
-static void set_4008c(ramctr_timing * ctrl)
-{
-	int channel, slotrank;
-	u32 reg;
-	FOR_ALL_POPULATED_CHANNELS {
-		u32 b20, b4_8_12;
-		int min_320c = 10000;
-		int max_320c = -10000;
-
-		FOR_ALL_POPULATED_RANKS {
-			max_320c = max(ctrl->timings[channel][slotrank].val_320c, max_320c);
-			min_320c = min(ctrl->timings[channel][slotrank].val_320c, min_320c);
-		}
-
-		if (max_320c - min_320c > 51)
-			b20 = 0;
-		else
-			b20 = ctrl->ref_card_offset[channel];
-
-		if (ctrl->reg_320c_range_threshold < max_320c - min_320c)
-			b4_8_12 = 0x3330;
-		else
-			b4_8_12 = 0x2220;
-
-		reg = read32(DEFAULT_MCHBAR + 0x400c + (channel << 10));
-		write32(DEFAULT_MCHBAR + 0x400c + (channel << 10),
-			(reg & 0xFFF0FFFF)
-			| (ctrl->ref_card_offset[channel] << 16)
-			| (ctrl->ref_card_offset[channel] << 18));
-		write32(DEFAULT_MCHBAR + 0x4008 + (channel << 10),
-			0x0a000000
-			| (b20 << 20)
-			| ((ctrl->ref_card_offset[channel] + 2) << 16)
-			| b4_8_12);
-	}
-}
-
-static void set_42a0(ramctr_timing * ctrl)
-{
-	int channel;
-	FOR_ALL_POPULATED_CHANNELS {
-		write32(DEFAULT_MCHBAR + (0x42a0 + 0x400 * channel),
-			0x00001000 | ctrl->rankmap[channel]);
-		MCHBAR32(0x4004 + 0x400 * channel) &= ~0x20000000;	// OK
-	}
-}
-
-static int encode_5d10(int ns)
-{
-  return (ns + 499) / 500;
-}
-
-/* FIXME: values in this function should be hardware revision-dependent.  */
-static void final_registers(ramctr_timing * ctrl)
-{
-	int channel;
-	int t1_cycles = 0, t1_ns = 0, t2_ns;
-	int t3_ns;
-	u32 r32;
-
-	write32(DEFAULT_MCHBAR + 0x4cd4, 0x00000046);
-
-	write32(DEFAULT_MCHBAR + 0x400c, (read32(DEFAULT_MCHBAR + 0x400c) & 0xFFFFCFFF) | 0x1000);	// OK
-	write32(DEFAULT_MCHBAR + 0x440c, (read32(DEFAULT_MCHBAR + 0x440c) & 0xFFFFCFFF) | 0x1000);	// OK
-	write32(DEFAULT_MCHBAR + 0x4cb0, 0x00000740);
-	write32(DEFAULT_MCHBAR + 0x4380, 0x00000aaa);	// OK
-	write32(DEFAULT_MCHBAR + 0x4780, 0x00000aaa);	// OK
-	write32(DEFAULT_MCHBAR + 0x4f88, 0x5f7003ff);	// OK
-	write32(DEFAULT_MCHBAR + 0x5064, 0x00073000 | ctrl->reg_5064b0); // OK
-
-	FOR_ALL_CHANNELS {
-		switch (ctrl->rankmap[channel]) {
-			/* Unpopulated channel.  */
-		case 0:
-			write32(DEFAULT_MCHBAR + 0x4384 + channel * 0x400, 0);
-			break;
-			/* Only single-ranked dimms.  */
-		case 1:
-		case 4:
-		case 5:
-			write32(DEFAULT_MCHBAR + 0x4384 + channel * 0x400, 0x373131);
-			break;
-			/* Dual-ranked dimms present.  */
-		default:
-			write32(DEFAULT_MCHBAR + 0x4384 + channel * 0x400, 0x9b6ea1);
-			break;
-		}
-	}
-
-	write32 (DEFAULT_MCHBAR + 0x5880, 0xca9171e5);
-	write32 (DEFAULT_MCHBAR + 0x5888,
-		 (read32 (DEFAULT_MCHBAR + 0x5888) & ~0xffffff) | 0xe4d5d0);
-	write32 (DEFAULT_MCHBAR + 0x58a8, read32 (DEFAULT_MCHBAR + 0x58a8) & ~0x1f);
-	write32 (DEFAULT_MCHBAR + 0x4294,
-		 (read32 (DEFAULT_MCHBAR + 0x4294) & ~0x30000)
-		 | (1 << 16));
-	write32 (DEFAULT_MCHBAR + 0x4694,
-		 (read32 (DEFAULT_MCHBAR + 0x4694) & ~0x30000)
-		 | (1 << 16));
-
-	MCHBAR32(0x5030) |= 1;	// OK
-	MCHBAR32(0x5030) |= 0x80;	// OK
-	MCHBAR32(0x5f18) = 0xfa;	// OK
-
-	/* Find a populated channel.  */
-	FOR_ALL_POPULATED_CHANNELS
-		break;
-
-	t1_cycles = ((read32(DEFAULT_MCHBAR + 0x4290 + channel * 0x400) >> 8) & 0xff);
-	r32 = read32(DEFAULT_MCHBAR + 0x5064);
-	if (r32 & 0x20000)
-		t1_cycles += (r32 & 0xfff);
-	t1_cycles += (read32(DEFAULT_MCHBAR + channel * 0x400 + 0x42a4) & 0xfff);
-	t1_ns = t1_cycles * ctrl->tCK / 256 + 544;
-	if (!(r32 & 0x20000))
-		t1_ns += 500;
-
-	t2_ns = 10 * ((read32(DEFAULT_MCHBAR + 0x5f10) >> 8) & 0xfff);
-	if ( read32(DEFAULT_MCHBAR + 0x5f00) & 8 )
-	{
-		t3_ns = 10 * ((read32(DEFAULT_MCHBAR + 0x5f20) >> 8) & 0xfff);
-		t3_ns += 10 * (read32(DEFAULT_MCHBAR + 0x5f18) & 0xff);
-	}
-	else
-	{
-		t3_ns = 500;
-	}
-	printk(BIOS_DEBUG, "t123: %d, %d, %d\n",
-	       t1_ns, t2_ns, t3_ns);
-	write32 (DEFAULT_MCHBAR + 0x5d10,
-		 ((encode_5d10(t1_ns) + encode_5d10(t2_ns)) << 16)
-		 | (encode_5d10(t1_ns) << 8)
-		 | ((encode_5d10(t3_ns) + encode_5d10(t2_ns) + encode_5d10(t1_ns)) << 24)
-		 | (read32(DEFAULT_MCHBAR + 0x5d10) & 0xC0C0C0C0)
-		 | 0xc);
-}
-
-static void save_timings(ramctr_timing * ctrl)
-{
-	struct mrc_data_container *mrcdata;
-	int output_len = ALIGN(sizeof (*ctrl), 16);
-
-	/* Save the MRC S3 restore data to cbmem */
-	mrcdata = cbmem_add
-		(CBMEM_ID_MRCDATA,
-		 output_len + sizeof(struct mrc_data_container));
-
-	printk(BIOS_DEBUG, "Relocate MRC DATA from %p to %p (%u bytes)\n",
-	       ctrl, mrcdata, output_len);
-
-	mrcdata->mrc_signature = MRC_DATA_SIGNATURE;
-	mrcdata->mrc_data_size = output_len;
-	mrcdata->reserved = 0;
-	memcpy(mrcdata->mrc_data, ctrl, sizeof (*ctrl));
-
-	/* Zero the unused space in aligned buffer. */
-	if (output_len > sizeof (*ctrl))
-		memset(mrcdata->mrc_data+sizeof (*ctrl), 0,
-		       output_len - sizeof (*ctrl));
-
-	mrcdata->mrc_checksum = compute_ip_checksum(mrcdata->mrc_data,
-						    mrcdata->mrc_data_size);
-}
-
-static void restore_timings(ramctr_timing * ctrl)
-{
-	int channel, slotrank, lane;
-
-	FOR_ALL_POPULATED_CHANNELS
-	    MCHBAR32(0x4004 + 0x400 * channel) =
-		ctrl->tRRD
-		| (ctrl->tRTP << 4)
-		| (ctrl->tCKE << 8)
-		| (ctrl->tWTR << 12)
-		| (ctrl->tFAW << 16)
-		| (ctrl->tWR << 24)
-		| (ctrl->cmd_stretch[channel] << 30);
-
-	udelay(1);
-
-	FOR_ALL_POPULATED_CHANNELS {
-		wait_428c(channel);
-	}
-
-	FOR_ALL_CHANNELS FOR_ALL_POPULATED_RANKS FOR_ALL_LANES {
-		write32(DEFAULT_MCHBAR + 0x4080 + 0x400 * channel
-			+ 4 * lane, 0);
-	}
-
-	FOR_ALL_POPULATED_CHANNELS
-	    write32(DEFAULT_MCHBAR + 0x4008 + 0x400 * channel,
-		    read32(DEFAULT_MCHBAR + 0x4008 +
-			   0x400 * channel) | 0x8000000);
-
-	FOR_ALL_POPULATED_CHANNELS {
-		udelay (1);
-		write32(DEFAULT_MCHBAR + 0x4020 + 0x400 * channel,
-			read32(DEFAULT_MCHBAR + 0x4020 +
-			       0x400 * channel) | 0x200000);
-	}
-
-	printram("CPE\n");
-
-	write32(DEFAULT_MCHBAR + 0x3400, 0);
-	write32(DEFAULT_MCHBAR + 0x4eb0, 0);
-
-	printram("CP5b\n");
-
-	FOR_ALL_POPULATED_CHANNELS {
-		program_timings(ctrl, channel);
-	}
-
-	u32 reg, addr;
-
-	while (!(MCHBAR32(0x5084) & 0x10000)) ;
-	do {
-		reg = MCHBAR32(0x428c);
-	} while ((reg & 0x14) == 0);
-
-	// Set state of memory controller
-	MCHBAR32(0x5030) = 0x116;
-	MCHBAR32(0x4ea0) = 0;
-
-	// Wait 500us
-	udelay(500);
-
-	FOR_ALL_CHANNELS {
-		// Set valid rank CKE
-		reg = 0;
-		reg = (reg & ~0xf) | ctrl->rankmap[channel];
-		addr = 0x400 * channel + 0x42a0;
-		MCHBAR32(addr) = reg;
-
-		// Wait 10ns for ranks to settle
-		//udelay(0.01);
-
-		reg = (reg & ~0xf0) | (ctrl->rankmap[channel] << 4);
-		MCHBAR32(addr) = reg;
-
-		// Write reset using a NOP
-		write_reset(ctrl);
-	}
-
-	/* mrs commands. */
-	dram_mrscommands(ctrl);
-
-	printram("CP5c\n");
-
-	write32(DEFAULT_MCHBAR + 0x3000, 0);
-
-	FOR_ALL_CHANNELS {
-		write32(DEFAULT_MCHBAR + (channel * 0x100) + 0xe3c,
-			0 | (read32(DEFAULT_MCHBAR + (channel * 0x100) + 0xe3c) &
-			     ~0x3f000000));
-		udelay(2);
-	}
-
-	write32(DEFAULT_MCHBAR + 0x4ea8, 0);
-}
-
-void init_dram_ddr3(spd_raw_data * spds, int mobile, int min_tck,
-	int s3resume)
-{
-	int me_uma_size;
-	int cbmem_was_inited;
-
-	MCHBAR32(0x5f00) |= 1;
-
-	report_platform_info();
-
-	/* Wait for ME to be ready */
-	intel_early_me_init();
-	me_uma_size = intel_early_me_uma_size();
-
-	printk(BIOS_DEBUG, "Starting native Platform init\n");
-
-	u32 reg_5d10;
-
-	wait_txt_clear();
-
-	wrmsr(0x000002e6, (msr_t) { .lo = 0, .hi = 0 });
-
-	reg_5d10 = read32(DEFAULT_MCHBAR + 0x5d10);	// !!! = 0x00000000
-	if ((pcie_read_config16(SOUTHBRIDGE, 0xa2) & 0xa0) == 0x20	/* 0x0004 */
-	    && reg_5d10 && !s3resume) {
-		write32(DEFAULT_MCHBAR + 0x5d10, 0);
-		/* Need reset.  */
-		outb(0x6, 0xcf9);
-
-		halt();
-	}
-
-	ramctr_timing ctrl;
-
-	memset(&ctrl, 0, sizeof (ctrl));
-
-	early_pch_init_native();
-	early_thermal_init();
-
-	ctrl.mobile = mobile;
-	ctrl.tCK = min_tck;
-
-	/* FIXME: for non-S3 we should be able to use timing caching with
-	   proper verification. Right now we use timings only for S3 case.
-	 */
-	if (s3resume) {
-		struct mrc_data_container *mrc_cache;
-
-		mrc_cache = find_current_mrc_cache();
-		if (!mrc_cache || mrc_cache->mrc_data_size < sizeof (ctrl)) {
-			/* Failed S3 resume, reset to come up cleanly */
-			outb(0x6, 0xcf9);
-			halt();
-		}
-		memcpy(&ctrl, mrc_cache->mrc_data, sizeof (ctrl));
-	}
-
-	if (!s3resume) {
-		dimm_info info;
-
-		/* Get DDR3 SPD data */
-		dram_find_spds_ddr3(spds, &info, &ctrl);
-
-		/* Find fastest common supported parameters */
-		dram_find_common_params(&info, &ctrl);
-
-		dram_dimm_mapping(&info, &ctrl);
-	}
-
-	/* Set MCU frequency */
-	dram_freq(&ctrl);
-
-	if (!s3resume) {
-		/* Calculate timings */
-		dram_timing(&ctrl);
-	}
-
-	/* Set version register */
-	MCHBAR32(0x5034) = 0xC04EB002;
-
-	/* Enable crossover */
-	dram_xover(&ctrl);
-
-	/* Set timing and refresh registers */
-	dram_timing_regs(&ctrl);
-
-	/* Power mode preset */
-	MCHBAR32(0x4e80) = 0x5500;
-
-	/* Set scheduler parameters */
-	MCHBAR32(0x4c20) = 0x10100005;
-
-	/* Set cpu specific register */
-	set_4f8c();
-
-	/* Clear IO reset bit */
-	MCHBAR32(0x5030) &= ~0x20;
-
-	/* Set MAD-DIMM registers */
-	dram_dimm_set_mapping(&ctrl);
-	printk(BIOS_DEBUG, "Done dimm mapping\n");
-
-	/* Zone config */
-	dram_zones(&ctrl, 1);
-
-	/* Set memory map */
-	dram_memorymap(&ctrl, me_uma_size);
-	printk(BIOS_DEBUG, "Done memory map\n");
-
-	/* Set IO registers */
-	dram_ioregs(&ctrl);
-	printk(BIOS_DEBUG, "Done io registers\n");
-
-	udelay(1);
-
-	if (s3resume) {
-		restore_timings(&ctrl);
-	} else {
-		/* Do jedec ddr3 reset sequence */
-		dram_jedecreset(&ctrl);
-		printk(BIOS_DEBUG, "Done jedec reset\n");
-
-		/* MRS commands */
-		dram_mrscommands(&ctrl);
-		printk(BIOS_DEBUG, "Done MRS commands\n");
-		dram_mrscommands(&ctrl);
-
-		/* Prepare for memory training */
-		prepare_training(&ctrl);
-
-		read_training(&ctrl);
-		write_training(&ctrl);
-
-		printram("CP5a\n");
-
-		discover_edges(&ctrl);
-
-		printram("CP5b\n");
-
-		command_training(&ctrl);
-
-		printram("CP5c\n");
-
-		discover_edges_write(&ctrl);
-
-		discover_timC_write(&ctrl);
-
-		normalize_training(&ctrl);
-	}
-
-	set_4008c(&ctrl);
-
-	write_controller_mr(&ctrl);
-
-	if (!s3resume) {
-		channel_test(&ctrl);
-	}
-
-	/* FIXME: should be hardware revision-dependent.  */
-	write32(DEFAULT_MCHBAR + 0x5024, 0x00a030ce);
-
-	set_scrambling_seed(&ctrl);
-
-	set_42a0(&ctrl);
-
-	final_registers(&ctrl);
-
-	/* Zone config */
-	dram_zones(&ctrl, 0);
-
-	if (!s3resume)
-		quick_ram_check();
-
-	intel_early_me_status();
-	intel_early_me_init_done(ME_INIT_STATUS_SUCCESS);
-	intel_early_me_status();
-
-	post_system_agent_init();
-	report_memory_config();
-
-	cbmem_was_inited = !cbmem_recovery(s3resume);
-	if (!s3resume)
-		save_timings(&ctrl);
-	if (s3resume && !cbmem_was_inited) {
-		/* Failed S3 resume, reset to come up cleanly */
-		outb(0x6, 0xcf9);
-		halt();
-	}
-}
diff --git a/src/northbridge/intel/sandybridge/romstage_native.c b/src/northbridge/intel/sandybridge/romstage.c
index a18c480d3c..a18c480d3c 100644
--- a/src/northbridge/intel/sandybridge/romstage_native.c
+++ b/src/northbridge/intel/sandybridge/romstage.c
diff --git a/src/southbridge/intel/bd82x6x/Makefile.inc b/src/southbridge/intel/bd82x6x/Makefile.inc
index a1256df2e3..ff140a092f 100644
--- a/src/southbridge/intel/bd82x6x/Makefile.inc
+++ b/src/southbridge/intel/bd82x6x/Makefile.inc
@@ -47,13 +47,13 @@ smm-$(CONFIG_HAVE_SMI_HANDLER) += smihandler.c me.c me_8.x.c finalize.c pch.c
 
 romstage-y += early_smbus.c me_status.c gpio.c
 romstage-y += reset.c
-romstage-y += early_spi.c early_pch.c
+romstage-y += early_spi.c early_pch_common.c
 romstage-y += early_rcba.c
 
-romstage-$(CONFIG_NORTHBRIDGE_INTEL_IVYBRIDGE) += early_me.c early_usb.c
-romstage-$(CONFIG_NORTHBRIDGE_INTEL_SANDYBRIDGE) += early_me.c early_usb.c
-romstage-$(CONFIG_NORTHBRIDGE_INTEL_IVYBRIDGE_NATIVE) += early_thermal.c early_pch_native.c early_me_native.c early_usb_native.c
-romstage-$(CONFIG_NORTHBRIDGE_INTEL_SANDYBRIDGE_NATIVE) += early_thermal.c early_pch_native.c early_me_native.c early_usb_native.c
+romstage-$(CONFIG_NORTHBRIDGE_INTEL_IVYBRIDGE_MRC) += early_me_mrc.c early_usb_mrc.c
+romstage-$(CONFIG_NORTHBRIDGE_INTEL_SANDYBRIDGE_MRC) += early_me_mrc.c early_usb_mrc.c
+romstage-$(CONFIG_NORTHBRIDGE_INTEL_IVYBRIDGE) += early_thermal.c early_pch.c early_me.c early_usb.c
+romstage-$(CONFIG_NORTHBRIDGE_INTEL_SANDYBRIDGE) += early_thermal.c early_pch.c early_me.c early_usb.c
 
 ramstage-y += madt.c
 
diff --git a/src/southbridge/intel/bd82x6x/early_me.c b/src/southbridge/intel/bd82x6x/early_me.c
index b7dbf3439f..ed9351fc16 100644
--- a/src/southbridge/intel/bd82x6x/early_me.c
+++ b/src/southbridge/intel/bd82x6x/early_me.c
@@ -44,13 +44,6 @@ static inline void pci_read_dword_ptr(void *ptr, int offset)
 	memcpy(ptr, &dword, sizeof(dword));
 }
 
-static inline void pci_write_dword_ptr(void *ptr, int offset)
-{
-	u32 dword = 0;
-	memcpy(&dword, ptr, sizeof(dword));
-	pci_write_config32(PCH_ME_DEV, offset, dword);
-}
-
 void intel_early_me_status(void)
 {
 	struct me_hfs hfs;
@@ -125,64 +118,136 @@ static inline void set_global_reset(int enable)
 
 int intel_early_me_init_done(u8 status)
 {
-	u8 reset;
-	int count;
+	u8 reset, errorcode, opmode;
+	u16 reg16;
 	u32 mebase_l, mebase_h;
-	struct me_hfs hfs;
+	u32 millisec;
+	u32 hfs, me_fws2;
 	struct me_did did = {
 		.init_done = ME_INIT_DONE,
 		.status = status
 	};
+	u32 meDID;
+
+	hfs = (pci_read_config32(PCI_DEV(0, 0x16, 0), PCI_ME_HFS) & 0xff000) >> 12;
+
+	opmode = (hfs & 0xf0) >> 4;
+	errorcode = hfs & 0xf;
+
+	if (opmode != ME_HFS_MODE_NORMAL) {
+		printk(BIOS_NOTICE, "ME: Wrong mode : %d\n", opmode);
+		//return 0;
+	}
+	if (errorcode) {
+		printk(BIOS_NOTICE, "ME: HFS error : %d\n", errorcode);
+		//return 0;
+	}
+
+	me_fws2 = pci_read_config32(PCI_DEV(0, 0x16, 0), 0x48);
+	printk(BIOS_NOTICE, "ME: FWS2: 0x%x\n", me_fws2);
+	printk(BIOS_NOTICE, "ME:  Bist in progress: 0x%x\n", me_fws2 & 0x1);
+	printk(BIOS_NOTICE, "ME:  ICC Status      : 0x%x\n", (me_fws2 & 0x6) >> 1);
+	printk(BIOS_NOTICE, "ME:  Invoke MEBx     : 0x%x\n", (me_fws2 & 0x8) >> 3);
+	printk(BIOS_NOTICE, "ME:  CPU replaced    : 0x%x\n", (me_fws2 & 0x10) >> 4);
+	printk(BIOS_NOTICE, "ME:  MBP ready       : 0x%x\n", (me_fws2 & 0x20) >> 5);
+	printk(BIOS_NOTICE, "ME:  MFS failure     : 0x%x\n", (me_fws2 & 0x40) >> 6);
+	printk(BIOS_NOTICE, "ME:  Warm reset req  : 0x%x\n", (me_fws2 & 0x80) >> 7);
+	printk(BIOS_NOTICE, "ME:  CPU repl valid  : 0x%x\n", (me_fws2 & 0x100) >> 8);
+	printk(BIOS_NOTICE, "ME:  (Reserved)      : 0x%x\n", (me_fws2 & 0x600) >> 9);
+	printk(BIOS_NOTICE, "ME:  FW update req   : 0x%x\n", (me_fws2 & 0x800) >> 11);
+	printk(BIOS_NOTICE, "ME:  (Reserved)      : 0x%x\n", (me_fws2 & 0xf000) >> 12);
+	printk(BIOS_NOTICE, "ME:  Current state   : 0x%x\n", (me_fws2 & 0xff0000) >> 16);
+	printk(BIOS_NOTICE, "ME:  Current PM event: 0x%x\n", (me_fws2 & 0xf000000) >> 24);
+	printk(BIOS_NOTICE, "ME:  Progress code   : 0x%x\n", (me_fws2 & 0xf0000000) >> 28);
+
+	// Poll cpu replaced for 50ms
+	millisec = 0;
+	while ((((me_fws2 & 0x100) >> 8) == 0) && millisec < 50) {
+		udelay(1000);
+		me_fws2 = pci_read_config32(PCI_DEV(0, 0x16, 0), 0x48);
+		millisec++;
+	}
+	if (millisec >= 50 || ((me_fws2 & 0x100) >> 8) == 0x0) {
+		printk(BIOS_NOTICE, "Waited long enough, or CPU was not replaced, continue...\n");
+	} else if ((me_fws2 & 0x100) == 0x100) {
+		if ((me_fws2 & 0x80) == 0x80) {
+			printk(BIOS_NOTICE, "CPU was replaced & warm reset required...\n");
+			reg16 = pcie_read_config16(PCI_DEV(0, 31, 0), 0xa2) & ~0x80;
+			pcie_write_config16(PCI_DEV(0, 31, 0), 0xa2, reg16);
+			set_global_reset(0);
+			outb(0x6, 0xcf9);
+			halt();
+		}
+
+		if (((me_fws2 & 0x10) == 0x10) && (me_fws2 & 0x80) == 0x00) {
+			printk(BIOS_NOTICE, "Full training required\n");
+		}
+	}
+
+	printk(BIOS_NOTICE, "PASSED! Tell ME that DRAM is ready\n");
 
 	/* MEBASE from MESEG_BASE[35:20] */
 	mebase_l = pci_read_config32(PCI_CPU_DEVICE, PCI_CPU_MEBASE_L);
 	mebase_h = pci_read_config32(PCI_CPU_DEVICE, PCI_CPU_MEBASE_H) & 0xf;
 	did.uma_base = (mebase_l >> 20) | (mebase_h << 12);
 
-	/* Send message to ME */
-	printk(BIOS_DEBUG, "ME: Sending Init Done with status: %d, "
-	       "UMA base: 0x%04x\n", status, did.uma_base);
+	meDID = did.uma_base | (1 << 28);// | (1 << 23);
+	pci_write_config32(PCI_DEV(0, 0x16, 0), PCI_ME_H_GS, meDID);
 
-	pci_write_dword_ptr(&did, PCI_ME_H_GS);
+	udelay(1100);
 
 	/* Must wait for ME acknowledgement */
-	for (count = ME_RETRY; count > 0; --count) {
-		pci_read_dword_ptr(&hfs, PCI_ME_HFS);
-		if (hfs.bios_msg_ack)
-			break;
-		udelay(ME_DELAY);
-	}
-	if (!count) {
-		printk(BIOS_ERR, "ERROR: ME failed to respond\n");
-		return -1;
+	millisec = 0;
+	hfs = (pci_read_config32(PCI_DEV(0, 0x16, 0), PCI_ME_HFS) & 0xfe000000) >> 24;
+	while ((((hfs & 0xf0) >> 4) != ME_HFS_BIOS_DRAM_ACK) && (millisec < 5000)) {
+		udelay(1000);
+		hfs = (pci_read_config32(PCI_DEV(0, 0x16, 0), PCI_ME_HFS) & 0xfe000000) >> 24;
+		millisec++;
 	}
 
+	me_fws2 = pci_read_config32(PCI_DEV(0, 0x16, 0), 0x48);
+	printk(BIOS_NOTICE, "ME: FWS2: 0x%x\n", me_fws2);
+	printk(BIOS_NOTICE, "ME:  Bist in progress: 0x%x\n", me_fws2 & 0x1);
+	printk(BIOS_NOTICE, "ME:  ICC Status      : 0x%x\n", (me_fws2 & 0x6) >> 1);
+	printk(BIOS_NOTICE, "ME:  Invoke MEBx     : 0x%x\n", (me_fws2 & 0x8) >> 3);
+	printk(BIOS_NOTICE, "ME:  CPU replaced    : 0x%x\n", (me_fws2 & 0x10) >> 4);
+	printk(BIOS_NOTICE, "ME:  MBP ready       : 0x%x\n", (me_fws2 & 0x20) >> 5);
+	printk(BIOS_NOTICE, "ME:  MFS failure     : 0x%x\n", (me_fws2 & 0x40) >> 6);
+	printk(BIOS_NOTICE, "ME:  Warm reset req  : 0x%x\n", (me_fws2 & 0x80) >> 7);
+	printk(BIOS_NOTICE, "ME:  CPU repl valid  : 0x%x\n", (me_fws2 & 0x100) >> 8);
+	printk(BIOS_NOTICE, "ME:  (Reserved)      : 0x%x\n", (me_fws2 & 0x600) >> 9);
+	printk(BIOS_NOTICE, "ME:  FW update req   : 0x%x\n", (me_fws2 & 0x800) >> 11);
+	printk(BIOS_NOTICE, "ME:  (Reserved)      : 0x%x\n", (me_fws2 & 0xf000) >> 12);
+	printk(BIOS_NOTICE, "ME:  Current state   : 0x%x\n", (me_fws2 & 0xff0000) >> 16);
+	printk(BIOS_NOTICE, "ME:  Current PM event: 0x%x\n", (me_fws2 & 0xf000000) >> 24);
+	printk(BIOS_NOTICE, "ME:  Progress code   : 0x%x\n", (me_fws2 & 0xf0000000) >> 28);
+
+
 	/* Return the requested BIOS action */
 	printk(BIOS_NOTICE, "ME: Requested BIOS Action: %s\n",
-	       me_ack_values[hfs.ack_data]);
-
-	/* Check status after acknowledgement */
-	intel_early_me_status();
+		me_ack_values[(hfs & 0xe) >> 1]);
 
-	reset = 0;
-	switch (hfs.ack_data) {
+	reset = inb(0xcf9);
+	reset &= 0xf1;
+	switch ((hfs & 0xe) >> 1) {
+	case ME_HFS_ACK_NO_DID:
 	case ME_HFS_ACK_CONTINUE:
 		/* Continue to boot */
 		return 0;
 	case ME_HFS_ACK_RESET:
 		/* Non-power cycle reset */
 		set_global_reset(0);
-		reset = 0x06;
+		reset |= 0x06;
 		break;
 	case ME_HFS_ACK_PWR_CYCLE:
 		/* Power cycle reset */
 		set_global_reset(0);
-		reset = 0x0e;
+		reset |= 0x0e;
 		break;
 	case ME_HFS_ACK_GBL_RESET:
 		/* Global reset */
 		set_global_reset(1);
-		reset = 0x0e;
+		reset |= 0x0e;
 		break;
 	case ME_HFS_ACK_S3:
 	case ME_HFS_ACK_S4:
diff --git a/src/southbridge/intel/bd82x6x/early_me_mrc.c b/src/southbridge/intel/bd82x6x/early_me_mrc.c
new file mode 100644
index 0000000000..b7dbf3439f
--- /dev/null
+++ b/src/southbridge/intel/bd82x6x/early_me_mrc.c
@@ -0,0 +1,199 @@
+/*
+ * This file is part of the coreboot project.
+ *
+ * Copyright (C) 2011 The Chromium OS Authors. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; version 2 of
+ * the License.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc.
+ */
+
+#include <arch/io.h>
+#include <console/console.h>
+#include <delay.h>
+#include <device/pci_ids.h>
+#include <halt.h>
+#include <string.h>
+#include "me.h"
+#include "pch.h"
+
+static const char *me_ack_values[] = {
+	[ME_HFS_ACK_NO_DID]	= "No DID Ack received",
+	[ME_HFS_ACK_RESET]	= "Non-power cycle reset",
+	[ME_HFS_ACK_PWR_CYCLE]	= "Power cycle reset",
+	[ME_HFS_ACK_S3]		= "Go to S3",
+	[ME_HFS_ACK_S4]		= "Go to S4",
+	[ME_HFS_ACK_S5]		= "Go to S5",
+	[ME_HFS_ACK_GBL_RESET]	= "Global Reset",
+	[ME_HFS_ACK_CONTINUE]	= "Continue to boot"
+};
+
+static inline void pci_read_dword_ptr(void *ptr, int offset)
+{
+	u32 dword = pci_read_config32(PCH_ME_DEV, offset);
+	memcpy(ptr, &dword, sizeof(dword));
+}
+
+static inline void pci_write_dword_ptr(void *ptr, int offset)
+{
+	u32 dword = 0;
+	memcpy(&dword, ptr, sizeof(dword));
+	pci_write_config32(PCH_ME_DEV, offset, dword);
+}
+
+void intel_early_me_status(void)
+{
+	struct me_hfs hfs;
+	struct me_gmes gmes;
+
+	pci_read_dword_ptr(&hfs, PCI_ME_HFS);
+	pci_read_dword_ptr(&gmes, PCI_ME_GMES);
+
+	intel_me_status(&hfs, &gmes);
+}
+
+int intel_early_me_init(void)
+{
+	int count;
+	struct me_uma uma;
+	struct me_hfs hfs;
+
+	printk(BIOS_INFO, "Intel ME early init\n");
+
+	/* Wait for ME UMA SIZE VALID bit to be set */
+	for (count = ME_RETRY; count > 0; --count) {
+		pci_read_dword_ptr(&uma, PCI_ME_UMA);
+		if (uma.valid)
+			break;
+		udelay(ME_DELAY);
+	}
+	if (!count) {
+		printk(BIOS_ERR, "ERROR: ME is not ready!\n");
+		return -1;
+	}
+
+	/* Check for valid firmware */
+	pci_read_dword_ptr(&hfs, PCI_ME_HFS);
+	if (hfs.fpt_bad) {
+		printk(BIOS_WARNING, "WARNING: ME has bad firmware\n");
+		return -1;
+	}
+
+	printk(BIOS_INFO, "Intel ME firmware is ready\n");
+	return 0;
+}
+
+int intel_early_me_uma_size(void)
+{
+	struct me_uma uma;
+
+	pci_read_dword_ptr(&uma, PCI_ME_UMA);
+	if (uma.valid) {
+		printk(BIOS_DEBUG, "ME: Requested %uMB UMA\n", uma.size);
+		return uma.size;
+	}
+
+	printk(BIOS_DEBUG, "ME: Invalid UMA size\n");
+	return 0;
+}
+
+static inline void set_global_reset(int enable)
+{
+	u32 etr3 = pci_read_config32(PCH_LPC_DEV, ETR3);
+
+	/* Clear CF9 Without Resume Well Reset Enable */
+	etr3 &= ~ETR3_CWORWRE;
+
+	/* CF9GR indicates a Global Reset */
+	if (enable)
+		etr3 |= ETR3_CF9GR;
+	else
+		etr3 &= ~ETR3_CF9GR;
+
+	pci_write_config32(PCH_LPC_DEV, ETR3, etr3);
+}
+
+int intel_early_me_init_done(u8 status)
+{
+	u8 reset;
+	int count;
+	u32 mebase_l, mebase_h;
+	struct me_hfs hfs;
+	struct me_did did = {
+		.init_done = ME_INIT_DONE,
+		.status = status
+	};
+
+	/* MEBASE from MESEG_BASE[35:20] */
+	mebase_l = pci_read_config32(PCI_CPU_DEVICE, PCI_CPU_MEBASE_L);
+	mebase_h = pci_read_config32(PCI_CPU_DEVICE, PCI_CPU_MEBASE_H) & 0xf;
+	did.uma_base = (mebase_l >> 20) | (mebase_h << 12);
+
+	/* Send message to ME */
+	printk(BIOS_DEBUG, "ME: Sending Init Done with status: %d, "
+	       "UMA base: 0x%04x\n", status, did.uma_base);
+
+	pci_write_dword_ptr(&did, PCI_ME_H_GS);
+
+	/* Must wait for ME acknowledgement */
+	for (count = ME_RETRY; count > 0; --count) {
+		pci_read_dword_ptr(&hfs, PCI_ME_HFS);
+		if (hfs.bios_msg_ack)
+			break;
+		udelay(ME_DELAY);
+	}
+	if (!count) {
+		printk(BIOS_ERR, "ERROR: ME failed to respond\n");
+		return -1;
+	}
+
+	/* Return the requested BIOS action */
+	printk(BIOS_NOTICE, "ME: Requested BIOS Action: %s\n",
+	       me_ack_values[hfs.ack_data]);
+
+	/* Check status after acknowledgement */
+	intel_early_me_status();
+
+	reset = 0;
+	switch (hfs.ack_data) {
+	case ME_HFS_ACK_CONTINUE:
+		/* Continue to boot */
+		return 0;
+	case ME_HFS_ACK_RESET:
+		/* Non-power cycle reset */
+		set_global_reset(0);
+		reset = 0x06;
+		break;
+	case ME_HFS_ACK_PWR_CYCLE:
+		/* Power cycle reset */
+		set_global_reset(0);
+		reset = 0x0e;
+		break;
+	case ME_HFS_ACK_GBL_RESET:
+		/* Global reset */
+		set_global_reset(1);
+		reset = 0x0e;
+		break;
+	case ME_HFS_ACK_S3:
+	case ME_HFS_ACK_S4:
+	case ME_HFS_ACK_S5:
+		break;
+	}
+
+	/* Perform the requested reset */
+	if (reset) {
+		outb(reset, 0xcf9);
+		halt();
+	}
+	return -1;
+}
diff --git a/src/southbridge/intel/bd82x6x/early_me_native.c b/src/southbridge/intel/bd82x6x/early_me_native.c
deleted file mode 100644
index ed9351fc16..0000000000
--- a/src/southbridge/intel/bd82x6x/early_me_native.c
+++ /dev/null
@@ -1,264 +0,0 @@
-/*
- * This file is part of the coreboot project.
- *
- * Copyright (C) 2011 The Chromium OS Authors. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License as
- * published by the Free Software Foundation; version 2 of
- * the License.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc.
- */
-
-#include <arch/io.h>
-#include <console/console.h>
-#include <delay.h>
-#include <device/pci_ids.h>
-#include <halt.h>
-#include <string.h>
-#include "me.h"
-#include "pch.h"
-
-static const char *me_ack_values[] = {
-	[ME_HFS_ACK_NO_DID]	= "No DID Ack received",
-	[ME_HFS_ACK_RESET]	= "Non-power cycle reset",
-	[ME_HFS_ACK_PWR_CYCLE]	= "Power cycle reset",
-	[ME_HFS_ACK_S3]		= "Go to S3",
-	[ME_HFS_ACK_S4]		= "Go to S4",
-	[ME_HFS_ACK_S5]		= "Go to S5",
-	[ME_HFS_ACK_GBL_RESET]	= "Global Reset",
-	[ME_HFS_ACK_CONTINUE]	= "Continue to boot"
-};
-
-static inline void pci_read_dword_ptr(void *ptr, int offset)
-{
-	u32 dword = pci_read_config32(PCH_ME_DEV, offset);
-	memcpy(ptr, &dword, sizeof(dword));
-}
-
-void intel_early_me_status(void)
-{
-	struct me_hfs hfs;
-	struct me_gmes gmes;
-
-	pci_read_dword_ptr(&hfs, PCI_ME_HFS);
-	pci_read_dword_ptr(&gmes, PCI_ME_GMES);
-
-	intel_me_status(&hfs, &gmes);
-}
-
-int intel_early_me_init(void)
-{
-	int count;
-	struct me_uma uma;
-	struct me_hfs hfs;
-
-	printk(BIOS_INFO, "Intel ME early init\n");
-
-	/* Wait for ME UMA SIZE VALID bit to be set */
-	for (count = ME_RETRY; count > 0; --count) {
-		pci_read_dword_ptr(&uma, PCI_ME_UMA);
-		if (uma.valid)
-			break;
-		udelay(ME_DELAY);
-	}
-	if (!count) {
-		printk(BIOS_ERR, "ERROR: ME is not ready!\n");
-		return -1;
-	}
-
-	/* Check for valid firmware */
-	pci_read_dword_ptr(&hfs, PCI_ME_HFS);
-	if (hfs.fpt_bad) {
-		printk(BIOS_WARNING, "WARNING: ME has bad firmware\n");
-		return -1;
-	}
-
-	printk(BIOS_INFO, "Intel ME firmware is ready\n");
-	return 0;
-}
-
-int intel_early_me_uma_size(void)
-{
-	struct me_uma uma;
-
-	pci_read_dword_ptr(&uma, PCI_ME_UMA);
-	if (uma.valid) {
-		printk(BIOS_DEBUG, "ME: Requested %uMB UMA\n", uma.size);
-		return uma.size;
-	}
-
-	printk(BIOS_DEBUG, "ME: Invalid UMA size\n");
-	return 0;
-}
-
-static inline void set_global_reset(int enable)
-{
-	u32 etr3 = pci_read_config32(PCH_LPC_DEV, ETR3);
-
-	/* Clear CF9 Without Resume Well Reset Enable */
-	etr3 &= ~ETR3_CWORWRE;
-
-	/* CF9GR indicates a Global Reset */
-	if (enable)
-		etr3 |= ETR3_CF9GR;
-	else
-		etr3 &= ~ETR3_CF9GR;
-
-	pci_write_config32(PCH_LPC_DEV, ETR3, etr3);
-}
-
-int intel_early_me_init_done(u8 status)
-{
-	u8 reset, errorcode, opmode;
-	u16 reg16;
-	u32 mebase_l, mebase_h;
-	u32 millisec;
-	u32 hfs, me_fws2;
-	struct me_did did = {
-		.init_done = ME_INIT_DONE,
-		.status = status
-	};
-	u32 meDID;
-
-	hfs = (pci_read_config32(PCI_DEV(0, 0x16, 0), PCI_ME_HFS) & 0xff000) >> 12;
-
-	opmode = (hfs & 0xf0) >> 4;
-	errorcode = hfs & 0xf;
-
-	if (opmode != ME_HFS_MODE_NORMAL) {
-		printk(BIOS_NOTICE, "ME: Wrong mode : %d\n", opmode);
-		//return 0;
-	}
-	if (errorcode) {
-		printk(BIOS_NOTICE, "ME: HFS error : %d\n", errorcode);
-		//return 0;
-	}
-
-	me_fws2 = pci_read_config32(PCI_DEV(0, 0x16, 0), 0x48);
-	printk(BIOS_NOTICE, "ME: FWS2: 0x%x\n", me_fws2);
-	printk(BIOS_NOTICE, "ME:  Bist in progress: 0x%x\n", me_fws2 & 0x1);
-	printk(BIOS_NOTICE, "ME:  ICC Status      : 0x%x\n", (me_fws2 & 0x6) >> 1);
-	printk(BIOS_NOTICE, "ME:  Invoke MEBx     : 0x%x\n", (me_fws2 & 0x8) >> 3);
-	printk(BIOS_NOTICE, "ME:  CPU replaced    : 0x%x\n", (me_fws2 & 0x10) >> 4);
-	printk(BIOS_NOTICE, "ME:  MBP ready       : 0x%x\n", (me_fws2 & 0x20) >> 5);
-	printk(BIOS_NOTICE, "ME:  MFS failure     : 0x%x\n", (me_fws2 & 0x40) >> 6);
-	printk(BIOS_NOTICE, "ME:  Warm reset req  : 0x%x\n", (me_fws2 & 0x80) >> 7);
-	printk(BIOS_NOTICE, "ME:  CPU repl valid  : 0x%x\n", (me_fws2 & 0x100) >> 8);
-	printk(BIOS_NOTICE, "ME:  (Reserved)      : 0x%x\n", (me_fws2 & 0x600) >> 9);
-	printk(BIOS_NOTICE, "ME:  FW update req   : 0x%x\n", (me_fws2 & 0x800) >> 11);
-	printk(BIOS_NOTICE, "ME:  (Reserved)      : 0x%x\n", (me_fws2 & 0xf000) >> 12);
-	printk(BIOS_NOTICE, "ME:  Current state   : 0x%x\n", (me_fws2 & 0xff0000) >> 16);
-	printk(BIOS_NOTICE, "ME:  Current PM event: 0x%x\n", (me_fws2 & 0xf000000) >> 24);
-	printk(BIOS_NOTICE, "ME:  Progress code   : 0x%x\n", (me_fws2 & 0xf0000000) >> 28);
-
-	// Poll cpu replaced for 50ms
-	millisec = 0;
-	while ((((me_fws2 & 0x100) >> 8) == 0) && millisec < 50) {
-		udelay(1000);
-		me_fws2 = pci_read_config32(PCI_DEV(0, 0x16, 0), 0x48);
-		millisec++;
-	}
-	if (millisec >= 50 || ((me_fws2 & 0x100) >> 8) == 0x0) {
-		printk(BIOS_NOTICE, "Waited long enough, or CPU was not replaced, continue...\n");
-	} else if ((me_fws2 & 0x100) == 0x100) {
-		if ((me_fws2 & 0x80) == 0x80) {
-			printk(BIOS_NOTICE, "CPU was replaced & warm reset required...\n");
-			reg16 = pcie_read_config16(PCI_DEV(0, 31, 0), 0xa2) & ~0x80;
-			pcie_write_config16(PCI_DEV(0, 31, 0), 0xa2, reg16);
-			set_global_reset(0);
-			outb(0x6, 0xcf9);
-			halt();
-		}
-
-		if (((me_fws2 & 0x10) == 0x10) && (me_fws2 & 0x80) == 0x00) {
-			printk(BIOS_NOTICE, "Full training required\n");
-		}
-	}
-
-	printk(BIOS_NOTICE, "PASSED! Tell ME that DRAM is ready\n");
-
-	/* MEBASE from MESEG_BASE[35:20] */
-	mebase_l = pci_read_config32(PCI_CPU_DEVICE, PCI_CPU_MEBASE_L);
-	mebase_h = pci_read_config32(PCI_CPU_DEVICE, PCI_CPU_MEBASE_H) & 0xf;
-	did.uma_base = (mebase_l >> 20) | (mebase_h << 12);
-
-	meDID = did.uma_base | (1 << 28);// | (1 << 23);
-	pci_write_config32(PCI_DEV(0, 0x16, 0), PCI_ME_H_GS, meDID);
-
-	udelay(1100);
-
-	/* Must wait for ME acknowledgement */
-	millisec = 0;
-	hfs = (pci_read_config32(PCI_DEV(0, 0x16, 0), PCI_ME_HFS) & 0xfe000000) >> 24;
-	while ((((hfs & 0xf0) >> 4) != ME_HFS_BIOS_DRAM_ACK) && (millisec < 5000)) {
-		udelay(1000);
-		hfs = (pci_read_config32(PCI_DEV(0, 0x16, 0), PCI_ME_HFS) & 0xfe000000) >> 24;
-		millisec++;
-	}
-
-	me_fws2 = pci_read_config32(PCI_DEV(0, 0x16, 0), 0x48);
-	printk(BIOS_NOTICE, "ME: FWS2: 0x%x\n", me_fws2);
-	printk(BIOS_NOTICE, "ME:  Bist in progress: 0x%x\n", me_fws2 & 0x1);
-	printk(BIOS_NOTICE, "ME:  ICC Status      : 0x%x\n", (me_fws2 & 0x6) >> 1);
-	printk(BIOS_NOTICE, "ME:  Invoke MEBx     : 0x%x\n", (me_fws2 & 0x8) >> 3);
-	printk(BIOS_NOTICE, "ME:  CPU replaced    : 0x%x\n", (me_fws2 & 0x10) >> 4);
-	printk(BIOS_NOTICE, "ME:  MBP ready       : 0x%x\n", (me_fws2 & 0x20) >> 5);
-	printk(BIOS_NOTICE, "ME:  MFS failure     : 0x%x\n", (me_fws2 & 0x40) >> 6);
-	printk(BIOS_NOTICE, "ME:  Warm reset req  : 0x%x\n", (me_fws2 & 0x80) >> 7);
-	printk(BIOS_NOTICE, "ME:  CPU repl valid  : 0x%x\n", (me_fws2 & 0x100) >> 8);
-	printk(BIOS_NOTICE, "ME:  (Reserved)      : 0x%x\n", (me_fws2 & 0x600) >> 9);
-	printk(BIOS_NOTICE, "ME:  FW update req   : 0x%x\n", (me_fws2 & 0x800) >> 11);
-	printk(BIOS_NOTICE, "ME:  (Reserved)      : 0x%x\n", (me_fws2 & 0xf000) >> 12);
-	printk(BIOS_NOTICE, "ME:  Current state   : 0x%x\n", (me_fws2 & 0xff0000) >> 16);
-	printk(BIOS_NOTICE, "ME:  Current PM event: 0x%x\n", (me_fws2 & 0xf000000) >> 24);
-	printk(BIOS_NOTICE, "ME:  Progress code   : 0x%x\n", (me_fws2 & 0xf0000000) >> 28);
-
-
-	/* Return the requested BIOS action */
-	printk(BIOS_NOTICE, "ME: Requested BIOS Action: %s\n",
-		me_ack_values[(hfs & 0xe) >> 1]);
-
-	reset = inb(0xcf9);
-	reset &= 0xf1;
-	switch ((hfs & 0xe) >> 1) {
-	case ME_HFS_ACK_NO_DID:
-	case ME_HFS_ACK_CONTINUE:
-		/* Continue to boot */
-		return 0;
-	case ME_HFS_ACK_RESET:
-		/* Non-power cycle reset */
-		set_global_reset(0);
-		reset |= 0x06;
-		break;
-	case ME_HFS_ACK_PWR_CYCLE:
-		/* Power cycle reset */
-		set_global_reset(0);
-		reset |= 0x0e;
-		break;
-	case ME_HFS_ACK_GBL_RESET:
-		/* Global reset */
-		set_global_reset(1);
-		reset |= 0x0e;
-		break;
-	case ME_HFS_ACK_S3:
-	case ME_HFS_ACK_S4:
-	case ME_HFS_ACK_S5:
-		break;
-	}
-
-	/* Perform the requested reset */
-	if (reset) {
-		outb(reset, 0xcf9);
-		halt();
-	}
-	return -1;
-}
diff --git a/src/southbridge/intel/bd82x6x/early_pch.c b/src/southbridge/intel/bd82x6x/early_pch.c
index 3852875b90..4dcdb8fff8 100644
--- a/src/southbridge/intel/bd82x6x/early_pch.c
+++ b/src/southbridge/intel/bd82x6x/early_pch.c
@@ -1,12 +1,11 @@
 /*
  * This file is part of the coreboot project.
  *
- * Copyright (C) 2008-2009 coresystems GmbH
+ * Copyright (C) 2014 Vladimir Serbinenko <phcoder@gmail.com>
  *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License as
- * published by the Free Software Foundation; version 2 of
- * the License.
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; version 2 of the License.
  *
  * This program is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
@@ -18,45 +17,358 @@
  * Foundation, Inc.
  */
 
+#include <console/console.h>
+#include <string.h>
 #include <arch/io.h>
-#include <timestamp.h>
-#include <cpu/x86/tsc.h>
+#include <cbmem.h>
+#include <arch/cbfs.h>
+#include <cbfs.h>
+#include <ip_checksum.h>
+#include <pc80/mc146818rtc.h>
+#include <device/pci_def.h>
+#include <delay.h>
+
 #include "pch.h"
-#include <arch/acpi.h>
-#include <console/console.h>
+/* For DMI bar.  */
+#include "northbridge/intel/sandybridge/sandybridge.h"
+
+#define SOUTHBRIDGE PCI_DEV(0, 0x1f, 0)
 
-uint64_t get_initial_timestamp(void)
+static void
+wait_2338 (void)
 {
-	tsc_t base_time = {
-		.lo = pci_read_config32(PCI_DEV(0, 0x00, 0), 0xdc),
-		.hi = pci_read_config32(PCI_DEV(0, 0x1f, 2), 0xd0)
-	};
-	return tsc_to_uint64(base_time);
+	while (read8 (DEFAULT_RCBA + 0x2338) & 1);
 }
 
-int southbridge_detect_s3_resume(void)
+static u32
+read_2338 (u32 edx)
+{
+	u32 ret;
+
+	write32 (DEFAULT_RCBA + 0x2330, edx);
+	write16 (DEFAULT_RCBA + 0x2338, (read16 (DEFAULT_RCBA + 0x2338)
+					 & 0x1ff) | 0x600);
+	wait_2338 ();
+	ret = read32 (DEFAULT_RCBA + 0x2334);
+	wait_2338 ();
+	read8 (DEFAULT_RCBA + 0x2338);
+	return ret;
+}
+
+static void
+write_2338 (u32 edx, u32 val)
+{
+	read_2338 (edx);
+	write16 (DEFAULT_RCBA + 0x2338, (read16 (DEFAULT_RCBA + 0x2338)
+					 & 0x1ff) | 0x600);
+	wait_2338 ();
+
+	write32 (DEFAULT_RCBA + 0x2334, val);
+	wait_2338 ();
+	write16 (DEFAULT_RCBA + 0x2338,
+		 (read16 (DEFAULT_RCBA + 0x2338) & 0x1ff) | 0x600);
+	read8 (DEFAULT_RCBA + 0x2338);
+}
+
+
+static void
+init_dmi (void)
 {
-	u32 pm1_cnt;
-	u16 pm1_sts;
-
-	/* Check PM1_STS[15] to see if we are waking from Sx */
-	pm1_sts = inw(DEFAULT_PMBASE + PM1_STS);
-
-	/* Read PM1_CNT[12:10] to determine which Sx state */
-	pm1_cnt = inl(DEFAULT_PMBASE + PM1_CNT);
-
-	if ((pm1_sts & WAK_STS) && ((pm1_cnt >> 10) & 7) == 5) {
-		if (acpi_s3_resume_allowed()) {
-			printk(BIOS_DEBUG, "Resume from S3 detected.\n");
-			/* Clear SLP_TYPE. This will break stage2 but
-			 * we care for that when we get there.
-			 */
-			outl(pm1_cnt & ~(7 << 10), DEFAULT_PMBASE + PM1_CNT);
-			return 1;
-		} else {
-			printk(BIOS_DEBUG, "Resume from S3 detected, but disabled.\n");
-		}
+	int i;
+
+	write32 (DEFAULT_DMIBAR + 0x0914,
+		 read32 (DEFAULT_DMIBAR + 0x0914) | 0x80000000);
+	write32 (DEFAULT_DMIBAR + 0x0934,
+		 read32 (DEFAULT_DMIBAR + 0x0934) | 0x80000000);
+	for (i = 0; i < 4; i++)
+	{
+		write32 (DEFAULT_DMIBAR + 0x0a00 + (i << 4),
+			 read32 (DEFAULT_DMIBAR + 0x0a00 + (i << 4)) & 0xf3ffffff);
+		write32 (DEFAULT_DMIBAR + 0x0a04 + (i << 4),
+			 read32 (DEFAULT_DMIBAR + 0x0a04 + (i << 4)) | 0x800);
+	}
+	write32 (DEFAULT_DMIBAR + 0x0c30, (read32 (DEFAULT_DMIBAR + 0x0c30)
+					   & 0xfffffff) | 0x40000000);
+	for (i = 0; i < 2; i++)
+	{
+		write32 (DEFAULT_DMIBAR + 0x0904 + (i << 5),
+			 read32 (DEFAULT_DMIBAR + 0x0904 + (i << 5)) & 0xfe3fffff);
+		write32 (DEFAULT_DMIBAR + 0x090c + (i << 5),
+			 read32 (DEFAULT_DMIBAR + 0x090c + (i << 5)) & 0xfff1ffff);
 	}
+	write32 (DEFAULT_DMIBAR + 0x090c,
+		 read32 (DEFAULT_DMIBAR + 0x090c) & 0xfe1fffff);
+	write32 (DEFAULT_DMIBAR + 0x092c,
+		 read32 (DEFAULT_DMIBAR + 0x092c) & 0xfe1fffff);
+	read32 (DEFAULT_DMIBAR + 0x0904);	// !!! = 0x7a1842ec
+	write32 (DEFAULT_DMIBAR + 0x0904, 0x7a1842ec);
+	read32 (DEFAULT_DMIBAR + 0x090c);	// !!! = 0x00000208
+	write32 (DEFAULT_DMIBAR + 0x090c, 0x00000128);
+	read32 (DEFAULT_DMIBAR + 0x0924);	// !!! = 0x7a1842ec
+	write32 (DEFAULT_DMIBAR + 0x0924, 0x7a1842ec);
+	read32 (DEFAULT_DMIBAR + 0x092c);	// !!! = 0x00000208
+	write32 (DEFAULT_DMIBAR + 0x092c, 0x00000128);
+	read32 (DEFAULT_DMIBAR + 0x0700);	// !!! = 0x46139008
+	write32 (DEFAULT_DMIBAR + 0x0700, 0x46139008);
+	read32 (DEFAULT_DMIBAR + 0x0720);	// !!! = 0x46139008
+	write32 (DEFAULT_DMIBAR + 0x0720, 0x46139008);
+	read32 (DEFAULT_DMIBAR + 0x0c04);	// !!! = 0x2e680008
+	write32 (DEFAULT_DMIBAR + 0x0c04, 0x2e680008);
+	read32 (DEFAULT_DMIBAR + 0x0904);	// !!! = 0x7a1842ec
+	write32 (DEFAULT_DMIBAR + 0x0904, 0x3a1842ec);
+	read32 (DEFAULT_DMIBAR + 0x0924);	// !!! = 0x7a1842ec
+	write32 (DEFAULT_DMIBAR + 0x0924, 0x3a1842ec);
+	read32 (DEFAULT_DMIBAR + 0x0910);	// !!! = 0x00006300
+	write32 (DEFAULT_DMIBAR + 0x0910, 0x00004300);
+	read32 (DEFAULT_DMIBAR + 0x0930);	// !!! = 0x00006300
+	write32 (DEFAULT_DMIBAR + 0x0930, 0x00004300);
+	read32 (DEFAULT_DMIBAR + 0x0a00);	// !!! = 0x03042010
+	write32 (DEFAULT_DMIBAR + 0x0a00, 0x03042018);
+	read32 (DEFAULT_DMIBAR + 0x0a10);	// !!! = 0x03042010
+	write32 (DEFAULT_DMIBAR + 0x0a10, 0x03042018);
+	read32 (DEFAULT_DMIBAR + 0x0a20);	// !!! = 0x03042010
+	write32 (DEFAULT_DMIBAR + 0x0a20, 0x03042018);
+	read32 (DEFAULT_DMIBAR + 0x0a30);	// !!! = 0x03042010
+	write32 (DEFAULT_DMIBAR + 0x0a30, 0x03042018);
+	read32 (DEFAULT_DMIBAR + 0x0c00);	// !!! = 0x29700c08
+	write32 (DEFAULT_DMIBAR + 0x0c00, 0x29700c08);
+	read32 (DEFAULT_DMIBAR + 0x0a04);	// !!! = 0x0c0708f0
+	write32 (DEFAULT_DMIBAR + 0x0a04, 0x0c0718f0);
+	read32 (DEFAULT_DMIBAR + 0x0a14);	// !!! = 0x0c0708f0
+	write32 (DEFAULT_DMIBAR + 0x0a14, 0x0c0718f0);
+	read32 (DEFAULT_DMIBAR + 0x0a24);	// !!! = 0x0c0708f0
+	write32 (DEFAULT_DMIBAR + 0x0a24, 0x0c0718f0);
+	read32 (DEFAULT_DMIBAR + 0x0a34);	// !!! = 0x0c0708f0
+	write32 (DEFAULT_DMIBAR + 0x0a34, 0x0c0718f0);
+	read32 (DEFAULT_DMIBAR + 0x0900);	// !!! = 0x50000000
+	write32 (DEFAULT_DMIBAR + 0x0900, 0x50000000);
+	read32 (DEFAULT_DMIBAR + 0x0920);	// !!! = 0x50000000
+	write32 (DEFAULT_DMIBAR + 0x0920, 0x50000000);
+	read32 (DEFAULT_DMIBAR + 0x0908);	// !!! = 0x51ffffff
+	write32 (DEFAULT_DMIBAR + 0x0908, 0x51ffffff);
+	read32 (DEFAULT_DMIBAR + 0x0928);	// !!! = 0x51ffffff
+	write32 (DEFAULT_DMIBAR + 0x0928, 0x51ffffff);
+	read32 (DEFAULT_DMIBAR + 0x0a00);	// !!! = 0x03042018
+	write32 (DEFAULT_DMIBAR + 0x0a00, 0x03042018);
+	read32 (DEFAULT_DMIBAR + 0x0a10);	// !!! = 0x03042018
+	write32 (DEFAULT_DMIBAR + 0x0a10, 0x03042018);
+	read32 (DEFAULT_DMIBAR + 0x0a20);	// !!! = 0x03042018
+	write32 (DEFAULT_DMIBAR + 0x0a20, 0x03042018);
+	read32 (DEFAULT_DMIBAR + 0x0a30);	// !!! = 0x03042018
+	write32 (DEFAULT_DMIBAR + 0x0a30, 0x03042018);
+	read32 (DEFAULT_DMIBAR + 0x0700);	// !!! = 0x46139008
+	write32 (DEFAULT_DMIBAR + 0x0700, 0x46139008);
+	read32 (DEFAULT_DMIBAR + 0x0720);	// !!! = 0x46139008
+	write32 (DEFAULT_DMIBAR + 0x0720, 0x46139008);
+	read32 (DEFAULT_DMIBAR + 0x0904);	// !!! = 0x3a1842ec
+	write32 (DEFAULT_DMIBAR + 0x0904, 0x3a1846ec);
+	read32 (DEFAULT_DMIBAR + 0x0924);	// !!! = 0x3a1842ec
+	write32 (DEFAULT_DMIBAR + 0x0924, 0x3a1846ec);
+	read32 (DEFAULT_DMIBAR + 0x0a00);	// !!! = 0x03042018
+	write32 (DEFAULT_DMIBAR + 0x0a00, 0x03042018);
+	read32 (DEFAULT_DMIBAR + 0x0a10);	// !!! = 0x03042018
+	write32 (DEFAULT_DMIBAR + 0x0a10, 0x03042018);
+	read32 (DEFAULT_DMIBAR + 0x0a20);	// !!! = 0x03042018
+	write32 (DEFAULT_DMIBAR + 0x0a20, 0x03042018);
+	read32 (DEFAULT_DMIBAR + 0x0a30);	// !!! = 0x03042018
+	write32 (DEFAULT_DMIBAR + 0x0a30, 0x03042018);
+	read32 (DEFAULT_DMIBAR + 0x0908);	// !!! = 0x51ffffff
+	write32 (DEFAULT_DMIBAR + 0x0908, 0x51ffffff);
+	read32 (DEFAULT_DMIBAR + 0x0928);	// !!! = 0x51ffffff
+	write32 (DEFAULT_DMIBAR + 0x0928, 0x51ffffff);
+	read32 (DEFAULT_DMIBAR + 0x0c00);	// !!! = 0x29700c08
+	write32 (DEFAULT_DMIBAR + 0x0c00, 0x29700c08);
+	read32 (DEFAULT_DMIBAR + 0x0c0c);	// !!! = 0x16063400
+	write32 (DEFAULT_DMIBAR + 0x0c0c, 0x00063400);
+	read32 (DEFAULT_DMIBAR + 0x0700);	// !!! = 0x46139008
+	write32 (DEFAULT_DMIBAR + 0x0700, 0x46339008);
+	read32 (DEFAULT_DMIBAR + 0x0720);	// !!! = 0x46139008
+	write32 (DEFAULT_DMIBAR + 0x0720, 0x46339008);
+	read32 (DEFAULT_DMIBAR + 0x0700);	// !!! = 0x46339008
+	write32 (DEFAULT_DMIBAR + 0x0700, 0x45339008);
+	read32 (DEFAULT_DMIBAR + 0x0720);	// !!! = 0x46339008
+	write32 (DEFAULT_DMIBAR + 0x0720, 0x45339008);
+	read32 (DEFAULT_DMIBAR + 0x0700);	// !!! = 0x45339008
+	write32 (DEFAULT_DMIBAR + 0x0700, 0x453b9008);
+	read32 (DEFAULT_DMIBAR + 0x0720);	// !!! = 0x45339008
+	write32 (DEFAULT_DMIBAR + 0x0720, 0x453b9008);
+	read32 (DEFAULT_DMIBAR + 0x0700);	// !!! = 0x453b9008
+	write32 (DEFAULT_DMIBAR + 0x0700, 0x45bb9008);
+	read32 (DEFAULT_DMIBAR + 0x0720);	// !!! = 0x453b9008
+	write32 (DEFAULT_DMIBAR + 0x0720, 0x45bb9008);
+	read32 (DEFAULT_DMIBAR + 0x0700);	// !!! = 0x45bb9008
+	write32 (DEFAULT_DMIBAR + 0x0700, 0x45fb9008);
+	read32 (DEFAULT_DMIBAR + 0x0720);	// !!! = 0x45bb9008
+	write32 (DEFAULT_DMIBAR + 0x0720, 0x45fb9008);
+	read32 (DEFAULT_DMIBAR + 0x0914);	// !!! = 0x9021a080
+	write32 (DEFAULT_DMIBAR + 0x0914, 0x9021a280);
+	read32 (DEFAULT_DMIBAR + 0x0934);	// !!! = 0x9021a080
+	write32 (DEFAULT_DMIBAR + 0x0934, 0x9021a280);
+	read32 (DEFAULT_DMIBAR + 0x0914);	// !!! = 0x9021a280
+	write32 (DEFAULT_DMIBAR + 0x0914, 0x9821a280);
+	read32 (DEFAULT_DMIBAR + 0x0934);	// !!! = 0x9021a280
+	write32 (DEFAULT_DMIBAR + 0x0934, 0x9821a280);
+	read32 (DEFAULT_DMIBAR + 0x0a00);	// !!! = 0x03042018
+	write32 (DEFAULT_DMIBAR + 0x0a00, 0x03242018);
+	read32 (DEFAULT_DMIBAR + 0x0a10);	// !!! = 0x03042018
+	write32 (DEFAULT_DMIBAR + 0x0a10, 0x03242018);
+	read32 (DEFAULT_DMIBAR + 0x0a20);	// !!! = 0x03042018
+	write32 (DEFAULT_DMIBAR + 0x0a20, 0x03242018);
+	read32 (DEFAULT_DMIBAR + 0x0a30);	// !!! = 0x03042018
+	write32 (DEFAULT_DMIBAR + 0x0a30, 0x03242018);
+	read32 (DEFAULT_DMIBAR + 0x0258);	// !!! = 0x40000600
+	write32 (DEFAULT_DMIBAR + 0x0258, 0x60000600);
+	read32 (DEFAULT_DMIBAR + 0x0904);	// !!! = 0x3a1846ec
+	write32 (DEFAULT_DMIBAR + 0x0904, 0x2a1846ec);
+	read32 (DEFAULT_DMIBAR + 0x0914);	// !!! = 0x9821a280
+	write32 (DEFAULT_DMIBAR + 0x0914, 0x98200280);
+	read32 (DEFAULT_DMIBAR + 0x0924);	// !!! = 0x3a1846ec
+	write32 (DEFAULT_DMIBAR + 0x0924, 0x2a1846ec);
+	read32 (DEFAULT_DMIBAR + 0x0934);	// !!! = 0x9821a280
+	write32 (DEFAULT_DMIBAR + 0x0934, 0x98200280);
+	read32 (DEFAULT_DMIBAR + 0x022c);	// !!! = 0x00c26460
+	write32 (DEFAULT_DMIBAR + 0x022c, 0x00c2403c);
+	read8 (DEFAULT_RCBA + 0x21a4);	// !!! = 0x42
+
+	read32 (DEFAULT_RCBA + 0x21a4);	// !!! = 0x00012c42
+	read32 (DEFAULT_RCBA + 0x2340);	// !!! = 0x0013001b
+	write32 (DEFAULT_RCBA + 0x2340, 0x003a001b);
+	read8 (DEFAULT_RCBA + 0x21b0);	// !!! = 0x01
+	write8 (DEFAULT_RCBA + 0x21b0, 0x02);
+	read32 (DEFAULT_DMIBAR + 0x0084);	// !!! = 0x0041ac41
+	write32 (DEFAULT_DMIBAR + 0x0084, 0x0041ac42);
+	read8 (DEFAULT_DMIBAR + 0x0088);	// !!! = 0x00
+	write8 (DEFAULT_DMIBAR + 0x0088, 0x20);
+	read16 (DEFAULT_DMIBAR + 0x008a);	// !!! = 0x0041
+	read8 (DEFAULT_DMIBAR + 0x0088);	// !!! = 0x00
+	write8 (DEFAULT_DMIBAR + 0x0088, 0x20);
+	read16 (DEFAULT_DMIBAR + 0x008a);	// !!! = 0x0042
+	read16 (DEFAULT_DMIBAR + 0x008a);	// !!! = 0x0042
+
+	read32 (DEFAULT_DMIBAR + 0x0014);	// !!! = 0x8000007f
+	write32 (DEFAULT_DMIBAR + 0x0014, 0x80000019);
+	read32 (DEFAULT_DMIBAR + 0x0020);	// !!! = 0x01000000
+	write32 (DEFAULT_DMIBAR + 0x0020, 0x81000022);
+	read32 (DEFAULT_DMIBAR + 0x002c);	// !!! = 0x02000000
+	write32 (DEFAULT_DMIBAR + 0x002c, 0x82000044);
+	read32 (DEFAULT_DMIBAR + 0x0038);	// !!! = 0x07000080
+	write32 (DEFAULT_DMIBAR + 0x0038, 0x87000080);
+	read8 (DEFAULT_DMIBAR + 0x0004);	// !!! = 0x00
+	write8 (DEFAULT_DMIBAR + 0x0004, 0x01);
+
+	read32 (DEFAULT_RCBA + 0x0050);	// !!! = 0x01200654
+	write32 (DEFAULT_RCBA + 0x0050, 0x01200654);
+	read32 (DEFAULT_RCBA + 0x0050);	// !!! = 0x01200654
+	write32 (DEFAULT_RCBA + 0x0050, 0x012a0654);
+	read32 (DEFAULT_RCBA + 0x0050);	// !!! = 0x012a0654
+	read8 (DEFAULT_RCBA + 0x1114);	// !!! = 0x00
+	write8 (DEFAULT_RCBA + 0x1114, 0x05);
+	read32 (DEFAULT_RCBA + 0x2014);	// !!! = 0x80000011
+	write32 (DEFAULT_RCBA + 0x2014, 0x80000019);
+	read32 (DEFAULT_RCBA + 0x2020);	// !!! = 0x00000000
+	write32 (DEFAULT_RCBA + 0x2020, 0x81000022);
+	read32 (DEFAULT_RCBA + 0x2020);	// !!! = 0x81000022
+	read32 (DEFAULT_RCBA + 0x2030);	// !!! = 0x00000000
+	write32 (DEFAULT_RCBA + 0x2030, 0x82000044);
+	read32 (DEFAULT_RCBA + 0x2030);	// !!! = 0x82000044
+	read32 (DEFAULT_RCBA + 0x2040);	// !!! = 0x00000000
+	write32 (DEFAULT_RCBA + 0x2040, 0x87000080);
+	read32 (DEFAULT_RCBA + 0x0050);	// !!! = 0x012a0654
+	write32 (DEFAULT_RCBA + 0x0050, 0x812a0654);
+	read32 (DEFAULT_RCBA + 0x0050);	// !!! = 0x812a0654
+	read16 (DEFAULT_RCBA + 0x201a);	// !!! = 0x0000
+	read16 (DEFAULT_RCBA + 0x2026);	// !!! = 0x0000
+	read16 (DEFAULT_RCBA + 0x2036);	// !!! = 0x0000
+	read16 (DEFAULT_RCBA + 0x2046);	// !!! = 0x0000
+	read16 (DEFAULT_DMIBAR + 0x001a);	// !!! = 0x0000
+	read16 (DEFAULT_DMIBAR + 0x0026);	// !!! = 0x0000
+	read16 (DEFAULT_DMIBAR + 0x0032);	// !!! = 0x0000
+	read16 (DEFAULT_DMIBAR + 0x003e);	// !!! = 0x0000
+}
+
+void
+early_pch_init_native (void)
+{
+	pcie_write_config8 (SOUTHBRIDGE, 0xa6,
+			    pcie_read_config8 (SOUTHBRIDGE, 0xa6) | 2);
+
+	write32 (DEFAULT_RCBA + 0x2088, 0x00109000);
+	read32 (DEFAULT_RCBA + 0x20ac);	// !!! = 0x00000000
+	write32 (DEFAULT_RCBA + 0x20ac, 0x40000000);
+	write32 (DEFAULT_RCBA + 0x100c, 0x01110000);
+	write8 (DEFAULT_RCBA + 0x2340, 0x1b);
+	read32 (DEFAULT_RCBA + 0x2314);	// !!! = 0x0a080000
+	write32 (DEFAULT_RCBA + 0x2314, 0x0a280000);
+	read32 (DEFAULT_RCBA + 0x2310);	// !!! = 0xc809605b
+	write32 (DEFAULT_RCBA + 0x2310, 0xa809605b);
+	write32 (DEFAULT_RCBA + 0x2324, 0x00854c74);
+	read8 (DEFAULT_RCBA + 0x0400);	// !!! = 0x00
+	read32 (DEFAULT_RCBA + 0x2310);	// !!! = 0xa809605b
+	write32 (DEFAULT_RCBA + 0x2310, 0xa809605b);
+	read32 (DEFAULT_RCBA + 0x2310);	// !!! = 0xa809605b
+	write32 (DEFAULT_RCBA + 0x2310, 0xa809605b);
+
+	write_2338 (0xea007f62, 0x00590133);
+	write_2338 (0xec007f62, 0x00590133);
+	write_2338 (0xec007f64, 0x59555588);
+	write_2338 (0xea0040b9, 0x0001051c);
+	write_2338 (0xeb0040a1, 0x800084ff);
+	write_2338 (0xec0040a1, 0x800084ff);
+	write_2338 (0xea004001, 0x00008400);
+	write_2338 (0xeb004002, 0x40201758);
+	write_2338 (0xec004002, 0x40201758);
+	write_2338 (0xea004002, 0x00601758);
+	write_2338 (0xea0040a1, 0x810084ff);
+	write_2338 (0xeb0040b1, 0x0001c598);
+	write_2338 (0xec0040b1, 0x0001c598);
+	write_2338 (0xeb0040b6, 0x0001c598);
+	write_2338 (0xea0000a9, 0x80ff969f);
+	write_2338 (0xea0001a9, 0x80ff969f);
+	write_2338 (0xeb0040b2, 0x0001c396);
+	write_2338 (0xeb0040b3, 0x0001c396);
+	write_2338 (0xec0040b2, 0x0001c396);
+	write_2338 (0xea0001a9, 0x80ff94ff);
+	write_2338 (0xea000151, 0x0088037f);
+	write_2338 (0xea0000a9, 0x80ff94ff);
+	write_2338 (0xea000051, 0x0088037f);
+
+	write_2338 (0xea007f05, 0x00010642);
+	write_2338 (0xea0040b7, 0x0001c91c);
+	write_2338 (0xea0040b8, 0x0001c91c);
+	write_2338 (0xeb0040a1, 0x820084ff);
+	write_2338 (0xec0040a1, 0x820084ff);
+	write_2338 (0xea007f0a, 0xc2480000);
+
+	write_2338 (0xec00404d, 0x1ff177f);
+	write_2338 (0xec000084, 0x5a600000);
+	write_2338 (0xec000184, 0x5a600000);
+	write_2338 (0xec000284, 0x5a600000);
+	write_2338 (0xec000384, 0x5a600000);
+	write_2338 (0xec000094, 0x000f0501);
+	write_2338 (0xec000194, 0x000f0501);
+	write_2338 (0xec000294, 0x000f0501);
+	write_2338 (0xec000394, 0x000f0501);
+	write_2338 (0xec000096, 0x00000001);
+	write_2338 (0xec000196, 0x00000001);
+	write_2338 (0xec000296, 0x00000001);
+	write_2338 (0xec000396, 0x00000001);
+	write_2338 (0xec000001, 0x00008c08);
+	write_2338 (0xec000101, 0x00008c08);
+	write_2338 (0xec000201, 0x00008c08);
+	write_2338 (0xec000301, 0x00008c08);
+	write_2338 (0xec0040b5, 0x0001c518);
+	write_2338 (0xec000087, 0x06077597);
+	write_2338 (0xec000187, 0x06077597);
+	write_2338 (0xec000287, 0x06077597);
+	write_2338 (0xec000387, 0x06077597);
+	write_2338 (0xea000050, 0x00bb0157);
+	write_2338 (0xea000150, 0x00bb0157);
+	write_2338 (0xec007f60, 0x77777d77);
+	write_2338 (0xea00008d, 0x01320000);
+	write_2338 (0xea00018d, 0x01320000);
+	write_2338 (0xec0007b2, 0x04514b5e);
+	write_2338 (0xec00078c, 0x40000200);
+	write_2338 (0xec000780, 0x02000020);
 
-	return 0;
+	init_dmi();
 }
diff --git a/src/southbridge/intel/bd82x6x/early_pch_common.c b/src/southbridge/intel/bd82x6x/early_pch_common.c
new file mode 100644
index 0000000000..3852875b90
--- /dev/null
+++ b/src/southbridge/intel/bd82x6x/early_pch_common.c
@@ -0,0 +1,62 @@
+/*
+ * This file is part of the coreboot project.
+ *
+ * Copyright (C) 2008-2009 coresystems GmbH
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; version 2 of
+ * the License.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc.
+ */
+
+#include <arch/io.h>
+#include <timestamp.h>
+#include <cpu/x86/tsc.h>
+#include "pch.h"
+#include <arch/acpi.h>
+#include <console/console.h>
+
+uint64_t get_initial_timestamp(void)
+{
+	tsc_t base_time = {
+		.lo = pci_read_config32(PCI_DEV(0, 0x00, 0), 0xdc),
+		.hi = pci_read_config32(PCI_DEV(0, 0x1f, 2), 0xd0)
+	};
+	return tsc_to_uint64(base_time);
+}
+
+int southbridge_detect_s3_resume(void)
+{
+	u32 pm1_cnt;
+	u16 pm1_sts;
+
+	/* Check PM1_STS[15] to see if we are waking from Sx */
+	pm1_sts = inw(DEFAULT_PMBASE + PM1_STS);
+
+	/* Read PM1_CNT[12:10] to determine which Sx state */
+	pm1_cnt = inl(DEFAULT_PMBASE + PM1_CNT);
+
+	if ((pm1_sts & WAK_STS) && ((pm1_cnt >> 10) & 7) == 5) {
+		if (acpi_s3_resume_allowed()) {
+			printk(BIOS_DEBUG, "Resume from S3 detected.\n");
+			/* Clear SLP_TYPE. This will break stage2 but
+			 * we care for that when we get there.
+			 */
+			outl(pm1_cnt & ~(7 << 10), DEFAULT_PMBASE + PM1_CNT);
+			return 1;
+		} else {
+			printk(BIOS_DEBUG, "Resume from S3 detected, but disabled.\n");
+		}
+	}
+
+	return 0;
+}
diff --git a/src/southbridge/intel/bd82x6x/early_pch_native.c b/src/southbridge/intel/bd82x6x/early_pch_native.c
deleted file mode 100644
index 4dcdb8fff8..0000000000
--- a/src/southbridge/intel/bd82x6x/early_pch_native.c
+++ /dev/null
@@ -1,374 +0,0 @@
-/*
- * This file is part of the coreboot project.
- *
- * Copyright (C) 2014 Vladimir Serbinenko <phcoder@gmail.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; version 2 of the License.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc.
- */
-
-#include <console/console.h>
-#include <string.h>
-#include <arch/io.h>
-#include <cbmem.h>
-#include <arch/cbfs.h>
-#include <cbfs.h>
-#include <ip_checksum.h>
-#include <pc80/mc146818rtc.h>
-#include <device/pci_def.h>
-#include <delay.h>
-
-#include "pch.h"
-/* For DMI bar.  */
-#include "northbridge/intel/sandybridge/sandybridge.h"
-
-#define SOUTHBRIDGE PCI_DEV(0, 0x1f, 0)
-
-static void
-wait_2338 (void)
-{
-	while (read8 (DEFAULT_RCBA + 0x2338) & 1);
-}
-
-static u32
-read_2338 (u32 edx)
-{
-	u32 ret;
-
-	write32 (DEFAULT_RCBA + 0x2330, edx);
-	write16 (DEFAULT_RCBA + 0x2338, (read16 (DEFAULT_RCBA + 0x2338)
-					 & 0x1ff) | 0x600);
-	wait_2338 ();
-	ret = read32 (DEFAULT_RCBA + 0x2334);
-	wait_2338 ();
-	read8 (DEFAULT_RCBA + 0x2338);
-	return ret;
-}
-
-static void
-write_2338 (u32 edx, u32 val)
-{
-	read_2338 (edx);
-	write16 (DEFAULT_RCBA + 0x2338, (read16 (DEFAULT_RCBA + 0x2338)
-					 & 0x1ff) | 0x600);
-	wait_2338 ();
-
-	write32 (DEFAULT_RCBA + 0x2334, val);
-	wait_2338 ();
-	write16 (DEFAULT_RCBA + 0x2338,
-		 (read16 (DEFAULT_RCBA + 0x2338) & 0x1ff) | 0x600);
-	read8 (DEFAULT_RCBA + 0x2338);
-}
-
-
-static void
-init_dmi (void)
-{
-	int i;
-
-	write32 (DEFAULT_DMIBAR + 0x0914,
-		 read32 (DEFAULT_DMIBAR + 0x0914) | 0x80000000);
-	write32 (DEFAULT_DMIBAR + 0x0934,
-		 read32 (DEFAULT_DMIBAR + 0x0934) | 0x80000000);
-	for (i = 0; i < 4; i++)
-	{
-		write32 (DEFAULT_DMIBAR + 0x0a00 + (i << 4),
-			 read32 (DEFAULT_DMIBAR + 0x0a00 + (i << 4)) & 0xf3ffffff);
-		write32 (DEFAULT_DMIBAR + 0x0a04 + (i << 4),
-			 read32 (DEFAULT_DMIBAR + 0x0a04 + (i << 4)) | 0x800);
-	}
-	write32 (DEFAULT_DMIBAR + 0x0c30, (read32 (DEFAULT_DMIBAR + 0x0c30)
-					   & 0xfffffff) | 0x40000000);
-	for (i = 0; i < 2; i++)
-	{
-		write32 (DEFAULT_DMIBAR + 0x0904 + (i << 5),
-			 read32 (DEFAULT_DMIBAR + 0x0904 + (i << 5)) & 0xfe3fffff);
-		write32 (DEFAULT_DMIBAR + 0x090c + (i << 5),
-			 read32 (DEFAULT_DMIBAR + 0x090c + (i << 5)) & 0xfff1ffff);
-	}
-	write32 (DEFAULT_DMIBAR + 0x090c,
-		 read32 (DEFAULT_DMIBAR + 0x090c) & 0xfe1fffff);
-	write32 (DEFAULT_DMIBAR + 0x092c,
-		 read32 (DEFAULT_DMIBAR + 0x092c) & 0xfe1fffff);
-	read32 (DEFAULT_DMIBAR + 0x0904);	// !!! = 0x7a1842ec
-	write32 (DEFAULT_DMIBAR + 0x0904, 0x7a1842ec);
-	read32 (DEFAULT_DMIBAR + 0x090c);	// !!! = 0x00000208
-	write32 (DEFAULT_DMIBAR + 0x090c, 0x00000128);
-	read32 (DEFAULT_DMIBAR + 0x0924);	// !!! = 0x7a1842ec
-	write32 (DEFAULT_DMIBAR + 0x0924, 0x7a1842ec);
-	read32 (DEFAULT_DMIBAR + 0x092c);	// !!! = 0x00000208
-	write32 (DEFAULT_DMIBAR + 0x092c, 0x00000128);
-	read32 (DEFAULT_DMIBAR + 0x0700);	// !!! = 0x46139008
-	write32 (DEFAULT_DMIBAR + 0x0700, 0x46139008);
-	read32 (DEFAULT_DMIBAR + 0x0720);	// !!! = 0x46139008
-	write32 (DEFAULT_DMIBAR + 0x0720, 0x46139008);
-	read32 (DEFAULT_DMIBAR + 0x0c04);	// !!! = 0x2e680008
-	write32 (DEFAULT_DMIBAR + 0x0c04, 0x2e680008);
-	read32 (DEFAULT_DMIBAR + 0x0904);	// !!! = 0x7a1842ec
-	write32 (DEFAULT_DMIBAR + 0x0904, 0x3a1842ec);
-	read32 (DEFAULT_DMIBAR + 0x0924);	// !!! = 0x7a1842ec
-	write32 (DEFAULT_DMIBAR + 0x0924, 0x3a1842ec);
-	read32 (DEFAULT_DMIBAR + 0x0910);	// !!! = 0x00006300
-	write32 (DEFAULT_DMIBAR + 0x0910, 0x00004300);
-	read32 (DEFAULT_DMIBAR + 0x0930);	// !!! = 0x00006300
-	write32 (DEFAULT_DMIBAR + 0x0930, 0x00004300);
-	read32 (DEFAULT_DMIBAR + 0x0a00);	// !!! = 0x03042010
-	write32 (DEFAULT_DMIBAR + 0x0a00, 0x03042018);
-	read32 (DEFAULT_DMIBAR + 0x0a10);	// !!! = 0x03042010
-	write32 (DEFAULT_DMIBAR + 0x0a10, 0x03042018);
-	read32 (DEFAULT_DMIBAR + 0x0a20);	// !!! = 0x03042010
-	write32 (DEFAULT_DMIBAR + 0x0a20, 0x03042018);
-	read32 (DEFAULT_DMIBAR + 0x0a30);	// !!! = 0x03042010
-	write32 (DEFAULT_DMIBAR + 0x0a30, 0x03042018);
-	read32 (DEFAULT_DMIBAR + 0x0c00);	// !!! = 0x29700c08
-	write32 (DEFAULT_DMIBAR + 0x0c00, 0x29700c08);
-	read32 (DEFAULT_DMIBAR + 0x0a04);	// !!! = 0x0c0708f0
-	write32 (DEFAULT_DMIBAR + 0x0a04, 0x0c0718f0);
-	read32 (DEFAULT_DMIBAR + 0x0a14);	// !!! = 0x0c0708f0
-	write32 (DEFAULT_DMIBAR + 0x0a14, 0x0c0718f0);
-	read32 (DEFAULT_DMIBAR + 0x0a24);	// !!! = 0x0c0708f0
-	write32 (DEFAULT_DMIBAR + 0x0a24, 0x0c0718f0);
-	read32 (DEFAULT_DMIBAR + 0x0a34);	// !!! = 0x0c0708f0
-	write32 (DEFAULT_DMIBAR + 0x0a34, 0x0c0718f0);
-	read32 (DEFAULT_DMIBAR + 0x0900);	// !!! = 0x50000000
-	write32 (DEFAULT_DMIBAR + 0x0900, 0x50000000);
-	read32 (DEFAULT_DMIBAR + 0x0920);	// !!! = 0x50000000
-	write32 (DEFAULT_DMIBAR + 0x0920, 0x50000000);
-	read32 (DEFAULT_DMIBAR + 0x0908);	// !!! = 0x51ffffff
-	write32 (DEFAULT_DMIBAR + 0x0908, 0x51ffffff);
-	read32 (DEFAULT_DMIBAR + 0x0928);	// !!! = 0x51ffffff
-	write32 (DEFAULT_DMIBAR + 0x0928, 0x51ffffff);
-	read32 (DEFAULT_DMIBAR + 0x0a00);	// !!! = 0x03042018
-	write32 (DEFAULT_DMIBAR + 0x0a00, 0x03042018);
-	read32 (DEFAULT_DMIBAR + 0x0a10);	// !!! = 0x03042018
-	write32 (DEFAULT_DMIBAR + 0x0a10, 0x03042018);
-	read32 (DEFAULT_DMIBAR + 0x0a20);	// !!! = 0x03042018
-	write32 (DEFAULT_DMIBAR + 0x0a20, 0x03042018);
-	read32 (DEFAULT_DMIBAR + 0x0a30);	// !!! = 0x03042018
-	write32 (DEFAULT_DMIBAR + 0x0a30, 0x03042018);
-	read32 (DEFAULT_DMIBAR + 0x0700);	// !!! = 0x46139008
-	write32 (DEFAULT_DMIBAR + 0x0700, 0x46139008);
-	read32 (DEFAULT_DMIBAR + 0x0720);	// !!! = 0x46139008
-	write32 (DEFAULT_DMIBAR + 0x0720, 0x46139008);
-	read32 (DEFAULT_DMIBAR + 0x0904);	// !!! = 0x3a1842ec
-	write32 (DEFAULT_DMIBAR + 0x0904, 0x3a1846ec);
-	read32 (DEFAULT_DMIBAR + 0x0924);	// !!! = 0x3a1842ec
-	write32 (DEFAULT_DMIBAR + 0x0924, 0x3a1846ec);
-	read32 (DEFAULT_DMIBAR + 0x0a00);	// !!! = 0x03042018
-	write32 (DEFAULT_DMIBAR + 0x0a00, 0x03042018);
-	read32 (DEFAULT_DMIBAR + 0x0a10);	// !!! = 0x03042018
-	write32 (DEFAULT_DMIBAR + 0x0a10, 0x03042018);
-	read32 (DEFAULT_DMIBAR + 0x0a20);	// !!! = 0x03042018
-	write32 (DEFAULT_DMIBAR + 0x0a20, 0x03042018);
-	read32 (DEFAULT_DMIBAR + 0x0a30);	// !!! = 0x03042018
-	write32 (DEFAULT_DMIBAR + 0x0a30, 0x03042018);
-	read32 (DEFAULT_DMIBAR + 0x0908);	// !!! = 0x51ffffff
-	write32 (DEFAULT_DMIBAR + 0x0908, 0x51ffffff);
-	read32 (DEFAULT_DMIBAR + 0x0928);	// !!! = 0x51ffffff
-	write32 (DEFAULT_DMIBAR + 0x0928, 0x51ffffff);
-	read32 (DEFAULT_DMIBAR + 0x0c00);	// !!! = 0x29700c08
-	write32 (DEFAULT_DMIBAR + 0x0c00, 0x29700c08);
-	read32 (DEFAULT_DMIBAR + 0x0c0c);	// !!! = 0x16063400
-	write32 (DEFAULT_DMIBAR + 0x0c0c, 0x00063400);
-	read32 (DEFAULT_DMIBAR + 0x0700);	// !!! = 0x46139008
-	write32 (DEFAULT_DMIBAR + 0x0700, 0x46339008);
-	read32 (DEFAULT_DMIBAR + 0x0720);	// !!! = 0x46139008
-	write32 (DEFAULT_DMIBAR + 0x0720, 0x46339008);
-	read32 (DEFAULT_DMIBAR + 0x0700);	// !!! = 0x46339008
-	write32 (DEFAULT_DMIBAR + 0x0700, 0x45339008);
-	read32 (DEFAULT_DMIBAR + 0x0720);	// !!! = 0x46339008
-	write32 (DEFAULT_DMIBAR + 0x0720, 0x45339008);
-	read32 (DEFAULT_DMIBAR + 0x0700);	// !!! = 0x45339008
-	write32 (DEFAULT_DMIBAR + 0x0700, 0x453b9008);
-	read32 (DEFAULT_DMIBAR + 0x0720);	// !!! = 0x45339008
-	write32 (DEFAULT_DMIBAR + 0x0720, 0x453b9008);
-	read32 (DEFAULT_DMIBAR + 0x0700);	// !!! = 0x453b9008
-	write32 (DEFAULT_DMIBAR + 0x0700, 0x45bb9008);
-	read32 (DEFAULT_DMIBAR + 0x0720);	// !!! = 0x453b9008
-	write32 (DEFAULT_DMIBAR + 0x0720, 0x45bb9008);
-	read32 (DEFAULT_DMIBAR + 0x0700);	// !!! = 0x45bb9008
-	write32 (DEFAULT_DMIBAR + 0x0700, 0x45fb9008);
-	read32 (DEFAULT_DMIBAR + 0x0720);	// !!! = 0x45bb9008
-	write32 (DEFAULT_DMIBAR + 0x0720, 0x45fb9008);
-	read32 (DEFAULT_DMIBAR + 0x0914);	// !!! = 0x9021a080
-	write32 (DEFAULT_DMIBAR + 0x0914, 0x9021a280);
-	read32 (DEFAULT_DMIBAR + 0x0934);	// !!! = 0x9021a080
-	write32 (DEFAULT_DMIBAR + 0x0934, 0x9021a280);
-	read32 (DEFAULT_DMIBAR + 0x0914);	// !!! = 0x9021a280
-	write32 (DEFAULT_DMIBAR + 0x0914, 0x9821a280);
-	read32 (DEFAULT_DMIBAR + 0x0934);	// !!! = 0x9021a280
-	write32 (DEFAULT_DMIBAR + 0x0934, 0x9821a280);
-	read32 (DEFAULT_DMIBAR + 0x0a00);	// !!! = 0x03042018
-	write32 (DEFAULT_DMIBAR + 0x0a00, 0x03242018);
-	read32 (DEFAULT_DMIBAR + 0x0a10);	// !!! = 0x03042018
-	write32 (DEFAULT_DMIBAR + 0x0a10, 0x03242018);
-	read32 (DEFAULT_DMIBAR + 0x0a20);	// !!! = 0x03042018
-	write32 (DEFAULT_DMIBAR + 0x0a20, 0x03242018);
-	read32 (DEFAULT_DMIBAR + 0x0a30);	// !!! = 0x03042018
-	write32 (DEFAULT_DMIBAR + 0x0a30, 0x03242018);
-	read32 (DEFAULT_DMIBAR + 0x0258);	// !!! = 0x40000600
-	write32 (DEFAULT_DMIBAR + 0x0258, 0x60000600);
-	read32 (DEFAULT_DMIBAR + 0x0904);	// !!! = 0x3a1846ec
-	write32 (DEFAULT_DMIBAR + 0x0904, 0x2a1846ec);
-	read32 (DEFAULT_DMIBAR + 0x0914);	// !!! = 0x9821a280
-	write32 (DEFAULT_DMIBAR + 0x0914, 0x98200280);
-	read32 (DEFAULT_DMIBAR + 0x0924);	// !!! = 0x3a1846ec
-	write32 (DEFAULT_DMIBAR + 0x0924, 0x2a1846ec);
-	read32 (DEFAULT_DMIBAR + 0x0934);	// !!! = 0x9821a280
-	write32 (DEFAULT_DMIBAR + 0x0934, 0x98200280);
-	read32 (DEFAULT_DMIBAR + 0x022c);	// !!! = 0x00c26460
-	write32 (DEFAULT_DMIBAR + 0x022c, 0x00c2403c);
-	read8 (DEFAULT_RCBA + 0x21a4);	// !!! = 0x42
-
-	read32 (DEFAULT_RCBA + 0x21a4);	// !!! = 0x00012c42
-	read32 (DEFAULT_RCBA + 0x2340);	// !!! = 0x0013001b
-	write32 (DEFAULT_RCBA + 0x2340, 0x003a001b);
-	read8 (DEFAULT_RCBA + 0x21b0);	// !!! = 0x01
-	write8 (DEFAULT_RCBA + 0x21b0, 0x02);
-	read32 (DEFAULT_DMIBAR + 0x0084);	// !!! = 0x0041ac41
-	write32 (DEFAULT_DMIBAR + 0x0084, 0x0041ac42);
-	read8 (DEFAULT_DMIBAR + 0x0088);	// !!! = 0x00
-	write8 (DEFAULT_DMIBAR + 0x0088, 0x20);
-	read16 (DEFAULT_DMIBAR + 0x008a);	// !!! = 0x0041
-	read8 (DEFAULT_DMIBAR + 0x0088);	// !!! = 0x00
-	write8 (DEFAULT_DMIBAR + 0x0088, 0x20);
-	read16 (DEFAULT_DMIBAR + 0x008a);	// !!! = 0x0042
-	read16 (DEFAULT_DMIBAR + 0x008a);	// !!! = 0x0042
-
-	read32 (DEFAULT_DMIBAR + 0x0014);	// !!! = 0x8000007f
-	write32 (DEFAULT_DMIBAR + 0x0014, 0x80000019);
-	read32 (DEFAULT_DMIBAR + 0x0020);	// !!! = 0x01000000
-	write32 (DEFAULT_DMIBAR + 0x0020, 0x81000022);
-	read32 (DEFAULT_DMIBAR + 0x002c);	// !!! = 0x02000000
-	write32 (DEFAULT_DMIBAR + 0x002c, 0x82000044);
-	read32 (DEFAULT_DMIBAR + 0x0038);	// !!! = 0x07000080
-	write32 (DEFAULT_DMIBAR + 0x0038, 0x87000080);
-	read8 (DEFAULT_DMIBAR + 0x0004);	// !!! = 0x00
-	write8 (DEFAULT_DMIBAR + 0x0004, 0x01);
-
-	read32 (DEFAULT_RCBA + 0x0050);	// !!! = 0x01200654
-	write32 (DEFAULT_RCBA + 0x0050, 0x01200654);
-	read32 (DEFAULT_RCBA + 0x0050);	// !!! = 0x01200654
-	write32 (DEFAULT_RCBA + 0x0050, 0x012a0654);
-	read32 (DEFAULT_RCBA + 0x0050);	// !!! = 0x012a0654
-	read8 (DEFAULT_RCBA + 0x1114);	// !!! = 0x00
-	write8 (DEFAULT_RCBA + 0x1114, 0x05);
-	read32 (DEFAULT_RCBA + 0x2014);	// !!! = 0x80000011
-	write32 (DEFAULT_RCBA + 0x2014, 0x80000019);
-	read32 (DEFAULT_RCBA + 0x2020);	// !!! = 0x00000000
-	write32 (DEFAULT_RCBA + 0x2020, 0x81000022);
-	read32 (DEFAULT_RCBA + 0x2020);	// !!! = 0x81000022
-	read32 (DEFAULT_RCBA + 0x2030);	// !!! = 0x00000000
-	write32 (DEFAULT_RCBA + 0x2030, 0x82000044);
-	read32 (DEFAULT_RCBA + 0x2030);	// !!! = 0x82000044
-	read32 (DEFAULT_RCBA + 0x2040);	// !!! = 0x00000000
-	write32 (DEFAULT_RCBA + 0x2040, 0x87000080);
-	read32 (DEFAULT_RCBA + 0x0050);	// !!! = 0x012a0654
-	write32 (DEFAULT_RCBA + 0x0050, 0x812a0654);
-	read32 (DEFAULT_RCBA + 0x0050);	// !!! = 0x812a0654
-	read16 (DEFAULT_RCBA + 0x201a);	// !!! = 0x0000
-	read16 (DEFAULT_RCBA + 0x2026);	// !!! = 0x0000
-	read16 (DEFAULT_RCBA + 0x2036);	// !!! = 0x0000
-	read16 (DEFAULT_RCBA + 0x2046);	// !!! = 0x0000
-	read16 (DEFAULT_DMIBAR + 0x001a);	// !!! = 0x0000
-	read16 (DEFAULT_DMIBAR + 0x0026);	// !!! = 0x0000
-	read16 (DEFAULT_DMIBAR + 0x0032);	// !!! = 0x0000
-	read16 (DEFAULT_DMIBAR + 0x003e);	// !!! = 0x0000
-}
-
-void
-early_pch_init_native (void)
-{
-	pcie_write_config8 (SOUTHBRIDGE, 0xa6,
-			    pcie_read_config8 (SOUTHBRIDGE, 0xa6) | 2);
-
-	write32 (DEFAULT_RCBA + 0x2088, 0x00109000);
-	read32 (DEFAULT_RCBA + 0x20ac);	// !!! = 0x00000000
-	write32 (DEFAULT_RCBA + 0x20ac, 0x40000000);
-	write32 (DEFAULT_RCBA + 0x100c, 0x01110000);
-	write8 (DEFAULT_RCBA + 0x2340, 0x1b);
-	read32 (DEFAULT_RCBA + 0x2314);	// !!! = 0x0a080000
-	write32 (DEFAULT_RCBA + 0x2314, 0x0a280000);
-	read32 (DEFAULT_RCBA + 0x2310);	// !!! = 0xc809605b
-	write32 (DEFAULT_RCBA + 0x2310, 0xa809605b);
-	write32 (DEFAULT_RCBA + 0x2324, 0x00854c74);
-	read8 (DEFAULT_RCBA + 0x0400);	// !!! = 0x00
-	read32 (DEFAULT_RCBA + 0x2310);	// !!! = 0xa809605b
-	write32 (DEFAULT_RCBA + 0x2310, 0xa809605b);
-	read32 (DEFAULT_RCBA + 0x2310);	// !!! = 0xa809605b
-	write32 (DEFAULT_RCBA + 0x2310, 0xa809605b);
-
-	write_2338 (0xea007f62, 0x00590133);
-	write_2338 (0xec007f62, 0x00590133);
-	write_2338 (0xec007f64, 0x59555588);
-	write_2338 (0xea0040b9, 0x0001051c);
-	write_2338 (0xeb0040a1, 0x800084ff);
-	write_2338 (0xec0040a1, 0x800084ff);
-	write_2338 (0xea004001, 0x00008400);
-	write_2338 (0xeb004002, 0x40201758);
-	write_2338 (0xec004002, 0x40201758);
-	write_2338 (0xea004002, 0x00601758);
-	write_2338 (0xea0040a1, 0x810084ff);
-	write_2338 (0xeb0040b1, 0x0001c598);
-	write_2338 (0xec0040b1, 0x0001c598);
-	write_2338 (0xeb0040b6, 0x0001c598);
-	write_2338 (0xea0000a9, 0x80ff969f);
-	write_2338 (0xea0001a9, 0x80ff969f);
-	write_2338 (0xeb0040b2, 0x0001c396);
-	write_2338 (0xeb0040b3, 0x0001c396);
-	write_2338 (0xec0040b2, 0x0001c396);
-	write_2338 (0xea0001a9, 0x80ff94ff);
-	write_2338 (0xea000151, 0x0088037f);
-	write_2338 (0xea0000a9, 0x80ff94ff);
-	write_2338 (0xea000051, 0x0088037f);
-
-	write_2338 (0xea007f05, 0x00010642);
-	write_2338 (0xea0040b7, 0x0001c91c);
-	write_2338 (0xea0040b8, 0x0001c91c);
-	write_2338 (0xeb0040a1, 0x820084ff);
-	write_2338 (0xec0040a1, 0x820084ff);
-	write_2338 (0xea007f0a, 0xc2480000);
-
-	write_2338 (0xec00404d, 0x1ff177f);
-	write_2338 (0xec000084, 0x5a600000);
-	write_2338 (0xec000184, 0x5a600000);
-	write_2338 (0xec000284, 0x5a600000);
-	write_2338 (0xec000384, 0x5a600000);
-	write_2338 (0xec000094, 0x000f0501);
-	write_2338 (0xec000194, 0x000f0501);
-	write_2338 (0xec000294, 0x000f0501);
-	write_2338 (0xec000394, 0x000f0501);
-	write_2338 (0xec000096, 0x00000001);
-	write_2338 (0xec000196, 0x00000001);
-	write_2338 (0xec000296, 0x00000001);
-	write_2338 (0xec000396, 0x00000001);
-	write_2338 (0xec000001, 0x00008c08);
-	write_2338 (0xec000101, 0x00008c08);
-	write_2338 (0xec000201, 0x00008c08);
-	write_2338 (0xec000301, 0x00008c08);
-	write_2338 (0xec0040b5, 0x0001c518);
-	write_2338 (0xec000087, 0x06077597);
-	write_2338 (0xec000187, 0x06077597);
-	write_2338 (0xec000287, 0x06077597);
-	write_2338 (0xec000387, 0x06077597);
-	write_2338 (0xea000050, 0x00bb0157);
-	write_2338 (0xea000150, 0x00bb0157);
-	write_2338 (0xec007f60, 0x77777d77);
-	write_2338 (0xea00008d, 0x01320000);
-	write_2338 (0xea00018d, 0x01320000);
-	write_2338 (0xec0007b2, 0x04514b5e);
-	write_2338 (0xec00078c, 0x40000200);
-	write_2338 (0xec000780, 0x02000020);
-
-	init_dmi();
-}
diff --git a/src/southbridge/intel/bd82x6x/early_usb.c b/src/southbridge/intel/bd82x6x/early_usb.c
index 0533ae7477..6973c6c805 100644
--- a/src/southbridge/intel/bd82x6x/early_usb.c
+++ b/src/southbridge/intel/bd82x6x/early_usb.c
@@ -1,7 +1,7 @@
 /*
  * This file is part of the coreboot project.
  *
- * Copyright (C) 2008-2009 coresystems GmbH
+ * Copyright (C) 2014 Vladimir Serbinenko
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License as
@@ -22,35 +22,56 @@
 #include <console/console.h>
 #include <device/pci_ids.h>
 #include <device/pci_def.h>
+#include "northbridge/intel/sandybridge/sandybridge.h" /* For DEFAULT_RCBABASE.  */
 #include "pch.h"
 
-#define PCH_EHCI1_TEMP_BAR0 0xe8000000
-#define PCH_EHCI2_TEMP_BAR0 0xe8000400
-
-/*
- * Setup USB controller MMIO BAR to prevent the
- * reference code from resetting the controller.
- *
- * The BAR will be re-assigned during device
- * enumeration so these are only temporary.
- */
-void enable_usb_bar(void)
+void
+early_usb_init (const struct southbridge_usb_port *portmap)
 {
-	device_t usb0 = PCH_EHCI1_DEV;
-	device_t usb1 = PCH_EHCI2_DEV;
-	u32 cmd;
+	u32 reg32;
+	const u32 rcba_dump[8] = {
+		/* 3560 */ 0x024c8001, 0x000024a3, 0x00040002, 0x01000050,
+		/* 3570 */ 0x02000772, 0x16000f9f, 0x1800ff4f, 0x0001d630,
+	};
+	const u32 currents[] = { 0x20000153, 0x20000f57, 0x2000055b, 0x20000f51, 0x2000094a, 0x2000035f };
+	int i;
+	/* Activate PMBAR.  */
+	pci_write_config32(PCI_DEV(0, 0x1f, 0), PMBASE, DEFAULT_PMBASE | 1);
+	pci_write_config32(PCI_DEV(0, 0x1f, 0), PMBASE + 4, 0);
+	pci_write_config8(PCI_DEV(0, 0x1f, 0), 0x44 /* ACPI_CNTL */ , 0x80); /* Enable ACPI BAR */
+
+	/* Unlock registers.  */
+	outw (inw (DEFAULT_PMBASE | 0x003c) | 2, DEFAULT_PMBASE | 0x003c);
+	for (i = 0; i < 14; i++)
+		write32 (DEFAULT_RCBABASE + (0x3500 + 4 * i),
+			 currents[portmap[i].current]);
+	for (i = 0; i < 10; i++)
+		write32 (DEFAULT_RCBABASE + (0x3538 + 4 * i), 0);
+
+	for (i = 0; i < 8; i++)
+		write32 (DEFAULT_RCBABASE + (0x3560 + 4 * i), rcba_dump[i]);
+	for (i = 0; i < 8; i++)
+		write32 (DEFAULT_RCBABASE + (0x3580 + 4 * i), 0);
+	reg32 = 0;
+	for (i = 0; i < 14; i++)
+		if (!portmap[i].enabled)
+			reg32 |= (1 << i);
+	write32 (DEFAULT_RCBABASE + USBPDO, reg32);
+	reg32 = 0;
+	for (i = 0; i < 8; i++)
+		if (portmap[i].enabled && portmap[i].oc_pin >= 0)
+			reg32 |= (1 << (i + 8 * portmap[i].oc_pin));
+	write32 (DEFAULT_RCBABASE + USBOCM1, reg32);
+	reg32 = 0;
+	for (i = 8; i < 14; i++)
+		if (portmap[i].enabled && portmap[i].oc_pin >= 4)
+			reg32 |= (1 << (i - 8 + 8 * (portmap[i].oc_pin - 4)));
+	write32 (DEFAULT_RCBABASE + USBOCM2, reg32);
+	for (i = 0; i < 22; i++)
+		write32 (DEFAULT_RCBABASE + (0x35a8 + 4 * i), 0);
 
-	/* USB Controller 1 */
-	pci_write_config32(usb0, PCI_BASE_ADDRESS_0,
-			   PCH_EHCI1_TEMP_BAR0);
-	cmd = pci_read_config32(usb0, PCI_COMMAND);
-	cmd |= PCI_COMMAND_MASTER | PCI_COMMAND_MEMORY;
-	pci_write_config32(usb0, PCI_COMMAND, cmd);
+	pcie_write_config32 (PCI_DEV (0, 0x14, 0), 0xe4, 0x00000000);
 
-	/* USB Controller 2 */
-	pci_write_config32(usb1, PCI_BASE_ADDRESS_0,
-			   PCH_EHCI2_TEMP_BAR0);
-	cmd = pci_read_config32(usb1, PCI_COMMAND);
-	cmd |= PCI_COMMAND_MASTER | PCI_COMMAND_MEMORY;
-	pci_write_config32(usb1, PCI_COMMAND, cmd);
+	/* Relock registers.  */
+	outw (0x0000, DEFAULT_PMBASE | 0x003c);
 }
diff --git a/src/southbridge/intel/bd82x6x/early_usb_mrc.c b/src/southbridge/intel/bd82x6x/early_usb_mrc.c
new file mode 100644
index 0000000000..0533ae7477
--- /dev/null
+++ b/src/southbridge/intel/bd82x6x/early_usb_mrc.c
@@ -0,0 +1,56 @@
+/*
+ * This file is part of the coreboot project.
+ *
+ * Copyright (C) 2008-2009 coresystems GmbH
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; version 2 of
+ * the License.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc.
+ */
+
+#include <arch/io.h>
+#include <console/console.h>
+#include <device/pci_ids.h>
+#include <device/pci_def.h>
+#include "pch.h"
+
+#define PCH_EHCI1_TEMP_BAR0 0xe8000000
+#define PCH_EHCI2_TEMP_BAR0 0xe8000400
+
+/*
+ * Setup USB controller MMIO BAR to prevent the
+ * reference code from resetting the controller.
+ *
+ * The BAR will be re-assigned during device
+ * enumeration so these are only temporary.
+ */
+void enable_usb_bar(void)
+{
+	device_t usb0 = PCH_EHCI1_DEV;
+	device_t usb1 = PCH_EHCI2_DEV;
+	u32 cmd;
+
+	/* USB Controller 1 */
+	pci_write_config32(usb0, PCI_BASE_ADDRESS_0,
+			   PCH_EHCI1_TEMP_BAR0);
+	cmd = pci_read_config32(usb0, PCI_COMMAND);
+	cmd |= PCI_COMMAND_MASTER | PCI_COMMAND_MEMORY;
+	pci_write_config32(usb0, PCI_COMMAND, cmd);
+
+	/* USB Controller 2 */
+	pci_write_config32(usb1, PCI_BASE_ADDRESS_0,
+			   PCH_EHCI2_TEMP_BAR0);
+	cmd = pci_read_config32(usb1, PCI_COMMAND);
+	cmd |= PCI_COMMAND_MASTER | PCI_COMMAND_MEMORY;
+	pci_write_config32(usb1, PCI_COMMAND, cmd);
+}
diff --git a/src/southbridge/intel/bd82x6x/early_usb_native.c b/src/southbridge/intel/bd82x6x/early_usb_native.c
deleted file mode 100644
index 6973c6c805..0000000000
--- a/src/southbridge/intel/bd82x6x/early_usb_native.c
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * This file is part of the coreboot project.
- *
- * Copyright (C) 2014 Vladimir Serbinenko
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License as
- * published by the Free Software Foundation; version 2 of
- * the License.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc.
- */
-
-#include <arch/io.h>
-#include <console/console.h>
-#include <device/pci_ids.h>
-#include <device/pci_def.h>
-#include "northbridge/intel/sandybridge/sandybridge.h" /* For DEFAULT_RCBABASE.  */
-#include "pch.h"
-
-void
-early_usb_init (const struct southbridge_usb_port *portmap)
-{
-	u32 reg32;
-	const u32 rcba_dump[8] = {
-		/* 3560 */ 0x024c8001, 0x000024a3, 0x00040002, 0x01000050,
-		/* 3570 */ 0x02000772, 0x16000f9f, 0x1800ff4f, 0x0001d630,
-	};
-	const u32 currents[] = { 0x20000153, 0x20000f57, 0x2000055b, 0x20000f51, 0x2000094a, 0x2000035f };
-	int i;
-	/* Activate PMBAR.  */
-	pci_write_config32(PCI_DEV(0, 0x1f, 0), PMBASE, DEFAULT_PMBASE | 1);
-	pci_write_config32(PCI_DEV(0, 0x1f, 0), PMBASE + 4, 0);
-	pci_write_config8(PCI_DEV(0, 0x1f, 0), 0x44 /* ACPI_CNTL */ , 0x80); /* Enable ACPI BAR */
-
-	/* Unlock registers.  */
-	outw (inw (DEFAULT_PMBASE | 0x003c) | 2, DEFAULT_PMBASE | 0x003c);
-	for (i = 0; i < 14; i++)
-		write32 (DEFAULT_RCBABASE + (0x3500 + 4 * i),
-			 currents[portmap[i].current]);
-	for (i = 0; i < 10; i++)
-		write32 (DEFAULT_RCBABASE + (0x3538 + 4 * i), 0);
-
-	for (i = 0; i < 8; i++)
-		write32 (DEFAULT_RCBABASE + (0x3560 + 4 * i), rcba_dump[i]);
-	for (i = 0; i < 8; i++)
-		write32 (DEFAULT_RCBABASE + (0x3580 + 4 * i), 0);
-	reg32 = 0;
-	for (i = 0; i < 14; i++)
-		if (!portmap[i].enabled)
-			reg32 |= (1 << i);
-	write32 (DEFAULT_RCBABASE + USBPDO, reg32);
-	reg32 = 0;
-	for (i = 0; i < 8; i++)
-		if (portmap[i].enabled && portmap[i].oc_pin >= 0)
-			reg32 |= (1 << (i + 8 * portmap[i].oc_pin));
-	write32 (DEFAULT_RCBABASE + USBOCM1, reg32);
-	reg32 = 0;
-	for (i = 8; i < 14; i++)
-		if (portmap[i].enabled && portmap[i].oc_pin >= 4)
-			reg32 |= (1 << (i - 8 + 8 * (portmap[i].oc_pin - 4)));
-	write32 (DEFAULT_RCBABASE + USBOCM2, reg32);
-	for (i = 0; i < 22; i++)
-		write32 (DEFAULT_RCBABASE + (0x35a8 + 4 * i), 0);
-
-	pcie_write_config32 (PCI_DEV (0, 0x14, 0), 0xe4, 0x00000000);
-
-	/* Relock registers.  */
-	outw (0x0000, DEFAULT_PMBASE | 0x003c);
-}
diff --git a/src/southbridge/intel/bd82x6x/usb_ehci.c b/src/southbridge/intel/bd82x6x/usb_ehci.c
index dc21dc1d3d..ec06dbd8cb 100644
--- a/src/southbridge/intel/bd82x6x/usb_ehci.c
+++ b/src/southbridge/intel/bd82x6x/usb_ehci.c
@@ -39,7 +39,7 @@ static void usb_ehci_init(struct device *dev)
 	printk(BIOS_DEBUG, "EHCI: Setting up controller.. ");
 
 	/* For others, done in MRC.  */
-#if IS_ENABLED(CONFIG_NORTHBRIDGE_INTEL_IVYBRIDGE_NATIVE) || IS_ENABLED(CONFIG_NORTHBRIDGE_INTEL_SANDYBRIDGE_NATIVE)
+#if IS_ENABLED(CONFIG_NORTHBRIDGE_INTEL_IVYBRIDGE) || IS_ENABLED(CONFIG_NORTHBRIDGE_INTEL_SANDYBRIDGE)
 	pci_write_config32(dev, 0x84, 0x930c8811);
 	pci_write_config32(dev, 0x88, 0x24000d30);
 	pci_write_config32(dev, 0xf4, 0x80408588);
@@ -54,7 +54,7 @@ static void usb_ehci_init(struct device *dev)
 	pci_write_config32(dev, PCI_COMMAND, reg32);
 
 	/* For others, done in MRC.  */
-#if IS_ENABLED(CONFIG_NORTHBRIDGE_INTEL_IVYBRIDGE_NATIVE) || IS_ENABLED(CONFIG_NORTHBRIDGE_INTEL_SANDYBRIDGE_NATIVE)
+#if IS_ENABLED(CONFIG_NORTHBRIDGE_INTEL_IVYBRIDGE) || IS_ENABLED(CONFIG_NORTHBRIDGE_INTEL_SANDYBRIDGE)
 	struct resource *res;
 	u8 access_cntl;
 
diff --git a/util/autoport/sandybridge.go b/util/autoport/sandybridge.go
index 838475ea5f..21c791969c 100644
--- a/util/autoport/sandybridge.go
+++ b/util/autoport/sandybridge.go
@@ -124,7 +124,7 @@ func (i sandybridgemc) Scan(ctx Context, addr PCIDevData) {
 	KconfigBool["VGA"] = true
 	KconfigBool["INTEL_EDID"] = true
 	KconfigBool["CPU_INTEL_SOCKET_RPGA989"] = true
-	KconfigBool["NORTHBRIDGE_INTEL_"+i.variant+"BRIDGE_NATIVE"] = true
+	KconfigBool["NORTHBRIDGE_INTEL_"+i.variant+"BRIDGE"] = true
 	KconfigBool["INTEL_INT15"] = true
 	KconfigBool["HAVE_ACPI_TABLES"] = true
 	KconfigBool["HAVE_ACPI_RESUME"] = true
diff --git a/util/board_status/to-wiki/towiki.sh b/util/board_status/to-wiki/towiki.sh
index 4a4f710faf..c1d2394a63 100755
--- a/util/board_status/to-wiki/towiki.sh
+++ b/util/board_status/to-wiki/towiki.sh
@@ -326,9 +326,9 @@ EOF
 				case $northbridge in
 					INTEL_HASWELL)
 						cpu_nice="Intel® 4th Gen (Haswell) Core i3/i5/i7";;
-					INTEL_IVYBRIDGE_NATIVE|INTEL_IVYBRIDGE|INTEL_FSP_IVYBRIDGE)
+					INTEL_IVYBRIDGE|INTEL_IVYBRIDGE_MRC|INTEL_FSP_IVYBRIDGE)
 						cpu_nice="Intel® 3rd Gen (Ivybridge) Core i3/i5/i7";;
-					INTEL_SANDYBRIDGE|INTEL_SANDYBRIDGE_NATIVE)
+					INTEL_SANDYBRIDGE|INTEL_SANDYBRIDGE_MRC)
 						cpu_nice="Intel® 2nd Gen (Sandybridge) Core i3/i5/i7";;
 					*)
 						cpu_nice="$northbridge";;