cpu/amd/fam10h-fam15h: Update Fam15h APIC config and startup sequence

This fixes Family 15h multiple package support; the previous code
hung in CAR setup and romstage when more than one CPU package was
installed for a variety of loosely related reasons.

TEST: Booted ASUS KGPE-D16 with two Opteron 6328 processors
and several different RDIMM configurations.

Change-Id: I171197c90f72d3496a385465937b7666cbf7e308
Signed-off-by: Timothy Pearson <tpearson@raptorengineeringinc.com>
Reviewed-on: http://review.coreboot.org/12020
Tested-by: build bot (Jenkins)
Reviewed-by: Alexandru Gagniuc <mr.nuke.me@gmail.com>

3 files changed, 204 insertions, 15 deletions

diff --git a/src/northbridge/amd/amdht/h3finit.c b/src/northbridge/amd/amdht/h3finit.c
index c9a310b9b8..bcdcfa2ae6 100644
--- a/src/northbridge/amd/amdht/h3finit.c
+++ b/src/northbridge/amd/amdht/h3finit.c
@@ -385,13 +385,49 @@ static u8 convertNodeToLink(u8 srcNode, u8 targetNode, sMainData *pDat)
  */
 static void htDiscoveryFloodFill(sMainData *pDat)
 {
-	u8 currentNode = 0;
-	u8 currentLink;
+	uint8_t currentNode = 0;
+	uint8_t currentLink;
+	uint8_t currentLinkID;
+
+	/* NOTE
+	 * Each node inside a dual node (socket G34) processor must share
+	 * an adjacent node ID.  Alter the link scan order such that the
+	 * other internal node is always scanned first...
+	 */
+	uint8_t currentLinkScanOrder_Default[8] = {0, 1, 2, 3, 4, 5, 6, 7};
+	uint8_t currentLinkScanOrder_G34_Fam10[8] = {1, 0, 2, 3, 4, 5, 6, 7};
+	uint8_t currentLinkScanOrder_G34_Fam15[8] = {2, 0, 1, 3, 4, 5, 6, 7};
+
+	uint8_t fam15h = 0;
+	uint8_t rev_gte_d = 0;
+	uint8_t dual_node = 0;
+	uint32_t f3xe8;
+	uint32_t family;
+	uint32_t model;
+
+	f3xe8 = pci_read_config32(NODE_PCI(0, 3), 0xe8);
+
+	family = model = cpuid_eax(0x80000001);
+	model = ((model & 0xf0000) >> 12) | ((model & 0xf0) >> 4);
+	family = ((family & 0xf00000) >> 16) | ((family & 0xf00) >> 8);
+
+	if (family >= 0x6f) {
+		/* Family 15h or later */
+		fam15h = 1;
+	}
+
+	if ((model >= 0x8) || fam15h)
+		/* Revision D or later */
+		rev_gte_d = 1;
+
+	if (rev_gte_d)
+		 /* Check for dual node capability */
+		if (f3xe8 & 0x20000000)
+			dual_node = 1;
 
 	/* Entries are always added in pairs, the even indices are the 'source'
 	 * side closest to the BSP, the odd indices are the 'destination' side
 	 */
-
 	while (currentNode <= pDat->NodesDiscovered)
 	{
 		u32 temp;
@@ -419,11 +455,24 @@ static void htDiscoveryFloodFill(sMainData *pDat)
 		/* Enable routing tables on currentNode*/
 		pDat->nb->enableRoutingTables(currentNode, pDat->nb);
 
-		for (currentLink = 0; currentLink < pDat->nb->maxLinks; currentLink++)
+		for (currentLinkID = 0; currentLinkID < pDat->nb->maxLinks; currentLinkID++)
 		{
 			BOOL linkfound;
 			u8 token;
 
+			if (currentLinkID < 8) {
+				if (dual_node) {
+					if (fam15h)
+						currentLink = currentLinkScanOrder_G34_Fam15[currentLinkID];
+					else
+						currentLink = currentLinkScanOrder_G34_Fam10[currentLinkID];
+				} else {
+					currentLink = currentLinkScanOrder_Default[currentLinkID];
+				}
+			} else {
+				currentLink = currentLinkID;
+			}
+
 			if (pDat->HtBlock->AMD_CB_IgnoreLink && pDat->HtBlock->AMD_CB_IgnoreLink(currentNode, currentLink))
 				continue;
 
diff --git a/src/northbridge/amd/amdht/h3ncmn.c b/src/northbridge/amd/amdht/h3ncmn.c
index ac7d393b24..01c0e48d0c 100644
--- a/src/northbridge/amd/amdht/h3ncmn.c
+++ b/src/northbridge/amd/amdht/h3ncmn.c
@@ -47,8 +47,9 @@
 #define REG_NODE_ID_0X60		0x60
 #define REG_UNIT_ID_0X64		0x64
 #define REG_LINK_TRANS_CONTROL_0X68	0x68
-#define REG_LINK_INIT_CONTROL_0X6C	0x6C
+#define REG_LINK_INIT_CONTROL_0X6C	0x6c
 #define REG_HT_CAP_BASE_0X80		0x80
+#define REG_NORTHBRIDGE_CFG_3X8C	0x8c
 #define REG_HT_LINK_RETRY0_0X130	0x130
 #define REG_HT_TRAFFIC_DIST_0X164	0x164
 #define REG_HT_LINK_EXT_CONTROL0_0X170	0x170
@@ -87,6 +88,21 @@
  ***			FAMILY/NORTHBRIDGE SPECIFIC FUNCTIONS		***
  ***************************************************************************/
 
+static inline uint8_t is_fam15h(void)
+{
+	uint8_t fam15h = 0;
+	uint32_t family;
+
+	family = cpuid_eax(0x80000001);
+	family = ((family & 0xf00000) >> 16) | ((family & 0xf00) >> 8);
+
+	if (family >= 0x6f)
+		/* Family 15h or later */
+		fam15h = 1;
+
+	return fam15h;
+}
+
 /***************************************************************************//**
  *
  * SBDFO
@@ -215,8 +231,18 @@ static void writeRoutingTable(u8 node, u8 target, u8 link, cNorthBridge *nb)
 
 static void writeNodeID(u8 node, u8 nodeID, cNorthBridge *nb)
 {
-	u32 temp = nodeID;
+	u32 temp;
 	ASSERT((node < nb->maxNodes) && (nodeID < nb->maxNodes));
+	if (is_fam15h()) {
+		temp = 1;
+		AmdPCIWriteBits(MAKE_SBDFO(makePCISegmentFromNode(node),
+					makePCIBusFromNode(node),
+					makePCIDeviceFromNode(node),
+					CPU_NB_FUNC_03,
+					REG_NORTHBRIDGE_CFG_3X8C),
+					22, 22, &temp);
+	}
+	temp = nodeID;
 	AmdPCIWriteBits(MAKE_SBDFO(makePCISegmentFromNode(node),
 				makePCIBusFromNode(node),
 				makePCIDeviceFromNode(node),
diff --git a/src/northbridge/amd/amdht/ht_wrapper.c b/src/northbridge/amd/amdht/ht_wrapper.c
index 1f38b0c032..18c0ac4456 100644
--- a/src/northbridge/amd/amdht/ht_wrapper.c
+++ b/src/northbridge/amd/amdht/ht_wrapper.c
@@ -82,22 +82,129 @@ static  u32 get_nodes(void)
 	return nodes;
 }
 
+static const char * event_class_string_decodes[] = {
+	[HT_EVENT_CLASS_CRITICAL] = "CRITICAL",
+	[HT_EVENT_CLASS_ERROR] = "ERROR",
+	[HT_EVENT_CLASS_HW_FAULT] = "HARDWARE FAULT",
+	[HT_EVENT_CLASS_WARNING] = "WARNING",
+	[HT_EVENT_CLASS_INFO] = "INFO"
+};
+
+static const char * event_string_decodes[] = {
+	[HT_EVENT_COH_EVENTS] = "HT_EVENT_COH_EVENTS",
+	[HT_EVENT_COH_NO_TOPOLOGY] = "HT_EVENT_COH_NO_TOPOLOGY",
+	[HT_EVENT_COH_LINK_EXCEED] = "HT_EVENT_COH_LINK_EXCEED",
+	[HT_EVENT_COH_FAMILY_FEUD] = "HT_EVENT_COH_FAMILY_FEUD",
+	[HT_EVENT_COH_NODE_DISCOVERED] = "HT_EVENT_COH_NODE_DISCOVERED",
+	[HT_EVENT_COH_MPCAP_MISMATCH] = "HT_EVENT_COH_MPCAP_MISMATCH",
+	[HT_EVENT_NCOH_EVENTS] = "HT_EVENT_NCOH_EVENTS",
+	[HT_EVENT_NCOH_BUID_EXCEED] = "HT_EVENT_NCOH_BUID_EXCEED",
+	[HT_EVENT_NCOH_LINK_EXCEED] = "HT_EVENT_NCOH_LINK_EXCEED",
+	[HT_EVENT_NCOH_BUS_MAX_EXCEED] = "HT_EVENT_NCOH_BUS_MAX_EXCEED",
+	[HT_EVENT_NCOH_CFG_MAP_EXCEED] = "HT_EVENT_NCOH_CFG_MAP_EXCEED",
+	[HT_EVENT_NCOH_DEVICE_FAILED] = "HT_EVENT_NCOH_DEVICE_FAILED",
+	[HT_EVENT_NCOH_AUTO_DEPTH] = "HT_EVENT_NCOH_AUTO_DEPTH",
+	[HT_EVENT_OPT_EVENTS] = "HT_EVENT_OPT_EVENTS",
+	[HT_EVENT_OPT_REQUIRED_CAP_RETRY] = "HT_EVENT_OPT_REQUIRED_CAP_RETRY",
+	[HT_EVENT_OPT_REQUIRED_CAP_GEN3] = "HT_EVENT_OPT_REQUIRED_CAP_GEN3",
+	[HT_EVENT_HW_EVENTS] = "HT_EVENT_HW_EVENTS",
+	[HT_EVENT_HW_SYNCHFLOOD] = "HT_EVENT_HW_SYNCHFLOOD",
+	[HT_EVENT_HW_HTCRC] = "HT_EVENT_HW_HTCRC"
+};
 
 /**
  * void AMD_CB_EventNotify (u8 evtClass, u16 event, const u8 *pEventData0)
  */
 static void AMD_CB_EventNotify (u8 evtClass, u16 event, const u8 *pEventData0)
 {
-	u8 i;
+	uint8_t i;
+	uint8_t log_level;
+	uint8_t dump_event_detail;
 
-	printk(BIOS_DEBUG, "AMD_CB_EventNotify()\n");
-	printk(BIOS_DEBUG, " event class: %02x\n event: %04x\n data: ", evtClass, event);
+	printk(BIOS_DEBUG, "AMD_CB_EventNotify(): ");
 
-	for (i = 0; i < *pEventData0; i++) {
-		printk(BIOS_DEBUG, " %02x ", *(pEventData0 + i));
+	/* Decode event */
+	dump_event_detail = 1;
+	switch (evtClass) {
+		case HT_EVENT_CLASS_CRITICAL:
+		case HT_EVENT_CLASS_ERROR:
+		case HT_EVENT_CLASS_HW_FAULT:
+		case HT_EVENT_CLASS_WARNING:
+		case HT_EVENT_CLASS_INFO:
+			log_level = BIOS_DEBUG;
+			printk(log_level, event_class_string_decodes[evtClass]);
+			break;
+		default:
+			log_level = BIOS_DEBUG;
+			printk(log_level, "UNKNOWN");
+			break;
 	}
-	printk(BIOS_DEBUG, "\n");
+	printk(log_level, ": ");
 
+	switch(event) {
+		case HT_EVENT_COH_EVENTS:
+		case HT_EVENT_COH_NO_TOPOLOGY:
+		case HT_EVENT_COH_LINK_EXCEED:
+		case HT_EVENT_COH_FAMILY_FEUD:
+			printk(log_level, event_string_decodes[event]);
+			break;
+		case HT_EVENT_COH_NODE_DISCOVERED:
+			{
+				printk(log_level, "HT_EVENT_COH_NODE_DISCOVERED");
+				sHtEventCohNodeDiscovered *evt = (sHtEventCohNodeDiscovered*)pEventData0;
+				printk(log_level, ": node %d link %d new node: %d",
+					evt->node, evt->link, evt->newNode);
+				dump_event_detail = 0;
+				break;
+			}
+		case HT_EVENT_COH_MPCAP_MISMATCH:
+		case HT_EVENT_NCOH_EVENTS:
+		case HT_EVENT_NCOH_BUID_EXCEED:
+		case HT_EVENT_NCOH_LINK_EXCEED:
+		case HT_EVENT_NCOH_BUS_MAX_EXCEED:
+		case HT_EVENT_NCOH_CFG_MAP_EXCEED:
+			printk(log_level, event_string_decodes[event]);
+			break;
+		case HT_EVENT_NCOH_DEVICE_FAILED:
+			{
+				printk(log_level, event_string_decodes[event]);
+				sHtEventNcohDeviceFailed *evt = (sHtEventNcohDeviceFailed*)pEventData0;
+				printk(log_level, ": node %d link %d depth: %d attemptedBUID: %d",
+					evt->node, evt->link, evt->depth, evt->attemptedBUID);
+				dump_event_detail = 0;
+				break;
+			}
+		case HT_EVENT_NCOH_AUTO_DEPTH:
+			{
+				printk(log_level, event_string_decodes[event]);
+				sHtEventNcohAutoDepth *evt = (sHtEventNcohAutoDepth*)pEventData0;
+				printk(log_level, ": node %d link %d depth: %d",
+					evt->node, evt->link, evt->depth);
+				dump_event_detail = 0;
+				break;
+			}
+		case HT_EVENT_OPT_EVENTS:
+		case HT_EVENT_OPT_REQUIRED_CAP_RETRY:
+		case HT_EVENT_OPT_REQUIRED_CAP_GEN3:
+		case HT_EVENT_HW_EVENTS:
+		case HT_EVENT_HW_SYNCHFLOOD:
+		case HT_EVENT_HW_HTCRC:
+			printk(log_level, event_string_decodes[event]);
+			break;
+		default:
+			printk(log_level, "HT_EVENT_UNKNOWN");
+			break;
+	}
+	printk(log_level, "\n");
+
+	if (dump_event_detail) {
+		printk(BIOS_DEBUG, " event class: %02x\n event: %04x\n data: ", evtClass, event);
+
+		for (i = 0; i < *pEventData0; i++) {
+			printk(BIOS_DEBUG, " %02x ", *(pEventData0 + i));
+		}
+		printk(BIOS_DEBUG, "\n");
+	}
 }
 
 /**
@@ -206,9 +313,10 @@ void amd_ht_fixup(struct sys_info *sysinfo) {
 				for (node = 0; node < node_count; node++) {
 					f3xe8 = pci_read_config32(NODE_PCI(node, 3), 0xe8);
 					uint8_t internal_node_number = ((f3xe8 & 0xc0000000) >> 30);
-					printk(BIOS_DEBUG, "amd_ht_fixup(): node %d (internal node ID %d): disabling defective HT link\n", node, internal_node_number);
+					printk(BIOS_DEBUG, "amd_ht_fixup(): node %d (internal node ID %d): disabling defective HT link", node, internal_node_number);
 					if (internal_node_number == 0) {
 						uint8_t package_link_3_connected = pci_read_config32(NODE_PCI(node, 0), (fam15h)?0x98:0xd8) & 0x1;
+						printk(BIOS_DEBUG, " (L3 connected: %d)\n", package_link_3_connected);
 						if (package_link_3_connected) {
 							/* Set WidthIn and WidthOut to 0 */
 							dword = pci_read_config32(NODE_PCI(node, 0), (fam15h)?0x84:0xc4);
@@ -230,15 +338,21 @@ void amd_ht_fixup(struct sys_info *sysinfo) {
 						}
 					} else if (internal_node_number == 1) {
 						uint8_t package_link_3_connected = pci_read_config32(NODE_PCI(node, 0), (fam15h)?0xf8:0xb8) & 0x1;
+						printk(BIOS_DEBUG, " (L3 connected: %d)\n", package_link_3_connected);
 						if (package_link_3_connected) {
 							/* Set WidthIn and WidthOut to 0 */
 							dword = pci_read_config32(NODE_PCI(node, 0), (fam15h)?0xe4:0xa4);
 							dword &= ~0x77000000;
 							pci_write_config32(NODE_PCI(node, 0), (fam15h)?0xe4:0xa4, dword);
 							/* Set Ganged to 1 */
-							dword = pci_read_config32(NODE_PCI(node, 0), (fam15h)?0x18c:0x174);
+							/* WARNING
+							 * The Family 15h BKDG states that 0x18c should be set,
+							 * however this is in error.  0x17c is the correct control
+							 * register (sublink 0) for these processors...
+							 */
+							dword = pci_read_config32(NODE_PCI(node, 0), (fam15h)?0x17c:0x174);
 							dword |= 0x00000001;
-							pci_write_config32(NODE_PCI(node, 0), (fam15h)?0x18c:0x174, dword);
+							pci_write_config32(NODE_PCI(node, 0), (fam15h)?0x17c:0x174, dword);
 						} else {
 							/* Set ConnDly to 1 */
 							dword = pci_read_config32(NODE_PCI(node, 0), 0x16c);