soc/cavium: import raw BDK sources

This imports common BDK sources that will be used in subsequent
patches.
The BDK is licensed under BSD and will be reduced in size and optimized to
compile under coreboot.

Change-Id: Icb32ee670d9fa9e5c10f9abb298cebf616fa67ad
Signed-off-by: David Hendricks <dhendricks@fb.com>
Reviewed-on: https://review.coreboot.org/25524
Tested-by: build bot (Jenkins) <no-reply@coreboot.org>
Reviewed-by: David Hendricks <david.hendricks@gmail.com>

1 files changed, 829 insertions, 0 deletions

diff --git a/src/vendorcode/cavium/bdk/libbdk-dram/bdk-dram-test-patfil.c b/src/vendorcode/cavium/bdk/libbdk-dram/bdk-dram-test-patfil.c
new file mode 100644
index 0000000000..e6c4b57721
--- /dev/null
+++ b/src/vendorcode/cavium/bdk/libbdk-dram/bdk-dram-test-patfil.c
@@ -0,0 +1,829 @@
+/***********************license start***********************************
+* Copyright (c) 2003-2017  Cavium Inc. (support@cavium.com). All rights
+* reserved.
+*
+*
+* Redistribution and use in source and binary forms, with or without
+* modification, are permitted provided that the following conditions are
+* met:
+*
+*   * Redistributions of source code must retain the above copyright
+*     notice, this list of conditions and the following disclaimer.
+*
+*   * Redistributions in binary form must reproduce the above
+*     copyright notice, this list of conditions and the following
+*     disclaimer in the documentation and/or other materials provided
+*     with the distribution.
+*
+*   * Neither the name of Cavium Inc. nor the names of
+*     its contributors may be used to endorse or promote products
+*     derived from this software without specific prior written
+*     permission.
+*
+* This Software, including technical data, may be subject to U.S. export
+* control laws, including the U.S. Export Administration Act and its
+* associated regulations, and may be subject to export or import
+* regulations in other countries.
+*
+* TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS"
+* AND WITH ALL FAULTS AND CAVIUM INC. MAKES NO PROMISES, REPRESENTATIONS OR
+* WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH RESPECT
+* TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY
+* REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT
+* DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) WARRANTIES
+* OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A PARTICULAR
+* PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET ENJOYMENT,
+* QUIET POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE ENTIRE  RISK
+* ARISING OUT OF USE OR PERFORMANCE OF THE SOFTWARE LIES WITH YOU.
+***********************license end**************************************/
+#include "bdk.h"
+
+// choose prediction-based algorithms for mem_xor and mem_rows tests
+#define USE_PREDICTION_CODE_VERSIONS 1   // change to 0 to go back to the original versions
+
+/* Used for all memory reads/writes related to the test */
+#define READ64(address) __bdk_dram_read64(address)
+#define WRITE64(address, data) __bdk_dram_write64(address, data)
+
+/**
+ * Fill an memory area with the address of each 64-bit word in the area.
+ * Reread to confirm the pattern.
+ *
+ * @param area   Start of the physical memory area
+ * @param max_address
+ *               End of the physical memory area (exclusive)
+ * @param bursts Number of time to repeat the test over the entire area
+ *
+ * @return Number of errors, zero on success
+ */
+int __bdk_dram_test_mem_self_addr(uint64_t area, uint64_t max_address, int bursts)
+{
+    int failures = 0;
+
+    for (int burst = 0; burst < bursts; burst++)
+    {
+        /* Write the pattern to memory. Each location receives the address
+         * of the location.
+         */
+        for (uint64_t address = area; address < max_address; address+=8)
+            WRITE64(address, address);
+        __bdk_dram_flush_to_mem_range(area, max_address);
+        BDK_DCACHE_INVALIDATE;
+
+        /* Read by ascending address the written memory and confirm that it
+         * has the expected data pattern.
+         */
+        for (uint64_t address = area; address < max_address; )
+        {
+            if (address + 256 < max_address)
+                BDK_PREFETCH(address + 256, 0);
+            for (int i=0; i<16; i++)
+            {
+                uint64_t data = READ64(address);
+                if (bdk_unlikely(data != address))
+                    failures += __bdk_dram_retry_failure(burst, address, data, address);
+                address += 8;
+            }
+        }
+        __bdk_dram_flush_to_mem_range(area, max_address);
+        BDK_DCACHE_INVALIDATE;
+
+        /* Read by descending address the written memory and confirm that it
+         * has the expected data pattern.
+         */
+        uint64_t end = max_address - sizeof(uint64_t);
+        for (uint64_t address = end; address >= area; )
+        {
+            if (address - 256 >= area)
+                BDK_PREFETCH(address - 256, 0);
+            for (int i=0; i<16; i++)
+            {
+                uint64_t data = READ64(address);
+                if (bdk_unlikely(data != address))
+                    failures += __bdk_dram_retry_failure(burst, address, data, address);
+                address -= 8;
+            }
+        }
+        __bdk_dram_flush_to_mem_range(area, max_address);
+        BDK_DCACHE_INVALIDATE;
+
+        /* Read from random addresses within the memory area.
+         */
+        uint64_t probes = (max_address - area) / 128;
+        uint64_t address_ahead1 = area;
+        uint64_t address_ahead2 = area;
+        for (uint64_t i = 0; i < probes; i++)
+        {
+            /* Create a pipeline of prefetches:
+               address = address read this loop
+               address_ahead1 = prefetch started last loop
+               address_ahead2 = prefetch started this loop */
+            uint64_t address = address_ahead1;
+            address_ahead1 = address_ahead2;
+            address_ahead2 = bdk_rng_get_random64() % (max_address - area);
+            address_ahead2 += area;
+            address_ahead2 &= -8;
+            BDK_PREFETCH(address_ahead2, 0);
+
+            uint64_t data = READ64(address);
+            if (bdk_unlikely(data != address))
+                failures += __bdk_dram_retry_failure(burst, address, data, address);
+        }
+    }
+    return failures;
+}
+
+/**
+ * Write "pattern" and its compliment to memory and verify it was written
+ * properly. Memory will be filled with DWORDs pattern, ~pattern, pattern,
+ * ~pattern, ...
+ *
+ * @param area    Start physical address of memory
+ * @param max_address
+ *                End of physical memory region
+ * @param pattern Pattern to write
+ * @param passes  Number of time to repeat the test
+ *
+ * @return Number of errors, zero on success
+ */
+static uint32_t test_mem_pattern(uint64_t area, uint64_t max_address, uint64_t pattern,
+    int passes)
+{
+    int failures = 0;
+
+    for (int pass = 0; pass < passes; pass++)
+    {
+        if (pass & 0x1)
+            pattern = ~pattern;
+
+        for (uint64_t address = area; address < max_address; address += 8)
+            WRITE64(address, pattern);
+        __bdk_dram_flush_to_mem_range(area, max_address);
+        BDK_DCACHE_INVALIDATE;
+
+        /* Read the written memory and confirm that it has the expected
+         * data pattern.
+         */
+        uint64_t address = area;
+        while (address < max_address)
+        {
+            if (address + 256 < max_address)
+                BDK_PREFETCH(address + 256, 0);
+            for (int i=0; i<16; i++)
+            {
+                uint64_t data = READ64(address);
+                if (bdk_unlikely(data != pattern))
+                    failures += __bdk_dram_retry_failure(pass, address, data, pattern);
+                address += 8;
+            }
+        }
+    }
+    return failures;
+}
+
+/**
+ * Walking zero written to memory, left shift
+ *
+ * @param area   Start of the physical memory area
+ * @param max_address
+ *               End of the physical memory area
+ * @param bursts Number of time to repeat the test over the entire area
+ *
+ * @return Number of errors, zero on success
+ */
+int __bdk_dram_test_mem_leftwalk0(uint64_t area, uint64_t max_address, int bursts)
+{
+    int failures = 0;
+    for (int burst = 0; burst < bursts; burst++)
+    {
+        for (uint64_t pattern = 1; pattern != 0; pattern = pattern << 1)
+            failures += test_mem_pattern(area, max_address, ~pattern, 1);
+    }
+    return failures;
+}
+
+/**
+ * Walking one written to memory, left shift
+ *
+ * @param area   Start of the physical memory area
+ * @param max_address
+ *               End of the physical memory area
+ * @param bursts Number of time to repeat the test over the entire area
+ *
+ * @return Number of errors, zero on success
+ */
+int __bdk_dram_test_mem_leftwalk1(uint64_t area, uint64_t max_address, int bursts)
+{
+    int failures = 0;
+    for (int burst = 0; burst < bursts; burst++)
+    {
+        for (uint64_t pattern = 1; pattern != 0; pattern = pattern << 1)
+            failures += test_mem_pattern(area, max_address, pattern, 1);
+    }
+    return failures;
+}
+
+/**
+ * Walking zero written to memory, right shift
+ *
+ * @param area   Start of the physical memory area
+ * @param max_address
+ *               End of the physical memory area
+ * @param bursts Number of time to repeat the test over the entire area
+ *
+ * @return Number of errors, zero on success
+ */
+int __bdk_dram_test_mem_rightwalk0(uint64_t area, uint64_t max_address, int bursts)
+{
+    int failures = 0;
+    for (int burst = 0; burst < bursts; burst++)
+    {
+        for (uint64_t pattern = 1ull << 63; pattern != 0; pattern = pattern >> 1)
+            failures += test_mem_pattern(area, max_address, ~pattern, 1);
+    }
+    return failures;
+}
+
+/**
+ * Walking one written to memory, right shift
+ *
+ * @param area   Start of the physical memory area
+ * @param max_address
+ *               End of the physical memory area
+ * @param bursts Number of time to repeat the test over the entire area
+ *
+ * @return Number of errors, zero on success
+ */
+int __bdk_dram_test_mem_rightwalk1(uint64_t area, uint64_t max_address, int bursts)
+{
+    int failures = 0;
+    for (int burst = 0; burst < bursts; burst++)
+    {
+        for (uint64_t pattern = 1ull<<63; pattern != 0; pattern = pattern >> 1)
+            failures += test_mem_pattern(area, max_address, pattern, 1);
+    }
+    return failures;
+}
+
+/**
+ * Apply the March C- testing algorithm to the given memory area.
+ * 1) Write "pattern" to memory.
+ * 2) Verify "pattern" and write "~pattern".
+ * 3) Verify "~pattern" and write "pattern".
+ * 4) Verify "pattern" and write "~pattern".
+ * 5) Verify "~pattern" and write "pattern".
+ * 6) Verify "pattern".
+ *
+ * @param area    Start of the physical memory area
+ * @param max_address
+ *                End of the physical memory area
+ * @param pattern
+ *
+ * @return Number of errors, zero on success
+ */
+static int test_mem_march_c(uint64_t area, uint64_t max_address, uint64_t pattern)
+{
+    int failures = 0;
+
+    /* Pass 1 ascending addresses, fill memory with pattern. */
+    BDK_TRACE(DRAM_TEST, "    [0x%016lx:0x%016lx] Phase1, address incrementing, pattern 0x%016lx\n", area, max_address-1, pattern);
+    for (uint64_t address = area; address < max_address; address += 8)
+        WRITE64(address, pattern);
+
+    __bdk_dram_flush_to_mem_range(area, max_address);
+    BDK_DCACHE_INVALIDATE;
+
+    /* Pass 2: ascending addresses, read pattern and write ~pattern */
+    BDK_TRACE(DRAM_TEST, "    [0x%016lx:0x%016lx] Phase2, address incrementing, pattern 0x%016lx\n", area, max_address-1, ~pattern);
+    for (uint64_t address = area; address < max_address; address += 8)
+    {
+        uint64_t data = READ64(address);
+        if (bdk_unlikely(data != pattern))
+            failures += __bdk_dram_retry_failure(1, address, data, pattern);
+        WRITE64(address, ~pattern);
+    }
+
+    __bdk_dram_flush_to_mem_range(area, max_address);
+    BDK_DCACHE_INVALIDATE;
+
+    /* Pass 3: ascending addresses, read ~pattern and write pattern. */
+    BDK_TRACE(DRAM_TEST, "    [0x%016lx:0x%016lx] Phase3, address incrementing, pattern 0x%016lx\n", area, max_address-1, pattern);
+    for (uint64_t address = area; address < max_address; address += 8)
+    {
+        uint64_t data = READ64(address);
+        if (bdk_unlikely(data != ~pattern))
+            failures += __bdk_dram_retry_failure(1, address, data, ~pattern);
+        WRITE64(address, pattern);
+    }
+
+    __bdk_dram_flush_to_mem_range(area, max_address);
+    BDK_DCACHE_INVALIDATE;
+
+    /* Pass 4: descending addresses, read pattern and write ~pattern. */
+    BDK_TRACE(DRAM_TEST, "    [0x%016lx:0x%016lx] Phase4, address decrementing, pattern 0x%016lx\n", area, max_address-1, ~pattern);
+    uint64_t end = max_address - sizeof(uint64_t);
+    for (uint64_t address = end; address >= area; address -= 8)
+    {
+        uint64_t data = READ64(address);
+        if (bdk_unlikely(data != pattern))
+            failures += __bdk_dram_retry_failure(1, address, data, pattern);
+        WRITE64(address, ~pattern);
+    }
+
+    __bdk_dram_flush_to_mem_range(area, max_address);
+    BDK_DCACHE_INVALIDATE;
+
+    /* Pass 5: descending addresses, read ~pattern and write pattern. */
+    BDK_TRACE(DRAM_TEST, "    [0x%016lx:0x%016lx] Phase5, address decrementing, pattern 0x%016lx\n", area, max_address-1, pattern);
+    for (uint64_t address = end; address >= area; address -= 8)
+    {
+        uint64_t data = READ64(address);
+        if (bdk_unlikely(data != ~pattern))
+            failures += __bdk_dram_retry_failure(1, address, data, ~pattern);
+        WRITE64(address, pattern);
+    }
+
+    __bdk_dram_flush_to_mem_range(area, max_address);
+    BDK_DCACHE_INVALIDATE;
+
+    /* Pass 6: ascending addresses, read pattern. */
+    BDK_TRACE(DRAM_TEST, "    [0x%016lx:0x%016lx] Phase6, address incrementing\n", area, max_address-1);
+    for (uint64_t address = area; address < max_address; address += 8)
+    {
+        uint64_t data = READ64(address);
+        if (bdk_unlikely(data != pattern))
+            failures += __bdk_dram_retry_failure(1, address, data, pattern);
+    }
+
+    return failures;
+}
+
+/**
+ * Use test_mem_march_c() with a all ones pattern
+ *
+ * @param area   Start of the physical memory area
+ * @param max_address
+ *               End of the physical memory area
+ * @param bursts Number of time to repeat the test over the entire area
+ *
+ * @return Number of errors, zero on success
+ */
+int __bdk_dram_test_mem_solid(uint64_t area, uint64_t max_address, int bursts)
+{
+    int failures = 0;
+    for (int burst = 0; burst < bursts; burst++)
+        failures += test_mem_march_c(area, max_address, -1);
+    return failures;
+}
+
+/**
+ * Use test_mem_march_c() with a 0x55 pattern
+ *
+ * @param area   Start of the physical memory area
+ * @param max_address
+ *               End of the physical memory area
+ * @param bursts Number of time to repeat the test over the entire area
+ *
+ * @return Number of errors, zero on success
+ */
+int __bdk_dram_test_mem_checkerboard(uint64_t area, uint64_t max_address, int bursts)
+{
+    int failures = 0;
+    for (int burst = 0; burst < bursts; burst++)
+        failures += test_mem_march_c(area, max_address, 0x5555555555555555L);
+    return failures;
+}
+
+/**
+ * Write a pseudo random pattern to memory and verify it
+ *
+ * @param area   Start of the physical memory area
+ * @param max_address
+ *               End of the physical memory area
+ * @param bursts Number of time to repeat the test over the entire area
+ *
+ * @return Number of errors, zero on success
+ */
+int __bdk_dram_test_mem_random(uint64_t area, uint64_t max_address, int bursts)
+{
+    /* This constant is used to increment the pattern after every DWORD. This
+       makes only the first DWORD truly random, but saves us processing
+       power generating the random values */
+    const uint64_t INC = 0x1010101010101010ULL;
+
+    int failures = 0;
+    for (int burst = 0; burst < bursts; burst++)
+    {
+        const uint64_t init_pattern = bdk_rng_get_random64();
+        uint64_t pattern = init_pattern;
+
+        /* Write the pattern to memory. Each location receives the address
+         * of the location. A second write pass is needed to force all of
+         * the cached memory out to the DDR.
+         */
+        for (uint64_t address = area; address < max_address; address += 8)
+        {
+            WRITE64(address, pattern);
+            pattern += INC;
+        }
+
+        __bdk_dram_flush_to_mem_range(area, max_address);
+        BDK_DCACHE_INVALIDATE;
+
+        /* Read the written memory and confirm that it has the expected
+         * data pattern.
+         */
+        pattern = init_pattern;
+        for (uint64_t address = area; address < max_address; address += 8)
+        {
+            uint64_t data = READ64(address);
+            if (bdk_unlikely(data != pattern))
+                failures += __bdk_dram_retry_failure(burst, address, data, pattern);
+            pattern += INC;
+        }
+    }
+    return failures;
+}
+
+#if !USE_PREDICTION_CODE_VERSIONS
+/**
+ * test_mem_xor
+ *
+ * @param area   Start of the physical memory area
+ * @param max_address
+ *               End of the physical memory area
+ * @param bursts Number of time to repeat the test over the entire area
+ *
+ * @return Number of errors, zero on success
+ */
+int __bdk_dram_test_mem_xor(uint64_t area, uint64_t max_address, int bursts)
+{
+    int failures = 0;
+
+    uint64_t extent = max_address - area;
+    uint64_t count = (extent / sizeof(uint64_t)) / 2;
+
+    /* Fill both halves of the memory area with identical randomized data.
+     */
+    uint64_t address1 = area;
+    uint64_t address2 = area + count * sizeof(uint64_t);
+
+    uint64_t pattern = bdk_rng_get_random64();
+
+    for (uint64_t j = 0; j < count; j++)
+    {
+        uint64_t p = pattern * address1;
+        WRITE64(address1, p);
+        WRITE64(address2, p);
+        address1 += 8;
+        address2 += 8;
+    }
+    __bdk_dram_flush_to_mem_range(area, max_address);
+    BDK_DCACHE_INVALIDATE;
+
+    /* Make a series of passes over the memory areas. */
+    for (int burst = 0; burst < bursts; burst++)
+    {
+        /* XOR the data with a random value, applying the change to both
+         * memory areas.
+         */
+        address1 = area;
+        address2 = area + count * sizeof(uint64_t);
+
+        pattern = bdk_rng_get_random64();
+
+        for (uint64_t j = 0; j < count; j++)
+        {
+            if ((address1 & BDK_CACHE_LINE_MASK) == 0)
+                BDK_PREFETCH(address1, BDK_CACHE_LINE_SIZE);
+            if ((address2 & BDK_CACHE_LINE_MASK) == 0)
+                BDK_PREFETCH(address2, BDK_CACHE_LINE_SIZE);
+            WRITE64(address1, READ64(address1) ^ pattern);
+            WRITE64(address2, READ64(address2) ^ pattern);
+            address1 += 8;
+            address2 += 8;
+        }
+
+        __bdk_dram_flush_to_mem_range(area, max_address);
+        BDK_DCACHE_INVALIDATE;
+
+        /* Look for differences in the areas. If there is a mismatch, reset
+         * both memory locations with the same pattern. Failing to do so
+         * means that on all subsequent passes the pair of locations remain
+         * out of sync giving spurious errors.
+         */
+        address1 = area;
+        address2 = area + count * sizeof(uint64_t);
+        for (uint64_t j = 0; j < count; j++)
+        {
+            if ((address1 & BDK_CACHE_LINE_MASK) == 0)
+                BDK_PREFETCH(address1, BDK_CACHE_LINE_SIZE);
+            if ((address2 & BDK_CACHE_LINE_MASK) == 0)
+                BDK_PREFETCH(address2, BDK_CACHE_LINE_SIZE);
+            uint64_t d1 = READ64(address1);
+            uint64_t d2 = READ64(address2);
+            if (bdk_unlikely(d1 != d2))
+            {
+		failures += __bdk_dram_retry_failure2(burst, address1, d1, address2, d2);
+
+                // Synchronize the two areas, adjusting for the error.
+                WRITE64(address1, d2);
+                WRITE64(address2, d2);
+            }
+            address1 += 8;
+            address2 += 8;
+        }
+    }
+    return failures;
+}
+
+/**
+ * test_mem_rows
+ *
+ * Write a pattern of alternating 64-bit words of all one bits and then all 0
+ * bits. This pattern generates the maximum amount of simultaneous switching
+ * activity on the memory channels. Each pass flips the pattern with words
+ * going from all ones to all zeros and vice versa.
+ *
+ * @param area   Start of the physical memory area
+ * @param max_address
+ *               End of the physical memory area
+ * @param bursts Number of times to repeat the test over the entire area
+ *
+ * @return Number of errors, zero on success
+ */
+int __bdk_dram_test_mem_rows(uint64_t area, uint64_t max_address, int bursts)
+{
+    int failures = 0;
+    uint64_t pattern = 0x0;
+    uint64_t extent = (max_address - area);
+    uint64_t count  = (extent / 2) / sizeof(uint64_t); // in terms of 64bit words
+
+    /* Fill both halves of the memory area with identical data pattern. Odd
+     * address 64-bit words get the pattern, while even address words get the
+     * inverted pattern.
+     */
+    uint64_t address1 = area;
+    uint64_t address2 = area + count * sizeof(uint64_t);
+
+    for (uint64_t j = 0; j < (count / 2); j++)
+    {
+        WRITE64(address1, pattern);
+        WRITE64(address2, pattern);
+        address1 += 8;
+        address2 += 8;
+        WRITE64(address1, ~pattern);
+        WRITE64(address2, ~pattern);
+        address1 += 8;
+        address2 += 8;
+    }
+    __bdk_dram_flush_to_mem_range(area, max_address);
+    BDK_DCACHE_INVALIDATE;
+
+    /* Make a series of passes over the memory areas. */
+    for (int burst = 0; burst < bursts; burst++)
+    {
+        /* Invert the data, applying the change to both memory areas. Thus on
+	 * alternate passes, the data flips from 0 to 1 and vice versa.
+         */
+        address1 = area;
+        address2 = area + count * sizeof(uint64_t);
+        for (uint64_t j = 0; j < count; j++)
+        {
+            WRITE64(address1, ~READ64(address1));
+            WRITE64(address2, ~READ64(address2));
+            address1 += 8;
+            address2 += 8;
+        }
+        __bdk_dram_flush_to_mem_range(area, max_address);
+        BDK_DCACHE_INVALIDATE;
+
+        /* Look for differences in the areas. If there is a mismatch, reset
+         * both memory locations with the same pattern. Failing to do so
+         * means that on all subsequent passes the pair of locations remain
+         * out of sync giving spurious errors.
+         */
+        address1 = area;
+        address2 = area + count * sizeof(uint64_t);
+        for (uint64_t j = 0; j < count; j++)
+        {
+            uint64_t d1 = READ64(address1);
+            uint64_t d2 = READ64(address2);
+            if (bdk_unlikely(d1 != d2))
+            {
+		failures += __bdk_dram_retry_failure2(burst, address1, d1, address2, d2);
+
+                // Synchronize the two areas, adjusting for the error.
+                WRITE64(address1, d2);
+                WRITE64(address2, d2);
+            }
+            address1 += 8;
+            address2 += 8;
+        }
+    }
+    return failures;
+}
+#endif /* !USE_PREDICTION_CODE_VERSIONS */
+
+#if USE_PREDICTION_CODE_VERSIONS
+//////////////////////////// this is the new code...
+
+int __bdk_dram_test_mem_xor(uint64_t area, uint64_t max_address, int bursts)
+{
+    int failures = 0;
+    int burst;
+
+    uint64_t extent = max_address - area;
+    uint64_t count  = (extent / sizeof(uint64_t)) / 2;
+    uint64_t offset = count * sizeof(uint64_t);
+    uint64_t area2  = area + offset;
+
+    /* Fill both halves of the memory area with identical randomized data.
+     */
+    uint64_t address1 = area;
+
+    uint64_t pattern1 = bdk_rng_get_random64();
+    uint64_t pattern2 = 0;
+    uint64_t this_pattern;
+
+    uint64_t p;
+    uint64_t d1, d2;
+
+    // move the multiplies outside the loop
+    uint64_t pbase = address1 * pattern1;
+    uint64_t pincr = 8 * pattern1;
+    uint64_t ppred;
+
+    p = pbase;
+    while (address1 < area2)
+    {
+        WRITE64(address1         , p);
+        WRITE64(address1 + offset, p);
+        address1 += 8;
+	p += pincr;
+    }
+    __bdk_dram_flush_to_mem_range(area, max_address);
+    BDK_DCACHE_INVALIDATE;
+
+    /* Make a series of passes over the memory areas. */
+    for (burst = 0; burst < bursts; burst++)
+    {
+        /* XOR the data with a random value, applying the change to both
+         * memory areas.
+         */
+        address1 = area;
+
+        this_pattern = bdk_rng_get_random64();
+	pattern2 ^= this_pattern;
+
+        while (address1 < area2)
+        {
+#if 1
+            if ((address1 & BDK_CACHE_LINE_MASK) == 0)
+                BDK_PREFETCH(address1, BDK_CACHE_LINE_SIZE);
+            if (((address1 + offset) & BDK_CACHE_LINE_MASK) == 0)
+                BDK_PREFETCH(address1 + offset, BDK_CACHE_LINE_SIZE);
+#endif
+            WRITE64(address1         , READ64(address1         ) ^ this_pattern);
+            WRITE64(address1 + offset, READ64(address1 + offset) ^ this_pattern);
+            address1 += 8;
+        }
+
+        __bdk_dram_flush_to_mem_range(area, max_address);
+        BDK_DCACHE_INVALIDATE;
+
+        /* Look for differences from the expected pattern in both areas.
+	 * If there is a mismatch, reset the appropriate memory location
+	 * with the correct pattern. Failing to do so
+         * means that on all subsequent passes the erroring locations
+	 * will be out of sync, giving spurious errors.
+         */
+        address1 = area;
+	ppred = pbase;
+
+        while (address1 < area2)
+        {
+#if 1
+            if ((address1 & BDK_CACHE_LINE_MASK) == 0)
+                BDK_PREFETCH(address1, BDK_CACHE_LINE_SIZE);
+            if (((address1 + offset) & BDK_CACHE_LINE_MASK) == 0)
+                BDK_PREFETCH(address1 + offset, BDK_CACHE_LINE_SIZE);
+#endif
+            d1 = READ64(address1         );
+            d2 = READ64(address1 + offset);
+
+	    p = ppred ^ pattern2;
+
+            if (bdk_unlikely(d1 != p)) {
+		failures += __bdk_dram_retry_failure(burst, address1, d1, p);
+                // Synchronize the area, adjusting for the error.
+                //WRITE64(address1, p); // retries should do this
+            }
+            if (bdk_unlikely(d2 != p)) {
+		failures += __bdk_dram_retry_failure(burst, address1 + offset, d2, p);
+                // Synchronize the area, adjusting for the error.
+                //WRITE64(address1 + offset, p); // retries should do this
+            }
+
+            address1 += 8;
+	    ppred += pincr;
+
+        } /* while (address1 < area2) */
+    } /* for (int burst = 0; burst < bursts; burst++) */
+    return failures;
+}
+
+//////////////// this is the new code...
+
+int __bdk_dram_test_mem_rows(uint64_t area, uint64_t max_address, int bursts)
+{
+    int failures = 0;
+
+    uint64_t pattern1 = 0x0;
+    uint64_t extent = (max_address - area);
+    uint64_t count  = (extent / 2) / sizeof(uint64_t); // in terms of 64bit words
+    uint64_t offset = count * sizeof(uint64_t);
+    uint64_t area2 = area + offset;
+    uint64_t pattern2;
+    uint64_t d1, d2;
+    int burst;
+
+    /* Fill both halves of the memory area with identical data pattern. Odd
+     * address 64-bit words get the pattern, while even address words get the
+     * inverted pattern.
+     */
+    uint64_t address1 = area;
+
+    pattern2 = pattern1; // start with original pattern
+
+    while (address1 < area2)
+    {
+        WRITE64(address1         , pattern2);
+        WRITE64(address1 + offset, pattern2);
+        address1 += 8;
+	pattern2 = ~pattern2; // flip for next slots
+    }
+
+    __bdk_dram_flush_to_mem_range(area, max_address);
+    BDK_DCACHE_INVALIDATE;
+
+    /* Make a series of passes over the memory areas. */
+    for (burst = 0; burst < bursts; burst++)
+    {
+        /* Invert the data, applying the change to both memory areas. Thus on
+	 * alternate passes, the data flips from 0 to 1 and vice versa.
+         */
+        address1 = area;
+
+        while (address1 < area2)
+        {
+            if ((address1 & BDK_CACHE_LINE_MASK) == 0)
+                BDK_PREFETCH(address1         , BDK_CACHE_LINE_SIZE);
+            if (((address1 + offset) & BDK_CACHE_LINE_MASK) == 0)
+                BDK_PREFETCH(address1 + offset, BDK_CACHE_LINE_SIZE);
+
+            WRITE64(address1         , ~READ64(address1         ));
+            WRITE64(address1 + offset, ~READ64(address1 + offset));
+            address1 += 8;
+        }
+
+        __bdk_dram_flush_to_mem_range(area, max_address);
+        BDK_DCACHE_INVALIDATE;
+
+        /* Look for differences in the areas. If there is a mismatch, reset
+         * both memory locations with the same pattern. Failing to do so
+         * means that on all subsequent passes the pair of locations remain
+         * out of sync giving spurious errors.
+         */
+        address1 = area;
+	pattern1 = ~pattern1; // flip the starting pattern to match above loop
+	pattern2 = pattern1;  // slots have been flipped by the above loop
+
+        while (address1 < area2)
+        {
+            if ((address1 & BDK_CACHE_LINE_MASK) == 0)
+                BDK_PREFETCH(address1         , BDK_CACHE_LINE_SIZE);
+            if (((address1 + offset) & BDK_CACHE_LINE_MASK) == 0)
+                BDK_PREFETCH(address1 + offset, BDK_CACHE_LINE_SIZE);
+
+            d1 = READ64(address1         );
+            d2 = READ64(address1 + offset);
+
+            if (bdk_unlikely(d1 != pattern2)) {
+		failures += __bdk_dram_retry_failure(burst, address1, d1, pattern2);
+                // Synchronize the area, adjusting for the error.
+                //WRITE64(address1, pattern2); // retries should do this
+            }
+            if (bdk_unlikely(d2 != pattern2)) {
+		failures += __bdk_dram_retry_failure(burst, address1 + offset, d2, pattern2);
+                // Synchronize the two areas, adjusting for the error.
+                //WRITE64(address1 + offset, pattern2); // retries should do this
+            }
+
+            address1 += 8;
+	    pattern2 = ~pattern2; // flip for next pair of slots
+        }
+    }
+    return failures;
+}
+#endif /* USE_PREDICTION_CODE_VERSIONS */