path: root/src/vendorcode/cavium/bdk/libbdk-dram/bdk-dram-test-patfil.c

/***********************license start***********************************
* Copyright (c) 2003-2017  Cavium Inc. (support@cavium.com). All rights
* reserved.
*
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are
* met:
*
*   * Redistributions of source code must retain the above copyright
*     notice, this list of conditions and the following disclaimer.
*
*   * Redistributions in binary form must reproduce the above
*     copyright notice, this list of conditions and the following
*     disclaimer in the documentation and/or other materials provided
*     with the distribution.
*
*   * Neither the name of Cavium Inc. nor the names of
*     its contributors may be used to endorse or promote products
*     derived from this software without specific prior written
*     permission.
*
* This Software, including technical data, may be subject to U.S. export
* control laws, including the U.S. Export Administration Act and its
* associated regulations, and may be subject to export or import
* regulations in other countries.
*
* TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS"
* AND WITH ALL FAULTS AND CAVIUM INC. MAKES NO PROMISES, REPRESENTATIONS OR
* WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH RESPECT
* TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY
* REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT
* DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) WARRANTIES
* OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A PARTICULAR
* PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET ENJOYMENT,
* QUIET POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE ENTIRE  RISK
* ARISING OUT OF USE OR PERFORMANCE OF THE SOFTWARE LIES WITH YOU.
***********************license end**************************************/
#include "bdk.h"
#include <libbdk-hal/bdk-rng.h>
#include <libbdk-hal/bdk-utils.h>

// choose prediction-based algorithms for mem_xor and mem_rows tests
#define USE_PREDICTION_CODE_VERSIONS 1   // change to 0 to go back to the original versions

/* Used for all memory reads/writes related to the test */
#define READ64(address) __bdk_dram_read64(address)
#define WRITE64(address, data) __bdk_dram_write64(address, data)

/**
 * Fill an memory area with the address of each 64-bit word in the area.
 * Reread to confirm the pattern.
 *
 * @param area   Start of the physical memory area
 * @param max_address
 *               End of the physical memory area (exclusive)
 * @param bursts Number of time to repeat the test over the entire area
 *
 * @return Number of errors, zero on success
 */
int __bdk_dram_test_mem_self_addr(uint64_t area, uint64_t max_address, int bursts)
{
    int failures = 0;

    for (int burst = 0; burst < bursts; burst++)
    {
        /* Write the pattern to memory. Each location receives the address
         * of the location.
         */
        for (uint64_t address = area; address < max_address; address+=8)
            WRITE64(address, address);
        __bdk_dram_flush_to_mem_range(area, max_address);
        BDK_DCACHE_INVALIDATE;

        /* Read by ascending address the written memory and confirm that it
         * has the expected data pattern.
         */
        for (uint64_t address = area; address < max_address; )
        {
            if (address + 256 < max_address)
                BDK_PREFETCH(address + 256, 0);
            for (int i=0; i<16; i++)
            {
                uint64_t data = READ64(address);
                if (bdk_unlikely(data != address))
                    failures += __bdk_dram_retry_failure(burst, address, data, address);
                address += 8;
            }
        }
        __bdk_dram_flush_to_mem_range(area, max_address);
        BDK_DCACHE_INVALIDATE;

        /* Read by descending address the written memory and confirm that it
         * has the expected data pattern.
         */
        uint64_t end = max_address - sizeof(uint64_t);
        for (uint64_t address = end; address >= area; )
        {
            if (address - 256 >= area)
                BDK_PREFETCH(address - 256, 0);
            for (int i=0; i<16; i++)
            {
                uint64_t data = READ64(address);
                if (bdk_unlikely(data != address))
                    failures += __bdk_dram_retry_failure(burst, address, data, address);
                address -= 8;
            }
        }
        __bdk_dram_flush_to_mem_range(area, max_address);
        BDK_DCACHE_INVALIDATE;

        /* Read from random addresses within the memory area.
         */
        uint64_t probes = (max_address - area) / 128;
        uint64_t address_ahead1 = area;
        uint64_t address_ahead2 = area;
        for (uint64_t i = 0; i < probes; i++)
        {
            /* Create a pipeline of prefetches:
               address = address read this loop
               address_ahead1 = prefetch started last loop
               address_ahead2 = prefetch started this loop */
            uint64_t address = address_ahead1;
            address_ahead1 = address_ahead2;
            address_ahead2 = bdk_rng_get_random64() % (max_address - area);
            address_ahead2 += area;
            address_ahead2 &= -8;
            BDK_PREFETCH(address_ahead2, 0);

            uint64_t data = READ64(address);
            if (bdk_unlikely(data != address))
                failures += __bdk_dram_retry_failure(burst, address, data, address);
        }
    }
    return failures;
}

/**
 * Write "pattern" and its compliment to memory and verify it was written
 * properly. Memory will be filled with DWORDs pattern, ~pattern, pattern,
 * ~pattern, ...
 *
 * @param area    Start physical address of memory
 * @param max_address
 *                End of physical memory region
 * @param pattern Pattern to write
 * @param passes  Number of time to repeat the test
 *
 * @return Number of errors, zero on success
 */
static uint32_t test_mem_pattern(uint64_t area, uint64_t max_address, uint64_t pattern,
    int passes)
{
    int failures = 0;

    for (int pass = 0; pass < passes; pass++)
    {
        if (pass & 0x1)
            pattern = ~pattern;

        for (uint64_t address = area; address < max_address; address += 8)
            WRITE64(address, pattern);
        __bdk_dram_flush_to_mem_range(area, max_address);
        BDK_DCACHE_INVALIDATE;

        /* Read the written memory and confirm that it has the expected
         * data pattern.
         */
        uint64_t address = area;
        while (address < max_address)
        {
            if (address + 256 < max_address)
                BDK_PREFETCH(address + 256, 0);
            for (int i=0; i<16; i++)
            {
                uint64_t data = READ64(address);
                if (bdk_unlikely(data != pattern))
                    failures += __bdk_dram_retry_failure(pass, address, data, pattern);
                address += 8;
            }
        }
    }
    return failures;
}

/**
 * Walking zero written to memory, left shift
 *
 * @param area   Start of the physical memory area
 * @param max_address
 *               End of the physical memory area
 * @param bursts Number of time to repeat the test over the entire area
 *
 * @return Number of errors, zero on success
 */
int __bdk_dram_test_mem_leftwalk0(uint64_t area, uint64_t max_address, int bursts)
{
    int failures = 0;
    for (int burst = 0; burst < bursts; burst++)
    {
        for (uint64_t pattern = 1; pattern != 0; pattern = pattern << 1)
            failures += test_mem_pattern(area, max_address, ~pattern, 1);
    }
    return failures;
}

/**
 * Walking one written to memory, left shift
 *
 * @param area   Start of the physical memory area
 * @param max_address
 *               End of the physical memory area
 * @param bursts Number of time to repeat the test over the entire area
 *
 * @return Number of errors, zero on success
 */
int __bdk_dram_test_mem_leftwalk1(uint64_t area, uint64_t max_address, int bursts)
{
    int failures = 0;
    for (int burst = 0; burst < bursts; burst++)
    {
        for (uint64_t pattern = 1; pattern != 0; pattern = pattern << 1)
            failures += test_mem_pattern(area, max_address, pattern, 1);
    }
    return failures;
}

/**
 * Walking zero written to memory, right shift
 *
 * @param area   Start of the physical memory area
 * @param max_address
 *               End of the physical memory area
 * @param bursts Number of time to repeat the test over the entire area
 *
 * @return Number of errors, zero on success
 */
int __bdk_dram_test_mem_rightwalk0(uint64_t area, uint64_t max_address, int bursts)
{
    int failures = 0;
    for (int burst = 0; burst < bursts; burst++)
    {
        for (uint64_t pattern = 1ull << 63; pattern != 0; pattern = pattern >> 1)
            failures += test_mem_pattern(area, max_address, ~pattern, 1);
    }
    return failures;
}

/**
 * Walking one written to memory, right shift
 *
 * @param area   Start of the physical memory area
 * @param max_address
 *               End of the physical memory area
 * @param bursts Number of time to repeat the test over the entire area
 *
 * @return Number of errors, zero on success
 */
int __bdk_dram_test_mem_rightwalk1(uint64_t area, uint64_t max_address, int bursts)
{
    int failures = 0;
    for (int burst = 0; burst < bursts; burst++)
    {
        for (uint64_t pattern = 1ull<<63; pattern != 0; pattern = pattern >> 1)
            failures += test_mem_pattern(area, max_address, pattern, 1);
    }
    return failures;
}

/**
 * Apply the March C- testing algorithm to the given memory area.
 * 1) Write "pattern" to memory.
 * 2) Verify "pattern" and write "~pattern".
 * 3) Verify "~pattern" and write "pattern".
 * 4) Verify "pattern" and write "~pattern".
 * 5) Verify "~pattern" and write "pattern".
 * 6) Verify "pattern".
 *
 * @param area    Start of the physical memory area
 * @param max_address
 *                End of the physical memory area
 * @param pattern
 *
 * @return Number of errors, zero on success
 */
static int test_mem_march_c(uint64_t area, uint64_t max_address, uint64_t pattern)
{
    int failures = 0;

    /* Pass 1 ascending addresses, fill memory with pattern. */
    BDK_TRACE(DRAM_TEST, "    [0x%016llx:0x%016llx] Phase1, address incrementing, pattern 0x%016llx\n", area, max_address-1, pattern);
    for (uint64_t address = area; address < max_address; address += 8)
        WRITE64(address, pattern);

    __bdk_dram_flush_to_mem_range(area, max_address);
    BDK_DCACHE_INVALIDATE;

    /* Pass 2: ascending addresses, read pattern and write ~pattern */
    BDK_TRACE(DRAM_TEST, "    [0x%016llx:0x%016llx] Phase2, address incrementing, pattern 0x%016llx\n", area, max_address-1, ~pattern);
    for (uint64_t address = area; address < max_address; address += 8)
    {
        uint64_t data = READ64(address);
        if (bdk_unlikely(data != pattern))
            failures += __bdk_dram_retry_failure(1, address, data, pattern);
        WRITE64(address, ~pattern);
    }

    __bdk_dram_flush_to_mem_range(area, max_address);
    BDK_DCACHE_INVALIDATE;

    /* Pass 3: ascending addresses, read ~pattern and write pattern. */
    BDK_TRACE(DRAM_TEST, "    [0x%016llx:0x%016llx] Phase3, address incrementing, pattern 0x%016llx\n", area, max_address-1, pattern);
    for (uint64_t address = area; address < max_address; address += 8)
    {
        uint64_t data = READ64(address);
        if (bdk_unlikely(data != ~pattern))
            failures += __bdk_dram_retry_failure(1, address, data, ~pattern);
        WRITE64(address, pattern);
    }

    __bdk_dram_flush_to_mem_range(area, max_address);
    BDK_DCACHE_INVALIDATE;

    /* Pass 4: descending addresses, read pattern and write ~pattern. */
    BDK_TRACE(DRAM_TEST, "    [0x%016llx:0x%016llx] Phase4, address decrementing, pattern 0x%016llx\n", area, max_address-1, ~pattern);
    uint64_t end = max_address - sizeof(uint64_t);
    for (uint64_t address = end; address >= area; address -= 8)
    {
        uint64_t data = READ64(address);
        if (bdk_unlikely(data != pattern))
            failures += __bdk_dram_retry_failure(1, address, data, pattern);
        WRITE64(address, ~pattern);
    }

    __bdk_dram_flush_to_mem_range(area, max_address);
    BDK_DCACHE_INVALIDATE;

    /* Pass 5: descending addresses, read ~pattern and write pattern. */
    BDK_TRACE(DRAM_TEST, "    [0x%016llx:0x%016llx] Phase5, address decrementing, pattern 0x%016llx\n", area, max_address-1, pattern);
    for (uint64_t address = end; address >= area; address -= 8)
    {
        uint64_t data = READ64(address);
        if (bdk_unlikely(data != ~pattern))
            failures += __bdk_dram_retry_failure(1, address, data, ~pattern);
        WRITE64(address, pattern);
    }

    __bdk_dram_flush_to_mem_range(area, max_address);
    BDK_DCACHE_INVALIDATE;

    /* Pass 6: ascending addresses, read pattern. */
    BDK_TRACE(DRAM_TEST, "    [0x%016llx:0x%016llx] Phase6, address incrementing\n", area, max_address-1);
    for (uint64_t address = area; address < max_address; address += 8)
    {
        uint64_t data = READ64(address);
        if (bdk_unlikely(data != pattern))
            failures += __bdk_dram_retry_failure(1, address, data, pattern);
    }

    return failures;
}

/**
 * Use test_mem_march_c() with a all ones pattern
 *
 * @param area   Start of the physical memory area
 * @param max_address
 *               End of the physical memory area
 * @param bursts Number of time to repeat the test over the entire area
 *
 * @return Number of errors, zero on success
 */
int __bdk_dram_test_mem_solid(uint64_t area, uint64_t max_address, int bursts)
{
    int failures = 0;
    for (int burst = 0; burst < bursts; burst++)
        failures += test_mem_march_c(area, max_address, -1);
    return failures;
}

/**
 * Use test_mem_march_c() with a 0x55 pattern
 *
 * @param area   Start of the physical memory area
 * @param max_address
 *               End of the physical memory area
 * @param bursts Number of time to repeat the test over the entire area
 *
 * @return Number of errors, zero on success
 */
int __bdk_dram_test_mem_checkerboard(uint64_t area, uint64_t max_address, int bursts)
{
    int failures = 0;
    for (int burst = 0; burst < bursts; burst++)
        failures += test_mem_march_c(area, max_address, 0x5555555555555555L);
    return failures;
}

/**
 * Write a pseudo random pattern to memory and verify it
 *
 * @param area   Start of the physical memory area
 * @param max_address
 *               End of the physical memory area
 * @param bursts Number of time to repeat the test over the entire area
 *
 * @return Number of errors, zero on success
 */
int __bdk_dram_test_mem_random(uint64_t area, uint64_t max_address, int bursts)
{
    /* This constant is used to increment the pattern after every DWORD. This
       makes only the first DWORD truly random, but saves us processing
       power generating the random values */
    const uint64_t INC = 0x1010101010101010ULL;

    int failures = 0;
    for (int burst = 0; burst < bursts; burst++)
    {
        const uint64_t init_pattern = bdk_rng_get_random64();
        uint64_t pattern = init_pattern;

        /* Write the pattern to memory. Each location receives the address
         * of the location. A second write pass is needed to force all of
         * the cached memory out to the DDR.
         */
        for (uint64_t address = area; address < max_address; address += 8)
        {
            WRITE64(address, pattern);
            pattern += INC;
        }

        __bdk_dram_flush_to_mem_range(area, max_address);
        BDK_DCACHE_INVALIDATE;

        /* Read the written memory and confirm that it has the expected
         * data pattern.
         */
        pattern = init_pattern;
        for (uint64_t address = area; address < max_address; address += 8)
        {
            uint64_t data = READ64(address);
            if (bdk_unlikely(data != pattern))
                failures += __bdk_dram_retry_failure(burst, address, data, pattern);
            pattern += INC;
        }
    }
    return failures;
}

#if !USE_PREDICTION_CODE_VERSIONS
/**
 * test_mem_xor
 *
 * @param area   Start of the physical memory area
 * @param max_address
 *               End of the physical memory area
 * @param bursts Number of time to repeat the test over the entire area
 *
 * @return Number of errors, zero on success
 */
int __bdk_dram_test_mem_xor(uint64_t area, uint64_t max_address, int bursts)
{
    int failures = 0;

    uint64_t extent = max_address - area;
    uint64_t count = (extent / sizeof(uint64_t)) / 2;

    /* Fill both halves of the memory area with identical randomized data.
     */
    uint64_t address1 = area;
    uint64_t address2 = area + count * sizeof(uint64_t);

    uint64_t pattern = bdk_rng_get_random64();

    for (uint64_t j = 0; j < count; j++)
    {
        uint64_t p = pattern * address1;
        WRITE64(address1, p);
        WRITE64(address2, p);
        address1 += 8;
        address2 += 8;
    }
    __bdk_dram_flush_to_mem_range(area, max_address);
    BDK_DCACHE_INVALIDATE;

    /* Make a series of passes over the memory areas. */
    for (int burst = 0; burst < bursts; burst++)
    {
        /* XOR the data with a random value, applying the change to both
         * memory areas.
         */
        address1 = area;
        address2 = area + count * sizeof(uint64_t);

        pattern = bdk_rng_get_random64();

        for (uint64_t j = 0; j < count; j++)
        {
            if ((address1 & BDK_CACHE_LINE_MASK) == 0)
                BDK_PREFETCH(address1, BDK_CACHE_LINE_SIZE);
            if ((address2 & BDK_CACHE_LINE_MASK) == 0)
                BDK_PREFETCH(address2, BDK_CACHE_LINE_SIZE);
            WRITE64(address1, READ64(address1) ^ pattern);
            WRITE64(address2, READ64(address2) ^ pattern);
            address1 += 8;
            address2 += 8;
        }

        __bdk_dram_flush_to_mem_range(area, max_address);
        BDK_DCACHE_INVALIDATE;

        /* Look for differences in the areas. If there is a mismatch, reset
         * both memory locations with the same pattern. Failing to do so
         * means that on all subsequent passes the pair of locations remain
         * out of sync giving spurious errors.
         */
        address1 = area;
        address2 = area + count * sizeof(uint64_t);
        for (uint64_t j = 0; j < count; j++)
        {
            if ((address1 & BDK_CACHE_LINE_MASK) == 0)
                BDK_PREFETCH(address1, BDK_CACHE_LINE_SIZE);
            if ((address2 & BDK_CACHE_LINE_MASK) == 0)
                BDK_PREFETCH(address2, BDK_CACHE_LINE_SIZE);
            uint64_t d1 = READ64(address1);
            uint64_t d2 = READ64(address2);
            if (bdk_unlikely(d1 != d2))
            {
		failures += __bdk_dram_retry_failure2(burst, address1, d1, address2, d2);

                // Synchronize the two areas, adjusting for the error.
                WRITE64(address1, d2);
                WRITE64(address2, d2);
            }
            address1 += 8;
            address2 += 8;
        }
    }
    return failures;
}

/**
 * test_mem_rows
 *
 * Write a pattern of alternating 64-bit words of all one bits and then all 0
 * bits. This pattern generates the maximum amount of simultaneous switching
 * activity on the memory channels. Each pass flips the pattern with words
 * going from all ones to all zeros and vice versa.
 *
 * @param area   Start of the physical memory area
 * @param max_address
 *               End of the physical memory area
 * @param bursts Number of times to repeat the test over the entire area
 *
 * @return Number of errors, zero on success
 */
int __bdk_dram_test_mem_rows(uint64_t area, uint64_t max_address, int bursts)
{
    int failures = 0;
    uint64_t pattern = 0x0;
    uint64_t extent = (max_address - area);
    uint64_t count  = (extent / 2) / sizeof(uint64_t); // in terms of 64bit words

    /* Fill both halves of the memory area with identical data pattern. Odd
     * address 64-bit words get the pattern, while even address words get the
     * inverted pattern.
     */
    uint64_t address1 = area;
    uint64_t address2 = area + count * sizeof(uint64_t);

    for (uint64_t j = 0; j < (count / 2); j++)
    {
        WRITE64(address1, pattern);
        WRITE64(address2, pattern);
        address1 += 8;
        address2 += 8;
        WRITE64(address1, ~pattern);
        WRITE64(address2, ~pattern);
        address1 += 8;
        address2 += 8;
    }
    __bdk_dram_flush_to_mem_range(area, max_address);
    BDK_DCACHE_INVALIDATE;

    /* Make a series of passes over the memory areas. */
    for (int burst = 0; burst < bursts; burst++)
    {
        /* Invert the data, applying the change to both memory areas. Thus on
	 * alternate passes, the data flips from 0 to 1 and vice versa.
         */
        address1 = area;
        address2 = area + count * sizeof(uint64_t);
        for (uint64_t j = 0; j < count; j++)
        {
            WRITE64(address1, ~READ64(address1));
            WRITE64(address2, ~READ64(address2));
            address1 += 8;
            address2 += 8;
        }
        __bdk_dram_flush_to_mem_range(area, max_address);
        BDK_DCACHE_INVALIDATE;

        /* Look for differences in the areas. If there is a mismatch, reset
         * both memory locations with the same pattern. Failing to do so
         * means that on all subsequent passes the pair of locations remain
         * out of sync giving spurious errors.
         */
        address1 = area;
        address2 = area + count * sizeof(uint64_t);
        for (uint64_t j = 0; j < count; j++)
        {
            uint64_t d1 = READ64(address1);
            uint64_t d2 = READ64(address2);
            if (bdk_unlikely(d1 != d2))
            {
		failures += __bdk_dram_retry_failure2(burst, address1, d1, address2, d2);

                // Synchronize the two areas, adjusting for the error.
                WRITE64(address1, d2);
                WRITE64(address2, d2);
            }
            address1 += 8;
            address2 += 8;
        }
    }
    return failures;
}
#endif /* !USE_PREDICTION_CODE_VERSIONS */

#if USE_PREDICTION_CODE_VERSIONS
//////////////////////////// this is the new code...

int __bdk_dram_test_mem_xor(uint64_t area, uint64_t max_address, int bursts)
{
    int failures = 0;
    int burst;

    uint64_t extent = max_address - area;
    uint64_t count  = (extent / sizeof(uint64_t)) / 2;
    uint64_t offset = count * sizeof(uint64_t);
    uint64_t area2  = area + offset;

    /* Fill both halves of the memory area with identical randomized data.
     */
    uint64_t address1 = area;

    uint64_t pattern1 = bdk_rng_get_random64();
    uint64_t pattern2 = 0;
    uint64_t this_pattern;

    uint64_t p;
    uint64_t d1, d2;

    // move the multiplies outside the loop
    uint64_t pbase = address1 * pattern1;
    uint64_t pincr = 8 * pattern1;
    uint64_t ppred;

    p = pbase;
    while (address1 < area2)
    {
        WRITE64(address1         , p);
        WRITE64(address1 + offset, p);
        address1 += 8;
        p += pincr;
    }
    __bdk_dram_flush_to_mem_range(area, max_address);
    BDK_DCACHE_INVALIDATE;

    /* Make a series of passes over the memory areas. */
    for (burst = 0; burst < bursts; burst++)
    {
        /* XOR the data with a random value, applying the change to both
         * memory areas.
         */
        address1 = area;

        this_pattern = bdk_rng_get_random64();
        pattern2 ^= this_pattern;

        while (address1 < area2)
        {
#if 1
            if ((address1 & BDK_CACHE_LINE_MASK) == 0)
                BDK_PREFETCH(address1, BDK_CACHE_LINE_SIZE);
            if (((address1 + offset) & BDK_CACHE_LINE_MASK) == 0)
                BDK_PREFETCH(address1 + offset, BDK_CACHE_LINE_SIZE);
#endif
            WRITE64(address1         , READ64(address1         ) ^ this_pattern);
            WRITE64(address1 + offset, READ64(address1 + offset) ^ this_pattern);
            address1 += 8;
        }

        __bdk_dram_flush_to_mem_range(area, max_address);
        BDK_DCACHE_INVALIDATE;

        /* Look for differences from the expected pattern in both areas.
         * If there is a mismatch, reset the appropriate memory location
         * with the correct pattern. Failing to do so
         * means that on all subsequent passes the erroring locations
         * will be out of sync, giving spurious errors.
         */
        address1 = area;
        ppred = pbase;

        while (address1 < area2)
        {
#if 1
            if ((address1 & BDK_CACHE_LINE_MASK) == 0)
                BDK_PREFETCH(address1, BDK_CACHE_LINE_SIZE);
            if (((address1 + offset) & BDK_CACHE_LINE_MASK) == 0)
                BDK_PREFETCH(address1 + offset, BDK_CACHE_LINE_SIZE);
#endif
            d1 = READ64(address1         );
            d2 = READ64(address1 + offset);

            p = ppred ^ pattern2;

            if (bdk_unlikely(d1 != p)) {
                failures += __bdk_dram_retry_failure(burst, address1, d1, p);
                // Synchronize the area, adjusting for the error.
                //WRITE64(address1, p); // retries should do this
            }
            if (bdk_unlikely(d2 != p)) {
                failures += __bdk_dram_retry_failure(burst, address1 + offset, d2, p);
                // Synchronize the area, adjusting for the error.
                //WRITE64(address1 + offset, p); // retries should do this
            }

            address1 += 8;
            ppred += pincr;

        } /* while (address1 < area2) */
    } /* for (int burst = 0; burst < bursts; burst++) */
    return failures;
}

//////////////// this is the new code...

int __bdk_dram_test_mem_rows(uint64_t area, uint64_t max_address, int bursts)
{
    int failures = 0;

    uint64_t pattern1 = 0x0;
    uint64_t extent = (max_address - area);
    uint64_t count  = (extent / 2) / sizeof(uint64_t); // in terms of 64bit words
    uint64_t offset = count * sizeof(uint64_t);
    uint64_t area2 = area + offset;
    uint64_t pattern2;
    uint64_t d1, d2;
    int burst;

    /* Fill both halves of the memory area with identical data pattern. Odd
     * address 64-bit words get the pattern, while even address words get the
     * inverted pattern.
     */
    uint64_t address1 = area;

    pattern2 = pattern1; // start with original pattern

    while (address1 < area2)
    {
        WRITE64(address1         , pattern2);
        WRITE64(address1 + offset, pattern2);
        address1 += 8;
        pattern2 = ~pattern2; // flip for next slots
    }

    __bdk_dram_flush_to_mem_range(area, max_address);
    BDK_DCACHE_INVALIDATE;

    /* Make a series of passes over the memory areas. */
    for (burst = 0; burst < bursts; burst++)
    {
        /* Invert the data, applying the change to both memory areas. Thus on
         * alternate passes, the data flips from 0 to 1 and vice versa.
         */
        address1 = area;

        while (address1 < area2)
        {
            if ((address1 & BDK_CACHE_LINE_MASK) == 0)
                BDK_PREFETCH(address1         , BDK_CACHE_LINE_SIZE);
            if (((address1 + offset) & BDK_CACHE_LINE_MASK) == 0)
                BDK_PREFETCH(address1 + offset, BDK_CACHE_LINE_SIZE);

            WRITE64(address1         , ~READ64(address1         ));
            WRITE64(address1 + offset, ~READ64(address1 + offset));
            address1 += 8;
        }

        __bdk_dram_flush_to_mem_range(area, max_address);
        BDK_DCACHE_INVALIDATE;

        /* Look for differences in the areas. If there is a mismatch, reset
         * both memory locations with the same pattern. Failing to do so
         * means that on all subsequent passes the pair of locations remain
         * out of sync giving spurious errors.
         */
        address1 = area;
        pattern1 = ~pattern1; // flip the starting pattern to match above loop
        pattern2 = pattern1;  // slots have been flipped by the above loop

        while (address1 < area2)
        {
            if ((address1 & BDK_CACHE_LINE_MASK) == 0)
                BDK_PREFETCH(address1         , BDK_CACHE_LINE_SIZE);
            if (((address1 + offset) & BDK_CACHE_LINE_MASK) == 0)
                BDK_PREFETCH(address1 + offset, BDK_CACHE_LINE_SIZE);

            d1 = READ64(address1         );
            d2 = READ64(address1 + offset);

            if (bdk_unlikely(d1 != pattern2)) {
                failures += __bdk_dram_retry_failure(burst, address1, d1, pattern2);
                // Synchronize the area, adjusting for the error.
                //WRITE64(address1, pattern2); // retries should do this
            }
            if (bdk_unlikely(d2 != pattern2)) {
                failures += __bdk_dram_retry_failure(burst, address1 + offset, d2, pattern2);
                // Synchronize the two areas, adjusting for the error.
                //WRITE64(address1 + offset, pattern2); // retries should do this
            }

            address1 += 8;
            pattern2 = ~pattern2; // flip for next pair of slots
        }
    }
    return failures;
}
#endif /* USE_PREDICTION_CODE_VERSIONS */