diff options
Diffstat (limited to 'src/vendorcode/cavium/bdk/libbdk-dram')
8 files changed, 2718 insertions, 0 deletions
diff --git a/src/vendorcode/cavium/bdk/libbdk-dram/bdk-dram-address.c b/src/vendorcode/cavium/bdk/libbdk-dram/bdk-dram-address.c new file mode 100644 index 0000000000..94d7d76752 --- /dev/null +++ b/src/vendorcode/cavium/bdk/libbdk-dram/bdk-dram-address.c @@ -0,0 +1,183 @@ +/***********************license start*********************************** +* Copyright (c) 2003-2017 Cavium Inc. (support@cavium.com). All rights +* reserved. +* +* +* Redistribution and use in source and binary forms, with or without +* modification, are permitted provided that the following conditions are +* met: +* +* * Redistributions of source code must retain the above copyright +* notice, this list of conditions and the following disclaimer. +* +* * Redistributions in binary form must reproduce the above +* copyright notice, this list of conditions and the following +* disclaimer in the documentation and/or other materials provided +* with the distribution. +* +* * Neither the name of Cavium Inc. nor the names of +* its contributors may be used to endorse or promote products +* derived from this software without specific prior written +* permission. +* +* This Software, including technical data, may be subject to U.S. export +* control laws, including the U.S. Export Administration Act and its +* associated regulations, and may be subject to export or import +* regulations in other countries. +* +* TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" +* AND WITH ALL FAULTS AND CAVIUM INC. MAKES NO PROMISES, REPRESENTATIONS OR +* WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH RESPECT +* TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY +* REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT +* DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) WARRANTIES +* OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A PARTICULAR +* PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET ENJOYMENT, +* QUIET POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE ENTIRE RISK +* ARISING OUT OF USE OR PERFORMANCE OF THE SOFTWARE LIES WITH YOU. +***********************license end**************************************/ +#include <bdk.h> +#include "libbdk-arch/bdk-csrs-l2c.h" + +#define EXTRACT(v, lsb, width) (((v) >> (lsb)) & ((1ull << (width)) - 1)) +#define INSERT(a, v, lsb, width) a|=(((v) & ((1ull << (width)) - 1)) << (lsb)) + +/** + * Given a physical DRAM address, extract information about the node, LMC, DIMM, + * prank, lrank, bank, row, and column that was accessed. + * + * @param address Physical address to decode + * @param node Node the address was for + * @param lmc LMC controller the address was for + * @param dimm DIMM the address was for + * @param prank Physical RANK on the DIMM + * @param lrank Logical RANK on the DIMM + * @param bank BANK on the DIMM + * @param row Row on the DIMM + * @param col Column on the DIMM + */ +void +bdk_dram_address_extract_info(uint64_t address, int *node, int *lmc, int *dimm, + int *prank, int *lrank, int *bank, int *row, int *col) +{ + int bitno = CAVIUM_IS_MODEL(CAVIUM_CN83XX) ? 19 : 20; + *node = EXTRACT(address, 40, 2); /* Address bits [41:40] */ + /* Determine the LMC controller */ + BDK_CSR_INIT(l2c_ctl, *node, BDK_L2C_CTL); + int bank_lsb, xbits; + + /* xbits depends on number of LMCs */ + xbits = __bdk_dram_get_num_lmc(*node) >> 1; // 4->2; 2->1; 1->0 + bank_lsb = 7 + xbits; + + /* LMC number is probably aliased */ + if (l2c_ctl.s.disidxalias) + *lmc = EXTRACT(address, 7, xbits); + else + *lmc = EXTRACT(address, 7, xbits) ^ EXTRACT(address, bitno, xbits) ^ EXTRACT(address, 12, xbits); + + /* Figure out the bank field width */ + BDK_CSR_INIT(lmcx_config, *node, BDK_LMCX_CONFIG(*lmc)); + int bank_width = __bdk_dram_get_num_bank_bits(*node, *lmc); + + /* Extract additional info from the LMC_CONFIG CSR */ + BDK_CSR_INIT(ext_config, *node, BDK_LMCX_EXT_CONFIG(*lmc)); + int dimm_lsb = 28 + lmcx_config.s.pbank_lsb + xbits; + int dimm_width = 40 - dimm_lsb; + int prank_lsb = dimm_lsb - lmcx_config.s.rank_ena; + int prank_width = dimm_lsb - prank_lsb; + int lrank_lsb = prank_lsb - ext_config.s.dimm0_cid; + int lrank_width = prank_lsb - lrank_lsb; + int row_lsb = 14 + lmcx_config.s.row_lsb + xbits; + int row_width = lrank_lsb - row_lsb; + int col_hi_lsb = bank_lsb + bank_width; + int col_hi_width= row_lsb - col_hi_lsb; + + /* Extract the parts of the address */ + *dimm = EXTRACT(address, dimm_lsb, dimm_width); + *prank = EXTRACT(address, prank_lsb, prank_width); + *lrank = EXTRACT(address, lrank_lsb, lrank_width); + *row = EXTRACT(address, row_lsb, row_width); + + /* bank calculation may be aliased... */ + BDK_CSR_INIT(lmcx_control, *node, BDK_LMCX_CONTROL(*lmc)); + if (lmcx_control.s.xor_bank) + *bank = EXTRACT(address, bank_lsb, bank_width) ^ EXTRACT(address, 12 + xbits, bank_width); + else + *bank = EXTRACT(address, bank_lsb, bank_width); + + /* LMC number already extracted */ + int col_hi = EXTRACT(address, col_hi_lsb, col_hi_width); + *col = EXTRACT(address, 3, 4) | (col_hi << 4); + /* Bus byte is address bits [2:0]. Unused here */ +} + +/** + * Construct a physical address given the node, LMC, DIMM, prank, lrank, bank, row, and column. + * + * @param node Node the address was for + * @param lmc LMC controller the address was for + * @param dimm DIMM the address was for + * @param prank Physical RANK on the DIMM + * @param lrank Logical RANK on the DIMM + * @param bank BANK on the DIMM + * @param row Row on the DIMM + * @param col Column on the DIMM + */ +uint64_t +bdk_dram_address_construct_info(bdk_node_t node, int lmc, int dimm, + int prank, int lrank, int bank, int row, int col) + +{ + uint64_t address = 0; + int bitno = CAVIUM_IS_MODEL(CAVIUM_CN83XX) ? 19 : 20; + + // insert node bits + INSERT(address, node, 40, 2); /* Address bits [41:40] */ + + /* xbits depends on number of LMCs */ + int xbits = __bdk_dram_get_num_lmc(node) >> 1; // 4->2; 2->1; 1->0 + int bank_lsb = 7 + xbits; + + /* Figure out the bank field width */ + int bank_width = __bdk_dram_get_num_bank_bits(node, lmc); + + /* Extract additional info from the LMC_CONFIG CSR */ + BDK_CSR_INIT(lmcx_config, node, BDK_LMCX_CONFIG(lmc)); + BDK_CSR_INIT(ext_config, node, BDK_LMCX_EXT_CONFIG(lmc)); + int dimm_lsb = 28 + lmcx_config.s.pbank_lsb + xbits; + int dimm_width = 40 - dimm_lsb; + int prank_lsb = dimm_lsb - lmcx_config.s.rank_ena; + int prank_width = dimm_lsb - prank_lsb; + int lrank_lsb = prank_lsb - ext_config.s.dimm0_cid; + int lrank_width = prank_lsb - lrank_lsb; + int row_lsb = 14 + lmcx_config.s.row_lsb + xbits; + int row_width = lrank_lsb - row_lsb; + int col_hi_lsb = bank_lsb + bank_width; + int col_hi_width = row_lsb - col_hi_lsb; + + /* Insert some other parts of the address */ + INSERT(address, dimm, dimm_lsb, dimm_width); + INSERT(address, prank, prank_lsb, prank_width); + INSERT(address, lrank, lrank_lsb, lrank_width); + INSERT(address, row, row_lsb, row_width); + INSERT(address, col >> 4, col_hi_lsb, col_hi_width); + INSERT(address, col, 3, 4); + + /* bank calculation may be aliased... */ + BDK_CSR_INIT(lmcx_control, node, BDK_LMCX_CONTROL(lmc)); + int new_bank = bank; + if (lmcx_control.s.xor_bank) + new_bank ^= EXTRACT(address, 12 + xbits, bank_width); + INSERT(address, new_bank, bank_lsb, bank_width); + + /* Determine the actual C bits from the input LMC controller arg */ + /* The input LMC number was probably aliased with other fields */ + BDK_CSR_INIT(l2c_ctl, node, BDK_L2C_CTL); + int new_lmc = lmc; + if (!l2c_ctl.s.disidxalias) + new_lmc ^= EXTRACT(address, bitno, xbits) ^ EXTRACT(address, 12, xbits); + INSERT(address, new_lmc, 7, xbits); + + return address; +} diff --git a/src/vendorcode/cavium/bdk/libbdk-dram/bdk-dram-config.c b/src/vendorcode/cavium/bdk/libbdk-dram/bdk-dram-config.c new file mode 100644 index 0000000000..3465c5d98b --- /dev/null +++ b/src/vendorcode/cavium/bdk/libbdk-dram/bdk-dram-config.c @@ -0,0 +1,163 @@ +/***********************license start*********************************** +* Copyright (c) 2003-2017 Cavium Inc. (support@cavium.com). All rights +* reserved. +* +* +* Redistribution and use in source and binary forms, with or without +* modification, are permitted provided that the following conditions are +* met: +* +* * Redistributions of source code must retain the above copyright +* notice, this list of conditions and the following disclaimer. +* +* * Redistributions in binary form must reproduce the above +* copyright notice, this list of conditions and the following +* disclaimer in the documentation and/or other materials provided +* with the distribution. +* +* * Neither the name of Cavium Inc. nor the names of +* its contributors may be used to endorse or promote products +* derived from this software without specific prior written +* permission. +* +* This Software, including technical data, may be subject to U.S. export +* control laws, including the U.S. Export Administration Act and its +* associated regulations, and may be subject to export or import +* regulations in other countries. +* +* TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" +* AND WITH ALL FAULTS AND CAVIUM INC. MAKES NO PROMISES, REPRESENTATIONS OR +* WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH RESPECT +* TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY +* REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT +* DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) WARRANTIES +* OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A PARTICULAR +* PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET ENJOYMENT, +* QUIET POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE ENTIRE RISK +* ARISING OUT OF USE OR PERFORMANCE OF THE SOFTWARE LIES WITH YOU. +***********************license end**************************************/ +#include <bdk.h> +#include <unistd.h> + +BDK_REQUIRE_DEFINE(DRAM_CONFIG); + +/** + * Lookup a DRAM configuration by name and initialize DRAM using it + * + * @param node Node to configure + * @param ddr_clock_override + * If non zero, override the DRAM frequency specified + * in the config with this value + * + * @return Amount of DRAM in MB, or negative on failure + */ +int bdk_dram_config(int node, int ddr_clock_override) +{ + const dram_config_t *config = libdram_config_load(node); + if (!config) + { + printf("N%d: No DRAM config specified, skipping DRAM init\n", node); + return 0; + } + + BDK_TRACE(DRAM, "N%d: Starting DRAM init (config=%p, ddr_clock_override=%d)\n", node, config, ddr_clock_override); + int mbytes = libdram_config(node, config, ddr_clock_override); + BDK_TRACE(DRAM, "N%d: DRAM init returned %d\n", node, mbytes); + if (mbytes <= 0) + { + printf("ERROR: DDR initialization failed\n"); + return -1; + } + + return mbytes; +} + +/** + * Do DRAM configuration tuning + * + * @param node Node to tune + * + * @return Success or Fail + */ +int bdk_dram_tune(int node) +{ + int ret; + BDK_TRACE(DRAM, "N%d: Starting DRAM tuning\n", node); + ret = libdram_tune(node); + BDK_TRACE(DRAM, "N%d: DRAM tuning returned %d\n", node, ret); + return ret; +} + +/** + * Do all the DRAM Margin tests + * + * @param node Node to test + * + * @return Success or Fail + */ +void bdk_dram_margin(int node) +{ + BDK_TRACE(DRAM, "N%d: Starting DRAM margining\n", node); + libdram_margin(node); + BDK_TRACE(DRAM, "N%d: Finished DRAM margining.\n", node); + return; +} + +/** + * Return the string of the DRAM configuration info at the specified node. + * If the node is not configured, NULL is returned. + * + * @param node node to retrieve + * + * @return string or NULL + */ +const char* bdk_dram_get_info_string(int node) +{ + #define INFO_STRING_LEN 40 + static char info_string[INFO_STRING_LEN]; + static const char *info_ptr = info_string; + + snprintf(info_string, INFO_STRING_LEN, + " %ld MB, %ld MT/s, %s %s", + bdk_dram_get_size_mbytes(node), + bdk_config_get_int(BDK_CONFIG_DDR_SPEED, node), + (__bdk_dram_is_ddr4(node, 0)) ? "DDR4" : "DDR3", + (__bdk_dram_is_rdimm(node, 0)) ? "RDIMM" : "UDIMM"); + + return info_ptr; +} + + +/** + * Return the highest address currently used by the BDK. This address will + * be about 4MB above the top of the BDK to make sure small growths between the + * call and its use don't cause corruption. Any call to memory allocation can + * change this value. + * + * @return Size of the BDK in bytes + */ +uint64_t bdk_dram_get_top_of_bdk(void) +{ + /* Make sure the start address is higher that the BDK's active range. + * + * As sbrk() returns a node address, mask off the node portion of + * the address to make it a physical offset. Doing this simplifies the + * address checks and calculations which only work with physical offsets. + */ + uint64_t top_of_bdk = (bdk_ptr_to_phys(sbrk(0)) & bdk_build_mask(40)); + uint64_t l2_size = bdk_l2c_get_cache_size_bytes(bdk_numa_master()); + if (top_of_bdk <= l2_size) + { + /* Early BDK code takes care of the first L2 sized area of memory */ + top_of_bdk = l2_size; + } + else + { + /* Give 4MB of extra so the BDK has room to grow */ + top_of_bdk += 4 << 20; + /* Align it on a 64KB boundary */ + top_of_bdk >>= 16; + top_of_bdk <<= 16; + } + return top_of_bdk; +} diff --git a/src/vendorcode/cavium/bdk/libbdk-dram/bdk-dram-size.c b/src/vendorcode/cavium/bdk/libbdk-dram/bdk-dram-size.c new file mode 100644 index 0000000000..122afb2a18 --- /dev/null +++ b/src/vendorcode/cavium/bdk/libbdk-dram/bdk-dram-size.c @@ -0,0 +1,213 @@ +/***********************license start*********************************** +* Copyright (c) 2003-2017 Cavium Inc. (support@cavium.com). All rights +* reserved. +* +* +* Redistribution and use in source and binary forms, with or without +* modification, are permitted provided that the following conditions are +* met: +* +* * Redistributions of source code must retain the above copyright +* notice, this list of conditions and the following disclaimer. +* +* * Redistributions in binary form must reproduce the above +* copyright notice, this list of conditions and the following +* disclaimer in the documentation and/or other materials provided +* with the distribution. +* +* * Neither the name of Cavium Inc. nor the names of +* its contributors may be used to endorse or promote products +* derived from this software without specific prior written +* permission. +* +* This Software, including technical data, may be subject to U.S. export +* control laws, including the U.S. Export Administration Act and its +* associated regulations, and may be subject to export or import +* regulations in other countries. +* +* TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" +* AND WITH ALL FAULTS AND CAVIUM INC. MAKES NO PROMISES, REPRESENTATIONS OR +* WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH RESPECT +* TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY +* REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT +* DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) WARRANTIES +* OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A PARTICULAR +* PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET ENJOYMENT, +* QUIET POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE ENTIRE RISK +* ARISING OUT OF USE OR PERFORMANCE OF THE SOFTWARE LIES WITH YOU. +***********************license end**************************************/ +#include <bdk.h> + +/** + * Return the number of LMC controllers in use + * + * @param node Node to probe + * + * @return 2 or 4 depending on the mode + */ +int __bdk_dram_get_num_lmc(bdk_node_t node) +{ + if (CAVIUM_IS_MODEL(CAVIUM_CN88XX)) + { + BDK_CSR_INIT(lmcx_dll_ctl2, node, BDK_LMCX_DLL_CTL2(2)); // sample LMC2 + return (lmcx_dll_ctl2.s.intf_en) ? 4 : 2; + } + else if (CAVIUM_IS_MODEL(CAVIUM_CN83XX)) + { + BDK_CSR_INIT(lmcx_dll_ctl1, node, BDK_LMCX_DLL_CTL2(1)); // sample LMC1 + return (lmcx_dll_ctl1.s.intf_en) ? 2 : 1; + } + else if (CAVIUM_IS_MODEL(CAVIUM_CN81XX)) + { + return 1; + } + else if (CAVIUM_IS_MODEL(CAVIUM_CN93XX)) + { + BDK_CSR_INIT(lmcx_dll_ctl1, node, BDK_LMCX_DLL_CTL2(2)); + if (lmcx_dll_ctl1.s.intf_en) + return 3; + lmcx_dll_ctl1.u = BDK_CSR_READ(node, BDK_LMCX_DLL_CTL2(1)); + return (lmcx_dll_ctl1.s.intf_en) ? 2 : 1; + } + bdk_error("__bdk_dram_get_num_lmc() needs update for this chip\n"); + return 1; +} + +/** + * Return whether the node/LMC is in DRESET + * + * @param node Node to probe + * @param node LMC to probe + * + * @return 1 or 0 + */ +static int __bdk_dram_is_lmc_in_dreset(bdk_node_t node, int lmc) +{ + BDK_CSR_INIT(lmcx_dll_ctl2, node, BDK_LMCX_DLL_CTL2(lmc)); // can always read this + return (lmcx_dll_ctl2.s.dreset != 0) ? 1 : 0; +} + +/** + * Return a mask of the number of row bits in use + * + * @param node Node to probe + * + */ +uint32_t __bdk_dram_get_row_mask(bdk_node_t node, int lmc) +{ + // PROTECT!!! + if (__bdk_dram_is_lmc_in_dreset(node, lmc)) // check LMCn + return 0; + BDK_CSR_INIT(lmcx_config, node, BDK_LMCX_CONFIG(lmc)); // sample LMCn + int numbits = 14 + lmcx_config.s.pbank_lsb - lmcx_config.s.row_lsb - lmcx_config.s.rank_ena; + return ((1ul << numbits) - 1); +} + +/** + * Return a mask of the number of column bits in use + * + * @param node Node to probe + * + */ +uint32_t __bdk_dram_get_col_mask(bdk_node_t node, int lmc) +{ + // PROTECT!!! + if (__bdk_dram_is_lmc_in_dreset(node, lmc)) // check LMCn + return 0; + BDK_CSR_INIT(lmcx_config, node, BDK_LMCX_CONFIG(lmc)); // sample LMCn + int numbits = 11 + lmcx_config.s.row_lsb - __bdk_dram_get_num_bank_bits(node, lmc); + return ((1ul << numbits) - 1); +} + +/** + * Return the number of bank bits in use + * + * @param node Node to probe + * + */ +// all DDR3, and DDR4 x16 today, use only 3 bank bits; DDR4 x4 and x8 always have 4 bank bits +// NOTE: this will change in the future, when DDR4 x16 devices can come with 16 banks!! FIXME!! +int __bdk_dram_get_num_bank_bits(bdk_node_t node, int lmc) +{ + // PROTECT!!! + if (__bdk_dram_is_lmc_in_dreset(node, lmc)) // check LMCn + return 0; + BDK_CSR_INIT(lmcx_config, node, BDK_LMCX_CONFIG(lmc)); // sample LMCn + int bank_width = (__bdk_dram_is_ddr4(node, lmc) && (lmcx_config.s.bg2_enable)) ? 4 : 3; + return bank_width; +} + +/** + * Return whether the node has DDR3 or DDR4 DRAM + * + * @param node Node to probe + * + * @return 0 (DDR3) or 1 (DDR4) + */ +int __bdk_dram_is_ddr4(bdk_node_t node, int lmc) +{ + // PROTECT!!! + if (__bdk_dram_is_lmc_in_dreset(node, lmc)) // check LMCn + return 0; + if (CAVIUM_IS_MODEL(CAVIUM_CN9XXX)) + return 1; + BDK_CSR_INIT(lmcx_ddr_pll_ctl, node, BDK_LMCX_DDR_PLL_CTL(lmc)); // sample LMCn + return (lmcx_ddr_pll_ctl.cn83xx.ddr4_mode != 0); +} + +/** + * Return whether the node has Registered DIMMs or Unbuffered DIMMs + * + * @param node Node to probe + * + * @return 0 (Unbuffered) or 1 (Registered) + */ +int __bdk_dram_is_rdimm(bdk_node_t node, int lmc) +{ + // PROTECT!!! + if (__bdk_dram_is_lmc_in_dreset(node, lmc)) // check LMCn + return 0; + BDK_CSR_INIT(lmcx_control, node, BDK_LMCX_CONTROL(lmc)); // sample LMCn + return (lmcx_control.s.rdimm_ena != 0); +} + +/** + * Get the amount of DRAM configured for a node. This is read from the LMC + * controller after DRAM is setup. + * + * @param node Node to query + * + * @return Size in megabytes + */ +uint64_t bdk_dram_get_size_mbytes(int node) +{ + if (bdk_is_platform(BDK_PLATFORM_EMULATOR)) + return 2 << 10; /* 2GB is available on t88 and t81 + ** some t83 models have 8gb, but it is too long to init */ + /* Return zero if dram isn't enabled */ + if (!__bdk_is_dram_enabled(node)) + return 0; + + uint64_t memsize = 0; + const int num_dram_controllers = __bdk_dram_get_num_lmc(node); + for (int lmc = 0; lmc < num_dram_controllers; lmc++) + { + if (bdk_is_platform(BDK_PLATFORM_ASIM)) + { + /* Asim doesn't simulate the rank detection, fake 4GB per controller */ + memsize += 4ull << 30; + } + else + { + // PROTECT!!! + if (__bdk_dram_is_lmc_in_dreset(node, lmc)) // check LMCn + return 0; + BDK_CSR_INIT(lmcx_config, node, BDK_LMCX_CONFIG(lmc)); + int num_ranks = bdk_pop(lmcx_config.s.init_status); + uint64_t rank_size = 1ull << (28 + lmcx_config.s.pbank_lsb - lmcx_config.s.rank_ena); + memsize += rank_size * num_ranks; + } + } + return memsize >> 20; +} + diff --git a/src/vendorcode/cavium/bdk/libbdk-dram/bdk-dram-test-addrbus.c b/src/vendorcode/cavium/bdk/libbdk-dram/bdk-dram-test-addrbus.c new file mode 100644 index 0000000000..9fe8570454 --- /dev/null +++ b/src/vendorcode/cavium/bdk/libbdk-dram/bdk-dram-test-addrbus.c @@ -0,0 +1,115 @@ +/***********************license start*********************************** +* Copyright (c) 2003-2017 Cavium Inc. (support@cavium.com). All rights +* reserved. +* +* +* Redistribution and use in source and binary forms, with or without +* modification, are permitted provided that the following conditions are +* met: +* +* * Redistributions of source code must retain the above copyright +* notice, this list of conditions and the following disclaimer. +* +* * Redistributions in binary form must reproduce the above +* copyright notice, this list of conditions and the following +* disclaimer in the documentation and/or other materials provided +* with the distribution. +* +* * Neither the name of Cavium Inc. nor the names of +* its contributors may be used to endorse or promote products +* derived from this software without specific prior written +* permission. +* +* This Software, including technical data, may be subject to U.S. export +* control laws, including the U.S. Export Administration Act and its +* associated regulations, and may be subject to export or import +* regulations in other countries. +* +* TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" +* AND WITH ALL FAULTS AND CAVIUM INC. MAKES NO PROMISES, REPRESENTATIONS OR +* WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH RESPECT +* TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY +* REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT +* DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) WARRANTIES +* OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A PARTICULAR +* PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET ENJOYMENT, +* QUIET POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE ENTIRE RISK +* ARISING OUT OF USE OR PERFORMANCE OF THE SOFTWARE LIES WITH YOU. +***********************license end**************************************/ +#include "bdk.h" + +/* Used for all memory reads/writes related to the test */ +#define READ64(address) __bdk_dram_read64(address) +#define WRITE64(address, data) __bdk_dram_write64(address, data) + +/** + * Address bus test. This test writes a single value to each power of two in the + * area, looking for false aliases that would be created by address lines being + * shorted or tied together. + * + * @param area + * @param max_address + * @param bursts + * + * @return + */ +int __bdk_dram_test_mem_address_bus(uint64_t area, uint64_t max_address, int bursts) +{ + int failures = 0; + + /* Clear our work area. Checking for aliases later could get false + positives if it matched stale data */ + void *ptr = (area) ? bdk_phys_to_ptr(area) : NULL; + bdk_zero_memory(ptr, max_address - area); + __bdk_dram_flush_to_mem_range(area, max_address); + + /* Each time we write, we'll write this pattern xored the address it is + written too */ + uint64_t pattern = 0x0fedcba987654321; + + /* Walk through the region incrementing our offset by a power of two. The + first few writes will be to the same cache line (offset 0x8, 0x10, 0x20, + and 0x40. Offset 0x80 and beyond will be to different cache lines */ + uint64_t offset = 0x8; + while (area + offset < max_address) + { + uint64_t address = area + offset; + /* Write one location with pattern xor address */ + uint64_t p = pattern ^ address; + WRITE64(address, p); + __bdk_dram_flush_to_mem(address); + offset <<= 1; + } + + /* Read all of the area to make sure no other locations were written */ + uint64_t a = area; + offset = 0x8; + uint64_t next_write = area + offset; + while (a < max_address) + { + if (a + 256 < max_address) + BDK_PREFETCH(a + 256, 0); + for (int i=0; i<16; i++) + { + uint64_t data = READ64(a); + uint64_t correct; + if (a == next_write) + { + correct = pattern ^ next_write; + offset <<= 1; + next_write = area + offset; + } + else + correct = 0; + if (bdk_unlikely(data != correct)) + { + failures++; + __bdk_dram_report_error(a, data, correct, 0, -1); + } + a += 8; + } + } + + return failures; +} + diff --git a/src/vendorcode/cavium/bdk/libbdk-dram/bdk-dram-test-databus.c b/src/vendorcode/cavium/bdk/libbdk-dram/bdk-dram-test-databus.c new file mode 100644 index 0000000000..c3fa1ffd8d --- /dev/null +++ b/src/vendorcode/cavium/bdk/libbdk-dram/bdk-dram-test-databus.c @@ -0,0 +1,252 @@ +/***********************license start*********************************** +* Copyright (c) 2003-2017 Cavium Inc. (support@cavium.com). All rights +* reserved. +* +* +* Redistribution and use in source and binary forms, with or without +* modification, are permitted provided that the following conditions are +* met: +* +* * Redistributions of source code must retain the above copyright +* notice, this list of conditions and the following disclaimer. +* +* * Redistributions in binary form must reproduce the above +* copyright notice, this list of conditions and the following +* disclaimer in the documentation and/or other materials provided +* with the distribution. +* +* * Neither the name of Cavium Inc. nor the names of +* its contributors may be used to endorse or promote products +* derived from this software without specific prior written +* permission. +* +* This Software, including technical data, may be subject to U.S. export +* control laws, including the U.S. Export Administration Act and its +* associated regulations, and may be subject to export or import +* regulations in other countries. +* +* TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" +* AND WITH ALL FAULTS AND CAVIUM INC. MAKES NO PROMISES, REPRESENTATIONS OR +* WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH RESPECT +* TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY +* REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT +* DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) WARRANTIES +* OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A PARTICULAR +* PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET ENJOYMENT, +* QUIET POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE ENTIRE RISK +* ARISING OUT OF USE OR PERFORMANCE OF THE SOFTWARE LIES WITH YOU. +***********************license end**************************************/ +#include "bdk.h" + +/* Used for all memory reads/writes related to the test */ +#define READ64(address) __bdk_dram_read64(address) +#define WRITE64(address, data) __bdk_dram_write64(address, data) + +/* Build a 64bit mask out of a single hex digit */ +#define REPEAT2(v) ((((uint64_t)v) << 4) | ((uint64_t)v)) +#define REPEAT4(v) ((REPEAT2(v) << 8) | REPEAT2(v)) +#define REPEAT8(v) ((REPEAT4(v) << 16) | REPEAT4(v)) +#define REPEAT16(v) ((REPEAT8(v) << 32) | REPEAT8(v)) + +/** + * Read memory and check that the data bus pattern is present. The pattern is a + * sequence if 16 dwords created from the 16 hex digits repeated in each word. + * + * @param address Physical address to read. This must be cache line aligned. + * @param bursts Number of time to repeat the read test to verify stability + * + * @return Number of errors, zero means success + */ +static int read_data_bus_burst(uint64_t address, int bursts) +{ + int failures = 0; + + /* Loop over the burst so people using a scope have time to capture + traces */ + for (int burst = 0; burst < bursts; burst++) + { + /* Invalidate all caches so we must read from DRAM */ + __bdk_dram_flush_to_mem(address); + BDK_DCACHE_INVALIDATE; + + for (uint64_t digit = 0; digit < 16; digit++) + { + uint64_t a = address + digit * 8; + uint64_t data = READ64(a); + uint64_t correct = REPEAT16(digit); + if (data != correct) + { + failures++; + __bdk_dram_report_error(a, data, correct, burst, -1); + } + } + } + return failures; +} + +/** + * Write memory with a data bus pattern and check that it can be read correctly. + * The pattern is a sequence if 16 dwords created from the 16 hex digits repeated + * in each word. + * + * @param address Physical address to write. This must be cache line aligned. 128 bytes will be + * written starting at this address. + * @param bursts Number of time to repeat the write+read test to verify stability + * + * @return Number of errors, zero means success + */ +static int write_data_bus_burst(uint64_t address, int bursts) +{ + BDK_TRACE(DRAM_TEST, "[0x%016lx:0x%016lx] Writing incrementing digits\n", + address, address + 127); + /* Loop over the burst so people using a scope have time to capture + traces */ + for (int burst = 0; burst < bursts; burst++) + { + /* Fill a cache line with an incrementing pattern. Each nibble + in the 64bit word increments from 0 to 0xf */ + for (uint64_t digit = 0; digit < 16; digit++) + WRITE64(address + digit * 8, REPEAT16(digit)); + /* Force the cache line out to memory */ + __bdk_dram_flush_to_mem(address); + } + return read_data_bus_burst(address, bursts); +} + +/** + * Read back the pattern written by write_data_bus_walk() nad + * make sure it was stored properly. + * + * @param address Physical address to read. This must be cache line aligned. + * @param bursts Number of time to repeat the read test to verify stability + * @param pattern Pattern basis for writes. See + * write_data_bus_walk() + * + * @return Number of errors, zero means success + */ +static int read_data_bus_walk(uint64_t address, int burst, uint64_t pattern) +{ + int failures = 0; + + /* Invalidate all caches so we must readfrom DRAM */ + __bdk_dram_flush_to_mem(address); + BDK_DCACHE_INVALIDATE; + + uint64_t correct = pattern; + for (uint64_t word = 0; word < 16; word++) + { + uint64_t a = address + word * 8; + uint64_t data = READ64(a); + if (data != correct) + { + failures++; + __bdk_dram_report_error(a, data, correct, burst, -1); + } + uint64_t tmp = correct >> 63; /* Save top bit */ + correct <<= 1; /* Shift left one bit */ + correct |= tmp; /* Restore the top bit as bit 0 */ + } + + return failures; +} + +/** + * Write a pattern to a cache line, rotating it one bit for each DWORD. Read back + * the pattern and make sure it was stored properly. The input pattern is rotated + * left by one bit for each DWORD written. + * + * @param address Physical address to write. This must be cache line aligned. 128 bytes will be + * written starting at this address. + * @param bursts Number of time to repeat the write+read test to verify stability + * @param pattern Pattern basis + * + * @return Number of errors, zero means success + */ +static void write_data_bus_walk(uint64_t address, int burst, uint64_t pattern) +{ + BDK_TRACE(DRAM_TEST, "[0x%016lx:0x%016lx] Writing walking pattern 0x%016lx\n", + address, address + 127, pattern); + + uint64_t a = address; + uint64_t d = pattern; + + /* Fill a cache line with pattern. Each 64bit work will have the + pattern rotated left one bit */ + for (uint64_t word = 0; word < 16; word++) + { + WRITE64(a, d); + a += 8; + uint64_t tmp = d >> 63; /* Save top bit */ + d <<= 1; /* Shift left one bit */ + d |= tmp; /* Restore the top bit as bit 0 */ + } + /* Force the cache line out to memory */ + __bdk_dram_flush_to_mem(address); +} + +/** + * The goal of these tests are to toggle every DDR data pin, one at a time or in + * related groups, to isolate any short circuits between the data pins or open + * circuits where the pin is not connected to the DDR memory. A board which fails + * one of these tests has severe problems and will not be able to run any of the + * later test patterns. + * + * @param start_address + * Physical address of a cache line to + * use for the test. Only this cache line is + * written. + * @param end_address + * Top end of the address range. Currently unused + * @param bursts Number of time to repeats writes+reads to insure stability + * + * @return Number of errors, zero means success + */ +int __bdk_dram_test_mem_data_bus(uint64_t start_address, uint64_t end_address, int bursts) +{ + int failures = 0; + + /* Incrementing pattern: 0x0 - 0xf in each nibble */ + failures += write_data_bus_burst(start_address, bursts); + + /* Walking ones. Run with 1, 2, and 3 bits walking */ + for (int bits = 1; bits <= 3; bits++) + { + for (int burst = 0; burst < bursts; burst++) + { + /* Each write_data_bus_walk() call write 16 dword, so step by 16 */ + for (int i = 0; i < 64; i += 16) + { + uint64_t pattern = bdk_build_mask(bits) << i; + write_data_bus_walk(start_address + i*8, burst, pattern); + } + /* Each read_data_bus_walk() call write 16 dword, so step by 16 */ + for (int i = 0; i < 64; i += 16) + { + uint64_t pattern = bdk_build_mask(bits) << i; + failures += read_data_bus_walk(start_address + i*8, burst, pattern); + } + } + } + + /* Walking zeros. Run with 1, 2, and 3 bits walking */ + for (int bits = 1; bits <= 3; bits++) + { + for (int burst = 0; burst < bursts; burst++) + { + /* Each write_data_bus_walk() call write 16 dword, so step by 16 */ + for (int i = 0; i < 64; i += 16) + { + uint64_t pattern = ~(bdk_build_mask(bits) << i); + write_data_bus_walk(start_address + i*8, burst, pattern); + } + /* Each read_data_bus_walk() call write 16 dword, so step by 16 */ + for (int i = 0; i < 64; i += 16) + { + uint64_t pattern = ~(bdk_build_mask(bits) << i); + failures += read_data_bus_walk(start_address + i*8, burst, pattern); + } + } + } + return failures; +} + diff --git a/src/vendorcode/cavium/bdk/libbdk-dram/bdk-dram-test-fastscan.c b/src/vendorcode/cavium/bdk/libbdk-dram/bdk-dram-test-fastscan.c new file mode 100644 index 0000000000..46e205dd80 --- /dev/null +++ b/src/vendorcode/cavium/bdk/libbdk-dram/bdk-dram-test-fastscan.c @@ -0,0 +1,103 @@ +/***********************license start*********************************** +* Copyright (c) 2003-2017 Cavium Inc. (support@cavium.com). All rights +* reserved. +* +* +* Redistribution and use in source and binary forms, with or without +* modification, are permitted provided that the following conditions are +* met: +* +* * Redistributions of source code must retain the above copyright +* notice, this list of conditions and the following disclaimer. +* +* * Redistributions in binary form must reproduce the above +* copyright notice, this list of conditions and the following +* disclaimer in the documentation and/or other materials provided +* with the distribution. +* +* * Neither the name of Cavium Inc. nor the names of +* its contributors may be used to endorse or promote products +* derived from this software without specific prior written +* permission. +* +* This Software, including technical data, may be subject to U.S. export +* control laws, including the U.S. Export Administration Act and its +* associated regulations, and may be subject to export or import +* regulations in other countries. +* +* TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" +* AND WITH ALL FAULTS AND CAVIUM INC. MAKES NO PROMISES, REPRESENTATIONS OR +* WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH RESPECT +* TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY +* REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT +* DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) WARRANTIES +* OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A PARTICULAR +* PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET ENJOYMENT, +* QUIET POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE ENTIRE RISK +* ARISING OUT OF USE OR PERFORMANCE OF THE SOFTWARE LIES WITH YOU. +***********************license end**************************************/ +#include "bdk.h" + +/* Used for all memory reads/writes related to the test */ +#define READ64(address) __bdk_dram_read64(address) +#define WRITE64(address, data) __bdk_dram_write64(address, data) + +/** + * Fast scan test. This test is meant to find gross errors caused by read/write + * level failing on a single rank or dimm. The idea is to scan through all of + * memory in large steps. The large steps hit each rank multiple times, but not + * every byte. If the whole rank has errors, his should find it quickly. This test + * is suitable for an alive test during early boot. + * + * @param area Starting physical address + * @param max_address + * Ending physical address, exclusive + * @param bursts Burst to run + * + * @return Number of errors + */ +int __bdk_dram_test_fast_scan(uint64_t area, uint64_t max_address, int bursts) +{ + int failures = 0; + const uint64_t step = 0x10008; /* The 8 is so we walk through cache lines too */ + const uint64_t pattern1 = 0xaaaaaaaaaaaaaaaa; + const uint64_t pattern2 = 0x5555555555555555; + + /* Walk through the region incrementing our offset by STEP */ + uint64_t a = area; + while (a + 16 <= max_address) + { + WRITE64(a, pattern1); + WRITE64(a+8, pattern2); + __bdk_dram_flush_to_mem_range(a, a + 16); + a += step; + } + + /* Read back, checking the writes */ + a = area; + while (a + 16 <= max_address) + { + /* Prefetch 3 ahead for better performance */ + uint64_t pre = a + step * 2; + if (pre + 16 < max_address) + BDK_PREFETCH(pre, 0); + /* Check pattern 1 */ + uint64_t data1 = READ64(a); + if (bdk_unlikely(data1 != pattern1)) + { + failures++; + __bdk_dram_report_error(a, data1, pattern1, 0, -1); + } + /* Check pattern 2 */ + uint64_t data2 = READ64(a+8); + if (bdk_unlikely(data2 != pattern2)) + { + failures++; + __bdk_dram_report_error(a+8, data2, pattern2, 0, -1); + } + a += step; + } + + return failures; +} + diff --git a/src/vendorcode/cavium/bdk/libbdk-dram/bdk-dram-test-patfil.c b/src/vendorcode/cavium/bdk/libbdk-dram/bdk-dram-test-patfil.c new file mode 100644 index 0000000000..e6c4b57721 --- /dev/null +++ b/src/vendorcode/cavium/bdk/libbdk-dram/bdk-dram-test-patfil.c @@ -0,0 +1,829 @@ +/***********************license start*********************************** +* Copyright (c) 2003-2017 Cavium Inc. (support@cavium.com). All rights +* reserved. +* +* +* Redistribution and use in source and binary forms, with or without +* modification, are permitted provided that the following conditions are +* met: +* +* * Redistributions of source code must retain the above copyright +* notice, this list of conditions and the following disclaimer. +* +* * Redistributions in binary form must reproduce the above +* copyright notice, this list of conditions and the following +* disclaimer in the documentation and/or other materials provided +* with the distribution. +* +* * Neither the name of Cavium Inc. nor the names of +* its contributors may be used to endorse or promote products +* derived from this software without specific prior written +* permission. +* +* This Software, including technical data, may be subject to U.S. export +* control laws, including the U.S. Export Administration Act and its +* associated regulations, and may be subject to export or import +* regulations in other countries. +* +* TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" +* AND WITH ALL FAULTS AND CAVIUM INC. MAKES NO PROMISES, REPRESENTATIONS OR +* WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH RESPECT +* TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY +* REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT +* DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) WARRANTIES +* OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A PARTICULAR +* PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET ENJOYMENT, +* QUIET POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE ENTIRE RISK +* ARISING OUT OF USE OR PERFORMANCE OF THE SOFTWARE LIES WITH YOU. +***********************license end**************************************/ +#include "bdk.h" + +// choose prediction-based algorithms for mem_xor and mem_rows tests +#define USE_PREDICTION_CODE_VERSIONS 1 // change to 0 to go back to the original versions + +/* Used for all memory reads/writes related to the test */ +#define READ64(address) __bdk_dram_read64(address) +#define WRITE64(address, data) __bdk_dram_write64(address, data) + +/** + * Fill an memory area with the address of each 64-bit word in the area. + * Reread to confirm the pattern. + * + * @param area Start of the physical memory area + * @param max_address + * End of the physical memory area (exclusive) + * @param bursts Number of time to repeat the test over the entire area + * + * @return Number of errors, zero on success + */ +int __bdk_dram_test_mem_self_addr(uint64_t area, uint64_t max_address, int bursts) +{ + int failures = 0; + + for (int burst = 0; burst < bursts; burst++) + { + /* Write the pattern to memory. Each location receives the address + * of the location. + */ + for (uint64_t address = area; address < max_address; address+=8) + WRITE64(address, address); + __bdk_dram_flush_to_mem_range(area, max_address); + BDK_DCACHE_INVALIDATE; + + /* Read by ascending address the written memory and confirm that it + * has the expected data pattern. + */ + for (uint64_t address = area; address < max_address; ) + { + if (address + 256 < max_address) + BDK_PREFETCH(address + 256, 0); + for (int i=0; i<16; i++) + { + uint64_t data = READ64(address); + if (bdk_unlikely(data != address)) + failures += __bdk_dram_retry_failure(burst, address, data, address); + address += 8; + } + } + __bdk_dram_flush_to_mem_range(area, max_address); + BDK_DCACHE_INVALIDATE; + + /* Read by descending address the written memory and confirm that it + * has the expected data pattern. + */ + uint64_t end = max_address - sizeof(uint64_t); + for (uint64_t address = end; address >= area; ) + { + if (address - 256 >= area) + BDK_PREFETCH(address - 256, 0); + for (int i=0; i<16; i++) + { + uint64_t data = READ64(address); + if (bdk_unlikely(data != address)) + failures += __bdk_dram_retry_failure(burst, address, data, address); + address -= 8; + } + } + __bdk_dram_flush_to_mem_range(area, max_address); + BDK_DCACHE_INVALIDATE; + + /* Read from random addresses within the memory area. + */ + uint64_t probes = (max_address - area) / 128; + uint64_t address_ahead1 = area; + uint64_t address_ahead2 = area; + for (uint64_t i = 0; i < probes; i++) + { + /* Create a pipeline of prefetches: + address = address read this loop + address_ahead1 = prefetch started last loop + address_ahead2 = prefetch started this loop */ + uint64_t address = address_ahead1; + address_ahead1 = address_ahead2; + address_ahead2 = bdk_rng_get_random64() % (max_address - area); + address_ahead2 += area; + address_ahead2 &= -8; + BDK_PREFETCH(address_ahead2, 0); + + uint64_t data = READ64(address); + if (bdk_unlikely(data != address)) + failures += __bdk_dram_retry_failure(burst, address, data, address); + } + } + return failures; +} + +/** + * Write "pattern" and its compliment to memory and verify it was written + * properly. Memory will be filled with DWORDs pattern, ~pattern, pattern, + * ~pattern, ... + * + * @param area Start physical address of memory + * @param max_address + * End of physical memory region + * @param pattern Pattern to write + * @param passes Number of time to repeat the test + * + * @return Number of errors, zero on success + */ +static uint32_t test_mem_pattern(uint64_t area, uint64_t max_address, uint64_t pattern, + int passes) +{ + int failures = 0; + + for (int pass = 0; pass < passes; pass++) + { + if (pass & 0x1) + pattern = ~pattern; + + for (uint64_t address = area; address < max_address; address += 8) + WRITE64(address, pattern); + __bdk_dram_flush_to_mem_range(area, max_address); + BDK_DCACHE_INVALIDATE; + + /* Read the written memory and confirm that it has the expected + * data pattern. + */ + uint64_t address = area; + while (address < max_address) + { + if (address + 256 < max_address) + BDK_PREFETCH(address + 256, 0); + for (int i=0; i<16; i++) + { + uint64_t data = READ64(address); + if (bdk_unlikely(data != pattern)) + failures += __bdk_dram_retry_failure(pass, address, data, pattern); + address += 8; + } + } + } + return failures; +} + +/** + * Walking zero written to memory, left shift + * + * @param area Start of the physical memory area + * @param max_address + * End of the physical memory area + * @param bursts Number of time to repeat the test over the entire area + * + * @return Number of errors, zero on success + */ +int __bdk_dram_test_mem_leftwalk0(uint64_t area, uint64_t max_address, int bursts) +{ + int failures = 0; + for (int burst = 0; burst < bursts; burst++) + { + for (uint64_t pattern = 1; pattern != 0; pattern = pattern << 1) + failures += test_mem_pattern(area, max_address, ~pattern, 1); + } + return failures; +} + +/** + * Walking one written to memory, left shift + * + * @param area Start of the physical memory area + * @param max_address + * End of the physical memory area + * @param bursts Number of time to repeat the test over the entire area + * + * @return Number of errors, zero on success + */ +int __bdk_dram_test_mem_leftwalk1(uint64_t area, uint64_t max_address, int bursts) +{ + int failures = 0; + for (int burst = 0; burst < bursts; burst++) + { + for (uint64_t pattern = 1; pattern != 0; pattern = pattern << 1) + failures += test_mem_pattern(area, max_address, pattern, 1); + } + return failures; +} + +/** + * Walking zero written to memory, right shift + * + * @param area Start of the physical memory area + * @param max_address + * End of the physical memory area + * @param bursts Number of time to repeat the test over the entire area + * + * @return Number of errors, zero on success + */ +int __bdk_dram_test_mem_rightwalk0(uint64_t area, uint64_t max_address, int bursts) +{ + int failures = 0; + for (int burst = 0; burst < bursts; burst++) + { + for (uint64_t pattern = 1ull << 63; pattern != 0; pattern = pattern >> 1) + failures += test_mem_pattern(area, max_address, ~pattern, 1); + } + return failures; +} + +/** + * Walking one written to memory, right shift + * + * @param area Start of the physical memory area + * @param max_address + * End of the physical memory area + * @param bursts Number of time to repeat the test over the entire area + * + * @return Number of errors, zero on success + */ +int __bdk_dram_test_mem_rightwalk1(uint64_t area, uint64_t max_address, int bursts) +{ + int failures = 0; + for (int burst = 0; burst < bursts; burst++) + { + for (uint64_t pattern = 1ull<<63; pattern != 0; pattern = pattern >> 1) + failures += test_mem_pattern(area, max_address, pattern, 1); + } + return failures; +} + +/** + * Apply the March C- testing algorithm to the given memory area. + * 1) Write "pattern" to memory. + * 2) Verify "pattern" and write "~pattern". + * 3) Verify "~pattern" and write "pattern". + * 4) Verify "pattern" and write "~pattern". + * 5) Verify "~pattern" and write "pattern". + * 6) Verify "pattern". + * + * @param area Start of the physical memory area + * @param max_address + * End of the physical memory area + * @param pattern + * + * @return Number of errors, zero on success + */ +static int test_mem_march_c(uint64_t area, uint64_t max_address, uint64_t pattern) +{ + int failures = 0; + + /* Pass 1 ascending addresses, fill memory with pattern. */ + BDK_TRACE(DRAM_TEST, " [0x%016lx:0x%016lx] Phase1, address incrementing, pattern 0x%016lx\n", area, max_address-1, pattern); + for (uint64_t address = area; address < max_address; address += 8) + WRITE64(address, pattern); + + __bdk_dram_flush_to_mem_range(area, max_address); + BDK_DCACHE_INVALIDATE; + + /* Pass 2: ascending addresses, read pattern and write ~pattern */ + BDK_TRACE(DRAM_TEST, " [0x%016lx:0x%016lx] Phase2, address incrementing, pattern 0x%016lx\n", area, max_address-1, ~pattern); + for (uint64_t address = area; address < max_address; address += 8) + { + uint64_t data = READ64(address); + if (bdk_unlikely(data != pattern)) + failures += __bdk_dram_retry_failure(1, address, data, pattern); + WRITE64(address, ~pattern); + } + + __bdk_dram_flush_to_mem_range(area, max_address); + BDK_DCACHE_INVALIDATE; + + /* Pass 3: ascending addresses, read ~pattern and write pattern. */ + BDK_TRACE(DRAM_TEST, " [0x%016lx:0x%016lx] Phase3, address incrementing, pattern 0x%016lx\n", area, max_address-1, pattern); + for (uint64_t address = area; address < max_address; address += 8) + { + uint64_t data = READ64(address); + if (bdk_unlikely(data != ~pattern)) + failures += __bdk_dram_retry_failure(1, address, data, ~pattern); + WRITE64(address, pattern); + } + + __bdk_dram_flush_to_mem_range(area, max_address); + BDK_DCACHE_INVALIDATE; + + /* Pass 4: descending addresses, read pattern and write ~pattern. */ + BDK_TRACE(DRAM_TEST, " [0x%016lx:0x%016lx] Phase4, address decrementing, pattern 0x%016lx\n", area, max_address-1, ~pattern); + uint64_t end = max_address - sizeof(uint64_t); + for (uint64_t address = end; address >= area; address -= 8) + { + uint64_t data = READ64(address); + if (bdk_unlikely(data != pattern)) + failures += __bdk_dram_retry_failure(1, address, data, pattern); + WRITE64(address, ~pattern); + } + + __bdk_dram_flush_to_mem_range(area, max_address); + BDK_DCACHE_INVALIDATE; + + /* Pass 5: descending addresses, read ~pattern and write pattern. */ + BDK_TRACE(DRAM_TEST, " [0x%016lx:0x%016lx] Phase5, address decrementing, pattern 0x%016lx\n", area, max_address-1, pattern); + for (uint64_t address = end; address >= area; address -= 8) + { + uint64_t data = READ64(address); + if (bdk_unlikely(data != ~pattern)) + failures += __bdk_dram_retry_failure(1, address, data, ~pattern); + WRITE64(address, pattern); + } + + __bdk_dram_flush_to_mem_range(area, max_address); + BDK_DCACHE_INVALIDATE; + + /* Pass 6: ascending addresses, read pattern. */ + BDK_TRACE(DRAM_TEST, " [0x%016lx:0x%016lx] Phase6, address incrementing\n", area, max_address-1); + for (uint64_t address = area; address < max_address; address += 8) + { + uint64_t data = READ64(address); + if (bdk_unlikely(data != pattern)) + failures += __bdk_dram_retry_failure(1, address, data, pattern); + } + + return failures; +} + +/** + * Use test_mem_march_c() with a all ones pattern + * + * @param area Start of the physical memory area + * @param max_address + * End of the physical memory area + * @param bursts Number of time to repeat the test over the entire area + * + * @return Number of errors, zero on success + */ +int __bdk_dram_test_mem_solid(uint64_t area, uint64_t max_address, int bursts) +{ + int failures = 0; + for (int burst = 0; burst < bursts; burst++) + failures += test_mem_march_c(area, max_address, -1); + return failures; +} + +/** + * Use test_mem_march_c() with a 0x55 pattern + * + * @param area Start of the physical memory area + * @param max_address + * End of the physical memory area + * @param bursts Number of time to repeat the test over the entire area + * + * @return Number of errors, zero on success + */ +int __bdk_dram_test_mem_checkerboard(uint64_t area, uint64_t max_address, int bursts) +{ + int failures = 0; + for (int burst = 0; burst < bursts; burst++) + failures += test_mem_march_c(area, max_address, 0x5555555555555555L); + return failures; +} + +/** + * Write a pseudo random pattern to memory and verify it + * + * @param area Start of the physical memory area + * @param max_address + * End of the physical memory area + * @param bursts Number of time to repeat the test over the entire area + * + * @return Number of errors, zero on success + */ +int __bdk_dram_test_mem_random(uint64_t area, uint64_t max_address, int bursts) +{ + /* This constant is used to increment the pattern after every DWORD. This + makes only the first DWORD truly random, but saves us processing + power generating the random values */ + const uint64_t INC = 0x1010101010101010ULL; + + int failures = 0; + for (int burst = 0; burst < bursts; burst++) + { + const uint64_t init_pattern = bdk_rng_get_random64(); + uint64_t pattern = init_pattern; + + /* Write the pattern to memory. Each location receives the address + * of the location. A second write pass is needed to force all of + * the cached memory out to the DDR. + */ + for (uint64_t address = area; address < max_address; address += 8) + { + WRITE64(address, pattern); + pattern += INC; + } + + __bdk_dram_flush_to_mem_range(area, max_address); + BDK_DCACHE_INVALIDATE; + + /* Read the written memory and confirm that it has the expected + * data pattern. + */ + pattern = init_pattern; + for (uint64_t address = area; address < max_address; address += 8) + { + uint64_t data = READ64(address); + if (bdk_unlikely(data != pattern)) + failures += __bdk_dram_retry_failure(burst, address, data, pattern); + pattern += INC; + } + } + return failures; +} + +#if !USE_PREDICTION_CODE_VERSIONS +/** + * test_mem_xor + * + * @param area Start of the physical memory area + * @param max_address + * End of the physical memory area + * @param bursts Number of time to repeat the test over the entire area + * + * @return Number of errors, zero on success + */ +int __bdk_dram_test_mem_xor(uint64_t area, uint64_t max_address, int bursts) +{ + int failures = 0; + + uint64_t extent = max_address - area; + uint64_t count = (extent / sizeof(uint64_t)) / 2; + + /* Fill both halves of the memory area with identical randomized data. + */ + uint64_t address1 = area; + uint64_t address2 = area + count * sizeof(uint64_t); + + uint64_t pattern = bdk_rng_get_random64(); + + for (uint64_t j = 0; j < count; j++) + { + uint64_t p = pattern * address1; + WRITE64(address1, p); + WRITE64(address2, p); + address1 += 8; + address2 += 8; + } + __bdk_dram_flush_to_mem_range(area, max_address); + BDK_DCACHE_INVALIDATE; + + /* Make a series of passes over the memory areas. */ + for (int burst = 0; burst < bursts; burst++) + { + /* XOR the data with a random value, applying the change to both + * memory areas. + */ + address1 = area; + address2 = area + count * sizeof(uint64_t); + + pattern = bdk_rng_get_random64(); + + for (uint64_t j = 0; j < count; j++) + { + if ((address1 & BDK_CACHE_LINE_MASK) == 0) + BDK_PREFETCH(address1, BDK_CACHE_LINE_SIZE); + if ((address2 & BDK_CACHE_LINE_MASK) == 0) + BDK_PREFETCH(address2, BDK_CACHE_LINE_SIZE); + WRITE64(address1, READ64(address1) ^ pattern); + WRITE64(address2, READ64(address2) ^ pattern); + address1 += 8; + address2 += 8; + } + + __bdk_dram_flush_to_mem_range(area, max_address); + BDK_DCACHE_INVALIDATE; + + /* Look for differences in the areas. If there is a mismatch, reset + * both memory locations with the same pattern. Failing to do so + * means that on all subsequent passes the pair of locations remain + * out of sync giving spurious errors. + */ + address1 = area; + address2 = area + count * sizeof(uint64_t); + for (uint64_t j = 0; j < count; j++) + { + if ((address1 & BDK_CACHE_LINE_MASK) == 0) + BDK_PREFETCH(address1, BDK_CACHE_LINE_SIZE); + if ((address2 & BDK_CACHE_LINE_MASK) == 0) + BDK_PREFETCH(address2, BDK_CACHE_LINE_SIZE); + uint64_t d1 = READ64(address1); + uint64_t d2 = READ64(address2); + if (bdk_unlikely(d1 != d2)) + { + failures += __bdk_dram_retry_failure2(burst, address1, d1, address2, d2); + + // Synchronize the two areas, adjusting for the error. + WRITE64(address1, d2); + WRITE64(address2, d2); + } + address1 += 8; + address2 += 8; + } + } + return failures; +} + +/** + * test_mem_rows + * + * Write a pattern of alternating 64-bit words of all one bits and then all 0 + * bits. This pattern generates the maximum amount of simultaneous switching + * activity on the memory channels. Each pass flips the pattern with words + * going from all ones to all zeros and vice versa. + * + * @param area Start of the physical memory area + * @param max_address + * End of the physical memory area + * @param bursts Number of times to repeat the test over the entire area + * + * @return Number of errors, zero on success + */ +int __bdk_dram_test_mem_rows(uint64_t area, uint64_t max_address, int bursts) +{ + int failures = 0; + uint64_t pattern = 0x0; + uint64_t extent = (max_address - area); + uint64_t count = (extent / 2) / sizeof(uint64_t); // in terms of 64bit words + + /* Fill both halves of the memory area with identical data pattern. Odd + * address 64-bit words get the pattern, while even address words get the + * inverted pattern. + */ + uint64_t address1 = area; + uint64_t address2 = area + count * sizeof(uint64_t); + + for (uint64_t j = 0; j < (count / 2); j++) + { + WRITE64(address1, pattern); + WRITE64(address2, pattern); + address1 += 8; + address2 += 8; + WRITE64(address1, ~pattern); + WRITE64(address2, ~pattern); + address1 += 8; + address2 += 8; + } + __bdk_dram_flush_to_mem_range(area, max_address); + BDK_DCACHE_INVALIDATE; + + /* Make a series of passes over the memory areas. */ + for (int burst = 0; burst < bursts; burst++) + { + /* Invert the data, applying the change to both memory areas. Thus on + * alternate passes, the data flips from 0 to 1 and vice versa. + */ + address1 = area; + address2 = area + count * sizeof(uint64_t); + for (uint64_t j = 0; j < count; j++) + { + WRITE64(address1, ~READ64(address1)); + WRITE64(address2, ~READ64(address2)); + address1 += 8; + address2 += 8; + } + __bdk_dram_flush_to_mem_range(area, max_address); + BDK_DCACHE_INVALIDATE; + + /* Look for differences in the areas. If there is a mismatch, reset + * both memory locations with the same pattern. Failing to do so + * means that on all subsequent passes the pair of locations remain + * out of sync giving spurious errors. + */ + address1 = area; + address2 = area + count * sizeof(uint64_t); + for (uint64_t j = 0; j < count; j++) + { + uint64_t d1 = READ64(address1); + uint64_t d2 = READ64(address2); + if (bdk_unlikely(d1 != d2)) + { + failures += __bdk_dram_retry_failure2(burst, address1, d1, address2, d2); + + // Synchronize the two areas, adjusting for the error. + WRITE64(address1, d2); + WRITE64(address2, d2); + } + address1 += 8; + address2 += 8; + } + } + return failures; +} +#endif /* !USE_PREDICTION_CODE_VERSIONS */ + +#if USE_PREDICTION_CODE_VERSIONS +//////////////////////////// this is the new code... + +int __bdk_dram_test_mem_xor(uint64_t area, uint64_t max_address, int bursts) +{ + int failures = 0; + int burst; + + uint64_t extent = max_address - area; + uint64_t count = (extent / sizeof(uint64_t)) / 2; + uint64_t offset = count * sizeof(uint64_t); + uint64_t area2 = area + offset; + + /* Fill both halves of the memory area with identical randomized data. + */ + uint64_t address1 = area; + + uint64_t pattern1 = bdk_rng_get_random64(); + uint64_t pattern2 = 0; + uint64_t this_pattern; + + uint64_t p; + uint64_t d1, d2; + + // move the multiplies outside the loop + uint64_t pbase = address1 * pattern1; + uint64_t pincr = 8 * pattern1; + uint64_t ppred; + + p = pbase; + while (address1 < area2) + { + WRITE64(address1 , p); + WRITE64(address1 + offset, p); + address1 += 8; + p += pincr; + } + __bdk_dram_flush_to_mem_range(area, max_address); + BDK_DCACHE_INVALIDATE; + + /* Make a series of passes over the memory areas. */ + for (burst = 0; burst < bursts; burst++) + { + /* XOR the data with a random value, applying the change to both + * memory areas. + */ + address1 = area; + + this_pattern = bdk_rng_get_random64(); + pattern2 ^= this_pattern; + + while (address1 < area2) + { +#if 1 + if ((address1 & BDK_CACHE_LINE_MASK) == 0) + BDK_PREFETCH(address1, BDK_CACHE_LINE_SIZE); + if (((address1 + offset) & BDK_CACHE_LINE_MASK) == 0) + BDK_PREFETCH(address1 + offset, BDK_CACHE_LINE_SIZE); +#endif + WRITE64(address1 , READ64(address1 ) ^ this_pattern); + WRITE64(address1 + offset, READ64(address1 + offset) ^ this_pattern); + address1 += 8; + } + + __bdk_dram_flush_to_mem_range(area, max_address); + BDK_DCACHE_INVALIDATE; + + /* Look for differences from the expected pattern in both areas. + * If there is a mismatch, reset the appropriate memory location + * with the correct pattern. Failing to do so + * means that on all subsequent passes the erroring locations + * will be out of sync, giving spurious errors. + */ + address1 = area; + ppred = pbase; + + while (address1 < area2) + { +#if 1 + if ((address1 & BDK_CACHE_LINE_MASK) == 0) + BDK_PREFETCH(address1, BDK_CACHE_LINE_SIZE); + if (((address1 + offset) & BDK_CACHE_LINE_MASK) == 0) + BDK_PREFETCH(address1 + offset, BDK_CACHE_LINE_SIZE); +#endif + d1 = READ64(address1 ); + d2 = READ64(address1 + offset); + + p = ppred ^ pattern2; + + if (bdk_unlikely(d1 != p)) { + failures += __bdk_dram_retry_failure(burst, address1, d1, p); + // Synchronize the area, adjusting for the error. + //WRITE64(address1, p); // retries should do this + } + if (bdk_unlikely(d2 != p)) { + failures += __bdk_dram_retry_failure(burst, address1 + offset, d2, p); + // Synchronize the area, adjusting for the error. + //WRITE64(address1 + offset, p); // retries should do this + } + + address1 += 8; + ppred += pincr; + + } /* while (address1 < area2) */ + } /* for (int burst = 0; burst < bursts; burst++) */ + return failures; +} + +//////////////// this is the new code... + +int __bdk_dram_test_mem_rows(uint64_t area, uint64_t max_address, int bursts) +{ + int failures = 0; + + uint64_t pattern1 = 0x0; + uint64_t extent = (max_address - area); + uint64_t count = (extent / 2) / sizeof(uint64_t); // in terms of 64bit words + uint64_t offset = count * sizeof(uint64_t); + uint64_t area2 = area + offset; + uint64_t pattern2; + uint64_t d1, d2; + int burst; + + /* Fill both halves of the memory area with identical data pattern. Odd + * address 64-bit words get the pattern, while even address words get the + * inverted pattern. + */ + uint64_t address1 = area; + + pattern2 = pattern1; // start with original pattern + + while (address1 < area2) + { + WRITE64(address1 , pattern2); + WRITE64(address1 + offset, pattern2); + address1 += 8; + pattern2 = ~pattern2; // flip for next slots + } + + __bdk_dram_flush_to_mem_range(area, max_address); + BDK_DCACHE_INVALIDATE; + + /* Make a series of passes over the memory areas. */ + for (burst = 0; burst < bursts; burst++) + { + /* Invert the data, applying the change to both memory areas. Thus on + * alternate passes, the data flips from 0 to 1 and vice versa. + */ + address1 = area; + + while (address1 < area2) + { + if ((address1 & BDK_CACHE_LINE_MASK) == 0) + BDK_PREFETCH(address1 , BDK_CACHE_LINE_SIZE); + if (((address1 + offset) & BDK_CACHE_LINE_MASK) == 0) + BDK_PREFETCH(address1 + offset, BDK_CACHE_LINE_SIZE); + + WRITE64(address1 , ~READ64(address1 )); + WRITE64(address1 + offset, ~READ64(address1 + offset)); + address1 += 8; + } + + __bdk_dram_flush_to_mem_range(area, max_address); + BDK_DCACHE_INVALIDATE; + + /* Look for differences in the areas. If there is a mismatch, reset + * both memory locations with the same pattern. Failing to do so + * means that on all subsequent passes the pair of locations remain + * out of sync giving spurious errors. + */ + address1 = area; + pattern1 = ~pattern1; // flip the starting pattern to match above loop + pattern2 = pattern1; // slots have been flipped by the above loop + + while (address1 < area2) + { + if ((address1 & BDK_CACHE_LINE_MASK) == 0) + BDK_PREFETCH(address1 , BDK_CACHE_LINE_SIZE); + if (((address1 + offset) & BDK_CACHE_LINE_MASK) == 0) + BDK_PREFETCH(address1 + offset, BDK_CACHE_LINE_SIZE); + + d1 = READ64(address1 ); + d2 = READ64(address1 + offset); + + if (bdk_unlikely(d1 != pattern2)) { + failures += __bdk_dram_retry_failure(burst, address1, d1, pattern2); + // Synchronize the area, adjusting for the error. + //WRITE64(address1, pattern2); // retries should do this + } + if (bdk_unlikely(d2 != pattern2)) { + failures += __bdk_dram_retry_failure(burst, address1 + offset, d2, pattern2); + // Synchronize the two areas, adjusting for the error. + //WRITE64(address1 + offset, pattern2); // retries should do this + } + + address1 += 8; + pattern2 = ~pattern2; // flip for next pair of slots + } + } + return failures; +} +#endif /* USE_PREDICTION_CODE_VERSIONS */ diff --git a/src/vendorcode/cavium/bdk/libbdk-dram/bdk-dram-test.c b/src/vendorcode/cavium/bdk/libbdk-dram/bdk-dram-test.c new file mode 100644 index 0000000000..53137502fc --- /dev/null +++ b/src/vendorcode/cavium/bdk/libbdk-dram/bdk-dram-test.c @@ -0,0 +1,860 @@ +/***********************license start*********************************** +* Copyright (c) 2003-2017 Cavium Inc. (support@cavium.com). All rights +* reserved. +* +* +* Redistribution and use in source and binary forms, with or without +* modification, are permitted provided that the following conditions are +* met: +* +* * Redistributions of source code must retain the above copyright +* notice, this list of conditions and the following disclaimer. +* +* * Redistributions in binary form must reproduce the above +* copyright notice, this list of conditions and the following +* disclaimer in the documentation and/or other materials provided +* with the distribution. +* +* * Neither the name of Cavium Inc. nor the names of +* its contributors may be used to endorse or promote products +* derived from this software without specific prior written +* permission. +* +* This Software, including technical data, may be subject to U.S. export +* control laws, including the U.S. Export Administration Act and its +* associated regulations, and may be subject to export or import +* regulations in other countries. +* +* TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" +* AND WITH ALL FAULTS AND CAVIUM INC. MAKES NO PROMISES, REPRESENTATIONS OR +* WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH RESPECT +* TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY +* REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT +* DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) WARRANTIES +* OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A PARTICULAR +* PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET ENJOYMENT, +* QUIET POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE ENTIRE RISK +* ARISING OUT OF USE OR PERFORMANCE OF THE SOFTWARE LIES WITH YOU. +***********************license end**************************************/ +#include <bdk.h> +#include "libbdk-arch/bdk-csrs-gti.h" +#include "libbdk-arch/bdk-csrs-ocx.h" + +/* This code is an optional part of the BDK. It is only linked in + if BDK_REQUIRE() needs it */ +BDK_REQUIRE_DEFINE(DRAM_TEST); + +#define MAX_ERRORS_TO_REPORT 50 +#define RETRY_LIMIT 1000 + +typedef struct +{ + const char * name; /* Friendly name for the test */ + __bdk_dram_test_t test_func; /* Function to call */ + int bursts; /* Bursts parameter to pass to the test */ + int max_cores; /* Maximum number of cores the test should be run on in parallel. Zero means all */ +} dram_test_info_t; + +static const dram_test_info_t TEST_INFO[] = { + /* Name, Test function, Bursts, Max Cores */ + { "Data Bus", __bdk_dram_test_mem_data_bus, 8, 1}, + { "Address Bus", __bdk_dram_test_mem_address_bus, 0, 1}, + { "Marching Rows", __bdk_dram_test_mem_rows, 16, 0}, + { "Random Data", __bdk_dram_test_mem_random, 32, 0}, + { "Random XOR (32 Burst)", __bdk_dram_test_mem_xor, 32, 0}, + { "Self Address", __bdk_dram_test_mem_self_addr, 1, 0}, + { "March C- Solid Bits", __bdk_dram_test_mem_solid, 1, 0}, + { "March C- Checkerboard", __bdk_dram_test_mem_checkerboard, 1, 0}, + { "Walking Ones Left", __bdk_dram_test_mem_leftwalk1, 1, 0}, + { "Walking Ones Right", __bdk_dram_test_mem_rightwalk1, 1, 0}, + { "Walking Zeros Left", __bdk_dram_test_mem_leftwalk0, 1, 0}, + { "Walking Zeros Right", __bdk_dram_test_mem_rightwalk0, 1, 0}, + { "Random XOR (224 Burst)", __bdk_dram_test_mem_xor, 224, 0}, + { "Fast Scan", __bdk_dram_test_fast_scan, 0, 0}, + { NULL, NULL, 0, 0} +}; + +/* These variables count the number of ECC errors. They should only be accessed atomically */ +int64_t __bdk_dram_ecc_single_bit_errors[BDK_MAX_MEM_CHANS]; +int64_t __bdk_dram_ecc_double_bit_errors[BDK_MAX_MEM_CHANS]; + +static int64_t dram_test_thread_done; +static int64_t dram_test_thread_errors; +static uint64_t dram_test_thread_start; +static uint64_t dram_test_thread_end; +static uint64_t dram_test_thread_size; + +/** + * Force the memory at the pointer location to be written to memory and evicted + * from L2. L1 will be unaffected. + * + * @param address Physical memory location + */ +void __bdk_dram_flush_to_mem(uint64_t address) +{ + BDK_MB; + /* The DRAM code doesn't use the normal bdk_phys_to_ptr() because of the + NULL check in it. This greatly slows down the memory tests */ + char *ptr = (void*)address; + BDK_CACHE_WBI_L2(ptr); +} + +/** + * Force a memory region to be written to DRAM and evicted from L2 + * + * @param area Start of the region + * @param max_address + * End of the region (exclusive) + */ +void __bdk_dram_flush_to_mem_range(uint64_t area, uint64_t max_address) +{ + /* The DRAM code doesn't use the normal bdk_phys_to_ptr() because of the + NULL check in it. This greatly slows down the memory tests */ + char *ptr = (void*)area; + char *end = (void*)max_address; + BDK_MB; + while (ptr < end) + { + BDK_CACHE_WBI_L2(ptr); + ptr += 128; + } +} + +/** + * Convert a test enumeration into a string + * + * @param test Test to convert + * + * @return String for display + */ +const char *bdk_dram_get_test_name(int test) +{ + if (test < (int)(sizeof(TEST_INFO) / sizeof(TEST_INFO[0]))) + return TEST_INFO[test].name; + else + return NULL; +} + +static bdk_dram_test_flags_t dram_test_flags; // FIXME: Don't use global +/** + * This function is run as a thread to perform memory tests over multiple cores. + * Each thread gets a section of memory to work on, which is controlled by global + * variables at the beginning of this file. + * + * @param arg Number of the region we should check + * @param arg1 Pointer the the test_info structure + */ +static void dram_test_thread(int arg, void *arg1) +{ + const dram_test_info_t *test_info = arg1; + const int bursts = test_info->bursts; + const int range_number = arg; + + /* Figure out our work memory range. + * + * Note start_address and end_address just provide the physical offset + * portion of the address and do not have the node bits set. This is + * to simplify address checks and calculations. Later, when about to run + * the memory test, the routines adds in the node bits to form the final + * addresses. + */ + uint64_t start_address = dram_test_thread_start + dram_test_thread_size * range_number; + uint64_t end_address = start_address + dram_test_thread_size; + if (end_address > dram_test_thread_end) + end_address = dram_test_thread_end; + + bdk_node_t test_node = bdk_numa_local(); + if (dram_test_flags & BDK_DRAM_TEST_USE_CCPI) + test_node ^= 1; + /* Insert the node part of the address */ + start_address = bdk_numa_get_address(test_node, start_address); + end_address = bdk_numa_get_address(test_node, end_address); + /* Test the region */ + BDK_TRACE(DRAM_TEST, " Node %d, core %d, Testing [0x%011lx:0x%011lx]\n", + bdk_numa_local(), bdk_get_core_num() & 127, start_address, end_address - 1); + test_info->test_func(start_address, end_address, bursts); + + /* Report that we're done */ + BDK_TRACE(DRAM_TEST, "Thread %d on node %d done with memory test\n", range_number, bdk_numa_local()); + bdk_atomic_add64_nosync(&dram_test_thread_done, 1); +} + +/** + * Run the memory test. + * + * @param test_info + * @param start_address + * Physical address to start at + * @param length Length of memory block + * @param flags Flags to control memory test options. Zero defaults to testing all + * node with statistics and progress output. + * + * @return Number of errors found. Zero is success. Negative means the test + * did not run due to some other failure. + */ +static int __bdk_dram_run_test(const dram_test_info_t *test_info, uint64_t start_address, + uint64_t length, bdk_dram_test_flags_t flags) +{ + /* Figure out the addess of the byte one off the top of memory */ + uint64_t max_address = bdk_dram_get_size_mbytes(bdk_numa_local()); + BDK_TRACE(DRAM_TEST, "DRAM available per node: %lu MB\n", max_address); + max_address <<= 20; + + /* Make sure we have enough */ + if (max_address < (16<<20)) + { + bdk_error("DRAM size is too small\n"); + return -1; + } + + /* Make sure the amount is sane */ + if (CAVIUM_IS_MODEL(CAVIUM_CN8XXX)) + { + if (max_address > (1ull << 40)) /* 40 bits in CN8XXX */ + max_address = 1ull << 40; + } + else + { + if (max_address > (1ull << 43)) /* 43 bits in CN9XXX */ + max_address = 1ull << 43; + } + BDK_TRACE(DRAM_TEST, "DRAM max address: 0x%011lx\n", max_address-1); + + /* Make sure the start address is lower than the top of memory */ + if (start_address >= max_address) + { + bdk_error("Start address is larger than the amount of memory: 0x%011lx versus 0x%011lx\n", + start_address, max_address); + return -1; + } + if (length == (uint64_t)-1) + length = max_address - start_address; + + /* Final range checks */ + uint64_t end_address = start_address + length; + if (end_address > max_address) + { + end_address = max_address; + length = end_address - start_address; + } + if (length == 0) + return 0; + + /* Ready to run the test. Figure out how many cores we need */ + int max_cores = test_info->max_cores; + int total_cores_all_nodes = max_cores; + + /* Figure out the number of cores available in the system */ + if (max_cores == 0) + { + max_cores += bdk_get_num_running_cores(bdk_numa_local()); + /* Calculate the total number of cores being used. The per node number + is confusing to people */ + for (bdk_node_t node = BDK_NODE_0; node < BDK_NUMA_MAX_NODES; node++) + if (flags & (1 << node)) + { + if (flags & BDK_DRAM_TEST_USE_CCPI) + total_cores_all_nodes += bdk_get_num_running_cores(node ^ 1); + else + total_cores_all_nodes += bdk_get_num_running_cores(node); + } + } + if (!(flags & BDK_DRAM_TEST_NO_BANNERS)) + printf("Starting Test \"%s\" for [0x%011lx:0x%011lx] using %d core(s)\n", + test_info->name, start_address, end_address - 1, total_cores_all_nodes); + + /* Remember the LMC perf counters for stats after the test */ + uint64_t start_dram_dclk[BDK_NUMA_MAX_NODES][4]; + uint64_t start_dram_ops[BDK_NUMA_MAX_NODES][4]; + uint64_t stop_dram_dclk[BDK_NUMA_MAX_NODES][4]; + uint64_t stop_dram_ops[BDK_NUMA_MAX_NODES][4]; + for (bdk_node_t node = BDK_NODE_0; node < BDK_NUMA_MAX_NODES; node++) + { + if (flags & (1 << node)) + { + const int num_dram_controllers = __bdk_dram_get_num_lmc(node); + for (int i = 0; i < num_dram_controllers; i++) + { + start_dram_dclk[node][i] = BDK_CSR_READ(node, BDK_LMCX_DCLK_CNT(i)); + start_dram_ops[node][i] = BDK_CSR_READ(node, BDK_LMCX_OPS_CNT(i)); + } + } + } + /* Remember the CCPI link counters for stats after the test */ + uint64_t start_ccpi_data[BDK_NUMA_MAX_NODES][3]; + uint64_t start_ccpi_idle[BDK_NUMA_MAX_NODES][3]; + uint64_t start_ccpi_err[BDK_NUMA_MAX_NODES][3]; + uint64_t stop_ccpi_data[BDK_NUMA_MAX_NODES][3]; + uint64_t stop_ccpi_idle[BDK_NUMA_MAX_NODES][3]; + uint64_t stop_ccpi_err[BDK_NUMA_MAX_NODES][3]; + if (!bdk_numa_is_only_one()) + { + for (bdk_node_t node = BDK_NODE_0; node < BDK_NUMA_MAX_NODES; node++) + { + if (flags & (1 << node)) + { + for (int link = 0; link < 3; link++) + { + start_ccpi_data[node][link] = BDK_CSR_READ(node, BDK_OCX_TLKX_STAT_DATA_CNT(link)); + start_ccpi_idle[node][link] = BDK_CSR_READ(node, BDK_OCX_TLKX_STAT_IDLE_CNT(link)); + start_ccpi_err[node][link] = BDK_CSR_READ(node, BDK_OCX_TLKX_STAT_ERR_CNT(link)); + } + } + } + } + + /* WARNING: This code assumes the same memory range is being tested on + all nodes. The same number of cores are used on each node to test + its local memory */ + uint64_t work_address = start_address; + dram_test_flags = flags; + bdk_atomic_set64(&dram_test_thread_errors, 0); + while ((work_address < end_address) && ((dram_test_thread_errors == 0) || (flags & BDK_DRAM_TEST_NO_STOP_ERROR))) + { + /* Check at most MAX_CHUNK_SIZE across each iteration. We only report + progress between chunks, so keep them reasonably small */ + const uint64_t MAX_CHUNK_SIZE = 1ull << 28; /* 256MB */ + uint64_t size = end_address - work_address; + if (size > MAX_CHUNK_SIZE) + size = MAX_CHUNK_SIZE; + + /* Divide memory evenly between the cores. Round the size up so that + all memory is covered. The last core may have slightly less memory to + test */ + uint64_t thread_size = (size + (max_cores - 1)) / max_cores; + thread_size += 127; + thread_size &= -128; + dram_test_thread_start = work_address; + dram_test_thread_end = work_address + size; + dram_test_thread_size = thread_size; + BDK_WMB; + + /* Poke the watchdog */ + BDK_CSR_WRITE(bdk_numa_local(), BDK_GTI_CWD_POKEX(0), 0); + + /* disable progress output when batch mode is ON */ + if (!(flags & BDK_DRAM_TEST_NO_PROGRESS)) { + + /* Report progress percentage */ + int percent_x10 = (work_address - start_address) * 1000 / (end_address - start_address); + printf(" %3d.%d%% complete, testing [0x%011lx:0x%011lx]\r", + percent_x10 / 10, percent_x10 % 10, work_address, work_address + size - 1); + fflush(stdout); + } + + work_address += size; + + /* Start threads for all the cores */ + int total_count = 0; + bdk_atomic_set64(&dram_test_thread_done, 0); + for (bdk_node_t node = BDK_NODE_0; node < BDK_NUMA_MAX_NODES; node++) + { + if (flags & (1 << node)) + { + const int num_cores = bdk_get_num_cores(node); + int per_node = 0; + for (int core = 0; core < num_cores; core++) + { + if (per_node >= max_cores) + break; + int run_node = (flags & BDK_DRAM_TEST_USE_CCPI) ? node ^ 1 : node; + BDK_TRACE(DRAM_TEST, "Starting thread %d on node %d for memory test\n", per_node, node); + if (bdk_thread_create(run_node, 0, dram_test_thread, per_node, (void *)test_info, 0)) + { + bdk_error("Failed to create thread %d for memory test on node %d\n", per_node, node); + } + else + { + per_node++; + total_count++; + } + } + } + } + +#if 0 + /* Wait for threads to finish */ + while (bdk_atomic_get64(&dram_test_thread_done) < total_count) + bdk_thread_yield(); +#else +#define TIMEOUT_SECS 30 // FIXME: long enough so multicore RXOR 224 should not print out + /* Wait for threads to finish, with progress */ + int cur_count; + uint64_t cur_time; + uint64_t period = bdk_clock_get_rate(bdk_numa_local(), BDK_CLOCK_TIME) * TIMEOUT_SECS; // FIXME? + uint64_t timeout = bdk_clock_get_count(BDK_CLOCK_TIME) + period; + do { + bdk_thread_yield(); + cur_count = bdk_atomic_get64(&dram_test_thread_done); + cur_time = bdk_clock_get_count(BDK_CLOCK_TIME); + if (cur_time >= timeout) { + BDK_TRACE(DRAM_TEST, "N%d: Waiting for %d cores\n", + bdk_numa_local(), total_count - cur_count); + timeout = cur_time + period; + } + } while (cur_count < total_count); +#endif + } + + /* Get the DRAM perf counters */ + for (bdk_node_t node = BDK_NODE_0; node < BDK_NUMA_MAX_NODES; node++) + { + if (flags & (1 << node)) + { + const int num_dram_controllers = __bdk_dram_get_num_lmc(node); + for (int i = 0; i < num_dram_controllers; i++) + { + stop_dram_dclk[node][i] = BDK_CSR_READ(node, BDK_LMCX_DCLK_CNT(i)); + stop_dram_ops[node][i] = BDK_CSR_READ(node, BDK_LMCX_OPS_CNT(i)); + } + } + } + /* Get the CCPI link counters */ + if (!bdk_numa_is_only_one()) + { + for (bdk_node_t node = BDK_NODE_0; node < BDK_NUMA_MAX_NODES; node++) + { + if (flags & (1 << node)) + { + for (int link = 0; link < 3; link++) + { + stop_ccpi_data[node][link] = BDK_CSR_READ(node, BDK_OCX_TLKX_STAT_DATA_CNT(link)); + stop_ccpi_idle[node][link] = BDK_CSR_READ(node, BDK_OCX_TLKX_STAT_IDLE_CNT(link)); + stop_ccpi_err[node][link] = BDK_CSR_READ(node, BDK_OCX_TLKX_STAT_ERR_CNT(link)); + } + } + } + } + + /* disable progress output when batch mode is ON */ + if (!(flags & BDK_DRAM_TEST_NO_PROGRESS)) { + + /* Report progress percentage as complete */ + printf(" %3d.%d%% complete, testing [0x%011lx:0x%011lx]\n", + 100, 0, start_address, end_address - 1); + fflush(stdout); + } + + if (!(flags & BDK_DRAM_TEST_NO_STATS)) + { + /* Display LMC load */ + for (bdk_node_t node = BDK_NODE_0; node < BDK_NUMA_MAX_NODES; node++) + { + if (flags & (1 << node)) + { + const int num_dram_controllers = __bdk_dram_get_num_lmc(node); + for (int i = 0; i < num_dram_controllers; i++) + { + uint64_t ops = stop_dram_ops[node][i] - start_dram_ops[node][i]; + uint64_t dclk = stop_dram_dclk[node][i] - start_dram_dclk[node][i]; + if (dclk == 0) + dclk = 1; + uint64_t percent_x10 = ops * 1000 / dclk; + printf(" Node %d, LMC%d: ops %lu, cycles %lu, used %lu.%lu%%\n", + node, i, ops, dclk, percent_x10 / 10, percent_x10 % 10); + } + } + } + if (flags & BDK_DRAM_TEST_USE_CCPI) + { + /* Display CCPI load */ + for (bdk_node_t node = BDK_NODE_0; node < BDK_NUMA_MAX_NODES; node++) + { + if (flags & (1 << node)) + { + for (int link = 0; link < 3; link++) + { + uint64_t busy = stop_ccpi_data[node][link] - start_ccpi_data[node][link]; + busy += stop_ccpi_err[node][link] - start_ccpi_err[node][link]; + uint64_t total = stop_ccpi_idle[node][link] - start_ccpi_idle[node][link]; + total += busy; + if (total == 0) + continue; + uint64_t percent_x10 = busy * 1000 / total; + printf(" Node %d, CCPI%d: busy %lu, total %lu, used %lu.%lu%%\n", + node, link, busy, total, percent_x10 / 10, percent_x10 % 10); + } + } + } + } + } + return dram_test_thread_errors; +} + +/** + * Perform a memory test. + * + * @param test Test type to run + * @param start_address + * Physical address to start at + * @param length Length of memory block + * @param flags Flags to control memory test options. Zero defaults to testing all + * node with statistics and progress output. + * + * @return Number of errors found. Zero is success. Negative means the test + * did not run due to some other failure. + */ +int bdk_dram_test(int test, uint64_t start_address, uint64_t length, bdk_dram_test_flags_t flags) +{ + /* These limits are arbitrary. They just make sure we aren't doing something + silly, like test a non cache line aligned memory region */ + if (start_address & 0xffff) + { + bdk_error("DRAM test start address must be aligned on a 64KB boundary\n"); + return -1; + } + if (length & 0xffff) + { + bdk_error("DRAM test length must be a multiple of 64KB\n"); + return -1; + } + + const char *name = bdk_dram_get_test_name(test); + if (name == NULL) + { + bdk_error("Invalid DRAM test number %d\n", test); + return -1; + } + + /* If no nodes are selected assume the user meant all nodes */ + if ((flags & (BDK_DRAM_TEST_NODE0 | BDK_DRAM_TEST_NODE1 | BDK_DRAM_TEST_NODE2 | BDK_DRAM_TEST_NODE3)) == 0) + flags |= BDK_DRAM_TEST_NODE0 | BDK_DRAM_TEST_NODE1 | BDK_DRAM_TEST_NODE2 | BDK_DRAM_TEST_NODE3; + + /* Remove nodes from the flags that don't exist */ + for (bdk_node_t node = BDK_NODE_0; node < BDK_NUMA_MAX_NODES; node++) + { + if (flags & BDK_DRAM_TEST_USE_CCPI) + { + if (!bdk_numa_exists(node ^ 1)) + flags &= ~(1 << node); + } + else + { + if (!bdk_numa_exists(node)) + flags &= ~(1 << node); + } + } + + + /* Make sure the start address is higher that the BDK's active range */ + uint64_t top_of_bdk = bdk_dram_get_top_of_bdk(); + if (start_address < top_of_bdk) + start_address = top_of_bdk; + + /* Clear ECC error counters before starting the test */ + for (int chan = 0; chan < BDK_MAX_MEM_CHANS; chan++) { + bdk_atomic_set64(&__bdk_dram_ecc_single_bit_errors[chan], 0); + bdk_atomic_set64(&__bdk_dram_ecc_double_bit_errors[chan], 0); + } + + /* Make sure at least one core from each node is running */ + for (bdk_node_t node = BDK_NODE_0; node < BDK_NUMA_MAX_NODES; node++) + { + if (flags & (1<<node)) + { + int use_node = (flags & BDK_DRAM_TEST_USE_CCPI) ? node ^ 1 : node; + if (bdk_get_running_coremask(use_node) == 0) + bdk_init_cores(use_node, 1); + } + } + + /* This returns any data compare errors found */ + int errors = __bdk_dram_run_test(&TEST_INFO[test], start_address, length, flags); + + /* Poll for any errors right now to make sure any ECC errors are reported */ + for (bdk_node_t node = BDK_NODE_0; node < BDK_NUMA_MAX_NODES; node++) + { + if (bdk_numa_exists(node) && bdk_error_check) + bdk_error_check(node); + } + + /* Check ECC error counters after the test */ + int64_t ecc_single = 0; + int64_t ecc_double = 0; + int64_t ecc_single_errs[BDK_MAX_MEM_CHANS]; + int64_t ecc_double_errs[BDK_MAX_MEM_CHANS]; + + for (int chan = 0; chan < BDK_MAX_MEM_CHANS; chan++) { + ecc_single += (ecc_single_errs[chan] = bdk_atomic_get64(&__bdk_dram_ecc_single_bit_errors[chan])); + ecc_double += (ecc_double_errs[chan] = bdk_atomic_get64(&__bdk_dram_ecc_double_bit_errors[chan])); + } + + /* Always print any ECC errors */ + if (ecc_single || ecc_double) + { + printf("Test \"%s\": ECC errors, %ld/%ld/%ld/%ld corrected, %ld/%ld/%ld/%ld uncorrected\n", + name, + ecc_single_errs[0], ecc_single_errs[1], ecc_single_errs[2], ecc_single_errs[3], + ecc_double_errs[0], ecc_double_errs[1], ecc_double_errs[2], ecc_double_errs[3]); + } + if (errors || ecc_double || ecc_single) { + printf("Test \"%s\": FAIL: %ld single, %ld double, %d compare errors\n", + name, ecc_single, ecc_double, errors); + } + else + BDK_TRACE(DRAM_TEST, "Test \"%s\": PASS\n", name); + + return (errors + ecc_double + ecc_single); +} + +/** + * Report a DRAM address in decoded format. + * + * @param address Physical address the error occurred at + * + */ +static void __bdk_dram_report_address_decode(uint64_t address, char *buffer, int len) +{ + int node, lmc, dimm, prank, lrank, bank, row, col; + + bdk_dram_address_extract_info(address, &node, &lmc, &dimm, &prank, &lrank, &bank, &row, &col); + + snprintf(buffer, len, "[0x%011lx] (N%d,LMC%d,DIMM%d,Rank%d/%d,Bank%02d,Row 0x%05x,Col 0x%04x)", + address, node, lmc, dimm, prank, lrank, bank, row, col); +} + +/** + * Report a DRAM address in a new decoded format. + * + * @param address Physical address the error occurred at + * @param xor XOR of data read vs expected data + * + */ +static void __bdk_dram_report_address_decode_new(uint64_t address, uint64_t orig_xor, char *buffer, int len) +{ + int node, lmc, dimm, prank, lrank, bank, row, col; + + int byte = 8; // means no byte-lanes in error, should not happen + uint64_t bits, print_bits = 0; + uint64_t xor = orig_xor; + + // find the byte-lane(s) with errors + for (int i = 0; i < 8; i++) { + bits = xor & 0xffULL; + xor >>= 8; + if (bits) { + if (byte != 8) { + byte = 9; // means more than 1 byte-lane was present + print_bits = orig_xor; // print the full original + break; // quit now + } else { + byte = i; // keep checking + print_bits = bits; + } + } + } + + bdk_dram_address_extract_info(address, &node, &lmc, &dimm, &prank, &lrank, &bank, &row, &col); + + snprintf(buffer, len, "N%d.LMC%d: CMP byte %d xor 0x%02lx (DIMM%d,Rank%d/%d,Bank%02d,Row 0x%05x,Col 0x%04x)[0x%011lx]", + node, lmc, byte, print_bits, dimm, prank, lrank, bank, row, col, address); +} + +/** + * Report a DRAM error. Errors are not shown after MAX_ERRORS_TO_REPORT is + * exceeded. Used when a single address is involved in the failure. + * + * @param address Physical address the error occurred at + * @param data Data read from memory + * @param correct Correct data + * @param burst Which burst this is from, informational only + * @param fails -1 for no retries done, >= 0 number of failures during retries + * + * @return Zero if a message was logged, non-zero if the error limit has been reached + */ +void __bdk_dram_report_error(uint64_t address, uint64_t data, uint64_t correct, int burst, int fails) +{ + char buffer[128]; + char failbuf[32]; + int64_t errors = bdk_atomic_fetch_and_add64(&dram_test_thread_errors, 1); + uint64_t xor = data ^ correct; + + if (errors < MAX_ERRORS_TO_REPORT) + { + if (fails < 0) { + snprintf(failbuf, sizeof(failbuf), " "); + } else { + int percent_x10 = fails * 1000 / RETRY_LIMIT; + snprintf(failbuf, sizeof(failbuf), ", retries failed %3d.%d%%", + percent_x10 / 10, percent_x10 % 10); + } + + __bdk_dram_report_address_decode_new(address, xor, buffer, sizeof(buffer)); + bdk_error("%s%s\n", buffer, failbuf); + + if (errors == MAX_ERRORS_TO_REPORT-1) + bdk_error("No further DRAM errors will be reported\n"); + } + return; +} + +/** + * Report a DRAM error. Errors are not shown after MAX_ERRORS_TO_REPORT is + * exceeded. Used when two addresses might be involved in the failure. + * + * @param address1 First address involved in the failure + * @param data1 Data from the first address + * @param address2 Second address involved in the failure + * @param data2 Data from second address + * @param burst Which burst this is from, informational only + * @param fails -1 for no retries done, >= 0 number of failures during retries + * + * @return Zero if a message was logged, non-zero if the error limit has been reached + */ +void __bdk_dram_report_error2(uint64_t address1, uint64_t data1, uint64_t address2, uint64_t data2, + int burst, int fails) +{ + int64_t errors = bdk_atomic_fetch_and_add64(&dram_test_thread_errors, 1); + if (errors < MAX_ERRORS_TO_REPORT) + { + char buffer1[80], buffer2[80]; + char failbuf[32]; + + if (fails < 0) { + snprintf(failbuf, sizeof(failbuf), " "); + } else { + snprintf(failbuf, sizeof(failbuf), ", retried %d failed %d", RETRY_LIMIT, fails); + } + __bdk_dram_report_address_decode(address1, buffer1, sizeof(buffer1)); + __bdk_dram_report_address_decode(address2, buffer2, sizeof(buffer2)); + + bdk_error("compare: data1: 0x%016lx, xor: 0x%016lx%s\n" + " %s\n %s\n", + data1, data1 ^ data2, failbuf, + buffer1, buffer2); + + if (errors == MAX_ERRORS_TO_REPORT-1) + bdk_error("No further DRAM errors will be reported\n"); + } + return; +} + +/* Report the circumstances of a failure and try re-reading the memory + * location to see if the error is transient or permanent. + * + * Note: re-reading requires using evicting addresses + */ +int __bdk_dram_retry_failure(int burst, uint64_t address, uint64_t data, uint64_t expected) +{ + int refail = 0; + + // bypass the retries if we are already over the limit... + if (bdk_atomic_get64(&dram_test_thread_errors) < MAX_ERRORS_TO_REPORT) { + + /* Try re-reading the memory location. A transient error may fail + * on one read and work on another. Keep on retrying even when a + * read succeeds. + */ + for (int i = 0; i < RETRY_LIMIT; i++) { + + __bdk_dram_flush_to_mem(address); + BDK_DCACHE_INVALIDATE; + + uint64_t new = __bdk_dram_read64(address); + + if (new != expected) { + refail++; + } + } + } else + refail = -1; + + // this will increment the errors always, but maybe not print... + __bdk_dram_report_error(address, data, expected, burst, refail); + + return 1; +} + +/** + * retry_failure2 + * + * @param burst + * @param address1 + * @param address2 + */ +int __bdk_dram_retry_failure2(int burst, uint64_t address1, uint64_t data1, uint64_t address2, uint64_t data2) +{ + int refail = 0; + + // bypass the retries if we are already over the limit... + if (bdk_atomic_get64(&dram_test_thread_errors) < MAX_ERRORS_TO_REPORT) { + + for (int i = 0; i < RETRY_LIMIT; i++) { + __bdk_dram_flush_to_mem(address1); + __bdk_dram_flush_to_mem(address2); + BDK_DCACHE_INVALIDATE; + + uint64_t d1 = __bdk_dram_read64(address1); + uint64_t d2 = __bdk_dram_read64(address2); + + if (d1 != d2) { + refail++; + } + } + } else + refail = -1; + + // this will increment the errors always, but maybe not print... + __bdk_dram_report_error2(address1, data1, address2, data2, burst, refail); + + return 1; +} + +/** + * Inject a DRAM error at a specific address in memory. The injection can either + * be a single bit inside the byte, or a double bit error in the ECC byte. Double + * bit errors may corrupt memory, causing software to crash. The corruption is + * written to memory and will continue to exist until the cache line is written + * again. After a call to this function, the BDK should report a ECC error. Double + * bit errors corrupt bits 0-1. + * + * @param address Physical address to corrupt. Any byte alignment is supported + * @param bit Bit to corrupt in the byte (0-7), or -1 to create a double bit fault in the ECC + * byte. + */ +void bdk_dram_test_inject_error(uint64_t address, int bit) +{ + uint64_t aligned_address = address & -16; + int corrupt_bit = -1; + if (bit >= 0) + corrupt_bit = (address & 0xf) * 8 + bit; + + /* Extract the DRAM controller information */ + int node, lmc, dimm, prank, lrank, bank, row, col; + bdk_dram_address_extract_info(address, &node, &lmc, &dimm, &prank, &lrank, &bank, &row, &col); + + /* Read the current data */ + uint64_t data = __bdk_dram_read64(aligned_address); + + /* Program LMC to inject the error */ + if ((corrupt_bit >= 0) && (corrupt_bit < 64)) + BDK_CSR_WRITE(node, BDK_LMCX_CHAR_MASK0(lmc), 1ull << corrupt_bit); + else if (bit == -1) + BDK_CSR_WRITE(node, BDK_LMCX_CHAR_MASK0(lmc), 3); + else + BDK_CSR_WRITE(node, BDK_LMCX_CHAR_MASK0(lmc), 0); + if (corrupt_bit >= 64) + BDK_CSR_WRITE(node, BDK_LMCX_CHAR_MASK2(lmc), 1ull << (corrupt_bit - 64)); + else + BDK_CSR_WRITE(node, BDK_LMCX_CHAR_MASK2(lmc), 0); + BDK_CSR_MODIFY(c, node, BDK_LMCX_ECC_PARITY_TEST(lmc), + c.s.ecc_corrupt_idx = (address & 0x7f) >> 4; + c.s.ecc_corrupt_ena = 1); + BDK_CSR_READ(node, BDK_LMCX_ECC_PARITY_TEST(lmc)); + + /* Perform a write and push it to DRAM. This creates the error */ + __bdk_dram_write64(aligned_address, data); + __bdk_dram_flush_to_mem(aligned_address); + + /* Disable error injection */ + BDK_CSR_MODIFY(c, node, BDK_LMCX_ECC_PARITY_TEST(lmc), + c.s.ecc_corrupt_ena = 0); + BDK_CSR_READ(node, BDK_LMCX_ECC_PARITY_TEST(lmc)); + BDK_CSR_WRITE(node, BDK_LMCX_CHAR_MASK0(lmc), 0); + BDK_CSR_WRITE(node, BDK_LMCX_CHAR_MASK2(lmc), 0); + + /* Read back the data, which should now cause an error */ + printf("Loading the injected error address 0x%lx, node=%d, lmc=%d, dimm=%d, rank=%d/%d, bank=%d, row=%d, col=%d\n", + address, node, lmc, dimm, prank, lrank, bank, row, col); + __bdk_dram_read64(aligned_address); +} |