diff options
| author | David Hendricks <dhendricks@fb.com> | 2018-03-09 13:58:27 -0800 |
|---|---|---|
| committer | Patrick Georgi <pgeorgi@google.com> | 2018-04-06 06:48:11 +0000 |
| commit | 2004b93aed993aa02bbc588b8d82c22418ac52ec (patch) | |
| tree | cdd5e95a154e2e0139474288262835a7f5847665 /src/vendorcode/cavium/bdk/libbdk-dram | |
| parent | 71cbd71eb5c0e8e13b25b5d5dd2f495e7d2967eb (diff) | |
soc/cavium: import raw BDK sources
This imports common BDK sources that will be used in subsequent
patches.
The BDK is licensed under BSD and will be reduced in size and optimized to
compile under coreboot.
Change-Id: Icb32ee670d9fa9e5c10f9abb298cebf616fa67ad
Signed-off-by: David Hendricks <dhendricks@fb.com>
Reviewed-on: https://review.coreboot.org/25524
Tested-by: build bot (Jenkins) <no-reply@coreboot.org>
Reviewed-by: David Hendricks <david.hendricks@gmail.com>
Diffstat (limited to 'src/vendorcode/cavium/bdk/libbdk-dram')
8 files changed, 2718 insertions, 0 deletions
diff --git a/src/vendorcode/cavium/bdk/libbdk-dram/bdk-dram-address.c b/src/vendorcode/cavium/bdk/libbdk-dram/bdk-dram-address.c new file mode 100644 index 0000000000..94d7d76752 --- /dev/null +++ b/src/vendorcode/cavium/bdk/libbdk-dram/bdk-dram-address.c @@ -0,0 +1,183 @@ +/***********************license start*********************************** +* Copyright (c) 2003-2017 Cavium Inc. (support@cavium.com). All rights +* reserved. +* +* +* Redistribution and use in source and binary forms, with or without +* modification, are permitted provided that the following conditions are +* met: +* +* * Redistributions of source code must retain the above copyright +* notice, this list of conditions and the following disclaimer. +* +* * Redistributions in binary form must reproduce the above +* copyright notice, this list of conditions and the following +* disclaimer in the documentation and/or other materials provided +* with the distribution. +* +* * Neither the name of Cavium Inc. nor the names of +* its contributors may be used to endorse or promote products +* derived from this software without specific prior written +* permission. +* +* This Software, including technical data, may be subject to U.S. export +* control laws, including the U.S. Export Administration Act and its +* associated regulations, and may be subject to export or import +* regulations in other countries. +* +* TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" +* AND WITH ALL FAULTS AND CAVIUM INC. MAKES NO PROMISES, REPRESENTATIONS OR +* WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH RESPECT +* TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY +* REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT +* DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) WARRANTIES +* OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A PARTICULAR +* PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET ENJOYMENT, +* QUIET POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE ENTIRE RISK +* ARISING OUT OF USE OR PERFORMANCE OF THE SOFTWARE LIES WITH YOU. +***********************license end**************************************/ +#include <bdk.h> +#include "libbdk-arch/bdk-csrs-l2c.h" + +#define EXTRACT(v, lsb, width) (((v) >> (lsb)) & ((1ull << (width)) - 1)) +#define INSERT(a, v, lsb, width) a|=(((v) & ((1ull << (width)) - 1)) << (lsb)) + +/** + * Given a physical DRAM address, extract information about the node, LMC, DIMM, + * prank, lrank, bank, row, and column that was accessed. + * + * @param address Physical address to decode + * @param node Node the address was for + * @param lmc LMC controller the address was for + * @param dimm DIMM the address was for + * @param prank Physical RANK on the DIMM + * @param lrank Logical RANK on the DIMM + * @param bank BANK on the DIMM + * @param row Row on the DIMM + * @param col Column on the DIMM + */ +void +bdk_dram_address_extract_info(uint64_t address, int *node, int *lmc, int *dimm, + int *prank, int *lrank, int *bank, int *row, int *col) +{ + int bitno = CAVIUM_IS_MODEL(CAVIUM_CN83XX) ? 19 : 20; + *node = EXTRACT(address, 40, 2); /* Address bits [41:40] */ + /* Determine the LMC controller */ + BDK_CSR_INIT(l2c_ctl, *node, BDK_L2C_CTL); + int bank_lsb, xbits; + + /* xbits depends on number of LMCs */ + xbits = __bdk_dram_get_num_lmc(*node) >> 1; // 4->2; 2->1; 1->0 + bank_lsb = 7 + xbits; + + /* LMC number is probably aliased */ + if (l2c_ctl.s.disidxalias) + *lmc = EXTRACT(address, 7, xbits); + else + *lmc = EXTRACT(address, 7, xbits) ^ EXTRACT(address, bitno, xbits) ^ EXTRACT(address, 12, xbits); + + /* Figure out the bank field width */ + BDK_CSR_INIT(lmcx_config, *node, BDK_LMCX_CONFIG(*lmc)); + int bank_width = __bdk_dram_get_num_bank_bits(*node, *lmc); + + /* Extract additional info from the LMC_CONFIG CSR */ + BDK_CSR_INIT(ext_config, *node, BDK_LMCX_EXT_CONFIG(*lmc)); + int dimm_lsb = 28 + lmcx_config.s.pbank_lsb + xbits; + int dimm_width = 40 - dimm_lsb; + int prank_lsb = dimm_lsb - lmcx_config.s.rank_ena; + int prank_width = dimm_lsb - prank_lsb; + int lrank_lsb = prank_lsb - ext_config.s.dimm0_cid; + int lrank_width = prank_lsb - lrank_lsb; + int row_lsb = 14 + lmcx_config.s.row_lsb + xbits; + int row_width = lrank_lsb - row_lsb; + int col_hi_lsb = bank_lsb + bank_width; + int col_hi_width= row_lsb - col_hi_lsb; + + /* Extract the parts of the address */ + *dimm = EXTRACT(address, dimm_lsb, dimm_width); + *prank = EXTRACT(address, prank_lsb, prank_width); + *lrank = EXTRACT(address, lrank_lsb, lrank_width); + *row = EXTRACT(address, row_lsb, row_width); + + /* bank calculation may be aliased... */ + BDK_CSR_INIT(lmcx_control, *node, BDK_LMCX_CONTROL(*lmc)); + if (lmcx_control.s.xor_bank) + *bank = EXTRACT(address, bank_lsb, bank_width) ^ EXTRACT(address, 12 + xbits, bank_width); + else + *bank = EXTRACT(address, bank_lsb, bank_width); + + /* LMC number already extracted */ + int col_hi = EXTRACT(address, col_hi_lsb, col_hi_width); + *col = EXTRACT(address, 3, 4) | (col_hi << 4); + /* Bus byte is address bits [2:0]. Unused here */ +} + +/** + * Construct a physical address given the node, LMC, DIMM, prank, lrank, bank, row, and column. + * + * @param node Node the address was for + * @param lmc LMC controller the address was for + * @param dimm DIMM the address was for + * @param prank Physical RANK on the DIMM + * @param lrank Logical RANK on the DIMM + * @param bank BANK on the DIMM + * @param row Row on the DIMM + * @param col Column on the DIMM + */ +uint64_t +bdk_dram_address_construct_info(bdk_node_t node, int lmc, int dimm, + int prank, int lrank, int bank, int row, int col) + +{ + uint64_t address = 0; + int bitno = CAVIUM_IS_MODEL(CAVIUM_CN83XX) ? 19 : 20; + + // insert node bits + INSERT(address, node, 40, 2); /* Address bits [41:40] */ + + /* xbits depends on number of LMCs */ + int xbits = __bdk_dram_get_num_lmc(node) >> 1; // 4->2; 2->1; 1->0 + int bank_lsb = 7 + xbits; + + /* Figure out the bank field width */ + int bank_width = __bdk_dram_get_num_bank_bits(node, lmc); + + /* Extract additional info from the LMC_CONFIG CSR */ + BDK_CSR_INIT(lmcx_config, node, BDK_LMCX_CONFIG(lmc)); + BDK_CSR_INIT(ext_config, node, BDK_LMCX_EXT_CONFIG(lmc)); + int dimm_lsb = 28 + lmcx_config.s.pbank_lsb + xbits; + int dimm_width = 40 - dimm_lsb; + int prank_lsb = dimm_lsb - lmcx_config.s.rank_ena; + int prank_width = dimm_lsb - prank_lsb; + int lrank_lsb = prank_lsb - ext_config.s.dimm0_cid; + int lrank_width = prank_lsb - lrank_lsb; + int row_lsb = 14 + lmcx_config.s.row_lsb + xbits; + int row_width = lrank_lsb - row_lsb; + int col_hi_lsb = bank_lsb + bank_width; + int col_hi_width = row_lsb - col_hi_lsb; + + /* Insert some other parts of the address */ + INSERT(address, dimm, dimm_lsb, dimm_width); + INSERT(address, prank, prank_lsb, prank_width); + INSERT(address, lrank, lrank_lsb, lrank_width); + INSERT(address, row, row_lsb, row_width); + INSERT(address, col >> 4, col_hi_lsb, col_hi_width); + INSERT(address, col, 3, 4); + + /* bank calculation may be aliased... */ + BDK_CSR_INIT(lmcx_control, node, BDK_LMCX_CONTROL(lmc)); + int new_bank = bank; + if (lmcx_control.s.xor_bank) + new_bank ^= EXTRACT(address, 12 + xbits, bank_width); + INSERT(address, new_bank, bank_lsb, bank_width); + + /* Determine the actual C bits from the input LMC controller arg */ + /* The input LMC number was probably aliased with other fields */ + BDK_CSR_INIT(l2c_ctl, node, BDK_L2C_CTL); + int new_lmc = lmc; + if (!l2c_ctl.s.disidxalias) + new_lmc ^= EXTRACT(address, bitno, xbits) ^ EXTRACT(address, 12, xbits); + INSERT(address, new_lmc, 7, xbits); + + return address; +} diff --git a/src/vendorcode/cavium/bdk/libbdk-dram/bdk-dram-config.c b/src/vendorcode/cavium/bdk/libbdk-dram/bdk-dram-config.c new file mode 100644 index 0000000000..3465c5d98b --- /dev/null +++ b/src/vendorcode/cavium/bdk/libbdk-dram/bdk-dram-config.c @@ -0,0 +1,163 @@ +/***********************license start*********************************** +* Copyright (c) 2003-2017 Cavium Inc. (support@cavium.com). All rights +* reserved. +* +* +* Redistribution and use in source and binary forms, with or without +* modification, are permitted provided that the following conditions are +* met: +* +* * Redistributions of source code must retain the above copyright +* notice, this list of conditions and the following disclaimer. +* +* * Redistributions in binary form must reproduce the above +* copyright notice, this list of conditions and the following +* disclaimer in the documentation and/or other materials provided +* with the distribution. +* +* * Neither the name of Cavium Inc. nor the names of +* its contributors may be used to endorse or promote products +* derived from this software without specific prior written +* permission. +* +* This Software, including technical data, may be subject to U.S. export +* control laws, including the U.S. Export Administration Act and its +* associated regulations, and may be subject to export or import +* regulations in other countries. +* +* TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" +* AND WITH ALL FAULTS AND CAVIUM INC. MAKES NO PROMISES, REPRESENTATIONS OR +* WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH RESPECT +* TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY +* REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT +* DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) WARRANTIES +* OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A PARTICULAR +* PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET ENJOYMENT, +* QUIET POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE ENTIRE RISK +* ARISING OUT OF USE OR PERFORMANCE OF THE SOFTWARE LIES WITH YOU. +***********************license end**************************************/ +#include <bdk.h> +#include <unistd.h> + +BDK_REQUIRE_DEFINE(DRAM_CONFIG); + +/** + * Lookup a DRAM configuration by name and initialize DRAM using it + * + * @param node Node to configure + * @param ddr_clock_override + * If non zero, override the DRAM frequency specified + * in the config with this value + * + * @return Amount of DRAM in MB, or negative on failure + */ +int bdk_dram_config(int node, int ddr_clock_override) +{ + const dram_config_t *config = libdram_config_load(node); + if (!config) + { + printf("N%d: No DRAM config specified, skipping DRAM init\n", node); + return 0; + } + + BDK_TRACE(DRAM, "N%d: Starting DRAM init (config=%p, ddr_clock_override=%d)\n", node, config, ddr_clock_override); + int mbytes = libdram_config(node, config, ddr_clock_override); + BDK_TRACE(DRAM, "N%d: DRAM init returned %d\n", node, mbytes); + if (mbytes <= 0) + { + printf("ERROR: DDR initialization failed\n"); + return -1; + } + + return mbytes; +} + +/** + * Do DRAM configuration tuning + * + * @param node Node to tune + * + * @return Success or Fail + */ +int bdk_dram_tune(int node) +{ + int ret; + BDK_TRACE(DRAM, "N%d: Starting DRAM tuning\n", node); + ret = libdram_tune(node); + BDK_TRACE(DRAM, "N%d: DRAM tuning returned %d\n", node, ret); + return ret; +} + +/** + * Do all the DRAM Margin tests + * + * @param node Node to test + * + * @return Success or Fail + */ +void bdk_dram_margin(int node) +{ + BDK_TRACE(DRAM, "N%d: Starting DRAM margining\n", node); + libdram_margin(node); + BDK_TRACE(DRAM, "N%d: Finished DRAM margining.\n", node); + return; +} + +/** + * Return the string of the DRAM configuration info at the specified node. + * If the node is not configured, NULL is returned. + * + * @param node node to retrieve + * + * @return string or NULL + */ +const char* bdk_dram_get_info_string(int node) +{ + #define INFO_STRING_LEN 40 + static char info_string[INFO_STRING_LEN]; + static const char *info_ptr = info_string; + + snprintf(info_string, INFO_STRING_LEN, + " %ld MB, %ld MT/s, %s %s", + bdk_dram_get_size_mbytes(node), + bdk_config_get_int(BDK_CONFIG_DDR_SPEED, node), + (__bdk_dram_is_ddr4(node, 0)) ? "DDR4" : "DDR3", + (__bdk_dram_is_rdimm(node, 0)) ? "RDIMM" : "UDIMM"); + + return info_ptr; +} + + +/** + * Return the highest address currently used by the BDK. This address will + * be about 4MB above the top of the BDK to make sure small growths between the + * call and its use don't cause corruption. Any call to memory allocation can + * change this value. + * + * @return Size of the BDK in bytes + */ +uint64_t bdk_dram_get_top_of_bdk(void) +{ + /* Make sure the start address is higher that the BDK's active range. + * + * As sbrk() returns a node address, mask off the node portion of + * the address to make it a physical offset. Doing this simplifies the + * address checks and calculations which only work with physical offsets. + */ + uint64_t top_of_bdk = (bdk_ptr_to_phys(sbrk(0)) & bdk_build_mask(40)); + uint64_t l2_size = bdk_l2c_get_cache_size_bytes(bdk_numa_master()); + if (top_of_bdk <= l2_size) + { + /* Early BDK code takes care of the first L2 sized area of memory */ + top_of_bdk = l2_size; + } + else + { + /* Give 4MB of extra so the BDK has room to grow */ + top_of_bdk += 4 << 20; + /* Align it on a 64KB boundary */ + top_of_bdk >>= 16; + top_of_bdk <<= 16; + } + return top_of_bdk; +} diff --git a/src/vendorcode/cavium/bdk/libbdk-dram/bdk-dram-size.c b/src/vendorcode/cavium/bdk/libbdk-dram/bdk-dram-size.c new file mode 100644 index 0000000000..122afb2a18 --- /dev/null +++ b/src/vendorcode/cavium/bdk/libbdk-dram/bdk-dram-size.c @@ -0,0 +1,213 @@ +/***********************license start*********************************** +* Copyright (c) 2003-2017 Cavium Inc. (support@cavium.com). All rights +* reserved. +* +* +* Redistribution and use in source and binary forms, with or without +* modification, are permitted provided that the following conditions are +* met: +* +* * Redistributions of source code must retain the above copyright +* notice, this list of conditions and the following disclaimer. +* +* * Redistributions in binary form must reproduce the above +* copyright notice, this list of conditions and the following +* disclaimer in the documentation and/or other materials provided +* with the distribution. +* +* * Neither the name of Cavium Inc. nor the names of +* its contributors may be used to endorse or promote products +* derived from this software without specific prior written +* permission. +* +* This Software, including technical data, may be subject to U.S. export +* control laws, including the U.S. Export Administration Act and its +* associated regulations, and may be subject to export or import +* regulations in other countries. +* +* TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" +* AND WITH ALL FAULTS AND CAVIUM INC. MAKES NO PROMISES, REPRESENTATIONS OR +* WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH RESPECT +* TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY +* REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT +* DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) WARRANTIES +* OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A PARTICULAR +* PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET ENJOYMENT, +* QUIET POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE ENTIRE RISK +* ARISING OUT OF USE OR PERFORMANCE OF THE SOFTWARE LIES WITH YOU. +***********************license end**************************************/ +#include <bdk.h> + +/** + * Return the number of LMC controllers in use + * + * @param node Node to probe + * + * @return 2 or 4 depending on the mode + */ +int __bdk_dram_get_num_lmc(bdk_node_t node) +{ + if (CAVIUM_IS_MODEL(CAVIUM_CN88XX)) + { + BDK_CSR_INIT(lmcx_dll_ctl2, node, BDK_LMCX_DLL_CTL2(2)); // sample LMC2 + return (lmcx_dll_ctl2.s.intf_en) ? 4 : 2; + } + else if (CAVIUM_IS_MODEL(CAVIUM_CN83XX)) + { + BDK_CSR_INIT(lmcx_dll_ctl1, node, BDK_LMCX_DLL_CTL2(1)); // sample LMC1 + return (lmcx_dll_ctl1.s.intf_en) ? 2 : 1; + } + else if (CAVIUM_IS_MODEL(CAVIUM_CN81XX)) + { + return 1; + } + else if (CAVIUM_IS_MODEL(CAVIUM_CN93XX)) + { + BDK_CSR_INIT(lmcx_dll_ctl1, node, BDK_LMCX_DLL_CTL2(2)); + if (lmcx_dll_ctl1.s.intf_en) + return 3; + lmcx_dll_ctl1.u = BDK_CSR_READ(node, BDK_LMCX_DLL_CTL2(1)); + return (lmcx_dll_ctl1.s.intf_en) ? 2 : 1; + } + bdk_error("__bdk_dram_get_num_lmc() needs update for this chip\n"); + return 1; +} + +/** + * Return whether the node/LMC is in DRESET + * + * @param node Node to probe + * @param node LMC to probe + * + * @return 1 or 0 + */ +static int __bdk_dram_is_lmc_in_dreset(bdk_node_t node, int lmc) +{ + BDK_CSR_INIT(lmcx_dll_ctl2, node, BDK_LMCX_DLL_CTL2(lmc)); // can always read this + return (lmcx_dll_ctl2.s.dreset != 0) ? 1 : 0; +} + +/** + * Return a mask of the number of row bits in use + * + * @param node Node to probe + * + */ +uint32_t __bdk_dram_get_row_mask(bdk_node_t node, int lmc) +{ + // PROTECT!!! + if (__bdk_dram_is_lmc_in_dreset(node, lmc)) // check LMCn + return 0; + BDK_CSR_INIT(lmcx_config, node, BDK_LMCX_CONFIG(lmc)); // sample LMCn + int numbits = 14 + lmcx_config.s.pbank_lsb - lmcx_config.s.row_lsb - lmcx_config.s.rank_ena; + return ((1ul << numbits) - 1); +} + +/** + * Return a mask of the number of column bits in use + * + * @param node Node to probe + * + */ +uint32_t __bdk_dram_get_col_mask(bdk_node_t node, int lmc) +{ + // PROTECT!!! + if (__bdk_dram_is_lmc_in_dreset(node, lmc)) // check LMCn + return 0; + BDK_CSR_INIT(lmcx_config, node, BDK_LMCX_CONFIG(lmc)); // sample LMCn + int numbits = 11 + lmcx_config.s.row_lsb - __bdk_dram_get_num_bank_bits(node, lmc); + return ((1ul << numbits) - 1); +} + +/** + * Return the number of bank bits in use + * + * @param node Node to probe + * + */ +// all DDR3, and DDR4 x16 today, use only 3 bank bits; DDR4 x4 and x8 always have 4 bank bits +// NOTE: this will change in the future, when DDR4 x16 devices can come with 16 banks!! FIXME!! +int __bdk_dram_get_num_bank_bits(bdk_node_t node, int lmc) +{ + // PROTECT!!! + if (__bdk_dram_is_lmc_in_dreset(node, lmc)) // check LMCn + return 0; + BDK_CSR_INIT(lmcx_config, node, BDK_LMCX_CONFIG(lmc)); // sample LMCn + int bank_width = (__bdk_dram_is_ddr4(node, lmc) && (lmcx_config.s.bg2_enable)) ? 4 : 3; + return bank_width; +} + +/** + * Return whether the node has DDR3 or DDR4 DRAM + * + * @param node Node to probe + * + * @return 0 (DDR3) or 1 (DDR4) + */ +int __bdk_dram_is_ddr4(bdk_node_t node, int lmc) +{ + // PROTECT!!! + if (__bdk_dram_is_lmc_in_dreset(node, lmc)) // check LMCn + return 0; + if (CAVIUM_IS_MODEL(CAVIUM_CN9XXX)) + return 1; + BDK_CSR_INIT(lmcx_ddr_pll_ctl, node, BDK_LMCX_DDR_PLL_CTL(lmc)); // sample LMCn + return (lmcx_ddr_pll_ctl.cn83xx.ddr4_mode != 0); +} + +/** + * Return whether the node has Registered DIMMs or Unbuffered DIMMs + * + * @param node Node to probe + * + * @return 0 (Unbuffered) or 1 (Registered) + */ +int __bdk_dram_is_rdimm(bdk_node_t node, int lmc) +{ + // PROTECT!!! + if (__bdk_dram_is_lmc_in_dreset(node, lmc)) // check LMCn + return 0; + BDK_CSR_INIT(lmcx_control, node, BDK_LMCX_CONTROL(lmc)); // sample LMCn + return (lmcx_control.s.rdimm_ena != 0); +} + +/** + * Get the amount of DRAM configured for a node. This is read from the LMC + * controller after DRAM is setup. + * + * @param node Node to query + * + * @return Size in megabytes + */ +uint64_t bdk_dram_get_size_mbytes(int node) +{ + if (bdk_is_platform(BDK_PLATFORM_EMULATOR)) + return 2 << 10; /* 2GB is available on t88 and t81 + ** some t83 models have 8gb, but it is too long to init */ + /* Return zero if dram isn't enabled */ + if (!__bdk_is_dram_enabled(node)) + return 0; + + uint64_t memsize = 0; + const int num_dram_controllers = __bdk_dram_get_num_lmc(node); + for (int lmc = 0; lmc < num_dram_controllers; lmc++) + { + if (bdk_is_platform(BDK_PLATFORM_ASIM)) + { + /* Asim doesn't simulate the rank detection, fake 4GB per controller */ + memsize += 4ull << 30; + } + else + { + // PROTECT!!! + if (__bdk_dram_is_lmc_in_dreset(node, lmc)) // check LMCn + return 0; + BDK_CSR_INIT(lmcx_config, node, BDK_LMCX_CONFIG(lmc)); + int num_ranks = bdk_pop(lmcx_config.s.init_status); + uint64_t rank_size = 1ull << (28 + lmcx_config.s.pbank_lsb - lmcx_config.s.rank_ena); + memsize += rank_size * num_ranks; + } + } + return memsize >> 20; +} + diff --git a/src/vendorcode/cavium/bdk/libbdk-dram/bdk-dram-test-addrbus.c b/src/vendorcode/cavium/bdk/libbdk-dram/bdk-dram-test-addrbus.c new file mode 100644 index 0000000000..9fe8570454 --- /dev/null +++ b/src/vendorcode/cavium/bdk/libbdk-dram/bdk-dram-test-addrbus.c @@ -0,0 +1,115 @@ +/***********************license start*********************************** +* Copyright (c) 2003-2017 Cavium Inc. (support@cavium.com). All rights +* reserved. +* +* +* Redistribution and use in source and binary forms, with or without +* modification, are permitted provided that the following conditions are +* met: +* +* * Redistributions of source code must retain the above copyright +* notice, this list of conditions and the following disclaimer. +* +* * Redistributions in binary form must reproduce the above +* copyright notice, this list of conditions and the following +* disclaimer in the documentation and/or other materials provided +* with the distribution. +* +* * Neither the name of Cavium Inc. nor the names of +* its contributors may be used to endorse or promote products +* derived from this software without specific prior written +* permission. +* +* This Software, including technical data, may be subject to U.S. export +* control laws, including the U.S. Export Administration Act and its +* associated regulations, and may be subject to export or import +* regulations in other countries. +* +* TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" +* AND WITH ALL FAULTS AND CAVIUM INC. MAKES NO PROMISES, REPRESENTATIONS OR +* WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH RESPECT +* TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY +* REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT +* DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) WARRANTIES +* OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A PARTICULAR +* PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET ENJOYMENT, +* QUIET POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE ENTIRE RISK +* ARISING OUT OF USE OR PERFORMANCE OF THE SOFTWARE LIES WITH YOU. +***********************license end**************************************/ +#include "bdk.h" + +/* Used for all memory reads/writes related to the test */ +#define READ64(address) __bdk_dram_read64(address) +#define WRITE64(address, data) __bdk_dram_write64(address, data) + +/** + * Address bus test. This test writes a single value to each power of two in the + * area, looking for false aliases that would be created by address lines being + * shorted or tied together. + * + * @param area + * @param max_address + * @param bursts + * + * @return + */ +int __bdk_dram_test_mem_address_bus(uint64_t area, uint64_t max_address, int bursts) +{ + int failures = 0; + + /* Clear our work area. Checking for aliases later could get false + positives if it matched stale data */ + void *ptr = (area) ? bdk_phys_to_ptr(area) : NULL; + bdk_zero_memory(ptr, max_address - area); + __bdk_dram_flush_to_mem_range(area, max_address); + + /* Each time we write, we'll write this pattern xored the address it is + written too */ + uint64_t pattern = 0x0fedcba987654321; + + /* Walk through the region incrementing our offset by a power of two. The + first few writes will be to the same cache line (offset 0x8, 0x10, 0x20, + and 0x40. Offset 0x80 and beyond will be to different cache lines */ + uint64_t offset = 0x8; + while (area + offset < max_address) + { + uint64_t address = area + offset; + /* Write one location with pattern xor address */ + uint64_t p = pattern ^ address; + WRITE64(address, p); + __bdk_dram_flush_to_mem(address); + offset <<= 1; + } + + /* Read all of the area to make sure no other locations were written */ + uint64_t a = area; + offset = 0x8; + uint64_t next_write = area + offset; + while (a < max_address) + { + if (a + 256 < max_address) + BDK_PREFETCH(a + 256, 0); + for (int i=0; i<16; i++) + { + uint64_t data = READ64(a); + uint64_t correct; + if (a == next_write) + { + correct = pattern ^ next_write; + offset <<= 1; + next_write = area + offset; + } + else + correct = 0; + if (bdk_unlikely(data != correct)) + { + failures++; + __bdk_dram_report_error(a, data, correct, 0, -1); + } + a += 8; + } + } + + return failures; +} + diff --git a/src/vendorcode/cavium/bdk/libbdk-dram/bdk-dram-test-databus.c b/src/vendorcode/cavium/bdk/libbdk-dram/bdk-dram-test-databus.c new file mode 100644 index 0000000000..c3fa1ffd8d --- /dev/null +++ b/src/vendorcode/cavium/bdk/libbdk-dram/bdk-dram-test-databus.c @@ -0,0 +1,252 @@ +/***********************license start*********************************** +* Copyright (c) 2003-2017 Cavium Inc. (support@cavium.com). All rights +* reserved. +* +* +* Redistribution and use in source and binary forms, with or without +* modification, are permitted provided that the following conditions are +* met: +* +* * Redistributions of source code must retain the above copyright +* notice, this list of conditions and the following disclaimer. +* +* * Redistributions in binary form must reproduce the above +* copyright notice, this list of conditions and the following +* disclaimer in the documentation and/or other materials provided +* with the distribution. +* +* * Neither the name of Cavium Inc. nor the names of +* its contributors may be used to endorse or promote products +* derived from this software without specific prior written +* permission. +* +* This Software, including technical data, may be subject to U.S. export +* control laws, including the U.S. Export Administration Act and its +* associated regulations, and may be subject to export or import +* regulations in other countries. +* +* TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" +* AND WITH ALL FAULTS AND CAVIUM INC. MAKES NO PROMISES, REPRESENTATIONS OR +* WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH RESPECT +* TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY +* REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT +* DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) WARRANTIES +* OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A PARTICULAR +* PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET ENJOYMENT, +* QUIET POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE ENTIRE RISK +* ARISING OUT OF USE OR PERFORMANCE OF THE SOFTWARE LIES WITH YOU. +***********************license end**************************************/ +#include "bdk.h" + +/* Used for all memory reads/writes related to the test */ +#define READ64(address) __bdk_dram_read64(address) +#define WRITE64(address, data) __bdk_dram_write64(address, data) + +/* Build a 64bit mask out of a single hex digit */ +#define REPEAT2(v) ((((uint64_t)v) << 4) | ((uint64_t)v)) +#define REPEAT4(v) ((REPEAT2(v) << 8) | REPEAT2(v)) +#define REPEAT8(v) ((REPEAT4(v) << 16) | REPEAT4(v)) +#define REPEAT16(v) ((REPEAT8(v) << 32) | REPEAT8(v)) + +/** + * Read memory and check that the data bus pattern is present. The pattern is a + * sequence if 16 dwords created from the 16 hex digits repeated in each word. + * + * @param address Physical address to read. This must be cache line aligned. + * @param bursts Number of time to repeat the read test to verify stability + * + * @return Number of errors, zero means success + */ +static int read_data_bus_burst(uint64_t address, int bursts) +{ + int failures = 0; + + /* Loop over the burst so people using a scope have time to capture + traces */ + for (int burst = 0; burst < bursts; burst++) + { + /* Invalidate all caches so we must read from DRAM */ + __bdk_dram_flush_to_mem(address); + BDK_DCACHE_INVALIDATE; + + for (uint64_t digit = 0; digit < 16; digit++) + { + uint64_t a = address + digit * 8; + uint64_t data = READ64(a); + uint64_t correct = REPEAT16(digit); + if (data != correct) + { + failures++; + __bdk_dram_report_error(a, data, correct, burst, -1); + } + } + } + return failures; +} + +/** + * Write memory with a data bus pattern and check that it can be read correctly. + * The pattern is a sequence if 16 dwords created from the 16 hex digits repeated + * in each word. + * + * @param address Physical address to write. This must be cache line aligned. 128 bytes will be + * written starting at this address. + * @param bursts Number of time to repeat the write+read test to verify stability + * + * @return Number of errors, zero means success + */ +static int write_data_bus_burst(uint64_t address, int bursts) +{ + BDK_TRACE(DRAM_TEST, "[0x%016lx:0x%016lx] Writing incrementing digits\n", + address, address + 127); + /* Loop over the burst so people using a scope have time to capture + traces */ + for (int burst = 0; burst < bursts; burst++) + { + /* Fill a cache line with an incrementing pattern. Each nibble + in the 64bit word increments from 0 to 0xf */ + for (uint64_t digit = 0; digit < 16; digit++) + WRITE64(address + digit * 8, REPEAT16(digit)); + /* Force the cache line out to memory */ + __bdk_dram_flush_to_mem(address); + } + return read_data_bus_burst(address, bursts); +} + +/** + * Read back the pattern written by write_data_bus_walk() nad + * make sure it was stored properly. + * + * @param address Physical address to read. This must be cache line aligned. + * @param bursts Number of time to repeat the read test to verify stability + * @param pattern Pattern basis for writes. See + * write_data_bus_walk() + * + * @return Number of errors, zero means success + */ +static int read_data_bus_walk(uint64_t address, int burst, uint64_t pattern) +{ + int failures = 0; + + /* Invalidate all caches so we must readfrom DRAM */ + __bdk_dram_flush_to_mem(address); + BDK_DCACHE_INVALIDATE; + + uint64_t correct = pattern; + for (uint64_t word = 0; word < 16; word++) + { + uint64_t a = address + word * 8; + uint64_t data = READ64(a); + if (data != correct) + { + failures++; + __bdk_dram_report_error(a, data, correct, burst, -1); + } + uint64_t tmp = correct >> 63; /* Save top bit */ + correct <<= 1; /* Shift left one bit */ + correct |= tmp; /* Restore the top bit as bit 0 */ + } + + return failures; +} + +/** + * Write a pattern to a cache line, rotating it one bit for each DWORD. Read back + * the pattern and make sure it was stored properly. The input pattern is rotated + * left by one bit for each DWORD written. + * + * @param address Physical address to write. This must be cache line aligned. 128 bytes will be + * written starting at this address. + * @param bursts Number of time to repeat the write+read test to verify stability + * @param pattern Pattern basis + * + * @return Number of errors, zero means success + */ +static void write_data_bus_walk(uint64_t address, int burst, uint64_t pattern) +{ + BDK_TRACE(DRAM_TEST, "[0x%016lx:0x%016lx] Writing walking pattern 0x%016lx\n", + address, address + 127, pattern); + + uint64_t a = address; + uint64_t d = pattern; + + /* Fill a cache line with pattern. Each 64bit work will have the + pattern rotated left one bit */ + for (uint64_t word = 0; word < 16; word++) + { + WRITE64(a, d); + a += 8; + uint64_t tmp = d >> 63; /* Save top bit */ + d <<= 1; /* Shift left one bit */ + d |= tmp; /* Restore the top bit as bit 0 */ + } + /* Force the cache line out to memory */ + __bdk_dram_flush_to_mem(address); +} + +/** + * The goal of these tests are to toggle every DDR data pin, one at a time or in + * related groups, to isolate any short circuits between the data pins or open + * circuits where the pin is not connected to the DDR memory. A board which fails + * one of these tests has severe problems and will not be able to run any of the + * later test patterns. + * + * @param start_address + * Physical address of a cache line to + * use for the test. Only this cache line is + * written. + * @param end_address + * Top end of the address range. Currently unused + * @param bursts Number of time to repeats writes+reads to insure stability + * + * @return Number of errors, zero means success + */ +int __bdk_dram_test_mem_data_bus(uint64_t start_address, uint64_t end_address, int bursts) +{ + int failures = 0; + + /* Incrementing pattern: 0x0 - 0xf in each nibble */ + failures += write_data_bus_burst(start_address, bursts); + + /* Walking ones. Run with 1, 2, and 3 bits walking */ + for (int bits = 1; bits <= 3; bits++) + { + for (int burst = 0; burst < bursts; burst++) + { + /* Each write_data_bus_walk() call write 16 dword, so step by 16 */ + for (int i = 0; i < 64; i += 16) + { + uint64_t pattern = bdk_build_mask(bits) << i; + write_data_bus_walk(start_address + i*8, burst, pattern); + } + /* Each read_data_bus_walk() call write 16 dword, so step by 16 */ + for (int i = 0; i < 64; i += 16) + { + uint64_t pattern = bdk_build_mask(bits) << i; + failures += read_data_bus_walk(start_address + i*8, burst, pattern); + } + } + } + + /* Walking zeros. Run with 1, 2, and 3 bits walking */ + for (int bits = 1; bits <= 3; bits++) + { + for (int burst = 0; burst < bursts; burst++) + { + /* Each write_data_bus_walk() call write 16 dword, so step by 16 */ + for (int i = 0; i < 64; i += 16) + { + uint64_t pattern = ~(bdk_build_mask(bits) << i); + write_data_bus_walk(start_address + i*8, burst, pattern); + } + /* Each read_data_bus_walk() call write 16 dword, so step by 16 */ + for (int i = 0; i < 64; i += 16) + { + uint64_t pattern = ~(bdk_build_mask(bits) << i); + failures += read_data_bus_walk(start_address + i*8, burst, pattern); + } + } + } + return failures; +} + diff --git a/src/vendorcode/cavium/bdk/libbdk-dram/bdk-dram-test-fastscan.c b/src/vendorcode/cavium/bdk/libbdk-dram/bdk-dram-test-fastscan.c new file mode 100644 index 0000000000..46e205dd80 --- /dev/null +++ b/src/vendorcode/cavium/bdk/libbdk-dram/bdk-dram-test-fastscan.c @@ -0,0 +1,103 @@ +/***********************license start*********************************** +* Copyright (c) 2003-2017 Cavium Inc. (support@cavium.com). All rights +* reserved. +* +* +* Redistribution and use in source and binary forms, with or without +* modification, are permitted provided that the following conditions are +* met: +* +* * Redistributions of source code must retain the above copyright +* notice, this list of conditions and the following disclaimer. +* +* * Redistributions in binary form must reproduce the above +* copyright notice, this list of conditions and the following +* disclaimer in the documentation and/or other materials provided +* with the distribution. +* +* * Neither the name of Cavium Inc. nor the names of +* its contributors may be used to endorse or promote products +* derived from this software without specific prior written +* permission. +* +* This Software, including technical data, may be subject to U.S. export +* control laws, including the U.S. Export Administration Act and its +* associated regulations, and may be subject to export or import +* regulations in other countries. +* +* TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" +* AND WITH ALL FAULTS AND CAVIUM INC. MAKES NO PROMISES, REPRESENTATIONS OR +* WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH RESPECT +* TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY +* REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT +* DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) WARRANTIES +* OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A PARTICULAR +* PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET ENJOYMENT, +* QUIET POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE ENTIRE RISK +* ARISING OUT OF USE OR PERFORMANCE OF THE SOFTWARE LIES WITH YOU. +***********************license end**************************************/ +#include "bdk.h" + +/* Used for all memory reads/writes related to the test */ +#define READ64(address) __bdk_dram_read64(address) +#define WRITE64(address, data) __bdk_dram_write64(address, data) + +/** + * Fast scan test. This test is meant to find gross errors caused by read/write + * level failing on a single rank or dimm. The idea is to scan through all of + * memory in large steps. The large steps hit each rank multiple times, but not + * every byte. If the whole rank has errors, his should find it quickly. This test + * is suitable for an alive test during early boot. + * + * @param area Starting physical address + * @param max_address + * Ending physical address, exclusive + * @param bursts Burst to run + * + * @return Number of errors + */ +int __bdk_dram_test_fast_scan(uint64_t area, uint64_t max_address, int bursts) +{ + int failures = 0; + const uint64_t step = 0x10008; /* The 8 is so we walk through cache lines too */ + const uint64_t pattern1 = 0xaaaaaaaaaaaaaaaa; + const uint64_t pattern2 = 0x5555555555555555; + + /* Walk through the region incrementing our offset by STEP */ + uint64_t a = area; + while (a + 16 <= max_address) + { + WRITE64(a, pattern1); + WRITE64(a+8, pattern2); + __bdk_dram_flush_to_mem_range(a, a + 16); + a += step; + } + + /* Read back, checking the writes */ + a = area; + while (a + 16 <= max_address) + { + /* Prefetch 3 ahead for better performance */ + uint64_t pre = a + step * 2; + if (pre + 16 < max_address) + BDK_PREFETCH(pre, 0); + /* Check pattern 1 */ + uint64_t data1 = READ64(a); + if (bdk_unlikely(data1 != pattern1)) + { + failures++; + __bdk_dram_report_error(a, data1, pattern1, 0, -1); + } + /* Check pattern 2 */ + uint64_t data2 = READ64(a+8); + if (bdk_unlikely(data2 != pattern2)) + { + failures++; + __bdk_dram_report_error(a+8, data2, pattern2, 0, -1); + } + a += step; + } + + return failures; +} + diff --git a/src/vendorcode/cavium/bdk/libbdk-dram/bdk-dram-test-patfil.c b/src/vendorcode/cavium/bdk/libbdk-dram/bdk-dram-test-patfil.c new file mode 100644 index 0000000000..e6c4b57721 --- /dev/null +++ b/src/vendorcode/cavium/bdk/libbdk-dram/bdk-dram-test-patfil.c @@ -0,0 +1,829 @@ +/***********************license start*********************************** +* Copyright (c) 2003-2017 Cavium Inc. (support@cavium.com). All rights +* reserved. +* +* +* Redistribution and use in source and binary forms, with or without +* modification, are permitted provided that the following conditions are +* met: +* +* * Redistributions of source code must retain the above copyright +* notice, this list of conditions and the following disclaimer. +* +* * Redistributions in binary form must reproduce the above +* copyright notice, this list of conditions and the following +* disclaimer in the documentation and/or other materials provided +* with the distribution. +* +* * Neither the name of Cavium Inc. nor the names of +* its contributors may be used to endorse or promote products +* derived from this software without specific prior written +* permission. +* +* This Software, including technical data, may be subject to U.S. export +* control laws, including the U.S. Export Administration Act and its +* associated regulations, and may be subject to export or import +* regulations in other countries. +* +* TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" +* AND WITH ALL FAULTS AND CAVIUM INC. MAKES NO PROMISES, REPRESENTATIONS OR +* WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH RESPECT +* TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY +* REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT +* DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) WARRANTIES +* OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A PARTICULAR +* PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET ENJOYMENT, +* QUIET POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE ENTIRE RISK +* ARISING OUT OF USE OR PERFORMANCE OF THE SOFTWARE LIES WITH YOU. +***********************license end**************************************/ +#include "bdk.h" + +// choose prediction-based algorithms for mem_xor and mem_rows tests +#define USE_PREDICTION_CODE_VERSIONS 1 // change to 0 to go back to the original versions + +/* Used for all memory reads/writes related to the test */ +#define READ64(address) __bdk_dram_read64(address) +#define WRITE64(address, data) __bdk_dram_write64(address, data) + +/** + * Fill an memory area with the address of each 64-bit word in the area. + * Reread to confirm the pattern. + * + * @param area Start of the physical memory area + * @param max_address + * End of the physical memory area (exclusive) + * @param bursts Number of time to repeat the test over the entire area + * + * @return Number of errors, zero on success + */ +int __bdk_dram_test_mem_self_addr(uint64_t area, uint64_t max_address, int bursts) +{ + int failures = 0; + + for (int burst = 0; burst < bursts; burst++) + { + /* Write the pattern to memory. Each location receives the address + * of the location. + */ + for (uint64_t address = area; address < max_address; address+=8) + WRITE64(address, address); + __bdk_dram_flush_to_mem_range(area, max_address); + BDK_DCACHE_INVALIDATE; + + /* Read by ascending address the written memory and confirm that it + * has the expected data pattern. + */ + for (uint64_t address = area; address < max_address; ) + { + if (address + 256 < max_address) + BDK_PREFETCH(address + 256, 0); + for (int i=0; i<16; i++) + { + uint64_t data = READ64(address); + if (bdk_unlikely(data != address)) + failures += __bdk_dram_retry_failure(burst, address, data, address); + address += 8; + } + } + __bdk_dram_flush_to_mem_range(area, max_address); + BDK_DCACHE_INVALIDATE; + + /* Read by descending address the written memory and confirm that it + * has the expected data pattern. + */ + uint64_t end = max_address - sizeof(uint64_t); + for (uint64_t address = end; address >= area; ) + { + if (address - 256 >= area) + BDK_PREFETCH(address - 256, 0); + for (int i=0; i<16; i++) + { + uint64_t data = READ64(address); + if (bdk_unlikely(data != address)) + failures += __bdk_dram_retry_failure(burst, address, data, address); + address -= 8; + } + } + __bdk_dram_flush_to_mem_range(area, max_address); + BDK_DCACHE_INVALIDATE; + + /* Read from random addresses within the memory area. + */ + uint64_t probes = (max_address - area) / 128; + uint64_t address_ahead1 = area; + uint64_t address_ahead2 = area; + for (uint64_t i = 0; i < probes; i++) + { + /* Create a pipeline of prefetches: + address = address read this loop + address_ahead1 = prefetch started last loop + address_ahead2 = prefetch started this loop */ + uint64_t address = address_ahead1; + address_ahead1 = address_ahead2; + address_ahead2 = bdk_rng_get_random64() % (max_address - area); + address_ahead2 += area; + address_ahead2 &= -8; + BDK_PREFETCH(address_ahead2, 0); + + uint64_t data = READ64(address); + if (bdk_unlikely(data != address)) + failures += __bdk_dram_retry_failure(burst, address, data, address); + } + } + return failures; +} + +/** + * Write "pattern" and its compliment to memory and verify it was written + * properly. Memory will be filled with DWORDs pattern, ~pattern, pattern, + * ~pattern, ... + * + * @param area Start physical address of memory + * @param max_address + * End of physical memory region + * @param pattern Pattern to write + * @param passes Number of time to repeat the test + * + * @return Number of errors, zero on success + */ +static uint32_t test_mem_pattern(uint64_t area, uint64_t max_address, uint64_t pattern, + int passes) +{ + int failures = 0; + + for (int pass = 0; pass < passes; pass++) + { + if (pass & 0x1) + pattern = ~pattern; + + for (uint64_t address = area; address < max_address; address += 8) + WRITE64(address, pattern); + __bdk_dram_flush_to_mem_range(area, max_address); + BDK_DCACHE_INVALIDATE; + + /* Read the written memory and confirm that it has the expected + * data pattern. + */ + uint64_t address = area; + while (address < max_address) + { + if (address + 256 < max_address) + BDK_PREFETCH(address + 256, 0); + for (int i=0; i<16; i++) + { + uint64_t data = READ64(address); + if (bdk_unlikely(data != pattern)) + failures += __bdk_dram_retry_failure(pass, address, data, pattern); + address += 8; + } + } + } + return failures; +} + +/** + * Walking zero written to memory, left shift + * + * @param area Start of the physical memory area + * @param max_address + * End of the physical memory area + * @param bursts Number of time to repeat the test over the entire area + * + * @return Number of errors, zero on success + */ +int __bdk_dram_test_mem_leftwalk0(uint64_t area, uint64_t max_address, int bursts) +{ + int failures = 0; + for (int burst = 0; burst < bursts; burst++) + { + for (uint64_t pattern = 1; pattern != 0; pattern = pattern << 1) + failures += test_mem_pattern(area, max_address, ~pattern, 1); + } + return failures; +} + +/** + * Walking one written to memory, left shift + * + * @param area Start of the physical memory area + * @param max_address + * End of the physical memory area + * @param bursts Number of time to repeat the test over the entire area + * + * @return Number of errors, zero on success + */ +int __bdk_dram_test_mem_leftwalk1(uint64_t area, uint64_t max_address, int bursts) +{ + int failures = 0; + for (int burst = 0; burst < bursts; burst++) + { + for (uint64_t pattern = 1; pattern != 0; pattern = pattern << 1) + failures += test_mem_pattern(area, max_address, pattern, 1); + } + return failures; +} + +/** + * Walking zero written to memory, right shift + * + * @param area Start of the physical memory area + * @param max_address + * End of the physical memory area + * @param bursts Number of time to repeat the test over the entire area + * + * @return Number of errors, zero on success + */ +int __bdk_dram_test_mem_rightwalk0(uint64_t area, uint64_t max_address, int bursts) +{ + int failures = 0; + for (int burst = 0; burst < bursts; burst++) + { + for (uint64_t pattern = 1ull << 63; pattern != 0; pattern = pattern >> 1) + failures += test_mem_pattern(area, max_address, ~pattern, 1); + } + return failures; +} + +/** + * Walking one written to memory, right shift + * + * @param area Start of the physical memory area + * @param max_address + * End of the physical memory area + * @param bursts Number of time to repeat the test over the entire area + * + * @return Number of errors, zero on success + */ +int __bdk_dram_test_mem_rightwalk1(uint64_t area, uint64_t max_address, int bursts) +{ + int failures = 0; + for (int burst = 0; burst < bursts; burst++) + { + for (uint64_t pattern = 1ull<<63; pattern != 0; pattern = pattern >> 1) + failures += test_mem_pattern(area, max_address, pattern, 1); + } + return failures; +} + +/** + * Apply the March C- testing algorithm to the given memory area. + * 1) Write "pattern" to memory. + * 2) Verify "pattern" and write "~pattern". + * 3) Verify "~pattern" and write "pattern". + * 4) Verify "pattern" and write "~pattern". + * 5) Verify "~pattern" and write "pattern". + * 6) Verify "pattern". + * + * @param area Start of the physical memory area + * @param max_address + * End of the physical memory area + * @param pattern + * + * @return Number of errors, zero on success + */ +static int test_mem_march_c(uint64_t area, uint64_t max_address, uint64_t pattern) +{ + int failures = 0; + + /* Pass 1 ascending addresses, fill memory with pattern. */ + BDK_TRACE(DRAM_TEST, " [0x%016lx:0x%016lx] Phase1, address incrementing, pattern 0x%016lx\n", area, max_address-1, pattern); + for (uint64_t address = area; address < max_address; address += 8) + WRITE64(address, pattern); + + __bdk_dram_flush_to_mem_range(area, max_address); + BDK_DCACHE_INVALIDATE; + + /* Pass 2: ascending addresses, read pattern and write ~pattern */ + BDK_TRACE(DRAM_TEST, " [0x%016lx:0x%016lx] Phase2, address incrementing, pattern 0x%016lx\n", area, max_address-1, ~pattern); + for (uint64_t address = area; address < max_address; address += 8) + { + uint64_t data = READ64(address); + if (bdk_unlikely(data != pattern)) + failures += __bdk_dram_retry_failure(1, address, data, pattern); + WRITE64(address, ~pattern); + } + + __bdk_dram_flush_to_mem_range(area, max_address); + BDK_DCACHE_INVALIDATE; + + /* Pass 3: ascending addresses, read ~pattern and write pattern. */ + BDK_TRACE(DRAM_TEST, " [0x%016lx:0x%016lx] Phase3, address incrementing, pattern 0x%016lx\n", area, max_address-1, pattern); + for (uint64_t address = area; address < max_address; address += 8) + { + uint64_t data = READ64(address); + if (bdk_unlikely(data != ~pattern)) + failures += __bdk_dram_retry_failure(1, address, data, ~pattern); + WRITE64(address, pattern); + } + + __bdk_dram_flush_to_mem_range(area, max_address); + BDK_DCACHE_INVALIDATE; + + /* Pass 4: descending addresses, read pattern and write ~pattern. */ + BDK_TRACE(DRAM_TEST, " [0x%016lx:0x%016lx] Phase4, address decrementing, pattern 0x%016lx\n", area, max_address-1, ~pattern); + uint64_t end = max_address - sizeof(uint64_t); + for (uint64_t address = end; address >= area; address -= 8) + { + uint64_t data = READ64(address); + if (bdk_unlikely(data != pattern)) + failures += __bdk_dram_retry_failure(1, address, data, pattern); + WRITE64(address, ~pattern); + } + + __bdk_dram_flush_to_mem_range(area, max_address); + BDK_DCACHE_INVALIDATE; + + /* Pass 5: descending addresses, read ~pattern and write pattern. */ + BDK_TRACE(DRAM_TEST, " [0x%016lx:0x%016lx] Phase5, address decrementing, pattern 0x%016lx\n", area, max_address-1, pattern); + for (uint64_t address = end; address >= area; address -= 8) + { + uint64_t data = READ64(address); + if (bdk_unlikely(data != ~pattern)) + failures += __bdk_dram_retry_failure(1, address, data, ~pattern); + WRITE64(address, pattern); + } + + __bdk_dram_flush_to_mem_range(area, max_address); + BDK_DCACHE_INVALIDATE; + + /* Pass 6: ascending addresses, read pattern. */ + BDK_TRACE(DRAM_TEST, " [0x%016lx:0x%016lx] Phase6, address incrementing\n", area, max_address-1); + for (uint64_t address = area; address < max_address; address += 8) + { + uint64_t data = READ64(address); + if (bdk_unlikely(data != pattern)) + failures += __bdk_dram_retry_failure(1, address, data, pattern); + } + + return failures; +} + +/** + * Use test_mem_march_c() with a all ones pattern + * + * @param area Start of the physical memory area + * @param max_address + * End of the physical memory area + * @param bursts Number of time to repeat the test over the entire area + * + * @return Number of errors, zero on success + */ +int __bdk_dram_test_mem_solid(uint64_t area, uint64_t max_address, int bursts) +{ + int failures = 0; + for (int burst = 0; burst < bursts; burst++) + failures += test_mem_march_c(area, max_address, -1); + return failures; +} + +/** + * Use test_mem_march_c() with a 0x55 pattern + * + * @param area Start of the physical memory area + * @param max_address + * End of the physical memory area + * @param bursts Number of time to repeat the test over the entire area + * + * @return Number of errors, zero on success + */ +int __bdk_dram_test_mem_checkerboard(uint64_t area, uint64_t max_address, int bursts) +{ + int failures = 0; + for (int burst = 0; burst < bursts; burst++) + failures += test_mem_march_c(area, max_address, 0x5555555555555555L); + return failures; +} + +/** + * Write a pseudo random pattern to memory and verify it + * + * @param area Start of the physical memory area + * @param max_address + * End of the physical memory area + * @param bursts Number of time to repeat the test over the entire area + * + * @return Number of errors, zero on success + */ +int __bdk_dram_test_mem_random(uint64_t area, uint64_t max_address, int bursts) +{ + /* This constant is used to increment the pattern after every DWORD. This + makes only the first DWORD truly random, but saves us processing + power generating the random values */ + const uint64_t INC = 0x1010101010101010ULL; + + int failures = 0; + for (int burst = 0; burst < bursts; burst++) + { + const uint64_t init_pattern = bdk_rng_get_random64(); + uint64_t pattern = init_pattern; + + /* Write the pattern to memory. Each location receives the address + * of the location. A second write pass is needed to force all of + * the cached memory out to the DDR. + */ + for (uint64_t address = area; address < max_address; address += 8) + { + WRITE64(address, pattern); + pattern += INC; + } + + __bdk_dram_flush_to_mem_range(area, max_address); + BDK_DCACHE_INVALIDATE; + + /* Read the written memory and confirm that it has the expected + * data pattern. + */ + pattern = init_pattern; + for (uint64_t address = area; address < max_address; address += 8) + { + uint64_t data = READ64(address); + if (bdk_unlikely(data != pattern)) + failures += __bdk_dram_retry_failure(burst, address, data, pattern); + pattern += INC; + } + } + return failures; +} + +#if !USE_PREDICTION_CODE_VERSIONS +/** + * test_mem_xor + * + * @param area Start of the physical memory area + * @param max_address + * End of the physical memory area + * @param bursts Number of time to repeat the test over the entire area + * + * @return Number of errors, zero on success + */ +int __bdk_dram_test_mem_xor(uint64_t area, uint64_t max_address, int bursts) +{ + int failures = 0; + + uint64_t extent = max_address - area; + uint64_t count = (extent / sizeof(uint64_t)) / 2; + + /* Fill both halves of the memory area with identical randomized data. + */ + uint64_t address1 = area; + uint64_t address2 = area + count * sizeof(uint64_t); + + uint64_t pattern = bdk_rng_get_random64(); + + for (uint64_t j = 0; j < count; j++) + { + uint64_t p = pattern * address1; + WRITE64(address1, p); + WRITE64(address2, p); + address1 += 8; + address2 += 8; + } + __bdk_dram_flush_to_mem_range(area, max_address); + BDK_DCACHE_INVALIDATE; + + /* Make a series of passes over the memory areas. */ + for (int burst = 0; burst < bursts; burst++) + { + /* XOR the data with a random value, applying the change to both + * memory areas. + */ + address1 = area; + address2 = area + count * sizeof(uint64_t); + + pattern = bdk_rng_get_random64(); + + for (uint64_t j = 0; j < count; j++) + { + if ((address1 & BDK_CACHE_LINE_MASK) == 0) + BDK_PREFETCH(address1, BDK_CACHE_LINE_SIZE); + if ((address2 & BDK_CACHE_LINE_MASK) == 0) + BDK_PREFETCH(address2, BDK_CACHE_LINE_SIZE); + WRITE64(address1, READ64(address1) ^ pattern); + WRITE64(address2, READ64(address2) ^ pattern); + address1 += 8; + address2 += 8; + } + + __bdk_dram_flush_to_mem_range(area, max_address); + BDK_DCACHE_INVALIDATE; + + /* Look for differences in the areas. If there is a mismatch, reset + * both memory locations with the same pattern. Failing to do so + * means that on all subsequent passes the pair of locations remain + * out of sync giving spurious errors. + */ + address1 = area; + address2 = area + count * sizeof(uint64_t); + for (uint64_t j = 0; j < count; j++) + { + if ((address1 & BDK_CACHE_LINE_MASK) == 0) + BDK_PREFETCH(address1, BDK_CACHE_LINE_SIZE); + if ((address2 & BDK_CACHE_LINE_MASK) == 0) + BDK_PREFETCH(address2, BDK_CACHE_LINE_SIZE); + uint64_t d1 = READ64(address1); + uint64_t d2 = READ64(address2); + if (bdk_unlikely(d1 != d2)) + { + failures += __bdk_dram_retry_failure2(burst, address1, d1, address2, d2); + + // Synchronize the two areas, adjusting for the error. + WRITE64(address1, d2); + WRITE64(address2, d2); + } + address1 += 8; + address2 += 8; + } + } + return failures; +} + +/** + * test_mem_rows + * + * Write a pattern of alternating 64-bit words of all one bits and then all 0 + * bits. This pattern generates the maximum amount of simultaneous switching + * activity on the memory channels. Each pass flips the pattern with words + * going from all ones to all zeros and vice versa. + * + * @param area Start of the physical memory area + * @param max_address + * End of the physical memory area + * @param bursts Number of times to repeat the test over the entire area + * + * @return Number of errors, zero on success + */ +int __bdk_dram_test_mem_rows(uint64_t area, uint64_t max_address, int bursts) +{ + int failures = 0; + uint64_t pattern = 0x0; + uint64_t extent = (max_address - area); + uint64_t count = (extent / 2) / sizeof(uint64_t); // in terms of 64bit words + + /* Fill both halves of the memory area with identical data pattern. Odd + * address 64-bit words get the pattern, while even address words get the + * inverted pattern. + */ + uint64_t address1 = area; + uint64_t address2 = area + count * sizeof(uint64_t); + + for (uint64_t j = 0; j < (count / 2); j++) + { + WRITE64(address1, pattern); + WRITE64(address2, pattern); + address1 += 8; + address2 += 8; + WRITE64(address1, ~pattern); + WRITE64(address2, ~pattern); + address1 += 8; + address2 += 8; + } + __bdk_dram_flush_to_mem_range(area, max_address); + BDK_DCACHE_INVALIDATE; + + /* Make a series of passes over the memory areas. */ + for (int burst = 0; burst < bursts; burst++) + { + /* Invert the data, applying the change to both memory areas. Thus on + * alternate passes, the data flips from 0 to 1 and vice versa. + */ + address1 = area; + address2 = area + count * sizeof(uint64_t); + for (uint64_t j = 0; j < count; j++) + { + WRITE64(address1, ~READ64(address1)); + WRITE64(address2, ~READ64(address2)); + address1 += 8; + address2 += 8; + } + __bdk_dram_flush_to_mem_range(area, max_address); + BDK_DCACHE_INVALIDATE; + + /* Look for differences in the areas. If there is a mismatch, reset + * both memory locations with the same pattern. Failing to do so + * means that on all subsequent passes the pair of locations remain + * out of sync giving spurious errors. + */ + address1 = area; + address2 = area + count * sizeof(uint64_t); + for (uint64_t j = 0; j < count; j++) + { + uint64_t d1 = READ64(address1); + uint64_t d2 = READ64(address2); + if (bdk_unlikely(d1 != d2)) + { + failures += __bdk_dram_retry_failure2(burst, address1, d1, address2, d2); + + // Synchronize the two areas, adjusting for the error. + WRITE64(address1, d2); + WRITE64(address2, d2); + } + address1 += 8; + address2 += 8; + } + } + return failures; +} +#endif /* !USE_PREDICTION_CODE_VERSIONS */ + +#if USE_PREDICTION_CODE_VERSIONS +//////////////////////////// this is the new code... + +int __bdk_dram_test_mem_xor(uint64_t area, uint64_t max_address, int bursts) +{ + int failures = 0; + int burst; + + uint64_t extent = max_address - area; + uint64_t count = (extent / sizeof(uint64_t)) / 2; + uint64_t offset = count * sizeof(uint64_t); + uint64_t area2 = area + offset; + + /* Fill both halves of the memory area with identical randomized data. + */ + uint64_t address1 = area; + + uint64_t pattern1 = bdk_rng_get_random64(); + uint64_t pattern2 = 0; + uint64_t this_pattern; + + uint64_t p; + uint64_t d1, d2; + + // move the multiplies outside the loop + uint64_t pbase = address1 * pattern1; + uint64_t pincr = 8 * pattern1; + uint64_t ppred; + + p = pbase; + while (address1 < area2) + { + WRITE64(address1 , p); + WRITE64(address1 + offset, p); + address1 += 8; + p += pincr; + } + __bdk_dram_flush_to_mem_range(area, max_address); + BDK_DCACHE_INVALIDATE; + + /* Make a series of passes over the memory areas. */ + for (burst = 0; burst < bursts; burst++) + { + /* XOR the data with a random value, applying the change to both + * memory areas. + */ + address1 = area; + + this_pattern = bdk_rng_get_random64(); + pattern2 ^= this_pattern; + + while (address1 < area2) + { +#if 1 + if ((address1 & BDK_CACHE_LINE_MASK) == 0) + BDK_PREFETCH(address1, BDK_CACHE_LINE_SIZE); + if (((address1 + offset) & BDK_CACHE_LINE_MASK) == 0) + BDK_PREFETCH(address1 + offset, BDK_CACHE_LINE_SIZE); +#endif + WRITE64(address1 , READ64(address1 ) ^ this_pattern); + WRITE64(address1 + offset, READ64(address1 + offset) ^ this_pattern); + address1 += 8; + } + + __bdk_dram_flush_to_mem_range(area, max_address); + BDK_DCACHE_INVALIDATE; + + /* Look for differences from the expected pattern in both areas. + * If there is a mismatch, reset the appropriate memory location + * with the correct pattern. Failing to do so + * means that on all subsequent passes the erroring locations + * will be out of sync, giving spurious errors. + */ + address1 = area; + ppred = pbase; + + while (address1 < area2) + { +#if 1 + if ((address1 & BDK_CACHE_LINE_MASK) == 0) + BDK_PREFETCH(address1, BDK_CACHE_LINE_SIZE); + if (((address1 + offset) & BDK_CACHE_LINE_MASK) == 0) + BDK_PREFETCH(address1 + offset, BDK_CACHE_LINE_SIZE); +#endif + d1 = READ64(address1 ); + d2 = READ64(address1 + offset); + + p = ppred ^ pattern2; + + if (bdk_unlikely(d1 != p)) { + failures += __bdk_dram_retry_failure(burst, address1, d1, p); + // Synchronize the area, adjusting for the error. + //WRITE64(address1, p); // retries should do this + } + if (bdk_unlikely(d2 != p)) { + failures += __bdk_dram_retry_failure(burst, address1 + offset, d2, p); + // Synchronize the area, adjusting for the error. + //WRITE64(address1 + offset, p); // retries should do this + } + + address1 += 8; + ppred += pincr; + + } /* while (address1 < area2) */ + } /* for (int burst = 0; burst < bursts; burst++) */ + return failures; +} + +//////////////// this is the new code... + +int __bdk_dram_test_mem_rows(uint64_t area, uint64_t max_address, int bursts) +{ + int failures = 0; + + uint64_t pattern1 = 0x0; + uint64_t extent = (max_address - area); + uint64_t count = (extent / 2) / sizeof(uint64_t); // in terms of 64bit words + uint64_t offset = count * sizeof(uint64_t); + uint64_t area2 = area + offset; + uint64_t pattern2; + uint64_t d1, d2; + int burst; + + /* Fill both halves of the memory area with identical data pattern. Odd + * address 64-bit words get the pattern, while even address words get the + * inverted pattern. + */ + uint64_t address1 = area; + + pattern2 = pattern1; // start with original pattern + + while (address1 < area2) + { + WRITE64(address1 , pattern2); + WRITE64(address1 + offset, pattern2); + address1 += 8; + pattern2 = ~pattern2; // flip for next slots + } + + __bdk_dram_flush_to_mem_range(area, max_address); + BDK_DCACHE_INVALIDATE; + + /* Make a series of passes over the memory areas. */ + for (burst = 0; burst < bursts; burst++) + { + /* Invert the data, applying the change to both memory areas. Thus on + * alternate passes, the data flips from 0 to 1 and vice versa. + */ + address1 = area; + + while (address1 < area2) + { + if ((address1 & BDK_CACHE_LINE_MASK) == 0) + BDK_PREFETCH(address1 , BDK_CACHE_LINE_SIZE); + if (((address1 + offset) & BDK_CACHE_LINE_MASK) == 0) + BDK_PREFETCH(address1 + offset, BDK_CACHE_LINE_SIZE); + + WRITE64(address1 , ~READ64(address1 )); + WRITE64(address1 + offset, ~READ64(address1 + offset)); + address1 += 8; + } + + __bdk_dram_flush_to_mem_range(area, max_address); + BDK_DCACHE_INVALIDATE; + + /* Look for differences in the areas. If there is a mismatch, reset + * both memory locations with the same pattern. Failing to do so + * means that on all subsequent passes the pair of locations remain + * out of sync giving spurious errors. + */ + address1 = area; + pattern1 = ~pattern1; // flip the starting pattern to match above loop + pattern2 = pattern1; // slots have been flipped by the above loop + + while (address1 < area2) + { + if ((address1 & BDK_CACHE_LINE_MASK) == 0) + BDK_PREFETCH(address1 , BDK_CACHE_LINE_SIZE); + if (((address1 + offset) & BDK_CACHE_LINE_MASK) == 0) + BDK_PREFETCH(address1 + offset, BDK_CACHE_LINE_SIZE); + + d1 = READ64(address1 ); + d2 = READ64(address1 + offset); + + if (bdk_unlikely(d1 != pattern2)) { + failures += __bdk_dram_retry_failure(burst, address1, d1, pattern2); + // Synchronize the area, adjusting for the error. + //WRITE64(address1, pattern2); // retries should do this + } + if (bdk_unlikely(d2 != pattern2)) { + failures += __bdk_dram_retry_failure(burst, address1 + offset, d2, pattern2); + // Synchronize the two areas, adjusting for the error. + //WRITE64(address1 + offset, pattern2); // retries should do this + } + + address1 += 8; + pattern2 = ~pattern2; // flip for next pair of slots + } + } + return failures; +} +#endif /* USE_PREDICTION_CODE_VERSIONS */ diff --git a/src/vendorcode/cavium/bdk/libbdk-dram/bdk-dram-test.c b/src/vendorcode/cavium/bdk/libbdk-dram/bdk-dram-test.c new file mode 100644 index 0000000000..53137502fc --- /dev/null +++ b/src/vendorcode/cavium/bdk/libbdk-dram/bdk-dram-test.c @@ -0,0 +1,860 @@ +/***********************license start*********************************** +* Copyright (c) 2003-2017 Cavium Inc. (support@cavium.com). All rights +* reserved. +* +* +* Redistribution and use in source and binary forms, with or without +* modification, are permitted provided that the following conditions are +* met: +* +* * Redistributions of source code must retain the above copyright +* notice, this list of conditions and the following disclaimer. +* +* * Redistributions in binary form must reproduce the above +* copyright notice, this list of conditions and the following +* disclaimer in the documentation and/or other materials provided +* with the distribution. +* +* * Neither the name of Cavium Inc. nor the names of +* its contributors may be used to endorse or promote products +* derived from this software without specific prior written +* permission. +* +* This Software, including technical data, may be subject to U.S. export +* control laws, including the U.S. Export Administration Act and its +* associated regulations, and may be subject to export or import +* regulations in other countries. +* +* TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" +* AND WITH ALL FAULTS AND CAVIUM INC. MAKES NO PROMISES, REPRESENTATIONS OR +* WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH RESPECT +* TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY +* REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT +* DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) WARRANTIES +* OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A PARTICULAR +* PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET ENJOYMENT, +* QUIET POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE ENTIRE RISK +* ARISING OUT OF USE OR PERFORMANCE OF THE SOFTWARE LIES WITH YOU. +***********************license end**************************************/ +#include <bdk.h> +#include "libbdk-arch/bdk-csrs-gti.h" +#include "libbdk-arch/bdk-csrs-ocx.h" + +/* This code is an optional part of the BDK. It is only linked in + if BDK_REQUIRE() needs it */ +BDK_REQUIRE_DEFINE(DRAM_TEST); + +#define MAX_ERRORS_TO_REPORT 50 +#define RETRY_LIMIT 1000 + +typedef struct +{ + const char * name; /* Friendly name for the test */ + __bdk_dram_test_t test_func; /* Function to call */ + int bursts; /* Bursts parameter to pass to the test */ + int max_cores; /* Maximum number of cores the test should be run on in parallel. Zero means all */ +} dram_test_info_t; + +static const dram_test_info_t TEST_INFO[] = { + /* Name, Test function, Bursts, Max Cores */ + { "Data Bus", __bdk_dram_test_mem_data_bus, 8, 1}, + { "Address Bus", __bdk_dram_test_mem_address_bus, 0, 1}, + { "Marching Rows", __bdk_dram_test_mem_rows, 16, 0}, + { "Random Data", __bdk_dram_test_mem_random, 32, 0}, + { "Random XOR (32 Burst)", __bdk_dram_test_mem_xor, 32, 0}, + { "Self Address", __bdk_dram_test_mem_self_addr, 1, 0}, + { "March C- Solid Bits", __bdk_dram_test_mem_solid, 1, 0}, + { "March C- Checkerboard", __bdk_dram_test_mem_checkerboard, 1, 0}, + { "Walking Ones Left", __bdk_dram_test_mem_leftwalk1, 1, 0}, + { "Walking Ones Right", __bdk_dram_test_mem_rightwalk1, 1, 0}, + { "Walking Zeros Left", __bdk_dram_test_mem_leftwalk0, 1, 0}, + { "Walking Zeros Right", __bdk_dram_test_mem_rightwalk0, 1, 0}, + { "Random XOR (224 Burst)", __bdk_dram_test_mem_xor, 224, 0}, + { "Fast Scan", __bdk_dram_test_fast_scan, 0, 0}, + { NULL, NULL, 0, 0} +}; + +/* These variables count the number of ECC errors. They should only be accessed atomically */ +int64_t __bdk_dram_ecc_single_bit_errors[BDK_MAX_MEM_CHANS]; +int64_t __bdk_dram_ecc_double_bit_errors[BDK_MAX_MEM_CHANS]; + +static int64_t dram_test_thread_done; +static int64_t dram_test_thread_errors; +static uint64_t dram_test_thread_start; +static uint64_t dram_test_thread_end; +static uint64_t dram_test_thread_size; + +/** + * Force the memory at the pointer location to be written to memory and evicted + * from L2. L1 will be unaffected. + * + * @param address Physical memory location + */ +void __bdk_dram_flush_to_mem(uint64_t address) +{ + BDK_MB; + /* The DRAM code doesn't use the normal bdk_phys_to_ptr() because of the + NULL check in it. This greatly slows down the memory tests */ + char *ptr = (void*)address; + BDK_CACHE_WBI_L2(ptr); +} + +/** + * Force a memory region to be written to DRAM and evicted from L2 + * + * @param area Start of the region + * @param max_address + * End of the region (exclusive) + */ +void __bdk_dram_flush_to_mem_range(uint64_t area, uint64_t max_address) +{ + /* The DRAM code doesn't use the normal bdk_phys_to_ptr() because of the + NULL check in it. This greatly slows down the memory tests */ + char *ptr = (void*)area; + char *end = (void*)max_address; + BDK_MB; + while (ptr < end) + { + BDK_CACHE_WBI_L2(ptr); + ptr += 128; + } +} + +/** + * Convert a test enumeration into a string + * + * @param test Test to convert + * + * @return String for display + */ +const char *bdk_dram_get_test_name(int test) +{ + if (test < (int)(sizeof(TEST_INFO) / sizeof(TEST_INFO[0]))) + return TEST_INFO[test].name; + else + return NULL; +} + +static bdk_dram_test_flags_t dram_test_flags; // FIXME: Don't use global +/** + * This function is run as a thread to perform memory tests over multiple cores. + * Each thread gets a section of memory to work on, which is controlled by global + * variables at the beginning of this file. + * + * @param arg Number of the region we should check + * @param arg1 Pointer the the test_info structure + */ +static void dram_test_thread(int arg, void *arg1) +{ + const dram_test_info_t *test_info = arg1; + const int bursts = test_info->bursts; + const int range_number = arg; + + /* Figure out our work memory range. + * + * Note start_address and end_address just provide the physical offset + * portion of the address and do not have the node bits set. This is + * to simplify address checks and calculations. Later, when about to run + * the memory test, the routines adds in the node bits to form the final + * addresses. + */ + uint64_t start_address = dram_test_thread_start + dram_test_thread_size * range_number; + uint64_t end_address = start_address + dram_test_thread_size; + if (end_address > dram_test_thread_end) + end_address = dram_test_thread_end; + + bdk_node_t test_node = bdk_numa_local(); + if (dram_test_flags & BDK_DRAM_TEST_USE_CCPI) + test_node ^= 1; + /* Insert the node part of the address */ + start_address = bdk_numa_get_address(test_node, start_address); + end_address = bdk_numa_get_address(test_node, end_address); + /* Test the region */ + BDK_TRACE(DRAM_TEST, " Node %d, core %d, Testing [0x%011lx:0x%011lx]\n", + bdk_numa_local(), bdk_get_core_num() & 127, start_address, end_address - 1); + test_info->test_func(start_address, end_address, bursts); + + /* Report that we're done */ + BDK_TRACE(DRAM_TEST, "Thread %d on node %d done with memory test\n", range_number, bdk_numa_local()); + bdk_atomic_add64_nosync(&dram_test_thread_done, 1); +} + +/** + * Run the memory test. + * + * @param test_info + * @param start_address + * Physical address to start at + * @param length Length of memory block + * @param flags Flags to control memory test options. Zero defaults to testing all + * node with statistics and progress output. + * + * @return Number of errors found. Zero is success. Negative means the test + * did not run due to some other failure. + */ +static int __bdk_dram_run_test(const dram_test_info_t *test_info, uint64_t start_address, + uint64_t length, bdk_dram_test_flags_t flags) +{ + /* Figure out the addess of the byte one off the top of memory */ + uint64_t max_address = bdk_dram_get_size_mbytes(bdk_numa_local()); + BDK_TRACE(DRAM_TEST, "DRAM available per node: %lu MB\n", max_address); + max_address <<= 20; + + /* Make sure we have enough */ + if (max_address < (16<<20)) + { + bdk_error("DRAM size is too small\n"); + return -1; + } + + /* Make sure the amount is sane */ + if (CAVIUM_IS_MODEL(CAVIUM_CN8XXX)) + { + if (max_address > (1ull << 40)) /* 40 bits in CN8XXX */ + max_address = 1ull << 40; + } + else + { + if (max_address > (1ull << 43)) /* 43 bits in CN9XXX */ + max_address = 1ull << 43; + } + BDK_TRACE(DRAM_TEST, "DRAM max address: 0x%011lx\n", max_address-1); + + /* Make sure the start address is lower than the top of memory */ + if (start_address >= max_address) + { + bdk_error("Start address is larger than the amount of memory: 0x%011lx versus 0x%011lx\n", + start_address, max_address); + return -1; + } + if (length == (uint64_t)-1) + length = max_address - start_address; + + /* Final range checks */ + uint64_t end_address = start_address + length; + if (end_address > max_address) + { + end_address = max_address; + length = end_address - start_address; + } + if (length == 0) + return 0; + + /* Ready to run the test. Figure out how many cores we need */ + int max_cores = test_info->max_cores; + int total_cores_all_nodes = max_cores; + + /* Figure out the number of cores available in the system */ + if (max_cores == 0) + { + max_cores += bdk_get_num_running_cores(bdk_numa_local()); + /* Calculate the total number of cores being used. The per node number + is confusing to people */ + for (bdk_node_t node = BDK_NODE_0; node < BDK_NUMA_MAX_NODES; node++) + if (flags & (1 << node)) + { + if (flags & BDK_DRAM_TEST_USE_CCPI) + total_cores_all_nodes += bdk_get_num_running_cores(node ^ 1); + else + total_cores_all_nodes += bdk_get_num_running_cores(node); + } + } + if (!(flags & BDK_DRAM_TEST_NO_BANNERS)) + printf("Starting Test \"%s\" for [0x%011lx:0x%011lx] using %d core(s)\n", + test_info->name, start_address, end_address - 1, total_cores_all_nodes); + + /* Remember the LMC perf counters for stats after the test */ + uint64_t start_dram_dclk[BDK_NUMA_MAX_NODES][4]; + uint64_t start_dram_ops[BDK_NUMA_MAX_NODES][4]; + uint64_t stop_dram_dclk[BDK_NUMA_MAX_NODES][4]; + uint64_t stop_dram_ops[BDK_NUMA_MAX_NODES][4]; + for (bdk_node_t node = BDK_NODE_0; node < BDK_NUMA_MAX_NODES; node++) + { + if (flags & (1 << node)) + { + const int num_dram_controllers = __bdk_dram_get_num_lmc(node); + for (int i = 0; i < num_dram_controllers; i++) + { + start_dram_dclk[node][i] = BDK_CSR_READ(node, BDK_LMCX_DCLK_CNT(i)); + start_dram_ops[node][i] = BDK_CSR_READ(node, BDK_LMCX_OPS_CNT(i)); + } + } + } + /* Remember the CCPI link counters for stats after the test */ + uint64_t start_ccpi_data[BDK_NUMA_MAX_NODES][3]; + uint64_t start_ccpi_idle[BDK_NUMA_MAX_NODES][3]; + uint64_t start_ccpi_err[BDK_NUMA_MAX_NODES][3]; + uint64_t stop_ccpi_data[BDK_NUMA_MAX_NODES][3]; + uint64_t stop_ccpi_idle[BDK_NUMA_MAX_NODES][3]; + uint64_t stop_ccpi_err[BDK_NUMA_MAX_NODES][3]; + if (!bdk_numa_is_only_one()) + { + for (bdk_node_t node = BDK_NODE_0; node < BDK_NUMA_MAX_NODES; node++) + { + if (flags & (1 << node)) + { + for (int link = 0; link < 3; link++) + { + start_ccpi_data[node][link] = BDK_CSR_READ(node, BDK_OCX_TLKX_STAT_DATA_CNT(link)); + start_ccpi_idle[node][link] = BDK_CSR_READ(node, BDK_OCX_TLKX_STAT_IDLE_CNT(link)); + start_ccpi_err[node][link] = BDK_CSR_READ(node, BDK_OCX_TLKX_STAT_ERR_CNT(link)); + } + } + } + } + + /* WARNING: This code assumes the same memory range is being tested on + all nodes. The same number of cores are used on each node to test + its local memory */ + uint64_t work_address = start_address; + dram_test_flags = flags; + bdk_atomic_set64(&dram_test_thread_errors, 0); + while ((work_address < end_address) && ((dram_test_thread_errors == 0) || (flags & BDK_DRAM_TEST_NO_STOP_ERROR))) + { + /* Check at most MAX_CHUNK_SIZE across each iteration. We only report + progress between chunks, so keep them reasonably small */ + const uint64_t MAX_CHUNK_SIZE = 1ull << 28; /* 256MB */ + uint64_t size = end_address - work_address; + if (size > MAX_CHUNK_SIZE) + size = MAX_CHUNK_SIZE; + + /* Divide memory evenly between the cores. Round the size up so that + all memory is covered. The last core may have slightly less memory to + test */ + uint64_t thread_size = (size + (max_cores - 1)) / max_cores; + thread_size += 127; + thread_size &= -128; + dram_test_thread_start = work_address; + dram_test_thread_end = work_address + size; + dram_test_thread_size = thread_size; + BDK_WMB; + + /* Poke the watchdog */ + BDK_CSR_WRITE(bdk_numa_local(), BDK_GTI_CWD_POKEX(0), 0); + + /* disable progress output when batch mode is ON */ + if (!(flags & BDK_DRAM_TEST_NO_PROGRESS)) { + + /* Report progress percentage */ + int percent_x10 = (work_address - start_address) * 1000 / (end_address - start_address); + printf(" %3d.%d%% complete, testing [0x%011lx:0x%011lx]\r", + percent_x10 / 10, percent_x10 % 10, work_address, work_address + size - 1); + fflush(stdout); + } + + work_address += size; + + /* Start threads for all the cores */ + int total_count = 0; + bdk_atomic_set64(&dram_test_thread_done, 0); + for (bdk_node_t node = BDK_NODE_0; node < BDK_NUMA_MAX_NODES; node++) + { + if (flags & (1 << node)) + { + const int num_cores = bdk_get_num_cores(node); + int per_node = 0; + for (int core = 0; core < num_cores; core++) + { + if (per_node >= max_cores) + break; + int run_node = (flags & BDK_DRAM_TEST_USE_CCPI) ? node ^ 1 : node; + BDK_TRACE(DRAM_TEST, "Starting thread %d on node %d for memory test\n", per_node, node); + if (bdk_thread_create(run_node, 0, dram_test_thread, per_node, (void *)test_info, 0)) + { + bdk_error("Failed to create thread %d for memory test on node %d\n", per_node, node); + } + else + { + per_node++; + total_count++; + } + } + } + } + +#if 0 + /* Wait for threads to finish */ + while (bdk_atomic_get64(&dram_test_thread_done) < total_count) + bdk_thread_yield(); +#else +#define TIMEOUT_SECS 30 // FIXME: long enough so multicore RXOR 224 should not print out + /* Wait for threads to finish, with progress */ + int cur_count; + uint64_t cur_time; + uint64_t period = bdk_clock_get_rate(bdk_numa_local(), BDK_CLOCK_TIME) * TIMEOUT_SECS; // FIXME? + uint64_t timeout = bdk_clock_get_count(BDK_CLOCK_TIME) + period; + do { + bdk_thread_yield(); + cur_count = bdk_atomic_get64(&dram_test_thread_done); + cur_time = bdk_clock_get_count(BDK_CLOCK_TIME); + if (cur_time >= timeout) { + BDK_TRACE(DRAM_TEST, "N%d: Waiting for %d cores\n", + bdk_numa_local(), total_count - cur_count); + timeout = cur_time + period; + } + } while (cur_count < total_count); +#endif + } + + /* Get the DRAM perf counters */ + for (bdk_node_t node = BDK_NODE_0; node < BDK_NUMA_MAX_NODES; node++) + { + if (flags & (1 << node)) + { + const int num_dram_controllers = __bdk_dram_get_num_lmc(node); + for (int i = 0; i < num_dram_controllers; i++) + { + stop_dram_dclk[node][i] = BDK_CSR_READ(node, BDK_LMCX_DCLK_CNT(i)); + stop_dram_ops[node][i] = BDK_CSR_READ(node, BDK_LMCX_OPS_CNT(i)); + } + } + } + /* Get the CCPI link counters */ + if (!bdk_numa_is_only_one()) + { + for (bdk_node_t node = BDK_NODE_0; node < BDK_NUMA_MAX_NODES; node++) + { + if (flags & (1 << node)) + { + for (int link = 0; link < 3; link++) + { + stop_ccpi_data[node][link] = BDK_CSR_READ(node, BDK_OCX_TLKX_STAT_DATA_CNT(link)); + stop_ccpi_idle[node][link] = BDK_CSR_READ(node, BDK_OCX_TLKX_STAT_IDLE_CNT(link)); + stop_ccpi_err[node][link] = BDK_CSR_READ(node, BDK_OCX_TLKX_STAT_ERR_CNT(link)); + } + } + } + } + + /* disable progress output when batch mode is ON */ + if (!(flags & BDK_DRAM_TEST_NO_PROGRESS)) { + + /* Report progress percentage as complete */ + printf(" %3d.%d%% complete, testing [0x%011lx:0x%011lx]\n", + 100, 0, start_address, end_address - 1); + fflush(stdout); + } + + if (!(flags & BDK_DRAM_TEST_NO_STATS)) + { + /* Display LMC load */ + for (bdk_node_t node = BDK_NODE_0; node < BDK_NUMA_MAX_NODES; node++) + { + if (flags & (1 << node)) + { + const int num_dram_controllers = __bdk_dram_get_num_lmc(node); + for (int i = 0; i < num_dram_controllers; i++) + { + uint64_t ops = stop_dram_ops[node][i] - start_dram_ops[node][i]; + uint64_t dclk = stop_dram_dclk[node][i] - start_dram_dclk[node][i]; + if (dclk == 0) + dclk = 1; + uint64_t percent_x10 = ops * 1000 / dclk; + printf(" Node %d, LMC%d: ops %lu, cycles %lu, used %lu.%lu%%\n", + node, i, ops, dclk, percent_x10 / 10, percent_x10 % 10); + } + } + } + if (flags & BDK_DRAM_TEST_USE_CCPI) + { + /* Display CCPI load */ + for (bdk_node_t node = BDK_NODE_0; node < BDK_NUMA_MAX_NODES; node++) + { + if (flags & (1 << node)) + { + for (int link = 0; link < 3; link++) + { + uint64_t busy = stop_ccpi_data[node][link] - start_ccpi_data[node][link]; + busy += stop_ccpi_err[node][link] - start_ccpi_err[node][link]; + uint64_t total = stop_ccpi_idle[node][link] - start_ccpi_idle[node][link]; + total += busy; + if (total == 0) + continue; + uint64_t percent_x10 = busy * 1000 / total; + printf(" Node %d, CCPI%d: busy %lu, total %lu, used %lu.%lu%%\n", + node, link, busy, total, percent_x10 / 10, percent_x10 % 10); + } + } + } + } + } + return dram_test_thread_errors; +} + +/** + * Perform a memory test. + * + * @param test Test type to run + * @param start_address + * Physical address to start at + * @param length Length of memory block + * @param flags Flags to control memory test options. Zero defaults to testing all + * node with statistics and progress output. + * + * @return Number of errors found. Zero is success. Negative means the test + * did not run due to some other failure. + */ +int bdk_dram_test(int test, uint64_t start_address, uint64_t length, bdk_dram_test_flags_t flags) +{ + /* These limits are arbitrary. They just make sure we aren't doing something + silly, like test a non cache line aligned memory region */ + if (start_address & 0xffff) + { + bdk_error("DRAM test start address must be aligned on a 64KB boundary\n"); + return -1; + } + if (length & 0xffff) + { + bdk_error("DRAM test length must be a multiple of 64KB\n"); + return -1; + } + + const char *name = bdk_dram_get_test_name(test); + if (name == NULL) + { + bdk_error("Invalid DRAM test number %d\n", test); + return -1; + } + + /* If no nodes are selected assume the user meant all nodes */ + if ((flags & (BDK_DRAM_TEST_NODE0 | BDK_DRAM_TEST_NODE1 | BDK_DRAM_TEST_NODE2 | BDK_DRAM_TEST_NODE3)) == 0) + flags |= BDK_DRAM_TEST_NODE0 | BDK_DRAM_TEST_NODE1 | BDK_DRAM_TEST_NODE2 | BDK_DRAM_TEST_NODE3; + + /* Remove nodes from the flags that don't exist */ + for (bdk_node_t node = BDK_NODE_0; node < BDK_NUMA_MAX_NODES; node++) + { + if (flags & BDK_DRAM_TEST_USE_CCPI) + { + if (!bdk_numa_exists(node ^ 1)) + flags &= ~(1 << node); + } + else + { + if (!bdk_numa_exists(node)) + flags &= ~(1 << node); + } + } + + + /* Make sure the start address is higher that the BDK's active range */ + uint64_t top_of_bdk = bdk_dram_get_top_of_bdk(); + if (start_address < top_of_bdk) + start_address = top_of_bdk; + + /* Clear ECC error counters before starting the test */ + for (int chan = 0; chan < BDK_MAX_MEM_CHANS; chan++) { + bdk_atomic_set64(&__bdk_dram_ecc_single_bit_errors[chan], 0); + bdk_atomic_set64(&__bdk_dram_ecc_double_bit_errors[chan], 0); + } + + /* Make sure at least one core from each node is running */ + for (bdk_node_t node = BDK_NODE_0; node < BDK_NUMA_MAX_NODES; node++) + { + if (flags & (1<<node)) + { + int use_node = (flags & BDK_DRAM_TEST_USE_CCPI) ? node ^ 1 : node; + if (bdk_get_running_coremask(use_node) == 0) + bdk_init_cores(use_node, 1); + } + } + + /* This returns any data compare errors found */ + int errors = __bdk_dram_run_test(&TEST_INFO[test], start_address, length, flags); + + /* Poll for any errors right now to make sure any ECC errors are reported */ + for (bdk_node_t node = BDK_NODE_0; node < BDK_NUMA_MAX_NODES; node++) + { + if (bdk_numa_exists(node) && bdk_error_check) + bdk_error_check(node); + } + + /* Check ECC error counters after the test */ + int64_t ecc_single = 0; + int64_t ecc_double = 0; + int64_t ecc_single_errs[BDK_MAX_MEM_CHANS]; + int64_t ecc_double_errs[BDK_MAX_MEM_CHANS]; + + for (int chan = 0; chan < BDK_MAX_MEM_CHANS; chan++) { + ecc_single += (ecc_single_errs[chan] = bdk_atomic_get64(&__bdk_dram_ecc_single_bit_errors[chan])); + ecc_double += (ecc_double_errs[chan] = bdk_atomic_get64(&__bdk_dram_ecc_double_bit_errors[chan])); + } + + /* Always print any ECC errors */ + if (ecc_single || ecc_double) + { + printf("Test \"%s\": ECC errors, %ld/%ld/%ld/%ld corrected, %ld/%ld/%ld/%ld uncorrected\n", + name, + ecc_single_errs[0], ecc_single_errs[1], ecc_single_errs[2], ecc_single_errs[3], + ecc_double_errs[0], ecc_double_errs[1], ecc_double_errs[2], ecc_double_errs[3]); + } + if (errors || ecc_double || ecc_single) { + printf("Test \"%s\": FAIL: %ld single, %ld double, %d compare errors\n", + name, ecc_single, ecc_double, errors); + } + else + BDK_TRACE(DRAM_TEST, "Test \"%s\": PASS\n", name); + + return (errors + ecc_double + ecc_single); +} + +/** + * Report a DRAM address in decoded format. + * + * @param address Physical address the error occurred at + * + */ +static void __bdk_dram_report_address_decode(uint64_t address, char *buffer, int len) +{ + int node, lmc, dimm, prank, lrank, bank, row, col; + + bdk_dram_address_extract_info(address, &node, &lmc, &dimm, &prank, &lrank, &bank, &row, &col); + + snprintf(buffer, len, "[0x%011lx] (N%d,LMC%d,DIMM%d,Rank%d/%d,Bank%02d,Row 0x%05x,Col 0x%04x)", + address, node, lmc, dimm, prank, lrank, bank, row, col); +} + +/** + * Report a DRAM address in a new decoded format. + * + * @param address Physical address the error occurred at + * @param xor XOR of data read vs expected data + * + */ +static void __bdk_dram_report_address_decode_new(uint64_t address, uint64_t orig_xor, char *buffer, int len) +{ + int node, lmc, dimm, prank, lrank, bank, row, col; + + int byte = 8; // means no byte-lanes in error, should not happen + uint64_t bits, print_bits = 0; + uint64_t xor = orig_xor; + + // find the byte-lane(s) with errors + for (int i = 0; i < 8; i++) { + bits = xor & 0xffULL; + xor >>= 8; + if (bits) { + if (byte != 8) { + byte = 9; // means more than 1 byte-lane was present + print_bits = orig_xor; // print the full original + break; // quit now + } else { + byte = i; // keep checking + print_bits = bits; + } + } + } + + bdk_dram_address_extract_info(address, &node, &lmc, &dimm, &prank, &lrank, &bank, &row, &col); + + snprintf(buffer, len, "N%d.LMC%d: CMP byte %d xor 0x%02lx (DIMM%d,Rank%d/%d,Bank%02d,Row 0x%05x,Col 0x%04x)[0x%011lx]", + node, lmc, byte, print_bits, dimm, prank, lrank, bank, row, col, address); +} + +/** + * Report a DRAM error. Errors are not shown after MAX_ERRORS_TO_REPORT is + * exceeded. Used when a single address is involved in the failure. + * + * @param address Physical address the error occurred at + * @param data Data read from memory + * @param correct Correct data + * @param burst Which burst this is from, informational only + * @param fails -1 for no retries done, >= 0 number of failures during retries + * + * @return Zero if a message was logged, non-zero if the error limit has been reached + */ +void __bdk_dram_report_error(uint64_t address, uint64_t data, uint64_t correct, int burst, int fails) +{ + char buffer[128]; + char failbuf[32]; + int64_t errors = bdk_atomic_fetch_and_add64(&dram_test_thread_errors, 1); + uint64_t xor = data ^ correct; + + if (errors < MAX_ERRORS_TO_REPORT) + { + if (fails < 0) { + snprintf(failbuf, sizeof(failbuf), " "); + } else { + int percent_x10 = fails * 1000 / RETRY_LIMIT; + snprintf(failbuf, sizeof(failbuf), ", retries failed %3d.%d%%", + percent_x10 / 10, percent_x10 % 10); + } + + __bdk_dram_report_address_decode_new(address, xor, buffer, sizeof(buffer)); + bdk_error("%s%s\n", buffer, failbuf); + + if (errors == MAX_ERRORS_TO_REPORT-1) + bdk_error("No further DRAM errors will be reported\n"); + } + return; +} + +/** + * Report a DRAM error. Errors are not shown after MAX_ERRORS_TO_REPORT is + * exceeded. Used when two addresses might be involved in the failure. + * + * @param address1 First address involved in the failure + * @param data1 Data from the first address + * @param address2 Second address involved in the failure + * @param data2 Data from second address + * @param burst Which burst this is from, informational only + * @param fails -1 for no retries done, >= 0 number of failures during retries + * + * @return Zero if a message was logged, non-zero if the error limit has been reached + */ +void __bdk_dram_report_error2(uint64_t address1, uint64_t data1, uint64_t address2, uint64_t data2, + int burst, int fails) +{ + int64_t errors = bdk_atomic_fetch_and_add64(&dram_test_thread_errors, 1); + if (errors < MAX_ERRORS_TO_REPORT) + { + char buffer1[80], buffer2[80]; + char failbuf[32]; + + if (fails < 0) { + snprintf(failbuf, sizeof(failbuf), " "); + } else { + snprintf(failbuf, sizeof(failbuf), ", retried %d failed %d", RETRY_LIMIT, fails); + } + __bdk_dram_report_address_decode(address1, buffer1, sizeof(buffer1)); + __bdk_dram_report_address_decode(address2, buffer2, sizeof(buffer2)); + + bdk_error("compare: data1: 0x%016lx, xor: 0x%016lx%s\n" + " %s\n %s\n", + data1, data1 ^ data2, failbuf, + buffer1, buffer2); + + if (errors == MAX_ERRORS_TO_REPORT-1) + bdk_error("No further DRAM errors will be reported\n"); + } + return; +} + +/* Report the circumstances of a failure and try re-reading the memory + * location to see if the error is transient or permanent. + * + * Note: re-reading requires using evicting addresses + */ +int __bdk_dram_retry_failure(int burst, uint64_t address, uint64_t data, uint64_t expected) +{ + int refail = 0; + + // bypass the retries if we are already over the limit... + if (bdk_atomic_get64(&dram_test_thread_errors) < MAX_ERRORS_TO_REPORT) { + + /* Try re-reading the memory location. A transient error may fail + * on one read and work on another. Keep on retrying even when a + * read succeeds. + */ + for (int i = 0; i < RETRY_LIMIT; i++) { + + __bdk_dram_flush_to_mem(address); + BDK_DCACHE_INVALIDATE; + + uint64_t new = __bdk_dram_read64(address); + + if (new != expected) { + refail++; + } + } + } else + refail = -1; + + // this will increment the errors always, but maybe not print... + __bdk_dram_report_error(address, data, expected, burst, refail); + + return 1; +} + +/** + * retry_failure2 + * + * @param burst + * @param address1 + * @param address2 + */ +int __bdk_dram_retry_failure2(int burst, uint64_t address1, uint64_t data1, uint64_t address2, uint64_t data2) +{ + int refail = 0; + + // bypass the retries if we are already over the limit... + if (bdk_atomic_get64(&dram_test_thread_errors) < MAX_ERRORS_TO_REPORT) { + + for (int i = 0; i < RETRY_LIMIT; i++) { + __bdk_dram_flush_to_mem(address1); + __bdk_dram_flush_to_mem(address2); + BDK_DCACHE_INVALIDATE; + + uint64_t d1 = __bdk_dram_read64(address1); + uint64_t d2 = __bdk_dram_read64(address2); + + if (d1 != d2) { + refail++; + } + } + } else + refail = -1; + + // this will increment the errors always, but maybe not print... + __bdk_dram_report_error2(address1, data1, address2, data2, burst, refail); + + return 1; +} + +/** + * Inject a DRAM error at a specific address in memory. The injection can either + * be a single bit inside the byte, or a double bit error in the ECC byte. Double + * bit errors may corrupt memory, causing software to crash. The corruption is + * written to memory and will continue to exist until the cache line is written + * again. After a call to this function, the BDK should report a ECC error. Double + * bit errors corrupt bits 0-1. + * + * @param address Physical address to corrupt. Any byte alignment is supported + * @param bit Bit to corrupt in the byte (0-7), or -1 to create a double bit fault in the ECC + * byte. + */ +void bdk_dram_test_inject_error(uint64_t address, int bit) +{ + uint64_t aligned_address = address & -16; + int corrupt_bit = -1; + if (bit >= 0) + corrupt_bit = (address & 0xf) * 8 + bit; + + /* Extract the DRAM controller information */ + int node, lmc, dimm, prank, lrank, bank, row, col; + bdk_dram_address_extract_info(address, &node, &lmc, &dimm, &prank, &lrank, &bank, &row, &col); + + /* Read the current data */ + uint64_t data = __bdk_dram_read64(aligned_address); + + /* Program LMC to inject the error */ + if ((corrupt_bit >= 0) && (corrupt_bit < 64)) + BDK_CSR_WRITE(node, BDK_LMCX_CHAR_MASK0(lmc), 1ull << corrupt_bit); + else if (bit == -1) + BDK_CSR_WRITE(node, BDK_LMCX_CHAR_MASK0(lmc), 3); + else + BDK_CSR_WRITE(node, BDK_LMCX_CHAR_MASK0(lmc), 0); + if (corrupt_bit >= 64) + BDK_CSR_WRITE(node, BDK_LMCX_CHAR_MASK2(lmc), 1ull << (corrupt_bit - 64)); + else + BDK_CSR_WRITE(node, BDK_LMCX_CHAR_MASK2(lmc), 0); + BDK_CSR_MODIFY(c, node, BDK_LMCX_ECC_PARITY_TEST(lmc), + c.s.ecc_corrupt_idx = (address & 0x7f) >> 4; + c.s.ecc_corrupt_ena = 1); + BDK_CSR_READ(node, BDK_LMCX_ECC_PARITY_TEST(lmc)); + + /* Perform a write and push it to DRAM. This creates the error */ + __bdk_dram_write64(aligned_address, data); + __bdk_dram_flush_to_mem(aligned_address); + + /* Disable error injection */ + BDK_CSR_MODIFY(c, node, BDK_LMCX_ECC_PARITY_TEST(lmc), + c.s.ecc_corrupt_ena = 0); + BDK_CSR_READ(node, BDK_LMCX_ECC_PARITY_TEST(lmc)); + BDK_CSR_WRITE(node, BDK_LMCX_CHAR_MASK0(lmc), 0); + BDK_CSR_WRITE(node, BDK_LMCX_CHAR_MASK2(lmc), 0); + + /* Read back the data, which should now cause an error */ + printf("Loading the injected error address 0x%lx, node=%d, lmc=%d, dimm=%d, rank=%d/%d, bank=%d, row=%d, col=%d\n", + address, node, lmc, dimm, prank, lrank, bank, row, col); + __bdk_dram_read64(aligned_address); +} |