1 #include "amd64_edac.h"
2 #include <asm/amd_nb.h>
4 static struct edac_pci_ctl_info *pci_ctl;
6 static int report_gart_errors;
7 module_param(report_gart_errors, int, 0644);
10 * Set by command line parameter. If BIOS has enabled the ECC, this override is
11 * cleared to prevent re-enabling the hardware by this driver.
13 static int ecc_enable_override;
14 module_param(ecc_enable_override, int, 0644);
16 static struct msr __percpu *msrs;
19 static struct ecc_settings **ecc_stngs;
22 * Valid scrub rates for the K8 hardware memory scrubber. We map the scrubbing
23 * bandwidth to a valid bit pattern. The 'set' operation finds the 'matching-
26 *FIXME: Produce a better mapping/linearisation.
28 static const struct scrubrate {
29 u32 scrubval; /* bit pattern for scrub rate */
30 u32 bandwidth; /* bandwidth consumed (bytes/sec) */
32 { 0x01, 1600000000UL},
54 { 0x00, 0UL}, /* scrubbing off */
57 int __amd64_read_pci_cfg_dword(struct pci_dev *pdev, int offset,
58 u32 *val, const char *func)
62 err = pci_read_config_dword(pdev, offset, val);
64 amd64_warn("%s: error reading F%dx%03x.\n",
65 func, PCI_FUNC(pdev->devfn), offset);
70 int __amd64_write_pci_cfg_dword(struct pci_dev *pdev, int offset,
71 u32 val, const char *func)
75 err = pci_write_config_dword(pdev, offset, val);
77 amd64_warn("%s: error writing to F%dx%03x.\n",
78 func, PCI_FUNC(pdev->devfn), offset);
84 * Select DCT to which PCI cfg accesses are routed
86 static void f15h_select_dct(struct amd64_pvt *pvt, u8 dct)
90 amd64_read_pci_cfg(pvt->F1, DCT_CFG_SEL, ®);
91 reg &= (pvt->model == 0x30) ? ~3 : ~1;
93 amd64_write_pci_cfg(pvt->F1, DCT_CFG_SEL, reg);
98 * Depending on the family, F2 DCT reads need special handling:
100 * K8: has a single DCT only and no address offsets >= 0x100
102 * F10h: each DCT has its own set of regs
106 * F16h: has only 1 DCT
108 * F15h: we select which DCT we access using F1x10C[DctCfgSel]
110 static inline int amd64_read_dct_pci_cfg(struct amd64_pvt *pvt, u8 dct,
111 int offset, u32 *val)
115 if (dct || offset >= 0x100)
122 * Note: If ganging is enabled, barring the regs
123 * F2x[1,0]98 and F2x[1,0]9C; reads reads to F2x1xx
124 * return 0. (cf. Section 2.8.1 F10h BKDG)
126 if (dct_ganging_enabled(pvt))
135 * F15h: F2x1xx addresses do not map explicitly to DCT1.
136 * We should select which DCT we access using F1x10C[DctCfgSel]
138 dct = (dct && pvt->model == 0x30) ? 3 : dct;
139 f15h_select_dct(pvt, dct);
150 return amd64_read_pci_cfg(pvt->F2, offset, val);
154 * Memory scrubber control interface. For K8, memory scrubbing is handled by
155 * hardware and can involve L2 cache, dcache as well as the main memory. With
156 * F10, this is extended to L3 cache scrubbing on CPU models sporting that
159 * This causes the "units" for the scrubbing speed to vary from 64 byte blocks
160 * (dram) over to cache lines. This is nasty, so we will use bandwidth in
161 * bytes/sec for the setting.
163 * Currently, we only do dram scrubbing. If the scrubbing is done in software on
164 * other archs, we might not have access to the caches directly.
168 * scan the scrub rate mapping table for a close or matching bandwidth value to
169 * issue. If requested is too big, then use last maximum value found.
171 static int __set_scrub_rate(struct amd64_pvt *pvt, u32 new_bw, u32 min_rate)
177 * map the configured rate (new_bw) to a value specific to the AMD64
178 * memory controller and apply to register. Search for the first
179 * bandwidth entry that is greater or equal than the setting requested
180 * and program that. If at last entry, turn off DRAM scrubbing.
182 * If no suitable bandwidth is found, turn off DRAM scrubbing entirely
183 * by falling back to the last element in scrubrates[].
185 for (i = 0; i < ARRAY_SIZE(scrubrates) - 1; i++) {
187 * skip scrub rates which aren't recommended
188 * (see F10 BKDG, F3x58)
190 if (scrubrates[i].scrubval < min_rate)
193 if (scrubrates[i].bandwidth <= new_bw)
197 scrubval = scrubrates[i].scrubval;
199 if (pvt->fam == 0x15 && pvt->model == 0x60) {
200 f15h_select_dct(pvt, 0);
201 pci_write_bits32(pvt->F2, F15H_M60H_SCRCTRL, scrubval, 0x001F);
202 f15h_select_dct(pvt, 1);
203 pci_write_bits32(pvt->F2, F15H_M60H_SCRCTRL, scrubval, 0x001F);
205 pci_write_bits32(pvt->F3, SCRCTRL, scrubval, 0x001F);
209 return scrubrates[i].bandwidth;
214 static int set_scrub_rate(struct mem_ctl_info *mci, u32 bw)
216 struct amd64_pvt *pvt = mci->pvt_info;
217 u32 min_scrubrate = 0x5;
222 if (pvt->fam == 0x15) {
224 if (pvt->model < 0x10)
225 f15h_select_dct(pvt, 0);
227 if (pvt->model == 0x60)
230 return __set_scrub_rate(pvt, bw, min_scrubrate);
233 static int get_scrub_rate(struct mem_ctl_info *mci)
235 struct amd64_pvt *pvt = mci->pvt_info;
237 int i, retval = -EINVAL;
239 if (pvt->fam == 0x15) {
241 if (pvt->model < 0x10)
242 f15h_select_dct(pvt, 0);
244 if (pvt->model == 0x60)
245 amd64_read_pci_cfg(pvt->F2, F15H_M60H_SCRCTRL, &scrubval);
247 amd64_read_pci_cfg(pvt->F3, SCRCTRL, &scrubval);
249 amd64_read_pci_cfg(pvt->F3, SCRCTRL, &scrubval);
251 scrubval = scrubval & 0x001F;
253 for (i = 0; i < ARRAY_SIZE(scrubrates); i++) {
254 if (scrubrates[i].scrubval == scrubval) {
255 retval = scrubrates[i].bandwidth;
263 * returns true if the SysAddr given by sys_addr matches the
264 * DRAM base/limit associated with node_id
266 static bool base_limit_match(struct amd64_pvt *pvt, u64 sys_addr, u8 nid)
270 /* The K8 treats this as a 40-bit value. However, bits 63-40 will be
271 * all ones if the most significant implemented address bit is 1.
272 * Here we discard bits 63-40. See section 3.4.2 of AMD publication
273 * 24592: AMD x86-64 Architecture Programmer's Manual Volume 1
274 * Application Programming.
276 addr = sys_addr & 0x000000ffffffffffull;
278 return ((addr >= get_dram_base(pvt, nid)) &&
279 (addr <= get_dram_limit(pvt, nid)));
283 * Attempt to map a SysAddr to a node. On success, return a pointer to the
284 * mem_ctl_info structure for the node that the SysAddr maps to.
286 * On failure, return NULL.
288 static struct mem_ctl_info *find_mc_by_sys_addr(struct mem_ctl_info *mci,
291 struct amd64_pvt *pvt;
296 * Here we use the DRAM Base (section 3.4.4.1) and DRAM Limit (section
297 * 3.4.4.2) registers to map the SysAddr to a node ID.
302 * The value of this field should be the same for all DRAM Base
303 * registers. Therefore we arbitrarily choose to read it from the
304 * register for node 0.
306 intlv_en = dram_intlv_en(pvt, 0);
309 for (node_id = 0; node_id < DRAM_RANGES; node_id++) {
310 if (base_limit_match(pvt, sys_addr, node_id))
316 if (unlikely((intlv_en != 0x01) &&
317 (intlv_en != 0x03) &&
318 (intlv_en != 0x07))) {
319 amd64_warn("DRAM Base[IntlvEn] junk value: 0x%x, BIOS bug?\n", intlv_en);
323 bits = (((u32) sys_addr) >> 12) & intlv_en;
325 for (node_id = 0; ; ) {
326 if ((dram_intlv_sel(pvt, node_id) & intlv_en) == bits)
327 break; /* intlv_sel field matches */
329 if (++node_id >= DRAM_RANGES)
333 /* sanity test for sys_addr */
334 if (unlikely(!base_limit_match(pvt, sys_addr, node_id))) {
335 amd64_warn("%s: sys_addr 0x%llx falls outside base/limit address"
336 "range for node %d with node interleaving enabled.\n",
337 __func__, sys_addr, node_id);
342 return edac_mc_find((int)node_id);
345 edac_dbg(2, "sys_addr 0x%lx doesn't match any node\n",
346 (unsigned long)sys_addr);
352 * compute the CS base address of the @csrow on the DRAM controller @dct.
353 * For details see F2x[5C:40] in the processor's BKDG
355 static void get_cs_base_and_mask(struct amd64_pvt *pvt, int csrow, u8 dct,
356 u64 *base, u64 *mask)
358 u64 csbase, csmask, base_bits, mask_bits;
361 if (pvt->fam == 0xf && pvt->ext_model < K8_REV_F) {
362 csbase = pvt->csels[dct].csbases[csrow];
363 csmask = pvt->csels[dct].csmasks[csrow];
364 base_bits = GENMASK_ULL(31, 21) | GENMASK_ULL(15, 9);
365 mask_bits = GENMASK_ULL(29, 21) | GENMASK_ULL(15, 9);
369 * F16h and F15h, models 30h and later need two addr_shift values:
370 * 8 for high and 6 for low (cf. F16h BKDG).
372 } else if (pvt->fam == 0x16 ||
373 (pvt->fam == 0x15 && pvt->model >= 0x30)) {
374 csbase = pvt->csels[dct].csbases[csrow];
375 csmask = pvt->csels[dct].csmasks[csrow >> 1];
377 *base = (csbase & GENMASK_ULL(15, 5)) << 6;
378 *base |= (csbase & GENMASK_ULL(30, 19)) << 8;
381 /* poke holes for the csmask */
382 *mask &= ~((GENMASK_ULL(15, 5) << 6) |
383 (GENMASK_ULL(30, 19) << 8));
385 *mask |= (csmask & GENMASK_ULL(15, 5)) << 6;
386 *mask |= (csmask & GENMASK_ULL(30, 19)) << 8;
390 csbase = pvt->csels[dct].csbases[csrow];
391 csmask = pvt->csels[dct].csmasks[csrow >> 1];
394 if (pvt->fam == 0x15)
395 base_bits = mask_bits =
396 GENMASK_ULL(30,19) | GENMASK_ULL(13,5);
398 base_bits = mask_bits =
399 GENMASK_ULL(28,19) | GENMASK_ULL(13,5);
402 *base = (csbase & base_bits) << addr_shift;
405 /* poke holes for the csmask */
406 *mask &= ~(mask_bits << addr_shift);
408 *mask |= (csmask & mask_bits) << addr_shift;
411 #define for_each_chip_select(i, dct, pvt) \
412 for (i = 0; i < pvt->csels[dct].b_cnt; i++)
414 #define chip_select_base(i, dct, pvt) \
415 pvt->csels[dct].csbases[i]
417 #define for_each_chip_select_mask(i, dct, pvt) \
418 for (i = 0; i < pvt->csels[dct].m_cnt; i++)
421 * @input_addr is an InputAddr associated with the node given by mci. Return the
422 * csrow that input_addr maps to, or -1 on failure (no csrow claims input_addr).
424 static int input_addr_to_csrow(struct mem_ctl_info *mci, u64 input_addr)
426 struct amd64_pvt *pvt;
432 for_each_chip_select(csrow, 0, pvt) {
433 if (!csrow_enabled(csrow, 0, pvt))
436 get_cs_base_and_mask(pvt, csrow, 0, &base, &mask);
440 if ((input_addr & mask) == (base & mask)) {
441 edac_dbg(2, "InputAddr 0x%lx matches csrow %d (node %d)\n",
442 (unsigned long)input_addr, csrow,
448 edac_dbg(2, "no matching csrow for InputAddr 0x%lx (MC node %d)\n",
449 (unsigned long)input_addr, pvt->mc_node_id);
455 * Obtain info from the DRAM Hole Address Register (section 3.4.8, pub #26094)
456 * for the node represented by mci. Info is passed back in *hole_base,
457 * *hole_offset, and *hole_size. Function returns 0 if info is valid or 1 if
458 * info is invalid. Info may be invalid for either of the following reasons:
460 * - The revision of the node is not E or greater. In this case, the DRAM Hole
461 * Address Register does not exist.
463 * - The DramHoleValid bit is cleared in the DRAM Hole Address Register,
464 * indicating that its contents are not valid.
466 * The values passed back in *hole_base, *hole_offset, and *hole_size are
467 * complete 32-bit values despite the fact that the bitfields in the DHAR
468 * only represent bits 31-24 of the base and offset values.
470 int amd64_get_dram_hole_info(struct mem_ctl_info *mci, u64 *hole_base,
471 u64 *hole_offset, u64 *hole_size)
473 struct amd64_pvt *pvt = mci->pvt_info;
475 /* only revE and later have the DRAM Hole Address Register */
476 if (pvt->fam == 0xf && pvt->ext_model < K8_REV_E) {
477 edac_dbg(1, " revision %d for node %d does not support DHAR\n",
478 pvt->ext_model, pvt->mc_node_id);
482 /* valid for Fam10h and above */
483 if (pvt->fam >= 0x10 && !dhar_mem_hoist_valid(pvt)) {
484 edac_dbg(1, " Dram Memory Hoisting is DISABLED on this system\n");
488 if (!dhar_valid(pvt)) {
489 edac_dbg(1, " Dram Memory Hoisting is DISABLED on this node %d\n",
494 /* This node has Memory Hoisting */
496 /* +------------------+--------------------+--------------------+-----
497 * | memory | DRAM hole | relocated |
498 * | [0, (x - 1)] | [x, 0xffffffff] | addresses from |
500 * | | | [0x100000000, |
501 * | | | (0x100000000+ |
502 * | | | (0xffffffff-x))] |
503 * +------------------+--------------------+--------------------+-----
505 * Above is a diagram of physical memory showing the DRAM hole and the
506 * relocated addresses from the DRAM hole. As shown, the DRAM hole
507 * starts at address x (the base address) and extends through address
508 * 0xffffffff. The DRAM Hole Address Register (DHAR) relocates the
509 * addresses in the hole so that they start at 0x100000000.
512 *hole_base = dhar_base(pvt);
513 *hole_size = (1ULL << 32) - *hole_base;
515 *hole_offset = (pvt->fam > 0xf) ? f10_dhar_offset(pvt)
516 : k8_dhar_offset(pvt);
518 edac_dbg(1, " DHAR info for node %d base 0x%lx offset 0x%lx size 0x%lx\n",
519 pvt->mc_node_id, (unsigned long)*hole_base,
520 (unsigned long)*hole_offset, (unsigned long)*hole_size);
524 EXPORT_SYMBOL_GPL(amd64_get_dram_hole_info);
527 * Return the DramAddr that the SysAddr given by @sys_addr maps to. It is
528 * assumed that sys_addr maps to the node given by mci.
530 * The first part of section 3.4.4 (p. 70) shows how the DRAM Base (section
531 * 3.4.4.1) and DRAM Limit (section 3.4.4.2) registers are used to translate a
532 * SysAddr to a DramAddr. If the DRAM Hole Address Register (DHAR) is enabled,
533 * then it is also involved in translating a SysAddr to a DramAddr. Sections
534 * 3.4.8 and 3.5.8.2 describe the DHAR and how it is used for memory hoisting.
535 * These parts of the documentation are unclear. I interpret them as follows:
537 * When node n receives a SysAddr, it processes the SysAddr as follows:
539 * 1. It extracts the DRAMBase and DRAMLimit values from the DRAM Base and DRAM
540 * Limit registers for node n. If the SysAddr is not within the range
541 * specified by the base and limit values, then node n ignores the Sysaddr
542 * (since it does not map to node n). Otherwise continue to step 2 below.
544 * 2. If the DramHoleValid bit of the DHAR for node n is clear, the DHAR is
545 * disabled so skip to step 3 below. Otherwise see if the SysAddr is within
546 * the range of relocated addresses (starting at 0x100000000) from the DRAM
547 * hole. If not, skip to step 3 below. Else get the value of the
548 * DramHoleOffset field from the DHAR. To obtain the DramAddr, subtract the
549 * offset defined by this value from the SysAddr.
551 * 3. Obtain the base address for node n from the DRAMBase field of the DRAM
552 * Base register for node n. To obtain the DramAddr, subtract the base
553 * address from the SysAddr, as shown near the start of section 3.4.4 (p.70).
555 static u64 sys_addr_to_dram_addr(struct mem_ctl_info *mci, u64 sys_addr)
557 struct amd64_pvt *pvt = mci->pvt_info;
558 u64 dram_base, hole_base, hole_offset, hole_size, dram_addr;
561 dram_base = get_dram_base(pvt, pvt->mc_node_id);
563 ret = amd64_get_dram_hole_info(mci, &hole_base, &hole_offset,
566 if ((sys_addr >= (1ULL << 32)) &&
567 (sys_addr < ((1ULL << 32) + hole_size))) {
568 /* use DHAR to translate SysAddr to DramAddr */
569 dram_addr = sys_addr - hole_offset;
571 edac_dbg(2, "using DHAR to translate SysAddr 0x%lx to DramAddr 0x%lx\n",
572 (unsigned long)sys_addr,
573 (unsigned long)dram_addr);
580 * Translate the SysAddr to a DramAddr as shown near the start of
581 * section 3.4.4 (p. 70). Although sys_addr is a 64-bit value, the k8
582 * only deals with 40-bit values. Therefore we discard bits 63-40 of
583 * sys_addr below. If bit 39 of sys_addr is 1 then the bits we
584 * discard are all 1s. Otherwise the bits we discard are all 0s. See
585 * section 3.4.2 of AMD publication 24592: AMD x86-64 Architecture
586 * Programmer's Manual Volume 1 Application Programming.
588 dram_addr = (sys_addr & GENMASK_ULL(39, 0)) - dram_base;
590 edac_dbg(2, "using DRAM Base register to translate SysAddr 0x%lx to DramAddr 0x%lx\n",
591 (unsigned long)sys_addr, (unsigned long)dram_addr);
596 * @intlv_en is the value of the IntlvEn field from a DRAM Base register
597 * (section 3.4.4.1). Return the number of bits from a SysAddr that are used
598 * for node interleaving.
600 static int num_node_interleave_bits(unsigned intlv_en)
602 static const int intlv_shift_table[] = { 0, 1, 0, 2, 0, 0, 0, 3 };
605 BUG_ON(intlv_en > 7);
606 n = intlv_shift_table[intlv_en];
610 /* Translate the DramAddr given by @dram_addr to an InputAddr. */
611 static u64 dram_addr_to_input_addr(struct mem_ctl_info *mci, u64 dram_addr)
613 struct amd64_pvt *pvt;
620 * See the start of section 3.4.4 (p. 70, BKDG #26094, K8, revA-E)
621 * concerning translating a DramAddr to an InputAddr.
623 intlv_shift = num_node_interleave_bits(dram_intlv_en(pvt, 0));
624 input_addr = ((dram_addr >> intlv_shift) & GENMASK_ULL(35, 12)) +
627 edac_dbg(2, " Intlv Shift=%d DramAddr=0x%lx maps to InputAddr=0x%lx\n",
628 intlv_shift, (unsigned long)dram_addr,
629 (unsigned long)input_addr);
635 * Translate the SysAddr represented by @sys_addr to an InputAddr. It is
636 * assumed that @sys_addr maps to the node given by mci.
638 static u64 sys_addr_to_input_addr(struct mem_ctl_info *mci, u64 sys_addr)
643 dram_addr_to_input_addr(mci, sys_addr_to_dram_addr(mci, sys_addr));
645 edac_dbg(2, "SysAddr 0x%lx translates to InputAddr 0x%lx\n",
646 (unsigned long)sys_addr, (unsigned long)input_addr);
651 /* Map the Error address to a PAGE and PAGE OFFSET. */
652 static inline void error_address_to_page_and_offset(u64 error_address,
653 struct err_info *err)
655 err->page = (u32) (error_address >> PAGE_SHIFT);
656 err->offset = ((u32) error_address) & ~PAGE_MASK;
660 * @sys_addr is an error address (a SysAddr) extracted from the MCA NB Address
661 * Low (section 3.6.4.5) and MCA NB Address High (section 3.6.4.6) registers
662 * of a node that detected an ECC memory error. mci represents the node that
663 * the error address maps to (possibly different from the node that detected
664 * the error). Return the number of the csrow that sys_addr maps to, or -1 on
667 static int sys_addr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr)
671 csrow = input_addr_to_csrow(mci, sys_addr_to_input_addr(mci, sys_addr));
674 amd64_mc_err(mci, "Failed to translate InputAddr to csrow for "
675 "address 0x%lx\n", (unsigned long)sys_addr);
679 static int get_channel_from_ecc_syndrome(struct mem_ctl_info *, u16);
682 * Determine if the DIMMs have ECC enabled. ECC is enabled ONLY if all the DIMMs
685 static unsigned long determine_edac_cap(struct amd64_pvt *pvt)
688 unsigned long edac_cap = EDAC_FLAG_NONE;
690 bit = (pvt->fam > 0xf || pvt->ext_model >= K8_REV_F)
694 if (pvt->dclr0 & BIT(bit))
695 edac_cap = EDAC_FLAG_SECDED;
700 static void debug_display_dimm_sizes(struct amd64_pvt *, u8);
702 static void debug_dump_dramcfg_low(struct amd64_pvt *pvt, u32 dclr, int chan)
704 edac_dbg(1, "F2x%d90 (DRAM Cfg Low): 0x%08x\n", chan, dclr);
706 if (pvt->dram_type == MEM_LRDDR3) {
707 u32 dcsm = pvt->csels[chan].csmasks[0];
709 * It's assumed all LRDIMMs in a DCT are going to be of
710 * same 'type' until proven otherwise. So, use a cs
711 * value of '0' here to get dcsm value.
713 edac_dbg(1, " LRDIMM %dx rank multiply\n", (dcsm & 0x3));
716 edac_dbg(1, "All DIMMs support ECC:%s\n",
717 (dclr & BIT(19)) ? "yes" : "no");
720 edac_dbg(1, " PAR/ERR parity: %s\n",
721 (dclr & BIT(8)) ? "enabled" : "disabled");
723 if (pvt->fam == 0x10)
724 edac_dbg(1, " DCT 128bit mode width: %s\n",
725 (dclr & BIT(11)) ? "128b" : "64b");
727 edac_dbg(1, " x4 logical DIMMs present: L0: %s L1: %s L2: %s L3: %s\n",
728 (dclr & BIT(12)) ? "yes" : "no",
729 (dclr & BIT(13)) ? "yes" : "no",
730 (dclr & BIT(14)) ? "yes" : "no",
731 (dclr & BIT(15)) ? "yes" : "no");
734 /* Display and decode various NB registers for debug purposes. */
735 static void dump_misc_regs(struct amd64_pvt *pvt)
737 edac_dbg(1, "F3xE8 (NB Cap): 0x%08x\n", pvt->nbcap);
739 edac_dbg(1, " NB two channel DRAM capable: %s\n",
740 (pvt->nbcap & NBCAP_DCT_DUAL) ? "yes" : "no");
742 edac_dbg(1, " ECC capable: %s, ChipKill ECC capable: %s\n",
743 (pvt->nbcap & NBCAP_SECDED) ? "yes" : "no",
744 (pvt->nbcap & NBCAP_CHIPKILL) ? "yes" : "no");
746 debug_dump_dramcfg_low(pvt, pvt->dclr0, 0);
748 edac_dbg(1, "F3xB0 (Online Spare): 0x%08x\n", pvt->online_spare);
750 edac_dbg(1, "F1xF0 (DRAM Hole Address): 0x%08x, base: 0x%08x, offset: 0x%08x\n",
751 pvt->dhar, dhar_base(pvt),
752 (pvt->fam == 0xf) ? k8_dhar_offset(pvt)
753 : f10_dhar_offset(pvt));
755 edac_dbg(1, " DramHoleValid: %s\n", dhar_valid(pvt) ? "yes" : "no");
757 debug_display_dimm_sizes(pvt, 0);
759 /* everything below this point is Fam10h and above */
763 debug_display_dimm_sizes(pvt, 1);
765 amd64_info("using %s syndromes.\n", ((pvt->ecc_sym_sz == 8) ? "x8" : "x4"));
767 /* Only if NOT ganged does dclr1 have valid info */
768 if (!dct_ganging_enabled(pvt))
769 debug_dump_dramcfg_low(pvt, pvt->dclr1, 1);
773 * See BKDG, F2x[1,0][5C:40], F2[1,0][6C:60]
775 static void prep_chip_selects(struct amd64_pvt *pvt)
777 if (pvt->fam == 0xf && pvt->ext_model < K8_REV_F) {
778 pvt->csels[0].b_cnt = pvt->csels[1].b_cnt = 8;
779 pvt->csels[0].m_cnt = pvt->csels[1].m_cnt = 8;
780 } else if (pvt->fam == 0x15 && pvt->model == 0x30) {
781 pvt->csels[0].b_cnt = pvt->csels[1].b_cnt = 4;
782 pvt->csels[0].m_cnt = pvt->csels[1].m_cnt = 2;
784 pvt->csels[0].b_cnt = pvt->csels[1].b_cnt = 8;
785 pvt->csels[0].m_cnt = pvt->csels[1].m_cnt = 4;
790 * Function 2 Offset F10_DCSB0; read in the DCS Base and DCS Mask registers
792 static void read_dct_base_mask(struct amd64_pvt *pvt)
796 prep_chip_selects(pvt);
798 for_each_chip_select(cs, 0, pvt) {
799 int reg0 = DCSB0 + (cs * 4);
800 int reg1 = DCSB1 + (cs * 4);
801 u32 *base0 = &pvt->csels[0].csbases[cs];
802 u32 *base1 = &pvt->csels[1].csbases[cs];
804 if (!amd64_read_dct_pci_cfg(pvt, 0, reg0, base0))
805 edac_dbg(0, " DCSB0[%d]=0x%08x reg: F2x%x\n",
811 if (!amd64_read_dct_pci_cfg(pvt, 1, reg0, base1))
812 edac_dbg(0, " DCSB1[%d]=0x%08x reg: F2x%x\n",
813 cs, *base1, (pvt->fam == 0x10) ? reg1
817 for_each_chip_select_mask(cs, 0, pvt) {
818 int reg0 = DCSM0 + (cs * 4);
819 int reg1 = DCSM1 + (cs * 4);
820 u32 *mask0 = &pvt->csels[0].csmasks[cs];
821 u32 *mask1 = &pvt->csels[1].csmasks[cs];
823 if (!amd64_read_dct_pci_cfg(pvt, 0, reg0, mask0))
824 edac_dbg(0, " DCSM0[%d]=0x%08x reg: F2x%x\n",
830 if (!amd64_read_dct_pci_cfg(pvt, 1, reg0, mask1))
831 edac_dbg(0, " DCSM1[%d]=0x%08x reg: F2x%x\n",
832 cs, *mask1, (pvt->fam == 0x10) ? reg1
837 static void determine_memory_type(struct amd64_pvt *pvt)
843 if (pvt->ext_model >= K8_REV_F)
846 pvt->dram_type = (pvt->dclr0 & BIT(18)) ? MEM_DDR : MEM_RDDR;
850 if (pvt->dchr0 & DDR3_MODE)
853 pvt->dram_type = (pvt->dclr0 & BIT(16)) ? MEM_DDR2 : MEM_RDDR2;
857 if (pvt->model < 0x60)
861 * Model 0x60h needs special handling:
863 * We use a Chip Select value of '0' to obtain dcsm.
864 * Theoretically, it is possible to populate LRDIMMs of different
865 * 'Rank' value on a DCT. But this is not the common case. So,
866 * it's reasonable to assume all DIMMs are going to be of same
867 * 'type' until proven otherwise.
869 amd64_read_dct_pci_cfg(pvt, 0, DRAM_CONTROL, &dram_ctrl);
870 dcsm = pvt->csels[0].csmasks[0];
872 if (((dram_ctrl >> 8) & 0x7) == 0x2)
873 pvt->dram_type = MEM_DDR4;
874 else if (pvt->dclr0 & BIT(16))
875 pvt->dram_type = MEM_DDR3;
877 pvt->dram_type = MEM_LRDDR3;
879 pvt->dram_type = MEM_RDDR3;
887 WARN(1, KERN_ERR "%s: Family??? 0x%x\n", __func__, pvt->fam);
888 pvt->dram_type = MEM_EMPTY;
893 pvt->dram_type = (pvt->dclr0 & BIT(16)) ? MEM_DDR3 : MEM_RDDR3;
896 /* Get the number of DCT channels the memory controller is using. */
897 static int k8_early_channel_count(struct amd64_pvt *pvt)
901 if (pvt->ext_model >= K8_REV_F)
902 /* RevF (NPT) and later */
903 flag = pvt->dclr0 & WIDTH_128;
905 /* RevE and earlier */
906 flag = pvt->dclr0 & REVE_WIDTH_128;
911 return (flag) ? 2 : 1;
914 /* On F10h and later ErrAddr is MC4_ADDR[47:1] */
915 static u64 get_error_address(struct amd64_pvt *pvt, struct mce *m)
917 u16 mce_nid = amd_get_nb_id(m->extcpu);
918 struct mem_ctl_info *mci;
923 mci = edac_mc_find(mce_nid);
929 if (pvt->fam == 0xf) {
934 addr = m->addr & GENMASK_ULL(end_bit, start_bit);
937 * Erratum 637 workaround
939 if (pvt->fam == 0x15) {
940 u64 cc6_base, tmp_addr;
944 if ((addr & GENMASK_ULL(47, 24)) >> 24 != 0x00fdf7)
948 amd64_read_pci_cfg(pvt->F1, DRAM_LOCAL_NODE_LIM, &tmp);
949 intlv_en = tmp >> 21 & 0x7;
951 /* add [47:27] + 3 trailing bits */
952 cc6_base = (tmp & GENMASK_ULL(20, 0)) << 3;
954 /* reverse and add DramIntlvEn */
955 cc6_base |= intlv_en ^ 0x7;
961 return cc6_base | (addr & GENMASK_ULL(23, 0));
963 amd64_read_pci_cfg(pvt->F1, DRAM_LOCAL_NODE_BASE, &tmp);
966 tmp_addr = (addr & GENMASK_ULL(23, 12)) << __fls(intlv_en + 1);
968 /* OR DramIntlvSel into bits [14:12] */
969 tmp_addr |= (tmp & GENMASK_ULL(23, 21)) >> 9;
971 /* add remaining [11:0] bits from original MC4_ADDR */
972 tmp_addr |= addr & GENMASK_ULL(11, 0);
974 return cc6_base | tmp_addr;
980 static struct pci_dev *pci_get_related_function(unsigned int vendor,
982 struct pci_dev *related)
984 struct pci_dev *dev = NULL;
986 while ((dev = pci_get_device(vendor, device, dev))) {
987 if (pci_domain_nr(dev->bus) == pci_domain_nr(related->bus) &&
988 (dev->bus->number == related->bus->number) &&
989 (PCI_SLOT(dev->devfn) == PCI_SLOT(related->devfn)))
996 static void read_dram_base_limit_regs(struct amd64_pvt *pvt, unsigned range)
998 struct amd_northbridge *nb;
999 struct pci_dev *f1 = NULL;
1000 unsigned int pci_func;
1001 int off = range << 3;
1004 amd64_read_pci_cfg(pvt->F1, DRAM_BASE_LO + off, &pvt->ranges[range].base.lo);
1005 amd64_read_pci_cfg(pvt->F1, DRAM_LIMIT_LO + off, &pvt->ranges[range].lim.lo);
1007 if (pvt->fam == 0xf)
1010 if (!dram_rw(pvt, range))
1013 amd64_read_pci_cfg(pvt->F1, DRAM_BASE_HI + off, &pvt->ranges[range].base.hi);
1014 amd64_read_pci_cfg(pvt->F1, DRAM_LIMIT_HI + off, &pvt->ranges[range].lim.hi);
1016 /* F15h: factor in CC6 save area by reading dst node's limit reg */
1017 if (pvt->fam != 0x15)
1020 nb = node_to_amd_nb(dram_dst_node(pvt, range));
1024 if (pvt->model == 0x60)
1025 pci_func = PCI_DEVICE_ID_AMD_15H_M60H_NB_F1;
1026 else if (pvt->model == 0x30)
1027 pci_func = PCI_DEVICE_ID_AMD_15H_M30H_NB_F1;
1029 pci_func = PCI_DEVICE_ID_AMD_15H_NB_F1;
1031 f1 = pci_get_related_function(nb->misc->vendor, pci_func, nb->misc);
1035 amd64_read_pci_cfg(f1, DRAM_LOCAL_NODE_LIM, &llim);
1037 pvt->ranges[range].lim.lo &= GENMASK_ULL(15, 0);
1039 /* {[39:27],111b} */
1040 pvt->ranges[range].lim.lo |= ((llim & 0x1fff) << 3 | 0x7) << 16;
1042 pvt->ranges[range].lim.hi &= GENMASK_ULL(7, 0);
1045 pvt->ranges[range].lim.hi |= llim >> 13;
1050 static void k8_map_sysaddr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr,
1051 struct err_info *err)
1053 struct amd64_pvt *pvt = mci->pvt_info;
1055 error_address_to_page_and_offset(sys_addr, err);
1058 * Find out which node the error address belongs to. This may be
1059 * different from the node that detected the error.
1061 err->src_mci = find_mc_by_sys_addr(mci, sys_addr);
1062 if (!err->src_mci) {
1063 amd64_mc_err(mci, "failed to map error addr 0x%lx to a node\n",
1064 (unsigned long)sys_addr);
1065 err->err_code = ERR_NODE;
1069 /* Now map the sys_addr to a CSROW */
1070 err->csrow = sys_addr_to_csrow(err->src_mci, sys_addr);
1071 if (err->csrow < 0) {
1072 err->err_code = ERR_CSROW;
1076 /* CHIPKILL enabled */
1077 if (pvt->nbcfg & NBCFG_CHIPKILL) {
1078 err->channel = get_channel_from_ecc_syndrome(mci, err->syndrome);
1079 if (err->channel < 0) {
1081 * Syndrome didn't map, so we don't know which of the
1082 * 2 DIMMs is in error. So we need to ID 'both' of them
1085 amd64_mc_warn(err->src_mci, "unknown syndrome 0x%04x - "
1086 "possible error reporting race\n",
1088 err->err_code = ERR_CHANNEL;
1093 * non-chipkill ecc mode
1095 * The k8 documentation is unclear about how to determine the
1096 * channel number when using non-chipkill memory. This method
1097 * was obtained from email communication with someone at AMD.
1098 * (Wish the email was placed in this comment - norsk)
1100 err->channel = ((sys_addr & BIT(3)) != 0);
1104 static int ddr2_cs_size(unsigned i, bool dct_width)
1110 else if (!(i & 0x1))
1113 shift = (i + 1) >> 1;
1115 return 128 << (shift + !!dct_width);
1118 static int k8_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,
1119 unsigned cs_mode, int cs_mask_nr)
1121 u32 dclr = dct ? pvt->dclr1 : pvt->dclr0;
1123 if (pvt->ext_model >= K8_REV_F) {
1124 WARN_ON(cs_mode > 11);
1125 return ddr2_cs_size(cs_mode, dclr & WIDTH_128);
1127 else if (pvt->ext_model >= K8_REV_D) {
1129 WARN_ON(cs_mode > 10);
1132 * the below calculation, besides trying to win an obfuscated C
1133 * contest, maps cs_mode values to DIMM chip select sizes. The
1136 * cs_mode CS size (mb)
1137 * ======= ============
1150 * Basically, it calculates a value with which to shift the
1151 * smallest CS size of 32MB.
1153 * ddr[23]_cs_size have a similar purpose.
1155 diff = cs_mode/3 + (unsigned)(cs_mode > 5);
1157 return 32 << (cs_mode - diff);
1160 WARN_ON(cs_mode > 6);
1161 return 32 << cs_mode;
1166 * Get the number of DCT channels in use.
1169 * number of Memory Channels in operation
1171 * contents of the DCL0_LOW register
1173 static int f1x_early_channel_count(struct amd64_pvt *pvt)
1175 int i, j, channels = 0;
1177 /* On F10h, if we are in 128 bit mode, then we are using 2 channels */
1178 if (pvt->fam == 0x10 && (pvt->dclr0 & WIDTH_128))
1182 * Need to check if in unganged mode: In such, there are 2 channels,
1183 * but they are not in 128 bit mode and thus the above 'dclr0' status
1186 * Need to check DCT0[0] and DCT1[0] to see if only one of them has
1187 * their CSEnable bit on. If so, then SINGLE DIMM case.
1189 edac_dbg(0, "Data width is not 128 bits - need more decoding\n");
1192 * Check DRAM Bank Address Mapping values for each DIMM to see if there
1193 * is more than just one DIMM present in unganged mode. Need to check
1194 * both controllers since DIMMs can be placed in either one.
1196 for (i = 0; i < 2; i++) {
1197 u32 dbam = (i ? pvt->dbam1 : pvt->dbam0);
1199 for (j = 0; j < 4; j++) {
1200 if (DBAM_DIMM(j, dbam) > 0) {
1210 amd64_info("MCT channel count: %d\n", channels);
1215 static int ddr3_cs_size(unsigned i, bool dct_width)
1220 if (i == 0 || i == 3 || i == 4)
1226 else if (!(i & 0x1))
1229 shift = (i + 1) >> 1;
1232 cs_size = (128 * (1 << !!dct_width)) << shift;
1237 static int ddr3_lrdimm_cs_size(unsigned i, unsigned rank_multiply)
1242 if (i < 4 || i == 6)
1246 else if (!(i & 0x1))
1249 shift = (i + 1) >> 1;
1252 cs_size = rank_multiply * (128 << shift);
1257 static int ddr4_cs_size(unsigned i)
1266 /* Min cs_size = 1G */
1267 cs_size = 1024 * (1 << (i >> 1));
1272 static int f10_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,
1273 unsigned cs_mode, int cs_mask_nr)
1275 u32 dclr = dct ? pvt->dclr1 : pvt->dclr0;
1277 WARN_ON(cs_mode > 11);
1279 if (pvt->dchr0 & DDR3_MODE || pvt->dchr1 & DDR3_MODE)
1280 return ddr3_cs_size(cs_mode, dclr & WIDTH_128);
1282 return ddr2_cs_size(cs_mode, dclr & WIDTH_128);
1286 * F15h supports only 64bit DCT interfaces
1288 static int f15_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,
1289 unsigned cs_mode, int cs_mask_nr)
1291 WARN_ON(cs_mode > 12);
1293 return ddr3_cs_size(cs_mode, false);
1296 /* F15h M60h supports DDR4 mapping as well.. */
1297 static int f15_m60h_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,
1298 unsigned cs_mode, int cs_mask_nr)
1301 u32 dcsm = pvt->csels[dct].csmasks[cs_mask_nr];
1303 WARN_ON(cs_mode > 12);
1305 if (pvt->dram_type == MEM_DDR4) {
1309 cs_size = ddr4_cs_size(cs_mode);
1310 } else if (pvt->dram_type == MEM_LRDDR3) {
1311 unsigned rank_multiply = dcsm & 0xf;
1313 if (rank_multiply == 3)
1315 cs_size = ddr3_lrdimm_cs_size(cs_mode, rank_multiply);
1317 /* Minimum cs size is 512mb for F15hM60h*/
1321 cs_size = ddr3_cs_size(cs_mode, false);
1328 * F16h and F15h model 30h have only limited cs_modes.
1330 static int f16_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,
1331 unsigned cs_mode, int cs_mask_nr)
1333 WARN_ON(cs_mode > 12);
1335 if (cs_mode == 6 || cs_mode == 8 ||
1336 cs_mode == 9 || cs_mode == 12)
1339 return ddr3_cs_size(cs_mode, false);
1342 static void read_dram_ctl_register(struct amd64_pvt *pvt)
1345 if (pvt->fam == 0xf)
1348 if (!amd64_read_pci_cfg(pvt->F2, DCT_SEL_LO, &pvt->dct_sel_lo)) {
1349 edac_dbg(0, "F2x110 (DCTSelLow): 0x%08x, High range addrs at: 0x%x\n",
1350 pvt->dct_sel_lo, dct_sel_baseaddr(pvt));
1352 edac_dbg(0, " DCTs operate in %s mode\n",
1353 (dct_ganging_enabled(pvt) ? "ganged" : "unganged"));
1355 if (!dct_ganging_enabled(pvt))
1356 edac_dbg(0, " Address range split per DCT: %s\n",
1357 (dct_high_range_enabled(pvt) ? "yes" : "no"));
1359 edac_dbg(0, " data interleave for ECC: %s, DRAM cleared since last warm reset: %s\n",
1360 (dct_data_intlv_enabled(pvt) ? "enabled" : "disabled"),
1361 (dct_memory_cleared(pvt) ? "yes" : "no"));
1363 edac_dbg(0, " channel interleave: %s, "
1364 "interleave bits selector: 0x%x\n",
1365 (dct_interleave_enabled(pvt) ? "enabled" : "disabled"),
1366 dct_sel_interleave_addr(pvt));
1369 amd64_read_pci_cfg(pvt->F2, DCT_SEL_HI, &pvt->dct_sel_hi);
1373 * Determine channel (DCT) based on the interleaving mode (see F15h M30h BKDG,
1374 * 2.10.12 Memory Interleaving Modes).
1376 static u8 f15_m30h_determine_channel(struct amd64_pvt *pvt, u64 sys_addr,
1377 u8 intlv_en, int num_dcts_intlv,
1384 return (u8)(dct_sel);
1386 if (num_dcts_intlv == 2) {
1387 select = (sys_addr >> 8) & 0x3;
1388 channel = select ? 0x3 : 0;
1389 } else if (num_dcts_intlv == 4) {
1390 u8 intlv_addr = dct_sel_interleave_addr(pvt);
1391 switch (intlv_addr) {
1393 channel = (sys_addr >> 8) & 0x3;
1396 channel = (sys_addr >> 9) & 0x3;
1404 * Determine channel (DCT) based on the interleaving mode: F10h BKDG, 2.8.9 Memory
1405 * Interleaving Modes.
1407 static u8 f1x_determine_channel(struct amd64_pvt *pvt, u64 sys_addr,
1408 bool hi_range_sel, u8 intlv_en)
1410 u8 dct_sel_high = (pvt->dct_sel_lo >> 1) & 1;
1412 if (dct_ganging_enabled(pvt))
1416 return dct_sel_high;
1419 * see F2x110[DctSelIntLvAddr] - channel interleave mode
1421 if (dct_interleave_enabled(pvt)) {
1422 u8 intlv_addr = dct_sel_interleave_addr(pvt);
1424 /* return DCT select function: 0=DCT0, 1=DCT1 */
1426 return sys_addr >> 6 & 1;
1428 if (intlv_addr & 0x2) {
1429 u8 shift = intlv_addr & 0x1 ? 9 : 6;
1430 u32 temp = hweight_long((u32) ((sys_addr >> 16) & 0x1F)) & 1;
1432 return ((sys_addr >> shift) & 1) ^ temp;
1435 if (intlv_addr & 0x4) {
1436 u8 shift = intlv_addr & 0x1 ? 9 : 8;
1438 return (sys_addr >> shift) & 1;
1441 return (sys_addr >> (12 + hweight8(intlv_en))) & 1;
1444 if (dct_high_range_enabled(pvt))
1445 return ~dct_sel_high & 1;
1450 /* Convert the sys_addr to the normalized DCT address */
1451 static u64 f1x_get_norm_dct_addr(struct amd64_pvt *pvt, u8 range,
1452 u64 sys_addr, bool hi_rng,
1453 u32 dct_sel_base_addr)
1456 u64 dram_base = get_dram_base(pvt, range);
1457 u64 hole_off = f10_dhar_offset(pvt);
1458 u64 dct_sel_base_off = (u64)(pvt->dct_sel_hi & 0xFFFFFC00) << 16;
1463 * base address of high range is below 4Gb
1464 * (bits [47:27] at [31:11])
1465 * DRAM address space on this DCT is hoisted above 4Gb &&
1468 * remove hole offset from sys_addr
1470 * remove high range offset from sys_addr
1472 if ((!(dct_sel_base_addr >> 16) ||
1473 dct_sel_base_addr < dhar_base(pvt)) &&
1475 (sys_addr >= BIT_64(32)))
1476 chan_off = hole_off;
1478 chan_off = dct_sel_base_off;
1482 * we have a valid hole &&
1487 * remove dram base to normalize to DCT address
1489 if (dhar_valid(pvt) && (sys_addr >= BIT_64(32)))
1490 chan_off = hole_off;
1492 chan_off = dram_base;
1495 return (sys_addr & GENMASK_ULL(47,6)) - (chan_off & GENMASK_ULL(47,23));
1499 * checks if the csrow passed in is marked as SPARED, if so returns the new
1502 static int f10_process_possible_spare(struct amd64_pvt *pvt, u8 dct, int csrow)
1506 if (online_spare_swap_done(pvt, dct) &&
1507 csrow == online_spare_bad_dramcs(pvt, dct)) {
1509 for_each_chip_select(tmp_cs, dct, pvt) {
1510 if (chip_select_base(tmp_cs, dct, pvt) & 0x2) {
1520 * Iterate over the DRAM DCT "base" and "mask" registers looking for a
1521 * SystemAddr match on the specified 'ChannelSelect' and 'NodeID'
1524 * -EINVAL: NOT FOUND
1525 * 0..csrow = Chip-Select Row
1527 static int f1x_lookup_addr_in_dct(u64 in_addr, u8 nid, u8 dct)
1529 struct mem_ctl_info *mci;
1530 struct amd64_pvt *pvt;
1531 u64 cs_base, cs_mask;
1532 int cs_found = -EINVAL;
1535 mci = edac_mc_find(nid);
1539 pvt = mci->pvt_info;
1541 edac_dbg(1, "input addr: 0x%llx, DCT: %d\n", in_addr, dct);
1543 for_each_chip_select(csrow, dct, pvt) {
1544 if (!csrow_enabled(csrow, dct, pvt))
1547 get_cs_base_and_mask(pvt, csrow, dct, &cs_base, &cs_mask);
1549 edac_dbg(1, " CSROW=%d CSBase=0x%llx CSMask=0x%llx\n",
1550 csrow, cs_base, cs_mask);
1554 edac_dbg(1, " (InputAddr & ~CSMask)=0x%llx (CSBase & ~CSMask)=0x%llx\n",
1555 (in_addr & cs_mask), (cs_base & cs_mask));
1557 if ((in_addr & cs_mask) == (cs_base & cs_mask)) {
1558 if (pvt->fam == 0x15 && pvt->model >= 0x30) {
1562 cs_found = f10_process_possible_spare(pvt, dct, csrow);
1564 edac_dbg(1, " MATCH csrow=%d\n", cs_found);
1572 * See F2x10C. Non-interleaved graphics framebuffer memory under the 16G is
1573 * swapped with a region located at the bottom of memory so that the GPU can use
1574 * the interleaved region and thus two channels.
1576 static u64 f1x_swap_interleaved_region(struct amd64_pvt *pvt, u64 sys_addr)
1578 u32 swap_reg, swap_base, swap_limit, rgn_size, tmp_addr;
1580 if (pvt->fam == 0x10) {
1581 /* only revC3 and revE have that feature */
1582 if (pvt->model < 4 || (pvt->model < 0xa && pvt->stepping < 3))
1586 amd64_read_pci_cfg(pvt->F2, SWAP_INTLV_REG, &swap_reg);
1588 if (!(swap_reg & 0x1))
1591 swap_base = (swap_reg >> 3) & 0x7f;
1592 swap_limit = (swap_reg >> 11) & 0x7f;
1593 rgn_size = (swap_reg >> 20) & 0x7f;
1594 tmp_addr = sys_addr >> 27;
1596 if (!(sys_addr >> 34) &&
1597 (((tmp_addr >= swap_base) &&
1598 (tmp_addr <= swap_limit)) ||
1599 (tmp_addr < rgn_size)))
1600 return sys_addr ^ (u64)swap_base << 27;
1605 /* For a given @dram_range, check if @sys_addr falls within it. */
1606 static int f1x_match_to_this_node(struct amd64_pvt *pvt, unsigned range,
1607 u64 sys_addr, int *chan_sel)
1609 int cs_found = -EINVAL;
1613 bool high_range = false;
1615 u8 node_id = dram_dst_node(pvt, range);
1616 u8 intlv_en = dram_intlv_en(pvt, range);
1617 u32 intlv_sel = dram_intlv_sel(pvt, range);
1619 edac_dbg(1, "(range %d) SystemAddr= 0x%llx Limit=0x%llx\n",
1620 range, sys_addr, get_dram_limit(pvt, range));
1622 if (dhar_valid(pvt) &&
1623 dhar_base(pvt) <= sys_addr &&
1624 sys_addr < BIT_64(32)) {
1625 amd64_warn("Huh? Address is in the MMIO hole: 0x%016llx\n",
1630 if (intlv_en && (intlv_sel != ((sys_addr >> 12) & intlv_en)))
1633 sys_addr = f1x_swap_interleaved_region(pvt, sys_addr);
1635 dct_sel_base = dct_sel_baseaddr(pvt);
1638 * check whether addresses >= DctSelBaseAddr[47:27] are to be used to
1639 * select between DCT0 and DCT1.
1641 if (dct_high_range_enabled(pvt) &&
1642 !dct_ganging_enabled(pvt) &&
1643 ((sys_addr >> 27) >= (dct_sel_base >> 11)))
1646 channel = f1x_determine_channel(pvt, sys_addr, high_range, intlv_en);
1648 chan_addr = f1x_get_norm_dct_addr(pvt, range, sys_addr,
1649 high_range, dct_sel_base);
1651 /* Remove node interleaving, see F1x120 */
1653 chan_addr = ((chan_addr >> (12 + hweight8(intlv_en))) << 12) |
1654 (chan_addr & 0xfff);
1656 /* remove channel interleave */
1657 if (dct_interleave_enabled(pvt) &&
1658 !dct_high_range_enabled(pvt) &&
1659 !dct_ganging_enabled(pvt)) {
1661 if (dct_sel_interleave_addr(pvt) != 1) {
1662 if (dct_sel_interleave_addr(pvt) == 0x3)
1664 chan_addr = ((chan_addr >> 10) << 9) |
1665 (chan_addr & 0x1ff);
1667 /* A[6] or hash 6 */
1668 chan_addr = ((chan_addr >> 7) << 6) |
1672 chan_addr = ((chan_addr >> 13) << 12) |
1673 (chan_addr & 0xfff);
1676 edac_dbg(1, " Normalized DCT addr: 0x%llx\n", chan_addr);
1678 cs_found = f1x_lookup_addr_in_dct(chan_addr, node_id, channel);
1681 *chan_sel = channel;
1686 static int f15_m30h_match_to_this_node(struct amd64_pvt *pvt, unsigned range,
1687 u64 sys_addr, int *chan_sel)
1689 int cs_found = -EINVAL;
1690 int num_dcts_intlv = 0;
1691 u64 chan_addr, chan_offset;
1692 u64 dct_base, dct_limit;
1693 u32 dct_cont_base_reg, dct_cont_limit_reg, tmp;
1694 u8 channel, alias_channel, leg_mmio_hole, dct_sel, dct_offset_en;
1696 u64 dhar_offset = f10_dhar_offset(pvt);
1697 u8 intlv_addr = dct_sel_interleave_addr(pvt);
1698 u8 node_id = dram_dst_node(pvt, range);
1699 u8 intlv_en = dram_intlv_en(pvt, range);
1701 amd64_read_pci_cfg(pvt->F1, DRAM_CONT_BASE, &dct_cont_base_reg);
1702 amd64_read_pci_cfg(pvt->F1, DRAM_CONT_LIMIT, &dct_cont_limit_reg);
1704 dct_offset_en = (u8) ((dct_cont_base_reg >> 3) & BIT(0));
1705 dct_sel = (u8) ((dct_cont_base_reg >> 4) & 0x7);
1707 edac_dbg(1, "(range %d) SystemAddr= 0x%llx Limit=0x%llx\n",
1708 range, sys_addr, get_dram_limit(pvt, range));
1710 if (!(get_dram_base(pvt, range) <= sys_addr) &&
1711 !(get_dram_limit(pvt, range) >= sys_addr))
1714 if (dhar_valid(pvt) &&
1715 dhar_base(pvt) <= sys_addr &&
1716 sys_addr < BIT_64(32)) {
1717 amd64_warn("Huh? Address is in the MMIO hole: 0x%016llx\n",
1722 /* Verify sys_addr is within DCT Range. */
1723 dct_base = (u64) dct_sel_baseaddr(pvt);
1724 dct_limit = (dct_cont_limit_reg >> 11) & 0x1FFF;
1726 if (!(dct_cont_base_reg & BIT(0)) &&
1727 !(dct_base <= (sys_addr >> 27) &&
1728 dct_limit >= (sys_addr >> 27)))
1731 /* Verify number of dct's that participate in channel interleaving. */
1732 num_dcts_intlv = (int) hweight8(intlv_en);
1734 if (!(num_dcts_intlv % 2 == 0) || (num_dcts_intlv > 4))
1737 if (pvt->model >= 0x60)
1738 channel = f1x_determine_channel(pvt, sys_addr, false, intlv_en);
1740 channel = f15_m30h_determine_channel(pvt, sys_addr, intlv_en,
1741 num_dcts_intlv, dct_sel);
1743 /* Verify we stay within the MAX number of channels allowed */
1747 leg_mmio_hole = (u8) (dct_cont_base_reg >> 1 & BIT(0));
1749 /* Get normalized DCT addr */
1750 if (leg_mmio_hole && (sys_addr >= BIT_64(32)))
1751 chan_offset = dhar_offset;
1753 chan_offset = dct_base << 27;
1755 chan_addr = sys_addr - chan_offset;
1757 /* remove channel interleave */
1758 if (num_dcts_intlv == 2) {
1759 if (intlv_addr == 0x4)
1760 chan_addr = ((chan_addr >> 9) << 8) |
1762 else if (intlv_addr == 0x5)
1763 chan_addr = ((chan_addr >> 10) << 9) |
1764 (chan_addr & 0x1ff);
1768 } else if (num_dcts_intlv == 4) {
1769 if (intlv_addr == 0x4)
1770 chan_addr = ((chan_addr >> 10) << 8) |
1772 else if (intlv_addr == 0x5)
1773 chan_addr = ((chan_addr >> 11) << 9) |
1774 (chan_addr & 0x1ff);
1779 if (dct_offset_en) {
1780 amd64_read_pci_cfg(pvt->F1,
1781 DRAM_CONT_HIGH_OFF + (int) channel * 4,
1783 chan_addr += (u64) ((tmp >> 11) & 0xfff) << 27;
1786 f15h_select_dct(pvt, channel);
1788 edac_dbg(1, " Normalized DCT addr: 0x%llx\n", chan_addr);
1792 * if channel = 3, then alias it to 1. This is because, in F15 M30h,
1793 * there is support for 4 DCT's, but only 2 are currently functional.
1794 * They are DCT0 and DCT3. But we have read all registers of DCT3 into
1795 * pvt->csels[1]. So we need to use '1' here to get correct info.
1796 * Refer F15 M30h BKDG Section 2.10 and 2.10.3 for clarifications.
1798 alias_channel = (channel == 3) ? 1 : channel;
1800 cs_found = f1x_lookup_addr_in_dct(chan_addr, node_id, alias_channel);
1803 *chan_sel = alias_channel;
1808 static int f1x_translate_sysaddr_to_cs(struct amd64_pvt *pvt,
1812 int cs_found = -EINVAL;
1815 for (range = 0; range < DRAM_RANGES; range++) {
1816 if (!dram_rw(pvt, range))
1819 if (pvt->fam == 0x15 && pvt->model >= 0x30)
1820 cs_found = f15_m30h_match_to_this_node(pvt, range,
1824 else if ((get_dram_base(pvt, range) <= sys_addr) &&
1825 (get_dram_limit(pvt, range) >= sys_addr)) {
1826 cs_found = f1x_match_to_this_node(pvt, range,
1827 sys_addr, chan_sel);
1836 * For reference see "2.8.5 Routing DRAM Requests" in F10 BKDG. This code maps
1837 * a @sys_addr to NodeID, DCT (channel) and chip select (CSROW).
1839 * The @sys_addr is usually an error address received from the hardware
1842 static void f1x_map_sysaddr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr,
1843 struct err_info *err)
1845 struct amd64_pvt *pvt = mci->pvt_info;
1847 error_address_to_page_and_offset(sys_addr, err);
1849 err->csrow = f1x_translate_sysaddr_to_cs(pvt, sys_addr, &err->channel);
1850 if (err->csrow < 0) {
1851 err->err_code = ERR_CSROW;
1856 * We need the syndromes for channel detection only when we're
1857 * ganged. Otherwise @chan should already contain the channel at
1860 if (dct_ganging_enabled(pvt))
1861 err->channel = get_channel_from_ecc_syndrome(mci, err->syndrome);
1865 * debug routine to display the memory sizes of all logical DIMMs and its
1868 static void debug_display_dimm_sizes(struct amd64_pvt *pvt, u8 ctrl)
1870 int dimm, size0, size1;
1871 u32 *dcsb = ctrl ? pvt->csels[1].csbases : pvt->csels[0].csbases;
1872 u32 dbam = ctrl ? pvt->dbam1 : pvt->dbam0;
1874 if (pvt->fam == 0xf) {
1875 /* K8 families < revF not supported yet */
1876 if (pvt->ext_model < K8_REV_F)
1882 if (pvt->fam == 0x10) {
1883 dbam = (ctrl && !dct_ganging_enabled(pvt)) ? pvt->dbam1
1885 dcsb = (ctrl && !dct_ganging_enabled(pvt)) ?
1886 pvt->csels[1].csbases :
1887 pvt->csels[0].csbases;
1890 dcsb = pvt->csels[1].csbases;
1892 edac_dbg(1, "F2x%d80 (DRAM Bank Address Mapping): 0x%08x\n",
1895 edac_printk(KERN_DEBUG, EDAC_MC, "DCT%d chip selects:\n", ctrl);
1897 /* Dump memory sizes for DIMM and its CSROWs */
1898 for (dimm = 0; dimm < 4; dimm++) {
1901 if (dcsb[dimm*2] & DCSB_CS_ENABLE)
1902 /* For f15m60h, need multiplier for LRDIMM cs_size
1903 * calculation. We pass 'dimm' value to the dbam_to_cs
1904 * mapper so we can find the multiplier from the
1905 * corresponding DCSM.
1907 size0 = pvt->ops->dbam_to_cs(pvt, ctrl,
1908 DBAM_DIMM(dimm, dbam),
1912 if (dcsb[dimm*2 + 1] & DCSB_CS_ENABLE)
1913 size1 = pvt->ops->dbam_to_cs(pvt, ctrl,
1914 DBAM_DIMM(dimm, dbam),
1917 amd64_info(EDAC_MC ": %d: %5dMB %d: %5dMB\n",
1919 dimm * 2 + 1, size1);
1923 static struct amd64_family_type family_types[] = {
1926 .f1_id = PCI_DEVICE_ID_AMD_K8_NB_ADDRMAP,
1927 .f2_id = PCI_DEVICE_ID_AMD_K8_NB_MEMCTL,
1929 .early_channel_count = k8_early_channel_count,
1930 .map_sysaddr_to_csrow = k8_map_sysaddr_to_csrow,
1931 .dbam_to_cs = k8_dbam_to_chip_select,
1936 .f1_id = PCI_DEVICE_ID_AMD_10H_NB_MAP,
1937 .f2_id = PCI_DEVICE_ID_AMD_10H_NB_DRAM,
1939 .early_channel_count = f1x_early_channel_count,
1940 .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow,
1941 .dbam_to_cs = f10_dbam_to_chip_select,
1946 .f1_id = PCI_DEVICE_ID_AMD_15H_NB_F1,
1947 .f2_id = PCI_DEVICE_ID_AMD_15H_NB_F2,
1949 .early_channel_count = f1x_early_channel_count,
1950 .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow,
1951 .dbam_to_cs = f15_dbam_to_chip_select,
1955 .ctl_name = "F15h_M30h",
1956 .f1_id = PCI_DEVICE_ID_AMD_15H_M30H_NB_F1,
1957 .f2_id = PCI_DEVICE_ID_AMD_15H_M30H_NB_F2,
1959 .early_channel_count = f1x_early_channel_count,
1960 .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow,
1961 .dbam_to_cs = f16_dbam_to_chip_select,
1965 .ctl_name = "F15h_M60h",
1966 .f1_id = PCI_DEVICE_ID_AMD_15H_M60H_NB_F1,
1967 .f2_id = PCI_DEVICE_ID_AMD_15H_M60H_NB_F2,
1969 .early_channel_count = f1x_early_channel_count,
1970 .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow,
1971 .dbam_to_cs = f15_m60h_dbam_to_chip_select,
1976 .f1_id = PCI_DEVICE_ID_AMD_16H_NB_F1,
1977 .f2_id = PCI_DEVICE_ID_AMD_16H_NB_F2,
1979 .early_channel_count = f1x_early_channel_count,
1980 .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow,
1981 .dbam_to_cs = f16_dbam_to_chip_select,
1985 .ctl_name = "F16h_M30h",
1986 .f1_id = PCI_DEVICE_ID_AMD_16H_M30H_NB_F1,
1987 .f2_id = PCI_DEVICE_ID_AMD_16H_M30H_NB_F2,
1989 .early_channel_count = f1x_early_channel_count,
1990 .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow,
1991 .dbam_to_cs = f16_dbam_to_chip_select,
1997 * These are tables of eigenvectors (one per line) which can be used for the
1998 * construction of the syndrome tables. The modified syndrome search algorithm
1999 * uses those to find the symbol in error and thus the DIMM.
2001 * Algorithm courtesy of Ross LaFetra from AMD.
2003 static const u16 x4_vectors[] = {
2004 0x2f57, 0x1afe, 0x66cc, 0xdd88,
2005 0x11eb, 0x3396, 0x7f4c, 0xeac8,
2006 0x0001, 0x0002, 0x0004, 0x0008,
2007 0x1013, 0x3032, 0x4044, 0x8088,
2008 0x106b, 0x30d6, 0x70fc, 0xe0a8,
2009 0x4857, 0xc4fe, 0x13cc, 0x3288,
2010 0x1ac5, 0x2f4a, 0x5394, 0xa1e8,
2011 0x1f39, 0x251e, 0xbd6c, 0x6bd8,
2012 0x15c1, 0x2a42, 0x89ac, 0x4758,
2013 0x2b03, 0x1602, 0x4f0c, 0xca08,
2014 0x1f07, 0x3a0e, 0x6b04, 0xbd08,
2015 0x8ba7, 0x465e, 0x244c, 0x1cc8,
2016 0x2b87, 0x164e, 0x642c, 0xdc18,
2017 0x40b9, 0x80de, 0x1094, 0x20e8,
2018 0x27db, 0x1eb6, 0x9dac, 0x7b58,
2019 0x11c1, 0x2242, 0x84ac, 0x4c58,
2020 0x1be5, 0x2d7a, 0x5e34, 0xa718,
2021 0x4b39, 0x8d1e, 0x14b4, 0x28d8,
2022 0x4c97, 0xc87e, 0x11fc, 0x33a8,
2023 0x8e97, 0x497e, 0x2ffc, 0x1aa8,
2024 0x16b3, 0x3d62, 0x4f34, 0x8518,
2025 0x1e2f, 0x391a, 0x5cac, 0xf858,
2026 0x1d9f, 0x3b7a, 0x572c, 0xfe18,
2027 0x15f5, 0x2a5a, 0x5264, 0xa3b8,
2028 0x1dbb, 0x3b66, 0x715c, 0xe3f8,
2029 0x4397, 0xc27e, 0x17fc, 0x3ea8,
2030 0x1617, 0x3d3e, 0x6464, 0xb8b8,
2031 0x23ff, 0x12aa, 0xab6c, 0x56d8,
2032 0x2dfb, 0x1ba6, 0x913c, 0x7328,
2033 0x185d, 0x2ca6, 0x7914, 0x9e28,
2034 0x171b, 0x3e36, 0x7d7c, 0xebe8,
2035 0x4199, 0x82ee, 0x19f4, 0x2e58,
2036 0x4807, 0xc40e, 0x130c, 0x3208,
2037 0x1905, 0x2e0a, 0x5804, 0xac08,
2038 0x213f, 0x132a, 0xadfc, 0x5ba8,
2039 0x19a9, 0x2efe, 0xb5cc, 0x6f88,
2042 static const u16 x8_vectors[] = {
2043 0x0145, 0x028a, 0x2374, 0x43c8, 0xa1f0, 0x0520, 0x0a40, 0x1480,
2044 0x0211, 0x0422, 0x0844, 0x1088, 0x01b0, 0x44e0, 0x23c0, 0xed80,
2045 0x1011, 0x0116, 0x022c, 0x0458, 0x08b0, 0x8c60, 0x2740, 0x4e80,
2046 0x0411, 0x0822, 0x1044, 0x0158, 0x02b0, 0x2360, 0x46c0, 0xab80,
2047 0x0811, 0x1022, 0x012c, 0x0258, 0x04b0, 0x4660, 0x8cc0, 0x2780,
2048 0x2071, 0x40e2, 0xa0c4, 0x0108, 0x0210, 0x0420, 0x0840, 0x1080,
2049 0x4071, 0x80e2, 0x0104, 0x0208, 0x0410, 0x0820, 0x1040, 0x2080,
2050 0x8071, 0x0102, 0x0204, 0x0408, 0x0810, 0x1020, 0x2040, 0x4080,
2051 0x019d, 0x03d6, 0x136c, 0x2198, 0x50b0, 0xb2e0, 0x0740, 0x0e80,
2052 0x0189, 0x03ea, 0x072c, 0x0e58, 0x1cb0, 0x56e0, 0x37c0, 0xf580,
2053 0x01fd, 0x0376, 0x06ec, 0x0bb8, 0x1110, 0x2220, 0x4440, 0x8880,
2054 0x0163, 0x02c6, 0x1104, 0x0758, 0x0eb0, 0x2be0, 0x6140, 0xc280,
2055 0x02fd, 0x01c6, 0x0b5c, 0x1108, 0x07b0, 0x25a0, 0x8840, 0x6180,
2056 0x0801, 0x012e, 0x025c, 0x04b8, 0x1370, 0x26e0, 0x57c0, 0xb580,
2057 0x0401, 0x0802, 0x015c, 0x02b8, 0x22b0, 0x13e0, 0x7140, 0xe280,
2058 0x0201, 0x0402, 0x0804, 0x01b8, 0x11b0, 0x31a0, 0x8040, 0x7180,
2059 0x0101, 0x0202, 0x0404, 0x0808, 0x1010, 0x2020, 0x4040, 0x8080,
2060 0x0001, 0x0002, 0x0004, 0x0008, 0x0010, 0x0020, 0x0040, 0x0080,
2061 0x0100, 0x0200, 0x0400, 0x0800, 0x1000, 0x2000, 0x4000, 0x8000,
2064 static int decode_syndrome(u16 syndrome, const u16 *vectors, unsigned num_vecs,
2067 unsigned int i, err_sym;
2069 for (err_sym = 0; err_sym < num_vecs / v_dim; err_sym++) {
2071 unsigned v_idx = err_sym * v_dim;
2072 unsigned v_end = (err_sym + 1) * v_dim;
2074 /* walk over all 16 bits of the syndrome */
2075 for (i = 1; i < (1U << 16); i <<= 1) {
2077 /* if bit is set in that eigenvector... */
2078 if (v_idx < v_end && vectors[v_idx] & i) {
2079 u16 ev_comp = vectors[v_idx++];
2081 /* ... and bit set in the modified syndrome, */
2091 /* can't get to zero, move to next symbol */
2096 edac_dbg(0, "syndrome(%x) not found\n", syndrome);
2100 static int map_err_sym_to_channel(int err_sym, int sym_size)
2113 return err_sym >> 4;
2119 /* imaginary bits not in a DIMM */
2121 WARN(1, KERN_ERR "Invalid error symbol: 0x%x\n",
2133 return err_sym >> 3;
2139 static int get_channel_from_ecc_syndrome(struct mem_ctl_info *mci, u16 syndrome)
2141 struct amd64_pvt *pvt = mci->pvt_info;
2144 if (pvt->ecc_sym_sz == 8)
2145 err_sym = decode_syndrome(syndrome, x8_vectors,
2146 ARRAY_SIZE(x8_vectors),
2148 else if (pvt->ecc_sym_sz == 4)
2149 err_sym = decode_syndrome(syndrome, x4_vectors,
2150 ARRAY_SIZE(x4_vectors),
2153 amd64_warn("Illegal syndrome type: %u\n", pvt->ecc_sym_sz);
2157 return map_err_sym_to_channel(err_sym, pvt->ecc_sym_sz);
2160 static void __log_bus_error(struct mem_ctl_info *mci, struct err_info *err,
2163 enum hw_event_mc_err_type err_type;
2167 err_type = HW_EVENT_ERR_CORRECTED;
2168 else if (ecc_type == 1)
2169 err_type = HW_EVENT_ERR_UNCORRECTED;
2171 WARN(1, "Something is rotten in the state of Denmark.\n");
2175 switch (err->err_code) {
2180 string = "Failed to map error addr to a node";
2183 string = "Failed to map error addr to a csrow";
2186 string = "unknown syndrome - possible error reporting race";
2189 string = "WTF error";
2193 edac_mc_handle_error(err_type, mci, 1,
2194 err->page, err->offset, err->syndrome,
2195 err->csrow, err->channel, -1,
2199 static inline void decode_bus_error(int node_id, struct mce *m)
2201 struct mem_ctl_info *mci;
2202 struct amd64_pvt *pvt;
2203 u8 ecc_type = (m->status >> 45) & 0x3;
2204 u8 xec = XEC(m->status, 0x1f);
2205 u16 ec = EC(m->status);
2207 struct err_info err;
2209 mci = edac_mc_find(node_id);
2213 pvt = mci->pvt_info;
2215 /* Bail out early if this was an 'observed' error */
2216 if (PP(ec) == NBSL_PP_OBS)
2219 /* Do only ECC errors */
2220 if (xec && xec != F10_NBSL_EXT_ERR_ECC)
2223 memset(&err, 0, sizeof(err));
2225 sys_addr = get_error_address(pvt, m);
2228 err.syndrome = extract_syndrome(m->status);
2230 pvt->ops->map_sysaddr_to_csrow(mci, sys_addr, &err);
2232 __log_bus_error(mci, &err, ecc_type);
2236 * Use pvt->F3 which contains the F3 CPU PCI device to get the related
2237 * F1 (AddrMap) and F2 (Dct) devices. Return negative value on error.
2239 static int reserve_mc_sibling_devs(struct amd64_pvt *pvt, u16 f1_id, u16 f2_id)
2241 /* Reserve the ADDRESS MAP Device */
2242 pvt->F1 = pci_get_related_function(pvt->F3->vendor, f1_id, pvt->F3);
2244 amd64_err("error address map device not found: "
2245 "vendor %x device 0x%x (broken BIOS?)\n",
2246 PCI_VENDOR_ID_AMD, f1_id);
2250 /* Reserve the DCT Device */
2251 pvt->F2 = pci_get_related_function(pvt->F3->vendor, f2_id, pvt->F3);
2253 pci_dev_put(pvt->F1);
2256 amd64_err("error F2 device not found: "
2257 "vendor %x device 0x%x (broken BIOS?)\n",
2258 PCI_VENDOR_ID_AMD, f2_id);
2262 edac_dbg(1, "F1: %s\n", pci_name(pvt->F1));
2263 edac_dbg(1, "F2: %s\n", pci_name(pvt->F2));
2264 edac_dbg(1, "F3: %s\n", pci_name(pvt->F3));
2269 static void free_mc_sibling_devs(struct amd64_pvt *pvt)
2271 pci_dev_put(pvt->F1);
2272 pci_dev_put(pvt->F2);
2276 * Retrieve the hardware registers of the memory controller (this includes the
2277 * 'Address Map' and 'Misc' device regs)
2279 static void read_mc_regs(struct amd64_pvt *pvt)
2286 * Retrieve TOP_MEM and TOP_MEM2; no masking off of reserved bits since
2287 * those are Read-As-Zero
2289 rdmsrl(MSR_K8_TOP_MEM1, pvt->top_mem);
2290 edac_dbg(0, " TOP_MEM: 0x%016llx\n", pvt->top_mem);
2292 /* check first whether TOP_MEM2 is enabled */
2293 rdmsrl(MSR_K8_SYSCFG, msr_val);
2294 if (msr_val & (1U << 21)) {
2295 rdmsrl(MSR_K8_TOP_MEM2, pvt->top_mem2);
2296 edac_dbg(0, " TOP_MEM2: 0x%016llx\n", pvt->top_mem2);
2298 edac_dbg(0, " TOP_MEM2 disabled\n");
2300 amd64_read_pci_cfg(pvt->F3, NBCAP, &pvt->nbcap);
2302 read_dram_ctl_register(pvt);
2304 for (range = 0; range < DRAM_RANGES; range++) {
2307 /* read settings for this DRAM range */
2308 read_dram_base_limit_regs(pvt, range);
2310 rw = dram_rw(pvt, range);
2314 edac_dbg(1, " DRAM range[%d], base: 0x%016llx; limit: 0x%016llx\n",
2316 get_dram_base(pvt, range),
2317 get_dram_limit(pvt, range));
2319 edac_dbg(1, " IntlvEn=%s; Range access: %s%s IntlvSel=%d DstNode=%d\n",
2320 dram_intlv_en(pvt, range) ? "Enabled" : "Disabled",
2321 (rw & 0x1) ? "R" : "-",
2322 (rw & 0x2) ? "W" : "-",
2323 dram_intlv_sel(pvt, range),
2324 dram_dst_node(pvt, range));
2327 read_dct_base_mask(pvt);
2329 amd64_read_pci_cfg(pvt->F1, DHAR, &pvt->dhar);
2330 amd64_read_dct_pci_cfg(pvt, 0, DBAM0, &pvt->dbam0);
2332 amd64_read_pci_cfg(pvt->F3, F10_ONLINE_SPARE, &pvt->online_spare);
2334 amd64_read_dct_pci_cfg(pvt, 0, DCLR0, &pvt->dclr0);
2335 amd64_read_dct_pci_cfg(pvt, 0, DCHR0, &pvt->dchr0);
2337 if (!dct_ganging_enabled(pvt)) {
2338 amd64_read_dct_pci_cfg(pvt, 1, DCLR0, &pvt->dclr1);
2339 amd64_read_dct_pci_cfg(pvt, 1, DCHR0, &pvt->dchr1);
2342 pvt->ecc_sym_sz = 4;
2343 determine_memory_type(pvt);
2344 edac_dbg(1, " DIMM type: %s\n", edac_mem_types[pvt->dram_type]);
2346 if (pvt->fam >= 0x10) {
2347 amd64_read_pci_cfg(pvt->F3, EXT_NB_MCA_CFG, &tmp);
2348 /* F16h has only DCT0, so no need to read dbam1 */
2349 if (pvt->fam != 0x16)
2350 amd64_read_dct_pci_cfg(pvt, 1, DBAM0, &pvt->dbam1);
2352 /* F10h, revD and later can do x8 ECC too */
2353 if ((pvt->fam > 0x10 || pvt->model > 7) && tmp & BIT(25))
2354 pvt->ecc_sym_sz = 8;
2356 dump_misc_regs(pvt);
2360 * NOTE: CPU Revision Dependent code
2363 * @csrow_nr ChipSelect Row Number (0..NUM_CHIPSELECTS-1)
2364 * k8 private pointer to -->
2365 * DRAM Bank Address mapping register
2367 * DCL register where dual_channel_active is
2369 * The DBAM register consists of 4 sets of 4 bits each definitions:
2372 * 0-3 CSROWs 0 and 1
2373 * 4-7 CSROWs 2 and 3
2374 * 8-11 CSROWs 4 and 5
2375 * 12-15 CSROWs 6 and 7
2377 * Values range from: 0 to 15
2378 * The meaning of the values depends on CPU revision and dual-channel state,
2379 * see relevant BKDG more info.
2381 * The memory controller provides for total of only 8 CSROWs in its current
2382 * architecture. Each "pair" of CSROWs normally represents just one DIMM in
2383 * single channel or two (2) DIMMs in dual channel mode.
2385 * The following code logic collapses the various tables for CSROW based on CPU
2389 * The number of PAGE_SIZE pages on the specified CSROW number it
2393 static u32 get_csrow_nr_pages(struct amd64_pvt *pvt, u8 dct, int csrow_nr)
2395 u32 cs_mode, nr_pages;
2396 u32 dbam = dct ? pvt->dbam1 : pvt->dbam0;
2400 * The math on this doesn't look right on the surface because x/2*4 can
2401 * be simplified to x*2 but this expression makes use of the fact that
2402 * it is integral math where 1/2=0. This intermediate value becomes the
2403 * number of bits to shift the DBAM register to extract the proper CSROW
2406 cs_mode = DBAM_DIMM(csrow_nr / 2, dbam);
2408 nr_pages = pvt->ops->dbam_to_cs(pvt, dct, cs_mode, (csrow_nr / 2))
2409 << (20 - PAGE_SHIFT);
2411 edac_dbg(0, "csrow: %d, channel: %d, DBAM idx: %d\n",
2412 csrow_nr, dct, cs_mode);
2413 edac_dbg(0, "nr_pages/channel: %u\n", nr_pages);
2419 * Initialize the array of csrow attribute instances, based on the values
2420 * from pci config hardware registers.
2422 static int init_csrows(struct mem_ctl_info *mci)
2424 struct amd64_pvt *pvt = mci->pvt_info;
2425 struct csrow_info *csrow;
2426 struct dimm_info *dimm;
2427 enum edac_type edac_mode;
2428 int i, j, empty = 1;
2432 amd64_read_pci_cfg(pvt->F3, NBCFG, &val);
2436 edac_dbg(0, "node %d, NBCFG=0x%08x[ChipKillEccCap: %d|DramEccEn: %d]\n",
2437 pvt->mc_node_id, val,
2438 !!(val & NBCFG_CHIPKILL), !!(val & NBCFG_ECC_ENABLE));
2441 * We iterate over DCT0 here but we look at DCT1 in parallel, if needed.
2443 for_each_chip_select(i, 0, pvt) {
2444 bool row_dct0 = !!csrow_enabled(i, 0, pvt);
2445 bool row_dct1 = false;
2447 if (pvt->fam != 0xf)
2448 row_dct1 = !!csrow_enabled(i, 1, pvt);
2450 if (!row_dct0 && !row_dct1)
2453 csrow = mci->csrows[i];
2456 edac_dbg(1, "MC node: %d, csrow: %d\n",
2457 pvt->mc_node_id, i);
2460 nr_pages = get_csrow_nr_pages(pvt, 0, i);
2461 csrow->channels[0]->dimm->nr_pages = nr_pages;
2464 /* K8 has only one DCT */
2465 if (pvt->fam != 0xf && row_dct1) {
2466 int row_dct1_pages = get_csrow_nr_pages(pvt, 1, i);
2468 csrow->channels[1]->dimm->nr_pages = row_dct1_pages;
2469 nr_pages += row_dct1_pages;
2472 edac_dbg(1, "Total csrow%d pages: %u\n", i, nr_pages);
2475 * determine whether CHIPKILL or JUST ECC or NO ECC is operating
2477 if (pvt->nbcfg & NBCFG_ECC_ENABLE)
2478 edac_mode = (pvt->nbcfg & NBCFG_CHIPKILL) ?
2479 EDAC_S4ECD4ED : EDAC_SECDED;
2481 edac_mode = EDAC_NONE;
2483 for (j = 0; j < pvt->channel_count; j++) {
2484 dimm = csrow->channels[j]->dimm;
2485 dimm->mtype = pvt->dram_type;
2486 dimm->edac_mode = edac_mode;
2493 /* get all cores on this DCT */
2494 static void get_cpus_on_this_dct_cpumask(struct cpumask *mask, u16 nid)
2498 for_each_online_cpu(cpu)
2499 if (amd_get_nb_id(cpu) == nid)
2500 cpumask_set_cpu(cpu, mask);
2503 /* check MCG_CTL on all the cpus on this node */
2504 static bool nb_mce_bank_enabled_on_node(u16 nid)
2510 if (!zalloc_cpumask_var(&mask, GFP_KERNEL)) {
2511 amd64_warn("%s: Error allocating mask\n", __func__);
2515 get_cpus_on_this_dct_cpumask(mask, nid);
2517 rdmsr_on_cpus(mask, MSR_IA32_MCG_CTL, msrs);
2519 for_each_cpu(cpu, mask) {
2520 struct msr *reg = per_cpu_ptr(msrs, cpu);
2521 nbe = reg->l & MSR_MCGCTL_NBE;
2523 edac_dbg(0, "core: %u, MCG_CTL: 0x%llx, NB MSR is %s\n",
2525 (nbe ? "enabled" : "disabled"));
2533 free_cpumask_var(mask);
2537 static int toggle_ecc_err_reporting(struct ecc_settings *s, u16 nid, bool on)
2539 cpumask_var_t cmask;
2542 if (!zalloc_cpumask_var(&cmask, GFP_KERNEL)) {
2543 amd64_warn("%s: error allocating mask\n", __func__);
2547 get_cpus_on_this_dct_cpumask(cmask, nid);
2549 rdmsr_on_cpus(cmask, MSR_IA32_MCG_CTL, msrs);
2551 for_each_cpu(cpu, cmask) {
2553 struct msr *reg = per_cpu_ptr(msrs, cpu);
2556 if (reg->l & MSR_MCGCTL_NBE)
2557 s->flags.nb_mce_enable = 1;
2559 reg->l |= MSR_MCGCTL_NBE;
2562 * Turn off NB MCE reporting only when it was off before
2564 if (!s->flags.nb_mce_enable)
2565 reg->l &= ~MSR_MCGCTL_NBE;
2568 wrmsr_on_cpus(cmask, MSR_IA32_MCG_CTL, msrs);
2570 free_cpumask_var(cmask);
2575 static bool enable_ecc_error_reporting(struct ecc_settings *s, u16 nid,
2579 u32 value, mask = 0x3; /* UECC/CECC enable */
2581 if (toggle_ecc_err_reporting(s, nid, ON)) {
2582 amd64_warn("Error enabling ECC reporting over MCGCTL!\n");
2586 amd64_read_pci_cfg(F3, NBCTL, &value);
2588 s->old_nbctl = value & mask;
2589 s->nbctl_valid = true;
2592 amd64_write_pci_cfg(F3, NBCTL, value);
2594 amd64_read_pci_cfg(F3, NBCFG, &value);
2596 edac_dbg(0, "1: node %d, NBCFG=0x%08x[DramEccEn: %d]\n",
2597 nid, value, !!(value & NBCFG_ECC_ENABLE));
2599 if (!(value & NBCFG_ECC_ENABLE)) {
2600 amd64_warn("DRAM ECC disabled on this node, enabling...\n");
2602 s->flags.nb_ecc_prev = 0;
2604 /* Attempt to turn on DRAM ECC Enable */
2605 value |= NBCFG_ECC_ENABLE;
2606 amd64_write_pci_cfg(F3, NBCFG, value);
2608 amd64_read_pci_cfg(F3, NBCFG, &value);
2610 if (!(value & NBCFG_ECC_ENABLE)) {
2611 amd64_warn("Hardware rejected DRAM ECC enable,"
2612 "check memory DIMM configuration.\n");
2615 amd64_info("Hardware accepted DRAM ECC Enable\n");
2618 s->flags.nb_ecc_prev = 1;
2621 edac_dbg(0, "2: node %d, NBCFG=0x%08x[DramEccEn: %d]\n",
2622 nid, value, !!(value & NBCFG_ECC_ENABLE));
2627 static void restore_ecc_error_reporting(struct ecc_settings *s, u16 nid,
2630 u32 value, mask = 0x3; /* UECC/CECC enable */
2633 if (!s->nbctl_valid)
2636 amd64_read_pci_cfg(F3, NBCTL, &value);
2638 value |= s->old_nbctl;
2640 amd64_write_pci_cfg(F3, NBCTL, value);
2642 /* restore previous BIOS DRAM ECC "off" setting we force-enabled */
2643 if (!s->flags.nb_ecc_prev) {
2644 amd64_read_pci_cfg(F3, NBCFG, &value);
2645 value &= ~NBCFG_ECC_ENABLE;
2646 amd64_write_pci_cfg(F3, NBCFG, value);
2649 /* restore the NB Enable MCGCTL bit */
2650 if (toggle_ecc_err_reporting(s, nid, OFF))
2651 amd64_warn("Error restoring NB MCGCTL settings!\n");
2655 * EDAC requires that the BIOS have ECC enabled before
2656 * taking over the processing of ECC errors. A command line
2657 * option allows to force-enable hardware ECC later in
2658 * enable_ecc_error_reporting().
2660 static const char *ecc_msg =
2661 "ECC disabled in the BIOS or no ECC capability, module will not load.\n"
2662 " Either enable ECC checking or force module loading by setting "
2663 "'ecc_enable_override'.\n"
2664 " (Note that use of the override may cause unknown side effects.)\n";
2666 static bool ecc_enabled(struct pci_dev *F3, u16 nid)
2670 bool nb_mce_en = false;
2672 amd64_read_pci_cfg(F3, NBCFG, &value);
2674 ecc_en = !!(value & NBCFG_ECC_ENABLE);
2675 amd64_info("DRAM ECC %s.\n", (ecc_en ? "enabled" : "disabled"));
2677 nb_mce_en = nb_mce_bank_enabled_on_node(nid);
2679 amd64_notice("NB MCE bank disabled, set MSR "
2680 "0x%08x[4] on node %d to enable.\n",
2681 MSR_IA32_MCG_CTL, nid);
2683 if (!ecc_en || !nb_mce_en) {
2684 amd64_notice("%s", ecc_msg);
2690 static void setup_mci_misc_attrs(struct mem_ctl_info *mci,
2691 struct amd64_family_type *fam)
2693 struct amd64_pvt *pvt = mci->pvt_info;
2695 mci->mtype_cap = MEM_FLAG_DDR2 | MEM_FLAG_RDDR2;
2696 mci->edac_ctl_cap = EDAC_FLAG_NONE;
2698 if (pvt->nbcap & NBCAP_SECDED)
2699 mci->edac_ctl_cap |= EDAC_FLAG_SECDED;
2701 if (pvt->nbcap & NBCAP_CHIPKILL)
2702 mci->edac_ctl_cap |= EDAC_FLAG_S4ECD4ED;
2704 mci->edac_cap = determine_edac_cap(pvt);
2705 mci->mod_name = EDAC_MOD_STR;
2706 mci->mod_ver = EDAC_AMD64_VERSION;
2707 mci->ctl_name = fam->ctl_name;
2708 mci->dev_name = pci_name(pvt->F2);
2709 mci->ctl_page_to_phys = NULL;
2711 /* memory scrubber interface */
2712 mci->set_sdram_scrub_rate = set_scrub_rate;
2713 mci->get_sdram_scrub_rate = get_scrub_rate;
2717 * returns a pointer to the family descriptor on success, NULL otherwise.
2719 static struct amd64_family_type *per_family_init(struct amd64_pvt *pvt)
2721 struct amd64_family_type *fam_type = NULL;
2723 pvt->ext_model = boot_cpu_data.x86_model >> 4;
2724 pvt->stepping = boot_cpu_data.x86_stepping;
2725 pvt->model = boot_cpu_data.x86_model;
2726 pvt->fam = boot_cpu_data.x86;
2730 fam_type = &family_types[K8_CPUS];
2731 pvt->ops = &family_types[K8_CPUS].ops;
2735 fam_type = &family_types[F10_CPUS];
2736 pvt->ops = &family_types[F10_CPUS].ops;
2740 if (pvt->model == 0x30) {
2741 fam_type = &family_types[F15_M30H_CPUS];
2742 pvt->ops = &family_types[F15_M30H_CPUS].ops;
2744 } else if (pvt->model == 0x60) {
2745 fam_type = &family_types[F15_M60H_CPUS];
2746 pvt->ops = &family_types[F15_M60H_CPUS].ops;
2750 fam_type = &family_types[F15_CPUS];
2751 pvt->ops = &family_types[F15_CPUS].ops;
2755 if (pvt->model == 0x30) {
2756 fam_type = &family_types[F16_M30H_CPUS];
2757 pvt->ops = &family_types[F16_M30H_CPUS].ops;
2760 fam_type = &family_types[F16_CPUS];
2761 pvt->ops = &family_types[F16_CPUS].ops;
2765 amd64_err("Unsupported family!\n");
2769 amd64_info("%s %sdetected (node %d).\n", fam_type->ctl_name,
2771 (pvt->ext_model >= K8_REV_F ? "revF or later "
2772 : "revE or earlier ")
2773 : ""), pvt->mc_node_id);
2777 static const struct attribute_group *amd64_edac_attr_groups[] = {
2778 #ifdef CONFIG_EDAC_DEBUG
2779 &amd64_edac_dbg_group,
2781 #ifdef CONFIG_EDAC_AMD64_ERROR_INJECTION
2782 &amd64_edac_inj_group,
2787 static int init_one_instance(unsigned int nid)
2789 struct pci_dev *F3 = node_to_amd_nb(nid)->misc;
2790 struct amd64_family_type *fam_type = NULL;
2791 struct mem_ctl_info *mci = NULL;
2792 struct edac_mc_layer layers[2];
2793 struct amd64_pvt *pvt = NULL;
2797 pvt = kzalloc(sizeof(struct amd64_pvt), GFP_KERNEL);
2801 pvt->mc_node_id = nid;
2805 fam_type = per_family_init(pvt);
2810 err = reserve_mc_sibling_devs(pvt, fam_type->f1_id, fam_type->f2_id);
2817 * We need to determine how many memory channels there are. Then use
2818 * that information for calculating the size of the dynamic instance
2819 * tables in the 'mci' structure.
2822 pvt->channel_count = pvt->ops->early_channel_count(pvt);
2823 if (pvt->channel_count < 0)
2827 layers[0].type = EDAC_MC_LAYER_CHIP_SELECT;
2828 layers[0].size = pvt->csels[0].b_cnt;
2829 layers[0].is_virt_csrow = true;
2830 layers[1].type = EDAC_MC_LAYER_CHANNEL;
2833 * Always allocate two channels since we can have setups with DIMMs on
2834 * only one channel. Also, this simplifies handling later for the price
2835 * of a couple of KBs tops.
2838 layers[1].is_virt_csrow = false;
2840 mci = edac_mc_alloc(nid, ARRAY_SIZE(layers), layers, 0);
2844 mci->pvt_info = pvt;
2845 mci->pdev = &pvt->F3->dev;
2847 setup_mci_misc_attrs(mci, fam_type);
2849 if (init_csrows(mci))
2850 mci->edac_cap = EDAC_FLAG_NONE;
2853 if (edac_mc_add_mc_with_groups(mci, amd64_edac_attr_groups)) {
2854 edac_dbg(1, "failed edac_mc_add_mc()\n");
2858 /* register stuff with EDAC MCE */
2859 if (report_gart_errors)
2860 amd_report_gart_errors(true);
2862 amd_register_ecc_decoder(decode_bus_error);
2870 free_mc_sibling_devs(pvt);
2879 static int probe_one_instance(unsigned int nid)
2881 struct pci_dev *F3 = node_to_amd_nb(nid)->misc;
2882 struct ecc_settings *s;
2886 s = kzalloc(sizeof(struct ecc_settings), GFP_KERNEL);
2892 if (!ecc_enabled(F3, nid)) {
2895 if (!ecc_enable_override)
2898 amd64_warn("Forcing ECC on!\n");
2900 if (!enable_ecc_error_reporting(s, nid, F3))
2904 ret = init_one_instance(nid);
2906 amd64_err("Error probing instance: %d\n", nid);
2907 restore_ecc_error_reporting(s, nid, F3);
2914 ecc_stngs[nid] = NULL;
2920 static void remove_one_instance(unsigned int nid)
2922 struct pci_dev *F3 = node_to_amd_nb(nid)->misc;
2923 struct ecc_settings *s = ecc_stngs[nid];
2924 struct mem_ctl_info *mci;
2925 struct amd64_pvt *pvt;
2927 mci = find_mci_by_dev(&F3->dev);
2930 /* Remove from EDAC CORE tracking list */
2931 mci = edac_mc_del_mc(&F3->dev);
2935 pvt = mci->pvt_info;
2937 restore_ecc_error_reporting(s, nid, F3);
2939 free_mc_sibling_devs(pvt);
2941 /* unregister from EDAC MCE */
2942 amd_report_gart_errors(false);
2943 amd_unregister_ecc_decoder(decode_bus_error);
2945 kfree(ecc_stngs[nid]);
2946 ecc_stngs[nid] = NULL;
2948 /* Free the EDAC CORE resources */
2949 mci->pvt_info = NULL;
2955 static void setup_pci_device(void)
2957 struct mem_ctl_info *mci;
2958 struct amd64_pvt *pvt;
2963 mci = edac_mc_find(0);
2967 pvt = mci->pvt_info;
2968 pci_ctl = edac_pci_create_generic_ctl(&pvt->F2->dev, EDAC_MOD_STR);
2970 pr_warn("%s(): Unable to create PCI control\n", __func__);
2971 pr_warn("%s(): PCI error report via EDAC not set\n", __func__);
2975 static const struct x86_cpu_id amd64_cpuids[] = {
2976 { X86_VENDOR_AMD, 0xF, X86_MODEL_ANY, X86_FEATURE_ANY, 0 },
2977 { X86_VENDOR_AMD, 0x10, X86_MODEL_ANY, X86_FEATURE_ANY, 0 },
2978 { X86_VENDOR_AMD, 0x15, X86_MODEL_ANY, X86_FEATURE_ANY, 0 },
2979 { X86_VENDOR_AMD, 0x16, X86_MODEL_ANY, X86_FEATURE_ANY, 0 },
2982 MODULE_DEVICE_TABLE(x86cpu, amd64_cpuids);
2984 static int __init amd64_edac_init(void)
2989 if (!x86_match_cpu(amd64_cpuids))
2992 if (amd_cache_northbridges() < 0)
2998 ecc_stngs = kzalloc(amd_nb_num() * sizeof(ecc_stngs[0]), GFP_KERNEL);
3002 msrs = msrs_alloc();
3006 for (i = 0; i < amd_nb_num(); i++) {
3007 err = probe_one_instance(i);
3009 /* unwind properly */
3011 remove_one_instance(i);
3019 #ifdef CONFIG_X86_32
3020 amd64_err("%s on 32-bit is unsupported. USE AT YOUR OWN RISK!\n", EDAC_MOD_STR);
3023 printk(KERN_INFO "AMD64 EDAC driver v%s\n", EDAC_AMD64_VERSION);
3038 static void __exit amd64_edac_exit(void)
3043 edac_pci_release_generic_ctl(pci_ctl);
3045 for (i = 0; i < amd_nb_num(); i++)
3046 remove_one_instance(i);
3055 module_init(amd64_edac_init);
3056 module_exit(amd64_edac_exit);
3058 MODULE_LICENSE("GPL");
3059 MODULE_AUTHOR("SoftwareBitMaker: Doug Thompson, "
3060 "Dave Peterson, Thayne Harbaugh");
3061 MODULE_DESCRIPTION("MC support for AMD64 memory controllers - "
3062 EDAC_AMD64_VERSION);
3064 module_param(edac_op_state, int, 0444);
3065 MODULE_PARM_DESC(edac_op_state, "EDAC Error Reporting state: 0=Poll,1=NMI");