2 * Copyright (C) 2016 Cavium, Inc.
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of version 2 of the GNU General Public License
6 * as published by the Free Software Foundation.
9 #include <linux/device.h>
10 #include <linux/firmware.h>
11 #include <linux/interrupt.h>
12 #include <linux/module.h>
13 #include <linux/moduleparam.h>
14 #include <linux/pci.h>
15 #include <linux/printk.h>
16 #include <linux/version.h>
20 #define DRV_NAME "thunder-cpt"
21 #define DRV_VERSION "1.0"
23 static u32 num_vfs = 4; /* Default 4 VF enabled */
24 module_param(num_vfs, uint, 0444);
25 MODULE_PARM_DESC(num_vfs, "Number of VFs to enable(1-16)");
28 * Disable cores specified by coremask
30 static void cpt_disable_cores(struct cpt_device *cpt, u64 coremask,
36 struct device *dev = &cpt->pdev->dev;
39 coremask = (coremask << cpt->max_se_cores);
41 /* Disengage the cores from groups */
42 grpmask = cpt_read_csr64(cpt->reg_base, CPTX_PF_GX_EN(0, grp));
43 cpt_write_csr64(cpt->reg_base, CPTX_PF_GX_EN(0, grp),
44 (grpmask & ~coremask));
46 grp = cpt_read_csr64(cpt->reg_base, CPTX_PF_EXEC_BUSY(0));
47 while (grp & coremask) {
48 dev_err(dev, "Cores still busy %llx", coremask);
49 grp = cpt_read_csr64(cpt->reg_base,
50 CPTX_PF_EXEC_BUSY(0));
57 /* Disable the cores */
58 pf_exe_ctl = cpt_read_csr64(cpt->reg_base, CPTX_PF_EXE_CTL(0));
59 cpt_write_csr64(cpt->reg_base, CPTX_PF_EXE_CTL(0),
60 (pf_exe_ctl & ~coremask));
65 * Enable cores specified by coremask
67 static void cpt_enable_cores(struct cpt_device *cpt, u64 coremask,
73 coremask = (coremask << cpt->max_se_cores);
75 pf_exe_ctl = cpt_read_csr64(cpt->reg_base, CPTX_PF_EXE_CTL(0));
76 cpt_write_csr64(cpt->reg_base, CPTX_PF_EXE_CTL(0),
77 (pf_exe_ctl | coremask));
81 static void cpt_configure_group(struct cpt_device *cpt, u8 grp,
82 u64 coremask, u8 type)
87 coremask = (coremask << cpt->max_se_cores);
89 pf_gx_en = cpt_read_csr64(cpt->reg_base, CPTX_PF_GX_EN(0, grp));
90 cpt_write_csr64(cpt->reg_base, CPTX_PF_GX_EN(0, grp),
91 (pf_gx_en | coremask));
95 static void cpt_disable_mbox_interrupts(struct cpt_device *cpt)
97 /* Clear mbox(0) interupts for all vfs */
98 cpt_write_csr64(cpt->reg_base, CPTX_PF_MBOX_ENA_W1CX(0, 0), ~0ull);
101 static void cpt_disable_ecc_interrupts(struct cpt_device *cpt)
103 /* Clear ecc(0) interupts for all vfs */
104 cpt_write_csr64(cpt->reg_base, CPTX_PF_ECC0_ENA_W1C(0), ~0ull);
107 static void cpt_disable_exec_interrupts(struct cpt_device *cpt)
109 /* Clear exec interupts for all vfs */
110 cpt_write_csr64(cpt->reg_base, CPTX_PF_EXEC_ENA_W1C(0), ~0ull);
113 static void cpt_disable_all_interrupts(struct cpt_device *cpt)
115 cpt_disable_mbox_interrupts(cpt);
116 cpt_disable_ecc_interrupts(cpt);
117 cpt_disable_exec_interrupts(cpt);
120 static void cpt_enable_mbox_interrupts(struct cpt_device *cpt)
122 /* Set mbox(0) interupts for all vfs */
123 cpt_write_csr64(cpt->reg_base, CPTX_PF_MBOX_ENA_W1SX(0, 0), ~0ull);
126 static int cpt_load_microcode(struct cpt_device *cpt, struct microcode *mcode)
128 int ret = 0, core = 0, shift = 0;
130 struct device *dev = &cpt->pdev->dev;
132 if (!mcode || !mcode->code) {
133 dev_err(dev, "Either the mcode is null or data is NULL\n");
137 if (mcode->code_size == 0) {
138 dev_err(dev, "microcode size is 0\n");
142 /* Assumes 0-9 are SE cores for UCODE_BASE registers and
143 * AE core bases follow
146 core = CPT_MAX_SE_CORES; /* start couting from 10 */
147 total_cores = CPT_MAX_TOTAL_CORES; /* upto 15 */
149 core = 0; /* start couting from 0 */
150 total_cores = CPT_MAX_SE_CORES; /* upto 9 */
153 /* Point to microcode for each core of the group */
154 for (; core < total_cores ; core++, shift++) {
155 if (mcode->core_mask & (1 << shift)) {
156 cpt_write_csr64(cpt->reg_base,
157 CPTX_PF_ENGX_UCODE_BASE(0, core),
158 (u64)mcode->phys_base);
164 static int do_cpt_init(struct cpt_device *cpt, struct microcode *mcode)
167 struct device *dev = &cpt->pdev->dev;
169 /* Make device not ready */
170 cpt->flags &= ~CPT_FLAG_DEVICE_READY;
171 /* Disable All PF interrupts */
172 cpt_disable_all_interrupts(cpt);
173 /* Calculate mcode group and coremasks */
175 if (mcode->num_cores > cpt->max_ae_cores) {
176 dev_err(dev, "Requested for more cores than available AE cores\n");
181 if (cpt->next_group >= CPT_MAX_CORE_GROUPS) {
182 dev_err(dev, "Can't load, all eight microcode groups in use");
186 mcode->group = cpt->next_group;
187 /* Convert requested cores to mask */
188 mcode->core_mask = GENMASK(mcode->num_cores, 0);
189 cpt_disable_cores(cpt, mcode->core_mask, AE_TYPES,
191 /* Load microcode for AE engines */
192 ret = cpt_load_microcode(cpt, mcode);
194 dev_err(dev, "Microcode load Failed for %s\n",
199 /* Configure group mask for the mcode */
200 cpt_configure_group(cpt, mcode->group, mcode->core_mask,
202 /* Enable AE cores for the group mask */
203 cpt_enable_cores(cpt, mcode->core_mask, AE_TYPES);
205 if (mcode->num_cores > cpt->max_se_cores) {
206 dev_err(dev, "Requested for more cores than available SE cores\n");
210 if (cpt->next_group >= CPT_MAX_CORE_GROUPS) {
211 dev_err(dev, "Can't load, all eight microcode groups in use");
215 mcode->group = cpt->next_group;
216 /* Covert requested cores to mask */
217 mcode->core_mask = GENMASK(mcode->num_cores, 0);
218 cpt_disable_cores(cpt, mcode->core_mask, SE_TYPES,
220 /* Load microcode for SE engines */
221 ret = cpt_load_microcode(cpt, mcode);
223 dev_err(dev, "Microcode load Failed for %s\n",
228 /* Configure group mask for the mcode */
229 cpt_configure_group(cpt, mcode->group, mcode->core_mask,
231 /* Enable SE cores for the group mask */
232 cpt_enable_cores(cpt, mcode->core_mask, SE_TYPES);
235 /* Enabled PF mailbox interrupts */
236 cpt_enable_mbox_interrupts(cpt);
237 cpt->flags |= CPT_FLAG_DEVICE_READY;
242 /* Enabled PF mailbox interrupts */
243 cpt_enable_mbox_interrupts(cpt);
248 struct ucode_header {
249 u8 version[CPT_UCODE_VERSION_SZ];
255 static int cpt_ucode_load_fw(struct cpt_device *cpt, const u8 *fw, bool is_ae)
257 const struct firmware *fw_entry;
258 struct device *dev = &cpt->pdev->dev;
259 struct ucode_header *ucode;
260 unsigned int code_length;
261 struct microcode *mcode;
264 ret = reject_firmware(&fw_entry, fw, dev);
268 ucode = (struct ucode_header *)fw_entry->data;
269 mcode = &cpt->mcode[cpt->next_mc_idx];
270 memcpy(mcode->version, (u8 *)fw_entry->data, CPT_UCODE_VERSION_SZ);
271 code_length = ntohl(ucode->code_length);
272 if (code_length == 0 || code_length >= INT_MAX / 2) {
276 mcode->code_size = code_length * 2;
278 mcode->is_ae = is_ae;
279 mcode->core_mask = 0ULL;
280 mcode->num_cores = is_ae ? 6 : 10;
282 /* Allocate DMAable space */
283 mcode->code = dma_zalloc_coherent(&cpt->pdev->dev, mcode->code_size,
284 &mcode->phys_base, GFP_KERNEL);
286 dev_err(dev, "Unable to allocate space for microcode");
291 memcpy((void *)mcode->code, (void *)(fw_entry->data + sizeof(*ucode)),
294 /* Byte swap 64-bit */
295 for (j = 0; j < (mcode->code_size / 8); j++)
296 ((u64 *)mcode->code)[j] = cpu_to_be64(((u64 *)mcode->code)[j]);
297 /* MC needs 16-bit swap */
298 for (j = 0; j < (mcode->code_size / 2); j++)
299 ((u16 *)mcode->code)[j] = cpu_to_be16(((u16 *)mcode->code)[j]);
301 dev_dbg(dev, "mcode->code_size = %u\n", mcode->code_size);
302 dev_dbg(dev, "mcode->is_ae = %u\n", mcode->is_ae);
303 dev_dbg(dev, "mcode->num_cores = %u\n", mcode->num_cores);
304 dev_dbg(dev, "mcode->code = %llx\n", (u64)mcode->code);
305 dev_dbg(dev, "mcode->phys_base = %llx\n", mcode->phys_base);
307 ret = do_cpt_init(cpt, mcode);
309 dev_err(dev, "do_cpt_init failed with ret: %d\n", ret);
313 dev_info(dev, "Microcode Loaded %s\n", mcode->version);
314 mcode->is_mc_valid = 1;
318 release_firmware(fw_entry);
323 static int cpt_ucode_load(struct cpt_device *cpt)
326 struct device *dev = &cpt->pdev->dev;
328 ret = cpt_ucode_load_fw(cpt, "/*(DEBLOBBED)*/", true);
330 dev_err(dev, "ae:cpt_ucode_load failed with ret: %d\n", ret);
333 ret = cpt_ucode_load_fw(cpt, "/*(DEBLOBBED)*/", false);
335 dev_err(dev, "se:cpt_ucode_load failed with ret: %d\n", ret);
342 static irqreturn_t cpt_mbx0_intr_handler(int irq, void *cpt_irq)
344 struct cpt_device *cpt = (struct cpt_device *)cpt_irq;
346 cpt_mbox_intr_handler(cpt, 0);
351 static void cpt_reset(struct cpt_device *cpt)
353 cpt_write_csr64(cpt->reg_base, CPTX_PF_RESET(0), 1);
356 static void cpt_find_max_enabled_cores(struct cpt_device *cpt)
358 union cptx_pf_constants pf_cnsts = {0};
360 pf_cnsts.u = cpt_read_csr64(cpt->reg_base, CPTX_PF_CONSTANTS(0));
361 cpt->max_se_cores = pf_cnsts.s.se;
362 cpt->max_ae_cores = pf_cnsts.s.ae;
365 static u32 cpt_check_bist_status(struct cpt_device *cpt)
367 union cptx_pf_bist_status bist_sts = {0};
369 bist_sts.u = cpt_read_csr64(cpt->reg_base,
370 CPTX_PF_BIST_STATUS(0));
375 static u64 cpt_check_exe_bist_status(struct cpt_device *cpt)
377 union cptx_pf_exe_bist_status bist_sts = {0};
379 bist_sts.u = cpt_read_csr64(cpt->reg_base,
380 CPTX_PF_EXE_BIST_STATUS(0));
385 static void cpt_disable_all_cores(struct cpt_device *cpt)
387 u32 grp, timeout = 100;
388 struct device *dev = &cpt->pdev->dev;
390 /* Disengage the cores from groups */
391 for (grp = 0; grp < CPT_MAX_CORE_GROUPS; grp++) {
392 cpt_write_csr64(cpt->reg_base, CPTX_PF_GX_EN(0, grp), 0);
396 grp = cpt_read_csr64(cpt->reg_base, CPTX_PF_EXEC_BUSY(0));
398 dev_err(dev, "Cores still busy");
399 grp = cpt_read_csr64(cpt->reg_base,
400 CPTX_PF_EXEC_BUSY(0));
406 /* Disable the cores */
407 cpt_write_csr64(cpt->reg_base, CPTX_PF_EXE_CTL(0), 0);
411 * Ensure all cores are disengaged from all groups by
412 * calling cpt_disable_all_cores() before calling this
415 static void cpt_unload_microcode(struct cpt_device *cpt)
419 /* Free microcode bases and reset group masks */
420 for (grp = 0; grp < CPT_MAX_CORE_GROUPS; grp++) {
421 struct microcode *mcode = &cpt->mcode[grp];
423 if (cpt->mcode[grp].code)
424 dma_free_coherent(&cpt->pdev->dev, mcode->code_size,
425 mcode->code, mcode->phys_base);
428 /* Clear UCODE_BASE registers for all engines */
429 for (core = 0; core < CPT_MAX_TOTAL_CORES; core++)
430 cpt_write_csr64(cpt->reg_base,
431 CPTX_PF_ENGX_UCODE_BASE(0, core), 0ull);
434 static int cpt_device_init(struct cpt_device *cpt)
437 struct device *dev = &cpt->pdev->dev;
439 /* Reset the PF when probed first */
443 /*Check BIST status*/
444 bist = (u64)cpt_check_bist_status(cpt);
446 dev_err(dev, "RAM BIST failed with code 0x%llx", bist);
450 bist = cpt_check_exe_bist_status(cpt);
452 dev_err(dev, "Engine BIST failed with code 0x%llx", bist);
456 /*Get CLK frequency*/
457 /*Get max enabled cores */
458 cpt_find_max_enabled_cores(cpt);
459 /*Disable all cores*/
460 cpt_disable_all_cores(cpt);
461 /*Reset device parameters*/
462 cpt->next_mc_idx = 0;
465 cpt->flags |= CPT_FLAG_DEVICE_READY;
470 static int cpt_register_interrupts(struct cpt_device *cpt)
473 struct device *dev = &cpt->pdev->dev;
476 ret = pci_alloc_irq_vectors(cpt->pdev, CPT_PF_MSIX_VECTORS,
477 CPT_PF_MSIX_VECTORS, PCI_IRQ_MSIX);
479 dev_err(&cpt->pdev->dev, "Request for #%d msix vectors failed\n",
480 CPT_PF_MSIX_VECTORS);
484 /* Register mailbox interrupt handlers */
485 ret = request_irq(pci_irq_vector(cpt->pdev, CPT_PF_INT_VEC_E_MBOXX(0)),
486 cpt_mbx0_intr_handler, 0, "CPT Mbox0", cpt);
490 /* Enable mailbox interrupt */
491 cpt_enable_mbox_interrupts(cpt);
495 dev_err(dev, "Request irq failed\n");
496 pci_disable_msix(cpt->pdev);
500 static void cpt_unregister_interrupts(struct cpt_device *cpt)
502 free_irq(pci_irq_vector(cpt->pdev, CPT_PF_INT_VEC_E_MBOXX(0)), cpt);
503 pci_disable_msix(cpt->pdev);
506 static int cpt_sriov_init(struct cpt_device *cpt, int num_vfs)
511 struct pci_dev *pdev = cpt->pdev;
513 pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_SRIOV);
515 dev_err(&pdev->dev, "SRIOV capability is not found in PCIe config space\n");
519 cpt->num_vf_en = num_vfs; /* User requested VFs */
520 pci_read_config_word(pdev, (pos + PCI_SRIOV_TOTAL_VF), &total_vf_cnt);
521 if (total_vf_cnt < cpt->num_vf_en)
522 cpt->num_vf_en = total_vf_cnt;
527 /*Enabled the available VFs */
528 err = pci_enable_sriov(pdev, cpt->num_vf_en);
530 dev_err(&pdev->dev, "SRIOV enable failed, num VF is %d\n",
536 /* TODO: Optionally enable static VQ priorities feature */
538 dev_info(&pdev->dev, "SRIOV enabled, number of VF available %d\n",
541 cpt->flags |= CPT_FLAG_SRIOV_ENABLED;
546 static int cpt_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
548 struct device *dev = &pdev->dev;
549 struct cpt_device *cpt;
552 if (num_vfs > 16 || num_vfs < 4) {
553 dev_warn(dev, "Invalid vf count %d, Resetting it to 4(default)\n",
558 cpt = devm_kzalloc(dev, sizeof(*cpt), GFP_KERNEL);
562 pci_set_drvdata(pdev, cpt);
564 err = pci_enable_device(pdev);
566 dev_err(dev, "Failed to enable PCI device\n");
567 pci_set_drvdata(pdev, NULL);
571 err = pci_request_regions(pdev, DRV_NAME);
573 dev_err(dev, "PCI request regions failed 0x%x\n", err);
574 goto cpt_err_disable_device;
577 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(48));
579 dev_err(dev, "Unable to get usable DMA configuration\n");
580 goto cpt_err_release_regions;
583 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(48));
585 dev_err(dev, "Unable to get 48-bit DMA for consistent allocations\n");
586 goto cpt_err_release_regions;
589 /* MAP PF's configuration registers */
590 cpt->reg_base = pcim_iomap(pdev, 0, 0);
591 if (!cpt->reg_base) {
592 dev_err(dev, "Cannot map config register space, aborting\n");
594 goto cpt_err_release_regions;
597 /* CPT device HW initialization */
598 cpt_device_init(cpt);
600 /* Register interrupts */
601 err = cpt_register_interrupts(cpt);
603 goto cpt_err_release_regions;
605 err = cpt_ucode_load(cpt);
607 goto cpt_err_unregister_interrupts;
609 /* Configure SRIOV */
610 err = cpt_sriov_init(cpt, num_vfs);
612 goto cpt_err_unregister_interrupts;
616 cpt_err_unregister_interrupts:
617 cpt_unregister_interrupts(cpt);
618 cpt_err_release_regions:
619 pci_release_regions(pdev);
620 cpt_err_disable_device:
621 pci_disable_device(pdev);
622 pci_set_drvdata(pdev, NULL);
626 static void cpt_remove(struct pci_dev *pdev)
628 struct cpt_device *cpt = pci_get_drvdata(pdev);
630 /* Disengage SE and AE cores from all groups*/
631 cpt_disable_all_cores(cpt);
632 /* Unload microcodes */
633 cpt_unload_microcode(cpt);
634 cpt_unregister_interrupts(cpt);
635 pci_disable_sriov(pdev);
636 pci_release_regions(pdev);
637 pci_disable_device(pdev);
638 pci_set_drvdata(pdev, NULL);
641 static void cpt_shutdown(struct pci_dev *pdev)
643 struct cpt_device *cpt = pci_get_drvdata(pdev);
648 dev_info(&pdev->dev, "Shutdown device %x:%x.\n",
649 (u32)pdev->vendor, (u32)pdev->device);
651 cpt_unregister_interrupts(cpt);
652 pci_release_regions(pdev);
653 pci_disable_device(pdev);
654 pci_set_drvdata(pdev, NULL);
657 /* Supported devices */
658 static const struct pci_device_id cpt_id_table[] = {
659 { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, CPT_81XX_PCI_PF_DEVICE_ID) },
660 { 0, } /* end of table */
663 static struct pci_driver cpt_pci_driver = {
665 .id_table = cpt_id_table,
667 .remove = cpt_remove,
668 .shutdown = cpt_shutdown,
671 module_pci_driver(cpt_pci_driver);
673 MODULE_AUTHOR("George Cherian <george.cherian@cavium.com>");
674 MODULE_DESCRIPTION("Cavium Thunder CPT Physical Function Driver");
675 MODULE_LICENSE("GPL v2");
676 MODULE_VERSION(DRV_VERSION);
677 MODULE_DEVICE_TABLE(pci, cpt_id_table);