drivers/s390/net/ism_drv.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * ISM driver for s390.
   4  *
   5  * Copyright IBM Corp. 2018
   6  */
   7 #define KMSG_COMPONENT "ism"
   8 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
   9
  10 #include <linux/module.h>
  11 #include <linux/types.h>
  12 #include <linux/interrupt.h>
  13 #include <linux/device.h>
  14 #include <linux/pci.h>
  15 #include <linux/err.h>
  16 #include <net/smc.h>
  17
  18 #include <asm/debug.h>
  19
  20 #include "ism.h"
  21
  22 MODULE_DESCRIPTION("ISM driver for s390");
  23 MODULE_LICENSE("GPL");
  24
  25 #define PCI_DEVICE_ID_IBM_ISM 0x04ED
  26 #define DRV_NAME "ism"
  27
  28 static const struct pci_device_id ism_device_table[] = {
  29         { PCI_VDEVICE(IBM, PCI_DEVICE_ID_IBM_ISM), 0 },
  30         { 0, }
  31 };
  32 MODULE_DEVICE_TABLE(pci, ism_device_table);
  33
  34 static debug_info_t *ism_debug_info;
  35
  36 static int ism_cmd(struct ism_dev *ism, void *cmd)
  37 {
  38         struct ism_req_hdr *req = cmd;
  39         struct ism_resp_hdr *resp = cmd;
  40
  41         memcpy_toio(ism->ctl + sizeof(*req), req + 1, req->len - sizeof(*req));
  42         memcpy_toio(ism->ctl, req, sizeof(*req));
  43
  44         WRITE_ONCE(resp->ret, ISM_ERROR);
  45
  46         memcpy_fromio(resp, ism->ctl, sizeof(*resp));
  47         if (resp->ret) {
  48                 debug_text_event(ism_debug_info, 0, "cmd failure");
  49                 debug_event(ism_debug_info, 0, resp, sizeof(*resp));
  50                 goto out;
  51         }
  52         memcpy_fromio(resp + 1, ism->ctl + sizeof(*resp),
  53                       resp->len - sizeof(*resp));
  54 out:
  55         return resp->ret;
  56 }
  57
  58 static int ism_cmd_simple(struct ism_dev *ism, u32 cmd_code)
  59 {
  60         union ism_cmd_simple cmd;
  61
  62         memset(&cmd, 0, sizeof(cmd));
  63         cmd.request.hdr.cmd = cmd_code;
  64         cmd.request.hdr.len = sizeof(cmd.request);
  65
  66         return ism_cmd(ism, &cmd);
  67 }
  68
  69 static int query_info(struct ism_dev *ism)
  70 {
  71         union ism_qi cmd;
  72
  73         memset(&cmd, 0, sizeof(cmd));
  74         cmd.request.hdr.cmd = ISM_QUERY_INFO;
  75         cmd.request.hdr.len = sizeof(cmd.request);
  76
  77         if (ism_cmd(ism, &cmd))
  78                 goto out;
  79
  80         debug_text_event(ism_debug_info, 3, "query info");
  81         debug_event(ism_debug_info, 3, &cmd.response, sizeof(cmd.response));
  82 out:
  83         return 0;
  84 }
  85
  86 static int register_sba(struct ism_dev *ism)
  87 {
  88         union ism_reg_sba cmd;
  89         dma_addr_t dma_handle;
  90         struct ism_sba *sba;
  91
  92         sba = dma_zalloc_coherent(&ism->pdev->dev, PAGE_SIZE,
  93                                   &dma_handle, GFP_KERNEL);
  94         if (!sba)
  95                 return -ENOMEM;
  96
  97         memset(&cmd, 0, sizeof(cmd));
  98         cmd.request.hdr.cmd = ISM_REG_SBA;
  99         cmd.request.hdr.len = sizeof(cmd.request);
 100         cmd.request.sba = dma_handle;
 101
 102         if (ism_cmd(ism, &cmd)) {
 103                 dma_free_coherent(&ism->pdev->dev, PAGE_SIZE, sba, dma_handle);
 104                 return -EIO;
 105         }
 106
 107         ism->sba = sba;
 108         ism->sba_dma_addr = dma_handle;
 109
 110         return 0;
 111 }
 112
 113 static int register_ieq(struct ism_dev *ism)
 114 {
 115         union ism_reg_ieq cmd;
 116         dma_addr_t dma_handle;
 117         struct ism_eq *ieq;
 118
 119         ieq = dma_zalloc_coherent(&ism->pdev->dev, PAGE_SIZE,
 120                                   &dma_handle, GFP_KERNEL);
 121         if (!ieq)
 122                 return -ENOMEM;
 123
 124         memset(&cmd, 0, sizeof(cmd));
 125         cmd.request.hdr.cmd = ISM_REG_IEQ;
 126         cmd.request.hdr.len = sizeof(cmd.request);
 127         cmd.request.ieq = dma_handle;
 128         cmd.request.len = sizeof(*ieq);
 129
 130         if (ism_cmd(ism, &cmd)) {
 131                 dma_free_coherent(&ism->pdev->dev, PAGE_SIZE, ieq, dma_handle);
 132                 return -EIO;
 133         }
 134
 135         ism->ieq = ieq;
 136         ism->ieq_idx = -1;
 137         ism->ieq_dma_addr = dma_handle;
 138
 139         return 0;
 140 }
 141
 142 static int unregister_sba(struct ism_dev *ism)
 143 {
 144         int ret;
 145
 146         if (!ism->sba)
 147                 return 0;
 148
 149         ret = ism_cmd_simple(ism, ISM_UNREG_SBA);
 150         if (ret && ret != ISM_ERROR)
 151                 return -EIO;
 152
 153         dma_free_coherent(&ism->pdev->dev, PAGE_SIZE,
 154                           ism->sba, ism->sba_dma_addr);
 155
 156         ism->sba = NULL;
 157         ism->sba_dma_addr = 0;
 158
 159         return 0;
 160 }
 161
 162 static int unregister_ieq(struct ism_dev *ism)
 163 {
 164         int ret;
 165
 166         if (!ism->ieq)
 167                 return 0;
 168
 169         ret = ism_cmd_simple(ism, ISM_UNREG_IEQ);
 170         if (ret && ret != ISM_ERROR)
 171                 return -EIO;
 172
 173         dma_free_coherent(&ism->pdev->dev, PAGE_SIZE,
 174                           ism->ieq, ism->ieq_dma_addr);
 175
 176         ism->ieq = NULL;
 177         ism->ieq_dma_addr = 0;
 178
 179         return 0;
 180 }
 181
 182 static int ism_read_local_gid(struct ism_dev *ism)
 183 {
 184         union ism_read_gid cmd;
 185         int ret;
 186
 187         memset(&cmd, 0, sizeof(cmd));
 188         cmd.request.hdr.cmd = ISM_READ_GID;
 189         cmd.request.hdr.len = sizeof(cmd.request);
 190
 191         ret = ism_cmd(ism, &cmd);
 192         if (ret)
 193                 goto out;
 194
 195         ism->smcd->local_gid = cmd.response.gid;
 196 out:
 197         return ret;
 198 }
 199
 200 static int ism_query_rgid(struct smcd_dev *smcd, u64 rgid, u32 vid_valid,
 201                           u32 vid)
 202 {
 203         struct ism_dev *ism = smcd->priv;
 204         union ism_query_rgid cmd;
 205
 206         memset(&cmd, 0, sizeof(cmd));
 207         cmd.request.hdr.cmd = ISM_QUERY_RGID;
 208         cmd.request.hdr.len = sizeof(cmd.request);
 209
 210         cmd.request.rgid = rgid;
 211         cmd.request.vlan_valid = vid_valid;
 212         cmd.request.vlan_id = vid;
 213
 214         return ism_cmd(ism, &cmd);
 215 }
 216
 217 static void ism_free_dmb(struct ism_dev *ism, struct smcd_dmb *dmb)
 218 {
 219         clear_bit(dmb->sba_idx, ism->sba_bitmap);
 220         dma_free_coherent(&ism->pdev->dev, dmb->dmb_len,
 221                           dmb->cpu_addr, dmb->dma_addr);
 222 }
 223
 224 static int ism_alloc_dmb(struct ism_dev *ism, struct smcd_dmb *dmb)
 225 {
 226         unsigned long bit;
 227
 228         if (PAGE_ALIGN(dmb->dmb_len) > dma_get_max_seg_size(&ism->pdev->dev))
 229                 return -EINVAL;
 230
 231         if (!dmb->sba_idx) {
 232                 bit = find_next_zero_bit(ism->sba_bitmap, ISM_NR_DMBS,
 233                                          ISM_DMB_BIT_OFFSET);
 234                 if (bit == ISM_NR_DMBS)
 235                         return -ENOMEM;
 236
 237                 dmb->sba_idx = bit;
 238         }
 239         if (dmb->sba_idx < ISM_DMB_BIT_OFFSET ||
 240             test_and_set_bit(dmb->sba_idx, ism->sba_bitmap))
 241                 return -EINVAL;
 242
 243         dmb->cpu_addr = dma_zalloc_coherent(&ism->pdev->dev, dmb->dmb_len,
 244                                             &dmb->dma_addr, GFP_KERNEL |
 245                                             __GFP_NOWARN | __GFP_NOMEMALLOC |
 246                                             __GFP_COMP | __GFP_NORETRY);
 247         if (!dmb->cpu_addr)
 248                 clear_bit(dmb->sba_idx, ism->sba_bitmap);
 249
 250         return dmb->cpu_addr ? 0 : -ENOMEM;
 251 }
 252
 253 static int ism_register_dmb(struct smcd_dev *smcd, struct smcd_dmb *dmb)
 254 {
 255         struct ism_dev *ism = smcd->priv;
 256         union ism_reg_dmb cmd;
 257         int ret;
 258
 259         ret = ism_alloc_dmb(ism, dmb);
 260         if (ret)
 261                 goto out;
 262
 263         memset(&cmd, 0, sizeof(cmd));
 264         cmd.request.hdr.cmd = ISM_REG_DMB;
 265         cmd.request.hdr.len = sizeof(cmd.request);
 266
 267         cmd.request.dmb = dmb->dma_addr;
 268         cmd.request.dmb_len = dmb->dmb_len;
 269         cmd.request.sba_idx = dmb->sba_idx;
 270         cmd.request.vlan_valid = dmb->vlan_valid;
 271         cmd.request.vlan_id = dmb->vlan_id;
 272         cmd.request.rgid = dmb->rgid;
 273
 274         ret = ism_cmd(ism, &cmd);
 275         if (ret) {
 276                 ism_free_dmb(ism, dmb);
 277                 goto out;
 278         }
 279         dmb->dmb_tok = cmd.response.dmb_tok;
 280 out:
 281         return ret;
 282 }
 283
 284 static int ism_unregister_dmb(struct smcd_dev *smcd, struct smcd_dmb *dmb)
 285 {
 286         struct ism_dev *ism = smcd->priv;
 287         union ism_unreg_dmb cmd;
 288         int ret;
 289
 290         memset(&cmd, 0, sizeof(cmd));
 291         cmd.request.hdr.cmd = ISM_UNREG_DMB;
 292         cmd.request.hdr.len = sizeof(cmd.request);
 293
 294         cmd.request.dmb_tok = dmb->dmb_tok;
 295
 296         ret = ism_cmd(ism, &cmd);
 297         if (ret && ret != ISM_ERROR)
 298                 goto out;
 299
 300         ism_free_dmb(ism, dmb);
 301 out:
 302         return ret;
 303 }
 304
 305 static int ism_add_vlan_id(struct smcd_dev *smcd, u64 vlan_id)
 306 {
 307         struct ism_dev *ism = smcd->priv;
 308         union ism_set_vlan_id cmd;
 309
 310         memset(&cmd, 0, sizeof(cmd));
 311         cmd.request.hdr.cmd = ISM_ADD_VLAN_ID;
 312         cmd.request.hdr.len = sizeof(cmd.request);
 313
 314         cmd.request.vlan_id = vlan_id;
 315
 316         return ism_cmd(ism, &cmd);
 317 }
 318
 319 static int ism_del_vlan_id(struct smcd_dev *smcd, u64 vlan_id)
 320 {
 321         struct ism_dev *ism = smcd->priv;
 322         union ism_set_vlan_id cmd;
 323
 324         memset(&cmd, 0, sizeof(cmd));
 325         cmd.request.hdr.cmd = ISM_DEL_VLAN_ID;
 326         cmd.request.hdr.len = sizeof(cmd.request);
 327
 328         cmd.request.vlan_id = vlan_id;
 329
 330         return ism_cmd(ism, &cmd);
 331 }
 332
 333 static int ism_set_vlan_required(struct smcd_dev *smcd)
 334 {
 335         return ism_cmd_simple(smcd->priv, ISM_SET_VLAN);
 336 }
 337
 338 static int ism_reset_vlan_required(struct smcd_dev *smcd)
 339 {
 340         return ism_cmd_simple(smcd->priv, ISM_RESET_VLAN);
 341 }
 342
 343 static int ism_signal_ieq(struct smcd_dev *smcd, u64 rgid, u32 trigger_irq,
 344                           u32 event_code, u64 info)
 345 {
 346         struct ism_dev *ism = smcd->priv;
 347         union ism_sig_ieq cmd;
 348
 349         memset(&cmd, 0, sizeof(cmd));
 350         cmd.request.hdr.cmd = ISM_SIGNAL_IEQ;
 351         cmd.request.hdr.len = sizeof(cmd.request);
 352
 353         cmd.request.rgid = rgid;
 354         cmd.request.trigger_irq = trigger_irq;
 355         cmd.request.event_code = event_code;
 356         cmd.request.info = info;
 357
 358         return ism_cmd(ism, &cmd);
 359 }
 360
 361 static unsigned int max_bytes(unsigned int start, unsigned int len,
 362                               unsigned int boundary)
 363 {
 364         return min(boundary - (start & (boundary - 1)), len);
 365 }
 366
 367 static int ism_move(struct smcd_dev *smcd, u64 dmb_tok, unsigned int idx,
 368                     bool sf, unsigned int offset, void *data, unsigned int size)
 369 {
 370         struct ism_dev *ism = smcd->priv;
 371         unsigned int bytes;
 372         u64 dmb_req;
 373         int ret;
 374
 375         while (size) {
 376                 bytes = max_bytes(offset, size, PAGE_SIZE);
 377                 dmb_req = ISM_CREATE_REQ(dmb_tok, idx, size == bytes ? sf : 0,
 378                                          offset);
 379
 380                 ret = __ism_move(ism, dmb_req, data, bytes);
 381                 if (ret)
 382                         return ret;
 383
 384                 size -= bytes;
 385                 data += bytes;
 386                 offset += bytes;
 387         }
 388
 389         return 0;
 390 }
 391
 392 static void ism_handle_event(struct ism_dev *ism)
 393 {
 394         struct smcd_event *entry;
 395
 396         while ((ism->ieq_idx + 1) != READ_ONCE(ism->ieq->header.idx)) {
 397                 if (++(ism->ieq_idx) == ARRAY_SIZE(ism->ieq->entry))
 398                         ism->ieq_idx = 0;
 399
 400                 entry = &ism->ieq->entry[ism->ieq_idx];
 401                 debug_event(ism_debug_info, 2, entry, sizeof(*entry));
 402                 smcd_handle_event(ism->smcd, entry);
 403         }
 404 }
 405
 406 static irqreturn_t ism_handle_irq(int irq, void *data)
 407 {
 408         struct ism_dev *ism = data;
 409         unsigned long bit, end;
 410         unsigned long *bv;
 411
 412         bv = (void *) &ism->sba->dmb_bits[ISM_DMB_WORD_OFFSET];
 413         end = sizeof(ism->sba->dmb_bits) * BITS_PER_BYTE - ISM_DMB_BIT_OFFSET;
 414
 415         spin_lock(&ism->lock);
 416         ism->sba->s = 0;
 417         barrier();
 418         for (bit = 0;;) {
 419                 bit = find_next_bit_inv(bv, end, bit);
 420                 if (bit >= end)
 421                         break;
 422
 423                 clear_bit_inv(bit, bv);
 424                 ism->sba->dmbe_mask[bit + ISM_DMB_BIT_OFFSET] = 0;
 425                 barrier();
 426                 smcd_handle_irq(ism->smcd, bit + ISM_DMB_BIT_OFFSET);
 427         }
 428
 429         if (ism->sba->e) {
 430                 ism->sba->e = 0;
 431                 barrier();
 432                 ism_handle_event(ism);
 433         }
 434         spin_unlock(&ism->lock);
 435         return IRQ_HANDLED;
 436 }
 437
 438 static const struct smcd_ops ism_ops = {
 439         .query_remote_gid = ism_query_rgid,
 440         .register_dmb = ism_register_dmb,
 441         .unregister_dmb = ism_unregister_dmb,
 442         .add_vlan_id = ism_add_vlan_id,
 443         .del_vlan_id = ism_del_vlan_id,
 444         .set_vlan_required = ism_set_vlan_required,
 445         .reset_vlan_required = ism_reset_vlan_required,
 446         .signal_event = ism_signal_ieq,
 447         .move_data = ism_move,
 448 };
 449
 450 static int ism_dev_init(struct ism_dev *ism)
 451 {
 452         struct pci_dev *pdev = ism->pdev;
 453         int ret;
 454
 455         ret = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_MSI);
 456         if (ret <= 0)
 457                 goto out;
 458
 459         ret = request_irq(pci_irq_vector(pdev, 0), ism_handle_irq, 0,
 460                           pci_name(pdev), ism);
 461         if (ret)
 462                 goto free_vectors;
 463
 464         ret = register_sba(ism);
 465         if (ret)
 466                 goto free_irq;
 467
 468         ret = register_ieq(ism);
 469         if (ret)
 470                 goto unreg_sba;
 471
 472         ret = ism_read_local_gid(ism);
 473         if (ret)
 474                 goto unreg_ieq;
 475
 476         ret = smcd_register_dev(ism->smcd);
 477         if (ret)
 478                 goto unreg_ieq;
 479
 480         query_info(ism);
 481         return 0;
 482
 483 unreg_ieq:
 484         unregister_ieq(ism);
 485 unreg_sba:
 486         unregister_sba(ism);
 487 free_irq:
 488         free_irq(pci_irq_vector(pdev, 0), ism);
 489 free_vectors:
 490         pci_free_irq_vectors(pdev);
 491 out:
 492         return ret;
 493 }
 494
 495 static int ism_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 496 {
 497         struct ism_dev *ism;
 498         int ret;
 499
 500         ism = kzalloc(sizeof(*ism), GFP_KERNEL);
 501         if (!ism)
 502                 return -ENOMEM;
 503
 504         spin_lock_init(&ism->lock);
 505         dev_set_drvdata(&pdev->dev, ism);
 506         ism->pdev = pdev;
 507
 508         ret = pci_enable_device_mem(pdev);
 509         if (ret)
 510                 goto err;
 511
 512         ret = pci_request_mem_regions(pdev, DRV_NAME);
 513         if (ret)
 514                 goto err_disable;
 515
 516         ism->ctl = pci_iomap(pdev, 2, 0);
 517         if (!ism->ctl)
 518                 goto err_resource;
 519
 520         ret = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
 521         if (ret)
 522                 goto err_unmap;
 523
 524         pci_set_dma_seg_boundary(pdev, SZ_1M - 1);
 525         pci_set_dma_max_seg_size(pdev, SZ_1M);
 526         pci_set_master(pdev);
 527
 528         ism->smcd = smcd_alloc_dev(&pdev->dev, dev_name(&pdev->dev), &ism_ops,
 529                                    ISM_NR_DMBS);
 530         if (!ism->smcd)
 531                 goto err_unmap;
 532
 533         ism->smcd->priv = ism;
 534         ret = ism_dev_init(ism);
 535         if (ret)
 536                 goto err_free;
 537
 538         return 0;
 539
 540 err_free:
 541         smcd_free_dev(ism->smcd);
 542 err_unmap:
 543         pci_iounmap(pdev, ism->ctl);
 544 err_resource:
 545         pci_release_mem_regions(pdev);
 546 err_disable:
 547         pci_disable_device(pdev);
 548 err:
 549         kfree(ism);
 550         dev_set_drvdata(&pdev->dev, NULL);
 551         return ret;
 552 }
 553
 554 static void ism_dev_exit(struct ism_dev *ism)
 555 {
 556         struct pci_dev *pdev = ism->pdev;
 557
 558         smcd_unregister_dev(ism->smcd);
 559         unregister_ieq(ism);
 560         unregister_sba(ism);
 561         free_irq(pci_irq_vector(pdev, 0), ism);
 562         pci_free_irq_vectors(pdev);
 563 }
 564
 565 static void ism_remove(struct pci_dev *pdev)
 566 {
 567         struct ism_dev *ism = dev_get_drvdata(&pdev->dev);
 568
 569         ism_dev_exit(ism);
 570
 571         smcd_free_dev(ism->smcd);
 572         pci_iounmap(pdev, ism->ctl);
 573         pci_release_mem_regions(pdev);
 574         pci_disable_device(pdev);
 575         dev_set_drvdata(&pdev->dev, NULL);
 576         kfree(ism);
 577 }
 578
 579 static int ism_suspend(struct device *dev)
 580 {
 581         struct ism_dev *ism = dev_get_drvdata(dev);
 582
 583         ism_dev_exit(ism);
 584         return 0;
 585 }
 586
 587 static int ism_resume(struct device *dev)
 588 {
 589         struct ism_dev *ism = dev_get_drvdata(dev);
 590
 591         return ism_dev_init(ism);
 592 }
 593
 594 static SIMPLE_DEV_PM_OPS(ism_pm_ops, ism_suspend, ism_resume);
 595
 596 static struct pci_driver ism_driver = {
 597         .name     = DRV_NAME,
 598         .id_table = ism_device_table,
 599         .probe    = ism_probe,
 600         .remove   = ism_remove,
 601         .driver   = {
 602                 .pm = &ism_pm_ops,
 603         },
 604 };
 605
 606 static int __init ism_init(void)
 607 {
 608         int ret;
 609
 610         ism_debug_info = debug_register("ism", 2, 1, 16);
 611         if (!ism_debug_info)
 612                 return -ENODEV;
 613
 614         debug_register_view(ism_debug_info, &debug_hex_ascii_view);
 615         ret = pci_register_driver(&ism_driver);
 616         if (ret)
 617                 debug_unregister(ism_debug_info);
 618
 619         return ret;
 620 }
 621
 622 static void __exit ism_exit(void)
 623 {
 624         pci_unregister_driver(&ism_driver);
 625         debug_unregister(ism_debug_info);
 626 }
 627
 628 module_init(ism_init);
 629 module_exit(ism_exit);