1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright(c) 2019 Intel Corporation. All rights rsvd. */
3 #include <linux/init.h>
4 #include <linux/kernel.h>
5 #include <linux/module.h>
7 #include <linux/device.h>
8 #include <linux/io-64-nonatomic-lo-hi.h>
9 #include <uapi/linux/idxd.h>
10 #include "registers.h"
13 static char *idxd_wq_type_names[] = {
14 [IDXD_WQT_NONE] = "none",
15 [IDXD_WQT_KERNEL] = "kernel",
16 [IDXD_WQT_USER] = "user",
19 static void idxd_conf_device_release(struct device *dev)
21 dev_dbg(dev, "%s for %s\n", __func__, dev_name(dev));
24 static struct device_type idxd_group_device_type = {
26 .release = idxd_conf_device_release,
29 static struct device_type idxd_wq_device_type = {
31 .release = idxd_conf_device_release,
34 static struct device_type idxd_engine_device_type = {
36 .release = idxd_conf_device_release,
39 static struct device_type dsa_device_type = {
41 .release = idxd_conf_device_release,
44 static inline bool is_dsa_dev(struct device *dev)
46 return dev ? dev->type == &dsa_device_type : false;
49 static inline bool is_idxd_dev(struct device *dev)
51 return is_dsa_dev(dev);
54 static inline bool is_idxd_wq_dev(struct device *dev)
56 return dev ? dev->type == &idxd_wq_device_type : false;
59 static inline bool is_idxd_wq_dmaengine(struct idxd_wq *wq)
61 if (wq->type == IDXD_WQT_KERNEL &&
62 strcmp(wq->name, "dmaengine") == 0)
67 static inline bool is_idxd_wq_cdev(struct idxd_wq *wq)
69 return wq->type == IDXD_WQT_USER;
72 static int idxd_config_bus_match(struct device *dev,
73 struct device_driver *drv)
77 if (is_idxd_dev(dev)) {
78 struct idxd_device *idxd = confdev_to_idxd(dev);
80 if (idxd->state != IDXD_DEV_CONF_READY)
83 } else if (is_idxd_wq_dev(dev)) {
84 struct idxd_wq *wq = confdev_to_wq(dev);
85 struct idxd_device *idxd = wq->idxd;
87 if (idxd->state < IDXD_DEV_CONF_READY)
90 if (wq->state != IDXD_WQ_DISABLED) {
91 dev_dbg(dev, "%s not disabled\n", dev_name(dev));
98 dev_dbg(dev, "%s matched\n", dev_name(dev));
103 static int idxd_config_bus_probe(struct device *dev)
108 dev_dbg(dev, "%s called\n", __func__);
110 if (is_idxd_dev(dev)) {
111 struct idxd_device *idxd = confdev_to_idxd(dev);
113 if (idxd->state != IDXD_DEV_CONF_READY) {
114 dev_warn(dev, "Device not ready for config\n");
118 if (!try_module_get(THIS_MODULE))
121 /* Perform IDXD configuration and enabling */
122 spin_lock_irqsave(&idxd->dev_lock, flags);
123 rc = idxd_device_config(idxd);
124 spin_unlock_irqrestore(&idxd->dev_lock, flags);
126 module_put(THIS_MODULE);
127 dev_warn(dev, "Device config failed: %d\n", rc);
132 rc = idxd_device_enable(idxd);
134 module_put(THIS_MODULE);
135 dev_warn(dev, "Device enable failed: %d\n", rc);
139 dev_info(dev, "Device %s enabled\n", dev_name(dev));
141 rc = idxd_register_dma_device(idxd);
143 module_put(THIS_MODULE);
144 dev_dbg(dev, "Failed to register dmaengine device\n");
148 } else if (is_idxd_wq_dev(dev)) {
149 struct idxd_wq *wq = confdev_to_wq(dev);
150 struct idxd_device *idxd = wq->idxd;
152 mutex_lock(&wq->wq_lock);
154 if (idxd->state != IDXD_DEV_ENABLED) {
155 mutex_unlock(&wq->wq_lock);
156 dev_warn(dev, "Enabling while device not enabled.\n");
160 if (wq->state != IDXD_WQ_DISABLED) {
161 mutex_unlock(&wq->wq_lock);
162 dev_warn(dev, "WQ %d already enabled.\n", wq->id);
167 mutex_unlock(&wq->wq_lock);
168 dev_warn(dev, "WQ not attached to group.\n");
172 if (strlen(wq->name) == 0) {
173 mutex_unlock(&wq->wq_lock);
174 dev_warn(dev, "WQ name not set.\n");
178 rc = idxd_wq_alloc_resources(wq);
180 mutex_unlock(&wq->wq_lock);
181 dev_warn(dev, "WQ resource alloc failed\n");
185 spin_lock_irqsave(&idxd->dev_lock, flags);
186 rc = idxd_device_config(idxd);
187 spin_unlock_irqrestore(&idxd->dev_lock, flags);
189 mutex_unlock(&wq->wq_lock);
190 dev_warn(dev, "Writing WQ %d config failed: %d\n",
195 rc = idxd_wq_enable(wq);
197 mutex_unlock(&wq->wq_lock);
198 dev_warn(dev, "WQ %d enabling failed: %d\n",
203 rc = idxd_wq_map_portal(wq);
205 dev_warn(dev, "wq portal mapping failed: %d\n", rc);
206 rc = idxd_wq_disable(wq);
208 dev_warn(dev, "IDXD wq disable failed\n");
209 mutex_unlock(&wq->wq_lock);
213 wq->client_count = 0;
215 dev_info(dev, "wq %s enabled\n", dev_name(&wq->conf_dev));
217 if (is_idxd_wq_dmaengine(wq)) {
218 rc = idxd_register_dma_channel(wq);
220 dev_dbg(dev, "DMA channel register failed\n");
221 mutex_unlock(&wq->wq_lock);
224 } else if (is_idxd_wq_cdev(wq)) {
225 rc = idxd_wq_add_cdev(wq);
227 dev_dbg(dev, "Cdev creation failed\n");
228 mutex_unlock(&wq->wq_lock);
233 mutex_unlock(&wq->wq_lock);
240 static void disable_wq(struct idxd_wq *wq)
242 struct idxd_device *idxd = wq->idxd;
243 struct device *dev = &idxd->pdev->dev;
245 mutex_lock(&wq->wq_lock);
246 dev_dbg(dev, "%s removing WQ %s\n", __func__, dev_name(&wq->conf_dev));
247 if (wq->state == IDXD_WQ_DISABLED) {
248 mutex_unlock(&wq->wq_lock);
252 if (is_idxd_wq_dmaengine(wq))
253 idxd_unregister_dma_channel(wq);
254 else if (is_idxd_wq_cdev(wq))
255 idxd_wq_del_cdev(wq);
257 if (idxd_wq_refcount(wq))
258 dev_warn(dev, "Clients has claim on wq %d: %d\n",
259 wq->id, idxd_wq_refcount(wq));
261 idxd_wq_unmap_portal(wq);
266 idxd_wq_free_resources(wq);
267 wq->client_count = 0;
268 mutex_unlock(&wq->wq_lock);
270 dev_info(dev, "wq %s disabled\n", dev_name(&wq->conf_dev));
273 static int idxd_config_bus_remove(struct device *dev)
277 dev_dbg(dev, "%s called for %s\n", __func__, dev_name(dev));
279 /* disable workqueue here */
280 if (is_idxd_wq_dev(dev)) {
281 struct idxd_wq *wq = confdev_to_wq(dev);
284 } else if (is_idxd_dev(dev)) {
285 struct idxd_device *idxd = confdev_to_idxd(dev);
288 dev_dbg(dev, "%s removing dev %s\n", __func__,
289 dev_name(&idxd->conf_dev));
290 for (i = 0; i < idxd->max_wqs; i++) {
291 struct idxd_wq *wq = &idxd->wqs[i];
293 if (wq->state == IDXD_WQ_DISABLED)
295 dev_warn(dev, "Active wq %d on disable %s.\n", i,
296 dev_name(&idxd->conf_dev));
297 device_release_driver(&wq->conf_dev);
300 idxd_unregister_dma_device(idxd);
301 rc = idxd_device_disable(idxd);
302 for (i = 0; i < idxd->max_wqs; i++) {
303 struct idxd_wq *wq = &idxd->wqs[i];
305 mutex_lock(&wq->wq_lock);
306 idxd_wq_disable_cleanup(wq);
307 mutex_unlock(&wq->wq_lock);
309 module_put(THIS_MODULE);
311 dev_warn(dev, "Device disable failed\n");
313 dev_info(dev, "Device %s disabled\n", dev_name(dev));
320 static void idxd_config_bus_shutdown(struct device *dev)
322 dev_dbg(dev, "%s called\n", __func__);
325 struct bus_type dsa_bus_type = {
327 .match = idxd_config_bus_match,
328 .probe = idxd_config_bus_probe,
329 .remove = idxd_config_bus_remove,
330 .shutdown = idxd_config_bus_shutdown,
333 static struct bus_type *idxd_bus_types[] = {
337 static struct idxd_device_driver dsa_drv = {
340 .bus = &dsa_bus_type,
341 .owner = THIS_MODULE,
342 .mod_name = KBUILD_MODNAME,
346 static struct idxd_device_driver *idxd_drvs[] = {
350 struct bus_type *idxd_get_bus_type(struct idxd_device *idxd)
352 return idxd_bus_types[idxd->type];
355 static struct device_type *idxd_get_device_type(struct idxd_device *idxd)
357 if (idxd->type == IDXD_TYPE_DSA)
358 return &dsa_device_type;
363 /* IDXD generic driver setup */
364 int idxd_register_driver(void)
368 for (i = 0; i < IDXD_TYPE_MAX; i++) {
369 rc = driver_register(&idxd_drvs[i]->drv);
378 driver_unregister(&idxd_drvs[i]->drv);
382 void idxd_unregister_driver(void)
386 for (i = 0; i < IDXD_TYPE_MAX; i++)
387 driver_unregister(&idxd_drvs[i]->drv);
390 /* IDXD engine attributes */
391 static ssize_t engine_group_id_show(struct device *dev,
392 struct device_attribute *attr, char *buf)
394 struct idxd_engine *engine =
395 container_of(dev, struct idxd_engine, conf_dev);
398 return sprintf(buf, "%d\n", engine->group->id);
400 return sprintf(buf, "%d\n", -1);
403 static ssize_t engine_group_id_store(struct device *dev,
404 struct device_attribute *attr,
405 const char *buf, size_t count)
407 struct idxd_engine *engine =
408 container_of(dev, struct idxd_engine, conf_dev);
409 struct idxd_device *idxd = engine->idxd;
412 struct idxd_group *prevg;
414 rc = kstrtol(buf, 10, &id);
418 if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags))
421 if (id > idxd->max_groups - 1 || id < -1)
426 engine->group->num_engines--;
427 engine->group = NULL;
432 prevg = engine->group;
435 prevg->num_engines--;
436 engine->group = &idxd->groups[id];
437 engine->group->num_engines++;
442 static struct device_attribute dev_attr_engine_group =
443 __ATTR(group_id, 0644, engine_group_id_show,
444 engine_group_id_store);
446 static struct attribute *idxd_engine_attributes[] = {
447 &dev_attr_engine_group.attr,
451 static const struct attribute_group idxd_engine_attribute_group = {
452 .attrs = idxd_engine_attributes,
455 static const struct attribute_group *idxd_engine_attribute_groups[] = {
456 &idxd_engine_attribute_group,
460 /* Group attributes */
462 static void idxd_set_free_tokens(struct idxd_device *idxd)
466 for (i = 0, tokens = 0; i < idxd->max_groups; i++) {
467 struct idxd_group *g = &idxd->groups[i];
469 tokens += g->tokens_reserved;
472 idxd->nr_tokens = idxd->max_tokens - tokens;
475 static ssize_t group_tokens_reserved_show(struct device *dev,
476 struct device_attribute *attr,
479 struct idxd_group *group =
480 container_of(dev, struct idxd_group, conf_dev);
482 return sprintf(buf, "%u\n", group->tokens_reserved);
485 static ssize_t group_tokens_reserved_store(struct device *dev,
486 struct device_attribute *attr,
487 const char *buf, size_t count)
489 struct idxd_group *group =
490 container_of(dev, struct idxd_group, conf_dev);
491 struct idxd_device *idxd = group->idxd;
495 rc = kstrtoul(buf, 10, &val);
499 if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags))
502 if (idxd->state == IDXD_DEV_ENABLED)
505 if (val > idxd->max_tokens)
508 if (val > idxd->nr_tokens + group->tokens_reserved)
511 group->tokens_reserved = val;
512 idxd_set_free_tokens(idxd);
516 static struct device_attribute dev_attr_group_tokens_reserved =
517 __ATTR(tokens_reserved, 0644, group_tokens_reserved_show,
518 group_tokens_reserved_store);
520 static ssize_t group_tokens_allowed_show(struct device *dev,
521 struct device_attribute *attr,
524 struct idxd_group *group =
525 container_of(dev, struct idxd_group, conf_dev);
527 return sprintf(buf, "%u\n", group->tokens_allowed);
530 static ssize_t group_tokens_allowed_store(struct device *dev,
531 struct device_attribute *attr,
532 const char *buf, size_t count)
534 struct idxd_group *group =
535 container_of(dev, struct idxd_group, conf_dev);
536 struct idxd_device *idxd = group->idxd;
540 rc = kstrtoul(buf, 10, &val);
544 if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags))
547 if (idxd->state == IDXD_DEV_ENABLED)
550 if (val < 4 * group->num_engines ||
551 val > group->tokens_reserved + idxd->nr_tokens)
554 group->tokens_allowed = val;
558 static struct device_attribute dev_attr_group_tokens_allowed =
559 __ATTR(tokens_allowed, 0644, group_tokens_allowed_show,
560 group_tokens_allowed_store);
562 static ssize_t group_use_token_limit_show(struct device *dev,
563 struct device_attribute *attr,
566 struct idxd_group *group =
567 container_of(dev, struct idxd_group, conf_dev);
569 return sprintf(buf, "%u\n", group->use_token_limit);
572 static ssize_t group_use_token_limit_store(struct device *dev,
573 struct device_attribute *attr,
574 const char *buf, size_t count)
576 struct idxd_group *group =
577 container_of(dev, struct idxd_group, conf_dev);
578 struct idxd_device *idxd = group->idxd;
582 rc = kstrtoul(buf, 10, &val);
586 if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags))
589 if (idxd->state == IDXD_DEV_ENABLED)
592 if (idxd->token_limit == 0)
595 group->use_token_limit = !!val;
599 static struct device_attribute dev_attr_group_use_token_limit =
600 __ATTR(use_token_limit, 0644, group_use_token_limit_show,
601 group_use_token_limit_store);
603 static ssize_t group_engines_show(struct device *dev,
604 struct device_attribute *attr, char *buf)
606 struct idxd_group *group =
607 container_of(dev, struct idxd_group, conf_dev);
610 struct idxd_device *idxd = group->idxd;
612 for (i = 0; i < idxd->max_engines; i++) {
613 struct idxd_engine *engine = &idxd->engines[i];
618 if (engine->group->id == group->id)
619 rc += sprintf(tmp + rc, "engine%d.%d ",
620 idxd->id, engine->id);
624 rc += sprintf(tmp + rc, "\n");
629 static struct device_attribute dev_attr_group_engines =
630 __ATTR(engines, 0444, group_engines_show, NULL);
632 static ssize_t group_work_queues_show(struct device *dev,
633 struct device_attribute *attr, char *buf)
635 struct idxd_group *group =
636 container_of(dev, struct idxd_group, conf_dev);
639 struct idxd_device *idxd = group->idxd;
641 for (i = 0; i < idxd->max_wqs; i++) {
642 struct idxd_wq *wq = &idxd->wqs[i];
647 if (wq->group->id == group->id)
648 rc += sprintf(tmp + rc, "wq%d.%d ",
653 rc += sprintf(tmp + rc, "\n");
658 static struct device_attribute dev_attr_group_work_queues =
659 __ATTR(work_queues, 0444, group_work_queues_show, NULL);
661 static ssize_t group_traffic_class_a_show(struct device *dev,
662 struct device_attribute *attr,
665 struct idxd_group *group =
666 container_of(dev, struct idxd_group, conf_dev);
668 return sprintf(buf, "%d\n", group->tc_a);
671 static ssize_t group_traffic_class_a_store(struct device *dev,
672 struct device_attribute *attr,
673 const char *buf, size_t count)
675 struct idxd_group *group =
676 container_of(dev, struct idxd_group, conf_dev);
677 struct idxd_device *idxd = group->idxd;
681 rc = kstrtol(buf, 10, &val);
685 if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags))
688 if (idxd->state == IDXD_DEV_ENABLED)
691 if (val < 0 || val > 7)
698 static struct device_attribute dev_attr_group_traffic_class_a =
699 __ATTR(traffic_class_a, 0644, group_traffic_class_a_show,
700 group_traffic_class_a_store);
702 static ssize_t group_traffic_class_b_show(struct device *dev,
703 struct device_attribute *attr,
706 struct idxd_group *group =
707 container_of(dev, struct idxd_group, conf_dev);
709 return sprintf(buf, "%d\n", group->tc_b);
712 static ssize_t group_traffic_class_b_store(struct device *dev,
713 struct device_attribute *attr,
714 const char *buf, size_t count)
716 struct idxd_group *group =
717 container_of(dev, struct idxd_group, conf_dev);
718 struct idxd_device *idxd = group->idxd;
722 rc = kstrtol(buf, 10, &val);
726 if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags))
729 if (idxd->state == IDXD_DEV_ENABLED)
732 if (val < 0 || val > 7)
739 static struct device_attribute dev_attr_group_traffic_class_b =
740 __ATTR(traffic_class_b, 0644, group_traffic_class_b_show,
741 group_traffic_class_b_store);
743 static struct attribute *idxd_group_attributes[] = {
744 &dev_attr_group_work_queues.attr,
745 &dev_attr_group_engines.attr,
746 &dev_attr_group_use_token_limit.attr,
747 &dev_attr_group_tokens_allowed.attr,
748 &dev_attr_group_tokens_reserved.attr,
749 &dev_attr_group_traffic_class_a.attr,
750 &dev_attr_group_traffic_class_b.attr,
754 static const struct attribute_group idxd_group_attribute_group = {
755 .attrs = idxd_group_attributes,
758 static const struct attribute_group *idxd_group_attribute_groups[] = {
759 &idxd_group_attribute_group,
763 /* IDXD work queue attribs */
764 static ssize_t wq_clients_show(struct device *dev,
765 struct device_attribute *attr, char *buf)
767 struct idxd_wq *wq = container_of(dev, struct idxd_wq, conf_dev);
769 return sprintf(buf, "%d\n", wq->client_count);
772 static struct device_attribute dev_attr_wq_clients =
773 __ATTR(clients, 0444, wq_clients_show, NULL);
775 static ssize_t wq_state_show(struct device *dev,
776 struct device_attribute *attr, char *buf)
778 struct idxd_wq *wq = container_of(dev, struct idxd_wq, conf_dev);
781 case IDXD_WQ_DISABLED:
782 return sprintf(buf, "disabled\n");
783 case IDXD_WQ_ENABLED:
784 return sprintf(buf, "enabled\n");
787 return sprintf(buf, "unknown\n");
790 static struct device_attribute dev_attr_wq_state =
791 __ATTR(state, 0444, wq_state_show, NULL);
793 static ssize_t wq_group_id_show(struct device *dev,
794 struct device_attribute *attr, char *buf)
796 struct idxd_wq *wq = container_of(dev, struct idxd_wq, conf_dev);
799 return sprintf(buf, "%u\n", wq->group->id);
801 return sprintf(buf, "-1\n");
804 static ssize_t wq_group_id_store(struct device *dev,
805 struct device_attribute *attr,
806 const char *buf, size_t count)
808 struct idxd_wq *wq = container_of(dev, struct idxd_wq, conf_dev);
809 struct idxd_device *idxd = wq->idxd;
812 struct idxd_group *prevg, *group;
814 rc = kstrtol(buf, 10, &id);
818 if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags))
821 if (wq->state != IDXD_WQ_DISABLED)
824 if (id > idxd->max_groups - 1 || id < -1)
829 wq->group->num_wqs--;
835 group = &idxd->groups[id];
845 static struct device_attribute dev_attr_wq_group_id =
846 __ATTR(group_id, 0644, wq_group_id_show, wq_group_id_store);
848 static ssize_t wq_mode_show(struct device *dev, struct device_attribute *attr,
851 struct idxd_wq *wq = container_of(dev, struct idxd_wq, conf_dev);
853 return sprintf(buf, "%s\n",
854 wq_dedicated(wq) ? "dedicated" : "shared");
857 static ssize_t wq_mode_store(struct device *dev,
858 struct device_attribute *attr, const char *buf,
861 struct idxd_wq *wq = container_of(dev, struct idxd_wq, conf_dev);
862 struct idxd_device *idxd = wq->idxd;
864 if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags))
867 if (wq->state != IDXD_WQ_DISABLED)
870 if (sysfs_streq(buf, "dedicated")) {
871 set_bit(WQ_FLAG_DEDICATED, &wq->flags);
880 static struct device_attribute dev_attr_wq_mode =
881 __ATTR(mode, 0644, wq_mode_show, wq_mode_store);
883 static ssize_t wq_size_show(struct device *dev, struct device_attribute *attr,
886 struct idxd_wq *wq = container_of(dev, struct idxd_wq, conf_dev);
888 return sprintf(buf, "%u\n", wq->size);
891 static int total_claimed_wq_size(struct idxd_device *idxd)
896 for (i = 0; i < idxd->max_wqs; i++) {
897 struct idxd_wq *wq = &idxd->wqs[i];
905 static ssize_t wq_size_store(struct device *dev,
906 struct device_attribute *attr, const char *buf,
909 struct idxd_wq *wq = container_of(dev, struct idxd_wq, conf_dev);
911 struct idxd_device *idxd = wq->idxd;
914 rc = kstrtoul(buf, 10, &size);
918 if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags))
921 if (idxd->state == IDXD_DEV_ENABLED)
924 if (size + total_claimed_wq_size(idxd) - wq->size > idxd->max_wq_size)
931 static struct device_attribute dev_attr_wq_size =
932 __ATTR(size, 0644, wq_size_show, wq_size_store);
934 static ssize_t wq_priority_show(struct device *dev,
935 struct device_attribute *attr, char *buf)
937 struct idxd_wq *wq = container_of(dev, struct idxd_wq, conf_dev);
939 return sprintf(buf, "%u\n", wq->priority);
942 static ssize_t wq_priority_store(struct device *dev,
943 struct device_attribute *attr,
944 const char *buf, size_t count)
946 struct idxd_wq *wq = container_of(dev, struct idxd_wq, conf_dev);
948 struct idxd_device *idxd = wq->idxd;
951 rc = kstrtoul(buf, 10, &prio);
955 if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags))
958 if (wq->state != IDXD_WQ_DISABLED)
961 if (prio > IDXD_MAX_PRIORITY)
968 static struct device_attribute dev_attr_wq_priority =
969 __ATTR(priority, 0644, wq_priority_show, wq_priority_store);
971 static ssize_t wq_type_show(struct device *dev,
972 struct device_attribute *attr, char *buf)
974 struct idxd_wq *wq = container_of(dev, struct idxd_wq, conf_dev);
977 case IDXD_WQT_KERNEL:
978 return sprintf(buf, "%s\n",
979 idxd_wq_type_names[IDXD_WQT_KERNEL]);
981 return sprintf(buf, "%s\n",
982 idxd_wq_type_names[IDXD_WQT_USER]);
985 return sprintf(buf, "%s\n",
986 idxd_wq_type_names[IDXD_WQT_NONE]);
992 static ssize_t wq_type_store(struct device *dev,
993 struct device_attribute *attr, const char *buf,
996 struct idxd_wq *wq = container_of(dev, struct idxd_wq, conf_dev);
997 enum idxd_wq_type old_type;
999 if (wq->state != IDXD_WQ_DISABLED)
1002 old_type = wq->type;
1003 if (sysfs_streq(buf, idxd_wq_type_names[IDXD_WQT_NONE]))
1004 wq->type = IDXD_WQT_NONE;
1005 else if (sysfs_streq(buf, idxd_wq_type_names[IDXD_WQT_KERNEL]))
1006 wq->type = IDXD_WQT_KERNEL;
1007 else if (sysfs_streq(buf, idxd_wq_type_names[IDXD_WQT_USER]))
1008 wq->type = IDXD_WQT_USER;
1012 /* If we are changing queue type, clear the name */
1013 if (wq->type != old_type)
1014 memset(wq->name, 0, WQ_NAME_SIZE + 1);
1019 static struct device_attribute dev_attr_wq_type =
1020 __ATTR(type, 0644, wq_type_show, wq_type_store);
1022 static ssize_t wq_name_show(struct device *dev,
1023 struct device_attribute *attr, char *buf)
1025 struct idxd_wq *wq = container_of(dev, struct idxd_wq, conf_dev);
1027 return sprintf(buf, "%s\n", wq->name);
1030 static ssize_t wq_name_store(struct device *dev,
1031 struct device_attribute *attr, const char *buf,
1034 struct idxd_wq *wq = container_of(dev, struct idxd_wq, conf_dev);
1036 if (wq->state != IDXD_WQ_DISABLED)
1039 if (strlen(buf) > WQ_NAME_SIZE || strlen(buf) == 0)
1042 memset(wq->name, 0, WQ_NAME_SIZE + 1);
1043 strncpy(wq->name, buf, WQ_NAME_SIZE);
1044 strreplace(wq->name, '\n', '\0');
1048 static struct device_attribute dev_attr_wq_name =
1049 __ATTR(name, 0644, wq_name_show, wq_name_store);
1051 static ssize_t wq_cdev_minor_show(struct device *dev,
1052 struct device_attribute *attr, char *buf)
1054 struct idxd_wq *wq = container_of(dev, struct idxd_wq, conf_dev);
1057 mutex_lock(&wq->wq_lock);
1059 minor = wq->idxd_cdev->minor;
1060 mutex_unlock(&wq->wq_lock);
1064 return sysfs_emit(buf, "%d\n", minor);
1067 static struct device_attribute dev_attr_wq_cdev_minor =
1068 __ATTR(cdev_minor, 0444, wq_cdev_minor_show, NULL);
1070 static int __get_sysfs_u64(const char *buf, u64 *val)
1074 rc = kstrtou64(buf, 0, val);
1081 *val = roundup_pow_of_two(*val);
1085 static ssize_t wq_max_transfer_size_show(struct device *dev, struct device_attribute *attr,
1088 struct idxd_wq *wq = container_of(dev, struct idxd_wq, conf_dev);
1090 return sprintf(buf, "%llu\n", wq->max_xfer_bytes);
1093 static ssize_t wq_max_transfer_size_store(struct device *dev, struct device_attribute *attr,
1094 const char *buf, size_t count)
1096 struct idxd_wq *wq = container_of(dev, struct idxd_wq, conf_dev);
1097 struct idxd_device *idxd = wq->idxd;
1101 if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags))
1104 if (wq->state != IDXD_WQ_DISABLED)
1107 rc = __get_sysfs_u64(buf, &xfer_size);
1111 if (xfer_size > idxd->max_xfer_bytes)
1114 wq->max_xfer_bytes = xfer_size;
1119 static struct device_attribute dev_attr_wq_max_transfer_size =
1120 __ATTR(max_transfer_size, 0644,
1121 wq_max_transfer_size_show, wq_max_transfer_size_store);
1123 static ssize_t wq_max_batch_size_show(struct device *dev, struct device_attribute *attr, char *buf)
1125 struct idxd_wq *wq = container_of(dev, struct idxd_wq, conf_dev);
1127 return sprintf(buf, "%u\n", wq->max_batch_size);
1130 static ssize_t wq_max_batch_size_store(struct device *dev, struct device_attribute *attr,
1131 const char *buf, size_t count)
1133 struct idxd_wq *wq = container_of(dev, struct idxd_wq, conf_dev);
1134 struct idxd_device *idxd = wq->idxd;
1138 if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags))
1141 if (wq->state != IDXD_WQ_DISABLED)
1144 rc = __get_sysfs_u64(buf, &batch_size);
1148 if (batch_size > idxd->max_batch_size)
1151 wq->max_batch_size = (u32)batch_size;
1156 static struct device_attribute dev_attr_wq_max_batch_size =
1157 __ATTR(max_batch_size, 0644, wq_max_batch_size_show, wq_max_batch_size_store);
1159 static struct attribute *idxd_wq_attributes[] = {
1160 &dev_attr_wq_clients.attr,
1161 &dev_attr_wq_state.attr,
1162 &dev_attr_wq_group_id.attr,
1163 &dev_attr_wq_mode.attr,
1164 &dev_attr_wq_size.attr,
1165 &dev_attr_wq_priority.attr,
1166 &dev_attr_wq_type.attr,
1167 &dev_attr_wq_name.attr,
1168 &dev_attr_wq_cdev_minor.attr,
1169 &dev_attr_wq_max_transfer_size.attr,
1170 &dev_attr_wq_max_batch_size.attr,
1174 static const struct attribute_group idxd_wq_attribute_group = {
1175 .attrs = idxd_wq_attributes,
1178 static const struct attribute_group *idxd_wq_attribute_groups[] = {
1179 &idxd_wq_attribute_group,
1183 /* IDXD device attribs */
1184 static ssize_t version_show(struct device *dev, struct device_attribute *attr,
1187 struct idxd_device *idxd =
1188 container_of(dev, struct idxd_device, conf_dev);
1190 return sprintf(buf, "%#x\n", idxd->hw.version);
1192 static DEVICE_ATTR_RO(version);
1194 static ssize_t max_work_queues_size_show(struct device *dev,
1195 struct device_attribute *attr,
1198 struct idxd_device *idxd =
1199 container_of(dev, struct idxd_device, conf_dev);
1201 return sprintf(buf, "%u\n", idxd->max_wq_size);
1203 static DEVICE_ATTR_RO(max_work_queues_size);
1205 static ssize_t max_groups_show(struct device *dev,
1206 struct device_attribute *attr, char *buf)
1208 struct idxd_device *idxd =
1209 container_of(dev, struct idxd_device, conf_dev);
1211 return sprintf(buf, "%u\n", idxd->max_groups);
1213 static DEVICE_ATTR_RO(max_groups);
1215 static ssize_t max_work_queues_show(struct device *dev,
1216 struct device_attribute *attr, char *buf)
1218 struct idxd_device *idxd =
1219 container_of(dev, struct idxd_device, conf_dev);
1221 return sprintf(buf, "%u\n", idxd->max_wqs);
1223 static DEVICE_ATTR_RO(max_work_queues);
1225 static ssize_t max_engines_show(struct device *dev,
1226 struct device_attribute *attr, char *buf)
1228 struct idxd_device *idxd =
1229 container_of(dev, struct idxd_device, conf_dev);
1231 return sprintf(buf, "%u\n", idxd->max_engines);
1233 static DEVICE_ATTR_RO(max_engines);
1235 static ssize_t numa_node_show(struct device *dev,
1236 struct device_attribute *attr, char *buf)
1238 struct idxd_device *idxd =
1239 container_of(dev, struct idxd_device, conf_dev);
1241 return sprintf(buf, "%d\n", dev_to_node(&idxd->pdev->dev));
1243 static DEVICE_ATTR_RO(numa_node);
1245 static ssize_t max_batch_size_show(struct device *dev,
1246 struct device_attribute *attr, char *buf)
1248 struct idxd_device *idxd =
1249 container_of(dev, struct idxd_device, conf_dev);
1251 return sprintf(buf, "%u\n", idxd->max_batch_size);
1253 static DEVICE_ATTR_RO(max_batch_size);
1255 static ssize_t max_transfer_size_show(struct device *dev,
1256 struct device_attribute *attr,
1259 struct idxd_device *idxd =
1260 container_of(dev, struct idxd_device, conf_dev);
1262 return sprintf(buf, "%llu\n", idxd->max_xfer_bytes);
1264 static DEVICE_ATTR_RO(max_transfer_size);
1266 static ssize_t op_cap_show(struct device *dev,
1267 struct device_attribute *attr, char *buf)
1269 struct idxd_device *idxd =
1270 container_of(dev, struct idxd_device, conf_dev);
1273 for (i = 0; i < 4; i++)
1274 rc += sysfs_emit_at(buf, rc, "%#llx ", idxd->hw.opcap.bits[i]);
1277 rc += sysfs_emit_at(buf, rc, "\n");
1280 static DEVICE_ATTR_RO(op_cap);
1282 static ssize_t gen_cap_show(struct device *dev,
1283 struct device_attribute *attr, char *buf)
1285 struct idxd_device *idxd =
1286 container_of(dev, struct idxd_device, conf_dev);
1288 return sprintf(buf, "%#llx\n", idxd->hw.gen_cap.bits);
1290 static DEVICE_ATTR_RO(gen_cap);
1292 static ssize_t configurable_show(struct device *dev,
1293 struct device_attribute *attr, char *buf)
1295 struct idxd_device *idxd =
1296 container_of(dev, struct idxd_device, conf_dev);
1298 return sprintf(buf, "%u\n",
1299 test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags));
1301 static DEVICE_ATTR_RO(configurable);
1303 static ssize_t clients_show(struct device *dev,
1304 struct device_attribute *attr, char *buf)
1306 struct idxd_device *idxd =
1307 container_of(dev, struct idxd_device, conf_dev);
1308 unsigned long flags;
1311 spin_lock_irqsave(&idxd->dev_lock, flags);
1312 for (i = 0; i < idxd->max_wqs; i++) {
1313 struct idxd_wq *wq = &idxd->wqs[i];
1315 count += wq->client_count;
1317 spin_unlock_irqrestore(&idxd->dev_lock, flags);
1319 return sprintf(buf, "%d\n", count);
1321 static DEVICE_ATTR_RO(clients);
1323 static ssize_t state_show(struct device *dev,
1324 struct device_attribute *attr, char *buf)
1326 struct idxd_device *idxd =
1327 container_of(dev, struct idxd_device, conf_dev);
1329 switch (idxd->state) {
1330 case IDXD_DEV_DISABLED:
1331 case IDXD_DEV_CONF_READY:
1332 return sprintf(buf, "disabled\n");
1333 case IDXD_DEV_ENABLED:
1334 return sprintf(buf, "enabled\n");
1335 case IDXD_DEV_HALTED:
1336 return sprintf(buf, "halted\n");
1339 return sprintf(buf, "unknown\n");
1341 static DEVICE_ATTR_RO(state);
1343 static ssize_t errors_show(struct device *dev,
1344 struct device_attribute *attr, char *buf)
1346 struct idxd_device *idxd =
1347 container_of(dev, struct idxd_device, conf_dev);
1349 unsigned long flags;
1351 spin_lock_irqsave(&idxd->dev_lock, flags);
1352 for (i = 0; i < 4; i++)
1353 out += sprintf(buf + out, "%#018llx ", idxd->sw_err.bits[i]);
1354 spin_unlock_irqrestore(&idxd->dev_lock, flags);
1356 out += sprintf(buf + out, "\n");
1359 static DEVICE_ATTR_RO(errors);
1361 static ssize_t max_tokens_show(struct device *dev,
1362 struct device_attribute *attr, char *buf)
1364 struct idxd_device *idxd =
1365 container_of(dev, struct idxd_device, conf_dev);
1367 return sprintf(buf, "%u\n", idxd->max_tokens);
1369 static DEVICE_ATTR_RO(max_tokens);
1371 static ssize_t token_limit_show(struct device *dev,
1372 struct device_attribute *attr, char *buf)
1374 struct idxd_device *idxd =
1375 container_of(dev, struct idxd_device, conf_dev);
1377 return sprintf(buf, "%u\n", idxd->token_limit);
1380 static ssize_t token_limit_store(struct device *dev,
1381 struct device_attribute *attr,
1382 const char *buf, size_t count)
1384 struct idxd_device *idxd =
1385 container_of(dev, struct idxd_device, conf_dev);
1389 rc = kstrtoul(buf, 10, &val);
1393 if (idxd->state == IDXD_DEV_ENABLED)
1396 if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags))
1399 if (!idxd->hw.group_cap.token_limit)
1402 if (val > idxd->hw.group_cap.total_tokens)
1405 idxd->token_limit = val;
1408 static DEVICE_ATTR_RW(token_limit);
1410 static ssize_t cdev_major_show(struct device *dev,
1411 struct device_attribute *attr, char *buf)
1413 struct idxd_device *idxd =
1414 container_of(dev, struct idxd_device, conf_dev);
1416 return sprintf(buf, "%u\n", idxd->major);
1418 static DEVICE_ATTR_RO(cdev_major);
1420 static ssize_t cmd_status_show(struct device *dev,
1421 struct device_attribute *attr, char *buf)
1423 struct idxd_device *idxd = container_of(dev, struct idxd_device, conf_dev);
1425 return sprintf(buf, "%#x\n", idxd->cmd_status);
1427 static DEVICE_ATTR_RO(cmd_status);
1429 static struct attribute *idxd_device_attributes[] = {
1430 &dev_attr_version.attr,
1431 &dev_attr_max_groups.attr,
1432 &dev_attr_max_work_queues.attr,
1433 &dev_attr_max_work_queues_size.attr,
1434 &dev_attr_max_engines.attr,
1435 &dev_attr_numa_node.attr,
1436 &dev_attr_max_batch_size.attr,
1437 &dev_attr_max_transfer_size.attr,
1438 &dev_attr_op_cap.attr,
1439 &dev_attr_gen_cap.attr,
1440 &dev_attr_configurable.attr,
1441 &dev_attr_clients.attr,
1442 &dev_attr_state.attr,
1443 &dev_attr_errors.attr,
1444 &dev_attr_max_tokens.attr,
1445 &dev_attr_token_limit.attr,
1446 &dev_attr_cdev_major.attr,
1447 &dev_attr_cmd_status.attr,
1451 static const struct attribute_group idxd_device_attribute_group = {
1452 .attrs = idxd_device_attributes,
1455 static const struct attribute_group *idxd_attribute_groups[] = {
1456 &idxd_device_attribute_group,
1460 static int idxd_setup_engine_sysfs(struct idxd_device *idxd)
1462 struct device *dev = &idxd->pdev->dev;
1465 for (i = 0; i < idxd->max_engines; i++) {
1466 struct idxd_engine *engine = &idxd->engines[i];
1468 engine->conf_dev.parent = &idxd->conf_dev;
1469 dev_set_name(&engine->conf_dev, "engine%d.%d",
1470 idxd->id, engine->id);
1471 engine->conf_dev.bus = idxd_get_bus_type(idxd);
1472 engine->conf_dev.groups = idxd_engine_attribute_groups;
1473 engine->conf_dev.type = &idxd_engine_device_type;
1474 dev_dbg(dev, "Engine device register: %s\n",
1475 dev_name(&engine->conf_dev));
1476 rc = device_register(&engine->conf_dev);
1478 put_device(&engine->conf_dev);
1487 struct idxd_engine *engine = &idxd->engines[i];
1489 device_unregister(&engine->conf_dev);
1494 static int idxd_setup_group_sysfs(struct idxd_device *idxd)
1496 struct device *dev = &idxd->pdev->dev;
1499 for (i = 0; i < idxd->max_groups; i++) {
1500 struct idxd_group *group = &idxd->groups[i];
1502 group->conf_dev.parent = &idxd->conf_dev;
1503 dev_set_name(&group->conf_dev, "group%d.%d",
1504 idxd->id, group->id);
1505 group->conf_dev.bus = idxd_get_bus_type(idxd);
1506 group->conf_dev.groups = idxd_group_attribute_groups;
1507 group->conf_dev.type = &idxd_group_device_type;
1508 dev_dbg(dev, "Group device register: %s\n",
1509 dev_name(&group->conf_dev));
1510 rc = device_register(&group->conf_dev);
1512 put_device(&group->conf_dev);
1521 struct idxd_group *group = &idxd->groups[i];
1523 device_unregister(&group->conf_dev);
1528 static int idxd_setup_wq_sysfs(struct idxd_device *idxd)
1530 struct device *dev = &idxd->pdev->dev;
1533 for (i = 0; i < idxd->max_wqs; i++) {
1534 struct idxd_wq *wq = &idxd->wqs[i];
1536 wq->conf_dev.parent = &idxd->conf_dev;
1537 dev_set_name(&wq->conf_dev, "wq%d.%d", idxd->id, wq->id);
1538 wq->conf_dev.bus = idxd_get_bus_type(idxd);
1539 wq->conf_dev.groups = idxd_wq_attribute_groups;
1540 wq->conf_dev.type = &idxd_wq_device_type;
1541 dev_dbg(dev, "WQ device register: %s\n",
1542 dev_name(&wq->conf_dev));
1543 rc = device_register(&wq->conf_dev);
1545 put_device(&wq->conf_dev);
1554 struct idxd_wq *wq = &idxd->wqs[i];
1556 device_unregister(&wq->conf_dev);
1561 static int idxd_setup_device_sysfs(struct idxd_device *idxd)
1563 struct device *dev = &idxd->pdev->dev;
1565 char devname[IDXD_NAME_SIZE];
1567 sprintf(devname, "%s%d", idxd_get_dev_name(idxd), idxd->id);
1568 idxd->conf_dev.parent = dev;
1569 dev_set_name(&idxd->conf_dev, "%s", devname);
1570 idxd->conf_dev.bus = idxd_get_bus_type(idxd);
1571 idxd->conf_dev.groups = idxd_attribute_groups;
1572 idxd->conf_dev.type = idxd_get_device_type(idxd);
1574 dev_dbg(dev, "IDXD device register: %s\n", dev_name(&idxd->conf_dev));
1575 rc = device_register(&idxd->conf_dev);
1577 put_device(&idxd->conf_dev);
1584 int idxd_setup_sysfs(struct idxd_device *idxd)
1586 struct device *dev = &idxd->pdev->dev;
1589 rc = idxd_setup_device_sysfs(idxd);
1591 dev_dbg(dev, "Device sysfs registering failed: %d\n", rc);
1595 rc = idxd_setup_wq_sysfs(idxd);
1597 /* unregister conf dev */
1598 dev_dbg(dev, "Work Queue sysfs registering failed: %d\n", rc);
1602 rc = idxd_setup_group_sysfs(idxd);
1604 /* unregister conf dev */
1605 dev_dbg(dev, "Group sysfs registering failed: %d\n", rc);
1609 rc = idxd_setup_engine_sysfs(idxd);
1611 /* unregister conf dev */
1612 dev_dbg(dev, "Engine sysfs registering failed: %d\n", rc);
1619 void idxd_cleanup_sysfs(struct idxd_device *idxd)
1623 for (i = 0; i < idxd->max_wqs; i++) {
1624 struct idxd_wq *wq = &idxd->wqs[i];
1626 device_unregister(&wq->conf_dev);
1629 for (i = 0; i < idxd->max_engines; i++) {
1630 struct idxd_engine *engine = &idxd->engines[i];
1632 device_unregister(&engine->conf_dev);
1635 for (i = 0; i < idxd->max_groups; i++) {
1636 struct idxd_group *group = &idxd->groups[i];
1638 device_unregister(&group->conf_dev);
1641 device_unregister(&idxd->conf_dev);
1644 int idxd_register_bus_type(void)
1648 for (i = 0; i < IDXD_TYPE_MAX; i++) {
1649 rc = bus_register(idxd_bus_types[i]);
1658 bus_unregister(idxd_bus_types[i]);
1662 void idxd_unregister_bus_type(void)
1666 for (i = 0; i < IDXD_TYPE_MAX; i++)
1667 bus_unregister(idxd_bus_types[i]);