1 // SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0
2 /* Copyright (c) 2020-2021 Marvell International Ltd. All rights reserved */
4 #include <linux/rhashtable.h>
6 #include "prestera_acl.h"
7 #include "prestera_flow.h"
8 #include "prestera_hw.h"
11 #define ACL_KEYMASK_SIZE \
12 (sizeof(__be32) * __PRESTERA_ACL_RULE_MATCH_TYPE_MAX)
15 struct prestera_switch *sw;
16 struct list_head vtcam_list;
17 struct list_head rules;
18 struct rhashtable ruleset_ht;
19 struct rhashtable acl_rule_entry_ht;
23 struct prestera_acl_ruleset_ht_key {
24 struct prestera_flow_block *block;
28 struct prestera_acl_rule_entry {
29 struct rhash_head ht_node;
30 struct prestera_acl_rule_entry_key key;
39 struct prestera_acl_action_police i;
42 struct prestera_acl_action_jump i;
47 struct prestera_counter_block *block;
52 struct prestera_acl_ruleset {
53 struct rhash_head ht_node; /* Member of acl HT */
54 struct prestera_acl_ruleset_ht_key ht_key;
55 struct rhashtable rule_ht;
56 struct prestera_acl *acl;
57 unsigned long rule_count;
66 struct prestera_acl_vtcam {
67 struct list_head list;
68 __be32 keymask[__PRESTERA_ACL_RULE_MATCH_TYPE_MAX];
75 static const struct rhashtable_params prestera_acl_ruleset_ht_params = {
76 .key_len = sizeof(struct prestera_acl_ruleset_ht_key),
77 .key_offset = offsetof(struct prestera_acl_ruleset, ht_key),
78 .head_offset = offsetof(struct prestera_acl_ruleset, ht_node),
79 .automatic_shrinking = true,
82 static const struct rhashtable_params prestera_acl_rule_ht_params = {
83 .key_len = sizeof(unsigned long),
84 .key_offset = offsetof(struct prestera_acl_rule, cookie),
85 .head_offset = offsetof(struct prestera_acl_rule, ht_node),
86 .automatic_shrinking = true,
89 static const struct rhashtable_params __prestera_acl_rule_entry_ht_params = {
90 .key_offset = offsetof(struct prestera_acl_rule_entry, key),
91 .head_offset = offsetof(struct prestera_acl_rule_entry, ht_node),
92 .key_len = sizeof(struct prestera_acl_rule_entry_key),
93 .automatic_shrinking = true,
96 int prestera_acl_chain_to_client(u32 chain_index, u32 *client)
98 static const u32 client_map[] = {
99 PRESTERA_HW_COUNTER_CLIENT_LOOKUP_0,
100 PRESTERA_HW_COUNTER_CLIENT_LOOKUP_1,
101 PRESTERA_HW_COUNTER_CLIENT_LOOKUP_2
104 if (chain_index >= ARRAY_SIZE(client_map))
107 *client = client_map[chain_index];
111 static bool prestera_acl_chain_is_supported(u32 chain_index)
113 return (chain_index & ~PRESTERA_ACL_CHAIN_MASK) == 0;
116 static struct prestera_acl_ruleset *
117 prestera_acl_ruleset_create(struct prestera_acl *acl,
118 struct prestera_flow_block *block,
121 struct prestera_acl_ruleset *ruleset;
125 if (!prestera_acl_chain_is_supported(chain_index))
126 return ERR_PTR(-EINVAL);
128 ruleset = kzalloc(sizeof(*ruleset), GFP_KERNEL);
130 return ERR_PTR(-ENOMEM);
133 ruleset->ht_key.block = block;
134 ruleset->ht_key.chain_index = chain_index;
135 refcount_set(&ruleset->refcount, 1);
137 err = rhashtable_init(&ruleset->rule_ht, &prestera_acl_rule_ht_params);
139 goto err_rhashtable_init;
141 err = idr_alloc_u32(&acl->uid, NULL, &uid, U8_MAX, GFP_KERNEL);
143 goto err_ruleset_create;
145 /* make pcl-id based on uid */
146 ruleset->pcl_id = PRESTERA_ACL_PCL_ID_MAKE((u8)uid, chain_index);
147 ruleset->index = uid;
149 err = rhashtable_insert_fast(&acl->ruleset_ht, &ruleset->ht_node,
150 prestera_acl_ruleset_ht_params);
152 goto err_ruleset_ht_insert;
156 err_ruleset_ht_insert:
157 idr_remove(&acl->uid, uid);
159 rhashtable_destroy(&ruleset->rule_ht);
165 void prestera_acl_ruleset_keymask_set(struct prestera_acl_ruleset *ruleset,
168 ruleset->keymask = kmemdup(keymask, ACL_KEYMASK_SIZE, GFP_KERNEL);
171 int prestera_acl_ruleset_offload(struct prestera_acl_ruleset *ruleset)
173 struct prestera_acl_iface iface;
177 if (ruleset->offload)
180 err = prestera_acl_vtcam_id_get(ruleset->acl,
181 ruleset->ht_key.chain_index,
182 ruleset->keymask, &vtcam_id);
184 goto err_vtcam_create;
186 if (ruleset->ht_key.chain_index) {
187 /* for chain > 0, bind iface index to pcl-id to be able
188 * to jump from any other ruleset to this one using the index.
190 iface.index = ruleset->index;
191 iface.type = PRESTERA_ACL_IFACE_TYPE_INDEX;
192 err = prestera_hw_vtcam_iface_bind(ruleset->acl->sw, &iface,
193 vtcam_id, ruleset->pcl_id);
195 goto err_ruleset_bind;
198 ruleset->vtcam_id = vtcam_id;
199 ruleset->offload = true;
203 prestera_acl_vtcam_id_put(ruleset->acl, ruleset->vtcam_id);
208 static void prestera_acl_ruleset_destroy(struct prestera_acl_ruleset *ruleset)
210 struct prestera_acl *acl = ruleset->acl;
211 u8 uid = ruleset->pcl_id & PRESTERA_ACL_KEYMASK_PCL_ID_USER;
214 rhashtable_remove_fast(&acl->ruleset_ht, &ruleset->ht_node,
215 prestera_acl_ruleset_ht_params);
217 if (ruleset->offload) {
218 if (ruleset->ht_key.chain_index) {
219 struct prestera_acl_iface iface = {
220 .type = PRESTERA_ACL_IFACE_TYPE_INDEX,
221 .index = ruleset->index
223 err = prestera_hw_vtcam_iface_unbind(acl->sw, &iface,
227 WARN_ON(prestera_acl_vtcam_id_put(acl, ruleset->vtcam_id));
230 idr_remove(&acl->uid, uid);
231 rhashtable_destroy(&ruleset->rule_ht);
232 kfree(ruleset->keymask);
236 static struct prestera_acl_ruleset *
237 __prestera_acl_ruleset_lookup(struct prestera_acl *acl,
238 struct prestera_flow_block *block,
241 struct prestera_acl_ruleset_ht_key ht_key;
243 memset(&ht_key, 0, sizeof(ht_key));
244 ht_key.block = block;
245 ht_key.chain_index = chain_index;
246 return rhashtable_lookup_fast(&acl->ruleset_ht, &ht_key,
247 prestera_acl_ruleset_ht_params);
250 struct prestera_acl_ruleset *
251 prestera_acl_ruleset_lookup(struct prestera_acl *acl,
252 struct prestera_flow_block *block,
255 struct prestera_acl_ruleset *ruleset;
257 ruleset = __prestera_acl_ruleset_lookup(acl, block, chain_index);
259 return ERR_PTR(-ENOENT);
261 refcount_inc(&ruleset->refcount);
265 struct prestera_acl_ruleset *
266 prestera_acl_ruleset_get(struct prestera_acl *acl,
267 struct prestera_flow_block *block,
270 struct prestera_acl_ruleset *ruleset;
272 ruleset = __prestera_acl_ruleset_lookup(acl, block, chain_index);
274 refcount_inc(&ruleset->refcount);
278 return prestera_acl_ruleset_create(acl, block, chain_index);
281 void prestera_acl_ruleset_put(struct prestera_acl_ruleset *ruleset)
283 if (!refcount_dec_and_test(&ruleset->refcount))
286 prestera_acl_ruleset_destroy(ruleset);
289 int prestera_acl_ruleset_bind(struct prestera_acl_ruleset *ruleset,
290 struct prestera_port *port)
292 struct prestera_acl_iface iface = {
293 .type = PRESTERA_ACL_IFACE_TYPE_PORT,
297 return prestera_hw_vtcam_iface_bind(port->sw, &iface, ruleset->vtcam_id,
301 int prestera_acl_ruleset_unbind(struct prestera_acl_ruleset *ruleset,
302 struct prestera_port *port)
304 struct prestera_acl_iface iface = {
305 .type = PRESTERA_ACL_IFACE_TYPE_PORT,
309 return prestera_hw_vtcam_iface_unbind(port->sw, &iface,
313 static int prestera_acl_ruleset_block_bind(struct prestera_acl_ruleset *ruleset,
314 struct prestera_flow_block *block)
316 struct prestera_flow_block_binding *binding;
319 block->ruleset_zero = ruleset;
320 list_for_each_entry(binding, &block->binding_list, list) {
321 err = prestera_acl_ruleset_bind(ruleset, binding->port);
328 list_for_each_entry_continue_reverse(binding, &block->binding_list,
330 err = prestera_acl_ruleset_unbind(ruleset, binding->port);
331 block->ruleset_zero = NULL;
337 prestera_acl_ruleset_block_unbind(struct prestera_acl_ruleset *ruleset,
338 struct prestera_flow_block *block)
340 struct prestera_flow_block_binding *binding;
342 list_for_each_entry(binding, &block->binding_list, list)
343 prestera_acl_ruleset_unbind(ruleset, binding->port);
344 block->ruleset_zero = NULL;
348 prestera_acl_rule_keymask_pcl_id_set(struct prestera_acl_rule *rule, u16 pcl_id)
350 struct prestera_acl_match *r_match = &rule->re_key.match;
351 __be16 pcl_id_mask = htons(PRESTERA_ACL_KEYMASK_PCL_ID);
352 __be16 pcl_id_key = htons(pcl_id);
354 rule_match_set(r_match->key, PCL_ID, pcl_id_key);
355 rule_match_set(r_match->mask, PCL_ID, pcl_id_mask);
358 struct prestera_acl_rule *
359 prestera_acl_rule_lookup(struct prestera_acl_ruleset *ruleset,
360 unsigned long cookie)
362 return rhashtable_lookup_fast(&ruleset->rule_ht, &cookie,
363 prestera_acl_rule_ht_params);
366 u32 prestera_acl_ruleset_index_get(const struct prestera_acl_ruleset *ruleset)
368 return ruleset->index;
371 bool prestera_acl_ruleset_is_offload(struct prestera_acl_ruleset *ruleset)
373 return ruleset->offload;
376 struct prestera_acl_rule *
377 prestera_acl_rule_create(struct prestera_acl_ruleset *ruleset,
378 unsigned long cookie, u32 chain_index)
380 struct prestera_acl_rule *rule;
382 rule = kzalloc(sizeof(*rule), GFP_KERNEL);
384 return ERR_PTR(-ENOMEM);
386 rule->ruleset = ruleset;
387 rule->cookie = cookie;
388 rule->chain_index = chain_index;
390 refcount_inc(&ruleset->refcount);
395 void prestera_acl_rule_priority_set(struct prestera_acl_rule *rule,
398 rule->priority = priority;
401 void prestera_acl_rule_destroy(struct prestera_acl_rule *rule)
403 if (rule->jump_ruleset)
404 /* release ruleset kept by jump action */
405 prestera_acl_ruleset_put(rule->jump_ruleset);
407 prestera_acl_ruleset_put(rule->ruleset);
411 int prestera_acl_rule_add(struct prestera_switch *sw,
412 struct prestera_acl_rule *rule)
415 struct prestera_acl_ruleset *ruleset = rule->ruleset;
416 struct prestera_flow_block *block = ruleset->ht_key.block;
418 /* try to add rule to hash table first */
419 err = rhashtable_insert_fast(&ruleset->rule_ht, &rule->ht_node,
420 prestera_acl_rule_ht_params);
424 prestera_acl_rule_keymask_pcl_id_set(rule, ruleset->pcl_id);
425 rule->re_arg.vtcam_id = ruleset->vtcam_id;
426 rule->re_key.prio = rule->priority;
428 rule->re = prestera_acl_rule_entry_find(sw->acl, &rule->re_key);
429 err = WARN_ON(rule->re) ? -EEXIST : 0;
433 rule->re = prestera_acl_rule_entry_create(sw->acl, &rule->re_key,
435 err = !rule->re ? -EINVAL : 0;
439 /* bind the block (all ports) to chain index 0, rest of
440 * the chains are bound to goto action
442 if (!ruleset->ht_key.chain_index && !ruleset->rule_count) {
443 err = prestera_acl_ruleset_block_bind(ruleset, block);
445 goto err_acl_block_bind;
448 list_add_tail(&rule->list, &sw->acl->rules);
449 ruleset->rule_count++;
453 prestera_acl_rule_entry_destroy(sw->acl, rule->re);
456 rhashtable_remove_fast(&ruleset->rule_ht, &rule->ht_node,
457 prestera_acl_rule_ht_params);
462 void prestera_acl_rule_del(struct prestera_switch *sw,
463 struct prestera_acl_rule *rule)
465 struct prestera_acl_ruleset *ruleset = rule->ruleset;
466 struct prestera_flow_block *block = ruleset->ht_key.block;
468 rhashtable_remove_fast(&ruleset->rule_ht, &rule->ht_node,
469 prestera_acl_rule_ht_params);
470 ruleset->rule_count--;
471 list_del(&rule->list);
473 prestera_acl_rule_entry_destroy(sw->acl, rule->re);
475 /* unbind block (all ports) */
476 if (!ruleset->ht_key.chain_index && !ruleset->rule_count)
477 prestera_acl_ruleset_block_unbind(ruleset, block);
480 int prestera_acl_rule_get_stats(struct prestera_acl *acl,
481 struct prestera_acl_rule *rule,
482 u64 *packets, u64 *bytes, u64 *last_use)
488 err = prestera_counter_stats_get(acl->sw->counter,
489 rule->re->counter.block,
490 rule->re->counter.id,
491 ¤t_packets, ¤t_bytes);
495 *packets = current_packets;
496 *bytes = current_bytes;
502 struct prestera_acl_rule_entry *
503 prestera_acl_rule_entry_find(struct prestera_acl *acl,
504 struct prestera_acl_rule_entry_key *key)
506 return rhashtable_lookup_fast(&acl->acl_rule_entry_ht, key,
507 __prestera_acl_rule_entry_ht_params);
510 static int __prestera_acl_rule_entry2hw_del(struct prestera_switch *sw,
511 struct prestera_acl_rule_entry *e)
513 return prestera_hw_vtcam_rule_del(sw, e->vtcam_id, e->hw_id);
516 static int __prestera_acl_rule_entry2hw_add(struct prestera_switch *sw,
517 struct prestera_acl_rule_entry *e)
519 struct prestera_acl_hw_action_info act_hw[PRESTERA_ACL_RULE_ACTION_MAX];
522 memset(&act_hw, 0, sizeof(act_hw));
526 if (e->accept.valid) {
527 act_hw[act_num].id = PRESTERA_ACL_RULE_ACTION_ACCEPT;
532 act_hw[act_num].id = PRESTERA_ACL_RULE_ACTION_DROP;
537 act_hw[act_num].id = PRESTERA_ACL_RULE_ACTION_TRAP;
541 if (e->police.valid) {
542 act_hw[act_num].id = PRESTERA_ACL_RULE_ACTION_POLICE;
543 act_hw[act_num].police = e->police.i;
548 act_hw[act_num].id = PRESTERA_ACL_RULE_ACTION_JUMP;
549 act_hw[act_num].jump = e->jump.i;
553 if (e->counter.block) {
554 act_hw[act_num].id = PRESTERA_ACL_RULE_ACTION_COUNT;
555 act_hw[act_num].count.id = e->counter.id;
559 return prestera_hw_vtcam_rule_add(sw, e->vtcam_id, e->key.prio,
560 e->key.match.key, e->key.match.mask,
561 act_hw, act_num, &e->hw_id);
565 __prestera_acl_rule_entry_act_destruct(struct prestera_switch *sw,
566 struct prestera_acl_rule_entry *e)
569 prestera_counter_put(sw->counter, e->counter.block, e->counter.id);
572 prestera_hw_policer_release(sw, e->police.i.id);
575 void prestera_acl_rule_entry_destroy(struct prestera_acl *acl,
576 struct prestera_acl_rule_entry *e)
580 rhashtable_remove_fast(&acl->acl_rule_entry_ht, &e->ht_node,
581 __prestera_acl_rule_entry_ht_params);
583 ret = __prestera_acl_rule_entry2hw_del(acl->sw, e);
584 WARN_ON(ret && ret != -ENODEV);
586 __prestera_acl_rule_entry_act_destruct(acl->sw, e);
591 __prestera_acl_rule_entry_act_construct(struct prestera_switch *sw,
592 struct prestera_acl_rule_entry *e,
593 struct prestera_acl_rule_entry_arg *arg)
598 e->accept.valid = arg->accept.valid;
600 e->drop.valid = arg->drop.valid;
602 e->trap.valid = arg->trap.valid;
604 e->jump.valid = arg->jump.valid;
605 e->jump.i = arg->jump.i;
607 if (arg->police.valid) {
608 u8 type = arg->police.ingress ? PRESTERA_POLICER_TYPE_INGRESS :
609 PRESTERA_POLICER_TYPE_EGRESS;
611 err = prestera_hw_policer_create(sw, type, &e->police.i.id);
615 err = prestera_hw_policer_sr_tcm_set(sw, e->police.i.id,
619 prestera_hw_policer_release(sw, e->police.i.id);
622 e->police.valid = arg->police.valid;
625 if (arg->count.valid) {
626 err = prestera_counter_get(sw->counter, arg->count.client,
636 __prestera_acl_rule_entry_act_destruct(sw, e);
640 struct prestera_acl_rule_entry *
641 prestera_acl_rule_entry_create(struct prestera_acl *acl,
642 struct prestera_acl_rule_entry_key *key,
643 struct prestera_acl_rule_entry_arg *arg)
645 struct prestera_acl_rule_entry *e;
648 e = kzalloc(sizeof(*e), GFP_KERNEL);
652 memcpy(&e->key, key, sizeof(*key));
653 e->vtcam_id = arg->vtcam_id;
654 err = __prestera_acl_rule_entry_act_construct(acl->sw, e, arg);
656 goto err_act_construct;
658 err = __prestera_acl_rule_entry2hw_add(acl->sw, e);
662 err = rhashtable_insert_fast(&acl->acl_rule_entry_ht, &e->ht_node,
663 __prestera_acl_rule_entry_ht_params);
670 WARN_ON(__prestera_acl_rule_entry2hw_del(acl->sw, e));
672 __prestera_acl_rule_entry_act_destruct(acl->sw, e);
679 static int __prestera_acl_vtcam_id_try_fit(struct prestera_acl *acl, u8 lookup,
680 void *keymask, u32 *vtcam_id)
682 struct prestera_acl_vtcam *vtcam;
685 list_for_each_entry(vtcam, &acl->vtcam_list, list) {
686 if (lookup != vtcam->lookup)
689 if (!keymask && !vtcam->is_keymask_set)
692 if (!(keymask && vtcam->is_keymask_set))
695 /* try to fit with vtcam keymask */
696 for (i = 0; i < __PRESTERA_ACL_RULE_MATCH_TYPE_MAX; i++) {
697 __be32 __keymask = ((__be32 *)keymask)[i];
700 /* vtcam keymask in not interested */
703 if (__keymask & ~vtcam->keymask[i])
704 /* keymask does not fit the vtcam keymask */
708 if (i == __PRESTERA_ACL_RULE_MATCH_TYPE_MAX)
709 /* keymask fits vtcam keymask, return it */
713 /* nothing is found */
717 refcount_inc(&vtcam->refcount);
718 *vtcam_id = vtcam->id;
722 int prestera_acl_vtcam_id_get(struct prestera_acl *acl, u8 lookup,
723 void *keymask, u32 *vtcam_id)
725 struct prestera_acl_vtcam *vtcam;
729 /* find the vtcam that suits keymask. We do not expect to have
730 * a big number of vtcams, so, the list type for vtcam list is
733 list_for_each_entry(vtcam, &acl->vtcam_list, list) {
734 if (lookup != vtcam->lookup)
737 if (!keymask && !vtcam->is_keymask_set) {
738 refcount_inc(&vtcam->refcount);
742 if (keymask && vtcam->is_keymask_set &&
743 !memcmp(keymask, vtcam->keymask, sizeof(vtcam->keymask))) {
744 refcount_inc(&vtcam->refcount);
749 /* vtcam not found, try to create new one */
750 vtcam = kzalloc(sizeof(*vtcam), GFP_KERNEL);
754 err = prestera_hw_vtcam_create(acl->sw, lookup, keymask, &new_vtcam_id,
755 PRESTERA_HW_VTCAM_DIR_INGRESS);
759 /* cannot create new, try to fit into existing vtcam */
760 if (__prestera_acl_vtcam_id_try_fit(acl, lookup,
761 keymask, &new_vtcam_id))
764 *vtcam_id = new_vtcam_id;
768 vtcam->id = new_vtcam_id;
769 vtcam->lookup = lookup;
771 memcpy(vtcam->keymask, keymask, sizeof(vtcam->keymask));
772 vtcam->is_keymask_set = true;
774 refcount_set(&vtcam->refcount, 1);
775 list_add_rcu(&vtcam->list, &acl->vtcam_list);
778 *vtcam_id = vtcam->id;
782 int prestera_acl_vtcam_id_put(struct prestera_acl *acl, u32 vtcam_id)
784 struct prestera_acl_vtcam *vtcam;
787 list_for_each_entry(vtcam, &acl->vtcam_list, list) {
788 if (vtcam_id != vtcam->id)
791 if (!refcount_dec_and_test(&vtcam->refcount))
794 err = prestera_hw_vtcam_destroy(acl->sw, vtcam->id);
795 if (err && err != -ENODEV) {
796 refcount_set(&vtcam->refcount, 1);
800 list_del(&vtcam->list);
808 int prestera_acl_init(struct prestera_switch *sw)
810 struct prestera_acl *acl;
813 acl = kzalloc(sizeof(*acl), GFP_KERNEL);
818 INIT_LIST_HEAD(&acl->rules);
819 INIT_LIST_HEAD(&acl->vtcam_list);
822 err = rhashtable_init(&acl->acl_rule_entry_ht,
823 &__prestera_acl_rule_entry_ht_params);
825 goto err_acl_rule_entry_ht_init;
827 err = rhashtable_init(&acl->ruleset_ht,
828 &prestera_acl_ruleset_ht_params);
830 goto err_ruleset_ht_init;
837 rhashtable_destroy(&acl->acl_rule_entry_ht);
838 err_acl_rule_entry_ht_init:
843 void prestera_acl_fini(struct prestera_switch *sw)
845 struct prestera_acl *acl = sw->acl;
847 WARN_ON(!idr_is_empty(&acl->uid));
848 idr_destroy(&acl->uid);
850 WARN_ON(!list_empty(&acl->vtcam_list));
851 WARN_ON(!list_empty(&acl->rules));
853 rhashtable_destroy(&acl->ruleset_ht);
854 rhashtable_destroy(&acl->acl_rule_entry_ht);