GNU Linux-libre 5.19-rc6-gnu
[releases.git] / lib / dim / net_dim.c
1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2 /*
3  * Copyright (c) 2018, Mellanox Technologies inc.  All rights reserved.
4  */
5
6 #include <linux/dim.h>
7
8 /*
9  * Net DIM profiles:
10  *        There are different set of profiles for each CQ period mode.
11  *        There are different set of profiles for RX/TX CQs.
12  *        Each profile size must be of NET_DIM_PARAMS_NUM_PROFILES
13  */
14 #define NET_DIM_PARAMS_NUM_PROFILES 5
15 #define NET_DIM_DEFAULT_RX_CQ_PKTS_FROM_EQE 256
16 #define NET_DIM_DEFAULT_TX_CQ_PKTS_FROM_EQE 128
17 #define NET_DIM_DEF_PROFILE_CQE 1
18 #define NET_DIM_DEF_PROFILE_EQE 1
19
20 #define NET_DIM_RX_EQE_PROFILES { \
21         {.usec = 1,   .pkts = NET_DIM_DEFAULT_RX_CQ_PKTS_FROM_EQE,}, \
22         {.usec = 8,   .pkts = NET_DIM_DEFAULT_RX_CQ_PKTS_FROM_EQE,}, \
23         {.usec = 64,  .pkts = NET_DIM_DEFAULT_RX_CQ_PKTS_FROM_EQE,}, \
24         {.usec = 128, .pkts = NET_DIM_DEFAULT_RX_CQ_PKTS_FROM_EQE,}, \
25         {.usec = 256, .pkts = NET_DIM_DEFAULT_RX_CQ_PKTS_FROM_EQE,}  \
26 }
27
28 #define NET_DIM_RX_CQE_PROFILES { \
29         {.usec = 2,  .pkts = 256,},             \
30         {.usec = 8,  .pkts = 128,},             \
31         {.usec = 16, .pkts = 64,},              \
32         {.usec = 32, .pkts = 64,},              \
33         {.usec = 64, .pkts = 64,}               \
34 }
35
36 #define NET_DIM_TX_EQE_PROFILES { \
37         {.usec = 1,   .pkts = NET_DIM_DEFAULT_TX_CQ_PKTS_FROM_EQE,},  \
38         {.usec = 8,   .pkts = NET_DIM_DEFAULT_TX_CQ_PKTS_FROM_EQE,},  \
39         {.usec = 32,  .pkts = NET_DIM_DEFAULT_TX_CQ_PKTS_FROM_EQE,},  \
40         {.usec = 64,  .pkts = NET_DIM_DEFAULT_TX_CQ_PKTS_FROM_EQE,},  \
41         {.usec = 128, .pkts = NET_DIM_DEFAULT_TX_CQ_PKTS_FROM_EQE,}   \
42 }
43
44 #define NET_DIM_TX_CQE_PROFILES { \
45         {.usec = 5,  .pkts = 128,},  \
46         {.usec = 8,  .pkts = 64,},  \
47         {.usec = 16, .pkts = 32,},  \
48         {.usec = 32, .pkts = 32,},  \
49         {.usec = 64, .pkts = 32,}   \
50 }
51
52 static const struct dim_cq_moder
53 rx_profile[DIM_CQ_PERIOD_NUM_MODES][NET_DIM_PARAMS_NUM_PROFILES] = {
54         NET_DIM_RX_EQE_PROFILES,
55         NET_DIM_RX_CQE_PROFILES,
56 };
57
58 static const struct dim_cq_moder
59 tx_profile[DIM_CQ_PERIOD_NUM_MODES][NET_DIM_PARAMS_NUM_PROFILES] = {
60         NET_DIM_TX_EQE_PROFILES,
61         NET_DIM_TX_CQE_PROFILES,
62 };
63
64 struct dim_cq_moder
65 net_dim_get_rx_moderation(u8 cq_period_mode, int ix)
66 {
67         struct dim_cq_moder cq_moder = rx_profile[cq_period_mode][ix];
68
69         cq_moder.cq_period_mode = cq_period_mode;
70         return cq_moder;
71 }
72 EXPORT_SYMBOL(net_dim_get_rx_moderation);
73
74 struct dim_cq_moder
75 net_dim_get_def_rx_moderation(u8 cq_period_mode)
76 {
77         u8 profile_ix = cq_period_mode == DIM_CQ_PERIOD_MODE_START_FROM_CQE ?
78                         NET_DIM_DEF_PROFILE_CQE : NET_DIM_DEF_PROFILE_EQE;
79
80         return net_dim_get_rx_moderation(cq_period_mode, profile_ix);
81 }
82 EXPORT_SYMBOL(net_dim_get_def_rx_moderation);
83
84 struct dim_cq_moder
85 net_dim_get_tx_moderation(u8 cq_period_mode, int ix)
86 {
87         struct dim_cq_moder cq_moder = tx_profile[cq_period_mode][ix];
88
89         cq_moder.cq_period_mode = cq_period_mode;
90         return cq_moder;
91 }
92 EXPORT_SYMBOL(net_dim_get_tx_moderation);
93
94 struct dim_cq_moder
95 net_dim_get_def_tx_moderation(u8 cq_period_mode)
96 {
97         u8 profile_ix = cq_period_mode == DIM_CQ_PERIOD_MODE_START_FROM_CQE ?
98                         NET_DIM_DEF_PROFILE_CQE : NET_DIM_DEF_PROFILE_EQE;
99
100         return net_dim_get_tx_moderation(cq_period_mode, profile_ix);
101 }
102 EXPORT_SYMBOL(net_dim_get_def_tx_moderation);
103
104 static int net_dim_step(struct dim *dim)
105 {
106         if (dim->tired == (NET_DIM_PARAMS_NUM_PROFILES * 2))
107                 return DIM_TOO_TIRED;
108
109         switch (dim->tune_state) {
110         case DIM_PARKING_ON_TOP:
111         case DIM_PARKING_TIRED:
112                 break;
113         case DIM_GOING_RIGHT:
114                 if (dim->profile_ix == (NET_DIM_PARAMS_NUM_PROFILES - 1))
115                         return DIM_ON_EDGE;
116                 dim->profile_ix++;
117                 dim->steps_right++;
118                 break;
119         case DIM_GOING_LEFT:
120                 if (dim->profile_ix == 0)
121                         return DIM_ON_EDGE;
122                 dim->profile_ix--;
123                 dim->steps_left++;
124                 break;
125         }
126
127         dim->tired++;
128         return DIM_STEPPED;
129 }
130
131 static void net_dim_exit_parking(struct dim *dim)
132 {
133         dim->tune_state = dim->profile_ix ? DIM_GOING_LEFT : DIM_GOING_RIGHT;
134         net_dim_step(dim);
135 }
136
137 static int net_dim_stats_compare(struct dim_stats *curr,
138                                  struct dim_stats *prev)
139 {
140         if (!prev->bpms)
141                 return curr->bpms ? DIM_STATS_BETTER : DIM_STATS_SAME;
142
143         if (IS_SIGNIFICANT_DIFF(curr->bpms, prev->bpms))
144                 return (curr->bpms > prev->bpms) ? DIM_STATS_BETTER :
145                                                    DIM_STATS_WORSE;
146
147         if (!prev->ppms)
148                 return curr->ppms ? DIM_STATS_BETTER :
149                                     DIM_STATS_SAME;
150
151         if (IS_SIGNIFICANT_DIFF(curr->ppms, prev->ppms))
152                 return (curr->ppms > prev->ppms) ? DIM_STATS_BETTER :
153                                                    DIM_STATS_WORSE;
154
155         if (!prev->epms)
156                 return DIM_STATS_SAME;
157
158         if (IS_SIGNIFICANT_DIFF(curr->epms, prev->epms))
159                 return (curr->epms < prev->epms) ? DIM_STATS_BETTER :
160                                                    DIM_STATS_WORSE;
161
162         return DIM_STATS_SAME;
163 }
164
165 static bool net_dim_decision(struct dim_stats *curr_stats, struct dim *dim)
166 {
167         int prev_state = dim->tune_state;
168         int prev_ix = dim->profile_ix;
169         int stats_res;
170         int step_res;
171
172         switch (dim->tune_state) {
173         case DIM_PARKING_ON_TOP:
174                 stats_res = net_dim_stats_compare(curr_stats,
175                                                   &dim->prev_stats);
176                 if (stats_res != DIM_STATS_SAME)
177                         net_dim_exit_parking(dim);
178                 break;
179
180         case DIM_PARKING_TIRED:
181                 dim->tired--;
182                 if (!dim->tired)
183                         net_dim_exit_parking(dim);
184                 break;
185
186         case DIM_GOING_RIGHT:
187         case DIM_GOING_LEFT:
188                 stats_res = net_dim_stats_compare(curr_stats,
189                                                   &dim->prev_stats);
190                 if (stats_res != DIM_STATS_BETTER)
191                         dim_turn(dim);
192
193                 if (dim_on_top(dim)) {
194                         dim_park_on_top(dim);
195                         break;
196                 }
197
198                 step_res = net_dim_step(dim);
199                 switch (step_res) {
200                 case DIM_ON_EDGE:
201                         dim_park_on_top(dim);
202                         break;
203                 case DIM_TOO_TIRED:
204                         dim_park_tired(dim);
205                         break;
206                 }
207
208                 break;
209         }
210
211         if (prev_state != DIM_PARKING_ON_TOP ||
212             dim->tune_state != DIM_PARKING_ON_TOP)
213                 dim->prev_stats = *curr_stats;
214
215         return dim->profile_ix != prev_ix;
216 }
217
218 void net_dim(struct dim *dim, struct dim_sample end_sample)
219 {
220         struct dim_stats curr_stats;
221         u16 nevents;
222
223         switch (dim->state) {
224         case DIM_MEASURE_IN_PROGRESS:
225                 nevents = BIT_GAP(BITS_PER_TYPE(u16),
226                                   end_sample.event_ctr,
227                                   dim->start_sample.event_ctr);
228                 if (nevents < DIM_NEVENTS)
229                         break;
230                 dim_calc_stats(&dim->start_sample, &end_sample, &curr_stats);
231                 if (net_dim_decision(&curr_stats, dim)) {
232                         dim->state = DIM_APPLY_NEW_PROFILE;
233                         schedule_work(&dim->work);
234                         break;
235                 }
236                 fallthrough;
237         case DIM_START_MEASURE:
238                 dim_update_sample(end_sample.event_ctr, end_sample.pkt_ctr,
239                                   end_sample.byte_ctr, &dim->start_sample);
240                 dim->state = DIM_MEASURE_IN_PROGRESS;
241                 break;
242         case DIM_APPLY_NEW_PROFILE:
243                 break;
244         }
245 }
246 EXPORT_SYMBOL(net_dim);