Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
tjh.dev
kernel
os
linux
1/*
2 * Copyright (c) 2016, Mellanox Technologies. All rights reserved.
3 * Copyright (c) 2017-2018, Broadcom Limited. All rights reserved.
4 *
5 * This software is available to you under a choice of one of two
6 * licenses. You may choose to be licensed under the terms of the GNU
7 * General Public License (GPL) Version 2, available from the file
8 * COPYING in the main directory of this source tree, or the
9 * OpenIB.org BSD license below:
10 *
11 * Redistribution and use in source and binary forms, with or
12 * without modification, are permitted provided that the following
13 * conditions are met:
14 *
15 * - Redistributions of source code must retain the above
16 * copyright notice, this list of conditions and the following
17 * disclaimer.
18 *
19 * - Redistributions in binary form must reproduce the above
20 * copyright notice, this list of conditions and the following
21 * disclaimer in the documentation and/or other materials
22 * provided with the distribution.
23 *
24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
31 * SOFTWARE.
32 */
33
34#ifndef NET_DIM_H
35#define NET_DIM_H
36
37#include <linux/module.h>
38
39struct net_dim_cq_moder {
40 u16 usec;
41 u16 pkts;
42 u8 cq_period_mode;
43};
44
45struct net_dim_sample {
46 ktime_t time;
47 u32 pkt_ctr;
48 u32 byte_ctr;
49 u16 event_ctr;
50};
51
52struct net_dim_stats {
53 int ppms; /* packets per msec */
54 int bpms; /* bytes per msec */
55 int epms; /* events per msec */
56};
57
58struct net_dim { /* Adaptive Moderation */
59 u8 state;
60 struct net_dim_stats prev_stats;
61 struct net_dim_sample start_sample;
62 struct work_struct work;
63 u8 profile_ix;
64 u8 mode;
65 u8 tune_state;
66 u8 steps_right;
67 u8 steps_left;
68 u8 tired;
69};
70
71enum {
72 NET_DIM_CQ_PERIOD_MODE_START_FROM_EQE = 0x0,
73 NET_DIM_CQ_PERIOD_MODE_START_FROM_CQE = 0x1,
74 NET_DIM_CQ_PERIOD_NUM_MODES
75};
76
77/* Adaptive moderation logic */
78enum {
79 NET_DIM_START_MEASURE,
80 NET_DIM_MEASURE_IN_PROGRESS,
81 NET_DIM_APPLY_NEW_PROFILE,
82};
83
84enum {
85 NET_DIM_PARKING_ON_TOP,
86 NET_DIM_PARKING_TIRED,
87 NET_DIM_GOING_RIGHT,
88 NET_DIM_GOING_LEFT,
89};
90
91enum {
92 NET_DIM_STATS_WORSE,
93 NET_DIM_STATS_SAME,
94 NET_DIM_STATS_BETTER,
95};
96
97enum {
98 NET_DIM_STEPPED,
99 NET_DIM_TOO_TIRED,
100 NET_DIM_ON_EDGE,
101};
102
103#define NET_DIM_PARAMS_NUM_PROFILES 5
104/* Adaptive moderation profiles */
105#define NET_DIM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE 256
106#define NET_DIM_DEFAULT_TX_CQ_MODERATION_PKTS_FROM_EQE 128
107#define NET_DIM_DEF_PROFILE_CQE 1
108#define NET_DIM_DEF_PROFILE_EQE 1
109
110/* All profiles sizes must be NET_PARAMS_DIM_NUM_PROFILES */
111#define NET_DIM_RX_EQE_PROFILES { \
112 {1, NET_DIM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \
113 {8, NET_DIM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \
114 {64, NET_DIM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \
115 {128, NET_DIM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \
116 {256, NET_DIM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \
117}
118
119#define NET_DIM_RX_CQE_PROFILES { \
120 {2, 256}, \
121 {8, 128}, \
122 {16, 64}, \
123 {32, 64}, \
124 {64, 64} \
125}
126
127#define NET_DIM_TX_EQE_PROFILES { \
128 {1, NET_DIM_DEFAULT_TX_CQ_MODERATION_PKTS_FROM_EQE}, \
129 {8, NET_DIM_DEFAULT_TX_CQ_MODERATION_PKTS_FROM_EQE}, \
130 {32, NET_DIM_DEFAULT_TX_CQ_MODERATION_PKTS_FROM_EQE}, \
131 {64, NET_DIM_DEFAULT_TX_CQ_MODERATION_PKTS_FROM_EQE}, \
132 {128, NET_DIM_DEFAULT_TX_CQ_MODERATION_PKTS_FROM_EQE} \
133}
134
135#define NET_DIM_TX_CQE_PROFILES { \
136 {5, 128}, \
137 {8, 64}, \
138 {16, 32}, \
139 {32, 32}, \
140 {64, 32} \
141}
142
143static const struct net_dim_cq_moder
144rx_profile[NET_DIM_CQ_PERIOD_NUM_MODES][NET_DIM_PARAMS_NUM_PROFILES] = {
145 NET_DIM_RX_EQE_PROFILES,
146 NET_DIM_RX_CQE_PROFILES,
147};
148
149static const struct net_dim_cq_moder
150tx_profile[NET_DIM_CQ_PERIOD_NUM_MODES][NET_DIM_PARAMS_NUM_PROFILES] = {
151 NET_DIM_TX_EQE_PROFILES,
152 NET_DIM_TX_CQE_PROFILES,
153};
154
155static inline struct net_dim_cq_moder
156net_dim_get_rx_moderation(u8 cq_period_mode, int ix)
157{
158 struct net_dim_cq_moder cq_moder = rx_profile[cq_period_mode][ix];
159
160 cq_moder.cq_period_mode = cq_period_mode;
161 return cq_moder;
162}
163
164static inline struct net_dim_cq_moder
165net_dim_get_def_rx_moderation(u8 cq_period_mode)
166{
167 u8 profile_ix = cq_period_mode == NET_DIM_CQ_PERIOD_MODE_START_FROM_CQE ?
168 NET_DIM_DEF_PROFILE_CQE : NET_DIM_DEF_PROFILE_EQE;
169
170 return net_dim_get_rx_moderation(cq_period_mode, profile_ix);
171}
172
173static inline struct net_dim_cq_moder
174net_dim_get_tx_moderation(u8 cq_period_mode, int ix)
175{
176 struct net_dim_cq_moder cq_moder = tx_profile[cq_period_mode][ix];
177
178 cq_moder.cq_period_mode = cq_period_mode;
179 return cq_moder;
180}
181
182static inline struct net_dim_cq_moder
183net_dim_get_def_tx_moderation(u8 cq_period_mode)
184{
185 u8 profile_ix = cq_period_mode == NET_DIM_CQ_PERIOD_MODE_START_FROM_CQE ?
186 NET_DIM_DEF_PROFILE_CQE : NET_DIM_DEF_PROFILE_EQE;
187
188 return net_dim_get_tx_moderation(cq_period_mode, profile_ix);
189}
190
191static inline bool net_dim_on_top(struct net_dim *dim)
192{
193 switch (dim->tune_state) {
194 case NET_DIM_PARKING_ON_TOP:
195 case NET_DIM_PARKING_TIRED:
196 return true;
197 case NET_DIM_GOING_RIGHT:
198 return (dim->steps_left > 1) && (dim->steps_right == 1);
199 default: /* NET_DIM_GOING_LEFT */
200 return (dim->steps_right > 1) && (dim->steps_left == 1);
201 }
202}
203
204static inline void net_dim_turn(struct net_dim *dim)
205{
206 switch (dim->tune_state) {
207 case NET_DIM_PARKING_ON_TOP:
208 case NET_DIM_PARKING_TIRED:
209 break;
210 case NET_DIM_GOING_RIGHT:
211 dim->tune_state = NET_DIM_GOING_LEFT;
212 dim->steps_left = 0;
213 break;
214 case NET_DIM_GOING_LEFT:
215 dim->tune_state = NET_DIM_GOING_RIGHT;
216 dim->steps_right = 0;
217 break;
218 }
219}
220
221static inline int net_dim_step(struct net_dim *dim)
222{
223 if (dim->tired == (NET_DIM_PARAMS_NUM_PROFILES * 2))
224 return NET_DIM_TOO_TIRED;
225
226 switch (dim->tune_state) {
227 case NET_DIM_PARKING_ON_TOP:
228 case NET_DIM_PARKING_TIRED:
229 break;
230 case NET_DIM_GOING_RIGHT:
231 if (dim->profile_ix == (NET_DIM_PARAMS_NUM_PROFILES - 1))
232 return NET_DIM_ON_EDGE;
233 dim->profile_ix++;
234 dim->steps_right++;
235 break;
236 case NET_DIM_GOING_LEFT:
237 if (dim->profile_ix == 0)
238 return NET_DIM_ON_EDGE;
239 dim->profile_ix--;
240 dim->steps_left++;
241 break;
242 }
243
244 dim->tired++;
245 return NET_DIM_STEPPED;
246}
247
248static inline void net_dim_park_on_top(struct net_dim *dim)
249{
250 dim->steps_right = 0;
251 dim->steps_left = 0;
252 dim->tired = 0;
253 dim->tune_state = NET_DIM_PARKING_ON_TOP;
254}
255
256static inline void net_dim_park_tired(struct net_dim *dim)
257{
258 dim->steps_right = 0;
259 dim->steps_left = 0;
260 dim->tune_state = NET_DIM_PARKING_TIRED;
261}
262
263static inline void net_dim_exit_parking(struct net_dim *dim)
264{
265 dim->tune_state = dim->profile_ix ? NET_DIM_GOING_LEFT :
266 NET_DIM_GOING_RIGHT;
267 net_dim_step(dim);
268}
269
270#define IS_SIGNIFICANT_DIFF(val, ref) \
271 (((100UL * abs((val) - (ref))) / (ref)) > 10) /* more than 10% difference */
272
273static inline int net_dim_stats_compare(struct net_dim_stats *curr,
274 struct net_dim_stats *prev)
275{
276 if (!prev->bpms)
277 return curr->bpms ? NET_DIM_STATS_BETTER :
278 NET_DIM_STATS_SAME;
279
280 if (IS_SIGNIFICANT_DIFF(curr->bpms, prev->bpms))
281 return (curr->bpms > prev->bpms) ? NET_DIM_STATS_BETTER :
282 NET_DIM_STATS_WORSE;
283
284 if (!prev->ppms)
285 return curr->ppms ? NET_DIM_STATS_BETTER :
286 NET_DIM_STATS_SAME;
287
288 if (IS_SIGNIFICANT_DIFF(curr->ppms, prev->ppms))
289 return (curr->ppms > prev->ppms) ? NET_DIM_STATS_BETTER :
290 NET_DIM_STATS_WORSE;
291
292 if (!prev->epms)
293 return NET_DIM_STATS_SAME;
294
295 if (IS_SIGNIFICANT_DIFF(curr->epms, prev->epms))
296 return (curr->epms < prev->epms) ? NET_DIM_STATS_BETTER :
297 NET_DIM_STATS_WORSE;
298
299 return NET_DIM_STATS_SAME;
300}
301
302static inline bool net_dim_decision(struct net_dim_stats *curr_stats,
303 struct net_dim *dim)
304{
305 int prev_state = dim->tune_state;
306 int prev_ix = dim->profile_ix;
307 int stats_res;
308 int step_res;
309
310 switch (dim->tune_state) {
311 case NET_DIM_PARKING_ON_TOP:
312 stats_res = net_dim_stats_compare(curr_stats, &dim->prev_stats);
313 if (stats_res != NET_DIM_STATS_SAME)
314 net_dim_exit_parking(dim);
315 break;
316
317 case NET_DIM_PARKING_TIRED:
318 dim->tired--;
319 if (!dim->tired)
320 net_dim_exit_parking(dim);
321 break;
322
323 case NET_DIM_GOING_RIGHT:
324 case NET_DIM_GOING_LEFT:
325 stats_res = net_dim_stats_compare(curr_stats, &dim->prev_stats);
326 if (stats_res != NET_DIM_STATS_BETTER)
327 net_dim_turn(dim);
328
329 if (net_dim_on_top(dim)) {
330 net_dim_park_on_top(dim);
331 break;
332 }
333
334 step_res = net_dim_step(dim);
335 switch (step_res) {
336 case NET_DIM_ON_EDGE:
337 net_dim_park_on_top(dim);
338 break;
339 case NET_DIM_TOO_TIRED:
340 net_dim_park_tired(dim);
341 break;
342 }
343
344 break;
345 }
346
347 if ((prev_state != NET_DIM_PARKING_ON_TOP) ||
348 (dim->tune_state != NET_DIM_PARKING_ON_TOP))
349 dim->prev_stats = *curr_stats;
350
351 return dim->profile_ix != prev_ix;
352}
353
354static inline void net_dim_sample(u16 event_ctr,
355 u64 packets,
356 u64 bytes,
357 struct net_dim_sample *s)
358{
359 s->time = ktime_get();
360 s->pkt_ctr = packets;
361 s->byte_ctr = bytes;
362 s->event_ctr = event_ctr;
363}
364
365#define NET_DIM_NEVENTS 64
366#define BITS_PER_TYPE(type) (sizeof(type) * BITS_PER_BYTE)
367#define BIT_GAP(bits, end, start) ((((end) - (start)) + BIT_ULL(bits)) & (BIT_ULL(bits) - 1))
368
369static inline void net_dim_calc_stats(struct net_dim_sample *start,
370 struct net_dim_sample *end,
371 struct net_dim_stats *curr_stats)
372{
373 /* u32 holds up to 71 minutes, should be enough */
374 u32 delta_us = ktime_us_delta(end->time, start->time);
375 u32 npkts = BIT_GAP(BITS_PER_TYPE(u32), end->pkt_ctr, start->pkt_ctr);
376 u32 nbytes = BIT_GAP(BITS_PER_TYPE(u32), end->byte_ctr,
377 start->byte_ctr);
378
379 if (!delta_us)
380 return;
381
382 curr_stats->ppms = DIV_ROUND_UP(npkts * USEC_PER_MSEC, delta_us);
383 curr_stats->bpms = DIV_ROUND_UP(nbytes * USEC_PER_MSEC, delta_us);
384 curr_stats->epms = DIV_ROUND_UP(NET_DIM_NEVENTS * USEC_PER_MSEC,
385 delta_us);
386}
387
388static inline void net_dim(struct net_dim *dim,
389 struct net_dim_sample end_sample)
390{
391 struct net_dim_stats curr_stats;
392 u16 nevents;
393
394 switch (dim->state) {
395 case NET_DIM_MEASURE_IN_PROGRESS:
396 nevents = BIT_GAP(BITS_PER_TYPE(u16),
397 end_sample.event_ctr,
398 dim->start_sample.event_ctr);
399 if (nevents < NET_DIM_NEVENTS)
400 break;
401 net_dim_calc_stats(&dim->start_sample, &end_sample,
402 &curr_stats);
403 if (net_dim_decision(&curr_stats, dim)) {
404 dim->state = NET_DIM_APPLY_NEW_PROFILE;
405 schedule_work(&dim->work);
406 break;
407 }
408 /* fall through */
409 case NET_DIM_START_MEASURE:
410 dim->state = NET_DIM_MEASURE_IN_PROGRESS;
411 break;
412 case NET_DIM_APPLY_NEW_PROFILE:
413 break;
414 }
415}
416
417#endif /* NET_DIM_H */