Something went wrong. Try again.
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB/* * Copyright (c) 2018, Mellanox Technologies inc. All rights reserved. */
#include <linux/dim.h>#include <linux/rtnetlink.h>
/* * Net DIM profiles: * There are different set of profiles for each CQ period mode. * There are different set of profiles for RX/TX CQs. * Each profile size must be of NET_DIM_PARAMS_NUM_PROFILES */#define NET_DIM_RX_EQE_PROFILES { \ {.usec = 1, .pkts = NET_DIM_DEFAULT_RX_CQ_PKTS_FROM_EQE,}, \ {.usec = 8, .pkts = NET_DIM_DEFAULT_RX_CQ_PKTS_FROM_EQE,}, \ {.usec = 64, .pkts = NET_DIM_DEFAULT_RX_CQ_PKTS_FROM_EQE,}, \ {.usec = 128, .pkts = NET_DIM_DEFAULT_RX_CQ_PKTS_FROM_EQE,}, \ {.usec = 256, .pkts = NET_DIM_DEFAULT_RX_CQ_PKTS_FROM_EQE,} \}
#define NET_DIM_RX_CQE_PROFILES { \ {.usec = 2, .pkts = 256,}, \ {.usec = 8, .pkts = 128,}, \ {.usec = 16, .pkts = 64,}, \ {.usec = 32, .pkts = 64,}, \ {.usec = 64, .pkts = 64,} \}
#define NET_DIM_TX_EQE_PROFILES { \ {.usec = 1, .pkts = NET_DIM_DEFAULT_TX_CQ_PKTS_FROM_EQE,}, \ {.usec = 8, .pkts = NET_DIM_DEFAULT_TX_CQ_PKTS_FROM_EQE,}, \ {.usec = 32, .pkts = NET_DIM_DEFAULT_TX_CQ_PKTS_FROM_EQE,}, \ {.usec = 64, .pkts = NET_DIM_DEFAULT_TX_CQ_PKTS_FROM_EQE,}, \ {.usec = 128, .pkts = NET_DIM_DEFAULT_TX_CQ_PKTS_FROM_EQE,} \}
#define NET_DIM_TX_CQE_PROFILES { \ {.usec = 5, .pkts = 128,}, \ {.usec = 8, .pkts = 64,}, \ {.usec = 16, .pkts = 32,}, \ {.usec = 32, .pkts = 32,}, \ {.usec = 64, .pkts = 32,} \}
static const struct dim_cq_moderrx_profile[DIM_CQ_PERIOD_NUM_MODES][NET_DIM_PARAMS_NUM_PROFILES] = { NET_DIM_RX_EQE_PROFILES, NET_DIM_RX_CQE_PROFILES,};
static const struct dim_cq_modertx_profile[DIM_CQ_PERIOD_NUM_MODES][NET_DIM_PARAMS_NUM_PROFILES] = { NET_DIM_TX_EQE_PROFILES, NET_DIM_TX_CQE_PROFILES,};
struct dim_cq_modernet_dim_get_rx_moderation(u8 cq_period_mode, int ix){ struct dim_cq_moder cq_moder = rx_profile[cq_period_mode][ix];
cq_moder.cq_period_mode = cq_period_mode; return cq_moder;}EXPORT_SYMBOL(net_dim_get_rx_moderation);
struct dim_cq_modernet_dim_get_def_rx_moderation(u8 cq_period_mode){ u8 profile_ix = cq_period_mode == DIM_CQ_PERIOD_MODE_START_FROM_CQE ? NET_DIM_DEF_PROFILE_CQE : NET_DIM_DEF_PROFILE_EQE;
return net_dim_get_rx_moderation(cq_period_mode, profile_ix);}EXPORT_SYMBOL(net_dim_get_def_rx_moderation);
struct dim_cq_modernet_dim_get_tx_moderation(u8 cq_period_mode, int ix){ struct dim_cq_moder cq_moder = tx_profile[cq_period_mode][ix];
cq_moder.cq_period_mode = cq_period_mode; return cq_moder;}EXPORT_SYMBOL(net_dim_get_tx_moderation);
struct dim_cq_modernet_dim_get_def_tx_moderation(u8 cq_period_mode){ u8 profile_ix = cq_period_mode == DIM_CQ_PERIOD_MODE_START_FROM_CQE ? NET_DIM_DEF_PROFILE_CQE : NET_DIM_DEF_PROFILE_EQE;
return net_dim_get_tx_moderation(cq_period_mode, profile_ix);}EXPORT_SYMBOL(net_dim_get_def_tx_moderation);
int net_dim_init_irq_moder(struct net_device *dev, u8 profile_flags, u8 coal_flags, u8 rx_mode, u8 tx_mode, void (*rx_dim_work)(struct work_struct *work), void (*tx_dim_work)(struct work_struct *work)){ struct dim_cq_moder *rxp = NULL, *txp; struct dim_irq_moder *moder; int len;
dev->irq_moder = kzalloc_obj(*dev->irq_moder); if (!dev->irq_moder) return -ENOMEM;
moder = dev->irq_moder; len = NET_DIM_PARAMS_NUM_PROFILES * sizeof(*moder->rx_profile);
moder->coal_flags = coal_flags; moder->profile_flags = profile_flags;
if (profile_flags & DIM_PROFILE_RX) { moder->rx_dim_work = rx_dim_work; moder->dim_rx_mode = rx_mode; rxp = kmemdup(rx_profile[rx_mode], len, GFP_KERNEL); if (!rxp) goto free_moder;
rcu_assign_pointer(moder->rx_profile, rxp); }
if (profile_flags & DIM_PROFILE_TX) { moder->tx_dim_work = tx_dim_work; moder->dim_tx_mode = tx_mode; txp = kmemdup(tx_profile[tx_mode], len, GFP_KERNEL); if (!txp) goto free_rxp;
rcu_assign_pointer(moder->tx_profile, txp); }
return 0;
free_rxp: kfree(rxp);free_moder: kfree(moder); return -ENOMEM;}EXPORT_SYMBOL(net_dim_init_irq_moder);
/* RTNL lock is held. */void net_dim_free_irq_moder(struct net_device *dev){ struct dim_cq_moder *rxp, *txp;
if (!dev->irq_moder) return;
rxp = rtnl_dereference(dev->irq_moder->rx_profile); txp = rtnl_dereference(dev->irq_moder->tx_profile);
rcu_assign_pointer(dev->irq_moder->rx_profile, NULL); rcu_assign_pointer(dev->irq_moder->tx_profile, NULL);
kfree_rcu(rxp, rcu); kfree_rcu(txp, rcu); kfree(dev->irq_moder);}EXPORT_SYMBOL(net_dim_free_irq_moder);
void net_dim_setting(struct net_device *dev, struct dim *dim, bool is_tx){ struct dim_irq_moder *irq_moder = dev->irq_moder;
if (!irq_moder) return;
if (is_tx) { INIT_WORK(&dim->work, irq_moder->tx_dim_work); dim->mode = READ_ONCE(irq_moder->dim_tx_mode); return; }
INIT_WORK(&dim->work, irq_moder->rx_dim_work); dim->mode = READ_ONCE(irq_moder->dim_rx_mode);}EXPORT_SYMBOL(net_dim_setting);
void net_dim_work_cancel(struct dim *dim){ cancel_work_sync(&dim->work);}EXPORT_SYMBOL(net_dim_work_cancel);
struct dim_cq_moder net_dim_get_rx_irq_moder(struct net_device *dev, struct dim *dim){ struct dim_cq_moder res, *profile;
rcu_read_lock(); profile = rcu_dereference(dev->irq_moder->rx_profile); res = profile[dim->profile_ix]; rcu_read_unlock();
res.cq_period_mode = dim->mode;
return res;}EXPORT_SYMBOL(net_dim_get_rx_irq_moder);
struct dim_cq_moder net_dim_get_tx_irq_moder(struct net_device *dev, struct dim *dim){ struct dim_cq_moder res, *profile;
rcu_read_lock(); profile = rcu_dereference(dev->irq_moder->tx_profile); res = profile[dim->profile_ix]; rcu_read_unlock();
res.cq_period_mode = dim->mode;
return res;}EXPORT_SYMBOL(net_dim_get_tx_irq_moder);
void net_dim_set_rx_mode(struct net_device *dev, u8 rx_mode){ WRITE_ONCE(dev->irq_moder->dim_rx_mode, rx_mode);}EXPORT_SYMBOL(net_dim_set_rx_mode);
void net_dim_set_tx_mode(struct net_device *dev, u8 tx_mode){ WRITE_ONCE(dev->irq_moder->dim_tx_mode, tx_mode);}EXPORT_SYMBOL(net_dim_set_tx_mode);
static int net_dim_step(struct dim *dim){ if (dim->tired == (NET_DIM_PARAMS_NUM_PROFILES * 2)) return DIM_TOO_TIRED;
switch (dim->tune_state) { case DIM_PARKING_ON_TOP: case DIM_PARKING_TIRED: break; case DIM_GOING_RIGHT: if (dim->profile_ix == (NET_DIM_PARAMS_NUM_PROFILES - 1)) return DIM_ON_EDGE; dim->profile_ix++; dim->steps_right++; break; case DIM_GOING_LEFT: if (dim->profile_ix == 0) return DIM_ON_EDGE; dim->profile_ix--; dim->steps_left++; break; }
dim->tired++; return DIM_STEPPED;}
static void net_dim_exit_parking(struct dim *dim){ dim->tune_state = dim->profile_ix ? DIM_GOING_LEFT : DIM_GOING_RIGHT; net_dim_step(dim);}
static int net_dim_stats_compare(struct dim_stats *curr, struct dim_stats *prev){ if (!prev->bpms) return curr->bpms ? DIM_STATS_BETTER : DIM_STATS_SAME;
if (IS_SIGNIFICANT_DIFF(curr->bpms, prev->bpms)) return (curr->bpms > prev->bpms) ? DIM_STATS_BETTER : DIM_STATS_WORSE;
if (!prev->ppms) return curr->ppms ? DIM_STATS_BETTER : DIM_STATS_SAME;
if (IS_SIGNIFICANT_DIFF(curr->ppms, prev->ppms)) return (curr->ppms > prev->ppms) ? DIM_STATS_BETTER : DIM_STATS_WORSE;
if (!prev->epms) return DIM_STATS_SAME;
if (IS_SIGNIFICANT_DIFF(curr->epms, prev->epms)) return (curr->epms < prev->epms) ? DIM_STATS_BETTER : DIM_STATS_WORSE;
return DIM_STATS_SAME;}
static bool net_dim_decision(struct dim_stats *curr_stats, struct dim *dim){ int prev_state = dim->tune_state; int prev_ix = dim->profile_ix; int stats_res; int step_res;
switch (dim->tune_state) { case DIM_PARKING_ON_TOP: stats_res = net_dim_stats_compare(curr_stats, &dim->prev_stats); if (stats_res != DIM_STATS_SAME) net_dim_exit_parking(dim); break;
case DIM_PARKING_TIRED: dim->tired--; if (!dim->tired) net_dim_exit_parking(dim); break;
case DIM_GOING_RIGHT: case DIM_GOING_LEFT: stats_res = net_dim_stats_compare(curr_stats, &dim->prev_stats); if (stats_res != DIM_STATS_BETTER) dim_turn(dim);
if (dim_on_top(dim)) { dim_park_on_top(dim); break; }
step_res = net_dim_step(dim); switch (step_res) { case DIM_ON_EDGE: dim_park_on_top(dim); break; case DIM_TOO_TIRED: dim_park_tired(dim); break; }
break; }
if (prev_state != DIM_PARKING_ON_TOP || dim->tune_state != DIM_PARKING_ON_TOP) dim->prev_stats = *curr_stats;
return dim->profile_ix != prev_ix;}
void net_dim(struct dim *dim, const struct dim_sample *end_sample){ struct dim_stats curr_stats; u16 nevents;
switch (dim->state) { case DIM_MEASURE_IN_PROGRESS: nevents = BIT_GAP(BITS_PER_TYPE(u16), end_sample->event_ctr, dim->start_sample.event_ctr); if (nevents < DIM_NEVENTS) break; if (!dim_calc_stats(&dim->start_sample, end_sample, &curr_stats)) break; if (net_dim_decision(&curr_stats, dim)) { dim->state = DIM_APPLY_NEW_PROFILE; schedule_work(&dim->work); break; } fallthrough; case DIM_START_MEASURE: dim_update_sample(end_sample->event_ctr, end_sample->pkt_ctr, end_sample->byte_ctr, &dim->start_sample); dim->state = DIM_MEASURE_IN_PROGRESS; break; case DIM_APPLY_NEW_PROFILE: break; }}EXPORT_SYMBOL(net_dim);