Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux

neighbour: Convert rwlock of struct neigh_table to spinlock.

Only neigh_for_each() and neigh_seq_start/stop() are on the
reader side of neigh_table.lock.

Let's convert rwlock to the plain spinlock.

Signed-off-by: Kuniyuki Iwashima <kuniyu@google.com>
Reviewed-by: Eric Dumazet <edumazet@google.com>
Link: https://patch.msgid.link/20251022054004.2514876-6-kuniyu@google.com
Signed-off-by: Jakub Kicinski <kuba@kernel.org>

authored by

Kuniyuki Iwashima and committed by
Jakub Kicinski
3064d0fe 55a6046b

+40 -38
+1 -1
include/net/neighbour.h
··· 238 238 atomic_t gc_entries; 239 239 struct list_head gc_list; 240 240 struct list_head managed_list; 241 - rwlock_t lock; 241 + spinlock_t lock; 242 242 unsigned long last_rand; 243 243 struct neigh_statistics __percpu *stats; 244 244 struct neigh_hash_table __rcu *nht;
+2 -2
net/atm/clip.c
··· 168 168 169 169 static void idle_timer_check(struct timer_list *unused) 170 170 { 171 - write_lock(&arp_tbl.lock); 171 + spin_lock(&arp_tbl.lock); 172 172 __neigh_for_each_release(&arp_tbl, neigh_check_cb); 173 173 mod_timer(&idle_timer, jiffies + CLIP_CHECK_INTERVAL * HZ); 174 - write_unlock(&arp_tbl.lock); 174 + spin_unlock(&arp_tbl.lock); 175 175 } 176 176 177 177 static int clip_arp_rcv(struct sk_buff *skb)
+35 -33
net/core/neighbour.c
··· 81 81 } 82 82 83 83 /* 84 - Neighbour hash table buckets are protected with rwlock tbl->lock. 84 + Neighbour hash table buckets are protected with tbl->lock. 85 85 86 86 - All the scans/updates to hash buckets MUST be made under this lock. 87 87 - NOTHING clever should be made under this lock: no callbacks ··· 149 149 { 150 150 bool on_gc_list, exempt_from_gc; 151 151 152 - write_lock_bh(&n->tbl->lock); 152 + spin_lock_bh(&n->tbl->lock); 153 153 write_lock(&n->lock); 154 154 if (n->dead) 155 155 goto out; ··· 172 172 } 173 173 out: 174 174 write_unlock(&n->lock); 175 - write_unlock_bh(&n->tbl->lock); 175 + spin_unlock_bh(&n->tbl->lock); 176 176 } 177 177 178 178 static void neigh_update_managed_list(struct neighbour *n) 179 179 { 180 180 bool on_managed_list, add_to_managed; 181 181 182 - write_lock_bh(&n->tbl->lock); 182 + spin_lock_bh(&n->tbl->lock); 183 183 write_lock(&n->lock); 184 184 if (n->dead) 185 185 goto out; ··· 193 193 list_add_tail(&n->managed_list, &n->tbl->managed_list); 194 194 out: 195 195 write_unlock(&n->lock); 196 - write_unlock_bh(&n->tbl->lock); 196 + spin_unlock_bh(&n->tbl->lock); 197 197 } 198 198 199 199 static void neigh_update_flags(struct neighbour *neigh, u32 flags, int *notify, ··· 263 263 264 264 NEIGH_CACHE_STAT_INC(tbl, forced_gc_runs); 265 265 266 - write_lock_bh(&tbl->lock); 266 + spin_lock_bh(&tbl->lock); 267 267 268 268 list_for_each_entry_safe(n, tmp, &tbl->gc_list, gc_list) { 269 269 if (refcount_read(&n->refcnt) == 1) { ··· 292 292 293 293 WRITE_ONCE(tbl->last_flush, jiffies); 294 294 unlock: 295 - write_unlock_bh(&tbl->lock); 295 + spin_unlock_bh(&tbl->lock); 296 296 297 297 return shrunk; 298 298 } ··· 454 454 455 455 void neigh_changeaddr(struct neigh_table *tbl, struct net_device *dev) 456 456 { 457 - write_lock_bh(&tbl->lock); 457 + spin_lock_bh(&tbl->lock); 458 458 neigh_flush_dev(tbl, dev, false); 459 - write_unlock_bh(&tbl->lock); 459 + spin_unlock_bh(&tbl->lock); 460 460 } 461 461 EXPORT_SYMBOL(neigh_changeaddr); 462 462 463 463 static int __neigh_ifdown(struct neigh_table *tbl, struct net_device *dev, 464 464 bool skip_perm) 465 465 { 466 - write_lock_bh(&tbl->lock); 466 + spin_lock_bh(&tbl->lock); 467 467 if (likely(dev)) { 468 468 neigh_flush_dev(tbl, dev, skip_perm); 469 469 } else { 470 470 DEBUG_NET_WARN_ON_ONCE(skip_perm); 471 471 neigh_flush_table(tbl); 472 472 } 473 - write_unlock_bh(&tbl->lock); 473 + spin_unlock_bh(&tbl->lock); 474 474 475 475 pneigh_ifdown(tbl, dev, skip_perm); 476 476 pneigh_queue_purge(&tbl->proxy_queue, dev ? dev_net(dev) : NULL, ··· 687 687 688 688 n->confirmed = jiffies - (NEIGH_VAR(n->parms, BASE_REACHABLE_TIME) << 1); 689 689 690 - write_lock_bh(&tbl->lock); 690 + spin_lock_bh(&tbl->lock); 691 691 nht = rcu_dereference_protected(tbl->nht, 692 692 lockdep_is_held(&tbl->lock)); 693 693 ··· 722 722 hlist_add_head_rcu(&n->dev_list, 723 723 neigh_get_dev_table(dev, tbl->family)); 724 724 725 - write_unlock_bh(&tbl->lock); 725 + spin_unlock_bh(&tbl->lock); 726 726 neigh_dbg(2, "neigh %p is created\n", n); 727 727 rc = n; 728 728 out: 729 729 return rc; 730 730 out_tbl_unlock: 731 - write_unlock_bh(&tbl->lock); 731 + spin_unlock_bh(&tbl->lock); 732 732 out_neigh_release: 733 733 if (!exempt_from_gc) 734 734 atomic_dec(&tbl->gc_entries); ··· 982 982 983 983 NEIGH_CACHE_STAT_INC(tbl, periodic_gc_runs); 984 984 985 - write_lock_bh(&tbl->lock); 985 + spin_lock_bh(&tbl->lock); 986 986 nht = rcu_dereference_protected(tbl->nht, 987 987 lockdep_is_held(&tbl->lock)); 988 988 ··· 1036 1036 * It's fine to release lock here, even if hash table 1037 1037 * grows while we are preempted. 1038 1038 */ 1039 - write_unlock_bh(&tbl->lock); 1039 + spin_unlock_bh(&tbl->lock); 1040 1040 cond_resched(); 1041 - write_lock_bh(&tbl->lock); 1041 + spin_lock_bh(&tbl->lock); 1042 1042 nht = rcu_dereference_protected(tbl->nht, 1043 1043 lockdep_is_held(&tbl->lock)); 1044 1044 } ··· 1049 1049 */ 1050 1050 queue_delayed_work(system_power_efficient_wq, &tbl->gc_work, 1051 1051 NEIGH_VAR(&tbl->parms, BASE_REACHABLE_TIME) >> 1); 1052 - write_unlock_bh(&tbl->lock); 1052 + spin_unlock_bh(&tbl->lock); 1053 1053 } 1054 1054 1055 1055 static __inline__ int neigh_max_probes(struct neighbour *n) ··· 1641 1641 managed_work.work); 1642 1642 struct neighbour *neigh; 1643 1643 1644 - write_lock_bh(&tbl->lock); 1644 + spin_lock_bh(&tbl->lock); 1645 1645 list_for_each_entry(neigh, &tbl->managed_list, managed_list) 1646 1646 neigh_event_send_probe(neigh, NULL, false); 1647 1647 queue_delayed_work(system_power_efficient_wq, &tbl->managed_work, 1648 1648 NEIGH_VAR(&tbl->parms, INTERVAL_PROBE_TIME_MS)); 1649 - write_unlock_bh(&tbl->lock); 1649 + spin_unlock_bh(&tbl->lock); 1650 1650 } 1651 1651 1652 1652 static void neigh_proxy_process(struct timer_list *t) ··· 1761 1761 return NULL; 1762 1762 } 1763 1763 1764 - write_lock_bh(&tbl->lock); 1764 + spin_lock_bh(&tbl->lock); 1765 1765 list_add_rcu(&p->list, &tbl->parms.list); 1766 - write_unlock_bh(&tbl->lock); 1766 + spin_unlock_bh(&tbl->lock); 1767 1767 1768 1768 neigh_parms_data_state_cleanall(p); 1769 1769 } ··· 1783 1783 { 1784 1784 if (!parms || parms == &tbl->parms) 1785 1785 return; 1786 - write_lock_bh(&tbl->lock); 1786 + 1787 + spin_lock_bh(&tbl->lock); 1787 1788 list_del_rcu(&parms->list); 1788 1789 parms->dead = 1; 1789 - write_unlock_bh(&tbl->lock); 1790 + spin_unlock_bh(&tbl->lock); 1791 + 1790 1792 netdev_put(parms->dev, &parms->dev_tracker); 1791 1793 call_rcu(&parms->rcu_head, neigh_rcu_free_parms); 1792 1794 } ··· 1837 1835 else 1838 1836 WARN_ON(tbl->entry_size % NEIGH_PRIV_ALIGN); 1839 1837 1840 - rwlock_init(&tbl->lock); 1838 + spin_lock_init(&tbl->lock); 1841 1839 mutex_init(&tbl->phash_lock); 1842 1840 1843 1841 INIT_DEFERRABLE_WORK(&tbl->gc_work, neigh_periodic_work); ··· 1980 1978 err = __neigh_update(neigh, NULL, NUD_FAILED, 1981 1979 NEIGH_UPDATE_F_OVERRIDE | NEIGH_UPDATE_F_ADMIN, 1982 1980 NETLINK_CB(skb).portid, extack); 1983 - write_lock_bh(&tbl->lock); 1981 + spin_lock_bh(&tbl->lock); 1984 1982 neigh_release(neigh); 1985 1983 neigh_remove_one(neigh); 1986 - write_unlock_bh(&tbl->lock); 1984 + spin_unlock_bh(&tbl->lock); 1987 1985 1988 1986 out: 1989 1987 return err; ··· 2408 2406 * We acquire tbl->lock to be nice to the periodic timers and 2409 2407 * make sure they always see a consistent set of values. 2410 2408 */ 2411 - write_lock_bh(&tbl->lock); 2409 + spin_lock_bh(&tbl->lock); 2412 2410 2413 2411 if (tb[NDTA_PARMS]) { 2414 2412 struct nlattr *tbp[NDTPA_MAX+1]; ··· 2527 2525 err = 0; 2528 2526 2529 2527 errout_tbl_lock: 2530 - write_unlock_bh(&tbl->lock); 2528 + spin_unlock_bh(&tbl->lock); 2531 2529 rcu_read_unlock(); 2532 2530 errout: 2533 2531 return err; ··· 3127 3125 rcu_read_lock(); 3128 3126 nht = rcu_dereference(tbl->nht); 3129 3127 3130 - read_lock_bh(&tbl->lock); /* avoid resizes */ 3128 + spin_lock_bh(&tbl->lock); /* avoid resizes */ 3131 3129 for (chain = 0; chain < (1 << nht->hash_shift); chain++) { 3132 3130 struct neighbour *n; 3133 3131 3134 3132 neigh_for_each_in_bucket(n, &nht->hash_heads[chain]) 3135 3133 cb(n, cookie); 3136 3134 } 3137 - read_unlock_bh(&tbl->lock); 3135 + spin_unlock_bh(&tbl->lock); 3138 3136 rcu_read_unlock(); 3139 3137 } 3140 3138 EXPORT_SYMBOL(neigh_for_each); ··· 3404 3402 3405 3403 rcu_read_lock(); 3406 3404 state->nht = rcu_dereference(tbl->nht); 3407 - read_lock_bh(&tbl->lock); 3405 + spin_lock_bh(&tbl->lock); 3408 3406 3409 3407 return *pos ? neigh_get_idx_any(seq, pos) : SEQ_START_TOKEN; 3410 3408 } ··· 3444 3442 struct neigh_seq_state *state = seq->private; 3445 3443 struct neigh_table *tbl = state->tbl; 3446 3444 3447 - read_unlock_bh(&tbl->lock); 3445 + spin_unlock_bh(&tbl->lock); 3448 3446 rcu_read_unlock(); 3449 3447 } 3450 3448 EXPORT_SYMBOL(neigh_seq_stop);
+2 -2
net/ipv4/arp.c
··· 1217 1217 err = neigh_update(neigh, NULL, NUD_FAILED, 1218 1218 NEIGH_UPDATE_F_OVERRIDE| 1219 1219 NEIGH_UPDATE_F_ADMIN, 0); 1220 - write_lock_bh(&tbl->lock); 1220 + spin_lock_bh(&tbl->lock); 1221 1221 neigh_release(neigh); 1222 1222 neigh_remove_one(neigh); 1223 - write_unlock_bh(&tbl->lock); 1223 + spin_unlock_bh(&tbl->lock); 1224 1224 } 1225 1225 1226 1226 return err;