tjh.dev
Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
net: add dst_cache support
This patch add a generic, lockless dst cache implementation.
The need for lock is avoided updating the dst cache fields
only in per cpu scope, and requiring that the cache manipulation
functions are invoked with the local bh disabled.
The refresh_ts and reset_ts fields are used to ensure the cache
consistency in case of cuncurrent cache update (dst_cache_set*) and
reset operation (dst_cache_reset).
Consider the following scenario:
CPU1: CPU2:
<cache lookup with emtpy cache: it fails>
<get dst via uncached route lookup>
<related configuration changes>
dst_cache_reset()
dst_cache_set()
The dst entry set passed to dst_cache_set() should not be used
for later dst cache lookup, because it's obtained using old
configuration values.
Since the refresh_ts is updated only on dst_cache lookup, the
cached value in the above scenario will be discarded on the next
lookup.
Signed-off-by: Paolo Abeni <pabeni@redhat.com>
Suggested-and-acked-by: Hannes Frederic Sowa <hannes@stressinduktion.org>
Signed-off-by: David S. Miller <davem@davemloft.net>
authored by
Paolo Abeni
and committed by
David S. Miller
911362c7
64f63d59
+270
+97
include/net/dst_cache.h
+97
include/net/dst_cache.h
···
1
+
#ifndef _NET_DST_CACHE_H
2
+
#define _NET_DST_CACHE_H
3
+
4
+
#include <linux/jiffies.h>
5
+
#include <net/dst.h>
6
+
#if IS_ENABLED(CONFIG_IPV6)
7
+
#include <net/ip6_fib.h>
8
+
#endif
9
+
10
+
struct dst_cache {
11
+
struct dst_cache_pcpu __percpu *cache;
12
+
unsigned long reset_ts;
13
+
};
14
+
15
+
/**
16
+
* dst_cache_get - perform cache lookup
17
+
* @dst_cache: the cache
18
+
*
19
+
* The caller should use dst_cache_get_ip4() if it need to retrieve the
20
+
* source address to be used when xmitting to the cached dst.
21
+
* local BH must be disabled.
22
+
*/
23
+
struct dst_entry *dst_cache_get(struct dst_cache *dst_cache);
24
+
25
+
/**
26
+
* dst_cache_get_ip4 - perform cache lookup and fetch ipv4 source address
27
+
* @dst_cache: the cache
28
+
* @saddr: return value for the retrieved source address
29
+
*
30
+
* local BH must be disabled.
31
+
*/
32
+
struct rtable *dst_cache_get_ip4(struct dst_cache *dst_cache, __be32 *saddr);
33
+
34
+
/**
35
+
* dst_cache_set_ip4 - store the ipv4 dst into the cache
36
+
* @dst_cache: the cache
37
+
* @dst: the entry to be cached
38
+
* @saddr: the source address to be stored inside the cache
39
+
*
40
+
* local BH must be disabled.
41
+
*/
42
+
void dst_cache_set_ip4(struct dst_cache *dst_cache, struct dst_entry *dst,
43
+
__be32 saddr);
44
+
45
+
#if IS_ENABLED(CONFIG_IPV6)
46
+
47
+
/**
48
+
* dst_cache_set_ip6 - store the ipv6 dst into the cache
49
+
* @dst_cache: the cache
50
+
* @dst: the entry to be cached
51
+
* @saddr: the source address to be stored inside the cache
52
+
*
53
+
* local BH must be disabled.
54
+
*/
55
+
void dst_cache_set_ip6(struct dst_cache *dst_cache, struct dst_entry *dst,
56
+
const struct in6_addr *addr);
57
+
58
+
/**
59
+
* dst_cache_get_ip6 - perform cache lookup and fetch ipv6 source address
60
+
* @dst_cache: the cache
61
+
* @saddr: return value for the retrieved source address
62
+
*
63
+
* local BH must be disabled.
64
+
*/
65
+
struct dst_entry *dst_cache_get_ip6(struct dst_cache *dst_cache,
66
+
struct in6_addr *saddr);
67
+
#endif
68
+
69
+
/**
70
+
* dst_cache_reset - invalidate the cache contents
71
+
* @dst_cache: the cache
72
+
*
73
+
* This do not free the cached dst to avoid races and contentions.
74
+
* the dst will be freed on later cache lookup.
75
+
*/
76
+
static inline void dst_cache_reset(struct dst_cache *dst_cache)
77
+
{
78
+
dst_cache->reset_ts = jiffies;
79
+
}
80
+
81
+
/**
82
+
* dst_cache_init - initialize the cache, allocating the required storage
83
+
* @dst_cache: the cache
84
+
* @gfp: allocation flags
85
+
*/
86
+
int dst_cache_init(struct dst_cache *dst_cache, gfp_t gfp);
87
+
88
+
/**
89
+
* dst_cache_destroy - empty the cache and free the allocated storage
90
+
* @dst_cache: the cache
91
+
*
92
+
* No synchronization is enforced: it must be called only when the cache
93
+
* is unsed.
94
+
*/
95
+
void dst_cache_destroy(struct dst_cache *dst_cache);
96
+
97
+
#endif
+4
net/Kconfig
+4
net/Kconfig
···
392
392
weight tunnel endpoint. Tunnel encapsulation parameters are stored
393
393
with light weight tunnel state associated with fib routes.
394
394
395
+
config DST_CACHE
396
+
bool "dst cache"
397
+
default n
398
+
395
399
endif # if NET
396
400
397
401
# Used by archs to tell that they support BPF_JIT
+1
net/core/Makefile
+1
net/core/Makefile
+168
net/core/dst_cache.c
+168
net/core/dst_cache.c
···
1
+
/*
2
+
* net/core/dst_cache.c - dst entry cache
3
+
*
4
+
* Copyright (c) 2016 Paolo Abeni <pabeni@redhat.com>
5
+
*
6
+
* This program is free software; you can redistribute it and/or modify
7
+
* it under the terms of the GNU General Public License as published by
8
+
* the Free Software Foundation; either version 2 of the License, or
9
+
* (at your option) any later version.
10
+
*/
11
+
12
+
#include <linux/kernel.h>
13
+
#include <linux/percpu.h>
14
+
#include <net/dst_cache.h>
15
+
#include <net/route.h>
16
+
#if IS_ENABLED(CONFIG_IPV6)
17
+
#include <net/ip6_fib.h>
18
+
#endif
19
+
#include <uapi/linux/in.h>
20
+
21
+
struct dst_cache_pcpu {
22
+
unsigned long refresh_ts;
23
+
struct dst_entry *dst;
24
+
u32 cookie;
25
+
union {
26
+
struct in_addr in_saddr;
27
+
struct in6_addr in6_saddr;
28
+
};
29
+
};
30
+
31
+
void dst_cache_per_cpu_dst_set(struct dst_cache_pcpu *dst_cache,
32
+
struct dst_entry *dst, u32 cookie)
33
+
{
34
+
dst_release(dst_cache->dst);
35
+
if (dst)
36
+
dst_hold(dst);
37
+
38
+
dst_cache->cookie = cookie;
39
+
dst_cache->dst = dst;
40
+
}
41
+
42
+
struct dst_entry *dst_cache_per_cpu_get(struct dst_cache *dst_cache,
43
+
struct dst_cache_pcpu *idst)
44
+
{
45
+
struct dst_entry *dst;
46
+
47
+
dst = idst->dst;
48
+
if (!dst)
49
+
goto fail;
50
+
51
+
/* the cache already hold a dst reference; it can't go away */
52
+
dst_hold(dst);
53
+
54
+
if (unlikely(!time_after(idst->refresh_ts, dst_cache->reset_ts) ||
55
+
(dst->obsolete && !dst->ops->check(dst, idst->cookie)))) {
56
+
dst_cache_per_cpu_dst_set(idst, NULL, 0);
57
+
dst_release(dst);
58
+
goto fail;
59
+
}
60
+
return dst;
61
+
62
+
fail:
63
+
idst->refresh_ts = jiffies;
64
+
return NULL;
65
+
}
66
+
67
+
struct dst_entry *dst_cache_get(struct dst_cache *dst_cache)
68
+
{
69
+
if (!dst_cache->cache)
70
+
return NULL;
71
+
72
+
return dst_cache_per_cpu_get(dst_cache, this_cpu_ptr(dst_cache->cache));
73
+
}
74
+
EXPORT_SYMBOL_GPL(dst_cache_get);
75
+
76
+
struct rtable *dst_cache_get_ip4(struct dst_cache *dst_cache, __be32 *saddr)
77
+
{
78
+
struct dst_cache_pcpu *idst;
79
+
struct dst_entry *dst;
80
+
81
+
if (!dst_cache->cache)
82
+
return NULL;
83
+
84
+
idst = this_cpu_ptr(dst_cache->cache);
85
+
dst = dst_cache_per_cpu_get(dst_cache, idst);
86
+
if (!dst)
87
+
return NULL;
88
+
89
+
*saddr = idst->in_saddr.s_addr;
90
+
return container_of(dst, struct rtable, dst);
91
+
}
92
+
EXPORT_SYMBOL_GPL(dst_cache_get_ip4);
93
+
94
+
void dst_cache_set_ip4(struct dst_cache *dst_cache, struct dst_entry *dst,
95
+
__be32 saddr)
96
+
{
97
+
struct dst_cache_pcpu *idst;
98
+
99
+
if (!dst_cache->cache)
100
+
return;
101
+
102
+
idst = this_cpu_ptr(dst_cache->cache);
103
+
dst_cache_per_cpu_dst_set(idst, dst, 0);
104
+
idst->in_saddr.s_addr = saddr;
105
+
}
106
+
EXPORT_SYMBOL_GPL(dst_cache_set_ip4);
107
+
108
+
#if IS_ENABLED(CONFIG_IPV6)
109
+
void dst_cache_set_ip6(struct dst_cache *dst_cache, struct dst_entry *dst,
110
+
const struct in6_addr *addr)
111
+
{
112
+
struct dst_cache_pcpu *idst;
113
+
114
+
if (!dst_cache->cache)
115
+
return;
116
+
117
+
idst = this_cpu_ptr(dst_cache->cache);
118
+
dst_cache_per_cpu_dst_set(this_cpu_ptr(dst_cache->cache), dst,
119
+
rt6_get_cookie((struct rt6_info *)dst));
120
+
idst->in6_saddr = *addr;
121
+
}
122
+
EXPORT_SYMBOL_GPL(dst_cache_set_ip6);
123
+
124
+
struct dst_entry *dst_cache_get_ip6(struct dst_cache *dst_cache,
125
+
struct in6_addr *saddr)
126
+
{
127
+
struct dst_cache_pcpu *idst;
128
+
struct dst_entry *dst;
129
+
130
+
if (!dst_cache->cache)
131
+
return NULL;
132
+
133
+
idst = this_cpu_ptr(dst_cache->cache);
134
+
dst = dst_cache_per_cpu_get(dst_cache, idst);
135
+
if (!dst)
136
+
return NULL;
137
+
138
+
*saddr = idst->in6_saddr;
139
+
return dst;
140
+
}
141
+
EXPORT_SYMBOL_GPL(dst_cache_get_ip6);
142
+
#endif
143
+
144
+
int dst_cache_init(struct dst_cache *dst_cache, gfp_t gfp)
145
+
{
146
+
dst_cache->cache = alloc_percpu_gfp(struct dst_cache_pcpu,
147
+
gfp | __GFP_ZERO);
148
+
if (!dst_cache->cache)
149
+
return -ENOMEM;
150
+
151
+
dst_cache_reset(dst_cache);
152
+
return 0;
153
+
}
154
+
EXPORT_SYMBOL_GPL(dst_cache_init);
155
+
156
+
void dst_cache_destroy(struct dst_cache *dst_cache)
157
+
{
158
+
int i;
159
+
160
+
if (!dst_cache->cache)
161
+
return;
162
+
163
+
for_each_possible_cpu(i)
164
+
dst_release(per_cpu_ptr(dst_cache->cache, i)->dst);
165
+
166
+
free_percpu(dst_cache->cache);
167
+
}
168
+
EXPORT_SYMBOL_GPL(dst_cache_destroy);