tjh.dev
Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
1/*
2 * Copyright (C) 2016 Red Hat, Inc.
3 * Author: Michael S. Tsirkin <mst@redhat.com>
4 * This work is licensed under the terms of the GNU GPL, version 2.
5 *
6 * Partial implementation of virtio 0.9. event index is used for signalling,
7 * unconditionally. Design roughly follows linux kernel implementation in order
8 * to be able to judge its performance.
9 */
10#define _GNU_SOURCE
11#include "main.h"
12#include <stdlib.h>
13#include <stdio.h>
14#include <assert.h>
15#include <string.h>
16#include <linux/virtio_ring.h>
17
18struct data {
19 void *data;
20} *data;
21
22struct vring ring;
23
24/* enabling the below activates experimental ring polling code
25 * (which skips index reads on consumer in favor of looking at
26 * high bits of ring id ^ 0x8000).
27 */
28/* #ifdef RING_POLL */
29/* enabling the below activates experimental in-order code
30 * (which skips ring updates and reads and writes len in descriptor).
31 */
32/* #ifdef INORDER */
33
34#if defined(RING_POLL) && defined(INORDER)
35#error "RING_POLL and INORDER are mutually exclusive"
36#endif
37
38/* how much padding is needed to avoid false cache sharing */
39#define HOST_GUEST_PADDING 0x80
40
41struct guest {
42 unsigned short avail_idx;
43 unsigned short last_used_idx;
44 unsigned short num_free;
45 unsigned short kicked_avail_idx;
46#ifndef INORDER
47 unsigned short free_head;
48#else
49 unsigned short reserved_free_head;
50#endif
51 unsigned char reserved[HOST_GUEST_PADDING - 10];
52} guest;
53
54struct host {
55 /* we do not need to track last avail index
56 * unless we have more than one in flight.
57 */
58 unsigned short used_idx;
59 unsigned short called_used_idx;
60 unsigned char reserved[HOST_GUEST_PADDING - 4];
61} host;
62
63/* implemented by ring */
64void alloc_ring(void)
65{
66 int ret;
67 int i;
68 void *p;
69
70 ret = posix_memalign(&p, 0x1000, vring_size(ring_size, 0x1000));
71 if (ret) {
72 perror("Unable to allocate ring buffer.\n");
73 exit(3);
74 }
75 memset(p, 0, vring_size(ring_size, 0x1000));
76 vring_init(&ring, ring_size, p, 0x1000);
77
78 guest.avail_idx = 0;
79 guest.kicked_avail_idx = -1;
80 guest.last_used_idx = 0;
81#ifndef INORDER
82 /* Put everything in free lists. */
83 guest.free_head = 0;
84#endif
85 for (i = 0; i < ring_size - 1; i++)
86 ring.desc[i].next = i + 1;
87 host.used_idx = 0;
88 host.called_used_idx = -1;
89 guest.num_free = ring_size;
90 data = malloc(ring_size * sizeof *data);
91 if (!data) {
92 perror("Unable to allocate data buffer.\n");
93 exit(3);
94 }
95 memset(data, 0, ring_size * sizeof *data);
96}
97
98/* guest side */
99int add_inbuf(unsigned len, void *buf, void *datap)
100{
101 unsigned head;
102#ifndef INORDER
103 unsigned avail;
104#endif
105 struct vring_desc *desc;
106
107 if (!guest.num_free)
108 return -1;
109
110#ifdef INORDER
111 head = (ring_size - 1) & (guest.avail_idx++);
112#else
113 head = guest.free_head;
114#endif
115 guest.num_free--;
116
117 desc = ring.desc;
118 desc[head].flags = VRING_DESC_F_NEXT;
119 desc[head].addr = (unsigned long)(void *)buf;
120 desc[head].len = len;
121 /* We do it like this to simulate the way
122 * we'd have to flip it if we had multiple
123 * descriptors.
124 */
125 desc[head].flags &= ~VRING_DESC_F_NEXT;
126#ifndef INORDER
127 guest.free_head = desc[head].next;
128#endif
129
130 data[head].data = datap;
131
132#ifdef RING_POLL
133 /* Barrier A (for pairing) */
134 smp_release();
135 avail = guest.avail_idx++;
136 ring.avail->ring[avail & (ring_size - 1)] =
137 (head | (avail & ~(ring_size - 1))) ^ 0x8000;
138#else
139#ifndef INORDER
140 /* Barrier A (for pairing) */
141 smp_release();
142 avail = (ring_size - 1) & (guest.avail_idx++);
143 ring.avail->ring[avail] = head;
144#endif
145 /* Barrier A (for pairing) */
146 smp_release();
147#endif
148 ring.avail->idx = guest.avail_idx;
149 return 0;
150}
151
152void *get_buf(unsigned *lenp, void **bufp)
153{
154 unsigned head;
155 unsigned index;
156 void *datap;
157
158#ifdef RING_POLL
159 head = (ring_size - 1) & guest.last_used_idx;
160 index = ring.used->ring[head].id;
161 if ((index ^ guest.last_used_idx ^ 0x8000) & ~(ring_size - 1))
162 return NULL;
163 /* Barrier B (for pairing) */
164 smp_acquire();
165 index &= ring_size - 1;
166#else
167 if (ring.used->idx == guest.last_used_idx)
168 return NULL;
169 /* Barrier B (for pairing) */
170 smp_acquire();
171#ifdef INORDER
172 head = (ring_size - 1) & guest.last_used_idx;
173 index = head;
174#else
175 head = (ring_size - 1) & guest.last_used_idx;
176 index = ring.used->ring[head].id;
177#endif
178
179#endif
180#ifdef INORDER
181 *lenp = ring.desc[index].len;
182#else
183 *lenp = ring.used->ring[head].len;
184#endif
185 datap = data[index].data;
186 *bufp = (void*)(unsigned long)ring.desc[index].addr;
187 data[index].data = NULL;
188#ifndef INORDER
189 ring.desc[index].next = guest.free_head;
190 guest.free_head = index;
191#endif
192 guest.num_free++;
193 guest.last_used_idx++;
194 return datap;
195}
196
197bool used_empty()
198{
199 unsigned short last_used_idx = guest.last_used_idx;
200#ifdef RING_POLL
201 unsigned short head = last_used_idx & (ring_size - 1);
202 unsigned index = ring.used->ring[head].id;
203
204 return (index ^ last_used_idx ^ 0x8000) & ~(ring_size - 1);
205#else
206 return ring.used->idx == last_used_idx;
207#endif
208}
209
210void disable_call()
211{
212 /* Doing nothing to disable calls might cause
213 * extra interrupts, but reduces the number of cache misses.
214 */
215}
216
217bool enable_call()
218{
219 vring_used_event(&ring) = guest.last_used_idx;
220 /* Flush call index write */
221 /* Barrier D (for pairing) */
222 smp_mb();
223 return used_empty();
224}
225
226void kick_available(void)
227{
228 /* Flush in previous flags write */
229 /* Barrier C (for pairing) */
230 smp_mb();
231 if (!vring_need_event(vring_avail_event(&ring),
232 guest.avail_idx,
233 guest.kicked_avail_idx))
234 return;
235
236 guest.kicked_avail_idx = guest.avail_idx;
237 kick();
238}
239
240/* host side */
241void disable_kick()
242{
243 /* Doing nothing to disable kicks might cause
244 * extra interrupts, but reduces the number of cache misses.
245 */
246}
247
248bool enable_kick()
249{
250 vring_avail_event(&ring) = host.used_idx;
251 /* Barrier C (for pairing) */
252 smp_mb();
253 return avail_empty();
254}
255
256bool avail_empty()
257{
258 unsigned head = host.used_idx;
259#ifdef RING_POLL
260 unsigned index = ring.avail->ring[head & (ring_size - 1)];
261
262 return ((index ^ head ^ 0x8000) & ~(ring_size - 1));
263#else
264 return head == ring.avail->idx;
265#endif
266}
267
268bool use_buf(unsigned *lenp, void **bufp)
269{
270 unsigned used_idx = host.used_idx;
271 struct vring_desc *desc;
272 unsigned head;
273
274#ifdef RING_POLL
275 head = ring.avail->ring[used_idx & (ring_size - 1)];
276 if ((used_idx ^ head ^ 0x8000) & ~(ring_size - 1))
277 return false;
278 /* Barrier A (for pairing) */
279 smp_acquire();
280
281 used_idx &= ring_size - 1;
282 desc = &ring.desc[head & (ring_size - 1)];
283#else
284 if (used_idx == ring.avail->idx)
285 return false;
286
287 /* Barrier A (for pairing) */
288 smp_acquire();
289
290 used_idx &= ring_size - 1;
291#ifdef INORDER
292 head = used_idx;
293#else
294 head = ring.avail->ring[used_idx];
295#endif
296 desc = &ring.desc[head];
297#endif
298
299 *lenp = desc->len;
300 *bufp = (void *)(unsigned long)desc->addr;
301
302#ifdef INORDER
303 desc->len = desc->len - 1;
304#else
305 /* now update used ring */
306 ring.used->ring[used_idx].id = head;
307 ring.used->ring[used_idx].len = desc->len - 1;
308#endif
309 /* Barrier B (for pairing) */
310 smp_release();
311 host.used_idx++;
312 ring.used->idx = host.used_idx;
313
314 return true;
315}
316
317void call_used(void)
318{
319 /* Flush in previous flags write */
320 /* Barrier D (for pairing) */
321 smp_mb();
322 if (!vring_need_event(vring_used_event(&ring),
323 host.used_idx,
324 host.called_used_idx))
325 return;
326
327 host.called_used_idx = host.used_idx;
328 call();
329}