tjh.dev
Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
1/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
2
3/* Authors: Cheng Xu <chengyou@linux.alibaba.com> */
4/* Kai Shen <kaishen@linux.alibaba.com> */
5/* Copyright (c) 2020-2022, Alibaba Group. */
6
7#ifndef __ERDMA_H__
8#define __ERDMA_H__
9
10#include <linux/bitfield.h>
11#include <linux/netdevice.h>
12#include <linux/pci.h>
13#include <linux/xarray.h>
14#include <rdma/ib_verbs.h>
15
16#include "erdma_hw.h"
17
18#define DRV_MODULE_NAME "erdma"
19#define ERDMA_NODE_DESC "Elastic RDMA Adapter stack"
20
21struct erdma_eq {
22 void *qbuf;
23 dma_addr_t qbuf_dma_addr;
24
25 spinlock_t lock;
26
27 u32 depth;
28
29 u16 ci;
30 u16 rsvd;
31
32 atomic64_t event_num;
33 atomic64_t notify_num;
34
35 void __iomem *db;
36 u64 *dbrec;
37 dma_addr_t dbrec_dma;
38};
39
40struct erdma_cmdq_sq {
41 void *qbuf;
42 dma_addr_t qbuf_dma_addr;
43
44 spinlock_t lock;
45
46 u32 depth;
47 u16 ci;
48 u16 pi;
49
50 u16 wqebb_cnt;
51
52 u64 *dbrec;
53 dma_addr_t dbrec_dma;
54};
55
56struct erdma_cmdq_cq {
57 void *qbuf;
58 dma_addr_t qbuf_dma_addr;
59
60 spinlock_t lock;
61
62 u32 depth;
63 u32 ci;
64 u32 cmdsn;
65
66 u64 *dbrec;
67 dma_addr_t dbrec_dma;
68
69 atomic64_t armed_num;
70};
71
72enum {
73 ERDMA_CMD_STATUS_INIT,
74 ERDMA_CMD_STATUS_ISSUED,
75 ERDMA_CMD_STATUS_FINISHED,
76 ERDMA_CMD_STATUS_TIMEOUT
77};
78
79struct erdma_comp_wait {
80 struct completion wait_event;
81 u32 cmd_status;
82 u32 ctx_id;
83 u16 sq_pi;
84 u8 comp_status;
85 u8 rsvd;
86 u32 comp_data[4];
87};
88
89enum {
90 ERDMA_CMDQ_STATE_OK_BIT = 0,
91 ERDMA_CMDQ_STATE_TIMEOUT_BIT = 1,
92 ERDMA_CMDQ_STATE_CTX_ERR_BIT = 2,
93};
94
95#define ERDMA_CMDQ_TIMEOUT_MS 15000
96#define ERDMA_REG_ACCESS_WAIT_MS 20
97#define ERDMA_WAIT_DEV_DONE_CNT 500
98
99struct erdma_cmdq {
100 unsigned long *comp_wait_bitmap;
101 struct erdma_comp_wait *wait_pool;
102 spinlock_t lock;
103
104 struct erdma_cmdq_sq sq;
105 struct erdma_cmdq_cq cq;
106 struct erdma_eq eq;
107
108 unsigned long state;
109
110 struct semaphore credits;
111 u16 max_outstandings;
112};
113
114#define COMPROMISE_CC ERDMA_CC_CUBIC
115enum erdma_cc_alg {
116 ERDMA_CC_NEWRENO = 0,
117 ERDMA_CC_CUBIC,
118 ERDMA_CC_HPCC_RTT,
119 ERDMA_CC_HPCC_ECN,
120 ERDMA_CC_HPCC_INT,
121 ERDMA_CC_METHODS_NUM
122};
123
124struct erdma_devattr {
125 u32 fw_version;
126
127 unsigned char peer_addr[ETH_ALEN];
128 unsigned long cap_flags;
129
130 enum erdma_cc_alg cc;
131 u32 irq_num;
132
133 u32 max_qp;
134 u32 max_send_wr;
135 u32 max_recv_wr;
136 u32 max_ord;
137 u32 max_ird;
138
139 u32 max_send_sge;
140 u32 max_recv_sge;
141 u32 max_sge_rd;
142 u32 max_cq;
143 u32 max_cqe;
144 u64 max_mr_size;
145 u32 max_mr;
146 u32 max_pd;
147 u32 max_mw;
148 u32 max_gid;
149 u32 max_ah;
150 u32 local_dma_key;
151};
152
153#define ERDMA_IRQNAME_SIZE 50
154
155struct erdma_irq {
156 char name[ERDMA_IRQNAME_SIZE];
157 u32 msix_vector;
158 cpumask_t affinity_hint_mask;
159};
160
161struct erdma_eq_cb {
162 bool ready;
163 void *dev; /* All EQs use this fields to get erdma_dev struct */
164 struct erdma_irq irq;
165 struct erdma_eq eq;
166 struct tasklet_struct tasklet;
167};
168
169struct erdma_resource_cb {
170 unsigned long *bitmap;
171 spinlock_t lock;
172 u32 next_alloc_idx;
173 u32 max_cap;
174};
175
176enum {
177 ERDMA_RES_TYPE_PD = 0,
178 ERDMA_RES_TYPE_STAG_IDX = 1,
179 ERDMA_RES_TYPE_AH = 2,
180 ERDMA_RES_CNT = 3,
181};
182
183struct erdma_dev {
184 struct ib_device ibdev;
185 struct net_device *netdev;
186 struct pci_dev *pdev;
187 struct notifier_block netdev_nb;
188 struct workqueue_struct *reflush_wq;
189
190 resource_size_t func_bar_addr;
191 resource_size_t func_bar_len;
192 u8 __iomem *func_bar;
193
194 struct erdma_devattr attrs;
195 u32 mtu;
196
197 /* cmdq and aeq use the same msix vector */
198 struct erdma_irq comm_irq;
199 struct erdma_cmdq cmdq;
200 struct erdma_eq aeq;
201 struct erdma_eq_cb ceqs[ERDMA_NUM_MSIX_VEC - 1];
202
203 spinlock_t lock;
204 struct erdma_resource_cb res_cb[ERDMA_RES_CNT];
205 struct xarray qp_xa;
206 struct xarray cq_xa;
207
208 u32 next_alloc_qpn;
209 u32 next_alloc_cqn;
210
211 atomic_t num_ctx;
212 struct list_head cep_list;
213
214 struct dma_pool *db_pool;
215 struct dma_pool *resp_pool;
216 enum erdma_proto_type proto;
217};
218
219static inline void *get_queue_entry(void *qbuf, u32 idx, u32 depth, u32 shift)
220{
221 idx &= (depth - 1);
222
223 return qbuf + (idx << shift);
224}
225
226static inline struct erdma_dev *to_edev(struct ib_device *ibdev)
227{
228 return container_of(ibdev, struct erdma_dev, ibdev);
229}
230
231static inline u32 erdma_reg_read32(struct erdma_dev *dev, u32 reg)
232{
233 return readl(dev->func_bar + reg);
234}
235
236static inline u64 erdma_reg_read64(struct erdma_dev *dev, u32 reg)
237{
238 return readq(dev->func_bar + reg);
239}
240
241static inline void erdma_reg_write32(struct erdma_dev *dev, u32 reg, u32 value)
242{
243 writel(value, dev->func_bar + reg);
244}
245
246static inline void erdma_reg_write64(struct erdma_dev *dev, u32 reg, u64 value)
247{
248 writeq(value, dev->func_bar + reg);
249}
250
251static inline u32 erdma_reg_read32_filed(struct erdma_dev *dev, u32 reg,
252 u32 filed_mask)
253{
254 u32 val = erdma_reg_read32(dev, reg);
255
256 return FIELD_GET(filed_mask, val);
257}
258
259#define ERDMA_GET(val, name) FIELD_GET(ERDMA_CMD_##name##_MASK, val)
260
261int erdma_cmdq_init(struct erdma_dev *dev);
262void erdma_finish_cmdq_init(struct erdma_dev *dev);
263void erdma_cmdq_destroy(struct erdma_dev *dev);
264
265void erdma_cmdq_build_reqhdr(u64 *hdr, u32 mod, u32 op);
266int erdma_post_cmd_wait(struct erdma_cmdq *cmdq, void *req, u32 req_size,
267 u64 *resp0, u64 *resp1, bool sleepable);
268void erdma_cmdq_completion_handler(struct erdma_cmdq *cmdq);
269
270int erdma_ceqs_init(struct erdma_dev *dev);
271void erdma_ceqs_uninit(struct erdma_dev *dev);
272void notify_eq(struct erdma_eq *eq);
273void *get_next_valid_eqe(struct erdma_eq *eq);
274
275int erdma_aeq_init(struct erdma_dev *dev);
276int erdma_eq_common_init(struct erdma_dev *dev, struct erdma_eq *eq, u32 depth);
277void erdma_eq_destroy(struct erdma_dev *dev, struct erdma_eq *eq);
278
279void erdma_aeq_event_handler(struct erdma_dev *dev);
280void erdma_ceq_completion_handler(struct erdma_eq_cb *ceq_cb);
281
282#endif