tjh.dev
Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
1// SPDX-License-Identifier: GPL-2.0
2/* Copyright(c) 2019 Intel Corporation. All rights rsvd. */
3#include <linux/init.h>
4#include <linux/kernel.h>
5#include <linux/module.h>
6#include <linux/pci.h>
7#include <linux/device.h>
8#include <linux/io-64-nonatomic-lo-hi.h>
9#include <linux/dmaengine.h>
10#include <uapi/linux/idxd.h>
11#include "../dmaengine.h"
12#include "registers.h"
13#include "idxd.h"
14
15static inline struct idxd_wq *to_idxd_wq(struct dma_chan *c)
16{
17 return container_of(c, struct idxd_wq, dma_chan);
18}
19
20void idxd_dma_complete_txd(struct idxd_desc *desc,
21 enum idxd_complete_type comp_type)
22{
23 struct dma_async_tx_descriptor *tx;
24 struct dmaengine_result res;
25 int complete = 1;
26
27 if (desc->completion->status == DSA_COMP_SUCCESS)
28 res.result = DMA_TRANS_NOERROR;
29 else if (desc->completion->status)
30 res.result = DMA_TRANS_WRITE_FAILED;
31 else if (comp_type == IDXD_COMPLETE_ABORT)
32 res.result = DMA_TRANS_ABORTED;
33 else
34 complete = 0;
35
36 tx = &desc->txd;
37 if (complete && tx->cookie) {
38 dma_cookie_complete(tx);
39 dma_descriptor_unmap(tx);
40 dmaengine_desc_get_callback_invoke(tx, &res);
41 tx->callback = NULL;
42 tx->callback_result = NULL;
43 }
44}
45
46static void op_flag_setup(unsigned long flags, u32 *desc_flags)
47{
48 *desc_flags = IDXD_OP_FLAG_CRAV | IDXD_OP_FLAG_RCR;
49 if (flags & DMA_PREP_INTERRUPT)
50 *desc_flags |= IDXD_OP_FLAG_RCI;
51}
52
53static inline void set_completion_address(struct idxd_desc *desc,
54 u64 *compl_addr)
55{
56 *compl_addr = desc->compl_dma;
57}
58
59static inline void idxd_prep_desc_common(struct idxd_wq *wq,
60 struct dsa_hw_desc *hw, char opcode,
61 u64 addr_f1, u64 addr_f2, u64 len,
62 u64 compl, u32 flags)
63{
64 hw->flags = flags;
65 hw->opcode = opcode;
66 hw->src_addr = addr_f1;
67 hw->dst_addr = addr_f2;
68 hw->xfer_size = len;
69 hw->priv = !!(wq->type == IDXD_WQT_KERNEL);
70 hw->completion_addr = compl;
71}
72
73static struct dma_async_tx_descriptor *
74idxd_dma_submit_memcpy(struct dma_chan *c, dma_addr_t dma_dest,
75 dma_addr_t dma_src, size_t len, unsigned long flags)
76{
77 struct idxd_wq *wq = to_idxd_wq(c);
78 u32 desc_flags;
79 struct idxd_device *idxd = wq->idxd;
80 struct idxd_desc *desc;
81
82 if (wq->state != IDXD_WQ_ENABLED)
83 return NULL;
84
85 if (len > idxd->max_xfer_bytes)
86 return NULL;
87
88 op_flag_setup(flags, &desc_flags);
89 desc = idxd_alloc_desc(wq, IDXD_OP_BLOCK);
90 if (IS_ERR(desc))
91 return NULL;
92
93 idxd_prep_desc_common(wq, desc->hw, DSA_OPCODE_MEMMOVE,
94 dma_src, dma_dest, len, desc->compl_dma,
95 desc_flags);
96
97 desc->txd.flags = flags;
98
99 return &desc->txd;
100}
101
102static int idxd_dma_alloc_chan_resources(struct dma_chan *chan)
103{
104 struct idxd_wq *wq = to_idxd_wq(chan);
105 struct device *dev = &wq->idxd->pdev->dev;
106
107 idxd_wq_get(wq);
108 dev_dbg(dev, "%s: client_count: %d\n", __func__,
109 idxd_wq_refcount(wq));
110 return 0;
111}
112
113static void idxd_dma_free_chan_resources(struct dma_chan *chan)
114{
115 struct idxd_wq *wq = to_idxd_wq(chan);
116 struct device *dev = &wq->idxd->pdev->dev;
117
118 idxd_wq_put(wq);
119 dev_dbg(dev, "%s: client_count: %d\n", __func__,
120 idxd_wq_refcount(wq));
121}
122
123static enum dma_status idxd_dma_tx_status(struct dma_chan *dma_chan,
124 dma_cookie_t cookie,
125 struct dma_tx_state *txstate)
126{
127 return DMA_OUT_OF_ORDER;
128}
129
130/*
131 * issue_pending() does not need to do anything since tx_submit() does the job
132 * already.
133 */
134static void idxd_dma_issue_pending(struct dma_chan *dma_chan)
135{
136}
137
138dma_cookie_t idxd_dma_tx_submit(struct dma_async_tx_descriptor *tx)
139{
140 struct dma_chan *c = tx->chan;
141 struct idxd_wq *wq = to_idxd_wq(c);
142 dma_cookie_t cookie;
143 int rc;
144 struct idxd_desc *desc = container_of(tx, struct idxd_desc, txd);
145
146 cookie = dma_cookie_assign(tx);
147
148 rc = idxd_submit_desc(wq, desc);
149 if (rc < 0) {
150 idxd_free_desc(wq, desc);
151 return rc;
152 }
153
154 return cookie;
155}
156
157static void idxd_dma_release(struct dma_device *device)
158{
159}
160
161int idxd_register_dma_device(struct idxd_device *idxd)
162{
163 struct dma_device *dma = &idxd->dma_dev;
164
165 INIT_LIST_HEAD(&dma->channels);
166 dma->dev = &idxd->pdev->dev;
167
168 dma_cap_set(DMA_COMPLETION_NO_ORDER, dma->cap_mask);
169 dma->device_release = idxd_dma_release;
170
171 if (idxd->hw.opcap.bits[0] & IDXD_OPCAP_MEMMOVE) {
172 dma_cap_set(DMA_MEMCPY, dma->cap_mask);
173 dma->device_prep_dma_memcpy = idxd_dma_submit_memcpy;
174 }
175
176 dma->device_tx_status = idxd_dma_tx_status;
177 dma->device_issue_pending = idxd_dma_issue_pending;
178 dma->device_alloc_chan_resources = idxd_dma_alloc_chan_resources;
179 dma->device_free_chan_resources = idxd_dma_free_chan_resources;
180
181 return dma_async_device_register(&idxd->dma_dev);
182}
183
184void idxd_unregister_dma_device(struct idxd_device *idxd)
185{
186 dma_async_device_unregister(&idxd->dma_dev);
187}
188
189int idxd_register_dma_channel(struct idxd_wq *wq)
190{
191 struct idxd_device *idxd = wq->idxd;
192 struct dma_device *dma = &idxd->dma_dev;
193 struct dma_chan *chan = &wq->dma_chan;
194 int rc;
195
196 memset(&wq->dma_chan, 0, sizeof(struct dma_chan));
197 chan->device = dma;
198 list_add_tail(&chan->device_node, &dma->channels);
199 rc = dma_async_device_channel_register(dma, chan);
200 if (rc < 0)
201 return rc;
202
203 return 0;
204}
205
206void idxd_unregister_dma_channel(struct idxd_wq *wq)
207{
208 dma_async_device_channel_unregister(&wq->idxd->dma_dev, &wq->dma_chan);
209}