tjh.dev
Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
1// SPDX-License-Identifier: GPL-2.0-only
2/* Copyright(c) 2022 Intel Corporation. All rights reserved. */
3#include <linux/io-64-nonatomic-hi-lo.h>
4#include <linux/device.h>
5#include <linux/delay.h>
6
7#include "cxlmem.h"
8#include "core.h"
9
10/**
11 * DOC: cxl core hdm
12 *
13 * Compute Express Link Host Managed Device Memory, starting with the
14 * CXL 2.0 specification, is managed by an array of HDM Decoder register
15 * instances per CXL port and per CXL endpoint. Define common helpers
16 * for enumerating these registers and capabilities.
17 */
18
19static int add_hdm_decoder(struct cxl_port *port, struct cxl_decoder *cxld,
20 int *target_map)
21{
22 int rc;
23
24 rc = cxl_decoder_add_locked(cxld, target_map);
25 if (rc) {
26 put_device(&cxld->dev);
27 dev_err(&port->dev, "Failed to add decoder\n");
28 return rc;
29 }
30
31 rc = cxl_decoder_autoremove(&port->dev, cxld);
32 if (rc)
33 return rc;
34
35 dev_dbg(&cxld->dev, "Added to port %s\n", dev_name(&port->dev));
36
37 return 0;
38}
39
40/*
41 * Per the CXL specification (8.2.5.12 CXL HDM Decoder Capability Structure)
42 * single ported host-bridges need not publish a decoder capability when a
43 * passthrough decode can be assumed, i.e. all transactions that the uport sees
44 * are claimed and passed to the single dport. Disable the range until the first
45 * CXL region is enumerated / activated.
46 */
47int devm_cxl_add_passthrough_decoder(struct cxl_port *port)
48{
49 struct cxl_decoder *cxld;
50 struct cxl_dport *dport;
51 int single_port_map[1];
52
53 cxld = cxl_switch_decoder_alloc(port, 1);
54 if (IS_ERR(cxld))
55 return PTR_ERR(cxld);
56
57 device_lock_assert(&port->dev);
58
59 dport = list_first_entry(&port->dports, typeof(*dport), list);
60 single_port_map[0] = dport->port_id;
61
62 return add_hdm_decoder(port, cxld, single_port_map);
63}
64EXPORT_SYMBOL_NS_GPL(devm_cxl_add_passthrough_decoder, CXL);
65
66static void parse_hdm_decoder_caps(struct cxl_hdm *cxlhdm)
67{
68 u32 hdm_cap;
69
70 hdm_cap = readl(cxlhdm->regs.hdm_decoder + CXL_HDM_DECODER_CAP_OFFSET);
71 cxlhdm->decoder_count = cxl_hdm_decoder_count(hdm_cap);
72 cxlhdm->target_count =
73 FIELD_GET(CXL_HDM_DECODER_TARGET_COUNT_MASK, hdm_cap);
74 if (FIELD_GET(CXL_HDM_DECODER_INTERLEAVE_11_8, hdm_cap))
75 cxlhdm->interleave_mask |= GENMASK(11, 8);
76 if (FIELD_GET(CXL_HDM_DECODER_INTERLEAVE_14_12, hdm_cap))
77 cxlhdm->interleave_mask |= GENMASK(14, 12);
78}
79
80static void __iomem *map_hdm_decoder_regs(struct cxl_port *port,
81 void __iomem *crb)
82{
83 struct cxl_component_reg_map map;
84
85 cxl_probe_component_regs(&port->dev, crb, &map);
86 if (!map.hdm_decoder.valid) {
87 dev_err(&port->dev, "HDM decoder registers invalid\n");
88 return IOMEM_ERR_PTR(-ENXIO);
89 }
90
91 return crb + map.hdm_decoder.offset;
92}
93
94/**
95 * devm_cxl_setup_hdm - map HDM decoder component registers
96 * @port: cxl_port to map
97 */
98struct cxl_hdm *devm_cxl_setup_hdm(struct cxl_port *port)
99{
100 struct device *dev = &port->dev;
101 void __iomem *crb, *hdm;
102 struct cxl_hdm *cxlhdm;
103
104 cxlhdm = devm_kzalloc(dev, sizeof(*cxlhdm), GFP_KERNEL);
105 if (!cxlhdm)
106 return ERR_PTR(-ENOMEM);
107
108 cxlhdm->port = port;
109 crb = devm_cxl_iomap_block(dev, port->component_reg_phys,
110 CXL_COMPONENT_REG_BLOCK_SIZE);
111 if (!crb) {
112 dev_err(dev, "No component registers mapped\n");
113 return ERR_PTR(-ENXIO);
114 }
115
116 hdm = map_hdm_decoder_regs(port, crb);
117 if (IS_ERR(hdm))
118 return ERR_CAST(hdm);
119 cxlhdm->regs.hdm_decoder = hdm;
120
121 parse_hdm_decoder_caps(cxlhdm);
122 if (cxlhdm->decoder_count == 0) {
123 dev_err(dev, "Spec violation. Caps invalid\n");
124 return ERR_PTR(-ENXIO);
125 }
126
127 return cxlhdm;
128}
129EXPORT_SYMBOL_NS_GPL(devm_cxl_setup_hdm, CXL);
130
131static int to_interleave_granularity(u32 ctrl)
132{
133 int val = FIELD_GET(CXL_HDM_DECODER0_CTRL_IG_MASK, ctrl);
134
135 return 256 << val;
136}
137
138static int to_interleave_ways(u32 ctrl)
139{
140 int val = FIELD_GET(CXL_HDM_DECODER0_CTRL_IW_MASK, ctrl);
141
142 switch (val) {
143 case 0 ... 4:
144 return 1 << val;
145 case 8 ... 10:
146 return 3 << (val - 8);
147 default:
148 return 0;
149 }
150}
151
152static int init_hdm_decoder(struct cxl_port *port, struct cxl_decoder *cxld,
153 int *target_map, void __iomem *hdm, int which)
154{
155 u64 size, base;
156 u32 ctrl;
157 int i;
158 union {
159 u64 value;
160 unsigned char target_id[8];
161 } target_list;
162
163 ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(which));
164 base = ioread64_hi_lo(hdm + CXL_HDM_DECODER0_BASE_LOW_OFFSET(which));
165 size = ioread64_hi_lo(hdm + CXL_HDM_DECODER0_SIZE_LOW_OFFSET(which));
166
167 if (!(ctrl & CXL_HDM_DECODER0_CTRL_COMMITTED))
168 size = 0;
169 if (base == U64_MAX || size == U64_MAX) {
170 dev_warn(&port->dev, "decoder%d.%d: Invalid resource range\n",
171 port->id, cxld->id);
172 return -ENXIO;
173 }
174
175 cxld->decoder_range = (struct range) {
176 .start = base,
177 .end = base + size - 1,
178 };
179
180 /* switch decoders are always enabled if committed */
181 if (ctrl & CXL_HDM_DECODER0_CTRL_COMMITTED) {
182 cxld->flags |= CXL_DECODER_F_ENABLE;
183 if (ctrl & CXL_HDM_DECODER0_CTRL_LOCK)
184 cxld->flags |= CXL_DECODER_F_LOCK;
185 }
186 cxld->interleave_ways = to_interleave_ways(ctrl);
187 if (!cxld->interleave_ways) {
188 dev_warn(&port->dev,
189 "decoder%d.%d: Invalid interleave ways (ctrl: %#x)\n",
190 port->id, cxld->id, ctrl);
191 return -ENXIO;
192 }
193 cxld->interleave_granularity = to_interleave_granularity(ctrl);
194
195 if (FIELD_GET(CXL_HDM_DECODER0_CTRL_TYPE, ctrl))
196 cxld->target_type = CXL_DECODER_EXPANDER;
197 else
198 cxld->target_type = CXL_DECODER_ACCELERATOR;
199
200 if (is_cxl_endpoint(to_cxl_port(cxld->dev.parent)))
201 return 0;
202
203 target_list.value =
204 ioread64_hi_lo(hdm + CXL_HDM_DECODER0_TL_LOW(which));
205 for (i = 0; i < cxld->interleave_ways; i++)
206 target_map[i] = target_list.target_id[i];
207
208 return 0;
209}
210
211/**
212 * devm_cxl_enumerate_decoders - add decoder objects per HDM register set
213 * @cxlhdm: Structure to populate with HDM capabilities
214 */
215int devm_cxl_enumerate_decoders(struct cxl_hdm *cxlhdm)
216{
217 void __iomem *hdm = cxlhdm->regs.hdm_decoder;
218 struct cxl_port *port = cxlhdm->port;
219 int i, committed, failed;
220 u32 ctrl;
221
222 /*
223 * Since the register resource was recently claimed via request_region()
224 * be careful about trusting the "not-committed" status until the commit
225 * timeout has elapsed. The commit timeout is 10ms (CXL 2.0
226 * 8.2.5.12.20), but double it to be tolerant of any clock skew between
227 * host and target.
228 */
229 for (i = 0, committed = 0; i < cxlhdm->decoder_count; i++) {
230 ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(i));
231 if (ctrl & CXL_HDM_DECODER0_CTRL_COMMITTED)
232 committed++;
233 }
234
235 /* ensure that future checks of committed can be trusted */
236 if (committed != cxlhdm->decoder_count)
237 msleep(20);
238
239 for (i = 0, failed = 0; i < cxlhdm->decoder_count; i++) {
240 int target_map[CXL_DECODER_MAX_INTERLEAVE] = { 0 };
241 int rc, target_count = cxlhdm->target_count;
242 struct cxl_decoder *cxld;
243
244 if (is_cxl_endpoint(port))
245 cxld = cxl_endpoint_decoder_alloc(port);
246 else
247 cxld = cxl_switch_decoder_alloc(port, target_count);
248 if (IS_ERR(cxld)) {
249 dev_warn(&port->dev,
250 "Failed to allocate the decoder\n");
251 return PTR_ERR(cxld);
252 }
253
254 rc = init_hdm_decoder(port, cxld, target_map,
255 cxlhdm->regs.hdm_decoder, i);
256 if (rc) {
257 put_device(&cxld->dev);
258 failed++;
259 continue;
260 }
261 rc = add_hdm_decoder(port, cxld, target_map);
262 if (rc) {
263 dev_warn(&port->dev,
264 "Failed to add decoder to port\n");
265 return rc;
266 }
267 }
268
269 if (failed == cxlhdm->decoder_count) {
270 dev_err(&port->dev, "No valid decoders found\n");
271 return -ENXIO;
272 }
273
274 return 0;
275}
276EXPORT_SYMBOL_NS_GPL(devm_cxl_enumerate_decoders, CXL);