tjh.dev
Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
1.. SPDX-License-Identifier: GPL-2.0
2
3============================
4Inter-Host-Bridge Interleave
5============================
6This cxl-cli configuration dump shows the following host configuration:
7
8* A single socket system with one CXL root
9* CXL Root has Four (4) CXL Host Bridges
10* Two CXL Host Bridges have a single CXL Memory Expander Attached
11* The CXL root is configured to interleave across the two host bridges.
12
13This output is generated by :code:`cxl list -v` and describes the relationships
14between objects exposed in :code:`/sys/bus/cxl/devices/`.
15
16::
17
18 [
19 {
20 "bus":"root0",
21 "provider":"ACPI.CXL",
22 "nr_dports":4,
23 "dports":[
24 {
25 "dport":"pci0000:00",
26 "alias":"ACPI0016:01",
27 "id":0
28 },
29 {
30 "dport":"pci0000:a8",
31 "alias":"ACPI0016:02",
32 "id":4
33 },
34 {
35 "dport":"pci0000:2a",
36 "alias":"ACPI0016:03",
37 "id":1
38 },
39 {
40 "dport":"pci0000:d2",
41 "alias":"ACPI0016:00",
42 "id":5
43 }
44 ],
45
46This chunk shows the CXL "bus" (root0) has 4 downstream ports attached to CXL
47Host Bridges. The `Root` can be considered the singular upstream port attached
48to the platform's memory controller - which routes memory requests to it.
49
50The `ports:root0` section lays out how each of these downstream ports are
51configured. If a port is not configured (id's 0 and 1), they are omitted.
52
53::
54
55 "ports:root0":[
56 {
57 "port":"port1",
58 "host":"pci0000:d2",
59 "depth":1,
60 "nr_dports":3,
61 "dports":[
62 {
63 "dport":"0000:d2:01.1",
64 "alias":"device:02",
65 "id":0
66 },
67 {
68 "dport":"0000:d2:01.3",
69 "alias":"device:05",
70 "id":2
71 },
72 {
73 "dport":"0000:d2:07.1",
74 "alias":"device:0d",
75 "id":113
76 }
77 ],
78
79This chunk shows the available downstream ports associated with the CXL Host
80Bridge :code:`port1`. In this case, :code:`port1` has 3 available downstream
81ports: :code:`dport1`, :code:`dport2`, and :code:`dport113`..
82
83::
84
85 "endpoints:port1":[
86 {
87 "endpoint":"endpoint5",
88 "host":"mem0",
89 "parent_dport":"0000:d2:01.1",
90 "depth":2,
91 "memdev":{
92 "memdev":"mem0",
93 "ram_size":137438953472,
94 "serial":0,
95 "numa_node":0,
96 "host":"0000:d3:00.0"
97 },
98 "decoders:endpoint5":[
99 {
100 "decoder":"decoder5.0",
101 "resource":825975898112,
102 "size":274877906944,
103 "interleave_ways":2,
104 "interleave_granularity":256,
105 "region":"region0",
106 "dpa_resource":0,
107 "dpa_size":137438953472,
108 "mode":"ram"
109 }
110 ]
111 }
112 ],
113
114This chunk shows the endpoints attached to the host bridge :code:`port1`.
115
116:code:`endpoint5` contains a single configured decoder :code:`decoder5.0`
117which has the same interleave configuration as :code:`region0` (shown later).
118
119Next we have the decodesr belonging to the host bridge:
120
121::
122
123 "decoders:port1":[
124 {
125 "decoder":"decoder1.0",
126 "resource":825975898112,
127 "size":274877906944,
128 "interleave_ways":1,
129 "region":"region0",
130 "nr_targets":1,
131 "targets":[
132 {
133 "target":"0000:d2:01.1",
134 "alias":"device:02",
135 "position":0,
136 "id":0
137 }
138 ]
139 }
140 ]
141 },
142
143Host Bridge :code:`port1` has a single decoder (:code:`decoder1.0`), whose only
144target is :code:`dport1` - which is attached to :code:`endpoint5`.
145
146The following chunk shows a similar configuration for Host Bridge :code:`port3`,
147the second host bridge with a memory device attached.
148
149::
150
151 {
152 "port":"port3",
153 "host":"pci0000:a8",
154 "depth":1,
155 "nr_dports":1,
156 "dports":[
157 {
158 "dport":"0000:a8:01.1",
159 "alias":"device:c3",
160 "id":0
161 }
162 ],
163 "endpoints:port3":[
164 {
165 "endpoint":"endpoint6",
166 "host":"mem1",
167 "parent_dport":"0000:a8:01.1",
168 "depth":2,
169 "memdev":{
170 "memdev":"mem1",
171 "ram_size":137438953472,
172 "serial":0,
173 "numa_node":0,
174 "host":"0000:a9:00.0"
175 },
176 "decoders:endpoint6":[
177 {
178 "decoder":"decoder6.0",
179 "resource":825975898112,
180 "size":274877906944,
181 "interleave_ways":2,
182 "interleave_granularity":256,
183 "region":"region0",
184 "dpa_resource":0,
185 "dpa_size":137438953472,
186 "mode":"ram"
187 }
188 ]
189 }
190 ],
191 "decoders:port3":[
192 {
193 "decoder":"decoder3.0",
194 "resource":825975898112,
195 "size":274877906944,
196 "interleave_ways":1,
197 "region":"region0",
198 "nr_targets":1,
199 "targets":[
200 {
201 "target":"0000:a8:01.1",
202 "alias":"device:c3",
203 "position":0,
204 "id":0
205 }
206 ]
207 }
208 ]
209 },
210
211
212The next chunk shows the two CXL host bridges without attached endpoints.
213
214::
215
216 {
217 "port":"port2",
218 "host":"pci0000:00",
219 "depth":1,
220 "nr_dports":2,
221 "dports":[
222 {
223 "dport":"0000:00:01.3",
224 "alias":"device:55",
225 "id":2
226 },
227 {
228 "dport":"0000:00:07.1",
229 "alias":"device:5d",
230 "id":113
231 }
232 ]
233 },
234 {
235 "port":"port4",
236 "host":"pci0000:2a",
237 "depth":1,
238 "nr_dports":1,
239 "dports":[
240 {
241 "dport":"0000:2a:01.1",
242 "alias":"device:d0",
243 "id":0
244 }
245 ]
246 }
247 ],
248
249Next we have the `Root Decoders` belonging to :code:`root0`. This root decoder
250applies the interleave across the downstream ports :code:`port1` and
251:code:`port3` - with a granularity of 256 bytes.
252
253This information is generated by the CXL driver reading the ACPI CEDT CMFWS.
254
255::
256
257 "decoders:root0":[
258 {
259 "decoder":"decoder0.0",
260 "resource":825975898112,
261 "size":274877906944,
262 "interleave_ways":2,
263 "interleave_granularity":256,
264 "max_available_extent":0,
265 "volatile_capable":true,
266 "nr_targets":2,
267 "targets":[
268 {
269 "target":"pci0000:a8",
270 "alias":"ACPI0016:02",
271 "position":1,
272 "id":4
273 },
274 {
275 "target":"pci0000:d2",
276 "alias":"ACPI0016:00",
277 "position":0,
278 "id":5
279 }
280 ],
281
282Finally we have the `Memory Region` associated with the `Root Decoder`
283:code:`decoder0.0`. This region describes the overall interleave configuration
284of the interleave set.
285
286::
287
288 "regions:decoder0.0":[
289 {
290 "region":"region0",
291 "resource":825975898112,
292 "size":274877906944,
293 "type":"ram",
294 "interleave_ways":2,
295 "interleave_granularity":256,
296 "decode_state":"commit",
297 "mappings":[
298 {
299 "position":1,
300 "memdev":"mem1",
301 "decoder":"decoder6.0"
302 },
303 {
304 "position":0,
305 "memdev":"mem0",
306 "decoder":"decoder5.0"
307 }
308 ]
309 }
310 ]
311 }
312 ]
313 }
314 ]