tjh.dev
Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
1.. SPDX-License-Identifier: GPL-2.0
2
3============================
4Intra-Host-Bridge Interleave
5============================
6This cxl-cli configuration dump shows the following host configuration:
7
8* A single socket system with one CXL root
9* CXL Root has Four (4) CXL Host Bridges
10* One (1) CXL Host Bridges has two CXL Memory Expanders Attached
11* The Host bridge decoder is programmed to interleave across the expanders.
12
13This output is generated by :code:`cxl list -v` and describes the relationships
14between objects exposed in :code:`/sys/bus/cxl/devices/`.
15
16::
17
18 [
19 {
20 "bus":"root0",
21 "provider":"ACPI.CXL",
22 "nr_dports":4,
23 "dports":[
24 {
25 "dport":"pci0000:00",
26 "alias":"ACPI0016:01",
27 "id":0
28 },
29 {
30 "dport":"pci0000:a8",
31 "alias":"ACPI0016:02",
32 "id":4
33 },
34 {
35 "dport":"pci0000:2a",
36 "alias":"ACPI0016:03",
37 "id":1
38 },
39 {
40 "dport":"pci0000:d2",
41 "alias":"ACPI0016:00",
42 "id":5
43 }
44 ],
45
46This chunk shows the CXL "bus" (root0) has 4 downstream ports attached to CXL
47Host Bridges. The `Root` can be considered the singular upstream port attached
48to the platform's memory controller - which routes memory requests to it.
49
50The `ports:root0` section lays out how each of these downstream ports are
51configured. If a port is not configured (id's 0 and 1), they are omitted.
52
53::
54
55 "ports:root0":[
56 {
57 "port":"port1",
58 "host":"pci0000:d2",
59 "depth":1,
60 "nr_dports":3,
61 "dports":[
62 {
63 "dport":"0000:d2:01.1",
64 "alias":"device:02",
65 "id":0
66 },
67 {
68 "dport":"0000:d2:01.3",
69 "alias":"device:05",
70 "id":2
71 },
72 {
73 "dport":"0000:d2:07.1",
74 "alias":"device:0d",
75 "id":113
76 }
77 ],
78
79This chunk shows the available downstream ports associated with the CXL Host
80Bridge :code:`port1`. In this case, :code:`port1` has 3 available downstream
81ports: :code:`dport1`, :code:`dport2`, and :code:`dport113`..
82
83::
84
85 "endpoints:port1":[
86 {
87 "endpoint":"endpoint5",
88 "host":"mem0",
89 "parent_dport":"0000:d2:01.1",
90 "depth":2,
91 "memdev":{
92 "memdev":"mem0",
93 "ram_size":137438953472,
94 "serial":0,
95 "numa_node":0,
96 "host":"0000:d3:00.0"
97 },
98 "decoders:endpoint5":[
99 {
100 "decoder":"decoder5.0",
101 "resource":825975898112,
102 "size":274877906944,
103 "interleave_ways":2,
104 "interleave_granularity":256,
105 "region":"region0",
106 "dpa_resource":0,
107 "dpa_size":137438953472,
108 "mode":"ram"
109 }
110 ]
111 },
112 {
113 "endpoint":"endpoint6",
114 "host":"mem1",
115 "parent_dport":"0000:d2:01.3,
116 "depth":2,
117 "memdev":{
118 "memdev":"mem1",
119 "ram_size":137438953472,
120 "serial":0,
121 "numa_node":0,
122 "host":"0000:a9:00.0"
123 },
124 "decoders:endpoint6":[
125 {
126 "decoder":"decoder6.0",
127 "resource":825975898112,
128 "size":274877906944,
129 "interleave_ways":2,
130 "interleave_granularity":256,
131 "region":"region0",
132 "dpa_resource":0,
133 "dpa_size":137438953472,
134 "mode":"ram"
135 }
136 ]
137 }
138 ],
139
140This chunk shows the endpoints attached to the host bridge :code:`port1`.
141
142:code:`endpoint5` contains a single configured decoder :code:`decoder5.0`
143which has the same interleave configuration memory region they belong to
144(show later).
145
146Next we have the decoders belonging to the host bridge:
147
148::
149
150 "decoders:port1":[
151 {
152 "decoder":"decoder1.0",
153 "resource":825975898112,
154 "size":274877906944,
155 "interleave_ways":2,
156 "interleave_granularity":256,
157 "region":"region0",
158 "nr_targets":2,
159 "targets":[
160 {
161 "target":"0000:d2:01.1",
162 "alias":"device:02",
163 "position":0,
164 "id":0
165 },
166 {
167 "target":"0000:d2:01.3",
168 "alias":"device:05",
169 "position":1,
170 "id":0
171 }
172 ]
173 }
174 ]
175 },
176
177Host Bridge :code:`port1` has a single decoder (:code:`decoder1.0`) with two
178targets: :code:`dport1` and :code:`dport3` - which are attached to
179:code:`endpoint5` and :code:`endpoint6` respectively.
180
181The host bridge decoder interleaves these devices at a 256 byte granularity.
182
183The next chunk shows the three CXL host bridges without attached endpoints.
184
185::
186
187 {
188 "port":"port2",
189 "host":"pci0000:00",
190 "depth":1,
191 "nr_dports":2,
192 "dports":[
193 {
194 "dport":"0000:00:01.3",
195 "alias":"device:55",
196 "id":2
197 },
198 {
199 "dport":"0000:00:07.1",
200 "alias":"device:5d",
201 "id":113
202 }
203 ]
204 },
205 {
206 "port":"port3",
207 "host":"pci0000:a8",
208 "depth":1,
209 "nr_dports":1,
210 "dports":[
211 {
212 "dport":"0000:a8:01.1",
213 "alias":"device:c3",
214 "id":0
215 }
216 ],
217 },
218 {
219 "port":"port4",
220 "host":"pci0000:2a",
221 "depth":1,
222 "nr_dports":1,
223 "dports":[
224 {
225 "dport":"0000:2a:01.1",
226 "alias":"device:d0",
227 "id":0
228 }
229 ]
230 }
231 ],
232
233Next we have the `Root Decoders` belonging to :code:`root0`. This root decoder
234applies the interleave across the downstream ports :code:`port1` and
235:code:`port3` - with a granularity of 256 bytes.
236
237This information is generated by the CXL driver reading the ACPI CEDT CMFWS.
238
239::
240
241 "decoders:root0":[
242 {
243 "decoder":"decoder0.0",
244 "resource":825975898112,
245 "size":274877906944,
246 "interleave_ways":1,
247 "max_available_extent":0,
248 "volatile_capable":true,
249 "nr_targets":2,
250 "targets":[
251 {
252 "target":"pci0000:a8",
253 "alias":"ACPI0016:02",
254 "position":1,
255 "id":4
256 },
257 ],
258
259Finally we have the `Memory Region` associated with the `Root Decoder`
260:code:`decoder0.0`. This region describes the overall interleave configuration
261of the interleave set.
262
263::
264
265 "regions:decoder0.0":[
266 {
267 "region":"region0",
268 "resource":825975898112,
269 "size":274877906944,
270 "type":"ram",
271 "interleave_ways":2,
272 "interleave_granularity":256,
273 "decode_state":"commit",
274 "mappings":[
275 {
276 "position":1,
277 "memdev":"mem1",
278 "decoder":"decoder6.0"
279 },
280 {
281 "position":0,
282 "memdev":"mem0",
283 "decoder":"decoder5.0"
284 }
285 ]
286 }
287 ]
288 }
289 ]
290 }
291 ]