]>
Commit | Line | Data |
---|---|---|
a6bed7a5 LG |
1 | // SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause) |
2 | ||
3 | #include <linux/module.h> | |
4 | #include <linux/debugfs.h> | |
5 | #include <linux/ntb.h> | |
6 | #include <linux/pci.h> | |
7 | #include <linux/radix-tree.h> | |
8 | #include <linux/workqueue.h> | |
9 | ||
10 | MODULE_LICENSE("Dual BSD/GPL"); | |
11 | MODULE_VERSION("0.1"); | |
12 | MODULE_AUTHOR("Logan Gunthorpe <logang@deltatee.com>"); | |
13 | MODULE_DESCRIPTION("Test for sending MSI interrupts over an NTB memory window"); | |
14 | ||
15 | static int num_irqs = 4; | |
16 | module_param(num_irqs, int, 0644); | |
17 | MODULE_PARM_DESC(num_irqs, "number of irqs to use"); | |
18 | ||
19 | struct ntb_msit_ctx { | |
20 | struct ntb_dev *ntb; | |
21 | struct dentry *dbgfs_dir; | |
22 | struct work_struct setup_work; | |
23 | ||
24 | struct ntb_msit_isr_ctx { | |
25 | int irq_idx; | |
26 | int irq_num; | |
27 | int occurrences; | |
28 | struct ntb_msit_ctx *nm; | |
29 | struct ntb_msi_desc desc; | |
30 | } *isr_ctx; | |
31 | ||
32 | struct ntb_msit_peer { | |
33 | struct ntb_msit_ctx *nm; | |
34 | int pidx; | |
35 | int num_irqs; | |
36 | struct completion init_comp; | |
37 | struct ntb_msi_desc *msi_desc; | |
38 | } peers[]; | |
39 | }; | |
40 | ||
41 | static struct dentry *ntb_msit_dbgfs_topdir; | |
42 | ||
43 | static irqreturn_t ntb_msit_isr(int irq, void *dev) | |
44 | { | |
45 | struct ntb_msit_isr_ctx *isr_ctx = dev; | |
46 | struct ntb_msit_ctx *nm = isr_ctx->nm; | |
47 | ||
48 | dev_dbg(&nm->ntb->dev, "Interrupt Occurred: %d", | |
49 | isr_ctx->irq_idx); | |
50 | ||
51 | isr_ctx->occurrences++; | |
52 | ||
53 | return IRQ_HANDLED; | |
54 | } | |
55 | ||
56 | static void ntb_msit_setup_work(struct work_struct *work) | |
57 | { | |
58 | struct ntb_msit_ctx *nm = container_of(work, struct ntb_msit_ctx, | |
59 | setup_work); | |
60 | int irq_count = 0; | |
61 | int irq; | |
62 | int ret; | |
63 | uintptr_t i; | |
64 | ||
65 | ret = ntb_msi_setup_mws(nm->ntb); | |
66 | if (ret) { | |
67 | dev_err(&nm->ntb->dev, "Unable to setup MSI windows: %d\n", | |
68 | ret); | |
69 | return; | |
70 | } | |
71 | ||
72 | for (i = 0; i < num_irqs; i++) { | |
73 | nm->isr_ctx[i].irq_idx = i; | |
74 | nm->isr_ctx[i].nm = nm; | |
75 | ||
76 | if (!nm->isr_ctx[i].irq_num) { | |
77 | irq = ntbm_msi_request_irq(nm->ntb, ntb_msit_isr, | |
78 | KBUILD_MODNAME, | |
79 | &nm->isr_ctx[i], | |
80 | &nm->isr_ctx[i].desc); | |
81 | if (irq < 0) | |
82 | break; | |
83 | ||
84 | nm->isr_ctx[i].irq_num = irq; | |
85 | } | |
86 | ||
87 | ret = ntb_spad_write(nm->ntb, 2 * i + 1, | |
88 | nm->isr_ctx[i].desc.addr_offset); | |
89 | if (ret) | |
90 | break; | |
91 | ||
92 | ret = ntb_spad_write(nm->ntb, 2 * i + 2, | |
93 | nm->isr_ctx[i].desc.data); | |
94 | if (ret) | |
95 | break; | |
96 | ||
97 | irq_count++; | |
98 | } | |
99 | ||
100 | ntb_spad_write(nm->ntb, 0, irq_count); | |
101 | ntb_peer_db_set(nm->ntb, BIT(ntb_port_number(nm->ntb))); | |
102 | } | |
103 | ||
104 | static void ntb_msit_desc_changed(void *ctx) | |
105 | { | |
106 | struct ntb_msit_ctx *nm = ctx; | |
107 | int i; | |
108 | ||
109 | dev_dbg(&nm->ntb->dev, "MSI Descriptors Changed\n"); | |
110 | ||
111 | for (i = 0; i < num_irqs; i++) { | |
112 | ntb_spad_write(nm->ntb, 2 * i + 1, | |
113 | nm->isr_ctx[i].desc.addr_offset); | |
114 | ntb_spad_write(nm->ntb, 2 * i + 2, | |
115 | nm->isr_ctx[i].desc.data); | |
116 | } | |
117 | ||
118 | ntb_peer_db_set(nm->ntb, BIT(ntb_port_number(nm->ntb))); | |
119 | } | |
120 | ||
121 | static void ntb_msit_link_event(void *ctx) | |
122 | { | |
123 | struct ntb_msit_ctx *nm = ctx; | |
124 | ||
125 | if (!ntb_link_is_up(nm->ntb, NULL, NULL)) | |
126 | return; | |
127 | ||
128 | schedule_work(&nm->setup_work); | |
129 | } | |
130 | ||
131 | static void ntb_msit_copy_peer_desc(struct ntb_msit_ctx *nm, int peer) | |
132 | { | |
133 | int i; | |
134 | struct ntb_msi_desc *desc = nm->peers[peer].msi_desc; | |
135 | int irq_count = nm->peers[peer].num_irqs; | |
136 | ||
137 | for (i = 0; i < irq_count; i++) { | |
138 | desc[i].addr_offset = ntb_peer_spad_read(nm->ntb, peer, | |
139 | 2 * i + 1); | |
140 | desc[i].data = ntb_peer_spad_read(nm->ntb, peer, 2 * i + 2); | |
141 | } | |
142 | ||
143 | dev_info(&nm->ntb->dev, "Found %d interrupts on peer %d\n", | |
144 | irq_count, peer); | |
145 | ||
146 | complete_all(&nm->peers[peer].init_comp); | |
147 | } | |
148 | ||
149 | static void ntb_msit_db_event(void *ctx, int vec) | |
150 | { | |
151 | struct ntb_msit_ctx *nm = ctx; | |
152 | struct ntb_msi_desc *desc; | |
153 | u64 peer_mask = ntb_db_read(nm->ntb); | |
154 | u32 irq_count; | |
155 | int peer; | |
156 | ||
157 | ntb_db_clear(nm->ntb, peer_mask); | |
158 | ||
159 | for (peer = 0; peer < sizeof(peer_mask) * 8; peer++) { | |
160 | if (!(peer_mask & BIT(peer))) | |
161 | continue; | |
162 | ||
163 | irq_count = ntb_peer_spad_read(nm->ntb, peer, 0); | |
164 | if (irq_count == -1) | |
165 | continue; | |
166 | ||
167 | desc = kcalloc(irq_count, sizeof(*desc), GFP_ATOMIC); | |
168 | if (!desc) | |
169 | continue; | |
170 | ||
171 | kfree(nm->peers[peer].msi_desc); | |
172 | nm->peers[peer].msi_desc = desc; | |
173 | nm->peers[peer].num_irqs = irq_count; | |
174 | ||
175 | ntb_msit_copy_peer_desc(nm, peer); | |
176 | } | |
177 | } | |
178 | ||
179 | static const struct ntb_ctx_ops ntb_msit_ops = { | |
180 | .link_event = ntb_msit_link_event, | |
181 | .db_event = ntb_msit_db_event, | |
182 | }; | |
183 | ||
184 | static int ntb_msit_dbgfs_trigger(void *data, u64 idx) | |
185 | { | |
186 | struct ntb_msit_peer *peer = data; | |
187 | ||
188 | if (idx >= peer->num_irqs) | |
189 | return -EINVAL; | |
190 | ||
191 | dev_dbg(&peer->nm->ntb->dev, "trigger irq %llu on peer %u\n", | |
192 | idx, peer->pidx); | |
193 | ||
194 | return ntb_msi_peer_trigger(peer->nm->ntb, peer->pidx, | |
195 | &peer->msi_desc[idx]); | |
196 | } | |
197 | ||
198 | DEFINE_DEBUGFS_ATTRIBUTE(ntb_msit_trigger_fops, NULL, | |
199 | ntb_msit_dbgfs_trigger, "%llu\n"); | |
200 | ||
201 | static int ntb_msit_dbgfs_port_get(void *data, u64 *port) | |
202 | { | |
203 | struct ntb_msit_peer *peer = data; | |
204 | ||
205 | *port = ntb_peer_port_number(peer->nm->ntb, peer->pidx); | |
206 | ||
207 | return 0; | |
208 | } | |
209 | ||
210 | DEFINE_DEBUGFS_ATTRIBUTE(ntb_msit_port_fops, ntb_msit_dbgfs_port_get, | |
211 | NULL, "%llu\n"); | |
212 | ||
213 | static int ntb_msit_dbgfs_count_get(void *data, u64 *count) | |
214 | { | |
215 | struct ntb_msit_peer *peer = data; | |
216 | ||
217 | *count = peer->num_irqs; | |
218 | ||
219 | return 0; | |
220 | } | |
221 | ||
222 | DEFINE_DEBUGFS_ATTRIBUTE(ntb_msit_count_fops, ntb_msit_dbgfs_count_get, | |
223 | NULL, "%llu\n"); | |
224 | ||
225 | static int ntb_msit_dbgfs_ready_get(void *data, u64 *ready) | |
226 | { | |
227 | struct ntb_msit_peer *peer = data; | |
228 | ||
229 | *ready = try_wait_for_completion(&peer->init_comp); | |
230 | ||
231 | return 0; | |
232 | } | |
233 | ||
234 | static int ntb_msit_dbgfs_ready_set(void *data, u64 ready) | |
235 | { | |
236 | struct ntb_msit_peer *peer = data; | |
237 | ||
238 | return wait_for_completion_interruptible(&peer->init_comp); | |
239 | } | |
240 | ||
241 | DEFINE_DEBUGFS_ATTRIBUTE(ntb_msit_ready_fops, ntb_msit_dbgfs_ready_get, | |
242 | ntb_msit_dbgfs_ready_set, "%llu\n"); | |
243 | ||
244 | static int ntb_msit_dbgfs_occurrences_get(void *data, u64 *occurrences) | |
245 | { | |
246 | struct ntb_msit_isr_ctx *isr_ctx = data; | |
247 | ||
248 | *occurrences = isr_ctx->occurrences; | |
249 | ||
250 | return 0; | |
251 | } | |
252 | ||
253 | DEFINE_DEBUGFS_ATTRIBUTE(ntb_msit_occurrences_fops, | |
254 | ntb_msit_dbgfs_occurrences_get, | |
255 | NULL, "%llu\n"); | |
256 | ||
257 | static int ntb_msit_dbgfs_local_port_get(void *data, u64 *port) | |
258 | { | |
259 | struct ntb_msit_ctx *nm = data; | |
260 | ||
261 | *port = ntb_port_number(nm->ntb); | |
262 | ||
263 | return 0; | |
264 | } | |
265 | ||
266 | DEFINE_DEBUGFS_ATTRIBUTE(ntb_msit_local_port_fops, | |
267 | ntb_msit_dbgfs_local_port_get, | |
268 | NULL, "%llu\n"); | |
269 | ||
270 | static void ntb_msit_create_dbgfs(struct ntb_msit_ctx *nm) | |
271 | { | |
272 | struct pci_dev *pdev = nm->ntb->pdev; | |
273 | char buf[32]; | |
274 | int i; | |
275 | struct dentry *peer_dir; | |
276 | ||
277 | nm->dbgfs_dir = debugfs_create_dir(pci_name(pdev), | |
278 | ntb_msit_dbgfs_topdir); | |
279 | debugfs_create_file("port", 0400, nm->dbgfs_dir, nm, | |
280 | &ntb_msit_local_port_fops); | |
281 | ||
282 | for (i = 0; i < ntb_peer_port_count(nm->ntb); i++) { | |
283 | nm->peers[i].pidx = i; | |
284 | nm->peers[i].nm = nm; | |
285 | init_completion(&nm->peers[i].init_comp); | |
286 | ||
287 | snprintf(buf, sizeof(buf), "peer%d", i); | |
288 | peer_dir = debugfs_create_dir(buf, nm->dbgfs_dir); | |
289 | ||
290 | debugfs_create_file_unsafe("trigger", 0200, peer_dir, | |
291 | &nm->peers[i], | |
292 | &ntb_msit_trigger_fops); | |
293 | ||
294 | debugfs_create_file_unsafe("port", 0400, peer_dir, | |
295 | &nm->peers[i], &ntb_msit_port_fops); | |
296 | ||
297 | debugfs_create_file_unsafe("count", 0400, peer_dir, | |
298 | &nm->peers[i], | |
299 | &ntb_msit_count_fops); | |
300 | ||
301 | debugfs_create_file_unsafe("ready", 0600, peer_dir, | |
302 | &nm->peers[i], | |
303 | &ntb_msit_ready_fops); | |
304 | } | |
305 | ||
306 | for (i = 0; i < num_irqs; i++) { | |
307 | snprintf(buf, sizeof(buf), "irq%d_occurrences", i); | |
308 | debugfs_create_file_unsafe(buf, 0400, nm->dbgfs_dir, | |
309 | &nm->isr_ctx[i], | |
310 | &ntb_msit_occurrences_fops); | |
311 | } | |
312 | } | |
313 | ||
314 | static void ntb_msit_remove_dbgfs(struct ntb_msit_ctx *nm) | |
315 | { | |
316 | debugfs_remove_recursive(nm->dbgfs_dir); | |
317 | } | |
318 | ||
319 | static int ntb_msit_probe(struct ntb_client *client, struct ntb_dev *ntb) | |
320 | { | |
321 | struct ntb_msit_ctx *nm; | |
a6bed7a5 LG |
322 | int peers; |
323 | int ret; | |
324 | ||
325 | peers = ntb_peer_port_count(ntb); | |
326 | if (peers <= 0) | |
327 | return -EINVAL; | |
328 | ||
329 | if (ntb_spad_is_unsafe(ntb) || ntb_spad_count(ntb) < 2 * num_irqs + 1) { | |
330 | dev_err(&ntb->dev, "NTB MSI test requires at least %d spads for %d irqs\n", | |
331 | 2 * num_irqs + 1, num_irqs); | |
332 | return -EFAULT; | |
333 | } | |
334 | ||
335 | ret = ntb_spad_write(ntb, 0, -1); | |
336 | if (ret) { | |
337 | dev_err(&ntb->dev, "Unable to write spads: %d\n", ret); | |
338 | return ret; | |
339 | } | |
340 | ||
341 | ret = ntb_db_clear_mask(ntb, GENMASK(peers - 1, 0)); | |
342 | if (ret) { | |
343 | dev_err(&ntb->dev, "Unable to clear doorbell mask: %d\n", ret); | |
344 | return ret; | |
345 | } | |
346 | ||
347 | ret = ntb_msi_init(ntb, ntb_msit_desc_changed); | |
348 | if (ret) { | |
349 | dev_err(&ntb->dev, "Unable to initialize MSI library: %d\n", | |
350 | ret); | |
351 | return ret; | |
352 | } | |
353 | ||
b8e2c8bb | 354 | nm = devm_kzalloc(&ntb->dev, struct_size(nm, peers, peers), GFP_KERNEL); |
a6bed7a5 LG |
355 | if (!nm) |
356 | return -ENOMEM; | |
357 | ||
358 | nm->isr_ctx = devm_kcalloc(&ntb->dev, num_irqs, sizeof(*nm->isr_ctx), | |
359 | GFP_KERNEL); | |
360 | if (!nm->isr_ctx) | |
361 | return -ENOMEM; | |
362 | ||
363 | INIT_WORK(&nm->setup_work, ntb_msit_setup_work); | |
364 | nm->ntb = ntb; | |
365 | ||
366 | ntb_msit_create_dbgfs(nm); | |
367 | ||
368 | ret = ntb_set_ctx(ntb, nm, &ntb_msit_ops); | |
369 | if (ret) | |
370 | goto remove_dbgfs; | |
371 | ||
319f83ac YL |
372 | if (!nm->isr_ctx) { |
373 | ret = -ENOMEM; | |
a6bed7a5 | 374 | goto remove_dbgfs; |
319f83ac | 375 | } |
a6bed7a5 LG |
376 | |
377 | ntb_link_enable(ntb, NTB_SPEED_AUTO, NTB_WIDTH_AUTO); | |
378 | ||
379 | return 0; | |
380 | ||
381 | remove_dbgfs: | |
382 | ntb_msit_remove_dbgfs(nm); | |
383 | devm_kfree(&ntb->dev, nm->isr_ctx); | |
384 | devm_kfree(&ntb->dev, nm); | |
385 | return ret; | |
386 | } | |
387 | ||
388 | static void ntb_msit_remove(struct ntb_client *client, struct ntb_dev *ntb) | |
389 | { | |
390 | struct ntb_msit_ctx *nm = ntb->ctx; | |
391 | int i; | |
392 | ||
393 | ntb_link_disable(ntb); | |
394 | ntb_db_set_mask(ntb, ntb_db_valid_mask(ntb)); | |
395 | ntb_msi_clear_mws(ntb); | |
396 | ||
397 | for (i = 0; i < ntb_peer_port_count(ntb); i++) | |
398 | kfree(nm->peers[i].msi_desc); | |
399 | ||
400 | ntb_clear_ctx(ntb); | |
401 | ntb_msit_remove_dbgfs(nm); | |
402 | } | |
403 | ||
404 | static struct ntb_client ntb_msit_client = { | |
405 | .ops = { | |
406 | .probe = ntb_msit_probe, | |
407 | .remove = ntb_msit_remove | |
408 | } | |
409 | }; | |
410 | ||
411 | static int __init ntb_msit_init(void) | |
412 | { | |
413 | int ret; | |
414 | ||
415 | if (debugfs_initialized()) | |
416 | ntb_msit_dbgfs_topdir = debugfs_create_dir(KBUILD_MODNAME, | |
417 | NULL); | |
418 | ||
419 | ret = ntb_register_client(&ntb_msit_client); | |
420 | if (ret) | |
421 | debugfs_remove_recursive(ntb_msit_dbgfs_topdir); | |
422 | ||
423 | return ret; | |
424 | } | |
425 | module_init(ntb_msit_init); | |
426 | ||
427 | static void __exit ntb_msit_exit(void) | |
428 | { | |
429 | ntb_unregister_client(&ntb_msit_client); | |
430 | debugfs_remove_recursive(ntb_msit_dbgfs_topdir); | |
431 | } | |
432 | module_exit(ntb_msit_exit); |