]>
Commit | Line | Data |
---|---|---|
1a58e196 SN |
1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* Copyright(c) 2017 - 2019 Pensando Systems, Inc */ | |
3 | ||
4 | #include <linux/netdevice.h> | |
5 | #include <linux/etherdevice.h> | |
6 | #include <linux/interrupt.h> | |
7 | #include <linux/pci.h> | |
8 | #include <linux/cpumask.h> | |
9 | ||
10 | #include "ionic.h" | |
11 | #include "ionic_bus.h" | |
12 | #include "ionic_lif.h" | |
13 | #include "ionic_debugfs.h" | |
14 | ||
1d062b7b SN |
15 | static irqreturn_t ionic_isr(int irq, void *data) |
16 | { | |
17 | struct napi_struct *napi = data; | |
18 | ||
19 | napi_schedule_irqoff(napi); | |
20 | ||
21 | return IRQ_HANDLED; | |
22 | } | |
23 | ||
24 | static int ionic_request_irq(struct ionic_lif *lif, struct ionic_qcq *qcq) | |
25 | { | |
26 | struct ionic_intr_info *intr = &qcq->intr; | |
27 | struct device *dev = lif->ionic->dev; | |
28 | struct ionic_queue *q = &qcq->q; | |
29 | const char *name; | |
30 | ||
31 | if (lif->registered) | |
32 | name = lif->netdev->name; | |
33 | else | |
34 | name = dev_name(dev); | |
35 | ||
36 | snprintf(intr->name, sizeof(intr->name), | |
37 | "%s-%s-%s", IONIC_DRV_NAME, name, q->name); | |
38 | ||
39 | return devm_request_irq(dev, intr->vector, ionic_isr, | |
40 | 0, intr->name, &qcq->napi); | |
41 | } | |
42 | ||
43 | static int ionic_intr_alloc(struct ionic_lif *lif, struct ionic_intr_info *intr) | |
44 | { | |
45 | struct ionic *ionic = lif->ionic; | |
46 | int index; | |
47 | ||
48 | index = find_first_zero_bit(ionic->intrs, ionic->nintrs); | |
49 | if (index == ionic->nintrs) { | |
50 | netdev_warn(lif->netdev, "%s: no intr, index=%d nintrs=%d\n", | |
51 | __func__, index, ionic->nintrs); | |
52 | return -ENOSPC; | |
53 | } | |
54 | ||
55 | set_bit(index, ionic->intrs); | |
56 | ionic_intr_init(&ionic->idev, intr, index); | |
57 | ||
58 | return 0; | |
59 | } | |
60 | ||
61 | static void ionic_intr_free(struct ionic_lif *lif, int index) | |
62 | { | |
63 | if (index != INTR_INDEX_NOT_ASSIGNED && index < lif->ionic->nintrs) | |
64 | clear_bit(index, lif->ionic->intrs); | |
65 | } | |
66 | ||
67 | static void ionic_lif_qcq_deinit(struct ionic_lif *lif, struct ionic_qcq *qcq) | |
68 | { | |
69 | struct ionic_dev *idev = &lif->ionic->idev; | |
70 | struct device *dev = lif->ionic->dev; | |
71 | ||
72 | if (!qcq) | |
73 | return; | |
74 | ||
75 | ionic_debugfs_del_qcq(qcq); | |
76 | ||
77 | if (!(qcq->flags & IONIC_QCQ_F_INITED)) | |
78 | return; | |
79 | ||
80 | if (qcq->flags & IONIC_QCQ_F_INTR) { | |
81 | ionic_intr_mask(idev->intr_ctrl, qcq->intr.index, | |
82 | IONIC_INTR_MASK_SET); | |
83 | devm_free_irq(dev, qcq->intr.vector, &qcq->napi); | |
84 | netif_napi_del(&qcq->napi); | |
85 | } | |
86 | ||
87 | qcq->flags &= ~IONIC_QCQ_F_INITED; | |
88 | } | |
89 | ||
90 | static void ionic_qcq_free(struct ionic_lif *lif, struct ionic_qcq *qcq) | |
91 | { | |
92 | struct device *dev = lif->ionic->dev; | |
93 | ||
94 | if (!qcq) | |
95 | return; | |
96 | ||
97 | dma_free_coherent(dev, qcq->total_size, qcq->base, qcq->base_pa); | |
98 | qcq->base = NULL; | |
99 | qcq->base_pa = 0; | |
100 | ||
101 | if (qcq->flags & IONIC_QCQ_F_INTR) | |
102 | ionic_intr_free(lif, qcq->intr.index); | |
103 | ||
104 | devm_kfree(dev, qcq->cq.info); | |
105 | qcq->cq.info = NULL; | |
106 | devm_kfree(dev, qcq->q.info); | |
107 | qcq->q.info = NULL; | |
108 | devm_kfree(dev, qcq); | |
109 | } | |
110 | ||
111 | static void ionic_qcqs_free(struct ionic_lif *lif) | |
112 | { | |
77ceb68e SN |
113 | if (lif->notifyqcq) { |
114 | ionic_qcq_free(lif, lif->notifyqcq); | |
115 | lif->notifyqcq = NULL; | |
116 | } | |
117 | ||
1d062b7b SN |
118 | if (lif->adminqcq) { |
119 | ionic_qcq_free(lif, lif->adminqcq); | |
120 | lif->adminqcq = NULL; | |
121 | } | |
122 | } | |
123 | ||
77ceb68e SN |
124 | static void ionic_link_qcq_interrupts(struct ionic_qcq *src_qcq, |
125 | struct ionic_qcq *n_qcq) | |
126 | { | |
127 | if (WARN_ON(n_qcq->flags & IONIC_QCQ_F_INTR)) { | |
128 | ionic_intr_free(n_qcq->cq.lif, n_qcq->intr.index); | |
129 | n_qcq->flags &= ~IONIC_QCQ_F_INTR; | |
130 | } | |
131 | ||
132 | n_qcq->intr.vector = src_qcq->intr.vector; | |
133 | n_qcq->intr.index = src_qcq->intr.index; | |
134 | } | |
135 | ||
1d062b7b SN |
136 | static int ionic_qcq_alloc(struct ionic_lif *lif, unsigned int type, |
137 | unsigned int index, | |
138 | const char *name, unsigned int flags, | |
139 | unsigned int num_descs, unsigned int desc_size, | |
140 | unsigned int cq_desc_size, | |
141 | unsigned int sg_desc_size, | |
142 | unsigned int pid, struct ionic_qcq **qcq) | |
143 | { | |
144 | struct ionic_dev *idev = &lif->ionic->idev; | |
145 | u32 q_size, cq_size, sg_size, total_size; | |
146 | struct device *dev = lif->ionic->dev; | |
147 | void *q_base, *cq_base, *sg_base; | |
148 | dma_addr_t cq_base_pa = 0; | |
149 | dma_addr_t sg_base_pa = 0; | |
150 | dma_addr_t q_base_pa = 0; | |
151 | struct ionic_qcq *new; | |
152 | int err; | |
153 | ||
154 | *qcq = NULL; | |
155 | ||
156 | q_size = num_descs * desc_size; | |
157 | cq_size = num_descs * cq_desc_size; | |
158 | sg_size = num_descs * sg_desc_size; | |
159 | ||
160 | total_size = ALIGN(q_size, PAGE_SIZE) + ALIGN(cq_size, PAGE_SIZE); | |
161 | /* Note: aligning q_size/cq_size is not enough due to cq_base | |
162 | * address aligning as q_base could be not aligned to the page. | |
163 | * Adding PAGE_SIZE. | |
164 | */ | |
165 | total_size += PAGE_SIZE; | |
166 | if (flags & IONIC_QCQ_F_SG) { | |
167 | total_size += ALIGN(sg_size, PAGE_SIZE); | |
168 | total_size += PAGE_SIZE; | |
169 | } | |
170 | ||
171 | new = devm_kzalloc(dev, sizeof(*new), GFP_KERNEL); | |
172 | if (!new) { | |
173 | netdev_err(lif->netdev, "Cannot allocate queue structure\n"); | |
174 | err = -ENOMEM; | |
175 | goto err_out; | |
176 | } | |
177 | ||
178 | new->flags = flags; | |
179 | ||
180 | new->q.info = devm_kzalloc(dev, sizeof(*new->q.info) * num_descs, | |
181 | GFP_KERNEL); | |
182 | if (!new->q.info) { | |
183 | netdev_err(lif->netdev, "Cannot allocate queue info\n"); | |
184 | err = -ENOMEM; | |
185 | goto err_out; | |
186 | } | |
187 | ||
188 | new->q.type = type; | |
189 | ||
190 | err = ionic_q_init(lif, idev, &new->q, index, name, num_descs, | |
191 | desc_size, sg_desc_size, pid); | |
192 | if (err) { | |
193 | netdev_err(lif->netdev, "Cannot initialize queue\n"); | |
194 | goto err_out; | |
195 | } | |
196 | ||
197 | if (flags & IONIC_QCQ_F_INTR) { | |
198 | err = ionic_intr_alloc(lif, &new->intr); | |
199 | if (err) { | |
200 | netdev_warn(lif->netdev, "no intr for %s: %d\n", | |
201 | name, err); | |
202 | goto err_out; | |
203 | } | |
204 | ||
205 | err = ionic_bus_get_irq(lif->ionic, new->intr.index); | |
206 | if (err < 0) { | |
207 | netdev_warn(lif->netdev, "no vector for %s: %d\n", | |
208 | name, err); | |
209 | goto err_out_free_intr; | |
210 | } | |
211 | new->intr.vector = err; | |
212 | ionic_intr_mask_assert(idev->intr_ctrl, new->intr.index, | |
213 | IONIC_INTR_MASK_SET); | |
214 | ||
215 | new->intr.cpu = new->intr.index % num_online_cpus(); | |
216 | if (cpu_online(new->intr.cpu)) | |
217 | cpumask_set_cpu(new->intr.cpu, | |
218 | &new->intr.affinity_mask); | |
219 | } else { | |
220 | new->intr.index = INTR_INDEX_NOT_ASSIGNED; | |
221 | } | |
222 | ||
223 | new->cq.info = devm_kzalloc(dev, sizeof(*new->cq.info) * num_descs, | |
224 | GFP_KERNEL); | |
225 | if (!new->cq.info) { | |
226 | netdev_err(lif->netdev, "Cannot allocate completion queue info\n"); | |
227 | err = -ENOMEM; | |
228 | goto err_out_free_intr; | |
229 | } | |
230 | ||
231 | err = ionic_cq_init(lif, &new->cq, &new->intr, num_descs, cq_desc_size); | |
232 | if (err) { | |
233 | netdev_err(lif->netdev, "Cannot initialize completion queue\n"); | |
234 | goto err_out_free_intr; | |
235 | } | |
236 | ||
237 | new->base = dma_alloc_coherent(dev, total_size, &new->base_pa, | |
238 | GFP_KERNEL); | |
239 | if (!new->base) { | |
240 | netdev_err(lif->netdev, "Cannot allocate queue DMA memory\n"); | |
241 | err = -ENOMEM; | |
242 | goto err_out_free_intr; | |
243 | } | |
244 | ||
245 | new->total_size = total_size; | |
246 | ||
247 | q_base = new->base; | |
248 | q_base_pa = new->base_pa; | |
249 | ||
250 | cq_base = (void *)ALIGN((uintptr_t)q_base + q_size, PAGE_SIZE); | |
251 | cq_base_pa = ALIGN(q_base_pa + q_size, PAGE_SIZE); | |
252 | ||
253 | if (flags & IONIC_QCQ_F_SG) { | |
254 | sg_base = (void *)ALIGN((uintptr_t)cq_base + cq_size, | |
255 | PAGE_SIZE); | |
256 | sg_base_pa = ALIGN(cq_base_pa + cq_size, PAGE_SIZE); | |
257 | ionic_q_sg_map(&new->q, sg_base, sg_base_pa); | |
258 | } | |
259 | ||
260 | ionic_q_map(&new->q, q_base, q_base_pa); | |
261 | ionic_cq_map(&new->cq, cq_base, cq_base_pa); | |
262 | ionic_cq_bind(&new->cq, &new->q); | |
263 | ||
264 | *qcq = new; | |
265 | ||
266 | return 0; | |
267 | ||
268 | err_out_free_intr: | |
269 | ionic_intr_free(lif, new->intr.index); | |
270 | err_out: | |
271 | dev_err(dev, "qcq alloc of %s%d failed %d\n", name, index, err); | |
272 | return err; | |
273 | } | |
274 | ||
275 | static int ionic_qcqs_alloc(struct ionic_lif *lif) | |
276 | { | |
277 | unsigned int flags; | |
278 | int err; | |
279 | ||
280 | flags = IONIC_QCQ_F_INTR; | |
281 | err = ionic_qcq_alloc(lif, IONIC_QTYPE_ADMINQ, 0, "admin", flags, | |
282 | IONIC_ADMINQ_LENGTH, | |
283 | sizeof(struct ionic_admin_cmd), | |
284 | sizeof(struct ionic_admin_comp), | |
285 | 0, lif->kern_pid, &lif->adminqcq); | |
286 | if (err) | |
287 | return err; | |
288 | ||
77ceb68e SN |
289 | if (lif->ionic->nnqs_per_lif) { |
290 | flags = IONIC_QCQ_F_NOTIFYQ; | |
291 | err = ionic_qcq_alloc(lif, IONIC_QTYPE_NOTIFYQ, 0, "notifyq", | |
292 | flags, IONIC_NOTIFYQ_LENGTH, | |
293 | sizeof(struct ionic_notifyq_cmd), | |
294 | sizeof(union ionic_notifyq_comp), | |
295 | 0, lif->kern_pid, &lif->notifyqcq); | |
296 | if (err) | |
297 | goto err_out_free_adminqcq; | |
298 | ||
299 | /* Let the notifyq ride on the adminq interrupt */ | |
300 | ionic_link_qcq_interrupts(lif->adminqcq, lif->notifyqcq); | |
301 | } | |
302 | ||
1d062b7b | 303 | return 0; |
77ceb68e SN |
304 | |
305 | err_out_free_adminqcq: | |
306 | ionic_qcq_free(lif, lif->adminqcq); | |
307 | lif->adminqcq = NULL; | |
308 | ||
309 | return err; | |
310 | } | |
311 | ||
312 | static bool ionic_notifyq_service(struct ionic_cq *cq, | |
313 | struct ionic_cq_info *cq_info) | |
314 | { | |
315 | union ionic_notifyq_comp *comp = cq_info->cq_desc; | |
316 | struct net_device *netdev; | |
317 | struct ionic_queue *q; | |
318 | struct ionic_lif *lif; | |
319 | u64 eid; | |
320 | ||
321 | q = cq->bound_q; | |
322 | lif = q->info[0].cb_arg; | |
323 | netdev = lif->netdev; | |
324 | eid = le64_to_cpu(comp->event.eid); | |
325 | ||
326 | /* Have we run out of new completions to process? */ | |
327 | if (eid <= lif->last_eid) | |
328 | return false; | |
329 | ||
330 | lif->last_eid = eid; | |
331 | ||
332 | dev_dbg(lif->ionic->dev, "notifyq event:\n"); | |
333 | dynamic_hex_dump("event ", DUMP_PREFIX_OFFSET, 16, 1, | |
334 | comp, sizeof(*comp), true); | |
335 | ||
336 | switch (le16_to_cpu(comp->event.ecode)) { | |
337 | case IONIC_EVENT_LINK_CHANGE: | |
338 | netdev_info(netdev, "Notifyq IONIC_EVENT_LINK_CHANGE eid=%lld\n", | |
339 | eid); | |
340 | netdev_info(netdev, | |
341 | " link_status=%d link_speed=%d\n", | |
342 | le16_to_cpu(comp->link_change.link_status), | |
343 | le32_to_cpu(comp->link_change.link_speed)); | |
344 | break; | |
345 | case IONIC_EVENT_RESET: | |
346 | netdev_info(netdev, "Notifyq IONIC_EVENT_RESET eid=%lld\n", | |
347 | eid); | |
348 | netdev_info(netdev, " reset_code=%d state=%d\n", | |
349 | comp->reset.reset_code, | |
350 | comp->reset.state); | |
351 | break; | |
352 | default: | |
353 | netdev_warn(netdev, "Notifyq unknown event ecode=%d eid=%lld\n", | |
354 | comp->event.ecode, eid); | |
355 | break; | |
356 | } | |
357 | ||
358 | return true; | |
359 | } | |
360 | ||
361 | static int ionic_notifyq_clean(struct ionic_lif *lif, int budget) | |
362 | { | |
363 | struct ionic_dev *idev = &lif->ionic->idev; | |
364 | struct ionic_cq *cq = &lif->notifyqcq->cq; | |
365 | u32 work_done; | |
366 | ||
367 | work_done = ionic_cq_service(cq, budget, ionic_notifyq_service, | |
368 | NULL, NULL); | |
369 | if (work_done) | |
370 | ionic_intr_credits(idev->intr_ctrl, cq->bound_intr->index, | |
371 | work_done, IONIC_INTR_CRED_RESET_COALESCE); | |
372 | ||
373 | return work_done; | |
1d062b7b SN |
374 | } |
375 | ||
376 | static bool ionic_adminq_service(struct ionic_cq *cq, | |
377 | struct ionic_cq_info *cq_info) | |
378 | { | |
379 | struct ionic_admin_comp *comp = cq_info->cq_desc; | |
380 | ||
381 | if (!color_match(comp->color, cq->done_color)) | |
382 | return false; | |
383 | ||
384 | ionic_q_service(cq->bound_q, cq_info, le16_to_cpu(comp->comp_index)); | |
385 | ||
386 | return true; | |
387 | } | |
388 | ||
389 | static int ionic_adminq_napi(struct napi_struct *napi, int budget) | |
390 | { | |
77ceb68e SN |
391 | struct ionic_lif *lif = napi_to_cq(napi)->lif; |
392 | int n_work = 0; | |
393 | int a_work = 0; | |
394 | ||
395 | if (likely(lif->notifyqcq && lif->notifyqcq->flags & IONIC_QCQ_F_INITED)) | |
396 | n_work = ionic_notifyq_clean(lif, budget); | |
397 | a_work = ionic_napi(napi, budget, ionic_adminq_service, NULL, NULL); | |
398 | ||
399 | return max(n_work, a_work); | |
1d062b7b SN |
400 | } |
401 | ||
1a58e196 SN |
402 | static struct ionic_lif *ionic_lif_alloc(struct ionic *ionic, unsigned int index) |
403 | { | |
404 | struct device *dev = ionic->dev; | |
405 | struct net_device *netdev; | |
406 | struct ionic_lif *lif; | |
407 | int err; | |
408 | ||
409 | netdev = alloc_etherdev_mqs(sizeof(*lif), | |
410 | ionic->ntxqs_per_lif, ionic->ntxqs_per_lif); | |
411 | if (!netdev) { | |
412 | dev_err(dev, "Cannot allocate netdev, aborting\n"); | |
413 | return ERR_PTR(-ENOMEM); | |
414 | } | |
415 | ||
416 | SET_NETDEV_DEV(netdev, dev); | |
417 | ||
418 | lif = netdev_priv(netdev); | |
419 | lif->netdev = netdev; | |
420 | ||
421 | lif->neqs = ionic->neqs_per_lif; | |
422 | lif->nxqs = ionic->ntxqs_per_lif; | |
423 | ||
424 | lif->ionic = ionic; | |
425 | lif->index = index; | |
426 | ||
427 | snprintf(lif->name, sizeof(lif->name), "lif%u", index); | |
428 | ||
1d062b7b SN |
429 | spin_lock_init(&lif->adminq_lock); |
430 | ||
1a58e196 SN |
431 | /* allocate lif info */ |
432 | lif->info_sz = ALIGN(sizeof(*lif->info), PAGE_SIZE); | |
433 | lif->info = dma_alloc_coherent(dev, lif->info_sz, | |
434 | &lif->info_pa, GFP_KERNEL); | |
435 | if (!lif->info) { | |
436 | dev_err(dev, "Failed to allocate lif info, aborting\n"); | |
437 | err = -ENOMEM; | |
438 | goto err_out_free_netdev; | |
439 | } | |
440 | ||
1d062b7b SN |
441 | /* allocate queues */ |
442 | err = ionic_qcqs_alloc(lif); | |
443 | if (err) | |
444 | goto err_out_free_lif_info; | |
445 | ||
1a58e196 SN |
446 | list_add_tail(&lif->list, &ionic->lifs); |
447 | ||
448 | return lif; | |
449 | ||
1d062b7b SN |
450 | err_out_free_lif_info: |
451 | dma_free_coherent(dev, lif->info_sz, lif->info, lif->info_pa); | |
452 | lif->info = NULL; | |
453 | lif->info_pa = 0; | |
1a58e196 SN |
454 | err_out_free_netdev: |
455 | free_netdev(lif->netdev); | |
456 | lif = NULL; | |
457 | ||
458 | return ERR_PTR(err); | |
459 | } | |
460 | ||
461 | int ionic_lifs_alloc(struct ionic *ionic) | |
462 | { | |
463 | struct ionic_lif *lif; | |
464 | ||
465 | INIT_LIST_HEAD(&ionic->lifs); | |
466 | ||
467 | /* only build the first lif, others are for later features */ | |
468 | set_bit(0, ionic->lifbits); | |
469 | lif = ionic_lif_alloc(ionic, 0); | |
470 | ||
471 | return PTR_ERR_OR_ZERO(lif); | |
472 | } | |
473 | ||
474 | static void ionic_lif_reset(struct ionic_lif *lif) | |
475 | { | |
476 | struct ionic_dev *idev = &lif->ionic->idev; | |
477 | ||
478 | mutex_lock(&lif->ionic->dev_cmd_lock); | |
479 | ionic_dev_cmd_lif_reset(idev, lif->index); | |
480 | ionic_dev_cmd_wait(lif->ionic, DEVCMD_TIMEOUT); | |
481 | mutex_unlock(&lif->ionic->dev_cmd_lock); | |
482 | } | |
483 | ||
484 | static void ionic_lif_free(struct ionic_lif *lif) | |
485 | { | |
486 | struct device *dev = lif->ionic->dev; | |
487 | ||
1d062b7b SN |
488 | /* free queues */ |
489 | ionic_qcqs_free(lif); | |
1a58e196 SN |
490 | ionic_lif_reset(lif); |
491 | ||
492 | /* free lif info */ | |
493 | dma_free_coherent(dev, lif->info_sz, lif->info, lif->info_pa); | |
494 | lif->info = NULL; | |
495 | lif->info_pa = 0; | |
496 | ||
6461b446 SN |
497 | /* unmap doorbell page */ |
498 | ionic_bus_unmap_dbpage(lif->ionic, lif->kern_dbpage); | |
499 | lif->kern_dbpage = NULL; | |
500 | kfree(lif->dbid_inuse); | |
501 | lif->dbid_inuse = NULL; | |
502 | ||
1a58e196 SN |
503 | /* free netdev & lif */ |
504 | ionic_debugfs_del_lif(lif); | |
505 | list_del(&lif->list); | |
506 | free_netdev(lif->netdev); | |
507 | } | |
508 | ||
509 | void ionic_lifs_free(struct ionic *ionic) | |
510 | { | |
511 | struct list_head *cur, *tmp; | |
512 | struct ionic_lif *lif; | |
513 | ||
514 | list_for_each_safe(cur, tmp, &ionic->lifs) { | |
515 | lif = list_entry(cur, struct ionic_lif, list); | |
516 | ||
517 | ionic_lif_free(lif); | |
518 | } | |
519 | } | |
520 | ||
521 | static void ionic_lif_deinit(struct ionic_lif *lif) | |
522 | { | |
523 | if (!test_bit(IONIC_LIF_INITED, lif->state)) | |
524 | return; | |
525 | ||
526 | clear_bit(IONIC_LIF_INITED, lif->state); | |
527 | ||
1d062b7b | 528 | napi_disable(&lif->adminqcq->napi); |
77ceb68e | 529 | ionic_lif_qcq_deinit(lif, lif->notifyqcq); |
1d062b7b SN |
530 | ionic_lif_qcq_deinit(lif, lif->adminqcq); |
531 | ||
1a58e196 SN |
532 | ionic_lif_reset(lif); |
533 | } | |
534 | ||
535 | void ionic_lifs_deinit(struct ionic *ionic) | |
536 | { | |
537 | struct list_head *cur, *tmp; | |
538 | struct ionic_lif *lif; | |
539 | ||
540 | list_for_each_safe(cur, tmp, &ionic->lifs) { | |
541 | lif = list_entry(cur, struct ionic_lif, list); | |
542 | ionic_lif_deinit(lif); | |
543 | } | |
544 | } | |
545 | ||
1d062b7b SN |
546 | static int ionic_lif_adminq_init(struct ionic_lif *lif) |
547 | { | |
548 | struct device *dev = lif->ionic->dev; | |
549 | struct ionic_q_init_comp comp; | |
550 | struct ionic_dev *idev; | |
551 | struct ionic_qcq *qcq; | |
552 | struct ionic_queue *q; | |
553 | int err; | |
554 | ||
555 | idev = &lif->ionic->idev; | |
556 | qcq = lif->adminqcq; | |
557 | q = &qcq->q; | |
558 | ||
559 | mutex_lock(&lif->ionic->dev_cmd_lock); | |
560 | ionic_dev_cmd_adminq_init(idev, qcq, lif->index, qcq->intr.index); | |
561 | err = ionic_dev_cmd_wait(lif->ionic, DEVCMD_TIMEOUT); | |
562 | ionic_dev_cmd_comp(idev, (union ionic_dev_cmd_comp *)&comp); | |
563 | mutex_unlock(&lif->ionic->dev_cmd_lock); | |
564 | if (err) { | |
565 | netdev_err(lif->netdev, "adminq init failed %d\n", err); | |
566 | return err; | |
567 | } | |
568 | ||
569 | q->hw_type = comp.hw_type; | |
570 | q->hw_index = le32_to_cpu(comp.hw_index); | |
571 | q->dbval = IONIC_DBELL_QID(q->hw_index); | |
572 | ||
573 | dev_dbg(dev, "adminq->hw_type %d\n", q->hw_type); | |
574 | dev_dbg(dev, "adminq->hw_index %d\n", q->hw_index); | |
575 | ||
576 | netif_napi_add(lif->netdev, &qcq->napi, ionic_adminq_napi, | |
577 | NAPI_POLL_WEIGHT); | |
578 | ||
579 | err = ionic_request_irq(lif, qcq); | |
580 | if (err) { | |
581 | netdev_warn(lif->netdev, "adminq irq request failed %d\n", err); | |
582 | netif_napi_del(&qcq->napi); | |
583 | return err; | |
584 | } | |
585 | ||
586 | napi_enable(&qcq->napi); | |
587 | ||
588 | if (qcq->flags & IONIC_QCQ_F_INTR) | |
589 | ionic_intr_mask(idev->intr_ctrl, qcq->intr.index, | |
590 | IONIC_INTR_MASK_CLEAR); | |
591 | ||
592 | qcq->flags |= IONIC_QCQ_F_INITED; | |
593 | ||
594 | ionic_debugfs_add_qcq(lif, qcq); | |
595 | ||
596 | return 0; | |
597 | } | |
598 | ||
77ceb68e SN |
599 | static int ionic_lif_notifyq_init(struct ionic_lif *lif) |
600 | { | |
601 | struct ionic_qcq *qcq = lif->notifyqcq; | |
602 | struct device *dev = lif->ionic->dev; | |
603 | struct ionic_queue *q = &qcq->q; | |
604 | int err; | |
605 | ||
606 | struct ionic_admin_ctx ctx = { | |
607 | .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work), | |
608 | .cmd.q_init = { | |
609 | .opcode = IONIC_CMD_Q_INIT, | |
610 | .lif_index = cpu_to_le16(lif->index), | |
611 | .type = q->type, | |
612 | .index = cpu_to_le32(q->index), | |
613 | .flags = cpu_to_le16(IONIC_QINIT_F_IRQ | | |
614 | IONIC_QINIT_F_ENA), | |
615 | .intr_index = cpu_to_le16(lif->adminqcq->intr.index), | |
616 | .pid = cpu_to_le16(q->pid), | |
617 | .ring_size = ilog2(q->num_descs), | |
618 | .ring_base = cpu_to_le64(q->base_pa), | |
619 | } | |
620 | }; | |
621 | ||
622 | dev_dbg(dev, "notifyq_init.pid %d\n", ctx.cmd.q_init.pid); | |
623 | dev_dbg(dev, "notifyq_init.index %d\n", ctx.cmd.q_init.index); | |
624 | dev_dbg(dev, "notifyq_init.ring_base 0x%llx\n", ctx.cmd.q_init.ring_base); | |
625 | dev_dbg(dev, "notifyq_init.ring_size %d\n", ctx.cmd.q_init.ring_size); | |
626 | ||
627 | err = ionic_adminq_post_wait(lif, &ctx); | |
628 | if (err) | |
629 | return err; | |
630 | ||
631 | q->hw_type = ctx.comp.q_init.hw_type; | |
632 | q->hw_index = le32_to_cpu(ctx.comp.q_init.hw_index); | |
633 | q->dbval = IONIC_DBELL_QID(q->hw_index); | |
634 | ||
635 | dev_dbg(dev, "notifyq->hw_type %d\n", q->hw_type); | |
636 | dev_dbg(dev, "notifyq->hw_index %d\n", q->hw_index); | |
637 | ||
638 | /* preset the callback info */ | |
639 | q->info[0].cb_arg = lif; | |
640 | ||
641 | qcq->flags |= IONIC_QCQ_F_INITED; | |
642 | ||
643 | ionic_debugfs_add_qcq(lif, qcq); | |
644 | ||
645 | return 0; | |
646 | } | |
647 | ||
1a58e196 SN |
648 | static int ionic_lif_init(struct ionic_lif *lif) |
649 | { | |
650 | struct ionic_dev *idev = &lif->ionic->idev; | |
6461b446 | 651 | struct device *dev = lif->ionic->dev; |
1a58e196 | 652 | struct ionic_lif_init_comp comp; |
6461b446 | 653 | int dbpage_num; |
1a58e196 SN |
654 | int err; |
655 | ||
656 | ionic_debugfs_add_lif(lif); | |
657 | ||
658 | mutex_lock(&lif->ionic->dev_cmd_lock); | |
659 | ionic_dev_cmd_lif_init(idev, lif->index, lif->info_pa); | |
660 | err = ionic_dev_cmd_wait(lif->ionic, DEVCMD_TIMEOUT); | |
661 | ionic_dev_cmd_comp(idev, (union ionic_dev_cmd_comp *)&comp); | |
662 | mutex_unlock(&lif->ionic->dev_cmd_lock); | |
663 | if (err) | |
664 | return err; | |
665 | ||
666 | lif->hw_index = le16_to_cpu(comp.hw_index); | |
667 | ||
6461b446 SN |
668 | /* now that we have the hw_index we can figure out our doorbell page */ |
669 | lif->dbid_count = le32_to_cpu(lif->ionic->ident.dev.ndbpgs_per_lif); | |
670 | if (!lif->dbid_count) { | |
671 | dev_err(dev, "No doorbell pages, aborting\n"); | |
672 | return -EINVAL; | |
673 | } | |
674 | ||
675 | lif->dbid_inuse = bitmap_alloc(lif->dbid_count, GFP_KERNEL); | |
676 | if (!lif->dbid_inuse) { | |
677 | dev_err(dev, "Failed alloc doorbell id bitmap, aborting\n"); | |
678 | return -ENOMEM; | |
679 | } | |
680 | ||
681 | /* first doorbell id reserved for kernel (dbid aka pid == zero) */ | |
682 | set_bit(0, lif->dbid_inuse); | |
683 | lif->kern_pid = 0; | |
684 | ||
685 | dbpage_num = ionic_db_page_num(lif, lif->kern_pid); | |
686 | lif->kern_dbpage = ionic_bus_map_dbpage(lif->ionic, dbpage_num); | |
687 | if (!lif->kern_dbpage) { | |
688 | dev_err(dev, "Cannot map dbpage, aborting\n"); | |
689 | err = -ENOMEM; | |
690 | goto err_out_free_dbid; | |
691 | } | |
692 | ||
1d062b7b SN |
693 | err = ionic_lif_adminq_init(lif); |
694 | if (err) | |
695 | goto err_out_adminq_deinit; | |
696 | ||
77ceb68e SN |
697 | if (lif->ionic->nnqs_per_lif) { |
698 | err = ionic_lif_notifyq_init(lif); | |
699 | if (err) | |
700 | goto err_out_notifyq_deinit; | |
701 | } | |
702 | ||
1a58e196 SN |
703 | set_bit(IONIC_LIF_INITED, lif->state); |
704 | ||
705 | return 0; | |
6461b446 | 706 | |
77ceb68e SN |
707 | err_out_notifyq_deinit: |
708 | ionic_lif_qcq_deinit(lif, lif->notifyqcq); | |
1d062b7b SN |
709 | err_out_adminq_deinit: |
710 | ionic_lif_qcq_deinit(lif, lif->adminqcq); | |
711 | ionic_lif_reset(lif); | |
712 | ionic_bus_unmap_dbpage(lif->ionic, lif->kern_dbpage); | |
713 | lif->kern_dbpage = NULL; | |
6461b446 SN |
714 | err_out_free_dbid: |
715 | kfree(lif->dbid_inuse); | |
716 | lif->dbid_inuse = NULL; | |
717 | ||
718 | return err; | |
1a58e196 SN |
719 | } |
720 | ||
721 | int ionic_lifs_init(struct ionic *ionic) | |
722 | { | |
723 | struct list_head *cur, *tmp; | |
724 | struct ionic_lif *lif; | |
725 | int err; | |
726 | ||
727 | list_for_each_safe(cur, tmp, &ionic->lifs) { | |
728 | lif = list_entry(cur, struct ionic_lif, list); | |
729 | err = ionic_lif_init(lif); | |
730 | if (err) | |
731 | return err; | |
732 | } | |
733 | ||
734 | return 0; | |
735 | } | |
736 | ||
737 | int ionic_lif_identify(struct ionic *ionic, u8 lif_type, | |
738 | union ionic_lif_identity *lid) | |
739 | { | |
740 | struct ionic_dev *idev = &ionic->idev; | |
741 | size_t sz; | |
742 | int err; | |
743 | ||
744 | sz = min(sizeof(*lid), sizeof(idev->dev_cmd_regs->data)); | |
745 | ||
746 | mutex_lock(&ionic->dev_cmd_lock); | |
747 | ionic_dev_cmd_lif_identify(idev, lif_type, IONIC_IDENTITY_VERSION_1); | |
748 | err = ionic_dev_cmd_wait(ionic, DEVCMD_TIMEOUT); | |
749 | memcpy_fromio(lid, &idev->dev_cmd_regs->data, sz); | |
750 | mutex_unlock(&ionic->dev_cmd_lock); | |
751 | if (err) | |
752 | return (err); | |
753 | ||
754 | dev_dbg(ionic->dev, "capabilities 0x%llx\n", | |
755 | le64_to_cpu(lid->capabilities)); | |
756 | ||
757 | dev_dbg(ionic->dev, "eth.max_ucast_filters %d\n", | |
758 | le32_to_cpu(lid->eth.max_ucast_filters)); | |
759 | dev_dbg(ionic->dev, "eth.max_mcast_filters %d\n", | |
760 | le32_to_cpu(lid->eth.max_mcast_filters)); | |
761 | dev_dbg(ionic->dev, "eth.features 0x%llx\n", | |
762 | le64_to_cpu(lid->eth.config.features)); | |
763 | dev_dbg(ionic->dev, "eth.queue_count[IONIC_QTYPE_ADMINQ] %d\n", | |
764 | le32_to_cpu(lid->eth.config.queue_count[IONIC_QTYPE_ADMINQ])); | |
765 | dev_dbg(ionic->dev, "eth.queue_count[IONIC_QTYPE_NOTIFYQ] %d\n", | |
766 | le32_to_cpu(lid->eth.config.queue_count[IONIC_QTYPE_NOTIFYQ])); | |
767 | dev_dbg(ionic->dev, "eth.queue_count[IONIC_QTYPE_RXQ] %d\n", | |
768 | le32_to_cpu(lid->eth.config.queue_count[IONIC_QTYPE_RXQ])); | |
769 | dev_dbg(ionic->dev, "eth.queue_count[IONIC_QTYPE_TXQ] %d\n", | |
770 | le32_to_cpu(lid->eth.config.queue_count[IONIC_QTYPE_TXQ])); | |
771 | dev_dbg(ionic->dev, "eth.config.name %s\n", lid->eth.config.name); | |
772 | dev_dbg(ionic->dev, "eth.config.mac %pM\n", lid->eth.config.mac); | |
773 | dev_dbg(ionic->dev, "eth.config.mtu %d\n", | |
774 | le32_to_cpu(lid->eth.config.mtu)); | |
775 | ||
776 | return 0; | |
777 | } | |
778 | ||
779 | int ionic_lifs_size(struct ionic *ionic) | |
780 | { | |
781 | struct ionic_identity *ident = &ionic->ident; | |
782 | unsigned int nintrs, dev_nintrs; | |
783 | union ionic_lif_config *lc; | |
784 | unsigned int ntxqs_per_lif; | |
785 | unsigned int nrxqs_per_lif; | |
786 | unsigned int neqs_per_lif; | |
787 | unsigned int nnqs_per_lif; | |
788 | unsigned int nxqs, neqs; | |
789 | unsigned int min_intrs; | |
790 | int err; | |
791 | ||
792 | lc = &ident->lif.eth.config; | |
793 | dev_nintrs = le32_to_cpu(ident->dev.nintrs); | |
794 | neqs_per_lif = le32_to_cpu(ident->lif.rdma.eq_qtype.qid_count); | |
795 | nnqs_per_lif = le32_to_cpu(lc->queue_count[IONIC_QTYPE_NOTIFYQ]); | |
796 | ntxqs_per_lif = le32_to_cpu(lc->queue_count[IONIC_QTYPE_TXQ]); | |
797 | nrxqs_per_lif = le32_to_cpu(lc->queue_count[IONIC_QTYPE_RXQ]); | |
798 | ||
799 | nxqs = min(ntxqs_per_lif, nrxqs_per_lif); | |
800 | nxqs = min(nxqs, num_online_cpus()); | |
801 | neqs = min(neqs_per_lif, num_online_cpus()); | |
802 | ||
803 | try_again: | |
804 | /* interrupt usage: | |
805 | * 1 for master lif adminq/notifyq | |
806 | * 1 for each CPU for master lif TxRx queue pairs | |
807 | * whatever's left is for RDMA queues | |
808 | */ | |
809 | nintrs = 1 + nxqs + neqs; | |
810 | min_intrs = 2; /* adminq + 1 TxRx queue pair */ | |
811 | ||
812 | if (nintrs > dev_nintrs) | |
813 | goto try_fewer; | |
814 | ||
815 | err = ionic_bus_alloc_irq_vectors(ionic, nintrs); | |
816 | if (err < 0 && err != -ENOSPC) { | |
817 | dev_err(ionic->dev, "Can't get intrs from OS: %d\n", err); | |
818 | return err; | |
819 | } | |
820 | if (err == -ENOSPC) | |
821 | goto try_fewer; | |
822 | ||
823 | if (err != nintrs) { | |
824 | ionic_bus_free_irq_vectors(ionic); | |
825 | goto try_fewer; | |
826 | } | |
827 | ||
828 | ionic->nnqs_per_lif = nnqs_per_lif; | |
829 | ionic->neqs_per_lif = neqs; | |
830 | ionic->ntxqs_per_lif = nxqs; | |
831 | ionic->nrxqs_per_lif = nxqs; | |
832 | ionic->nintrs = nintrs; | |
833 | ||
834 | ionic_debugfs_add_sizes(ionic); | |
835 | ||
836 | return 0; | |
837 | ||
838 | try_fewer: | |
839 | if (nnqs_per_lif > 1) { | |
840 | nnqs_per_lif >>= 1; | |
841 | goto try_again; | |
842 | } | |
843 | if (neqs > 1) { | |
844 | neqs >>= 1; | |
845 | goto try_again; | |
846 | } | |
847 | if (nxqs > 1) { | |
848 | nxqs >>= 1; | |
849 | goto try_again; | |
850 | } | |
851 | dev_err(ionic->dev, "Can't get minimum %d intrs from OS\n", min_intrs); | |
852 | return -ENOSPC; | |
853 | } |