]>
Commit | Line | Data |
---|---|---|
1ac5a404 SX |
1 | /* |
2 | * Broadcom NetXtreme-E RoCE driver. | |
3 | * | |
4 | * Copyright (c) 2016 - 2017, Broadcom. All rights reserved. The term | |
5 | * Broadcom refers to Broadcom Limited and/or its subsidiaries. | |
6 | * | |
7 | * This software is available to you under a choice of one of two | |
8 | * licenses. You may choose to be licensed under the terms of the GNU | |
9 | * General Public License (GPL) Version 2, available from the file | |
10 | * COPYING in the main directory of this source tree, or the | |
11 | * BSD license below: | |
12 | * | |
13 | * Redistribution and use in source and binary forms, with or without | |
14 | * modification, are permitted provided that the following conditions | |
15 | * are met: | |
16 | * | |
17 | * 1. Redistributions of source code must retain the above copyright | |
18 | * notice, this list of conditions and the following disclaimer. | |
19 | * 2. Redistributions in binary form must reproduce the above copyright | |
20 | * notice, this list of conditions and the following disclaimer in | |
21 | * the documentation and/or other materials provided with the | |
22 | * distribution. | |
23 | * | |
24 | * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' | |
25 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, | |
26 | * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR | |
27 | * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS | |
28 | * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR | |
29 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF | |
30 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR | |
31 | * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, | |
32 | * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE | |
33 | * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN | |
34 | * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | |
35 | * | |
36 | * Description: Main component of the bnxt_re driver | |
37 | */ | |
38 | ||
39 | #include <linux/module.h> | |
40 | #include <linux/netdevice.h> | |
41 | #include <linux/ethtool.h> | |
42 | #include <linux/mutex.h> | |
43 | #include <linux/list.h> | |
44 | #include <linux/rculist.h> | |
45 | #include <linux/spinlock.h> | |
46 | #include <linux/pci.h> | |
47 | #include <net/dcbnl.h> | |
48 | #include <net/ipv6.h> | |
49 | #include <net/addrconf.h> | |
50 | #include <linux/if_ether.h> | |
51 | ||
52 | #include <rdma/ib_verbs.h> | |
53 | #include <rdma/ib_user_verbs.h> | |
54 | #include <rdma/ib_umem.h> | |
55 | #include <rdma/ib_addr.h> | |
56 | ||
57 | #include "bnxt_ulp.h" | |
58 | #include "roce_hsi.h" | |
59 | #include "qplib_res.h" | |
60 | #include "qplib_sp.h" | |
61 | #include "qplib_fp.h" | |
62 | #include "qplib_rcfw.h" | |
63 | #include "bnxt_re.h" | |
64 | #include "ib_verbs.h" | |
65 | #include <rdma/bnxt_re-abi.h> | |
66 | #include "bnxt.h" | |
67 | static char version[] = | |
68 | BNXT_RE_DESC " v" ROCE_DRV_MODULE_VERSION "\n"; | |
69 | ||
70 | MODULE_AUTHOR("Eddie Wai <eddie.wai@broadcom.com>"); | |
71 | MODULE_DESCRIPTION(BNXT_RE_DESC " Driver"); | |
72 | MODULE_LICENSE("Dual BSD/GPL"); | |
73 | MODULE_VERSION(ROCE_DRV_MODULE_VERSION); | |
74 | ||
75 | /* globals */ | |
76 | static struct list_head bnxt_re_dev_list = LIST_HEAD_INIT(bnxt_re_dev_list); | |
77 | /* Mutex to protect the list of bnxt_re devices added */ | |
78 | static DEFINE_MUTEX(bnxt_re_dev_lock); | |
79 | static struct workqueue_struct *bnxt_re_wq; | |
80 | ||
81 | /* for handling bnxt_en callbacks later */ | |
82 | static void bnxt_re_stop(void *p) | |
83 | { | |
84 | } | |
85 | ||
86 | static void bnxt_re_start(void *p) | |
87 | { | |
88 | } | |
89 | ||
90 | static void bnxt_re_sriov_config(void *p, int num_vfs) | |
91 | { | |
92 | } | |
93 | ||
94 | static struct bnxt_ulp_ops bnxt_re_ulp_ops = { | |
95 | .ulp_async_notifier = NULL, | |
96 | .ulp_stop = bnxt_re_stop, | |
97 | .ulp_start = bnxt_re_start, | |
98 | .ulp_sriov_config = bnxt_re_sriov_config | |
99 | }; | |
100 | ||
101 | /* RoCE -> Net driver */ | |
102 | ||
103 | /* Driver registration routines used to let the networking driver (bnxt_en) | |
104 | * to know that the RoCE driver is now installed | |
105 | */ | |
106 | static int bnxt_re_unregister_netdev(struct bnxt_re_dev *rdev, bool lock_wait) | |
107 | { | |
108 | struct bnxt_en_dev *en_dev; | |
109 | int rc; | |
110 | ||
111 | if (!rdev) | |
112 | return -EINVAL; | |
113 | ||
114 | en_dev = rdev->en_dev; | |
115 | /* Acquire rtnl lock if it is not invokded from netdev event */ | |
116 | if (lock_wait) | |
117 | rtnl_lock(); | |
118 | ||
119 | rc = en_dev->en_ops->bnxt_unregister_device(rdev->en_dev, | |
120 | BNXT_ROCE_ULP); | |
121 | if (lock_wait) | |
122 | rtnl_unlock(); | |
123 | return rc; | |
124 | } | |
125 | ||
126 | static int bnxt_re_register_netdev(struct bnxt_re_dev *rdev) | |
127 | { | |
128 | struct bnxt_en_dev *en_dev; | |
129 | int rc = 0; | |
130 | ||
131 | if (!rdev) | |
132 | return -EINVAL; | |
133 | ||
134 | en_dev = rdev->en_dev; | |
135 | ||
136 | rtnl_lock(); | |
137 | rc = en_dev->en_ops->bnxt_register_device(en_dev, BNXT_ROCE_ULP, | |
138 | &bnxt_re_ulp_ops, rdev); | |
139 | rtnl_unlock(); | |
140 | return rc; | |
141 | } | |
142 | ||
143 | static int bnxt_re_free_msix(struct bnxt_re_dev *rdev, bool lock_wait) | |
144 | { | |
145 | struct bnxt_en_dev *en_dev; | |
146 | int rc; | |
147 | ||
148 | if (!rdev) | |
149 | return -EINVAL; | |
150 | ||
151 | en_dev = rdev->en_dev; | |
152 | ||
153 | if (lock_wait) | |
154 | rtnl_lock(); | |
155 | ||
156 | rc = en_dev->en_ops->bnxt_free_msix(rdev->en_dev, BNXT_ROCE_ULP); | |
157 | ||
158 | if (lock_wait) | |
159 | rtnl_unlock(); | |
160 | return rc; | |
161 | } | |
162 | ||
163 | static int bnxt_re_request_msix(struct bnxt_re_dev *rdev) | |
164 | { | |
165 | int rc = 0, num_msix_want = BNXT_RE_MIN_MSIX, num_msix_got; | |
166 | struct bnxt_en_dev *en_dev; | |
167 | ||
168 | if (!rdev) | |
169 | return -EINVAL; | |
170 | ||
171 | en_dev = rdev->en_dev; | |
172 | ||
173 | rtnl_lock(); | |
174 | num_msix_got = en_dev->en_ops->bnxt_request_msix(en_dev, BNXT_ROCE_ULP, | |
175 | rdev->msix_entries, | |
176 | num_msix_want); | |
177 | if (num_msix_got < BNXT_RE_MIN_MSIX) { | |
178 | rc = -EINVAL; | |
179 | goto done; | |
180 | } | |
181 | if (num_msix_got != num_msix_want) { | |
182 | dev_warn(rdev_to_dev(rdev), | |
183 | "Requested %d MSI-X vectors, got %d\n", | |
184 | num_msix_want, num_msix_got); | |
185 | } | |
186 | rdev->num_msix = num_msix_got; | |
187 | done: | |
188 | rtnl_unlock(); | |
189 | return rc; | |
190 | } | |
191 | ||
192 | static void bnxt_re_init_hwrm_hdr(struct bnxt_re_dev *rdev, struct input *hdr, | |
193 | u16 opcd, u16 crid, u16 trid) | |
194 | { | |
195 | hdr->req_type = cpu_to_le16(opcd); | |
196 | hdr->cmpl_ring = cpu_to_le16(crid); | |
197 | hdr->target_id = cpu_to_le16(trid); | |
198 | } | |
199 | ||
200 | static void bnxt_re_fill_fw_msg(struct bnxt_fw_msg *fw_msg, void *msg, | |
201 | int msg_len, void *resp, int resp_max_len, | |
202 | int timeout) | |
203 | { | |
204 | fw_msg->msg = msg; | |
205 | fw_msg->msg_len = msg_len; | |
206 | fw_msg->resp = resp; | |
207 | fw_msg->resp_max_len = resp_max_len; | |
208 | fw_msg->timeout = timeout; | |
209 | } | |
210 | ||
211 | static int bnxt_re_net_ring_free(struct bnxt_re_dev *rdev, u16 fw_ring_id, | |
212 | bool lock_wait) | |
213 | { | |
214 | struct bnxt_en_dev *en_dev = rdev->en_dev; | |
215 | struct hwrm_ring_free_input req = {0}; | |
216 | struct hwrm_ring_free_output resp; | |
217 | struct bnxt_fw_msg fw_msg; | |
218 | bool do_unlock = false; | |
219 | int rc = -EINVAL; | |
220 | ||
221 | if (!en_dev) | |
222 | return rc; | |
223 | ||
224 | memset(&fw_msg, 0, sizeof(fw_msg)); | |
225 | if (lock_wait) { | |
226 | rtnl_lock(); | |
227 | do_unlock = true; | |
228 | } | |
229 | ||
230 | bnxt_re_init_hwrm_hdr(rdev, (void *)&req, HWRM_RING_FREE, -1, -1); | |
db690328 | 231 | req.ring_type = RING_ALLOC_REQ_RING_TYPE_L2_CMPL; |
1ac5a404 SX |
232 | req.ring_id = cpu_to_le16(fw_ring_id); |
233 | bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, | |
234 | sizeof(resp), DFLT_HWRM_CMD_TIMEOUT); | |
235 | rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); | |
236 | if (rc) | |
237 | dev_err(rdev_to_dev(rdev), | |
238 | "Failed to free HW ring:%d :%#x", req.ring_id, rc); | |
239 | if (do_unlock) | |
240 | rtnl_unlock(); | |
241 | return rc; | |
242 | } | |
243 | ||
244 | static int bnxt_re_net_ring_alloc(struct bnxt_re_dev *rdev, dma_addr_t *dma_arr, | |
245 | int pages, int type, u32 ring_mask, | |
246 | u32 map_index, u16 *fw_ring_id) | |
247 | { | |
248 | struct bnxt_en_dev *en_dev = rdev->en_dev; | |
249 | struct hwrm_ring_alloc_input req = {0}; | |
250 | struct hwrm_ring_alloc_output resp; | |
251 | struct bnxt_fw_msg fw_msg; | |
252 | int rc = -EINVAL; | |
253 | ||
254 | if (!en_dev) | |
255 | return rc; | |
256 | ||
257 | memset(&fw_msg, 0, sizeof(fw_msg)); | |
258 | rtnl_lock(); | |
259 | bnxt_re_init_hwrm_hdr(rdev, (void *)&req, HWRM_RING_ALLOC, -1, -1); | |
260 | req.enables = 0; | |
261 | req.page_tbl_addr = cpu_to_le64(dma_arr[0]); | |
262 | if (pages > 1) { | |
263 | /* Page size is in log2 units */ | |
264 | req.page_size = BNXT_PAGE_SHIFT; | |
265 | req.page_tbl_depth = 1; | |
266 | } | |
267 | req.fbo = 0; | |
268 | /* Association of ring index with doorbell index and MSIX number */ | |
269 | req.logical_id = cpu_to_le16(map_index); | |
270 | req.length = cpu_to_le32(ring_mask + 1); | |
db690328 | 271 | req.ring_type = RING_ALLOC_REQ_RING_TYPE_L2_CMPL; |
1ac5a404 SX |
272 | req.int_mode = RING_ALLOC_REQ_INT_MODE_MSIX; |
273 | bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, | |
274 | sizeof(resp), DFLT_HWRM_CMD_TIMEOUT); | |
275 | rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); | |
276 | if (!rc) | |
277 | *fw_ring_id = le16_to_cpu(resp.ring_id); | |
278 | ||
279 | rtnl_unlock(); | |
280 | return rc; | |
281 | } | |
282 | ||
283 | static int bnxt_re_net_stats_ctx_free(struct bnxt_re_dev *rdev, | |
284 | u32 fw_stats_ctx_id, bool lock_wait) | |
285 | { | |
286 | struct bnxt_en_dev *en_dev = rdev->en_dev; | |
287 | struct hwrm_stat_ctx_free_input req = {0}; | |
288 | struct bnxt_fw_msg fw_msg; | |
289 | bool do_unlock = false; | |
290 | int rc = -EINVAL; | |
291 | ||
292 | if (!en_dev) | |
293 | return rc; | |
294 | ||
295 | memset(&fw_msg, 0, sizeof(fw_msg)); | |
296 | if (lock_wait) { | |
297 | rtnl_lock(); | |
298 | do_unlock = true; | |
299 | } | |
300 | ||
301 | bnxt_re_init_hwrm_hdr(rdev, (void *)&req, HWRM_STAT_CTX_FREE, -1, -1); | |
302 | req.stat_ctx_id = cpu_to_le32(fw_stats_ctx_id); | |
303 | bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&req, | |
304 | sizeof(req), DFLT_HWRM_CMD_TIMEOUT); | |
305 | rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); | |
306 | if (rc) | |
307 | dev_err(rdev_to_dev(rdev), | |
308 | "Failed to free HW stats context %#x", rc); | |
309 | ||
310 | if (do_unlock) | |
311 | rtnl_unlock(); | |
312 | return rc; | |
313 | } | |
314 | ||
315 | static int bnxt_re_net_stats_ctx_alloc(struct bnxt_re_dev *rdev, | |
316 | dma_addr_t dma_map, | |
317 | u32 *fw_stats_ctx_id) | |
318 | { | |
319 | struct hwrm_stat_ctx_alloc_output resp = {0}; | |
320 | struct hwrm_stat_ctx_alloc_input req = {0}; | |
321 | struct bnxt_en_dev *en_dev = rdev->en_dev; | |
322 | struct bnxt_fw_msg fw_msg; | |
323 | int rc = -EINVAL; | |
324 | ||
325 | *fw_stats_ctx_id = INVALID_STATS_CTX_ID; | |
326 | ||
327 | if (!en_dev) | |
328 | return rc; | |
329 | ||
330 | memset(&fw_msg, 0, sizeof(fw_msg)); | |
331 | rtnl_lock(); | |
332 | ||
333 | bnxt_re_init_hwrm_hdr(rdev, (void *)&req, HWRM_STAT_CTX_ALLOC, -1, -1); | |
334 | req.update_period_ms = cpu_to_le32(1000); | |
335 | req.stats_dma_addr = cpu_to_le64(dma_map); | |
536f0928 | 336 | req.stat_ctx_flags = STAT_CTX_ALLOC_REQ_STAT_CTX_FLAGS_ROCE; |
1ac5a404 SX |
337 | bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, |
338 | sizeof(resp), DFLT_HWRM_CMD_TIMEOUT); | |
339 | rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); | |
340 | if (!rc) | |
341 | *fw_stats_ctx_id = le32_to_cpu(resp.stat_ctx_id); | |
342 | ||
343 | rtnl_unlock(); | |
344 | return rc; | |
345 | } | |
346 | ||
347 | /* Device */ | |
348 | ||
349 | static bool is_bnxt_re_dev(struct net_device *netdev) | |
350 | { | |
351 | struct ethtool_drvinfo drvinfo; | |
352 | ||
353 | if (netdev->ethtool_ops && netdev->ethtool_ops->get_drvinfo) { | |
354 | memset(&drvinfo, 0, sizeof(drvinfo)); | |
355 | netdev->ethtool_ops->get_drvinfo(netdev, &drvinfo); | |
356 | ||
357 | if (strcmp(drvinfo.driver, "bnxt_en")) | |
358 | return false; | |
359 | return true; | |
360 | } | |
361 | return false; | |
362 | } | |
363 | ||
364 | static struct bnxt_re_dev *bnxt_re_from_netdev(struct net_device *netdev) | |
365 | { | |
366 | struct bnxt_re_dev *rdev; | |
367 | ||
368 | rcu_read_lock(); | |
369 | list_for_each_entry_rcu(rdev, &bnxt_re_dev_list, list) { | |
370 | if (rdev->netdev == netdev) { | |
371 | rcu_read_unlock(); | |
372 | return rdev; | |
373 | } | |
374 | } | |
375 | rcu_read_unlock(); | |
376 | return NULL; | |
377 | } | |
378 | ||
379 | static void bnxt_re_dev_unprobe(struct net_device *netdev, | |
380 | struct bnxt_en_dev *en_dev) | |
381 | { | |
382 | dev_put(netdev); | |
383 | module_put(en_dev->pdev->driver->driver.owner); | |
384 | } | |
385 | ||
386 | static struct bnxt_en_dev *bnxt_re_dev_probe(struct net_device *netdev) | |
387 | { | |
388 | struct bnxt *bp = netdev_priv(netdev); | |
389 | struct bnxt_en_dev *en_dev; | |
390 | struct pci_dev *pdev; | |
391 | ||
392 | /* Call bnxt_en's RoCE probe via indirect API */ | |
393 | if (!bp->ulp_probe) | |
394 | return ERR_PTR(-EINVAL); | |
395 | ||
396 | en_dev = bp->ulp_probe(netdev); | |
397 | if (IS_ERR(en_dev)) | |
398 | return en_dev; | |
399 | ||
400 | pdev = en_dev->pdev; | |
401 | if (!pdev) | |
402 | return ERR_PTR(-EINVAL); | |
403 | ||
404 | if (!(en_dev->flags & BNXT_EN_FLAG_ROCE_CAP)) { | |
405 | dev_dbg(&pdev->dev, | |
406 | "%s: probe error: RoCE is not supported on this device", | |
407 | ROCE_DRV_MODULE_NAME); | |
408 | return ERR_PTR(-ENODEV); | |
409 | } | |
410 | ||
411 | /* Bump net device reference count */ | |
412 | if (!try_module_get(pdev->driver->driver.owner)) | |
413 | return ERR_PTR(-ENODEV); | |
414 | ||
415 | dev_hold(netdev); | |
416 | ||
417 | return en_dev; | |
418 | } | |
419 | ||
420 | static void bnxt_re_unregister_ib(struct bnxt_re_dev *rdev) | |
421 | { | |
422 | ib_unregister_device(&rdev->ibdev); | |
423 | } | |
424 | ||
425 | static int bnxt_re_register_ib(struct bnxt_re_dev *rdev) | |
426 | { | |
427 | struct ib_device *ibdev = &rdev->ibdev; | |
428 | ||
429 | /* ib device init */ | |
430 | ibdev->owner = THIS_MODULE; | |
431 | ibdev->node_type = RDMA_NODE_IB_CA; | |
432 | strlcpy(ibdev->name, "bnxt_re%d", IB_DEVICE_NAME_MAX); | |
433 | strlcpy(ibdev->node_desc, BNXT_RE_DESC " HCA", | |
434 | strlen(BNXT_RE_DESC) + 5); | |
435 | ibdev->phys_port_cnt = 1; | |
436 | ||
437 | bnxt_qplib_get_guid(rdev->netdev->dev_addr, (u8 *)&ibdev->node_guid); | |
438 | ||
439 | ibdev->num_comp_vectors = 1; | |
ac1820fb | 440 | ibdev->dev.parent = &rdev->en_dev->pdev->dev; |
1ac5a404 SX |
441 | ibdev->local_dma_lkey = BNXT_QPLIB_RSVD_LKEY; |
442 | ||
443 | /* User space */ | |
444 | ibdev->uverbs_abi_ver = BNXT_RE_ABI_VERSION; | |
445 | ibdev->uverbs_cmd_mask = | |
446 | (1ull << IB_USER_VERBS_CMD_GET_CONTEXT) | | |
447 | (1ull << IB_USER_VERBS_CMD_QUERY_DEVICE) | | |
448 | (1ull << IB_USER_VERBS_CMD_QUERY_PORT) | | |
449 | (1ull << IB_USER_VERBS_CMD_ALLOC_PD) | | |
450 | (1ull << IB_USER_VERBS_CMD_DEALLOC_PD) | | |
451 | (1ull << IB_USER_VERBS_CMD_REG_MR) | | |
452 | (1ull << IB_USER_VERBS_CMD_REREG_MR) | | |
453 | (1ull << IB_USER_VERBS_CMD_DEREG_MR) | | |
454 | (1ull << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) | | |
455 | (1ull << IB_USER_VERBS_CMD_CREATE_CQ) | | |
456 | (1ull << IB_USER_VERBS_CMD_RESIZE_CQ) | | |
457 | (1ull << IB_USER_VERBS_CMD_DESTROY_CQ) | | |
458 | (1ull << IB_USER_VERBS_CMD_CREATE_QP) | | |
459 | (1ull << IB_USER_VERBS_CMD_MODIFY_QP) | | |
460 | (1ull << IB_USER_VERBS_CMD_QUERY_QP) | | |
461 | (1ull << IB_USER_VERBS_CMD_DESTROY_QP) | | |
462 | (1ull << IB_USER_VERBS_CMD_CREATE_SRQ) | | |
463 | (1ull << IB_USER_VERBS_CMD_MODIFY_SRQ) | | |
464 | (1ull << IB_USER_VERBS_CMD_QUERY_SRQ) | | |
465 | (1ull << IB_USER_VERBS_CMD_DESTROY_SRQ) | | |
466 | (1ull << IB_USER_VERBS_CMD_CREATE_AH) | | |
467 | (1ull << IB_USER_VERBS_CMD_MODIFY_AH) | | |
468 | (1ull << IB_USER_VERBS_CMD_QUERY_AH) | | |
469 | (1ull << IB_USER_VERBS_CMD_DESTROY_AH); | |
470 | /* POLL_CQ and REQ_NOTIFY_CQ is directly handled in libbnxt_re */ | |
471 | ||
472 | /* Kernel verbs */ | |
473 | ibdev->query_device = bnxt_re_query_device; | |
474 | ibdev->modify_device = bnxt_re_modify_device; | |
475 | ||
476 | ibdev->query_port = bnxt_re_query_port; | |
477 | ibdev->modify_port = bnxt_re_modify_port; | |
478 | ibdev->get_port_immutable = bnxt_re_get_port_immutable; | |
479 | ibdev->query_pkey = bnxt_re_query_pkey; | |
480 | ibdev->query_gid = bnxt_re_query_gid; | |
481 | ibdev->get_netdev = bnxt_re_get_netdev; | |
482 | ibdev->add_gid = bnxt_re_add_gid; | |
483 | ibdev->del_gid = bnxt_re_del_gid; | |
484 | ibdev->get_link_layer = bnxt_re_get_link_layer; | |
485 | ||
486 | ibdev->alloc_pd = bnxt_re_alloc_pd; | |
487 | ibdev->dealloc_pd = bnxt_re_dealloc_pd; | |
488 | ||
489 | ibdev->create_ah = bnxt_re_create_ah; | |
490 | ibdev->modify_ah = bnxt_re_modify_ah; | |
491 | ibdev->query_ah = bnxt_re_query_ah; | |
492 | ibdev->destroy_ah = bnxt_re_destroy_ah; | |
493 | ||
494 | ibdev->create_qp = bnxt_re_create_qp; | |
495 | ibdev->modify_qp = bnxt_re_modify_qp; | |
496 | ibdev->query_qp = bnxt_re_query_qp; | |
497 | ibdev->destroy_qp = bnxt_re_destroy_qp; | |
498 | ||
499 | ibdev->post_send = bnxt_re_post_send; | |
500 | ibdev->post_recv = bnxt_re_post_recv; | |
501 | ||
502 | ibdev->create_cq = bnxt_re_create_cq; | |
503 | ibdev->destroy_cq = bnxt_re_destroy_cq; | |
504 | ibdev->poll_cq = bnxt_re_poll_cq; | |
505 | ibdev->req_notify_cq = bnxt_re_req_notify_cq; | |
506 | ||
507 | ibdev->get_dma_mr = bnxt_re_get_dma_mr; | |
508 | ibdev->dereg_mr = bnxt_re_dereg_mr; | |
509 | ibdev->alloc_mr = bnxt_re_alloc_mr; | |
510 | ibdev->map_mr_sg = bnxt_re_map_mr_sg; | |
1ac5a404 SX |
511 | |
512 | ibdev->reg_user_mr = bnxt_re_reg_user_mr; | |
513 | ibdev->alloc_ucontext = bnxt_re_alloc_ucontext; | |
514 | ibdev->dealloc_ucontext = bnxt_re_dealloc_ucontext; | |
515 | ibdev->mmap = bnxt_re_mmap; | |
516 | ||
517 | return ib_register_device(ibdev, NULL); | |
518 | } | |
519 | ||
520 | static ssize_t show_rev(struct device *device, struct device_attribute *attr, | |
521 | char *buf) | |
522 | { | |
523 | struct bnxt_re_dev *rdev = to_bnxt_re_dev(device, ibdev.dev); | |
524 | ||
525 | return scnprintf(buf, PAGE_SIZE, "0x%x\n", rdev->en_dev->pdev->vendor); | |
526 | } | |
527 | ||
528 | static ssize_t show_fw_ver(struct device *device, struct device_attribute *attr, | |
529 | char *buf) | |
530 | { | |
531 | struct bnxt_re_dev *rdev = to_bnxt_re_dev(device, ibdev.dev); | |
532 | ||
533 | return scnprintf(buf, PAGE_SIZE, "%s\n", rdev->dev_attr.fw_ver); | |
534 | } | |
535 | ||
536 | static ssize_t show_hca(struct device *device, struct device_attribute *attr, | |
537 | char *buf) | |
538 | { | |
539 | struct bnxt_re_dev *rdev = to_bnxt_re_dev(device, ibdev.dev); | |
540 | ||
541 | return scnprintf(buf, PAGE_SIZE, "%s\n", rdev->ibdev.node_desc); | |
542 | } | |
543 | ||
544 | static DEVICE_ATTR(hw_rev, 0444, show_rev, NULL); | |
545 | static DEVICE_ATTR(fw_rev, 0444, show_fw_ver, NULL); | |
546 | static DEVICE_ATTR(hca_type, 0444, show_hca, NULL); | |
547 | ||
548 | static struct device_attribute *bnxt_re_attributes[] = { | |
549 | &dev_attr_hw_rev, | |
550 | &dev_attr_fw_rev, | |
551 | &dev_attr_hca_type | |
552 | }; | |
553 | ||
554 | static void bnxt_re_dev_remove(struct bnxt_re_dev *rdev) | |
555 | { | |
556 | dev_put(rdev->netdev); | |
557 | rdev->netdev = NULL; | |
558 | ||
559 | mutex_lock(&bnxt_re_dev_lock); | |
560 | list_del_rcu(&rdev->list); | |
561 | mutex_unlock(&bnxt_re_dev_lock); | |
562 | ||
563 | synchronize_rcu(); | |
564 | flush_workqueue(bnxt_re_wq); | |
565 | ||
566 | ib_dealloc_device(&rdev->ibdev); | |
567 | /* rdev is gone */ | |
568 | } | |
569 | ||
570 | static struct bnxt_re_dev *bnxt_re_dev_add(struct net_device *netdev, | |
571 | struct bnxt_en_dev *en_dev) | |
572 | { | |
573 | struct bnxt_re_dev *rdev; | |
574 | ||
575 | /* Allocate bnxt_re_dev instance here */ | |
576 | rdev = (struct bnxt_re_dev *)ib_alloc_device(sizeof(*rdev)); | |
577 | if (!rdev) { | |
578 | dev_err(NULL, "%s: bnxt_re_dev allocation failure!", | |
579 | ROCE_DRV_MODULE_NAME); | |
580 | return NULL; | |
581 | } | |
582 | /* Default values */ | |
583 | rdev->netdev = netdev; | |
584 | dev_hold(rdev->netdev); | |
585 | rdev->en_dev = en_dev; | |
586 | rdev->id = rdev->en_dev->pdev->devfn; | |
587 | INIT_LIST_HEAD(&rdev->qp_list); | |
588 | mutex_init(&rdev->qp_lock); | |
589 | atomic_set(&rdev->qp_count, 0); | |
590 | atomic_set(&rdev->cq_count, 0); | |
591 | atomic_set(&rdev->srq_count, 0); | |
592 | atomic_set(&rdev->mr_count, 0); | |
593 | atomic_set(&rdev->mw_count, 0); | |
594 | rdev->cosq[0] = 0xFFFF; | |
595 | rdev->cosq[1] = 0xFFFF; | |
596 | ||
597 | mutex_lock(&bnxt_re_dev_lock); | |
598 | list_add_tail_rcu(&rdev->list, &bnxt_re_dev_list); | |
599 | mutex_unlock(&bnxt_re_dev_lock); | |
600 | return rdev; | |
601 | } | |
602 | ||
603 | static int bnxt_re_aeq_handler(struct bnxt_qplib_rcfw *rcfw, | |
604 | struct creq_func_event *aeqe) | |
605 | { | |
606 | switch (aeqe->event) { | |
607 | case CREQ_FUNC_EVENT_EVENT_TX_WQE_ERROR: | |
608 | break; | |
609 | case CREQ_FUNC_EVENT_EVENT_TX_DATA_ERROR: | |
610 | break; | |
611 | case CREQ_FUNC_EVENT_EVENT_RX_WQE_ERROR: | |
612 | break; | |
613 | case CREQ_FUNC_EVENT_EVENT_RX_DATA_ERROR: | |
614 | break; | |
615 | case CREQ_FUNC_EVENT_EVENT_CQ_ERROR: | |
616 | break; | |
617 | case CREQ_FUNC_EVENT_EVENT_TQM_ERROR: | |
618 | break; | |
619 | case CREQ_FUNC_EVENT_EVENT_CFCQ_ERROR: | |
620 | break; | |
621 | case CREQ_FUNC_EVENT_EVENT_CFCS_ERROR: | |
622 | break; | |
623 | case CREQ_FUNC_EVENT_EVENT_CFCC_ERROR: | |
624 | break; | |
625 | case CREQ_FUNC_EVENT_EVENT_CFCM_ERROR: | |
626 | break; | |
627 | case CREQ_FUNC_EVENT_EVENT_TIM_ERROR: | |
628 | break; | |
629 | default: | |
630 | return -EINVAL; | |
631 | } | |
632 | return 0; | |
633 | } | |
634 | ||
635 | static int bnxt_re_cqn_handler(struct bnxt_qplib_nq *nq, | |
636 | struct bnxt_qplib_cq *handle) | |
637 | { | |
638 | struct bnxt_re_cq *cq = container_of(handle, struct bnxt_re_cq, | |
639 | qplib_cq); | |
640 | ||
641 | if (!cq) { | |
642 | dev_err(NULL, "%s: CQ is NULL, CQN not handled", | |
643 | ROCE_DRV_MODULE_NAME); | |
644 | return -EINVAL; | |
645 | } | |
646 | if (cq->ib_cq.comp_handler) { | |
647 | /* Lock comp_handler? */ | |
648 | (*cq->ib_cq.comp_handler)(&cq->ib_cq, cq->ib_cq.cq_context); | |
649 | } | |
650 | ||
651 | return 0; | |
652 | } | |
653 | ||
654 | static void bnxt_re_cleanup_res(struct bnxt_re_dev *rdev) | |
655 | { | |
656 | if (rdev->nq.hwq.max_elements) | |
657 | bnxt_qplib_disable_nq(&rdev->nq); | |
658 | ||
659 | if (rdev->qplib_res.rcfw) | |
660 | bnxt_qplib_cleanup_res(&rdev->qplib_res); | |
661 | } | |
662 | ||
663 | static int bnxt_re_init_res(struct bnxt_re_dev *rdev) | |
664 | { | |
665 | int rc = 0; | |
666 | ||
667 | bnxt_qplib_init_res(&rdev->qplib_res); | |
668 | ||
669 | if (rdev->msix_entries[BNXT_RE_NQ_IDX].vector <= 0) | |
670 | return -EINVAL; | |
671 | ||
672 | rc = bnxt_qplib_enable_nq(rdev->en_dev->pdev, &rdev->nq, | |
673 | rdev->msix_entries[BNXT_RE_NQ_IDX].vector, | |
674 | rdev->msix_entries[BNXT_RE_NQ_IDX].db_offset, | |
675 | &bnxt_re_cqn_handler, | |
676 | NULL); | |
677 | ||
678 | if (rc) | |
679 | dev_err(rdev_to_dev(rdev), "Failed to enable NQ: %#x", rc); | |
680 | ||
681 | return rc; | |
682 | } | |
683 | ||
684 | static void bnxt_re_free_res(struct bnxt_re_dev *rdev, bool lock_wait) | |
685 | { | |
686 | if (rdev->nq.hwq.max_elements) { | |
687 | bnxt_re_net_ring_free(rdev, rdev->nq.ring_id, lock_wait); | |
688 | bnxt_qplib_free_nq(&rdev->nq); | |
689 | } | |
690 | if (rdev->qplib_res.dpi_tbl.max) { | |
691 | bnxt_qplib_dealloc_dpi(&rdev->qplib_res, | |
692 | &rdev->qplib_res.dpi_tbl, | |
693 | &rdev->dpi_privileged); | |
694 | } | |
695 | if (rdev->qplib_res.rcfw) { | |
696 | bnxt_qplib_free_res(&rdev->qplib_res); | |
697 | rdev->qplib_res.rcfw = NULL; | |
698 | } | |
699 | } | |
700 | ||
701 | static int bnxt_re_alloc_res(struct bnxt_re_dev *rdev) | |
702 | { | |
703 | int rc = 0; | |
704 | ||
705 | /* Configure and allocate resources for qplib */ | |
706 | rdev->qplib_res.rcfw = &rdev->rcfw; | |
707 | rc = bnxt_qplib_get_dev_attr(&rdev->rcfw, &rdev->dev_attr); | |
708 | if (rc) | |
709 | goto fail; | |
710 | ||
711 | rc = bnxt_qplib_alloc_res(&rdev->qplib_res, rdev->en_dev->pdev, | |
712 | rdev->netdev, &rdev->dev_attr); | |
713 | if (rc) | |
714 | goto fail; | |
715 | ||
716 | rc = bnxt_qplib_alloc_dpi(&rdev->qplib_res.dpi_tbl, | |
717 | &rdev->dpi_privileged, | |
718 | rdev); | |
719 | if (rc) | |
720 | goto fail; | |
721 | ||
722 | rdev->nq.hwq.max_elements = BNXT_RE_MAX_CQ_COUNT + | |
723 | BNXT_RE_MAX_SRQC_COUNT + 2; | |
724 | rc = bnxt_qplib_alloc_nq(rdev->en_dev->pdev, &rdev->nq); | |
725 | if (rc) { | |
726 | dev_err(rdev_to_dev(rdev), | |
727 | "Failed to allocate NQ memory: %#x", rc); | |
728 | goto fail; | |
729 | } | |
730 | rc = bnxt_re_net_ring_alloc | |
731 | (rdev, rdev->nq.hwq.pbl[PBL_LVL_0].pg_map_arr, | |
732 | rdev->nq.hwq.pbl[rdev->nq.hwq.level].pg_count, | |
733 | HWRM_RING_ALLOC_CMPL, BNXT_QPLIB_NQE_MAX_CNT - 1, | |
734 | rdev->msix_entries[BNXT_RE_NQ_IDX].ring_idx, | |
735 | &rdev->nq.ring_id); | |
736 | if (rc) { | |
737 | dev_err(rdev_to_dev(rdev), | |
738 | "Failed to allocate NQ ring: %#x", rc); | |
739 | goto free_nq; | |
740 | } | |
741 | return 0; | |
742 | free_nq: | |
743 | bnxt_qplib_free_nq(&rdev->nq); | |
744 | fail: | |
745 | rdev->qplib_res.rcfw = NULL; | |
746 | return rc; | |
747 | } | |
748 | ||
749 | static void bnxt_re_dispatch_event(struct ib_device *ibdev, struct ib_qp *qp, | |
750 | u8 port_num, enum ib_event_type event) | |
751 | { | |
752 | struct ib_event ib_event; | |
753 | ||
754 | ib_event.device = ibdev; | |
755 | if (qp) | |
756 | ib_event.element.qp = qp; | |
757 | else | |
758 | ib_event.element.port_num = port_num; | |
759 | ib_event.event = event; | |
760 | ib_dispatch_event(&ib_event); | |
761 | } | |
762 | ||
763 | #define HWRM_QUEUE_PRI2COS_QCFG_INPUT_FLAGS_IVLAN 0x02 | |
764 | static int bnxt_re_query_hwrm_pri2cos(struct bnxt_re_dev *rdev, u8 dir, | |
765 | u64 *cid_map) | |
766 | { | |
767 | struct hwrm_queue_pri2cos_qcfg_input req = {0}; | |
768 | struct bnxt *bp = netdev_priv(rdev->netdev); | |
769 | struct hwrm_queue_pri2cos_qcfg_output resp; | |
770 | struct bnxt_en_dev *en_dev = rdev->en_dev; | |
771 | struct bnxt_fw_msg fw_msg; | |
772 | u32 flags = 0; | |
773 | u8 *qcfgmap, *tmp_map; | |
774 | int rc = 0, i; | |
775 | ||
776 | if (!cid_map) | |
777 | return -EINVAL; | |
778 | ||
779 | memset(&fw_msg, 0, sizeof(fw_msg)); | |
780 | bnxt_re_init_hwrm_hdr(rdev, (void *)&req, | |
781 | HWRM_QUEUE_PRI2COS_QCFG, -1, -1); | |
782 | flags |= (dir & 0x01); | |
783 | flags |= HWRM_QUEUE_PRI2COS_QCFG_INPUT_FLAGS_IVLAN; | |
784 | req.flags = cpu_to_le32(flags); | |
785 | req.port_id = bp->pf.port_id; | |
786 | ||
787 | bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, | |
788 | sizeof(resp), DFLT_HWRM_CMD_TIMEOUT); | |
789 | rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); | |
790 | if (rc) | |
791 | return rc; | |
792 | ||
793 | if (resp.queue_cfg_info) { | |
794 | dev_warn(rdev_to_dev(rdev), | |
795 | "Asymmetric cos queue configuration detected"); | |
796 | dev_warn(rdev_to_dev(rdev), | |
797 | " on device, QoS may not be fully functional\n"); | |
798 | } | |
799 | qcfgmap = &resp.pri0_cos_queue_id; | |
800 | tmp_map = (u8 *)cid_map; | |
801 | for (i = 0; i < IEEE_8021QAZ_MAX_TCS; i++) | |
802 | tmp_map[i] = qcfgmap[i]; | |
803 | ||
804 | return rc; | |
805 | } | |
806 | ||
807 | static bool bnxt_re_is_qp1_or_shadow_qp(struct bnxt_re_dev *rdev, | |
808 | struct bnxt_re_qp *qp) | |
809 | { | |
810 | return (qp->ib_qp.qp_type == IB_QPT_GSI) || (qp == rdev->qp1_sqp); | |
811 | } | |
812 | ||
813 | static void bnxt_re_dev_stop(struct bnxt_re_dev *rdev) | |
814 | { | |
815 | int mask = IB_QP_STATE; | |
816 | struct ib_qp_attr qp_attr; | |
817 | struct bnxt_re_qp *qp; | |
818 | ||
819 | qp_attr.qp_state = IB_QPS_ERR; | |
820 | mutex_lock(&rdev->qp_lock); | |
821 | list_for_each_entry(qp, &rdev->qp_list, list) { | |
822 | /* Modify the state of all QPs except QP1/Shadow QP */ | |
823 | if (!bnxt_re_is_qp1_or_shadow_qp(rdev, qp)) { | |
824 | if (qp->qplib_qp.state != | |
825 | CMDQ_MODIFY_QP_NEW_STATE_RESET && | |
826 | qp->qplib_qp.state != | |
827 | CMDQ_MODIFY_QP_NEW_STATE_ERR) { | |
828 | bnxt_re_dispatch_event(&rdev->ibdev, &qp->ib_qp, | |
829 | 1, IB_EVENT_QP_FATAL); | |
830 | bnxt_re_modify_qp(&qp->ib_qp, &qp_attr, mask, | |
831 | NULL); | |
832 | } | |
833 | } | |
834 | } | |
835 | mutex_unlock(&rdev->qp_lock); | |
836 | } | |
837 | ||
838 | static u32 bnxt_re_get_priority_mask(struct bnxt_re_dev *rdev) | |
839 | { | |
840 | u32 prio_map = 0, tmp_map = 0; | |
841 | struct net_device *netdev; | |
842 | struct dcb_app app; | |
843 | ||
844 | netdev = rdev->netdev; | |
845 | ||
846 | memset(&app, 0, sizeof(app)); | |
847 | app.selector = IEEE_8021QAZ_APP_SEL_ETHERTYPE; | |
848 | app.protocol = ETH_P_IBOE; | |
849 | tmp_map = dcb_ieee_getapp_mask(netdev, &app); | |
850 | prio_map = tmp_map; | |
851 | ||
852 | app.selector = IEEE_8021QAZ_APP_SEL_DGRAM; | |
853 | app.protocol = ROCE_V2_UDP_DPORT; | |
854 | tmp_map = dcb_ieee_getapp_mask(netdev, &app); | |
855 | prio_map |= tmp_map; | |
856 | ||
857 | if (!prio_map) | |
858 | prio_map = -EFAULT; | |
859 | return prio_map; | |
860 | } | |
861 | ||
862 | static void bnxt_re_parse_cid_map(u8 prio_map, u8 *cid_map, u16 *cosq) | |
863 | { | |
864 | u16 prio; | |
865 | u8 id; | |
866 | ||
867 | for (prio = 0, id = 0; prio < 8; prio++) { | |
868 | if (prio_map & (1 << prio)) { | |
869 | cosq[id] = cid_map[prio]; | |
870 | id++; | |
871 | if (id == 2) /* Max 2 tcs supported */ | |
872 | break; | |
873 | } | |
874 | } | |
875 | } | |
876 | ||
877 | static int bnxt_re_setup_qos(struct bnxt_re_dev *rdev) | |
878 | { | |
879 | u8 prio_map = 0; | |
880 | u64 cid_map; | |
881 | int rc; | |
882 | ||
883 | /* Get priority for roce */ | |
884 | rc = bnxt_re_get_priority_mask(rdev); | |
885 | if (rc < 0) | |
886 | return rc; | |
887 | prio_map = (u8)rc; | |
888 | ||
889 | if (prio_map == rdev->cur_prio_map) | |
890 | return 0; | |
891 | rdev->cur_prio_map = prio_map; | |
892 | /* Get cosq id for this priority */ | |
893 | rc = bnxt_re_query_hwrm_pri2cos(rdev, 0, &cid_map); | |
894 | if (rc) { | |
895 | dev_warn(rdev_to_dev(rdev), "no cos for p_mask %x\n", prio_map); | |
896 | return rc; | |
897 | } | |
898 | /* Parse CoS IDs for app priority */ | |
899 | bnxt_re_parse_cid_map(prio_map, (u8 *)&cid_map, rdev->cosq); | |
900 | ||
901 | /* Config BONO. */ | |
902 | rc = bnxt_qplib_map_tc2cos(&rdev->qplib_res, rdev->cosq); | |
903 | if (rc) { | |
904 | dev_warn(rdev_to_dev(rdev), "no tc for cos{%x, %x}\n", | |
905 | rdev->cosq[0], rdev->cosq[1]); | |
906 | return rc; | |
907 | } | |
908 | ||
909 | return 0; | |
910 | } | |
911 | ||
912 | static void bnxt_re_ib_unreg(struct bnxt_re_dev *rdev, bool lock_wait) | |
913 | { | |
914 | int i, rc; | |
915 | ||
916 | if (test_and_clear_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags)) { | |
917 | for (i = 0; i < ARRAY_SIZE(bnxt_re_attributes); i++) | |
918 | device_remove_file(&rdev->ibdev.dev, | |
919 | bnxt_re_attributes[i]); | |
920 | /* Cleanup ib dev */ | |
921 | bnxt_re_unregister_ib(rdev); | |
922 | } | |
923 | if (test_and_clear_bit(BNXT_RE_FLAG_QOS_WORK_REG, &rdev->flags)) | |
924 | cancel_delayed_work(&rdev->worker); | |
925 | ||
926 | bnxt_re_cleanup_res(rdev); | |
927 | bnxt_re_free_res(rdev, lock_wait); | |
928 | ||
929 | if (test_and_clear_bit(BNXT_RE_FLAG_RCFW_CHANNEL_EN, &rdev->flags)) { | |
930 | rc = bnxt_qplib_deinit_rcfw(&rdev->rcfw); | |
931 | if (rc) | |
932 | dev_warn(rdev_to_dev(rdev), | |
933 | "Failed to deinitialize RCFW: %#x", rc); | |
934 | bnxt_re_net_stats_ctx_free(rdev, rdev->qplib_ctx.stats.fw_id, | |
935 | lock_wait); | |
936 | bnxt_qplib_free_ctx(rdev->en_dev->pdev, &rdev->qplib_ctx); | |
937 | bnxt_qplib_disable_rcfw_channel(&rdev->rcfw); | |
938 | bnxt_re_net_ring_free(rdev, rdev->rcfw.creq_ring_id, lock_wait); | |
939 | bnxt_qplib_free_rcfw_channel(&rdev->rcfw); | |
940 | } | |
941 | if (test_and_clear_bit(BNXT_RE_FLAG_GOT_MSIX, &rdev->flags)) { | |
942 | rc = bnxt_re_free_msix(rdev, lock_wait); | |
943 | if (rc) | |
944 | dev_warn(rdev_to_dev(rdev), | |
945 | "Failed to free MSI-X vectors: %#x", rc); | |
946 | } | |
947 | if (test_and_clear_bit(BNXT_RE_FLAG_NETDEV_REGISTERED, &rdev->flags)) { | |
948 | rc = bnxt_re_unregister_netdev(rdev, lock_wait); | |
949 | if (rc) | |
950 | dev_warn(rdev_to_dev(rdev), | |
951 | "Failed to unregister with netdev: %#x", rc); | |
952 | } | |
953 | } | |
954 | ||
955 | static void bnxt_re_set_resource_limits(struct bnxt_re_dev *rdev) | |
956 | { | |
957 | u32 i; | |
958 | ||
959 | rdev->qplib_ctx.qpc_count = BNXT_RE_MAX_QPC_COUNT; | |
960 | rdev->qplib_ctx.mrw_count = BNXT_RE_MAX_MRW_COUNT; | |
961 | rdev->qplib_ctx.srqc_count = BNXT_RE_MAX_SRQC_COUNT; | |
962 | rdev->qplib_ctx.cq_count = BNXT_RE_MAX_CQ_COUNT; | |
963 | for (i = 0; i < MAX_TQM_ALLOC_REQ; i++) | |
964 | rdev->qplib_ctx.tqm_count[i] = | |
965 | rdev->dev_attr.tqm_alloc_reqs[i]; | |
966 | } | |
967 | ||
968 | /* worker thread for polling periodic events. Now used for QoS programming*/ | |
969 | static void bnxt_re_worker(struct work_struct *work) | |
970 | { | |
971 | struct bnxt_re_dev *rdev = container_of(work, struct bnxt_re_dev, | |
972 | worker.work); | |
973 | ||
974 | bnxt_re_setup_qos(rdev); | |
975 | schedule_delayed_work(&rdev->worker, msecs_to_jiffies(30000)); | |
976 | } | |
977 | ||
978 | static int bnxt_re_ib_reg(struct bnxt_re_dev *rdev) | |
979 | { | |
980 | int i, j, rc; | |
981 | ||
982 | /* Registered a new RoCE device instance to netdev */ | |
983 | rc = bnxt_re_register_netdev(rdev); | |
984 | if (rc) { | |
985 | pr_err("Failed to register with netedev: %#x\n", rc); | |
986 | return -EINVAL; | |
987 | } | |
988 | set_bit(BNXT_RE_FLAG_NETDEV_REGISTERED, &rdev->flags); | |
989 | ||
990 | rc = bnxt_re_request_msix(rdev); | |
991 | if (rc) { | |
992 | pr_err("Failed to get MSI-X vectors: %#x\n", rc); | |
993 | rc = -EINVAL; | |
994 | goto fail; | |
995 | } | |
996 | set_bit(BNXT_RE_FLAG_GOT_MSIX, &rdev->flags); | |
997 | ||
998 | /* Establish RCFW Communication Channel to initialize the context | |
999 | * memory for the function and all child VFs | |
1000 | */ | |
1001 | rc = bnxt_qplib_alloc_rcfw_channel(rdev->en_dev->pdev, &rdev->rcfw); | |
1002 | if (rc) | |
1003 | goto fail; | |
1004 | ||
1005 | rc = bnxt_re_net_ring_alloc | |
1006 | (rdev, rdev->rcfw.creq.pbl[PBL_LVL_0].pg_map_arr, | |
1007 | rdev->rcfw.creq.pbl[rdev->rcfw.creq.level].pg_count, | |
1008 | HWRM_RING_ALLOC_CMPL, BNXT_QPLIB_CREQE_MAX_CNT - 1, | |
1009 | rdev->msix_entries[BNXT_RE_AEQ_IDX].ring_idx, | |
1010 | &rdev->rcfw.creq_ring_id); | |
1011 | if (rc) { | |
1012 | pr_err("Failed to allocate CREQ: %#x\n", rc); | |
1013 | goto free_rcfw; | |
1014 | } | |
1015 | rc = bnxt_qplib_enable_rcfw_channel | |
1016 | (rdev->en_dev->pdev, &rdev->rcfw, | |
1017 | rdev->msix_entries[BNXT_RE_AEQ_IDX].vector, | |
1018 | rdev->msix_entries[BNXT_RE_AEQ_IDX].db_offset, | |
1019 | 0, &bnxt_re_aeq_handler); | |
1020 | if (rc) { | |
1021 | pr_err("Failed to enable RCFW channel: %#x\n", rc); | |
1022 | goto free_ring; | |
1023 | } | |
1024 | ||
1025 | rc = bnxt_qplib_get_dev_attr(&rdev->rcfw, &rdev->dev_attr); | |
1026 | if (rc) | |
1027 | goto disable_rcfw; | |
1028 | bnxt_re_set_resource_limits(rdev); | |
1029 | ||
1030 | rc = bnxt_qplib_alloc_ctx(rdev->en_dev->pdev, &rdev->qplib_ctx, 0); | |
1031 | if (rc) { | |
1032 | pr_err("Failed to allocate QPLIB context: %#x\n", rc); | |
1033 | goto disable_rcfw; | |
1034 | } | |
1035 | rc = bnxt_re_net_stats_ctx_alloc(rdev, | |
1036 | rdev->qplib_ctx.stats.dma_map, | |
1037 | &rdev->qplib_ctx.stats.fw_id); | |
1038 | if (rc) { | |
1039 | pr_err("Failed to allocate stats context: %#x\n", rc); | |
1040 | goto free_ctx; | |
1041 | } | |
1042 | ||
1043 | rc = bnxt_qplib_init_rcfw(&rdev->rcfw, &rdev->qplib_ctx, 0); | |
1044 | if (rc) { | |
1045 | pr_err("Failed to initialize RCFW: %#x\n", rc); | |
1046 | goto free_sctx; | |
1047 | } | |
1048 | set_bit(BNXT_RE_FLAG_RCFW_CHANNEL_EN, &rdev->flags); | |
1049 | ||
1050 | /* Resources based on the 'new' device caps */ | |
1051 | rc = bnxt_re_alloc_res(rdev); | |
1052 | if (rc) { | |
1053 | pr_err("Failed to allocate resources: %#x\n", rc); | |
1054 | goto fail; | |
1055 | } | |
1056 | rc = bnxt_re_init_res(rdev); | |
1057 | if (rc) { | |
1058 | pr_err("Failed to initialize resources: %#x\n", rc); | |
1059 | goto fail; | |
1060 | } | |
1061 | ||
1062 | rc = bnxt_re_setup_qos(rdev); | |
1063 | if (rc) | |
1064 | pr_info("RoCE priority not yet configured\n"); | |
1065 | ||
1066 | INIT_DELAYED_WORK(&rdev->worker, bnxt_re_worker); | |
1067 | set_bit(BNXT_RE_FLAG_QOS_WORK_REG, &rdev->flags); | |
1068 | schedule_delayed_work(&rdev->worker, msecs_to_jiffies(30000)); | |
1069 | ||
1070 | /* Register ib dev */ | |
1071 | rc = bnxt_re_register_ib(rdev); | |
1072 | if (rc) { | |
1073 | pr_err("Failed to register with IB: %#x\n", rc); | |
1074 | goto fail; | |
1075 | } | |
1076 | dev_info(rdev_to_dev(rdev), "Device registered successfully"); | |
1077 | for (i = 0; i < ARRAY_SIZE(bnxt_re_attributes); i++) { | |
1078 | rc = device_create_file(&rdev->ibdev.dev, | |
1079 | bnxt_re_attributes[i]); | |
1080 | if (rc) { | |
1081 | dev_err(rdev_to_dev(rdev), | |
1082 | "Failed to create IB sysfs: %#x", rc); | |
1083 | /* Must clean up all created device files */ | |
1084 | for (j = 0; j < i; j++) | |
1085 | device_remove_file(&rdev->ibdev.dev, | |
1086 | bnxt_re_attributes[j]); | |
1087 | bnxt_re_unregister_ib(rdev); | |
1088 | goto fail; | |
1089 | } | |
1090 | } | |
1091 | set_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags); | |
1092 | bnxt_re_dispatch_event(&rdev->ibdev, NULL, 1, IB_EVENT_PORT_ACTIVE); | |
1093 | bnxt_re_dispatch_event(&rdev->ibdev, NULL, 1, IB_EVENT_GID_CHANGE); | |
1094 | ||
1095 | return 0; | |
1096 | free_sctx: | |
1097 | bnxt_re_net_stats_ctx_free(rdev, rdev->qplib_ctx.stats.fw_id, true); | |
1098 | free_ctx: | |
1099 | bnxt_qplib_free_ctx(rdev->en_dev->pdev, &rdev->qplib_ctx); | |
1100 | disable_rcfw: | |
1101 | bnxt_qplib_disable_rcfw_channel(&rdev->rcfw); | |
1102 | free_ring: | |
1103 | bnxt_re_net_ring_free(rdev, rdev->rcfw.creq_ring_id, true); | |
1104 | free_rcfw: | |
1105 | bnxt_qplib_free_rcfw_channel(&rdev->rcfw); | |
1106 | fail: | |
1107 | bnxt_re_ib_unreg(rdev, true); | |
1108 | return rc; | |
1109 | } | |
1110 | ||
1111 | static void bnxt_re_dev_unreg(struct bnxt_re_dev *rdev) | |
1112 | { | |
1113 | struct bnxt_en_dev *en_dev = rdev->en_dev; | |
1114 | struct net_device *netdev = rdev->netdev; | |
1115 | ||
1116 | bnxt_re_dev_remove(rdev); | |
1117 | ||
1118 | if (netdev) | |
1119 | bnxt_re_dev_unprobe(netdev, en_dev); | |
1120 | } | |
1121 | ||
1122 | static int bnxt_re_dev_reg(struct bnxt_re_dev **rdev, struct net_device *netdev) | |
1123 | { | |
1124 | struct bnxt_en_dev *en_dev; | |
1125 | int rc = 0; | |
1126 | ||
1127 | if (!is_bnxt_re_dev(netdev)) | |
1128 | return -ENODEV; | |
1129 | ||
1130 | en_dev = bnxt_re_dev_probe(netdev); | |
1131 | if (IS_ERR(en_dev)) { | |
1132 | if (en_dev != ERR_PTR(-ENODEV)) | |
1133 | pr_err("%s: Failed to probe\n", ROCE_DRV_MODULE_NAME); | |
1134 | rc = PTR_ERR(en_dev); | |
1135 | goto exit; | |
1136 | } | |
1137 | *rdev = bnxt_re_dev_add(netdev, en_dev); | |
1138 | if (!*rdev) { | |
1139 | rc = -ENOMEM; | |
1140 | bnxt_re_dev_unprobe(netdev, en_dev); | |
1141 | goto exit; | |
1142 | } | |
1143 | exit: | |
1144 | return rc; | |
1145 | } | |
1146 | ||
1147 | static void bnxt_re_remove_one(struct bnxt_re_dev *rdev) | |
1148 | { | |
1149 | pci_dev_put(rdev->en_dev->pdev); | |
1150 | } | |
1151 | ||
1152 | /* Handle all deferred netevents tasks */ | |
1153 | static void bnxt_re_task(struct work_struct *work) | |
1154 | { | |
1155 | struct bnxt_re_work *re_work; | |
1156 | struct bnxt_re_dev *rdev; | |
1157 | int rc = 0; | |
1158 | ||
1159 | re_work = container_of(work, struct bnxt_re_work, work); | |
1160 | rdev = re_work->rdev; | |
1161 | ||
1162 | if (re_work->event != NETDEV_REGISTER && | |
1163 | !test_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags)) | |
1164 | return; | |
1165 | ||
1166 | switch (re_work->event) { | |
1167 | case NETDEV_REGISTER: | |
1168 | rc = bnxt_re_ib_reg(rdev); | |
1169 | if (rc) | |
1170 | dev_err(rdev_to_dev(rdev), | |
1171 | "Failed to register with IB: %#x", rc); | |
1172 | break; | |
1173 | case NETDEV_UP: | |
1174 | bnxt_re_dispatch_event(&rdev->ibdev, NULL, 1, | |
1175 | IB_EVENT_PORT_ACTIVE); | |
1176 | break; | |
1177 | case NETDEV_DOWN: | |
1178 | bnxt_re_dev_stop(rdev); | |
1179 | break; | |
1180 | case NETDEV_CHANGE: | |
1181 | if (!netif_carrier_ok(rdev->netdev)) | |
1182 | bnxt_re_dev_stop(rdev); | |
1183 | else if (netif_carrier_ok(rdev->netdev)) | |
1184 | bnxt_re_dispatch_event(&rdev->ibdev, NULL, 1, | |
1185 | IB_EVENT_PORT_ACTIVE); | |
1186 | break; | |
1187 | default: | |
1188 | break; | |
1189 | } | |
1190 | kfree(re_work); | |
1191 | } | |
1192 | ||
1193 | static void bnxt_re_init_one(struct bnxt_re_dev *rdev) | |
1194 | { | |
1195 | pci_dev_get(rdev->en_dev->pdev); | |
1196 | } | |
1197 | ||
1198 | /* | |
1199 | * "Notifier chain callback can be invoked for the same chain from | |
1200 | * different CPUs at the same time". | |
1201 | * | |
1202 | * For cases when the netdev is already present, our call to the | |
1203 | * register_netdevice_notifier() will actually get the rtnl_lock() | |
1204 | * before sending NETDEV_REGISTER and (if up) NETDEV_UP | |
1205 | * events. | |
1206 | * | |
1207 | * But for cases when the netdev is not already present, the notifier | |
1208 | * chain is subjected to be invoked from different CPUs simultaneously. | |
1209 | * | |
1210 | * This is protected by the netdev_mutex. | |
1211 | */ | |
1212 | static int bnxt_re_netdev_event(struct notifier_block *notifier, | |
1213 | unsigned long event, void *ptr) | |
1214 | { | |
1215 | struct net_device *real_dev, *netdev = netdev_notifier_info_to_dev(ptr); | |
1216 | struct bnxt_re_work *re_work; | |
1217 | struct bnxt_re_dev *rdev; | |
1218 | int rc = 0; | |
1219 | bool sch_work = false; | |
1220 | ||
1221 | real_dev = rdma_vlan_dev_real_dev(netdev); | |
1222 | if (!real_dev) | |
1223 | real_dev = netdev; | |
1224 | ||
1225 | rdev = bnxt_re_from_netdev(real_dev); | |
1226 | if (!rdev && event != NETDEV_REGISTER) | |
1227 | goto exit; | |
1228 | if (real_dev != netdev) | |
1229 | goto exit; | |
1230 | ||
1231 | switch (event) { | |
1232 | case NETDEV_REGISTER: | |
1233 | if (rdev) | |
1234 | break; | |
1235 | rc = bnxt_re_dev_reg(&rdev, real_dev); | |
1236 | if (rc == -ENODEV) | |
1237 | break; | |
1238 | if (rc) { | |
1239 | pr_err("Failed to register with the device %s: %#x\n", | |
1240 | real_dev->name, rc); | |
1241 | break; | |
1242 | } | |
1243 | bnxt_re_init_one(rdev); | |
1244 | sch_work = true; | |
1245 | break; | |
1246 | ||
1247 | case NETDEV_UNREGISTER: | |
1248 | bnxt_re_ib_unreg(rdev, false); | |
1249 | bnxt_re_remove_one(rdev); | |
1250 | bnxt_re_dev_unreg(rdev); | |
1251 | break; | |
1252 | ||
1253 | default: | |
1254 | sch_work = true; | |
1255 | break; | |
1256 | } | |
1257 | if (sch_work) { | |
1258 | /* Allocate for the deferred task */ | |
1259 | re_work = kzalloc(sizeof(*re_work), GFP_ATOMIC); | |
1260 | if (re_work) { | |
1261 | re_work->rdev = rdev; | |
1262 | re_work->event = event; | |
1263 | re_work->vlan_dev = (real_dev == netdev ? | |
1264 | NULL : netdev); | |
1265 | INIT_WORK(&re_work->work, bnxt_re_task); | |
1266 | queue_work(bnxt_re_wq, &re_work->work); | |
1267 | } | |
1268 | } | |
1269 | ||
1270 | exit: | |
1271 | return NOTIFY_DONE; | |
1272 | } | |
1273 | ||
1274 | static struct notifier_block bnxt_re_netdev_notifier = { | |
1275 | .notifier_call = bnxt_re_netdev_event | |
1276 | }; | |
1277 | ||
1278 | static int __init bnxt_re_mod_init(void) | |
1279 | { | |
1280 | int rc = 0; | |
1281 | ||
1282 | pr_info("%s: %s", ROCE_DRV_MODULE_NAME, version); | |
1283 | ||
1284 | bnxt_re_wq = create_singlethread_workqueue("bnxt_re"); | |
1285 | if (!bnxt_re_wq) | |
1286 | return -ENOMEM; | |
1287 | ||
1288 | INIT_LIST_HEAD(&bnxt_re_dev_list); | |
1289 | ||
1290 | rc = register_netdevice_notifier(&bnxt_re_netdev_notifier); | |
1291 | if (rc) { | |
1292 | pr_err("%s: Cannot register to netdevice_notifier", | |
1293 | ROCE_DRV_MODULE_NAME); | |
1294 | goto err_netdev; | |
1295 | } | |
1296 | return 0; | |
1297 | ||
1298 | err_netdev: | |
1299 | destroy_workqueue(bnxt_re_wq); | |
1300 | ||
1301 | return rc; | |
1302 | } | |
1303 | ||
1304 | static void __exit bnxt_re_mod_exit(void) | |
1305 | { | |
1306 | unregister_netdevice_notifier(&bnxt_re_netdev_notifier); | |
1307 | if (bnxt_re_wq) | |
1308 | destroy_workqueue(bnxt_re_wq); | |
1309 | } | |
1310 | ||
1311 | module_init(bnxt_re_mod_init); | |
1312 | module_exit(bnxt_re_mod_exit); |