]>
Commit | Line | Data |
---|---|---|
1ac5a404 SX |
1 | /* |
2 | * Broadcom NetXtreme-E RoCE driver. | |
3 | * | |
4 | * Copyright (c) 2016 - 2017, Broadcom. All rights reserved. The term | |
5 | * Broadcom refers to Broadcom Limited and/or its subsidiaries. | |
6 | * | |
7 | * This software is available to you under a choice of one of two | |
8 | * licenses. You may choose to be licensed under the terms of the GNU | |
9 | * General Public License (GPL) Version 2, available from the file | |
10 | * COPYING in the main directory of this source tree, or the | |
11 | * BSD license below: | |
12 | * | |
13 | * Redistribution and use in source and binary forms, with or without | |
14 | * modification, are permitted provided that the following conditions | |
15 | * are met: | |
16 | * | |
17 | * 1. Redistributions of source code must retain the above copyright | |
18 | * notice, this list of conditions and the following disclaimer. | |
19 | * 2. Redistributions in binary form must reproduce the above copyright | |
20 | * notice, this list of conditions and the following disclaimer in | |
21 | * the documentation and/or other materials provided with the | |
22 | * distribution. | |
23 | * | |
24 | * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' | |
25 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, | |
26 | * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR | |
27 | * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS | |
28 | * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR | |
29 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF | |
30 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR | |
31 | * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, | |
32 | * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE | |
33 | * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN | |
34 | * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | |
35 | * | |
36 | * Description: Main component of the bnxt_re driver | |
37 | */ | |
38 | ||
39 | #include <linux/module.h> | |
40 | #include <linux/netdevice.h> | |
41 | #include <linux/ethtool.h> | |
42 | #include <linux/mutex.h> | |
43 | #include <linux/list.h> | |
44 | #include <linux/rculist.h> | |
45 | #include <linux/spinlock.h> | |
46 | #include <linux/pci.h> | |
47 | #include <net/dcbnl.h> | |
48 | #include <net/ipv6.h> | |
49 | #include <net/addrconf.h> | |
50 | #include <linux/if_ether.h> | |
51 | ||
52 | #include <rdma/ib_verbs.h> | |
53 | #include <rdma/ib_user_verbs.h> | |
54 | #include <rdma/ib_umem.h> | |
55 | #include <rdma/ib_addr.h> | |
56 | ||
57 | #include "bnxt_ulp.h" | |
58 | #include "roce_hsi.h" | |
59 | #include "qplib_res.h" | |
60 | #include "qplib_sp.h" | |
61 | #include "qplib_fp.h" | |
62 | #include "qplib_rcfw.h" | |
63 | #include "bnxt_re.h" | |
64 | #include "ib_verbs.h" | |
65 | #include <rdma/bnxt_re-abi.h> | |
66 | #include "bnxt.h" | |
67 | static char version[] = | |
68 | BNXT_RE_DESC " v" ROCE_DRV_MODULE_VERSION "\n"; | |
69 | ||
70 | MODULE_AUTHOR("Eddie Wai <eddie.wai@broadcom.com>"); | |
71 | MODULE_DESCRIPTION(BNXT_RE_DESC " Driver"); | |
72 | MODULE_LICENSE("Dual BSD/GPL"); | |
73 | MODULE_VERSION(ROCE_DRV_MODULE_VERSION); | |
74 | ||
75 | /* globals */ | |
76 | static struct list_head bnxt_re_dev_list = LIST_HEAD_INIT(bnxt_re_dev_list); | |
77 | /* Mutex to protect the list of bnxt_re devices added */ | |
78 | static DEFINE_MUTEX(bnxt_re_dev_lock); | |
79 | static struct workqueue_struct *bnxt_re_wq; | |
80 | ||
81 | /* for handling bnxt_en callbacks later */ | |
82 | static void bnxt_re_stop(void *p) | |
83 | { | |
84 | } | |
85 | ||
86 | static void bnxt_re_start(void *p) | |
87 | { | |
88 | } | |
89 | ||
90 | static void bnxt_re_sriov_config(void *p, int num_vfs) | |
91 | { | |
92 | } | |
93 | ||
94 | static struct bnxt_ulp_ops bnxt_re_ulp_ops = { | |
95 | .ulp_async_notifier = NULL, | |
96 | .ulp_stop = bnxt_re_stop, | |
97 | .ulp_start = bnxt_re_start, | |
98 | .ulp_sriov_config = bnxt_re_sriov_config | |
99 | }; | |
100 | ||
101 | /* RoCE -> Net driver */ | |
102 | ||
103 | /* Driver registration routines used to let the networking driver (bnxt_en) | |
104 | * to know that the RoCE driver is now installed | |
105 | */ | |
106 | static int bnxt_re_unregister_netdev(struct bnxt_re_dev *rdev, bool lock_wait) | |
107 | { | |
108 | struct bnxt_en_dev *en_dev; | |
109 | int rc; | |
110 | ||
111 | if (!rdev) | |
112 | return -EINVAL; | |
113 | ||
114 | en_dev = rdev->en_dev; | |
115 | /* Acquire rtnl lock if it is not invokded from netdev event */ | |
116 | if (lock_wait) | |
117 | rtnl_lock(); | |
118 | ||
119 | rc = en_dev->en_ops->bnxt_unregister_device(rdev->en_dev, | |
120 | BNXT_ROCE_ULP); | |
121 | if (lock_wait) | |
122 | rtnl_unlock(); | |
123 | return rc; | |
124 | } | |
125 | ||
126 | static int bnxt_re_register_netdev(struct bnxt_re_dev *rdev) | |
127 | { | |
128 | struct bnxt_en_dev *en_dev; | |
129 | int rc = 0; | |
130 | ||
131 | if (!rdev) | |
132 | return -EINVAL; | |
133 | ||
134 | en_dev = rdev->en_dev; | |
135 | ||
136 | rtnl_lock(); | |
137 | rc = en_dev->en_ops->bnxt_register_device(en_dev, BNXT_ROCE_ULP, | |
138 | &bnxt_re_ulp_ops, rdev); | |
139 | rtnl_unlock(); | |
140 | return rc; | |
141 | } | |
142 | ||
143 | static int bnxt_re_free_msix(struct bnxt_re_dev *rdev, bool lock_wait) | |
144 | { | |
145 | struct bnxt_en_dev *en_dev; | |
146 | int rc; | |
147 | ||
148 | if (!rdev) | |
149 | return -EINVAL; | |
150 | ||
151 | en_dev = rdev->en_dev; | |
152 | ||
153 | if (lock_wait) | |
154 | rtnl_lock(); | |
155 | ||
156 | rc = en_dev->en_ops->bnxt_free_msix(rdev->en_dev, BNXT_ROCE_ULP); | |
157 | ||
158 | if (lock_wait) | |
159 | rtnl_unlock(); | |
160 | return rc; | |
161 | } | |
162 | ||
163 | static int bnxt_re_request_msix(struct bnxt_re_dev *rdev) | |
164 | { | |
165 | int rc = 0, num_msix_want = BNXT_RE_MIN_MSIX, num_msix_got; | |
166 | struct bnxt_en_dev *en_dev; | |
167 | ||
168 | if (!rdev) | |
169 | return -EINVAL; | |
170 | ||
171 | en_dev = rdev->en_dev; | |
172 | ||
173 | rtnl_lock(); | |
174 | num_msix_got = en_dev->en_ops->bnxt_request_msix(en_dev, BNXT_ROCE_ULP, | |
175 | rdev->msix_entries, | |
176 | num_msix_want); | |
177 | if (num_msix_got < BNXT_RE_MIN_MSIX) { | |
178 | rc = -EINVAL; | |
179 | goto done; | |
180 | } | |
181 | if (num_msix_got != num_msix_want) { | |
182 | dev_warn(rdev_to_dev(rdev), | |
183 | "Requested %d MSI-X vectors, got %d\n", | |
184 | num_msix_want, num_msix_got); | |
185 | } | |
186 | rdev->num_msix = num_msix_got; | |
187 | done: | |
188 | rtnl_unlock(); | |
189 | return rc; | |
190 | } | |
191 | ||
192 | static void bnxt_re_init_hwrm_hdr(struct bnxt_re_dev *rdev, struct input *hdr, | |
193 | u16 opcd, u16 crid, u16 trid) | |
194 | { | |
195 | hdr->req_type = cpu_to_le16(opcd); | |
196 | hdr->cmpl_ring = cpu_to_le16(crid); | |
197 | hdr->target_id = cpu_to_le16(trid); | |
198 | } | |
199 | ||
200 | static void bnxt_re_fill_fw_msg(struct bnxt_fw_msg *fw_msg, void *msg, | |
201 | int msg_len, void *resp, int resp_max_len, | |
202 | int timeout) | |
203 | { | |
204 | fw_msg->msg = msg; | |
205 | fw_msg->msg_len = msg_len; | |
206 | fw_msg->resp = resp; | |
207 | fw_msg->resp_max_len = resp_max_len; | |
208 | fw_msg->timeout = timeout; | |
209 | } | |
210 | ||
211 | static int bnxt_re_net_ring_free(struct bnxt_re_dev *rdev, u16 fw_ring_id, | |
212 | bool lock_wait) | |
213 | { | |
214 | struct bnxt_en_dev *en_dev = rdev->en_dev; | |
215 | struct hwrm_ring_free_input req = {0}; | |
216 | struct hwrm_ring_free_output resp; | |
217 | struct bnxt_fw_msg fw_msg; | |
218 | bool do_unlock = false; | |
219 | int rc = -EINVAL; | |
220 | ||
221 | if (!en_dev) | |
222 | return rc; | |
223 | ||
224 | memset(&fw_msg, 0, sizeof(fw_msg)); | |
225 | if (lock_wait) { | |
226 | rtnl_lock(); | |
227 | do_unlock = true; | |
228 | } | |
229 | ||
230 | bnxt_re_init_hwrm_hdr(rdev, (void *)&req, HWRM_RING_FREE, -1, -1); | |
db690328 | 231 | req.ring_type = RING_ALLOC_REQ_RING_TYPE_L2_CMPL; |
1ac5a404 SX |
232 | req.ring_id = cpu_to_le16(fw_ring_id); |
233 | bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, | |
234 | sizeof(resp), DFLT_HWRM_CMD_TIMEOUT); | |
235 | rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); | |
236 | if (rc) | |
237 | dev_err(rdev_to_dev(rdev), | |
238 | "Failed to free HW ring:%d :%#x", req.ring_id, rc); | |
239 | if (do_unlock) | |
240 | rtnl_unlock(); | |
241 | return rc; | |
242 | } | |
243 | ||
244 | static int bnxt_re_net_ring_alloc(struct bnxt_re_dev *rdev, dma_addr_t *dma_arr, | |
245 | int pages, int type, u32 ring_mask, | |
246 | u32 map_index, u16 *fw_ring_id) | |
247 | { | |
248 | struct bnxt_en_dev *en_dev = rdev->en_dev; | |
249 | struct hwrm_ring_alloc_input req = {0}; | |
250 | struct hwrm_ring_alloc_output resp; | |
251 | struct bnxt_fw_msg fw_msg; | |
252 | int rc = -EINVAL; | |
253 | ||
254 | if (!en_dev) | |
255 | return rc; | |
256 | ||
257 | memset(&fw_msg, 0, sizeof(fw_msg)); | |
258 | rtnl_lock(); | |
259 | bnxt_re_init_hwrm_hdr(rdev, (void *)&req, HWRM_RING_ALLOC, -1, -1); | |
260 | req.enables = 0; | |
261 | req.page_tbl_addr = cpu_to_le64(dma_arr[0]); | |
262 | if (pages > 1) { | |
263 | /* Page size is in log2 units */ | |
264 | req.page_size = BNXT_PAGE_SHIFT; | |
265 | req.page_tbl_depth = 1; | |
266 | } | |
267 | req.fbo = 0; | |
268 | /* Association of ring index with doorbell index and MSIX number */ | |
269 | req.logical_id = cpu_to_le16(map_index); | |
270 | req.length = cpu_to_le32(ring_mask + 1); | |
db690328 | 271 | req.ring_type = RING_ALLOC_REQ_RING_TYPE_L2_CMPL; |
1ac5a404 SX |
272 | req.int_mode = RING_ALLOC_REQ_INT_MODE_MSIX; |
273 | bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, | |
274 | sizeof(resp), DFLT_HWRM_CMD_TIMEOUT); | |
275 | rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); | |
276 | if (!rc) | |
277 | *fw_ring_id = le16_to_cpu(resp.ring_id); | |
278 | ||
279 | rtnl_unlock(); | |
280 | return rc; | |
281 | } | |
282 | ||
283 | static int bnxt_re_net_stats_ctx_free(struct bnxt_re_dev *rdev, | |
284 | u32 fw_stats_ctx_id, bool lock_wait) | |
285 | { | |
286 | struct bnxt_en_dev *en_dev = rdev->en_dev; | |
287 | struct hwrm_stat_ctx_free_input req = {0}; | |
288 | struct bnxt_fw_msg fw_msg; | |
289 | bool do_unlock = false; | |
290 | int rc = -EINVAL; | |
291 | ||
292 | if (!en_dev) | |
293 | return rc; | |
294 | ||
295 | memset(&fw_msg, 0, sizeof(fw_msg)); | |
296 | if (lock_wait) { | |
297 | rtnl_lock(); | |
298 | do_unlock = true; | |
299 | } | |
300 | ||
301 | bnxt_re_init_hwrm_hdr(rdev, (void *)&req, HWRM_STAT_CTX_FREE, -1, -1); | |
302 | req.stat_ctx_id = cpu_to_le32(fw_stats_ctx_id); | |
303 | bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&req, | |
304 | sizeof(req), DFLT_HWRM_CMD_TIMEOUT); | |
305 | rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); | |
306 | if (rc) | |
307 | dev_err(rdev_to_dev(rdev), | |
308 | "Failed to free HW stats context %#x", rc); | |
309 | ||
310 | if (do_unlock) | |
311 | rtnl_unlock(); | |
312 | return rc; | |
313 | } | |
314 | ||
315 | static int bnxt_re_net_stats_ctx_alloc(struct bnxt_re_dev *rdev, | |
316 | dma_addr_t dma_map, | |
317 | u32 *fw_stats_ctx_id) | |
318 | { | |
319 | struct hwrm_stat_ctx_alloc_output resp = {0}; | |
320 | struct hwrm_stat_ctx_alloc_input req = {0}; | |
321 | struct bnxt_en_dev *en_dev = rdev->en_dev; | |
322 | struct bnxt_fw_msg fw_msg; | |
323 | int rc = -EINVAL; | |
324 | ||
325 | *fw_stats_ctx_id = INVALID_STATS_CTX_ID; | |
326 | ||
327 | if (!en_dev) | |
328 | return rc; | |
329 | ||
330 | memset(&fw_msg, 0, sizeof(fw_msg)); | |
331 | rtnl_lock(); | |
332 | ||
333 | bnxt_re_init_hwrm_hdr(rdev, (void *)&req, HWRM_STAT_CTX_ALLOC, -1, -1); | |
334 | req.update_period_ms = cpu_to_le32(1000); | |
335 | req.stats_dma_addr = cpu_to_le64(dma_map); | |
336 | bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, | |
337 | sizeof(resp), DFLT_HWRM_CMD_TIMEOUT); | |
338 | rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); | |
339 | if (!rc) | |
340 | *fw_stats_ctx_id = le32_to_cpu(resp.stat_ctx_id); | |
341 | ||
342 | rtnl_unlock(); | |
343 | return rc; | |
344 | } | |
345 | ||
346 | /* Device */ | |
347 | ||
348 | static bool is_bnxt_re_dev(struct net_device *netdev) | |
349 | { | |
350 | struct ethtool_drvinfo drvinfo; | |
351 | ||
352 | if (netdev->ethtool_ops && netdev->ethtool_ops->get_drvinfo) { | |
353 | memset(&drvinfo, 0, sizeof(drvinfo)); | |
354 | netdev->ethtool_ops->get_drvinfo(netdev, &drvinfo); | |
355 | ||
356 | if (strcmp(drvinfo.driver, "bnxt_en")) | |
357 | return false; | |
358 | return true; | |
359 | } | |
360 | return false; | |
361 | } | |
362 | ||
363 | static struct bnxt_re_dev *bnxt_re_from_netdev(struct net_device *netdev) | |
364 | { | |
365 | struct bnxt_re_dev *rdev; | |
366 | ||
367 | rcu_read_lock(); | |
368 | list_for_each_entry_rcu(rdev, &bnxt_re_dev_list, list) { | |
369 | if (rdev->netdev == netdev) { | |
370 | rcu_read_unlock(); | |
371 | return rdev; | |
372 | } | |
373 | } | |
374 | rcu_read_unlock(); | |
375 | return NULL; | |
376 | } | |
377 | ||
378 | static void bnxt_re_dev_unprobe(struct net_device *netdev, | |
379 | struct bnxt_en_dev *en_dev) | |
380 | { | |
381 | dev_put(netdev); | |
382 | module_put(en_dev->pdev->driver->driver.owner); | |
383 | } | |
384 | ||
385 | static struct bnxt_en_dev *bnxt_re_dev_probe(struct net_device *netdev) | |
386 | { | |
387 | struct bnxt *bp = netdev_priv(netdev); | |
388 | struct bnxt_en_dev *en_dev; | |
389 | struct pci_dev *pdev; | |
390 | ||
391 | /* Call bnxt_en's RoCE probe via indirect API */ | |
392 | if (!bp->ulp_probe) | |
393 | return ERR_PTR(-EINVAL); | |
394 | ||
395 | en_dev = bp->ulp_probe(netdev); | |
396 | if (IS_ERR(en_dev)) | |
397 | return en_dev; | |
398 | ||
399 | pdev = en_dev->pdev; | |
400 | if (!pdev) | |
401 | return ERR_PTR(-EINVAL); | |
402 | ||
403 | if (!(en_dev->flags & BNXT_EN_FLAG_ROCE_CAP)) { | |
404 | dev_dbg(&pdev->dev, | |
405 | "%s: probe error: RoCE is not supported on this device", | |
406 | ROCE_DRV_MODULE_NAME); | |
407 | return ERR_PTR(-ENODEV); | |
408 | } | |
409 | ||
410 | /* Bump net device reference count */ | |
411 | if (!try_module_get(pdev->driver->driver.owner)) | |
412 | return ERR_PTR(-ENODEV); | |
413 | ||
414 | dev_hold(netdev); | |
415 | ||
416 | return en_dev; | |
417 | } | |
418 | ||
419 | static void bnxt_re_unregister_ib(struct bnxt_re_dev *rdev) | |
420 | { | |
421 | ib_unregister_device(&rdev->ibdev); | |
422 | } | |
423 | ||
424 | static int bnxt_re_register_ib(struct bnxt_re_dev *rdev) | |
425 | { | |
426 | struct ib_device *ibdev = &rdev->ibdev; | |
427 | ||
428 | /* ib device init */ | |
429 | ibdev->owner = THIS_MODULE; | |
430 | ibdev->node_type = RDMA_NODE_IB_CA; | |
431 | strlcpy(ibdev->name, "bnxt_re%d", IB_DEVICE_NAME_MAX); | |
432 | strlcpy(ibdev->node_desc, BNXT_RE_DESC " HCA", | |
433 | strlen(BNXT_RE_DESC) + 5); | |
434 | ibdev->phys_port_cnt = 1; | |
435 | ||
436 | bnxt_qplib_get_guid(rdev->netdev->dev_addr, (u8 *)&ibdev->node_guid); | |
437 | ||
438 | ibdev->num_comp_vectors = 1; | |
ac1820fb | 439 | ibdev->dev.parent = &rdev->en_dev->pdev->dev; |
1ac5a404 SX |
440 | ibdev->local_dma_lkey = BNXT_QPLIB_RSVD_LKEY; |
441 | ||
442 | /* User space */ | |
443 | ibdev->uverbs_abi_ver = BNXT_RE_ABI_VERSION; | |
444 | ibdev->uverbs_cmd_mask = | |
445 | (1ull << IB_USER_VERBS_CMD_GET_CONTEXT) | | |
446 | (1ull << IB_USER_VERBS_CMD_QUERY_DEVICE) | | |
447 | (1ull << IB_USER_VERBS_CMD_QUERY_PORT) | | |
448 | (1ull << IB_USER_VERBS_CMD_ALLOC_PD) | | |
449 | (1ull << IB_USER_VERBS_CMD_DEALLOC_PD) | | |
450 | (1ull << IB_USER_VERBS_CMD_REG_MR) | | |
451 | (1ull << IB_USER_VERBS_CMD_REREG_MR) | | |
452 | (1ull << IB_USER_VERBS_CMD_DEREG_MR) | | |
453 | (1ull << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) | | |
454 | (1ull << IB_USER_VERBS_CMD_CREATE_CQ) | | |
455 | (1ull << IB_USER_VERBS_CMD_RESIZE_CQ) | | |
456 | (1ull << IB_USER_VERBS_CMD_DESTROY_CQ) | | |
457 | (1ull << IB_USER_VERBS_CMD_CREATE_QP) | | |
458 | (1ull << IB_USER_VERBS_CMD_MODIFY_QP) | | |
459 | (1ull << IB_USER_VERBS_CMD_QUERY_QP) | | |
460 | (1ull << IB_USER_VERBS_CMD_DESTROY_QP) | | |
461 | (1ull << IB_USER_VERBS_CMD_CREATE_SRQ) | | |
462 | (1ull << IB_USER_VERBS_CMD_MODIFY_SRQ) | | |
463 | (1ull << IB_USER_VERBS_CMD_QUERY_SRQ) | | |
464 | (1ull << IB_USER_VERBS_CMD_DESTROY_SRQ) | | |
465 | (1ull << IB_USER_VERBS_CMD_CREATE_AH) | | |
466 | (1ull << IB_USER_VERBS_CMD_MODIFY_AH) | | |
467 | (1ull << IB_USER_VERBS_CMD_QUERY_AH) | | |
468 | (1ull << IB_USER_VERBS_CMD_DESTROY_AH); | |
469 | /* POLL_CQ and REQ_NOTIFY_CQ is directly handled in libbnxt_re */ | |
470 | ||
471 | /* Kernel verbs */ | |
472 | ibdev->query_device = bnxt_re_query_device; | |
473 | ibdev->modify_device = bnxt_re_modify_device; | |
474 | ||
475 | ibdev->query_port = bnxt_re_query_port; | |
476 | ibdev->modify_port = bnxt_re_modify_port; | |
477 | ibdev->get_port_immutable = bnxt_re_get_port_immutable; | |
478 | ibdev->query_pkey = bnxt_re_query_pkey; | |
479 | ibdev->query_gid = bnxt_re_query_gid; | |
480 | ibdev->get_netdev = bnxt_re_get_netdev; | |
481 | ibdev->add_gid = bnxt_re_add_gid; | |
482 | ibdev->del_gid = bnxt_re_del_gid; | |
483 | ibdev->get_link_layer = bnxt_re_get_link_layer; | |
484 | ||
485 | ibdev->alloc_pd = bnxt_re_alloc_pd; | |
486 | ibdev->dealloc_pd = bnxt_re_dealloc_pd; | |
487 | ||
488 | ibdev->create_ah = bnxt_re_create_ah; | |
489 | ibdev->modify_ah = bnxt_re_modify_ah; | |
490 | ibdev->query_ah = bnxt_re_query_ah; | |
491 | ibdev->destroy_ah = bnxt_re_destroy_ah; | |
492 | ||
493 | ibdev->create_qp = bnxt_re_create_qp; | |
494 | ibdev->modify_qp = bnxt_re_modify_qp; | |
495 | ibdev->query_qp = bnxt_re_query_qp; | |
496 | ibdev->destroy_qp = bnxt_re_destroy_qp; | |
497 | ||
498 | ibdev->post_send = bnxt_re_post_send; | |
499 | ibdev->post_recv = bnxt_re_post_recv; | |
500 | ||
501 | ibdev->create_cq = bnxt_re_create_cq; | |
502 | ibdev->destroy_cq = bnxt_re_destroy_cq; | |
503 | ibdev->poll_cq = bnxt_re_poll_cq; | |
504 | ibdev->req_notify_cq = bnxt_re_req_notify_cq; | |
505 | ||
506 | ibdev->get_dma_mr = bnxt_re_get_dma_mr; | |
507 | ibdev->dereg_mr = bnxt_re_dereg_mr; | |
508 | ibdev->alloc_mr = bnxt_re_alloc_mr; | |
509 | ibdev->map_mr_sg = bnxt_re_map_mr_sg; | |
1ac5a404 SX |
510 | |
511 | ibdev->reg_user_mr = bnxt_re_reg_user_mr; | |
512 | ibdev->alloc_ucontext = bnxt_re_alloc_ucontext; | |
513 | ibdev->dealloc_ucontext = bnxt_re_dealloc_ucontext; | |
514 | ibdev->mmap = bnxt_re_mmap; | |
515 | ||
516 | return ib_register_device(ibdev, NULL); | |
517 | } | |
518 | ||
519 | static ssize_t show_rev(struct device *device, struct device_attribute *attr, | |
520 | char *buf) | |
521 | { | |
522 | struct bnxt_re_dev *rdev = to_bnxt_re_dev(device, ibdev.dev); | |
523 | ||
524 | return scnprintf(buf, PAGE_SIZE, "0x%x\n", rdev->en_dev->pdev->vendor); | |
525 | } | |
526 | ||
527 | static ssize_t show_fw_ver(struct device *device, struct device_attribute *attr, | |
528 | char *buf) | |
529 | { | |
530 | struct bnxt_re_dev *rdev = to_bnxt_re_dev(device, ibdev.dev); | |
531 | ||
532 | return scnprintf(buf, PAGE_SIZE, "%s\n", rdev->dev_attr.fw_ver); | |
533 | } | |
534 | ||
535 | static ssize_t show_hca(struct device *device, struct device_attribute *attr, | |
536 | char *buf) | |
537 | { | |
538 | struct bnxt_re_dev *rdev = to_bnxt_re_dev(device, ibdev.dev); | |
539 | ||
540 | return scnprintf(buf, PAGE_SIZE, "%s\n", rdev->ibdev.node_desc); | |
541 | } | |
542 | ||
543 | static DEVICE_ATTR(hw_rev, 0444, show_rev, NULL); | |
544 | static DEVICE_ATTR(fw_rev, 0444, show_fw_ver, NULL); | |
545 | static DEVICE_ATTR(hca_type, 0444, show_hca, NULL); | |
546 | ||
547 | static struct device_attribute *bnxt_re_attributes[] = { | |
548 | &dev_attr_hw_rev, | |
549 | &dev_attr_fw_rev, | |
550 | &dev_attr_hca_type | |
551 | }; | |
552 | ||
553 | static void bnxt_re_dev_remove(struct bnxt_re_dev *rdev) | |
554 | { | |
555 | dev_put(rdev->netdev); | |
556 | rdev->netdev = NULL; | |
557 | ||
558 | mutex_lock(&bnxt_re_dev_lock); | |
559 | list_del_rcu(&rdev->list); | |
560 | mutex_unlock(&bnxt_re_dev_lock); | |
561 | ||
562 | synchronize_rcu(); | |
563 | flush_workqueue(bnxt_re_wq); | |
564 | ||
565 | ib_dealloc_device(&rdev->ibdev); | |
566 | /* rdev is gone */ | |
567 | } | |
568 | ||
569 | static struct bnxt_re_dev *bnxt_re_dev_add(struct net_device *netdev, | |
570 | struct bnxt_en_dev *en_dev) | |
571 | { | |
572 | struct bnxt_re_dev *rdev; | |
573 | ||
574 | /* Allocate bnxt_re_dev instance here */ | |
575 | rdev = (struct bnxt_re_dev *)ib_alloc_device(sizeof(*rdev)); | |
576 | if (!rdev) { | |
577 | dev_err(NULL, "%s: bnxt_re_dev allocation failure!", | |
578 | ROCE_DRV_MODULE_NAME); | |
579 | return NULL; | |
580 | } | |
581 | /* Default values */ | |
582 | rdev->netdev = netdev; | |
583 | dev_hold(rdev->netdev); | |
584 | rdev->en_dev = en_dev; | |
585 | rdev->id = rdev->en_dev->pdev->devfn; | |
586 | INIT_LIST_HEAD(&rdev->qp_list); | |
587 | mutex_init(&rdev->qp_lock); | |
588 | atomic_set(&rdev->qp_count, 0); | |
589 | atomic_set(&rdev->cq_count, 0); | |
590 | atomic_set(&rdev->srq_count, 0); | |
591 | atomic_set(&rdev->mr_count, 0); | |
592 | atomic_set(&rdev->mw_count, 0); | |
593 | rdev->cosq[0] = 0xFFFF; | |
594 | rdev->cosq[1] = 0xFFFF; | |
595 | ||
596 | mutex_lock(&bnxt_re_dev_lock); | |
597 | list_add_tail_rcu(&rdev->list, &bnxt_re_dev_list); | |
598 | mutex_unlock(&bnxt_re_dev_lock); | |
599 | return rdev; | |
600 | } | |
601 | ||
602 | static int bnxt_re_aeq_handler(struct bnxt_qplib_rcfw *rcfw, | |
603 | struct creq_func_event *aeqe) | |
604 | { | |
605 | switch (aeqe->event) { | |
606 | case CREQ_FUNC_EVENT_EVENT_TX_WQE_ERROR: | |
607 | break; | |
608 | case CREQ_FUNC_EVENT_EVENT_TX_DATA_ERROR: | |
609 | break; | |
610 | case CREQ_FUNC_EVENT_EVENT_RX_WQE_ERROR: | |
611 | break; | |
612 | case CREQ_FUNC_EVENT_EVENT_RX_DATA_ERROR: | |
613 | break; | |
614 | case CREQ_FUNC_EVENT_EVENT_CQ_ERROR: | |
615 | break; | |
616 | case CREQ_FUNC_EVENT_EVENT_TQM_ERROR: | |
617 | break; | |
618 | case CREQ_FUNC_EVENT_EVENT_CFCQ_ERROR: | |
619 | break; | |
620 | case CREQ_FUNC_EVENT_EVENT_CFCS_ERROR: | |
621 | break; | |
622 | case CREQ_FUNC_EVENT_EVENT_CFCC_ERROR: | |
623 | break; | |
624 | case CREQ_FUNC_EVENT_EVENT_CFCM_ERROR: | |
625 | break; | |
626 | case CREQ_FUNC_EVENT_EVENT_TIM_ERROR: | |
627 | break; | |
628 | default: | |
629 | return -EINVAL; | |
630 | } | |
631 | return 0; | |
632 | } | |
633 | ||
634 | static int bnxt_re_cqn_handler(struct bnxt_qplib_nq *nq, | |
635 | struct bnxt_qplib_cq *handle) | |
636 | { | |
637 | struct bnxt_re_cq *cq = container_of(handle, struct bnxt_re_cq, | |
638 | qplib_cq); | |
639 | ||
640 | if (!cq) { | |
641 | dev_err(NULL, "%s: CQ is NULL, CQN not handled", | |
642 | ROCE_DRV_MODULE_NAME); | |
643 | return -EINVAL; | |
644 | } | |
645 | if (cq->ib_cq.comp_handler) { | |
646 | /* Lock comp_handler? */ | |
647 | (*cq->ib_cq.comp_handler)(&cq->ib_cq, cq->ib_cq.cq_context); | |
648 | } | |
649 | ||
650 | return 0; | |
651 | } | |
652 | ||
653 | static void bnxt_re_cleanup_res(struct bnxt_re_dev *rdev) | |
654 | { | |
655 | if (rdev->nq.hwq.max_elements) | |
656 | bnxt_qplib_disable_nq(&rdev->nq); | |
657 | ||
658 | if (rdev->qplib_res.rcfw) | |
659 | bnxt_qplib_cleanup_res(&rdev->qplib_res); | |
660 | } | |
661 | ||
662 | static int bnxt_re_init_res(struct bnxt_re_dev *rdev) | |
663 | { | |
664 | int rc = 0; | |
665 | ||
666 | bnxt_qplib_init_res(&rdev->qplib_res); | |
667 | ||
668 | if (rdev->msix_entries[BNXT_RE_NQ_IDX].vector <= 0) | |
669 | return -EINVAL; | |
670 | ||
671 | rc = bnxt_qplib_enable_nq(rdev->en_dev->pdev, &rdev->nq, | |
672 | rdev->msix_entries[BNXT_RE_NQ_IDX].vector, | |
673 | rdev->msix_entries[BNXT_RE_NQ_IDX].db_offset, | |
674 | &bnxt_re_cqn_handler, | |
675 | NULL); | |
676 | ||
677 | if (rc) | |
678 | dev_err(rdev_to_dev(rdev), "Failed to enable NQ: %#x", rc); | |
679 | ||
680 | return rc; | |
681 | } | |
682 | ||
683 | static void bnxt_re_free_res(struct bnxt_re_dev *rdev, bool lock_wait) | |
684 | { | |
685 | if (rdev->nq.hwq.max_elements) { | |
686 | bnxt_re_net_ring_free(rdev, rdev->nq.ring_id, lock_wait); | |
687 | bnxt_qplib_free_nq(&rdev->nq); | |
688 | } | |
689 | if (rdev->qplib_res.dpi_tbl.max) { | |
690 | bnxt_qplib_dealloc_dpi(&rdev->qplib_res, | |
691 | &rdev->qplib_res.dpi_tbl, | |
692 | &rdev->dpi_privileged); | |
693 | } | |
694 | if (rdev->qplib_res.rcfw) { | |
695 | bnxt_qplib_free_res(&rdev->qplib_res); | |
696 | rdev->qplib_res.rcfw = NULL; | |
697 | } | |
698 | } | |
699 | ||
700 | static int bnxt_re_alloc_res(struct bnxt_re_dev *rdev) | |
701 | { | |
702 | int rc = 0; | |
703 | ||
704 | /* Configure and allocate resources for qplib */ | |
705 | rdev->qplib_res.rcfw = &rdev->rcfw; | |
706 | rc = bnxt_qplib_get_dev_attr(&rdev->rcfw, &rdev->dev_attr); | |
707 | if (rc) | |
708 | goto fail; | |
709 | ||
710 | rc = bnxt_qplib_alloc_res(&rdev->qplib_res, rdev->en_dev->pdev, | |
711 | rdev->netdev, &rdev->dev_attr); | |
712 | if (rc) | |
713 | goto fail; | |
714 | ||
715 | rc = bnxt_qplib_alloc_dpi(&rdev->qplib_res.dpi_tbl, | |
716 | &rdev->dpi_privileged, | |
717 | rdev); | |
718 | if (rc) | |
719 | goto fail; | |
720 | ||
721 | rdev->nq.hwq.max_elements = BNXT_RE_MAX_CQ_COUNT + | |
722 | BNXT_RE_MAX_SRQC_COUNT + 2; | |
723 | rc = bnxt_qplib_alloc_nq(rdev->en_dev->pdev, &rdev->nq); | |
724 | if (rc) { | |
725 | dev_err(rdev_to_dev(rdev), | |
726 | "Failed to allocate NQ memory: %#x", rc); | |
727 | goto fail; | |
728 | } | |
729 | rc = bnxt_re_net_ring_alloc | |
730 | (rdev, rdev->nq.hwq.pbl[PBL_LVL_0].pg_map_arr, | |
731 | rdev->nq.hwq.pbl[rdev->nq.hwq.level].pg_count, | |
732 | HWRM_RING_ALLOC_CMPL, BNXT_QPLIB_NQE_MAX_CNT - 1, | |
733 | rdev->msix_entries[BNXT_RE_NQ_IDX].ring_idx, | |
734 | &rdev->nq.ring_id); | |
735 | if (rc) { | |
736 | dev_err(rdev_to_dev(rdev), | |
737 | "Failed to allocate NQ ring: %#x", rc); | |
738 | goto free_nq; | |
739 | } | |
740 | return 0; | |
741 | free_nq: | |
742 | bnxt_qplib_free_nq(&rdev->nq); | |
743 | fail: | |
744 | rdev->qplib_res.rcfw = NULL; | |
745 | return rc; | |
746 | } | |
747 | ||
748 | static void bnxt_re_dispatch_event(struct ib_device *ibdev, struct ib_qp *qp, | |
749 | u8 port_num, enum ib_event_type event) | |
750 | { | |
751 | struct ib_event ib_event; | |
752 | ||
753 | ib_event.device = ibdev; | |
754 | if (qp) | |
755 | ib_event.element.qp = qp; | |
756 | else | |
757 | ib_event.element.port_num = port_num; | |
758 | ib_event.event = event; | |
759 | ib_dispatch_event(&ib_event); | |
760 | } | |
761 | ||
762 | #define HWRM_QUEUE_PRI2COS_QCFG_INPUT_FLAGS_IVLAN 0x02 | |
763 | static int bnxt_re_query_hwrm_pri2cos(struct bnxt_re_dev *rdev, u8 dir, | |
764 | u64 *cid_map) | |
765 | { | |
766 | struct hwrm_queue_pri2cos_qcfg_input req = {0}; | |
767 | struct bnxt *bp = netdev_priv(rdev->netdev); | |
768 | struct hwrm_queue_pri2cos_qcfg_output resp; | |
769 | struct bnxt_en_dev *en_dev = rdev->en_dev; | |
770 | struct bnxt_fw_msg fw_msg; | |
771 | u32 flags = 0; | |
772 | u8 *qcfgmap, *tmp_map; | |
773 | int rc = 0, i; | |
774 | ||
775 | if (!cid_map) | |
776 | return -EINVAL; | |
777 | ||
778 | memset(&fw_msg, 0, sizeof(fw_msg)); | |
779 | bnxt_re_init_hwrm_hdr(rdev, (void *)&req, | |
780 | HWRM_QUEUE_PRI2COS_QCFG, -1, -1); | |
781 | flags |= (dir & 0x01); | |
782 | flags |= HWRM_QUEUE_PRI2COS_QCFG_INPUT_FLAGS_IVLAN; | |
783 | req.flags = cpu_to_le32(flags); | |
784 | req.port_id = bp->pf.port_id; | |
785 | ||
786 | bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, | |
787 | sizeof(resp), DFLT_HWRM_CMD_TIMEOUT); | |
788 | rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); | |
789 | if (rc) | |
790 | return rc; | |
791 | ||
792 | if (resp.queue_cfg_info) { | |
793 | dev_warn(rdev_to_dev(rdev), | |
794 | "Asymmetric cos queue configuration detected"); | |
795 | dev_warn(rdev_to_dev(rdev), | |
796 | " on device, QoS may not be fully functional\n"); | |
797 | } | |
798 | qcfgmap = &resp.pri0_cos_queue_id; | |
799 | tmp_map = (u8 *)cid_map; | |
800 | for (i = 0; i < IEEE_8021QAZ_MAX_TCS; i++) | |
801 | tmp_map[i] = qcfgmap[i]; | |
802 | ||
803 | return rc; | |
804 | } | |
805 | ||
806 | static bool bnxt_re_is_qp1_or_shadow_qp(struct bnxt_re_dev *rdev, | |
807 | struct bnxt_re_qp *qp) | |
808 | { | |
809 | return (qp->ib_qp.qp_type == IB_QPT_GSI) || (qp == rdev->qp1_sqp); | |
810 | } | |
811 | ||
812 | static void bnxt_re_dev_stop(struct bnxt_re_dev *rdev) | |
813 | { | |
814 | int mask = IB_QP_STATE; | |
815 | struct ib_qp_attr qp_attr; | |
816 | struct bnxt_re_qp *qp; | |
817 | ||
818 | qp_attr.qp_state = IB_QPS_ERR; | |
819 | mutex_lock(&rdev->qp_lock); | |
820 | list_for_each_entry(qp, &rdev->qp_list, list) { | |
821 | /* Modify the state of all QPs except QP1/Shadow QP */ | |
822 | if (!bnxt_re_is_qp1_or_shadow_qp(rdev, qp)) { | |
823 | if (qp->qplib_qp.state != | |
824 | CMDQ_MODIFY_QP_NEW_STATE_RESET && | |
825 | qp->qplib_qp.state != | |
826 | CMDQ_MODIFY_QP_NEW_STATE_ERR) { | |
827 | bnxt_re_dispatch_event(&rdev->ibdev, &qp->ib_qp, | |
828 | 1, IB_EVENT_QP_FATAL); | |
829 | bnxt_re_modify_qp(&qp->ib_qp, &qp_attr, mask, | |
830 | NULL); | |
831 | } | |
832 | } | |
833 | } | |
834 | mutex_unlock(&rdev->qp_lock); | |
835 | } | |
836 | ||
837 | static u32 bnxt_re_get_priority_mask(struct bnxt_re_dev *rdev) | |
838 | { | |
839 | u32 prio_map = 0, tmp_map = 0; | |
840 | struct net_device *netdev; | |
841 | struct dcb_app app; | |
842 | ||
843 | netdev = rdev->netdev; | |
844 | ||
845 | memset(&app, 0, sizeof(app)); | |
846 | app.selector = IEEE_8021QAZ_APP_SEL_ETHERTYPE; | |
847 | app.protocol = ETH_P_IBOE; | |
848 | tmp_map = dcb_ieee_getapp_mask(netdev, &app); | |
849 | prio_map = tmp_map; | |
850 | ||
851 | app.selector = IEEE_8021QAZ_APP_SEL_DGRAM; | |
852 | app.protocol = ROCE_V2_UDP_DPORT; | |
853 | tmp_map = dcb_ieee_getapp_mask(netdev, &app); | |
854 | prio_map |= tmp_map; | |
855 | ||
856 | if (!prio_map) | |
857 | prio_map = -EFAULT; | |
858 | return prio_map; | |
859 | } | |
860 | ||
861 | static void bnxt_re_parse_cid_map(u8 prio_map, u8 *cid_map, u16 *cosq) | |
862 | { | |
863 | u16 prio; | |
864 | u8 id; | |
865 | ||
866 | for (prio = 0, id = 0; prio < 8; prio++) { | |
867 | if (prio_map & (1 << prio)) { | |
868 | cosq[id] = cid_map[prio]; | |
869 | id++; | |
870 | if (id == 2) /* Max 2 tcs supported */ | |
871 | break; | |
872 | } | |
873 | } | |
874 | } | |
875 | ||
876 | static int bnxt_re_setup_qos(struct bnxt_re_dev *rdev) | |
877 | { | |
878 | u8 prio_map = 0; | |
879 | u64 cid_map; | |
880 | int rc; | |
881 | ||
882 | /* Get priority for roce */ | |
883 | rc = bnxt_re_get_priority_mask(rdev); | |
884 | if (rc < 0) | |
885 | return rc; | |
886 | prio_map = (u8)rc; | |
887 | ||
888 | if (prio_map == rdev->cur_prio_map) | |
889 | return 0; | |
890 | rdev->cur_prio_map = prio_map; | |
891 | /* Get cosq id for this priority */ | |
892 | rc = bnxt_re_query_hwrm_pri2cos(rdev, 0, &cid_map); | |
893 | if (rc) { | |
894 | dev_warn(rdev_to_dev(rdev), "no cos for p_mask %x\n", prio_map); | |
895 | return rc; | |
896 | } | |
897 | /* Parse CoS IDs for app priority */ | |
898 | bnxt_re_parse_cid_map(prio_map, (u8 *)&cid_map, rdev->cosq); | |
899 | ||
900 | /* Config BONO. */ | |
901 | rc = bnxt_qplib_map_tc2cos(&rdev->qplib_res, rdev->cosq); | |
902 | if (rc) { | |
903 | dev_warn(rdev_to_dev(rdev), "no tc for cos{%x, %x}\n", | |
904 | rdev->cosq[0], rdev->cosq[1]); | |
905 | return rc; | |
906 | } | |
907 | ||
908 | return 0; | |
909 | } | |
910 | ||
911 | static void bnxt_re_ib_unreg(struct bnxt_re_dev *rdev, bool lock_wait) | |
912 | { | |
913 | int i, rc; | |
914 | ||
915 | if (test_and_clear_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags)) { | |
916 | for (i = 0; i < ARRAY_SIZE(bnxt_re_attributes); i++) | |
917 | device_remove_file(&rdev->ibdev.dev, | |
918 | bnxt_re_attributes[i]); | |
919 | /* Cleanup ib dev */ | |
920 | bnxt_re_unregister_ib(rdev); | |
921 | } | |
922 | if (test_and_clear_bit(BNXT_RE_FLAG_QOS_WORK_REG, &rdev->flags)) | |
923 | cancel_delayed_work(&rdev->worker); | |
924 | ||
925 | bnxt_re_cleanup_res(rdev); | |
926 | bnxt_re_free_res(rdev, lock_wait); | |
927 | ||
928 | if (test_and_clear_bit(BNXT_RE_FLAG_RCFW_CHANNEL_EN, &rdev->flags)) { | |
929 | rc = bnxt_qplib_deinit_rcfw(&rdev->rcfw); | |
930 | if (rc) | |
931 | dev_warn(rdev_to_dev(rdev), | |
932 | "Failed to deinitialize RCFW: %#x", rc); | |
933 | bnxt_re_net_stats_ctx_free(rdev, rdev->qplib_ctx.stats.fw_id, | |
934 | lock_wait); | |
935 | bnxt_qplib_free_ctx(rdev->en_dev->pdev, &rdev->qplib_ctx); | |
936 | bnxt_qplib_disable_rcfw_channel(&rdev->rcfw); | |
937 | bnxt_re_net_ring_free(rdev, rdev->rcfw.creq_ring_id, lock_wait); | |
938 | bnxt_qplib_free_rcfw_channel(&rdev->rcfw); | |
939 | } | |
940 | if (test_and_clear_bit(BNXT_RE_FLAG_GOT_MSIX, &rdev->flags)) { | |
941 | rc = bnxt_re_free_msix(rdev, lock_wait); | |
942 | if (rc) | |
943 | dev_warn(rdev_to_dev(rdev), | |
944 | "Failed to free MSI-X vectors: %#x", rc); | |
945 | } | |
946 | if (test_and_clear_bit(BNXT_RE_FLAG_NETDEV_REGISTERED, &rdev->flags)) { | |
947 | rc = bnxt_re_unregister_netdev(rdev, lock_wait); | |
948 | if (rc) | |
949 | dev_warn(rdev_to_dev(rdev), | |
950 | "Failed to unregister with netdev: %#x", rc); | |
951 | } | |
952 | } | |
953 | ||
954 | static void bnxt_re_set_resource_limits(struct bnxt_re_dev *rdev) | |
955 | { | |
956 | u32 i; | |
957 | ||
958 | rdev->qplib_ctx.qpc_count = BNXT_RE_MAX_QPC_COUNT; | |
959 | rdev->qplib_ctx.mrw_count = BNXT_RE_MAX_MRW_COUNT; | |
960 | rdev->qplib_ctx.srqc_count = BNXT_RE_MAX_SRQC_COUNT; | |
961 | rdev->qplib_ctx.cq_count = BNXT_RE_MAX_CQ_COUNT; | |
962 | for (i = 0; i < MAX_TQM_ALLOC_REQ; i++) | |
963 | rdev->qplib_ctx.tqm_count[i] = | |
964 | rdev->dev_attr.tqm_alloc_reqs[i]; | |
965 | } | |
966 | ||
967 | /* worker thread for polling periodic events. Now used for QoS programming*/ | |
968 | static void bnxt_re_worker(struct work_struct *work) | |
969 | { | |
970 | struct bnxt_re_dev *rdev = container_of(work, struct bnxt_re_dev, | |
971 | worker.work); | |
972 | ||
973 | bnxt_re_setup_qos(rdev); | |
974 | schedule_delayed_work(&rdev->worker, msecs_to_jiffies(30000)); | |
975 | } | |
976 | ||
977 | static int bnxt_re_ib_reg(struct bnxt_re_dev *rdev) | |
978 | { | |
979 | int i, j, rc; | |
980 | ||
981 | /* Registered a new RoCE device instance to netdev */ | |
982 | rc = bnxt_re_register_netdev(rdev); | |
983 | if (rc) { | |
984 | pr_err("Failed to register with netedev: %#x\n", rc); | |
985 | return -EINVAL; | |
986 | } | |
987 | set_bit(BNXT_RE_FLAG_NETDEV_REGISTERED, &rdev->flags); | |
988 | ||
989 | rc = bnxt_re_request_msix(rdev); | |
990 | if (rc) { | |
991 | pr_err("Failed to get MSI-X vectors: %#x\n", rc); | |
992 | rc = -EINVAL; | |
993 | goto fail; | |
994 | } | |
995 | set_bit(BNXT_RE_FLAG_GOT_MSIX, &rdev->flags); | |
996 | ||
997 | /* Establish RCFW Communication Channel to initialize the context | |
998 | * memory for the function and all child VFs | |
999 | */ | |
1000 | rc = bnxt_qplib_alloc_rcfw_channel(rdev->en_dev->pdev, &rdev->rcfw); | |
1001 | if (rc) | |
1002 | goto fail; | |
1003 | ||
1004 | rc = bnxt_re_net_ring_alloc | |
1005 | (rdev, rdev->rcfw.creq.pbl[PBL_LVL_0].pg_map_arr, | |
1006 | rdev->rcfw.creq.pbl[rdev->rcfw.creq.level].pg_count, | |
1007 | HWRM_RING_ALLOC_CMPL, BNXT_QPLIB_CREQE_MAX_CNT - 1, | |
1008 | rdev->msix_entries[BNXT_RE_AEQ_IDX].ring_idx, | |
1009 | &rdev->rcfw.creq_ring_id); | |
1010 | if (rc) { | |
1011 | pr_err("Failed to allocate CREQ: %#x\n", rc); | |
1012 | goto free_rcfw; | |
1013 | } | |
1014 | rc = bnxt_qplib_enable_rcfw_channel | |
1015 | (rdev->en_dev->pdev, &rdev->rcfw, | |
1016 | rdev->msix_entries[BNXT_RE_AEQ_IDX].vector, | |
1017 | rdev->msix_entries[BNXT_RE_AEQ_IDX].db_offset, | |
1018 | 0, &bnxt_re_aeq_handler); | |
1019 | if (rc) { | |
1020 | pr_err("Failed to enable RCFW channel: %#x\n", rc); | |
1021 | goto free_ring; | |
1022 | } | |
1023 | ||
1024 | rc = bnxt_qplib_get_dev_attr(&rdev->rcfw, &rdev->dev_attr); | |
1025 | if (rc) | |
1026 | goto disable_rcfw; | |
1027 | bnxt_re_set_resource_limits(rdev); | |
1028 | ||
1029 | rc = bnxt_qplib_alloc_ctx(rdev->en_dev->pdev, &rdev->qplib_ctx, 0); | |
1030 | if (rc) { | |
1031 | pr_err("Failed to allocate QPLIB context: %#x\n", rc); | |
1032 | goto disable_rcfw; | |
1033 | } | |
1034 | rc = bnxt_re_net_stats_ctx_alloc(rdev, | |
1035 | rdev->qplib_ctx.stats.dma_map, | |
1036 | &rdev->qplib_ctx.stats.fw_id); | |
1037 | if (rc) { | |
1038 | pr_err("Failed to allocate stats context: %#x\n", rc); | |
1039 | goto free_ctx; | |
1040 | } | |
1041 | ||
1042 | rc = bnxt_qplib_init_rcfw(&rdev->rcfw, &rdev->qplib_ctx, 0); | |
1043 | if (rc) { | |
1044 | pr_err("Failed to initialize RCFW: %#x\n", rc); | |
1045 | goto free_sctx; | |
1046 | } | |
1047 | set_bit(BNXT_RE_FLAG_RCFW_CHANNEL_EN, &rdev->flags); | |
1048 | ||
1049 | /* Resources based on the 'new' device caps */ | |
1050 | rc = bnxt_re_alloc_res(rdev); | |
1051 | if (rc) { | |
1052 | pr_err("Failed to allocate resources: %#x\n", rc); | |
1053 | goto fail; | |
1054 | } | |
1055 | rc = bnxt_re_init_res(rdev); | |
1056 | if (rc) { | |
1057 | pr_err("Failed to initialize resources: %#x\n", rc); | |
1058 | goto fail; | |
1059 | } | |
1060 | ||
1061 | rc = bnxt_re_setup_qos(rdev); | |
1062 | if (rc) | |
1063 | pr_info("RoCE priority not yet configured\n"); | |
1064 | ||
1065 | INIT_DELAYED_WORK(&rdev->worker, bnxt_re_worker); | |
1066 | set_bit(BNXT_RE_FLAG_QOS_WORK_REG, &rdev->flags); | |
1067 | schedule_delayed_work(&rdev->worker, msecs_to_jiffies(30000)); | |
1068 | ||
1069 | /* Register ib dev */ | |
1070 | rc = bnxt_re_register_ib(rdev); | |
1071 | if (rc) { | |
1072 | pr_err("Failed to register with IB: %#x\n", rc); | |
1073 | goto fail; | |
1074 | } | |
1075 | dev_info(rdev_to_dev(rdev), "Device registered successfully"); | |
1076 | for (i = 0; i < ARRAY_SIZE(bnxt_re_attributes); i++) { | |
1077 | rc = device_create_file(&rdev->ibdev.dev, | |
1078 | bnxt_re_attributes[i]); | |
1079 | if (rc) { | |
1080 | dev_err(rdev_to_dev(rdev), | |
1081 | "Failed to create IB sysfs: %#x", rc); | |
1082 | /* Must clean up all created device files */ | |
1083 | for (j = 0; j < i; j++) | |
1084 | device_remove_file(&rdev->ibdev.dev, | |
1085 | bnxt_re_attributes[j]); | |
1086 | bnxt_re_unregister_ib(rdev); | |
1087 | goto fail; | |
1088 | } | |
1089 | } | |
1090 | set_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags); | |
1091 | bnxt_re_dispatch_event(&rdev->ibdev, NULL, 1, IB_EVENT_PORT_ACTIVE); | |
1092 | bnxt_re_dispatch_event(&rdev->ibdev, NULL, 1, IB_EVENT_GID_CHANGE); | |
1093 | ||
1094 | return 0; | |
1095 | free_sctx: | |
1096 | bnxt_re_net_stats_ctx_free(rdev, rdev->qplib_ctx.stats.fw_id, true); | |
1097 | free_ctx: | |
1098 | bnxt_qplib_free_ctx(rdev->en_dev->pdev, &rdev->qplib_ctx); | |
1099 | disable_rcfw: | |
1100 | bnxt_qplib_disable_rcfw_channel(&rdev->rcfw); | |
1101 | free_ring: | |
1102 | bnxt_re_net_ring_free(rdev, rdev->rcfw.creq_ring_id, true); | |
1103 | free_rcfw: | |
1104 | bnxt_qplib_free_rcfw_channel(&rdev->rcfw); | |
1105 | fail: | |
1106 | bnxt_re_ib_unreg(rdev, true); | |
1107 | return rc; | |
1108 | } | |
1109 | ||
1110 | static void bnxt_re_dev_unreg(struct bnxt_re_dev *rdev) | |
1111 | { | |
1112 | struct bnxt_en_dev *en_dev = rdev->en_dev; | |
1113 | struct net_device *netdev = rdev->netdev; | |
1114 | ||
1115 | bnxt_re_dev_remove(rdev); | |
1116 | ||
1117 | if (netdev) | |
1118 | bnxt_re_dev_unprobe(netdev, en_dev); | |
1119 | } | |
1120 | ||
1121 | static int bnxt_re_dev_reg(struct bnxt_re_dev **rdev, struct net_device *netdev) | |
1122 | { | |
1123 | struct bnxt_en_dev *en_dev; | |
1124 | int rc = 0; | |
1125 | ||
1126 | if (!is_bnxt_re_dev(netdev)) | |
1127 | return -ENODEV; | |
1128 | ||
1129 | en_dev = bnxt_re_dev_probe(netdev); | |
1130 | if (IS_ERR(en_dev)) { | |
1131 | if (en_dev != ERR_PTR(-ENODEV)) | |
1132 | pr_err("%s: Failed to probe\n", ROCE_DRV_MODULE_NAME); | |
1133 | rc = PTR_ERR(en_dev); | |
1134 | goto exit; | |
1135 | } | |
1136 | *rdev = bnxt_re_dev_add(netdev, en_dev); | |
1137 | if (!*rdev) { | |
1138 | rc = -ENOMEM; | |
1139 | bnxt_re_dev_unprobe(netdev, en_dev); | |
1140 | goto exit; | |
1141 | } | |
1142 | exit: | |
1143 | return rc; | |
1144 | } | |
1145 | ||
1146 | static void bnxt_re_remove_one(struct bnxt_re_dev *rdev) | |
1147 | { | |
1148 | pci_dev_put(rdev->en_dev->pdev); | |
1149 | } | |
1150 | ||
1151 | /* Handle all deferred netevents tasks */ | |
1152 | static void bnxt_re_task(struct work_struct *work) | |
1153 | { | |
1154 | struct bnxt_re_work *re_work; | |
1155 | struct bnxt_re_dev *rdev; | |
1156 | int rc = 0; | |
1157 | ||
1158 | re_work = container_of(work, struct bnxt_re_work, work); | |
1159 | rdev = re_work->rdev; | |
1160 | ||
1161 | if (re_work->event != NETDEV_REGISTER && | |
1162 | !test_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags)) | |
1163 | return; | |
1164 | ||
1165 | switch (re_work->event) { | |
1166 | case NETDEV_REGISTER: | |
1167 | rc = bnxt_re_ib_reg(rdev); | |
1168 | if (rc) | |
1169 | dev_err(rdev_to_dev(rdev), | |
1170 | "Failed to register with IB: %#x", rc); | |
1171 | break; | |
1172 | case NETDEV_UP: | |
1173 | bnxt_re_dispatch_event(&rdev->ibdev, NULL, 1, | |
1174 | IB_EVENT_PORT_ACTIVE); | |
1175 | break; | |
1176 | case NETDEV_DOWN: | |
1177 | bnxt_re_dev_stop(rdev); | |
1178 | break; | |
1179 | case NETDEV_CHANGE: | |
1180 | if (!netif_carrier_ok(rdev->netdev)) | |
1181 | bnxt_re_dev_stop(rdev); | |
1182 | else if (netif_carrier_ok(rdev->netdev)) | |
1183 | bnxt_re_dispatch_event(&rdev->ibdev, NULL, 1, | |
1184 | IB_EVENT_PORT_ACTIVE); | |
1185 | break; | |
1186 | default: | |
1187 | break; | |
1188 | } | |
1189 | kfree(re_work); | |
1190 | } | |
1191 | ||
1192 | static void bnxt_re_init_one(struct bnxt_re_dev *rdev) | |
1193 | { | |
1194 | pci_dev_get(rdev->en_dev->pdev); | |
1195 | } | |
1196 | ||
1197 | /* | |
1198 | * "Notifier chain callback can be invoked for the same chain from | |
1199 | * different CPUs at the same time". | |
1200 | * | |
1201 | * For cases when the netdev is already present, our call to the | |
1202 | * register_netdevice_notifier() will actually get the rtnl_lock() | |
1203 | * before sending NETDEV_REGISTER and (if up) NETDEV_UP | |
1204 | * events. | |
1205 | * | |
1206 | * But for cases when the netdev is not already present, the notifier | |
1207 | * chain is subjected to be invoked from different CPUs simultaneously. | |
1208 | * | |
1209 | * This is protected by the netdev_mutex. | |
1210 | */ | |
1211 | static int bnxt_re_netdev_event(struct notifier_block *notifier, | |
1212 | unsigned long event, void *ptr) | |
1213 | { | |
1214 | struct net_device *real_dev, *netdev = netdev_notifier_info_to_dev(ptr); | |
1215 | struct bnxt_re_work *re_work; | |
1216 | struct bnxt_re_dev *rdev; | |
1217 | int rc = 0; | |
1218 | bool sch_work = false; | |
1219 | ||
1220 | real_dev = rdma_vlan_dev_real_dev(netdev); | |
1221 | if (!real_dev) | |
1222 | real_dev = netdev; | |
1223 | ||
1224 | rdev = bnxt_re_from_netdev(real_dev); | |
1225 | if (!rdev && event != NETDEV_REGISTER) | |
1226 | goto exit; | |
1227 | if (real_dev != netdev) | |
1228 | goto exit; | |
1229 | ||
1230 | switch (event) { | |
1231 | case NETDEV_REGISTER: | |
1232 | if (rdev) | |
1233 | break; | |
1234 | rc = bnxt_re_dev_reg(&rdev, real_dev); | |
1235 | if (rc == -ENODEV) | |
1236 | break; | |
1237 | if (rc) { | |
1238 | pr_err("Failed to register with the device %s: %#x\n", | |
1239 | real_dev->name, rc); | |
1240 | break; | |
1241 | } | |
1242 | bnxt_re_init_one(rdev); | |
1243 | sch_work = true; | |
1244 | break; | |
1245 | ||
1246 | case NETDEV_UNREGISTER: | |
1247 | bnxt_re_ib_unreg(rdev, false); | |
1248 | bnxt_re_remove_one(rdev); | |
1249 | bnxt_re_dev_unreg(rdev); | |
1250 | break; | |
1251 | ||
1252 | default: | |
1253 | sch_work = true; | |
1254 | break; | |
1255 | } | |
1256 | if (sch_work) { | |
1257 | /* Allocate for the deferred task */ | |
1258 | re_work = kzalloc(sizeof(*re_work), GFP_ATOMIC); | |
1259 | if (re_work) { | |
1260 | re_work->rdev = rdev; | |
1261 | re_work->event = event; | |
1262 | re_work->vlan_dev = (real_dev == netdev ? | |
1263 | NULL : netdev); | |
1264 | INIT_WORK(&re_work->work, bnxt_re_task); | |
1265 | queue_work(bnxt_re_wq, &re_work->work); | |
1266 | } | |
1267 | } | |
1268 | ||
1269 | exit: | |
1270 | return NOTIFY_DONE; | |
1271 | } | |
1272 | ||
1273 | static struct notifier_block bnxt_re_netdev_notifier = { | |
1274 | .notifier_call = bnxt_re_netdev_event | |
1275 | }; | |
1276 | ||
1277 | static int __init bnxt_re_mod_init(void) | |
1278 | { | |
1279 | int rc = 0; | |
1280 | ||
1281 | pr_info("%s: %s", ROCE_DRV_MODULE_NAME, version); | |
1282 | ||
1283 | bnxt_re_wq = create_singlethread_workqueue("bnxt_re"); | |
1284 | if (!bnxt_re_wq) | |
1285 | return -ENOMEM; | |
1286 | ||
1287 | INIT_LIST_HEAD(&bnxt_re_dev_list); | |
1288 | ||
1289 | rc = register_netdevice_notifier(&bnxt_re_netdev_notifier); | |
1290 | if (rc) { | |
1291 | pr_err("%s: Cannot register to netdevice_notifier", | |
1292 | ROCE_DRV_MODULE_NAME); | |
1293 | goto err_netdev; | |
1294 | } | |
1295 | return 0; | |
1296 | ||
1297 | err_netdev: | |
1298 | destroy_workqueue(bnxt_re_wq); | |
1299 | ||
1300 | return rc; | |
1301 | } | |
1302 | ||
1303 | static void __exit bnxt_re_mod_exit(void) | |
1304 | { | |
1305 | unregister_netdevice_notifier(&bnxt_re_netdev_notifier); | |
1306 | if (bnxt_re_wq) | |
1307 | destroy_workqueue(bnxt_re_wq); | |
1308 | } | |
1309 | ||
1310 | module_init(bnxt_re_mod_init); | |
1311 | module_exit(bnxt_re_mod_exit); |