2 * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved.
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
36 #include <linux/kernel.h>
37 #include <linux/sched.h>
38 #include <rdma/ib_verbs.h>
39 #include <rdma/ib_umem.h>
40 #include <rdma/ib_smi.h>
41 #include <linux/mlx5/driver.h>
42 #include <linux/mlx5/cq.h>
43 #include <linux/mlx5/fs.h>
44 #include <linux/mlx5/qp.h>
45 #include <linux/types.h>
46 #include <linux/mlx5/transobj.h>
47 #include <rdma/ib_user_verbs.h>
48 #include <rdma/mlx5-abi.h>
49 #include <rdma/uverbs_ioctl.h>
50 #include <rdma/mlx5_user_ioctl_cmds.h>
51 #include <rdma/mlx5_user_ioctl_verbs.h>
55 #define mlx5_ib_dbg(_dev, format, arg...) \
56 dev_dbg(&(_dev)->ib_dev.dev, "%s:%d:(pid %d): " format, __func__, \
57 __LINE__, current->pid, ##arg)
59 #define mlx5_ib_err(_dev, format, arg...) \
60 dev_err(&(_dev)->ib_dev.dev, "%s:%d:(pid %d): " format, __func__, \
61 __LINE__, current->pid, ##arg)
63 #define mlx5_ib_warn(_dev, format, arg...) \
64 dev_warn(&(_dev)->ib_dev.dev, "%s:%d:(pid %d): " format, __func__, \
65 __LINE__, current->pid, ##arg)
67 #define field_avail(type, fld, sz) (offsetof(type, fld) + \
68 sizeof(((type *)0)->fld) <= (sz))
69 #define MLX5_IB_DEFAULT_UIDX 0xffffff
70 #define MLX5_USER_ASSIGNED_UIDX_MASK __mlx5_mask(qpc, user_index)
72 #define MLX5_MKEY_PAGE_SHIFT_MASK __mlx5_mask(mkc, log_page_size)
75 MLX5_IB_MMAP_CMD_SHIFT
= 8,
76 MLX5_IB_MMAP_CMD_MASK
= 0xff,
80 MLX5_RES_SCAT_DATA32_CQE
= 0x1,
81 MLX5_RES_SCAT_DATA64_CQE
= 0x2,
82 MLX5_REQ_SCAT_DATA32_CQE
= 0x11,
83 MLX5_REQ_SCAT_DATA64_CQE
= 0x22,
86 enum mlx5_ib_mad_ifc_flags
{
87 MLX5_MAD_IFC_IGNORE_MKEY
= 1,
88 MLX5_MAD_IFC_IGNORE_BKEY
= 2,
89 MLX5_MAD_IFC_NET_VIEW
= 4,
93 MLX5_CROSS_CHANNEL_BFREG
= 0,
102 MLX5_TM_MAX_RNDV_MSG_SIZE
= 64,
107 MLX5_IB_INVALID_UAR_INDEX
= BIT(31),
108 MLX5_IB_INVALID_BFREG
= BIT(31),
112 MLX5_MAX_MEMIC_PAGES
= 0x100,
113 MLX5_MEMIC_ALLOC_SIZE_MASK
= 0x3f,
117 MLX5_MEMIC_BASE_ALIGN
= 6,
118 MLX5_MEMIC_BASE_SIZE
= 1 << MLX5_MEMIC_BASE_ALIGN
,
121 enum mlx5_ib_mmap_type
{
122 MLX5_IB_MMAP_TYPE_MEMIC
= 1,
125 #define MLX5_LOG_SW_ICM_BLOCK_SIZE(dev) \
126 (MLX5_CAP_DEV_MEM(dev, log_sw_icm_alloc_granularity))
127 #define MLX5_SW_ICM_BLOCK_SIZE(dev) (1 << MLX5_LOG_SW_ICM_BLOCK_SIZE(dev))
129 struct mlx5_ib_ucontext
{
130 struct ib_ucontext ibucontext
;
131 struct list_head db_page_list
;
133 /* protect doorbell record alloc/free
135 struct mutex db_page_mutex
;
136 struct mlx5_bfreg_info bfregi
;
138 /* Transport Domain number */
143 /* For RoCE LAG TX affinity */
144 atomic_t tx_port_affinity
;
147 static inline struct mlx5_ib_ucontext
*to_mucontext(struct ib_ucontext
*ibucontext
)
149 return container_of(ibucontext
, struct mlx5_ib_ucontext
, ibucontext
);
159 MLX5_IB_FLOW_ACTION_MODIFY_HEADER
,
160 MLX5_IB_FLOW_ACTION_PACKET_REFORMAT
,
161 MLX5_IB_FLOW_ACTION_DECAP
,
164 #define MLX5_IB_FLOW_MCAST_PRIO (MLX5_BY_PASS_NUM_PRIOS - 1)
165 #define MLX5_IB_FLOW_LAST_PRIO (MLX5_BY_PASS_NUM_REGULAR_PRIOS - 1)
166 #if (MLX5_IB_FLOW_LAST_PRIO <= 0)
167 #error "Invalid number of bypass priorities"
169 #define MLX5_IB_FLOW_LEFTOVERS_PRIO (MLX5_IB_FLOW_MCAST_PRIO + 1)
171 #define MLX5_IB_NUM_FLOW_FT (MLX5_IB_FLOW_LEFTOVERS_PRIO + 1)
172 #define MLX5_IB_NUM_SNIFFER_FTS 2
173 #define MLX5_IB_NUM_EGRESS_FTS 1
174 struct mlx5_ib_flow_prio
{
175 struct mlx5_flow_table
*flow_table
;
176 unsigned int refcount
;
179 struct mlx5_ib_flow_handler
{
180 struct list_head list
;
181 struct ib_flow ibflow
;
182 struct mlx5_ib_flow_prio
*prio
;
183 struct mlx5_flow_handle
*rule
;
184 struct ib_counters
*ibcounters
;
185 struct mlx5_ib_dev
*dev
;
186 struct mlx5_ib_flow_matcher
*flow_matcher
;
189 struct mlx5_ib_flow_matcher
{
190 struct mlx5_ib_match_params matcher_mask
;
192 enum mlx5_ib_flow_type flow_type
;
193 enum mlx5_flow_namespace_type ns_type
;
195 struct mlx5_core_dev
*mdev
;
197 u8 match_criteria_enable
;
200 struct mlx5_ib_flow_db
{
201 struct mlx5_ib_flow_prio prios
[MLX5_IB_NUM_FLOW_FT
];
202 struct mlx5_ib_flow_prio egress_prios
[MLX5_IB_NUM_FLOW_FT
];
203 struct mlx5_ib_flow_prio sniffer
[MLX5_IB_NUM_SNIFFER_FTS
];
204 struct mlx5_ib_flow_prio egress
[MLX5_IB_NUM_EGRESS_FTS
];
205 struct mlx5_ib_flow_prio fdb
;
206 struct mlx5_ib_flow_prio rdma_rx
[MLX5_IB_NUM_FLOW_FT
];
207 struct mlx5_flow_table
*lag_demux_ft
;
208 /* Protect flow steering bypass flow tables
209 * when add/del flow rules.
210 * only single add/removal of flow steering rule could be done
216 /* Use macros here so that don't have to duplicate
217 * enum ib_send_flags and enum ib_qp_type for low-level driver
220 #define MLX5_IB_SEND_UMR_ENABLE_MR (IB_SEND_RESERVED_START << 0)
221 #define MLX5_IB_SEND_UMR_DISABLE_MR (IB_SEND_RESERVED_START << 1)
222 #define MLX5_IB_SEND_UMR_FAIL_IF_FREE (IB_SEND_RESERVED_START << 2)
223 #define MLX5_IB_SEND_UMR_UPDATE_XLT (IB_SEND_RESERVED_START << 3)
224 #define MLX5_IB_SEND_UMR_UPDATE_TRANSLATION (IB_SEND_RESERVED_START << 4)
225 #define MLX5_IB_SEND_UMR_UPDATE_PD_ACCESS IB_SEND_RESERVED_END
227 #define MLX5_IB_QPT_REG_UMR IB_QPT_RESERVED1
229 * IB_QPT_GSI creates the software wrapper around GSI, and MLX5_IB_QPT_HW_GSI
230 * creates the actual hardware QP.
232 #define MLX5_IB_QPT_HW_GSI IB_QPT_RESERVED2
233 #define MLX5_IB_QPT_DCI IB_QPT_RESERVED3
234 #define MLX5_IB_QPT_DCT IB_QPT_RESERVED4
235 #define MLX5_IB_WR_UMR IB_WR_RESERVED1
237 #define MLX5_IB_UMR_OCTOWORD 16
238 #define MLX5_IB_UMR_XLT_ALIGNMENT 64
240 #define MLX5_IB_UPD_XLT_ZAP BIT(0)
241 #define MLX5_IB_UPD_XLT_ENABLE BIT(1)
242 #define MLX5_IB_UPD_XLT_ATOMIC BIT(2)
243 #define MLX5_IB_UPD_XLT_ADDR BIT(3)
244 #define MLX5_IB_UPD_XLT_PD BIT(4)
245 #define MLX5_IB_UPD_XLT_ACCESS BIT(5)
246 #define MLX5_IB_UPD_XLT_INDIRECT BIT(6)
248 /* Private QP creation flags to be passed in ib_qp_init_attr.create_flags.
250 * These flags are intended for internal use by the mlx5_ib driver, and they
251 * rely on the range reserved for that use in the ib_qp_create_flags enum.
253 #define MLX5_IB_QP_CREATE_SQPN_QP1 IB_QP_CREATE_RESERVED_START
254 #define MLX5_IB_QP_CREATE_WC_TEST (IB_QP_CREATE_RESERVED_START << 1)
261 enum mlx5_ib_rq_flags
{
262 MLX5_IB_RQ_CVLAN_STRIPPING
= 1 << 0,
263 MLX5_IB_RQ_PCI_WRITE_END_PADDING
= 1 << 1,
267 struct mlx5_frag_buf_ctrl fbc
;
270 struct wr_list
*w_list
;
274 /* serialize post to the work queue
288 enum mlx5_ib_wq_flags
{
289 MLX5_IB_WQ_FLAGS_DELAY_DROP
= 0x1,
290 MLX5_IB_WQ_FLAGS_STRIDING_RQ
= 0x2,
293 #define MLX5_MIN_SINGLE_WQE_LOG_NUM_STRIDES 9
294 #define MLX5_MAX_SINGLE_WQE_LOG_NUM_STRIDES 16
295 #define MLX5_MIN_SINGLE_STRIDE_LOG_NUM_BYTES 6
296 #define MLX5_MAX_SINGLE_STRIDE_LOG_NUM_BYTES 13
297 #define MLX5_EXT_MIN_SINGLE_WQE_LOG_NUM_STRIDES 3
301 struct mlx5_core_qp core_qp
;
308 u32 two_byte_shift_en
;
309 u32 single_stride_log_num_of_bytes
;
310 struct ib_umem
*umem
;
312 unsigned int page_shift
;
319 u32 create_flags
; /* Use enum mlx5_ib_wq_flags */
333 struct mlx5_ib_rwq_ind_table
{
334 struct ib_rwq_ind_table ib_rwq_ind_tbl
;
339 struct mlx5_ib_ubuffer
{
340 struct ib_umem
*umem
;
345 struct mlx5_ib_qp_base
{
346 struct mlx5_ib_qp
*container_mibqp
;
347 struct mlx5_core_qp mqp
;
348 struct mlx5_ib_ubuffer ubuffer
;
351 struct mlx5_ib_qp_trans
{
352 struct mlx5_ib_qp_base base
;
359 struct mlx5_ib_rss_qp
{
364 struct mlx5_ib_qp_base base
;
365 struct mlx5_ib_wq
*rq
;
366 struct mlx5_ib_ubuffer ubuffer
;
367 struct mlx5_db
*doorbell
;
374 struct mlx5_ib_qp_base base
;
375 struct mlx5_ib_wq
*sq
;
376 struct mlx5_ib_ubuffer ubuffer
;
377 struct mlx5_db
*doorbell
;
378 struct mlx5_flow_handle
*flow_rule
;
383 struct mlx5_ib_raw_packet_qp
{
384 struct mlx5_ib_sq sq
;
385 struct mlx5_ib_rq rq
;
390 unsigned long offset
;
391 struct mlx5_sq_bfreg
*bfreg
;
395 struct mlx5_core_dct mdct
;
402 struct mlx5_ib_qp_trans trans_qp
;
403 struct mlx5_ib_raw_packet_qp raw_packet_qp
;
404 struct mlx5_ib_rss_qp rss_qp
;
405 struct mlx5_ib_dct dct
;
407 struct mlx5_frag_buf buf
;
410 struct mlx5_ib_wq rq
;
414 struct mlx5_ib_wq sq
;
416 /* serialize qp state modifications
428 /* only for user space QPs. For kernel
429 * we have it from the bf object
435 struct list_head qps_list
;
436 struct list_head cq_recv_list
;
437 struct list_head cq_send_list
;
438 struct mlx5_rate_limit rl
;
441 /* storage for qp sub type when core qp type is IB_QPT_DRIVER */
442 enum ib_qp_type qp_sub_type
;
443 /* A flag to indicate if there's a new counter is configured
444 * but not take effective
449 struct mlx5_ib_cq_buf
{
450 struct mlx5_frag_buf_ctrl fbc
;
451 struct mlx5_frag_buf frag_buf
;
452 struct ib_umem
*umem
;
457 enum mlx5_ib_qp_flags
{
458 MLX5_IB_QP_LSO
= IB_QP_CREATE_IPOIB_UD_LSO
,
459 MLX5_IB_QP_BLOCK_MULTICAST_LOOPBACK
= IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK
,
460 MLX5_IB_QP_CROSS_CHANNEL
= IB_QP_CREATE_CROSS_CHANNEL
,
461 MLX5_IB_QP_MANAGED_SEND
= IB_QP_CREATE_MANAGED_SEND
,
462 MLX5_IB_QP_MANAGED_RECV
= IB_QP_CREATE_MANAGED_RECV
,
463 MLX5_IB_QP_SIGNATURE_HANDLING
= 1 << 5,
464 /* QP uses 1 as its source QP number */
465 MLX5_IB_QP_SQPN_QP1
= 1 << 6,
466 MLX5_IB_QP_CAP_SCATTER_FCS
= 1 << 7,
467 MLX5_IB_QP_RSS
= 1 << 8,
468 MLX5_IB_QP_CVLAN_STRIPPING
= 1 << 9,
469 MLX5_IB_QP_UNDERLAY
= 1 << 10,
470 MLX5_IB_QP_PCI_WRITE_END_PADDING
= 1 << 11,
471 MLX5_IB_QP_TUNNEL_OFFLOAD
= 1 << 12,
472 MLX5_IB_QP_PACKET_BASED_CREDIT
= 1 << 13,
476 struct ib_send_wr wr
;
480 unsigned int page_shift
;
481 unsigned int xlt_size
;
485 u8 ignore_free_state
:1;
488 static inline const struct mlx5_umr_wr
*umr_wr(const struct ib_send_wr
*wr
)
490 return container_of(wr
, struct mlx5_umr_wr
, wr
);
493 struct mlx5_shared_mr_info
{
495 struct ib_umem
*umem
;
498 enum mlx5_ib_cq_pr_flags
{
499 MLX5_IB_CQ_PR_FLAGS_CQE_128_PAD
= 1 << 0,
504 struct mlx5_core_cq mcq
;
505 struct mlx5_ib_cq_buf buf
;
508 /* serialize access to the CQ
514 struct mutex resize_mutex
;
515 struct mlx5_ib_cq_buf
*resize_buf
;
516 struct ib_umem
*resize_umem
;
518 struct list_head list_send_qp
;
519 struct list_head list_recv_qp
;
521 struct list_head wc_list
;
522 enum ib_cq_notify_flags notify_flags
;
523 struct work_struct notify_work
;
524 u16 private_flags
; /* Use mlx5_ib_cq_pr_flags */
529 struct list_head list
;
534 struct mlx5_core_srq msrq
;
535 struct mlx5_frag_buf buf
;
537 struct mlx5_frag_buf_ctrl fbc
;
539 /* protect SRQ hanlding
545 struct ib_umem
*umem
;
546 /* serialize arming a SRQ
552 struct mlx5_ib_xrcd
{
553 struct ib_xrcd ibxrcd
;
557 enum mlx5_ib_mtt_access_flags
{
558 MLX5_IB_MTT_READ
= (1 << 0),
559 MLX5_IB_MTT_WRITE
= (1 << 1),
562 struct mlx5_user_mmap_entry
{
563 struct rdma_user_mmap_entry rdma_entry
;
570 phys_addr_t dev_addr
;
577 /* other dm types specific params should be added here */
579 struct mlx5_user_mmap_entry mentry
;
582 #define MLX5_IB_MTT_PRESENT (MLX5_IB_MTT_READ | MLX5_IB_MTT_WRITE)
584 #define MLX5_IB_DM_MEMIC_ALLOWED_ACCESS (IB_ACCESS_LOCAL_WRITE |\
585 IB_ACCESS_REMOTE_WRITE |\
586 IB_ACCESS_REMOTE_READ |\
587 IB_ACCESS_REMOTE_ATOMIC |\
590 #define MLX5_IB_DM_SW_ICM_ALLOWED_ACCESS (IB_ACCESS_LOCAL_WRITE |\
591 IB_ACCESS_REMOTE_WRITE |\
592 IB_ACCESS_REMOTE_READ |\
595 #define mlx5_update_odp_stats(mr, counter_name, value) \
596 atomic64_add(value, &((mr)->odp_stats.counter_name))
609 struct mlx5_core_mkey mmkey
;
610 struct ib_umem
*umem
;
611 struct mlx5_shared_mr_info
*smr_info
;
612 struct list_head list
;
614 bool allocated_from_cache
;
616 struct mlx5_ib_dev
*dev
;
617 u32 out
[MLX5_ST_SZ_DW(create_mkey_out
)];
618 struct mlx5_core_sig_ctx
*sig
;
620 int access_flags
; /* Needed for rereg MR */
622 struct mlx5_ib_mr
*parent
;
623 /* Needed for IB_MR_TYPE_INTEGRITY */
624 struct mlx5_ib_mr
*pi_mr
;
625 struct mlx5_ib_mr
*klm_mr
;
626 struct mlx5_ib_mr
*mtt_mr
;
630 /* For ODP and implicit */
631 atomic_t num_deferred_work
;
632 struct xarray implicit_children
;
635 struct list_head elm
;
636 struct work_struct work
;
638 struct ib_odp_counters odp_stats
;
639 bool is_odp_implicit
;
641 struct mlx5_async_work cb_work
;
644 static inline bool is_odp_mr(struct mlx5_ib_mr
*mr
)
646 return IS_ENABLED(CONFIG_INFINIBAND_ON_DEMAND_PAGING
) && mr
->umem
&&
652 struct mlx5_core_mkey mmkey
;
656 struct mlx5_ib_devx_mr
{
657 struct mlx5_core_mkey mmkey
;
661 struct mlx5_ib_umr_context
{
663 enum ib_wc_status status
;
664 struct completion done
;
671 /* control access to UMR QP
673 struct semaphore sem
;
682 struct mlx5_cache_ent
{
683 struct list_head head
;
684 /* sync access to the cahce entry
700 struct mlx5_ib_dev
*dev
;
701 struct work_struct work
;
702 struct delayed_work dwork
;
704 struct completion
compl;
707 struct mlx5_mr_cache
{
708 struct workqueue_struct
*wq
;
709 struct mlx5_cache_ent ent
[MAX_MR_CACHE_ENTRIES
];
712 unsigned long last_add
;
715 struct mlx5_ib_gsi_qp
;
717 struct mlx5_ib_port_resources
{
718 struct mlx5_ib_resources
*devr
;
719 struct mlx5_ib_gsi_qp
*gsi
;
720 struct work_struct pkey_change_work
;
723 struct mlx5_ib_resources
{
730 struct mlx5_ib_port_resources ports
[2];
731 /* Protects changes to the port resources */
735 struct mlx5_ib_counters
{
739 u32 num_cong_counters
;
740 u32 num_ext_ppcnt_counters
;
745 struct mlx5_ib_multiport_info
;
747 struct mlx5_ib_multiport
{
748 struct mlx5_ib_multiport_info
*mpi
;
749 /* To be held when accessing the multiport info */
754 /* Protect mlx5_ib_get_netdev from invoking dev_hold() with a NULL
757 rwlock_t netdev_lock
;
758 struct net_device
*netdev
;
759 struct notifier_block nb
;
760 atomic_t tx_port_affinity
;
761 enum ib_port_state last_port_state
;
762 struct mlx5_ib_dev
*dev
;
766 struct mlx5_ib_port
{
767 struct mlx5_ib_counters cnts
;
768 struct mlx5_ib_multiport mp
;
769 struct mlx5_ib_dbg_cc_params
*dbg_cc_params
;
770 struct mlx5_roce roce
;
771 struct mlx5_eswitch_rep
*rep
;
774 struct mlx5_ib_dbg_param
{
776 struct mlx5_ib_dev
*dev
;
777 struct dentry
*dentry
;
781 enum mlx5_ib_dbg_cc_types
{
782 MLX5_IB_DBG_CC_RP_CLAMP_TGT_RATE
,
783 MLX5_IB_DBG_CC_RP_CLAMP_TGT_RATE_ATI
,
784 MLX5_IB_DBG_CC_RP_TIME_RESET
,
785 MLX5_IB_DBG_CC_RP_BYTE_RESET
,
786 MLX5_IB_DBG_CC_RP_THRESHOLD
,
787 MLX5_IB_DBG_CC_RP_AI_RATE
,
788 MLX5_IB_DBG_CC_RP_HAI_RATE
,
789 MLX5_IB_DBG_CC_RP_MIN_DEC_FAC
,
790 MLX5_IB_DBG_CC_RP_MIN_RATE
,
791 MLX5_IB_DBG_CC_RP_RATE_TO_SET_ON_FIRST_CNP
,
792 MLX5_IB_DBG_CC_RP_DCE_TCP_G
,
793 MLX5_IB_DBG_CC_RP_DCE_TCP_RTT
,
794 MLX5_IB_DBG_CC_RP_RATE_REDUCE_MONITOR_PERIOD
,
795 MLX5_IB_DBG_CC_RP_INITIAL_ALPHA_VALUE
,
796 MLX5_IB_DBG_CC_RP_GD
,
797 MLX5_IB_DBG_CC_NP_CNP_DSCP
,
798 MLX5_IB_DBG_CC_NP_CNP_PRIO_MODE
,
799 MLX5_IB_DBG_CC_NP_CNP_PRIO
,
803 struct mlx5_ib_dbg_cc_params
{
805 struct mlx5_ib_dbg_param params
[MLX5_IB_DBG_CC_MAX
];
809 MLX5_MAX_DELAY_DROP_TIMEOUT_MS
= 100,
812 struct mlx5_ib_delay_drop
{
813 struct mlx5_ib_dev
*dev
;
814 struct work_struct delay_drop_work
;
815 /* serialize setting of delay drop */
821 struct dentry
*dir_debugfs
;
824 enum mlx5_ib_stages
{
826 MLX5_IB_STAGE_FLOW_DB
,
828 MLX5_IB_STAGE_NON_DEFAULT_CB
,
831 MLX5_IB_STAGE_DEVICE_RESOURCES
,
832 MLX5_IB_STAGE_DEVICE_NOTIFIER
,
834 MLX5_IB_STAGE_COUNTERS
,
835 MLX5_IB_STAGE_CONG_DEBUGFS
,
838 MLX5_IB_STAGE_PRE_IB_REG_UMR
,
839 MLX5_IB_STAGE_WHITELIST_UID
,
840 MLX5_IB_STAGE_IB_REG
,
841 MLX5_IB_STAGE_POST_IB_REG_UMR
,
842 MLX5_IB_STAGE_DELAY_DROP
,
843 MLX5_IB_STAGE_CLASS_ATTR
,
847 struct mlx5_ib_stage
{
848 int (*init
)(struct mlx5_ib_dev
*dev
);
849 void (*cleanup
)(struct mlx5_ib_dev
*dev
);
852 #define STAGE_CREATE(_stage, _init, _cleanup) \
853 .stage[_stage] = {.init = _init, .cleanup = _cleanup}
855 struct mlx5_ib_profile
{
856 struct mlx5_ib_stage stage
[MLX5_IB_STAGE_MAX
];
859 struct mlx5_ib_multiport_info
{
860 struct list_head list
;
861 struct mlx5_ib_dev
*ibdev
;
862 struct mlx5_core_dev
*mdev
;
863 struct notifier_block mdev_events
;
864 struct completion unref_comp
;
871 struct mlx5_ib_flow_action
{
872 struct ib_flow_action ib_action
;
876 struct mlx5_accel_esp_xfrm
*ctx
;
879 struct mlx5_ib_dev
*dev
;
882 struct mlx5_modify_hdr
*modify_hdr
;
883 struct mlx5_pkt_reformat
*pkt_reformat
;
890 struct mlx5_core_dev
*dev
;
891 /* This lock is used to protect the access to the shared
892 * allocation map when concurrent requests by different
893 * processes are handled.
896 DECLARE_BITMAP(memic_alloc_pages
, MLX5_MAX_MEMIC_PAGES
);
899 struct mlx5_read_counters_attr
{
900 struct mlx5_fc
*hw_cntrs_hndl
;
905 enum mlx5_ib_counters_type
{
906 MLX5_IB_COUNTERS_FLOW
,
909 struct mlx5_ib_mcounters
{
910 struct ib_counters ibcntrs
;
911 enum mlx5_ib_counters_type type
;
912 /* number of counters supported for this counters type */
914 struct mlx5_fc
*hw_cntrs_hndl
;
915 /* read function for this counters type */
916 int (*read_counters
)(struct ib_device
*ibdev
,
917 struct mlx5_read_counters_attr
*read_attr
);
918 /* max index set as part of create_flow */
920 /* number of counters data entries (<description,index> pair) */
922 /* counters data array for descriptions and indexes */
923 struct mlx5_ib_flow_counters_desc
*counters_data
;
924 /* protects access to mcounters internal data */
925 struct mutex mcntrs_mutex
;
928 static inline struct mlx5_ib_mcounters
*
929 to_mcounters(struct ib_counters
*ibcntrs
)
931 return container_of(ibcntrs
, struct mlx5_ib_mcounters
, ibcntrs
);
934 int parse_flow_flow_action(struct mlx5_ib_flow_action
*maction
,
936 struct mlx5_flow_act
*action
);
937 struct mlx5_ib_lb_state
{
938 /* protect the user_td */
945 struct mlx5_ib_pf_eq
{
946 struct notifier_block irq_nb
;
947 struct mlx5_ib_dev
*dev
;
948 struct mlx5_eq
*core
;
949 struct work_struct work
;
950 spinlock_t lock
; /* Pagefaults spinlock */
951 struct workqueue_struct
*wq
;
955 struct mlx5_devx_event_table
{
956 struct mlx5_nb devx_nb
;
957 /* serialize updating the event_xa */
958 struct mutex event_xa_lock
;
959 struct xarray event_xa
;
963 struct ib_device ib_dev
;
964 struct mlx5_core_dev
*mdev
;
965 struct notifier_block mdev_events
;
967 /* serialize update of capability mask
969 struct mutex cap_mask_mutex
;
975 struct umr_common umrc
;
976 /* sync used page count stats
978 struct mlx5_ib_resources devr
;
979 struct mlx5_mr_cache cache
;
980 struct timer_list delay_timer
;
981 /* Prevents soft lock on massive reg MRs */
982 struct mutex slow_path_mutex
;
983 struct ib_odp_caps odp_caps
;
985 struct mlx5_ib_pf_eq odp_pf_eq
;
988 * Sleepable RCU that prevents destruction of MRs while they are still
989 * being used by a page fault handler.
991 struct srcu_struct odp_srcu
;
992 struct xarray odp_mkeys
;
995 struct mlx5_ib_flow_db
*flow_db
;
996 /* protect resources needed as part of reset flow */
997 spinlock_t reset_flow_resource_lock
;
998 struct list_head qp_list
;
999 /* Array with num_ports elements */
1000 struct mlx5_ib_port
*port
;
1001 struct mlx5_sq_bfreg bfreg
;
1002 struct mlx5_sq_bfreg wc_bfreg
;
1003 struct mlx5_sq_bfreg fp_bfreg
;
1004 struct mlx5_ib_delay_drop delay_drop
;
1005 const struct mlx5_ib_profile
*profile
;
1007 struct mlx5_ib_lb_state lb
;
1009 struct list_head ib_dev_list
;
1012 u16 devx_whitelist_uid
;
1013 struct mlx5_srq_table srq_table
;
1014 struct mlx5_async_ctx async_ctx
;
1015 struct mlx5_devx_event_table devx_event_table
;
1017 struct xarray sig_mrs
;
1020 static inline struct mlx5_ib_cq
*to_mibcq(struct mlx5_core_cq
*mcq
)
1022 return container_of(mcq
, struct mlx5_ib_cq
, mcq
);
1025 static inline struct mlx5_ib_xrcd
*to_mxrcd(struct ib_xrcd
*ibxrcd
)
1027 return container_of(ibxrcd
, struct mlx5_ib_xrcd
, ibxrcd
);
1030 static inline struct mlx5_ib_dev
*to_mdev(struct ib_device
*ibdev
)
1032 return container_of(ibdev
, struct mlx5_ib_dev
, ib_dev
);
1035 static inline struct mlx5_ib_dev
*mlx5_udata_to_mdev(struct ib_udata
*udata
)
1037 struct mlx5_ib_ucontext
*context
= rdma_udata_to_drv_context(
1038 udata
, struct mlx5_ib_ucontext
, ibucontext
);
1040 return to_mdev(context
->ibucontext
.device
);
1043 static inline struct mlx5_ib_cq
*to_mcq(struct ib_cq
*ibcq
)
1045 return container_of(ibcq
, struct mlx5_ib_cq
, ibcq
);
1048 static inline struct mlx5_ib_qp
*to_mibqp(struct mlx5_core_qp
*mqp
)
1050 return container_of(mqp
, struct mlx5_ib_qp_base
, mqp
)->container_mibqp
;
1053 static inline struct mlx5_ib_rwq
*to_mibrwq(struct mlx5_core_qp
*core_qp
)
1055 return container_of(core_qp
, struct mlx5_ib_rwq
, core_qp
);
1058 static inline struct mlx5_ib_mr
*to_mibmr(struct mlx5_core_mkey
*mmkey
)
1060 return container_of(mmkey
, struct mlx5_ib_mr
, mmkey
);
1063 static inline struct mlx5_ib_pd
*to_mpd(struct ib_pd
*ibpd
)
1065 return container_of(ibpd
, struct mlx5_ib_pd
, ibpd
);
1068 static inline struct mlx5_ib_srq
*to_msrq(struct ib_srq
*ibsrq
)
1070 return container_of(ibsrq
, struct mlx5_ib_srq
, ibsrq
);
1073 static inline struct mlx5_ib_qp
*to_mqp(struct ib_qp
*ibqp
)
1075 return container_of(ibqp
, struct mlx5_ib_qp
, ibqp
);
1078 static inline struct mlx5_ib_rwq
*to_mrwq(struct ib_wq
*ibwq
)
1080 return container_of(ibwq
, struct mlx5_ib_rwq
, ibwq
);
1083 static inline struct mlx5_ib_rwq_ind_table
*to_mrwq_ind_table(struct ib_rwq_ind_table
*ib_rwq_ind_tbl
)
1085 return container_of(ib_rwq_ind_tbl
, struct mlx5_ib_rwq_ind_table
, ib_rwq_ind_tbl
);
1088 static inline struct mlx5_ib_srq
*to_mibsrq(struct mlx5_core_srq
*msrq
)
1090 return container_of(msrq
, struct mlx5_ib_srq
, msrq
);
1093 static inline struct mlx5_ib_dm
*to_mdm(struct ib_dm
*ibdm
)
1095 return container_of(ibdm
, struct mlx5_ib_dm
, ibdm
);
1098 static inline struct mlx5_ib_mr
*to_mmr(struct ib_mr
*ibmr
)
1100 return container_of(ibmr
, struct mlx5_ib_mr
, ibmr
);
1103 static inline struct mlx5_ib_mw
*to_mmw(struct ib_mw
*ibmw
)
1105 return container_of(ibmw
, struct mlx5_ib_mw
, ibmw
);
1108 static inline struct mlx5_ib_flow_action
*
1109 to_mflow_act(struct ib_flow_action
*ibact
)
1111 return container_of(ibact
, struct mlx5_ib_flow_action
, ib_action
);
1114 static inline struct mlx5_user_mmap_entry
*
1115 to_mmmap(struct rdma_user_mmap_entry
*rdma_entry
)
1117 return container_of(rdma_entry
,
1118 struct mlx5_user_mmap_entry
, rdma_entry
);
1121 int mlx5_ib_db_map_user(struct mlx5_ib_ucontext
*context
,
1122 struct ib_udata
*udata
, unsigned long virt
,
1123 struct mlx5_db
*db
);
1124 void mlx5_ib_db_unmap_user(struct mlx5_ib_ucontext
*context
, struct mlx5_db
*db
);
1125 void __mlx5_ib_cq_clean(struct mlx5_ib_cq
*cq
, u32 qpn
, struct mlx5_ib_srq
*srq
);
1126 void mlx5_ib_cq_clean(struct mlx5_ib_cq
*cq
, u32 qpn
, struct mlx5_ib_srq
*srq
);
1127 void mlx5_ib_free_srq_wqe(struct mlx5_ib_srq
*srq
, int wqe_index
);
1128 int mlx5_ib_create_ah(struct ib_ah
*ah
, struct rdma_ah_attr
*ah_attr
, u32 flags
,
1129 struct ib_udata
*udata
);
1130 int mlx5_ib_query_ah(struct ib_ah
*ibah
, struct rdma_ah_attr
*ah_attr
);
1131 void mlx5_ib_destroy_ah(struct ib_ah
*ah
, u32 flags
);
1132 int mlx5_ib_create_srq(struct ib_srq
*srq
, struct ib_srq_init_attr
*init_attr
,
1133 struct ib_udata
*udata
);
1134 int mlx5_ib_modify_srq(struct ib_srq
*ibsrq
, struct ib_srq_attr
*attr
,
1135 enum ib_srq_attr_mask attr_mask
, struct ib_udata
*udata
);
1136 int mlx5_ib_query_srq(struct ib_srq
*ibsrq
, struct ib_srq_attr
*srq_attr
);
1137 void mlx5_ib_destroy_srq(struct ib_srq
*srq
, struct ib_udata
*udata
);
1138 int mlx5_ib_post_srq_recv(struct ib_srq
*ibsrq
, const struct ib_recv_wr
*wr
,
1139 const struct ib_recv_wr
**bad_wr
);
1140 int mlx5_ib_enable_lb(struct mlx5_ib_dev
*dev
, bool td
, bool qp
);
1141 void mlx5_ib_disable_lb(struct mlx5_ib_dev
*dev
, bool td
, bool qp
);
1142 struct ib_qp
*mlx5_ib_create_qp(struct ib_pd
*pd
,
1143 struct ib_qp_init_attr
*init_attr
,
1144 struct ib_udata
*udata
);
1145 int mlx5_ib_modify_qp(struct ib_qp
*ibqp
, struct ib_qp_attr
*attr
,
1146 int attr_mask
, struct ib_udata
*udata
);
1147 int mlx5_ib_query_qp(struct ib_qp
*ibqp
, struct ib_qp_attr
*qp_attr
, int qp_attr_mask
,
1148 struct ib_qp_init_attr
*qp_init_attr
);
1149 int mlx5_ib_destroy_qp(struct ib_qp
*qp
, struct ib_udata
*udata
);
1150 void mlx5_ib_drain_sq(struct ib_qp
*qp
);
1151 void mlx5_ib_drain_rq(struct ib_qp
*qp
);
1152 int mlx5_ib_post_send(struct ib_qp
*ibqp
, const struct ib_send_wr
*wr
,
1153 const struct ib_send_wr
**bad_wr
);
1154 int mlx5_ib_post_recv(struct ib_qp
*ibqp
, const struct ib_recv_wr
*wr
,
1155 const struct ib_recv_wr
**bad_wr
);
1156 int mlx5_ib_read_user_wqe_sq(struct mlx5_ib_qp
*qp
, int wqe_index
, void *buffer
,
1157 int buflen
, size_t *bc
);
1158 int mlx5_ib_read_user_wqe_rq(struct mlx5_ib_qp
*qp
, int wqe_index
, void *buffer
,
1159 int buflen
, size_t *bc
);
1160 int mlx5_ib_read_user_wqe_srq(struct mlx5_ib_srq
*srq
, int wqe_index
,
1161 void *buffer
, int buflen
, size_t *bc
);
1162 int mlx5_ib_create_cq(struct ib_cq
*ibcq
, const struct ib_cq_init_attr
*attr
,
1163 struct ib_udata
*udata
);
1164 void mlx5_ib_destroy_cq(struct ib_cq
*cq
, struct ib_udata
*udata
);
1165 int mlx5_ib_poll_cq(struct ib_cq
*ibcq
, int num_entries
, struct ib_wc
*wc
);
1166 int mlx5_ib_arm_cq(struct ib_cq
*ibcq
, enum ib_cq_notify_flags flags
);
1167 int mlx5_ib_modify_cq(struct ib_cq
*cq
, u16 cq_count
, u16 cq_period
);
1168 int mlx5_ib_resize_cq(struct ib_cq
*ibcq
, int entries
, struct ib_udata
*udata
);
1169 struct ib_mr
*mlx5_ib_get_dma_mr(struct ib_pd
*pd
, int acc
);
1170 struct ib_mr
*mlx5_ib_reg_user_mr(struct ib_pd
*pd
, u64 start
, u64 length
,
1171 u64 virt_addr
, int access_flags
,
1172 struct ib_udata
*udata
);
1173 int mlx5_ib_advise_mr(struct ib_pd
*pd
,
1174 enum ib_uverbs_advise_mr_advice advice
,
1176 struct ib_sge
*sg_list
,
1178 struct uverbs_attr_bundle
*attrs
);
1179 struct ib_mw
*mlx5_ib_alloc_mw(struct ib_pd
*pd
, enum ib_mw_type type
,
1180 struct ib_udata
*udata
);
1181 int mlx5_ib_dealloc_mw(struct ib_mw
*mw
);
1182 int mlx5_ib_update_xlt(struct mlx5_ib_mr
*mr
, u64 idx
, int npages
,
1183 int page_shift
, int flags
);
1184 struct mlx5_ib_mr
*mlx5_ib_alloc_implicit_mr(struct mlx5_ib_pd
*pd
,
1185 struct ib_udata
*udata
,
1187 void mlx5_ib_free_implicit_mr(struct mlx5_ib_mr
*mr
);
1188 void mlx5_ib_fence_odp_mr(struct mlx5_ib_mr
*mr
);
1189 int mlx5_ib_rereg_user_mr(struct ib_mr
*ib_mr
, int flags
, u64 start
,
1190 u64 length
, u64 virt_addr
, int access_flags
,
1191 struct ib_pd
*pd
, struct ib_udata
*udata
);
1192 int mlx5_ib_dereg_mr(struct ib_mr
*ibmr
, struct ib_udata
*udata
);
1193 struct ib_mr
*mlx5_ib_alloc_mr(struct ib_pd
*pd
, enum ib_mr_type mr_type
,
1194 u32 max_num_sg
, struct ib_udata
*udata
);
1195 struct ib_mr
*mlx5_ib_alloc_mr_integrity(struct ib_pd
*pd
,
1197 u32 max_num_meta_sg
);
1198 int mlx5_ib_map_mr_sg(struct ib_mr
*ibmr
, struct scatterlist
*sg
, int sg_nents
,
1199 unsigned int *sg_offset
);
1200 int mlx5_ib_map_mr_sg_pi(struct ib_mr
*ibmr
, struct scatterlist
*data_sg
,
1201 int data_sg_nents
, unsigned int *data_sg_offset
,
1202 struct scatterlist
*meta_sg
, int meta_sg_nents
,
1203 unsigned int *meta_sg_offset
);
1204 int mlx5_ib_process_mad(struct ib_device
*ibdev
, int mad_flags
, u8 port_num
,
1205 const struct ib_wc
*in_wc
, const struct ib_grh
*in_grh
,
1206 const struct ib_mad
*in
, struct ib_mad
*out
,
1207 size_t *out_mad_size
, u16
*out_mad_pkey_index
);
1208 struct ib_xrcd
*mlx5_ib_alloc_xrcd(struct ib_device
*ibdev
,
1209 struct ib_udata
*udata
);
1210 int mlx5_ib_dealloc_xrcd(struct ib_xrcd
*xrcd
, struct ib_udata
*udata
);
1211 int mlx5_ib_get_buf_offset(u64 addr
, int page_shift
, u32
*offset
);
1212 int mlx5_query_ext_port_caps(struct mlx5_ib_dev
*dev
, u8 port
);
1213 int mlx5_query_mad_ifc_smp_attr_node_info(struct ib_device
*ibdev
,
1214 struct ib_smp
*out_mad
);
1215 int mlx5_query_mad_ifc_system_image_guid(struct ib_device
*ibdev
,
1216 __be64
*sys_image_guid
);
1217 int mlx5_query_mad_ifc_max_pkeys(struct ib_device
*ibdev
,
1219 int mlx5_query_mad_ifc_vendor_id(struct ib_device
*ibdev
,
1221 int mlx5_query_mad_ifc_node_desc(struct mlx5_ib_dev
*dev
, char *node_desc
);
1222 int mlx5_query_mad_ifc_node_guid(struct mlx5_ib_dev
*dev
, __be64
*node_guid
);
1223 int mlx5_query_mad_ifc_pkey(struct ib_device
*ibdev
, u8 port
, u16 index
,
1225 int mlx5_query_mad_ifc_gids(struct ib_device
*ibdev
, u8 port
, int index
,
1227 int mlx5_query_mad_ifc_port(struct ib_device
*ibdev
, u8 port
,
1228 struct ib_port_attr
*props
);
1229 int mlx5_ib_query_port(struct ib_device
*ibdev
, u8 port
,
1230 struct ib_port_attr
*props
);
1231 int mlx5_ib_init_fmr(struct mlx5_ib_dev
*dev
);
1232 void mlx5_ib_cleanup_fmr(struct mlx5_ib_dev
*dev
);
1233 void mlx5_ib_cont_pages(struct ib_umem
*umem
, u64 addr
,
1234 unsigned long max_page_shift
,
1235 int *count
, int *shift
,
1236 int *ncont
, int *order
);
1237 void __mlx5_ib_populate_pas(struct mlx5_ib_dev
*dev
, struct ib_umem
*umem
,
1238 int page_shift
, size_t offset
, size_t num_pages
,
1239 __be64
*pas
, int access_flags
);
1240 void mlx5_ib_populate_pas(struct mlx5_ib_dev
*dev
, struct ib_umem
*umem
,
1241 int page_shift
, __be64
*pas
, int access_flags
);
1242 void mlx5_ib_copy_pas(u64
*old
, u64
*new, int step
, int num
);
1243 int mlx5_ib_get_cqe_size(struct ib_cq
*ibcq
);
1244 int mlx5_mr_cache_init(struct mlx5_ib_dev
*dev
);
1245 int mlx5_mr_cache_cleanup(struct mlx5_ib_dev
*dev
);
1247 struct mlx5_ib_mr
*mlx5_mr_cache_alloc(struct mlx5_ib_dev
*dev
, int entry
);
1248 void mlx5_mr_cache_free(struct mlx5_ib_dev
*dev
, struct mlx5_ib_mr
*mr
);
1249 int mlx5_mr_cache_invalidate(struct mlx5_ib_mr
*mr
);
1251 int mlx5_ib_check_mr_status(struct ib_mr
*ibmr
, u32 check_mask
,
1252 struct ib_mr_status
*mr_status
);
1253 struct ib_wq
*mlx5_ib_create_wq(struct ib_pd
*pd
,
1254 struct ib_wq_init_attr
*init_attr
,
1255 struct ib_udata
*udata
);
1256 void mlx5_ib_destroy_wq(struct ib_wq
*wq
, struct ib_udata
*udata
);
1257 int mlx5_ib_modify_wq(struct ib_wq
*wq
, struct ib_wq_attr
*wq_attr
,
1258 u32 wq_attr_mask
, struct ib_udata
*udata
);
1259 struct ib_rwq_ind_table
*mlx5_ib_create_rwq_ind_table(struct ib_device
*device
,
1260 struct ib_rwq_ind_table_init_attr
*init_attr
,
1261 struct ib_udata
*udata
);
1262 int mlx5_ib_destroy_rwq_ind_table(struct ib_rwq_ind_table
*wq_ind_table
);
1263 struct ib_dm
*mlx5_ib_alloc_dm(struct ib_device
*ibdev
,
1264 struct ib_ucontext
*context
,
1265 struct ib_dm_alloc_attr
*attr
,
1266 struct uverbs_attr_bundle
*attrs
);
1267 int mlx5_ib_dealloc_dm(struct ib_dm
*ibdm
, struct uverbs_attr_bundle
*attrs
);
1268 struct ib_mr
*mlx5_ib_reg_dm_mr(struct ib_pd
*pd
, struct ib_dm
*dm
,
1269 struct ib_dm_mr_attr
*attr
,
1270 struct uverbs_attr_bundle
*attrs
);
1272 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
1273 void mlx5_ib_internal_fill_odp_caps(struct mlx5_ib_dev
*dev
);
1274 int mlx5_ib_odp_init_one(struct mlx5_ib_dev
*ibdev
);
1275 void mlx5_ib_odp_cleanup_one(struct mlx5_ib_dev
*ibdev
);
1276 int __init
mlx5_ib_odp_init(void);
1277 void mlx5_ib_odp_cleanup(void);
1278 void mlx5_odp_init_mr_cache_entry(struct mlx5_cache_ent
*ent
);
1279 void mlx5_odp_populate_klm(struct mlx5_klm
*pklm
, size_t offset
,
1280 size_t nentries
, struct mlx5_ib_mr
*mr
, int flags
);
1282 int mlx5_ib_advise_mr_prefetch(struct ib_pd
*pd
,
1283 enum ib_uverbs_advise_mr_advice advice
,
1284 u32 flags
, struct ib_sge
*sg_list
, u32 num_sge
);
1285 #else /* CONFIG_INFINIBAND_ON_DEMAND_PAGING */
1286 static inline void mlx5_ib_internal_fill_odp_caps(struct mlx5_ib_dev
*dev
)
1291 static inline int mlx5_ib_odp_init_one(struct mlx5_ib_dev
*ibdev
) { return 0; }
1292 static inline void mlx5_ib_odp_cleanup_one(struct mlx5_ib_dev
*ibdev
) {}
1293 static inline int mlx5_ib_odp_init(void) { return 0; }
1294 static inline void mlx5_ib_odp_cleanup(void) {}
1295 static inline void mlx5_odp_init_mr_cache_entry(struct mlx5_cache_ent
*ent
) {}
1296 static inline void mlx5_odp_populate_klm(struct mlx5_klm
*pklm
, size_t offset
,
1297 size_t nentries
, struct mlx5_ib_mr
*mr
,
1301 mlx5_ib_advise_mr_prefetch(struct ib_pd
*pd
,
1302 enum ib_uverbs_advise_mr_advice advice
, u32 flags
,
1303 struct ib_sge
*sg_list
, u32 num_sge
)
1307 #endif /* CONFIG_INFINIBAND_ON_DEMAND_PAGING */
1309 extern const struct mmu_interval_notifier_ops mlx5_mn_ops
;
1311 /* Needed for rep profile */
1312 void __mlx5_ib_remove(struct mlx5_ib_dev
*dev
,
1313 const struct mlx5_ib_profile
*profile
,
1315 void *__mlx5_ib_add(struct mlx5_ib_dev
*dev
,
1316 const struct mlx5_ib_profile
*profile
);
1318 int mlx5_ib_get_vf_config(struct ib_device
*device
, int vf
,
1319 u8 port
, struct ifla_vf_info
*info
);
1320 int mlx5_ib_set_vf_link_state(struct ib_device
*device
, int vf
,
1321 u8 port
, int state
);
1322 int mlx5_ib_get_vf_stats(struct ib_device
*device
, int vf
,
1323 u8 port
, struct ifla_vf_stats
*stats
);
1324 int mlx5_ib_get_vf_guid(struct ib_device
*device
, int vf
, u8 port
,
1325 struct ifla_vf_guid
*node_guid
,
1326 struct ifla_vf_guid
*port_guid
);
1327 int mlx5_ib_set_vf_guid(struct ib_device
*device
, int vf
, u8 port
,
1328 u64 guid
, int type
);
1330 __be16
mlx5_get_roce_udp_sport(struct mlx5_ib_dev
*dev
,
1331 const struct ib_gid_attr
*attr
);
1333 void mlx5_ib_cleanup_cong_debugfs(struct mlx5_ib_dev
*dev
, u8 port_num
);
1334 void mlx5_ib_init_cong_debugfs(struct mlx5_ib_dev
*dev
, u8 port_num
);
1336 /* GSI QP helper functions */
1337 struct ib_qp
*mlx5_ib_gsi_create_qp(struct ib_pd
*pd
,
1338 struct ib_qp_init_attr
*init_attr
);
1339 int mlx5_ib_gsi_destroy_qp(struct ib_qp
*qp
);
1340 int mlx5_ib_gsi_modify_qp(struct ib_qp
*qp
, struct ib_qp_attr
*attr
,
1342 int mlx5_ib_gsi_query_qp(struct ib_qp
*qp
, struct ib_qp_attr
*qp_attr
,
1344 struct ib_qp_init_attr
*qp_init_attr
);
1345 int mlx5_ib_gsi_post_send(struct ib_qp
*qp
, const struct ib_send_wr
*wr
,
1346 const struct ib_send_wr
**bad_wr
);
1347 int mlx5_ib_gsi_post_recv(struct ib_qp
*qp
, const struct ib_recv_wr
*wr
,
1348 const struct ib_recv_wr
**bad_wr
);
1349 void mlx5_ib_gsi_pkey_change(struct mlx5_ib_gsi_qp
*gsi
);
1351 int mlx5_ib_generate_wc(struct ib_cq
*ibcq
, struct ib_wc
*wc
);
1353 void mlx5_ib_free_bfreg(struct mlx5_ib_dev
*dev
, struct mlx5_bfreg_info
*bfregi
,
1355 struct mlx5_ib_dev
*mlx5_ib_get_ibdev_from_mpi(struct mlx5_ib_multiport_info
*mpi
);
1356 struct mlx5_core_dev
*mlx5_ib_get_native_port_mdev(struct mlx5_ib_dev
*dev
,
1358 u8
*native_port_num
);
1359 void mlx5_ib_put_native_port_mdev(struct mlx5_ib_dev
*dev
,
1361 int mlx5_ib_fill_res_entry(struct sk_buff
*msg
,
1362 struct rdma_restrack_entry
*res
);
1363 int mlx5_ib_fill_stat_entry(struct sk_buff
*msg
,
1364 struct rdma_restrack_entry
*res
);
1366 #if IS_ENABLED(CONFIG_INFINIBAND_USER_ACCESS)
1367 int mlx5_ib_devx_create(struct mlx5_ib_dev
*dev
, bool is_user
);
1368 void mlx5_ib_devx_destroy(struct mlx5_ib_dev
*dev
, u16 uid
);
1369 void mlx5_ib_devx_init_event_table(struct mlx5_ib_dev
*dev
);
1370 void mlx5_ib_devx_cleanup_event_table(struct mlx5_ib_dev
*dev
);
1371 const struct uverbs_object_tree_def
*mlx5_ib_get_devx_tree(void);
1372 extern const struct uapi_definition mlx5_ib_devx_defs
[];
1373 extern const struct uapi_definition mlx5_ib_flow_defs
[];
1374 struct mlx5_ib_flow_handler
*mlx5_ib_raw_fs_rule_add(
1375 struct mlx5_ib_dev
*dev
, struct mlx5_ib_flow_matcher
*fs_matcher
,
1376 struct mlx5_flow_context
*flow_context
,
1377 struct mlx5_flow_act
*flow_act
, u32 counter_id
,
1378 void *cmd_in
, int inlen
, int dest_id
, int dest_type
);
1379 bool mlx5_ib_devx_is_flow_dest(void *obj
, int *dest_id
, int *dest_type
);
1380 bool mlx5_ib_devx_is_flow_counter(void *obj
, u32 offset
, u32
*counter_id
);
1381 int mlx5_ib_get_flow_trees(const struct uverbs_object_tree_def
**root
);
1382 void mlx5_ib_destroy_flow_action_raw(struct mlx5_ib_flow_action
*maction
);
1385 mlx5_ib_devx_create(struct mlx5_ib_dev
*dev
,
1386 bool is_user
) { return -EOPNOTSUPP
; }
1387 static inline void mlx5_ib_devx_destroy(struct mlx5_ib_dev
*dev
, u16 uid
) {}
1388 static inline void mlx5_ib_devx_init_event_table(struct mlx5_ib_dev
*dev
) {}
1389 static inline void mlx5_ib_devx_cleanup_event_table(struct mlx5_ib_dev
*dev
) {}
1390 static inline bool mlx5_ib_devx_is_flow_dest(void *obj
, int *dest_id
,
1396 mlx5_ib_destroy_flow_action_raw(struct mlx5_ib_flow_action
*maction
)
1401 static inline void init_query_mad(struct ib_smp
*mad
)
1403 mad
->base_version
= 1;
1404 mad
->mgmt_class
= IB_MGMT_CLASS_SUBN_LID_ROUTED
;
1405 mad
->class_version
= 1;
1406 mad
->method
= IB_MGMT_METHOD_GET
;
1409 static inline u8
convert_access(int acc
)
1411 return (acc
& IB_ACCESS_REMOTE_ATOMIC
? MLX5_PERM_ATOMIC
: 0) |
1412 (acc
& IB_ACCESS_REMOTE_WRITE
? MLX5_PERM_REMOTE_WRITE
: 0) |
1413 (acc
& IB_ACCESS_REMOTE_READ
? MLX5_PERM_REMOTE_READ
: 0) |
1414 (acc
& IB_ACCESS_LOCAL_WRITE
? MLX5_PERM_LOCAL_WRITE
: 0) |
1415 MLX5_PERM_LOCAL_READ
;
1418 static inline int is_qp1(enum ib_qp_type qp_type
)
1420 return qp_type
== MLX5_IB_QPT_HW_GSI
;
1423 #define MLX5_MAX_UMR_SHIFT 16
1424 #define MLX5_MAX_UMR_PAGES (1 << MLX5_MAX_UMR_SHIFT)
1426 static inline u32
check_cq_create_flags(u32 flags
)
1429 * It returns non-zero value for unsupported CQ
1430 * create flags, otherwise it returns zero.
1432 return (flags
& ~(IB_UVERBS_CQ_FLAGS_IGNORE_OVERRUN
|
1433 IB_UVERBS_CQ_FLAGS_TIMESTAMP_COMPLETION
));
1436 static inline int verify_assign_uidx(u8 cqe_version
, u32 cmd_uidx
,
1440 if ((cmd_uidx
== MLX5_IB_DEFAULT_UIDX
) ||
1441 (cmd_uidx
& ~MLX5_USER_ASSIGNED_UIDX_MASK
))
1443 *user_index
= cmd_uidx
;
1445 *user_index
= MLX5_IB_DEFAULT_UIDX
;
1451 static inline int get_qp_user_index(struct mlx5_ib_ucontext
*ucontext
,
1452 struct mlx5_ib_create_qp
*ucmd
,
1456 u8 cqe_version
= ucontext
->cqe_version
;
1458 if (field_avail(struct mlx5_ib_create_qp
, uidx
, inlen
) &&
1459 !cqe_version
&& (ucmd
->uidx
== MLX5_IB_DEFAULT_UIDX
))
1462 if (!!(field_avail(struct mlx5_ib_create_qp
, uidx
, inlen
) !=
1466 return verify_assign_uidx(cqe_version
, ucmd
->uidx
, user_index
);
1469 static inline int get_srq_user_index(struct mlx5_ib_ucontext
*ucontext
,
1470 struct mlx5_ib_create_srq
*ucmd
,
1474 u8 cqe_version
= ucontext
->cqe_version
;
1476 if (field_avail(struct mlx5_ib_create_srq
, uidx
, inlen
) &&
1477 !cqe_version
&& (ucmd
->uidx
== MLX5_IB_DEFAULT_UIDX
))
1480 if (!!(field_avail(struct mlx5_ib_create_srq
, uidx
, inlen
) !=
1484 return verify_assign_uidx(cqe_version
, ucmd
->uidx
, user_index
);
1487 static inline int get_uars_per_sys_page(struct mlx5_ib_dev
*dev
, bool lib_support
)
1489 return lib_support
&& MLX5_CAP_GEN(dev
->mdev
, uar_4k
) ?
1490 MLX5_UARS_IN_PAGE
: 1;
1493 static inline int get_num_static_uars(struct mlx5_ib_dev
*dev
,
1494 struct mlx5_bfreg_info
*bfregi
)
1496 return get_uars_per_sys_page(dev
, bfregi
->lib_uar_4k
) * bfregi
->num_static_sys_pages
;
1499 unsigned long mlx5_ib_get_xlt_emergency_page(void);
1500 void mlx5_ib_put_xlt_emergency_page(void);
1502 int bfregn_to_uar_index(struct mlx5_ib_dev
*dev
,
1503 struct mlx5_bfreg_info
*bfregi
, u32 bfregn
,
1506 int mlx5_ib_qp_set_counter(struct ib_qp
*qp
, struct rdma_counter
*counter
);
1507 u16
mlx5_ib_get_counters_id(struct mlx5_ib_dev
*dev
, u8 port_num
);
1509 static inline bool mlx5_ib_can_use_umr(struct mlx5_ib_dev
*dev
,
1510 bool do_modify_atomic
)
1512 if (MLX5_CAP_GEN(dev
->mdev
, umr_modify_entity_size_disabled
))
1515 if (do_modify_atomic
&&
1516 MLX5_CAP_GEN(dev
->mdev
, atomic
) &&
1517 MLX5_CAP_GEN(dev
->mdev
, umr_modify_atomic_disabled
))
1523 int mlx5_ib_enable_driver(struct ib_device
*dev
);
1524 int mlx5_ib_test_wc(struct mlx5_ib_dev
*dev
);
1525 #endif /* MLX5_IB_H */