]> git.proxmox.com Git - mirror_ubuntu-eoan-kernel.git/blame - drivers/net/ethernet/mellanox/mlx4/main.c
projects / mirror_ubuntu-eoan-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
UBUNTU: Ubuntu-5.3.0-29.31
[mirror_ubuntu-eoan-kernel.git] / drivers / net / ethernet / mellanox / mlx4 / main.c
CommitLineData
225c7b1f
RD		 1/*
2 * Copyright (c) 2004, 2005 Topspin Communications. All rights reserved.
3 * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.
51a379d0 4 * Copyright (c) 2005, 2006, 2007, 2008 Mellanox Technologies. All rights reserved.
225c7b1f
RD		 5 * Copyright (c) 2006, 2007 Cisco Systems, Inc. All rights reserved.
6 *
7 * This software is available to you under a choice of one of two
8 * licenses. You may choose to be licensed under the terms of the GNU
9 * General Public License (GPL) Version 2, available from the file
10 * COPYING in the main directory of this source tree, or the
11 * OpenIB.org BSD license below:
12 *
13 * Redistribution and use in source and binary forms, with or
14 * without modification, are permitted provided that the following
15 * conditions are met:
16 *
17 * - Redistributions of source code must retain the above
18 * copyright notice, this list of conditions and the following
19 * disclaimer.
20 *
21 * - Redistributions in binary form must reproduce the above
22 * copyright notice, this list of conditions and the following
23 * disclaimer in the documentation and/or other materials
24 * provided with the distribution.
25 *
26 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
27 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
28 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
29 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
30 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
31 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
32 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
33 * SOFTWARE.
34 */
35
36#include <linux/module.h>
691223ec 37#include <linux/kernel.h>
225c7b1f
RD		 38#include <linux/init.h>
39#include <linux/errno.h>
40#include <linux/pci.h>
41#include <linux/dma-mapping.h>
5a0e3ad6 42#include <linux/slab.h>
c1b43dca 43#include <linux/io-mapping.h>
ab9c17a0 44#include <linux/delay.h>
b046ffe5 45#include <linux/kmod.h>
10b1c04e 46#include <linux/etherdevice.h>
09d4d087 47#include <net/devlink.h>
225c7b1f 48
48962f5c 49#include <uapi/rdma/mlx4-abi.h>
225c7b1f
RD		 50#include <linux/mlx4/device.h>
51#include <linux/mlx4/doorbell.h>
52
53#include "mlx4.h"
54#include "fw.h"
55#include "icm.h"
56
57MODULE_AUTHOR("Roland Dreier");
58MODULE_DESCRIPTION("Mellanox ConnectX HCA low-level driver");
59MODULE_LICENSE("Dual BSD/GPL");
60MODULE_VERSION(DRV_VERSION);
61
27bf91d6
YP		 62struct workqueue_struct *mlx4_wq;
63
225c7b1f
RD		 64#ifdef CONFIG_MLX4_DEBUG
65
92a59ad0 66int mlx4_debug_level; /* 0 by default */
225c7b1f
RD		 67module_param_named(debug_level, mlx4_debug_level, int, 0644);
68MODULE_PARM_DESC(debug_level, "Enable debug tracing if > 0");
69
70#endif /* CONFIG_MLX4_DEBUG */
71
72#ifdef CONFIG_PCI_MSI
73
08fb1055 74static int msi_x = 1;
225c7b1f 75module_param(msi_x, int, 0444);
e5732838 76MODULE_PARM_DESC(msi_x, "0 - don't use MSI-X, 1 - use MSI-X, >1 - limit number of MSI-X irqs to msi_x");
225c7b1f
RD		 77
78#else /* CONFIG_PCI_MSI */
79
80#define msi_x (0)
81
82#endif /* CONFIG_PCI_MSI */
83
dd41cc3b 84static uint8_t num_vfs[3] = {0, 0, 0};
effa4bc4 85static int num_vfs_argc;
92a59ad0 86module_param_array(num_vfs, byte, &num_vfs_argc, 0444);
dd41cc3b
MB		 87MODULE_PARM_DESC(num_vfs, "enable #num_vfs functions if num_vfs > 0\n"
88 "num_vfs=port1,port2,port1+2");
89
90static uint8_t probe_vf[3] = {0, 0, 0};
effa4bc4 91static int probe_vfs_argc;
dd41cc3b
MB		 92module_param_array(probe_vf, byte, &probe_vfs_argc, 0444);
93MODULE_PARM_DESC(probe_vf, "number of vfs to probe by pf driver (num_vfs > 0)\n"
94 "probe_vf=port1,port2,port1+2");
ab9c17a0 95
3b68067b 96static int mlx4_log_num_mgm_entry_size = MLX4_DEFAULT_MGM_LOG_ENTRY_SIZE;
0ec2c0f8
EE		 97module_param_named(log_num_mgm_entry_size,
98 mlx4_log_num_mgm_entry_size, int, 0444);
99MODULE_PARM_DESC(log_num_mgm_entry_size, "log mgm size, that defines the num"
100 " of qp per mcg, for example:"
3c439b55 101 " 10 gives 248.range: 7 <="
0ff1fb65 102 " log_num_mgm_entry_size <= 12."
3c439b55
JM		 103 " To activate device managed"
104 " flow steering when available, set to -1");
0ec2c0f8 105
be902ab1 106static bool enable_64b_cqe_eqe = true;
08ff3235
OG		 107module_param(enable_64b_cqe_eqe, bool, 0444);
108MODULE_PARM_DESC(enable_64b_cqe_eqe,
be902ab1 109 "Enable 64 byte CQEs/EQEs when the FW supports this (default: True)");
08ff3235 110
76e39ccf
EC		 111static bool enable_4k_uar;
112module_param(enable_4k_uar, bool, 0444);
113MODULE_PARM_DESC(enable_4k_uar,
114 "Enable using 4K UAR. Should not be enabled if have VFs which do not support 4K UARs (default: false)");
115
77507aa2 116#define PF_CONTEXT_BEHAVIOUR_MASK (MLX4_FUNC_CAP_64B_EQE_CQE | \
7d077cd3
MB		 117 MLX4_FUNC_CAP_EQE_CQE_STRIDE | \
118 MLX4_FUNC_CAP_DMFS_A0_STATIC)
ab9c17a0 119
55ad3592
YH		 120#define RESET_PERSIST_MASK_FLAGS (MLX4_FLAG_SRIOV)
121
f57e6848 122static char mlx4_version[] =
225c7b1f 123 DRV_NAME ": Mellanox ConnectX core driver v"
cea2a6d8 124 DRV_VERSION "\n";
225c7b1f 125
3f2c5fb2 126static const struct mlx4_profile default_profile = {
ab9c17a0 127 .num_qp = 1 << 18,
225c7b1f 128 .num_srq = 1 << 16,
c9f2ba5e 129 .rdmarc_per_qp = 1 << 4,
225c7b1f
RD		 130 .num_cq = 1 << 16,
131 .num_mcg = 1 << 13,
ab9c17a0 132 .num_mpt = 1 << 19,
9fd7a1e1 133 .num_mtt = 1 << 20, /* It is really num mtt segements */
225c7b1f
RD		 134};
135
3f2c5fb2 136static const struct mlx4_profile low_mem_profile = {
2599d858
AV		 137 .num_qp = 1 << 17,
138 .num_srq = 1 << 6,
139 .rdmarc_per_qp = 1 << 4,
140 .num_cq = 1 << 8,
141 .num_mcg = 1 << 8,
142 .num_mpt = 1 << 9,
143 .num_mtt = 1 << 7,
144};
145
ab9c17a0 146static int log_num_mac = 7;
93fc9e1b
YP		 147module_param_named(log_num_mac, log_num_mac, int, 0444);
148MODULE_PARM_DESC(log_num_mac, "Log2 max number of MACs per ETH port (1-7)");
149
150static int log_num_vlan;
151module_param_named(log_num_vlan, log_num_vlan, int, 0444);
152MODULE_PARM_DESC(log_num_vlan, "Log2 max number of VLANs per ETH port (0-7)");
cb29688a
OG		 153/* Log2 max number of VLANs per ETH port (0-7) */
154#define MLX4_LOG_NUM_VLANS 7
2599d858
AV		 155#define MLX4_MIN_LOG_NUM_VLANS 0
156#define MLX4_MIN_LOG_NUM_MAC 1
93fc9e1b 157
eb939922 158static bool use_prio;
93fc9e1b 159module_param_named(use_prio, use_prio, bool, 0444);
ecc8fb11 160MODULE_PARM_DESC(use_prio, "Enable steering by VLAN priority on ETH ports (deprecated)");
93fc9e1b 161
7cc77bf4 162int log_mtts_per_seg = ilog2(1);
ab6bf42e 163module_param_named(log_mtts_per_seg, log_mtts_per_seg, int, 0444);
7cc77bf4
TT		 164MODULE_PARM_DESC(log_mtts_per_seg, "Log2 number of MTT entries per segment "
165 "(0-7) (default: 0)");
ab6bf42e 166
8d0fc7b6 167static int port_type_array[2] = {MLX4_PORT_TYPE_NONE, MLX4_PORT_TYPE_NONE};
ab9c17a0
JM		 168static int arr_argc = 2;
169module_param_array(port_type_array, int, &arr_argc, 0444);
8d0fc7b6
YP		 170MODULE_PARM_DESC(port_type_array, "Array of port types: HW_DEFAULT (0) is default "
171 "1 for IB, 2 for Ethernet");
ab9c17a0
JM		 172
173struct mlx4_port_config {
174 struct list_head list;
175 enum mlx4_port_type port_type[MLX4_MAX_PORTS + 1];
176 struct pci_dev *pdev;
177};
178
97989356
AV		 179static atomic_t pf_loading = ATOMIC_INIT(0);
180
bd1b51dc
MS		 181static int mlx4_devlink_ierr_reset_get(struct devlink *devlink, u32 id,
182 struct devlink_param_gset_ctx *ctx)
183{
184 ctx->val.vbool = !!mlx4_internal_err_reset;
185 return 0;
186}
187
188static int mlx4_devlink_ierr_reset_set(struct devlink *devlink, u32 id,
189 struct devlink_param_gset_ctx *ctx)
190{
191 mlx4_internal_err_reset = ctx->val.vbool;
192 return 0;
193}
194
3c641ba4
AV		 195static int mlx4_devlink_crdump_snapshot_get(struct devlink *devlink, u32 id,
196 struct devlink_param_gset_ctx *ctx)
197{
198 struct mlx4_priv *priv = devlink_priv(devlink);
199 struct mlx4_dev *dev = &priv->dev;
200
201 ctx->val.vbool = dev->persist->crdump.snapshot_enable;
202 return 0;
203}
204
205static int mlx4_devlink_crdump_snapshot_set(struct devlink *devlink, u32 id,
206 struct devlink_param_gset_ctx *ctx)
207{
208 struct mlx4_priv *priv = devlink_priv(devlink);
209 struct mlx4_dev *dev = &priv->dev;
210
211 dev->persist->crdump.snapshot_enable = ctx->val.vbool;
212 return 0;
213}
214
bd1b51dc
MS		 215static int
216mlx4_devlink_max_macs_validate(struct devlink *devlink, u32 id,
217 union devlink_param_value val,
218 struct netlink_ext_ack *extack)
219{
220 u32 value = val.vu32;
221
222 if (value < 1 || value > 128)
223 return -ERANGE;
224
225 if (!is_power_of_2(value)) {
226 NL_SET_ERR_MSG_MOD(extack, "max_macs supported must be power of 2");
227 return -EINVAL;
228 }
229
230 return 0;
231}
232
233enum mlx4_devlink_param_id {
234 MLX4_DEVLINK_PARAM_ID_BASE = DEVLINK_PARAM_GENERIC_ID_MAX,
235 MLX4_DEVLINK_PARAM_ID_ENABLE_64B_CQE_EQE,
236 MLX4_DEVLINK_PARAM_ID_ENABLE_4K_UAR,
237};
238
239static const struct devlink_param mlx4_devlink_params[] = {
240 DEVLINK_PARAM_GENERIC(INT_ERR_RESET,
241 BIT(DEVLINK_PARAM_CMODE_RUNTIME) |
242 BIT(DEVLINK_PARAM_CMODE_DRIVERINIT),
243 mlx4_devlink_ierr_reset_get,
244 mlx4_devlink_ierr_reset_set, NULL),
245 DEVLINK_PARAM_GENERIC(MAX_MACS,
246 BIT(DEVLINK_PARAM_CMODE_DRIVERINIT),
247 NULL, NULL, mlx4_devlink_max_macs_validate),
3c641ba4
AV		 248 DEVLINK_PARAM_GENERIC(REGION_SNAPSHOT,
249 BIT(DEVLINK_PARAM_CMODE_RUNTIME) |
250 BIT(DEVLINK_PARAM_CMODE_DRIVERINIT),
251 mlx4_devlink_crdump_snapshot_get,
252 mlx4_devlink_crdump_snapshot_set, NULL),
bd1b51dc
MS		 253 DEVLINK_PARAM_DRIVER(MLX4_DEVLINK_PARAM_ID_ENABLE_64B_CQE_EQE,
254 "enable_64b_cqe_eqe", DEVLINK_PARAM_TYPE_BOOL,
255 BIT(DEVLINK_PARAM_CMODE_DRIVERINIT),
256 NULL, NULL, NULL),
257 DEVLINK_PARAM_DRIVER(MLX4_DEVLINK_PARAM_ID_ENABLE_4K_UAR,
258 "enable_4k_uar", DEVLINK_PARAM_TYPE_BOOL,
259 BIT(DEVLINK_PARAM_CMODE_DRIVERINIT),
260 NULL, NULL, NULL),
261};
262
bd1b51dc
MS		 263static void mlx4_devlink_set_params_init_values(struct devlink *devlink)
264{
265 union devlink_param_value value;
266
267 value.vbool = !!mlx4_internal_err_reset;
26450608
MS		 268 devlink_param_driverinit_value_set(devlink,
269 DEVLINK_PARAM_GENERIC_ID_INT_ERR_RESET,
270 value);
bd1b51dc
MS		 271
272 value.vu32 = 1UL << log_num_mac;
26450608
MS		 273 devlink_param_driverinit_value_set(devlink,
274 DEVLINK_PARAM_GENERIC_ID_MAX_MACS,
275 value);
bd1b51dc
MS		 276
277 value.vbool = enable_64b_cqe_eqe;
26450608
MS		 278 devlink_param_driverinit_value_set(devlink,
279 MLX4_DEVLINK_PARAM_ID_ENABLE_64B_CQE_EQE,
280 value);
bd1b51dc
MS		 281
282 value.vbool = enable_4k_uar;
26450608
MS		 283 devlink_param_driverinit_value_set(devlink,
284 MLX4_DEVLINK_PARAM_ID_ENABLE_4K_UAR,
285 value);
3c641ba4
AV		 286
287 value.vbool = false;
26450608
MS		 288 devlink_param_driverinit_value_set(devlink,
289 DEVLINK_PARAM_GENERIC_ID_REGION_SNAPSHOT,
290 value);
bd1b51dc
MS		 291}
292
85743f1e
HN		 293static inline void mlx4_set_num_reserved_uars(struct mlx4_dev *dev,
294 struct mlx4_dev_cap *dev_cap)
295{
296 /* The reserved_uars is calculated by system page size unit.
297 * Therefore, adjustment is added when the uar page size is less
298 * than the system page size
299 */
300 dev->caps.reserved_uars =
301 max_t(int,
302 mlx4_get_num_reserved_uar(dev),
303 dev_cap->reserved_uars /
304 (1 << (PAGE_SHIFT - dev->uar_page_shift)));
305}
306
27bf91d6
YP		 307int mlx4_check_port_params(struct mlx4_dev *dev,
308 enum mlx4_port_type *port_type)
7ff93f8b
YP		 309{
310 int i;
311
0b997657
YS		 312 if (!(dev->caps.flags & MLX4_DEV_CAP_FLAG_DPDP)) {
313 for (i = 0; i < dev->caps.num_ports - 1; i++) {
314 if (port_type[i] != port_type[i + 1]) {
1a91de28 315 mlx4_err(dev, "Only same port types supported on this HCA, aborting\n");
95aac2cd 316 return -EOPNOTSUPP;
27bf91d6 317 }
7ff93f8b
YP		 318 }
319 }
7ff93f8b
YP		 320
321 for (i = 0; i < dev->caps.num_ports; i++) {
322 if (!(port_type[i] & dev->caps.supported_type[i+1])) {
1a91de28
JP		 323 mlx4_err(dev, "Requested port type for port %d is not supported on this HCA\n",
324 i + 1);
95aac2cd 325 return -EOPNOTSUPP;
7ff93f8b
YP		 326 }
327 }
328 return 0;
329}
330
331static void mlx4_set_port_mask(struct mlx4_dev *dev)
332{
333 int i;
334
7ff93f8b 335 for (i = 1; i <= dev->caps.num_ports; ++i)
65dab25d 336 dev->caps.port_mask[i] = dev->caps.port_type[i];
7ff93f8b 337}
f2a3f6a3 338
7ae0e400
MB		 339enum {
340 MLX4_QUERY_FUNC_NUM_SYS_EQS = 1 << 0,
341};
342
343static int mlx4_query_func(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap)
344{
345 int err = 0;
346 struct mlx4_func func;
347
348 if (dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_SYS_EQS) {
349 err = mlx4_QUERY_FUNC(dev, &func, 0);
350 if (err) {
351 mlx4_err(dev, "QUERY_DEV_CAP command failed, aborting.\n");
352 return err;
353 }
354 dev_cap->max_eqs = func.max_eq;
355 dev_cap->reserved_eqs = func.rsvd_eqs;
356 dev_cap->reserved_uars = func.rsvd_uars;
357 err |= MLX4_QUERY_FUNC_NUM_SYS_EQS;
358 }
359 return err;
360}
361
77507aa2
IS		 362static void mlx4_enable_cqe_eqe_stride(struct mlx4_dev *dev)
363{
364 struct mlx4_caps *dev_cap = &dev->caps;
365
366 /* FW not supporting or cancelled by user */
367 if (!(dev_cap->flags2 & MLX4_DEV_CAP_FLAG2_EQE_STRIDE) ||
368 !(dev_cap->flags2 & MLX4_DEV_CAP_FLAG2_CQE_STRIDE))
369 return;
370
371 /* Must have 64B CQE_EQE enabled by FW to use bigger stride
372 * When FW has NCSI it may decide not to report 64B CQE/EQEs
373 */
374 if (!(dev_cap->flags & MLX4_DEV_CAP_FLAG_64B_EQE) ||
375 !(dev_cap->flags & MLX4_DEV_CAP_FLAG_64B_CQE)) {
376 dev_cap->flags2 &= ~MLX4_DEV_CAP_FLAG2_CQE_STRIDE;
377 dev_cap->flags2 &= ~MLX4_DEV_CAP_FLAG2_EQE_STRIDE;
378 return;
379 }
380
381 if (cache_line_size() == 128 || cache_line_size() == 256) {
382 mlx4_dbg(dev, "Enabling CQE stride cacheLine supported\n");
383 /* Changing the real data inside CQE size to 32B */
384 dev_cap->flags &= ~MLX4_DEV_CAP_FLAG_64B_CQE;
385 dev_cap->flags &= ~MLX4_DEV_CAP_FLAG_64B_EQE;
386
387 if (mlx4_is_master(dev))
388 dev_cap->function_caps |= MLX4_FUNC_CAP_EQE_CQE_STRIDE;
389 } else {
0fab541a
OG		 390 if (cache_line_size() != 32 && cache_line_size() != 64)
391 mlx4_dbg(dev, "Disabling CQE stride, cacheLine size unsupported\n");
77507aa2
IS		 392 dev_cap->flags2 &= ~MLX4_DEV_CAP_FLAG2_CQE_STRIDE;
393 dev_cap->flags2 &= ~MLX4_DEV_CAP_FLAG2_EQE_STRIDE;
394 }
395}
396
431df8c7
MB		 397static int _mlx4_dev_port(struct mlx4_dev *dev, int port,
398 struct mlx4_port_cap *port_cap)
399{
400 dev->caps.vl_cap[port] = port_cap->max_vl;
401 dev->caps.ib_mtu_cap[port] = port_cap->ib_mtu;
402 dev->phys_caps.gid_phys_table_len[port] = port_cap->max_gids;
403 dev->phys_caps.pkey_phys_table_len[port] = port_cap->max_pkeys;
404 /* set gid and pkey table operating lengths by default
405 * to non-sriov values
406 */
407 dev->caps.gid_table_len[port] = port_cap->max_gids;
408 dev->caps.pkey_table_len[port] = port_cap->max_pkeys;
409 dev->caps.port_width_cap[port] = port_cap->max_port_width;
410 dev->caps.eth_mtu_cap[port] = port_cap->eth_mtu;
af7d5185 411 dev->caps.max_tc_eth = port_cap->max_tc_eth;
431df8c7
MB		 412 dev->caps.def_mac[port] = port_cap->def_mac;
413 dev->caps.supported_type[port] = port_cap->supported_port_types;
414 dev->caps.suggested_type[port] = port_cap->suggested_type;
415 dev->caps.default_sense[port] = port_cap->default_sense;
416 dev->caps.trans_type[port] = port_cap->trans_type;
417 dev->caps.vendor_oui[port] = port_cap->vendor_oui;
418 dev->caps.wavelength[port] = port_cap->wavelength;
419 dev->caps.trans_code[port] = port_cap->trans_code;
420
421 return 0;
422}
423
424static int mlx4_dev_port(struct mlx4_dev *dev, int port,
425 struct mlx4_port_cap *port_cap)
426{
427 int err = 0;
428
429 err = mlx4_QUERY_PORT(dev, port, port_cap);
430
431 if (err)
432 mlx4_err(dev, "QUERY_PORT command failed.\n");
433
434 return err;
435}
436
78500b8c
MM		 437static inline void mlx4_enable_ignore_fcs(struct mlx4_dev *dev)
438{
439 if (!(dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_IGNORE_FCS))
440 return;
441
442 if (mlx4_is_mfunc(dev)) {
443 mlx4_dbg(dev, "SRIOV mode - Disabling Ignore FCS");
444 dev->caps.flags2 &= ~MLX4_DEV_CAP_FLAG2_IGNORE_FCS;
445 return;
446 }
447
448 if (!(dev->caps.flags & MLX4_DEV_CAP_FLAG_FCS_KEEP)) {
449 mlx4_dbg(dev,
450 "Keep FCS is not supported - Disabling Ignore FCS");
451 dev->caps.flags2 &= ~MLX4_DEV_CAP_FLAG2_IGNORE_FCS;
452 return;
453 }
454}
455
431df8c7 456#define MLX4_A0_STEERING_TABLE_SIZE 256
3d73c288 457static int mlx4_dev_cap(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap)
225c7b1f
RD		 458{
459 int err;
5ae2a7a8 460 int i;
225c7b1f
RD		 461
462 err = mlx4_QUERY_DEV_CAP(dev, dev_cap);
463 if (err) {
1a91de28 464 mlx4_err(dev, "QUERY_DEV_CAP command failed, aborting\n");
225c7b1f
RD		 465 return err;
466 }
c78e25ed 467 mlx4_dev_cap_dump(dev, dev_cap);
225c7b1f
RD		 468
469 if (dev_cap->min_page_sz > PAGE_SIZE) {
1a91de28 470 mlx4_err(dev, "HCA minimum page size of %d bigger than kernel PAGE_SIZE of %ld, aborting\n",
225c7b1f
RD		 471 dev_cap->min_page_sz, PAGE_SIZE);
472 return -ENODEV;
473 }
474 if (dev_cap->num_ports > MLX4_MAX_PORTS) {
1a91de28 475 mlx4_err(dev, "HCA has %d ports, but we only support %d, aborting\n",
225c7b1f
RD		 476 dev_cap->num_ports, MLX4_MAX_PORTS);
477 return -ENODEV;
478 }
479
872bf2fb 480 if (dev_cap->uar_size > pci_resource_len(dev->persist->pdev, 2)) {
1a91de28 481 mlx4_err(dev, "HCA reported UAR size of 0x%x bigger than PCI resource 2 size of 0x%llx, aborting\n",
225c7b1f 482 dev_cap->uar_size,
872bf2fb
YH		 483 (unsigned long long)
484 pci_resource_len(dev->persist->pdev, 2));
225c7b1f
RD		 485 return -ENODEV;
486 }
487
488 dev->caps.num_ports = dev_cap->num_ports;
7ae0e400
MB		 489 dev->caps.num_sys_eqs = dev_cap->num_sys_eqs;
490 dev->phys_caps.num_phys_eqs = dev_cap->flags2 & MLX4_DEV_CAP_FLAG2_SYS_EQS ?
491 dev->caps.num_sys_eqs :
492 MLX4_MAX_EQ_NUM;
5ae2a7a8 493 for (i = 1; i <= dev->caps.num_ports; ++i) {
431df8c7
MB		 494 err = _mlx4_dev_port(dev, i, dev_cap->port_cap + i);
495 if (err) {
496 mlx4_err(dev, "QUERY_PORT command failed, aborting\n");
497 return err;
498 }
5ae2a7a8
RD		 499 }
500
ab9c17a0 501 dev->caps.uar_page_size = PAGE_SIZE;
225c7b1f 502 dev->caps.num_uars = dev_cap->uar_size / PAGE_SIZE;
225c7b1f
RD		 503 dev->caps.local_ca_ack_delay = dev_cap->local_ca_ack_delay;
504 dev->caps.bf_reg_size = dev_cap->bf_reg_size;
505 dev->caps.bf_regs_per_page = dev_cap->bf_regs_per_page;
506 dev->caps.max_sq_sg = dev_cap->max_sq_sg;
507 dev->caps.max_rq_sg = dev_cap->max_rq_sg;
508 dev->caps.max_wqes = dev_cap->max_qp_sz;
509 dev->caps.max_qp_init_rdma = dev_cap->max_requester_per_qp;
225c7b1f
RD		 510 dev->caps.max_srq_wqes = dev_cap->max_srq_sz;
511 dev->caps.max_srq_sge = dev_cap->max_rq_sg - 1;
512 dev->caps.reserved_srqs = dev_cap->reserved_srqs;
513 dev->caps.max_sq_desc_sz = dev_cap->max_sq_desc_sz;
514 dev->caps.max_rq_desc_sz = dev_cap->max_rq_desc_sz;
225c7b1f
RD		 515 /*
516 * Subtract 1 from the limit because we need to allocate a
517 * spare CQE so the HCA HW can tell the difference between an
518 * empty CQ and a full CQ.
519 */
520 dev->caps.max_cqes = dev_cap->max_cq_sz - 1;
521 dev->caps.reserved_cqs = dev_cap->reserved_cqs;
522 dev->caps.reserved_eqs = dev_cap->reserved_eqs;
2b8fb286 523 dev->caps.reserved_mtts = dev_cap->reserved_mtts;
225c7b1f 524 dev->caps.reserved_mrws = dev_cap->reserved_mrws;
ab9c17a0 525
225c7b1f 526 dev->caps.reserved_pds = dev_cap->reserved_pds;
012a8ff5
SH		 527 dev->caps.reserved_xrcds = (dev->caps.flags & MLX4_DEV_CAP_FLAG_XRC) ?
528 dev_cap->reserved_xrcds : 0;
529 dev->caps.max_xrcds = (dev->caps.flags & MLX4_DEV_CAP_FLAG_XRC) ?
530 dev_cap->max_xrcds : 0;
2b8fb286
MA		 531 dev->caps.mtt_entry_sz = dev_cap->mtt_entry_sz;
532
149983af 533 dev->caps.max_msg_sz = dev_cap->max_msg_sz;
225c7b1f
RD		 534 dev->caps.page_size_cap = ~(u32) (dev_cap->min_page_sz - 1);
535 dev->caps.flags = dev_cap->flags;
b3416f44 536 dev->caps.flags2 = dev_cap->flags2;
95d04f07
RD		 537 dev->caps.bmme_flags = dev_cap->bmme_flags;
538 dev->caps.reserved_lkey = dev_cap->reserved_lkey;
225c7b1f 539 dev->caps.stat_rate_support = dev_cap->stat_rate_support;
b832be1e 540 dev->caps.max_gso_sz = dev_cap->max_gso_sz;
b3416f44 541 dev->caps.max_rss_tbl_sz = dev_cap->max_rss_tbl_sz;
c994f778
IK		 542 dev->caps.wol_port[1] = dev_cap->wol_port[1];
543 dev->caps.wol_port[2] = dev_cap->wol_port[2];
523f9eb1 544 dev->caps.health_buffer_addrs = dev_cap->health_buffer_addrs;
225c7b1f 545
85743f1e
HN		 546 /* Save uar page shift */
547 if (!mlx4_is_slave(dev)) {
548 /* Virtual PCI function needs to determine UAR page size from
549 * firmware. Only master PCI function can set the uar page size
550 */
ca3d89a3 551 if (enable_4k_uar || !dev->persist->num_vfs)
76e39ccf
EC		 552 dev->uar_page_shift = DEFAULT_UAR_PAGE_SHIFT;
553 else
554 dev->uar_page_shift = PAGE_SHIFT;
555
85743f1e
HN		 556 mlx4_set_num_reserved_uars(dev, dev_cap);
557 }
558
77fc29c4
HHZ		 559 if (dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_PHV_EN) {
560 struct mlx4_init_hca_param hca_param;
561
562 memset(&hca_param, 0, sizeof(hca_param));
563 err = mlx4_QUERY_HCA(dev, &hca_param);
564 /* Turn off PHV_EN flag in case phv_check_en is set.
565 * phv_check_en is a HW check that parse the packet and verify
566 * phv bit was reported correctly in the wqe. To allow QinQ
567 * PHV_EN flag should be set and phv_check_en must be cleared
568 * otherwise QinQ packets will be drop by the HW.
569 */
570 if (err || hca_param.phv_check_en)
571 dev->caps.flags2 &= ~MLX4_DEV_CAP_FLAG2_PHV_EN;
572 }
573
ca3e57a5
RD		 574 /* Sense port always allowed on supported devices for ConnectX-1 and -2 */
575 if (mlx4_priv(dev)->pci_dev_data & MLX4_PCI_DEV_FORCE_SENSE_PORT)
58a60168 576 dev->caps.flags |= MLX4_DEV_CAP_FLAG_SENSE_SUPPORT;
aadf4f3f
RD		 577 /* Don't do sense port on multifunction devices (for now at least) */
578 if (mlx4_is_mfunc(dev))
579 dev->caps.flags &= ~MLX4_DEV_CAP_FLAG_SENSE_SUPPORT;
58a60168 580
2599d858
AV		 581 if (mlx4_low_memory_profile()) {
582 dev->caps.log_num_macs = MLX4_MIN_LOG_NUM_MAC;
583 dev->caps.log_num_vlans = MLX4_MIN_LOG_NUM_VLANS;
584 } else {
585 dev->caps.log_num_macs = log_num_mac;
586 dev->caps.log_num_vlans = MLX4_LOG_NUM_VLANS;
587 }
93fc9e1b
YP		 588
589 for (i = 1; i <= dev->caps.num_ports; ++i) {
ab9c17a0
JM		 590 dev->caps.port_type[i] = MLX4_PORT_TYPE_NONE;
591 if (dev->caps.supported_type[i]) {
592 /* if only ETH is supported - assign ETH */
593 if (dev->caps.supported_type[i] == MLX4_PORT_TYPE_ETH)
594 dev->caps.port_type[i] = MLX4_PORT_TYPE_ETH;
105c320f 595 /* if only IB is supported, assign IB */
ab9c17a0 596 else if (dev->caps.supported_type[i] ==
105c320f
JM		 597 MLX4_PORT_TYPE_IB)
598 dev->caps.port_type[i] = MLX4_PORT_TYPE_IB;
ab9c17a0 599 else {
105c320f
JM		 600 /* if IB and ETH are supported, we set the port
601 * type according to user selection of port type;
602 * if user selected none, take the FW hint */
603 if (port_type_array[i - 1] == MLX4_PORT_TYPE_NONE)
8d0fc7b6
YP		 604 dev->caps.port_type[i] = dev->caps.suggested_type[i] ?
605 MLX4_PORT_TYPE_ETH : MLX4_PORT_TYPE_IB;
ab9c17a0 606 else
105c320f 607 dev->caps.port_type[i] = port_type_array[i - 1];
ab9c17a0
JM		 608 }
609 }
8d0fc7b6
YP		 610 /*
611 * Link sensing is allowed on the port if 3 conditions are true:
612 * 1. Both protocols are supported on the port.
613 * 2. Different types are supported on the port
614 * 3. FW declared that it supports link sensing
615 */
27bf91d6 616 mlx4_priv(dev)->sense.sense_allowed[i] =
58a60168 617 ((dev->caps.supported_type[i] == MLX4_PORT_TYPE_AUTO) &&
8d0fc7b6 618 (dev->caps.flags & MLX4_DEV_CAP_FLAG_DPDP) &&
58a60168 619 (dev->caps.flags & MLX4_DEV_CAP_FLAG_SENSE_SUPPORT));
7ff93f8b 620
8d0fc7b6
YP		 621 /*
622 * If "default_sense" bit is set, we move the port to "AUTO" mode
623 * and perform sense_port FW command to try and set the correct
624 * port type from beginning
625 */
46c46747 626 if (mlx4_priv(dev)->sense.sense_allowed[i] && dev->caps.default_sense[i]) {
8d0fc7b6
YP		 627 enum mlx4_port_type sensed_port = MLX4_PORT_TYPE_NONE;
628 dev->caps.possible_type[i] = MLX4_PORT_TYPE_AUTO;
629 mlx4_SENSE_PORT(dev, i, &sensed_port);
630 if (sensed_port != MLX4_PORT_TYPE_NONE)
631 dev->caps.port_type[i] = sensed_port;
632 } else {
633 dev->caps.possible_type[i] = dev->caps.port_type[i];
634 }
635
431df8c7
MB		 636 if (dev->caps.log_num_macs > dev_cap->port_cap[i].log_max_macs) {
637 dev->caps.log_num_macs = dev_cap->port_cap[i].log_max_macs;
1a91de28 638 mlx4_warn(dev, "Requested number of MACs is too much for port %d, reducing to %d\n",
93fc9e1b
YP		 639 i, 1 << dev->caps.log_num_macs);
640 }
431df8c7
MB		 641 if (dev->caps.log_num_vlans > dev_cap->port_cap[i].log_max_vlans) {
642 dev->caps.log_num_vlans = dev_cap->port_cap[i].log_max_vlans;
1a91de28 643 mlx4_warn(dev, "Requested number of VLANs is too much for port %d, reducing to %d\n",
93fc9e1b
YP		 644 i, 1 << dev->caps.log_num_vlans);
645 }
646 }
647
ac0a72a3
OG		 648 if (mlx4_is_master(dev) && (dev->caps.num_ports == 2) &&
649 (port_type_array[0] == MLX4_PORT_TYPE_IB) &&
650 (port_type_array[1] == MLX4_PORT_TYPE_ETH)) {
651 mlx4_warn(dev,
652 "Granular QoS per VF not supported with IB/Eth configuration\n");
653 dev->caps.flags2 &= ~MLX4_DEV_CAP_FLAG2_QOS_VPP;
654 }
655
47d8417f 656 dev->caps.max_counters = dev_cap->max_counters;
f2a3f6a3 657
93fc9e1b
YP		 658 dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FW] = dev_cap->reserved_qps;
659 dev->caps.reserved_qps_cnt[MLX4_QP_REGION_ETH_ADDR] =
660 dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FC_ADDR] =
661 (1 << dev->caps.log_num_macs) *
662 (1 << dev->caps.log_num_vlans) *
93fc9e1b
YP		 663 dev->caps.num_ports;
664 dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FC_EXCH] = MLX4_NUM_FEXCH;
7d077cd3
MB		 665
666 if (dev_cap->dmfs_high_rate_qpn_base > 0 &&
667 dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_FS_EN)
668 dev->caps.dmfs_high_rate_qpn_base = dev_cap->dmfs_high_rate_qpn_base;
669 else
670 dev->caps.dmfs_high_rate_qpn_base =
671 dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FW];
672
673 if (dev_cap->dmfs_high_rate_qpn_range > 0 &&
674 dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_FS_EN) {
675 dev->caps.dmfs_high_rate_qpn_range = dev_cap->dmfs_high_rate_qpn_range;
676 dev->caps.dmfs_high_steer_mode = MLX4_STEERING_DMFS_A0_DEFAULT;
677 dev->caps.flags2 |= MLX4_DEV_CAP_FLAG2_FS_A0;
678 } else {
679 dev->caps.dmfs_high_steer_mode = MLX4_STEERING_DMFS_A0_NOT_SUPPORTED;
680 dev->caps.dmfs_high_rate_qpn_base =
681 dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FW];
682 dev->caps.dmfs_high_rate_qpn_range = MLX4_A0_STEERING_TABLE_SIZE;
683 }
684
fc31e256
OG		 685 dev->caps.rl_caps = dev_cap->rl_caps;
686
d57febe1 687 dev->caps.reserved_qps_cnt[MLX4_QP_REGION_RSS_RAW_ETH] =
7d077cd3 688 dev->caps.dmfs_high_rate_qpn_range;
93fc9e1b
YP		 689
690 dev->caps.reserved_qps = dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FW] +
691 dev->caps.reserved_qps_cnt[MLX4_QP_REGION_ETH_ADDR] +
692 dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FC_ADDR] +
693 dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FC_EXCH];
694
e2c76824 695 dev->caps.sqp_demux = (mlx4_is_master(dev)) ? MLX4_MAX_NUM_SLAVES : 0;
08ff3235 696
b3051320 697 if (!enable_64b_cqe_eqe && !mlx4_is_slave(dev)) {
08ff3235
OG		 698 if (dev_cap->flags &
699 (MLX4_DEV_CAP_FLAG_64B_CQE | MLX4_DEV_CAP_FLAG_64B_EQE)) {
700 mlx4_warn(dev, "64B EQEs/CQEs supported by the device but not enabled\n");
701 dev->caps.flags &= ~MLX4_DEV_CAP_FLAG_64B_CQE;
702 dev->caps.flags &= ~MLX4_DEV_CAP_FLAG_64B_EQE;
703 }
77507aa2
IS		 704
705 if (dev_cap->flags2 &
706 (MLX4_DEV_CAP_FLAG2_CQE_STRIDE |
707 MLX4_DEV_CAP_FLAG2_EQE_STRIDE)) {
708 mlx4_warn(dev, "Disabling EQE/CQE stride per user request\n");
709 dev_cap->flags2 &= ~MLX4_DEV_CAP_FLAG2_CQE_STRIDE;
710 dev_cap->flags2 &= ~MLX4_DEV_CAP_FLAG2_EQE_STRIDE;
711 }
08ff3235
OG		 712 }
713
f97b4b5d 714 if ((dev->caps.flags &
08ff3235
OG		 715 (MLX4_DEV_CAP_FLAG_64B_CQE | MLX4_DEV_CAP_FLAG_64B_EQE)) &&
716 mlx4_is_master(dev))
717 dev->caps.function_caps |= MLX4_FUNC_CAP_64B_EQE_CQE;
718
ddae0349 719 if (!mlx4_is_slave(dev)) {
77507aa2 720 mlx4_enable_cqe_eqe_stride(dev);
ddae0349 721 dev->caps.alloc_res_qp_mask =
d57febe1
MB		 722 (dev->caps.bf_reg_size ? MLX4_RESERVE_ETH_BF_QP : 0) |
723 MLX4_RESERVE_A0_QP;
3742cc65
IS		 724
725 if (!(dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_ETS_CFG) &&
726 dev->caps.flags & MLX4_DEV_CAP_FLAG_SET_ETH_SCHED) {
727 mlx4_warn(dev, "Old device ETS support detected\n");
728 mlx4_warn(dev, "Consider upgrading device FW.\n");
729 dev->caps.flags2 |= MLX4_DEV_CAP_FLAG2_ETS_CFG;
730 }
731
ddae0349
EE		 732 } else {
733 dev->caps.alloc_res_qp_mask = 0;
734 }
77507aa2 735
78500b8c
MM		 736 mlx4_enable_ignore_fcs(dev);
737
225c7b1f
RD		 738 return 0;
739}
b912b2f8 740
ab9c17a0
JM		 741/*The function checks if there are live vf, return the num of them*/
742static int mlx4_how_many_lives_vf(struct mlx4_dev *dev)
743{
744 struct mlx4_priv *priv = mlx4_priv(dev);
745 struct mlx4_slave_state *s_state;
746 int i;
747 int ret = 0;
748
749 for (i = 1/*the ppf is 0*/; i < dev->num_slaves; ++i) {
750 s_state = &priv->mfunc.master.slave_state[i];
751 if (s_state->active && s_state->last_cmd !=
752 MLX4_COMM_CMD_RESET) {
753 mlx4_warn(dev, "%s: slave: %d is still active\n",
754 __func__, i);
755 ret++;
756 }
757 }
758 return ret;
759}
760
396f2feb
JM		 761int mlx4_get_parav_qkey(struct mlx4_dev *dev, u32 qpn, u32 *qkey)
762{
763 u32 qk = MLX4_RESERVED_QKEY_BASE;
47605df9
JM		 764
765 if (qpn >= dev->phys_caps.base_tunnel_sqpn + 8 * MLX4_MFUNC_MAX ||
766 qpn < dev->phys_caps.base_proxy_sqpn)
396f2feb
JM		 767 return -EINVAL;
768
47605df9 769 if (qpn >= dev->phys_caps.base_tunnel_sqpn)
396f2feb 770 /* tunnel qp */
47605df9 771 qk += qpn - dev->phys_caps.base_tunnel_sqpn;
396f2feb 772 else
47605df9 773 qk += qpn - dev->phys_caps.base_proxy_sqpn;
396f2feb
JM		 774 *qkey = qk;
775 return 0;
776}
777EXPORT_SYMBOL(mlx4_get_parav_qkey);
778
54679e14
JM		 779void mlx4_sync_pkey_table(struct mlx4_dev *dev, int slave, int port, int i, int val)
780{
781 struct mlx4_priv *priv = container_of(dev, struct mlx4_priv, dev);
782
783 if (!mlx4_is_master(dev))
784 return;
785
786 priv->virt2phys_pkey[slave][port - 1][i] = val;
787}
788EXPORT_SYMBOL(mlx4_sync_pkey_table);
789
afa8fd1d
JM		 790void mlx4_put_slave_node_guid(struct mlx4_dev *dev, int slave, __be64 guid)
791{
792 struct mlx4_priv *priv = container_of(dev, struct mlx4_priv, dev);
793
794 if (!mlx4_is_master(dev))
795 return;
796
797 priv->slave_node_guids[slave] = guid;
798}
799EXPORT_SYMBOL(mlx4_put_slave_node_guid);
800
801__be64 mlx4_get_slave_node_guid(struct mlx4_dev *dev, int slave)
802{
803 struct mlx4_priv *priv = container_of(dev, struct mlx4_priv, dev);
804
805 if (!mlx4_is_master(dev))
806 return 0;
807
808 return priv->slave_node_guids[slave];
809}
810EXPORT_SYMBOL(mlx4_get_slave_node_guid);
811
e10903b0 812int mlx4_is_slave_active(struct mlx4_dev *dev, int slave)
ab9c17a0
JM		 813{
814 struct mlx4_priv *priv = mlx4_priv(dev);
815 struct mlx4_slave_state *s_slave;
816
817 if (!mlx4_is_master(dev))
818 return 0;
819
820 s_slave = &priv->mfunc.master.slave_state[slave];
821 return !!s_slave->active;
822}
823EXPORT_SYMBOL(mlx4_is_slave_active);
824
10b1c04e
JM		 825void mlx4_handle_eth_header_mcast_prio(struct mlx4_net_trans_rule_hw_ctrl *ctrl,
826 struct _rule_hw *eth_header)
827{
828 if (is_multicast_ether_addr(eth_header->eth.dst_mac) ||
829 is_broadcast_ether_addr(eth_header->eth.dst_mac)) {
830 struct mlx4_net_trans_rule_hw_eth *eth =
831 (struct mlx4_net_trans_rule_hw_eth *)eth_header;
832 struct _rule_hw *next_rule = (struct _rule_hw *)(eth + 1);
833 bool last_rule = next_rule->size == 0 && next_rule->id == 0 &&
834 next_rule->rsvd == 0;
835
836 if (last_rule)
837 ctrl->prio = cpu_to_be16(MLX4_DOMAIN_NIC);
838 }
839}
840EXPORT_SYMBOL(mlx4_handle_eth_header_mcast_prio);
841
7b8157be
JM		 842static void slave_adjust_steering_mode(struct mlx4_dev *dev,
843 struct mlx4_dev_cap *dev_cap,
844 struct mlx4_init_hca_param *hca_param)
845{
846 dev->caps.steering_mode = hca_param->steering_mode;
847 if (dev->caps.steering_mode == MLX4_STEERING_MODE_DEVICE_MANAGED) {
848 dev->caps.num_qp_per_mgm = dev_cap->fs_max_num_qp_per_entry;
849 dev->caps.fs_log_max_ucast_qp_range_size =
850 dev_cap->fs_log_max_ucast_qp_range_size;
851 } else
852 dev->caps.num_qp_per_mgm =
853 4 * ((1 << hca_param->log_mc_entry_sz)/16 - 2);
854
855 mlx4_dbg(dev, "Steering mode is: %s\n",
856 mlx4_steering_mode_str(dev->caps.steering_mode));
857}
858
c73c8b1e
EBE		 859static void mlx4_slave_destroy_special_qp_cap(struct mlx4_dev *dev)
860{
861 kfree(dev->caps.spec_qps);
862 dev->caps.spec_qps = NULL;
863}
864
865static int mlx4_slave_special_qp_cap(struct mlx4_dev *dev)
866{
867 struct mlx4_func_cap *func_cap = NULL;
868 struct mlx4_caps *caps = &dev->caps;
869 int i, err = 0;
870
871 func_cap = kzalloc(sizeof(*func_cap), GFP_KERNEL);
872 caps->spec_qps = kcalloc(caps->num_ports, sizeof(*caps->spec_qps), GFP_KERNEL);
873
874 if (!func_cap || !caps->spec_qps) {
875 mlx4_err(dev, "Failed to allocate memory for special qps cap\n");
876 err = -ENOMEM;
877 goto err_mem;
878 }
879
880 for (i = 1; i <= caps->num_ports; ++i) {
881 err = mlx4_QUERY_FUNC_CAP(dev, i, func_cap);
882 if (err) {
883 mlx4_err(dev, "QUERY_FUNC_CAP port command failed for port %d, aborting (%d)\n",
884 i, err);
885 goto err_mem;
886 }
887 caps->spec_qps[i - 1] = func_cap->spec_qps;
888 caps->port_mask[i] = caps->port_type[i];
889 caps->phys_port_id[i] = func_cap->phys_port_id;
890 err = mlx4_get_slave_pkey_gid_tbl_len(dev, i,
891 &caps->gid_table_len[i],
892 &caps->pkey_table_len[i]);
893 if (err) {
894 mlx4_err(dev, "QUERY_PORT command failed for port %d, aborting (%d)\n",
895 i, err);
896 goto err_mem;
897 }
898 }
899
900err_mem:
901 if (err)
902 mlx4_slave_destroy_special_qp_cap(dev);
903 kfree(func_cap);
904 return err;
905}
906
ab9c17a0
JM		 907static int mlx4_slave_cap(struct mlx4_dev *dev)
908{
909 int err;
910 u32 page_size;
c73c8b1e
EBE		 911 struct mlx4_dev_cap *dev_cap = NULL;
912 struct mlx4_func_cap *func_cap = NULL;
913 struct mlx4_init_hca_param *hca_param = NULL;
914
915 hca_param = kzalloc(sizeof(*hca_param), GFP_KERNEL);
916 func_cap = kzalloc(sizeof(*func_cap), GFP_KERNEL);
917 dev_cap = kzalloc(sizeof(*dev_cap), GFP_KERNEL);
918 if (!hca_param || !func_cap || !dev_cap) {
919 mlx4_err(dev, "Failed to allocate memory for slave_cap\n");
920 err = -ENOMEM;
921 goto free_mem;
922 }
ab9c17a0 923
c73c8b1e 924 err = mlx4_QUERY_HCA(dev, hca_param);
ab9c17a0 925 if (err) {
1a91de28 926 mlx4_err(dev, "QUERY_HCA command failed, aborting\n");
c73c8b1e 927 goto free_mem;
ab9c17a0
JM		 928 }
929
483e0132
EP		 930 /* fail if the hca has an unknown global capability
931 * at this time global_caps should be always zeroed
932 */
c73c8b1e 933 if (hca_param->global_caps) {
ab9c17a0 934 mlx4_err(dev, "Unknown hca global capabilities\n");
c73c8b1e
EBE		 935 err = -EINVAL;
936 goto free_mem;
ab9c17a0
JM		 937 }
938
c73c8b1e 939 dev->caps.hca_core_clock = hca_param->hca_core_clock;
ddd8a6c1 940
c73c8b1e
EBE		 941 dev->caps.max_qp_dest_rdma = 1 << hca_param->log_rd_per_qp;
942 err = mlx4_dev_cap(dev, dev_cap);
ab9c17a0 943 if (err) {
1a91de28 944 mlx4_err(dev, "QUERY_DEV_CAP command failed, aborting\n");
c73c8b1e 945 goto free_mem;
ab9c17a0
JM		 946 }
947
b91cb3eb
JM		 948 err = mlx4_QUERY_FW(dev);
949 if (err)
1a91de28 950 mlx4_err(dev, "QUERY_FW command failed: could not get FW version\n");
b91cb3eb 951
ab9c17a0
JM		 952 page_size = ~dev->caps.page_size_cap + 1;
953 mlx4_warn(dev, "HCA minimum page size:%d\n", page_size);
954 if (page_size > PAGE_SIZE) {
1a91de28 955 mlx4_err(dev, "HCA minimum page size of %d bigger than kernel PAGE_SIZE of %ld, aborting\n",
ab9c17a0 956 page_size, PAGE_SIZE);
c73c8b1e
EBE		 957 err = -ENODEV;
958 goto free_mem;
ab9c17a0
JM		 959 }
960
85743f1e 961 /* Set uar_page_shift for VF */
c73c8b1e 962 dev->uar_page_shift = hca_param->uar_page_sz + 12;
ab9c17a0 963
85743f1e
HN		 964 /* Make sure the master uar page size is valid */
965 if (dev->uar_page_shift > PAGE_SHIFT) {
966 mlx4_err(dev,
967 "Invalid configuration: uar page size is larger than system page size\n");
c73c8b1e
EBE		 968 err = -ENODEV;
969 goto free_mem;
ab9c17a0
JM		 970 }
971
85743f1e 972 /* Set reserved_uars based on the uar_page_shift */
c73c8b1e 973 mlx4_set_num_reserved_uars(dev, dev_cap);
85743f1e
HN		 974
975 /* Although uar page size in FW differs from system page size,
976 * upper software layers (mlx4_ib, mlx4_en and part of mlx4_core)
977 * still works with assumption that uar page size == system page size
978 */
979 dev->caps.uar_page_size = PAGE_SIZE;
980
c73c8b1e 981 err = mlx4_QUERY_FUNC_CAP(dev, 0, func_cap);
ab9c17a0 982 if (err) {
1a91de28
JP		 983 mlx4_err(dev, "QUERY_FUNC_CAP general command failed, aborting (%d)\n",
984 err);
c73c8b1e 985 goto free_mem;
ab9c17a0
JM		 986 }
987
c73c8b1e 988 if ((func_cap->pf_context_behaviour | PF_CONTEXT_BEHAVIOUR_MASK) !=
ab9c17a0 989 PF_CONTEXT_BEHAVIOUR_MASK) {
7d077cd3 990 mlx4_err(dev, "Unknown pf context behaviour %x known flags %x\n",
c73c8b1e
EBE		 991 func_cap->pf_context_behaviour,
992 PF_CONTEXT_BEHAVIOUR_MASK);
993 err = -EINVAL;
994 goto free_mem;
995 }
996
997 dev->caps.num_ports = func_cap->num_ports;
998 dev->quotas.qp = func_cap->qp_quota;
999 dev->quotas.srq = func_cap->srq_quota;
1000 dev->quotas.cq = func_cap->cq_quota;
1001 dev->quotas.mpt = func_cap->mpt_quota;
1002 dev->quotas.mtt = func_cap->mtt_quota;
1003 dev->caps.num_qps = 1 << hca_param->log_num_qps;
1004 dev->caps.num_srqs = 1 << hca_param->log_num_srqs;
1005 dev->caps.num_cqs = 1 << hca_param->log_num_cqs;
1006 dev->caps.num_mpts = 1 << hca_param->log_mpt_sz;
1007 dev->caps.num_eqs = func_cap->max_eq;
1008 dev->caps.reserved_eqs = func_cap->reserved_eq;
1009 dev->caps.reserved_lkey = func_cap->reserved_lkey;
ab9c17a0
JM		 1010 dev->caps.num_pds = MLX4_NUM_PDS;
1011 dev->caps.num_mgms = 0;
1012 dev->caps.num_amgms = 0;
1013
ab9c17a0 1014 if (dev->caps.num_ports > MLX4_MAX_PORTS) {
1a91de28
JP		 1015 mlx4_err(dev, "HCA has %d ports, but we only support %d, aborting\n",
1016 dev->caps.num_ports, MLX4_MAX_PORTS);
542deb88
CIK		 1017 err = -ENODEV;
1018 goto free_mem;
ab9c17a0
JM		 1019 }
1020
2b3ddf27
JM		 1021 mlx4_replace_zero_macs(dev);
1022
c73c8b1e
EBE		 1023 err = mlx4_slave_special_qp_cap(dev);
1024 if (err) {
1025 mlx4_err(dev, "Set special QP caps failed. aborting\n");
1026 goto free_mem;
6634961c 1027 }
6230bb23 1028
ab9c17a0
JM		 1029 if (dev->caps.uar_page_size * (dev->caps.num_uars -
1030 dev->caps.reserved_uars) >
872bf2fb
YH		 1031 pci_resource_len(dev->persist->pdev,
1032 2)) {
1a91de28 1033 mlx4_err(dev, "HCA reported UAR region size of 0x%x bigger than PCI resource 2 size of 0x%llx, aborting\n",
ab9c17a0 1034 dev->caps.uar_page_size * dev->caps.num_uars,
872bf2fb
YH		 1035 (unsigned long long)
1036 pci_resource_len(dev->persist->pdev, 2));
d49c2197 1037 err = -ENOMEM;
47605df9 1038 goto err_mem;
ab9c17a0
JM		 1039 }
1040
c73c8b1e 1041 if (hca_param->dev_cap_enabled & MLX4_DEV_CAP_64B_EQE_ENABLED) {
08ff3235
OG		 1042 dev->caps.eqe_size = 64;
1043 dev->caps.eqe_factor = 1;
1044 } else {
1045 dev->caps.eqe_size = 32;
1046 dev->caps.eqe_factor = 0;
1047 }
1048
c73c8b1e 1049 if (hca_param->dev_cap_enabled & MLX4_DEV_CAP_64B_CQE_ENABLED) {
08ff3235 1050 dev->caps.cqe_size = 64;
77507aa2 1051 dev->caps.userspace_caps |= MLX4_USER_DEV_CAP_LARGE_CQE;
08ff3235
OG		 1052 } else {
1053 dev->caps.cqe_size = 32;
1054 }
1055
c73c8b1e
EBE		 1056 if (hca_param->dev_cap_enabled & MLX4_DEV_CAP_EQE_STRIDE_ENABLED) {
1057 dev->caps.eqe_size = hca_param->eqe_size;
77507aa2
IS		 1058 dev->caps.eqe_factor = 0;
1059 }
1060
c73c8b1e
EBE		 1061 if (hca_param->dev_cap_enabled & MLX4_DEV_CAP_CQE_STRIDE_ENABLED) {
1062 dev->caps.cqe_size = hca_param->cqe_size;
77507aa2
IS		 1063 /* User still need to know when CQE > 32B */
1064 dev->caps.userspace_caps |= MLX4_USER_DEV_CAP_LARGE_CQE;
1065 }
1066
f9bd2d7f 1067 dev->caps.flags2 &= ~MLX4_DEV_CAP_FLAG2_TS;
1a91de28 1068 mlx4_warn(dev, "Timestamping is not supported in slave mode\n");
f9bd2d7f 1069
be599603
MS		 1070 dev->caps.flags2 &= ~MLX4_DEV_CAP_FLAG2_USER_MAC_EN;
1071 mlx4_dbg(dev, "User MAC FW update is not supported in slave mode\n");
1072
c73c8b1e 1073 slave_adjust_steering_mode(dev, dev_cap, hca_param);
802f42a8 1074 mlx4_dbg(dev, "RSS support for IP fragments is %s\n",
c73c8b1e 1075 hca_param->rss_ip_frags ? "on" : "off");
7b8157be 1076
c73c8b1e 1077 if (func_cap->extra_flags & MLX4_QUERY_FUNC_FLAGS_BF_RES_QP &&
ddae0349
EE		 1078 dev->caps.bf_reg_size)
1079 dev->caps.alloc_res_qp_mask |= MLX4_RESERVE_ETH_BF_QP;
1080
c73c8b1e 1081 if (func_cap->extra_flags & MLX4_QUERY_FUNC_FLAGS_A0_RES_QP)
d57febe1
MB		 1082 dev->caps.alloc_res_qp_mask |= MLX4_RESERVE_A0_QP;
1083
47605df9 1084err_mem:
c73c8b1e
EBE		 1085 if (err)
1086 mlx4_slave_destroy_special_qp_cap(dev);
1087free_mem:
1088 kfree(hca_param);
1089 kfree(func_cap);
1090 kfree(dev_cap);
47605df9 1091 return err;
ab9c17a0 1092}
225c7b1f 1093
b046ffe5
EP		 1094static void mlx4_request_modules(struct mlx4_dev *dev)
1095{
1096 int port;
1097 int has_ib_port = false;
1098 int has_eth_port = false;
1099#define EN_DRV_NAME "mlx4_en"
1100#define IB_DRV_NAME "mlx4_ib"
1101
1102 for (port = 1; port <= dev->caps.num_ports; port++) {
1103 if (dev->caps.port_type[port] == MLX4_PORT_TYPE_IB)
1104 has_ib_port = true;
1105 else if (dev->caps.port_type[port] == MLX4_PORT_TYPE_ETH)
1106 has_eth_port = true;
1107 }
1108
b046ffe5
EP		 1109 if (has_eth_port)
1110 request_module_nowait(EN_DRV_NAME);
f24f790f
OG		 1111 if (has_ib_port || (dev->caps.flags & MLX4_DEV_CAP_FLAG_IBOE))
1112 request_module_nowait(IB_DRV_NAME);
b046ffe5
EP		 1113}
1114
7ff93f8b
YP		 1115/*
1116 * Change the port configuration of the device.
1117 * Every user of this function must hold the port mutex.
1118 */
27bf91d6
YP		 1119int mlx4_change_port_types(struct mlx4_dev *dev,
1120 enum mlx4_port_type *port_types)
7ff93f8b
YP		 1121{
1122 int err = 0;
1123 int change = 0;
1124 int port;
1125
1126 for (port = 0; port < dev->caps.num_ports; port++) {
27bf91d6
YP		 1127 /* Change the port type only if the new type is different
1128 * from the current, and not set to Auto */
3d8f9308 1129 if (port_types[port] != dev->caps.port_type[port + 1])
7ff93f8b 1130 change = 1;
7ff93f8b
YP		 1131 }
1132 if (change) {
1133 mlx4_unregister_device(dev);
1134 for (port = 1; port <= dev->caps.num_ports; port++) {
1135 mlx4_CLOSE_PORT(dev, port);
1e0f03d5 1136 dev->caps.port_type[port] = port_types[port - 1];
6634961c 1137 err = mlx4_SET_PORT(dev, port, -1);
7ff93f8b 1138 if (err) {
1a91de28
JP		 1139 mlx4_err(dev, "Failed to set port %d, aborting\n",
1140 port);
7ff93f8b
YP		 1141 goto out;
1142 }
1143 }
1144 mlx4_set_port_mask(dev);
1145 err = mlx4_register_device(dev);
b046ffe5
EP		 1146 if (err) {
1147 mlx4_err(dev, "Failed to register device\n");
1148 goto out;
1149 }
1150 mlx4_request_modules(dev);
7ff93f8b
YP		 1151 }
1152
1153out:
1154 return err;
1155}
1156
1157static ssize_t show_port_type(struct device *dev,
1158 struct device_attribute *attr,
1159 char *buf)
1160{
1161 struct mlx4_port_info *info = container_of(attr, struct mlx4_port_info,
1162 port_attr);
1163 struct mlx4_dev *mdev = info->dev;
27bf91d6
YP		 1164 char type[8];
1165
1166 sprintf(type, "%s",
1167 (mdev->caps.port_type[info->port] == MLX4_PORT_TYPE_IB) ?
1168 "ib" : "eth");
1169 if (mdev->caps.possible_type[info->port] == MLX4_PORT_TYPE_AUTO)
1170 sprintf(buf, "auto (%s)\n", type);
1171 else
1172 sprintf(buf, "%s\n", type);
7ff93f8b 1173
27bf91d6 1174 return strlen(buf);
7ff93f8b
YP		 1175}
1176
b2facd95
JP		 1177static int __set_port_type(struct mlx4_port_info *info,
1178 enum mlx4_port_type port_type)
7ff93f8b 1179{
7ff93f8b
YP		 1180 struct mlx4_dev *mdev = info->dev;
1181 struct mlx4_priv *priv = mlx4_priv(mdev);
1182 enum mlx4_port_type types[MLX4_MAX_PORTS];
27bf91d6 1183 enum mlx4_port_type new_types[MLX4_MAX_PORTS];
7ff93f8b
YP		 1184 int i;
1185 int err = 0;
1186
33a1f8b1
MG		 1187 if ((port_type & mdev->caps.supported_type[info->port]) != port_type) {
1188 mlx4_err(mdev,
1189 "Requested port type for port %d is not supported on this HCA\n",
1190 info->port);
95aac2cd 1191 return -EOPNOTSUPP;
33a1f8b1
MG		 1192 }
1193
27bf91d6 1194 mlx4_stop_sense(mdev);
7ff93f8b 1195 mutex_lock(&priv->port_mutex);
b2facd95
JP		 1196 info->tmp_type = port_type;
1197
27bf91d6
YP		 1198 /* Possible type is always the one that was delivered */
1199 mdev->caps.possible_type[info->port] = info->tmp_type;
1200
1201 for (i = 0; i < mdev->caps.num_ports; i++) {
7ff93f8b 1202 types[i] = priv->port[i+1].tmp_type ? priv->port[i+1].tmp_type :
27bf91d6
YP		 1203 mdev->caps.possible_type[i+1];
1204 if (types[i] == MLX4_PORT_TYPE_AUTO)
1205 types[i] = mdev->caps.port_type[i+1];
1206 }
7ff93f8b 1207
58a60168
YP		 1208 if (!(mdev->caps.flags & MLX4_DEV_CAP_FLAG_DPDP) &&
1209 !(mdev->caps.flags & MLX4_DEV_CAP_FLAG_SENSE_SUPPORT)) {
27bf91d6
YP		 1210 for (i = 1; i <= mdev->caps.num_ports; i++) {
1211 if (mdev->caps.possible_type[i] == MLX4_PORT_TYPE_AUTO) {
1212 mdev->caps.possible_type[i] = mdev->caps.port_type[i];
95aac2cd 1213 err = -EOPNOTSUPP;
27bf91d6
YP		 1214 }
1215 }
1216 }
1217 if (err) {
1a91de28 1218 mlx4_err(mdev, "Auto sensing is not supported on this HCA. Set only 'eth' or 'ib' for both ports (should be the same)\n");
27bf91d6
YP		 1219 goto out;
1220 }
1221
1222 mlx4_do_sense_ports(mdev, new_types, types);
1223
1224 err = mlx4_check_port_params(mdev, new_types);
7ff93f8b
YP		 1225 if (err)
1226 goto out;
1227
27bf91d6
YP		 1228 /* We are about to apply the changes after the configuration
1229 * was verified, no need to remember the temporary types
1230 * any more */
1231 for (i = 0; i < mdev->caps.num_ports; i++)
1232 priv->port[i + 1].tmp_type = 0;
7ff93f8b 1233
27bf91d6 1234 err = mlx4_change_port_types(mdev, new_types);
7ff93f8b
YP		 1235
1236out:
27bf91d6 1237 mlx4_start_sense(mdev);
7ff93f8b 1238 mutex_unlock(&priv->port_mutex);
95aac2cd 1239
b2facd95
JP		 1240 return err;
1241}
1242
1243static ssize_t set_port_type(struct device *dev,
1244 struct device_attribute *attr,
1245 const char *buf, size_t count)
1246{
1247 struct mlx4_port_info *info = container_of(attr, struct mlx4_port_info,
1248 port_attr);
1249 struct mlx4_dev *mdev = info->dev;
1250 enum mlx4_port_type port_type;
1251 static DEFINE_MUTEX(set_port_type_mutex);
1252 int err;
1253
1254 mutex_lock(&set_port_type_mutex);
1255
1256 if (!strcmp(buf, "ib\n")) {
1257 port_type = MLX4_PORT_TYPE_IB;
1258 } else if (!strcmp(buf, "eth\n")) {
1259 port_type = MLX4_PORT_TYPE_ETH;
1260 } else if (!strcmp(buf, "auto\n")) {
1261 port_type = MLX4_PORT_TYPE_AUTO;
1262 } else {
1263 mlx4_err(mdev, "%s is not supported port type\n", buf);
1264 err = -EINVAL;
1265 goto err_out;
1266 }
1267
1268 err = __set_port_type(info, port_type);
1269
0a984556
AV		 1270err_out:
1271 mutex_unlock(&set_port_type_mutex);
1272
7ff93f8b
YP		 1273 return err ? err : count;
1274}
1275
096335b3
OG		 1276enum ibta_mtu {
1277 IB_MTU_256 = 1,
1278 IB_MTU_512 = 2,
1279 IB_MTU_1024 = 3,
1280 IB_MTU_2048 = 4,
1281 IB_MTU_4096 = 5
1282};
1283
1284static inline int int_to_ibta_mtu(int mtu)
1285{
1286 switch (mtu) {
1287 case 256: return IB_MTU_256;
1288 case 512: return IB_MTU_512;
1289 case 1024: return IB_MTU_1024;
1290 case 2048: return IB_MTU_2048;
1291 case 4096: return IB_MTU_4096;
1292 default: return -1;
1293 }
1294}
1295
1296static inline int ibta_mtu_to_int(enum ibta_mtu mtu)
1297{
1298 switch (mtu) {
1299 case IB_MTU_256: return 256;
1300 case IB_MTU_512: return 512;
1301 case IB_MTU_1024: return 1024;
1302 case IB_MTU_2048: return 2048;
1303 case IB_MTU_4096: return 4096;
1304 default: return -1;
1305 }
1306}
1307
1308static ssize_t show_port_ib_mtu(struct device *dev,
1309 struct device_attribute *attr,
1310 char *buf)
1311{
1312 struct mlx4_port_info *info = container_of(attr, struct mlx4_port_info,
1313 port_mtu_attr);
1314 struct mlx4_dev *mdev = info->dev;
1315
1316 if (mdev->caps.port_type[info->port] == MLX4_PORT_TYPE_ETH)
1317 mlx4_warn(mdev, "port level mtu is only used for IB ports\n");
1318
1319 sprintf(buf, "%d\n",
1320 ibta_mtu_to_int(mdev->caps.port_ib_mtu[info->port]));
1321 return strlen(buf);
1322}
1323
1324static ssize_t set_port_ib_mtu(struct device *dev,
1325 struct device_attribute *attr,
1326 const char *buf, size_t count)
1327{
1328 struct mlx4_port_info *info = container_of(attr, struct mlx4_port_info,
1329 port_mtu_attr);
1330 struct mlx4_dev *mdev = info->dev;
1331 struct mlx4_priv *priv = mlx4_priv(mdev);
1332 int err, port, mtu, ibta_mtu = -1;
1333
1334 if (mdev->caps.port_type[info->port] == MLX4_PORT_TYPE_ETH) {
1335 mlx4_warn(mdev, "port level mtu is only used for IB ports\n");
1336 return -EINVAL;
1337 }
1338
618fad95
DB		 1339 err = kstrtoint(buf, 0, &mtu);
1340 if (!err)
096335b3
OG		 1341 ibta_mtu = int_to_ibta_mtu(mtu);
1342
618fad95 1343 if (err || ibta_mtu < 0) {
096335b3
OG		 1344 mlx4_err(mdev, "%s is invalid IBTA mtu\n", buf);
1345 return -EINVAL;
1346 }
1347
1348 mdev->caps.port_ib_mtu[info->port] = ibta_mtu;
1349
1350 mlx4_stop_sense(mdev);
1351 mutex_lock(&priv->port_mutex);
1352 mlx4_unregister_device(mdev);
1353 for (port = 1; port <= mdev->caps.num_ports; port++) {
1354 mlx4_CLOSE_PORT(mdev, port);
6634961c 1355 err = mlx4_SET_PORT(mdev, port, -1);
096335b3 1356 if (err) {
1a91de28
JP		 1357 mlx4_err(mdev, "Failed to set port %d, aborting\n",
1358 port);
096335b3
OG		 1359 goto err_set_port;
1360 }
1361 }
1362 err = mlx4_register_device(mdev);
1363err_set_port:
1364 mutex_unlock(&priv->port_mutex);
1365 mlx4_start_sense(mdev);
1366 return err ? err : count;
1367}
1368
e57968a1
MS		 1369/* bond for multi-function device */
1370#define MAX_MF_BOND_ALLOWED_SLAVES 63
1371static int mlx4_mf_bond(struct mlx4_dev *dev)
1372{
1373 int err = 0;
00ada910 1374 int nvfs;
e57968a1
MS		 1375 struct mlx4_slaves_pport slaves_port1;
1376 struct mlx4_slaves_pport slaves_port2;
1377 DECLARE_BITMAP(slaves_port_1_2, MLX4_MFUNC_MAX);
1378
1379 slaves_port1 = mlx4_phys_to_slaves_pport(dev, 1);
1380 slaves_port2 = mlx4_phys_to_slaves_pport(dev, 2);
1381 bitmap_and(slaves_port_1_2,
1382 slaves_port1.slaves, slaves_port2.slaves,
1383 dev->persist->num_vfs + 1);
1384
1385 /* only single port vfs are allowed */
1386 if (bitmap_weight(slaves_port_1_2, dev->persist->num_vfs + 1) > 1) {
1387 mlx4_warn(dev, "HA mode unsupported for dual ported VFs\n");
1388 return -EINVAL;
1389 }
1390
00ada910
MS		 1391 /* number of virtual functions is number of total functions minus one
1392 * physical function for each port.
1393 */
1394 nvfs = bitmap_weight(slaves_port1.slaves, dev->persist->num_vfs + 1) +
1395 bitmap_weight(slaves_port2.slaves, dev->persist->num_vfs + 1) - 2;
1396
e57968a1 1397 /* limit on maximum allowed VFs */
00ada910
MS		 1398 if (nvfs > MAX_MF_BOND_ALLOWED_SLAVES) {
1399 mlx4_warn(dev, "HA mode is not supported for %d VFs (max %d are allowed)\n",
1400 nvfs, MAX_MF_BOND_ALLOWED_SLAVES);
e57968a1 1401 return -EINVAL;
00ada910 1402 }
e57968a1
MS		 1403
1404 if (dev->caps.steering_mode != MLX4_STEERING_MODE_DEVICE_MANAGED) {
1405 mlx4_warn(dev, "HA mode unsupported for NON DMFS steering\n");
1406 return -EINVAL;
1407 }
1408
1409 err = mlx4_bond_mac_table(dev);
1410 if (err)
1411 return err;
1412 err = mlx4_bond_vlan_table(dev);
1413 if (err)
1414 goto err1;
1415 err = mlx4_bond_fs_rules(dev);
1416 if (err)
1417 goto err2;
1418
1419 return 0;
1420err2:
1421 (void)mlx4_unbond_vlan_table(dev);
1422err1:
1423 (void)mlx4_unbond_mac_table(dev);
1424 return err;
1425}
1426
1427static int mlx4_mf_unbond(struct mlx4_dev *dev)
1428{
1429 int ret, ret1;
1430
1431 ret = mlx4_unbond_fs_rules(dev);
1432 if (ret)
26ff7585 1433 mlx4_warn(dev, "multifunction unbond for flow rules failed (%d)\n", ret);
e57968a1
MS		 1434 ret1 = mlx4_unbond_mac_table(dev);
1435 if (ret1) {
1436 mlx4_warn(dev, "multifunction unbond for MAC table failed (%d)\n", ret1);
1437 ret = ret1;
1438 }
1439 ret1 = mlx4_unbond_vlan_table(dev);
1440 if (ret1) {
1441 mlx4_warn(dev, "multifunction unbond for VLAN table failed (%d)\n", ret1);
1442 ret = ret1;
1443 }
1444 return ret;
1445}
1446
53f33ae2
MS		 1447int mlx4_bond(struct mlx4_dev *dev)
1448{
1449 int ret = 0;
1450 struct mlx4_priv *priv = mlx4_priv(dev);
1451
1452 mutex_lock(&priv->bond_mutex);
1453
e57968a1 1454 if (!mlx4_is_bonded(dev)) {
53f33ae2 1455 ret = mlx4_do_bond(dev, true);
e57968a1
MS		 1456 if (ret)
1457 mlx4_err(dev, "Failed to bond device: %d\n", ret);
1458 if (!ret && mlx4_is_master(dev)) {
1459 ret = mlx4_mf_bond(dev);
1460 if (ret) {
1461 mlx4_err(dev, "bond for multifunction failed\n");
1462 mlx4_do_bond(dev, false);
1463 }
1464 }
1465 }
53f33ae2
MS		 1466
1467 mutex_unlock(&priv->bond_mutex);
e57968a1 1468 if (!ret)
53f33ae2 1469 mlx4_dbg(dev, "Device is bonded\n");
e57968a1 1470
53f33ae2
MS		 1471 return ret;
1472}
1473EXPORT_SYMBOL_GPL(mlx4_bond);
1474
1475int mlx4_unbond(struct mlx4_dev *dev)
1476{
1477 int ret = 0;
1478 struct mlx4_priv *priv = mlx4_priv(dev);
1479
1480 mutex_lock(&priv->bond_mutex);
1481
e57968a1
MS		 1482 if (mlx4_is_bonded(dev)) {
1483 int ret2 = 0;
1484
53f33ae2 1485 ret = mlx4_do_bond(dev, false);
e57968a1
MS		 1486 if (ret)
1487 mlx4_err(dev, "Failed to unbond device: %d\n", ret);
1488 if (mlx4_is_master(dev))
1489 ret2 = mlx4_mf_unbond(dev);
1490 if (ret2) {
1491 mlx4_warn(dev, "Failed to unbond device for multifunction (%d)\n", ret2);
1492 ret = ret2;
1493 }
1494 }
53f33ae2
MS		 1495
1496 mutex_unlock(&priv->bond_mutex);
e57968a1 1497 if (!ret)
53f33ae2 1498 mlx4_dbg(dev, "Device is unbonded\n");
e57968a1 1499
53f33ae2
MS		 1500 return ret;
1501}
1502EXPORT_SYMBOL_GPL(mlx4_unbond);
1503
1504
1505int mlx4_port_map_set(struct mlx4_dev *dev, struct mlx4_port_map *v2p)
1506{
1507 u8 port1 = v2p->port1;
1508 u8 port2 = v2p->port2;
1509 struct mlx4_priv *priv = mlx4_priv(dev);
1510 int err;
1511
1512 if (!(dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_PORT_REMAP))
423b3aec 1513 return -EOPNOTSUPP;
53f33ae2
MS		 1514
1515 mutex_lock(&priv->bond_mutex);
1516
1517 /* zero means keep current mapping for this port */
1518 if (port1 == 0)
1519 port1 = priv->v2p.port1;
1520 if (port2 == 0)
1521 port2 = priv->v2p.port2;
1522
1523 if ((port1 < 1) || (port1 > MLX4_MAX_PORTS) ||
1524 (port2 < 1) || (port2 > MLX4_MAX_PORTS) ||
1525 (port1 == 2 && port2 == 1)) {
1526 /* besides boundary checks cross mapping makes
1527 * no sense and therefore not allowed */
1528 err = -EINVAL;
1529 } else if ((port1 == priv->v2p.port1) &&
1530 (port2 == priv->v2p.port2)) {
1531 err = 0;
1532 } else {
1533 err = mlx4_virt2phy_port_map(dev, port1, port2);
1534 if (!err) {
1535 mlx4_dbg(dev, "port map changed: [%d][%d]\n",
1536 port1, port2);
1537 priv->v2p.port1 = port1;
1538 priv->v2p.port2 = port2;
1539 } else {
1540 mlx4_err(dev, "Failed to change port mape: %d\n", err);
1541 }
1542 }
1543
1544 mutex_unlock(&priv->bond_mutex);
1545 return err;
1546}
1547EXPORT_SYMBOL_GPL(mlx4_port_map_set);
1548
e8f9b2ed 1549static int mlx4_load_fw(struct mlx4_dev *dev)
225c7b1f
RD		 1550{
1551 struct mlx4_priv *priv = mlx4_priv(dev);
1552 int err;
1553
1554 priv->fw.fw_icm = mlx4_alloc_icm(dev, priv->fw.fw_pages,
5b0bf5e2 1555 GFP_HIGHUSER | __GFP_NOWARN, 0);
225c7b1f 1556 if (!priv->fw.fw_icm) {
1a91de28 1557 mlx4_err(dev, "Couldn't allocate FW area, aborting\n");
225c7b1f
RD		 1558 return -ENOMEM;
1559 }
1560
1561 err = mlx4_MAP_FA(dev, priv->fw.fw_icm);
1562 if (err) {
1a91de28 1563 mlx4_err(dev, "MAP_FA command failed, aborting\n");
225c7b1f
RD		 1564 goto err_free;
1565 }
1566
1567 err = mlx4_RUN_FW(dev);
1568 if (err) {
1a91de28 1569 mlx4_err(dev, "RUN_FW command failed, aborting\n");
225c7b1f
RD		 1570 goto err_unmap_fa;
1571 }
1572
1573 return 0;
1574
1575err_unmap_fa:
1576 mlx4_UNMAP_FA(dev);
1577
1578err_free:
5b0bf5e2 1579 mlx4_free_icm(dev, priv->fw.fw_icm, 0);
225c7b1f
RD		 1580 return err;
1581}
1582
e8f9b2ed
RD		 1583static int mlx4_init_cmpt_table(struct mlx4_dev *dev, u64 cmpt_base,
1584 int cmpt_entry_sz)
225c7b1f
RD		 1585{
1586 struct mlx4_priv *priv = mlx4_priv(dev);
1587 int err;
ab9c17a0 1588 int num_eqs;
225c7b1f
RD		 1589
1590 err = mlx4_init_icm_table(dev, &priv->qp_table.cmpt_table,
1591 cmpt_base +
1592 ((u64) (MLX4_CMPT_TYPE_QP *
1593 cmpt_entry_sz) << MLX4_CMPT_SHIFT),
1594 cmpt_entry_sz, dev->caps.num_qps,
93fc9e1b
YP		 1595 dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FW],
1596 0, 0);
225c7b1f
RD		 1597 if (err)
1598 goto err;
1599
1600 err = mlx4_init_icm_table(dev, &priv->srq_table.cmpt_table,
1601 cmpt_base +
1602 ((u64) (MLX4_CMPT_TYPE_SRQ *
1603 cmpt_entry_sz) << MLX4_CMPT_SHIFT),
1604 cmpt_entry_sz, dev->caps.num_srqs,
5b0bf5e2 1605 dev->caps.reserved_srqs, 0, 0);
225c7b1f
RD		 1606 if (err)
1607 goto err_qp;
1608
1609 err = mlx4_init_icm_table(dev, &priv->cq_table.cmpt_table,
1610 cmpt_base +
1611 ((u64) (MLX4_CMPT_TYPE_CQ *
1612 cmpt_entry_sz) << MLX4_CMPT_SHIFT),
1613 cmpt_entry_sz, dev->caps.num_cqs,
5b0bf5e2 1614 dev->caps.reserved_cqs, 0, 0);
225c7b1f
RD		 1615 if (err)
1616 goto err_srq;
1617
7ae0e400 1618 num_eqs = dev->phys_caps.num_phys_eqs;
225c7b1f
RD		 1619 err = mlx4_init_icm_table(dev, &priv->eq_table.cmpt_table,
1620 cmpt_base +
1621 ((u64) (MLX4_CMPT_TYPE_EQ *
1622 cmpt_entry_sz) << MLX4_CMPT_SHIFT),
ab9c17a0 1623 cmpt_entry_sz, num_eqs, num_eqs, 0, 0);
225c7b1f
RD		 1624 if (err)
1625 goto err_cq;
1626
1627 return 0;
1628
1629err_cq:
1630 mlx4_cleanup_icm_table(dev, &priv->cq_table.cmpt_table);
1631
1632err_srq:
1633 mlx4_cleanup_icm_table(dev, &priv->srq_table.cmpt_table);
1634
1635err_qp:
1636 mlx4_cleanup_icm_table(dev, &priv->qp_table.cmpt_table);
1637
1638err:
1639 return err;
1640}
1641
3d73c288
RD		 1642static int mlx4_init_icm(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap,
1643 struct mlx4_init_hca_param *init_hca, u64 icm_size)
225c7b1f
RD		 1644{
1645 struct mlx4_priv *priv = mlx4_priv(dev);
1646 u64 aux_pages;
ab9c17a0 1647 int num_eqs;
225c7b1f
RD		 1648 int err;
1649
1650 err = mlx4_SET_ICM_SIZE(dev, icm_size, &aux_pages);
1651 if (err) {
1a91de28 1652 mlx4_err(dev, "SET_ICM_SIZE command failed, aborting\n");
225c7b1f
RD		 1653 return err;
1654 }
1655
1a91de28 1656 mlx4_dbg(dev, "%lld KB of HCA context requires %lld KB aux memory\n",
225c7b1f
RD		 1657 (unsigned long long) icm_size >> 10,
1658 (unsigned long long) aux_pages << 2);
1659
1660 priv->fw.aux_icm = mlx4_alloc_icm(dev, aux_pages,
5b0bf5e2 1661 GFP_HIGHUSER | __GFP_NOWARN, 0);
225c7b1f 1662 if (!priv->fw.aux_icm) {
1a91de28 1663 mlx4_err(dev, "Couldn't allocate aux memory, aborting\n");
225c7b1f
RD		 1664 return -ENOMEM;
1665 }
1666
1667 err = mlx4_MAP_ICM_AUX(dev, priv->fw.aux_icm);
1668 if (err) {
1a91de28 1669 mlx4_err(dev, "MAP_ICM_AUX command failed, aborting\n");
225c7b1f
RD		 1670 goto err_free_aux;
1671 }
1672
1673 err = mlx4_init_cmpt_table(dev, init_hca->cmpt_base, dev_cap->cmpt_entry_sz);
1674 if (err) {
1a91de28 1675 mlx4_err(dev, "Failed to map cMPT context memory, aborting\n");
225c7b1f
RD		 1676 goto err_unmap_aux;
1677 }
1678
ab9c17a0 1679
7ae0e400 1680 num_eqs = dev->phys_caps.num_phys_eqs;
fa0681d2
RD		 1681 err = mlx4_init_icm_table(dev, &priv->eq_table.table,
1682 init_hca->eqc_base, dev_cap->eqc_entry_sz,
ab9c17a0 1683 num_eqs, num_eqs, 0, 0);
225c7b1f 1684 if (err) {
1a91de28 1685 mlx4_err(dev, "Failed to map EQ context memory, aborting\n");
225c7b1f
RD		 1686 goto err_unmap_cmpt;
1687 }
1688
d7bb58fb
JM		 1689 /*
1690 * Reserved MTT entries must be aligned up to a cacheline
1691 * boundary, since the FW will write to them, while the driver
1692 * writes to all other MTT entries. (The variable
1693 * dev->caps.mtt_entry_sz below is really the MTT segment
1694 * size, not the raw entry size)
1695 */
1696 dev->caps.reserved_mtts =
1697 ALIGN(dev->caps.reserved_mtts * dev->caps.mtt_entry_sz,
1698 dma_get_cache_alignment()) / dev->caps.mtt_entry_sz;
1699
225c7b1f
RD		 1700 err = mlx4_init_icm_table(dev, &priv->mr_table.mtt_table,
1701 init_hca->mtt_base,
1702 dev->caps.mtt_entry_sz,
2b8fb286 1703 dev->caps.num_mtts,
5b0bf5e2 1704 dev->caps.reserved_mtts, 1, 0);
225c7b1f 1705 if (err) {
1a91de28 1706 mlx4_err(dev, "Failed to map MTT context memory, aborting\n");
225c7b1f
RD		 1707 goto err_unmap_eq;
1708 }
1709
1710 err = mlx4_init_icm_table(dev, &priv->mr_table.dmpt_table,
1711 init_hca->dmpt_base,
1712 dev_cap->dmpt_entry_sz,
1713 dev->caps.num_mpts,
5b0bf5e2 1714 dev->caps.reserved_mrws, 1, 1);
225c7b1f 1715 if (err) {
1a91de28 1716 mlx4_err(dev, "Failed to map dMPT context memory, aborting\n");
225c7b1f
RD		 1717 goto err_unmap_mtt;
1718 }
1719
1720 err = mlx4_init_icm_table(dev, &priv->qp_table.qp_table,
1721 init_hca->qpc_base,
1722 dev_cap->qpc_entry_sz,
1723 dev->caps.num_qps,
93fc9e1b
YP		 1724 dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FW],
1725 0, 0);
225c7b1f 1726 if (err) {
1a91de28 1727 mlx4_err(dev, "Failed to map QP context memory, aborting\n");
225c7b1f
RD		 1728 goto err_unmap_dmpt;
1729 }
1730
1731 err = mlx4_init_icm_table(dev, &priv->qp_table.auxc_table,
1732 init_hca->auxc_base,
1733 dev_cap->aux_entry_sz,
1734 dev->caps.num_qps,
93fc9e1b
YP		 1735 dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FW],
1736 0, 0);
225c7b1f 1737 if (err) {
1a91de28 1738 mlx4_err(dev, "Failed to map AUXC context memory, aborting\n");
225c7b1f
RD		 1739 goto err_unmap_qp;
1740 }
1741
1742 err = mlx4_init_icm_table(dev, &priv->qp_table.altc_table,
1743 init_hca->altc_base,
1744 dev_cap->altc_entry_sz,
1745 dev->caps.num_qps,
93fc9e1b
YP		 1746 dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FW],
1747 0, 0);
225c7b1f 1748 if (err) {
1a91de28 1749 mlx4_err(dev, "Failed to map ALTC context memory, aborting\n");
225c7b1f
RD		 1750 goto err_unmap_auxc;
1751 }
1752
1753 err = mlx4_init_icm_table(dev, &priv->qp_table.rdmarc_table,
1754 init_hca->rdmarc_base,
1755 dev_cap->rdmarc_entry_sz << priv->qp_table.rdmarc_shift,
1756 dev->caps.num_qps,
93fc9e1b
YP		 1757 dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FW],
1758 0, 0);
225c7b1f
RD		 1759 if (err) {
1760 mlx4_err(dev, "Failed to map RDMARC context memory, aborting\n");
1761 goto err_unmap_altc;
1762 }
1763
1764 err = mlx4_init_icm_table(dev, &priv->cq_table.table,
1765 init_hca->cqc_base,
1766 dev_cap->cqc_entry_sz,
1767 dev->caps.num_cqs,
5b0bf5e2 1768 dev->caps.reserved_cqs, 0, 0);
225c7b1f 1769 if (err) {
1a91de28 1770 mlx4_err(dev, "Failed to map CQ context memory, aborting\n");
225c7b1f
RD		 1771 goto err_unmap_rdmarc;
1772 }
1773
1774 err = mlx4_init_icm_table(dev, &priv->srq_table.table,
1775 init_hca->srqc_base,
1776 dev_cap->srq_entry_sz,
1777 dev->caps.num_srqs,
5b0bf5e2 1778 dev->caps.reserved_srqs, 0, 0);
225c7b1f 1779 if (err) {
1a91de28 1780 mlx4_err(dev, "Failed to map SRQ context memory, aborting\n");
225c7b1f
RD		 1781 goto err_unmap_cq;
1782 }
1783
1784 /*
0ff1fb65
HHZ		 1785 * For flow steering device managed mode it is required to use
1786 * mlx4_init_icm_table. For B0 steering mode it's not strictly
1787 * required, but for simplicity just map the whole multicast
1788 * group table now. The table isn't very big and it's a lot
1789 * easier than trying to track ref counts.
225c7b1f
RD		 1790 */
1791 err = mlx4_init_icm_table(dev, &priv->mcg_table.table,
0ec2c0f8
EE		 1792 init_hca->mc_base,
1793 mlx4_get_mgm_entry_size(dev),
225c7b1f
RD		 1794 dev->caps.num_mgms + dev->caps.num_amgms,
1795 dev->caps.num_mgms + dev->caps.num_amgms,
5b0bf5e2 1796 0, 0);
225c7b1f 1797 if (err) {
1a91de28 1798 mlx4_err(dev, "Failed to map MCG context memory, aborting\n");
225c7b1f
RD		 1799 goto err_unmap_srq;
1800 }
1801
1802 return 0;
1803
1804err_unmap_srq:
1805 mlx4_cleanup_icm_table(dev, &priv->srq_table.table);
1806
1807err_unmap_cq:
1808 mlx4_cleanup_icm_table(dev, &priv->cq_table.table);
1809
1810err_unmap_rdmarc:
1811 mlx4_cleanup_icm_table(dev, &priv->qp_table.rdmarc_table);
1812
1813err_unmap_altc:
1814 mlx4_cleanup_icm_table(dev, &priv->qp_table.altc_table);
1815
1816err_unmap_auxc:
1817 mlx4_cleanup_icm_table(dev, &priv->qp_table.auxc_table);
1818
1819err_unmap_qp:
1820 mlx4_cleanup_icm_table(dev, &priv->qp_table.qp_table);
1821
1822err_unmap_dmpt:
1823 mlx4_cleanup_icm_table(dev, &priv->mr_table.dmpt_table);
1824
1825err_unmap_mtt:
1826 mlx4_cleanup_icm_table(dev, &priv->mr_table.mtt_table);
1827
1828err_unmap_eq:
fa0681d2 1829 mlx4_cleanup_icm_table(dev, &priv->eq_table.table);
225c7b1f
RD		 1830
1831err_unmap_cmpt:
1832 mlx4_cleanup_icm_table(dev, &priv->eq_table.cmpt_table);
1833 mlx4_cleanup_icm_table(dev, &priv->cq_table.cmpt_table);
1834 mlx4_cleanup_icm_table(dev, &priv->srq_table.cmpt_table);
1835 mlx4_cleanup_icm_table(dev, &priv->qp_table.cmpt_table);
1836
1837err_unmap_aux:
1838 mlx4_UNMAP_ICM_AUX(dev);
1839
1840err_free_aux:
5b0bf5e2 1841 mlx4_free_icm(dev, priv->fw.aux_icm, 0);
225c7b1f
RD		 1842
1843 return err;
1844}
1845
1846static void mlx4_free_icms(struct mlx4_dev *dev)
1847{
1848 struct mlx4_priv *priv = mlx4_priv(dev);
1849
1850 mlx4_cleanup_icm_table(dev, &priv->mcg_table.table);
1851 mlx4_cleanup_icm_table(dev, &priv->srq_table.table);
1852 mlx4_cleanup_icm_table(dev, &priv->cq_table.table);
1853 mlx4_cleanup_icm_table(dev, &priv->qp_table.rdmarc_table);
1854 mlx4_cleanup_icm_table(dev, &priv->qp_table.altc_table);
1855 mlx4_cleanup_icm_table(dev, &priv->qp_table.auxc_table);
1856 mlx4_cleanup_icm_table(dev, &priv->qp_table.qp_table);
1857 mlx4_cleanup_icm_table(dev, &priv->mr_table.dmpt_table);
1858 mlx4_cleanup_icm_table(dev, &priv->mr_table.mtt_table);
fa0681d2 1859 mlx4_cleanup_icm_table(dev, &priv->eq_table.table);
225c7b1f
RD		 1860 mlx4_cleanup_icm_table(dev, &priv->eq_table.cmpt_table);
1861 mlx4_cleanup_icm_table(dev, &priv->cq_table.cmpt_table);
1862 mlx4_cleanup_icm_table(dev, &priv->srq_table.cmpt_table);
1863 mlx4_cleanup_icm_table(dev, &priv->qp_table.cmpt_table);
225c7b1f
RD		 1864
1865 mlx4_UNMAP_ICM_AUX(dev);
5b0bf5e2 1866 mlx4_free_icm(dev, priv->fw.aux_icm, 0);
225c7b1f
RD		 1867}
1868
ab9c17a0
JM		 1869static void mlx4_slave_exit(struct mlx4_dev *dev)
1870{
1871 struct mlx4_priv *priv = mlx4_priv(dev);
1872
f3d4c89e 1873 mutex_lock(&priv->cmd.slave_cmd_mutex);
0cd93027
YH		 1874 if (mlx4_comm_cmd(dev, MLX4_COMM_CMD_RESET, 0, MLX4_COMM_CMD_NA_OP,
1875 MLX4_COMM_TIME))
1a91de28 1876 mlx4_warn(dev, "Failed to close slave function\n");
f3d4c89e 1877 mutex_unlock(&priv->cmd.slave_cmd_mutex);
ab9c17a0
JM		 1878}
1879
c1b43dca
EC		 1880static int map_bf_area(struct mlx4_dev *dev)
1881{
1882 struct mlx4_priv *priv = mlx4_priv(dev);
1883 resource_size_t bf_start;
1884 resource_size_t bf_len;
1885 int err = 0;
1886
3d747473
JM		 1887 if (!dev->caps.bf_reg_size)
1888 return -ENXIO;
1889
872bf2fb 1890 bf_start = pci_resource_start(dev->persist->pdev, 2) +
ab9c17a0 1891 (dev->caps.num_uars << PAGE_SHIFT);
872bf2fb 1892 bf_len = pci_resource_len(dev->persist->pdev, 2) -
ab9c17a0 1893 (dev->caps.num_uars << PAGE_SHIFT);
c1b43dca
EC		 1894 priv->bf_mapping = io_mapping_create_wc(bf_start, bf_len);
1895 if (!priv->bf_mapping)
1896 err = -ENOMEM;
1897
1898 return err;
1899}
1900
1901static void unmap_bf_area(struct mlx4_dev *dev)
1902{
1903 if (mlx4_priv(dev)->bf_mapping)
1904 io_mapping_free(mlx4_priv(dev)->bf_mapping);
1905}
1906
a5a1d1c2 1907u64 mlx4_read_clock(struct mlx4_dev *dev)
ec693d47
AV		 1908{
1909 u32 clockhi, clocklo, clockhi1;
a5a1d1c2 1910 u64 cycles;
ec693d47
AV		 1911 int i;
1912 struct mlx4_priv *priv = mlx4_priv(dev);
1913
1914 for (i = 0; i < 10; i++) {
1915 clockhi = swab32(readl(priv->clock_mapping));
1916 clocklo = swab32(readl(priv->clock_mapping + 4));
1917 clockhi1 = swab32(readl(priv->clock_mapping));
1918 if (clockhi == clockhi1)
1919 break;
1920 }
1921
1922 cycles = (u64) clockhi << 32 | (u64) clocklo;
1923
1924 return cycles;
1925}
1926EXPORT_SYMBOL_GPL(mlx4_read_clock);
1927
1928
ddd8a6c1
EE		 1929static int map_internal_clock(struct mlx4_dev *dev)
1930{
1931 struct mlx4_priv *priv = mlx4_priv(dev);
1932
1933 priv->clock_mapping =
872bf2fb
YH		 1934 ioremap(pci_resource_start(dev->persist->pdev,
1935 priv->fw.clock_bar) +
ddd8a6c1
EE		 1936 priv->fw.clock_offset, MLX4_CLOCK_SIZE);
1937
1938 if (!priv->clock_mapping)
1939 return -ENOMEM;
1940
1941 return 0;
1942}
1943
52033cfb
MB		 1944int mlx4_get_internal_clock_params(struct mlx4_dev *dev,
1945 struct mlx4_clock_params *params)
1946{
1947 struct mlx4_priv *priv = mlx4_priv(dev);
1948
1949 if (mlx4_is_slave(dev))
423b3aec 1950 return -EOPNOTSUPP;
52033cfb
MB		 1951
1952 if (!params)
1953 return -EINVAL;
1954
1955 params->bar = priv->fw.clock_bar;
1956 params->offset = priv->fw.clock_offset;
1957 params->size = MLX4_CLOCK_SIZE;
1958
1959 return 0;
1960}
1961EXPORT_SYMBOL_GPL(mlx4_get_internal_clock_params);
1962
ddd8a6c1
EE		 1963static void unmap_internal_clock(struct mlx4_dev *dev)
1964{
1965 struct mlx4_priv *priv = mlx4_priv(dev);
1966
1967 if (priv->clock_mapping)
1968 iounmap(priv->clock_mapping);
1969}
1970
225c7b1f
RD		 1971static void mlx4_close_hca(struct mlx4_dev *dev)
1972{
ddd8a6c1 1973 unmap_internal_clock(dev);
c1b43dca 1974 unmap_bf_area(dev);
ab9c17a0
JM		 1975 if (mlx4_is_slave(dev))
1976 mlx4_slave_exit(dev);
1977 else {
1978 mlx4_CLOSE_HCA(dev, 0);
1979 mlx4_free_icms(dev);
a0eacca9
MB		 1980 }
1981}
1982
1983static void mlx4_close_fw(struct mlx4_dev *dev)
1984{
1985 if (!mlx4_is_slave(dev)) {
ab9c17a0
JM		 1986 mlx4_UNMAP_FA(dev);
1987 mlx4_free_icm(dev, mlx4_priv(dev)->fw.fw_icm, 0);
1988 }
1989}
1990
55ad3592
YH		 1991static int mlx4_comm_check_offline(struct mlx4_dev *dev)
1992{
1993#define COMM_CHAN_OFFLINE_OFFSET 0x09
1994
1995 u32 comm_flags;
1996 u32 offline_bit;
1997 unsigned long end;
1998 struct mlx4_priv *priv = mlx4_priv(dev);
1999
2000 end = msecs_to_jiffies(MLX4_COMM_OFFLINE_TIME_OUT) + jiffies;
2001 while (time_before(jiffies, end)) {
2002 comm_flags = swab32(readl((__iomem char *)priv->mfunc.comm +
2003 MLX4_COMM_CHAN_FLAGS));
2004 offline_bit = (comm_flags &
2005 (u32)(1 << COMM_CHAN_OFFLINE_OFFSET));
2006 if (!offline_bit)
2007 return 0;
4cbe4dac
JM		 2008
2009 /* If device removal has been requested,
2010 * do not continue retrying.
2011 */
2012 if (dev->persist->interface_state &
2013 MLX4_INTERFACE_STATE_NOWAIT)
2014 break;
2015
55ad3592
YH		 2016 /* There are cases as part of AER/Reset flow that PF needs
2017 * around 100 msec to load. We therefore sleep for 100 msec
2018 * to allow other tasks to make use of that CPU during this
2019 * time interval.
2020 */
2021 msleep(100);
2022 }
2023 mlx4_err(dev, "Communication channel is offline.\n");
2024 return -EIO;
2025}
2026
2027static void mlx4_reset_vf_support(struct mlx4_dev *dev)
2028{
2029#define COMM_CHAN_RST_OFFSET 0x1e
2030
2031 struct mlx4_priv *priv = mlx4_priv(dev);
2032 u32 comm_rst;
2033 u32 comm_caps;
2034
2035 comm_caps = swab32(readl((__iomem char *)priv->mfunc.comm +
2036 MLX4_COMM_CHAN_CAPS));
2037 comm_rst = (comm_caps & (u32)(1 << COMM_CHAN_RST_OFFSET));
2038
2039 if (comm_rst)
2040 dev->caps.vf_caps |= MLX4_VF_CAP_FLAG_RESET;
2041}
2042
ab9c17a0
JM		 2043static int mlx4_init_slave(struct mlx4_dev *dev)
2044{
2045 struct mlx4_priv *priv = mlx4_priv(dev);
2046 u64 dma = (u64) priv->mfunc.vhcr_dma;
ab9c17a0
JM		 2047 int ret_from_reset = 0;
2048 u32 slave_read;
2049 u32 cmd_channel_ver;
2050
97989356 2051 if (atomic_read(&pf_loading)) {
1a91de28 2052 mlx4_warn(dev, "PF is not ready - Deferring probe\n");
97989356
AV		 2053 return -EPROBE_DEFER;
2054 }
2055
f3d4c89e 2056 mutex_lock(&priv->cmd.slave_cmd_mutex);
ab9c17a0 2057 priv->cmd.max_cmds = 1;
55ad3592
YH		 2058 if (mlx4_comm_check_offline(dev)) {
2059 mlx4_err(dev, "PF is not responsive, skipping initialization\n");
2060 goto err_offline;
2061 }
2062
2063 mlx4_reset_vf_support(dev);
ab9c17a0
JM		 2064 mlx4_warn(dev, "Sending reset\n");
2065 ret_from_reset = mlx4_comm_cmd(dev, MLX4_COMM_CMD_RESET, 0,
0cd93027 2066 MLX4_COMM_CMD_NA_OP, MLX4_COMM_TIME);
ab9c17a0
JM		 2067 /* if we are in the middle of flr the slave will try
2068 * NUM_OF_RESET_RETRIES times before leaving.*/
2069 if (ret_from_reset) {
2070 if (MLX4_DELAY_RESET_SLAVE == ret_from_reset) {
1a91de28 2071 mlx4_warn(dev, "slave is currently in the middle of FLR - Deferring probe\n");
5efe5355
JM		 2072 mutex_unlock(&priv->cmd.slave_cmd_mutex);
2073 return -EPROBE_DEFER;
ab9c17a0
JM		 2074 } else
2075 goto err;
2076 }
2077
2078 /* check the driver version - the slave I/F revision
2079 * must match the master's */
2080 slave_read = swab32(readl(&priv->mfunc.comm->slave_read));
2081 cmd_channel_ver = mlx4_comm_get_version();
2082
2083 if (MLX4_COMM_GET_IF_REV(cmd_channel_ver) !=
2084 MLX4_COMM_GET_IF_REV(slave_read)) {
1a91de28 2085 mlx4_err(dev, "slave driver version is not supported by the master\n");
ab9c17a0
JM		 2086 goto err;
2087 }
2088
2089 mlx4_warn(dev, "Sending vhcr0\n");
2090 if (mlx4_comm_cmd(dev, MLX4_COMM_CMD_VHCR0, dma >> 48,
0cd93027 2091 MLX4_COMM_CMD_NA_OP, MLX4_COMM_TIME))
ab9c17a0
JM		 2092 goto err;
2093 if (mlx4_comm_cmd(dev, MLX4_COMM_CMD_VHCR1, dma >> 32,
0cd93027 2094 MLX4_COMM_CMD_NA_OP, MLX4_COMM_TIME))
ab9c17a0
JM		 2095 goto err;
2096 if (mlx4_comm_cmd(dev, MLX4_COMM_CMD_VHCR2, dma >> 16,
0cd93027 2097 MLX4_COMM_CMD_NA_OP, MLX4_COMM_TIME))
ab9c17a0 2098 goto err;
0cd93027
YH		 2099 if (mlx4_comm_cmd(dev, MLX4_COMM_CMD_VHCR_EN, dma,
2100 MLX4_COMM_CMD_NA_OP, MLX4_COMM_TIME))
ab9c17a0 2101 goto err;
f3d4c89e
RD		 2102
2103 mutex_unlock(&priv->cmd.slave_cmd_mutex);
ab9c17a0
JM		 2104 return 0;
2105
2106err:
0cd93027 2107 mlx4_comm_cmd(dev, MLX4_COMM_CMD_RESET, 0, MLX4_COMM_CMD_NA_OP, 0);
55ad3592 2108err_offline:
f3d4c89e 2109 mutex_unlock(&priv->cmd.slave_cmd_mutex);
ab9c17a0 2110 return -EIO;
225c7b1f
RD		 2111}
2112
6634961c
JM		 2113static void mlx4_parav_master_pf_caps(struct mlx4_dev *dev)
2114{
2115 int i;
2116
2117 for (i = 1; i <= dev->caps.num_ports; i++) {
b6ffaeff
JM		 2118 if (dev->caps.port_type[i] == MLX4_PORT_TYPE_ETH)
2119 dev->caps.gid_table_len[i] =
449fc488 2120 mlx4_get_slave_num_gids(dev, 0, i);
b6ffaeff
JM		 2121 else
2122 dev->caps.gid_table_len[i] = 1;
6634961c
JM		 2123 dev->caps.pkey_table_len[i] =
2124 dev->phys_caps.pkey_phys_table_len[i] - 1;
2125 }
2126}
2127
3c439b55
JM		 2128static int choose_log_fs_mgm_entry_size(int qp_per_entry)
2129{
2130 int i = MLX4_MIN_MGM_LOG_ENTRY_SIZE;
2131
2132 for (i = MLX4_MIN_MGM_LOG_ENTRY_SIZE; i <= MLX4_MAX_MGM_LOG_ENTRY_SIZE;
2133 i++) {
2134 if (qp_per_entry <= 4 * ((1 << i) / 16 - 2))
2135 break;
2136 }
2137
2138 return (i <= MLX4_MAX_MGM_LOG_ENTRY_SIZE) ? i : -1;
2139}
2140
7d077cd3
MB		 2141static const char *dmfs_high_rate_steering_mode_str(int dmfs_high_steer_mode)
2142{
2143 switch (dmfs_high_steer_mode) {
2144 case MLX4_STEERING_DMFS_A0_DEFAULT:
2145 return "default performance";
2146
2147 case MLX4_STEERING_DMFS_A0_DYNAMIC:
2148 return "dynamic hybrid mode";
2149
2150 case MLX4_STEERING_DMFS_A0_STATIC:
2151 return "performance optimized for limited rule configuration (static)";
2152
2153 case MLX4_STEERING_DMFS_A0_DISABLE:
2154 return "disabled performance optimized steering";
2155
2156 case MLX4_STEERING_DMFS_A0_NOT_SUPPORTED:
2157 return "performance optimized steering not supported";
2158
2159 default:
2160 return "Unrecognized mode";
2161 }
2162}
2163
2164#define MLX4_DMFS_A0_STEERING (1UL << 2)
2165
7b8157be
JM		 2166static void choose_steering_mode(struct mlx4_dev *dev,
2167 struct mlx4_dev_cap *dev_cap)
2168{
7d077cd3
MB		 2169 if (mlx4_log_num_mgm_entry_size <= 0) {
2170 if ((-mlx4_log_num_mgm_entry_size) & MLX4_DMFS_A0_STEERING) {
2171 if (dev->caps.dmfs_high_steer_mode ==
2172 MLX4_STEERING_DMFS_A0_NOT_SUPPORTED)
2173 mlx4_err(dev, "DMFS high rate mode not supported\n");
2174 else
2175 dev->caps.dmfs_high_steer_mode =
2176 MLX4_STEERING_DMFS_A0_STATIC;
2177 }
2178 }
2179
2180 if (mlx4_log_num_mgm_entry_size <= 0 &&
3c439b55 2181 dev_cap->flags2 & MLX4_DEV_CAP_FLAG2_FS_EN &&
7b8157be 2182 (!mlx4_is_mfunc(dev) ||
872bf2fb
YH		 2183 (dev_cap->fs_max_num_qp_per_entry >=
2184 (dev->persist->num_vfs + 1))) &&
3c439b55
JM		 2185 choose_log_fs_mgm_entry_size(dev_cap->fs_max_num_qp_per_entry) >=
2186 MLX4_MIN_MGM_LOG_ENTRY_SIZE) {
2187 dev->oper_log_mgm_entry_size =
2188 choose_log_fs_mgm_entry_size(dev_cap->fs_max_num_qp_per_entry);
7b8157be
JM		 2189 dev->caps.steering_mode = MLX4_STEERING_MODE_DEVICE_MANAGED;
2190 dev->caps.num_qp_per_mgm = dev_cap->fs_max_num_qp_per_entry;
2191 dev->caps.fs_log_max_ucast_qp_range_size =
2192 dev_cap->fs_log_max_ucast_qp_range_size;
2193 } else {
7d077cd3
MB		 2194 if (dev->caps.dmfs_high_steer_mode !=
2195 MLX4_STEERING_DMFS_A0_NOT_SUPPORTED)
2196 dev->caps.dmfs_high_steer_mode = MLX4_STEERING_DMFS_A0_DISABLE;
7b8157be
JM		 2197 if (dev->caps.flags & MLX4_DEV_CAP_FLAG_VEP_UC_STEER &&
2198 dev->caps.flags & MLX4_DEV_CAP_FLAG_VEP_MC_STEER)
2199 dev->caps.steering_mode = MLX4_STEERING_MODE_B0;
2200 else {
2201 dev->caps.steering_mode = MLX4_STEERING_MODE_A0;
2202
2203 if (dev->caps.flags & MLX4_DEV_CAP_FLAG_VEP_UC_STEER ||
2204 dev->caps.flags & MLX4_DEV_CAP_FLAG_VEP_MC_STEER)
1a91de28 2205 mlx4_warn(dev, "Must have both UC_STEER and MC_STEER flags set to use B0 steering - falling back to A0 steering mode\n");
7b8157be 2206 }
3c439b55
JM		 2207 dev->oper_log_mgm_entry_size =
2208 mlx4_log_num_mgm_entry_size > 0 ?
2209 mlx4_log_num_mgm_entry_size :
2210 MLX4_DEFAULT_MGM_LOG_ENTRY_SIZE;
7b8157be
JM		 2211 dev->caps.num_qp_per_mgm = mlx4_get_qp_per_mgm(dev);
2212 }
1a91de28 2213 mlx4_dbg(dev, "Steering mode is: %s, oper_log_mgm_entry_size = %d, modparam log_num_mgm_entry_size = %d\n",
3c439b55
JM		 2214 mlx4_steering_mode_str(dev->caps.steering_mode),
2215 dev->oper_log_mgm_entry_size,
2216 mlx4_log_num_mgm_entry_size);
7b8157be
JM		 2217}
2218
7ffdf726
OG		 2219static void choose_tunnel_offload_mode(struct mlx4_dev *dev,
2220 struct mlx4_dev_cap *dev_cap)
2221{
2222 if (dev->caps.steering_mode == MLX4_STEERING_MODE_DEVICE_MANAGED &&
5eff6dad 2223 dev_cap->flags2 & MLX4_DEV_CAP_FLAG2_VXLAN_OFFLOADS)
7ffdf726
OG		 2224 dev->caps.tunnel_offload_mode = MLX4_TUNNEL_OFFLOAD_MODE_VXLAN;
2225 else
2226 dev->caps.tunnel_offload_mode = MLX4_TUNNEL_OFFLOAD_MODE_NONE;
2227
2228 mlx4_dbg(dev, "Tunneling offload mode is: %s\n", (dev->caps.tunnel_offload_mode
2229 == MLX4_TUNNEL_OFFLOAD_MODE_VXLAN) ? "vxlan" : "none");
2230}
2231
7d077cd3
MB		 2232static int mlx4_validate_optimized_steering(struct mlx4_dev *dev)
2233{
2234 int i;
2235 struct mlx4_port_cap port_cap;
2236
2237 if (dev->caps.dmfs_high_steer_mode == MLX4_STEERING_DMFS_A0_NOT_SUPPORTED)
2238 return -EINVAL;
2239
2240 for (i = 1; i <= dev->caps.num_ports; i++) {
2241 if (mlx4_dev_port(dev, i, &port_cap)) {
2242 mlx4_err(dev,
f4b752a6 2243 "QUERY_DEV_CAP command failed, can't verify DMFS high rate steering.\n");
7d077cd3
MB		 2244 } else if ((dev->caps.dmfs_high_steer_mode !=
2245 MLX4_STEERING_DMFS_A0_DEFAULT) &&
2246 (port_cap.dmfs_optimized_state ==
2247 !!(dev->caps.dmfs_high_steer_mode ==
2248 MLX4_STEERING_DMFS_A0_DISABLE))) {
2249 mlx4_err(dev,
2250 "DMFS high rate steer mode differ, driver requested %s but %s in FW.\n",
2251 dmfs_high_rate_steering_mode_str(
2252 dev->caps.dmfs_high_steer_mode),
2253 (port_cap.dmfs_optimized_state ?
2254 "enabled" : "disabled"));
2255 }
2256 }
2257
2258 return 0;
2259}
2260
a0eacca9 2261static int mlx4_init_fw(struct mlx4_dev *dev)
225c7b1f 2262{
2d928651 2263 struct mlx4_mod_stat_cfg mlx4_cfg;
a0eacca9 2264 int err = 0;
225c7b1f 2265
ab9c17a0
JM		 2266 if (!mlx4_is_slave(dev)) {
2267 err = mlx4_QUERY_FW(dev);
2268 if (err) {
2269 if (err == -EACCES)
1a91de28 2270 mlx4_info(dev, "non-primary physical function, skipping\n");
ab9c17a0 2271 else
1a91de28 2272 mlx4_err(dev, "QUERY_FW command failed, aborting\n");
bef772eb 2273 return err;
ab9c17a0 2274 }
225c7b1f 2275
ab9c17a0
JM		 2276 err = mlx4_load_fw(dev);
2277 if (err) {
1a91de28 2278 mlx4_err(dev, "Failed to start FW, aborting\n");
bef772eb 2279 return err;
ab9c17a0 2280 }
225c7b1f 2281
ab9c17a0
JM		 2282 mlx4_cfg.log_pg_sz_m = 1;
2283 mlx4_cfg.log_pg_sz = 0;
2284 err = mlx4_MOD_STAT_CFG(dev, &mlx4_cfg);
2285 if (err)
2286 mlx4_warn(dev, "Failed to override log_pg_sz parameter\n");
a0eacca9 2287 }
2d928651 2288
a0eacca9
MB		 2289 return err;
2290}
2291
2292static int mlx4_init_hca(struct mlx4_dev *dev)
2293{
2294 struct mlx4_priv *priv = mlx4_priv(dev);
2295 struct mlx4_adapter adapter;
2296 struct mlx4_dev_cap dev_cap;
2297 struct mlx4_profile profile;
2298 struct mlx4_init_hca_param init_hca;
2299 u64 icm_size;
2300 struct mlx4_config_dev_params params;
2301 int err;
2302
2303 if (!mlx4_is_slave(dev)) {
ab9c17a0
JM		 2304 err = mlx4_dev_cap(dev, &dev_cap);
2305 if (err) {
1a91de28 2306 mlx4_err(dev, "QUERY_DEV_CAP command failed, aborting\n");
d0d01250 2307 return err;
ab9c17a0 2308 }
225c7b1f 2309
7b8157be 2310 choose_steering_mode(dev, &dev_cap);
7ffdf726 2311 choose_tunnel_offload_mode(dev, &dev_cap);
7b8157be 2312
7d077cd3
MB		 2313 if (dev->caps.dmfs_high_steer_mode == MLX4_STEERING_DMFS_A0_STATIC &&
2314 mlx4_is_master(dev))
2315 dev->caps.function_caps |= MLX4_FUNC_CAP_DMFS_A0_STATIC;
2316
8e1a28e8
HHZ		 2317 err = mlx4_get_phys_port_id(dev);
2318 if (err)
2319 mlx4_err(dev, "Fail to get physical port id\n");
2320
6634961c
JM		 2321 if (mlx4_is_master(dev))
2322 mlx4_parav_master_pf_caps(dev);
2323
2599d858
AV		 2324 if (mlx4_low_memory_profile()) {
2325 mlx4_info(dev, "Running from within kdump kernel. Using low memory profile\n");
2326 profile = low_mem_profile;
2327 } else {
2328 profile = default_profile;
2329 }
0ff1fb65
HHZ		 2330 if (dev->caps.steering_mode ==
2331 MLX4_STEERING_MODE_DEVICE_MANAGED)
2332 profile.num_mcg = MLX4_FS_NUM_MCG;
225c7b1f 2333
ab9c17a0
JM		 2334 icm_size = mlx4_make_profile(dev, &profile, &dev_cap,
2335 &init_hca);
2336 if ((long long) icm_size < 0) {
2337 err = icm_size;
d0d01250 2338 return err;
ab9c17a0 2339 }
225c7b1f 2340
a5bbe892
EC		 2341 dev->caps.max_fmr_maps = (1 << (32 - ilog2(dev->caps.num_mpts))) - 1;
2342
ca3d89a3 2343 if (enable_4k_uar || !dev->persist->num_vfs) {
76e39ccf
EC		 2344 init_hca.log_uar_sz = ilog2(dev->caps.num_uars) +
2345 PAGE_SHIFT - DEFAULT_UAR_PAGE_SHIFT;
2346 init_hca.uar_page_sz = DEFAULT_UAR_PAGE_SHIFT - 12;
2347 } else {
2348 init_hca.log_uar_sz = ilog2(dev->caps.num_uars);
2349 init_hca.uar_page_sz = PAGE_SHIFT - 12;
2350 }
85743f1e 2351
e448834e
SM		 2352 init_hca.mw_enabled = 0;
2353 if (dev->caps.flags & MLX4_DEV_CAP_FLAG_MEM_WINDOW ||
2354 dev->caps.bmme_flags & MLX4_BMME_FLAG_TYPE_2_WIN)
2355 init_hca.mw_enabled = INIT_HCA_TPT_MW_ENABLE;
c1b43dca 2356
ab9c17a0
JM		 2357 err = mlx4_init_icm(dev, &dev_cap, &init_hca, icm_size);
2358 if (err)
d0d01250 2359 return err;
225c7b1f 2360
ab9c17a0
JM		 2361 err = mlx4_INIT_HCA(dev, &init_hca);
2362 if (err) {
1a91de28 2363 mlx4_err(dev, "INIT_HCA command failed, aborting\n");
ab9c17a0
JM		 2364 goto err_free_icm;
2365 }
7ae0e400
MB		 2366
2367 if (dev_cap.flags2 & MLX4_DEV_CAP_FLAG2_SYS_EQS) {
2368 err = mlx4_query_func(dev, &dev_cap);
2369 if (err < 0) {
2370 mlx4_err(dev, "QUERY_FUNC command failed, aborting.\n");
d0d01250 2371 goto err_close;
7ae0e400
MB		 2372 } else if (err & MLX4_QUERY_FUNC_NUM_SYS_EQS) {
2373 dev->caps.num_eqs = dev_cap.max_eqs;
2374 dev->caps.reserved_eqs = dev_cap.reserved_eqs;
2375 dev->caps.reserved_uars = dev_cap.reserved_uars;
2376 }
2377 }
2378
ddd8a6c1
EE		 2379 /*
2380 * If TS is supported by FW
2381 * read HCA frequency by QUERY_HCA command
2382 */
2383 if (dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_TS) {
2384 memset(&init_hca, 0, sizeof(init_hca));
2385 err = mlx4_QUERY_HCA(dev, &init_hca);
2386 if (err) {
1a91de28 2387 mlx4_err(dev, "QUERY_HCA command failed, disable timestamp\n");
ddd8a6c1
EE		 2388 dev->caps.flags2 &= ~MLX4_DEV_CAP_FLAG2_TS;
2389 } else {
2390 dev->caps.hca_core_clock =
2391 init_hca.hca_core_clock;
2392 }
2393
2394 /* In case we got HCA frequency 0 - disable timestamping
2395 * to avoid dividing by zero
2396 */
2397 if (!dev->caps.hca_core_clock) {
2398 dev->caps.flags2 &= ~MLX4_DEV_CAP_FLAG2_TS;
2399 mlx4_err(dev,
1a91de28 2400 "HCA frequency is 0 - timestamping is not supported\n");
ddd8a6c1
EE		 2401 } else if (map_internal_clock(dev)) {
2402 /*
2403 * Map internal clock,
2404 * in case of failure disable timestamping
2405 */
2406 dev->caps.flags2 &= ~MLX4_DEV_CAP_FLAG2_TS;
1a91de28 2407 mlx4_err(dev, "Failed to map internal clock. Timestamping is not supported\n");
ddd8a6c1
EE		 2408 }
2409 }
7d077cd3
MB		 2410
2411 if (dev->caps.dmfs_high_steer_mode !=
2412 MLX4_STEERING_DMFS_A0_NOT_SUPPORTED) {
2413 if (mlx4_validate_optimized_steering(dev))
2414 mlx4_warn(dev, "Optimized steering validation failed\n");
2415
2416 if (dev->caps.dmfs_high_steer_mode ==
2417 MLX4_STEERING_DMFS_A0_DISABLE) {
2418 dev->caps.dmfs_high_rate_qpn_base =
2419 dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FW];
2420 dev->caps.dmfs_high_rate_qpn_range =
2421 MLX4_A0_STEERING_TABLE_SIZE;
2422 }
2423
4931c6ef
SM		 2424 mlx4_info(dev, "DMFS high rate steer mode is: %s\n",
2425 dmfs_high_rate_steering_mode_str(
7d077cd3
MB		 2426 dev->caps.dmfs_high_steer_mode));
2427 }
ab9c17a0
JM		 2428 } else {
2429 err = mlx4_init_slave(dev);
2430 if (err) {
5efe5355
JM		 2431 if (err != -EPROBE_DEFER)
2432 mlx4_err(dev, "Failed to initialize slave\n");
bef772eb 2433 return err;
ab9c17a0 2434 }
225c7b1f 2435
ab9c17a0
JM		 2436 err = mlx4_slave_cap(dev);
2437 if (err) {
2438 mlx4_err(dev, "Failed to obtain slave caps\n");
2439 goto err_close;
2440 }
225c7b1f
RD		 2441 }
2442
ab9c17a0
JM		 2443 if (map_bf_area(dev))
2444 mlx4_dbg(dev, "Failed to map blue flame area\n");
2445
2446 /*Only the master set the ports, all the rest got it from it.*/
2447 if (!mlx4_is_slave(dev))
2448 mlx4_set_port_mask(dev);
2449
225c7b1f
RD		 2450 err = mlx4_QUERY_ADAPTER(dev, &adapter);
2451 if (err) {
1a91de28 2452 mlx4_err(dev, "QUERY_ADAPTER command failed, aborting\n");
bef772eb 2453 goto unmap_bf;
225c7b1f
RD		 2454 }
2455
f8c6455b
SM		 2456 /* Query CONFIG_DEV parameters */
2457 err = mlx4_config_dev_retrieval(dev, &params);
423b3aec 2458 if (err && err != -EOPNOTSUPP) {
f8c6455b
SM		 2459 mlx4_err(dev, "Failed to query CONFIG_DEV parameters\n");
2460 } else if (!err) {
2461 dev->caps.rx_checksum_flags_port[1] = params.rx_csum_flags_port_1;
2462 dev->caps.rx_checksum_flags_port[2] = params.rx_csum_flags_port_2;
2463 }
225c7b1f 2464 priv->eq_table.inta_pin = adapter.inta_pin;
31975e27 2465 memcpy(dev->board_id, adapter.board_id, sizeof(dev->board_id));
225c7b1f
RD		 2466
2467 return 0;
2468
bef772eb 2469unmap_bf:
ddd8a6c1 2470 unmap_internal_clock(dev);
bef772eb
AY		 2471 unmap_bf_area(dev);
2472
c73c8b1e
EBE		 2473 if (mlx4_is_slave(dev))
2474 mlx4_slave_destroy_special_qp_cap(dev);
b38f2879 2475
225c7b1f 2476err_close:
41929ed2
DB		 2477 if (mlx4_is_slave(dev))
2478 mlx4_slave_exit(dev);
2479 else
2480 mlx4_CLOSE_HCA(dev, 0);
225c7b1f
RD		 2481
2482err_free_icm:
ab9c17a0
JM		 2483 if (!mlx4_is_slave(dev))
2484 mlx4_free_icms(dev);
225c7b1f 2485
225c7b1f
RD		 2486 return err;
2487}
2488
f2a3f6a3
OG		 2489static int mlx4_init_counters_table(struct mlx4_dev *dev)
2490{
2491 struct mlx4_priv *priv = mlx4_priv(dev);
47d8417f 2492 int nent_pow2;
f2a3f6a3
OG		 2493
2494 if (!(dev->caps.flags & MLX4_DEV_CAP_FLAG_COUNTERS))
2495 return -ENOENT;
2496
2632d18d
EBE		 2497 if (!dev->caps.max_counters)
2498 return -ENOSPC;
2499
47d8417f
EBE		 2500 nent_pow2 = roundup_pow_of_two(dev->caps.max_counters);
2501 /* reserve last counter index for sink counter */
2502 return mlx4_bitmap_init(&priv->counters_bitmap, nent_pow2,
2503 nent_pow2 - 1, 0,
2504 nent_pow2 - dev->caps.max_counters + 1);
f2a3f6a3
OG		 2505}
2506
2507static void mlx4_cleanup_counters_table(struct mlx4_dev *dev)
2508{
efa6bc91
EBE		 2509 if (!(dev->caps.flags & MLX4_DEV_CAP_FLAG_COUNTERS))
2510 return;
2511
2632d18d
EBE		 2512 if (!dev->caps.max_counters)
2513 return;
2514
f2a3f6a3
OG		 2515 mlx4_bitmap_cleanup(&mlx4_priv(dev)->counters_bitmap);
2516}
2517
6de5f7f6
EBE		 2518static void mlx4_cleanup_default_counters(struct mlx4_dev *dev)
2519{
2520 struct mlx4_priv *priv = mlx4_priv(dev);
2521 int port;
2522
2523 for (port = 0; port < dev->caps.num_ports; port++)
2524 if (priv->def_counter[port] != -1)
2525 mlx4_counter_free(dev, priv->def_counter[port]);
2526}
2527
2528static int mlx4_allocate_default_counters(struct mlx4_dev *dev)
2529{
2530 struct mlx4_priv *priv = mlx4_priv(dev);
2531 int port, err = 0;
2532 u32 idx;
2533
2534 for (port = 0; port < dev->caps.num_ports; port++)
2535 priv->def_counter[port] = -1;
2536
2537 for (port = 0; port < dev->caps.num_ports; port++) {
f3301870 2538 err = mlx4_counter_alloc(dev, &idx, MLX4_RES_USAGE_DRIVER);
6de5f7f6
EBE		 2539
2540 if (!err || err == -ENOSPC) {
2541 priv->def_counter[port] = idx;
2542 } else if (err == -ENOENT) {
2543 err = 0;
2544 continue;
178d23e3
OG		 2545 } else if (mlx4_is_slave(dev) && err == -EINVAL) {
2546 priv->def_counter[port] = MLX4_SINK_COUNTER_INDEX(dev);
2547 mlx4_warn(dev, "can't allocate counter from old PF driver, using index %d\n",
2548 MLX4_SINK_COUNTER_INDEX(dev));
2549 err = 0;
6de5f7f6
EBE		 2550 } else {
2551 mlx4_err(dev, "%s: failed to allocate default counter port %d err %d\n",
2552 __func__, port + 1, err);
2553 mlx4_cleanup_default_counters(dev);
2554 return err;
2555 }
2556
2557 mlx4_dbg(dev, "%s: default counter index %d for port %d\n",
2558 __func__, priv->def_counter[port], port + 1);
2559 }
2560
2561 return err;
2562}
2563
ba062d52 2564int __mlx4_counter_alloc(struct mlx4_dev *dev, u32 *idx)
f2a3f6a3
OG		 2565{
2566 struct mlx4_priv *priv = mlx4_priv(dev);
2567
2568 if (!(dev->caps.flags & MLX4_DEV_CAP_FLAG_COUNTERS))
2569 return -ENOENT;
2570
2571 *idx = mlx4_bitmap_alloc(&priv->counters_bitmap);
6de5f7f6
EBE		 2572 if (*idx == -1) {
2573 *idx = MLX4_SINK_COUNTER_INDEX(dev);
2574 return -ENOSPC;
2575 }
f2a3f6a3
OG		 2576
2577 return 0;
2578}
ba062d52 2579
f3301870 2580int mlx4_counter_alloc(struct mlx4_dev *dev, u32 *idx, u8 usage)
ba062d52 2581{
f3301870 2582 u32 in_modifier = RES_COUNTER | (((u32)usage & 3) << 30);
ba062d52
JM		 2583 u64 out_param;
2584 int err;
2585
2586 if (mlx4_is_mfunc(dev)) {
f3301870 2587 err = mlx4_cmd_imm(dev, 0, &out_param, in_modifier,
ba062d52
JM		 2588 RES_OP_RESERVE, MLX4_CMD_ALLOC_RES,
2589 MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED);
2590 if (!err)
2591 *idx = get_param_l(&out_param);
2592
2593 return err;
2594 }
2595 return __mlx4_counter_alloc(dev, idx);
2596}
f2a3f6a3
OG		 2597EXPORT_SYMBOL_GPL(mlx4_counter_alloc);
2598
b72ca7e9
EBE		 2599static int __mlx4_clear_if_stat(struct mlx4_dev *dev,
2600 u8 counter_index)
2601{
2602 struct mlx4_cmd_mailbox *if_stat_mailbox;
2603 int err;
2604 u32 if_stat_in_mod = (counter_index & 0xff) | MLX4_QUERY_IF_STAT_RESET;
2605
2606 if_stat_mailbox = mlx4_alloc_cmd_mailbox(dev);
2607 if (IS_ERR(if_stat_mailbox))
2608 return PTR_ERR(if_stat_mailbox);
2609
2610 err = mlx4_cmd_box(dev, 0, if_stat_mailbox->dma, if_stat_in_mod, 0,
2611 MLX4_CMD_QUERY_IF_STAT, MLX4_CMD_TIME_CLASS_C,
2612 MLX4_CMD_NATIVE);
2613
2614 mlx4_free_cmd_mailbox(dev, if_stat_mailbox);
2615 return err;
2616}
2617
ba062d52 2618void __mlx4_counter_free(struct mlx4_dev *dev, u32 idx)
f2a3f6a3 2619{
efa6bc91
EBE		 2620 if (!(dev->caps.flags & MLX4_DEV_CAP_FLAG_COUNTERS))
2621 return;
2622
6de5f7f6
EBE		 2623 if (idx == MLX4_SINK_COUNTER_INDEX(dev))
2624 return;
2625
b72ca7e9
EBE		 2626 __mlx4_clear_if_stat(dev, idx);
2627
7c6d74d2 2628 mlx4_bitmap_free(&mlx4_priv(dev)->counters_bitmap, idx, MLX4_USE_RR);
f2a3f6a3
OG		 2629 return;
2630}
ba062d52
JM		 2631
2632void mlx4_counter_free(struct mlx4_dev *dev, u32 idx)
2633{
e7dbeba8 2634 u64 in_param = 0;
ba062d52
JM		 2635
2636 if (mlx4_is_mfunc(dev)) {
2637 set_param_l(&in_param, idx);
2638 mlx4_cmd(dev, in_param, RES_COUNTER, RES_OP_RESERVE,
2639 MLX4_CMD_FREE_RES, MLX4_CMD_TIME_CLASS_A,
2640 MLX4_CMD_WRAPPED);
2641 return;
2642 }
2643 __mlx4_counter_free(dev, idx);
2644}
f2a3f6a3
OG		 2645EXPORT_SYMBOL_GPL(mlx4_counter_free);
2646
6de5f7f6
EBE		 2647int mlx4_get_default_counter_index(struct mlx4_dev *dev, int port)
2648{
2649 struct mlx4_priv *priv = mlx4_priv(dev);
2650
2651 return priv->def_counter[port - 1];
2652}
2653EXPORT_SYMBOL_GPL(mlx4_get_default_counter_index);
2654
773af94e
YH		 2655void mlx4_set_admin_guid(struct mlx4_dev *dev, __be64 guid, int entry, int port)
2656{
2657 struct mlx4_priv *priv = mlx4_priv(dev);
2658
2659 priv->mfunc.master.vf_admin[entry].vport[port].guid = guid;
2660}
2661EXPORT_SYMBOL_GPL(mlx4_set_admin_guid);
2662
2663__be64 mlx4_get_admin_guid(struct mlx4_dev *dev, int entry, int port)
2664{
2665 struct mlx4_priv *priv = mlx4_priv(dev);
2666
2667 return priv->mfunc.master.vf_admin[entry].vport[port].guid;
2668}
2669EXPORT_SYMBOL_GPL(mlx4_get_admin_guid);
2670
fb517a4f
YH		 2671void mlx4_set_random_admin_guid(struct mlx4_dev *dev, int entry, int port)
2672{
2673 struct mlx4_priv *priv = mlx4_priv(dev);
2674 __be64 guid;
2675
2676 /* hw GUID */
2677 if (entry == 0)
2678 return;
2679
2680 get_random_bytes((char *)&guid, sizeof(guid));
2681 guid &= ~(cpu_to_be64(1ULL << 56));
2682 guid |= cpu_to_be64(1ULL << 57);
2683 priv->mfunc.master.vf_admin[entry].vport[port].guid = guid;
2684}
2685
3d73c288 2686static int mlx4_setup_hca(struct mlx4_dev *dev)
225c7b1f
RD		 2687{
2688 struct mlx4_priv *priv = mlx4_priv(dev);
2689 int err;
7ff93f8b 2690 int port;
9a5aa622 2691 __be32 ib_port_default_caps;
225c7b1f 2692
225c7b1f
RD		 2693 err = mlx4_init_uar_table(dev);
2694 if (err) {
1a91de28 2695 mlx4_err(dev, "Failed to initialize user access region table, aborting\n");
5d4de16c 2696 return err;
225c7b1f
RD		 2697 }
2698
2699 err = mlx4_uar_alloc(dev, &priv->driver_uar);
2700 if (err) {
1a91de28 2701 mlx4_err(dev, "Failed to allocate driver access region, aborting\n");
225c7b1f
RD		 2702 goto err_uar_table_free;
2703 }
2704
4979d18f 2705 priv->kar = ioremap((phys_addr_t) priv->driver_uar.pfn << PAGE_SHIFT, PAGE_SIZE);
225c7b1f 2706 if (!priv->kar) {
1a91de28 2707 mlx4_err(dev, "Couldn't map kernel access region, aborting\n");
225c7b1f
RD		 2708 err = -ENOMEM;
2709 goto err_uar_free;
2710 }
2711
2712 err = mlx4_init_pd_table(dev);
2713 if (err) {
1a91de28 2714 mlx4_err(dev, "Failed to initialize protection domain table, aborting\n");
225c7b1f
RD		 2715 goto err_kar_unmap;
2716 }
2717
012a8ff5
SH		 2718 err = mlx4_init_xrcd_table(dev);
2719 if (err) {
1a91de28 2720 mlx4_err(dev, "Failed to initialize reliable connection domain table, aborting\n");
012a8ff5
SH		 2721 goto err_pd_table_free;
2722 }
2723
225c7b1f
RD		 2724 err = mlx4_init_mr_table(dev);
2725 if (err) {
1a91de28 2726 mlx4_err(dev, "Failed to initialize memory region table, aborting\n");
012a8ff5 2727 goto err_xrcd_table_free;
225c7b1f
RD		 2728 }
2729
fe6f700d
YP		 2730 if (!mlx4_is_slave(dev)) {
2731 err = mlx4_init_mcg_table(dev);
2732 if (err) {
1a91de28 2733 mlx4_err(dev, "Failed to initialize multicast group table, aborting\n");
fe6f700d
YP		 2734 goto err_mr_table_free;
2735 }
114840c3
JM		 2736 err = mlx4_config_mad_demux(dev);
2737 if (err) {
2738 mlx4_err(dev, "Failed in config_mad_demux, aborting\n");
2739 goto err_mcg_table_free;
2740 }
fe6f700d
YP		 2741 }
2742
225c7b1f
RD		 2743 err = mlx4_init_eq_table(dev);
2744 if (err) {
1a91de28 2745 mlx4_err(dev, "Failed to initialize event queue table, aborting\n");
fe6f700d 2746 goto err_mcg_table_free;
225c7b1f
RD		 2747 }
2748
2749 err = mlx4_cmd_use_events(dev);
2750 if (err) {
1a91de28 2751 mlx4_err(dev, "Failed to switch to event-driven firmware commands, aborting\n");
225c7b1f
RD		 2752 goto err_eq_table_free;
2753 }
2754
2755 err = mlx4_NOP(dev);
2756 if (err) {
08fb1055 2757 if (dev->flags & MLX4_FLAG_MSI_X) {
1a91de28 2758 mlx4_warn(dev, "NOP command failed to generate MSI-X interrupt IRQ %d)\n",
c66fa19c 2759 priv->eq_table.eq[MLX4_EQ_ASYNC].irq);
1a91de28 2760 mlx4_warn(dev, "Trying again without MSI-X\n");
08fb1055 2761 } else {
1a91de28 2762 mlx4_err(dev, "NOP command failed to generate interrupt (IRQ %d), aborting\n",
c66fa19c 2763 priv->eq_table.eq[MLX4_EQ_ASYNC].irq);
225c7b1f 2764 mlx4_err(dev, "BIOS or ACPI interrupt routing problem?\n");
08fb1055 2765 }
225c7b1f
RD		 2766
2767 goto err_cmd_poll;
2768 }
2769
2770 mlx4_dbg(dev, "NOP command IRQ test passed\n");
2771
2772 err = mlx4_init_cq_table(dev);
2773 if (err) {
1a91de28 2774 mlx4_err(dev, "Failed to initialize completion queue table, aborting\n");
225c7b1f
RD		 2775 goto err_cmd_poll;
2776 }
2777
2778 err = mlx4_init_srq_table(dev);
2779 if (err) {
1a91de28 2780 mlx4_err(dev, "Failed to initialize shared receive queue table, aborting\n");
225c7b1f
RD		 2781 goto err_cq_table_free;
2782 }
2783
2784 err = mlx4_init_qp_table(dev);
2785 if (err) {
1a91de28 2786 mlx4_err(dev, "Failed to initialize queue pair table, aborting\n");
225c7b1f
RD		 2787 goto err_srq_table_free;
2788 }
2789
2632d18d
EBE		 2790 if (!mlx4_is_slave(dev)) {
2791 err = mlx4_init_counters_table(dev);
2792 if (err && err != -ENOENT) {
2793 mlx4_err(dev, "Failed to initialize counters table, aborting\n");
2794 goto err_qp_table_free;
2795 }
f2a3f6a3
OG		 2796 }
2797
6de5f7f6
EBE		 2798 err = mlx4_allocate_default_counters(dev);
2799 if (err) {
2800 mlx4_err(dev, "Failed to allocate default counters, aborting\n");
2801 goto err_counters_table_free;
f2a3f6a3
OG		 2802 }
2803
ab9c17a0
JM		 2804 if (!mlx4_is_slave(dev)) {
2805 for (port = 1; port <= dev->caps.num_ports; port++) {
ab9c17a0
JM		 2806 ib_port_default_caps = 0;
2807 err = mlx4_get_port_ib_caps(dev, port,
2808 &ib_port_default_caps);
2809 if (err)
1a91de28
JP		 2810 mlx4_warn(dev, "failed to get port %d default ib capabilities (%d). Continuing with caps = 0\n",
2811 port, err);
ab9c17a0
JM		 2812 dev->caps.ib_port_def_cap[port] = ib_port_default_caps;
2813
2aca1172
JM		 2814 /* initialize per-slave default ib port capabilities */
2815 if (mlx4_is_master(dev)) {
2816 int i;
2817 for (i = 0; i < dev->num_slaves; i++) {
2818 if (i == mlx4_master_func_num(dev))
2819 continue;
2820 priv->mfunc.master.slave_state[i].ib_cap_mask[port] =
1a91de28 2821 ib_port_default_caps;
2aca1172
JM		 2822 }
2823 }
2824
096335b3
OG		 2825 if (mlx4_is_mfunc(dev))
2826 dev->caps.port_ib_mtu[port] = IB_MTU_2048;
2827 else
2828 dev->caps.port_ib_mtu[port] = IB_MTU_4096;
97285b78 2829
6634961c
JM		 2830 err = mlx4_SET_PORT(dev, port, mlx4_is_master(dev) ?
2831 dev->caps.pkey_table_len[port] : -1);
ab9c17a0
JM		 2832 if (err) {
2833 mlx4_err(dev, "Failed to set port %d, aborting\n",
1a91de28 2834 port);
6de5f7f6 2835 goto err_default_countes_free;
ab9c17a0 2836 }
7ff93f8b
YP		 2837 }
2838 }
2839
225c7b1f
RD		 2840 return 0;
2841
6de5f7f6
EBE		 2842err_default_countes_free:
2843 mlx4_cleanup_default_counters(dev);
2844
f2a3f6a3 2845err_counters_table_free:
2632d18d
EBE		 2846 if (!mlx4_is_slave(dev))
2847 mlx4_cleanup_counters_table(dev);
f2a3f6a3 2848
225c7b1f
RD		 2849err_qp_table_free:
2850 mlx4_cleanup_qp_table(dev);
2851
2852err_srq_table_free:
2853 mlx4_cleanup_srq_table(dev);
2854
2855err_cq_table_free:
2856 mlx4_cleanup_cq_table(dev);
2857
2858err_cmd_poll:
2859 mlx4_cmd_use_polling(dev);
2860
2861err_eq_table_free:
2862 mlx4_cleanup_eq_table(dev);
2863
fe6f700d
YP		 2864err_mcg_table_free:
2865 if (!mlx4_is_slave(dev))
2866 mlx4_cleanup_mcg_table(dev);
2867
ee49bd93 2868err_mr_table_free:
225c7b1f
RD		 2869 mlx4_cleanup_mr_table(dev);
2870
012a8ff5
SH		 2871err_xrcd_table_free:
2872 mlx4_cleanup_xrcd_table(dev);
2873
225c7b1f
RD		 2874err_pd_table_free:
2875 mlx4_cleanup_pd_table(dev);
2876
2877err_kar_unmap:
2878 iounmap(priv->kar);
2879
2880err_uar_free:
2881 mlx4_uar_free(dev, &priv->driver_uar);
2882
2883err_uar_table_free:
2884 mlx4_cleanup_uar_table(dev);
2885 return err;
2886}
2887
de161803
IS		 2888static int mlx4_init_affinity_hint(struct mlx4_dev *dev, int port, int eqn)
2889{
2890 int requested_cpu = 0;
2891 struct mlx4_priv *priv = mlx4_priv(dev);
2892 struct mlx4_eq *eq;
2893 int off = 0;
2894 int i;
2895
2896 if (eqn > dev->caps.num_comp_vectors)
2897 return -EINVAL;
2898
2899 for (i = 1; i < port; i++)
2900 off += mlx4_get_eqs_per_port(dev, i);
2901
2902 requested_cpu = eqn - off - !!(eqn > MLX4_EQ_ASYNC);
2903
2904 /* Meaning EQs are shared, and this call comes from the second port */
2905 if (requested_cpu < 0)
2906 return 0;
2907
2908 eq = &priv->eq_table.eq[eqn];
2909
2910 if (!zalloc_cpumask_var(&eq->affinity_mask, GFP_KERNEL))
2911 return -ENOMEM;
2912
2913 cpumask_set_cpu(requested_cpu, eq->affinity_mask);
2914
2915 return 0;
2916}
2917
e8f9b2ed 2918static void mlx4_enable_msi_x(struct mlx4_dev *dev)
225c7b1f
RD		 2919{
2920 struct mlx4_priv *priv = mlx4_priv(dev);
b8dd786f 2921 struct msix_entry *entries;
225c7b1f 2922 int i;
c66fa19c 2923 int port = 0;
225c7b1f
RD		 2924
2925 if (msi_x) {
4762010f 2926 int nreq = min3(dev->caps.num_ports *
2927 (int)num_online_cpus() + 1,
2928 dev->caps.num_eqs - dev->caps.reserved_eqs,
2929 MAX_MSIX);
ab9c17a0 2930
e5732838
TT		 2931 if (msi_x > 1)
2932 nreq = min_t(int, nreq, msi_x);
2933
31975e27 2934 entries = kcalloc(nreq, sizeof(*entries), GFP_KERNEL);
b8dd786f
YP		 2935 if (!entries)
2936 goto no_msi;
2937
2938 for (i = 0; i < nreq; ++i)
225c7b1f
RD		 2939 entries[i].entry = i;
2940
872bf2fb
YH		 2941 nreq = pci_enable_msix_range(dev->persist->pdev, entries, 2,
2942 nreq);
66e2f9c1 2943
c66fa19c 2944 if (nreq < 0 || nreq < MLX4_EQ_ASYNC) {
5bf0da7d 2945 kfree(entries);
225c7b1f 2946 goto no_msi;
0b7ca5a9 2947 }
c66fa19c
MB		 2948 /* 1 is reserved for events (asyncrounous EQ) */
2949 dev->caps.num_comp_vectors = nreq - 1;
2950
2951 priv->eq_table.eq[MLX4_EQ_ASYNC].irq = entries[0].vector;
2952 bitmap_zero(priv->eq_table.eq[MLX4_EQ_ASYNC].actv_ports.ports,
2953 dev->caps.num_ports);
2954
2955 for (i = 0; i < dev->caps.num_comp_vectors + 1; i++) {
2956 if (i == MLX4_EQ_ASYNC)
2957 continue;
2958
2959 priv->eq_table.eq[i].irq =
2960 entries[i + 1 - !!(i > MLX4_EQ_ASYNC)].vector;
2961
85121d6e 2962 if (MLX4_IS_LEGACY_EQ_MODE(dev->caps)) {
c66fa19c
MB		 2963 bitmap_fill(priv->eq_table.eq[i].actv_ports.ports,
2964 dev->caps.num_ports);
de161803
IS		 2965 /* We don't set affinity hint when there
2966 * aren't enough EQs
2967 */
c66fa19c
MB		 2968 } else {
2969 set_bit(port,
2970 priv->eq_table.eq[i].actv_ports.ports);
de161803
IS		 2971 if (mlx4_init_affinity_hint(dev, port + 1, i))
2972 mlx4_warn(dev, "Couldn't init hint cpumask for EQ %d\n",
2973 i);
c66fa19c
MB		 2974 }
2975 /* We divide the Eqs evenly between the two ports.
2976 * (dev->caps.num_comp_vectors / dev->caps.num_ports)
2977 * refers to the number of Eqs per port
2978 * (i.e eqs_per_port). Theoretically, we would like to
2979 * write something like (i + 1) % eqs_per_port == 0.
2980 * However, since there's an asynchronous Eq, we have
2981 * to skip over it by comparing this condition to
2982 * !!((i + 1) > MLX4_EQ_ASYNC).
2983 */
2984 if ((dev->caps.num_comp_vectors > dev->caps.num_ports) &&
2985 ((i + 1) %
2986 (dev->caps.num_comp_vectors / dev->caps.num_ports)) ==
2987 !!((i + 1) > MLX4_EQ_ASYNC))
2988 /* If dev->caps.num_comp_vectors < dev->caps.num_ports,
2989 * everything is shared anyway.
2990 */
2991 port++;
2992 }
225c7b1f
RD		 2993
2994 dev->flags |= MLX4_FLAG_MSI_X;
b8dd786f
YP		 2995
2996 kfree(entries);
225c7b1f
RD		 2997 return;
2998 }
2999
3000no_msi:
b8dd786f
YP		 3001 dev->caps.num_comp_vectors = 1;
3002
c66fa19c
MB		 3003 BUG_ON(MLX4_EQ_ASYNC >= 2);
3004 for (i = 0; i < 2; ++i) {
872bf2fb 3005 priv->eq_table.eq[i].irq = dev->persist->pdev->irq;
c66fa19c
MB		 3006 if (i != MLX4_EQ_ASYNC) {
3007 bitmap_fill(priv->eq_table.eq[i].actv_ports.ports,
3008 dev->caps.num_ports);
3009 }
3010 }
225c7b1f
RD		 3011}
3012
7ff93f8b 3013static int mlx4_init_port_info(struct mlx4_dev *dev, int port)
2a2336f8 3014{
09d4d087 3015 struct devlink *devlink = priv_to_devlink(mlx4_priv(dev));
2a2336f8 3016 struct mlx4_port_info *info = &mlx4_priv(dev)->port[port];
09d4d087
JP		 3017 int err;
3018
3019 err = devlink_port_register(devlink, &info->devlink_port, port);
3020 if (err)
3021 return err;
2a2336f8
YP		 3022
3023 info->dev = dev;
3024 info->port = port;
ab9c17a0 3025 if (!mlx4_is_slave(dev)) {
ab9c17a0
JM		 3026 mlx4_init_mac_table(dev, &info->mac_table);
3027 mlx4_init_vlan_table(dev, &info->vlan_table);
111c6094 3028 mlx4_init_roce_gid_table(dev, &info->gid_table);
16a10ffd 3029 info->base_qpn = mlx4_get_base_qpn(dev, port);
ab9c17a0 3030 }
7ff93f8b
YP		 3031
3032 sprintf(info->dev_name, "mlx4_port%d", port);
3033 info->port_attr.attr.name = info->dev_name;
d3757ba4
JP		 3034 if (mlx4_is_mfunc(dev)) {
3035 info->port_attr.attr.mode = 0444;
3036 } else {
3037 info->port_attr.attr.mode = 0644;
ab9c17a0
JM		 3038 info->port_attr.store = set_port_type;
3039 }
7ff93f8b 3040 info->port_attr.show = show_port_type;
3691c964 3041 sysfs_attr_init(&info->port_attr.attr);
7ff93f8b 3042
872bf2fb 3043 err = device_create_file(&dev->persist->pdev->dev, &info->port_attr);
7ff93f8b
YP		 3044 if (err) {
3045 mlx4_err(dev, "Failed to create file for port %d\n", port);
09d4d087 3046 devlink_port_unregister(&info->devlink_port);
7ff93f8b 3047 info->port = -1;
57f6f99f 3048 return err;
7ff93f8b
YP		 3049 }
3050
096335b3
OG		 3051 sprintf(info->dev_mtu_name, "mlx4_port%d_mtu", port);
3052 info->port_mtu_attr.attr.name = info->dev_mtu_name;
d3757ba4
JP		 3053 if (mlx4_is_mfunc(dev)) {
3054 info->port_mtu_attr.attr.mode = 0444;
3055 } else {
3056 info->port_mtu_attr.attr.mode = 0644;
096335b3
OG		 3057 info->port_mtu_attr.store = set_port_ib_mtu;
3058 }
3059 info->port_mtu_attr.show = show_port_ib_mtu;
3060 sysfs_attr_init(&info->port_mtu_attr.attr);
3061
872bf2fb
YH		 3062 err = device_create_file(&dev->persist->pdev->dev,
3063 &info->port_mtu_attr);
096335b3
OG		 3064 if (err) {
3065 mlx4_err(dev, "Failed to create mtu file for port %d\n", port);
872bf2fb
YH		 3066 device_remove_file(&info->dev->persist->pdev->dev,
3067 &info->port_attr);
fba12966 3068 devlink_port_unregister(&info->devlink_port);
096335b3 3069 info->port = -1;
57f6f99f 3070 return err;
096335b3
OG		 3071 }
3072
57f6f99f 3073 return 0;
7ff93f8b
YP		 3074}
3075
3076static void mlx4_cleanup_port_info(struct mlx4_port_info *info)
3077{
3078 if (info->port < 0)
3079 return;
3080
872bf2fb
YH		 3081 device_remove_file(&info->dev->persist->pdev->dev, &info->port_attr);
3082 device_remove_file(&info->dev->persist->pdev->dev,
3083 &info->port_mtu_attr);
fba12966
KH		 3084 devlink_port_unregister(&info->devlink_port);
3085
c66fa19c
MB		 3086#ifdef CONFIG_RFS_ACCEL
3087 free_irq_cpu_rmap(info->rmap);
3088 info->rmap = NULL;
3089#endif
2a2336f8
YP		 3090}
3091
b12d93d6
YP		 3092static int mlx4_init_steering(struct mlx4_dev *dev)
3093{
3094 struct mlx4_priv *priv = mlx4_priv(dev);
3095 int num_entries = dev->caps.num_ports;
3096 int i, j;
3097
6396bb22
KC		 3098 priv->steer = kcalloc(num_entries, sizeof(struct mlx4_steer),
3099 GFP_KERNEL);
b12d93d6
YP		 3100 if (!priv->steer)
3101 return -ENOMEM;
3102
45b51365 3103 for (i = 0; i < num_entries; i++)
b12d93d6
YP		 3104 for (j = 0; j < MLX4_NUM_STEERS; j++) {
3105 INIT_LIST_HEAD(&priv->steer[i].promisc_qps[j]);
3106 INIT_LIST_HEAD(&priv->steer[i].steer_entries[j]);
3107 }
b12d93d6
YP		 3108 return 0;
3109}
3110
3111static void mlx4_clear_steering(struct mlx4_dev *dev)
3112{
3113 struct mlx4_priv *priv = mlx4_priv(dev);
3114 struct mlx4_steer_index *entry, *tmp_entry;
3115 struct mlx4_promisc_qp *pqp, *tmp_pqp;
3116 int num_entries = dev->caps.num_ports;
3117 int i, j;
3118
3119 for (i = 0; i < num_entries; i++) {
3120 for (j = 0; j < MLX4_NUM_STEERS; j++) {
3121 list_for_each_entry_safe(pqp, tmp_pqp,
3122 &priv->steer[i].promisc_qps[j],
3123 list) {
3124 list_del(&pqp->list);
3125 kfree(pqp);
3126 }
3127 list_for_each_entry_safe(entry, tmp_entry,
3128 &priv->steer[i].steer_entries[j],
3129 list) {
3130 list_del(&entry->list);
3131 list_for_each_entry_safe(pqp, tmp_pqp,
3132 &entry->duplicates,
3133 list) {
3134 list_del(&pqp->list);
3135 kfree(pqp);
3136 }
3137 kfree(entry);
3138 }
3139 }
3140 }
3141 kfree(priv->steer);
3142}
3143
ab9c17a0
JM		 3144static int extended_func_num(struct pci_dev *pdev)
3145{
3146 return PCI_SLOT(pdev->devfn) * 8 + PCI_FUNC(pdev->devfn);
3147}
3148
3149#define MLX4_OWNER_BASE 0x8069c
3150#define MLX4_OWNER_SIZE 4
3151
3152static int mlx4_get_ownership(struct mlx4_dev *dev)
3153{
3154 void __iomem *owner;
3155 u32 ret;
3156
872bf2fb 3157 if (pci_channel_offline(dev->persist->pdev))
57dbf29a
KSS		 3158 return -EIO;
3159
872bf2fb
YH		 3160 owner = ioremap(pci_resource_start(dev->persist->pdev, 0) +
3161 MLX4_OWNER_BASE,
ab9c17a0
JM		 3162 MLX4_OWNER_SIZE);
3163 if (!owner) {
3164 mlx4_err(dev, "Failed to obtain ownership bit\n");
3165 return -ENOMEM;
3166 }
3167
3168 ret = readl(owner);
3169 iounmap(owner);
3170 return (int) !!ret;
3171}
3172
3173static void mlx4_free_ownership(struct mlx4_dev *dev)
3174{
3175 void __iomem *owner;
3176
872bf2fb 3177 if (pci_channel_offline(dev->persist->pdev))
57dbf29a
KSS		 3178 return;
3179
872bf2fb
YH		 3180 owner = ioremap(pci_resource_start(dev->persist->pdev, 0) +
3181 MLX4_OWNER_BASE,
ab9c17a0
JM		 3182 MLX4_OWNER_SIZE);
3183 if (!owner) {
3184 mlx4_err(dev, "Failed to obtain ownership bit\n");
3185 return;
3186 }
3187 writel(0, owner);
3188 msleep(1000);
3189 iounmap(owner);
3190}
3191
a0eacca9
MB		 3192#define SRIOV_VALID_STATE(flags) (!!((flags) & MLX4_FLAG_SRIOV) ==\
3193 !!((flags) & MLX4_FLAG_MASTER))
3194
3195static u64 mlx4_enable_sriov(struct mlx4_dev *dev, struct pci_dev *pdev,
55ad3592 3196 u8 total_vfs, int existing_vfs, int reset_flow)
a0eacca9
MB		 3197{
3198 u64 dev_flags = dev->flags;
da315679 3199 int err = 0;
0beb44b0
CS		 3200 int fw_enabled_sriov_vfs = min(pci_sriov_get_totalvfs(pdev),
3201 MLX4_MAX_NUM_VF);
a0eacca9 3202
55ad3592
YH		 3203 if (reset_flow) {
3204 dev->dev_vfs = kcalloc(total_vfs, sizeof(*dev->dev_vfs),
3205 GFP_KERNEL);
3206 if (!dev->dev_vfs)
3207 goto free_mem;
3208 return dev_flags;
3209 }
3210
da315679
MB		 3211 atomic_inc(&pf_loading);
3212 if (dev->flags & MLX4_FLAG_SRIOV) {
3213 if (existing_vfs != total_vfs) {
3214 mlx4_err(dev, "SR-IOV was already enabled, but with num_vfs (%d) different than requested (%d)\n",
3215 existing_vfs, total_vfs);
3216 total_vfs = existing_vfs;
3217 }
3218 }
3219
6396bb22 3220 dev->dev_vfs = kcalloc(total_vfs, sizeof(*dev->dev_vfs), GFP_KERNEL);
a0eacca9
MB		 3221 if (NULL == dev->dev_vfs) {
3222 mlx4_err(dev, "Failed to allocate memory for VFs\n");
3223 goto disable_sriov;
da315679
MB		 3224 }
3225
3226 if (!(dev->flags & MLX4_FLAG_SRIOV)) {
0beb44b0
CS		 3227 if (total_vfs > fw_enabled_sriov_vfs) {
3228 mlx4_err(dev, "requested vfs (%d) > available vfs (%d). Continuing without SR_IOV\n",
3229 total_vfs, fw_enabled_sriov_vfs);
3230 err = -ENOMEM;
3231 goto disable_sriov;
3232 }
da315679
MB		 3233 mlx4_warn(dev, "Enabling SR-IOV with %d VFs\n", total_vfs);
3234 err = pci_enable_sriov(pdev, total_vfs);
3235 }
3236 if (err) {
3237 mlx4_err(dev, "Failed to enable SR-IOV, continuing without SR-IOV (err = %d)\n",
3238 err);
3239 goto disable_sriov;
3240 } else {
3241 mlx4_warn(dev, "Running in master mode\n");
3242 dev_flags |= MLX4_FLAG_SRIOV |
3243 MLX4_FLAG_MASTER;
3244 dev_flags &= ~MLX4_FLAG_SLAVE;
872bf2fb 3245 dev->persist->num_vfs = total_vfs;
a0eacca9
MB		 3246 }
3247 return dev_flags;
3248
3249disable_sriov:
da315679 3250 atomic_dec(&pf_loading);
55ad3592 3251free_mem:
872bf2fb 3252 dev->persist->num_vfs = 0;
a0eacca9 3253 kfree(dev->dev_vfs);
92a59ad0 3254 dev->dev_vfs = NULL;
a0eacca9
MB		 3255 return dev_flags & ~MLX4_FLAG_MASTER;
3256}
3257
de966c59
MB		 3258enum {
3259 MLX4_DEV_CAP_CHECK_NUM_VFS_ABOVE_64 = -1,
3260};
3261
3262static int mlx4_check_dev_cap(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap,
3263 int *nvfs)
3264{
3265 int requested_vfs = nvfs[0] + nvfs[1] + nvfs[2];
3266 /* Checking for 64 VFs as a limitation of CX2 */
3267 if (!(dev_cap->flags2 & MLX4_DEV_CAP_FLAG2_80_VFS) &&
3268 requested_vfs >= 64) {
3269 mlx4_err(dev, "Requested %d VFs, but FW does not support more than 64\n",
3270 requested_vfs);
3271 return MLX4_DEV_CAP_CHECK_NUM_VFS_ABOVE_64;
3272 }
3273 return 0;
3274}
3275
4bfd2e6e
DJ		 3276static int mlx4_pci_enable_device(struct mlx4_dev *dev)
3277{
3278 struct pci_dev *pdev = dev->persist->pdev;
3279 int err = 0;
3280
3281 mutex_lock(&dev->persist->pci_status_mutex);
3282 if (dev->persist->pci_status == MLX4_PCI_STATUS_DISABLED) {
3283 err = pci_enable_device(pdev);
3284 if (!err)
3285 dev->persist->pci_status = MLX4_PCI_STATUS_ENABLED;
3286 }
3287 mutex_unlock(&dev->persist->pci_status_mutex);
3288
3289 return err;
3290}
3291
3292static void mlx4_pci_disable_device(struct mlx4_dev *dev)
3293{
3294 struct pci_dev *pdev = dev->persist->pdev;
3295
3296 mutex_lock(&dev->persist->pci_status_mutex);
3297 if (dev->persist->pci_status == MLX4_PCI_STATUS_ENABLED) {
3298 pci_disable_device(pdev);
3299 dev->persist->pci_status = MLX4_PCI_STATUS_DISABLED;
3300 }
3301 mutex_unlock(&dev->persist->pci_status_mutex);
3302}
3303
e1c00e10 3304static int mlx4_load_one(struct pci_dev *pdev, int pci_dev_data,
55ad3592
YH		 3305 int total_vfs, int *nvfs, struct mlx4_priv *priv,
3306 int reset_flow)
225c7b1f 3307{
225c7b1f 3308 struct mlx4_dev *dev;
e1c00e10 3309 unsigned sum = 0;
225c7b1f 3310 int err;
2a2336f8 3311 int port;
e1c00e10 3312 int i;
7ae0e400 3313 struct mlx4_dev_cap *dev_cap = NULL;
bbb07af4 3314 int existing_vfs = 0;
225c7b1f 3315
e1c00e10 3316 dev = &priv->dev;
225c7b1f 3317
b581401e
RD		 3318 INIT_LIST_HEAD(&priv->ctx_list);
3319 spin_lock_init(&priv->ctx_lock);
225c7b1f 3320
7ff93f8b 3321 mutex_init(&priv->port_mutex);
53f33ae2 3322 mutex_init(&priv->bond_mutex);
7ff93f8b 3323
6296883c
YP		 3324 INIT_LIST_HEAD(&priv->pgdir_list);
3325 mutex_init(&priv->pgdir_mutex);
0c5ddb51 3326 spin_lock_init(&priv->cmd.context_lock);
6296883c 3327
c1b43dca
EC		 3328 INIT_LIST_HEAD(&priv->bf_list);
3329 mutex_init(&priv->bf_mutex);
3330
aca7a3ac 3331 dev->rev_id = pdev->revision;
6e7136ed 3332 dev->numa_node = dev_to_node(&pdev->dev);
e1c00e10 3333
ab9c17a0 3334 /* Detect if this device is a virtual function */
839f1243 3335 if (pci_dev_data & MLX4_PCI_DEV_IS_VF) {
ab9c17a0
JM		 3336 mlx4_warn(dev, "Detected virtual function - running in slave mode\n");
3337 dev->flags |= MLX4_FLAG_SLAVE;
3338 } else {
3339 /* We reset the device and enable SRIOV only for physical
3340 * devices. Try to claim ownership on the device;
3341 * if already taken, skip -- do not allow multiple PFs */
3342 err = mlx4_get_ownership(dev);
3343 if (err) {
3344 if (err < 0)
e1c00e10 3345 return err;
ab9c17a0 3346 else {
1a91de28 3347 mlx4_warn(dev, "Multiple PFs not yet supported - Skipping PF\n");
e1c00e10 3348 return -EINVAL;
ab9c17a0
JM		 3349 }
3350 }
aca7a3ac 3351
fe6f700d
YP		 3352 atomic_set(&priv->opreq_count, 0);
3353 INIT_WORK(&priv->opreq_task, mlx4_opreq_action);
3354
ab9c17a0
JM		 3355 /*
3356 * Now reset the HCA before we touch the PCI capabilities or
3357 * attempt a firmware command, since a boot ROM may have left
3358 * the HCA in an undefined state.
3359 */
3360 err = mlx4_reset(dev);
3361 if (err) {
1a91de28 3362 mlx4_err(dev, "Failed to reset HCA, aborting\n");
e1c00e10 3363 goto err_sriov;
ab9c17a0 3364 }
7ae0e400
MB		 3365
3366 if (total_vfs) {
7ae0e400 3367 dev->flags = MLX4_FLAG_MASTER;
da315679
MB		 3368 existing_vfs = pci_num_vf(pdev);
3369 if (existing_vfs)
3370 dev->flags |= MLX4_FLAG_SRIOV;
872bf2fb 3371 dev->persist->num_vfs = total_vfs;
7ae0e400 3372 }
225c7b1f
RD		 3373 }
3374
f6bc11e4
YH		 3375 /* on load remove any previous indication of internal error,
3376 * device is up.
3377 */
3378 dev->persist->state = MLX4_DEVICE_STATE_UP;
3379
ab9c17a0 3380slave_start:
521130d1
EE		 3381 err = mlx4_cmd_init(dev);
3382 if (err) {
1a91de28 3383 mlx4_err(dev, "Failed to init command interface, aborting\n");
ab9c17a0
JM		 3384 goto err_sriov;
3385 }
3386
3387 /* In slave functions, the communication channel must be initialized
3388 * before posting commands. Also, init num_slaves before calling
3389 * mlx4_init_hca */
3390 if (mlx4_is_mfunc(dev)) {
7ae0e400 3391 if (mlx4_is_master(dev)) {
ab9c17a0 3392 dev->num_slaves = MLX4_MAX_NUM_SLAVES;
7ae0e400
MB		 3393
3394 } else {
ab9c17a0 3395 dev->num_slaves = 0;
f356fcbe
JM		 3396 err = mlx4_multi_func_init(dev);
3397 if (err) {
1a91de28 3398 mlx4_err(dev, "Failed to init slave mfunc interface, aborting\n");
ab9c17a0
JM		 3399 goto err_cmd;
3400 }
3401 }
225c7b1f
RD		 3402 }
3403
a0eacca9
MB		 3404 err = mlx4_init_fw(dev);
3405 if (err) {
3406 mlx4_err(dev, "Failed to init fw, aborting.\n");
3407 goto err_mfunc;
3408 }
3409
7ae0e400 3410 if (mlx4_is_master(dev)) {
da315679 3411 /* when we hit the goto slave_start below, dev_cap already initialized */
7ae0e400
MB		 3412 if (!dev_cap) {
3413 dev_cap = kzalloc(sizeof(*dev_cap), GFP_KERNEL);
3414
3415 if (!dev_cap) {
3416 err = -ENOMEM;
3417 goto err_fw;
3418 }
3419
3420 err = mlx4_QUERY_DEV_CAP(dev, dev_cap);
3421 if (err) {
3422 mlx4_err(dev, "QUERY_DEV_CAP command failed, aborting.\n");
3423 goto err_fw;
3424 }
3425
de966c59
MB		 3426 if (mlx4_check_dev_cap(dev, dev_cap, nvfs))
3427 goto err_fw;
3428
7ae0e400 3429 if (!(dev_cap->flags2 & MLX4_DEV_CAP_FLAG2_SYS_EQS)) {
55ad3592
YH		 3430 u64 dev_flags = mlx4_enable_sriov(dev, pdev,
3431 total_vfs,
3432 existing_vfs,
3433 reset_flow);
7ae0e400 3434
ed3d2276 3435 mlx4_close_fw(dev);
7ae0e400
MB		 3436 mlx4_cmd_cleanup(dev, MLX4_CMD_CLEANUP_ALL);
3437 dev->flags = dev_flags;
3438 if (!SRIOV_VALID_STATE(dev->flags)) {
3439 mlx4_err(dev, "Invalid SRIOV state\n");
3440 goto err_sriov;
3441 }
3442 err = mlx4_reset(dev);
3443 if (err) {
3444 mlx4_err(dev, "Failed to reset HCA, aborting.\n");
3445 goto err_sriov;
3446 }
3447 goto slave_start;
3448 }
3449 } else {
3450 /* Legacy mode FW requires SRIOV to be enabled before
3451 * doing QUERY_DEV_CAP, since max_eq's value is different if
3452 * SRIOV is enabled.
3453 */
3454 memset(dev_cap, 0, sizeof(*dev_cap));
3455 err = mlx4_QUERY_DEV_CAP(dev, dev_cap);
3456 if (err) {
3457 mlx4_err(dev, "QUERY_DEV_CAP command failed, aborting.\n");
3458 goto err_fw;
3459 }
de966c59
MB		 3460
3461 if (mlx4_check_dev_cap(dev, dev_cap, nvfs))
3462 goto err_fw;
7ae0e400
MB		 3463 }
3464 }
3465
225c7b1f 3466 err = mlx4_init_hca(dev);
ab9c17a0
JM		 3467 if (err) {
3468 if (err == -EACCES) {
3469 /* Not primary Physical function
3470 * Running in slave mode */
ffc39f6d 3471 mlx4_cmd_cleanup(dev, MLX4_CMD_CLEANUP_ALL);
a0eacca9
MB		 3472 /* We're not a PF */
3473 if (dev->flags & MLX4_FLAG_SRIOV) {
3474 if (!existing_vfs)
3475 pci_disable_sriov(pdev);
55ad3592 3476 if (mlx4_is_master(dev) && !reset_flow)
a0eacca9
MB		 3477 atomic_dec(&pf_loading);
3478 dev->flags &= ~MLX4_FLAG_SRIOV;
3479 }
3480 if (!mlx4_is_slave(dev))
3481 mlx4_free_ownership(dev);
ab9c17a0
JM		 3482 dev->flags |= MLX4_FLAG_SLAVE;
3483 dev->flags &= ~MLX4_FLAG_MASTER;
3484 goto slave_start;
3485 } else
a0eacca9 3486 goto err_fw;
ab9c17a0
JM		 3487 }
3488
7ae0e400 3489 if (mlx4_is_master(dev) && (dev_cap->flags2 & MLX4_DEV_CAP_FLAG2_SYS_EQS)) {
55ad3592
YH		 3490 u64 dev_flags = mlx4_enable_sriov(dev, pdev, total_vfs,
3491 existing_vfs, reset_flow);
7ae0e400
MB		 3492
3493 if ((dev->flags ^ dev_flags) & (MLX4_FLAG_MASTER | MLX4_FLAG_SLAVE)) {
3494 mlx4_cmd_cleanup(dev, MLX4_CMD_CLEANUP_VHCR);
3495 dev->flags = dev_flags;
3496 err = mlx4_cmd_init(dev);
3497 if (err) {
3498 /* Only VHCR is cleaned up, so could still
3499 * send FW commands
3500 */
3501 mlx4_err(dev, "Failed to init VHCR command interface, aborting\n");
3502 goto err_close;
3503 }
3504 } else {
3505 dev->flags = dev_flags;
3506 }
3507
3508 if (!SRIOV_VALID_STATE(dev->flags)) {
3509 mlx4_err(dev, "Invalid SRIOV state\n");
3510 goto err_close;
3511 }
3512 }
3513
b912b2f8
EP		 3514 /* check if the device is functioning at its maximum possible speed.
3515 * No return code for this call, just warn the user in case of PCI
3516 * express device capabilities are under-satisfied by the bus.
3517 */
83d3459a 3518 if (!mlx4_is_slave(dev))
190b509c 3519 pcie_print_link_status(dev->persist->pdev);
b912b2f8 3520
ab9c17a0
JM		 3521 /* In master functions, the communication channel must be initialized
3522 * after obtaining its address from fw */
3523 if (mlx4_is_master(dev)) {
e1c00e10
MD		 3524 if (dev->caps.num_ports < 2 &&
3525 num_vfs_argc > 1) {
3526 err = -EINVAL;
3527 mlx4_err(dev,
3528 "Error: Trying to configure VFs on port 2, but HCA has only %d physical ports\n",
3529 dev->caps.num_ports);
ab9c17a0
JM		 3530 goto err_close;
3531 }
872bf2fb 3532 memcpy(dev->persist->nvfs, nvfs, sizeof(dev->persist->nvfs));
dd41cc3b 3533
872bf2fb
YH		 3534 for (i = 0;
3535 i < sizeof(dev->persist->nvfs)/
3536 sizeof(dev->persist->nvfs[0]); i++) {
e1c00e10
MD		 3537 unsigned j;
3538
872bf2fb 3539 for (j = 0; j < dev->persist->nvfs[i]; ++sum, ++j) {
e1c00e10
MD		 3540 dev->dev_vfs[sum].min_port = i < 2 ? i + 1 : 1;
3541 dev->dev_vfs[sum].n_ports = i < 2 ? 1 :
3542 dev->caps.num_ports;
1ab95d37
MB		 3543 }
3544 }
e1c00e10
MD		 3545
3546 /* In master functions, the communication channel
3547 * must be initialized after obtaining its address from fw
3548 */
3549 err = mlx4_multi_func_init(dev);
3550 if (err) {
3551 mlx4_err(dev, "Failed to init master mfunc interface, aborting.\n");
3552 goto err_close;
3553 }
ab9c17a0 3554 }
225c7b1f 3555
b8dd786f
YP		 3556 err = mlx4_alloc_eq_table(dev);
3557 if (err)
ab9c17a0 3558 goto err_master_mfunc;
b8dd786f 3559
c66fa19c 3560 bitmap_zero(priv->msix_ctl.pool_bm, MAX_MSIX);
730c41d5 3561 mutex_init(&priv->msix_ctl.pool_lock);
0b7ca5a9 3562
08fb1055 3563 mlx4_enable_msi_x(dev);
ab9c17a0
JM		 3564 if ((mlx4_is_mfunc(dev)) &&
3565 !(dev->flags & MLX4_FLAG_MSI_X)) {
72b8eaab 3566 err = -EOPNOTSUPP;
1a91de28 3567 mlx4_err(dev, "INTx is not supported in multi-function mode, aborting\n");
b12d93d6 3568 goto err_free_eq;
ab9c17a0
JM		 3569 }
3570
3571 if (!mlx4_is_slave(dev)) {
3572 err = mlx4_init_steering(dev);
3573 if (err)
e1c00e10 3574 goto err_disable_msix;
ab9c17a0 3575 }
b12d93d6 3576
6ed63d84
JM		 3577 mlx4_init_quotas(dev);
3578
225c7b1f 3579 err = mlx4_setup_hca(dev);
ab9c17a0
JM		 3580 if (err == -EBUSY && (dev->flags & MLX4_FLAG_MSI_X) &&
3581 !mlx4_is_mfunc(dev)) {
08fb1055 3582 dev->flags &= ~MLX4_FLAG_MSI_X;
9858d2d1 3583 dev->caps.num_comp_vectors = 1;
08fb1055
MT		 3584 pci_disable_msix(pdev);
3585 err = mlx4_setup_hca(dev);
3586 }
3587
225c7b1f 3588 if (err)
b12d93d6 3589 goto err_steer;
225c7b1f 3590
55ad3592
YH		 3591 /* When PF resources are ready arm its comm channel to enable
3592 * getting commands
3593 */
3594 if (mlx4_is_master(dev)) {
3595 err = mlx4_ARM_COMM_CHANNEL(dev);
3596 if (err) {
3597 mlx4_err(dev, " Failed to arm comm channel eq: %x\n",
3598 err);
3599 goto err_steer;
3600 }
3601 }
5a0d0a61 3602
7ff93f8b
YP		 3603 for (port = 1; port <= dev->caps.num_ports; port++) {
3604 err = mlx4_init_port_info(dev, port);
3605 if (err)
3606 goto err_port;
3607 }
2a2336f8 3608
53f33ae2
MS		 3609 priv->v2p.port1 = 1;
3610 priv->v2p.port2 = 2;
3611
225c7b1f
RD		 3612 err = mlx4_register_device(dev);
3613 if (err)
7ff93f8b 3614 goto err_port;
225c7b1f 3615
b046ffe5
EP		 3616 mlx4_request_modules(dev);
3617
27bf91d6
YP		 3618 mlx4_sense_init(dev);
3619 mlx4_start_sense(dev);
3620
befdf897 3621 priv->removed = 0;
225c7b1f 3622
55ad3592 3623 if (mlx4_is_master(dev) && dev->persist->num_vfs && !reset_flow)
e1a5ddc5
AV		 3624 atomic_dec(&pf_loading);
3625
da315679 3626 kfree(dev_cap);
225c7b1f
RD		 3627 return 0;
3628
7ff93f8b 3629err_port:
b4f77264 3630 for (--port; port >= 1; --port)
7ff93f8b
YP		 3631 mlx4_cleanup_port_info(&priv->port[port]);
3632
6de5f7f6 3633 mlx4_cleanup_default_counters(dev);
2632d18d
EBE		 3634 if (!mlx4_is_slave(dev))
3635 mlx4_cleanup_counters_table(dev);
225c7b1f
RD		 3636 mlx4_cleanup_qp_table(dev);
3637 mlx4_cleanup_srq_table(dev);
3638 mlx4_cleanup_cq_table(dev);
3639 mlx4_cmd_use_polling(dev);
3640 mlx4_cleanup_eq_table(dev);
fe6f700d 3641 mlx4_cleanup_mcg_table(dev);
225c7b1f 3642 mlx4_cleanup_mr_table(dev);
012a8ff5 3643 mlx4_cleanup_xrcd_table(dev);
225c7b1f
RD		 3644 mlx4_cleanup_pd_table(dev);
3645 mlx4_cleanup_uar_table(dev);
3646
b12d93d6 3647err_steer:
ab9c17a0
JM		 3648 if (!mlx4_is_slave(dev))
3649 mlx4_clear_steering(dev);
b12d93d6 3650
e1c00e10
MD		 3651err_disable_msix:
3652 if (dev->flags & MLX4_FLAG_MSI_X)
3653 pci_disable_msix(pdev);
3654
b8dd786f
YP		 3655err_free_eq:
3656 mlx4_free_eq_table(dev);
3657
ab9c17a0 3658err_master_mfunc:
772103e6
JM		 3659 if (mlx4_is_master(dev)) {
3660 mlx4_free_resource_tracker(dev, RES_TR_FREE_STRUCTS_ONLY);
ab9c17a0 3661 mlx4_multi_func_cleanup(dev);
772103e6 3662 }
ab9c17a0 3663
c73c8b1e
EBE		 3664 if (mlx4_is_slave(dev))
3665 mlx4_slave_destroy_special_qp_cap(dev);
b38f2879 3666
225c7b1f
RD		 3667err_close:
3668 mlx4_close_hca(dev);
3669
a0eacca9
MB		 3670err_fw:
3671 mlx4_close_fw(dev);
3672
ab9c17a0
JM		 3673err_mfunc:
3674 if (mlx4_is_slave(dev))
3675 mlx4_multi_func_cleanup(dev);
3676
225c7b1f 3677err_cmd:
ffc39f6d 3678 mlx4_cmd_cleanup(dev, MLX4_CMD_CLEANUP_ALL);
225c7b1f 3679
ab9c17a0 3680err_sriov:
55ad3592 3681 if (dev->flags & MLX4_FLAG_SRIOV && !existing_vfs) {
ab9c17a0 3682 pci_disable_sriov(pdev);
55ad3592
YH		 3683 dev->flags &= ~MLX4_FLAG_SRIOV;
3684 }
ab9c17a0 3685
55ad3592 3686 if (mlx4_is_master(dev) && dev->persist->num_vfs && !reset_flow)
e1a5ddc5
AV		 3687 atomic_dec(&pf_loading);
3688
1ab95d37
MB		 3689 kfree(priv->dev.dev_vfs);
3690
e1c00e10
MD		 3691 if (!mlx4_is_slave(dev))
3692 mlx4_free_ownership(dev);
3693
7ae0e400 3694 kfree(dev_cap);
e1c00e10
MD		 3695 return err;
3696}
3697
3698static int __mlx4_init_one(struct pci_dev *pdev, int pci_dev_data,
3699 struct mlx4_priv *priv)
3700{
3701 int err;
3702 int nvfs[MLX4_MAX_PORTS + 1] = {0, 0, 0};
3703 int prb_vf[MLX4_MAX_PORTS + 1] = {0, 0, 0};
3704 const int param_map[MLX4_MAX_PORTS + 1][MLX4_MAX_PORTS + 1] = {
3705 {2, 0, 0}, {0, 1, 2}, {0, 1, 2} };
3706 unsigned total_vfs = 0;
3707 unsigned int i;
3708
3709 pr_info(DRV_NAME ": Initializing %s\n", pci_name(pdev));
3710
4bfd2e6e 3711 err = mlx4_pci_enable_device(&priv->dev);
e1c00e10
MD		 3712 if (err) {
3713 dev_err(&pdev->dev, "Cannot enable PCI device, aborting\n");
3714 return err;
3715 }
3716
3717 /* Due to requirement that all VFs and the PF are *guaranteed* 2 MACS
3718 * per port, we must limit the number of VFs to 63 (since their are
3719 * 128 MACs)
3720 */
691223ec 3721 for (i = 0; i < ARRAY_SIZE(nvfs) && i < num_vfs_argc;
e1c00e10
MD		 3722 total_vfs += nvfs[param_map[num_vfs_argc - 1][i]], i++) {
3723 nvfs[param_map[num_vfs_argc - 1][i]] = num_vfs[i];
3724 if (nvfs[i] < 0) {
3725 dev_err(&pdev->dev, "num_vfs module parameter cannot be negative\n");
3726 err = -EINVAL;
3727 goto err_disable_pdev;
3728 }
3729 }
691223ec 3730 for (i = 0; i < ARRAY_SIZE(prb_vf) && i < probe_vfs_argc;
e1c00e10
MD		 3731 i++) {
3732 prb_vf[param_map[probe_vfs_argc - 1][i]] = probe_vf[i];
3733 if (prb_vf[i] < 0 || prb_vf[i] > nvfs[i]) {
3734 dev_err(&pdev->dev, "probe_vf module parameter cannot be negative or greater than num_vfs\n");
3735 err = -EINVAL;
3736 goto err_disable_pdev;
3737 }
3738 }
0beb44b0 3739 if (total_vfs > MLX4_MAX_NUM_VF) {
e1c00e10 3740 dev_err(&pdev->dev,
0beb44b0
CS		 3741 "Requested more VF's (%d) than allowed by hw (%d)\n",
3742 total_vfs, MLX4_MAX_NUM_VF);
e1c00e10
MD		 3743 err = -EINVAL;
3744 goto err_disable_pdev;
3745 }
3746
3747 for (i = 0; i < MLX4_MAX_PORTS; i++) {
0beb44b0 3748 if (nvfs[i] + nvfs[2] > MLX4_MAX_NUM_VF_P_PORT) {
e1c00e10 3749 dev_err(&pdev->dev,
0beb44b0 3750 "Requested more VF's (%d) for port (%d) than allowed by driver (%d)\n",
e1c00e10 3751 nvfs[i] + nvfs[2], i + 1,
0beb44b0 3752 MLX4_MAX_NUM_VF_P_PORT);
e1c00e10
MD		 3753 err = -EINVAL;
3754 goto err_disable_pdev;
3755 }
3756 }
3757
3758 /* Check for BARs. */
3759 if (!(pci_dev_data & MLX4_PCI_DEV_IS_VF) &&
3760 !(pci_resource_flags(pdev, 0) & IORESOURCE_MEM)) {
3761 dev_err(&pdev->dev, "Missing DCS, aborting (driver_data: 0x%x, pci_resource_flags(pdev, 0):0x%lx)\n",
3762 pci_dev_data, pci_resource_flags(pdev, 0));
3763 err = -ENODEV;
3764 goto err_disable_pdev;
3765 }
3766 if (!(pci_resource_flags(pdev, 2) & IORESOURCE_MEM)) {
3767 dev_err(&pdev->dev, "Missing UAR, aborting\n");
3768 err = -ENODEV;
3769 goto err_disable_pdev;
3770 }
3771
3772 err = pci_request_regions(pdev, DRV_NAME);
3773 if (err) {
3774 dev_err(&pdev->dev, "Couldn't get PCI resources, aborting\n");
3775 goto err_disable_pdev;
3776 }
3777
3778 pci_set_master(pdev);
3779
3780 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
3781 if (err) {
3782 dev_warn(&pdev->dev, "Warning: couldn't set 64-bit PCI DMA mask\n");
3783 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
3784 if (err) {
3785 dev_err(&pdev->dev, "Can't set PCI DMA mask, aborting\n");
3786 goto err_release_regions;
3787 }
3788 }
3789 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
3790 if (err) {
3791 dev_warn(&pdev->dev, "Warning: couldn't set 64-bit consistent PCI DMA mask\n");
3792 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
3793 if (err) {
3794 dev_err(&pdev->dev, "Can't set consistent PCI DMA mask, aborting\n");
3795 goto err_release_regions;
3796 }
3797 }
3798
3799 /* Allow large DMA segments, up to the firmware limit of 1 GB */
3800 dma_set_max_seg_size(&pdev->dev, 1024 * 1024 * 1024);
3801 /* Detect if this device is a virtual function */
3802 if (pci_dev_data & MLX4_PCI_DEV_IS_VF) {
3803 /* When acting as pf, we normally skip vfs unless explicitly
3804 * requested to probe them.
3805 */
3806 if (total_vfs) {
3807 unsigned vfs_offset = 0;
3808
691223ec 3809 for (i = 0; i < ARRAY_SIZE(nvfs) &&
e1c00e10
MD		 3810 vfs_offset + nvfs[i] < extended_func_num(pdev);
3811 vfs_offset += nvfs[i], i++)
3812 ;
691223ec 3813 if (i == ARRAY_SIZE(nvfs)) {
e1c00e10
MD		 3814 err = -ENODEV;
3815 goto err_release_regions;
3816 }
3817 if ((extended_func_num(pdev) - vfs_offset)
3818 > prb_vf[i]) {
3819 dev_warn(&pdev->dev, "Skipping virtual function:%d\n",
3820 extended_func_num(pdev));
3821 err = -ENODEV;
3822 goto err_release_regions;
3823 }
3824 }
3825 }
3826
bedc989b 3827 err = mlx4_crdump_init(&priv->dev);
e1c00e10
MD		 3828 if (err)
3829 goto err_release_regions;
ad9a0bf0 3830
bedc989b
AV		 3831 err = mlx4_catas_init(&priv->dev);
3832 if (err)
3833 goto err_crdump;
3834
55ad3592 3835 err = mlx4_load_one(pdev, pci_dev_data, total_vfs, nvfs, priv, 0);
ad9a0bf0
YH		 3836 if (err)
3837 goto err_catas;
3838
e1c00e10 3839 return 0;
225c7b1f 3840
ad9a0bf0
YH		 3841err_catas:
3842 mlx4_catas_end(&priv->dev);
3843
bedc989b
AV		 3844err_crdump:
3845 mlx4_crdump_end(&priv->dev);
3846
a01df0fe
RD		 3847err_release_regions:
3848 pci_release_regions(pdev);
225c7b1f
RD		 3849
3850err_disable_pdev:
4bfd2e6e 3851 mlx4_pci_disable_device(&priv->dev);
225c7b1f
RD		 3852 return err;
3853}
3854
b2facd95
JP		 3855static int mlx4_devlink_port_type_set(struct devlink_port *devlink_port,
3856 enum devlink_port_type port_type)
3857{
3858 struct mlx4_port_info *info = container_of(devlink_port,
3859 struct mlx4_port_info,
3860 devlink_port);
3861 enum mlx4_port_type mlx4_port_type;
3862
3863 switch (port_type) {
3864 case DEVLINK_PORT_TYPE_AUTO:
3865 mlx4_port_type = MLX4_PORT_TYPE_AUTO;
3866 break;
3867 case DEVLINK_PORT_TYPE_ETH:
3868 mlx4_port_type = MLX4_PORT_TYPE_ETH;
3869 break;
3870 case DEVLINK_PORT_TYPE_IB:
3871 mlx4_port_type = MLX4_PORT_TYPE_IB;
3872 break;
3873 default:
3874 return -EOPNOTSUPP;
3875 }
3876
3877 return __set_port_type(info, mlx4_port_type);
3878}
3879
dfb3c082
MS		 3880static void mlx4_devlink_param_load_driverinit_values(struct devlink *devlink)
3881{
3c641ba4
AV		 3882 struct mlx4_priv *priv = devlink_priv(devlink);
3883 struct mlx4_dev *dev = &priv->dev;
3884 struct mlx4_fw_crdump *crdump = &dev->persist->crdump;
dfb3c082
MS		 3885 union devlink_param_value saved_value;
3886 int err;
3887
3888 err = devlink_param_driverinit_value_get(devlink,
3889 DEVLINK_PARAM_GENERIC_ID_INT_ERR_RESET,
3890 &saved_value);
3891 if (!err && mlx4_internal_err_reset != saved_value.vbool) {
3892 mlx4_internal_err_reset = saved_value.vbool;
3893 /* Notify on value changed on runtime configuration mode */
3894 devlink_param_value_changed(devlink,
3895 DEVLINK_PARAM_GENERIC_ID_INT_ERR_RESET);
3896 }
3897 err = devlink_param_driverinit_value_get(devlink,
3898 DEVLINK_PARAM_GENERIC_ID_MAX_MACS,
3899 &saved_value);
3900 if (!err)
3901 log_num_mac = order_base_2(saved_value.vu32);
3902 err = devlink_param_driverinit_value_get(devlink,
3903 MLX4_DEVLINK_PARAM_ID_ENABLE_64B_CQE_EQE,
3904 &saved_value);
3905 if (!err)
3906 enable_64b_cqe_eqe = saved_value.vbool;
3907 err = devlink_param_driverinit_value_get(devlink,
3908 MLX4_DEVLINK_PARAM_ID_ENABLE_4K_UAR,
3909 &saved_value);
3910 if (!err)
3911 enable_4k_uar = saved_value.vbool;
3c641ba4
AV		 3912 err = devlink_param_driverinit_value_get(devlink,
3913 DEVLINK_PARAM_GENERIC_ID_REGION_SNAPSHOT,
3914 &saved_value);
3915 if (!err && crdump->snapshot_enable != saved_value.vbool) {
3916 crdump->snapshot_enable = saved_value.vbool;
3917 devlink_param_value_changed(devlink,
3918 DEVLINK_PARAM_GENERIC_ID_REGION_SNAPSHOT);
3919 }
dfb3c082
MS		 3920}
3921
3922static int mlx4_devlink_reload(struct devlink *devlink,
3923 struct netlink_ext_ack *extack)
3924{
3925 struct mlx4_priv *priv = devlink_priv(devlink);
3926 struct mlx4_dev *dev = &priv->dev;
3927 struct mlx4_dev_persistent *persist = dev->persist;
3928 int err;
3929
3930 if (persist->num_vfs)
3931 mlx4_warn(persist->dev, "Reload performed on PF, will cause reset on operating Virtual Functions\n");
3932 err = mlx4_restart_one(persist->pdev, true, devlink);
3933 if (err)
3934 mlx4_err(persist->dev, "mlx4_restart_one failed, ret=%d\n", err);
3935
3936 return err;
3937}
3938
b2facd95
JP		 3939static const struct devlink_ops mlx4_devlink_ops = {
3940 .port_type_set = mlx4_devlink_port_type_set,
dfb3c082 3941 .reload = mlx4_devlink_reload,
b2facd95
JP		 3942};
3943
1dd06ae8 3944static int mlx4_init_one(struct pci_dev *pdev, const struct pci_device_id *id)
3d73c288 3945{
09d4d087 3946 struct devlink *devlink;
befdf897
WY		 3947 struct mlx4_priv *priv;
3948 struct mlx4_dev *dev;
e1c00e10 3949 int ret;
befdf897 3950
0a645e80 3951 printk_once(KERN_INFO "%s", mlx4_version);
3d73c288 3952
b2facd95 3953 devlink = devlink_alloc(&mlx4_devlink_ops, sizeof(*priv));
09d4d087 3954 if (!devlink)
befdf897 3955 return -ENOMEM;
09d4d087 3956 priv = devlink_priv(devlink);
befdf897
WY		 3957
3958 dev = &priv->dev;
872bf2fb
YH		 3959 dev->persist = kzalloc(sizeof(*dev->persist), GFP_KERNEL);
3960 if (!dev->persist) {
09d4d087
JP		 3961 ret = -ENOMEM;
3962 goto err_devlink_free;
872bf2fb
YH		 3963 }
3964 dev->persist->pdev = pdev;
3965 dev->persist->dev = dev;
3966 pci_set_drvdata(pdev, dev->persist);
befdf897 3967 priv->pci_dev_data = id->driver_data;
f6bc11e4 3968 mutex_init(&dev->persist->device_state_mutex);
c69453e2 3969 mutex_init(&dev->persist->interface_state_mutex);
4bfd2e6e 3970 mutex_init(&dev->persist->pci_status_mutex);
befdf897 3971
09d4d087
JP		 3972 ret = devlink_register(devlink, &pdev->dev);
3973 if (ret)
3974 goto err_persist_free;
bd1b51dc
MS		 3975 ret = devlink_params_register(devlink, mlx4_devlink_params,
3976 ARRAY_SIZE(mlx4_devlink_params));
09d4d087
JP		 3977 if (ret)
3978 goto err_devlink_unregister;
bd1b51dc
MS		 3979 mlx4_devlink_set_params_init_values(devlink);
3980 ret = __mlx4_init_one(pdev, id->driver_data, priv);
3981 if (ret)
3982 goto err_params_unregister;
2ba5fbd6 3983
7c62cfb8 3984 devlink_params_publish(devlink);
c1bdb295 3985 devlink_reload_enable(devlink);
09d4d087
JP		 3986 pci_save_state(pdev);
3987 return 0;
3988
bd1b51dc
MS		 3989err_params_unregister:
3990 devlink_params_unregister(devlink, mlx4_devlink_params,
3991 ARRAY_SIZE(mlx4_devlink_params));
09d4d087
JP		 3992err_devlink_unregister:
3993 devlink_unregister(devlink);
3994err_persist_free:
3995 kfree(dev->persist);
3996err_devlink_free:
3997 devlink_free(devlink);
e1c00e10 3998 return ret;
3d73c288
RD		 3999}
4000
dd0eefe3
YH		 4001static void mlx4_clean_dev(struct mlx4_dev *dev)
4002{
4003 struct mlx4_dev_persistent *persist = dev->persist;
4004 struct mlx4_priv *priv = mlx4_priv(dev);
55ad3592 4005 unsigned long flags = (dev->flags & RESET_PERSIST_MASK_FLAGS);
dd0eefe3
YH		 4006
4007 memset(priv, 0, sizeof(*priv));
4008 priv->dev.persist = persist;
55ad3592 4009 priv->dev.flags = flags;
dd0eefe3
YH		 4010}
4011
e1c00e10 4012static void mlx4_unload_one(struct pci_dev *pdev)
225c7b1f 4013{
872bf2fb
YH		 4014 struct mlx4_dev_persistent *persist = pci_get_drvdata(pdev);
4015 struct mlx4_dev *dev = persist->dev;
225c7b1f 4016 struct mlx4_priv *priv = mlx4_priv(dev);
befdf897 4017 int pci_dev_data;
dd0eefe3 4018 int p, i;
225c7b1f 4019
befdf897
WY		 4020 if (priv->removed)
4021 return;
225c7b1f 4022
dd0eefe3
YH		 4023 /* saving current ports type for further use */
4024 for (i = 0; i < dev->caps.num_ports; i++) {
4025 dev->persist->curr_port_type[i] = dev->caps.port_type[i + 1];
4026 dev->persist->curr_port_poss_type[i] = dev->caps.
4027 possible_type[i + 1];
4028 }
4029
befdf897 4030 pci_dev_data = priv->pci_dev_data;
225c7b1f 4031
befdf897
WY		 4032 mlx4_stop_sense(dev);
4033 mlx4_unregister_device(dev);
225c7b1f 4034
befdf897
WY		 4035 for (p = 1; p <= dev->caps.num_ports; p++) {
4036 mlx4_cleanup_port_info(&priv->port[p]);
4037 mlx4_CLOSE_PORT(dev, p);
4038 }
4039
4040 if (mlx4_is_master(dev))
4041 mlx4_free_resource_tracker(dev,
4042 RES_TR_FREE_SLAVES_ONLY);
4043
6de5f7f6 4044 mlx4_cleanup_default_counters(dev);
2632d18d
EBE		 4045 if (!mlx4_is_slave(dev))
4046 mlx4_cleanup_counters_table(dev);
befdf897
WY		 4047 mlx4_cleanup_qp_table(dev);
4048 mlx4_cleanup_srq_table(dev);
4049 mlx4_cleanup_cq_table(dev);
4050 mlx4_cmd_use_polling(dev);
4051 mlx4_cleanup_eq_table(dev);
4052 mlx4_cleanup_mcg_table(dev);
4053 mlx4_cleanup_mr_table(dev);
4054 mlx4_cleanup_xrcd_table(dev);
4055 mlx4_cleanup_pd_table(dev);
225c7b1f 4056
befdf897
WY		 4057 if (mlx4_is_master(dev))
4058 mlx4_free_resource_tracker(dev,
4059 RES_TR_FREE_STRUCTS_ONLY);
47605df9 4060
befdf897
WY		 4061 iounmap(priv->kar);
4062 mlx4_uar_free(dev, &priv->driver_uar);
4063 mlx4_cleanup_uar_table(dev);
4064 if (!mlx4_is_slave(dev))
4065 mlx4_clear_steering(dev);
4066 mlx4_free_eq_table(dev);
4067 if (mlx4_is_master(dev))
4068 mlx4_multi_func_cleanup(dev);
4069 mlx4_close_hca(dev);
a0eacca9 4070 mlx4_close_fw(dev);
befdf897
WY		 4071 if (mlx4_is_slave(dev))
4072 mlx4_multi_func_cleanup(dev);
ffc39f6d 4073 mlx4_cmd_cleanup(dev, MLX4_CMD_CLEANUP_ALL);
47605df9 4074
befdf897
WY		 4075 if (dev->flags & MLX4_FLAG_MSI_X)
4076 pci_disable_msix(pdev);
befdf897
WY		 4077
4078 if (!mlx4_is_slave(dev))
4079 mlx4_free_ownership(dev);
4080
c73c8b1e 4081 mlx4_slave_destroy_special_qp_cap(dev);
befdf897
WY		 4082 kfree(dev->dev_vfs);
4083
dd0eefe3 4084 mlx4_clean_dev(dev);
befdf897
WY		 4085 priv->pci_dev_data = pci_dev_data;
4086 priv->removed = 1;
4087}
4088
4089static void mlx4_remove_one(struct pci_dev *pdev)
4090{
872bf2fb
YH		 4091 struct mlx4_dev_persistent *persist = pci_get_drvdata(pdev);
4092 struct mlx4_dev *dev = persist->dev;
befdf897 4093 struct mlx4_priv *priv = mlx4_priv(dev);
09d4d087 4094 struct devlink *devlink = priv_to_devlink(priv);
55ad3592 4095 int active_vfs = 0;
befdf897 4096
c1bdb295
JP		 4097 devlink_reload_disable(devlink);
4098
4cbe4dac
JM		 4099 if (mlx4_is_slave(dev))
4100 persist->interface_state |= MLX4_INTERFACE_STATE_NOWAIT;
4101
c69453e2
YH		 4102 mutex_lock(&persist->interface_state_mutex);
4103 persist->interface_state |= MLX4_INTERFACE_STATE_DELETION;
4104 mutex_unlock(&persist->interface_state_mutex);
4105
55ad3592
YH		 4106 /* Disabling SR-IOV is not allowed while there are active vf's */
4107 if (mlx4_is_master(dev) && dev->flags & MLX4_FLAG_SRIOV) {
4108 active_vfs = mlx4_how_many_lives_vf(dev);
4109 if (active_vfs) {
4110 pr_warn("Removing PF when there are active VF's !!\n");
4111 pr_warn("Will not disable SR-IOV.\n");
4112 }
4113 }
4114
c69453e2
YH		 4115 /* device marked to be under deletion running now without the lock
4116 * letting other tasks to be terminated
4117 */
4118 if (persist->interface_state & MLX4_INTERFACE_STATE_UP)
4119 mlx4_unload_one(pdev);
4120 else
4121 mlx4_info(dev, "%s: interface is down\n", __func__);
ad9a0bf0 4122 mlx4_catas_end(dev);
bedc989b 4123 mlx4_crdump_end(dev);
55ad3592
YH		 4124 if (dev->flags & MLX4_FLAG_SRIOV && !active_vfs) {
4125 mlx4_warn(dev, "Disabling SR-IOV\n");
4126 pci_disable_sriov(pdev);
4127 }
4128
e1c00e10 4129 pci_release_regions(pdev);
4bfd2e6e 4130 mlx4_pci_disable_device(dev);
bd1b51dc
MS		 4131 devlink_params_unregister(devlink, mlx4_devlink_params,
4132 ARRAY_SIZE(mlx4_devlink_params));
09d4d087 4133 devlink_unregister(devlink);
872bf2fb 4134 kfree(dev->persist);
09d4d087 4135 devlink_free(devlink);
225c7b1f
RD		 4136}
4137
dd0eefe3
YH		 4138static int restore_current_port_types(struct mlx4_dev *dev,
4139 enum mlx4_port_type *types,
4140 enum mlx4_port_type *poss_types)
4141{
4142 struct mlx4_priv *priv = mlx4_priv(dev);
4143 int err, i;
4144
4145 mlx4_stop_sense(dev);
4146
4147 mutex_lock(&priv->port_mutex);
4148 for (i = 0; i < dev->caps.num_ports; i++)
4149 dev->caps.possible_type[i + 1] = poss_types[i];
4150 err = mlx4_change_port_types(dev, types);
4151 mlx4_start_sense(dev);
4152 mutex_unlock(&priv->port_mutex);
4153
4154 return err;
4155}
4156
dfb3c082 4157int mlx4_restart_one(struct pci_dev *pdev, bool reload, struct devlink *devlink)
ee49bd93 4158{
872bf2fb
YH		 4159 struct mlx4_dev_persistent *persist = pci_get_drvdata(pdev);
4160 struct mlx4_dev *dev = persist->dev;
839f1243 4161 struct mlx4_priv *priv = mlx4_priv(dev);
e1c00e10
MD		 4162 int nvfs[MLX4_MAX_PORTS + 1] = {0, 0, 0};
4163 int pci_dev_data, err, total_vfs;
839f1243
RD		 4164
4165 pci_dev_data = priv->pci_dev_data;
872bf2fb
YH		 4166 total_vfs = dev->persist->num_vfs;
4167 memcpy(nvfs, dev->persist->nvfs, sizeof(dev->persist->nvfs));
e1c00e10
MD		 4168
4169 mlx4_unload_one(pdev);
dfb3c082
MS		 4170 if (reload)
4171 mlx4_devlink_param_load_driverinit_values(devlink);
55ad3592 4172 err = mlx4_load_one(pdev, pci_dev_data, total_vfs, nvfs, priv, 1);
e1c00e10
MD		 4173 if (err) {
4174 mlx4_err(dev, "%s: ERROR: mlx4_load_one failed, pci_name=%s, err=%d\n",
4175 __func__, pci_name(pdev), err);
4176 return err;
4177 }
4178
dd0eefe3
YH		 4179 err = restore_current_port_types(dev, dev->persist->curr_port_type,
4180 dev->persist->curr_port_poss_type);
4181 if (err)
4182 mlx4_err(dev, "could not restore original port types (%d)\n",
4183 err);
4184
e1c00e10 4185 return err;
ee49bd93
JM		 4186}
4187
c19e4b90
BH		 4188#define MLX_SP(id) { PCI_VDEVICE(MELLANOX, id), MLX4_PCI_DEV_FORCE_SENSE_PORT }
4189#define MLX_VF(id) { PCI_VDEVICE(MELLANOX, id), MLX4_PCI_DEV_IS_VF }
4190#define MLX_GN(id) { PCI_VDEVICE(MELLANOX, id), 0 }
4191
9baa3c34 4192static const struct pci_device_id mlx4_pci_table[] = {
a1b87145 4193#ifdef CONFIG_MLX4_CORE_GEN2
c19e4b90
BH		 4194 /* MT25408 "Hermon" */
4195 MLX_SP(PCI_DEVICE_ID_MELLANOX_HERMON_SDR), /* SDR */
4196 MLX_SP(PCI_DEVICE_ID_MELLANOX_HERMON_DDR), /* DDR */
4197 MLX_SP(PCI_DEVICE_ID_MELLANOX_HERMON_QDR), /* QDR */
4198 MLX_SP(PCI_DEVICE_ID_MELLANOX_HERMON_DDR_GEN2), /* DDR Gen2 */
4199 MLX_SP(PCI_DEVICE_ID_MELLANOX_HERMON_QDR_GEN2), /* QDR Gen2 */
4200 MLX_SP(PCI_DEVICE_ID_MELLANOX_HERMON_EN), /* EN 10GigE */
4201 MLX_SP(PCI_DEVICE_ID_MELLANOX_HERMON_EN_GEN2), /* EN 10GigE Gen2 */
4202 /* MT25458 ConnectX EN 10GBASE-T */
4203 MLX_SP(PCI_DEVICE_ID_MELLANOX_CONNECTX_EN),
4204 MLX_SP(PCI_DEVICE_ID_MELLANOX_CONNECTX_EN_T_GEN2), /* Gen2 */
4205 /* MT26468 ConnectX EN 10GigE PCIe Gen2*/
4206 MLX_SP(PCI_DEVICE_ID_MELLANOX_CONNECTX_EN_GEN2),
4207 /* MT26438 ConnectX EN 40GigE PCIe Gen2 5GT/s */
4208 MLX_SP(PCI_DEVICE_ID_MELLANOX_CONNECTX_EN_5_GEN2),
4209 /* MT26478 ConnectX2 40GigE PCIe Gen2 */
4210 MLX_SP(PCI_DEVICE_ID_MELLANOX_CONNECTX2),
4211 /* MT25400 Family [ConnectX-2] */
4212 MLX_VF(0x1002), /* Virtual Function */
a1b87145 4213#endif /* CONFIG_MLX4_CORE_GEN2 */
ab9c17a0 4214 /* MT27500 Family [ConnectX-3] */
c19e4b90
BH		 4215 MLX_GN(PCI_DEVICE_ID_MELLANOX_CONNECTX3),
4216 MLX_VF(0x1004), /* Virtual Function */
4217 MLX_GN(0x1005), /* MT27510 Family */
4218 MLX_GN(0x1006), /* MT27511 Family */
4219 MLX_GN(PCI_DEVICE_ID_MELLANOX_CONNECTX3_PRO), /* MT27520 Family */
4220 MLX_GN(0x1008), /* MT27521 Family */
4221 MLX_GN(0x1009), /* MT27530 Family */
4222 MLX_GN(0x100a), /* MT27531 Family */
4223 MLX_GN(0x100b), /* MT27540 Family */
4224 MLX_GN(0x100c), /* MT27541 Family */
4225 MLX_GN(0x100d), /* MT27550 Family */
4226 MLX_GN(0x100e), /* MT27551 Family */
4227 MLX_GN(0x100f), /* MT27560 Family */
4228 MLX_GN(0x1010), /* MT27561 Family */
4229
4230 /*
4231 * See the mellanox_check_broken_intx_masking() quirk when
4232 * adding devices
4233 */
4234
225c7b1f
RD		 4235 { 0, }
4236};
4237
4238MODULE_DEVICE_TABLE(pci, mlx4_pci_table);
4239
57dbf29a
KSS		 4240static pci_ers_result_t mlx4_pci_err_detected(struct pci_dev *pdev,
4241 pci_channel_state_t state)
4242{
2ba5fbd6
YH		 4243 struct mlx4_dev_persistent *persist = pci_get_drvdata(pdev);
4244
4245 mlx4_err(persist->dev, "mlx4_pci_err_detected was called\n");
4246 mlx4_enter_error_state(persist);
57dbf29a 4247
2ba5fbd6
YH		 4248 mutex_lock(&persist->interface_state_mutex);
4249 if (persist->interface_state & MLX4_INTERFACE_STATE_UP)
4250 mlx4_unload_one(pdev);
4251
4252 mutex_unlock(&persist->interface_state_mutex);
4253 if (state == pci_channel_io_perm_failure)
4254 return PCI_ERS_RESULT_DISCONNECT;
4255
4bfd2e6e 4256 mlx4_pci_disable_device(persist->dev);
2ba5fbd6 4257 return PCI_ERS_RESULT_NEED_RESET;
57dbf29a
KSS		 4258}
4259
4260static pci_ers_result_t mlx4_pci_slot_reset(struct pci_dev *pdev)
4261{
2ba5fbd6
YH		 4262 struct mlx4_dev_persistent *persist = pci_get_drvdata(pdev);
4263 struct mlx4_dev *dev = persist->dev;
c12833ac 4264 int err;
97a5221f 4265
2ba5fbd6 4266 mlx4_err(dev, "mlx4_pci_slot_reset was called\n");
4bfd2e6e 4267 err = mlx4_pci_enable_device(dev);
c12833ac
DJ		 4268 if (err) {
4269 mlx4_err(dev, "Can not re-enable device, err=%d\n", err);
2ba5fbd6
YH		 4270 return PCI_ERS_RESULT_DISCONNECT;
4271 }
4272
4273 pci_set_master(pdev);
4274 pci_restore_state(pdev);
4275 pci_save_state(pdev);
c12833ac
DJ		 4276 return PCI_ERS_RESULT_RECOVERED;
4277}
4278
4279static void mlx4_pci_resume(struct pci_dev *pdev)
4280{
4281 struct mlx4_dev_persistent *persist = pci_get_drvdata(pdev);
4282 struct mlx4_dev *dev = persist->dev;
4283 struct mlx4_priv *priv = mlx4_priv(dev);
4284 int nvfs[MLX4_MAX_PORTS + 1] = {0, 0, 0};
4285 int total_vfs;
4286 int err;
2ba5fbd6 4287
c12833ac 4288 mlx4_err(dev, "%s was called\n", __func__);
2ba5fbd6
YH		 4289 total_vfs = dev->persist->num_vfs;
4290 memcpy(nvfs, dev->persist->nvfs, sizeof(dev->persist->nvfs));
4291
4292 mutex_lock(&persist->interface_state_mutex);
4293 if (!(persist->interface_state & MLX4_INTERFACE_STATE_UP)) {
c12833ac 4294 err = mlx4_load_one(pdev, priv->pci_dev_data, total_vfs, nvfs,
55ad3592 4295 priv, 1);
c12833ac
DJ		 4296 if (err) {
4297 mlx4_err(dev, "%s: mlx4_load_one failed, err=%d\n",
4298 __func__, err);
2ba5fbd6
YH		 4299 goto end;
4300 }
4301
c12833ac 4302 err = restore_current_port_types(dev, dev->persist->
2ba5fbd6
YH		 4303 curr_port_type, dev->persist->
4304 curr_port_poss_type);
c12833ac
DJ		 4305 if (err)
4306 mlx4_err(dev, "could not restore original port types (%d)\n", err);
2ba5fbd6
YH		 4307 }
4308end:
4309 mutex_unlock(&persist->interface_state_mutex);
57dbf29a 4310
57dbf29a
KSS		 4311}
4312
2ba5fbd6
YH		 4313static void mlx4_shutdown(struct pci_dev *pdev)
4314{
4315 struct mlx4_dev_persistent *persist = pci_get_drvdata(pdev);
4316
4317 mlx4_info(persist->dev, "mlx4_shutdown was called\n");
4318 mutex_lock(&persist->interface_state_mutex);
b4353708 4319 if (persist->interface_state & MLX4_INTERFACE_STATE_UP)
2ba5fbd6
YH		 4320 mlx4_unload_one(pdev);
4321 mutex_unlock(&persist->interface_state_mutex);
4322}
4323
3646f0e5 4324static const struct pci_error_handlers mlx4_err_handler = {
57dbf29a
KSS		 4325 .error_detected = mlx4_pci_err_detected,
4326 .slot_reset = mlx4_pci_slot_reset,
c12833ac 4327 .resume = mlx4_pci_resume,
57dbf29a
KSS		 4328};
4329
86a3e5d0
YH		 4330static int mlx4_suspend(struct pci_dev *pdev, pm_message_t state)
4331{
4332 struct mlx4_dev_persistent *persist = pci_get_drvdata(pdev);
4333 struct mlx4_dev *dev = persist->dev;
4334
4335 mlx4_err(dev, "suspend was called\n");
4336 mutex_lock(&persist->interface_state_mutex);
4337 if (persist->interface_state & MLX4_INTERFACE_STATE_UP)
4338 mlx4_unload_one(pdev);
4339 mutex_unlock(&persist->interface_state_mutex);
4340
4341 return 0;
4342}
4343
4344static int mlx4_resume(struct pci_dev *pdev)
4345{
4346 struct mlx4_dev_persistent *persist = pci_get_drvdata(pdev);
4347 struct mlx4_dev *dev = persist->dev;
4348 struct mlx4_priv *priv = mlx4_priv(dev);
4349 int nvfs[MLX4_MAX_PORTS + 1] = {0, 0, 0};
4350 int total_vfs;
4351 int ret = 0;
4352
4353 mlx4_err(dev, "resume was called\n");
4354 total_vfs = dev->persist->num_vfs;
4355 memcpy(nvfs, dev->persist->nvfs, sizeof(dev->persist->nvfs));
4356
4357 mutex_lock(&persist->interface_state_mutex);
4358 if (!(persist->interface_state & MLX4_INTERFACE_STATE_UP)) {
4359 ret = mlx4_load_one(pdev, priv->pci_dev_data, total_vfs,
4360 nvfs, priv, 1);
4361 if (!ret) {
4362 ret = restore_current_port_types(dev,
4363 dev->persist->curr_port_type,
4364 dev->persist->curr_port_poss_type);
4365 if (ret)
4366 mlx4_err(dev, "resume: could not restore original port types (%d)\n", ret);
4367 }
4368 }
4369 mutex_unlock(&persist->interface_state_mutex);
4370
4371 return ret;
4372}
4373
225c7b1f
RD		 4374static struct pci_driver mlx4_driver = {
4375 .name = DRV_NAME,
4376 .id_table = mlx4_pci_table,
4377 .probe = mlx4_init_one,
2ba5fbd6 4378 .shutdown = mlx4_shutdown,
f57e6848 4379 .remove = mlx4_remove_one,
86a3e5d0
YH		 4380 .suspend = mlx4_suspend,
4381 .resume = mlx4_resume,
57dbf29a 4382 .err_handler = &mlx4_err_handler,
225c7b1f
RD		 4383};
4384
7ff93f8b
YP		 4385static int __init mlx4_verify_params(void)
4386{
e5732838
TT		 4387 if (msi_x < 0) {
4388 pr_warn("mlx4_core: bad msi_x: %d\n", msi_x);
4389 return -1;
4390 }
4391
7ff93f8b 4392 if ((log_num_mac < 0) || (log_num_mac > 7)) {
c20862c8 4393 pr_warn("mlx4_core: bad num_mac: %d\n", log_num_mac);
7ff93f8b
YP		 4394 return -1;
4395 }
4396
cb29688a 4397 if (log_num_vlan != 0)
c20862c8
AV		 4398 pr_warn("mlx4_core: log_num_vlan - obsolete module param, using %d\n",
4399 MLX4_LOG_NUM_VLANS);
7ff93f8b 4400
ecc8fb11
AV		 4401 if (use_prio != 0)
4402 pr_warn("mlx4_core: use_prio - obsolete module param, ignored\n");
7ff93f8b 4403
7cc77bf4 4404 if ((log_mtts_per_seg < 0) || (log_mtts_per_seg > 7)) {
c20862c8
AV		 4405 pr_warn("mlx4_core: bad log_mtts_per_seg: %d\n",
4406 log_mtts_per_seg);
ab6bf42e
EC		 4407 return -1;
4408 }
4409
ab9c17a0
JM		 4410 /* Check if module param for ports type has legal combination */
4411 if (port_type_array[0] == false && port_type_array[1] == true) {
c20862c8 4412 pr_warn("Module parameter configuration ETH/IB is not supported. Switching to default configuration IB/IB\n");
ab9c17a0
JM		 4413 port_type_array[0] = true;
4414 }
4415
7d077cd3
MB		 4416 if (mlx4_log_num_mgm_entry_size < -7 ||
4417 (mlx4_log_num_mgm_entry_size > 0 &&
4418 (mlx4_log_num_mgm_entry_size < MLX4_MIN_MGM_LOG_ENTRY_SIZE ||
4419 mlx4_log_num_mgm_entry_size > MLX4_MAX_MGM_LOG_ENTRY_SIZE))) {
4420 pr_warn("mlx4_core: mlx4_log_num_mgm_entry_size (%d) not in legal range (-7..0 or %d..%d)\n",
1a91de28
JP		 4421 mlx4_log_num_mgm_entry_size,
4422 MLX4_MIN_MGM_LOG_ENTRY_SIZE,
4423 MLX4_MAX_MGM_LOG_ENTRY_SIZE);
3c439b55
JM		 4424 return -1;
4425 }
4426
7ff93f8b
YP		 4427 return 0;
4428}
4429
225c7b1f
RD		 4430static int __init mlx4_init(void)
4431{
4432 int ret;
4433
7ff93f8b
YP		 4434 if (mlx4_verify_params())
4435 return -EINVAL;
4436
27bf91d6
YP		 4437
4438 mlx4_wq = create_singlethread_workqueue("mlx4");
4439 if (!mlx4_wq)
4440 return -ENOMEM;
ee49bd93 4441
225c7b1f 4442 ret = pci_register_driver(&mlx4_driver);
1b85ee09
WY		 4443 if (ret < 0)
4444 destroy_workqueue(mlx4_wq);
225c7b1f
RD		 4445 return ret < 0 ? ret : 0;
4446}
4447
4448static void __exit mlx4_cleanup(void)
4449{
4450 pci_unregister_driver(&mlx4_driver);
27bf91d6 4451 destroy_workqueue(mlx4_wq);
225c7b1f
RD		 4452}
4453
4454module_init(mlx4_init);
4455module_exit(mlx4_cleanup);
Ubuntu Eoan Linux kernel mirror