]> git.proxmox.com Git - mirror_ubuntu-bionic-kernel.git/blob - drivers/infiniband/hw/nes/nes_hw.c
projects / mirror_ubuntu-bionic-kernel.git / blob
? search:


4713dd7ed76432b6d11b042e6dfd24c268f82325
[mirror_ubuntu-bionic-kernel.git] / drivers / infiniband / hw / nes / nes_hw.c
1 /*
2 * Copyright (c) 2006 - 2011 Intel Corporation. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 *
32 */
33
34 #include <linux/module.h>
35 #include <linux/moduleparam.h>
36 #include <linux/netdevice.h>
37 #include <linux/etherdevice.h>
38 #include <linux/ip.h>
39 #include <linux/tcp.h>
40 #include <linux/if_vlan.h>
41 #include <linux/inet_lro.h>
42 #include <linux/slab.h>
43
44 #include "nes.h"
45
46 static unsigned int nes_lro_max_aggr = NES_LRO_MAX_AGGR;
47 module_param(nes_lro_max_aggr, uint, 0444);
48 MODULE_PARM_DESC(nes_lro_max_aggr, "NIC LRO max packet aggregation");
49
50 static int wide_ppm_offset;
51 module_param(wide_ppm_offset, int, 0644);
52 MODULE_PARM_DESC(wide_ppm_offset, "Increase CX4 interface clock ppm offset, 0=100ppm (default), 1=300ppm");
53
54 static u32 crit_err_count;
55 u32 int_mod_timer_init;
56 u32 int_mod_cq_depth_256;
57 u32 int_mod_cq_depth_128;
58 u32 int_mod_cq_depth_32;
59 u32 int_mod_cq_depth_24;
60 u32 int_mod_cq_depth_16;
61 u32 int_mod_cq_depth_4;
62 u32 int_mod_cq_depth_1;
63 static const u8 nes_max_critical_error_count = 100;
64 #include "nes_cm.h"
65
66 static void nes_cqp_ce_handler(struct nes_device *nesdev, struct nes_hw_cq *cq);
67 static void nes_init_csr_ne020(struct nes_device *nesdev, u8 hw_rev, u8 port_count);
68 static int nes_init_serdes(struct nes_device *nesdev, u8 hw_rev, u8 port_count,
69 struct nes_adapter *nesadapter, u8 OneG_Mode);
70 static void nes_nic_napi_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq);
71 static void nes_process_aeq(struct nes_device *nesdev, struct nes_hw_aeq *aeq);
72 static void nes_process_ceq(struct nes_device *nesdev, struct nes_hw_ceq *ceq);
73 static void nes_process_iwarp_aeqe(struct nes_device *nesdev,
74 struct nes_hw_aeqe *aeqe);
75 static void process_critical_error(struct nes_device *nesdev);
76 static void nes_process_mac_intr(struct nes_device *nesdev, u32 mac_number);
77 static unsigned int nes_reset_adapter_ne020(struct nes_device *nesdev, u8 *OneG_Mode);
78 static void nes_terminate_start_timer(struct nes_qp *nesqp);
79
80 #ifdef CONFIG_INFINIBAND_NES_DEBUG
81 static unsigned char *nes_iwarp_state_str[] = {
82 "Non-Existent",
83 "Idle",
84 "RTS",
85 "Closing",
86 "RSVD1",
87 "Terminate",
88 "Error",
89 "RSVD2",
90 };
91
92 static unsigned char *nes_tcp_state_str[] = {
93 "Non-Existent",
94 "Closed",
95 "Listen",
96 "SYN Sent",
97 "SYN Rcvd",
98 "Established",
99 "Close Wait",
100 "FIN Wait 1",
101 "Closing",
102 "Last Ack",
103 "FIN Wait 2",
104 "Time Wait",
105 "RSVD1",
106 "RSVD2",
107 "RSVD3",
108 "RSVD4",
109 };
110 #endif
111
112 static inline void print_ip(struct nes_cm_node *cm_node)
113 {
114 unsigned char *rem_addr;
115 if (cm_node) {
116 rem_addr = (unsigned char *)&cm_node->rem_addr;
117 printk(KERN_ERR PFX "Remote IP addr: %pI4\n", rem_addr);
118 }
119 }
120
121 /**
122 * nes_nic_init_timer_defaults
123 */
124 void nes_nic_init_timer_defaults(struct nes_device *nesdev, u8 jumbomode)
125 {
126 unsigned long flags;
127 struct nes_adapter *nesadapter = nesdev->nesadapter;
128 struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer;
129
130 spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags);
131
132 shared_timer->timer_in_use_min = NES_NIC_FAST_TIMER_LOW;
133 shared_timer->timer_in_use_max = NES_NIC_FAST_TIMER_HIGH;
134 if (jumbomode) {
135 shared_timer->threshold_low = DEFAULT_JUMBO_NES_QL_LOW;
136 shared_timer->threshold_target = DEFAULT_JUMBO_NES_QL_TARGET;
137 shared_timer->threshold_high = DEFAULT_JUMBO_NES_QL_HIGH;
138 } else {
139 shared_timer->threshold_low = DEFAULT_NES_QL_LOW;
140 shared_timer->threshold_target = DEFAULT_NES_QL_TARGET;
141 shared_timer->threshold_high = DEFAULT_NES_QL_HIGH;
142 }
143
144 /* todo use netdev->mtu to set thresholds */
145 spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
146 }
147
148
149 /**
150 * nes_nic_init_timer
151 */
152 static void nes_nic_init_timer(struct nes_device *nesdev)
153 {
154 unsigned long flags;
155 struct nes_adapter *nesadapter = nesdev->nesadapter;
156 struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer;
157
158 spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags);
159
160 if (shared_timer->timer_in_use_old == 0) {
161 nesdev->deepcq_count = 0;
162 shared_timer->timer_direction_upward = 0;
163 shared_timer->timer_direction_downward = 0;
164 shared_timer->timer_in_use = NES_NIC_FAST_TIMER;
165 shared_timer->timer_in_use_old = 0;
166
167 }
168 if (shared_timer->timer_in_use != shared_timer->timer_in_use_old) {
169 shared_timer->timer_in_use_old = shared_timer->timer_in_use;
170 nes_write32(nesdev->regs+NES_PERIODIC_CONTROL,
171 0x80000000 | ((u32)(shared_timer->timer_in_use*8)));
172 }
173 /* todo use netdev->mtu to set thresholds */
174 spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
175 }
176
177
178 /**
179 * nes_nic_tune_timer
180 */
181 static void nes_nic_tune_timer(struct nes_device *nesdev)
182 {
183 unsigned long flags;
184 struct nes_adapter *nesadapter = nesdev->nesadapter;
185 struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer;
186 u16 cq_count = nesdev->currcq_count;
187
188 spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags);
189
190 if (shared_timer->cq_count_old <= cq_count)
191 shared_timer->cq_direction_downward = 0;
192 else
193 shared_timer->cq_direction_downward++;
194 shared_timer->cq_count_old = cq_count;
195 if (shared_timer->cq_direction_downward > NES_NIC_CQ_DOWNWARD_TREND) {
196 if (cq_count <= shared_timer->threshold_low &&
197 shared_timer->threshold_low > 4) {
198 shared_timer->threshold_low = shared_timer->threshold_low/2;
199 shared_timer->cq_direction_downward=0;
200 nesdev->currcq_count = 0;
201 spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
202 return;
203 }
204 }
205
206 if (cq_count > 1) {
207 nesdev->deepcq_count += cq_count;
208 if (cq_count <= shared_timer->threshold_low) { /* increase timer gently */
209 shared_timer->timer_direction_upward++;
210 shared_timer->timer_direction_downward = 0;
211 } else if (cq_count <= shared_timer->threshold_target) { /* balanced */
212 shared_timer->timer_direction_upward = 0;
213 shared_timer->timer_direction_downward = 0;
214 } else if (cq_count <= shared_timer->threshold_high) { /* decrease timer gently */
215 shared_timer->timer_direction_downward++;
216 shared_timer->timer_direction_upward = 0;
217 } else if (cq_count <= (shared_timer->threshold_high) * 2) {
218 shared_timer->timer_in_use -= 2;
219 shared_timer->timer_direction_upward = 0;
220 shared_timer->timer_direction_downward++;
221 } else {
222 shared_timer->timer_in_use -= 4;
223 shared_timer->timer_direction_upward = 0;
224 shared_timer->timer_direction_downward++;
225 }
226
227 if (shared_timer->timer_direction_upward > 3 ) { /* using history */
228 shared_timer->timer_in_use += 3;
229 shared_timer->timer_direction_upward = 0;
230 shared_timer->timer_direction_downward = 0;
231 }
232 if (shared_timer->timer_direction_downward > 5) { /* using history */
233 shared_timer->timer_in_use -= 4 ;
234 shared_timer->timer_direction_downward = 0;
235 shared_timer->timer_direction_upward = 0;
236 }
237 }
238
239 /* boundary checking */
240 if (shared_timer->timer_in_use > shared_timer->threshold_high)
241 shared_timer->timer_in_use = shared_timer->threshold_high;
242 else if (shared_timer->timer_in_use < shared_timer->threshold_low)
243 shared_timer->timer_in_use = shared_timer->threshold_low;
244
245 nesdev->currcq_count = 0;
246
247 spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
248 }
249
250
251 /**
252 * nes_init_adapter - initialize adapter
253 */
254 struct nes_adapter *nes_init_adapter(struct nes_device *nesdev, u8 hw_rev) {
255 struct nes_adapter *nesadapter = NULL;
256 unsigned long num_pds;
257 u32 u32temp;
258 u32 port_count;
259 u16 max_rq_wrs;
260 u16 max_sq_wrs;
261 u32 max_mr;
262 u32 max_256pbl;
263 u32 max_4kpbl;
264 u32 max_qp;
265 u32 max_irrq;
266 u32 max_cq;
267 u32 hte_index_mask;
268 u32 adapter_size;
269 u32 arp_table_size;
270 u16 vendor_id;
271 u16 device_id;
272 u8 OneG_Mode;
273 u8 func_index;
274
275 /* search the list of existing adapters */
276 list_for_each_entry(nesadapter, &nes_adapter_list, list) {
277 nes_debug(NES_DBG_INIT, "Searching Adapter list for PCI devfn = 0x%X,"
278 " adapter PCI slot/bus = %u/%u, pci devices PCI slot/bus = %u/%u, .\n",
279 nesdev->pcidev->devfn,
280 PCI_SLOT(nesadapter->devfn),
281 nesadapter->bus_number,
282 PCI_SLOT(nesdev->pcidev->devfn),
283 nesdev->pcidev->bus->number );
284 if ((PCI_SLOT(nesadapter->devfn) == PCI_SLOT(nesdev->pcidev->devfn)) &&
285 (nesadapter->bus_number == nesdev->pcidev->bus->number)) {
286 nesadapter->ref_count++;
287 return nesadapter;
288 }
289 }
290
291 /* no adapter found */
292 num_pds = pci_resource_len(nesdev->pcidev, BAR_1) >> PAGE_SHIFT;
293 if ((hw_rev != NE020_REV) && (hw_rev != NE020_REV1)) {
294 nes_debug(NES_DBG_INIT, "NE020 driver detected unknown hardware revision 0x%x\n",
295 hw_rev);
296 return NULL;
297 }
298
299 nes_debug(NES_DBG_INIT, "Determine Soft Reset, QP_control=0x%x, CPU0=0x%x, CPU1=0x%x, CPU2=0x%x\n",
300 nes_read_indexed(nesdev, NES_IDX_QP_CONTROL + PCI_FUNC(nesdev->pcidev->devfn) * 8),
301 nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS),
302 nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS + 4),
303 nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS + 8));
304
305 nes_debug(NES_DBG_INIT, "Reset and init NE020\n");
306
307
308 if ((port_count = nes_reset_adapter_ne020(nesdev, &OneG_Mode)) == 0)
309 return NULL;
310
311 max_qp = nes_read_indexed(nesdev, NES_IDX_QP_CTX_SIZE);
312 nes_debug(NES_DBG_INIT, "QP_CTX_SIZE=%u\n", max_qp);
313
314 u32temp = nes_read_indexed(nesdev, NES_IDX_QUAD_HASH_TABLE_SIZE);
315 if (max_qp > ((u32)1 << (u32temp & 0x001f))) {
316 nes_debug(NES_DBG_INIT, "Reducing Max QPs to %u due to hash table size = 0x%08X\n",
317 max_qp, u32temp);
318 max_qp = (u32)1 << (u32temp & 0x001f);
319 }
320
321 hte_index_mask = ((u32)1 << ((u32temp & 0x001f)+1))-1;
322 nes_debug(NES_DBG_INIT, "Max QP = %u, hte_index_mask = 0x%08X.\n",
323 max_qp, hte_index_mask);
324
325 u32temp = nes_read_indexed(nesdev, NES_IDX_IRRQ_COUNT);
326
327 max_irrq = 1 << (u32temp & 0x001f);
328
329 if (max_qp > max_irrq) {
330 max_qp = max_irrq;
331 nes_debug(NES_DBG_INIT, "Reducing Max QPs to %u due to Available Q1s.\n",
332 max_qp);
333 }
334
335 /* there should be no reason to allocate more pds than qps */
336 if (num_pds > max_qp)
337 num_pds = max_qp;
338
339 u32temp = nes_read_indexed(nesdev, NES_IDX_MRT_SIZE);
340 max_mr = (u32)8192 << (u32temp & 0x7);
341
342 u32temp = nes_read_indexed(nesdev, NES_IDX_PBL_REGION_SIZE);
343 max_256pbl = (u32)1 << (u32temp & 0x0000001f);
344 max_4kpbl = (u32)1 << ((u32temp >> 16) & 0x0000001f);
345 max_cq = nes_read_indexed(nesdev, NES_IDX_CQ_CTX_SIZE);
346
347 u32temp = nes_read_indexed(nesdev, NES_IDX_ARP_CACHE_SIZE);
348 arp_table_size = 1 << u32temp;
349
350 adapter_size = (sizeof(struct nes_adapter) +
351 (sizeof(unsigned long)-1)) & (~(sizeof(unsigned long)-1));
352 adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(max_qp);
353 adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(max_mr);
354 adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(max_cq);
355 adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(num_pds);
356 adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(arp_table_size);
357 adapter_size += sizeof(struct nes_qp **) * max_qp;
358
359 /* allocate a new adapter struct */
360 nesadapter = kzalloc(adapter_size, GFP_KERNEL);
361 if (nesadapter == NULL) {
362 return NULL;
363 }
364
365 nes_debug(NES_DBG_INIT, "Allocating new nesadapter @ %p, size = %u (actual size = %u).\n",
366 nesadapter, (u32)sizeof(struct nes_adapter), adapter_size);
367
368 if (nes_read_eeprom_values(nesdev, nesadapter)) {
369 printk(KERN_ERR PFX "Unable to read EEPROM data.\n");
370 kfree(nesadapter);
371 return NULL;
372 }
373
374 nesadapter->vendor_id = (((u32) nesadapter->mac_addr_high) << 8) |
375 (nesadapter->mac_addr_low >> 24);
376
377 pci_bus_read_config_word(nesdev->pcidev->bus, nesdev->pcidev->devfn,
378 PCI_DEVICE_ID, &device_id);
379 nesadapter->vendor_part_id = device_id;
380
381 if (nes_init_serdes(nesdev, hw_rev, port_count, nesadapter,
382 OneG_Mode)) {
383 kfree(nesadapter);
384 return NULL;
385 }
386 nes_init_csr_ne020(nesdev, hw_rev, port_count);
387
388 memset(nesadapter->pft_mcast_map, 255,
389 sizeof nesadapter->pft_mcast_map);
390
391 /* populate the new nesadapter */
392 nesadapter->devfn = nesdev->pcidev->devfn;
393 nesadapter->bus_number = nesdev->pcidev->bus->number;
394 nesadapter->ref_count = 1;
395 nesadapter->timer_int_req = 0xffff0000;
396 nesadapter->OneG_Mode = OneG_Mode;
397 nesadapter->doorbell_start = nesdev->doorbell_region;
398
399 /* nesadapter->tick_delta = clk_divisor; */
400 nesadapter->hw_rev = hw_rev;
401 nesadapter->port_count = port_count;
402
403 nesadapter->max_qp = max_qp;
404 nesadapter->hte_index_mask = hte_index_mask;
405 nesadapter->max_irrq = max_irrq;
406 nesadapter->max_mr = max_mr;
407 nesadapter->max_256pbl = max_256pbl - 1;
408 nesadapter->max_4kpbl = max_4kpbl - 1;
409 nesadapter->max_cq = max_cq;
410 nesadapter->free_256pbl = max_256pbl - 1;
411 nesadapter->free_4kpbl = max_4kpbl - 1;
412 nesadapter->max_pd = num_pds;
413 nesadapter->arp_table_size = arp_table_size;
414
415 nesadapter->et_pkt_rate_low = NES_TIMER_ENABLE_LIMIT;
416 if (nes_drv_opt & NES_DRV_OPT_DISABLE_INT_MOD) {
417 nesadapter->et_use_adaptive_rx_coalesce = 0;
418 nesadapter->timer_int_limit = NES_TIMER_INT_LIMIT;
419 nesadapter->et_rx_coalesce_usecs_irq = interrupt_mod_interval;
420 } else {
421 nesadapter->et_use_adaptive_rx_coalesce = 1;
422 nesadapter->timer_int_limit = NES_TIMER_INT_LIMIT_DYNAMIC;
423 nesadapter->et_rx_coalesce_usecs_irq = 0;
424 printk(PFX "%s: Using Adaptive Interrupt Moderation\n", __func__);
425 }
426 /* Setup and enable the periodic timer */
427 if (nesadapter->et_rx_coalesce_usecs_irq)
428 nes_write32(nesdev->regs+NES_PERIODIC_CONTROL, 0x80000000 |
429 ((u32)(nesadapter->et_rx_coalesce_usecs_irq * 8)));
430 else
431 nes_write32(nesdev->regs+NES_PERIODIC_CONTROL, 0x00000000);
432
433 nesadapter->base_pd = 1;
434
435 nesadapter->device_cap_flags = IB_DEVICE_LOCAL_DMA_LKEY |
436 IB_DEVICE_MEM_WINDOW |
437 IB_DEVICE_MEM_MGT_EXTENSIONS;
438
439 nesadapter->allocated_qps = (unsigned long *)&(((unsigned char *)nesadapter)
440 [(sizeof(struct nes_adapter)+(sizeof(unsigned long)-1))&(~(sizeof(unsigned long)-1))]);
441 nesadapter->allocated_cqs = &nesadapter->allocated_qps[BITS_TO_LONGS(max_qp)];
442 nesadapter->allocated_mrs = &nesadapter->allocated_cqs[BITS_TO_LONGS(max_cq)];
443 nesadapter->allocated_pds = &nesadapter->allocated_mrs[BITS_TO_LONGS(max_mr)];
444 nesadapter->allocated_arps = &nesadapter->allocated_pds[BITS_TO_LONGS(num_pds)];
445 nesadapter->qp_table = (struct nes_qp **)(&nesadapter->allocated_arps[BITS_TO_LONGS(arp_table_size)]);
446
447
448 /* mark the usual suspect QPs, MR and CQs as in use */
449 for (u32temp = 0; u32temp < NES_FIRST_QPN; u32temp++) {
450 set_bit(u32temp, nesadapter->allocated_qps);
451 set_bit(u32temp, nesadapter->allocated_cqs);
452 }
453 set_bit(0, nesadapter->allocated_mrs);
454
455 for (u32temp = 0; u32temp < 20; u32temp++)
456 set_bit(u32temp, nesadapter->allocated_pds);
457 u32temp = nes_read_indexed(nesdev, NES_IDX_QP_MAX_CFG_SIZES);
458
459 max_rq_wrs = ((u32temp >> 8) & 3);
460 switch (max_rq_wrs) {
461 case 0:
462 max_rq_wrs = 4;
463 break;
464 case 1:
465 max_rq_wrs = 16;
466 break;
467 case 2:
468 max_rq_wrs = 32;
469 break;
470 case 3:
471 max_rq_wrs = 512;
472 break;
473 }
474
475 max_sq_wrs = (u32temp & 3);
476 switch (max_sq_wrs) {
477 case 0:
478 max_sq_wrs = 4;
479 break;
480 case 1:
481 max_sq_wrs = 16;
482 break;
483 case 2:
484 max_sq_wrs = 32;
485 break;
486 case 3:
487 max_sq_wrs = 512;
488 break;
489 }
490 nesadapter->max_qp_wr = min(max_rq_wrs, max_sq_wrs);
491 nesadapter->max_irrq_wr = (u32temp >> 16) & 3;
492
493 nesadapter->max_sge = 4;
494 nesadapter->max_cqe = 32766;
495
496 if (nes_read_eeprom_values(nesdev, nesadapter)) {
497 printk(KERN_ERR PFX "Unable to read EEPROM data.\n");
498 kfree(nesadapter);
499 return NULL;
500 }
501
502 u32temp = nes_read_indexed(nesdev, NES_IDX_TCP_TIMER_CONFIG);
503 nes_write_indexed(nesdev, NES_IDX_TCP_TIMER_CONFIG,
504 (u32temp & 0xff000000) | (nesadapter->tcp_timer_core_clk_divisor & 0x00ffffff));
505
506 /* setup port configuration */
507 if (nesadapter->port_count == 1) {
508 nesadapter->log_port = 0x00000000;
509 if (nes_drv_opt & NES_DRV_OPT_DUAL_LOGICAL_PORT)
510 nes_write_indexed(nesdev, NES_IDX_TX_POOL_SIZE, 0x00000002);
511 else
512 nes_write_indexed(nesdev, NES_IDX_TX_POOL_SIZE, 0x00000003);
513 } else {
514 if (nesadapter->phy_type[0] == NES_PHY_TYPE_PUMA_1G) {
515 nesadapter->log_port = 0x000000D8;
516 } else {
517 if (nesadapter->port_count == 2)
518 nesadapter->log_port = 0x00000044;
519 else
520 nesadapter->log_port = 0x000000e4;
521 }
522 nes_write_indexed(nesdev, NES_IDX_TX_POOL_SIZE, 0x00000003);
523 }
524
525 nes_write_indexed(nesdev, NES_IDX_NIC_LOGPORT_TO_PHYPORT,
526 nesadapter->log_port);
527 nes_debug(NES_DBG_INIT, "Probe time, LOG2PHY=%u\n",
528 nes_read_indexed(nesdev, NES_IDX_NIC_LOGPORT_TO_PHYPORT));
529
530 spin_lock_init(&nesadapter->resource_lock);
531 spin_lock_init(&nesadapter->phy_lock);
532 spin_lock_init(&nesadapter->pbl_lock);
533 spin_lock_init(&nesadapter->periodic_timer_lock);
534
535 INIT_LIST_HEAD(&nesadapter->nesvnic_list[0]);
536 INIT_LIST_HEAD(&nesadapter->nesvnic_list[1]);
537 INIT_LIST_HEAD(&nesadapter->nesvnic_list[2]);
538 INIT_LIST_HEAD(&nesadapter->nesvnic_list[3]);
539
540 if ((!nesadapter->OneG_Mode) && (nesadapter->port_count == 2)) {
541 u32 pcs_control_status0, pcs_control_status1;
542 u32 reset_value;
543 u32 i = 0;
544 u32 int_cnt = 0;
545 u32 ext_cnt = 0;
546 unsigned long flags;
547 u32 j = 0;
548
549 pcs_control_status0 = nes_read_indexed(nesdev,
550 NES_IDX_PHY_PCS_CONTROL_STATUS0);
551 pcs_control_status1 = nes_read_indexed(nesdev,
552 NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
553
554 for (i = 0; i < NES_MAX_LINK_CHECK; i++) {
555 pcs_control_status0 = nes_read_indexed(nesdev,
556 NES_IDX_PHY_PCS_CONTROL_STATUS0);
557 pcs_control_status1 = nes_read_indexed(nesdev,
558 NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
559 if ((0x0F000100 == (pcs_control_status0 & 0x0F000100))
560 || (0x0F000100 == (pcs_control_status1 & 0x0F000100)))
561 int_cnt++;
562 msleep(1);
563 }
564 if (int_cnt > 1) {
565 spin_lock_irqsave(&nesadapter->phy_lock, flags);
566 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F0C8);
567 mh_detected++;
568 reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
569 reset_value |= 0x0000003d;
570 nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
571
572 while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET)
573 & 0x00000040) != 0x00000040) && (j++ < 5000));
574 spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
575
576 pcs_control_status0 = nes_read_indexed(nesdev,
577 NES_IDX_PHY_PCS_CONTROL_STATUS0);
578 pcs_control_status1 = nes_read_indexed(nesdev,
579 NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
580
581 for (i = 0; i < NES_MAX_LINK_CHECK; i++) {
582 pcs_control_status0 = nes_read_indexed(nesdev,
583 NES_IDX_PHY_PCS_CONTROL_STATUS0);
584 pcs_control_status1 = nes_read_indexed(nesdev,
585 NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
586 if ((0x0F000100 == (pcs_control_status0 & 0x0F000100))
587 || (0x0F000100 == (pcs_control_status1 & 0x0F000100))) {
588 if (++ext_cnt > int_cnt) {
589 spin_lock_irqsave(&nesadapter->phy_lock, flags);
590 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1,
591 0x0000F088);
592 mh_detected++;
593 reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
594 reset_value |= 0x0000003d;
595 nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
596
597 while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET)
598 & 0x00000040) != 0x00000040) && (j++ < 5000));
599 spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
600 break;
601 }
602 }
603 msleep(1);
604 }
605 }
606 }
607
608 if (nesadapter->hw_rev == NE020_REV) {
609 init_timer(&nesadapter->mh_timer);
610 nesadapter->mh_timer.function = nes_mh_fix;
611 nesadapter->mh_timer.expires = jiffies + (HZ/5); /* 1 second */
612 nesadapter->mh_timer.data = (unsigned long)nesdev;
613 add_timer(&nesadapter->mh_timer);
614 } else {
615 nes_write32(nesdev->regs+NES_INTF_INT_STAT, 0x0f000000);
616 }
617
618 init_timer(&nesadapter->lc_timer);
619 nesadapter->lc_timer.function = nes_clc;
620 nesadapter->lc_timer.expires = jiffies + 3600 * HZ; /* 1 hour */
621 nesadapter->lc_timer.data = (unsigned long)nesdev;
622 add_timer(&nesadapter->lc_timer);
623
624 list_add_tail(&nesadapter->list, &nes_adapter_list);
625
626 for (func_index = 0; func_index < 8; func_index++) {
627 pci_bus_read_config_word(nesdev->pcidev->bus,
628 PCI_DEVFN(PCI_SLOT(nesdev->pcidev->devfn),
629 func_index), 0, &vendor_id);
630 if (vendor_id == 0xffff)
631 break;
632 }
633 nes_debug(NES_DBG_INIT, "%s %d functions found for %s.\n", __func__,
634 func_index, pci_name(nesdev->pcidev));
635 nesadapter->adapter_fcn_count = func_index;
636
637 return nesadapter;
638 }
639
640
641 /**
642 * nes_reset_adapter_ne020
643 */
644 static unsigned int nes_reset_adapter_ne020(struct nes_device *nesdev, u8 *OneG_Mode)
645 {
646 u32 port_count;
647 u32 u32temp;
648 u32 i;
649
650 u32temp = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
651 port_count = ((u32temp & 0x00000300) >> 8) + 1;
652 /* TODO: assuming that both SERDES are set the same for now */
653 *OneG_Mode = (u32temp & 0x00003c00) ? 0 : 1;
654 nes_debug(NES_DBG_INIT, "Initial Software Reset = 0x%08X, port_count=%u\n",
655 u32temp, port_count);
656 if (*OneG_Mode)
657 nes_debug(NES_DBG_INIT, "Running in 1G mode.\n");
658 u32temp &= 0xff00ffc0;
659 switch (port_count) {
660 case 1:
661 u32temp |= 0x00ee0000;
662 break;
663 case 2:
664 u32temp |= 0x00cc0000;
665 break;
666 case 4:
667 u32temp |= 0x00000000;
668 break;
669 default:
670 return 0;
671 break;
672 }
673
674 /* check and do full reset if needed */
675 if (nes_read_indexed(nesdev, NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8))) {
676 nes_debug(NES_DBG_INIT, "Issuing Full Soft reset = 0x%08X\n", u32temp | 0xd);
677 nes_write32(nesdev->regs+NES_SOFTWARE_RESET, u32temp | 0xd);
678
679 i = 0;
680 while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET) & 0x00000040) == 0) && i++ < 10000)
681 mdelay(1);
682 if (i > 10000) {
683 nes_debug(NES_DBG_INIT, "Did not see full soft reset done.\n");
684 return 0;
685 }
686
687 i = 0;
688 while ((nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS) != 0x80) && i++ < 10000)
689 mdelay(1);
690 if (i > 10000) {
691 printk(KERN_ERR PFX "Internal CPU not ready, status = %02X\n",
692 nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS));
693 return 0;
694 }
695 }
696
697 /* port reset */
698 switch (port_count) {
699 case 1:
700 u32temp |= 0x00ee0010;
701 break;
702 case 2:
703 u32temp |= 0x00cc0030;
704 break;
705 case 4:
706 u32temp |= 0x00000030;
707 break;
708 }
709
710 nes_debug(NES_DBG_INIT, "Issuing Port Soft reset = 0x%08X\n", u32temp | 0xd);
711 nes_write32(nesdev->regs+NES_SOFTWARE_RESET, u32temp | 0xd);
712
713 i = 0;
714 while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET) & 0x00000040) == 0) && i++ < 10000)
715 mdelay(1);
716 if (i > 10000) {
717 nes_debug(NES_DBG_INIT, "Did not see port soft reset done.\n");
718 return 0;
719 }
720
721 /* serdes 0 */
722 i = 0;
723 while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0)
724 & 0x0000000f)) != 0x0000000f) && i++ < 5000)
725 mdelay(1);
726 if (i > 5000) {
727 nes_debug(NES_DBG_INIT, "Serdes 0 not ready, status=%x\n", u32temp);
728 return 0;
729 }
730
731 /* serdes 1 */
732 if (port_count > 1) {
733 i = 0;
734 while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS1)
735 & 0x0000000f)) != 0x0000000f) && i++ < 5000)
736 mdelay(1);
737 if (i > 5000) {
738 nes_debug(NES_DBG_INIT, "Serdes 1 not ready, status=%x\n", u32temp);
739 return 0;
740 }
741 }
742
743 return port_count;
744 }
745
746
747 /**
748 * nes_init_serdes
749 */
750 static int nes_init_serdes(struct nes_device *nesdev, u8 hw_rev, u8 port_count,
751 struct nes_adapter *nesadapter, u8 OneG_Mode)
752 {
753 int i;
754 u32 u32temp;
755 u32 sds;
756
757 if (hw_rev != NE020_REV) {
758 /* init serdes 0 */
759 switch (nesadapter->phy_type[0]) {
760 case NES_PHY_TYPE_CX4:
761 if (wide_ppm_offset)
762 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000FFFAA);
763 else
764 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000FF);
765 break;
766 case NES_PHY_TYPE_KR:
767 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000FF);
768 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP0, 0x00000000);
769 break;
770 case NES_PHY_TYPE_PUMA_1G:
771 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000FF);
772 sds = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0);
773 sds |= 0x00000100;
774 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0, sds);
775 break;
776 default:
777 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000FF);
778 break;
779 }
780
781 if (!OneG_Mode)
782 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_HIGHZ_LANE_MODE0, 0x11110000);
783
784 if (port_count < 2)
785 return 0;
786
787 /* init serdes 1 */
788 if (!(OneG_Mode && (nesadapter->phy_type[1] != NES_PHY_TYPE_PUMA_1G)))
789 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL1, 0x000000FF);
790
791 switch (nesadapter->phy_type[1]) {
792 case NES_PHY_TYPE_ARGUS:
793 case NES_PHY_TYPE_SFP_D:
794 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP0, 0x00000000);
795 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP1, 0x00000000);
796 break;
797 case NES_PHY_TYPE_CX4:
798 if (wide_ppm_offset)
799 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL1, 0x000FFFAA);
800 break;
801 case NES_PHY_TYPE_KR:
802 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP1, 0x00000000);
803 break;
804 case NES_PHY_TYPE_PUMA_1G:
805 sds = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
806 sds |= 0x000000100;
807 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, sds);
808 }
809 if (!OneG_Mode) {
810 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_HIGHZ_LANE_MODE1, 0x11110000);
811 sds = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
812 sds &= 0xFFFFFFBF;
813 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, sds);
814 }
815 } else {
816 /* init serdes 0 */
817 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0, 0x00000008);
818 i = 0;
819 while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0)
820 & 0x0000000f)) != 0x0000000f) && i++ < 5000)
821 mdelay(1);
822 if (i > 5000) {
823 nes_debug(NES_DBG_PHY, "Init: serdes 0 not ready, status=%x\n", u32temp);
824 return 1;
825 }
826 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP0, 0x000bdef7);
827 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_DRIVE0, 0x9ce73000);
828 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_MODE0, 0x0ff00000);
829 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_SIGDET0, 0x00000000);
830 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_BYPASS0, 0x00000000);
831 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_LOOPBACK_CONTROL0, 0x00000000);
832 if (OneG_Mode)
833 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL0, 0xf0182222);
834 else
835 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL0, 0xf0042222);
836
837 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000ff);
838 if (port_count > 1) {
839 /* init serdes 1 */
840 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x00000048);
841 i = 0;
842 while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS1)
843 & 0x0000000f)) != 0x0000000f) && (i++ < 5000))
844 mdelay(1);
845 if (i > 5000) {
846 printk("%s: Init: serdes 1 not ready, status=%x\n", __func__, u32temp);
847 /* return 1; */
848 }
849 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP1, 0x000bdef7);
850 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_DRIVE1, 0x9ce73000);
851 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_MODE1, 0x0ff00000);
852 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_SIGDET1, 0x00000000);
853 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_BYPASS1, 0x00000000);
854 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_LOOPBACK_CONTROL1, 0x00000000);
855 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL1, 0xf0002222);
856 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL1, 0x000000ff);
857 }
858 }
859 return 0;
860 }
861
862
863 /**
864 * nes_init_csr_ne020
865 * Initialize registers for ne020 hardware
866 */
867 static void nes_init_csr_ne020(struct nes_device *nesdev, u8 hw_rev, u8 port_count)
868 {
869 u32 u32temp;
870
871 nes_debug(NES_DBG_INIT, "port_count=%d\n", port_count);
872
873 nes_write_indexed(nesdev, 0x000001E4, 0x00000007);
874 /* nes_write_indexed(nesdev, 0x000001E8, 0x000208C4); */
875 nes_write_indexed(nesdev, 0x000001E8, 0x00020874);
876 nes_write_indexed(nesdev, 0x000001D8, 0x00048002);
877 /* nes_write_indexed(nesdev, 0x000001D8, 0x0004B002); */
878 nes_write_indexed(nesdev, 0x000001FC, 0x00050005);
879 nes_write_indexed(nesdev, 0x00000600, 0x55555555);
880 nes_write_indexed(nesdev, 0x00000604, 0x55555555);
881
882 /* TODO: move these MAC register settings to NIC bringup */
883 nes_write_indexed(nesdev, 0x00002000, 0x00000001);
884 nes_write_indexed(nesdev, 0x00002004, 0x00000001);
885 nes_write_indexed(nesdev, 0x00002008, 0x0000FFFF);
886 nes_write_indexed(nesdev, 0x0000200C, 0x00000001);
887 nes_write_indexed(nesdev, 0x00002010, 0x000003c1);
888 nes_write_indexed(nesdev, 0x0000201C, 0x75345678);
889 if (port_count > 1) {
890 nes_write_indexed(nesdev, 0x00002200, 0x00000001);
891 nes_write_indexed(nesdev, 0x00002204, 0x00000001);
892 nes_write_indexed(nesdev, 0x00002208, 0x0000FFFF);
893 nes_write_indexed(nesdev, 0x0000220C, 0x00000001);
894 nes_write_indexed(nesdev, 0x00002210, 0x000003c1);
895 nes_write_indexed(nesdev, 0x0000221C, 0x75345678);
896 nes_write_indexed(nesdev, 0x00000908, 0x20000001);
897 }
898 if (port_count > 2) {
899 nes_write_indexed(nesdev, 0x00002400, 0x00000001);
900 nes_write_indexed(nesdev, 0x00002404, 0x00000001);
901 nes_write_indexed(nesdev, 0x00002408, 0x0000FFFF);
902 nes_write_indexed(nesdev, 0x0000240C, 0x00000001);
903 nes_write_indexed(nesdev, 0x00002410, 0x000003c1);
904 nes_write_indexed(nesdev, 0x0000241C, 0x75345678);
905 nes_write_indexed(nesdev, 0x00000910, 0x20000001);
906
907 nes_write_indexed(nesdev, 0x00002600, 0x00000001);
908 nes_write_indexed(nesdev, 0x00002604, 0x00000001);
909 nes_write_indexed(nesdev, 0x00002608, 0x0000FFFF);
910 nes_write_indexed(nesdev, 0x0000260C, 0x00000001);
911 nes_write_indexed(nesdev, 0x00002610, 0x000003c1);
912 nes_write_indexed(nesdev, 0x0000261C, 0x75345678);
913 nes_write_indexed(nesdev, 0x00000918, 0x20000001);
914 }
915
916 nes_write_indexed(nesdev, 0x00005000, 0x00018000);
917 /* nes_write_indexed(nesdev, 0x00005000, 0x00010000); */
918 nes_write_indexed(nesdev, NES_IDX_WQM_CONFIG1, (wqm_quanta << 1) |
919 0x00000001);
920 nes_write_indexed(nesdev, 0x00005008, 0x1F1F1F1F);
921 nes_write_indexed(nesdev, 0x00005010, 0x1F1F1F1F);
922 nes_write_indexed(nesdev, 0x00005018, 0x1F1F1F1F);
923 nes_write_indexed(nesdev, 0x00005020, 0x1F1F1F1F);
924 nes_write_indexed(nesdev, 0x00006090, 0xFFFFFFFF);
925
926 /* TODO: move this to code, get from EEPROM */
927 nes_write_indexed(nesdev, 0x00000900, 0x20000001);
928 nes_write_indexed(nesdev, 0x000060C0, 0x0000028e);
929 nes_write_indexed(nesdev, 0x000060C8, 0x00000020);
930
931 nes_write_indexed(nesdev, 0x000001EC, 0x7b2625a0);
932 /* nes_write_indexed(nesdev, 0x000001EC, 0x5f2625a0); */
933
934 if (hw_rev != NE020_REV) {
935 u32temp = nes_read_indexed(nesdev, 0x000008e8);
936 u32temp |= 0x80000000;
937 nes_write_indexed(nesdev, 0x000008e8, u32temp);
938 u32temp = nes_read_indexed(nesdev, 0x000021f8);
939 u32temp &= 0x7fffffff;
940 u32temp |= 0x7fff0010;
941 nes_write_indexed(nesdev, 0x000021f8, u32temp);
942 if (port_count > 1) {
943 u32temp = nes_read_indexed(nesdev, 0x000023f8);
944 u32temp &= 0x7fffffff;
945 u32temp |= 0x7fff0010;
946 nes_write_indexed(nesdev, 0x000023f8, u32temp);
947 }
948 }
949 }
950
951
952 /**
953 * nes_destroy_adapter - destroy the adapter structure
954 */
955 void nes_destroy_adapter(struct nes_adapter *nesadapter)
956 {
957 struct nes_adapter *tmp_adapter;
958
959 list_for_each_entry(tmp_adapter, &nes_adapter_list, list) {
960 nes_debug(NES_DBG_SHUTDOWN, "Nes Adapter list entry = 0x%p.\n",
961 tmp_adapter);
962 }
963
964 nesadapter->ref_count--;
965 if (!nesadapter->ref_count) {
966 if (nesadapter->hw_rev == NE020_REV) {
967 del_timer(&nesadapter->mh_timer);
968 }
969 del_timer(&nesadapter->lc_timer);
970
971 list_del(&nesadapter->list);
972 kfree(nesadapter);
973 }
974 }
975
976
977 /**
978 * nes_init_cqp
979 */
980 int nes_init_cqp(struct nes_device *nesdev)
981 {
982 struct nes_adapter *nesadapter = nesdev->nesadapter;
983 struct nes_hw_cqp_qp_context *cqp_qp_context;
984 struct nes_hw_cqp_wqe *cqp_wqe;
985 struct nes_hw_ceq *ceq;
986 struct nes_hw_ceq *nic_ceq;
987 struct nes_hw_aeq *aeq;
988 void *vmem;
989 dma_addr_t pmem;
990 u32 count=0;
991 u32 cqp_head;
992 u64 u64temp;
993 u32 u32temp;
994
995 /* allocate CQP memory */
996 /* Need to add max_cq to the aeq size once cq overflow checking is added back */
997 /* SQ is 512 byte aligned, others are 256 byte aligned */
998 nesdev->cqp_mem_size = 512 +
999 (sizeof(struct nes_hw_cqp_wqe) * NES_CQP_SQ_SIZE) +
1000 (sizeof(struct nes_hw_cqe) * NES_CCQ_SIZE) +
1001 max(((u32)sizeof(struct nes_hw_ceqe) * NES_CCEQ_SIZE), (u32)256) +
1002 max(((u32)sizeof(struct nes_hw_ceqe) * NES_NIC_CEQ_SIZE), (u32)256) +
1003 (sizeof(struct nes_hw_aeqe) * nesadapter->max_qp) +
1004 sizeof(struct nes_hw_cqp_qp_context);
1005
1006 nesdev->cqp_vbase = pci_zalloc_consistent(nesdev->pcidev,
1007 nesdev->cqp_mem_size,
1008 &nesdev->cqp_pbase);
1009 if (!nesdev->cqp_vbase) {
1010 nes_debug(NES_DBG_INIT, "Unable to allocate memory for host descriptor rings\n");
1011 return -ENOMEM;
1012 }
1013
1014 /* Allocate a twice the number of CQP requests as the SQ size */
1015 nesdev->nes_cqp_requests = kzalloc(sizeof(struct nes_cqp_request) *
1016 2 * NES_CQP_SQ_SIZE, GFP_KERNEL);
1017 if (nesdev->nes_cqp_requests == NULL) {
1018 nes_debug(NES_DBG_INIT, "Unable to allocate memory CQP request entries.\n");
1019 pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size, nesdev->cqp.sq_vbase,
1020 nesdev->cqp.sq_pbase);
1021 return -ENOMEM;
1022 }
1023
1024 nes_debug(NES_DBG_INIT, "Allocated CQP structures at %p (phys = %016lX), size = %u.\n",
1025 nesdev->cqp_vbase, (unsigned long)nesdev->cqp_pbase, nesdev->cqp_mem_size);
1026
1027 spin_lock_init(&nesdev->cqp.lock);
1028 init_waitqueue_head(&nesdev->cqp.waitq);
1029
1030 /* Setup Various Structures */
1031 vmem = (void *)(((unsigned long)nesdev->cqp_vbase + (512 - 1)) &
1032 ~(unsigned long)(512 - 1));
1033 pmem = (dma_addr_t)(((unsigned long long)nesdev->cqp_pbase + (512 - 1)) &
1034 ~(unsigned long long)(512 - 1));
1035
1036 nesdev->cqp.sq_vbase = vmem;
1037 nesdev->cqp.sq_pbase = pmem;
1038 nesdev->cqp.sq_size = NES_CQP_SQ_SIZE;
1039 nesdev->cqp.sq_head = 0;
1040 nesdev->cqp.sq_tail = 0;
1041 nesdev->cqp.qp_id = PCI_FUNC(nesdev->pcidev->devfn);
1042
1043 vmem += (sizeof(struct nes_hw_cqp_wqe) * nesdev->cqp.sq_size);
1044 pmem += (sizeof(struct nes_hw_cqp_wqe) * nesdev->cqp.sq_size);
1045
1046 nesdev->ccq.cq_vbase = vmem;
1047 nesdev->ccq.cq_pbase = pmem;
1048 nesdev->ccq.cq_size = NES_CCQ_SIZE;
1049 nesdev->ccq.cq_head = 0;
1050 nesdev->ccq.ce_handler = nes_cqp_ce_handler;
1051 nesdev->ccq.cq_number = PCI_FUNC(nesdev->pcidev->devfn);
1052
1053 vmem += (sizeof(struct nes_hw_cqe) * nesdev->ccq.cq_size);
1054 pmem += (sizeof(struct nes_hw_cqe) * nesdev->ccq.cq_size);
1055
1056 nesdev->ceq_index = PCI_FUNC(nesdev->pcidev->devfn);
1057 ceq = &nesadapter->ceq[nesdev->ceq_index];
1058 ceq->ceq_vbase = vmem;
1059 ceq->ceq_pbase = pmem;
1060 ceq->ceq_size = NES_CCEQ_SIZE;
1061 ceq->ceq_head = 0;
1062
1063 vmem += max(((u32)sizeof(struct nes_hw_ceqe) * ceq->ceq_size), (u32)256);
1064 pmem += max(((u32)sizeof(struct nes_hw_ceqe) * ceq->ceq_size), (u32)256);
1065
1066 nesdev->nic_ceq_index = PCI_FUNC(nesdev->pcidev->devfn) + 8;
1067 nic_ceq = &nesadapter->ceq[nesdev->nic_ceq_index];
1068 nic_ceq->ceq_vbase = vmem;
1069 nic_ceq->ceq_pbase = pmem;
1070 nic_ceq->ceq_size = NES_NIC_CEQ_SIZE;
1071 nic_ceq->ceq_head = 0;
1072
1073 vmem += max(((u32)sizeof(struct nes_hw_ceqe) * nic_ceq->ceq_size), (u32)256);
1074 pmem += max(((u32)sizeof(struct nes_hw_ceqe) * nic_ceq->ceq_size), (u32)256);
1075
1076 aeq = &nesadapter->aeq[PCI_FUNC(nesdev->pcidev->devfn)];
1077 aeq->aeq_vbase = vmem;
1078 aeq->aeq_pbase = pmem;
1079 aeq->aeq_size = nesadapter->max_qp;
1080 aeq->aeq_head = 0;
1081
1082 /* Setup QP Context */
1083 vmem += (sizeof(struct nes_hw_aeqe) * aeq->aeq_size);
1084 pmem += (sizeof(struct nes_hw_aeqe) * aeq->aeq_size);
1085
1086 cqp_qp_context = vmem;
1087 cqp_qp_context->context_words[0] =
1088 cpu_to_le32((PCI_FUNC(nesdev->pcidev->devfn) << 12) + (2 << 10));
1089 cqp_qp_context->context_words[1] = 0;
1090 cqp_qp_context->context_words[2] = cpu_to_le32((u32)nesdev->cqp.sq_pbase);
1091 cqp_qp_context->context_words[3] = cpu_to_le32(((u64)nesdev->cqp.sq_pbase) >> 32);
1092
1093
1094 /* Write the address to Create CQP */
1095 if ((sizeof(dma_addr_t) > 4)) {
1096 nes_write_indexed(nesdev,
1097 NES_IDX_CREATE_CQP_HIGH + (PCI_FUNC(nesdev->pcidev->devfn) * 8),
1098 ((u64)pmem) >> 32);
1099 } else {
1100 nes_write_indexed(nesdev,
1101 NES_IDX_CREATE_CQP_HIGH + (PCI_FUNC(nesdev->pcidev->devfn) * 8), 0);
1102 }
1103 nes_write_indexed(nesdev,
1104 NES_IDX_CREATE_CQP_LOW + (PCI_FUNC(nesdev->pcidev->devfn) * 8),
1105 (u32)pmem);
1106
1107 INIT_LIST_HEAD(&nesdev->cqp_avail_reqs);
1108 INIT_LIST_HEAD(&nesdev->cqp_pending_reqs);
1109
1110 for (count = 0; count < 2*NES_CQP_SQ_SIZE; count++) {
1111 init_waitqueue_head(&nesdev->nes_cqp_requests[count].waitq);
1112 list_add_tail(&nesdev->nes_cqp_requests[count].list, &nesdev->cqp_avail_reqs);
1113 }
1114
1115 /* Write Create CCQ WQE */
1116 cqp_head = nesdev->cqp.sq_head++;
1117 cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1118 nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1119 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1120 (NES_CQP_CREATE_CQ | NES_CQP_CQ_CEQ_VALID |
1121 NES_CQP_CQ_CHK_OVERFLOW | ((u32)nesdev->ccq.cq_size << 16)));
1122 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
1123 (nesdev->ccq.cq_number |
1124 ((u32)nesdev->ceq_index << 16)));
1125 u64temp = (u64)nesdev->ccq.cq_pbase;
1126 set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1127 cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] = 0;
1128 u64temp = (unsigned long)&nesdev->ccq;
1129 cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_LOW_IDX] =
1130 cpu_to_le32((u32)(u64temp >> 1));
1131 cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] =
1132 cpu_to_le32(((u32)((u64temp) >> 33)) & 0x7FFFFFFF);
1133 cqp_wqe->wqe_words[NES_CQP_CQ_WQE_DOORBELL_INDEX_HIGH_IDX] = 0;
1134
1135 /* Write Create CEQ WQE */
1136 cqp_head = nesdev->cqp.sq_head++;
1137 cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1138 nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1139 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1140 (NES_CQP_CREATE_CEQ + ((u32)nesdev->ceq_index << 8)));
1141 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_CEQ_WQE_ELEMENT_COUNT_IDX, ceq->ceq_size);
1142 u64temp = (u64)ceq->ceq_pbase;
1143 set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1144
1145 /* Write Create AEQ WQE */
1146 cqp_head = nesdev->cqp.sq_head++;
1147 cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1148 nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1149 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1150 (NES_CQP_CREATE_AEQ + ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 8)));
1151 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_AEQ_WQE_ELEMENT_COUNT_IDX, aeq->aeq_size);
1152 u64temp = (u64)aeq->aeq_pbase;
1153 set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1154
1155 /* Write Create NIC CEQ WQE */
1156 cqp_head = nesdev->cqp.sq_head++;
1157 cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1158 nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1159 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1160 (NES_CQP_CREATE_CEQ + ((u32)nesdev->nic_ceq_index << 8)));
1161 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_CEQ_WQE_ELEMENT_COUNT_IDX, nic_ceq->ceq_size);
1162 u64temp = (u64)nic_ceq->ceq_pbase;
1163 set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1164
1165 /* Poll until CCQP done */
1166 count = 0;
1167 do {
1168 if (count++ > 1000) {
1169 printk(KERN_ERR PFX "Error creating CQP\n");
1170 pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size,
1171 nesdev->cqp_vbase, nesdev->cqp_pbase);
1172 return -1;
1173 }
1174 udelay(10);
1175 } while (!(nes_read_indexed(nesdev,
1176 NES_IDX_QP_CONTROL + (PCI_FUNC(nesdev->pcidev->devfn) * 8)) & (1 << 8)));
1177
1178 nes_debug(NES_DBG_INIT, "CQP Status = 0x%08X\n", nes_read_indexed(nesdev,
1179 NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)));
1180
1181 u32temp = 0x04800000;
1182 nes_write32(nesdev->regs+NES_WQE_ALLOC, u32temp | nesdev->cqp.qp_id);
1183
1184 /* wait for the CCQ, CEQ, and AEQ to get created */
1185 count = 0;
1186 do {
1187 if (count++ > 1000) {
1188 printk(KERN_ERR PFX "Error creating CCQ, CEQ, and AEQ\n");
1189 pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size,
1190 nesdev->cqp_vbase, nesdev->cqp_pbase);
1191 return -1;
1192 }
1193 udelay(10);
1194 } while (((nes_read_indexed(nesdev,
1195 NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)) & (15<<8)) != (15<<8)));
1196
1197 /* dump the QP status value */
1198 nes_debug(NES_DBG_INIT, "QP Status = 0x%08X\n", nes_read_indexed(nesdev,
1199 NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)));
1200
1201 nesdev->cqp.sq_tail++;
1202
1203 return 0;
1204 }
1205
1206
1207 /**
1208 * nes_destroy_cqp
1209 */
1210 int nes_destroy_cqp(struct nes_device *nesdev)
1211 {
1212 struct nes_hw_cqp_wqe *cqp_wqe;
1213 u32 count = 0;
1214 u32 cqp_head;
1215 unsigned long flags;
1216
1217 do {
1218 if (count++ > 1000)
1219 break;
1220 udelay(10);
1221 } while (!(nesdev->cqp.sq_head == nesdev->cqp.sq_tail));
1222
1223 /* Reset CCQ */
1224 nes_write32(nesdev->regs+NES_CQE_ALLOC, NES_CQE_ALLOC_RESET |
1225 nesdev->ccq.cq_number);
1226
1227 /* Disable device interrupts */
1228 nes_write32(nesdev->regs+NES_INT_MASK, 0x7fffffff);
1229
1230 spin_lock_irqsave(&nesdev->cqp.lock, flags);
1231
1232 /* Destroy the AEQ */
1233 cqp_head = nesdev->cqp.sq_head++;
1234 nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1235 cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1236 cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_AEQ |
1237 ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 8));
1238 cqp_wqe->wqe_words[NES_CQP_WQE_COMP_CTX_HIGH_IDX] = 0;
1239
1240 /* Destroy the NIC CEQ */
1241 cqp_head = nesdev->cqp.sq_head++;
1242 nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1243 cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1244 cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_CEQ |
1245 ((u32)nesdev->nic_ceq_index << 8));
1246
1247 /* Destroy the CEQ */
1248 cqp_head = nesdev->cqp.sq_head++;
1249 nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1250 cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1251 cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_CEQ |
1252 (nesdev->ceq_index << 8));
1253
1254 /* Destroy the CCQ */
1255 cqp_head = nesdev->cqp.sq_head++;
1256 nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1257 cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1258 cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_CQ);
1259 cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesdev->ccq.cq_number |
1260 ((u32)nesdev->ceq_index << 16));
1261
1262 /* Destroy CQP */
1263 cqp_head = nesdev->cqp.sq_head++;
1264 nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1265 cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1266 cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_QP |
1267 NES_CQP_QP_TYPE_CQP);
1268 cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesdev->cqp.qp_id);
1269
1270 barrier();
1271 /* Ring doorbell (5 WQEs) */
1272 nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x05800000 | nesdev->cqp.qp_id);
1273
1274 spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
1275
1276 /* wait for the CCQ, CEQ, and AEQ to get destroyed */
1277 count = 0;
1278 do {
1279 if (count++ > 1000) {
1280 printk(KERN_ERR PFX "Function%d: Error destroying CCQ, CEQ, and AEQ\n",
1281 PCI_FUNC(nesdev->pcidev->devfn));
1282 break;
1283 }
1284 udelay(10);
1285 } while (((nes_read_indexed(nesdev,
1286 NES_IDX_QP_CONTROL + (PCI_FUNC(nesdev->pcidev->devfn)*8)) & (15 << 8)) != 0));
1287
1288 /* dump the QP status value */
1289 nes_debug(NES_DBG_SHUTDOWN, "Function%d: QP Status = 0x%08X\n",
1290 PCI_FUNC(nesdev->pcidev->devfn),
1291 nes_read_indexed(nesdev,
1292 NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)));
1293
1294 kfree(nesdev->nes_cqp_requests);
1295
1296 /* Free the control structures */
1297 pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size, nesdev->cqp.sq_vbase,
1298 nesdev->cqp.sq_pbase);
1299
1300 return 0;
1301 }
1302
1303
1304 /**
1305 * nes_init_1g_phy
1306 */
1307 static int nes_init_1g_phy(struct nes_device *nesdev, u8 phy_type, u8 phy_index)
1308 {
1309 u32 counter = 0;
1310 u16 phy_data;
1311 int ret = 0;
1312
1313 nes_read_1G_phy_reg(nesdev, 1, phy_index, &phy_data);
1314 nes_write_1G_phy_reg(nesdev, 23, phy_index, 0xb000);
1315
1316 /* Reset the PHY */
1317 nes_write_1G_phy_reg(nesdev, 0, phy_index, 0x8000);
1318 udelay(100);
1319 counter = 0;
1320 do {
1321 nes_read_1G_phy_reg(nesdev, 0, phy_index, &phy_data);
1322 if (counter++ > 100) {
1323 ret = -1;
1324 break;
1325 }
1326 } while (phy_data & 0x8000);
1327
1328 /* Setting no phy loopback */
1329 phy_data &= 0xbfff;
1330 phy_data |= 0x1140;
1331 nes_write_1G_phy_reg(nesdev, 0, phy_index, phy_data);
1332 nes_read_1G_phy_reg(nesdev, 0, phy_index, &phy_data);
1333 nes_read_1G_phy_reg(nesdev, 0x17, phy_index, &phy_data);
1334 nes_read_1G_phy_reg(nesdev, 0x1e, phy_index, &phy_data);
1335
1336 /* Setting the interrupt mask */
1337 nes_read_1G_phy_reg(nesdev, 0x19, phy_index, &phy_data);
1338 nes_write_1G_phy_reg(nesdev, 0x19, phy_index, 0xffee);
1339 nes_read_1G_phy_reg(nesdev, 0x19, phy_index, &phy_data);
1340
1341 /* turning on flow control */
1342 nes_read_1G_phy_reg(nesdev, 4, phy_index, &phy_data);
1343 nes_write_1G_phy_reg(nesdev, 4, phy_index, (phy_data & ~(0x03E0)) | 0xc00);
1344 nes_read_1G_phy_reg(nesdev, 4, phy_index, &phy_data);
1345
1346 /* Clear Half duplex */
1347 nes_read_1G_phy_reg(nesdev, 9, phy_index, &phy_data);
1348 nes_write_1G_phy_reg(nesdev, 9, phy_index, phy_data & ~(0x0100));
1349 nes_read_1G_phy_reg(nesdev, 9, phy_index, &phy_data);
1350
1351 nes_read_1G_phy_reg(nesdev, 0, phy_index, &phy_data);
1352 nes_write_1G_phy_reg(nesdev, 0, phy_index, phy_data | 0x0300);
1353
1354 return ret;
1355 }
1356
1357
1358 /**
1359 * nes_init_2025_phy
1360 */
1361 static int nes_init_2025_phy(struct nes_device *nesdev, u8 phy_type, u8 phy_index)
1362 {
1363 u32 temp_phy_data = 0;
1364 u32 temp_phy_data2 = 0;
1365 u32 counter = 0;
1366 u32 sds;
1367 u32 mac_index = nesdev->mac_index;
1368 int ret = 0;
1369 unsigned int first_attempt = 1;
1370
1371 /* Check firmware heartbeat */
1372 nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7ee);
1373 temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1374 udelay(1500);
1375 nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7ee);
1376 temp_phy_data2 = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1377
1378 if (temp_phy_data != temp_phy_data2) {
1379 nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7fd);
1380 temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1381 if ((temp_phy_data & 0xff) > 0x20)
1382 return 0;
1383 printk(PFX "Reinitialize external PHY\n");
1384 }
1385
1386 /* no heartbeat, configure the PHY */
1387 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0x0000, 0x8000);
1388 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc300, 0x0000);
1389 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc316, 0x000A);
1390 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc318, 0x0052);
1391
1392 switch (phy_type) {
1393 case NES_PHY_TYPE_ARGUS:
1394 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc316, 0x000A);
1395 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc318, 0x0052);
1396 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc302, 0x000C);
1397 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc319, 0x0008);
1398 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0027, 0x0001);
1399 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc31a, 0x0098);
1400 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0026, 0x0E00);
1401
1402 /* setup LEDs */
1403 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd006, 0x0007);
1404 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd007, 0x000A);
1405 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd008, 0x0009);
1406 break;
1407
1408 case NES_PHY_TYPE_SFP_D:
1409 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc316, 0x000A);
1410 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc318, 0x0052);
1411 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc302, 0x0004);
1412 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc319, 0x0038);
1413 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0027, 0x0013);
1414 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc31a, 0x0098);
1415 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0026, 0x0E00);
1416
1417 /* setup LEDs */
1418 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd006, 0x0007);
1419 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd007, 0x000A);
1420 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd008, 0x0009);
1421 break;
1422
1423 case NES_PHY_TYPE_KR:
1424 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc316, 0x000A);
1425 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc318, 0x0052);
1426 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc302, 0x000C);
1427 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc319, 0x0010);
1428 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0027, 0x0013);
1429 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc31a, 0x0080);
1430 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0026, 0x0E00);
1431
1432 /* setup LEDs */
1433 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd006, 0x000B);
1434 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd007, 0x0003);
1435 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd008, 0x0004);
1436
1437 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0022, 0x406D);
1438 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0023, 0x0020);
1439 break;
1440 }
1441
1442 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0028, 0xA528);
1443
1444 /* Bring PHY out of reset */
1445 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc300, 0x0002);
1446
1447 /* Check for heartbeat */
1448 counter = 0;
1449 mdelay(690);
1450 nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7ee);
1451 temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1452 do {
1453 if (counter++ > 150) {
1454 printk(PFX "No PHY heartbeat\n");
1455 break;
1456 }
1457 mdelay(1);
1458 nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7ee);
1459 temp_phy_data2 = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1460 } while ((temp_phy_data2 == temp_phy_data));
1461
1462 /* wait for tracking */
1463 counter = 0;
1464 do {
1465 nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7fd);
1466 temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1467 if (counter++ > 300) {
1468 if (((temp_phy_data & 0xff) == 0x0) && first_attempt) {
1469 first_attempt = 0;
1470 counter = 0;
1471 /* reset AMCC PHY and try again */
1472 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0xe854, 0x00c0);
1473 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0xe854, 0x0040);
1474 continue;
1475 } else {
1476 ret = 1;
1477 break;
1478 }
1479 }
1480 mdelay(10);
1481 } while ((temp_phy_data & 0xff) < 0x30);
1482
1483 /* setup signal integrity */
1484 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd003, 0x0000);
1485 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xF00D, 0x00FE);
1486 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xF00E, 0x0032);
1487 if (phy_type == NES_PHY_TYPE_KR) {
1488 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xF00F, 0x000C);
1489 } else {
1490 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xF00F, 0x0002);
1491 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc314, 0x0063);
1492 }
1493
1494 /* reset serdes */
1495 sds = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0 + mac_index * 0x200);
1496 sds |= 0x1;
1497 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0 + mac_index * 0x200, sds);
1498 sds &= 0xfffffffe;
1499 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0 + mac_index * 0x200, sds);
1500
1501 counter = 0;
1502 while (((nes_read32(nesdev->regs + NES_SOFTWARE_RESET) & 0x00000040) != 0x00000040)
1503 && (counter++ < 5000))
1504 ;
1505
1506 return ret;
1507 }
1508
1509
1510 /**
1511 * nes_init_phy
1512 */
1513 int nes_init_phy(struct nes_device *nesdev)
1514 {
1515 struct nes_adapter *nesadapter = nesdev->nesadapter;
1516 u32 mac_index = nesdev->mac_index;
1517 u32 tx_config = 0;
1518 unsigned long flags;
1519 u8 phy_type = nesadapter->phy_type[mac_index];
1520 u8 phy_index = nesadapter->phy_index[mac_index];
1521 int ret = 0;
1522
1523 tx_config = nes_read_indexed(nesdev, NES_IDX_MAC_TX_CONFIG);
1524 if (phy_type == NES_PHY_TYPE_1G) {
1525 /* setup 1G MDIO operation */
1526 tx_config &= 0xFFFFFFE3;
1527 tx_config |= 0x04;
1528 } else {
1529 /* setup 10G MDIO operation */
1530 tx_config &= 0xFFFFFFE3;
1531 tx_config |= 0x1D;
1532 }
1533 nes_write_indexed(nesdev, NES_IDX_MAC_TX_CONFIG, tx_config);
1534
1535 spin_lock_irqsave(&nesdev->nesadapter->phy_lock, flags);
1536
1537 switch (phy_type) {
1538 case NES_PHY_TYPE_1G:
1539 ret = nes_init_1g_phy(nesdev, phy_type, phy_index);
1540 break;
1541 case NES_PHY_TYPE_ARGUS:
1542 case NES_PHY_TYPE_SFP_D:
1543 case NES_PHY_TYPE_KR:
1544 ret = nes_init_2025_phy(nesdev, phy_type, phy_index);
1545 break;
1546 }
1547
1548 spin_unlock_irqrestore(&nesdev->nesadapter->phy_lock, flags);
1549
1550 return ret;
1551 }
1552
1553
1554 /**
1555 * nes_replenish_nic_rq
1556 */
1557 static void nes_replenish_nic_rq(struct nes_vnic *nesvnic)
1558 {
1559 unsigned long flags;
1560 dma_addr_t bus_address;
1561 struct sk_buff *skb;
1562 struct nes_hw_nic_rq_wqe *nic_rqe;
1563 struct nes_hw_nic *nesnic;
1564 struct nes_device *nesdev;
1565 struct nes_rskb_cb *cb;
1566 u32 rx_wqes_posted = 0;
1567
1568 nesnic = &nesvnic->nic;
1569 nesdev = nesvnic->nesdev;
1570 spin_lock_irqsave(&nesnic->rq_lock, flags);
1571 if (nesnic->replenishing_rq !=0) {
1572 if (((nesnic->rq_size-1) == atomic_read(&nesvnic->rx_skbs_needed)) &&
1573 (atomic_read(&nesvnic->rx_skb_timer_running) == 0)) {
1574 atomic_set(&nesvnic->rx_skb_timer_running, 1);
1575 spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1576 nesvnic->rq_wqes_timer.expires = jiffies + (HZ/2); /* 1/2 second */
1577 add_timer(&nesvnic->rq_wqes_timer);
1578 } else
1579 spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1580 return;
1581 }
1582 nesnic->replenishing_rq = 1;
1583 spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1584 do {
1585 skb = dev_alloc_skb(nesvnic->max_frame_size);
1586 if (skb) {
1587 skb->dev = nesvnic->netdev;
1588
1589 bus_address = pci_map_single(nesdev->pcidev,
1590 skb->data, nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
1591 cb = (struct nes_rskb_cb *)&skb->cb[0];
1592 cb->busaddr = bus_address;
1593 cb->maplen = nesvnic->max_frame_size;
1594
1595 nic_rqe = &nesnic->rq_vbase[nesvnic->nic.rq_head];
1596 nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_1_0_IDX] =
1597 cpu_to_le32(nesvnic->max_frame_size);
1598 nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_3_2_IDX] = 0;
1599 nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX] =
1600 cpu_to_le32((u32)bus_address);
1601 nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX] =
1602 cpu_to_le32((u32)((u64)bus_address >> 32));
1603 nesnic->rx_skb[nesnic->rq_head] = skb;
1604 nesnic->rq_head++;
1605 nesnic->rq_head &= nesnic->rq_size - 1;
1606 atomic_dec(&nesvnic->rx_skbs_needed);
1607 barrier();
1608 if (++rx_wqes_posted == 255) {
1609 nes_write32(nesdev->regs+NES_WQE_ALLOC, (rx_wqes_posted << 24) | nesnic->qp_id);
1610 rx_wqes_posted = 0;
1611 }
1612 } else {
1613 spin_lock_irqsave(&nesnic->rq_lock, flags);
1614 if (((nesnic->rq_size-1) == atomic_read(&nesvnic->rx_skbs_needed)) &&
1615 (atomic_read(&nesvnic->rx_skb_timer_running) == 0)) {
1616 atomic_set(&nesvnic->rx_skb_timer_running, 1);
1617 spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1618 nesvnic->rq_wqes_timer.expires = jiffies + (HZ/2); /* 1/2 second */
1619 add_timer(&nesvnic->rq_wqes_timer);
1620 } else
1621 spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1622 break;
1623 }
1624 } while (atomic_read(&nesvnic->rx_skbs_needed));
1625 barrier();
1626 if (rx_wqes_posted)
1627 nes_write32(nesdev->regs+NES_WQE_ALLOC, (rx_wqes_posted << 24) | nesnic->qp_id);
1628 nesnic->replenishing_rq = 0;
1629 }
1630
1631
1632 /**
1633 * nes_rq_wqes_timeout
1634 */
1635 static void nes_rq_wqes_timeout(unsigned long parm)
1636 {
1637 struct nes_vnic *nesvnic = (struct nes_vnic *)parm;
1638 printk("%s: Timer fired.\n", __func__);
1639 atomic_set(&nesvnic->rx_skb_timer_running, 0);
1640 if (atomic_read(&nesvnic->rx_skbs_needed))
1641 nes_replenish_nic_rq(nesvnic);
1642 }
1643
1644
1645 static int nes_lro_get_skb_hdr(struct sk_buff *skb, void **iphdr,
1646 void **tcph, u64 *hdr_flags, void *priv)
1647 {
1648 unsigned int ip_len;
1649 struct iphdr *iph;
1650 skb_reset_network_header(skb);
1651 iph = ip_hdr(skb);
1652 if (iph->protocol != IPPROTO_TCP)
1653 return -1;
1654 ip_len = ip_hdrlen(skb);
1655 skb_set_transport_header(skb, ip_len);
1656 *tcph = tcp_hdr(skb);
1657
1658 *hdr_flags = LRO_IPV4 | LRO_TCP;
1659 *iphdr = iph;
1660 return 0;
1661 }
1662
1663
1664 /**
1665 * nes_init_nic_qp
1666 */
1667 int nes_init_nic_qp(struct nes_device *nesdev, struct net_device *netdev)
1668 {
1669 struct nes_hw_cqp_wqe *cqp_wqe;
1670 struct nes_hw_nic_sq_wqe *nic_sqe;
1671 struct nes_hw_nic_qp_context *nic_context;
1672 struct sk_buff *skb;
1673 struct nes_hw_nic_rq_wqe *nic_rqe;
1674 struct nes_vnic *nesvnic = netdev_priv(netdev);
1675 unsigned long flags;
1676 void *vmem;
1677 dma_addr_t pmem;
1678 u64 u64temp;
1679 int ret;
1680 u32 cqp_head;
1681 u32 counter;
1682 u32 wqe_count;
1683 struct nes_rskb_cb *cb;
1684 u8 jumbomode=0;
1685
1686 /* Allocate fragment, SQ, RQ, and CQ; Reuse CEQ based on the PCI function */
1687 nesvnic->nic_mem_size = 256 +
1688 (NES_NIC_WQ_SIZE * sizeof(struct nes_first_frag)) +
1689 (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_sq_wqe)) +
1690 (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_rq_wqe)) +
1691 (NES_NIC_WQ_SIZE * 2 * sizeof(struct nes_hw_nic_cqe)) +
1692 sizeof(struct nes_hw_nic_qp_context);
1693
1694 nesvnic->nic_vbase = pci_zalloc_consistent(nesdev->pcidev,
1695 nesvnic->nic_mem_size,
1696 &nesvnic->nic_pbase);
1697 if (!nesvnic->nic_vbase) {
1698 nes_debug(NES_DBG_INIT, "Unable to allocate memory for NIC host descriptor rings\n");
1699 return -ENOMEM;
1700 }
1701 nes_debug(NES_DBG_INIT, "Allocated NIC QP structures at %p (phys = %016lX), size = %u.\n",
1702 nesvnic->nic_vbase, (unsigned long)nesvnic->nic_pbase, nesvnic->nic_mem_size);
1703
1704 vmem = (void *)(((unsigned long)nesvnic->nic_vbase + (256 - 1)) &
1705 ~(unsigned long)(256 - 1));
1706 pmem = (dma_addr_t)(((unsigned long long)nesvnic->nic_pbase + (256 - 1)) &
1707 ~(unsigned long long)(256 - 1));
1708
1709 /* Setup the first Fragment buffers */
1710 nesvnic->nic.first_frag_vbase = vmem;
1711
1712 for (counter = 0; counter < NES_NIC_WQ_SIZE; counter++) {
1713 nesvnic->nic.frag_paddr[counter] = pmem;
1714 pmem += sizeof(struct nes_first_frag);
1715 }
1716
1717 /* setup the SQ */
1718 vmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_first_frag));
1719
1720 nesvnic->nic.sq_vbase = (void *)vmem;
1721 nesvnic->nic.sq_pbase = pmem;
1722 nesvnic->nic.sq_head = 0;
1723 nesvnic->nic.sq_tail = 0;
1724 nesvnic->nic.sq_size = NES_NIC_WQ_SIZE;
1725 for (counter = 0; counter < NES_NIC_WQ_SIZE; counter++) {
1726 nic_sqe = &nesvnic->nic.sq_vbase[counter];
1727 nic_sqe->wqe_words[NES_NIC_SQ_WQE_MISC_IDX] =
1728 cpu_to_le32(NES_NIC_SQ_WQE_DISABLE_CHKSUM |
1729 NES_NIC_SQ_WQE_COMPLETION);
1730 nic_sqe->wqe_words[NES_NIC_SQ_WQE_LENGTH_0_TAG_IDX] =
1731 cpu_to_le32((u32)NES_FIRST_FRAG_SIZE << 16);
1732 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX] =
1733 cpu_to_le32((u32)nesvnic->nic.frag_paddr[counter]);
1734 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX] =
1735 cpu_to_le32((u32)((u64)nesvnic->nic.frag_paddr[counter] >> 32));
1736 }
1737
1738 nesvnic->get_cqp_request = nes_get_cqp_request;
1739 nesvnic->post_cqp_request = nes_post_cqp_request;
1740 nesvnic->mcrq_mcast_filter = NULL;
1741
1742 spin_lock_init(&nesvnic->nic.rq_lock);
1743
1744 /* setup the RQ */
1745 vmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_sq_wqe));
1746 pmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_sq_wqe));
1747
1748
1749 nesvnic->nic.rq_vbase = vmem;
1750 nesvnic->nic.rq_pbase = pmem;
1751 nesvnic->nic.rq_head = 0;
1752 nesvnic->nic.rq_tail = 0;
1753 nesvnic->nic.rq_size = NES_NIC_WQ_SIZE;
1754
1755 /* setup the CQ */
1756 vmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_rq_wqe));
1757 pmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_rq_wqe));
1758
1759 if (nesdev->nesadapter->netdev_count > 2)
1760 nesvnic->mcrq_qp_id = nesvnic->nic_index + 32;
1761 else
1762 nesvnic->mcrq_qp_id = nesvnic->nic.qp_id + 4;
1763
1764 nesvnic->nic_cq.cq_vbase = vmem;
1765 nesvnic->nic_cq.cq_pbase = pmem;
1766 nesvnic->nic_cq.cq_head = 0;
1767 nesvnic->nic_cq.cq_size = NES_NIC_WQ_SIZE * 2;
1768
1769 nesvnic->nic_cq.ce_handler = nes_nic_napi_ce_handler;
1770
1771 /* Send CreateCQ request to CQP */
1772 spin_lock_irqsave(&nesdev->cqp.lock, flags);
1773 cqp_head = nesdev->cqp.sq_head;
1774
1775 cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1776 nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1777
1778 cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(
1779 NES_CQP_CREATE_CQ | NES_CQP_CQ_CEQ_VALID |
1780 ((u32)nesvnic->nic_cq.cq_size << 16));
1781 cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(
1782 nesvnic->nic_cq.cq_number | ((u32)nesdev->nic_ceq_index << 16));
1783 u64temp = (u64)nesvnic->nic_cq.cq_pbase;
1784 set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1785 cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] = 0;
1786 u64temp = (unsigned long)&nesvnic->nic_cq;
1787 cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_LOW_IDX] = cpu_to_le32((u32)(u64temp >> 1));
1788 cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] =
1789 cpu_to_le32(((u32)((u64temp) >> 33)) & 0x7FFFFFFF);
1790 cqp_wqe->wqe_words[NES_CQP_CQ_WQE_DOORBELL_INDEX_HIGH_IDX] = 0;
1791 if (++cqp_head >= nesdev->cqp.sq_size)
1792 cqp_head = 0;
1793 cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1794 nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1795
1796 /* Send CreateQP request to CQP */
1797 nic_context = (void *)(&nesvnic->nic_cq.cq_vbase[nesvnic->nic_cq.cq_size]);
1798 nic_context->context_words[NES_NIC_CTX_MISC_IDX] =
1799 cpu_to_le32((u32)NES_NIC_CTX_SIZE |
1800 ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 12));
1801 nes_debug(NES_DBG_INIT, "RX_WINDOW_BUFFER_PAGE_TABLE_SIZE = 0x%08X, RX_WINDOW_BUFFER_SIZE = 0x%08X\n",
1802 nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_PAGE_TABLE_SIZE),
1803 nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_SIZE));
1804 if (nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_SIZE) != 0) {
1805 nic_context->context_words[NES_NIC_CTX_MISC_IDX] |= cpu_to_le32(NES_NIC_BACK_STORE);
1806 }
1807
1808 u64temp = (u64)nesvnic->nic.sq_pbase;
1809 nic_context->context_words[NES_NIC_CTX_SQ_LOW_IDX] = cpu_to_le32((u32)u64temp);
1810 nic_context->context_words[NES_NIC_CTX_SQ_HIGH_IDX] = cpu_to_le32((u32)(u64temp >> 32));
1811 u64temp = (u64)nesvnic->nic.rq_pbase;
1812 nic_context->context_words[NES_NIC_CTX_RQ_LOW_IDX] = cpu_to_le32((u32)u64temp);
1813 nic_context->context_words[NES_NIC_CTX_RQ_HIGH_IDX] = cpu_to_le32((u32)(u64temp >> 32));
1814
1815 cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_CREATE_QP |
1816 NES_CQP_QP_TYPE_NIC);
1817 cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesvnic->nic.qp_id);
1818 u64temp = (u64)nesvnic->nic_cq.cq_pbase +
1819 (nesvnic->nic_cq.cq_size * sizeof(struct nes_hw_nic_cqe));
1820 set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_QP_WQE_CONTEXT_LOW_IDX, u64temp);
1821
1822 if (++cqp_head >= nesdev->cqp.sq_size)
1823 cqp_head = 0;
1824 nesdev->cqp.sq_head = cqp_head;
1825
1826 barrier();
1827
1828 /* Ring doorbell (2 WQEs) */
1829 nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x02800000 | nesdev->cqp.qp_id);
1830
1831 spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
1832 nes_debug(NES_DBG_INIT, "Waiting for create NIC QP%u to complete.\n",
1833 nesvnic->nic.qp_id);
1834
1835 ret = wait_event_timeout(nesdev->cqp.waitq, (nesdev->cqp.sq_tail == cqp_head),
1836 NES_EVENT_TIMEOUT);
1837 nes_debug(NES_DBG_INIT, "Create NIC QP%u completed, wait_event_timeout ret = %u.\n",
1838 nesvnic->nic.qp_id, ret);
1839 if (!ret) {
1840 nes_debug(NES_DBG_INIT, "NIC QP%u create timeout expired\n", nesvnic->nic.qp_id);
1841 pci_free_consistent(nesdev->pcidev, nesvnic->nic_mem_size, nesvnic->nic_vbase,
1842 nesvnic->nic_pbase);
1843 return -EIO;
1844 }
1845
1846 /* Populate the RQ */
1847 for (counter = 0; counter < (NES_NIC_WQ_SIZE - 1); counter++) {
1848 skb = dev_alloc_skb(nesvnic->max_frame_size);
1849 if (!skb) {
1850 nes_debug(NES_DBG_INIT, "%s: out of memory for receive skb\n", netdev->name);
1851
1852 nes_destroy_nic_qp(nesvnic);
1853 return -ENOMEM;
1854 }
1855
1856 skb->dev = netdev;
1857
1858 pmem = pci_map_single(nesdev->pcidev, skb->data,
1859 nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
1860 cb = (struct nes_rskb_cb *)&skb->cb[0];
1861 cb->busaddr = pmem;
1862 cb->maplen = nesvnic->max_frame_size;
1863
1864 nic_rqe = &nesvnic->nic.rq_vbase[counter];
1865 nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_1_0_IDX] = cpu_to_le32(nesvnic->max_frame_size);
1866 nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_3_2_IDX] = 0;
1867 nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX] = cpu_to_le32((u32)pmem);
1868 nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX] = cpu_to_le32((u32)((u64)pmem >> 32));
1869 nesvnic->nic.rx_skb[counter] = skb;
1870 }
1871
1872 wqe_count = NES_NIC_WQ_SIZE - 1;
1873 nesvnic->nic.rq_head = wqe_count;
1874 barrier();
1875 do {
1876 counter = min(wqe_count, ((u32)255));
1877 wqe_count -= counter;
1878 nes_write32(nesdev->regs+NES_WQE_ALLOC, (counter << 24) | nesvnic->nic.qp_id);
1879 } while (wqe_count);
1880 init_timer(&nesvnic->rq_wqes_timer);
1881 nesvnic->rq_wqes_timer.function = nes_rq_wqes_timeout;
1882 nesvnic->rq_wqes_timer.data = (unsigned long)nesvnic;
1883 nes_debug(NES_DBG_INIT, "NAPI support Enabled\n");
1884 if (nesdev->nesadapter->et_use_adaptive_rx_coalesce)
1885 {
1886 nes_nic_init_timer(nesdev);
1887 if (netdev->mtu > 1500)
1888 jumbomode = 1;
1889 nes_nic_init_timer_defaults(nesdev, jumbomode);
1890 }
1891 if ((nesdev->nesadapter->allow_unaligned_fpdus) &&
1892 (nes_init_mgt_qp(nesdev, netdev, nesvnic))) {
1893 nes_debug(NES_DBG_INIT, "%s: Out of memory for pau nic\n", netdev->name);
1894 nes_destroy_nic_qp(nesvnic);
1895 return -ENOMEM;
1896 }
1897
1898 nesvnic->lro_mgr.max_aggr = nes_lro_max_aggr;
1899 nesvnic->lro_mgr.max_desc = NES_MAX_LRO_DESCRIPTORS;
1900 nesvnic->lro_mgr.lro_arr = nesvnic->lro_desc;
1901 nesvnic->lro_mgr.get_skb_header = nes_lro_get_skb_hdr;
1902 nesvnic->lro_mgr.features = LRO_F_NAPI | LRO_F_EXTRACT_VLAN_ID;
1903 nesvnic->lro_mgr.dev = netdev;
1904 nesvnic->lro_mgr.ip_summed = CHECKSUM_UNNECESSARY;
1905 nesvnic->lro_mgr.ip_summed_aggr = CHECKSUM_UNNECESSARY;
1906 return 0;
1907 }
1908
1909
1910 /**
1911 * nes_destroy_nic_qp
1912 */
1913 void nes_destroy_nic_qp(struct nes_vnic *nesvnic)
1914 {
1915 u64 u64temp;
1916 dma_addr_t bus_address;
1917 struct nes_device *nesdev = nesvnic->nesdev;
1918 struct nes_hw_cqp_wqe *cqp_wqe;
1919 struct nes_hw_nic_sq_wqe *nic_sqe;
1920 __le16 *wqe_fragment_length;
1921 u16 wqe_fragment_index;
1922 u32 cqp_head;
1923 u32 wqm_cfg0;
1924 unsigned long flags;
1925 struct sk_buff *rx_skb;
1926 struct nes_rskb_cb *cb;
1927 int ret;
1928
1929 if (nesdev->nesadapter->allow_unaligned_fpdus)
1930 nes_destroy_mgt(nesvnic);
1931
1932 /* clear wqe stall before destroying NIC QP */
1933 wqm_cfg0 = nes_read_indexed(nesdev, NES_IDX_WQM_CONFIG0);
1934 nes_write_indexed(nesdev, NES_IDX_WQM_CONFIG0, wqm_cfg0 & 0xFFFF7FFF);
1935
1936 /* Free remaining NIC receive buffers */
1937 while (nesvnic->nic.rq_head != nesvnic->nic.rq_tail) {
1938 rx_skb = nesvnic->nic.rx_skb[nesvnic->nic.rq_tail];
1939 cb = (struct nes_rskb_cb *)&rx_skb->cb[0];
1940 pci_unmap_single(nesdev->pcidev, cb->busaddr, cb->maplen,
1941 PCI_DMA_FROMDEVICE);
1942
1943 dev_kfree_skb(nesvnic->nic.rx_skb[nesvnic->nic.rq_tail++]);
1944 nesvnic->nic.rq_tail &= (nesvnic->nic.rq_size - 1);
1945 }
1946
1947 /* Free remaining NIC transmit buffers */
1948 while (nesvnic->nic.sq_head != nesvnic->nic.sq_tail) {
1949 nic_sqe = &nesvnic->nic.sq_vbase[nesvnic->nic.sq_tail];
1950 wqe_fragment_index = 1;
1951 wqe_fragment_length = (__le16 *)
1952 &nic_sqe->wqe_words[NES_NIC_SQ_WQE_LENGTH_0_TAG_IDX];
1953 /* bump past the vlan tag */
1954 wqe_fragment_length++;
1955 if (le16_to_cpu(wqe_fragment_length[wqe_fragment_index]) != 0) {
1956 u64temp = (u64)le32_to_cpu(
1957 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX+
1958 wqe_fragment_index*2]);
1959 u64temp += ((u64)le32_to_cpu(
1960 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX
1961 + wqe_fragment_index*2]))<<32;
1962 bus_address = (dma_addr_t)u64temp;
1963 if (test_and_clear_bit(nesvnic->nic.sq_tail,
1964 nesvnic->nic.first_frag_overflow)) {
1965 pci_unmap_single(nesdev->pcidev,
1966 bus_address,
1967 le16_to_cpu(wqe_fragment_length[
1968 wqe_fragment_index++]),
1969 PCI_DMA_TODEVICE);
1970 }
1971 for (; wqe_fragment_index < 5; wqe_fragment_index++) {
1972 if (wqe_fragment_length[wqe_fragment_index]) {
1973 u64temp = le32_to_cpu(
1974 nic_sqe->wqe_words[
1975 NES_NIC_SQ_WQE_FRAG0_LOW_IDX+
1976 wqe_fragment_index*2]);
1977 u64temp += ((u64)le32_to_cpu(
1978 nic_sqe->wqe_words[
1979 NES_NIC_SQ_WQE_FRAG0_HIGH_IDX+
1980 wqe_fragment_index*2]))<<32;
1981 bus_address = (dma_addr_t)u64temp;
1982 pci_unmap_page(nesdev->pcidev,
1983 bus_address,
1984 le16_to_cpu(
1985 wqe_fragment_length[
1986 wqe_fragment_index]),
1987 PCI_DMA_TODEVICE);
1988 } else
1989 break;
1990 }
1991 }
1992 if (nesvnic->nic.tx_skb[nesvnic->nic.sq_tail])
1993 dev_kfree_skb(
1994 nesvnic->nic.tx_skb[nesvnic->nic.sq_tail]);
1995
1996 nesvnic->nic.sq_tail = (nesvnic->nic.sq_tail + 1)
1997 & (nesvnic->nic.sq_size - 1);
1998 }
1999
2000 spin_lock_irqsave(&nesdev->cqp.lock, flags);
2001
2002 /* Destroy NIC QP */
2003 cqp_head = nesdev->cqp.sq_head;
2004 cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
2005 nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
2006
2007 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
2008 (NES_CQP_DESTROY_QP | NES_CQP_QP_TYPE_NIC));
2009 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
2010 nesvnic->nic.qp_id);
2011
2012 if (++cqp_head >= nesdev->cqp.sq_size)
2013 cqp_head = 0;
2014
2015 cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
2016
2017 /* Destroy NIC CQ */
2018 nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
2019 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
2020 (NES_CQP_DESTROY_CQ | ((u32)nesvnic->nic_cq.cq_size << 16)));
2021 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
2022 (nesvnic->nic_cq.cq_number | ((u32)nesdev->nic_ceq_index << 16)));
2023
2024 if (++cqp_head >= nesdev->cqp.sq_size)
2025 cqp_head = 0;
2026
2027 nesdev->cqp.sq_head = cqp_head;
2028 barrier();
2029
2030 /* Ring doorbell (2 WQEs) */
2031 nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x02800000 | nesdev->cqp.qp_id);
2032
2033 spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
2034 nes_debug(NES_DBG_SHUTDOWN, "Waiting for CQP, cqp_head=%u, cqp.sq_head=%u,"
2035 " cqp.sq_tail=%u, cqp.sq_size=%u\n",
2036 cqp_head, nesdev->cqp.sq_head,
2037 nesdev->cqp.sq_tail, nesdev->cqp.sq_size);
2038
2039 ret = wait_event_timeout(nesdev->cqp.waitq, (nesdev->cqp.sq_tail == cqp_head),
2040 NES_EVENT_TIMEOUT);
2041
2042 nes_debug(NES_DBG_SHUTDOWN, "Destroy NIC QP returned, wait_event_timeout ret = %u, cqp_head=%u,"
2043 " cqp.sq_head=%u, cqp.sq_tail=%u\n",
2044 ret, cqp_head, nesdev->cqp.sq_head, nesdev->cqp.sq_tail);
2045 if (!ret) {
2046 nes_debug(NES_DBG_SHUTDOWN, "NIC QP%u destroy timeout expired\n",
2047 nesvnic->nic.qp_id);
2048 }
2049
2050 pci_free_consistent(nesdev->pcidev, nesvnic->nic_mem_size, nesvnic->nic_vbase,
2051 nesvnic->nic_pbase);
2052
2053 /* restore old wqm_cfg0 value */
2054 nes_write_indexed(nesdev, NES_IDX_WQM_CONFIG0, wqm_cfg0);
2055 }
2056
2057 /**
2058 * nes_napi_isr
2059 */
2060 int nes_napi_isr(struct nes_device *nesdev)
2061 {
2062 struct nes_adapter *nesadapter = nesdev->nesadapter;
2063 u32 int_stat;
2064
2065 if (nesdev->napi_isr_ran) {
2066 /* interrupt status has already been read in ISR */
2067 int_stat = nesdev->int_stat;
2068 } else {
2069 int_stat = nes_read32(nesdev->regs + NES_INT_STAT);
2070 nesdev->int_stat = int_stat;
2071 nesdev->napi_isr_ran = 1;
2072 }
2073
2074 int_stat &= nesdev->int_req;
2075 /* iff NIC, process here, else wait for DPC */
2076 if ((int_stat) && ((int_stat & 0x0000ff00) == int_stat)) {
2077 nesdev->napi_isr_ran = 0;
2078 nes_write32(nesdev->regs + NES_INT_STAT,
2079 (int_stat &
2080 ~(NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0 | NES_INT_MAC1 | NES_INT_MAC2 | NES_INT_MAC3)));
2081
2082 /* Process the CEQs */
2083 nes_process_ceq(nesdev, &nesdev->nesadapter->ceq[nesdev->nic_ceq_index]);
2084
2085 if (unlikely((((nesadapter->et_rx_coalesce_usecs_irq) &&
2086 (!nesadapter->et_use_adaptive_rx_coalesce)) ||
2087 ((nesadapter->et_use_adaptive_rx_coalesce) &&
2088 (nesdev->deepcq_count > nesadapter->et_pkt_rate_low))))) {
2089 if ((nesdev->int_req & NES_INT_TIMER) == 0) {
2090 /* Enable Periodic timer interrupts */
2091 nesdev->int_req |= NES_INT_TIMER;
2092 /* ack any pending periodic timer interrupts so we don't get an immediate interrupt */
2093 /* TODO: need to also ack other unused periodic timer values, get from nesadapter */
2094 nes_write32(nesdev->regs+NES_TIMER_STAT,
2095 nesdev->timer_int_req | ~(nesdev->nesadapter->timer_int_req));
2096 nes_write32(nesdev->regs+NES_INTF_INT_MASK,
2097 ~(nesdev->intf_int_req | NES_INTF_PERIODIC_TIMER));
2098 }
2099
2100 if (unlikely(nesadapter->et_use_adaptive_rx_coalesce))
2101 {
2102 nes_nic_init_timer(nesdev);
2103 }
2104 /* Enable interrupts, except CEQs */
2105 nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
2106 } else {
2107 /* Enable interrupts, make sure timer is off */
2108 nesdev->int_req &= ~NES_INT_TIMER;
2109 nes_write32(nesdev->regs+NES_INTF_INT_MASK, ~(nesdev->intf_int_req));
2110 nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
2111 }
2112 nesdev->deepcq_count = 0;
2113 return 1;
2114 } else {
2115 return 0;
2116 }
2117 }
2118
2119 static void process_critical_error(struct nes_device *nesdev)
2120 {
2121 u32 debug_error;
2122 u32 nes_idx_debug_error_masks0 = 0;
2123 u16 error_module = 0;
2124
2125 debug_error = nes_read_indexed(nesdev, NES_IDX_DEBUG_ERROR_CONTROL_STATUS);
2126 printk(KERN_ERR PFX "Critical Error reported by device!!! 0x%02X\n",
2127 (u16)debug_error);
2128 nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_CONTROL_STATUS,
2129 0x01010000 | (debug_error & 0x0000ffff));
2130 if (crit_err_count++ > 10)
2131 nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_MASKS1, 1 << 0x17);
2132 error_module = (u16) (debug_error & 0x1F00) >> 8;
2133 if (++nesdev->nesadapter->crit_error_count[error_module-1] >=
2134 nes_max_critical_error_count) {
2135 printk(KERN_ERR PFX "Masking off critical error for module "
2136 "0x%02X\n", (u16)error_module);
2137 nes_idx_debug_error_masks0 = nes_read_indexed(nesdev,
2138 NES_IDX_DEBUG_ERROR_MASKS0);
2139 nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_MASKS0,
2140 nes_idx_debug_error_masks0 | (1 << error_module));
2141 }
2142 }
2143 /**
2144 * nes_dpc
2145 */
2146 void nes_dpc(unsigned long param)
2147 {
2148 struct nes_device *nesdev = (struct nes_device *)param;
2149 struct nes_adapter *nesadapter = nesdev->nesadapter;
2150 u32 counter;
2151 u32 loop_counter = 0;
2152 u32 int_status_bit;
2153 u32 int_stat;
2154 u32 timer_stat;
2155 u32 temp_int_stat;
2156 u32 intf_int_stat;
2157 u32 processed_intf_int = 0;
2158 u16 processed_timer_int = 0;
2159 u16 completion_ints = 0;
2160 u16 timer_ints = 0;
2161
2162 /* nes_debug(NES_DBG_ISR, "\n"); */
2163
2164 do {
2165 timer_stat = 0;
2166 if (nesdev->napi_isr_ran) {
2167 nesdev->napi_isr_ran = 0;
2168 int_stat = nesdev->int_stat;
2169 } else
2170 int_stat = nes_read32(nesdev->regs+NES_INT_STAT);
2171 if (processed_intf_int != 0)
2172 int_stat &= nesdev->int_req & ~NES_INT_INTF;
2173 else
2174 int_stat &= nesdev->int_req;
2175 if (processed_timer_int == 0) {
2176 processed_timer_int = 1;
2177 if (int_stat & NES_INT_TIMER) {
2178 timer_stat = nes_read32(nesdev->regs + NES_TIMER_STAT);
2179 if ((timer_stat & nesdev->timer_int_req) == 0) {
2180 int_stat &= ~NES_INT_TIMER;
2181 }
2182 }
2183 } else {
2184 int_stat &= ~NES_INT_TIMER;
2185 }
2186
2187 if (int_stat) {
2188 if (int_stat & ~(NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0|
2189 NES_INT_MAC1|NES_INT_MAC2 | NES_INT_MAC3)) {
2190 /* Ack the interrupts */
2191 nes_write32(nesdev->regs+NES_INT_STAT,
2192 (int_stat & ~(NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0|
2193 NES_INT_MAC1 | NES_INT_MAC2 | NES_INT_MAC3)));
2194 }
2195
2196 temp_int_stat = int_stat;
2197 for (counter = 0, int_status_bit = 1; counter < 16; counter++) {
2198 if (int_stat & int_status_bit) {
2199 nes_process_ceq(nesdev, &nesadapter->ceq[counter]);
2200 temp_int_stat &= ~int_status_bit;
2201 completion_ints = 1;
2202 }
2203 if (!(temp_int_stat & 0x0000ffff))
2204 break;
2205 int_status_bit <<= 1;
2206 }
2207
2208 /* Process the AEQ for this pci function */
2209 int_status_bit = 1 << (16 + PCI_FUNC(nesdev->pcidev->devfn));
2210 if (int_stat & int_status_bit) {
2211 nes_process_aeq(nesdev, &nesadapter->aeq[PCI_FUNC(nesdev->pcidev->devfn)]);
2212 }
2213
2214 /* Process the MAC interrupt for this pci function */
2215 int_status_bit = 1 << (24 + nesdev->mac_index);
2216 if (int_stat & int_status_bit) {
2217 nes_process_mac_intr(nesdev, nesdev->mac_index);
2218 }
2219
2220 if (int_stat & NES_INT_TIMER) {
2221 if (timer_stat & nesdev->timer_int_req) {
2222 nes_write32(nesdev->regs + NES_TIMER_STAT,
2223 (timer_stat & nesdev->timer_int_req) |
2224 ~(nesdev->nesadapter->timer_int_req));
2225 timer_ints = 1;
2226 }
2227 }
2228
2229 if (int_stat & NES_INT_INTF) {
2230 processed_intf_int = 1;
2231 intf_int_stat = nes_read32(nesdev->regs+NES_INTF_INT_STAT);
2232 intf_int_stat &= nesdev->intf_int_req;
2233 if (NES_INTF_INT_CRITERR & intf_int_stat) {
2234 process_critical_error(nesdev);
2235 }
2236 if (NES_INTF_INT_PCIERR & intf_int_stat) {
2237 printk(KERN_ERR PFX "PCI Error reported by device!!!\n");
2238 BUG();
2239 }
2240 if (NES_INTF_INT_AEQ_OFLOW & intf_int_stat) {
2241 printk(KERN_ERR PFX "AEQ Overflow reported by device!!!\n");
2242 BUG();
2243 }
2244 nes_write32(nesdev->regs+NES_INTF_INT_STAT, intf_int_stat);
2245 }
2246
2247 if (int_stat & NES_INT_TSW) {
2248 }
2249 }
2250 /* Don't use the interface interrupt bit stay in loop */
2251 int_stat &= ~NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0 |
2252 NES_INT_MAC1 | NES_INT_MAC2 | NES_INT_MAC3;
2253 } while ((int_stat != 0) && (loop_counter++ < MAX_DPC_ITERATIONS));
2254
2255 if (timer_ints == 1) {
2256 if ((nesadapter->et_rx_coalesce_usecs_irq) || (nesadapter->et_use_adaptive_rx_coalesce)) {
2257 if (completion_ints == 0) {
2258 nesdev->timer_only_int_count++;
2259 if (nesdev->timer_only_int_count>=nesadapter->timer_int_limit) {
2260 nesdev->timer_only_int_count = 0;
2261 nesdev->int_req &= ~NES_INT_TIMER;
2262 nes_write32(nesdev->regs + NES_INTF_INT_MASK, ~(nesdev->intf_int_req));
2263 nes_write32(nesdev->regs + NES_INT_MASK, ~nesdev->int_req);
2264 } else {
2265 nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
2266 }
2267 } else {
2268 if (unlikely(nesadapter->et_use_adaptive_rx_coalesce))
2269 {
2270 nes_nic_init_timer(nesdev);
2271 }
2272 nesdev->timer_only_int_count = 0;
2273 nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
2274 }
2275 } else {
2276 nesdev->timer_only_int_count = 0;
2277 nesdev->int_req &= ~NES_INT_TIMER;
2278 nes_write32(nesdev->regs+NES_INTF_INT_MASK, ~(nesdev->intf_int_req));
2279 nes_write32(nesdev->regs+NES_TIMER_STAT,
2280 nesdev->timer_int_req | ~(nesdev->nesadapter->timer_int_req));
2281 nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
2282 }
2283 } else {
2284 if ( (completion_ints == 1) &&
2285 (((nesadapter->et_rx_coalesce_usecs_irq) &&
2286 (!nesadapter->et_use_adaptive_rx_coalesce)) ||
2287 ((nesdev->deepcq_count > nesadapter->et_pkt_rate_low) &&
2288 (nesadapter->et_use_adaptive_rx_coalesce) )) ) {
2289 /* nes_debug(NES_DBG_ISR, "Enabling periodic timer interrupt.\n" ); */
2290 nesdev->timer_only_int_count = 0;
2291 nesdev->int_req |= NES_INT_TIMER;
2292 nes_write32(nesdev->regs+NES_TIMER_STAT,
2293 nesdev->timer_int_req | ~(nesdev->nesadapter->timer_int_req));
2294 nes_write32(nesdev->regs+NES_INTF_INT_MASK,
2295 ~(nesdev->intf_int_req | NES_INTF_PERIODIC_TIMER));
2296 nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
2297 } else {
2298 nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
2299 }
2300 }
2301 nesdev->deepcq_count = 0;
2302 }
2303
2304
2305 /**
2306 * nes_process_ceq
2307 */
2308 static void nes_process_ceq(struct nes_device *nesdev, struct nes_hw_ceq *ceq)
2309 {
2310 u64 u64temp;
2311 struct nes_hw_cq *cq;
2312 u32 head;
2313 u32 ceq_size;
2314
2315 /* nes_debug(NES_DBG_CQ, "\n"); */
2316 head = ceq->ceq_head;
2317 ceq_size = ceq->ceq_size;
2318
2319 do {
2320 if (le32_to_cpu(ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_HIGH_IDX]) &
2321 NES_CEQE_VALID) {
2322 u64temp = (((u64)(le32_to_cpu(ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_HIGH_IDX]))) << 32) |
2323 ((u64)(le32_to_cpu(ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_LOW_IDX])));
2324 u64temp <<= 1;
2325 cq = *((struct nes_hw_cq **)&u64temp);
2326 /* nes_debug(NES_DBG_CQ, "pCQ = %p\n", cq); */
2327 barrier();
2328 ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_HIGH_IDX] = 0;
2329
2330 /* call the event handler */
2331 cq->ce_handler(nesdev, cq);
2332
2333 if (++head >= ceq_size)
2334 head = 0;
2335 } else {
2336 break;
2337 }
2338
2339 } while (1);
2340
2341 ceq->ceq_head = head;
2342 }
2343
2344
2345 /**
2346 * nes_process_aeq
2347 */
2348 static void nes_process_aeq(struct nes_device *nesdev, struct nes_hw_aeq *aeq)
2349 {
2350 /* u64 u64temp; */
2351 u32 head;
2352 u32 aeq_size;
2353 u32 aeqe_misc;
2354 u32 aeqe_cq_id;
2355 struct nes_hw_aeqe volatile *aeqe;
2356
2357 head = aeq->aeq_head;
2358 aeq_size = aeq->aeq_size;
2359
2360 do {
2361 aeqe = &aeq->aeq_vbase[head];
2362 if ((le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]) & NES_AEQE_VALID) == 0)
2363 break;
2364 aeqe_misc = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
2365 aeqe_cq_id = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]);
2366 if (aeqe_misc & (NES_AEQE_QP|NES_AEQE_CQ)) {
2367 if (aeqe_cq_id >= NES_FIRST_QPN) {
2368 /* dealing with an accelerated QP related AE */
2369 /*
2370 * u64temp = (((u64)(le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_HIGH_IDX]))) << 32) |
2371 * ((u64)(le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_LOW_IDX])));
2372 */
2373 nes_process_iwarp_aeqe(nesdev, (struct nes_hw_aeqe *)aeqe);
2374 } else {
2375 /* TODO: dealing with a CQP related AE */
2376 nes_debug(NES_DBG_AEQ, "Processing CQP related AE, misc = 0x%04X\n",
2377 (u16)(aeqe_misc >> 16));
2378 }
2379 }
2380
2381 aeqe->aeqe_words[NES_AEQE_MISC_IDX] = 0;
2382
2383 if (++head >= aeq_size)
2384 head = 0;
2385
2386 nes_write32(nesdev->regs + NES_AEQ_ALLOC, 1 << 16);
2387 }
2388 while (1);
2389 aeq->aeq_head = head;
2390 }
2391
2392 static void nes_reset_link(struct nes_device *nesdev, u32 mac_index)
2393 {
2394 struct nes_adapter *nesadapter = nesdev->nesadapter;
2395 u32 reset_value;
2396 u32 i=0;
2397 u32 u32temp;
2398
2399 if (nesadapter->hw_rev == NE020_REV) {
2400 return;
2401 }
2402 mh_detected++;
2403
2404 reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
2405
2406 if ((mac_index == 0) || ((mac_index == 1) && (nesadapter->OneG_Mode)))
2407 reset_value |= 0x0000001d;
2408 else
2409 reset_value |= 0x0000002d;
2410
2411 if (4 <= (nesadapter->link_interrupt_count[mac_index] / ((u16)NES_MAX_LINK_INTERRUPTS))) {
2412 if ((!nesadapter->OneG_Mode) && (nesadapter->port_count == 2)) {
2413 nesadapter->link_interrupt_count[0] = 0;
2414 nesadapter->link_interrupt_count[1] = 0;
2415 u32temp = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
2416 if (0x00000040 & u32temp)
2417 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F088);
2418 else
2419 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F0C8);
2420
2421 reset_value |= 0x0000003d;
2422 }
2423 nesadapter->link_interrupt_count[mac_index] = 0;
2424 }
2425
2426 nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
2427
2428 while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET)
2429 & 0x00000040) != 0x00000040) && (i++ < 5000));
2430
2431 if (0x0000003d == (reset_value & 0x0000003d)) {
2432 u32 pcs_control_status0, pcs_control_status1;
2433
2434 for (i = 0; i < 10; i++) {
2435 pcs_control_status0 = nes_read_indexed(nesdev, NES_IDX_PHY_PCS_CONTROL_STATUS0);
2436 pcs_control_status1 = nes_read_indexed(nesdev, NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
2437 if (((0x0F000000 == (pcs_control_status0 & 0x0F000000))
2438 && (pcs_control_status0 & 0x00100000))
2439 || ((0x0F000000 == (pcs_control_status1 & 0x0F000000))
2440 && (pcs_control_status1 & 0x00100000)))
2441 continue;
2442 else
2443 break;
2444 }
2445 if (10 == i) {
2446 u32temp = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
2447 if (0x00000040 & u32temp)
2448 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F088);
2449 else
2450 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F0C8);
2451
2452 nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
2453
2454 while (((nes_read32(nesdev->regs + NES_SOFTWARE_RESET)
2455 & 0x00000040) != 0x00000040) && (i++ < 5000));
2456 }
2457 }
2458 }
2459
2460 /**
2461 * nes_process_mac_intr
2462 */
2463 static void nes_process_mac_intr(struct nes_device *nesdev, u32 mac_number)
2464 {
2465 unsigned long flags;
2466 u32 pcs_control_status;
2467 struct nes_adapter *nesadapter = nesdev->nesadapter;
2468 struct nes_vnic *nesvnic;
2469 u32 mac_status;
2470 u32 mac_index = nesdev->mac_index;
2471 u32 u32temp;
2472 u16 phy_data;
2473 u16 temp_phy_data;
2474 u32 pcs_val = 0x0f0f0000;
2475 u32 pcs_mask = 0x0f1f0000;
2476 u32 cdr_ctrl;
2477
2478 spin_lock_irqsave(&nesadapter->phy_lock, flags);
2479 if (nesadapter->mac_sw_state[mac_number] != NES_MAC_SW_IDLE) {
2480 spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
2481 return;
2482 }
2483 nesadapter->mac_sw_state[mac_number] = NES_MAC_SW_INTERRUPT;
2484
2485 /* ack the MAC interrupt */
2486 mac_status = nes_read_indexed(nesdev, NES_IDX_MAC_INT_STATUS + (mac_index * 0x200));
2487 /* Clear the interrupt */
2488 nes_write_indexed(nesdev, NES_IDX_MAC_INT_STATUS + (mac_index * 0x200), mac_status);
2489
2490 nes_debug(NES_DBG_PHY, "MAC%u interrupt status = 0x%X.\n", mac_number, mac_status);
2491
2492 if (mac_status & (NES_MAC_INT_LINK_STAT_CHG | NES_MAC_INT_XGMII_EXT)) {
2493 nesdev->link_status_interrupts++;
2494 if (0 == (++nesadapter->link_interrupt_count[mac_index] % ((u16)NES_MAX_LINK_INTERRUPTS)))
2495 nes_reset_link(nesdev, mac_index);
2496
2497 /* read the PHY interrupt status register */
2498 if ((nesadapter->OneG_Mode) &&
2499 (nesadapter->phy_type[mac_index] != NES_PHY_TYPE_PUMA_1G)) {
2500 do {
2501 nes_read_1G_phy_reg(nesdev, 0x1a,
2502 nesadapter->phy_index[mac_index], &phy_data);
2503 nes_debug(NES_DBG_PHY, "Phy%d data from register 0x1a = 0x%X.\n",
2504 nesadapter->phy_index[mac_index], phy_data);
2505 } while (phy_data&0x8000);
2506
2507 temp_phy_data = 0;
2508 do {
2509 nes_read_1G_phy_reg(nesdev, 0x11,
2510 nesadapter->phy_index[mac_index], &phy_data);
2511 nes_debug(NES_DBG_PHY, "Phy%d data from register 0x11 = 0x%X.\n",
2512 nesadapter->phy_index[mac_index], phy_data);
2513 if (temp_phy_data == phy_data)
2514 break;
2515 temp_phy_data = phy_data;
2516 } while (1);
2517
2518 nes_read_1G_phy_reg(nesdev, 0x1e,
2519 nesadapter->phy_index[mac_index], &phy_data);
2520 nes_debug(NES_DBG_PHY, "Phy%d data from register 0x1e = 0x%X.\n",
2521 nesadapter->phy_index[mac_index], phy_data);
2522
2523 nes_read_1G_phy_reg(nesdev, 1,
2524 nesadapter->phy_index[mac_index], &phy_data);
2525 nes_debug(NES_DBG_PHY, "1G phy%u data from register 1 = 0x%X\n",
2526 nesadapter->phy_index[mac_index], phy_data);
2527
2528 if (temp_phy_data & 0x1000) {
2529 nes_debug(NES_DBG_PHY, "The Link is up according to the PHY\n");
2530 phy_data = 4;
2531 } else {
2532 nes_debug(NES_DBG_PHY, "The Link is down according to the PHY\n");
2533 }
2534 }
2535 nes_debug(NES_DBG_PHY, "Eth SERDES Common Status: 0=0x%08X, 1=0x%08X\n",
2536 nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0),
2537 nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0+0x200));
2538
2539 if (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_PUMA_1G) {
2540 switch (mac_index) {
2541 case 1:
2542 case 3:
2543 pcs_control_status = nes_read_indexed(nesdev,
2544 NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
2545 break;
2546 default:
2547 pcs_control_status = nes_read_indexed(nesdev,
2548 NES_IDX_PHY_PCS_CONTROL_STATUS0);
2549 break;
2550 }
2551 } else {
2552 pcs_control_status = nes_read_indexed(nesdev,
2553 NES_IDX_PHY_PCS_CONTROL_STATUS0 + ((mac_index & 1) * 0x200));
2554 pcs_control_status = nes_read_indexed(nesdev,
2555 NES_IDX_PHY_PCS_CONTROL_STATUS0 + ((mac_index & 1) * 0x200));
2556 }
2557
2558 nes_debug(NES_DBG_PHY, "PCS PHY Control/Status%u: 0x%08X\n",
2559 mac_index, pcs_control_status);
2560 if ((nesadapter->OneG_Mode) &&
2561 (nesadapter->phy_type[mac_index] != NES_PHY_TYPE_PUMA_1G)) {
2562 u32temp = 0x01010000;
2563 if (nesadapter->port_count > 2) {
2564 u32temp |= 0x02020000;
2565 }
2566 if ((pcs_control_status & u32temp)!= u32temp) {
2567 phy_data = 0;
2568 nes_debug(NES_DBG_PHY, "PCS says the link is down\n");
2569 }
2570 } else {
2571 switch (nesadapter->phy_type[mac_index]) {
2572 case NES_PHY_TYPE_ARGUS:
2573 case NES_PHY_TYPE_SFP_D:
2574 case NES_PHY_TYPE_KR:
2575 /* clear the alarms */
2576 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0x0008);
2577 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc001);
2578 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc002);
2579 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc005);
2580 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc006);
2581 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9003);
2582 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9004);
2583 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9005);
2584 /* check link status */
2585 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9003);
2586 temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2587
2588 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 3, 0x0021);
2589 nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2590 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 3, 0x0021);
2591 phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2592
2593 phy_data = (!temp_phy_data && (phy_data == 0x8000)) ? 0x4 : 0x0;
2594
2595 nes_debug(NES_DBG_PHY, "%s: Phy data = 0x%04X, link was %s.\n",
2596 __func__, phy_data, nesadapter->mac_link_down[mac_index] ? "DOWN" : "UP");
2597 break;
2598
2599 case NES_PHY_TYPE_PUMA_1G:
2600 if (mac_index < 2)
2601 pcs_val = pcs_mask = 0x01010000;
2602 else
2603 pcs_val = pcs_mask = 0x02020000;
2604 /* fall through */
2605 default:
2606 phy_data = (pcs_val == (pcs_control_status & pcs_mask)) ? 0x4 : 0x0;
2607 break;
2608 }
2609 }
2610
2611 if (phy_data & 0x0004) {
2612 if (wide_ppm_offset &&
2613 (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_CX4) &&
2614 (nesadapter->hw_rev != NE020_REV)) {
2615 cdr_ctrl = nes_read_indexed(nesdev,
2616 NES_IDX_ETH_SERDES_CDR_CONTROL0 +
2617 mac_index * 0x200);
2618 nes_write_indexed(nesdev,
2619 NES_IDX_ETH_SERDES_CDR_CONTROL0 +
2620 mac_index * 0x200,
2621 cdr_ctrl | 0x000F0000);
2622 }
2623 nesadapter->mac_link_down[mac_index] = 0;
2624 list_for_each_entry(nesvnic, &nesadapter->nesvnic_list[mac_index], list) {
2625 nes_debug(NES_DBG_PHY, "The Link is UP!!. linkup was %d\n",
2626 nesvnic->linkup);
2627 if (nesvnic->linkup == 0) {
2628 printk(PFX "The Link is now up for port %s, netdev %p.\n",
2629 nesvnic->netdev->name, nesvnic->netdev);
2630 if (netif_queue_stopped(nesvnic->netdev))
2631 netif_start_queue(nesvnic->netdev);
2632 nesvnic->linkup = 1;
2633 netif_carrier_on(nesvnic->netdev);
2634
2635 spin_lock(&nesvnic->port_ibevent_lock);
2636 if (nesvnic->of_device_registered) {
2637 if (nesdev->iw_status == 0) {
2638 nesdev->iw_status = 1;
2639 nes_port_ibevent(nesvnic);
2640 }
2641 }
2642 spin_unlock(&nesvnic->port_ibevent_lock);
2643 }
2644 }
2645 } else {
2646 if (wide_ppm_offset &&
2647 (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_CX4) &&
2648 (nesadapter->hw_rev != NE020_REV)) {
2649 cdr_ctrl = nes_read_indexed(nesdev,
2650 NES_IDX_ETH_SERDES_CDR_CONTROL0 +
2651 mac_index * 0x200);
2652 nes_write_indexed(nesdev,
2653 NES_IDX_ETH_SERDES_CDR_CONTROL0 +
2654 mac_index * 0x200,
2655 cdr_ctrl & 0xFFF0FFFF);
2656 }
2657 nesadapter->mac_link_down[mac_index] = 1;
2658 list_for_each_entry(nesvnic, &nesadapter->nesvnic_list[mac_index], list) {
2659 nes_debug(NES_DBG_PHY, "The Link is Down!!. linkup was %d\n",
2660 nesvnic->linkup);
2661 if (nesvnic->linkup == 1) {
2662 printk(PFX "The Link is now down for port %s, netdev %p.\n",
2663 nesvnic->netdev->name, nesvnic->netdev);
2664 if (!(netif_queue_stopped(nesvnic->netdev)))
2665 netif_stop_queue(nesvnic->netdev);
2666 nesvnic->linkup = 0;
2667 netif_carrier_off(nesvnic->netdev);
2668
2669 spin_lock(&nesvnic->port_ibevent_lock);
2670 if (nesvnic->of_device_registered) {
2671 if (nesdev->iw_status == 1) {
2672 nesdev->iw_status = 0;
2673 nes_port_ibevent(nesvnic);
2674 }
2675 }
2676 spin_unlock(&nesvnic->port_ibevent_lock);
2677 }
2678 }
2679 }
2680 if (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_SFP_D) {
2681 nesdev->link_recheck = 1;
2682 mod_delayed_work(system_wq, &nesdev->work,
2683 NES_LINK_RECHECK_DELAY);
2684 }
2685 }
2686
2687 spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
2688
2689 nesadapter->mac_sw_state[mac_number] = NES_MAC_SW_IDLE;
2690 }
2691
2692 void nes_recheck_link_status(struct work_struct *work)
2693 {
2694 unsigned long flags;
2695 struct nes_device *nesdev = container_of(work, struct nes_device, work.work);
2696 struct nes_adapter *nesadapter = nesdev->nesadapter;
2697 struct nes_vnic *nesvnic;
2698 u32 mac_index = nesdev->mac_index;
2699 u16 phy_data;
2700 u16 temp_phy_data;
2701
2702 spin_lock_irqsave(&nesadapter->phy_lock, flags);
2703
2704 /* check link status */
2705 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9003);
2706 temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2707
2708 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 3, 0x0021);
2709 nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2710 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 3, 0x0021);
2711 phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2712
2713 phy_data = (!temp_phy_data && (phy_data == 0x8000)) ? 0x4 : 0x0;
2714
2715 nes_debug(NES_DBG_PHY, "%s: Phy data = 0x%04X, link was %s.\n",
2716 __func__, phy_data,
2717 nesadapter->mac_link_down[mac_index] ? "DOWN" : "UP");
2718
2719 if (phy_data & 0x0004) {
2720 nesadapter->mac_link_down[mac_index] = 0;
2721 list_for_each_entry(nesvnic, &nesadapter->nesvnic_list[mac_index], list) {
2722 if (nesvnic->linkup == 0) {
2723 printk(PFX "The Link is now up for port %s, netdev %p.\n",
2724 nesvnic->netdev->name, nesvnic->netdev);
2725 if (netif_queue_stopped(nesvnic->netdev))
2726 netif_start_queue(nesvnic->netdev);
2727 nesvnic->linkup = 1;
2728 netif_carrier_on(nesvnic->netdev);
2729
2730 spin_lock(&nesvnic->port_ibevent_lock);
2731 if (nesvnic->of_device_registered) {
2732 if (nesdev->iw_status == 0) {
2733 nesdev->iw_status = 1;
2734 nes_port_ibevent(nesvnic);
2735 }
2736 }
2737 spin_unlock(&nesvnic->port_ibevent_lock);
2738 }
2739 }
2740
2741 } else {
2742 nesadapter->mac_link_down[mac_index] = 1;
2743 list_for_each_entry(nesvnic, &nesadapter->nesvnic_list[mac_index], list) {
2744 if (nesvnic->linkup == 1) {
2745 printk(PFX "The Link is now down for port %s, netdev %p.\n",
2746 nesvnic->netdev->name, nesvnic->netdev);
2747 if (!(netif_queue_stopped(nesvnic->netdev)))
2748 netif_stop_queue(nesvnic->netdev);
2749 nesvnic->linkup = 0;
2750 netif_carrier_off(nesvnic->netdev);
2751
2752 spin_lock(&nesvnic->port_ibevent_lock);
2753 if (nesvnic->of_device_registered) {
2754 if (nesdev->iw_status == 1) {
2755 nesdev->iw_status = 0;
2756 nes_port_ibevent(nesvnic);
2757 }
2758 }
2759 spin_unlock(&nesvnic->port_ibevent_lock);
2760 }
2761 }
2762 }
2763 if (nesdev->link_recheck++ < NES_LINK_RECHECK_MAX)
2764 schedule_delayed_work(&nesdev->work, NES_LINK_RECHECK_DELAY);
2765 else
2766 nesdev->link_recheck = 0;
2767
2768 spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
2769 }
2770
2771
2772 static void nes_nic_napi_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq)
2773 {
2774 struct nes_vnic *nesvnic = container_of(cq, struct nes_vnic, nic_cq);
2775
2776 napi_schedule(&nesvnic->napi);
2777 }
2778
2779
2780 /* The MAX_RQES_TO_PROCESS defines how many max read requests to complete before
2781 * getting out of nic_ce_handler
2782 */
2783 #define MAX_RQES_TO_PROCESS 384
2784
2785 /**
2786 * nes_nic_ce_handler
2787 */
2788 void nes_nic_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq)
2789 {
2790 u64 u64temp;
2791 dma_addr_t bus_address;
2792 struct nes_hw_nic *nesnic;
2793 struct nes_vnic *nesvnic = container_of(cq, struct nes_vnic, nic_cq);
2794 struct nes_adapter *nesadapter = nesdev->nesadapter;
2795 struct nes_hw_nic_rq_wqe *nic_rqe;
2796 struct nes_hw_nic_sq_wqe *nic_sqe;
2797 struct sk_buff *skb;
2798 struct sk_buff *rx_skb;
2799 struct nes_rskb_cb *cb;
2800 __le16 *wqe_fragment_length;
2801 u32 head;
2802 u32 cq_size;
2803 u32 rx_pkt_size;
2804 u32 cqe_count=0;
2805 u32 cqe_errv;
2806 u32 cqe_misc;
2807 u16 wqe_fragment_index = 1; /* first fragment (0) is used by copy buffer */
2808 u16 vlan_tag;
2809 u16 pkt_type;
2810 u16 rqes_processed = 0;
2811 u8 sq_cqes = 0;
2812 u8 nes_use_lro = 0;
2813
2814 head = cq->cq_head;
2815 cq_size = cq->cq_size;
2816 cq->cqes_pending = 1;
2817 if (nesvnic->netdev->features & NETIF_F_LRO)
2818 nes_use_lro = 1;
2819 do {
2820 if (le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX]) &
2821 NES_NIC_CQE_VALID) {
2822 nesnic = &nesvnic->nic;
2823 cqe_misc = le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX]);
2824 if (cqe_misc & NES_NIC_CQE_SQ) {
2825 sq_cqes++;
2826 wqe_fragment_index = 1;
2827 nic_sqe = &nesnic->sq_vbase[nesnic->sq_tail];
2828 skb = nesnic->tx_skb[nesnic->sq_tail];
2829 wqe_fragment_length = (__le16 *)&nic_sqe->wqe_words[NES_NIC_SQ_WQE_LENGTH_0_TAG_IDX];
2830 /* bump past the vlan tag */
2831 wqe_fragment_length++;
2832 if (le16_to_cpu(wqe_fragment_length[wqe_fragment_index]) != 0) {
2833 u64temp = (u64) le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX +
2834 wqe_fragment_index * 2]);
2835 u64temp += ((u64)le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX +
2836 wqe_fragment_index * 2])) << 32;
2837 bus_address = (dma_addr_t)u64temp;
2838 if (test_and_clear_bit(nesnic->sq_tail, nesnic->first_frag_overflow)) {
2839 pci_unmap_single(nesdev->pcidev,
2840 bus_address,
2841 le16_to_cpu(wqe_fragment_length[wqe_fragment_index++]),
2842 PCI_DMA_TODEVICE);
2843 }
2844 for (; wqe_fragment_index < 5; wqe_fragment_index++) {
2845 if (wqe_fragment_length[wqe_fragment_index]) {
2846 u64temp = le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX +
2847 wqe_fragment_index * 2]);
2848 u64temp += ((u64)le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX
2849 + wqe_fragment_index * 2])) <<32;
2850 bus_address = (dma_addr_t)u64temp;
2851 pci_unmap_page(nesdev->pcidev,
2852 bus_address,
2853 le16_to_cpu(wqe_fragment_length[wqe_fragment_index]),
2854 PCI_DMA_TODEVICE);
2855 } else
2856 break;
2857 }
2858 }
2859 if (skb)
2860 dev_kfree_skb_any(skb);
2861 nesnic->sq_tail++;
2862 nesnic->sq_tail &= nesnic->sq_size-1;
2863 if (sq_cqes > 128) {
2864 barrier();
2865 /* restart the queue if it had been stopped */
2866 if (netif_queue_stopped(nesvnic->netdev))
2867 netif_wake_queue(nesvnic->netdev);
2868 sq_cqes = 0;
2869 }
2870 } else {
2871 rqes_processed ++;
2872
2873 cq->rx_cqes_completed++;
2874 cq->rx_pkts_indicated++;
2875 rx_pkt_size = cqe_misc & 0x0000ffff;
2876 nic_rqe = &nesnic->rq_vbase[nesnic->rq_tail];
2877 /* Get the skb */
2878 rx_skb = nesnic->rx_skb[nesnic->rq_tail];
2879 nic_rqe = &nesnic->rq_vbase[nesvnic->nic.rq_tail];
2880 bus_address = (dma_addr_t)le32_to_cpu(nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX]);
2881 bus_address += ((u64)le32_to_cpu(nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX])) << 32;
2882 pci_unmap_single(nesdev->pcidev, bus_address,
2883 nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
2884 cb = (struct nes_rskb_cb *)&rx_skb->cb[0];
2885 cb->busaddr = 0;
2886 /* rx_skb->tail = rx_skb->data + rx_pkt_size; */
2887 /* rx_skb->len = rx_pkt_size; */
2888 rx_skb->len = 0; /* TODO: see if this is necessary */
2889 skb_put(rx_skb, rx_pkt_size);
2890 rx_skb->protocol = eth_type_trans(rx_skb, nesvnic->netdev);
2891 nesnic->rq_tail++;
2892 nesnic->rq_tail &= nesnic->rq_size - 1;
2893
2894 atomic_inc(&nesvnic->rx_skbs_needed);
2895 if (atomic_read(&nesvnic->rx_skbs_needed) > (nesvnic->nic.rq_size>>1)) {
2896 nes_write32(nesdev->regs+NES_CQE_ALLOC,
2897 cq->cq_number | (cqe_count << 16));
2898 /* nesadapter->tune_timer.cq_count += cqe_count; */
2899 nesdev->currcq_count += cqe_count;
2900 cqe_count = 0;
2901 nes_replenish_nic_rq(nesvnic);
2902 }
2903 pkt_type = (u16)(le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_TAG_PKT_TYPE_IDX]));
2904 cqe_errv = (cqe_misc & NES_NIC_CQE_ERRV_MASK) >> NES_NIC_CQE_ERRV_SHIFT;
2905 rx_skb->ip_summed = CHECKSUM_NONE;
2906
2907 if ((NES_PKT_TYPE_TCPV4_BITS == (pkt_type & NES_PKT_TYPE_TCPV4_MASK)) ||
2908 (NES_PKT_TYPE_UDPV4_BITS == (pkt_type & NES_PKT_TYPE_UDPV4_MASK))) {
2909 if ((cqe_errv &
2910 (NES_NIC_ERRV_BITS_IPV4_CSUM_ERR | NES_NIC_ERRV_BITS_TCPUDP_CSUM_ERR |
2911 NES_NIC_ERRV_BITS_IPH_ERR | NES_NIC_ERRV_BITS_WQE_OVERRUN)) == 0) {
2912 if (nesvnic->netdev->features & NETIF_F_RXCSUM)
2913 rx_skb->ip_summed = CHECKSUM_UNNECESSARY;
2914 } else
2915 nes_debug(NES_DBG_CQ, "%s: unsuccessfully checksummed TCP or UDP packet."
2916 " errv = 0x%X, pkt_type = 0x%X.\n",
2917 nesvnic->netdev->name, cqe_errv, pkt_type);
2918
2919 } else if ((pkt_type & NES_PKT_TYPE_IPV4_MASK) == NES_PKT_TYPE_IPV4_BITS) {
2920 if ((cqe_errv &
2921 (NES_NIC_ERRV_BITS_IPV4_CSUM_ERR | NES_NIC_ERRV_BITS_IPH_ERR |
2922 NES_NIC_ERRV_BITS_WQE_OVERRUN)) == 0) {
2923 if (nesvnic->netdev->features & NETIF_F_RXCSUM) {
2924 rx_skb->ip_summed = CHECKSUM_UNNECESSARY;
2925 /* nes_debug(NES_DBG_CQ, "%s: Reporting successfully checksummed IPv4 packet.\n",
2926 nesvnic->netdev->name); */
2927 }
2928 } else
2929 nes_debug(NES_DBG_CQ, "%s: unsuccessfully checksummed TCP or UDP packet."
2930 " errv = 0x%X, pkt_type = 0x%X.\n",
2931 nesvnic->netdev->name, cqe_errv, pkt_type);
2932 }
2933 /* nes_debug(NES_DBG_CQ, "pkt_type=%x, APBVT_MASK=%x\n",
2934 pkt_type, (pkt_type & NES_PKT_TYPE_APBVT_MASK)); */
2935
2936 if ((pkt_type & NES_PKT_TYPE_APBVT_MASK) == NES_PKT_TYPE_APBVT_BITS) {
2937 if (nes_cm_recv(rx_skb, nesvnic->netdev))
2938 rx_skb = NULL;
2939 }
2940 if (rx_skb == NULL)
2941 goto skip_rx_indicate0;
2942
2943
2944 if (cqe_misc & NES_NIC_CQE_TAG_VALID) {
2945 vlan_tag = (u16)(le32_to_cpu(
2946 cq->cq_vbase[head].cqe_words[NES_NIC_CQE_TAG_PKT_TYPE_IDX])
2947 >> 16);
2948 nes_debug(NES_DBG_CQ, "%s: Reporting stripped VLAN packet. Tag = 0x%04X\n",
2949 nesvnic->netdev->name, vlan_tag);
2950
2951 __vlan_hwaccel_put_tag(rx_skb, htons(ETH_P_8021Q), vlan_tag);
2952 }
2953 if (nes_use_lro)
2954 lro_receive_skb(&nesvnic->lro_mgr, rx_skb, NULL);
2955 else
2956 netif_receive_skb(rx_skb);
2957
2958 skip_rx_indicate0:
2959 ;
2960 /* nesvnic->netstats.rx_packets++; */
2961 /* nesvnic->netstats.rx_bytes += rx_pkt_size; */
2962 }
2963
2964 cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX] = 0;
2965 /* Accounting... */
2966 cqe_count++;
2967 if (++head >= cq_size)
2968 head = 0;
2969 if (cqe_count == 255) {
2970 /* Replenish Nic CQ */
2971 nes_write32(nesdev->regs+NES_CQE_ALLOC,
2972 cq->cq_number | (cqe_count << 16));
2973 /* nesdev->nesadapter->tune_timer.cq_count += cqe_count; */
2974 nesdev->currcq_count += cqe_count;
2975 cqe_count = 0;
2976 }
2977
2978 if (cq->rx_cqes_completed >= nesvnic->budget)
2979 break;
2980 } else {
2981 cq->cqes_pending = 0;
2982 break;
2983 }
2984
2985 } while (1);
2986
2987 if (nes_use_lro)
2988 lro_flush_all(&nesvnic->lro_mgr);
2989 if (sq_cqes) {
2990 barrier();
2991 /* restart the queue if it had been stopped */
2992 if (netif_queue_stopped(nesvnic->netdev))
2993 netif_wake_queue(nesvnic->netdev);
2994 }
2995 cq->cq_head = head;
2996 /* nes_debug(NES_DBG_CQ, "CQ%u Processed = %u cqes, new head = %u.\n",
2997 cq->cq_number, cqe_count, cq->cq_head); */
2998 cq->cqe_allocs_pending = cqe_count;
2999 if (unlikely(nesadapter->et_use_adaptive_rx_coalesce))
3000 {
3001 /* nesdev->nesadapter->tune_timer.cq_count += cqe_count; */
3002 nesdev->currcq_count += cqe_count;
3003 nes_nic_tune_timer(nesdev);
3004 }
3005 if (atomic_read(&nesvnic->rx_skbs_needed))
3006 nes_replenish_nic_rq(nesvnic);
3007 }
3008
3009
3010
3011 /**
3012 * nes_cqp_ce_handler
3013 */
3014 static void nes_cqp_ce_handler(struct nes_device *nesdev, struct nes_hw_cq *cq)
3015 {
3016 u64 u64temp;
3017 unsigned long flags;
3018 struct nes_hw_cqp *cqp = NULL;
3019 struct nes_cqp_request *cqp_request;
3020 struct nes_hw_cqp_wqe *cqp_wqe;
3021 u32 head;
3022 u32 cq_size;
3023 u32 cqe_count=0;
3024 u32 error_code;
3025 u32 opcode;
3026 u32 ctx_index;
3027 /* u32 counter; */
3028
3029 head = cq->cq_head;
3030 cq_size = cq->cq_size;
3031
3032 do {
3033 /* process the CQE */
3034 /* nes_debug(NES_DBG_CQP, "head=%u cqe_words=%08X\n", head,
3035 le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX])); */
3036
3037 opcode = le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX]);
3038 if (opcode & NES_CQE_VALID) {
3039 cqp = &nesdev->cqp;
3040
3041 error_code = le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_ERROR_CODE_IDX]);
3042 if (error_code) {
3043 nes_debug(NES_DBG_CQP, "Bad Completion code for opcode 0x%02X from CQP,"
3044 " Major/Minor codes = 0x%04X:%04X.\n",
3045 le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX])&0x3f,
3046 (u16)(error_code >> 16),
3047 (u16)error_code);
3048 }
3049
3050 u64temp = (((u64)(le32_to_cpu(cq->cq_vbase[head].
3051 cqe_words[NES_CQE_COMP_COMP_CTX_HIGH_IDX]))) << 32) |
3052 ((u64)(le32_to_cpu(cq->cq_vbase[head].
3053 cqe_words[NES_CQE_COMP_COMP_CTX_LOW_IDX])));
3054
3055 cqp_request = (struct nes_cqp_request *)(unsigned long)u64temp;
3056 if (cqp_request) {
3057 if (cqp_request->waiting) {
3058 /* nes_debug(NES_DBG_CQP, "%s: Waking up requestor\n"); */
3059 cqp_request->major_code = (u16)(error_code >> 16);
3060 cqp_request->minor_code = (u16)error_code;
3061 barrier();
3062 cqp_request->request_done = 1;
3063 wake_up(&cqp_request->waitq);
3064 nes_put_cqp_request(nesdev, cqp_request);
3065 } else {
3066 if (cqp_request->callback)
3067 cqp_request->cqp_callback(nesdev, cqp_request);
3068 nes_free_cqp_request(nesdev, cqp_request);
3069 }
3070 } else {
3071 wake_up(&nesdev->cqp.waitq);
3072 }
3073
3074 cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX] = 0;
3075 nes_write32(nesdev->regs + NES_CQE_ALLOC, cq->cq_number | (1 << 16));
3076 if (++cqp->sq_tail >= cqp->sq_size)
3077 cqp->sq_tail = 0;
3078
3079 /* Accounting... */
3080 cqe_count++;
3081 if (++head >= cq_size)
3082 head = 0;
3083 } else {
3084 break;
3085 }
3086 } while (1);
3087 cq->cq_head = head;
3088
3089 spin_lock_irqsave(&nesdev->cqp.lock, flags);
3090 while ((!list_empty(&nesdev->cqp_pending_reqs)) &&
3091 ((((nesdev->cqp.sq_tail+nesdev->cqp.sq_size)-nesdev->cqp.sq_head) &
3092 (nesdev->cqp.sq_size - 1)) != 1)) {
3093 cqp_request = list_entry(nesdev->cqp_pending_reqs.next,
3094 struct nes_cqp_request, list);
3095 list_del_init(&cqp_request->list);
3096 head = nesdev->cqp.sq_head++;
3097 nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
3098 cqp_wqe = &nesdev->cqp.sq_vbase[head];
3099 memcpy(cqp_wqe, &cqp_request->cqp_wqe, sizeof(*cqp_wqe));
3100 barrier();
3101
3102 opcode = cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX];
3103 if ((opcode & NES_CQP_OPCODE_MASK) == NES_CQP_DOWNLOAD_SEGMENT)
3104 ctx_index = NES_CQP_WQE_DL_COMP_CTX_LOW_IDX;
3105 else
3106 ctx_index = NES_CQP_WQE_COMP_CTX_LOW_IDX;
3107 cqp_wqe->wqe_words[ctx_index] =
3108 cpu_to_le32((u32)((unsigned long)cqp_request));
3109 cqp_wqe->wqe_words[ctx_index + 1] =
3110 cpu_to_le32((u32)(upper_32_bits((unsigned long)cqp_request)));
3111 nes_debug(NES_DBG_CQP, "CQP request %p (opcode 0x%02X) put on CQPs SQ wqe%u.\n",
3112 cqp_request, le32_to_cpu(cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX])&0x3f, head);
3113 /* Ring doorbell (1 WQEs) */
3114 barrier();
3115 nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x01800000 | nesdev->cqp.qp_id);
3116 }
3117 spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
3118
3119 /* Arm the CCQ */
3120 nes_write32(nesdev->regs+NES_CQE_ALLOC, NES_CQE_ALLOC_NOTIFY_NEXT |
3121 cq->cq_number);
3122 nes_read32(nesdev->regs+NES_CQE_ALLOC);
3123 }
3124
3125 static u8 *locate_mpa(u8 *pkt, u32 aeq_info)
3126 {
3127 if (aeq_info & NES_AEQE_Q2_DATA_ETHERNET) {
3128 /* skip over ethernet header */
3129 pkt += ETH_HLEN;
3130
3131 /* Skip over IP and TCP headers */
3132 pkt += 4 * (pkt[0] & 0x0f);
3133 pkt += 4 * ((pkt[12] >> 4) & 0x0f);
3134 }
3135 return pkt;
3136 }
3137
3138 /* Determine if incoming error pkt is rdma layer */
3139 static u32 iwarp_opcode(struct nes_qp *nesqp, u32 aeq_info)
3140 {
3141 u8 *pkt;
3142 u16 *mpa;
3143 u32 opcode = 0xffffffff;
3144
3145 if (aeq_info & NES_AEQE_Q2_DATA_WRITTEN) {
3146 pkt = nesqp->hwqp.q2_vbase + BAD_FRAME_OFFSET;
3147 mpa = (u16 *)locate_mpa(pkt, aeq_info);
3148 opcode = be16_to_cpu(mpa[1]) & 0xf;
3149 }
3150
3151 return opcode;
3152 }
3153
3154 /* Build iWARP terminate header */
3155 static int nes_bld_terminate_hdr(struct nes_qp *nesqp, u16 async_event_id, u32 aeq_info)
3156 {
3157 u8 *pkt = nesqp->hwqp.q2_vbase + BAD_FRAME_OFFSET;
3158 u16 ddp_seg_len;
3159 int copy_len = 0;
3160 u8 is_tagged = 0;
3161 u8 flush_code = 0;
3162 struct nes_terminate_hdr *termhdr;
3163
3164 termhdr = (struct nes_terminate_hdr *)nesqp->hwqp.q2_vbase;
3165 memset(termhdr, 0, 64);
3166
3167 if (aeq_info & NES_AEQE_Q2_DATA_WRITTEN) {
3168
3169 /* Use data from offending packet to fill in ddp & rdma hdrs */
3170 pkt = locate_mpa(pkt, aeq_info);
3171 ddp_seg_len = be16_to_cpu(*(u16 *)pkt);
3172 if (ddp_seg_len) {
3173 copy_len = 2;
3174 termhdr->hdrct = DDP_LEN_FLAG;
3175 if (pkt[2] & 0x80) {
3176 is_tagged = 1;
3177 if (ddp_seg_len >= TERM_DDP_LEN_TAGGED) {
3178 copy_len += TERM_DDP_LEN_TAGGED;
3179 termhdr->hdrct |= DDP_HDR_FLAG;
3180 }
3181 } else {
3182 if (ddp_seg_len >= TERM_DDP_LEN_UNTAGGED) {
3183 copy_len += TERM_DDP_LEN_UNTAGGED;
3184 termhdr->hdrct |= DDP_HDR_FLAG;
3185 }
3186
3187 if (ddp_seg_len >= (TERM_DDP_LEN_UNTAGGED + TERM_RDMA_LEN)) {
3188 if ((pkt[3] & RDMA_OPCODE_MASK) == RDMA_READ_REQ_OPCODE) {
3189 copy_len += TERM_RDMA_LEN;
3190 termhdr->hdrct |= RDMA_HDR_FLAG;
3191 }
3192 }
3193 }
3194 }
3195 }
3196
3197 switch (async_event_id) {
3198 case NES_AEQE_AEID_AMP_UNALLOCATED_STAG:
3199 switch (iwarp_opcode(nesqp, aeq_info)) {
3200 case IWARP_OPCODE_WRITE:
3201 flush_code = IB_WC_LOC_PROT_ERR;
3202 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_TAGGED_BUFFER;
3203 termhdr->error_code = DDP_TAGGED_INV_STAG;
3204 break;
3205 default:
3206 flush_code = IB_WC_REM_ACCESS_ERR;
3207 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3208 termhdr->error_code = RDMAP_INV_STAG;
3209 }
3210 break;
3211 case NES_AEQE_AEID_AMP_INVALID_STAG:
3212 flush_code = IB_WC_REM_ACCESS_ERR;
3213 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3214 termhdr->error_code = RDMAP_INV_STAG;
3215 break;
3216 case NES_AEQE_AEID_AMP_BAD_QP:
3217 flush_code = IB_WC_LOC_QP_OP_ERR;
3218 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3219 termhdr->error_code = DDP_UNTAGGED_INV_QN;
3220 break;
3221 case NES_AEQE_AEID_AMP_BAD_STAG_KEY:
3222 case NES_AEQE_AEID_AMP_BAD_STAG_INDEX:
3223 switch (iwarp_opcode(nesqp, aeq_info)) {
3224 case IWARP_OPCODE_SEND_INV:
3225 case IWARP_OPCODE_SEND_SE_INV:
3226 flush_code = IB_WC_REM_OP_ERR;
3227 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_OP;
3228 termhdr->error_code = RDMAP_CANT_INV_STAG;
3229 break;
3230 default:
3231 flush_code = IB_WC_REM_ACCESS_ERR;
3232 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3233 termhdr->error_code = RDMAP_INV_STAG;
3234 }
3235 break;
3236 case NES_AEQE_AEID_AMP_BOUNDS_VIOLATION:
3237 if (aeq_info & (NES_AEQE_Q2_DATA_ETHERNET | NES_AEQE_Q2_DATA_MPA)) {
3238 flush_code = IB_WC_LOC_PROT_ERR;
3239 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_TAGGED_BUFFER;
3240 termhdr->error_code = DDP_TAGGED_BOUNDS;
3241 } else {
3242 flush_code = IB_WC_REM_ACCESS_ERR;
3243 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3244 termhdr->error_code = RDMAP_INV_BOUNDS;
3245 }
3246 break;
3247 case NES_AEQE_AEID_AMP_RIGHTS_VIOLATION:
3248 case NES_AEQE_AEID_AMP_INVALIDATE_NO_REMOTE_ACCESS_RIGHTS:
3249 case NES_AEQE_AEID_PRIV_OPERATION_DENIED:
3250 flush_code = IB_WC_REM_ACCESS_ERR;
3251 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3252 termhdr->error_code = RDMAP_ACCESS;
3253 break;
3254 case NES_AEQE_AEID_AMP_TO_WRAP:
3255 flush_code = IB_WC_REM_ACCESS_ERR;
3256 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3257 termhdr->error_code = RDMAP_TO_WRAP;
3258 break;
3259 case NES_AEQE_AEID_AMP_BAD_PD:
3260 switch (iwarp_opcode(nesqp, aeq_info)) {
3261 case IWARP_OPCODE_WRITE:
3262 flush_code = IB_WC_LOC_PROT_ERR;
3263 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_TAGGED_BUFFER;
3264 termhdr->error_code = DDP_TAGGED_UNASSOC_STAG;
3265 break;
3266 case IWARP_OPCODE_SEND_INV:
3267 case IWARP_OPCODE_SEND_SE_INV:
3268 flush_code = IB_WC_REM_ACCESS_ERR;
3269 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3270 termhdr->error_code = RDMAP_CANT_INV_STAG;
3271 break;
3272 default:
3273 flush_code = IB_WC_REM_ACCESS_ERR;
3274 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3275 termhdr->error_code = RDMAP_UNASSOC_STAG;
3276 }
3277 break;
3278 case NES_AEQE_AEID_LLP_RECEIVED_MARKER_AND_LENGTH_FIELDS_DONT_MATCH:
3279 flush_code = IB_WC_LOC_LEN_ERR;
3280 termhdr->layer_etype = (LAYER_MPA << 4) | DDP_LLP;
3281 termhdr->error_code = MPA_MARKER;
3282 break;
3283 case NES_AEQE_AEID_LLP_RECEIVED_MPA_CRC_ERROR:
3284 flush_code = IB_WC_GENERAL_ERR;
3285 termhdr->layer_etype = (LAYER_MPA << 4) | DDP_LLP;
3286 termhdr->error_code = MPA_CRC;
3287 break;
3288 case NES_AEQE_AEID_LLP_SEGMENT_TOO_LARGE:
3289 case NES_AEQE_AEID_LLP_SEGMENT_TOO_SMALL:
3290 flush_code = IB_WC_LOC_LEN_ERR;
3291 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_CATASTROPHIC;
3292 termhdr->error_code = DDP_CATASTROPHIC_LOCAL;
3293 break;
3294 case NES_AEQE_AEID_DDP_LCE_LOCAL_CATASTROPHIC:
3295 case NES_AEQE_AEID_DDP_NO_L_BIT:
3296 flush_code = IB_WC_FATAL_ERR;
3297 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_CATASTROPHIC;
3298 termhdr->error_code = DDP_CATASTROPHIC_LOCAL;
3299 break;
3300 case NES_AEQE_AEID_DDP_INVALID_MSN_GAP_IN_MSN:
3301 case NES_AEQE_AEID_DDP_INVALID_MSN_RANGE_IS_NOT_VALID:
3302 flush_code = IB_WC_GENERAL_ERR;
3303 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3304 termhdr->error_code = DDP_UNTAGGED_INV_MSN_RANGE;
3305 break;
3306 case NES_AEQE_AEID_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER:
3307 flush_code = IB_WC_LOC_LEN_ERR;
3308 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3309 termhdr->error_code = DDP_UNTAGGED_INV_TOO_LONG;
3310 break;
3311 case NES_AEQE_AEID_DDP_UBE_INVALID_DDP_VERSION:
3312 flush_code = IB_WC_GENERAL_ERR;
3313 if (is_tagged) {
3314 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_TAGGED_BUFFER;
3315 termhdr->error_code = DDP_TAGGED_INV_DDP_VER;
3316 } else {
3317 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3318 termhdr->error_code = DDP_UNTAGGED_INV_DDP_VER;
3319 }
3320 break;
3321 case NES_AEQE_AEID_DDP_UBE_INVALID_MO:
3322 flush_code = IB_WC_GENERAL_ERR;
3323 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3324 termhdr->error_code = DDP_UNTAGGED_INV_MO;
3325 break;
3326 case NES_AEQE_AEID_DDP_UBE_INVALID_MSN_NO_BUFFER_AVAILABLE:
3327 flush_code = IB_WC_REM_OP_ERR;
3328 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3329 termhdr->error_code = DDP_UNTAGGED_INV_MSN_NO_BUF;
3330 break;
3331 case NES_AEQE_AEID_DDP_UBE_INVALID_QN:
3332 flush_code = IB_WC_GENERAL_ERR;
3333 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3334 termhdr->error_code = DDP_UNTAGGED_INV_QN;
3335 break;
3336 case NES_AEQE_AEID_RDMAP_ROE_INVALID_RDMAP_VERSION:
3337 flush_code = IB_WC_GENERAL_ERR;
3338 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_OP;
3339 termhdr->error_code = RDMAP_INV_RDMAP_VER;
3340 break;
3341 case NES_AEQE_AEID_RDMAP_ROE_UNEXPECTED_OPCODE:
3342 flush_code = IB_WC_LOC_QP_OP_ERR;
3343 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_OP;
3344 termhdr->error_code = RDMAP_UNEXPECTED_OP;
3345 break;
3346 default:
3347 flush_code = IB_WC_FATAL_ERR;
3348 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_OP;
3349 termhdr->error_code = RDMAP_UNSPECIFIED;
3350 break;
3351 }
3352
3353 if (copy_len)
3354 memcpy(termhdr + 1, pkt, copy_len);
3355
3356 if ((flush_code) && ((NES_AEQE_INBOUND_RDMA & aeq_info) == 0)) {
3357 if (aeq_info & NES_AEQE_SQ)
3358 nesqp->term_sq_flush_code = flush_code;
3359 else
3360 nesqp->term_rq_flush_code = flush_code;
3361 }
3362
3363 return sizeof(struct nes_terminate_hdr) + copy_len;
3364 }
3365
3366 static void nes_terminate_connection(struct nes_device *nesdev, struct nes_qp *nesqp,
3367 struct nes_hw_aeqe *aeqe, enum ib_event_type eventtype)
3368 {
3369 u64 context;
3370 unsigned long flags;
3371 u32 aeq_info;
3372 u16 async_event_id;
3373 u8 tcp_state;
3374 u8 iwarp_state;
3375 u32 termlen = 0;
3376 u32 mod_qp_flags = NES_CQP_QP_IWARP_STATE_TERMINATE |
3377 NES_CQP_QP_TERM_DONT_SEND_FIN;
3378 struct nes_adapter *nesadapter = nesdev->nesadapter;
3379
3380 if (nesqp->term_flags & NES_TERM_SENT)
3381 return; /* Sanity check */
3382
3383 aeq_info = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
3384 tcp_state = (aeq_info & NES_AEQE_TCP_STATE_MASK) >> NES_AEQE_TCP_STATE_SHIFT;
3385 iwarp_state = (aeq_info & NES_AEQE_IWARP_STATE_MASK) >> NES_AEQE_IWARP_STATE_SHIFT;
3386 async_event_id = (u16)aeq_info;
3387
3388 context = (unsigned long)nesadapter->qp_table[le32_to_cpu(
3389 aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]) - NES_FIRST_QPN];
3390 if (!context) {
3391 WARN_ON(!context);
3392 return;
3393 }
3394
3395 nesqp = (struct nes_qp *)(unsigned long)context;
3396 spin_lock_irqsave(&nesqp->lock, flags);
3397 nesqp->hw_iwarp_state = iwarp_state;
3398 nesqp->hw_tcp_state = tcp_state;
3399 nesqp->last_aeq = async_event_id;
3400 nesqp->terminate_eventtype = eventtype;
3401 spin_unlock_irqrestore(&nesqp->lock, flags);
3402
3403 if (nesadapter->send_term_ok)
3404 termlen = nes_bld_terminate_hdr(nesqp, async_event_id, aeq_info);
3405 else
3406 mod_qp_flags |= NES_CQP_QP_TERM_DONT_SEND_TERM_MSG;
3407
3408 if (!nesdev->iw_status) {
3409 nesqp->term_flags = NES_TERM_DONE;
3410 nes_hw_modify_qp(nesdev, nesqp, NES_CQP_QP_IWARP_STATE_ERROR, 0, 0);
3411 nes_cm_disconn(nesqp);
3412 } else {
3413 nes_terminate_start_timer(nesqp);
3414 nesqp->term_flags |= NES_TERM_SENT;
3415 nes_hw_modify_qp(nesdev, nesqp, mod_qp_flags, termlen, 0);
3416 }
3417 }
3418
3419 static void nes_terminate_send_fin(struct nes_device *nesdev,
3420 struct nes_qp *nesqp, struct nes_hw_aeqe *aeqe)
3421 {
3422 u32 aeq_info;
3423 u16 async_event_id;
3424 u8 tcp_state;
3425 u8 iwarp_state;
3426 unsigned long flags;
3427
3428 aeq_info = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
3429 tcp_state = (aeq_info & NES_AEQE_TCP_STATE_MASK) >> NES_AEQE_TCP_STATE_SHIFT;
3430 iwarp_state = (aeq_info & NES_AEQE_IWARP_STATE_MASK) >> NES_AEQE_IWARP_STATE_SHIFT;
3431 async_event_id = (u16)aeq_info;
3432
3433 spin_lock_irqsave(&nesqp->lock, flags);
3434 nesqp->hw_iwarp_state = iwarp_state;
3435 nesqp->hw_tcp_state = tcp_state;
3436 nesqp->last_aeq = async_event_id;
3437 spin_unlock_irqrestore(&nesqp->lock, flags);
3438
3439 /* Send the fin only */
3440 nes_hw_modify_qp(nesdev, nesqp, NES_CQP_QP_IWARP_STATE_TERMINATE |
3441 NES_CQP_QP_TERM_DONT_SEND_TERM_MSG, 0, 0);
3442 }
3443
3444 /* Cleanup after a terminate sent or received */
3445 static void nes_terminate_done(struct nes_qp *nesqp, int timeout_occurred)
3446 {
3447 u32 next_iwarp_state = NES_CQP_QP_IWARP_STATE_ERROR;
3448 unsigned long flags;
3449 struct nes_vnic *nesvnic = to_nesvnic(nesqp->ibqp.device);
3450 struct nes_device *nesdev = nesvnic->nesdev;
3451 u8 first_time = 0;
3452
3453 spin_lock_irqsave(&nesqp->lock, flags);
3454 if (nesqp->hte_added) {
3455 nesqp->hte_added = 0;
3456 next_iwarp_state |= NES_CQP_QP_DEL_HTE;
3457 }
3458
3459 first_time = (nesqp->term_flags & NES_TERM_DONE) == 0;
3460 nesqp->term_flags |= NES_TERM_DONE;
3461 spin_unlock_irqrestore(&nesqp->lock, flags);
3462
3463 /* Make sure we go through this only once */
3464 if (first_time) {
3465 if (timeout_occurred == 0)
3466 del_timer(&nesqp->terminate_timer);
3467 else
3468 next_iwarp_state |= NES_CQP_QP_RESET;
3469
3470 nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0, 0);
3471 nes_cm_disconn(nesqp);
3472 }
3473 }
3474
3475 static void nes_terminate_received(struct nes_device *nesdev,
3476 struct nes_qp *nesqp, struct nes_hw_aeqe *aeqe)
3477 {
3478 u32 aeq_info;
3479 u8 *pkt;
3480 u32 *mpa;
3481 u8 ddp_ctl;
3482 u8 rdma_ctl;
3483 u16 aeq_id = 0;
3484
3485 aeq_info = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
3486 if (aeq_info & NES_AEQE_Q2_DATA_WRITTEN) {
3487 /* Terminate is not a performance path so the silicon */
3488 /* did not validate the frame - do it now */
3489 pkt = nesqp->hwqp.q2_vbase + BAD_FRAME_OFFSET;
3490 mpa = (u32 *)locate_mpa(pkt, aeq_info);
3491 ddp_ctl = (be32_to_cpu(mpa[0]) >> 8) & 0xff;
3492 rdma_ctl = be32_to_cpu(mpa[0]) & 0xff;
3493 if ((ddp_ctl & 0xc0) != 0x40)
3494 aeq_id = NES_AEQE_AEID_DDP_LCE_LOCAL_CATASTROPHIC;
3495 else if ((ddp_ctl & 0x03) != 1)
3496 aeq_id = NES_AEQE_AEID_DDP_UBE_INVALID_DDP_VERSION;
3497 else if (be32_to_cpu(mpa[2]) != 2)
3498 aeq_id = NES_AEQE_AEID_DDP_UBE_INVALID_QN;
3499 else if (be32_to_cpu(mpa[3]) != 1)
3500 aeq_id = NES_AEQE_AEID_DDP_INVALID_MSN_GAP_IN_MSN;
3501 else if (be32_to_cpu(mpa[4]) != 0)
3502 aeq_id = NES_AEQE_AEID_DDP_UBE_INVALID_MO;
3503 else if ((rdma_ctl & 0xc0) != 0x40)
3504 aeq_id = NES_AEQE_AEID_RDMAP_ROE_INVALID_RDMAP_VERSION;
3505
3506 if (aeq_id) {
3507 /* Bad terminate recvd - send back a terminate */
3508 aeq_info = (aeq_info & 0xffff0000) | aeq_id;
3509 aeqe->aeqe_words[NES_AEQE_MISC_IDX] = cpu_to_le32(aeq_info);
3510 nes_terminate_connection(nesdev, nesqp, aeqe, IB_EVENT_QP_FATAL);
3511 return;
3512 }
3513 }
3514
3515 nesqp->term_flags |= NES_TERM_RCVD;
3516 nesqp->terminate_eventtype = IB_EVENT_QP_FATAL;
3517 nes_terminate_start_timer(nesqp);
3518 nes_terminate_send_fin(nesdev, nesqp, aeqe);
3519 }
3520
3521 /* Timeout routine in case terminate fails to complete */
3522 void nes_terminate_timeout(unsigned long context)
3523 {
3524 struct nes_qp *nesqp = (struct nes_qp *)(unsigned long)context;
3525
3526 nes_terminate_done(nesqp, 1);
3527 }
3528
3529 /* Set a timer in case hw cannot complete the terminate sequence */
3530 static void nes_terminate_start_timer(struct nes_qp *nesqp)
3531 {
3532 mod_timer(&nesqp->terminate_timer, (jiffies + HZ));
3533 }
3534
3535 /**
3536 * nes_process_iwarp_aeqe
3537 */
3538 static void nes_process_iwarp_aeqe(struct nes_device *nesdev,
3539 struct nes_hw_aeqe *aeqe)
3540 {
3541 u64 context;
3542 unsigned long flags;
3543 struct nes_qp *nesqp;
3544 struct nes_hw_cq *hw_cq;
3545 struct nes_cq *nescq;
3546 int resource_allocated;
3547 struct nes_adapter *nesadapter = nesdev->nesadapter;
3548 u32 aeq_info;
3549 u32 next_iwarp_state = 0;
3550 u32 aeqe_cq_id;
3551 u16 async_event_id;
3552 u8 tcp_state;
3553 u8 iwarp_state;
3554 struct ib_event ibevent;
3555
3556 nes_debug(NES_DBG_AEQ, "\n");
3557 aeq_info = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
3558 if ((NES_AEQE_INBOUND_RDMA & aeq_info) || (!(NES_AEQE_QP & aeq_info))) {
3559 context = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_LOW_IDX]);
3560 context += ((u64)le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_HIGH_IDX])) << 32;
3561 } else {
3562 context = (unsigned long)nesadapter->qp_table[le32_to_cpu(
3563 aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]) - NES_FIRST_QPN];
3564 BUG_ON(!context);
3565 }
3566
3567 /* context is nesqp unless async_event_id == CQ ERROR */
3568 nesqp = (struct nes_qp *)(unsigned long)context;
3569 async_event_id = (u16)aeq_info;
3570 tcp_state = (aeq_info & NES_AEQE_TCP_STATE_MASK) >> NES_AEQE_TCP_STATE_SHIFT;
3571 iwarp_state = (aeq_info & NES_AEQE_IWARP_STATE_MASK) >> NES_AEQE_IWARP_STATE_SHIFT;
3572 nes_debug(NES_DBG_AEQ, "aeid = 0x%04X, qp-cq id = %d, aeqe = %p,"
3573 " Tcp state = %s, iWARP state = %s\n",
3574 async_event_id,
3575 le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]), aeqe,
3576 nes_tcp_state_str[tcp_state], nes_iwarp_state_str[iwarp_state]);
3577
3578 aeqe_cq_id = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]);
3579 if (aeq_info & NES_AEQE_QP) {
3580 if (!nes_is_resource_allocated(nesadapter,
3581 nesadapter->allocated_qps,
3582 aeqe_cq_id))
3583 return;
3584 }
3585
3586 switch (async_event_id) {
3587 case NES_AEQE_AEID_LLP_FIN_RECEIVED:
3588 if (nesqp->term_flags)
3589 return; /* Ignore it, wait for close complete */
3590
3591 if (atomic_inc_return(&nesqp->close_timer_started) == 1) {
3592 if ((tcp_state == NES_AEQE_TCP_STATE_CLOSE_WAIT) &&
3593 (nesqp->ibqp_state == IB_QPS_RTS)) {
3594 spin_lock_irqsave(&nesqp->lock, flags);
3595 nesqp->hw_iwarp_state = iwarp_state;
3596 nesqp->hw_tcp_state = tcp_state;
3597 nesqp->last_aeq = async_event_id;
3598 next_iwarp_state = NES_CQP_QP_IWARP_STATE_CLOSING;
3599 nesqp->hw_iwarp_state = NES_AEQE_IWARP_STATE_CLOSING;
3600 spin_unlock_irqrestore(&nesqp->lock, flags);
3601 nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0, 0);
3602 nes_cm_disconn(nesqp);
3603 }
3604 nesqp->cm_id->add_ref(nesqp->cm_id);
3605 schedule_nes_timer(nesqp->cm_node, (struct sk_buff *)nesqp,
3606 NES_TIMER_TYPE_CLOSE, 1, 0);
3607 nes_debug(NES_DBG_AEQ, "QP%u Not decrementing QP refcount (%d),"
3608 " need ae to finish up, original_last_aeq = 0x%04X."
3609 " last_aeq = 0x%04X, scheduling timer. TCP state = %d\n",
3610 nesqp->hwqp.qp_id, atomic_read(&nesqp->refcount),
3611 async_event_id, nesqp->last_aeq, tcp_state);
3612 }
3613 break;
3614 case NES_AEQE_AEID_LLP_CLOSE_COMPLETE:
3615 spin_lock_irqsave(&nesqp->lock, flags);
3616 nesqp->hw_iwarp_state = iwarp_state;
3617 nesqp->hw_tcp_state = tcp_state;
3618 nesqp->last_aeq = async_event_id;
3619 spin_unlock_irqrestore(&nesqp->lock, flags);
3620 nes_cm_disconn(nesqp);
3621 break;
3622
3623 case NES_AEQE_AEID_RESET_SENT:
3624 tcp_state = NES_AEQE_TCP_STATE_CLOSED;
3625 spin_lock_irqsave(&nesqp->lock, flags);
3626 nesqp->hw_iwarp_state = iwarp_state;
3627 nesqp->hw_tcp_state = tcp_state;
3628 nesqp->last_aeq = async_event_id;
3629 nesqp->hte_added = 0;
3630 spin_unlock_irqrestore(&nesqp->lock, flags);
3631 next_iwarp_state = NES_CQP_QP_IWARP_STATE_ERROR | NES_CQP_QP_DEL_HTE;
3632 nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0, 0);
3633 nes_cm_disconn(nesqp);
3634 break;
3635
3636 case NES_AEQE_AEID_LLP_CONNECTION_RESET:
3637 if (atomic_read(&nesqp->close_timer_started))
3638 return;
3639 spin_lock_irqsave(&nesqp->lock, flags);
3640 nesqp->hw_iwarp_state = iwarp_state;
3641 nesqp->hw_tcp_state = tcp_state;
3642 nesqp->last_aeq = async_event_id;
3643 spin_unlock_irqrestore(&nesqp->lock, flags);
3644 nes_cm_disconn(nesqp);
3645 break;
3646
3647 case NES_AEQE_AEID_TERMINATE_SENT:
3648 nes_terminate_send_fin(nesdev, nesqp, aeqe);
3649 break;
3650
3651 case NES_AEQE_AEID_LLP_TERMINATE_RECEIVED:
3652 nes_terminate_received(nesdev, nesqp, aeqe);
3653 break;
3654
3655 case NES_AEQE_AEID_AMP_BAD_STAG_KEY:
3656 case NES_AEQE_AEID_AMP_BAD_STAG_INDEX:
3657 case NES_AEQE_AEID_AMP_UNALLOCATED_STAG:
3658 case NES_AEQE_AEID_AMP_INVALID_STAG:
3659 case NES_AEQE_AEID_AMP_RIGHTS_VIOLATION:
3660 case NES_AEQE_AEID_AMP_INVALIDATE_NO_REMOTE_ACCESS_RIGHTS:
3661 case NES_AEQE_AEID_PRIV_OPERATION_DENIED:
3662 case NES_AEQE_AEID_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER:
3663 case NES_AEQE_AEID_AMP_BOUNDS_VIOLATION:
3664 case NES_AEQE_AEID_AMP_TO_WRAP:
3665 printk(KERN_ERR PFX "QP[%u] async_event_id=0x%04X IB_EVENT_QP_ACCESS_ERR\n",
3666 nesqp->hwqp.qp_id, async_event_id);
3667 nes_terminate_connection(nesdev, nesqp, aeqe, IB_EVENT_QP_ACCESS_ERR);
3668 break;
3669
3670 case NES_AEQE_AEID_LLP_SEGMENT_TOO_LARGE:
3671 case NES_AEQE_AEID_LLP_SEGMENT_TOO_SMALL:
3672 case NES_AEQE_AEID_DDP_UBE_INVALID_MO:
3673 case NES_AEQE_AEID_DDP_UBE_INVALID_QN:
3674 if (iwarp_opcode(nesqp, aeq_info) > IWARP_OPCODE_TERM) {
3675 aeq_info &= 0xffff0000;
3676 aeq_info |= NES_AEQE_AEID_RDMAP_ROE_UNEXPECTED_OPCODE;
3677 aeqe->aeqe_words[NES_AEQE_MISC_IDX] = cpu_to_le32(aeq_info);
3678 }
3679
3680 case NES_AEQE_AEID_RDMAP_ROE_BAD_LLP_CLOSE:
3681 case NES_AEQE_AEID_LLP_TOO_MANY_RETRIES:
3682 case NES_AEQE_AEID_DDP_UBE_INVALID_MSN_NO_BUFFER_AVAILABLE:
3683 case NES_AEQE_AEID_LLP_RECEIVED_MPA_CRC_ERROR:
3684 case NES_AEQE_AEID_AMP_BAD_QP:
3685 case NES_AEQE_AEID_LLP_RECEIVED_MARKER_AND_LENGTH_FIELDS_DONT_MATCH:
3686 case NES_AEQE_AEID_DDP_LCE_LOCAL_CATASTROPHIC:
3687 case NES_AEQE_AEID_DDP_NO_L_BIT:
3688 case NES_AEQE_AEID_DDP_INVALID_MSN_GAP_IN_MSN:
3689 case NES_AEQE_AEID_DDP_INVALID_MSN_RANGE_IS_NOT_VALID:
3690 case NES_AEQE_AEID_DDP_UBE_INVALID_DDP_VERSION:
3691 case NES_AEQE_AEID_RDMAP_ROE_INVALID_RDMAP_VERSION:
3692 case NES_AEQE_AEID_RDMAP_ROE_UNEXPECTED_OPCODE:
3693 case NES_AEQE_AEID_AMP_BAD_PD:
3694 case NES_AEQE_AEID_AMP_FASTREG_SHARED:
3695 case NES_AEQE_AEID_AMP_FASTREG_VALID_STAG:
3696 case NES_AEQE_AEID_AMP_FASTREG_MW_STAG:
3697 case NES_AEQE_AEID_AMP_FASTREG_INVALID_RIGHTS:
3698 case NES_AEQE_AEID_AMP_FASTREG_PBL_TABLE_OVERFLOW:
3699 case NES_AEQE_AEID_AMP_FASTREG_INVALID_LENGTH:
3700 case NES_AEQE_AEID_AMP_INVALIDATE_SHARED:
3701 case NES_AEQE_AEID_AMP_INVALIDATE_MR_WITH_BOUND_WINDOWS:
3702 case NES_AEQE_AEID_AMP_MWBIND_VALID_STAG:
3703 case NES_AEQE_AEID_AMP_MWBIND_OF_MR_STAG:
3704 case NES_AEQE_AEID_AMP_MWBIND_TO_ZERO_BASED_STAG:
3705 case NES_AEQE_AEID_AMP_MWBIND_TO_MW_STAG:
3706 case NES_AEQE_AEID_AMP_MWBIND_INVALID_RIGHTS:
3707 case NES_AEQE_AEID_AMP_MWBIND_INVALID_BOUNDS:
3708 case NES_AEQE_AEID_AMP_MWBIND_TO_INVALID_PARENT:
3709 case NES_AEQE_AEID_AMP_MWBIND_BIND_DISABLED:
3710 case NES_AEQE_AEID_BAD_CLOSE:
3711 case NES_AEQE_AEID_RDMA_READ_WHILE_ORD_ZERO:
3712 case NES_AEQE_AEID_STAG_ZERO_INVALID:
3713 case NES_AEQE_AEID_ROE_INVALID_RDMA_READ_REQUEST:
3714 case NES_AEQE_AEID_ROE_INVALID_RDMA_WRITE_OR_READ_RESP:
3715 printk(KERN_ERR PFX "QP[%u] async_event_id=0x%04X IB_EVENT_QP_FATAL\n",
3716 nesqp->hwqp.qp_id, async_event_id);
3717 print_ip(nesqp->cm_node);
3718 if (!atomic_read(&nesqp->close_timer_started))
3719 nes_terminate_connection(nesdev, nesqp, aeqe, IB_EVENT_QP_FATAL);
3720 break;
3721
3722 case NES_AEQE_AEID_CQ_OPERATION_ERROR:
3723 context <<= 1;
3724 nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_CQ_OPERATION_ERROR event on CQ%u, %p\n",
3725 le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]), (void *)(unsigned long)context);
3726 resource_allocated = nes_is_resource_allocated(nesadapter, nesadapter->allocated_cqs,
3727 le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]));
3728 if (resource_allocated) {
3729 printk(KERN_ERR PFX "%s: Processing an NES_AEQE_AEID_CQ_OPERATION_ERROR event on CQ%u\n",
3730 __func__, le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]));
3731 hw_cq = (struct nes_hw_cq *)(unsigned long)context;
3732 if (hw_cq) {
3733 nescq = container_of(hw_cq, struct nes_cq, hw_cq);
3734 if (nescq->ibcq.event_handler) {
3735 ibevent.device = nescq->ibcq.device;
3736 ibevent.event = IB_EVENT_CQ_ERR;
3737 ibevent.element.cq = &nescq->ibcq;
3738 nescq->ibcq.event_handler(&ibevent, nescq->ibcq.cq_context);
3739 }
3740 }
3741 }
3742 break;
3743
3744 default:
3745 nes_debug(NES_DBG_AEQ, "Processing an iWARP related AE for QP, misc = 0x%04X\n",
3746 async_event_id);
3747 break;
3748 }
3749
3750 }
3751
3752 /**
3753 * nes_iwarp_ce_handler
3754 */
3755 void nes_iwarp_ce_handler(struct nes_device *nesdev, struct nes_hw_cq *hw_cq)
3756 {
3757 struct nes_cq *nescq = container_of(hw_cq, struct nes_cq, hw_cq);
3758
3759 /* nes_debug(NES_DBG_CQ, "Processing completion event for iWARP CQ%u.\n",
3760 nescq->hw_cq.cq_number); */
3761 nes_write32(nesdev->regs+NES_CQ_ACK, nescq->hw_cq.cq_number);
3762
3763 if (nescq->ibcq.comp_handler)
3764 nescq->ibcq.comp_handler(&nescq->ibcq, nescq->ibcq.cq_context);
3765
3766 return;
3767 }
3768
3769
3770 /**
3771 * nes_manage_apbvt()
3772 */
3773 int nes_manage_apbvt(struct nes_vnic *nesvnic, u32 accel_local_port,
3774 u32 nic_index, u32 add_port)
3775 {
3776 struct nes_device *nesdev = nesvnic->nesdev;
3777 struct nes_hw_cqp_wqe *cqp_wqe;
3778 struct nes_cqp_request *cqp_request;
3779 int ret = 0;
3780 u16 major_code;
3781
3782 /* Send manage APBVT request to CQP */
3783 cqp_request = nes_get_cqp_request(nesdev);
3784 if (cqp_request == NULL) {
3785 nes_debug(NES_DBG_QP, "Failed to get a cqp_request.\n");
3786 return -ENOMEM;
3787 }
3788 cqp_request->waiting = 1;
3789 cqp_wqe = &cqp_request->cqp_wqe;
3790
3791 nes_debug(NES_DBG_QP, "%s APBV for local port=%u(0x%04x), nic_index=%u\n",
3792 (add_port == NES_MANAGE_APBVT_ADD) ? "ADD" : "DEL",
3793 accel_local_port, accel_local_port, nic_index);
3794
3795 nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
3796 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX, (NES_CQP_MANAGE_APBVT |
3797 ((add_port == NES_MANAGE_APBVT_ADD) ? NES_CQP_APBVT_ADD : 0)));
3798 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
3799 ((nic_index << NES_CQP_APBVT_NIC_SHIFT) | accel_local_port));
3800
3801 nes_debug(NES_DBG_QP, "Waiting for CQP completion for APBVT.\n");
3802
3803 atomic_set(&cqp_request->refcount, 2);
3804 nes_post_cqp_request(nesdev, cqp_request);
3805
3806 if (add_port == NES_MANAGE_APBVT_ADD)
3807 ret = wait_event_timeout(cqp_request->waitq, (cqp_request->request_done != 0),
3808 NES_EVENT_TIMEOUT);
3809 nes_debug(NES_DBG_QP, "Completed, ret=%u, CQP Major:Minor codes = 0x%04X:0x%04X\n",
3810 ret, cqp_request->major_code, cqp_request->minor_code);
3811 major_code = cqp_request->major_code;
3812
3813 nes_put_cqp_request(nesdev, cqp_request);
3814
3815 if (!ret)
3816 return -ETIME;
3817 else if (major_code)
3818 return -EIO;
3819 else
3820 return 0;
3821 }
3822
3823
3824 /**
3825 * nes_manage_arp_cache
3826 */
3827 void nes_manage_arp_cache(struct net_device *netdev, unsigned char *mac_addr,
3828 u32 ip_addr, u32 action)
3829 {
3830 struct nes_hw_cqp_wqe *cqp_wqe;
3831 struct nes_vnic *nesvnic = netdev_priv(netdev);
3832 struct nes_device *nesdev;
3833 struct nes_cqp_request *cqp_request;
3834 int arp_index;
3835
3836 nesdev = nesvnic->nesdev;
3837 arp_index = nes_arp_table(nesdev, ip_addr, mac_addr, action);
3838 if (arp_index == -1) {
3839 return;
3840 }
3841
3842 /* update the ARP entry */
3843 cqp_request = nes_get_cqp_request(nesdev);
3844 if (cqp_request == NULL) {
3845 nes_debug(NES_DBG_NETDEV, "Failed to get a cqp_request.\n");
3846 return;
3847 }
3848 cqp_request->waiting = 0;
3849 cqp_wqe = &cqp_request->cqp_wqe;
3850 nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
3851
3852 cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(
3853 NES_CQP_MANAGE_ARP_CACHE | NES_CQP_ARP_PERM);
3854 cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] |= cpu_to_le32(
3855 (u32)PCI_FUNC(nesdev->pcidev->devfn) << NES_CQP_ARP_AEQ_INDEX_SHIFT);
3856 cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(arp_index);
3857
3858 if (action == NES_ARP_ADD) {
3859 cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] |= cpu_to_le32(NES_CQP_ARP_VALID);
3860 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_ADDR_LOW_IDX] = cpu_to_le32(
3861 (((u32)mac_addr[2]) << 24) | (((u32)mac_addr[3]) << 16) |
3862 (((u32)mac_addr[4]) << 8) | (u32)mac_addr[5]);
3863 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_HIGH_IDX] = cpu_to_le32(
3864 (((u32)mac_addr[0]) << 8) | (u32)mac_addr[1]);
3865 } else {
3866 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_ADDR_LOW_IDX] = 0;
3867 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_HIGH_IDX] = 0;
3868 }
3869
3870 nes_debug(NES_DBG_NETDEV, "Not waiting for CQP, cqp.sq_head=%u, cqp.sq_tail=%u\n",
3871 nesdev->cqp.sq_head, nesdev->cqp.sq_tail);
3872
3873 atomic_set(&cqp_request->refcount, 1);
3874 nes_post_cqp_request(nesdev, cqp_request);
3875 }
3876
3877
3878 /**
3879 * flush_wqes
3880 */
3881 void flush_wqes(struct nes_device *nesdev, struct nes_qp *nesqp,
3882 u32 which_wq, u32 wait_completion)
3883 {
3884 struct nes_cqp_request *cqp_request;
3885 struct nes_hw_cqp_wqe *cqp_wqe;
3886 u32 sq_code = (NES_IWARP_CQE_MAJOR_FLUSH << 16) | NES_IWARP_CQE_MINOR_FLUSH;
3887 u32 rq_code = (NES_IWARP_CQE_MAJOR_FLUSH << 16) | NES_IWARP_CQE_MINOR_FLUSH;
3888 int ret;
3889
3890 cqp_request = nes_get_cqp_request(nesdev);
3891 if (cqp_request == NULL) {
3892 nes_debug(NES_DBG_QP, "Failed to get a cqp_request.\n");
3893 return;
3894 }
3895 if (wait_completion) {
3896 cqp_request->waiting = 1;
3897 atomic_set(&cqp_request->refcount, 2);
3898 } else {
3899 cqp_request->waiting = 0;
3900 }
3901 cqp_wqe = &cqp_request->cqp_wqe;
3902 nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
3903
3904 /* If wqe in error was identified, set code to be put into cqe */
3905 if ((nesqp->term_sq_flush_code) && (which_wq & NES_CQP_FLUSH_SQ)) {
3906 which_wq |= NES_CQP_FLUSH_MAJ_MIN;
3907 sq_code = (CQE_MAJOR_DRV << 16) | nesqp->term_sq_flush_code;
3908 nesqp->term_sq_flush_code = 0;
3909 }
3910
3911 if ((nesqp->term_rq_flush_code) && (which_wq & NES_CQP_FLUSH_RQ)) {
3912 which_wq |= NES_CQP_FLUSH_MAJ_MIN;
3913 rq_code = (CQE_MAJOR_DRV << 16) | nesqp->term_rq_flush_code;
3914 nesqp->term_rq_flush_code = 0;
3915 }
3916
3917 if (which_wq & NES_CQP_FLUSH_MAJ_MIN) {
3918 cqp_wqe->wqe_words[NES_CQP_QP_WQE_FLUSH_SQ_CODE] = cpu_to_le32(sq_code);
3919 cqp_wqe->wqe_words[NES_CQP_QP_WQE_FLUSH_RQ_CODE] = cpu_to_le32(rq_code);
3920 }
3921
3922 cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] =
3923 cpu_to_le32(NES_CQP_FLUSH_WQES | which_wq);
3924 cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesqp->hwqp.qp_id);
3925
3926 nes_post_cqp_request(nesdev, cqp_request);
3927
3928 if (wait_completion) {
3929 /* Wait for CQP */
3930 ret = wait_event_timeout(cqp_request->waitq, (cqp_request->request_done != 0),
3931 NES_EVENT_TIMEOUT);
3932 nes_debug(NES_DBG_QP, "Flush SQ QP WQEs completed, ret=%u,"
3933 " CQP Major:Minor codes = 0x%04X:0x%04X\n",
3934 ret, cqp_request->major_code, cqp_request->minor_code);
3935 nes_put_cqp_request(nesdev, cqp_request);
3936 }
3937 }
ubuntu-bionic-kernel mirror