]> git.proxmox.com Git - mirror_ubuntu-bionic-kernel.git/blob - drivers/infiniband/hw/nes/nes_hw.c
projects / mirror_ubuntu-bionic-kernel.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
RDMA/nes: Replace LRO with GRO
[mirror_ubuntu-bionic-kernel.git] / drivers / infiniband / hw / nes / nes_hw.c
1 /*
2 * Copyright (c) 2006 - 2011 Intel Corporation. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 *
32 */
33
34 #include <linux/module.h>
35 #include <linux/moduleparam.h>
36 #include <linux/netdevice.h>
37 #include <linux/etherdevice.h>
38 #include <linux/if_vlan.h>
39 #include <linux/slab.h>
40
41 #include "nes.h"
42
43 static int wide_ppm_offset;
44 module_param(wide_ppm_offset, int, 0644);
45 MODULE_PARM_DESC(wide_ppm_offset, "Increase CX4 interface clock ppm offset, 0=100ppm (default), 1=300ppm");
46
47 static u32 crit_err_count;
48 u32 int_mod_timer_init;
49 u32 int_mod_cq_depth_256;
50 u32 int_mod_cq_depth_128;
51 u32 int_mod_cq_depth_32;
52 u32 int_mod_cq_depth_24;
53 u32 int_mod_cq_depth_16;
54 u32 int_mod_cq_depth_4;
55 u32 int_mod_cq_depth_1;
56 static const u8 nes_max_critical_error_count = 100;
57 #include "nes_cm.h"
58
59 static void nes_cqp_ce_handler(struct nes_device *nesdev, struct nes_hw_cq *cq);
60 static void nes_init_csr_ne020(struct nes_device *nesdev, u8 hw_rev, u8 port_count);
61 static int nes_init_serdes(struct nes_device *nesdev, u8 hw_rev, u8 port_count,
62 struct nes_adapter *nesadapter, u8 OneG_Mode);
63 static void nes_nic_napi_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq);
64 static void nes_process_aeq(struct nes_device *nesdev, struct nes_hw_aeq *aeq);
65 static void nes_process_ceq(struct nes_device *nesdev, struct nes_hw_ceq *ceq);
66 static void nes_process_iwarp_aeqe(struct nes_device *nesdev,
67 struct nes_hw_aeqe *aeqe);
68 static void process_critical_error(struct nes_device *nesdev);
69 static void nes_process_mac_intr(struct nes_device *nesdev, u32 mac_number);
70 static unsigned int nes_reset_adapter_ne020(struct nes_device *nesdev, u8 *OneG_Mode);
71 static void nes_terminate_start_timer(struct nes_qp *nesqp);
72
73 #ifdef CONFIG_INFINIBAND_NES_DEBUG
74 static unsigned char *nes_iwarp_state_str[] = {
75 "Non-Existent",
76 "Idle",
77 "RTS",
78 "Closing",
79 "RSVD1",
80 "Terminate",
81 "Error",
82 "RSVD2",
83 };
84
85 static unsigned char *nes_tcp_state_str[] = {
86 "Non-Existent",
87 "Closed",
88 "Listen",
89 "SYN Sent",
90 "SYN Rcvd",
91 "Established",
92 "Close Wait",
93 "FIN Wait 1",
94 "Closing",
95 "Last Ack",
96 "FIN Wait 2",
97 "Time Wait",
98 "RSVD1",
99 "RSVD2",
100 "RSVD3",
101 "RSVD4",
102 };
103 #endif
104
105 static inline void print_ip(struct nes_cm_node *cm_node)
106 {
107 unsigned char *rem_addr;
108 if (cm_node) {
109 rem_addr = (unsigned char *)&cm_node->rem_addr;
110 printk(KERN_ERR PFX "Remote IP addr: %pI4\n", rem_addr);
111 }
112 }
113
114 /**
115 * nes_nic_init_timer_defaults
116 */
117 void nes_nic_init_timer_defaults(struct nes_device *nesdev, u8 jumbomode)
118 {
119 unsigned long flags;
120 struct nes_adapter *nesadapter = nesdev->nesadapter;
121 struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer;
122
123 spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags);
124
125 shared_timer->timer_in_use_min = NES_NIC_FAST_TIMER_LOW;
126 shared_timer->timer_in_use_max = NES_NIC_FAST_TIMER_HIGH;
127 if (jumbomode) {
128 shared_timer->threshold_low = DEFAULT_JUMBO_NES_QL_LOW;
129 shared_timer->threshold_target = DEFAULT_JUMBO_NES_QL_TARGET;
130 shared_timer->threshold_high = DEFAULT_JUMBO_NES_QL_HIGH;
131 } else {
132 shared_timer->threshold_low = DEFAULT_NES_QL_LOW;
133 shared_timer->threshold_target = DEFAULT_NES_QL_TARGET;
134 shared_timer->threshold_high = DEFAULT_NES_QL_HIGH;
135 }
136
137 /* todo use netdev->mtu to set thresholds */
138 spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
139 }
140
141
142 /**
143 * nes_nic_init_timer
144 */
145 static void nes_nic_init_timer(struct nes_device *nesdev)
146 {
147 unsigned long flags;
148 struct nes_adapter *nesadapter = nesdev->nesadapter;
149 struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer;
150
151 spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags);
152
153 if (shared_timer->timer_in_use_old == 0) {
154 nesdev->deepcq_count = 0;
155 shared_timer->timer_direction_upward = 0;
156 shared_timer->timer_direction_downward = 0;
157 shared_timer->timer_in_use = NES_NIC_FAST_TIMER;
158 shared_timer->timer_in_use_old = 0;
159
160 }
161 if (shared_timer->timer_in_use != shared_timer->timer_in_use_old) {
162 shared_timer->timer_in_use_old = shared_timer->timer_in_use;
163 nes_write32(nesdev->regs+NES_PERIODIC_CONTROL,
164 0x80000000 | ((u32)(shared_timer->timer_in_use*8)));
165 }
166 /* todo use netdev->mtu to set thresholds */
167 spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
168 }
169
170
171 /**
172 * nes_nic_tune_timer
173 */
174 static void nes_nic_tune_timer(struct nes_device *nesdev)
175 {
176 unsigned long flags;
177 struct nes_adapter *nesadapter = nesdev->nesadapter;
178 struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer;
179 u16 cq_count = nesdev->currcq_count;
180
181 spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags);
182
183 if (shared_timer->cq_count_old <= cq_count)
184 shared_timer->cq_direction_downward = 0;
185 else
186 shared_timer->cq_direction_downward++;
187 shared_timer->cq_count_old = cq_count;
188 if (shared_timer->cq_direction_downward > NES_NIC_CQ_DOWNWARD_TREND) {
189 if (cq_count <= shared_timer->threshold_low &&
190 shared_timer->threshold_low > 4) {
191 shared_timer->threshold_low = shared_timer->threshold_low/2;
192 shared_timer->cq_direction_downward=0;
193 nesdev->currcq_count = 0;
194 spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
195 return;
196 }
197 }
198
199 if (cq_count > 1) {
200 nesdev->deepcq_count += cq_count;
201 if (cq_count <= shared_timer->threshold_low) { /* increase timer gently */
202 shared_timer->timer_direction_upward++;
203 shared_timer->timer_direction_downward = 0;
204 } else if (cq_count <= shared_timer->threshold_target) { /* balanced */
205 shared_timer->timer_direction_upward = 0;
206 shared_timer->timer_direction_downward = 0;
207 } else if (cq_count <= shared_timer->threshold_high) { /* decrease timer gently */
208 shared_timer->timer_direction_downward++;
209 shared_timer->timer_direction_upward = 0;
210 } else if (cq_count <= (shared_timer->threshold_high) * 2) {
211 shared_timer->timer_in_use -= 2;
212 shared_timer->timer_direction_upward = 0;
213 shared_timer->timer_direction_downward++;
214 } else {
215 shared_timer->timer_in_use -= 4;
216 shared_timer->timer_direction_upward = 0;
217 shared_timer->timer_direction_downward++;
218 }
219
220 if (shared_timer->timer_direction_upward > 3 ) { /* using history */
221 shared_timer->timer_in_use += 3;
222 shared_timer->timer_direction_upward = 0;
223 shared_timer->timer_direction_downward = 0;
224 }
225 if (shared_timer->timer_direction_downward > 5) { /* using history */
226 shared_timer->timer_in_use -= 4 ;
227 shared_timer->timer_direction_downward = 0;
228 shared_timer->timer_direction_upward = 0;
229 }
230 }
231
232 /* boundary checking */
233 if (shared_timer->timer_in_use > shared_timer->threshold_high)
234 shared_timer->timer_in_use = shared_timer->threshold_high;
235 else if (shared_timer->timer_in_use < shared_timer->threshold_low)
236 shared_timer->timer_in_use = shared_timer->threshold_low;
237
238 nesdev->currcq_count = 0;
239
240 spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
241 }
242
243
244 /**
245 * nes_init_adapter - initialize adapter
246 */
247 struct nes_adapter *nes_init_adapter(struct nes_device *nesdev, u8 hw_rev) {
248 struct nes_adapter *nesadapter = NULL;
249 unsigned long num_pds;
250 u32 u32temp;
251 u32 port_count;
252 u16 max_rq_wrs;
253 u16 max_sq_wrs;
254 u32 max_mr;
255 u32 max_256pbl;
256 u32 max_4kpbl;
257 u32 max_qp;
258 u32 max_irrq;
259 u32 max_cq;
260 u32 hte_index_mask;
261 u32 adapter_size;
262 u32 arp_table_size;
263 u16 vendor_id;
264 u16 device_id;
265 u8 OneG_Mode;
266 u8 func_index;
267
268 /* search the list of existing adapters */
269 list_for_each_entry(nesadapter, &nes_adapter_list, list) {
270 nes_debug(NES_DBG_INIT, "Searching Adapter list for PCI devfn = 0x%X,"
271 " adapter PCI slot/bus = %u/%u, pci devices PCI slot/bus = %u/%u, .\n",
272 nesdev->pcidev->devfn,
273 PCI_SLOT(nesadapter->devfn),
274 nesadapter->bus_number,
275 PCI_SLOT(nesdev->pcidev->devfn),
276 nesdev->pcidev->bus->number );
277 if ((PCI_SLOT(nesadapter->devfn) == PCI_SLOT(nesdev->pcidev->devfn)) &&
278 (nesadapter->bus_number == nesdev->pcidev->bus->number)) {
279 nesadapter->ref_count++;
280 return nesadapter;
281 }
282 }
283
284 /* no adapter found */
285 num_pds = pci_resource_len(nesdev->pcidev, BAR_1) >> PAGE_SHIFT;
286 if ((hw_rev != NE020_REV) && (hw_rev != NE020_REV1)) {
287 nes_debug(NES_DBG_INIT, "NE020 driver detected unknown hardware revision 0x%x\n",
288 hw_rev);
289 return NULL;
290 }
291
292 nes_debug(NES_DBG_INIT, "Determine Soft Reset, QP_control=0x%x, CPU0=0x%x, CPU1=0x%x, CPU2=0x%x\n",
293 nes_read_indexed(nesdev, NES_IDX_QP_CONTROL + PCI_FUNC(nesdev->pcidev->devfn) * 8),
294 nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS),
295 nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS + 4),
296 nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS + 8));
297
298 nes_debug(NES_DBG_INIT, "Reset and init NE020\n");
299
300
301 if ((port_count = nes_reset_adapter_ne020(nesdev, &OneG_Mode)) == 0)
302 return NULL;
303
304 max_qp = nes_read_indexed(nesdev, NES_IDX_QP_CTX_SIZE);
305 nes_debug(NES_DBG_INIT, "QP_CTX_SIZE=%u\n", max_qp);
306
307 u32temp = nes_read_indexed(nesdev, NES_IDX_QUAD_HASH_TABLE_SIZE);
308 if (max_qp > ((u32)1 << (u32temp & 0x001f))) {
309 nes_debug(NES_DBG_INIT, "Reducing Max QPs to %u due to hash table size = 0x%08X\n",
310 max_qp, u32temp);
311 max_qp = (u32)1 << (u32temp & 0x001f);
312 }
313
314 hte_index_mask = ((u32)1 << ((u32temp & 0x001f)+1))-1;
315 nes_debug(NES_DBG_INIT, "Max QP = %u, hte_index_mask = 0x%08X.\n",
316 max_qp, hte_index_mask);
317
318 u32temp = nes_read_indexed(nesdev, NES_IDX_IRRQ_COUNT);
319
320 max_irrq = 1 << (u32temp & 0x001f);
321
322 if (max_qp > max_irrq) {
323 max_qp = max_irrq;
324 nes_debug(NES_DBG_INIT, "Reducing Max QPs to %u due to Available Q1s.\n",
325 max_qp);
326 }
327
328 /* there should be no reason to allocate more pds than qps */
329 if (num_pds > max_qp)
330 num_pds = max_qp;
331
332 u32temp = nes_read_indexed(nesdev, NES_IDX_MRT_SIZE);
333 max_mr = (u32)8192 << (u32temp & 0x7);
334
335 u32temp = nes_read_indexed(nesdev, NES_IDX_PBL_REGION_SIZE);
336 max_256pbl = (u32)1 << (u32temp & 0x0000001f);
337 max_4kpbl = (u32)1 << ((u32temp >> 16) & 0x0000001f);
338 max_cq = nes_read_indexed(nesdev, NES_IDX_CQ_CTX_SIZE);
339
340 u32temp = nes_read_indexed(nesdev, NES_IDX_ARP_CACHE_SIZE);
341 arp_table_size = 1 << u32temp;
342
343 adapter_size = (sizeof(struct nes_adapter) +
344 (sizeof(unsigned long)-1)) & (~(sizeof(unsigned long)-1));
345 adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(max_qp);
346 adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(max_mr);
347 adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(max_cq);
348 adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(num_pds);
349 adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(arp_table_size);
350 adapter_size += sizeof(struct nes_qp **) * max_qp;
351
352 /* allocate a new adapter struct */
353 nesadapter = kzalloc(adapter_size, GFP_KERNEL);
354 if (nesadapter == NULL) {
355 return NULL;
356 }
357
358 nes_debug(NES_DBG_INIT, "Allocating new nesadapter @ %p, size = %u (actual size = %u).\n",
359 nesadapter, (u32)sizeof(struct nes_adapter), adapter_size);
360
361 if (nes_read_eeprom_values(nesdev, nesadapter)) {
362 printk(KERN_ERR PFX "Unable to read EEPROM data.\n");
363 kfree(nesadapter);
364 return NULL;
365 }
366
367 nesadapter->vendor_id = (((u32) nesadapter->mac_addr_high) << 8) |
368 (nesadapter->mac_addr_low >> 24);
369
370 pci_bus_read_config_word(nesdev->pcidev->bus, nesdev->pcidev->devfn,
371 PCI_DEVICE_ID, &device_id);
372 nesadapter->vendor_part_id = device_id;
373
374 if (nes_init_serdes(nesdev, hw_rev, port_count, nesadapter,
375 OneG_Mode)) {
376 kfree(nesadapter);
377 return NULL;
378 }
379 nes_init_csr_ne020(nesdev, hw_rev, port_count);
380
381 memset(nesadapter->pft_mcast_map, 255,
382 sizeof nesadapter->pft_mcast_map);
383
384 /* populate the new nesadapter */
385 nesadapter->devfn = nesdev->pcidev->devfn;
386 nesadapter->bus_number = nesdev->pcidev->bus->number;
387 nesadapter->ref_count = 1;
388 nesadapter->timer_int_req = 0xffff0000;
389 nesadapter->OneG_Mode = OneG_Mode;
390 nesadapter->doorbell_start = nesdev->doorbell_region;
391
392 /* nesadapter->tick_delta = clk_divisor; */
393 nesadapter->hw_rev = hw_rev;
394 nesadapter->port_count = port_count;
395
396 nesadapter->max_qp = max_qp;
397 nesadapter->hte_index_mask = hte_index_mask;
398 nesadapter->max_irrq = max_irrq;
399 nesadapter->max_mr = max_mr;
400 nesadapter->max_256pbl = max_256pbl - 1;
401 nesadapter->max_4kpbl = max_4kpbl - 1;
402 nesadapter->max_cq = max_cq;
403 nesadapter->free_256pbl = max_256pbl - 1;
404 nesadapter->free_4kpbl = max_4kpbl - 1;
405 nesadapter->max_pd = num_pds;
406 nesadapter->arp_table_size = arp_table_size;
407
408 nesadapter->et_pkt_rate_low = NES_TIMER_ENABLE_LIMIT;
409 if (nes_drv_opt & NES_DRV_OPT_DISABLE_INT_MOD) {
410 nesadapter->et_use_adaptive_rx_coalesce = 0;
411 nesadapter->timer_int_limit = NES_TIMER_INT_LIMIT;
412 nesadapter->et_rx_coalesce_usecs_irq = interrupt_mod_interval;
413 } else {
414 nesadapter->et_use_adaptive_rx_coalesce = 1;
415 nesadapter->timer_int_limit = NES_TIMER_INT_LIMIT_DYNAMIC;
416 nesadapter->et_rx_coalesce_usecs_irq = 0;
417 printk(PFX "%s: Using Adaptive Interrupt Moderation\n", __func__);
418 }
419 /* Setup and enable the periodic timer */
420 if (nesadapter->et_rx_coalesce_usecs_irq)
421 nes_write32(nesdev->regs+NES_PERIODIC_CONTROL, 0x80000000 |
422 ((u32)(nesadapter->et_rx_coalesce_usecs_irq * 8)));
423 else
424 nes_write32(nesdev->regs+NES_PERIODIC_CONTROL, 0x00000000);
425
426 nesadapter->base_pd = 1;
427
428 nesadapter->device_cap_flags = IB_DEVICE_LOCAL_DMA_LKEY |
429 IB_DEVICE_MEM_WINDOW |
430 IB_DEVICE_MEM_MGT_EXTENSIONS;
431
432 nesadapter->allocated_qps = (unsigned long *)&(((unsigned char *)nesadapter)
433 [(sizeof(struct nes_adapter)+(sizeof(unsigned long)-1))&(~(sizeof(unsigned long)-1))]);
434 nesadapter->allocated_cqs = &nesadapter->allocated_qps[BITS_TO_LONGS(max_qp)];
435 nesadapter->allocated_mrs = &nesadapter->allocated_cqs[BITS_TO_LONGS(max_cq)];
436 nesadapter->allocated_pds = &nesadapter->allocated_mrs[BITS_TO_LONGS(max_mr)];
437 nesadapter->allocated_arps = &nesadapter->allocated_pds[BITS_TO_LONGS(num_pds)];
438 nesadapter->qp_table = (struct nes_qp **)(&nesadapter->allocated_arps[BITS_TO_LONGS(arp_table_size)]);
439
440
441 /* mark the usual suspect QPs, MR and CQs as in use */
442 for (u32temp = 0; u32temp < NES_FIRST_QPN; u32temp++) {
443 set_bit(u32temp, nesadapter->allocated_qps);
444 set_bit(u32temp, nesadapter->allocated_cqs);
445 }
446 set_bit(0, nesadapter->allocated_mrs);
447
448 for (u32temp = 0; u32temp < 20; u32temp++)
449 set_bit(u32temp, nesadapter->allocated_pds);
450 u32temp = nes_read_indexed(nesdev, NES_IDX_QP_MAX_CFG_SIZES);
451
452 max_rq_wrs = ((u32temp >> 8) & 3);
453 switch (max_rq_wrs) {
454 case 0:
455 max_rq_wrs = 4;
456 break;
457 case 1:
458 max_rq_wrs = 16;
459 break;
460 case 2:
461 max_rq_wrs = 32;
462 break;
463 case 3:
464 max_rq_wrs = 512;
465 break;
466 }
467
468 max_sq_wrs = (u32temp & 3);
469 switch (max_sq_wrs) {
470 case 0:
471 max_sq_wrs = 4;
472 break;
473 case 1:
474 max_sq_wrs = 16;
475 break;
476 case 2:
477 max_sq_wrs = 32;
478 break;
479 case 3:
480 max_sq_wrs = 512;
481 break;
482 }
483 nesadapter->max_qp_wr = min(max_rq_wrs, max_sq_wrs);
484 nesadapter->max_irrq_wr = (u32temp >> 16) & 3;
485
486 nesadapter->max_sge = 4;
487 nesadapter->max_cqe = 32766;
488
489 if (nes_read_eeprom_values(nesdev, nesadapter)) {
490 printk(KERN_ERR PFX "Unable to read EEPROM data.\n");
491 kfree(nesadapter);
492 return NULL;
493 }
494
495 u32temp = nes_read_indexed(nesdev, NES_IDX_TCP_TIMER_CONFIG);
496 nes_write_indexed(nesdev, NES_IDX_TCP_TIMER_CONFIG,
497 (u32temp & 0xff000000) | (nesadapter->tcp_timer_core_clk_divisor & 0x00ffffff));
498
499 /* setup port configuration */
500 if (nesadapter->port_count == 1) {
501 nesadapter->log_port = 0x00000000;
502 if (nes_drv_opt & NES_DRV_OPT_DUAL_LOGICAL_PORT)
503 nes_write_indexed(nesdev, NES_IDX_TX_POOL_SIZE, 0x00000002);
504 else
505 nes_write_indexed(nesdev, NES_IDX_TX_POOL_SIZE, 0x00000003);
506 } else {
507 if (nesadapter->phy_type[0] == NES_PHY_TYPE_PUMA_1G) {
508 nesadapter->log_port = 0x000000D8;
509 } else {
510 if (nesadapter->port_count == 2)
511 nesadapter->log_port = 0x00000044;
512 else
513 nesadapter->log_port = 0x000000e4;
514 }
515 nes_write_indexed(nesdev, NES_IDX_TX_POOL_SIZE, 0x00000003);
516 }
517
518 nes_write_indexed(nesdev, NES_IDX_NIC_LOGPORT_TO_PHYPORT,
519 nesadapter->log_port);
520 nes_debug(NES_DBG_INIT, "Probe time, LOG2PHY=%u\n",
521 nes_read_indexed(nesdev, NES_IDX_NIC_LOGPORT_TO_PHYPORT));
522
523 spin_lock_init(&nesadapter->resource_lock);
524 spin_lock_init(&nesadapter->phy_lock);
525 spin_lock_init(&nesadapter->pbl_lock);
526 spin_lock_init(&nesadapter->periodic_timer_lock);
527
528 INIT_LIST_HEAD(&nesadapter->nesvnic_list[0]);
529 INIT_LIST_HEAD(&nesadapter->nesvnic_list[1]);
530 INIT_LIST_HEAD(&nesadapter->nesvnic_list[2]);
531 INIT_LIST_HEAD(&nesadapter->nesvnic_list[3]);
532
533 if ((!nesadapter->OneG_Mode) && (nesadapter->port_count == 2)) {
534 u32 pcs_control_status0, pcs_control_status1;
535 u32 reset_value;
536 u32 i = 0;
537 u32 int_cnt = 0;
538 u32 ext_cnt = 0;
539 unsigned long flags;
540 u32 j = 0;
541
542 pcs_control_status0 = nes_read_indexed(nesdev,
543 NES_IDX_PHY_PCS_CONTROL_STATUS0);
544 pcs_control_status1 = nes_read_indexed(nesdev,
545 NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
546
547 for (i = 0; i < NES_MAX_LINK_CHECK; i++) {
548 pcs_control_status0 = nes_read_indexed(nesdev,
549 NES_IDX_PHY_PCS_CONTROL_STATUS0);
550 pcs_control_status1 = nes_read_indexed(nesdev,
551 NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
552 if ((0x0F000100 == (pcs_control_status0 & 0x0F000100))
553 || (0x0F000100 == (pcs_control_status1 & 0x0F000100)))
554 int_cnt++;
555 msleep(1);
556 }
557 if (int_cnt > 1) {
558 spin_lock_irqsave(&nesadapter->phy_lock, flags);
559 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F0C8);
560 mh_detected++;
561 reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
562 reset_value |= 0x0000003d;
563 nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
564
565 while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET)
566 & 0x00000040) != 0x00000040) && (j++ < 5000));
567 spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
568
569 pcs_control_status0 = nes_read_indexed(nesdev,
570 NES_IDX_PHY_PCS_CONTROL_STATUS0);
571 pcs_control_status1 = nes_read_indexed(nesdev,
572 NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
573
574 for (i = 0; i < NES_MAX_LINK_CHECK; i++) {
575 pcs_control_status0 = nes_read_indexed(nesdev,
576 NES_IDX_PHY_PCS_CONTROL_STATUS0);
577 pcs_control_status1 = nes_read_indexed(nesdev,
578 NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
579 if ((0x0F000100 == (pcs_control_status0 & 0x0F000100))
580 || (0x0F000100 == (pcs_control_status1 & 0x0F000100))) {
581 if (++ext_cnt > int_cnt) {
582 spin_lock_irqsave(&nesadapter->phy_lock, flags);
583 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1,
584 0x0000F088);
585 mh_detected++;
586 reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
587 reset_value |= 0x0000003d;
588 nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
589
590 while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET)
591 & 0x00000040) != 0x00000040) && (j++ < 5000));
592 spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
593 break;
594 }
595 }
596 msleep(1);
597 }
598 }
599 }
600
601 if (nesadapter->hw_rev == NE020_REV) {
602 init_timer(&nesadapter->mh_timer);
603 nesadapter->mh_timer.function = nes_mh_fix;
604 nesadapter->mh_timer.expires = jiffies + (HZ/5); /* 1 second */
605 nesadapter->mh_timer.data = (unsigned long)nesdev;
606 add_timer(&nesadapter->mh_timer);
607 } else {
608 nes_write32(nesdev->regs+NES_INTF_INT_STAT, 0x0f000000);
609 }
610
611 init_timer(&nesadapter->lc_timer);
612 nesadapter->lc_timer.function = nes_clc;
613 nesadapter->lc_timer.expires = jiffies + 3600 * HZ; /* 1 hour */
614 nesadapter->lc_timer.data = (unsigned long)nesdev;
615 add_timer(&nesadapter->lc_timer);
616
617 list_add_tail(&nesadapter->list, &nes_adapter_list);
618
619 for (func_index = 0; func_index < 8; func_index++) {
620 pci_bus_read_config_word(nesdev->pcidev->bus,
621 PCI_DEVFN(PCI_SLOT(nesdev->pcidev->devfn),
622 func_index), 0, &vendor_id);
623 if (vendor_id == 0xffff)
624 break;
625 }
626 nes_debug(NES_DBG_INIT, "%s %d functions found for %s.\n", __func__,
627 func_index, pci_name(nesdev->pcidev));
628 nesadapter->adapter_fcn_count = func_index;
629
630 return nesadapter;
631 }
632
633
634 /**
635 * nes_reset_adapter_ne020
636 */
637 static unsigned int nes_reset_adapter_ne020(struct nes_device *nesdev, u8 *OneG_Mode)
638 {
639 u32 port_count;
640 u32 u32temp;
641 u32 i;
642
643 u32temp = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
644 port_count = ((u32temp & 0x00000300) >> 8) + 1;
645 /* TODO: assuming that both SERDES are set the same for now */
646 *OneG_Mode = (u32temp & 0x00003c00) ? 0 : 1;
647 nes_debug(NES_DBG_INIT, "Initial Software Reset = 0x%08X, port_count=%u\n",
648 u32temp, port_count);
649 if (*OneG_Mode)
650 nes_debug(NES_DBG_INIT, "Running in 1G mode.\n");
651 u32temp &= 0xff00ffc0;
652 switch (port_count) {
653 case 1:
654 u32temp |= 0x00ee0000;
655 break;
656 case 2:
657 u32temp |= 0x00cc0000;
658 break;
659 case 4:
660 u32temp |= 0x00000000;
661 break;
662 default:
663 return 0;
664 break;
665 }
666
667 /* check and do full reset if needed */
668 if (nes_read_indexed(nesdev, NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8))) {
669 nes_debug(NES_DBG_INIT, "Issuing Full Soft reset = 0x%08X\n", u32temp | 0xd);
670 nes_write32(nesdev->regs+NES_SOFTWARE_RESET, u32temp | 0xd);
671
672 i = 0;
673 while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET) & 0x00000040) == 0) && i++ < 10000)
674 mdelay(1);
675 if (i > 10000) {
676 nes_debug(NES_DBG_INIT, "Did not see full soft reset done.\n");
677 return 0;
678 }
679
680 i = 0;
681 while ((nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS) != 0x80) && i++ < 10000)
682 mdelay(1);
683 if (i > 10000) {
684 printk(KERN_ERR PFX "Internal CPU not ready, status = %02X\n",
685 nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS));
686 return 0;
687 }
688 }
689
690 /* port reset */
691 switch (port_count) {
692 case 1:
693 u32temp |= 0x00ee0010;
694 break;
695 case 2:
696 u32temp |= 0x00cc0030;
697 break;
698 case 4:
699 u32temp |= 0x00000030;
700 break;
701 }
702
703 nes_debug(NES_DBG_INIT, "Issuing Port Soft reset = 0x%08X\n", u32temp | 0xd);
704 nes_write32(nesdev->regs+NES_SOFTWARE_RESET, u32temp | 0xd);
705
706 i = 0;
707 while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET) & 0x00000040) == 0) && i++ < 10000)
708 mdelay(1);
709 if (i > 10000) {
710 nes_debug(NES_DBG_INIT, "Did not see port soft reset done.\n");
711 return 0;
712 }
713
714 /* serdes 0 */
715 i = 0;
716 while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0)
717 & 0x0000000f)) != 0x0000000f) && i++ < 5000)
718 mdelay(1);
719 if (i > 5000) {
720 nes_debug(NES_DBG_INIT, "Serdes 0 not ready, status=%x\n", u32temp);
721 return 0;
722 }
723
724 /* serdes 1 */
725 if (port_count > 1) {
726 i = 0;
727 while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS1)
728 & 0x0000000f)) != 0x0000000f) && i++ < 5000)
729 mdelay(1);
730 if (i > 5000) {
731 nes_debug(NES_DBG_INIT, "Serdes 1 not ready, status=%x\n", u32temp);
732 return 0;
733 }
734 }
735
736 return port_count;
737 }
738
739
740 /**
741 * nes_init_serdes
742 */
743 static int nes_init_serdes(struct nes_device *nesdev, u8 hw_rev, u8 port_count,
744 struct nes_adapter *nesadapter, u8 OneG_Mode)
745 {
746 int i;
747 u32 u32temp;
748 u32 sds;
749
750 if (hw_rev != NE020_REV) {
751 /* init serdes 0 */
752 switch (nesadapter->phy_type[0]) {
753 case NES_PHY_TYPE_CX4:
754 if (wide_ppm_offset)
755 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000FFFAA);
756 else
757 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000FF);
758 break;
759 case NES_PHY_TYPE_KR:
760 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000FF);
761 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP0, 0x00000000);
762 break;
763 case NES_PHY_TYPE_PUMA_1G:
764 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000FF);
765 sds = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0);
766 sds |= 0x00000100;
767 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0, sds);
768 break;
769 default:
770 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000FF);
771 break;
772 }
773
774 if (!OneG_Mode)
775 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_HIGHZ_LANE_MODE0, 0x11110000);
776
777 if (port_count < 2)
778 return 0;
779
780 /* init serdes 1 */
781 if (!(OneG_Mode && (nesadapter->phy_type[1] != NES_PHY_TYPE_PUMA_1G)))
782 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL1, 0x000000FF);
783
784 switch (nesadapter->phy_type[1]) {
785 case NES_PHY_TYPE_ARGUS:
786 case NES_PHY_TYPE_SFP_D:
787 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP0, 0x00000000);
788 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP1, 0x00000000);
789 break;
790 case NES_PHY_TYPE_CX4:
791 if (wide_ppm_offset)
792 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL1, 0x000FFFAA);
793 break;
794 case NES_PHY_TYPE_KR:
795 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP1, 0x00000000);
796 break;
797 case NES_PHY_TYPE_PUMA_1G:
798 sds = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
799 sds |= 0x000000100;
800 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, sds);
801 }
802 if (!OneG_Mode) {
803 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_HIGHZ_LANE_MODE1, 0x11110000);
804 sds = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
805 sds &= 0xFFFFFFBF;
806 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, sds);
807 }
808 } else {
809 /* init serdes 0 */
810 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0, 0x00000008);
811 i = 0;
812 while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0)
813 & 0x0000000f)) != 0x0000000f) && i++ < 5000)
814 mdelay(1);
815 if (i > 5000) {
816 nes_debug(NES_DBG_PHY, "Init: serdes 0 not ready, status=%x\n", u32temp);
817 return 1;
818 }
819 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP0, 0x000bdef7);
820 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_DRIVE0, 0x9ce73000);
821 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_MODE0, 0x0ff00000);
822 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_SIGDET0, 0x00000000);
823 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_BYPASS0, 0x00000000);
824 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_LOOPBACK_CONTROL0, 0x00000000);
825 if (OneG_Mode)
826 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL0, 0xf0182222);
827 else
828 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL0, 0xf0042222);
829
830 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000ff);
831 if (port_count > 1) {
832 /* init serdes 1 */
833 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x00000048);
834 i = 0;
835 while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS1)
836 & 0x0000000f)) != 0x0000000f) && (i++ < 5000))
837 mdelay(1);
838 if (i > 5000) {
839 printk("%s: Init: serdes 1 not ready, status=%x\n", __func__, u32temp);
840 /* return 1; */
841 }
842 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP1, 0x000bdef7);
843 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_DRIVE1, 0x9ce73000);
844 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_MODE1, 0x0ff00000);
845 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_SIGDET1, 0x00000000);
846 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_BYPASS1, 0x00000000);
847 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_LOOPBACK_CONTROL1, 0x00000000);
848 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL1, 0xf0002222);
849 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL1, 0x000000ff);
850 }
851 }
852 return 0;
853 }
854
855
856 /**
857 * nes_init_csr_ne020
858 * Initialize registers for ne020 hardware
859 */
860 static void nes_init_csr_ne020(struct nes_device *nesdev, u8 hw_rev, u8 port_count)
861 {
862 u32 u32temp;
863
864 nes_debug(NES_DBG_INIT, "port_count=%d\n", port_count);
865
866 nes_write_indexed(nesdev, 0x000001E4, 0x00000007);
867 /* nes_write_indexed(nesdev, 0x000001E8, 0x000208C4); */
868 nes_write_indexed(nesdev, 0x000001E8, 0x00020874);
869 nes_write_indexed(nesdev, 0x000001D8, 0x00048002);
870 /* nes_write_indexed(nesdev, 0x000001D8, 0x0004B002); */
871 nes_write_indexed(nesdev, 0x000001FC, 0x00050005);
872 nes_write_indexed(nesdev, 0x00000600, 0x55555555);
873 nes_write_indexed(nesdev, 0x00000604, 0x55555555);
874
875 /* TODO: move these MAC register settings to NIC bringup */
876 nes_write_indexed(nesdev, 0x00002000, 0x00000001);
877 nes_write_indexed(nesdev, 0x00002004, 0x00000001);
878 nes_write_indexed(nesdev, 0x00002008, 0x0000FFFF);
879 nes_write_indexed(nesdev, 0x0000200C, 0x00000001);
880 nes_write_indexed(nesdev, 0x00002010, 0x000003c1);
881 nes_write_indexed(nesdev, 0x0000201C, 0x75345678);
882 if (port_count > 1) {
883 nes_write_indexed(nesdev, 0x00002200, 0x00000001);
884 nes_write_indexed(nesdev, 0x00002204, 0x00000001);
885 nes_write_indexed(nesdev, 0x00002208, 0x0000FFFF);
886 nes_write_indexed(nesdev, 0x0000220C, 0x00000001);
887 nes_write_indexed(nesdev, 0x00002210, 0x000003c1);
888 nes_write_indexed(nesdev, 0x0000221C, 0x75345678);
889 nes_write_indexed(nesdev, 0x00000908, 0x20000001);
890 }
891 if (port_count > 2) {
892 nes_write_indexed(nesdev, 0x00002400, 0x00000001);
893 nes_write_indexed(nesdev, 0x00002404, 0x00000001);
894 nes_write_indexed(nesdev, 0x00002408, 0x0000FFFF);
895 nes_write_indexed(nesdev, 0x0000240C, 0x00000001);
896 nes_write_indexed(nesdev, 0x00002410, 0x000003c1);
897 nes_write_indexed(nesdev, 0x0000241C, 0x75345678);
898 nes_write_indexed(nesdev, 0x00000910, 0x20000001);
899
900 nes_write_indexed(nesdev, 0x00002600, 0x00000001);
901 nes_write_indexed(nesdev, 0x00002604, 0x00000001);
902 nes_write_indexed(nesdev, 0x00002608, 0x0000FFFF);
903 nes_write_indexed(nesdev, 0x0000260C, 0x00000001);
904 nes_write_indexed(nesdev, 0x00002610, 0x000003c1);
905 nes_write_indexed(nesdev, 0x0000261C, 0x75345678);
906 nes_write_indexed(nesdev, 0x00000918, 0x20000001);
907 }
908
909 nes_write_indexed(nesdev, 0x00005000, 0x00018000);
910 /* nes_write_indexed(nesdev, 0x00005000, 0x00010000); */
911 nes_write_indexed(nesdev, NES_IDX_WQM_CONFIG1, (wqm_quanta << 1) |
912 0x00000001);
913 nes_write_indexed(nesdev, 0x00005008, 0x1F1F1F1F);
914 nes_write_indexed(nesdev, 0x00005010, 0x1F1F1F1F);
915 nes_write_indexed(nesdev, 0x00005018, 0x1F1F1F1F);
916 nes_write_indexed(nesdev, 0x00005020, 0x1F1F1F1F);
917 nes_write_indexed(nesdev, 0x00006090, 0xFFFFFFFF);
918
919 /* TODO: move this to code, get from EEPROM */
920 nes_write_indexed(nesdev, 0x00000900, 0x20000001);
921 nes_write_indexed(nesdev, 0x000060C0, 0x0000028e);
922 nes_write_indexed(nesdev, 0x000060C8, 0x00000020);
923
924 nes_write_indexed(nesdev, 0x000001EC, 0x7b2625a0);
925 /* nes_write_indexed(nesdev, 0x000001EC, 0x5f2625a0); */
926
927 if (hw_rev != NE020_REV) {
928 u32temp = nes_read_indexed(nesdev, 0x000008e8);
929 u32temp |= 0x80000000;
930 nes_write_indexed(nesdev, 0x000008e8, u32temp);
931 u32temp = nes_read_indexed(nesdev, 0x000021f8);
932 u32temp &= 0x7fffffff;
933 u32temp |= 0x7fff0010;
934 nes_write_indexed(nesdev, 0x000021f8, u32temp);
935 if (port_count > 1) {
936 u32temp = nes_read_indexed(nesdev, 0x000023f8);
937 u32temp &= 0x7fffffff;
938 u32temp |= 0x7fff0010;
939 nes_write_indexed(nesdev, 0x000023f8, u32temp);
940 }
941 }
942 }
943
944
945 /**
946 * nes_destroy_adapter - destroy the adapter structure
947 */
948 void nes_destroy_adapter(struct nes_adapter *nesadapter)
949 {
950 struct nes_adapter *tmp_adapter;
951
952 list_for_each_entry(tmp_adapter, &nes_adapter_list, list) {
953 nes_debug(NES_DBG_SHUTDOWN, "Nes Adapter list entry = 0x%p.\n",
954 tmp_adapter);
955 }
956
957 nesadapter->ref_count--;
958 if (!nesadapter->ref_count) {
959 if (nesadapter->hw_rev == NE020_REV) {
960 del_timer(&nesadapter->mh_timer);
961 }
962 del_timer(&nesadapter->lc_timer);
963
964 list_del(&nesadapter->list);
965 kfree(nesadapter);
966 }
967 }
968
969
970 /**
971 * nes_init_cqp
972 */
973 int nes_init_cqp(struct nes_device *nesdev)
974 {
975 struct nes_adapter *nesadapter = nesdev->nesadapter;
976 struct nes_hw_cqp_qp_context *cqp_qp_context;
977 struct nes_hw_cqp_wqe *cqp_wqe;
978 struct nes_hw_ceq *ceq;
979 struct nes_hw_ceq *nic_ceq;
980 struct nes_hw_aeq *aeq;
981 void *vmem;
982 dma_addr_t pmem;
983 u32 count=0;
984 u32 cqp_head;
985 u64 u64temp;
986 u32 u32temp;
987
988 /* allocate CQP memory */
989 /* Need to add max_cq to the aeq size once cq overflow checking is added back */
990 /* SQ is 512 byte aligned, others are 256 byte aligned */
991 nesdev->cqp_mem_size = 512 +
992 (sizeof(struct nes_hw_cqp_wqe) * NES_CQP_SQ_SIZE) +
993 (sizeof(struct nes_hw_cqe) * NES_CCQ_SIZE) +
994 max(((u32)sizeof(struct nes_hw_ceqe) * NES_CCEQ_SIZE), (u32)256) +
995 max(((u32)sizeof(struct nes_hw_ceqe) * NES_NIC_CEQ_SIZE), (u32)256) +
996 (sizeof(struct nes_hw_aeqe) * nesadapter->max_qp) +
997 sizeof(struct nes_hw_cqp_qp_context);
998
999 nesdev->cqp_vbase = pci_zalloc_consistent(nesdev->pcidev,
1000 nesdev->cqp_mem_size,
1001 &nesdev->cqp_pbase);
1002 if (!nesdev->cqp_vbase) {
1003 nes_debug(NES_DBG_INIT, "Unable to allocate memory for host descriptor rings\n");
1004 return -ENOMEM;
1005 }
1006
1007 /* Allocate a twice the number of CQP requests as the SQ size */
1008 nesdev->nes_cqp_requests = kzalloc(sizeof(struct nes_cqp_request) *
1009 2 * NES_CQP_SQ_SIZE, GFP_KERNEL);
1010 if (nesdev->nes_cqp_requests == NULL) {
1011 nes_debug(NES_DBG_INIT, "Unable to allocate memory CQP request entries.\n");
1012 pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size, nesdev->cqp.sq_vbase,
1013 nesdev->cqp.sq_pbase);
1014 return -ENOMEM;
1015 }
1016
1017 nes_debug(NES_DBG_INIT, "Allocated CQP structures at %p (phys = %016lX), size = %u.\n",
1018 nesdev->cqp_vbase, (unsigned long)nesdev->cqp_pbase, nesdev->cqp_mem_size);
1019
1020 spin_lock_init(&nesdev->cqp.lock);
1021 init_waitqueue_head(&nesdev->cqp.waitq);
1022
1023 /* Setup Various Structures */
1024 vmem = (void *)(((unsigned long)nesdev->cqp_vbase + (512 - 1)) &
1025 ~(unsigned long)(512 - 1));
1026 pmem = (dma_addr_t)(((unsigned long long)nesdev->cqp_pbase + (512 - 1)) &
1027 ~(unsigned long long)(512 - 1));
1028
1029 nesdev->cqp.sq_vbase = vmem;
1030 nesdev->cqp.sq_pbase = pmem;
1031 nesdev->cqp.sq_size = NES_CQP_SQ_SIZE;
1032 nesdev->cqp.sq_head = 0;
1033 nesdev->cqp.sq_tail = 0;
1034 nesdev->cqp.qp_id = PCI_FUNC(nesdev->pcidev->devfn);
1035
1036 vmem += (sizeof(struct nes_hw_cqp_wqe) * nesdev->cqp.sq_size);
1037 pmem += (sizeof(struct nes_hw_cqp_wqe) * nesdev->cqp.sq_size);
1038
1039 nesdev->ccq.cq_vbase = vmem;
1040 nesdev->ccq.cq_pbase = pmem;
1041 nesdev->ccq.cq_size = NES_CCQ_SIZE;
1042 nesdev->ccq.cq_head = 0;
1043 nesdev->ccq.ce_handler = nes_cqp_ce_handler;
1044 nesdev->ccq.cq_number = PCI_FUNC(nesdev->pcidev->devfn);
1045
1046 vmem += (sizeof(struct nes_hw_cqe) * nesdev->ccq.cq_size);
1047 pmem += (sizeof(struct nes_hw_cqe) * nesdev->ccq.cq_size);
1048
1049 nesdev->ceq_index = PCI_FUNC(nesdev->pcidev->devfn);
1050 ceq = &nesadapter->ceq[nesdev->ceq_index];
1051 ceq->ceq_vbase = vmem;
1052 ceq->ceq_pbase = pmem;
1053 ceq->ceq_size = NES_CCEQ_SIZE;
1054 ceq->ceq_head = 0;
1055
1056 vmem += max(((u32)sizeof(struct nes_hw_ceqe) * ceq->ceq_size), (u32)256);
1057 pmem += max(((u32)sizeof(struct nes_hw_ceqe) * ceq->ceq_size), (u32)256);
1058
1059 nesdev->nic_ceq_index = PCI_FUNC(nesdev->pcidev->devfn) + 8;
1060 nic_ceq = &nesadapter->ceq[nesdev->nic_ceq_index];
1061 nic_ceq->ceq_vbase = vmem;
1062 nic_ceq->ceq_pbase = pmem;
1063 nic_ceq->ceq_size = NES_NIC_CEQ_SIZE;
1064 nic_ceq->ceq_head = 0;
1065
1066 vmem += max(((u32)sizeof(struct nes_hw_ceqe) * nic_ceq->ceq_size), (u32)256);
1067 pmem += max(((u32)sizeof(struct nes_hw_ceqe) * nic_ceq->ceq_size), (u32)256);
1068
1069 aeq = &nesadapter->aeq[PCI_FUNC(nesdev->pcidev->devfn)];
1070 aeq->aeq_vbase = vmem;
1071 aeq->aeq_pbase = pmem;
1072 aeq->aeq_size = nesadapter->max_qp;
1073 aeq->aeq_head = 0;
1074
1075 /* Setup QP Context */
1076 vmem += (sizeof(struct nes_hw_aeqe) * aeq->aeq_size);
1077 pmem += (sizeof(struct nes_hw_aeqe) * aeq->aeq_size);
1078
1079 cqp_qp_context = vmem;
1080 cqp_qp_context->context_words[0] =
1081 cpu_to_le32((PCI_FUNC(nesdev->pcidev->devfn) << 12) + (2 << 10));
1082 cqp_qp_context->context_words[1] = 0;
1083 cqp_qp_context->context_words[2] = cpu_to_le32((u32)nesdev->cqp.sq_pbase);
1084 cqp_qp_context->context_words[3] = cpu_to_le32(((u64)nesdev->cqp.sq_pbase) >> 32);
1085
1086
1087 /* Write the address to Create CQP */
1088 if ((sizeof(dma_addr_t) > 4)) {
1089 nes_write_indexed(nesdev,
1090 NES_IDX_CREATE_CQP_HIGH + (PCI_FUNC(nesdev->pcidev->devfn) * 8),
1091 ((u64)pmem) >> 32);
1092 } else {
1093 nes_write_indexed(nesdev,
1094 NES_IDX_CREATE_CQP_HIGH + (PCI_FUNC(nesdev->pcidev->devfn) * 8), 0);
1095 }
1096 nes_write_indexed(nesdev,
1097 NES_IDX_CREATE_CQP_LOW + (PCI_FUNC(nesdev->pcidev->devfn) * 8),
1098 (u32)pmem);
1099
1100 INIT_LIST_HEAD(&nesdev->cqp_avail_reqs);
1101 INIT_LIST_HEAD(&nesdev->cqp_pending_reqs);
1102
1103 for (count = 0; count < 2*NES_CQP_SQ_SIZE; count++) {
1104 init_waitqueue_head(&nesdev->nes_cqp_requests[count].waitq);
1105 list_add_tail(&nesdev->nes_cqp_requests[count].list, &nesdev->cqp_avail_reqs);
1106 }
1107
1108 /* Write Create CCQ WQE */
1109 cqp_head = nesdev->cqp.sq_head++;
1110 cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1111 nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1112 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1113 (NES_CQP_CREATE_CQ | NES_CQP_CQ_CEQ_VALID |
1114 NES_CQP_CQ_CHK_OVERFLOW | ((u32)nesdev->ccq.cq_size << 16)));
1115 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
1116 (nesdev->ccq.cq_number |
1117 ((u32)nesdev->ceq_index << 16)));
1118 u64temp = (u64)nesdev->ccq.cq_pbase;
1119 set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1120 cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] = 0;
1121 u64temp = (unsigned long)&nesdev->ccq;
1122 cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_LOW_IDX] =
1123 cpu_to_le32((u32)(u64temp >> 1));
1124 cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] =
1125 cpu_to_le32(((u32)((u64temp) >> 33)) & 0x7FFFFFFF);
1126 cqp_wqe->wqe_words[NES_CQP_CQ_WQE_DOORBELL_INDEX_HIGH_IDX] = 0;
1127
1128 /* Write Create CEQ WQE */
1129 cqp_head = nesdev->cqp.sq_head++;
1130 cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1131 nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1132 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1133 (NES_CQP_CREATE_CEQ + ((u32)nesdev->ceq_index << 8)));
1134 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_CEQ_WQE_ELEMENT_COUNT_IDX, ceq->ceq_size);
1135 u64temp = (u64)ceq->ceq_pbase;
1136 set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1137
1138 /* Write Create AEQ WQE */
1139 cqp_head = nesdev->cqp.sq_head++;
1140 cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1141 nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1142 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1143 (NES_CQP_CREATE_AEQ + ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 8)));
1144 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_AEQ_WQE_ELEMENT_COUNT_IDX, aeq->aeq_size);
1145 u64temp = (u64)aeq->aeq_pbase;
1146 set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1147
1148 /* Write Create NIC CEQ WQE */
1149 cqp_head = nesdev->cqp.sq_head++;
1150 cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1151 nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1152 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1153 (NES_CQP_CREATE_CEQ + ((u32)nesdev->nic_ceq_index << 8)));
1154 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_CEQ_WQE_ELEMENT_COUNT_IDX, nic_ceq->ceq_size);
1155 u64temp = (u64)nic_ceq->ceq_pbase;
1156 set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1157
1158 /* Poll until CCQP done */
1159 count = 0;
1160 do {
1161 if (count++ > 1000) {
1162 printk(KERN_ERR PFX "Error creating CQP\n");
1163 pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size,
1164 nesdev->cqp_vbase, nesdev->cqp_pbase);
1165 return -1;
1166 }
1167 udelay(10);
1168 } while (!(nes_read_indexed(nesdev,
1169 NES_IDX_QP_CONTROL + (PCI_FUNC(nesdev->pcidev->devfn) * 8)) & (1 << 8)));
1170
1171 nes_debug(NES_DBG_INIT, "CQP Status = 0x%08X\n", nes_read_indexed(nesdev,
1172 NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)));
1173
1174 u32temp = 0x04800000;
1175 nes_write32(nesdev->regs+NES_WQE_ALLOC, u32temp | nesdev->cqp.qp_id);
1176
1177 /* wait for the CCQ, CEQ, and AEQ to get created */
1178 count = 0;
1179 do {
1180 if (count++ > 1000) {
1181 printk(KERN_ERR PFX "Error creating CCQ, CEQ, and AEQ\n");
1182 pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size,
1183 nesdev->cqp_vbase, nesdev->cqp_pbase);
1184 return -1;
1185 }
1186 udelay(10);
1187 } while (((nes_read_indexed(nesdev,
1188 NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)) & (15<<8)) != (15<<8)));
1189
1190 /* dump the QP status value */
1191 nes_debug(NES_DBG_INIT, "QP Status = 0x%08X\n", nes_read_indexed(nesdev,
1192 NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)));
1193
1194 nesdev->cqp.sq_tail++;
1195
1196 return 0;
1197 }
1198
1199
1200 /**
1201 * nes_destroy_cqp
1202 */
1203 int nes_destroy_cqp(struct nes_device *nesdev)
1204 {
1205 struct nes_hw_cqp_wqe *cqp_wqe;
1206 u32 count = 0;
1207 u32 cqp_head;
1208 unsigned long flags;
1209
1210 do {
1211 if (count++ > 1000)
1212 break;
1213 udelay(10);
1214 } while (!(nesdev->cqp.sq_head == nesdev->cqp.sq_tail));
1215
1216 /* Reset CCQ */
1217 nes_write32(nesdev->regs+NES_CQE_ALLOC, NES_CQE_ALLOC_RESET |
1218 nesdev->ccq.cq_number);
1219
1220 /* Disable device interrupts */
1221 nes_write32(nesdev->regs+NES_INT_MASK, 0x7fffffff);
1222
1223 spin_lock_irqsave(&nesdev->cqp.lock, flags);
1224
1225 /* Destroy the AEQ */
1226 cqp_head = nesdev->cqp.sq_head++;
1227 nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1228 cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1229 cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_AEQ |
1230 ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 8));
1231 cqp_wqe->wqe_words[NES_CQP_WQE_COMP_CTX_HIGH_IDX] = 0;
1232
1233 /* Destroy the NIC CEQ */
1234 cqp_head = nesdev->cqp.sq_head++;
1235 nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1236 cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1237 cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_CEQ |
1238 ((u32)nesdev->nic_ceq_index << 8));
1239
1240 /* Destroy the CEQ */
1241 cqp_head = nesdev->cqp.sq_head++;
1242 nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1243 cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1244 cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_CEQ |
1245 (nesdev->ceq_index << 8));
1246
1247 /* Destroy the CCQ */
1248 cqp_head = nesdev->cqp.sq_head++;
1249 nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1250 cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1251 cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_CQ);
1252 cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesdev->ccq.cq_number |
1253 ((u32)nesdev->ceq_index << 16));
1254
1255 /* Destroy CQP */
1256 cqp_head = nesdev->cqp.sq_head++;
1257 nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1258 cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1259 cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_QP |
1260 NES_CQP_QP_TYPE_CQP);
1261 cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesdev->cqp.qp_id);
1262
1263 barrier();
1264 /* Ring doorbell (5 WQEs) */
1265 nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x05800000 | nesdev->cqp.qp_id);
1266
1267 spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
1268
1269 /* wait for the CCQ, CEQ, and AEQ to get destroyed */
1270 count = 0;
1271 do {
1272 if (count++ > 1000) {
1273 printk(KERN_ERR PFX "Function%d: Error destroying CCQ, CEQ, and AEQ\n",
1274 PCI_FUNC(nesdev->pcidev->devfn));
1275 break;
1276 }
1277 udelay(10);
1278 } while (((nes_read_indexed(nesdev,
1279 NES_IDX_QP_CONTROL + (PCI_FUNC(nesdev->pcidev->devfn)*8)) & (15 << 8)) != 0));
1280
1281 /* dump the QP status value */
1282 nes_debug(NES_DBG_SHUTDOWN, "Function%d: QP Status = 0x%08X\n",
1283 PCI_FUNC(nesdev->pcidev->devfn),
1284 nes_read_indexed(nesdev,
1285 NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)));
1286
1287 kfree(nesdev->nes_cqp_requests);
1288
1289 /* Free the control structures */
1290 pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size, nesdev->cqp.sq_vbase,
1291 nesdev->cqp.sq_pbase);
1292
1293 return 0;
1294 }
1295
1296
1297 /**
1298 * nes_init_1g_phy
1299 */
1300 static int nes_init_1g_phy(struct nes_device *nesdev, u8 phy_type, u8 phy_index)
1301 {
1302 u32 counter = 0;
1303 u16 phy_data;
1304 int ret = 0;
1305
1306 nes_read_1G_phy_reg(nesdev, 1, phy_index, &phy_data);
1307 nes_write_1G_phy_reg(nesdev, 23, phy_index, 0xb000);
1308
1309 /* Reset the PHY */
1310 nes_write_1G_phy_reg(nesdev, 0, phy_index, 0x8000);
1311 udelay(100);
1312 counter = 0;
1313 do {
1314 nes_read_1G_phy_reg(nesdev, 0, phy_index, &phy_data);
1315 if (counter++ > 100) {
1316 ret = -1;
1317 break;
1318 }
1319 } while (phy_data & 0x8000);
1320
1321 /* Setting no phy loopback */
1322 phy_data &= 0xbfff;
1323 phy_data |= 0x1140;
1324 nes_write_1G_phy_reg(nesdev, 0, phy_index, phy_data);
1325 nes_read_1G_phy_reg(nesdev, 0, phy_index, &phy_data);
1326 nes_read_1G_phy_reg(nesdev, 0x17, phy_index, &phy_data);
1327 nes_read_1G_phy_reg(nesdev, 0x1e, phy_index, &phy_data);
1328
1329 /* Setting the interrupt mask */
1330 nes_read_1G_phy_reg(nesdev, 0x19, phy_index, &phy_data);
1331 nes_write_1G_phy_reg(nesdev, 0x19, phy_index, 0xffee);
1332 nes_read_1G_phy_reg(nesdev, 0x19, phy_index, &phy_data);
1333
1334 /* turning on flow control */
1335 nes_read_1G_phy_reg(nesdev, 4, phy_index, &phy_data);
1336 nes_write_1G_phy_reg(nesdev, 4, phy_index, (phy_data & ~(0x03E0)) | 0xc00);
1337 nes_read_1G_phy_reg(nesdev, 4, phy_index, &phy_data);
1338
1339 /* Clear Half duplex */
1340 nes_read_1G_phy_reg(nesdev, 9, phy_index, &phy_data);
1341 nes_write_1G_phy_reg(nesdev, 9, phy_index, phy_data & ~(0x0100));
1342 nes_read_1G_phy_reg(nesdev, 9, phy_index, &phy_data);
1343
1344 nes_read_1G_phy_reg(nesdev, 0, phy_index, &phy_data);
1345 nes_write_1G_phy_reg(nesdev, 0, phy_index, phy_data | 0x0300);
1346
1347 return ret;
1348 }
1349
1350
1351 /**
1352 * nes_init_2025_phy
1353 */
1354 static int nes_init_2025_phy(struct nes_device *nesdev, u8 phy_type, u8 phy_index)
1355 {
1356 u32 temp_phy_data = 0;
1357 u32 temp_phy_data2 = 0;
1358 u32 counter = 0;
1359 u32 sds;
1360 u32 mac_index = nesdev->mac_index;
1361 int ret = 0;
1362 unsigned int first_attempt = 1;
1363
1364 /* Check firmware heartbeat */
1365 nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7ee);
1366 temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1367 udelay(1500);
1368 nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7ee);
1369 temp_phy_data2 = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1370
1371 if (temp_phy_data != temp_phy_data2) {
1372 nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7fd);
1373 temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1374 if ((temp_phy_data & 0xff) > 0x20)
1375 return 0;
1376 printk(PFX "Reinitialize external PHY\n");
1377 }
1378
1379 /* no heartbeat, configure the PHY */
1380 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0x0000, 0x8000);
1381 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc300, 0x0000);
1382 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc316, 0x000A);
1383 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc318, 0x0052);
1384
1385 switch (phy_type) {
1386 case NES_PHY_TYPE_ARGUS:
1387 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc316, 0x000A);
1388 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc318, 0x0052);
1389 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc302, 0x000C);
1390 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc319, 0x0008);
1391 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0027, 0x0001);
1392 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc31a, 0x0098);
1393 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0026, 0x0E00);
1394
1395 /* setup LEDs */
1396 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd006, 0x0007);
1397 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd007, 0x000A);
1398 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd008, 0x0009);
1399 break;
1400
1401 case NES_PHY_TYPE_SFP_D:
1402 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc316, 0x000A);
1403 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc318, 0x0052);
1404 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc302, 0x0004);
1405 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc319, 0x0038);
1406 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0027, 0x0013);
1407 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc31a, 0x0098);
1408 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0026, 0x0E00);
1409
1410 /* setup LEDs */
1411 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd006, 0x0007);
1412 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd007, 0x000A);
1413 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd008, 0x0009);
1414 break;
1415
1416 case NES_PHY_TYPE_KR:
1417 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc316, 0x000A);
1418 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc318, 0x0052);
1419 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc302, 0x000C);
1420 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc319, 0x0010);
1421 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0027, 0x0013);
1422 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc31a, 0x0080);
1423 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0026, 0x0E00);
1424
1425 /* setup LEDs */
1426 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd006, 0x000B);
1427 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd007, 0x0003);
1428 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd008, 0x0004);
1429
1430 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0022, 0x406D);
1431 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0023, 0x0020);
1432 break;
1433 }
1434
1435 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0028, 0xA528);
1436
1437 /* Bring PHY out of reset */
1438 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc300, 0x0002);
1439
1440 /* Check for heartbeat */
1441 counter = 0;
1442 mdelay(690);
1443 nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7ee);
1444 temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1445 do {
1446 if (counter++ > 150) {
1447 printk(PFX "No PHY heartbeat\n");
1448 break;
1449 }
1450 mdelay(1);
1451 nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7ee);
1452 temp_phy_data2 = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1453 } while ((temp_phy_data2 == temp_phy_data));
1454
1455 /* wait for tracking */
1456 counter = 0;
1457 do {
1458 nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7fd);
1459 temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1460 if (counter++ > 300) {
1461 if (((temp_phy_data & 0xff) == 0x0) && first_attempt) {
1462 first_attempt = 0;
1463 counter = 0;
1464 /* reset AMCC PHY and try again */
1465 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0xe854, 0x00c0);
1466 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0xe854, 0x0040);
1467 continue;
1468 } else {
1469 ret = 1;
1470 break;
1471 }
1472 }
1473 mdelay(10);
1474 } while ((temp_phy_data & 0xff) < 0x30);
1475
1476 /* setup signal integrity */
1477 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd003, 0x0000);
1478 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xF00D, 0x00FE);
1479 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xF00E, 0x0032);
1480 if (phy_type == NES_PHY_TYPE_KR) {
1481 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xF00F, 0x000C);
1482 } else {
1483 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xF00F, 0x0002);
1484 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc314, 0x0063);
1485 }
1486
1487 /* reset serdes */
1488 sds = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0 + mac_index * 0x200);
1489 sds |= 0x1;
1490 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0 + mac_index * 0x200, sds);
1491 sds &= 0xfffffffe;
1492 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0 + mac_index * 0x200, sds);
1493
1494 counter = 0;
1495 while (((nes_read32(nesdev->regs + NES_SOFTWARE_RESET) & 0x00000040) != 0x00000040)
1496 && (counter++ < 5000))
1497 ;
1498
1499 return ret;
1500 }
1501
1502
1503 /**
1504 * nes_init_phy
1505 */
1506 int nes_init_phy(struct nes_device *nesdev)
1507 {
1508 struct nes_adapter *nesadapter = nesdev->nesadapter;
1509 u32 mac_index = nesdev->mac_index;
1510 u32 tx_config = 0;
1511 unsigned long flags;
1512 u8 phy_type = nesadapter->phy_type[mac_index];
1513 u8 phy_index = nesadapter->phy_index[mac_index];
1514 int ret = 0;
1515
1516 tx_config = nes_read_indexed(nesdev, NES_IDX_MAC_TX_CONFIG);
1517 if (phy_type == NES_PHY_TYPE_1G) {
1518 /* setup 1G MDIO operation */
1519 tx_config &= 0xFFFFFFE3;
1520 tx_config |= 0x04;
1521 } else {
1522 /* setup 10G MDIO operation */
1523 tx_config &= 0xFFFFFFE3;
1524 tx_config |= 0x1D;
1525 }
1526 nes_write_indexed(nesdev, NES_IDX_MAC_TX_CONFIG, tx_config);
1527
1528 spin_lock_irqsave(&nesdev->nesadapter->phy_lock, flags);
1529
1530 switch (phy_type) {
1531 case NES_PHY_TYPE_1G:
1532 ret = nes_init_1g_phy(nesdev, phy_type, phy_index);
1533 break;
1534 case NES_PHY_TYPE_ARGUS:
1535 case NES_PHY_TYPE_SFP_D:
1536 case NES_PHY_TYPE_KR:
1537 ret = nes_init_2025_phy(nesdev, phy_type, phy_index);
1538 break;
1539 }
1540
1541 spin_unlock_irqrestore(&nesdev->nesadapter->phy_lock, flags);
1542
1543 return ret;
1544 }
1545
1546
1547 /**
1548 * nes_replenish_nic_rq
1549 */
1550 static void nes_replenish_nic_rq(struct nes_vnic *nesvnic)
1551 {
1552 unsigned long flags;
1553 dma_addr_t bus_address;
1554 struct sk_buff *skb;
1555 struct nes_hw_nic_rq_wqe *nic_rqe;
1556 struct nes_hw_nic *nesnic;
1557 struct nes_device *nesdev;
1558 struct nes_rskb_cb *cb;
1559 u32 rx_wqes_posted = 0;
1560
1561 nesnic = &nesvnic->nic;
1562 nesdev = nesvnic->nesdev;
1563 spin_lock_irqsave(&nesnic->rq_lock, flags);
1564 if (nesnic->replenishing_rq !=0) {
1565 if (((nesnic->rq_size-1) == atomic_read(&nesvnic->rx_skbs_needed)) &&
1566 (atomic_read(&nesvnic->rx_skb_timer_running) == 0)) {
1567 atomic_set(&nesvnic->rx_skb_timer_running, 1);
1568 spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1569 nesvnic->rq_wqes_timer.expires = jiffies + (HZ/2); /* 1/2 second */
1570 add_timer(&nesvnic->rq_wqes_timer);
1571 } else
1572 spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1573 return;
1574 }
1575 nesnic->replenishing_rq = 1;
1576 spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1577 do {
1578 skb = dev_alloc_skb(nesvnic->max_frame_size);
1579 if (skb) {
1580 skb->dev = nesvnic->netdev;
1581
1582 bus_address = pci_map_single(nesdev->pcidev,
1583 skb->data, nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
1584 cb = (struct nes_rskb_cb *)&skb->cb[0];
1585 cb->busaddr = bus_address;
1586 cb->maplen = nesvnic->max_frame_size;
1587
1588 nic_rqe = &nesnic->rq_vbase[nesvnic->nic.rq_head];
1589 nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_1_0_IDX] =
1590 cpu_to_le32(nesvnic->max_frame_size);
1591 nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_3_2_IDX] = 0;
1592 nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX] =
1593 cpu_to_le32((u32)bus_address);
1594 nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX] =
1595 cpu_to_le32((u32)((u64)bus_address >> 32));
1596 nesnic->rx_skb[nesnic->rq_head] = skb;
1597 nesnic->rq_head++;
1598 nesnic->rq_head &= nesnic->rq_size - 1;
1599 atomic_dec(&nesvnic->rx_skbs_needed);
1600 barrier();
1601 if (++rx_wqes_posted == 255) {
1602 nes_write32(nesdev->regs+NES_WQE_ALLOC, (rx_wqes_posted << 24) | nesnic->qp_id);
1603 rx_wqes_posted = 0;
1604 }
1605 } else {
1606 spin_lock_irqsave(&nesnic->rq_lock, flags);
1607 if (((nesnic->rq_size-1) == atomic_read(&nesvnic->rx_skbs_needed)) &&
1608 (atomic_read(&nesvnic->rx_skb_timer_running) == 0)) {
1609 atomic_set(&nesvnic->rx_skb_timer_running, 1);
1610 spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1611 nesvnic->rq_wqes_timer.expires = jiffies + (HZ/2); /* 1/2 second */
1612 add_timer(&nesvnic->rq_wqes_timer);
1613 } else
1614 spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1615 break;
1616 }
1617 } while (atomic_read(&nesvnic->rx_skbs_needed));
1618 barrier();
1619 if (rx_wqes_posted)
1620 nes_write32(nesdev->regs+NES_WQE_ALLOC, (rx_wqes_posted << 24) | nesnic->qp_id);
1621 nesnic->replenishing_rq = 0;
1622 }
1623
1624
1625 /**
1626 * nes_rq_wqes_timeout
1627 */
1628 static void nes_rq_wqes_timeout(unsigned long parm)
1629 {
1630 struct nes_vnic *nesvnic = (struct nes_vnic *)parm;
1631 printk("%s: Timer fired.\n", __func__);
1632 atomic_set(&nesvnic->rx_skb_timer_running, 0);
1633 if (atomic_read(&nesvnic->rx_skbs_needed))
1634 nes_replenish_nic_rq(nesvnic);
1635 }
1636
1637
1638 /**
1639 * nes_init_nic_qp
1640 */
1641 int nes_init_nic_qp(struct nes_device *nesdev, struct net_device *netdev)
1642 {
1643 struct nes_hw_cqp_wqe *cqp_wqe;
1644 struct nes_hw_nic_sq_wqe *nic_sqe;
1645 struct nes_hw_nic_qp_context *nic_context;
1646 struct sk_buff *skb;
1647 struct nes_hw_nic_rq_wqe *nic_rqe;
1648 struct nes_vnic *nesvnic = netdev_priv(netdev);
1649 unsigned long flags;
1650 void *vmem;
1651 dma_addr_t pmem;
1652 u64 u64temp;
1653 int ret;
1654 u32 cqp_head;
1655 u32 counter;
1656 u32 wqe_count;
1657 struct nes_rskb_cb *cb;
1658 u8 jumbomode=0;
1659
1660 /* Allocate fragment, SQ, RQ, and CQ; Reuse CEQ based on the PCI function */
1661 nesvnic->nic_mem_size = 256 +
1662 (NES_NIC_WQ_SIZE * sizeof(struct nes_first_frag)) +
1663 (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_sq_wqe)) +
1664 (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_rq_wqe)) +
1665 (NES_NIC_WQ_SIZE * 2 * sizeof(struct nes_hw_nic_cqe)) +
1666 sizeof(struct nes_hw_nic_qp_context);
1667
1668 nesvnic->nic_vbase = pci_zalloc_consistent(nesdev->pcidev,
1669 nesvnic->nic_mem_size,
1670 &nesvnic->nic_pbase);
1671 if (!nesvnic->nic_vbase) {
1672 nes_debug(NES_DBG_INIT, "Unable to allocate memory for NIC host descriptor rings\n");
1673 return -ENOMEM;
1674 }
1675 nes_debug(NES_DBG_INIT, "Allocated NIC QP structures at %p (phys = %016lX), size = %u.\n",
1676 nesvnic->nic_vbase, (unsigned long)nesvnic->nic_pbase, nesvnic->nic_mem_size);
1677
1678 vmem = (void *)(((unsigned long)nesvnic->nic_vbase + (256 - 1)) &
1679 ~(unsigned long)(256 - 1));
1680 pmem = (dma_addr_t)(((unsigned long long)nesvnic->nic_pbase + (256 - 1)) &
1681 ~(unsigned long long)(256 - 1));
1682
1683 /* Setup the first Fragment buffers */
1684 nesvnic->nic.first_frag_vbase = vmem;
1685
1686 for (counter = 0; counter < NES_NIC_WQ_SIZE; counter++) {
1687 nesvnic->nic.frag_paddr[counter] = pmem;
1688 pmem += sizeof(struct nes_first_frag);
1689 }
1690
1691 /* setup the SQ */
1692 vmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_first_frag));
1693
1694 nesvnic->nic.sq_vbase = (void *)vmem;
1695 nesvnic->nic.sq_pbase = pmem;
1696 nesvnic->nic.sq_head = 0;
1697 nesvnic->nic.sq_tail = 0;
1698 nesvnic->nic.sq_size = NES_NIC_WQ_SIZE;
1699 for (counter = 0; counter < NES_NIC_WQ_SIZE; counter++) {
1700 nic_sqe = &nesvnic->nic.sq_vbase[counter];
1701 nic_sqe->wqe_words[NES_NIC_SQ_WQE_MISC_IDX] =
1702 cpu_to_le32(NES_NIC_SQ_WQE_DISABLE_CHKSUM |
1703 NES_NIC_SQ_WQE_COMPLETION);
1704 nic_sqe->wqe_words[NES_NIC_SQ_WQE_LENGTH_0_TAG_IDX] =
1705 cpu_to_le32((u32)NES_FIRST_FRAG_SIZE << 16);
1706 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX] =
1707 cpu_to_le32((u32)nesvnic->nic.frag_paddr[counter]);
1708 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX] =
1709 cpu_to_le32((u32)((u64)nesvnic->nic.frag_paddr[counter] >> 32));
1710 }
1711
1712 nesvnic->get_cqp_request = nes_get_cqp_request;
1713 nesvnic->post_cqp_request = nes_post_cqp_request;
1714 nesvnic->mcrq_mcast_filter = NULL;
1715
1716 spin_lock_init(&nesvnic->nic.rq_lock);
1717
1718 /* setup the RQ */
1719 vmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_sq_wqe));
1720 pmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_sq_wqe));
1721
1722
1723 nesvnic->nic.rq_vbase = vmem;
1724 nesvnic->nic.rq_pbase = pmem;
1725 nesvnic->nic.rq_head = 0;
1726 nesvnic->nic.rq_tail = 0;
1727 nesvnic->nic.rq_size = NES_NIC_WQ_SIZE;
1728
1729 /* setup the CQ */
1730 vmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_rq_wqe));
1731 pmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_rq_wqe));
1732
1733 if (nesdev->nesadapter->netdev_count > 2)
1734 nesvnic->mcrq_qp_id = nesvnic->nic_index + 32;
1735 else
1736 nesvnic->mcrq_qp_id = nesvnic->nic.qp_id + 4;
1737
1738 nesvnic->nic_cq.cq_vbase = vmem;
1739 nesvnic->nic_cq.cq_pbase = pmem;
1740 nesvnic->nic_cq.cq_head = 0;
1741 nesvnic->nic_cq.cq_size = NES_NIC_WQ_SIZE * 2;
1742
1743 nesvnic->nic_cq.ce_handler = nes_nic_napi_ce_handler;
1744
1745 /* Send CreateCQ request to CQP */
1746 spin_lock_irqsave(&nesdev->cqp.lock, flags);
1747 cqp_head = nesdev->cqp.sq_head;
1748
1749 cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1750 nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1751
1752 cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(
1753 NES_CQP_CREATE_CQ | NES_CQP_CQ_CEQ_VALID |
1754 ((u32)nesvnic->nic_cq.cq_size << 16));
1755 cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(
1756 nesvnic->nic_cq.cq_number | ((u32)nesdev->nic_ceq_index << 16));
1757 u64temp = (u64)nesvnic->nic_cq.cq_pbase;
1758 set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1759 cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] = 0;
1760 u64temp = (unsigned long)&nesvnic->nic_cq;
1761 cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_LOW_IDX] = cpu_to_le32((u32)(u64temp >> 1));
1762 cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] =
1763 cpu_to_le32(((u32)((u64temp) >> 33)) & 0x7FFFFFFF);
1764 cqp_wqe->wqe_words[NES_CQP_CQ_WQE_DOORBELL_INDEX_HIGH_IDX] = 0;
1765 if (++cqp_head >= nesdev->cqp.sq_size)
1766 cqp_head = 0;
1767 cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1768 nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1769
1770 /* Send CreateQP request to CQP */
1771 nic_context = (void *)(&nesvnic->nic_cq.cq_vbase[nesvnic->nic_cq.cq_size]);
1772 nic_context->context_words[NES_NIC_CTX_MISC_IDX] =
1773 cpu_to_le32((u32)NES_NIC_CTX_SIZE |
1774 ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 12));
1775 nes_debug(NES_DBG_INIT, "RX_WINDOW_BUFFER_PAGE_TABLE_SIZE = 0x%08X, RX_WINDOW_BUFFER_SIZE = 0x%08X\n",
1776 nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_PAGE_TABLE_SIZE),
1777 nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_SIZE));
1778 if (nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_SIZE) != 0) {
1779 nic_context->context_words[NES_NIC_CTX_MISC_IDX] |= cpu_to_le32(NES_NIC_BACK_STORE);
1780 }
1781
1782 u64temp = (u64)nesvnic->nic.sq_pbase;
1783 nic_context->context_words[NES_NIC_CTX_SQ_LOW_IDX] = cpu_to_le32((u32)u64temp);
1784 nic_context->context_words[NES_NIC_CTX_SQ_HIGH_IDX] = cpu_to_le32((u32)(u64temp >> 32));
1785 u64temp = (u64)nesvnic->nic.rq_pbase;
1786 nic_context->context_words[NES_NIC_CTX_RQ_LOW_IDX] = cpu_to_le32((u32)u64temp);
1787 nic_context->context_words[NES_NIC_CTX_RQ_HIGH_IDX] = cpu_to_le32((u32)(u64temp >> 32));
1788
1789 cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_CREATE_QP |
1790 NES_CQP_QP_TYPE_NIC);
1791 cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesvnic->nic.qp_id);
1792 u64temp = (u64)nesvnic->nic_cq.cq_pbase +
1793 (nesvnic->nic_cq.cq_size * sizeof(struct nes_hw_nic_cqe));
1794 set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_QP_WQE_CONTEXT_LOW_IDX, u64temp);
1795
1796 if (++cqp_head >= nesdev->cqp.sq_size)
1797 cqp_head = 0;
1798 nesdev->cqp.sq_head = cqp_head;
1799
1800 barrier();
1801
1802 /* Ring doorbell (2 WQEs) */
1803 nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x02800000 | nesdev->cqp.qp_id);
1804
1805 spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
1806 nes_debug(NES_DBG_INIT, "Waiting for create NIC QP%u to complete.\n",
1807 nesvnic->nic.qp_id);
1808
1809 ret = wait_event_timeout(nesdev->cqp.waitq, (nesdev->cqp.sq_tail == cqp_head),
1810 NES_EVENT_TIMEOUT);
1811 nes_debug(NES_DBG_INIT, "Create NIC QP%u completed, wait_event_timeout ret = %u.\n",
1812 nesvnic->nic.qp_id, ret);
1813 if (!ret) {
1814 nes_debug(NES_DBG_INIT, "NIC QP%u create timeout expired\n", nesvnic->nic.qp_id);
1815 pci_free_consistent(nesdev->pcidev, nesvnic->nic_mem_size, nesvnic->nic_vbase,
1816 nesvnic->nic_pbase);
1817 return -EIO;
1818 }
1819
1820 /* Populate the RQ */
1821 for (counter = 0; counter < (NES_NIC_WQ_SIZE - 1); counter++) {
1822 skb = dev_alloc_skb(nesvnic->max_frame_size);
1823 if (!skb) {
1824 nes_debug(NES_DBG_INIT, "%s: out of memory for receive skb\n", netdev->name);
1825
1826 nes_destroy_nic_qp(nesvnic);
1827 return -ENOMEM;
1828 }
1829
1830 skb->dev = netdev;
1831
1832 pmem = pci_map_single(nesdev->pcidev, skb->data,
1833 nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
1834 cb = (struct nes_rskb_cb *)&skb->cb[0];
1835 cb->busaddr = pmem;
1836 cb->maplen = nesvnic->max_frame_size;
1837
1838 nic_rqe = &nesvnic->nic.rq_vbase[counter];
1839 nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_1_0_IDX] = cpu_to_le32(nesvnic->max_frame_size);
1840 nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_3_2_IDX] = 0;
1841 nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX] = cpu_to_le32((u32)pmem);
1842 nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX] = cpu_to_le32((u32)((u64)pmem >> 32));
1843 nesvnic->nic.rx_skb[counter] = skb;
1844 }
1845
1846 wqe_count = NES_NIC_WQ_SIZE - 1;
1847 nesvnic->nic.rq_head = wqe_count;
1848 barrier();
1849 do {
1850 counter = min(wqe_count, ((u32)255));
1851 wqe_count -= counter;
1852 nes_write32(nesdev->regs+NES_WQE_ALLOC, (counter << 24) | nesvnic->nic.qp_id);
1853 } while (wqe_count);
1854 init_timer(&nesvnic->rq_wqes_timer);
1855 nesvnic->rq_wqes_timer.function = nes_rq_wqes_timeout;
1856 nesvnic->rq_wqes_timer.data = (unsigned long)nesvnic;
1857 nes_debug(NES_DBG_INIT, "NAPI support Enabled\n");
1858 if (nesdev->nesadapter->et_use_adaptive_rx_coalesce)
1859 {
1860 nes_nic_init_timer(nesdev);
1861 if (netdev->mtu > 1500)
1862 jumbomode = 1;
1863 nes_nic_init_timer_defaults(nesdev, jumbomode);
1864 }
1865 if ((nesdev->nesadapter->allow_unaligned_fpdus) &&
1866 (nes_init_mgt_qp(nesdev, netdev, nesvnic))) {
1867 nes_debug(NES_DBG_INIT, "%s: Out of memory for pau nic\n", netdev->name);
1868 nes_destroy_nic_qp(nesvnic);
1869 return -ENOMEM;
1870 }
1871
1872 return 0;
1873 }
1874
1875
1876 /**
1877 * nes_destroy_nic_qp
1878 */
1879 void nes_destroy_nic_qp(struct nes_vnic *nesvnic)
1880 {
1881 u64 u64temp;
1882 dma_addr_t bus_address;
1883 struct nes_device *nesdev = nesvnic->nesdev;
1884 struct nes_hw_cqp_wqe *cqp_wqe;
1885 struct nes_hw_nic_sq_wqe *nic_sqe;
1886 __le16 *wqe_fragment_length;
1887 u16 wqe_fragment_index;
1888 u32 cqp_head;
1889 u32 wqm_cfg0;
1890 unsigned long flags;
1891 struct sk_buff *rx_skb;
1892 struct nes_rskb_cb *cb;
1893 int ret;
1894
1895 if (nesdev->nesadapter->allow_unaligned_fpdus)
1896 nes_destroy_mgt(nesvnic);
1897
1898 /* clear wqe stall before destroying NIC QP */
1899 wqm_cfg0 = nes_read_indexed(nesdev, NES_IDX_WQM_CONFIG0);
1900 nes_write_indexed(nesdev, NES_IDX_WQM_CONFIG0, wqm_cfg0 & 0xFFFF7FFF);
1901
1902 /* Free remaining NIC receive buffers */
1903 while (nesvnic->nic.rq_head != nesvnic->nic.rq_tail) {
1904 rx_skb = nesvnic->nic.rx_skb[nesvnic->nic.rq_tail];
1905 cb = (struct nes_rskb_cb *)&rx_skb->cb[0];
1906 pci_unmap_single(nesdev->pcidev, cb->busaddr, cb->maplen,
1907 PCI_DMA_FROMDEVICE);
1908
1909 dev_kfree_skb(nesvnic->nic.rx_skb[nesvnic->nic.rq_tail++]);
1910 nesvnic->nic.rq_tail &= (nesvnic->nic.rq_size - 1);
1911 }
1912
1913 /* Free remaining NIC transmit buffers */
1914 while (nesvnic->nic.sq_head != nesvnic->nic.sq_tail) {
1915 nic_sqe = &nesvnic->nic.sq_vbase[nesvnic->nic.sq_tail];
1916 wqe_fragment_index = 1;
1917 wqe_fragment_length = (__le16 *)
1918 &nic_sqe->wqe_words[NES_NIC_SQ_WQE_LENGTH_0_TAG_IDX];
1919 /* bump past the vlan tag */
1920 wqe_fragment_length++;
1921 if (le16_to_cpu(wqe_fragment_length[wqe_fragment_index]) != 0) {
1922 u64temp = (u64)le32_to_cpu(
1923 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX+
1924 wqe_fragment_index*2]);
1925 u64temp += ((u64)le32_to_cpu(
1926 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX
1927 + wqe_fragment_index*2]))<<32;
1928 bus_address = (dma_addr_t)u64temp;
1929 if (test_and_clear_bit(nesvnic->nic.sq_tail,
1930 nesvnic->nic.first_frag_overflow)) {
1931 pci_unmap_single(nesdev->pcidev,
1932 bus_address,
1933 le16_to_cpu(wqe_fragment_length[
1934 wqe_fragment_index++]),
1935 PCI_DMA_TODEVICE);
1936 }
1937 for (; wqe_fragment_index < 5; wqe_fragment_index++) {
1938 if (wqe_fragment_length[wqe_fragment_index]) {
1939 u64temp = le32_to_cpu(
1940 nic_sqe->wqe_words[
1941 NES_NIC_SQ_WQE_FRAG0_LOW_IDX+
1942 wqe_fragment_index*2]);
1943 u64temp += ((u64)le32_to_cpu(
1944 nic_sqe->wqe_words[
1945 NES_NIC_SQ_WQE_FRAG0_HIGH_IDX+
1946 wqe_fragment_index*2]))<<32;
1947 bus_address = (dma_addr_t)u64temp;
1948 pci_unmap_page(nesdev->pcidev,
1949 bus_address,
1950 le16_to_cpu(
1951 wqe_fragment_length[
1952 wqe_fragment_index]),
1953 PCI_DMA_TODEVICE);
1954 } else
1955 break;
1956 }
1957 }
1958 if (nesvnic->nic.tx_skb[nesvnic->nic.sq_tail])
1959 dev_kfree_skb(
1960 nesvnic->nic.tx_skb[nesvnic->nic.sq_tail]);
1961
1962 nesvnic->nic.sq_tail = (nesvnic->nic.sq_tail + 1)
1963 & (nesvnic->nic.sq_size - 1);
1964 }
1965
1966 spin_lock_irqsave(&nesdev->cqp.lock, flags);
1967
1968 /* Destroy NIC QP */
1969 cqp_head = nesdev->cqp.sq_head;
1970 cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1971 nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1972
1973 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1974 (NES_CQP_DESTROY_QP | NES_CQP_QP_TYPE_NIC));
1975 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
1976 nesvnic->nic.qp_id);
1977
1978 if (++cqp_head >= nesdev->cqp.sq_size)
1979 cqp_head = 0;
1980
1981 cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1982
1983 /* Destroy NIC CQ */
1984 nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1985 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1986 (NES_CQP_DESTROY_CQ | ((u32)nesvnic->nic_cq.cq_size << 16)));
1987 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
1988 (nesvnic->nic_cq.cq_number | ((u32)nesdev->nic_ceq_index << 16)));
1989
1990 if (++cqp_head >= nesdev->cqp.sq_size)
1991 cqp_head = 0;
1992
1993 nesdev->cqp.sq_head = cqp_head;
1994 barrier();
1995
1996 /* Ring doorbell (2 WQEs) */
1997 nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x02800000 | nesdev->cqp.qp_id);
1998
1999 spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
2000 nes_debug(NES_DBG_SHUTDOWN, "Waiting for CQP, cqp_head=%u, cqp.sq_head=%u,"
2001 " cqp.sq_tail=%u, cqp.sq_size=%u\n",
2002 cqp_head, nesdev->cqp.sq_head,
2003 nesdev->cqp.sq_tail, nesdev->cqp.sq_size);
2004
2005 ret = wait_event_timeout(nesdev->cqp.waitq, (nesdev->cqp.sq_tail == cqp_head),
2006 NES_EVENT_TIMEOUT);
2007
2008 nes_debug(NES_DBG_SHUTDOWN, "Destroy NIC QP returned, wait_event_timeout ret = %u, cqp_head=%u,"
2009 " cqp.sq_head=%u, cqp.sq_tail=%u\n",
2010 ret, cqp_head, nesdev->cqp.sq_head, nesdev->cqp.sq_tail);
2011 if (!ret) {
2012 nes_debug(NES_DBG_SHUTDOWN, "NIC QP%u destroy timeout expired\n",
2013 nesvnic->nic.qp_id);
2014 }
2015
2016 pci_free_consistent(nesdev->pcidev, nesvnic->nic_mem_size, nesvnic->nic_vbase,
2017 nesvnic->nic_pbase);
2018
2019 /* restore old wqm_cfg0 value */
2020 nes_write_indexed(nesdev, NES_IDX_WQM_CONFIG0, wqm_cfg0);
2021 }
2022
2023 /**
2024 * nes_napi_isr
2025 */
2026 int nes_napi_isr(struct nes_device *nesdev)
2027 {
2028 struct nes_adapter *nesadapter = nesdev->nesadapter;
2029 u32 int_stat;
2030
2031 if (nesdev->napi_isr_ran) {
2032 /* interrupt status has already been read in ISR */
2033 int_stat = nesdev->int_stat;
2034 } else {
2035 int_stat = nes_read32(nesdev->regs + NES_INT_STAT);
2036 nesdev->int_stat = int_stat;
2037 nesdev->napi_isr_ran = 1;
2038 }
2039
2040 int_stat &= nesdev->int_req;
2041 /* iff NIC, process here, else wait for DPC */
2042 if ((int_stat) && ((int_stat & 0x0000ff00) == int_stat)) {
2043 nesdev->napi_isr_ran = 0;
2044 nes_write32(nesdev->regs + NES_INT_STAT,
2045 (int_stat &
2046 ~(NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0 | NES_INT_MAC1 | NES_INT_MAC2 | NES_INT_MAC3)));
2047
2048 /* Process the CEQs */
2049 nes_process_ceq(nesdev, &nesdev->nesadapter->ceq[nesdev->nic_ceq_index]);
2050
2051 if (unlikely((((nesadapter->et_rx_coalesce_usecs_irq) &&
2052 (!nesadapter->et_use_adaptive_rx_coalesce)) ||
2053 ((nesadapter->et_use_adaptive_rx_coalesce) &&
2054 (nesdev->deepcq_count > nesadapter->et_pkt_rate_low))))) {
2055 if ((nesdev->int_req & NES_INT_TIMER) == 0) {
2056 /* Enable Periodic timer interrupts */
2057 nesdev->int_req |= NES_INT_TIMER;
2058 /* ack any pending periodic timer interrupts so we don't get an immediate interrupt */
2059 /* TODO: need to also ack other unused periodic timer values, get from nesadapter */
2060 nes_write32(nesdev->regs+NES_TIMER_STAT,
2061 nesdev->timer_int_req | ~(nesdev->nesadapter->timer_int_req));
2062 nes_write32(nesdev->regs+NES_INTF_INT_MASK,
2063 ~(nesdev->intf_int_req | NES_INTF_PERIODIC_TIMER));
2064 }
2065
2066 if (unlikely(nesadapter->et_use_adaptive_rx_coalesce))
2067 {
2068 nes_nic_init_timer(nesdev);
2069 }
2070 /* Enable interrupts, except CEQs */
2071 nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
2072 } else {
2073 /* Enable interrupts, make sure timer is off */
2074 nesdev->int_req &= ~NES_INT_TIMER;
2075 nes_write32(nesdev->regs+NES_INTF_INT_MASK, ~(nesdev->intf_int_req));
2076 nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
2077 }
2078 nesdev->deepcq_count = 0;
2079 return 1;
2080 } else {
2081 return 0;
2082 }
2083 }
2084
2085 static void process_critical_error(struct nes_device *nesdev)
2086 {
2087 u32 debug_error;
2088 u32 nes_idx_debug_error_masks0 = 0;
2089 u16 error_module = 0;
2090
2091 debug_error = nes_read_indexed(nesdev, NES_IDX_DEBUG_ERROR_CONTROL_STATUS);
2092 printk(KERN_ERR PFX "Critical Error reported by device!!! 0x%02X\n",
2093 (u16)debug_error);
2094 nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_CONTROL_STATUS,
2095 0x01010000 | (debug_error & 0x0000ffff));
2096 if (crit_err_count++ > 10)
2097 nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_MASKS1, 1 << 0x17);
2098 error_module = (u16) (debug_error & 0x1F00) >> 8;
2099 if (++nesdev->nesadapter->crit_error_count[error_module-1] >=
2100 nes_max_critical_error_count) {
2101 printk(KERN_ERR PFX "Masking off critical error for module "
2102 "0x%02X\n", (u16)error_module);
2103 nes_idx_debug_error_masks0 = nes_read_indexed(nesdev,
2104 NES_IDX_DEBUG_ERROR_MASKS0);
2105 nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_MASKS0,
2106 nes_idx_debug_error_masks0 | (1 << error_module));
2107 }
2108 }
2109 /**
2110 * nes_dpc
2111 */
2112 void nes_dpc(unsigned long param)
2113 {
2114 struct nes_device *nesdev = (struct nes_device *)param;
2115 struct nes_adapter *nesadapter = nesdev->nesadapter;
2116 u32 counter;
2117 u32 loop_counter = 0;
2118 u32 int_status_bit;
2119 u32 int_stat;
2120 u32 timer_stat;
2121 u32 temp_int_stat;
2122 u32 intf_int_stat;
2123 u32 processed_intf_int = 0;
2124 u16 processed_timer_int = 0;
2125 u16 completion_ints = 0;
2126 u16 timer_ints = 0;
2127
2128 /* nes_debug(NES_DBG_ISR, "\n"); */
2129
2130 do {
2131 timer_stat = 0;
2132 if (nesdev->napi_isr_ran) {
2133 nesdev->napi_isr_ran = 0;
2134 int_stat = nesdev->int_stat;
2135 } else
2136 int_stat = nes_read32(nesdev->regs+NES_INT_STAT);
2137 if (processed_intf_int != 0)
2138 int_stat &= nesdev->int_req & ~NES_INT_INTF;
2139 else
2140 int_stat &= nesdev->int_req;
2141 if (processed_timer_int == 0) {
2142 processed_timer_int = 1;
2143 if (int_stat & NES_INT_TIMER) {
2144 timer_stat = nes_read32(nesdev->regs + NES_TIMER_STAT);
2145 if ((timer_stat & nesdev->timer_int_req) == 0) {
2146 int_stat &= ~NES_INT_TIMER;
2147 }
2148 }
2149 } else {
2150 int_stat &= ~NES_INT_TIMER;
2151 }
2152
2153 if (int_stat) {
2154 if (int_stat & ~(NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0|
2155 NES_INT_MAC1|NES_INT_MAC2 | NES_INT_MAC3)) {
2156 /* Ack the interrupts */
2157 nes_write32(nesdev->regs+NES_INT_STAT,
2158 (int_stat & ~(NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0|
2159 NES_INT_MAC1 | NES_INT_MAC2 | NES_INT_MAC3)));
2160 }
2161
2162 temp_int_stat = int_stat;
2163 for (counter = 0, int_status_bit = 1; counter < 16; counter++) {
2164 if (int_stat & int_status_bit) {
2165 nes_process_ceq(nesdev, &nesadapter->ceq[counter]);
2166 temp_int_stat &= ~int_status_bit;
2167 completion_ints = 1;
2168 }
2169 if (!(temp_int_stat & 0x0000ffff))
2170 break;
2171 int_status_bit <<= 1;
2172 }
2173
2174 /* Process the AEQ for this pci function */
2175 int_status_bit = 1 << (16 + PCI_FUNC(nesdev->pcidev->devfn));
2176 if (int_stat & int_status_bit) {
2177 nes_process_aeq(nesdev, &nesadapter->aeq[PCI_FUNC(nesdev->pcidev->devfn)]);
2178 }
2179
2180 /* Process the MAC interrupt for this pci function */
2181 int_status_bit = 1 << (24 + nesdev->mac_index);
2182 if (int_stat & int_status_bit) {
2183 nes_process_mac_intr(nesdev, nesdev->mac_index);
2184 }
2185
2186 if (int_stat & NES_INT_TIMER) {
2187 if (timer_stat & nesdev->timer_int_req) {
2188 nes_write32(nesdev->regs + NES_TIMER_STAT,
2189 (timer_stat & nesdev->timer_int_req) |
2190 ~(nesdev->nesadapter->timer_int_req));
2191 timer_ints = 1;
2192 }
2193 }
2194
2195 if (int_stat & NES_INT_INTF) {
2196 processed_intf_int = 1;
2197 intf_int_stat = nes_read32(nesdev->regs+NES_INTF_INT_STAT);
2198 intf_int_stat &= nesdev->intf_int_req;
2199 if (NES_INTF_INT_CRITERR & intf_int_stat) {
2200 process_critical_error(nesdev);
2201 }
2202 if (NES_INTF_INT_PCIERR & intf_int_stat) {
2203 printk(KERN_ERR PFX "PCI Error reported by device!!!\n");
2204 BUG();
2205 }
2206 if (NES_INTF_INT_AEQ_OFLOW & intf_int_stat) {
2207 printk(KERN_ERR PFX "AEQ Overflow reported by device!!!\n");
2208 BUG();
2209 }
2210 nes_write32(nesdev->regs+NES_INTF_INT_STAT, intf_int_stat);
2211 }
2212
2213 if (int_stat & NES_INT_TSW) {
2214 }
2215 }
2216 /* Don't use the interface interrupt bit stay in loop */
2217 int_stat &= ~NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0 |
2218 NES_INT_MAC1 | NES_INT_MAC2 | NES_INT_MAC3;
2219 } while ((int_stat != 0) && (loop_counter++ < MAX_DPC_ITERATIONS));
2220
2221 if (timer_ints == 1) {
2222 if ((nesadapter->et_rx_coalesce_usecs_irq) || (nesadapter->et_use_adaptive_rx_coalesce)) {
2223 if (completion_ints == 0) {
2224 nesdev->timer_only_int_count++;
2225 if (nesdev->timer_only_int_count>=nesadapter->timer_int_limit) {
2226 nesdev->timer_only_int_count = 0;
2227 nesdev->int_req &= ~NES_INT_TIMER;
2228 nes_write32(nesdev->regs + NES_INTF_INT_MASK, ~(nesdev->intf_int_req));
2229 nes_write32(nesdev->regs + NES_INT_MASK, ~nesdev->int_req);
2230 } else {
2231 nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
2232 }
2233 } else {
2234 if (unlikely(nesadapter->et_use_adaptive_rx_coalesce))
2235 {
2236 nes_nic_init_timer(nesdev);
2237 }
2238 nesdev->timer_only_int_count = 0;
2239 nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
2240 }
2241 } else {
2242 nesdev->timer_only_int_count = 0;
2243 nesdev->int_req &= ~NES_INT_TIMER;
2244 nes_write32(nesdev->regs+NES_INTF_INT_MASK, ~(nesdev->intf_int_req));
2245 nes_write32(nesdev->regs+NES_TIMER_STAT,
2246 nesdev->timer_int_req | ~(nesdev->nesadapter->timer_int_req));
2247 nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
2248 }
2249 } else {
2250 if ( (completion_ints == 1) &&
2251 (((nesadapter->et_rx_coalesce_usecs_irq) &&
2252 (!nesadapter->et_use_adaptive_rx_coalesce)) ||
2253 ((nesdev->deepcq_count > nesadapter->et_pkt_rate_low) &&
2254 (nesadapter->et_use_adaptive_rx_coalesce) )) ) {
2255 /* nes_debug(NES_DBG_ISR, "Enabling periodic timer interrupt.\n" ); */
2256 nesdev->timer_only_int_count = 0;
2257 nesdev->int_req |= NES_INT_TIMER;
2258 nes_write32(nesdev->regs+NES_TIMER_STAT,
2259 nesdev->timer_int_req | ~(nesdev->nesadapter->timer_int_req));
2260 nes_write32(nesdev->regs+NES_INTF_INT_MASK,
2261 ~(nesdev->intf_int_req | NES_INTF_PERIODIC_TIMER));
2262 nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
2263 } else {
2264 nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
2265 }
2266 }
2267 nesdev->deepcq_count = 0;
2268 }
2269
2270
2271 /**
2272 * nes_process_ceq
2273 */
2274 static void nes_process_ceq(struct nes_device *nesdev, struct nes_hw_ceq *ceq)
2275 {
2276 u64 u64temp;
2277 struct nes_hw_cq *cq;
2278 u32 head;
2279 u32 ceq_size;
2280
2281 /* nes_debug(NES_DBG_CQ, "\n"); */
2282 head = ceq->ceq_head;
2283 ceq_size = ceq->ceq_size;
2284
2285 do {
2286 if (le32_to_cpu(ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_HIGH_IDX]) &
2287 NES_CEQE_VALID) {
2288 u64temp = (((u64)(le32_to_cpu(ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_HIGH_IDX]))) << 32) |
2289 ((u64)(le32_to_cpu(ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_LOW_IDX])));
2290 u64temp <<= 1;
2291 cq = *((struct nes_hw_cq **)&u64temp);
2292 /* nes_debug(NES_DBG_CQ, "pCQ = %p\n", cq); */
2293 barrier();
2294 ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_HIGH_IDX] = 0;
2295
2296 /* call the event handler */
2297 cq->ce_handler(nesdev, cq);
2298
2299 if (++head >= ceq_size)
2300 head = 0;
2301 } else {
2302 break;
2303 }
2304
2305 } while (1);
2306
2307 ceq->ceq_head = head;
2308 }
2309
2310
2311 /**
2312 * nes_process_aeq
2313 */
2314 static void nes_process_aeq(struct nes_device *nesdev, struct nes_hw_aeq *aeq)
2315 {
2316 /* u64 u64temp; */
2317 u32 head;
2318 u32 aeq_size;
2319 u32 aeqe_misc;
2320 u32 aeqe_cq_id;
2321 struct nes_hw_aeqe volatile *aeqe;
2322
2323 head = aeq->aeq_head;
2324 aeq_size = aeq->aeq_size;
2325
2326 do {
2327 aeqe = &aeq->aeq_vbase[head];
2328 if ((le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]) & NES_AEQE_VALID) == 0)
2329 break;
2330 aeqe_misc = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
2331 aeqe_cq_id = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]);
2332 if (aeqe_misc & (NES_AEQE_QP|NES_AEQE_CQ)) {
2333 if (aeqe_cq_id >= NES_FIRST_QPN) {
2334 /* dealing with an accelerated QP related AE */
2335 /*
2336 * u64temp = (((u64)(le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_HIGH_IDX]))) << 32) |
2337 * ((u64)(le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_LOW_IDX])));
2338 */
2339 nes_process_iwarp_aeqe(nesdev, (struct nes_hw_aeqe *)aeqe);
2340 } else {
2341 /* TODO: dealing with a CQP related AE */
2342 nes_debug(NES_DBG_AEQ, "Processing CQP related AE, misc = 0x%04X\n",
2343 (u16)(aeqe_misc >> 16));
2344 }
2345 }
2346
2347 aeqe->aeqe_words[NES_AEQE_MISC_IDX] = 0;
2348
2349 if (++head >= aeq_size)
2350 head = 0;
2351
2352 nes_write32(nesdev->regs + NES_AEQ_ALLOC, 1 << 16);
2353 }
2354 while (1);
2355 aeq->aeq_head = head;
2356 }
2357
2358 static void nes_reset_link(struct nes_device *nesdev, u32 mac_index)
2359 {
2360 struct nes_adapter *nesadapter = nesdev->nesadapter;
2361 u32 reset_value;
2362 u32 i=0;
2363 u32 u32temp;
2364
2365 if (nesadapter->hw_rev == NE020_REV) {
2366 return;
2367 }
2368 mh_detected++;
2369
2370 reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
2371
2372 if ((mac_index == 0) || ((mac_index == 1) && (nesadapter->OneG_Mode)))
2373 reset_value |= 0x0000001d;
2374 else
2375 reset_value |= 0x0000002d;
2376
2377 if (4 <= (nesadapter->link_interrupt_count[mac_index] / ((u16)NES_MAX_LINK_INTERRUPTS))) {
2378 if ((!nesadapter->OneG_Mode) && (nesadapter->port_count == 2)) {
2379 nesadapter->link_interrupt_count[0] = 0;
2380 nesadapter->link_interrupt_count[1] = 0;
2381 u32temp = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
2382 if (0x00000040 & u32temp)
2383 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F088);
2384 else
2385 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F0C8);
2386
2387 reset_value |= 0x0000003d;
2388 }
2389 nesadapter->link_interrupt_count[mac_index] = 0;
2390 }
2391
2392 nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
2393
2394 while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET)
2395 & 0x00000040) != 0x00000040) && (i++ < 5000));
2396
2397 if (0x0000003d == (reset_value & 0x0000003d)) {
2398 u32 pcs_control_status0, pcs_control_status1;
2399
2400 for (i = 0; i < 10; i++) {
2401 pcs_control_status0 = nes_read_indexed(nesdev, NES_IDX_PHY_PCS_CONTROL_STATUS0);
2402 pcs_control_status1 = nes_read_indexed(nesdev, NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
2403 if (((0x0F000000 == (pcs_control_status0 & 0x0F000000))
2404 && (pcs_control_status0 & 0x00100000))
2405 || ((0x0F000000 == (pcs_control_status1 & 0x0F000000))
2406 && (pcs_control_status1 & 0x00100000)))
2407 continue;
2408 else
2409 break;
2410 }
2411 if (10 == i) {
2412 u32temp = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
2413 if (0x00000040 & u32temp)
2414 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F088);
2415 else
2416 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F0C8);
2417
2418 nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
2419
2420 while (((nes_read32(nesdev->regs + NES_SOFTWARE_RESET)
2421 & 0x00000040) != 0x00000040) && (i++ < 5000));
2422 }
2423 }
2424 }
2425
2426 /**
2427 * nes_process_mac_intr
2428 */
2429 static void nes_process_mac_intr(struct nes_device *nesdev, u32 mac_number)
2430 {
2431 unsigned long flags;
2432 u32 pcs_control_status;
2433 struct nes_adapter *nesadapter = nesdev->nesadapter;
2434 struct nes_vnic *nesvnic;
2435 u32 mac_status;
2436 u32 mac_index = nesdev->mac_index;
2437 u32 u32temp;
2438 u16 phy_data;
2439 u16 temp_phy_data;
2440 u32 pcs_val = 0x0f0f0000;
2441 u32 pcs_mask = 0x0f1f0000;
2442 u32 cdr_ctrl;
2443
2444 spin_lock_irqsave(&nesadapter->phy_lock, flags);
2445 if (nesadapter->mac_sw_state[mac_number] != NES_MAC_SW_IDLE) {
2446 spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
2447 return;
2448 }
2449 nesadapter->mac_sw_state[mac_number] = NES_MAC_SW_INTERRUPT;
2450
2451 /* ack the MAC interrupt */
2452 mac_status = nes_read_indexed(nesdev, NES_IDX_MAC_INT_STATUS + (mac_index * 0x200));
2453 /* Clear the interrupt */
2454 nes_write_indexed(nesdev, NES_IDX_MAC_INT_STATUS + (mac_index * 0x200), mac_status);
2455
2456 nes_debug(NES_DBG_PHY, "MAC%u interrupt status = 0x%X.\n", mac_number, mac_status);
2457
2458 if (mac_status & (NES_MAC_INT_LINK_STAT_CHG | NES_MAC_INT_XGMII_EXT)) {
2459 nesdev->link_status_interrupts++;
2460 if (0 == (++nesadapter->link_interrupt_count[mac_index] % ((u16)NES_MAX_LINK_INTERRUPTS)))
2461 nes_reset_link(nesdev, mac_index);
2462
2463 /* read the PHY interrupt status register */
2464 if ((nesadapter->OneG_Mode) &&
2465 (nesadapter->phy_type[mac_index] != NES_PHY_TYPE_PUMA_1G)) {
2466 do {
2467 nes_read_1G_phy_reg(nesdev, 0x1a,
2468 nesadapter->phy_index[mac_index], &phy_data);
2469 nes_debug(NES_DBG_PHY, "Phy%d data from register 0x1a = 0x%X.\n",
2470 nesadapter->phy_index[mac_index], phy_data);
2471 } while (phy_data&0x8000);
2472
2473 temp_phy_data = 0;
2474 do {
2475 nes_read_1G_phy_reg(nesdev, 0x11,
2476 nesadapter->phy_index[mac_index], &phy_data);
2477 nes_debug(NES_DBG_PHY, "Phy%d data from register 0x11 = 0x%X.\n",
2478 nesadapter->phy_index[mac_index], phy_data);
2479 if (temp_phy_data == phy_data)
2480 break;
2481 temp_phy_data = phy_data;
2482 } while (1);
2483
2484 nes_read_1G_phy_reg(nesdev, 0x1e,
2485 nesadapter->phy_index[mac_index], &phy_data);
2486 nes_debug(NES_DBG_PHY, "Phy%d data from register 0x1e = 0x%X.\n",
2487 nesadapter->phy_index[mac_index], phy_data);
2488
2489 nes_read_1G_phy_reg(nesdev, 1,
2490 nesadapter->phy_index[mac_index], &phy_data);
2491 nes_debug(NES_DBG_PHY, "1G phy%u data from register 1 = 0x%X\n",
2492 nesadapter->phy_index[mac_index], phy_data);
2493
2494 if (temp_phy_data & 0x1000) {
2495 nes_debug(NES_DBG_PHY, "The Link is up according to the PHY\n");
2496 phy_data = 4;
2497 } else {
2498 nes_debug(NES_DBG_PHY, "The Link is down according to the PHY\n");
2499 }
2500 }
2501 nes_debug(NES_DBG_PHY, "Eth SERDES Common Status: 0=0x%08X, 1=0x%08X\n",
2502 nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0),
2503 nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0+0x200));
2504
2505 if (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_PUMA_1G) {
2506 switch (mac_index) {
2507 case 1:
2508 case 3:
2509 pcs_control_status = nes_read_indexed(nesdev,
2510 NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
2511 break;
2512 default:
2513 pcs_control_status = nes_read_indexed(nesdev,
2514 NES_IDX_PHY_PCS_CONTROL_STATUS0);
2515 break;
2516 }
2517 } else {
2518 pcs_control_status = nes_read_indexed(nesdev,
2519 NES_IDX_PHY_PCS_CONTROL_STATUS0 + ((mac_index & 1) * 0x200));
2520 pcs_control_status = nes_read_indexed(nesdev,
2521 NES_IDX_PHY_PCS_CONTROL_STATUS0 + ((mac_index & 1) * 0x200));
2522 }
2523
2524 nes_debug(NES_DBG_PHY, "PCS PHY Control/Status%u: 0x%08X\n",
2525 mac_index, pcs_control_status);
2526 if ((nesadapter->OneG_Mode) &&
2527 (nesadapter->phy_type[mac_index] != NES_PHY_TYPE_PUMA_1G)) {
2528 u32temp = 0x01010000;
2529 if (nesadapter->port_count > 2) {
2530 u32temp |= 0x02020000;
2531 }
2532 if ((pcs_control_status & u32temp)!= u32temp) {
2533 phy_data = 0;
2534 nes_debug(NES_DBG_PHY, "PCS says the link is down\n");
2535 }
2536 } else {
2537 switch (nesadapter->phy_type[mac_index]) {
2538 case NES_PHY_TYPE_ARGUS:
2539 case NES_PHY_TYPE_SFP_D:
2540 case NES_PHY_TYPE_KR:
2541 /* clear the alarms */
2542 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0x0008);
2543 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc001);
2544 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc002);
2545 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc005);
2546 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc006);
2547 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9003);
2548 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9004);
2549 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9005);
2550 /* check link status */
2551 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9003);
2552 temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2553
2554 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 3, 0x0021);
2555 nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2556 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 3, 0x0021);
2557 phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2558
2559 phy_data = (!temp_phy_data && (phy_data == 0x8000)) ? 0x4 : 0x0;
2560
2561 nes_debug(NES_DBG_PHY, "%s: Phy data = 0x%04X, link was %s.\n",
2562 __func__, phy_data, nesadapter->mac_link_down[mac_index] ? "DOWN" : "UP");
2563 break;
2564
2565 case NES_PHY_TYPE_PUMA_1G:
2566 if (mac_index < 2)
2567 pcs_val = pcs_mask = 0x01010000;
2568 else
2569 pcs_val = pcs_mask = 0x02020000;
2570 /* fall through */
2571 default:
2572 phy_data = (pcs_val == (pcs_control_status & pcs_mask)) ? 0x4 : 0x0;
2573 break;
2574 }
2575 }
2576
2577 if (phy_data & 0x0004) {
2578 if (wide_ppm_offset &&
2579 (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_CX4) &&
2580 (nesadapter->hw_rev != NE020_REV)) {
2581 cdr_ctrl = nes_read_indexed(nesdev,
2582 NES_IDX_ETH_SERDES_CDR_CONTROL0 +
2583 mac_index * 0x200);
2584 nes_write_indexed(nesdev,
2585 NES_IDX_ETH_SERDES_CDR_CONTROL0 +
2586 mac_index * 0x200,
2587 cdr_ctrl | 0x000F0000);
2588 }
2589 nesadapter->mac_link_down[mac_index] = 0;
2590 list_for_each_entry(nesvnic, &nesadapter->nesvnic_list[mac_index], list) {
2591 nes_debug(NES_DBG_PHY, "The Link is UP!!. linkup was %d\n",
2592 nesvnic->linkup);
2593 if (nesvnic->linkup == 0) {
2594 printk(PFX "The Link is now up for port %s, netdev %p.\n",
2595 nesvnic->netdev->name, nesvnic->netdev);
2596 if (netif_queue_stopped(nesvnic->netdev))
2597 netif_start_queue(nesvnic->netdev);
2598 nesvnic->linkup = 1;
2599 netif_carrier_on(nesvnic->netdev);
2600
2601 spin_lock(&nesvnic->port_ibevent_lock);
2602 if (nesvnic->of_device_registered) {
2603 if (nesdev->iw_status == 0) {
2604 nesdev->iw_status = 1;
2605 nes_port_ibevent(nesvnic);
2606 }
2607 }
2608 spin_unlock(&nesvnic->port_ibevent_lock);
2609 }
2610 }
2611 } else {
2612 if (wide_ppm_offset &&
2613 (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_CX4) &&
2614 (nesadapter->hw_rev != NE020_REV)) {
2615 cdr_ctrl = nes_read_indexed(nesdev,
2616 NES_IDX_ETH_SERDES_CDR_CONTROL0 +
2617 mac_index * 0x200);
2618 nes_write_indexed(nesdev,
2619 NES_IDX_ETH_SERDES_CDR_CONTROL0 +
2620 mac_index * 0x200,
2621 cdr_ctrl & 0xFFF0FFFF);
2622 }
2623 nesadapter->mac_link_down[mac_index] = 1;
2624 list_for_each_entry(nesvnic, &nesadapter->nesvnic_list[mac_index], list) {
2625 nes_debug(NES_DBG_PHY, "The Link is Down!!. linkup was %d\n",
2626 nesvnic->linkup);
2627 if (nesvnic->linkup == 1) {
2628 printk(PFX "The Link is now down for port %s, netdev %p.\n",
2629 nesvnic->netdev->name, nesvnic->netdev);
2630 if (!(netif_queue_stopped(nesvnic->netdev)))
2631 netif_stop_queue(nesvnic->netdev);
2632 nesvnic->linkup = 0;
2633 netif_carrier_off(nesvnic->netdev);
2634
2635 spin_lock(&nesvnic->port_ibevent_lock);
2636 if (nesvnic->of_device_registered) {
2637 if (nesdev->iw_status == 1) {
2638 nesdev->iw_status = 0;
2639 nes_port_ibevent(nesvnic);
2640 }
2641 }
2642 spin_unlock(&nesvnic->port_ibevent_lock);
2643 }
2644 }
2645 }
2646 if (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_SFP_D) {
2647 nesdev->link_recheck = 1;
2648 mod_delayed_work(system_wq, &nesdev->work,
2649 NES_LINK_RECHECK_DELAY);
2650 }
2651 }
2652
2653 spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
2654
2655 nesadapter->mac_sw_state[mac_number] = NES_MAC_SW_IDLE;
2656 }
2657
2658 void nes_recheck_link_status(struct work_struct *work)
2659 {
2660 unsigned long flags;
2661 struct nes_device *nesdev = container_of(work, struct nes_device, work.work);
2662 struct nes_adapter *nesadapter = nesdev->nesadapter;
2663 struct nes_vnic *nesvnic;
2664 u32 mac_index = nesdev->mac_index;
2665 u16 phy_data;
2666 u16 temp_phy_data;
2667
2668 spin_lock_irqsave(&nesadapter->phy_lock, flags);
2669
2670 /* check link status */
2671 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9003);
2672 temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2673
2674 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 3, 0x0021);
2675 nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2676 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 3, 0x0021);
2677 phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2678
2679 phy_data = (!temp_phy_data && (phy_data == 0x8000)) ? 0x4 : 0x0;
2680
2681 nes_debug(NES_DBG_PHY, "%s: Phy data = 0x%04X, link was %s.\n",
2682 __func__, phy_data,
2683 nesadapter->mac_link_down[mac_index] ? "DOWN" : "UP");
2684
2685 if (phy_data & 0x0004) {
2686 nesadapter->mac_link_down[mac_index] = 0;
2687 list_for_each_entry(nesvnic, &nesadapter->nesvnic_list[mac_index], list) {
2688 if (nesvnic->linkup == 0) {
2689 printk(PFX "The Link is now up for port %s, netdev %p.\n",
2690 nesvnic->netdev->name, nesvnic->netdev);
2691 if (netif_queue_stopped(nesvnic->netdev))
2692 netif_start_queue(nesvnic->netdev);
2693 nesvnic->linkup = 1;
2694 netif_carrier_on(nesvnic->netdev);
2695
2696 spin_lock(&nesvnic->port_ibevent_lock);
2697 if (nesvnic->of_device_registered) {
2698 if (nesdev->iw_status == 0) {
2699 nesdev->iw_status = 1;
2700 nes_port_ibevent(nesvnic);
2701 }
2702 }
2703 spin_unlock(&nesvnic->port_ibevent_lock);
2704 }
2705 }
2706
2707 } else {
2708 nesadapter->mac_link_down[mac_index] = 1;
2709 list_for_each_entry(nesvnic, &nesadapter->nesvnic_list[mac_index], list) {
2710 if (nesvnic->linkup == 1) {
2711 printk(PFX "The Link is now down for port %s, netdev %p.\n",
2712 nesvnic->netdev->name, nesvnic->netdev);
2713 if (!(netif_queue_stopped(nesvnic->netdev)))
2714 netif_stop_queue(nesvnic->netdev);
2715 nesvnic->linkup = 0;
2716 netif_carrier_off(nesvnic->netdev);
2717
2718 spin_lock(&nesvnic->port_ibevent_lock);
2719 if (nesvnic->of_device_registered) {
2720 if (nesdev->iw_status == 1) {
2721 nesdev->iw_status = 0;
2722 nes_port_ibevent(nesvnic);
2723 }
2724 }
2725 spin_unlock(&nesvnic->port_ibevent_lock);
2726 }
2727 }
2728 }
2729 if (nesdev->link_recheck++ < NES_LINK_RECHECK_MAX)
2730 schedule_delayed_work(&nesdev->work, NES_LINK_RECHECK_DELAY);
2731 else
2732 nesdev->link_recheck = 0;
2733
2734 spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
2735 }
2736
2737
2738 static void nes_nic_napi_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq)
2739 {
2740 struct nes_vnic *nesvnic = container_of(cq, struct nes_vnic, nic_cq);
2741
2742 napi_schedule(&nesvnic->napi);
2743 }
2744
2745
2746 /* The MAX_RQES_TO_PROCESS defines how many max read requests to complete before
2747 * getting out of nic_ce_handler
2748 */
2749 #define MAX_RQES_TO_PROCESS 384
2750
2751 /**
2752 * nes_nic_ce_handler
2753 */
2754 void nes_nic_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq)
2755 {
2756 u64 u64temp;
2757 dma_addr_t bus_address;
2758 struct nes_hw_nic *nesnic;
2759 struct nes_vnic *nesvnic = container_of(cq, struct nes_vnic, nic_cq);
2760 struct nes_adapter *nesadapter = nesdev->nesadapter;
2761 struct nes_hw_nic_rq_wqe *nic_rqe;
2762 struct nes_hw_nic_sq_wqe *nic_sqe;
2763 struct sk_buff *skb;
2764 struct sk_buff *rx_skb;
2765 struct nes_rskb_cb *cb;
2766 __le16 *wqe_fragment_length;
2767 u32 head;
2768 u32 cq_size;
2769 u32 rx_pkt_size;
2770 u32 cqe_count=0;
2771 u32 cqe_errv;
2772 u32 cqe_misc;
2773 u16 wqe_fragment_index = 1; /* first fragment (0) is used by copy buffer */
2774 u16 vlan_tag;
2775 u16 pkt_type;
2776 u16 rqes_processed = 0;
2777 u8 sq_cqes = 0;
2778
2779 head = cq->cq_head;
2780 cq_size = cq->cq_size;
2781 cq->cqes_pending = 1;
2782 do {
2783 if (le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX]) &
2784 NES_NIC_CQE_VALID) {
2785 nesnic = &nesvnic->nic;
2786 cqe_misc = le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX]);
2787 if (cqe_misc & NES_NIC_CQE_SQ) {
2788 sq_cqes++;
2789 wqe_fragment_index = 1;
2790 nic_sqe = &nesnic->sq_vbase[nesnic->sq_tail];
2791 skb = nesnic->tx_skb[nesnic->sq_tail];
2792 wqe_fragment_length = (__le16 *)&nic_sqe->wqe_words[NES_NIC_SQ_WQE_LENGTH_0_TAG_IDX];
2793 /* bump past the vlan tag */
2794 wqe_fragment_length++;
2795 if (le16_to_cpu(wqe_fragment_length[wqe_fragment_index]) != 0) {
2796 u64temp = (u64) le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX +
2797 wqe_fragment_index * 2]);
2798 u64temp += ((u64)le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX +
2799 wqe_fragment_index * 2])) << 32;
2800 bus_address = (dma_addr_t)u64temp;
2801 if (test_and_clear_bit(nesnic->sq_tail, nesnic->first_frag_overflow)) {
2802 pci_unmap_single(nesdev->pcidev,
2803 bus_address,
2804 le16_to_cpu(wqe_fragment_length[wqe_fragment_index++]),
2805 PCI_DMA_TODEVICE);
2806 }
2807 for (; wqe_fragment_index < 5; wqe_fragment_index++) {
2808 if (wqe_fragment_length[wqe_fragment_index]) {
2809 u64temp = le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX +
2810 wqe_fragment_index * 2]);
2811 u64temp += ((u64)le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX
2812 + wqe_fragment_index * 2])) <<32;
2813 bus_address = (dma_addr_t)u64temp;
2814 pci_unmap_page(nesdev->pcidev,
2815 bus_address,
2816 le16_to_cpu(wqe_fragment_length[wqe_fragment_index]),
2817 PCI_DMA_TODEVICE);
2818 } else
2819 break;
2820 }
2821 }
2822 if (skb)
2823 dev_kfree_skb_any(skb);
2824 nesnic->sq_tail++;
2825 nesnic->sq_tail &= nesnic->sq_size-1;
2826 if (sq_cqes > 128) {
2827 barrier();
2828 /* restart the queue if it had been stopped */
2829 if (netif_queue_stopped(nesvnic->netdev))
2830 netif_wake_queue(nesvnic->netdev);
2831 sq_cqes = 0;
2832 }
2833 } else {
2834 rqes_processed ++;
2835
2836 cq->rx_cqes_completed++;
2837 cq->rx_pkts_indicated++;
2838 rx_pkt_size = cqe_misc & 0x0000ffff;
2839 nic_rqe = &nesnic->rq_vbase[nesnic->rq_tail];
2840 /* Get the skb */
2841 rx_skb = nesnic->rx_skb[nesnic->rq_tail];
2842 nic_rqe = &nesnic->rq_vbase[nesvnic->nic.rq_tail];
2843 bus_address = (dma_addr_t)le32_to_cpu(nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX]);
2844 bus_address += ((u64)le32_to_cpu(nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX])) << 32;
2845 pci_unmap_single(nesdev->pcidev, bus_address,
2846 nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
2847 cb = (struct nes_rskb_cb *)&rx_skb->cb[0];
2848 cb->busaddr = 0;
2849 /* rx_skb->tail = rx_skb->data + rx_pkt_size; */
2850 /* rx_skb->len = rx_pkt_size; */
2851 rx_skb->len = 0; /* TODO: see if this is necessary */
2852 skb_put(rx_skb, rx_pkt_size);
2853 rx_skb->protocol = eth_type_trans(rx_skb, nesvnic->netdev);
2854 nesnic->rq_tail++;
2855 nesnic->rq_tail &= nesnic->rq_size - 1;
2856
2857 atomic_inc(&nesvnic->rx_skbs_needed);
2858 if (atomic_read(&nesvnic->rx_skbs_needed) > (nesvnic->nic.rq_size>>1)) {
2859 nes_write32(nesdev->regs+NES_CQE_ALLOC,
2860 cq->cq_number | (cqe_count << 16));
2861 /* nesadapter->tune_timer.cq_count += cqe_count; */
2862 nesdev->currcq_count += cqe_count;
2863 cqe_count = 0;
2864 nes_replenish_nic_rq(nesvnic);
2865 }
2866 pkt_type = (u16)(le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_TAG_PKT_TYPE_IDX]));
2867 cqe_errv = (cqe_misc & NES_NIC_CQE_ERRV_MASK) >> NES_NIC_CQE_ERRV_SHIFT;
2868 rx_skb->ip_summed = CHECKSUM_NONE;
2869
2870 if ((NES_PKT_TYPE_TCPV4_BITS == (pkt_type & NES_PKT_TYPE_TCPV4_MASK)) ||
2871 (NES_PKT_TYPE_UDPV4_BITS == (pkt_type & NES_PKT_TYPE_UDPV4_MASK))) {
2872 if ((cqe_errv &
2873 (NES_NIC_ERRV_BITS_IPV4_CSUM_ERR | NES_NIC_ERRV_BITS_TCPUDP_CSUM_ERR |
2874 NES_NIC_ERRV_BITS_IPH_ERR | NES_NIC_ERRV_BITS_WQE_OVERRUN)) == 0) {
2875 if (nesvnic->netdev->features & NETIF_F_RXCSUM)
2876 rx_skb->ip_summed = CHECKSUM_UNNECESSARY;
2877 } else
2878 nes_debug(NES_DBG_CQ, "%s: unsuccessfully checksummed TCP or UDP packet."
2879 " errv = 0x%X, pkt_type = 0x%X.\n",
2880 nesvnic->netdev->name, cqe_errv, pkt_type);
2881
2882 } else if ((pkt_type & NES_PKT_TYPE_IPV4_MASK) == NES_PKT_TYPE_IPV4_BITS) {
2883 if ((cqe_errv &
2884 (NES_NIC_ERRV_BITS_IPV4_CSUM_ERR | NES_NIC_ERRV_BITS_IPH_ERR |
2885 NES_NIC_ERRV_BITS_WQE_OVERRUN)) == 0) {
2886 if (nesvnic->netdev->features & NETIF_F_RXCSUM) {
2887 rx_skb->ip_summed = CHECKSUM_UNNECESSARY;
2888 /* nes_debug(NES_DBG_CQ, "%s: Reporting successfully checksummed IPv4 packet.\n",
2889 nesvnic->netdev->name); */
2890 }
2891 } else
2892 nes_debug(NES_DBG_CQ, "%s: unsuccessfully checksummed TCP or UDP packet."
2893 " errv = 0x%X, pkt_type = 0x%X.\n",
2894 nesvnic->netdev->name, cqe_errv, pkt_type);
2895 }
2896 /* nes_debug(NES_DBG_CQ, "pkt_type=%x, APBVT_MASK=%x\n",
2897 pkt_type, (pkt_type & NES_PKT_TYPE_APBVT_MASK)); */
2898
2899 if ((pkt_type & NES_PKT_TYPE_APBVT_MASK) == NES_PKT_TYPE_APBVT_BITS) {
2900 if (nes_cm_recv(rx_skb, nesvnic->netdev))
2901 rx_skb = NULL;
2902 }
2903 if (rx_skb == NULL)
2904 goto skip_rx_indicate0;
2905
2906
2907 if (cqe_misc & NES_NIC_CQE_TAG_VALID) {
2908 vlan_tag = (u16)(le32_to_cpu(
2909 cq->cq_vbase[head].cqe_words[NES_NIC_CQE_TAG_PKT_TYPE_IDX])
2910 >> 16);
2911 nes_debug(NES_DBG_CQ, "%s: Reporting stripped VLAN packet. Tag = 0x%04X\n",
2912 nesvnic->netdev->name, vlan_tag);
2913
2914 __vlan_hwaccel_put_tag(rx_skb, htons(ETH_P_8021Q), vlan_tag);
2915 }
2916 napi_gro_receive(&nesvnic->napi, rx_skb);
2917
2918 skip_rx_indicate0:
2919 ;
2920 /* nesvnic->netstats.rx_packets++; */
2921 /* nesvnic->netstats.rx_bytes += rx_pkt_size; */
2922 }
2923
2924 cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX] = 0;
2925 /* Accounting... */
2926 cqe_count++;
2927 if (++head >= cq_size)
2928 head = 0;
2929 if (cqe_count == 255) {
2930 /* Replenish Nic CQ */
2931 nes_write32(nesdev->regs+NES_CQE_ALLOC,
2932 cq->cq_number | (cqe_count << 16));
2933 /* nesdev->nesadapter->tune_timer.cq_count += cqe_count; */
2934 nesdev->currcq_count += cqe_count;
2935 cqe_count = 0;
2936 }
2937
2938 if (cq->rx_cqes_completed >= nesvnic->budget)
2939 break;
2940 } else {
2941 cq->cqes_pending = 0;
2942 break;
2943 }
2944
2945 } while (1);
2946
2947 if (sq_cqes) {
2948 barrier();
2949 /* restart the queue if it had been stopped */
2950 if (netif_queue_stopped(nesvnic->netdev))
2951 netif_wake_queue(nesvnic->netdev);
2952 }
2953 cq->cq_head = head;
2954 /* nes_debug(NES_DBG_CQ, "CQ%u Processed = %u cqes, new head = %u.\n",
2955 cq->cq_number, cqe_count, cq->cq_head); */
2956 cq->cqe_allocs_pending = cqe_count;
2957 if (unlikely(nesadapter->et_use_adaptive_rx_coalesce))
2958 {
2959 /* nesdev->nesadapter->tune_timer.cq_count += cqe_count; */
2960 nesdev->currcq_count += cqe_count;
2961 nes_nic_tune_timer(nesdev);
2962 }
2963 if (atomic_read(&nesvnic->rx_skbs_needed))
2964 nes_replenish_nic_rq(nesvnic);
2965 }
2966
2967
2968
2969 /**
2970 * nes_cqp_ce_handler
2971 */
2972 static void nes_cqp_ce_handler(struct nes_device *nesdev, struct nes_hw_cq *cq)
2973 {
2974 u64 u64temp;
2975 unsigned long flags;
2976 struct nes_hw_cqp *cqp = NULL;
2977 struct nes_cqp_request *cqp_request;
2978 struct nes_hw_cqp_wqe *cqp_wqe;
2979 u32 head;
2980 u32 cq_size;
2981 u32 cqe_count=0;
2982 u32 error_code;
2983 u32 opcode;
2984 u32 ctx_index;
2985 /* u32 counter; */
2986
2987 head = cq->cq_head;
2988 cq_size = cq->cq_size;
2989
2990 do {
2991 /* process the CQE */
2992 /* nes_debug(NES_DBG_CQP, "head=%u cqe_words=%08X\n", head,
2993 le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX])); */
2994
2995 opcode = le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX]);
2996 if (opcode & NES_CQE_VALID) {
2997 cqp = &nesdev->cqp;
2998
2999 error_code = le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_ERROR_CODE_IDX]);
3000 if (error_code) {
3001 nes_debug(NES_DBG_CQP, "Bad Completion code for opcode 0x%02X from CQP,"
3002 " Major/Minor codes = 0x%04X:%04X.\n",
3003 le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX])&0x3f,
3004 (u16)(error_code >> 16),
3005 (u16)error_code);
3006 }
3007
3008 u64temp = (((u64)(le32_to_cpu(cq->cq_vbase[head].
3009 cqe_words[NES_CQE_COMP_COMP_CTX_HIGH_IDX]))) << 32) |
3010 ((u64)(le32_to_cpu(cq->cq_vbase[head].
3011 cqe_words[NES_CQE_COMP_COMP_CTX_LOW_IDX])));
3012
3013 cqp_request = (struct nes_cqp_request *)(unsigned long)u64temp;
3014 if (cqp_request) {
3015 if (cqp_request->waiting) {
3016 /* nes_debug(NES_DBG_CQP, "%s: Waking up requestor\n"); */
3017 cqp_request->major_code = (u16)(error_code >> 16);
3018 cqp_request->minor_code = (u16)error_code;
3019 barrier();
3020 cqp_request->request_done = 1;
3021 wake_up(&cqp_request->waitq);
3022 nes_put_cqp_request(nesdev, cqp_request);
3023 } else {
3024 if (cqp_request->callback)
3025 cqp_request->cqp_callback(nesdev, cqp_request);
3026 nes_free_cqp_request(nesdev, cqp_request);
3027 }
3028 } else {
3029 wake_up(&nesdev->cqp.waitq);
3030 }
3031
3032 cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX] = 0;
3033 nes_write32(nesdev->regs + NES_CQE_ALLOC, cq->cq_number | (1 << 16));
3034 if (++cqp->sq_tail >= cqp->sq_size)
3035 cqp->sq_tail = 0;
3036
3037 /* Accounting... */
3038 cqe_count++;
3039 if (++head >= cq_size)
3040 head = 0;
3041 } else {
3042 break;
3043 }
3044 } while (1);
3045 cq->cq_head = head;
3046
3047 spin_lock_irqsave(&nesdev->cqp.lock, flags);
3048 while ((!list_empty(&nesdev->cqp_pending_reqs)) &&
3049 ((((nesdev->cqp.sq_tail+nesdev->cqp.sq_size)-nesdev->cqp.sq_head) &
3050 (nesdev->cqp.sq_size - 1)) != 1)) {
3051 cqp_request = list_entry(nesdev->cqp_pending_reqs.next,
3052 struct nes_cqp_request, list);
3053 list_del_init(&cqp_request->list);
3054 head = nesdev->cqp.sq_head++;
3055 nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
3056 cqp_wqe = &nesdev->cqp.sq_vbase[head];
3057 memcpy(cqp_wqe, &cqp_request->cqp_wqe, sizeof(*cqp_wqe));
3058 barrier();
3059
3060 opcode = cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX];
3061 if ((opcode & NES_CQP_OPCODE_MASK) == NES_CQP_DOWNLOAD_SEGMENT)
3062 ctx_index = NES_CQP_WQE_DL_COMP_CTX_LOW_IDX;
3063 else
3064 ctx_index = NES_CQP_WQE_COMP_CTX_LOW_IDX;
3065 cqp_wqe->wqe_words[ctx_index] =
3066 cpu_to_le32((u32)((unsigned long)cqp_request));
3067 cqp_wqe->wqe_words[ctx_index + 1] =
3068 cpu_to_le32((u32)(upper_32_bits((unsigned long)cqp_request)));
3069 nes_debug(NES_DBG_CQP, "CQP request %p (opcode 0x%02X) put on CQPs SQ wqe%u.\n",
3070 cqp_request, le32_to_cpu(cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX])&0x3f, head);
3071 /* Ring doorbell (1 WQEs) */
3072 barrier();
3073 nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x01800000 | nesdev->cqp.qp_id);
3074 }
3075 spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
3076
3077 /* Arm the CCQ */
3078 nes_write32(nesdev->regs+NES_CQE_ALLOC, NES_CQE_ALLOC_NOTIFY_NEXT |
3079 cq->cq_number);
3080 nes_read32(nesdev->regs+NES_CQE_ALLOC);
3081 }
3082
3083 static u8 *locate_mpa(u8 *pkt, u32 aeq_info)
3084 {
3085 if (aeq_info & NES_AEQE_Q2_DATA_ETHERNET) {
3086 /* skip over ethernet header */
3087 pkt += ETH_HLEN;
3088
3089 /* Skip over IP and TCP headers */
3090 pkt += 4 * (pkt[0] & 0x0f);
3091 pkt += 4 * ((pkt[12] >> 4) & 0x0f);
3092 }
3093 return pkt;
3094 }
3095
3096 /* Determine if incoming error pkt is rdma layer */
3097 static u32 iwarp_opcode(struct nes_qp *nesqp, u32 aeq_info)
3098 {
3099 u8 *pkt;
3100 u16 *mpa;
3101 u32 opcode = 0xffffffff;
3102
3103 if (aeq_info & NES_AEQE_Q2_DATA_WRITTEN) {
3104 pkt = nesqp->hwqp.q2_vbase + BAD_FRAME_OFFSET;
3105 mpa = (u16 *)locate_mpa(pkt, aeq_info);
3106 opcode = be16_to_cpu(mpa[1]) & 0xf;
3107 }
3108
3109 return opcode;
3110 }
3111
3112 /* Build iWARP terminate header */
3113 static int nes_bld_terminate_hdr(struct nes_qp *nesqp, u16 async_event_id, u32 aeq_info)
3114 {
3115 u8 *pkt = nesqp->hwqp.q2_vbase + BAD_FRAME_OFFSET;
3116 u16 ddp_seg_len;
3117 int copy_len = 0;
3118 u8 is_tagged = 0;
3119 u8 flush_code = 0;
3120 struct nes_terminate_hdr *termhdr;
3121
3122 termhdr = (struct nes_terminate_hdr *)nesqp->hwqp.q2_vbase;
3123 memset(termhdr, 0, 64);
3124
3125 if (aeq_info & NES_AEQE_Q2_DATA_WRITTEN) {
3126
3127 /* Use data from offending packet to fill in ddp & rdma hdrs */
3128 pkt = locate_mpa(pkt, aeq_info);
3129 ddp_seg_len = be16_to_cpu(*(u16 *)pkt);
3130 if (ddp_seg_len) {
3131 copy_len = 2;
3132 termhdr->hdrct = DDP_LEN_FLAG;
3133 if (pkt[2] & 0x80) {
3134 is_tagged = 1;
3135 if (ddp_seg_len >= TERM_DDP_LEN_TAGGED) {
3136 copy_len += TERM_DDP_LEN_TAGGED;
3137 termhdr->hdrct |= DDP_HDR_FLAG;
3138 }
3139 } else {
3140 if (ddp_seg_len >= TERM_DDP_LEN_UNTAGGED) {
3141 copy_len += TERM_DDP_LEN_UNTAGGED;
3142 termhdr->hdrct |= DDP_HDR_FLAG;
3143 }
3144
3145 if (ddp_seg_len >= (TERM_DDP_LEN_UNTAGGED + TERM_RDMA_LEN)) {
3146 if ((pkt[3] & RDMA_OPCODE_MASK) == RDMA_READ_REQ_OPCODE) {
3147 copy_len += TERM_RDMA_LEN;
3148 termhdr->hdrct |= RDMA_HDR_FLAG;
3149 }
3150 }
3151 }
3152 }
3153 }
3154
3155 switch (async_event_id) {
3156 case NES_AEQE_AEID_AMP_UNALLOCATED_STAG:
3157 switch (iwarp_opcode(nesqp, aeq_info)) {
3158 case IWARP_OPCODE_WRITE:
3159 flush_code = IB_WC_LOC_PROT_ERR;
3160 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_TAGGED_BUFFER;
3161 termhdr->error_code = DDP_TAGGED_INV_STAG;
3162 break;
3163 default:
3164 flush_code = IB_WC_REM_ACCESS_ERR;
3165 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3166 termhdr->error_code = RDMAP_INV_STAG;
3167 }
3168 break;
3169 case NES_AEQE_AEID_AMP_INVALID_STAG:
3170 flush_code = IB_WC_REM_ACCESS_ERR;
3171 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3172 termhdr->error_code = RDMAP_INV_STAG;
3173 break;
3174 case NES_AEQE_AEID_AMP_BAD_QP:
3175 flush_code = IB_WC_LOC_QP_OP_ERR;
3176 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3177 termhdr->error_code = DDP_UNTAGGED_INV_QN;
3178 break;
3179 case NES_AEQE_AEID_AMP_BAD_STAG_KEY:
3180 case NES_AEQE_AEID_AMP_BAD_STAG_INDEX:
3181 switch (iwarp_opcode(nesqp, aeq_info)) {
3182 case IWARP_OPCODE_SEND_INV:
3183 case IWARP_OPCODE_SEND_SE_INV:
3184 flush_code = IB_WC_REM_OP_ERR;
3185 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_OP;
3186 termhdr->error_code = RDMAP_CANT_INV_STAG;
3187 break;
3188 default:
3189 flush_code = IB_WC_REM_ACCESS_ERR;
3190 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3191 termhdr->error_code = RDMAP_INV_STAG;
3192 }
3193 break;
3194 case NES_AEQE_AEID_AMP_BOUNDS_VIOLATION:
3195 if (aeq_info & (NES_AEQE_Q2_DATA_ETHERNET | NES_AEQE_Q2_DATA_MPA)) {
3196 flush_code = IB_WC_LOC_PROT_ERR;
3197 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_TAGGED_BUFFER;
3198 termhdr->error_code = DDP_TAGGED_BOUNDS;
3199 } else {
3200 flush_code = IB_WC_REM_ACCESS_ERR;
3201 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3202 termhdr->error_code = RDMAP_INV_BOUNDS;
3203 }
3204 break;
3205 case NES_AEQE_AEID_AMP_RIGHTS_VIOLATION:
3206 case NES_AEQE_AEID_AMP_INVALIDATE_NO_REMOTE_ACCESS_RIGHTS:
3207 case NES_AEQE_AEID_PRIV_OPERATION_DENIED:
3208 flush_code = IB_WC_REM_ACCESS_ERR;
3209 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3210 termhdr->error_code = RDMAP_ACCESS;
3211 break;
3212 case NES_AEQE_AEID_AMP_TO_WRAP:
3213 flush_code = IB_WC_REM_ACCESS_ERR;
3214 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3215 termhdr->error_code = RDMAP_TO_WRAP;
3216 break;
3217 case NES_AEQE_AEID_AMP_BAD_PD:
3218 switch (iwarp_opcode(nesqp, aeq_info)) {
3219 case IWARP_OPCODE_WRITE:
3220 flush_code = IB_WC_LOC_PROT_ERR;
3221 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_TAGGED_BUFFER;
3222 termhdr->error_code = DDP_TAGGED_UNASSOC_STAG;
3223 break;
3224 case IWARP_OPCODE_SEND_INV:
3225 case IWARP_OPCODE_SEND_SE_INV:
3226 flush_code = IB_WC_REM_ACCESS_ERR;
3227 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3228 termhdr->error_code = RDMAP_CANT_INV_STAG;
3229 break;
3230 default:
3231 flush_code = IB_WC_REM_ACCESS_ERR;
3232 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3233 termhdr->error_code = RDMAP_UNASSOC_STAG;
3234 }
3235 break;
3236 case NES_AEQE_AEID_LLP_RECEIVED_MARKER_AND_LENGTH_FIELDS_DONT_MATCH:
3237 flush_code = IB_WC_LOC_LEN_ERR;
3238 termhdr->layer_etype = (LAYER_MPA << 4) | DDP_LLP;
3239 termhdr->error_code = MPA_MARKER;
3240 break;
3241 case NES_AEQE_AEID_LLP_RECEIVED_MPA_CRC_ERROR:
3242 flush_code = IB_WC_GENERAL_ERR;
3243 termhdr->layer_etype = (LAYER_MPA << 4) | DDP_LLP;
3244 termhdr->error_code = MPA_CRC;
3245 break;
3246 case NES_AEQE_AEID_LLP_SEGMENT_TOO_LARGE:
3247 case NES_AEQE_AEID_LLP_SEGMENT_TOO_SMALL:
3248 flush_code = IB_WC_LOC_LEN_ERR;
3249 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_CATASTROPHIC;
3250 termhdr->error_code = DDP_CATASTROPHIC_LOCAL;
3251 break;
3252 case NES_AEQE_AEID_DDP_LCE_LOCAL_CATASTROPHIC:
3253 case NES_AEQE_AEID_DDP_NO_L_BIT:
3254 flush_code = IB_WC_FATAL_ERR;
3255 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_CATASTROPHIC;
3256 termhdr->error_code = DDP_CATASTROPHIC_LOCAL;
3257 break;
3258 case NES_AEQE_AEID_DDP_INVALID_MSN_GAP_IN_MSN:
3259 case NES_AEQE_AEID_DDP_INVALID_MSN_RANGE_IS_NOT_VALID:
3260 flush_code = IB_WC_GENERAL_ERR;
3261 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3262 termhdr->error_code = DDP_UNTAGGED_INV_MSN_RANGE;
3263 break;
3264 case NES_AEQE_AEID_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER:
3265 flush_code = IB_WC_LOC_LEN_ERR;
3266 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3267 termhdr->error_code = DDP_UNTAGGED_INV_TOO_LONG;
3268 break;
3269 case NES_AEQE_AEID_DDP_UBE_INVALID_DDP_VERSION:
3270 flush_code = IB_WC_GENERAL_ERR;
3271 if (is_tagged) {
3272 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_TAGGED_BUFFER;
3273 termhdr->error_code = DDP_TAGGED_INV_DDP_VER;
3274 } else {
3275 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3276 termhdr->error_code = DDP_UNTAGGED_INV_DDP_VER;
3277 }
3278 break;
3279 case NES_AEQE_AEID_DDP_UBE_INVALID_MO:
3280 flush_code = IB_WC_GENERAL_ERR;
3281 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3282 termhdr->error_code = DDP_UNTAGGED_INV_MO;
3283 break;
3284 case NES_AEQE_AEID_DDP_UBE_INVALID_MSN_NO_BUFFER_AVAILABLE:
3285 flush_code = IB_WC_REM_OP_ERR;
3286 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3287 termhdr->error_code = DDP_UNTAGGED_INV_MSN_NO_BUF;
3288 break;
3289 case NES_AEQE_AEID_DDP_UBE_INVALID_QN:
3290 flush_code = IB_WC_GENERAL_ERR;
3291 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3292 termhdr->error_code = DDP_UNTAGGED_INV_QN;
3293 break;
3294 case NES_AEQE_AEID_RDMAP_ROE_INVALID_RDMAP_VERSION:
3295 flush_code = IB_WC_GENERAL_ERR;
3296 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_OP;
3297 termhdr->error_code = RDMAP_INV_RDMAP_VER;
3298 break;
3299 case NES_AEQE_AEID_RDMAP_ROE_UNEXPECTED_OPCODE:
3300 flush_code = IB_WC_LOC_QP_OP_ERR;
3301 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_OP;
3302 termhdr->error_code = RDMAP_UNEXPECTED_OP;
3303 break;
3304 default:
3305 flush_code = IB_WC_FATAL_ERR;
3306 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_OP;
3307 termhdr->error_code = RDMAP_UNSPECIFIED;
3308 break;
3309 }
3310
3311 if (copy_len)
3312 memcpy(termhdr + 1, pkt, copy_len);
3313
3314 if ((flush_code) && ((NES_AEQE_INBOUND_RDMA & aeq_info) == 0)) {
3315 if (aeq_info & NES_AEQE_SQ)
3316 nesqp->term_sq_flush_code = flush_code;
3317 else
3318 nesqp->term_rq_flush_code = flush_code;
3319 }
3320
3321 return sizeof(struct nes_terminate_hdr) + copy_len;
3322 }
3323
3324 static void nes_terminate_connection(struct nes_device *nesdev, struct nes_qp *nesqp,
3325 struct nes_hw_aeqe *aeqe, enum ib_event_type eventtype)
3326 {
3327 u64 context;
3328 unsigned long flags;
3329 u32 aeq_info;
3330 u16 async_event_id;
3331 u8 tcp_state;
3332 u8 iwarp_state;
3333 u32 termlen = 0;
3334 u32 mod_qp_flags = NES_CQP_QP_IWARP_STATE_TERMINATE |
3335 NES_CQP_QP_TERM_DONT_SEND_FIN;
3336 struct nes_adapter *nesadapter = nesdev->nesadapter;
3337
3338 if (nesqp->term_flags & NES_TERM_SENT)
3339 return; /* Sanity check */
3340
3341 aeq_info = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
3342 tcp_state = (aeq_info & NES_AEQE_TCP_STATE_MASK) >> NES_AEQE_TCP_STATE_SHIFT;
3343 iwarp_state = (aeq_info & NES_AEQE_IWARP_STATE_MASK) >> NES_AEQE_IWARP_STATE_SHIFT;
3344 async_event_id = (u16)aeq_info;
3345
3346 context = (unsigned long)nesadapter->qp_table[le32_to_cpu(
3347 aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]) - NES_FIRST_QPN];
3348 if (!context) {
3349 WARN_ON(!context);
3350 return;
3351 }
3352
3353 nesqp = (struct nes_qp *)(unsigned long)context;
3354 spin_lock_irqsave(&nesqp->lock, flags);
3355 nesqp->hw_iwarp_state = iwarp_state;
3356 nesqp->hw_tcp_state = tcp_state;
3357 nesqp->last_aeq = async_event_id;
3358 nesqp->terminate_eventtype = eventtype;
3359 spin_unlock_irqrestore(&nesqp->lock, flags);
3360
3361 if (nesadapter->send_term_ok)
3362 termlen = nes_bld_terminate_hdr(nesqp, async_event_id, aeq_info);
3363 else
3364 mod_qp_flags |= NES_CQP_QP_TERM_DONT_SEND_TERM_MSG;
3365
3366 if (!nesdev->iw_status) {
3367 nesqp->term_flags = NES_TERM_DONE;
3368 nes_hw_modify_qp(nesdev, nesqp, NES_CQP_QP_IWARP_STATE_ERROR, 0, 0);
3369 nes_cm_disconn(nesqp);
3370 } else {
3371 nes_terminate_start_timer(nesqp);
3372 nesqp->term_flags |= NES_TERM_SENT;
3373 nes_hw_modify_qp(nesdev, nesqp, mod_qp_flags, termlen, 0);
3374 }
3375 }
3376
3377 static void nes_terminate_send_fin(struct nes_device *nesdev,
3378 struct nes_qp *nesqp, struct nes_hw_aeqe *aeqe)
3379 {
3380 u32 aeq_info;
3381 u16 async_event_id;
3382 u8 tcp_state;
3383 u8 iwarp_state;
3384 unsigned long flags;
3385
3386 aeq_info = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
3387 tcp_state = (aeq_info & NES_AEQE_TCP_STATE_MASK) >> NES_AEQE_TCP_STATE_SHIFT;
3388 iwarp_state = (aeq_info & NES_AEQE_IWARP_STATE_MASK) >> NES_AEQE_IWARP_STATE_SHIFT;
3389 async_event_id = (u16)aeq_info;
3390
3391 spin_lock_irqsave(&nesqp->lock, flags);
3392 nesqp->hw_iwarp_state = iwarp_state;
3393 nesqp->hw_tcp_state = tcp_state;
3394 nesqp->last_aeq = async_event_id;
3395 spin_unlock_irqrestore(&nesqp->lock, flags);
3396
3397 /* Send the fin only */
3398 nes_hw_modify_qp(nesdev, nesqp, NES_CQP_QP_IWARP_STATE_TERMINATE |
3399 NES_CQP_QP_TERM_DONT_SEND_TERM_MSG, 0, 0);
3400 }
3401
3402 /* Cleanup after a terminate sent or received */
3403 static void nes_terminate_done(struct nes_qp *nesqp, int timeout_occurred)
3404 {
3405 u32 next_iwarp_state = NES_CQP_QP_IWARP_STATE_ERROR;
3406 unsigned long flags;
3407 struct nes_vnic *nesvnic = to_nesvnic(nesqp->ibqp.device);
3408 struct nes_device *nesdev = nesvnic->nesdev;
3409 u8 first_time = 0;
3410
3411 spin_lock_irqsave(&nesqp->lock, flags);
3412 if (nesqp->hte_added) {
3413 nesqp->hte_added = 0;
3414 next_iwarp_state |= NES_CQP_QP_DEL_HTE;
3415 }
3416
3417 first_time = (nesqp->term_flags & NES_TERM_DONE) == 0;
3418 nesqp->term_flags |= NES_TERM_DONE;
3419 spin_unlock_irqrestore(&nesqp->lock, flags);
3420
3421 /* Make sure we go through this only once */
3422 if (first_time) {
3423 if (timeout_occurred == 0)
3424 del_timer(&nesqp->terminate_timer);
3425 else
3426 next_iwarp_state |= NES_CQP_QP_RESET;
3427
3428 nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0, 0);
3429 nes_cm_disconn(nesqp);
3430 }
3431 }
3432
3433 static void nes_terminate_received(struct nes_device *nesdev,
3434 struct nes_qp *nesqp, struct nes_hw_aeqe *aeqe)
3435 {
3436 u32 aeq_info;
3437 u8 *pkt;
3438 u32 *mpa;
3439 u8 ddp_ctl;
3440 u8 rdma_ctl;
3441 u16 aeq_id = 0;
3442
3443 aeq_info = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
3444 if (aeq_info & NES_AEQE_Q2_DATA_WRITTEN) {
3445 /* Terminate is not a performance path so the silicon */
3446 /* did not validate the frame - do it now */
3447 pkt = nesqp->hwqp.q2_vbase + BAD_FRAME_OFFSET;
3448 mpa = (u32 *)locate_mpa(pkt, aeq_info);
3449 ddp_ctl = (be32_to_cpu(mpa[0]) >> 8) & 0xff;
3450 rdma_ctl = be32_to_cpu(mpa[0]) & 0xff;
3451 if ((ddp_ctl & 0xc0) != 0x40)
3452 aeq_id = NES_AEQE_AEID_DDP_LCE_LOCAL_CATASTROPHIC;
3453 else if ((ddp_ctl & 0x03) != 1)
3454 aeq_id = NES_AEQE_AEID_DDP_UBE_INVALID_DDP_VERSION;
3455 else if (be32_to_cpu(mpa[2]) != 2)
3456 aeq_id = NES_AEQE_AEID_DDP_UBE_INVALID_QN;
3457 else if (be32_to_cpu(mpa[3]) != 1)
3458 aeq_id = NES_AEQE_AEID_DDP_INVALID_MSN_GAP_IN_MSN;
3459 else if (be32_to_cpu(mpa[4]) != 0)
3460 aeq_id = NES_AEQE_AEID_DDP_UBE_INVALID_MO;
3461 else if ((rdma_ctl & 0xc0) != 0x40)
3462 aeq_id = NES_AEQE_AEID_RDMAP_ROE_INVALID_RDMAP_VERSION;
3463
3464 if (aeq_id) {
3465 /* Bad terminate recvd - send back a terminate */
3466 aeq_info = (aeq_info & 0xffff0000) | aeq_id;
3467 aeqe->aeqe_words[NES_AEQE_MISC_IDX] = cpu_to_le32(aeq_info);
3468 nes_terminate_connection(nesdev, nesqp, aeqe, IB_EVENT_QP_FATAL);
3469 return;
3470 }
3471 }
3472
3473 nesqp->term_flags |= NES_TERM_RCVD;
3474 nesqp->terminate_eventtype = IB_EVENT_QP_FATAL;
3475 nes_terminate_start_timer(nesqp);
3476 nes_terminate_send_fin(nesdev, nesqp, aeqe);
3477 }
3478
3479 /* Timeout routine in case terminate fails to complete */
3480 void nes_terminate_timeout(unsigned long context)
3481 {
3482 struct nes_qp *nesqp = (struct nes_qp *)(unsigned long)context;
3483
3484 nes_terminate_done(nesqp, 1);
3485 }
3486
3487 /* Set a timer in case hw cannot complete the terminate sequence */
3488 static void nes_terminate_start_timer(struct nes_qp *nesqp)
3489 {
3490 mod_timer(&nesqp->terminate_timer, (jiffies + HZ));
3491 }
3492
3493 /**
3494 * nes_process_iwarp_aeqe
3495 */
3496 static void nes_process_iwarp_aeqe(struct nes_device *nesdev,
3497 struct nes_hw_aeqe *aeqe)
3498 {
3499 u64 context;
3500 unsigned long flags;
3501 struct nes_qp *nesqp;
3502 struct nes_hw_cq *hw_cq;
3503 struct nes_cq *nescq;
3504 int resource_allocated;
3505 struct nes_adapter *nesadapter = nesdev->nesadapter;
3506 u32 aeq_info;
3507 u32 next_iwarp_state = 0;
3508 u32 aeqe_cq_id;
3509 u16 async_event_id;
3510 u8 tcp_state;
3511 u8 iwarp_state;
3512 struct ib_event ibevent;
3513
3514 nes_debug(NES_DBG_AEQ, "\n");
3515 aeq_info = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
3516 if ((NES_AEQE_INBOUND_RDMA & aeq_info) || (!(NES_AEQE_QP & aeq_info))) {
3517 context = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_LOW_IDX]);
3518 context += ((u64)le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_HIGH_IDX])) << 32;
3519 } else {
3520 context = (unsigned long)nesadapter->qp_table[le32_to_cpu(
3521 aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]) - NES_FIRST_QPN];
3522 BUG_ON(!context);
3523 }
3524
3525 /* context is nesqp unless async_event_id == CQ ERROR */
3526 nesqp = (struct nes_qp *)(unsigned long)context;
3527 async_event_id = (u16)aeq_info;
3528 tcp_state = (aeq_info & NES_AEQE_TCP_STATE_MASK) >> NES_AEQE_TCP_STATE_SHIFT;
3529 iwarp_state = (aeq_info & NES_AEQE_IWARP_STATE_MASK) >> NES_AEQE_IWARP_STATE_SHIFT;
3530 nes_debug(NES_DBG_AEQ, "aeid = 0x%04X, qp-cq id = %d, aeqe = %p,"
3531 " Tcp state = %s, iWARP state = %s\n",
3532 async_event_id,
3533 le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]), aeqe,
3534 nes_tcp_state_str[tcp_state], nes_iwarp_state_str[iwarp_state]);
3535
3536 aeqe_cq_id = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]);
3537 if (aeq_info & NES_AEQE_QP) {
3538 if (!nes_is_resource_allocated(nesadapter,
3539 nesadapter->allocated_qps,
3540 aeqe_cq_id))
3541 return;
3542 }
3543
3544 switch (async_event_id) {
3545 case NES_AEQE_AEID_LLP_FIN_RECEIVED:
3546 if (nesqp->term_flags)
3547 return; /* Ignore it, wait for close complete */
3548
3549 if (atomic_inc_return(&nesqp->close_timer_started) == 1) {
3550 if ((tcp_state == NES_AEQE_TCP_STATE_CLOSE_WAIT) &&
3551 (nesqp->ibqp_state == IB_QPS_RTS)) {
3552 spin_lock_irqsave(&nesqp->lock, flags);
3553 nesqp->hw_iwarp_state = iwarp_state;
3554 nesqp->hw_tcp_state = tcp_state;
3555 nesqp->last_aeq = async_event_id;
3556 next_iwarp_state = NES_CQP_QP_IWARP_STATE_CLOSING;
3557 nesqp->hw_iwarp_state = NES_AEQE_IWARP_STATE_CLOSING;
3558 spin_unlock_irqrestore(&nesqp->lock, flags);
3559 nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0, 0);
3560 nes_cm_disconn(nesqp);
3561 }
3562 nesqp->cm_id->add_ref(nesqp->cm_id);
3563 schedule_nes_timer(nesqp->cm_node, (struct sk_buff *)nesqp,
3564 NES_TIMER_TYPE_CLOSE, 1, 0);
3565 nes_debug(NES_DBG_AEQ, "QP%u Not decrementing QP refcount (%d),"
3566 " need ae to finish up, original_last_aeq = 0x%04X."
3567 " last_aeq = 0x%04X, scheduling timer. TCP state = %d\n",
3568 nesqp->hwqp.qp_id, atomic_read(&nesqp->refcount),
3569 async_event_id, nesqp->last_aeq, tcp_state);
3570 }
3571 break;
3572 case NES_AEQE_AEID_LLP_CLOSE_COMPLETE:
3573 spin_lock_irqsave(&nesqp->lock, flags);
3574 nesqp->hw_iwarp_state = iwarp_state;
3575 nesqp->hw_tcp_state = tcp_state;
3576 nesqp->last_aeq = async_event_id;
3577 spin_unlock_irqrestore(&nesqp->lock, flags);
3578 nes_cm_disconn(nesqp);
3579 break;
3580
3581 case NES_AEQE_AEID_RESET_SENT:
3582 tcp_state = NES_AEQE_TCP_STATE_CLOSED;
3583 spin_lock_irqsave(&nesqp->lock, flags);
3584 nesqp->hw_iwarp_state = iwarp_state;
3585 nesqp->hw_tcp_state = tcp_state;
3586 nesqp->last_aeq = async_event_id;
3587 nesqp->hte_added = 0;
3588 spin_unlock_irqrestore(&nesqp->lock, flags);
3589 next_iwarp_state = NES_CQP_QP_IWARP_STATE_ERROR | NES_CQP_QP_DEL_HTE;
3590 nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0, 0);
3591 nes_cm_disconn(nesqp);
3592 break;
3593
3594 case NES_AEQE_AEID_LLP_CONNECTION_RESET:
3595 if (atomic_read(&nesqp->close_timer_started))
3596 return;
3597 spin_lock_irqsave(&nesqp->lock, flags);
3598 nesqp->hw_iwarp_state = iwarp_state;
3599 nesqp->hw_tcp_state = tcp_state;
3600 nesqp->last_aeq = async_event_id;
3601 spin_unlock_irqrestore(&nesqp->lock, flags);
3602 nes_cm_disconn(nesqp);
3603 break;
3604
3605 case NES_AEQE_AEID_TERMINATE_SENT:
3606 nes_terminate_send_fin(nesdev, nesqp, aeqe);
3607 break;
3608
3609 case NES_AEQE_AEID_LLP_TERMINATE_RECEIVED:
3610 nes_terminate_received(nesdev, nesqp, aeqe);
3611 break;
3612
3613 case NES_AEQE_AEID_AMP_BAD_STAG_KEY:
3614 case NES_AEQE_AEID_AMP_BAD_STAG_INDEX:
3615 case NES_AEQE_AEID_AMP_UNALLOCATED_STAG:
3616 case NES_AEQE_AEID_AMP_INVALID_STAG:
3617 case NES_AEQE_AEID_AMP_RIGHTS_VIOLATION:
3618 case NES_AEQE_AEID_AMP_INVALIDATE_NO_REMOTE_ACCESS_RIGHTS:
3619 case NES_AEQE_AEID_PRIV_OPERATION_DENIED:
3620 case NES_AEQE_AEID_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER:
3621 case NES_AEQE_AEID_AMP_BOUNDS_VIOLATION:
3622 case NES_AEQE_AEID_AMP_TO_WRAP:
3623 printk(KERN_ERR PFX "QP[%u] async_event_id=0x%04X IB_EVENT_QP_ACCESS_ERR\n",
3624 nesqp->hwqp.qp_id, async_event_id);
3625 nes_terminate_connection(nesdev, nesqp, aeqe, IB_EVENT_QP_ACCESS_ERR);
3626 break;
3627
3628 case NES_AEQE_AEID_LLP_SEGMENT_TOO_LARGE:
3629 case NES_AEQE_AEID_LLP_SEGMENT_TOO_SMALL:
3630 case NES_AEQE_AEID_DDP_UBE_INVALID_MO:
3631 case NES_AEQE_AEID_DDP_UBE_INVALID_QN:
3632 if (iwarp_opcode(nesqp, aeq_info) > IWARP_OPCODE_TERM) {
3633 aeq_info &= 0xffff0000;
3634 aeq_info |= NES_AEQE_AEID_RDMAP_ROE_UNEXPECTED_OPCODE;
3635 aeqe->aeqe_words[NES_AEQE_MISC_IDX] = cpu_to_le32(aeq_info);
3636 }
3637
3638 case NES_AEQE_AEID_RDMAP_ROE_BAD_LLP_CLOSE:
3639 case NES_AEQE_AEID_LLP_TOO_MANY_RETRIES:
3640 case NES_AEQE_AEID_DDP_UBE_INVALID_MSN_NO_BUFFER_AVAILABLE:
3641 case NES_AEQE_AEID_LLP_RECEIVED_MPA_CRC_ERROR:
3642 case NES_AEQE_AEID_AMP_BAD_QP:
3643 case NES_AEQE_AEID_LLP_RECEIVED_MARKER_AND_LENGTH_FIELDS_DONT_MATCH:
3644 case NES_AEQE_AEID_DDP_LCE_LOCAL_CATASTROPHIC:
3645 case NES_AEQE_AEID_DDP_NO_L_BIT:
3646 case NES_AEQE_AEID_DDP_INVALID_MSN_GAP_IN_MSN:
3647 case NES_AEQE_AEID_DDP_INVALID_MSN_RANGE_IS_NOT_VALID:
3648 case NES_AEQE_AEID_DDP_UBE_INVALID_DDP_VERSION:
3649 case NES_AEQE_AEID_RDMAP_ROE_INVALID_RDMAP_VERSION:
3650 case NES_AEQE_AEID_RDMAP_ROE_UNEXPECTED_OPCODE:
3651 case NES_AEQE_AEID_AMP_BAD_PD:
3652 case NES_AEQE_AEID_AMP_FASTREG_SHARED:
3653 case NES_AEQE_AEID_AMP_FASTREG_VALID_STAG:
3654 case NES_AEQE_AEID_AMP_FASTREG_MW_STAG:
3655 case NES_AEQE_AEID_AMP_FASTREG_INVALID_RIGHTS:
3656 case NES_AEQE_AEID_AMP_FASTREG_PBL_TABLE_OVERFLOW:
3657 case NES_AEQE_AEID_AMP_FASTREG_INVALID_LENGTH:
3658 case NES_AEQE_AEID_AMP_INVALIDATE_SHARED:
3659 case NES_AEQE_AEID_AMP_INVALIDATE_MR_WITH_BOUND_WINDOWS:
3660 case NES_AEQE_AEID_AMP_MWBIND_VALID_STAG:
3661 case NES_AEQE_AEID_AMP_MWBIND_OF_MR_STAG:
3662 case NES_AEQE_AEID_AMP_MWBIND_TO_ZERO_BASED_STAG:
3663 case NES_AEQE_AEID_AMP_MWBIND_TO_MW_STAG:
3664 case NES_AEQE_AEID_AMP_MWBIND_INVALID_RIGHTS:
3665 case NES_AEQE_AEID_AMP_MWBIND_INVALID_BOUNDS:
3666 case NES_AEQE_AEID_AMP_MWBIND_TO_INVALID_PARENT:
3667 case NES_AEQE_AEID_AMP_MWBIND_BIND_DISABLED:
3668 case NES_AEQE_AEID_BAD_CLOSE:
3669 case NES_AEQE_AEID_RDMA_READ_WHILE_ORD_ZERO:
3670 case NES_AEQE_AEID_STAG_ZERO_INVALID:
3671 case NES_AEQE_AEID_ROE_INVALID_RDMA_READ_REQUEST:
3672 case NES_AEQE_AEID_ROE_INVALID_RDMA_WRITE_OR_READ_RESP:
3673 printk(KERN_ERR PFX "QP[%u] async_event_id=0x%04X IB_EVENT_QP_FATAL\n",
3674 nesqp->hwqp.qp_id, async_event_id);
3675 print_ip(nesqp->cm_node);
3676 if (!atomic_read(&nesqp->close_timer_started))
3677 nes_terminate_connection(nesdev, nesqp, aeqe, IB_EVENT_QP_FATAL);
3678 break;
3679
3680 case NES_AEQE_AEID_CQ_OPERATION_ERROR:
3681 context <<= 1;
3682 nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_CQ_OPERATION_ERROR event on CQ%u, %p\n",
3683 le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]), (void *)(unsigned long)context);
3684 resource_allocated = nes_is_resource_allocated(nesadapter, nesadapter->allocated_cqs,
3685 le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]));
3686 if (resource_allocated) {
3687 printk(KERN_ERR PFX "%s: Processing an NES_AEQE_AEID_CQ_OPERATION_ERROR event on CQ%u\n",
3688 __func__, le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]));
3689 hw_cq = (struct nes_hw_cq *)(unsigned long)context;
3690 if (hw_cq) {
3691 nescq = container_of(hw_cq, struct nes_cq, hw_cq);
3692 if (nescq->ibcq.event_handler) {
3693 ibevent.device = nescq->ibcq.device;
3694 ibevent.event = IB_EVENT_CQ_ERR;
3695 ibevent.element.cq = &nescq->ibcq;
3696 nescq->ibcq.event_handler(&ibevent, nescq->ibcq.cq_context);
3697 }
3698 }
3699 }
3700 break;
3701
3702 default:
3703 nes_debug(NES_DBG_AEQ, "Processing an iWARP related AE for QP, misc = 0x%04X\n",
3704 async_event_id);
3705 break;
3706 }
3707
3708 }
3709
3710 /**
3711 * nes_iwarp_ce_handler
3712 */
3713 void nes_iwarp_ce_handler(struct nes_device *nesdev, struct nes_hw_cq *hw_cq)
3714 {
3715 struct nes_cq *nescq = container_of(hw_cq, struct nes_cq, hw_cq);
3716
3717 /* nes_debug(NES_DBG_CQ, "Processing completion event for iWARP CQ%u.\n",
3718 nescq->hw_cq.cq_number); */
3719 nes_write32(nesdev->regs+NES_CQ_ACK, nescq->hw_cq.cq_number);
3720
3721 if (nescq->ibcq.comp_handler)
3722 nescq->ibcq.comp_handler(&nescq->ibcq, nescq->ibcq.cq_context);
3723
3724 return;
3725 }
3726
3727
3728 /**
3729 * nes_manage_apbvt()
3730 */
3731 int nes_manage_apbvt(struct nes_vnic *nesvnic, u32 accel_local_port,
3732 u32 nic_index, u32 add_port)
3733 {
3734 struct nes_device *nesdev = nesvnic->nesdev;
3735 struct nes_hw_cqp_wqe *cqp_wqe;
3736 struct nes_cqp_request *cqp_request;
3737 int ret = 0;
3738 u16 major_code;
3739
3740 /* Send manage APBVT request to CQP */
3741 cqp_request = nes_get_cqp_request(nesdev);
3742 if (cqp_request == NULL) {
3743 nes_debug(NES_DBG_QP, "Failed to get a cqp_request.\n");
3744 return -ENOMEM;
3745 }
3746 cqp_request->waiting = 1;
3747 cqp_wqe = &cqp_request->cqp_wqe;
3748
3749 nes_debug(NES_DBG_QP, "%s APBV for local port=%u(0x%04x), nic_index=%u\n",
3750 (add_port == NES_MANAGE_APBVT_ADD) ? "ADD" : "DEL",
3751 accel_local_port, accel_local_port, nic_index);
3752
3753 nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
3754 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX, (NES_CQP_MANAGE_APBVT |
3755 ((add_port == NES_MANAGE_APBVT_ADD) ? NES_CQP_APBVT_ADD : 0)));
3756 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
3757 ((nic_index << NES_CQP_APBVT_NIC_SHIFT) | accel_local_port));
3758
3759 nes_debug(NES_DBG_QP, "Waiting for CQP completion for APBVT.\n");
3760
3761 atomic_set(&cqp_request->refcount, 2);
3762 nes_post_cqp_request(nesdev, cqp_request);
3763
3764 if (add_port == NES_MANAGE_APBVT_ADD)
3765 ret = wait_event_timeout(cqp_request->waitq, (cqp_request->request_done != 0),
3766 NES_EVENT_TIMEOUT);
3767 nes_debug(NES_DBG_QP, "Completed, ret=%u, CQP Major:Minor codes = 0x%04X:0x%04X\n",
3768 ret, cqp_request->major_code, cqp_request->minor_code);
3769 major_code = cqp_request->major_code;
3770
3771 nes_put_cqp_request(nesdev, cqp_request);
3772
3773 if (!ret)
3774 return -ETIME;
3775 else if (major_code)
3776 return -EIO;
3777 else
3778 return 0;
3779 }
3780
3781
3782 /**
3783 * nes_manage_arp_cache
3784 */
3785 void nes_manage_arp_cache(struct net_device *netdev, unsigned char *mac_addr,
3786 u32 ip_addr, u32 action)
3787 {
3788 struct nes_hw_cqp_wqe *cqp_wqe;
3789 struct nes_vnic *nesvnic = netdev_priv(netdev);
3790 struct nes_device *nesdev;
3791 struct nes_cqp_request *cqp_request;
3792 int arp_index;
3793
3794 nesdev = nesvnic->nesdev;
3795 arp_index = nes_arp_table(nesdev, ip_addr, mac_addr, action);
3796 if (arp_index == -1) {
3797 return;
3798 }
3799
3800 /* update the ARP entry */
3801 cqp_request = nes_get_cqp_request(nesdev);
3802 if (cqp_request == NULL) {
3803 nes_debug(NES_DBG_NETDEV, "Failed to get a cqp_request.\n");
3804 return;
3805 }
3806 cqp_request->waiting = 0;
3807 cqp_wqe = &cqp_request->cqp_wqe;
3808 nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
3809
3810 cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(
3811 NES_CQP_MANAGE_ARP_CACHE | NES_CQP_ARP_PERM);
3812 cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] |= cpu_to_le32(
3813 (u32)PCI_FUNC(nesdev->pcidev->devfn) << NES_CQP_ARP_AEQ_INDEX_SHIFT);
3814 cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(arp_index);
3815
3816 if (action == NES_ARP_ADD) {
3817 cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] |= cpu_to_le32(NES_CQP_ARP_VALID);
3818 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_ADDR_LOW_IDX] = cpu_to_le32(
3819 (((u32)mac_addr[2]) << 24) | (((u32)mac_addr[3]) << 16) |
3820 (((u32)mac_addr[4]) << 8) | (u32)mac_addr[5]);
3821 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_HIGH_IDX] = cpu_to_le32(
3822 (((u32)mac_addr[0]) << 8) | (u32)mac_addr[1]);
3823 } else {
3824 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_ADDR_LOW_IDX] = 0;
3825 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_HIGH_IDX] = 0;
3826 }
3827
3828 nes_debug(NES_DBG_NETDEV, "Not waiting for CQP, cqp.sq_head=%u, cqp.sq_tail=%u\n",
3829 nesdev->cqp.sq_head, nesdev->cqp.sq_tail);
3830
3831 atomic_set(&cqp_request->refcount, 1);
3832 nes_post_cqp_request(nesdev, cqp_request);
3833 }
3834
3835
3836 /**
3837 * flush_wqes
3838 */
3839 void flush_wqes(struct nes_device *nesdev, struct nes_qp *nesqp,
3840 u32 which_wq, u32 wait_completion)
3841 {
3842 struct nes_cqp_request *cqp_request;
3843 struct nes_hw_cqp_wqe *cqp_wqe;
3844 u32 sq_code = (NES_IWARP_CQE_MAJOR_FLUSH << 16) | NES_IWARP_CQE_MINOR_FLUSH;
3845 u32 rq_code = (NES_IWARP_CQE_MAJOR_FLUSH << 16) | NES_IWARP_CQE_MINOR_FLUSH;
3846 int ret;
3847
3848 cqp_request = nes_get_cqp_request(nesdev);
3849 if (cqp_request == NULL) {
3850 nes_debug(NES_DBG_QP, "Failed to get a cqp_request.\n");
3851 return;
3852 }
3853 if (wait_completion) {
3854 cqp_request->waiting = 1;
3855 atomic_set(&cqp_request->refcount, 2);
3856 } else {
3857 cqp_request->waiting = 0;
3858 }
3859 cqp_wqe = &cqp_request->cqp_wqe;
3860 nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
3861
3862 /* If wqe in error was identified, set code to be put into cqe */
3863 if ((nesqp->term_sq_flush_code) && (which_wq & NES_CQP_FLUSH_SQ)) {
3864 which_wq |= NES_CQP_FLUSH_MAJ_MIN;
3865 sq_code = (CQE_MAJOR_DRV << 16) | nesqp->term_sq_flush_code;
3866 nesqp->term_sq_flush_code = 0;
3867 }
3868
3869 if ((nesqp->term_rq_flush_code) && (which_wq & NES_CQP_FLUSH_RQ)) {
3870 which_wq |= NES_CQP_FLUSH_MAJ_MIN;
3871 rq_code = (CQE_MAJOR_DRV << 16) | nesqp->term_rq_flush_code;
3872 nesqp->term_rq_flush_code = 0;
3873 }
3874
3875 if (which_wq & NES_CQP_FLUSH_MAJ_MIN) {
3876 cqp_wqe->wqe_words[NES_CQP_QP_WQE_FLUSH_SQ_CODE] = cpu_to_le32(sq_code);
3877 cqp_wqe->wqe_words[NES_CQP_QP_WQE_FLUSH_RQ_CODE] = cpu_to_le32(rq_code);
3878 }
3879
3880 cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] =
3881 cpu_to_le32(NES_CQP_FLUSH_WQES | which_wq);
3882 cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesqp->hwqp.qp_id);
3883
3884 nes_post_cqp_request(nesdev, cqp_request);
3885
3886 if (wait_completion) {
3887 /* Wait for CQP */
3888 ret = wait_event_timeout(cqp_request->waitq, (cqp_request->request_done != 0),
3889 NES_EVENT_TIMEOUT);
3890 nes_debug(NES_DBG_QP, "Flush SQ QP WQEs completed, ret=%u,"
3891 " CQP Major:Minor codes = 0x%04X:0x%04X\n",
3892 ret, cqp_request->major_code, cqp_request->minor_code);
3893 nes_put_cqp_request(nesdev, cqp_request);
3894 }
3895 }
ubuntu-bionic-kernel mirror