1 /*******************************************************************************
3 * Intel Ethernet Controller XL710 Family Linux Driver
4 * Copyright(c) 2013 - 2014 Intel Corporation.
6 * This program is free software; you can redistribute it and/or modify it
7 * under the terms and conditions of the GNU General Public License,
8 * version 2, as published by the Free Software Foundation.
10 * This program is distributed in the hope it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
15 * You should have received a copy of the GNU General Public License along
16 * with this program. If not, see <http://www.gnu.org/licenses/>.
18 * The full GNU General Public License is included in this distribution in
19 * the file called "COPYING".
21 * Contact Information:
22 * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
23 * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
25 ******************************************************************************/
27 #include "i40e_status.h"
28 #include "i40e_type.h"
29 #include "i40e_register.h"
30 #include "i40e_adminq.h"
31 #include "i40e_prototype.h"
33 static void i40e_resume_aq(struct i40e_hw
*hw
);
36 * i40e_is_nvm_update_op - return true if this is an NVM update operation
37 * @desc: API request descriptor
39 static inline bool i40e_is_nvm_update_op(struct i40e_aq_desc
*desc
)
41 return (desc
->opcode
== cpu_to_le16(i40e_aqc_opc_nvm_erase
)) ||
42 (desc
->opcode
== cpu_to_le16(i40e_aqc_opc_nvm_update
));
46 * i40e_adminq_init_regs - Initialize AdminQ registers
47 * @hw: pointer to the hardware structure
49 * This assumes the alloc_asq and alloc_arq functions have already been called
51 static void i40e_adminq_init_regs(struct i40e_hw
*hw
)
53 /* set head and tail registers in our local struct */
55 hw
->aq
.asq
.tail
= I40E_VF_ATQT1
;
56 hw
->aq
.asq
.head
= I40E_VF_ATQH1
;
57 hw
->aq
.asq
.len
= I40E_VF_ATQLEN1
;
58 hw
->aq
.asq
.bal
= I40E_VF_ATQBAL1
;
59 hw
->aq
.asq
.bah
= I40E_VF_ATQBAH1
;
60 hw
->aq
.arq
.tail
= I40E_VF_ARQT1
;
61 hw
->aq
.arq
.head
= I40E_VF_ARQH1
;
62 hw
->aq
.arq
.len
= I40E_VF_ARQLEN1
;
63 hw
->aq
.arq
.bal
= I40E_VF_ARQBAL1
;
64 hw
->aq
.arq
.bah
= I40E_VF_ARQBAH1
;
66 hw
->aq
.asq
.tail
= I40E_PF_ATQT
;
67 hw
->aq
.asq
.head
= I40E_PF_ATQH
;
68 hw
->aq
.asq
.len
= I40E_PF_ATQLEN
;
69 hw
->aq
.asq
.bal
= I40E_PF_ATQBAL
;
70 hw
->aq
.asq
.bah
= I40E_PF_ATQBAH
;
71 hw
->aq
.arq
.tail
= I40E_PF_ARQT
;
72 hw
->aq
.arq
.head
= I40E_PF_ARQH
;
73 hw
->aq
.arq
.len
= I40E_PF_ARQLEN
;
74 hw
->aq
.arq
.bal
= I40E_PF_ARQBAL
;
75 hw
->aq
.arq
.bah
= I40E_PF_ARQBAH
;
80 * i40e_alloc_adminq_asq_ring - Allocate Admin Queue send rings
81 * @hw: pointer to the hardware structure
83 static i40e_status
i40e_alloc_adminq_asq_ring(struct i40e_hw
*hw
)
87 ret_code
= i40e_allocate_dma_mem(hw
, &hw
->aq
.asq
.desc_buf
,
89 (hw
->aq
.num_asq_entries
*
90 sizeof(struct i40e_aq_desc
)),
91 I40E_ADMINQ_DESC_ALIGNMENT
);
95 ret_code
= i40e_allocate_virt_mem(hw
, &hw
->aq
.asq
.cmd_buf
,
96 (hw
->aq
.num_asq_entries
*
97 sizeof(struct i40e_asq_cmd_details
)));
99 i40e_free_dma_mem(hw
, &hw
->aq
.asq
.desc_buf
);
107 * i40e_alloc_adminq_arq_ring - Allocate Admin Queue receive rings
108 * @hw: pointer to the hardware structure
110 static i40e_status
i40e_alloc_adminq_arq_ring(struct i40e_hw
*hw
)
112 i40e_status ret_code
;
114 ret_code
= i40e_allocate_dma_mem(hw
, &hw
->aq
.arq
.desc_buf
,
116 (hw
->aq
.num_arq_entries
*
117 sizeof(struct i40e_aq_desc
)),
118 I40E_ADMINQ_DESC_ALIGNMENT
);
124 * i40e_free_adminq_asq - Free Admin Queue send rings
125 * @hw: pointer to the hardware structure
127 * This assumes the posted send buffers have already been cleaned
130 static void i40e_free_adminq_asq(struct i40e_hw
*hw
)
132 i40e_free_dma_mem(hw
, &hw
->aq
.asq
.desc_buf
);
136 * i40e_free_adminq_arq - Free Admin Queue receive rings
137 * @hw: pointer to the hardware structure
139 * This assumes the posted receive buffers have already been cleaned
142 static void i40e_free_adminq_arq(struct i40e_hw
*hw
)
144 i40e_free_dma_mem(hw
, &hw
->aq
.arq
.desc_buf
);
148 * i40e_alloc_arq_bufs - Allocate pre-posted buffers for the receive queue
149 * @hw: pointer to the hardware structure
151 static i40e_status
i40e_alloc_arq_bufs(struct i40e_hw
*hw
)
153 i40e_status ret_code
;
154 struct i40e_aq_desc
*desc
;
155 struct i40e_dma_mem
*bi
;
158 /* We'll be allocating the buffer info memory first, then we can
159 * allocate the mapped buffers for the event processing
162 /* buffer_info structures do not need alignment */
163 ret_code
= i40e_allocate_virt_mem(hw
, &hw
->aq
.arq
.dma_head
,
164 (hw
->aq
.num_arq_entries
* sizeof(struct i40e_dma_mem
)));
167 hw
->aq
.arq
.r
.arq_bi
= (struct i40e_dma_mem
*)hw
->aq
.arq
.dma_head
.va
;
169 /* allocate the mapped buffers */
170 for (i
= 0; i
< hw
->aq
.num_arq_entries
; i
++) {
171 bi
= &hw
->aq
.arq
.r
.arq_bi
[i
];
172 ret_code
= i40e_allocate_dma_mem(hw
, bi
,
175 I40E_ADMINQ_DESC_ALIGNMENT
);
177 goto unwind_alloc_arq_bufs
;
179 /* now configure the descriptors for use */
180 desc
= I40E_ADMINQ_DESC(hw
->aq
.arq
, i
);
182 desc
->flags
= cpu_to_le16(I40E_AQ_FLAG_BUF
);
183 if (hw
->aq
.arq_buf_size
> I40E_AQ_LARGE_BUF
)
184 desc
->flags
|= cpu_to_le16(I40E_AQ_FLAG_LB
);
186 /* This is in accordance with Admin queue design, there is no
187 * register for buffer size configuration
189 desc
->datalen
= cpu_to_le16((u16
)bi
->size
);
191 desc
->cookie_high
= 0;
192 desc
->cookie_low
= 0;
193 desc
->params
.external
.addr_high
=
194 cpu_to_le32(upper_32_bits(bi
->pa
));
195 desc
->params
.external
.addr_low
=
196 cpu_to_le32(lower_32_bits(bi
->pa
));
197 desc
->params
.external
.param0
= 0;
198 desc
->params
.external
.param1
= 0;
204 unwind_alloc_arq_bufs
:
205 /* don't try to free the one that failed... */
208 i40e_free_dma_mem(hw
, &hw
->aq
.arq
.r
.arq_bi
[i
]);
209 i40e_free_virt_mem(hw
, &hw
->aq
.arq
.dma_head
);
215 * i40e_alloc_asq_bufs - Allocate empty buffer structs for the send queue
216 * @hw: pointer to the hardware structure
218 static i40e_status
i40e_alloc_asq_bufs(struct i40e_hw
*hw
)
220 i40e_status ret_code
;
221 struct i40e_dma_mem
*bi
;
224 /* No mapped memory needed yet, just the buffer info structures */
225 ret_code
= i40e_allocate_virt_mem(hw
, &hw
->aq
.asq
.dma_head
,
226 (hw
->aq
.num_asq_entries
* sizeof(struct i40e_dma_mem
)));
229 hw
->aq
.asq
.r
.asq_bi
= (struct i40e_dma_mem
*)hw
->aq
.asq
.dma_head
.va
;
231 /* allocate the mapped buffers */
232 for (i
= 0; i
< hw
->aq
.num_asq_entries
; i
++) {
233 bi
= &hw
->aq
.asq
.r
.asq_bi
[i
];
234 ret_code
= i40e_allocate_dma_mem(hw
, bi
,
237 I40E_ADMINQ_DESC_ALIGNMENT
);
239 goto unwind_alloc_asq_bufs
;
244 unwind_alloc_asq_bufs
:
245 /* don't try to free the one that failed... */
248 i40e_free_dma_mem(hw
, &hw
->aq
.asq
.r
.asq_bi
[i
]);
249 i40e_free_virt_mem(hw
, &hw
->aq
.asq
.dma_head
);
255 * i40e_free_arq_bufs - Free receive queue buffer info elements
256 * @hw: pointer to the hardware structure
258 static void i40e_free_arq_bufs(struct i40e_hw
*hw
)
262 /* free descriptors */
263 for (i
= 0; i
< hw
->aq
.num_arq_entries
; i
++)
264 i40e_free_dma_mem(hw
, &hw
->aq
.arq
.r
.arq_bi
[i
]);
266 /* free the descriptor memory */
267 i40e_free_dma_mem(hw
, &hw
->aq
.arq
.desc_buf
);
269 /* free the dma header */
270 i40e_free_virt_mem(hw
, &hw
->aq
.arq
.dma_head
);
274 * i40e_free_asq_bufs - Free send queue buffer info elements
275 * @hw: pointer to the hardware structure
277 static void i40e_free_asq_bufs(struct i40e_hw
*hw
)
281 /* only unmap if the address is non-NULL */
282 for (i
= 0; i
< hw
->aq
.num_asq_entries
; i
++)
283 if (hw
->aq
.asq
.r
.asq_bi
[i
].pa
)
284 i40e_free_dma_mem(hw
, &hw
->aq
.asq
.r
.asq_bi
[i
]);
286 /* free the buffer info list */
287 i40e_free_virt_mem(hw
, &hw
->aq
.asq
.cmd_buf
);
289 /* free the descriptor memory */
290 i40e_free_dma_mem(hw
, &hw
->aq
.asq
.desc_buf
);
292 /* free the dma header */
293 i40e_free_virt_mem(hw
, &hw
->aq
.asq
.dma_head
);
297 * i40e_config_asq_regs - configure ASQ registers
298 * @hw: pointer to the hardware structure
300 * Configure base address and length registers for the transmit queue
302 static i40e_status
i40e_config_asq_regs(struct i40e_hw
*hw
)
304 i40e_status ret_code
= 0;
307 /* Clear Head and Tail */
308 wr32(hw
, hw
->aq
.asq
.head
, 0);
309 wr32(hw
, hw
->aq
.asq
.tail
, 0);
311 /* set starting point */
312 wr32(hw
, hw
->aq
.asq
.len
, (hw
->aq
.num_asq_entries
|
313 I40E_PF_ATQLEN_ATQENABLE_MASK
));
314 wr32(hw
, hw
->aq
.asq
.bal
, lower_32_bits(hw
->aq
.asq
.desc_buf
.pa
));
315 wr32(hw
, hw
->aq
.asq
.bah
, upper_32_bits(hw
->aq
.asq
.desc_buf
.pa
));
317 /* Check one register to verify that config was applied */
318 reg
= rd32(hw
, hw
->aq
.asq
.bal
);
319 if (reg
!= lower_32_bits(hw
->aq
.asq
.desc_buf
.pa
))
320 ret_code
= I40E_ERR_ADMIN_QUEUE_ERROR
;
326 * i40e_config_arq_regs - ARQ register configuration
327 * @hw: pointer to the hardware structure
329 * Configure base address and length registers for the receive (event queue)
331 static i40e_status
i40e_config_arq_regs(struct i40e_hw
*hw
)
333 i40e_status ret_code
= 0;
336 /* Clear Head and Tail */
337 wr32(hw
, hw
->aq
.arq
.head
, 0);
338 wr32(hw
, hw
->aq
.arq
.tail
, 0);
340 /* set starting point */
341 wr32(hw
, hw
->aq
.arq
.len
, (hw
->aq
.num_arq_entries
|
342 I40E_PF_ARQLEN_ARQENABLE_MASK
));
343 wr32(hw
, hw
->aq
.arq
.bal
, lower_32_bits(hw
->aq
.arq
.desc_buf
.pa
));
344 wr32(hw
, hw
->aq
.arq
.bah
, upper_32_bits(hw
->aq
.arq
.desc_buf
.pa
));
346 /* Update tail in the HW to post pre-allocated buffers */
347 wr32(hw
, hw
->aq
.arq
.tail
, hw
->aq
.num_arq_entries
- 1);
349 /* Check one register to verify that config was applied */
350 reg
= rd32(hw
, hw
->aq
.arq
.bal
);
351 if (reg
!= lower_32_bits(hw
->aq
.arq
.desc_buf
.pa
))
352 ret_code
= I40E_ERR_ADMIN_QUEUE_ERROR
;
358 * i40e_init_asq - main initialization routine for ASQ
359 * @hw: pointer to the hardware structure
361 * This is the main initialization routine for the Admin Send Queue
362 * Prior to calling this function, drivers *MUST* set the following fields
363 * in the hw->aq structure:
364 * - hw->aq.num_asq_entries
365 * - hw->aq.arq_buf_size
367 * Do *NOT* hold the lock when calling this as the memory allocation routines
368 * called are not going to be atomic context safe
370 static i40e_status
i40e_init_asq(struct i40e_hw
*hw
)
372 i40e_status ret_code
= 0;
374 if (hw
->aq
.asq
.count
> 0) {
375 /* queue already initialized */
376 ret_code
= I40E_ERR_NOT_READY
;
377 goto init_adminq_exit
;
380 /* verify input for valid configuration */
381 if ((hw
->aq
.num_asq_entries
== 0) ||
382 (hw
->aq
.asq_buf_size
== 0)) {
383 ret_code
= I40E_ERR_CONFIG
;
384 goto init_adminq_exit
;
387 hw
->aq
.asq
.next_to_use
= 0;
388 hw
->aq
.asq
.next_to_clean
= 0;
389 hw
->aq
.asq
.count
= hw
->aq
.num_asq_entries
;
391 /* allocate the ring memory */
392 ret_code
= i40e_alloc_adminq_asq_ring(hw
);
394 goto init_adminq_exit
;
396 /* allocate buffers in the rings */
397 ret_code
= i40e_alloc_asq_bufs(hw
);
399 goto init_adminq_free_rings
;
401 /* initialize base registers */
402 ret_code
= i40e_config_asq_regs(hw
);
404 goto init_adminq_free_rings
;
407 goto init_adminq_exit
;
409 init_adminq_free_rings
:
410 i40e_free_adminq_asq(hw
);
417 * i40e_init_arq - initialize ARQ
418 * @hw: pointer to the hardware structure
420 * The main initialization routine for the Admin Receive (Event) Queue.
421 * Prior to calling this function, drivers *MUST* set the following fields
422 * in the hw->aq structure:
423 * - hw->aq.num_asq_entries
424 * - hw->aq.arq_buf_size
426 * Do *NOT* hold the lock when calling this as the memory allocation routines
427 * called are not going to be atomic context safe
429 static i40e_status
i40e_init_arq(struct i40e_hw
*hw
)
431 i40e_status ret_code
= 0;
433 if (hw
->aq
.arq
.count
> 0) {
434 /* queue already initialized */
435 ret_code
= I40E_ERR_NOT_READY
;
436 goto init_adminq_exit
;
439 /* verify input for valid configuration */
440 if ((hw
->aq
.num_arq_entries
== 0) ||
441 (hw
->aq
.arq_buf_size
== 0)) {
442 ret_code
= I40E_ERR_CONFIG
;
443 goto init_adminq_exit
;
446 hw
->aq
.arq
.next_to_use
= 0;
447 hw
->aq
.arq
.next_to_clean
= 0;
448 hw
->aq
.arq
.count
= hw
->aq
.num_arq_entries
;
450 /* allocate the ring memory */
451 ret_code
= i40e_alloc_adminq_arq_ring(hw
);
453 goto init_adminq_exit
;
455 /* allocate buffers in the rings */
456 ret_code
= i40e_alloc_arq_bufs(hw
);
458 goto init_adminq_free_rings
;
460 /* initialize base registers */
461 ret_code
= i40e_config_arq_regs(hw
);
463 goto init_adminq_free_rings
;
466 goto init_adminq_exit
;
468 init_adminq_free_rings
:
469 i40e_free_adminq_arq(hw
);
476 * i40e_shutdown_asq - shutdown the ASQ
477 * @hw: pointer to the hardware structure
479 * The main shutdown routine for the Admin Send Queue
481 static i40e_status
i40e_shutdown_asq(struct i40e_hw
*hw
)
483 i40e_status ret_code
= 0;
485 mutex_lock(&hw
->aq
.asq_mutex
);
487 if (hw
->aq
.asq
.count
== 0) {
488 ret_code
= I40E_ERR_NOT_READY
;
489 goto shutdown_asq_out
;
492 /* Stop firmware AdminQ processing */
493 wr32(hw
, hw
->aq
.asq
.head
, 0);
494 wr32(hw
, hw
->aq
.asq
.tail
, 0);
495 wr32(hw
, hw
->aq
.asq
.len
, 0);
496 wr32(hw
, hw
->aq
.asq
.bal
, 0);
497 wr32(hw
, hw
->aq
.asq
.bah
, 0);
499 hw
->aq
.asq
.count
= 0; /* to indicate uninitialized queue */
501 /* free ring buffers */
502 i40e_free_asq_bufs(hw
);
505 mutex_unlock(&hw
->aq
.asq_mutex
);
510 * i40e_shutdown_arq - shutdown ARQ
511 * @hw: pointer to the hardware structure
513 * The main shutdown routine for the Admin Receive Queue
515 static i40e_status
i40e_shutdown_arq(struct i40e_hw
*hw
)
517 i40e_status ret_code
= 0;
519 mutex_lock(&hw
->aq
.arq_mutex
);
521 if (hw
->aq
.arq
.count
== 0) {
522 ret_code
= I40E_ERR_NOT_READY
;
523 goto shutdown_arq_out
;
526 /* Stop firmware AdminQ processing */
527 wr32(hw
, hw
->aq
.arq
.head
, 0);
528 wr32(hw
, hw
->aq
.arq
.tail
, 0);
529 wr32(hw
, hw
->aq
.arq
.len
, 0);
530 wr32(hw
, hw
->aq
.arq
.bal
, 0);
531 wr32(hw
, hw
->aq
.arq
.bah
, 0);
533 hw
->aq
.arq
.count
= 0; /* to indicate uninitialized queue */
535 /* free ring buffers */
536 i40e_free_arq_bufs(hw
);
539 mutex_unlock(&hw
->aq
.arq_mutex
);
544 * i40e_init_adminq - main initialization routine for Admin Queue
545 * @hw: pointer to the hardware structure
547 * Prior to calling this function, drivers *MUST* set the following fields
548 * in the hw->aq structure:
549 * - hw->aq.num_asq_entries
550 * - hw->aq.num_arq_entries
551 * - hw->aq.arq_buf_size
552 * - hw->aq.asq_buf_size
554 i40e_status
i40e_init_adminq(struct i40e_hw
*hw
)
556 i40e_status ret_code
;
557 u16 eetrack_lo
, eetrack_hi
;
560 /* verify input for valid configuration */
561 if ((hw
->aq
.num_arq_entries
== 0) ||
562 (hw
->aq
.num_asq_entries
== 0) ||
563 (hw
->aq
.arq_buf_size
== 0) ||
564 (hw
->aq
.asq_buf_size
== 0)) {
565 ret_code
= I40E_ERR_CONFIG
;
566 goto init_adminq_exit
;
569 /* initialize locks */
570 mutex_init(&hw
->aq
.asq_mutex
);
571 mutex_init(&hw
->aq
.arq_mutex
);
573 /* Set up register offsets */
574 i40e_adminq_init_regs(hw
);
576 /* setup ASQ command write back timeout */
577 hw
->aq
.asq_cmd_timeout
= I40E_ASQ_CMD_TIMEOUT
;
579 /* allocate the ASQ */
580 ret_code
= i40e_init_asq(hw
);
582 goto init_adminq_destroy_locks
;
584 /* allocate the ARQ */
585 ret_code
= i40e_init_arq(hw
);
587 goto init_adminq_free_asq
;
589 /* There are some cases where the firmware may not be quite ready
590 * for AdminQ operations, so we retry the AdminQ setup a few times
591 * if we see timeouts in this first AQ call.
594 ret_code
= i40e_aq_get_firmware_version(hw
,
601 if (ret_code
!= I40E_ERR_ADMIN_QUEUE_TIMEOUT
)
606 } while (retry
< 10);
607 if (ret_code
!= I40E_SUCCESS
)
608 goto init_adminq_free_arq
;
610 /* get the NVM version info */
611 i40e_read_nvm_word(hw
, I40E_SR_NVM_DEV_STARTER_VERSION
,
613 i40e_read_nvm_word(hw
, I40E_SR_NVM_EETRACK_LO
, &eetrack_lo
);
614 i40e_read_nvm_word(hw
, I40E_SR_NVM_EETRACK_HI
, &eetrack_hi
);
615 hw
->nvm
.eetrack
= (eetrack_hi
<< 16) | eetrack_lo
;
617 if (hw
->aq
.api_maj_ver
> I40E_FW_API_VERSION_MAJOR
) {
618 ret_code
= I40E_ERR_FIRMWARE_API_VERSION
;
619 goto init_adminq_free_arq
;
622 /* pre-emptive resource lock release */
623 i40e_aq_release_resource(hw
, I40E_NVM_RESOURCE_ID
, 0, NULL
);
624 hw
->aq
.nvm_release_on_done
= false;
625 hw
->nvmupd_state
= I40E_NVMUPD_STATE_INIT
;
627 ret_code
= i40e_aq_set_hmc_resource_profile(hw
,
628 I40E_HMC_PROFILE_DEFAULT
,
634 goto init_adminq_exit
;
636 init_adminq_free_arq
:
637 i40e_shutdown_arq(hw
);
638 init_adminq_free_asq
:
639 i40e_shutdown_asq(hw
);
640 init_adminq_destroy_locks
:
647 * i40e_shutdown_adminq - shutdown routine for the Admin Queue
648 * @hw: pointer to the hardware structure
650 i40e_status
i40e_shutdown_adminq(struct i40e_hw
*hw
)
652 i40e_status ret_code
= 0;
654 if (i40e_check_asq_alive(hw
))
655 i40e_aq_queue_shutdown(hw
, true);
657 i40e_shutdown_asq(hw
);
658 i40e_shutdown_arq(hw
);
660 /* destroy the locks */
663 i40e_free_virt_mem(hw
, &hw
->nvm_buff
);
669 * i40e_clean_asq - cleans Admin send queue
670 * @hw: pointer to the hardware structure
672 * returns the number of free desc
674 static u16
i40e_clean_asq(struct i40e_hw
*hw
)
676 struct i40e_adminq_ring
*asq
= &(hw
->aq
.asq
);
677 struct i40e_asq_cmd_details
*details
;
678 u16 ntc
= asq
->next_to_clean
;
679 struct i40e_aq_desc desc_cb
;
680 struct i40e_aq_desc
*desc
;
682 desc
= I40E_ADMINQ_DESC(*asq
, ntc
);
683 details
= I40E_ADMINQ_DETAILS(*asq
, ntc
);
684 while (rd32(hw
, hw
->aq
.asq
.head
) != ntc
) {
685 i40e_debug(hw
, I40E_DEBUG_AQ_MESSAGE
,
686 "ntc %d head %d.\n", ntc
, rd32(hw
, hw
->aq
.asq
.head
));
688 if (details
->callback
) {
689 I40E_ADMINQ_CALLBACK cb_func
=
690 (I40E_ADMINQ_CALLBACK
)details
->callback
;
692 cb_func(hw
, &desc_cb
);
694 memset(desc
, 0, sizeof(*desc
));
695 memset(details
, 0, sizeof(*details
));
697 if (ntc
== asq
->count
)
699 desc
= I40E_ADMINQ_DESC(*asq
, ntc
);
700 details
= I40E_ADMINQ_DETAILS(*asq
, ntc
);
703 asq
->next_to_clean
= ntc
;
705 return I40E_DESC_UNUSED(asq
);
709 * i40e_asq_done - check if FW has processed the Admin Send Queue
710 * @hw: pointer to the hw struct
712 * Returns true if the firmware has processed all descriptors on the
713 * admin send queue. Returns false if there are still requests pending.
715 static bool i40e_asq_done(struct i40e_hw
*hw
)
717 /* AQ designers suggest use of head for better
718 * timing reliability than DD bit
720 return rd32(hw
, hw
->aq
.asq
.head
) == hw
->aq
.asq
.next_to_use
;
725 * i40e_asq_send_command - send command to Admin Queue
726 * @hw: pointer to the hw struct
727 * @desc: prefilled descriptor describing the command (non DMA mem)
728 * @buff: buffer to use for indirect commands
729 * @buff_size: size of buffer for indirect commands
730 * @cmd_details: pointer to command details structure
732 * This is the main send command driver routine for the Admin Queue send
733 * queue. It runs the queue, cleans the queue, etc
735 i40e_status
i40e_asq_send_command(struct i40e_hw
*hw
,
736 struct i40e_aq_desc
*desc
,
737 void *buff
, /* can be NULL */
739 struct i40e_asq_cmd_details
*cmd_details
)
741 i40e_status status
= 0;
742 struct i40e_dma_mem
*dma_buff
= NULL
;
743 struct i40e_asq_cmd_details
*details
;
744 struct i40e_aq_desc
*desc_on_ring
;
745 bool cmd_completed
= false;
749 mutex_lock(&hw
->aq
.asq_mutex
);
751 if (hw
->aq
.asq
.count
== 0) {
752 i40e_debug(hw
, I40E_DEBUG_AQ_MESSAGE
,
753 "AQTX: Admin queue not initialized.\n");
754 status
= I40E_ERR_QUEUE_EMPTY
;
755 goto asq_send_command_error
;
758 hw
->aq
.asq_last_status
= I40E_AQ_RC_OK
;
760 val
= rd32(hw
, hw
->aq
.asq
.head
);
761 if (val
>= hw
->aq
.num_asq_entries
) {
762 i40e_debug(hw
, I40E_DEBUG_AQ_MESSAGE
,
763 "AQTX: head overrun at %d\n", val
);
764 status
= I40E_ERR_QUEUE_EMPTY
;
765 goto asq_send_command_error
;
768 details
= I40E_ADMINQ_DETAILS(hw
->aq
.asq
, hw
->aq
.asq
.next_to_use
);
770 *details
= *cmd_details
;
772 /* If the cmd_details are defined copy the cookie. The
773 * cpu_to_le32 is not needed here because the data is ignored
774 * by the FW, only used by the driver
776 if (details
->cookie
) {
778 cpu_to_le32(upper_32_bits(details
->cookie
));
780 cpu_to_le32(lower_32_bits(details
->cookie
));
783 memset(details
, 0, sizeof(struct i40e_asq_cmd_details
));
786 /* clear requested flags and then set additional flags if defined */
787 desc
->flags
&= ~cpu_to_le16(details
->flags_dis
);
788 desc
->flags
|= cpu_to_le16(details
->flags_ena
);
790 if (buff_size
> hw
->aq
.asq_buf_size
) {
792 I40E_DEBUG_AQ_MESSAGE
,
793 "AQTX: Invalid buffer size: %d.\n",
795 status
= I40E_ERR_INVALID_SIZE
;
796 goto asq_send_command_error
;
799 if (details
->postpone
&& !details
->async
) {
801 I40E_DEBUG_AQ_MESSAGE
,
802 "AQTX: Async flag not set along with postpone flag");
803 status
= I40E_ERR_PARAM
;
804 goto asq_send_command_error
;
807 /* call clean and check queue available function to reclaim the
808 * descriptors that were processed by FW, the function returns the
809 * number of desc available
811 /* the clean function called here could be called in a separate thread
812 * in case of asynchronous completions
814 if (i40e_clean_asq(hw
) == 0) {
816 I40E_DEBUG_AQ_MESSAGE
,
817 "AQTX: Error queue is full.\n");
818 status
= I40E_ERR_ADMIN_QUEUE_FULL
;
819 goto asq_send_command_error
;
822 /* initialize the temp desc pointer with the right desc */
823 desc_on_ring
= I40E_ADMINQ_DESC(hw
->aq
.asq
, hw
->aq
.asq
.next_to_use
);
825 /* if the desc is available copy the temp desc to the right place */
826 *desc_on_ring
= *desc
;
828 /* if buff is not NULL assume indirect command */
830 dma_buff
= &(hw
->aq
.asq
.r
.asq_bi
[hw
->aq
.asq
.next_to_use
]);
831 /* copy the user buff into the respective DMA buff */
832 memcpy(dma_buff
->va
, buff
, buff_size
);
833 desc_on_ring
->datalen
= cpu_to_le16(buff_size
);
835 /* Update the address values in the desc with the pa value
836 * for respective buffer
838 desc_on_ring
->params
.external
.addr_high
=
839 cpu_to_le32(upper_32_bits(dma_buff
->pa
));
840 desc_on_ring
->params
.external
.addr_low
=
841 cpu_to_le32(lower_32_bits(dma_buff
->pa
));
845 i40e_debug(hw
, I40E_DEBUG_AQ_MESSAGE
, "AQTX: desc and buffer:\n");
846 i40e_debug_aq(hw
, I40E_DEBUG_AQ_COMMAND
, (void *)desc_on_ring
,
848 (hw
->aq
.asq
.next_to_use
)++;
849 if (hw
->aq
.asq
.next_to_use
== hw
->aq
.asq
.count
)
850 hw
->aq
.asq
.next_to_use
= 0;
851 if (!details
->postpone
)
852 wr32(hw
, hw
->aq
.asq
.tail
, hw
->aq
.asq
.next_to_use
);
854 /* if cmd_details are not defined or async flag is not set,
855 * we need to wait for desc write back
857 if (!details
->async
&& !details
->postpone
) {
861 /* AQ designers suggest use of head for better
862 * timing reliability than DD bit
864 if (i40e_asq_done(hw
))
866 usleep_range(1000, 2000);
868 } while (total_delay
< hw
->aq
.asq_cmd_timeout
);
871 /* if ready, copy the desc back to temp */
872 if (i40e_asq_done(hw
)) {
873 *desc
= *desc_on_ring
;
875 memcpy(buff
, dma_buff
->va
, buff_size
);
876 retval
= le16_to_cpu(desc
->retval
);
879 I40E_DEBUG_AQ_MESSAGE
,
880 "AQTX: Command completed with error 0x%X.\n",
883 /* strip off FW internal code */
886 cmd_completed
= true;
887 if ((enum i40e_admin_queue_err
)retval
== I40E_AQ_RC_OK
)
890 status
= I40E_ERR_ADMIN_QUEUE_ERROR
;
891 hw
->aq
.asq_last_status
= (enum i40e_admin_queue_err
)retval
;
894 i40e_debug(hw
, I40E_DEBUG_AQ_MESSAGE
,
895 "AQTX: desc and buffer writeback:\n");
896 i40e_debug_aq(hw
, I40E_DEBUG_AQ_COMMAND
, (void *)desc
, buff
, buff_size
);
898 /* save writeback aq if requested */
899 if (details
->wb_desc
)
900 *details
->wb_desc
= *desc_on_ring
;
902 /* update the error if time out occurred */
903 if ((!cmd_completed
) &&
904 (!details
->async
&& !details
->postpone
)) {
906 I40E_DEBUG_AQ_MESSAGE
,
907 "AQTX: Writeback timeout.\n");
908 status
= I40E_ERR_ADMIN_QUEUE_TIMEOUT
;
911 asq_send_command_error
:
912 mutex_unlock(&hw
->aq
.asq_mutex
);
917 * i40e_fill_default_direct_cmd_desc - AQ descriptor helper function
918 * @desc: pointer to the temp descriptor (non DMA mem)
919 * @opcode: the opcode can be used to decide which flags to turn off or on
921 * Fill the desc with default values
923 void i40e_fill_default_direct_cmd_desc(struct i40e_aq_desc
*desc
,
926 /* zero out the desc */
927 memset((void *)desc
, 0, sizeof(struct i40e_aq_desc
));
928 desc
->opcode
= cpu_to_le16(opcode
);
929 desc
->flags
= cpu_to_le16(I40E_AQ_FLAG_SI
);
933 * i40e_clean_arq_element
934 * @hw: pointer to the hw struct
935 * @e: event info from the receive descriptor, includes any buffers
936 * @pending: number of events that could be left to process
938 * This function cleans one Admin Receive Queue element and returns
939 * the contents through e. It can also return how many events are
940 * left to process through 'pending'
942 i40e_status
i40e_clean_arq_element(struct i40e_hw
*hw
,
943 struct i40e_arq_event_info
*e
,
946 i40e_status ret_code
= 0;
947 u16 ntc
= hw
->aq
.arq
.next_to_clean
;
948 struct i40e_aq_desc
*desc
;
949 struct i40e_dma_mem
*bi
;
955 /* take the lock before we start messing with the ring */
956 mutex_lock(&hw
->aq
.arq_mutex
);
958 if (hw
->aq
.arq
.count
== 0) {
959 i40e_debug(hw
, I40E_DEBUG_AQ_MESSAGE
,
960 "AQRX: Admin queue not initialized.\n");
961 ret_code
= I40E_ERR_QUEUE_EMPTY
;
962 goto clean_arq_element_err
;
965 /* set next_to_use to head */
966 ntu
= (rd32(hw
, hw
->aq
.arq
.head
) & I40E_PF_ARQH_ARQH_MASK
);
968 /* nothing to do - shouldn't need to update ring's values */
969 ret_code
= I40E_ERR_ADMIN_QUEUE_NO_WORK
;
970 goto clean_arq_element_out
;
973 /* now clean the next descriptor */
974 desc
= I40E_ADMINQ_DESC(hw
->aq
.arq
, ntc
);
977 flags
= le16_to_cpu(desc
->flags
);
978 if (flags
& I40E_AQ_FLAG_ERR
) {
979 ret_code
= I40E_ERR_ADMIN_QUEUE_ERROR
;
980 hw
->aq
.arq_last_status
=
981 (enum i40e_admin_queue_err
)le16_to_cpu(desc
->retval
);
983 I40E_DEBUG_AQ_MESSAGE
,
984 "AQRX: Event received with error 0x%X.\n",
985 hw
->aq
.arq_last_status
);
989 datalen
= le16_to_cpu(desc
->datalen
);
990 e
->msg_len
= min(datalen
, e
->buf_len
);
991 if (e
->msg_buf
!= NULL
&& (e
->msg_len
!= 0))
992 memcpy(e
->msg_buf
, hw
->aq
.arq
.r
.arq_bi
[desc_idx
].va
,
995 i40e_debug(hw
, I40E_DEBUG_AQ_MESSAGE
, "AQRX: desc and buffer:\n");
996 i40e_debug_aq(hw
, I40E_DEBUG_AQ_COMMAND
, (void *)desc
, e
->msg_buf
,
997 hw
->aq
.arq_buf_size
);
999 /* Restore the original datalen and buffer address in the desc,
1000 * FW updates datalen to indicate the event message
1003 bi
= &hw
->aq
.arq
.r
.arq_bi
[ntc
];
1004 memset((void *)desc
, 0, sizeof(struct i40e_aq_desc
));
1006 desc
->flags
= cpu_to_le16(I40E_AQ_FLAG_BUF
);
1007 if (hw
->aq
.arq_buf_size
> I40E_AQ_LARGE_BUF
)
1008 desc
->flags
|= cpu_to_le16(I40E_AQ_FLAG_LB
);
1009 desc
->datalen
= cpu_to_le16((u16
)bi
->size
);
1010 desc
->params
.external
.addr_high
= cpu_to_le32(upper_32_bits(bi
->pa
));
1011 desc
->params
.external
.addr_low
= cpu_to_le32(lower_32_bits(bi
->pa
));
1013 /* set tail = the last cleaned desc index. */
1014 wr32(hw
, hw
->aq
.arq
.tail
, ntc
);
1015 /* ntc is updated to tail + 1 */
1017 if (ntc
== hw
->aq
.num_arq_entries
)
1019 hw
->aq
.arq
.next_to_clean
= ntc
;
1020 hw
->aq
.arq
.next_to_use
= ntu
;
1022 clean_arq_element_out
:
1023 /* Set pending if needed, unlock and return */
1024 if (pending
!= NULL
)
1025 *pending
= (ntc
> ntu
? hw
->aq
.arq
.count
: 0) + (ntu
- ntc
);
1027 clean_arq_element_err
:
1028 mutex_unlock(&hw
->aq
.arq_mutex
);
1030 if (i40e_is_nvm_update_op(&e
->desc
)) {
1031 if (hw
->aq
.nvm_release_on_done
) {
1032 i40e_release_nvm(hw
);
1033 hw
->aq
.nvm_release_on_done
= false;
1036 switch (hw
->nvmupd_state
) {
1037 case I40E_NVMUPD_STATE_INIT_WAIT
:
1038 hw
->nvmupd_state
= I40E_NVMUPD_STATE_INIT
;
1041 case I40E_NVMUPD_STATE_WRITE_WAIT
:
1042 hw
->nvmupd_state
= I40E_NVMUPD_STATE_WRITING
;
1053 static void i40e_resume_aq(struct i40e_hw
*hw
)
1055 /* Registers are reset after PF reset */
1056 hw
->aq
.asq
.next_to_use
= 0;
1057 hw
->aq
.asq
.next_to_clean
= 0;
1059 i40e_config_asq_regs(hw
);
1061 hw
->aq
.arq
.next_to_use
= 0;
1062 hw
->aq
.arq
.next_to_clean
= 0;
1064 i40e_config_arq_regs(hw
);