1 /*******************************************************************************
3 * Intel Ethernet Controller XL710 Family Linux Driver
4 * Copyright(c) 2013 - 2016 Intel Corporation.
6 * This program is free software; you can redistribute it and/or modify it
7 * under the terms and conditions of the GNU General Public License,
8 * version 2, as published by the Free Software Foundation.
10 * This program is distributed in the hope it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
15 * You should have received a copy of the GNU General Public License along
16 * with this program. If not, see <http://www.gnu.org/licenses/>.
18 * The full GNU General Public License is included in this distribution in
19 * the file called "COPYING".
21 * Contact Information:
22 * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
23 * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
25 ******************************************************************************/
27 #include "i40e_status.h"
28 #include "i40e_type.h"
29 #include "i40e_register.h"
30 #include "i40e_adminq.h"
31 #include "i40e_prototype.h"
33 static void i40e_resume_aq(struct i40e_hw
*hw
);
36 * i40e_adminq_init_regs - Initialize AdminQ registers
37 * @hw: pointer to the hardware structure
39 * This assumes the alloc_asq and alloc_arq functions have already been called
41 static void i40e_adminq_init_regs(struct i40e_hw
*hw
)
43 /* set head and tail registers in our local struct */
45 hw
->aq
.asq
.tail
= I40E_VF_ATQT1
;
46 hw
->aq
.asq
.head
= I40E_VF_ATQH1
;
47 hw
->aq
.asq
.len
= I40E_VF_ATQLEN1
;
48 hw
->aq
.asq
.bal
= I40E_VF_ATQBAL1
;
49 hw
->aq
.asq
.bah
= I40E_VF_ATQBAH1
;
50 hw
->aq
.arq
.tail
= I40E_VF_ARQT1
;
51 hw
->aq
.arq
.head
= I40E_VF_ARQH1
;
52 hw
->aq
.arq
.len
= I40E_VF_ARQLEN1
;
53 hw
->aq
.arq
.bal
= I40E_VF_ARQBAL1
;
54 hw
->aq
.arq
.bah
= I40E_VF_ARQBAH1
;
56 hw
->aq
.asq
.tail
= I40E_PF_ATQT
;
57 hw
->aq
.asq
.head
= I40E_PF_ATQH
;
58 hw
->aq
.asq
.len
= I40E_PF_ATQLEN
;
59 hw
->aq
.asq
.bal
= I40E_PF_ATQBAL
;
60 hw
->aq
.asq
.bah
= I40E_PF_ATQBAH
;
61 hw
->aq
.arq
.tail
= I40E_PF_ARQT
;
62 hw
->aq
.arq
.head
= I40E_PF_ARQH
;
63 hw
->aq
.arq
.len
= I40E_PF_ARQLEN
;
64 hw
->aq
.arq
.bal
= I40E_PF_ARQBAL
;
65 hw
->aq
.arq
.bah
= I40E_PF_ARQBAH
;
70 * i40e_alloc_adminq_asq_ring - Allocate Admin Queue send rings
71 * @hw: pointer to the hardware structure
73 static i40e_status
i40e_alloc_adminq_asq_ring(struct i40e_hw
*hw
)
77 ret_code
= i40e_allocate_dma_mem(hw
, &hw
->aq
.asq
.desc_buf
,
79 (hw
->aq
.num_asq_entries
*
80 sizeof(struct i40e_aq_desc
)),
81 I40E_ADMINQ_DESC_ALIGNMENT
);
85 ret_code
= i40e_allocate_virt_mem(hw
, &hw
->aq
.asq
.cmd_buf
,
86 (hw
->aq
.num_asq_entries
*
87 sizeof(struct i40e_asq_cmd_details
)));
89 i40e_free_dma_mem(hw
, &hw
->aq
.asq
.desc_buf
);
97 * i40e_alloc_adminq_arq_ring - Allocate Admin Queue receive rings
98 * @hw: pointer to the hardware structure
100 static i40e_status
i40e_alloc_adminq_arq_ring(struct i40e_hw
*hw
)
102 i40e_status ret_code
;
104 ret_code
= i40e_allocate_dma_mem(hw
, &hw
->aq
.arq
.desc_buf
,
106 (hw
->aq
.num_arq_entries
*
107 sizeof(struct i40e_aq_desc
)),
108 I40E_ADMINQ_DESC_ALIGNMENT
);
114 * i40e_free_adminq_asq - Free Admin Queue send rings
115 * @hw: pointer to the hardware structure
117 * This assumes the posted send buffers have already been cleaned
120 static void i40e_free_adminq_asq(struct i40e_hw
*hw
)
122 i40e_free_dma_mem(hw
, &hw
->aq
.asq
.desc_buf
);
126 * i40e_free_adminq_arq - Free Admin Queue receive rings
127 * @hw: pointer to the hardware structure
129 * This assumes the posted receive buffers have already been cleaned
132 static void i40e_free_adminq_arq(struct i40e_hw
*hw
)
134 i40e_free_dma_mem(hw
, &hw
->aq
.arq
.desc_buf
);
138 * i40e_alloc_arq_bufs - Allocate pre-posted buffers for the receive queue
139 * @hw: pointer to the hardware structure
141 static i40e_status
i40e_alloc_arq_bufs(struct i40e_hw
*hw
)
143 i40e_status ret_code
;
144 struct i40e_aq_desc
*desc
;
145 struct i40e_dma_mem
*bi
;
148 /* We'll be allocating the buffer info memory first, then we can
149 * allocate the mapped buffers for the event processing
152 /* buffer_info structures do not need alignment */
153 ret_code
= i40e_allocate_virt_mem(hw
, &hw
->aq
.arq
.dma_head
,
154 (hw
->aq
.num_arq_entries
* sizeof(struct i40e_dma_mem
)));
157 hw
->aq
.arq
.r
.arq_bi
= (struct i40e_dma_mem
*)hw
->aq
.arq
.dma_head
.va
;
159 /* allocate the mapped buffers */
160 for (i
= 0; i
< hw
->aq
.num_arq_entries
; i
++) {
161 bi
= &hw
->aq
.arq
.r
.arq_bi
[i
];
162 ret_code
= i40e_allocate_dma_mem(hw
, bi
,
165 I40E_ADMINQ_DESC_ALIGNMENT
);
167 goto unwind_alloc_arq_bufs
;
169 /* now configure the descriptors for use */
170 desc
= I40E_ADMINQ_DESC(hw
->aq
.arq
, i
);
172 desc
->flags
= cpu_to_le16(I40E_AQ_FLAG_BUF
);
173 if (hw
->aq
.arq_buf_size
> I40E_AQ_LARGE_BUF
)
174 desc
->flags
|= cpu_to_le16(I40E_AQ_FLAG_LB
);
176 /* This is in accordance with Admin queue design, there is no
177 * register for buffer size configuration
179 desc
->datalen
= cpu_to_le16((u16
)bi
->size
);
181 desc
->cookie_high
= 0;
182 desc
->cookie_low
= 0;
183 desc
->params
.external
.addr_high
=
184 cpu_to_le32(upper_32_bits(bi
->pa
));
185 desc
->params
.external
.addr_low
=
186 cpu_to_le32(lower_32_bits(bi
->pa
));
187 desc
->params
.external
.param0
= 0;
188 desc
->params
.external
.param1
= 0;
194 unwind_alloc_arq_bufs
:
195 /* don't try to free the one that failed... */
198 i40e_free_dma_mem(hw
, &hw
->aq
.arq
.r
.arq_bi
[i
]);
199 i40e_free_virt_mem(hw
, &hw
->aq
.arq
.dma_head
);
205 * i40e_alloc_asq_bufs - Allocate empty buffer structs for the send queue
206 * @hw: pointer to the hardware structure
208 static i40e_status
i40e_alloc_asq_bufs(struct i40e_hw
*hw
)
210 i40e_status ret_code
;
211 struct i40e_dma_mem
*bi
;
214 /* No mapped memory needed yet, just the buffer info structures */
215 ret_code
= i40e_allocate_virt_mem(hw
, &hw
->aq
.asq
.dma_head
,
216 (hw
->aq
.num_asq_entries
* sizeof(struct i40e_dma_mem
)));
219 hw
->aq
.asq
.r
.asq_bi
= (struct i40e_dma_mem
*)hw
->aq
.asq
.dma_head
.va
;
221 /* allocate the mapped buffers */
222 for (i
= 0; i
< hw
->aq
.num_asq_entries
; i
++) {
223 bi
= &hw
->aq
.asq
.r
.asq_bi
[i
];
224 ret_code
= i40e_allocate_dma_mem(hw
, bi
,
227 I40E_ADMINQ_DESC_ALIGNMENT
);
229 goto unwind_alloc_asq_bufs
;
234 unwind_alloc_asq_bufs
:
235 /* don't try to free the one that failed... */
238 i40e_free_dma_mem(hw
, &hw
->aq
.asq
.r
.asq_bi
[i
]);
239 i40e_free_virt_mem(hw
, &hw
->aq
.asq
.dma_head
);
245 * i40e_free_arq_bufs - Free receive queue buffer info elements
246 * @hw: pointer to the hardware structure
248 static void i40e_free_arq_bufs(struct i40e_hw
*hw
)
252 /* free descriptors */
253 for (i
= 0; i
< hw
->aq
.num_arq_entries
; i
++)
254 i40e_free_dma_mem(hw
, &hw
->aq
.arq
.r
.arq_bi
[i
]);
256 /* free the descriptor memory */
257 i40e_free_dma_mem(hw
, &hw
->aq
.arq
.desc_buf
);
259 /* free the dma header */
260 i40e_free_virt_mem(hw
, &hw
->aq
.arq
.dma_head
);
264 * i40e_free_asq_bufs - Free send queue buffer info elements
265 * @hw: pointer to the hardware structure
267 static void i40e_free_asq_bufs(struct i40e_hw
*hw
)
271 /* only unmap if the address is non-NULL */
272 for (i
= 0; i
< hw
->aq
.num_asq_entries
; i
++)
273 if (hw
->aq
.asq
.r
.asq_bi
[i
].pa
)
274 i40e_free_dma_mem(hw
, &hw
->aq
.asq
.r
.asq_bi
[i
]);
276 /* free the buffer info list */
277 i40e_free_virt_mem(hw
, &hw
->aq
.asq
.cmd_buf
);
279 /* free the descriptor memory */
280 i40e_free_dma_mem(hw
, &hw
->aq
.asq
.desc_buf
);
282 /* free the dma header */
283 i40e_free_virt_mem(hw
, &hw
->aq
.asq
.dma_head
);
287 * i40e_config_asq_regs - configure ASQ registers
288 * @hw: pointer to the hardware structure
290 * Configure base address and length registers for the transmit queue
292 static i40e_status
i40e_config_asq_regs(struct i40e_hw
*hw
)
294 i40e_status ret_code
= 0;
297 /* Clear Head and Tail */
298 wr32(hw
, hw
->aq
.asq
.head
, 0);
299 wr32(hw
, hw
->aq
.asq
.tail
, 0);
301 /* set starting point */
302 wr32(hw
, hw
->aq
.asq
.len
, (hw
->aq
.num_asq_entries
|
303 I40E_PF_ATQLEN_ATQENABLE_MASK
));
304 wr32(hw
, hw
->aq
.asq
.bal
, lower_32_bits(hw
->aq
.asq
.desc_buf
.pa
));
305 wr32(hw
, hw
->aq
.asq
.bah
, upper_32_bits(hw
->aq
.asq
.desc_buf
.pa
));
307 /* Check one register to verify that config was applied */
308 reg
= rd32(hw
, hw
->aq
.asq
.bal
);
309 if (reg
!= lower_32_bits(hw
->aq
.asq
.desc_buf
.pa
))
310 ret_code
= I40E_ERR_ADMIN_QUEUE_ERROR
;
316 * i40e_config_arq_regs - ARQ register configuration
317 * @hw: pointer to the hardware structure
319 * Configure base address and length registers for the receive (event queue)
321 static i40e_status
i40e_config_arq_regs(struct i40e_hw
*hw
)
323 i40e_status ret_code
= 0;
326 /* Clear Head and Tail */
327 wr32(hw
, hw
->aq
.arq
.head
, 0);
328 wr32(hw
, hw
->aq
.arq
.tail
, 0);
330 /* set starting point */
331 wr32(hw
, hw
->aq
.arq
.len
, (hw
->aq
.num_arq_entries
|
332 I40E_PF_ARQLEN_ARQENABLE_MASK
));
333 wr32(hw
, hw
->aq
.arq
.bal
, lower_32_bits(hw
->aq
.arq
.desc_buf
.pa
));
334 wr32(hw
, hw
->aq
.arq
.bah
, upper_32_bits(hw
->aq
.arq
.desc_buf
.pa
));
336 /* Update tail in the HW to post pre-allocated buffers */
337 wr32(hw
, hw
->aq
.arq
.tail
, hw
->aq
.num_arq_entries
- 1);
339 /* Check one register to verify that config was applied */
340 reg
= rd32(hw
, hw
->aq
.arq
.bal
);
341 if (reg
!= lower_32_bits(hw
->aq
.arq
.desc_buf
.pa
))
342 ret_code
= I40E_ERR_ADMIN_QUEUE_ERROR
;
348 * i40e_init_asq - main initialization routine for ASQ
349 * @hw: pointer to the hardware structure
351 * This is the main initialization routine for the Admin Send Queue
352 * Prior to calling this function, drivers *MUST* set the following fields
353 * in the hw->aq structure:
354 * - hw->aq.num_asq_entries
355 * - hw->aq.arq_buf_size
357 * Do *NOT* hold the lock when calling this as the memory allocation routines
358 * called are not going to be atomic context safe
360 static i40e_status
i40e_init_asq(struct i40e_hw
*hw
)
362 i40e_status ret_code
= 0;
364 if (hw
->aq
.asq
.count
> 0) {
365 /* queue already initialized */
366 ret_code
= I40E_ERR_NOT_READY
;
367 goto init_adminq_exit
;
370 /* verify input for valid configuration */
371 if ((hw
->aq
.num_asq_entries
== 0) ||
372 (hw
->aq
.asq_buf_size
== 0)) {
373 ret_code
= I40E_ERR_CONFIG
;
374 goto init_adminq_exit
;
377 hw
->aq
.asq
.next_to_use
= 0;
378 hw
->aq
.asq
.next_to_clean
= 0;
380 /* allocate the ring memory */
381 ret_code
= i40e_alloc_adminq_asq_ring(hw
);
383 goto init_adminq_exit
;
385 /* allocate buffers in the rings */
386 ret_code
= i40e_alloc_asq_bufs(hw
);
388 goto init_adminq_free_rings
;
390 /* initialize base registers */
391 ret_code
= i40e_config_asq_regs(hw
);
393 goto init_adminq_free_rings
;
396 hw
->aq
.asq
.count
= hw
->aq
.num_asq_entries
;
397 goto init_adminq_exit
;
399 init_adminq_free_rings
:
400 i40e_free_adminq_asq(hw
);
407 * i40e_init_arq - initialize ARQ
408 * @hw: pointer to the hardware structure
410 * The main initialization routine for the Admin Receive (Event) Queue.
411 * Prior to calling this function, drivers *MUST* set the following fields
412 * in the hw->aq structure:
413 * - hw->aq.num_asq_entries
414 * - hw->aq.arq_buf_size
416 * Do *NOT* hold the lock when calling this as the memory allocation routines
417 * called are not going to be atomic context safe
419 static i40e_status
i40e_init_arq(struct i40e_hw
*hw
)
421 i40e_status ret_code
= 0;
423 if (hw
->aq
.arq
.count
> 0) {
424 /* queue already initialized */
425 ret_code
= I40E_ERR_NOT_READY
;
426 goto init_adminq_exit
;
429 /* verify input for valid configuration */
430 if ((hw
->aq
.num_arq_entries
== 0) ||
431 (hw
->aq
.arq_buf_size
== 0)) {
432 ret_code
= I40E_ERR_CONFIG
;
433 goto init_adminq_exit
;
436 hw
->aq
.arq
.next_to_use
= 0;
437 hw
->aq
.arq
.next_to_clean
= 0;
439 /* allocate the ring memory */
440 ret_code
= i40e_alloc_adminq_arq_ring(hw
);
442 goto init_adminq_exit
;
444 /* allocate buffers in the rings */
445 ret_code
= i40e_alloc_arq_bufs(hw
);
447 goto init_adminq_free_rings
;
449 /* initialize base registers */
450 ret_code
= i40e_config_arq_regs(hw
);
452 goto init_adminq_free_rings
;
455 hw
->aq
.arq
.count
= hw
->aq
.num_arq_entries
;
456 goto init_adminq_exit
;
458 init_adminq_free_rings
:
459 i40e_free_adminq_arq(hw
);
466 * i40e_shutdown_asq - shutdown the ASQ
467 * @hw: pointer to the hardware structure
469 * The main shutdown routine for the Admin Send Queue
471 static i40e_status
i40e_shutdown_asq(struct i40e_hw
*hw
)
473 i40e_status ret_code
= 0;
475 mutex_lock(&hw
->aq
.asq_mutex
);
477 if (hw
->aq
.asq
.count
== 0) {
478 ret_code
= I40E_ERR_NOT_READY
;
479 goto shutdown_asq_out
;
482 /* Stop firmware AdminQ processing */
483 wr32(hw
, hw
->aq
.asq
.head
, 0);
484 wr32(hw
, hw
->aq
.asq
.tail
, 0);
485 wr32(hw
, hw
->aq
.asq
.len
, 0);
486 wr32(hw
, hw
->aq
.asq
.bal
, 0);
487 wr32(hw
, hw
->aq
.asq
.bah
, 0);
489 hw
->aq
.asq
.count
= 0; /* to indicate uninitialized queue */
491 /* free ring buffers */
492 i40e_free_asq_bufs(hw
);
495 mutex_unlock(&hw
->aq
.asq_mutex
);
500 * i40e_shutdown_arq - shutdown ARQ
501 * @hw: pointer to the hardware structure
503 * The main shutdown routine for the Admin Receive Queue
505 static i40e_status
i40e_shutdown_arq(struct i40e_hw
*hw
)
507 i40e_status ret_code
= 0;
509 mutex_lock(&hw
->aq
.arq_mutex
);
511 if (hw
->aq
.arq
.count
== 0) {
512 ret_code
= I40E_ERR_NOT_READY
;
513 goto shutdown_arq_out
;
516 /* Stop firmware AdminQ processing */
517 wr32(hw
, hw
->aq
.arq
.head
, 0);
518 wr32(hw
, hw
->aq
.arq
.tail
, 0);
519 wr32(hw
, hw
->aq
.arq
.len
, 0);
520 wr32(hw
, hw
->aq
.arq
.bal
, 0);
521 wr32(hw
, hw
->aq
.arq
.bah
, 0);
523 hw
->aq
.arq
.count
= 0; /* to indicate uninitialized queue */
525 /* free ring buffers */
526 i40e_free_arq_bufs(hw
);
529 mutex_unlock(&hw
->aq
.arq_mutex
);
534 * i40e_init_adminq - main initialization routine for Admin Queue
535 * @hw: pointer to the hardware structure
537 * Prior to calling this function, drivers *MUST* set the following fields
538 * in the hw->aq structure:
539 * - hw->aq.num_asq_entries
540 * - hw->aq.num_arq_entries
541 * - hw->aq.arq_buf_size
542 * - hw->aq.asq_buf_size
544 i40e_status
i40e_init_adminq(struct i40e_hw
*hw
)
546 u16 cfg_ptr
, oem_hi
, oem_lo
;
547 u16 eetrack_lo
, eetrack_hi
;
548 i40e_status ret_code
;
551 /* verify input for valid configuration */
552 if ((hw
->aq
.num_arq_entries
== 0) ||
553 (hw
->aq
.num_asq_entries
== 0) ||
554 (hw
->aq
.arq_buf_size
== 0) ||
555 (hw
->aq
.asq_buf_size
== 0)) {
556 ret_code
= I40E_ERR_CONFIG
;
557 goto init_adminq_exit
;
560 /* Set up register offsets */
561 i40e_adminq_init_regs(hw
);
563 /* setup ASQ command write back timeout */
564 hw
->aq
.asq_cmd_timeout
= I40E_ASQ_CMD_TIMEOUT
;
566 /* allocate the ASQ */
567 ret_code
= i40e_init_asq(hw
);
569 goto init_adminq_destroy_locks
;
571 /* allocate the ARQ */
572 ret_code
= i40e_init_arq(hw
);
574 goto init_adminq_free_asq
;
576 /* There are some cases where the firmware may not be quite ready
577 * for AdminQ operations, so we retry the AdminQ setup a few times
578 * if we see timeouts in this first AQ call.
581 ret_code
= i40e_aq_get_firmware_version(hw
,
588 if (ret_code
!= I40E_ERR_ADMIN_QUEUE_TIMEOUT
)
593 } while (retry
< 10);
594 if (ret_code
!= I40E_SUCCESS
)
595 goto init_adminq_free_arq
;
597 /* get the NVM version info */
598 i40e_read_nvm_word(hw
, I40E_SR_NVM_DEV_STARTER_VERSION
,
600 i40e_read_nvm_word(hw
, I40E_SR_NVM_EETRACK_LO
, &eetrack_lo
);
601 i40e_read_nvm_word(hw
, I40E_SR_NVM_EETRACK_HI
, &eetrack_hi
);
602 hw
->nvm
.eetrack
= (eetrack_hi
<< 16) | eetrack_lo
;
603 i40e_read_nvm_word(hw
, I40E_SR_BOOT_CONFIG_PTR
, &cfg_ptr
);
604 i40e_read_nvm_word(hw
, (cfg_ptr
+ I40E_NVM_OEM_VER_OFF
),
606 i40e_read_nvm_word(hw
, (cfg_ptr
+ (I40E_NVM_OEM_VER_OFF
+ 1)),
608 hw
->nvm
.oem_ver
= ((u32
)oem_hi
<< 16) | oem_lo
;
610 if (hw
->aq
.api_maj_ver
> I40E_FW_API_VERSION_MAJOR
) {
611 ret_code
= I40E_ERR_FIRMWARE_API_VERSION
;
612 goto init_adminq_free_arq
;
615 /* pre-emptive resource lock release */
616 i40e_aq_release_resource(hw
, I40E_NVM_RESOURCE_ID
, 0, NULL
);
617 hw
->nvm_release_on_done
= false;
618 hw
->nvmupd_state
= I40E_NVMUPD_STATE_INIT
;
623 goto init_adminq_exit
;
625 init_adminq_free_arq
:
626 i40e_shutdown_arq(hw
);
627 init_adminq_free_asq
:
628 i40e_shutdown_asq(hw
);
629 init_adminq_destroy_locks
:
636 * i40e_shutdown_adminq - shutdown routine for the Admin Queue
637 * @hw: pointer to the hardware structure
639 i40e_status
i40e_shutdown_adminq(struct i40e_hw
*hw
)
641 i40e_status ret_code
= 0;
643 if (i40e_check_asq_alive(hw
))
644 i40e_aq_queue_shutdown(hw
, true);
646 i40e_shutdown_asq(hw
);
647 i40e_shutdown_arq(hw
);
650 i40e_free_virt_mem(hw
, &hw
->nvm_buff
);
656 * i40e_clean_asq - cleans Admin send queue
657 * @hw: pointer to the hardware structure
659 * returns the number of free desc
661 static u16
i40e_clean_asq(struct i40e_hw
*hw
)
663 struct i40e_adminq_ring
*asq
= &(hw
->aq
.asq
);
664 struct i40e_asq_cmd_details
*details
;
665 u16 ntc
= asq
->next_to_clean
;
666 struct i40e_aq_desc desc_cb
;
667 struct i40e_aq_desc
*desc
;
669 desc
= I40E_ADMINQ_DESC(*asq
, ntc
);
670 details
= I40E_ADMINQ_DETAILS(*asq
, ntc
);
671 while (rd32(hw
, hw
->aq
.asq
.head
) != ntc
) {
672 i40e_debug(hw
, I40E_DEBUG_AQ_MESSAGE
,
673 "ntc %d head %d.\n", ntc
, rd32(hw
, hw
->aq
.asq
.head
));
675 if (details
->callback
) {
676 I40E_ADMINQ_CALLBACK cb_func
=
677 (I40E_ADMINQ_CALLBACK
)details
->callback
;
679 cb_func(hw
, &desc_cb
);
681 memset(desc
, 0, sizeof(*desc
));
682 memset(details
, 0, sizeof(*details
));
684 if (ntc
== asq
->count
)
686 desc
= I40E_ADMINQ_DESC(*asq
, ntc
);
687 details
= I40E_ADMINQ_DETAILS(*asq
, ntc
);
690 asq
->next_to_clean
= ntc
;
692 return I40E_DESC_UNUSED(asq
);
696 * i40e_asq_done - check if FW has processed the Admin Send Queue
697 * @hw: pointer to the hw struct
699 * Returns true if the firmware has processed all descriptors on the
700 * admin send queue. Returns false if there are still requests pending.
702 static bool i40e_asq_done(struct i40e_hw
*hw
)
704 /* AQ designers suggest use of head for better
705 * timing reliability than DD bit
707 return rd32(hw
, hw
->aq
.asq
.head
) == hw
->aq
.asq
.next_to_use
;
712 * i40e_asq_send_command - send command to Admin Queue
713 * @hw: pointer to the hw struct
714 * @desc: prefilled descriptor describing the command (non DMA mem)
715 * @buff: buffer to use for indirect commands
716 * @buff_size: size of buffer for indirect commands
717 * @cmd_details: pointer to command details structure
719 * This is the main send command driver routine for the Admin Queue send
720 * queue. It runs the queue, cleans the queue, etc
722 i40e_status
i40e_asq_send_command(struct i40e_hw
*hw
,
723 struct i40e_aq_desc
*desc
,
724 void *buff
, /* can be NULL */
726 struct i40e_asq_cmd_details
*cmd_details
)
728 i40e_status status
= 0;
729 struct i40e_dma_mem
*dma_buff
= NULL
;
730 struct i40e_asq_cmd_details
*details
;
731 struct i40e_aq_desc
*desc_on_ring
;
732 bool cmd_completed
= false;
736 mutex_lock(&hw
->aq
.asq_mutex
);
738 if (hw
->aq
.asq
.count
== 0) {
739 i40e_debug(hw
, I40E_DEBUG_AQ_MESSAGE
,
740 "AQTX: Admin queue not initialized.\n");
741 status
= I40E_ERR_QUEUE_EMPTY
;
742 goto asq_send_command_error
;
745 hw
->aq
.asq_last_status
= I40E_AQ_RC_OK
;
747 val
= rd32(hw
, hw
->aq
.asq
.head
);
748 if (val
>= hw
->aq
.num_asq_entries
) {
749 i40e_debug(hw
, I40E_DEBUG_AQ_MESSAGE
,
750 "AQTX: head overrun at %d\n", val
);
751 status
= I40E_ERR_QUEUE_EMPTY
;
752 goto asq_send_command_error
;
755 details
= I40E_ADMINQ_DETAILS(hw
->aq
.asq
, hw
->aq
.asq
.next_to_use
);
757 *details
= *cmd_details
;
759 /* If the cmd_details are defined copy the cookie. The
760 * cpu_to_le32 is not needed here because the data is ignored
761 * by the FW, only used by the driver
763 if (details
->cookie
) {
765 cpu_to_le32(upper_32_bits(details
->cookie
));
767 cpu_to_le32(lower_32_bits(details
->cookie
));
770 memset(details
, 0, sizeof(struct i40e_asq_cmd_details
));
773 /* clear requested flags and then set additional flags if defined */
774 desc
->flags
&= ~cpu_to_le16(details
->flags_dis
);
775 desc
->flags
|= cpu_to_le16(details
->flags_ena
);
777 if (buff_size
> hw
->aq
.asq_buf_size
) {
779 I40E_DEBUG_AQ_MESSAGE
,
780 "AQTX: Invalid buffer size: %d.\n",
782 status
= I40E_ERR_INVALID_SIZE
;
783 goto asq_send_command_error
;
786 if (details
->postpone
&& !details
->async
) {
788 I40E_DEBUG_AQ_MESSAGE
,
789 "AQTX: Async flag not set along with postpone flag");
790 status
= I40E_ERR_PARAM
;
791 goto asq_send_command_error
;
794 /* call clean and check queue available function to reclaim the
795 * descriptors that were processed by FW, the function returns the
796 * number of desc available
798 /* the clean function called here could be called in a separate thread
799 * in case of asynchronous completions
801 if (i40e_clean_asq(hw
) == 0) {
803 I40E_DEBUG_AQ_MESSAGE
,
804 "AQTX: Error queue is full.\n");
805 status
= I40E_ERR_ADMIN_QUEUE_FULL
;
806 goto asq_send_command_error
;
809 /* initialize the temp desc pointer with the right desc */
810 desc_on_ring
= I40E_ADMINQ_DESC(hw
->aq
.asq
, hw
->aq
.asq
.next_to_use
);
812 /* if the desc is available copy the temp desc to the right place */
813 *desc_on_ring
= *desc
;
815 /* if buff is not NULL assume indirect command */
817 dma_buff
= &(hw
->aq
.asq
.r
.asq_bi
[hw
->aq
.asq
.next_to_use
]);
818 /* copy the user buff into the respective DMA buff */
819 memcpy(dma_buff
->va
, buff
, buff_size
);
820 desc_on_ring
->datalen
= cpu_to_le16(buff_size
);
822 /* Update the address values in the desc with the pa value
823 * for respective buffer
825 desc_on_ring
->params
.external
.addr_high
=
826 cpu_to_le32(upper_32_bits(dma_buff
->pa
));
827 desc_on_ring
->params
.external
.addr_low
=
828 cpu_to_le32(lower_32_bits(dma_buff
->pa
));
832 i40e_debug(hw
, I40E_DEBUG_AQ_MESSAGE
, "AQTX: desc and buffer:\n");
833 i40e_debug_aq(hw
, I40E_DEBUG_AQ_COMMAND
, (void *)desc_on_ring
,
835 (hw
->aq
.asq
.next_to_use
)++;
836 if (hw
->aq
.asq
.next_to_use
== hw
->aq
.asq
.count
)
837 hw
->aq
.asq
.next_to_use
= 0;
838 if (!details
->postpone
)
839 wr32(hw
, hw
->aq
.asq
.tail
, hw
->aq
.asq
.next_to_use
);
841 /* if cmd_details are not defined or async flag is not set,
842 * we need to wait for desc write back
844 if (!details
->async
&& !details
->postpone
) {
848 /* AQ designers suggest use of head for better
849 * timing reliability than DD bit
851 if (i40e_asq_done(hw
))
853 usleep_range(1000, 2000);
855 } while (total_delay
< hw
->aq
.asq_cmd_timeout
);
858 /* if ready, copy the desc back to temp */
859 if (i40e_asq_done(hw
)) {
860 *desc
= *desc_on_ring
;
862 memcpy(buff
, dma_buff
->va
, buff_size
);
863 retval
= le16_to_cpu(desc
->retval
);
866 I40E_DEBUG_AQ_MESSAGE
,
867 "AQTX: Command completed with error 0x%X.\n",
870 /* strip off FW internal code */
873 cmd_completed
= true;
874 if ((enum i40e_admin_queue_err
)retval
== I40E_AQ_RC_OK
)
877 status
= I40E_ERR_ADMIN_QUEUE_ERROR
;
878 hw
->aq
.asq_last_status
= (enum i40e_admin_queue_err
)retval
;
881 i40e_debug(hw
, I40E_DEBUG_AQ_MESSAGE
,
882 "AQTX: desc and buffer writeback:\n");
883 i40e_debug_aq(hw
, I40E_DEBUG_AQ_COMMAND
, (void *)desc
, buff
, buff_size
);
885 /* save writeback aq if requested */
886 if (details
->wb_desc
)
887 *details
->wb_desc
= *desc_on_ring
;
889 /* update the error if time out occurred */
890 if ((!cmd_completed
) &&
891 (!details
->async
&& !details
->postpone
)) {
893 I40E_DEBUG_AQ_MESSAGE
,
894 "AQTX: Writeback timeout.\n");
895 status
= I40E_ERR_ADMIN_QUEUE_TIMEOUT
;
898 asq_send_command_error
:
899 mutex_unlock(&hw
->aq
.asq_mutex
);
904 * i40e_fill_default_direct_cmd_desc - AQ descriptor helper function
905 * @desc: pointer to the temp descriptor (non DMA mem)
906 * @opcode: the opcode can be used to decide which flags to turn off or on
908 * Fill the desc with default values
910 void i40e_fill_default_direct_cmd_desc(struct i40e_aq_desc
*desc
,
913 /* zero out the desc */
914 memset((void *)desc
, 0, sizeof(struct i40e_aq_desc
));
915 desc
->opcode
= cpu_to_le16(opcode
);
916 desc
->flags
= cpu_to_le16(I40E_AQ_FLAG_SI
);
920 * i40e_clean_arq_element
921 * @hw: pointer to the hw struct
922 * @e: event info from the receive descriptor, includes any buffers
923 * @pending: number of events that could be left to process
925 * This function cleans one Admin Receive Queue element and returns
926 * the contents through e. It can also return how many events are
927 * left to process through 'pending'
929 i40e_status
i40e_clean_arq_element(struct i40e_hw
*hw
,
930 struct i40e_arq_event_info
*e
,
933 i40e_status ret_code
= 0;
934 u16 ntc
= hw
->aq
.arq
.next_to_clean
;
935 struct i40e_aq_desc
*desc
;
936 struct i40e_dma_mem
*bi
;
942 /* pre-clean the event info */
943 memset(&e
->desc
, 0, sizeof(e
->desc
));
945 /* take the lock before we start messing with the ring */
946 mutex_lock(&hw
->aq
.arq_mutex
);
948 if (hw
->aq
.arq
.count
== 0) {
949 i40e_debug(hw
, I40E_DEBUG_AQ_MESSAGE
,
950 "AQRX: Admin queue not initialized.\n");
951 ret_code
= I40E_ERR_QUEUE_EMPTY
;
952 goto clean_arq_element_err
;
955 /* set next_to_use to head */
956 ntu
= (rd32(hw
, hw
->aq
.arq
.head
) & I40E_PF_ARQH_ARQH_MASK
);
958 /* nothing to do - shouldn't need to update ring's values */
959 ret_code
= I40E_ERR_ADMIN_QUEUE_NO_WORK
;
960 goto clean_arq_element_out
;
963 /* now clean the next descriptor */
964 desc
= I40E_ADMINQ_DESC(hw
->aq
.arq
, ntc
);
967 flags
= le16_to_cpu(desc
->flags
);
968 if (flags
& I40E_AQ_FLAG_ERR
) {
969 ret_code
= I40E_ERR_ADMIN_QUEUE_ERROR
;
970 hw
->aq
.arq_last_status
=
971 (enum i40e_admin_queue_err
)le16_to_cpu(desc
->retval
);
973 I40E_DEBUG_AQ_MESSAGE
,
974 "AQRX: Event received with error 0x%X.\n",
975 hw
->aq
.arq_last_status
);
979 datalen
= le16_to_cpu(desc
->datalen
);
980 e
->msg_len
= min(datalen
, e
->buf_len
);
981 if (e
->msg_buf
!= NULL
&& (e
->msg_len
!= 0))
982 memcpy(e
->msg_buf
, hw
->aq
.arq
.r
.arq_bi
[desc_idx
].va
,
985 i40e_debug(hw
, I40E_DEBUG_AQ_MESSAGE
, "AQRX: desc and buffer:\n");
986 i40e_debug_aq(hw
, I40E_DEBUG_AQ_COMMAND
, (void *)desc
, e
->msg_buf
,
987 hw
->aq
.arq_buf_size
);
989 /* Restore the original datalen and buffer address in the desc,
990 * FW updates datalen to indicate the event message
993 bi
= &hw
->aq
.arq
.r
.arq_bi
[ntc
];
994 memset((void *)desc
, 0, sizeof(struct i40e_aq_desc
));
996 desc
->flags
= cpu_to_le16(I40E_AQ_FLAG_BUF
);
997 if (hw
->aq
.arq_buf_size
> I40E_AQ_LARGE_BUF
)
998 desc
->flags
|= cpu_to_le16(I40E_AQ_FLAG_LB
);
999 desc
->datalen
= cpu_to_le16((u16
)bi
->size
);
1000 desc
->params
.external
.addr_high
= cpu_to_le32(upper_32_bits(bi
->pa
));
1001 desc
->params
.external
.addr_low
= cpu_to_le32(lower_32_bits(bi
->pa
));
1003 /* set tail = the last cleaned desc index. */
1004 wr32(hw
, hw
->aq
.arq
.tail
, ntc
);
1005 /* ntc is updated to tail + 1 */
1007 if (ntc
== hw
->aq
.num_arq_entries
)
1009 hw
->aq
.arq
.next_to_clean
= ntc
;
1010 hw
->aq
.arq
.next_to_use
= ntu
;
1012 i40e_nvmupd_check_wait_event(hw
, le16_to_cpu(e
->desc
.opcode
));
1013 clean_arq_element_out
:
1014 /* Set pending if needed, unlock and return */
1016 *pending
= (ntc
> ntu
? hw
->aq
.arq
.count
: 0) + (ntu
- ntc
);
1017 clean_arq_element_err
:
1018 mutex_unlock(&hw
->aq
.arq_mutex
);
1023 static void i40e_resume_aq(struct i40e_hw
*hw
)
1025 /* Registers are reset after PF reset */
1026 hw
->aq
.asq
.next_to_use
= 0;
1027 hw
->aq
.asq
.next_to_clean
= 0;
1029 i40e_config_asq_regs(hw
);
1031 hw
->aq
.arq
.next_to_use
= 0;
1032 hw
->aq
.arq
.next_to_clean
= 0;
1034 i40e_config_arq_regs(hw
);