1 /******************************************************************************
3 * This file is provided under a dual BSD/GPLv2 license. When using or
4 * redistributing this file, you may do so under either license.
8 * Copyright(c) 2017 Intel Deutschland GmbH
10 * This program is free software; you can redistribute it and/or modify
11 * it under the terms of version 2 of the GNU General Public License as
12 * published by the Free Software Foundation.
14 * This program is distributed in the hope that it will be useful, but
15 * WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
17 * General Public License for more details.
21 * Copyright(c) 2017 Intel Deutschland GmbH
22 * All rights reserved.
24 * Redistribution and use in source and binary forms, with or without
25 * modification, are permitted provided that the following conditions
28 * * Redistributions of source code must retain the above copyright
29 * notice, this list of conditions and the following disclaimer.
30 * * Redistributions in binary form must reproduce the above copyright
31 * notice, this list of conditions and the following disclaimer in
32 * the documentation and/or other materials provided with the
34 * * Neither the name Intel Corporation nor the names of its
35 * contributors may be used to endorse or promote products derived
36 * from this software without specific prior written permission.
38 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
39 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
40 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
41 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
42 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
43 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
44 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
45 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
46 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
47 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
48 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
50 *****************************************************************************/
51 #include <linux/pm_runtime.h>
54 #include "iwl-debug.h"
58 #include "fw/api/tx.h"
61 * iwl_pcie_gen2_tx_stop - Stop all Tx DMA channels
63 void iwl_pcie_gen2_tx_stop(struct iwl_trans
*trans
)
65 struct iwl_trans_pcie
*trans_pcie
= IWL_TRANS_GET_PCIE_TRANS(trans
);
69 * This function can be called before the op_mode disabled the
70 * queues. This happens when we have an rfkill interrupt.
71 * Since we stop Tx altogether - mark the queues as stopped.
73 memset(trans_pcie
->queue_stopped
, 0, sizeof(trans_pcie
->queue_stopped
));
74 memset(trans_pcie
->queue_used
, 0, sizeof(trans_pcie
->queue_used
));
76 /* Unmap DMA from host system and free skb's */
77 for (txq_id
= 0; txq_id
< ARRAY_SIZE(trans_pcie
->txq
); txq_id
++) {
78 if (!trans_pcie
->txq
[txq_id
])
80 iwl_pcie_gen2_txq_unmap(trans
, txq_id
);
85 * iwl_pcie_txq_update_byte_tbl - Set up entry in Tx byte-count array
87 static void iwl_pcie_gen2_update_byte_tbl(struct iwl_txq
*txq
, u16 byte_cnt
,
90 struct iwlagn_scd_bc_tbl
*scd_bc_tbl
= txq
->bc_tbl
.addr
;
91 int idx
= iwl_pcie_get_cmd_index(txq
, txq
->write_ptr
);
92 u8 filled_tfd_size
, num_fetch_chunks
;
96 len
= DIV_ROUND_UP(len
, 4);
98 if (WARN_ON(len
> 0xFFF || idx
>= txq
->n_window
))
101 filled_tfd_size
= offsetof(struct iwl_tfh_tfd
, tbs
) +
102 num_tbs
* sizeof(struct iwl_tfh_tb
);
104 * filled_tfd_size contains the number of filled bytes in the TFD.
105 * Dividing it by 64 will give the number of chunks to fetch
106 * to SRAM- 0 for one chunk, 1 for 2 and so on.
107 * If, for example, TFD contains only 3 TBs then 32 bytes
108 * of the TFD are used, and only one chunk of 64 bytes should
111 num_fetch_chunks
= DIV_ROUND_UP(filled_tfd_size
, 64) - 1;
113 bc_ent
= cpu_to_le16(len
| (num_fetch_chunks
<< 12));
114 scd_bc_tbl
->tfd_offset
[idx
] = bc_ent
;
118 * iwl_pcie_gen2_txq_inc_wr_ptr - Send new write index to hardware
120 static void iwl_pcie_gen2_txq_inc_wr_ptr(struct iwl_trans
*trans
,
123 lockdep_assert_held(&txq
->lock
);
125 IWL_DEBUG_TX(trans
, "Q:%d WR: 0x%x\n", txq
->id
, txq
->write_ptr
);
128 * if not in power-save mode, uCode will never sleep when we're
129 * trying to tx (during RFKILL, we're not trying to tx).
131 iwl_write32(trans
, HBUS_TARG_WRPTR
, txq
->write_ptr
| (txq
->id
<< 16));
134 static u8
iwl_pcie_gen2_get_num_tbs(struct iwl_trans
*trans
,
135 struct iwl_tfh_tfd
*tfd
)
137 return le16_to_cpu(tfd
->num_tbs
) & 0x1f;
140 static void iwl_pcie_gen2_tfd_unmap(struct iwl_trans
*trans
,
141 struct iwl_cmd_meta
*meta
,
142 struct iwl_tfh_tfd
*tfd
)
144 struct iwl_trans_pcie
*trans_pcie
= IWL_TRANS_GET_PCIE_TRANS(trans
);
147 /* Sanity check on number of chunks */
148 num_tbs
= iwl_pcie_gen2_get_num_tbs(trans
, tfd
);
150 if (num_tbs
>= trans_pcie
->max_tbs
) {
151 IWL_ERR(trans
, "Too many chunks: %i\n", num_tbs
);
155 /* first TB is never freed - it's the bidirectional DMA data */
156 for (i
= 1; i
< num_tbs
; i
++) {
157 if (meta
->tbs
& BIT(i
))
158 dma_unmap_page(trans
->dev
,
159 le64_to_cpu(tfd
->tbs
[i
].addr
),
160 le16_to_cpu(tfd
->tbs
[i
].tb_len
),
163 dma_unmap_single(trans
->dev
,
164 le64_to_cpu(tfd
->tbs
[i
].addr
),
165 le16_to_cpu(tfd
->tbs
[i
].tb_len
),
172 static void iwl_pcie_gen2_free_tfd(struct iwl_trans
*trans
, struct iwl_txq
*txq
)
174 struct iwl_trans_pcie
*trans_pcie
= IWL_TRANS_GET_PCIE_TRANS(trans
);
176 /* rd_ptr is bounded by TFD_QUEUE_SIZE_MAX and
177 * idx is bounded by n_window
179 int idx
= iwl_pcie_get_cmd_index(txq
, txq
->read_ptr
);
181 lockdep_assert_held(&txq
->lock
);
183 iwl_pcie_gen2_tfd_unmap(trans
, &txq
->entries
[idx
].meta
,
184 iwl_pcie_get_tfd(trans_pcie
, txq
, idx
));
190 skb
= txq
->entries
[idx
].skb
;
192 /* Can be called from irqs-disabled context
193 * If skb is not NULL, it means that the whole queue is being
194 * freed and that the queue is not empty - free the skb
197 iwl_op_mode_free_skb(trans
->op_mode
, skb
);
198 txq
->entries
[idx
].skb
= NULL
;
203 static int iwl_pcie_gen2_set_tb(struct iwl_trans
*trans
,
204 struct iwl_tfh_tfd
*tfd
, dma_addr_t addr
,
207 struct iwl_trans_pcie
*trans_pcie
= IWL_TRANS_GET_PCIE_TRANS(trans
);
208 int idx
= iwl_pcie_gen2_get_num_tbs(trans
, tfd
);
209 struct iwl_tfh_tb
*tb
= &tfd
->tbs
[idx
];
211 /* Each TFD can point to a maximum max_tbs Tx buffers */
212 if (le16_to_cpu(tfd
->num_tbs
) >= trans_pcie
->max_tbs
) {
213 IWL_ERR(trans
, "Error can not send more than %d chunks\n",
214 trans_pcie
->max_tbs
);
218 put_unaligned_le64(addr
, &tb
->addr
);
219 tb
->tb_len
= cpu_to_le16(len
);
221 tfd
->num_tbs
= cpu_to_le16(idx
+ 1);
226 static int iwl_pcie_gen2_build_amsdu(struct iwl_trans
*trans
,
228 struct iwl_tfh_tfd
*tfd
, int start_len
,
229 u8 hdr_len
, struct iwl_device_cmd
*dev_cmd
)
232 struct iwl_trans_pcie
*trans_pcie
= IWL_TRANS_GET_PCIE_TRANS(trans
);
233 struct iwl_tx_cmd
*tx_cmd
= (void *)dev_cmd
->payload
;
234 struct ieee80211_hdr
*hdr
= (void *)skb
->data
;
235 unsigned int snap_ip_tcp_hdrlen
, ip_hdrlen
, total_len
, hdr_room
;
236 unsigned int mss
= skb_shinfo(skb
)->gso_size
;
237 u16 length
, iv_len
, amsdu_pad
;
239 struct iwl_tso_hdr_page
*hdr_page
;
240 struct page
**page_ptr
;
243 /* if the packet is protected, then it must be CCMP or GCMP */
244 iv_len
= ieee80211_has_protected(hdr
->frame_control
) ?
245 IEEE80211_CCMP_HDR_LEN
: 0;
247 trace_iwlwifi_dev_tx(trans
->dev
, skb
, tfd
, sizeof(*tfd
),
248 &dev_cmd
->hdr
, start_len
, 0);
250 ip_hdrlen
= skb_transport_header(skb
) - skb_network_header(skb
);
251 snap_ip_tcp_hdrlen
= 8 + ip_hdrlen
+ tcp_hdrlen(skb
);
252 total_len
= skb
->len
- snap_ip_tcp_hdrlen
- hdr_len
- iv_len
;
255 /* total amount of header we may need for this A-MSDU */
256 hdr_room
= DIV_ROUND_UP(total_len
, mss
) *
257 (3 + snap_ip_tcp_hdrlen
+ sizeof(struct ethhdr
)) + iv_len
;
259 /* Our device supports 9 segments at most, it will fit in 1 page */
260 hdr_page
= get_page_hdr(trans
, hdr_room
);
264 get_page(hdr_page
->page
);
265 start_hdr
= hdr_page
->pos
;
266 page_ptr
= (void *)((u8
*)skb
->cb
+ trans_pcie
->page_offs
);
267 *page_ptr
= hdr_page
->page
;
268 memcpy(hdr_page
->pos
, skb
->data
+ hdr_len
, iv_len
);
269 hdr_page
->pos
+= iv_len
;
272 * Pull the ieee80211 header + IV to be able to use TSO core,
273 * we will restore it for the tx_status flow.
275 skb_pull(skb
, hdr_len
+ iv_len
);
278 * Remove the length of all the headers that we don't actually
279 * have in the MPDU by themselves, but that we duplicate into
280 * all the different MSDUs inside the A-MSDU.
282 le16_add_cpu(&tx_cmd
->len
, -snap_ip_tcp_hdrlen
);
284 tso_start(skb
, &tso
);
287 /* this is the data left for this subframe */
288 unsigned int data_left
= min_t(unsigned int, mss
, total_len
);
289 struct sk_buff
*csum_skb
= NULL
;
292 u8
*subf_hdrs_start
= hdr_page
->pos
;
294 total_len
-= data_left
;
296 memset(hdr_page
->pos
, 0, amsdu_pad
);
297 hdr_page
->pos
+= amsdu_pad
;
298 amsdu_pad
= (4 - (sizeof(struct ethhdr
) + snap_ip_tcp_hdrlen
+
300 ether_addr_copy(hdr_page
->pos
, ieee80211_get_DA(hdr
));
301 hdr_page
->pos
+= ETH_ALEN
;
302 ether_addr_copy(hdr_page
->pos
, ieee80211_get_SA(hdr
));
303 hdr_page
->pos
+= ETH_ALEN
;
305 length
= snap_ip_tcp_hdrlen
+ data_left
;
306 *((__be16
*)hdr_page
->pos
) = cpu_to_be16(length
);
307 hdr_page
->pos
+= sizeof(length
);
310 * This will copy the SNAP as well which will be considered
313 tso_build_hdr(skb
, hdr_page
->pos
, &tso
, data_left
, !total_len
);
315 hdr_page
->pos
+= snap_ip_tcp_hdrlen
;
317 tb_len
= hdr_page
->pos
- start_hdr
;
318 tb_phys
= dma_map_single(trans
->dev
, start_hdr
,
319 tb_len
, DMA_TO_DEVICE
);
320 if (unlikely(dma_mapping_error(trans
->dev
, tb_phys
))) {
321 dev_kfree_skb(csum_skb
);
324 iwl_pcie_gen2_set_tb(trans
, tfd
, tb_phys
, tb_len
);
325 trace_iwlwifi_dev_tx_tso_chunk(trans
->dev
, start_hdr
, tb_len
);
326 /* add this subframe's headers' length to the tx_cmd */
327 le16_add_cpu(&tx_cmd
->len
, hdr_page
->pos
- subf_hdrs_start
);
329 /* prepare the start_hdr for the next subframe */
330 start_hdr
= hdr_page
->pos
;
332 /* put the payload */
334 tb_len
= min_t(unsigned int, tso
.size
, data_left
);
335 tb_phys
= dma_map_single(trans
->dev
, tso
.data
,
336 tb_len
, DMA_TO_DEVICE
);
337 if (unlikely(dma_mapping_error(trans
->dev
, tb_phys
))) {
338 dev_kfree_skb(csum_skb
);
341 iwl_pcie_gen2_set_tb(trans
, tfd
, tb_phys
, tb_len
);
342 trace_iwlwifi_dev_tx_tso_chunk(trans
->dev
, tso
.data
,
346 tso_build_data(skb
, &tso
, tb_len
);
350 /* re -add the WiFi header and IV */
351 skb_push(skb
, hdr_len
+ iv_len
);
361 struct iwl_tfh_tfd
*iwl_pcie_gen2_build_tfd(struct iwl_trans
*trans
,
363 struct iwl_device_cmd
*dev_cmd
,
365 struct iwl_cmd_meta
*out_meta
)
367 struct iwl_trans_pcie
*trans_pcie
= IWL_TRANS_GET_PCIE_TRANS(trans
);
368 struct ieee80211_hdr
*hdr
= (struct ieee80211_hdr
*)skb
->data
;
369 int idx
= iwl_pcie_get_cmd_index(txq
, txq
->write_ptr
);
370 struct iwl_tfh_tfd
*tfd
=
371 iwl_pcie_get_tfd(trans_pcie
, txq
, idx
);
374 int i
, len
, tb1_len
, tb2_len
, hdr_len
;
377 memset(tfd
, 0, sizeof(*tfd
));
379 amsdu
= ieee80211_is_data_qos(hdr
->frame_control
) &&
380 (*ieee80211_get_qos_ctl(hdr
) &
381 IEEE80211_QOS_CTL_A_MSDU_PRESENT
);
383 tb_phys
= iwl_pcie_get_first_tb_dma(txq
, idx
);
384 /* The first TB points to bi-directional DMA data */
386 memcpy(&txq
->first_tb_bufs
[idx
], &dev_cmd
->hdr
,
389 iwl_pcie_gen2_set_tb(trans
, tfd
, tb_phys
, IWL_FIRST_TB_SIZE
);
391 /* there must be data left over for TB1 or this code must be changed */
392 BUILD_BUG_ON(sizeof(struct iwl_tx_cmd_gen2
) < IWL_FIRST_TB_SIZE
);
395 * The second TB (tb1) points to the remainder of the TX command
396 * and the 802.11 header - dword aligned size
397 * (This calculation modifies the TX command, so do it before the
398 * setup of the first TB)
400 len
= sizeof(struct iwl_tx_cmd_gen2
) + sizeof(struct iwl_cmd_header
) +
401 ieee80211_hdrlen(hdr
->frame_control
) - IWL_FIRST_TB_SIZE
;
403 /* do not align A-MSDU to dword as the subframe header aligns it */
407 tb1_len
= ALIGN(len
, 4);
409 /* map the data for TB1 */
410 tb1_addr
= ((u8
*)&dev_cmd
->hdr
) + IWL_FIRST_TB_SIZE
;
411 tb_phys
= dma_map_single(trans
->dev
, tb1_addr
, tb1_len
, DMA_TO_DEVICE
);
412 if (unlikely(dma_mapping_error(trans
->dev
, tb_phys
)))
414 iwl_pcie_gen2_set_tb(trans
, tfd
, tb_phys
, tb1_len
);
416 hdr_len
= ieee80211_hdrlen(hdr
->frame_control
);
419 if (iwl_pcie_gen2_build_amsdu(trans
, skb
, tfd
,
420 tb1_len
+ IWL_FIRST_TB_SIZE
,
425 * building the A-MSDU might have changed this data, so memcpy
428 memcpy(&txq
->first_tb_bufs
[idx
], &dev_cmd
->hdr
,
433 /* set up TFD's third entry to point to remainder of skb's head */
434 tb2_len
= skb_headlen(skb
) - hdr_len
;
437 tb_phys
= dma_map_single(trans
->dev
, skb
->data
+ hdr_len
,
438 tb2_len
, DMA_TO_DEVICE
);
439 if (unlikely(dma_mapping_error(trans
->dev
, tb_phys
)))
441 iwl_pcie_gen2_set_tb(trans
, tfd
, tb_phys
, tb2_len
);
444 /* set up the remaining entries to point to the data */
445 for (i
= 0; i
< skb_shinfo(skb
)->nr_frags
; i
++) {
446 const skb_frag_t
*frag
= &skb_shinfo(skb
)->frags
[i
];
449 if (!skb_frag_size(frag
))
452 tb_phys
= skb_frag_dma_map(trans
->dev
, frag
, 0,
453 skb_frag_size(frag
), DMA_TO_DEVICE
);
455 if (unlikely(dma_mapping_error(trans
->dev
, tb_phys
)))
457 tb_idx
= iwl_pcie_gen2_set_tb(trans
, tfd
, tb_phys
,
458 skb_frag_size(frag
));
460 out_meta
->tbs
|= BIT(tb_idx
);
463 trace_iwlwifi_dev_tx(trans
->dev
, skb
, tfd
, sizeof(*tfd
), &dev_cmd
->hdr
,
464 IWL_FIRST_TB_SIZE
+ tb1_len
, hdr_len
);
465 trace_iwlwifi_dev_tx_data(trans
->dev
, skb
, hdr_len
);
470 iwl_pcie_gen2_tfd_unmap(trans
, out_meta
, tfd
);
474 int iwl_trans_pcie_gen2_tx(struct iwl_trans
*trans
, struct sk_buff
*skb
,
475 struct iwl_device_cmd
*dev_cmd
, int txq_id
)
477 struct iwl_trans_pcie
*trans_pcie
= IWL_TRANS_GET_PCIE_TRANS(trans
);
478 struct iwl_tx_cmd_gen2
*tx_cmd
= (void *)dev_cmd
->payload
;
479 struct iwl_cmd_meta
*out_meta
;
480 struct iwl_txq
*txq
= trans_pcie
->txq
[txq_id
];
484 if (WARN_ONCE(!test_bit(txq_id
, trans_pcie
->queue_used
),
485 "TX on unused queue %d\n", txq_id
))
488 if (skb_is_nonlinear(skb
) &&
489 skb_shinfo(skb
)->nr_frags
> IWL_PCIE_MAX_FRAGS(trans_pcie
) &&
490 __skb_linearize(skb
))
493 spin_lock(&txq
->lock
);
495 idx
= iwl_pcie_get_cmd_index(txq
, txq
->write_ptr
);
497 /* Set up driver data for this TFD */
498 txq
->entries
[idx
].skb
= skb
;
499 txq
->entries
[idx
].cmd
= dev_cmd
;
501 dev_cmd
->hdr
.sequence
=
502 cpu_to_le16((u16
)(QUEUE_TO_SEQ(txq_id
) |
505 /* Set up first empty entry in queue's array of Tx/cmd buffers */
506 out_meta
= &txq
->entries
[idx
].meta
;
509 tfd
= iwl_pcie_gen2_build_tfd(trans
, txq
, dev_cmd
, skb
, out_meta
);
511 spin_unlock(&txq
->lock
);
515 /* Set up entry for this TFD in Tx byte-count array */
516 iwl_pcie_gen2_update_byte_tbl(txq
, le16_to_cpu(tx_cmd
->len
),
517 iwl_pcie_gen2_get_num_tbs(trans
, tfd
));
519 /* start timer if queue currently empty */
520 if (txq
->read_ptr
== txq
->write_ptr
) {
522 mod_timer(&txq
->stuck_timer
, jiffies
+ txq
->wd_timeout
);
523 IWL_DEBUG_RPM(trans
, "Q: %d first tx - take ref\n", txq
->id
);
524 iwl_trans_ref(trans
);
527 /* Tell device the write index *just past* this latest filled TFD */
528 txq
->write_ptr
= iwl_queue_inc_wrap(txq
->write_ptr
);
529 iwl_pcie_gen2_txq_inc_wr_ptr(trans
, txq
);
530 if (iwl_queue_space(txq
) < txq
->high_mark
)
531 iwl_stop_queue(trans
, txq
);
534 * At this point the frame is "transmitted" successfully
535 * and we will get a TX status notification eventually.
537 spin_unlock(&txq
->lock
);
541 /*************** HOST COMMAND QUEUE FUNCTIONS *****/
544 * iwl_pcie_gen2_enqueue_hcmd - enqueue a uCode command
545 * @priv: device private data point
546 * @cmd: a pointer to the ucode command structure
548 * The function returns < 0 values to indicate the operation
549 * failed. On success, it returns the index (>= 0) of command in the
552 static int iwl_pcie_gen2_enqueue_hcmd(struct iwl_trans
*trans
,
553 struct iwl_host_cmd
*cmd
)
555 struct iwl_trans_pcie
*trans_pcie
= IWL_TRANS_GET_PCIE_TRANS(trans
);
556 struct iwl_txq
*txq
= trans_pcie
->txq
[trans_pcie
->cmd_queue
];
557 struct iwl_device_cmd
*out_cmd
;
558 struct iwl_cmd_meta
*out_meta
;
560 void *dup_buf
= NULL
;
561 dma_addr_t phys_addr
;
562 int i
, cmd_pos
, idx
= iwl_pcie_get_cmd_index(txq
, txq
->write_ptr
);
563 u16 copy_size
, cmd_size
, tb0_size
;
564 bool had_nocopy
= false;
565 u8 group_id
= iwl_cmd_groupid(cmd
->id
);
566 const u8
*cmddata
[IWL_MAX_CMD_TBS_PER_TFD
];
567 u16 cmdlen
[IWL_MAX_CMD_TBS_PER_TFD
];
568 struct iwl_tfh_tfd
*tfd
=
569 iwl_pcie_get_tfd(trans_pcie
, txq
, txq
->write_ptr
);
571 memset(tfd
, 0, sizeof(*tfd
));
573 copy_size
= sizeof(struct iwl_cmd_header_wide
);
574 cmd_size
= sizeof(struct iwl_cmd_header_wide
);
576 for (i
= 0; i
< IWL_MAX_CMD_TBS_PER_TFD
; i
++) {
577 cmddata
[i
] = cmd
->data
[i
];
578 cmdlen
[i
] = cmd
->len
[i
];
583 /* need at least IWL_FIRST_TB_SIZE copied */
584 if (copy_size
< IWL_FIRST_TB_SIZE
) {
585 int copy
= IWL_FIRST_TB_SIZE
- copy_size
;
587 if (copy
> cmdlen
[i
])
594 if (cmd
->dataflags
[i
] & IWL_HCMD_DFL_NOCOPY
) {
596 if (WARN_ON(cmd
->dataflags
[i
] & IWL_HCMD_DFL_DUP
)) {
600 } else if (cmd
->dataflags
[i
] & IWL_HCMD_DFL_DUP
) {
602 * This is also a chunk that isn't copied
603 * to the static buffer so set had_nocopy.
607 /* only allowed once */
608 if (WARN_ON(dup_buf
)) {
613 dup_buf
= kmemdup(cmddata
[i
], cmdlen
[i
],
618 /* NOCOPY must not be followed by normal! */
619 if (WARN_ON(had_nocopy
)) {
623 copy_size
+= cmdlen
[i
];
625 cmd_size
+= cmd
->len
[i
];
629 * If any of the command structures end up being larger than the
630 * TFD_MAX_PAYLOAD_SIZE and they aren't dynamically allocated into
631 * separate TFDs, then we will need to increase the size of the buffers
633 if (WARN(copy_size
> TFD_MAX_PAYLOAD_SIZE
,
634 "Command %s (%#x) is too large (%d bytes)\n",
635 iwl_get_cmd_string(trans
, cmd
->id
), cmd
->id
, copy_size
)) {
640 spin_lock_bh(&txq
->lock
);
642 if (iwl_queue_space(txq
) < ((cmd
->flags
& CMD_ASYNC
) ? 2 : 1)) {
643 spin_unlock_bh(&txq
->lock
);
645 IWL_ERR(trans
, "No space in command queue\n");
646 iwl_op_mode_cmd_queue_full(trans
->op_mode
);
651 out_cmd
= txq
->entries
[idx
].cmd
;
652 out_meta
= &txq
->entries
[idx
].meta
;
654 /* re-initialize to NULL */
655 memset(out_meta
, 0, sizeof(*out_meta
));
656 if (cmd
->flags
& CMD_WANT_SKB
)
657 out_meta
->source
= cmd
;
659 /* set up the header */
660 out_cmd
->hdr_wide
.cmd
= iwl_cmd_opcode(cmd
->id
);
661 out_cmd
->hdr_wide
.group_id
= group_id
;
662 out_cmd
->hdr_wide
.version
= iwl_cmd_version(cmd
->id
);
663 out_cmd
->hdr_wide
.length
=
664 cpu_to_le16(cmd_size
- sizeof(struct iwl_cmd_header_wide
));
665 out_cmd
->hdr_wide
.reserved
= 0;
666 out_cmd
->hdr_wide
.sequence
=
667 cpu_to_le16(QUEUE_TO_SEQ(trans_pcie
->cmd_queue
) |
668 INDEX_TO_SEQ(txq
->write_ptr
));
670 cmd_pos
= sizeof(struct iwl_cmd_header_wide
);
671 copy_size
= sizeof(struct iwl_cmd_header_wide
);
673 /* and copy the data that needs to be copied */
674 for (i
= 0; i
< IWL_MAX_CMD_TBS_PER_TFD
; i
++) {
680 /* copy everything if not nocopy/dup */
681 if (!(cmd
->dataflags
[i
] & (IWL_HCMD_DFL_NOCOPY
|
682 IWL_HCMD_DFL_DUP
))) {
685 memcpy((u8
*)out_cmd
+ cmd_pos
, cmd
->data
[i
], copy
);
692 * Otherwise we need at least IWL_FIRST_TB_SIZE copied
693 * in total (for bi-directional DMA), but copy up to what
694 * we can fit into the payload for debug dump purposes.
696 copy
= min_t(int, TFD_MAX_PAYLOAD_SIZE
- cmd_pos
, cmd
->len
[i
]);
698 memcpy((u8
*)out_cmd
+ cmd_pos
, cmd
->data
[i
], copy
);
701 /* However, treat copy_size the proper way, we need it below */
702 if (copy_size
< IWL_FIRST_TB_SIZE
) {
703 copy
= IWL_FIRST_TB_SIZE
- copy_size
;
705 if (copy
> cmd
->len
[i
])
712 "Sending command %s (%.2x.%.2x), seq: 0x%04X, %d bytes at %d[%d]:%d\n",
713 iwl_get_cmd_string(trans
, cmd
->id
), group_id
,
714 out_cmd
->hdr
.cmd
, le16_to_cpu(out_cmd
->hdr
.sequence
),
715 cmd_size
, txq
->write_ptr
, idx
, trans_pcie
->cmd_queue
);
717 /* start the TFD with the minimum copy bytes */
718 tb0_size
= min_t(int, copy_size
, IWL_FIRST_TB_SIZE
);
719 memcpy(&txq
->first_tb_bufs
[idx
], &out_cmd
->hdr
, tb0_size
);
720 iwl_pcie_gen2_set_tb(trans
, tfd
, iwl_pcie_get_first_tb_dma(txq
, idx
),
723 /* map first command fragment, if any remains */
724 if (copy_size
> tb0_size
) {
725 phys_addr
= dma_map_single(trans
->dev
,
726 ((u8
*)&out_cmd
->hdr
) + tb0_size
,
727 copy_size
- tb0_size
,
729 if (dma_mapping_error(trans
->dev
, phys_addr
)) {
731 iwl_pcie_gen2_tfd_unmap(trans
, out_meta
, tfd
);
734 iwl_pcie_gen2_set_tb(trans
, tfd
, phys_addr
,
735 copy_size
- tb0_size
);
738 /* map the remaining (adjusted) nocopy/dup fragments */
739 for (i
= 0; i
< IWL_MAX_CMD_TBS_PER_TFD
; i
++) {
740 const void *data
= cmddata
[i
];
744 if (!(cmd
->dataflags
[i
] & (IWL_HCMD_DFL_NOCOPY
|
747 if (cmd
->dataflags
[i
] & IWL_HCMD_DFL_DUP
)
749 phys_addr
= dma_map_single(trans
->dev
, (void *)data
,
750 cmdlen
[i
], DMA_TO_DEVICE
);
751 if (dma_mapping_error(trans
->dev
, phys_addr
)) {
753 iwl_pcie_gen2_tfd_unmap(trans
, out_meta
, tfd
);
756 iwl_pcie_gen2_set_tb(trans
, tfd
, phys_addr
, cmdlen
[i
]);
759 BUILD_BUG_ON(IWL_TFH_NUM_TBS
> sizeof(out_meta
->tbs
) * BITS_PER_BYTE
);
760 out_meta
->flags
= cmd
->flags
;
761 if (WARN_ON_ONCE(txq
->entries
[idx
].free_buf
))
762 kzfree(txq
->entries
[idx
].free_buf
);
763 txq
->entries
[idx
].free_buf
= dup_buf
;
765 trace_iwlwifi_dev_hcmd(trans
->dev
, cmd
, cmd_size
, &out_cmd
->hdr_wide
);
767 /* start timer if queue currently empty */
768 if (txq
->read_ptr
== txq
->write_ptr
&& txq
->wd_timeout
)
769 mod_timer(&txq
->stuck_timer
, jiffies
+ txq
->wd_timeout
);
771 spin_lock_irqsave(&trans_pcie
->reg_lock
, flags
);
772 if (!(cmd
->flags
& CMD_SEND_IN_IDLE
) &&
773 !trans_pcie
->ref_cmd_in_flight
) {
774 trans_pcie
->ref_cmd_in_flight
= true;
775 IWL_DEBUG_RPM(trans
, "set ref_cmd_in_flight - ref\n");
776 iwl_trans_ref(trans
);
778 /* Increment and update queue's write index */
779 txq
->write_ptr
= iwl_queue_inc_wrap(txq
->write_ptr
);
780 iwl_pcie_gen2_txq_inc_wr_ptr(trans
, txq
);
781 spin_unlock_irqrestore(&trans_pcie
->reg_lock
, flags
);
784 spin_unlock_bh(&txq
->lock
);
791 #define HOST_COMPLETE_TIMEOUT (2 * HZ)
793 static int iwl_pcie_gen2_send_hcmd_sync(struct iwl_trans
*trans
,
794 struct iwl_host_cmd
*cmd
)
796 struct iwl_trans_pcie
*trans_pcie
= IWL_TRANS_GET_PCIE_TRANS(trans
);
797 const char *cmd_str
= iwl_get_cmd_string(trans
, cmd
->id
);
798 struct iwl_txq
*txq
= trans_pcie
->txq
[trans_pcie
->cmd_queue
];
802 IWL_DEBUG_INFO(trans
, "Attempting to send sync command %s\n", cmd_str
);
804 if (WARN(test_and_set_bit(STATUS_SYNC_HCMD_ACTIVE
,
806 "Command %s: a command is already active!\n", cmd_str
))
809 IWL_DEBUG_INFO(trans
, "Setting HCMD_ACTIVE for command %s\n", cmd_str
);
811 if (pm_runtime_suspended(&trans_pcie
->pci_dev
->dev
)) {
812 ret
= wait_event_timeout(trans_pcie
->d0i3_waitq
,
813 pm_runtime_active(&trans_pcie
->pci_dev
->dev
),
814 msecs_to_jiffies(IWL_TRANS_IDLE_TIMEOUT
));
816 IWL_ERR(trans
, "Timeout exiting D0i3 before hcmd\n");
821 cmd_idx
= iwl_pcie_gen2_enqueue_hcmd(trans
, cmd
);
824 clear_bit(STATUS_SYNC_HCMD_ACTIVE
, &trans
->status
);
825 IWL_ERR(trans
, "Error sending %s: enqueue_hcmd failed: %d\n",
830 ret
= wait_event_timeout(trans_pcie
->wait_command_queue
,
831 !test_bit(STATUS_SYNC_HCMD_ACTIVE
,
833 HOST_COMPLETE_TIMEOUT
);
835 IWL_ERR(trans
, "Error sending %s: time out after %dms.\n",
836 cmd_str
, jiffies_to_msecs(HOST_COMPLETE_TIMEOUT
));
838 IWL_ERR(trans
, "Current CMD queue read_ptr %d write_ptr %d\n",
839 txq
->read_ptr
, txq
->write_ptr
);
841 clear_bit(STATUS_SYNC_HCMD_ACTIVE
, &trans
->status
);
842 IWL_DEBUG_INFO(trans
, "Clearing HCMD_ACTIVE for command %s\n",
846 iwl_force_nmi(trans
);
847 iwl_trans_fw_error(trans
);
852 if (test_bit(STATUS_FW_ERROR
, &trans
->status
)) {
853 IWL_ERR(trans
, "FW error in SYNC CMD %s\n", cmd_str
);
859 if (!(cmd
->flags
& CMD_SEND_IN_RFKILL
) &&
860 test_bit(STATUS_RFKILL_OPMODE
, &trans
->status
)) {
861 IWL_DEBUG_RF_KILL(trans
, "RFKILL in SYNC CMD... no rsp\n");
866 if ((cmd
->flags
& CMD_WANT_SKB
) && !cmd
->resp_pkt
) {
867 IWL_ERR(trans
, "Error: Response NULL in '%s'\n", cmd_str
);
875 if (cmd
->flags
& CMD_WANT_SKB
) {
877 * Cancel the CMD_WANT_SKB flag for the cmd in the
878 * TX cmd queue. Otherwise in case the cmd comes
879 * in later, it will possibly set an invalid
880 * address (cmd->meta.source).
882 txq
->entries
[cmd_idx
].meta
.flags
&= ~CMD_WANT_SKB
;
887 cmd
->resp_pkt
= NULL
;
893 int iwl_trans_pcie_gen2_send_hcmd(struct iwl_trans
*trans
,
894 struct iwl_host_cmd
*cmd
)
896 if (!(cmd
->flags
& CMD_SEND_IN_RFKILL
) &&
897 test_bit(STATUS_RFKILL_OPMODE
, &trans
->status
)) {
898 IWL_DEBUG_RF_KILL(trans
, "Dropping CMD 0x%x: RF KILL\n",
903 if (cmd
->flags
& CMD_ASYNC
) {
906 /* An asynchronous command can not expect an SKB to be set. */
907 if (WARN_ON(cmd
->flags
& CMD_WANT_SKB
))
910 ret
= iwl_pcie_gen2_enqueue_hcmd(trans
, cmd
);
913 "Error sending %s: enqueue_hcmd failed: %d\n",
914 iwl_get_cmd_string(trans
, cmd
->id
), ret
);
920 return iwl_pcie_gen2_send_hcmd_sync(trans
, cmd
);
924 * iwl_pcie_gen2_txq_unmap - Unmap any remaining DMA mappings and free skb's
926 void iwl_pcie_gen2_txq_unmap(struct iwl_trans
*trans
, int txq_id
)
928 struct iwl_trans_pcie
*trans_pcie
= IWL_TRANS_GET_PCIE_TRANS(trans
);
929 struct iwl_txq
*txq
= trans_pcie
->txq
[txq_id
];
931 spin_lock_bh(&txq
->lock
);
932 while (txq
->write_ptr
!= txq
->read_ptr
) {
933 IWL_DEBUG_TX_REPLY(trans
, "Q %d Free %d\n",
934 txq_id
, txq
->read_ptr
);
936 if (txq_id
!= trans_pcie
->cmd_queue
) {
937 int idx
= iwl_pcie_get_cmd_index(txq
, txq
->read_ptr
);
938 struct sk_buff
*skb
= txq
->entries
[idx
].skb
;
940 if (WARN_ON_ONCE(!skb
))
943 iwl_pcie_free_tso_page(trans_pcie
, skb
);
945 iwl_pcie_gen2_free_tfd(trans
, txq
);
946 txq
->read_ptr
= iwl_queue_inc_wrap(txq
->read_ptr
);
948 if (txq
->read_ptr
== txq
->write_ptr
) {
951 spin_lock_irqsave(&trans_pcie
->reg_lock
, flags
);
952 if (txq_id
!= trans_pcie
->cmd_queue
) {
953 IWL_DEBUG_RPM(trans
, "Q %d - last tx freed\n",
955 iwl_trans_unref(trans
);
956 } else if (trans_pcie
->ref_cmd_in_flight
) {
957 trans_pcie
->ref_cmd_in_flight
= false;
959 "clear ref_cmd_in_flight\n");
960 iwl_trans_unref(trans
);
962 spin_unlock_irqrestore(&trans_pcie
->reg_lock
, flags
);
965 spin_unlock_bh(&txq
->lock
);
967 /* just in case - this queue may have been stopped */
968 iwl_wake_queue(trans
, txq
);
971 static void iwl_pcie_gen2_txq_free_memory(struct iwl_trans
*trans
,
974 struct iwl_trans_pcie
*trans_pcie
= IWL_TRANS_GET_PCIE_TRANS(trans
);
975 struct device
*dev
= trans
->dev
;
977 /* De-alloc circular buffer of TFDs */
979 dma_free_coherent(dev
,
980 trans_pcie
->tfd_size
* TFD_QUEUE_SIZE_MAX
,
981 txq
->tfds
, txq
->dma_addr
);
982 dma_free_coherent(dev
,
983 sizeof(*txq
->first_tb_bufs
) * txq
->n_window
,
984 txq
->first_tb_bufs
, txq
->first_tb_dma
);
988 iwl_pcie_free_dma_ptr(trans
, &txq
->bc_tbl
);
993 * iwl_pcie_txq_free - Deallocate DMA queue.
994 * @txq: Transmit queue to deallocate.
996 * Empty queue by removing and destroying all BD's.
998 * 0-fill, but do not free "txq" descriptor structure.
1000 static void iwl_pcie_gen2_txq_free(struct iwl_trans
*trans
, int txq_id
)
1002 struct iwl_trans_pcie
*trans_pcie
= IWL_TRANS_GET_PCIE_TRANS(trans
);
1003 struct iwl_txq
*txq
= trans_pcie
->txq
[txq_id
];
1009 iwl_pcie_gen2_txq_unmap(trans
, txq_id
);
1011 /* De-alloc array of command/tx buffers */
1012 if (txq_id
== trans_pcie
->cmd_queue
)
1013 for (i
= 0; i
< txq
->n_window
; i
++) {
1014 kzfree(txq
->entries
[i
].cmd
);
1015 kzfree(txq
->entries
[i
].free_buf
);
1017 del_timer_sync(&txq
->stuck_timer
);
1019 iwl_pcie_gen2_txq_free_memory(trans
, txq
);
1021 trans_pcie
->txq
[txq_id
] = NULL
;
1023 clear_bit(txq_id
, trans_pcie
->queue_used
);
1026 int iwl_trans_pcie_dyn_txq_alloc(struct iwl_trans
*trans
,
1027 struct iwl_tx_queue_cfg_cmd
*cmd
,
1029 unsigned int timeout
)
1031 struct iwl_trans_pcie
*trans_pcie
= IWL_TRANS_GET_PCIE_TRANS(trans
);
1032 struct iwl_tx_queue_cfg_rsp
*rsp
;
1033 struct iwl_txq
*txq
;
1034 struct iwl_host_cmd hcmd
= {
1036 .len
= { sizeof(*cmd
) },
1038 .flags
= CMD_WANT_SKB
,
1043 txq
= kzalloc(sizeof(*txq
), GFP_KERNEL
);
1046 ret
= iwl_pcie_alloc_dma_ptr(trans
, &txq
->bc_tbl
,
1047 sizeof(struct iwlagn_scd_bc_tbl
));
1049 IWL_ERR(trans
, "Scheduler BC Table allocation failed\n");
1054 ret
= iwl_pcie_txq_alloc(trans
, txq
, TFD_TX_CMD_SLOTS
, false);
1056 IWL_ERR(trans
, "Tx queue alloc failed\n");
1059 ret
= iwl_pcie_txq_init(trans
, txq
, TFD_TX_CMD_SLOTS
, false);
1061 IWL_ERR(trans
, "Tx queue init failed\n");
1065 txq
->wd_timeout
= msecs_to_jiffies(timeout
);
1067 cmd
->tfdq_addr
= cpu_to_le64(txq
->dma_addr
);
1068 cmd
->byte_cnt_addr
= cpu_to_le64(txq
->bc_tbl
.dma
);
1069 cmd
->cb_size
= cpu_to_le32(TFD_QUEUE_CB_SIZE(TFD_TX_CMD_SLOTS
));
1071 ret
= iwl_trans_send_cmd(trans
, &hcmd
);
1075 if (WARN_ON(iwl_rx_packet_payload_len(hcmd
.resp_pkt
) != sizeof(*rsp
))) {
1077 goto error_free_resp
;
1080 rsp
= (void *)hcmd
.resp_pkt
->data
;
1081 qid
= le16_to_cpu(rsp
->queue_number
);
1082 wr_ptr
= le16_to_cpu(rsp
->write_pointer
);
1084 if (qid
>= ARRAY_SIZE(trans_pcie
->txq
)) {
1085 WARN_ONCE(1, "queue index %d unsupported", qid
);
1087 goto error_free_resp
;
1090 if (test_and_set_bit(qid
, trans_pcie
->queue_used
)) {
1091 WARN_ONCE(1, "queue %d already used", qid
);
1093 goto error_free_resp
;
1097 trans_pcie
->txq
[qid
] = txq
;
1098 wr_ptr
&= (TFD_QUEUE_SIZE_MAX
- 1);
1100 /* Place first TFD at index corresponding to start sequence number */
1101 txq
->read_ptr
= wr_ptr
;
1102 txq
->write_ptr
= wr_ptr
;
1103 iwl_write_direct32(trans
, HBUS_TARG_WRPTR
,
1104 (txq
->write_ptr
) | (qid
<< 16));
1105 IWL_DEBUG_TX_QUEUES(trans
, "Activate queue %d\n", qid
);
1107 iwl_free_resp(&hcmd
);
1111 iwl_free_resp(&hcmd
);
1113 iwl_pcie_gen2_txq_free_memory(trans
, txq
);
1117 void iwl_trans_pcie_dyn_txq_free(struct iwl_trans
*trans
, int queue
)
1119 struct iwl_trans_pcie
*trans_pcie
= IWL_TRANS_GET_PCIE_TRANS(trans
);
1122 * Upon HW Rfkill - we stop the device, and then stop the queues
1123 * in the op_mode. Just for the sake of the simplicity of the op_mode,
1124 * allow the op_mode to call txq_disable after it already called
1127 if (!test_and_clear_bit(queue
, trans_pcie
->queue_used
)) {
1128 WARN_ONCE(test_bit(STATUS_DEVICE_ENABLED
, &trans
->status
),
1129 "queue %d not used", queue
);
1133 iwl_pcie_gen2_txq_unmap(trans
, queue
);
1135 IWL_DEBUG_TX_QUEUES(trans
, "Deactivate queue %d\n", queue
);
1138 void iwl_pcie_gen2_tx_free(struct iwl_trans
*trans
)
1140 struct iwl_trans_pcie
*trans_pcie
= IWL_TRANS_GET_PCIE_TRANS(trans
);
1143 memset(trans_pcie
->queue_used
, 0, sizeof(trans_pcie
->queue_used
));
1145 /* Free all TX queues */
1146 for (i
= 0; i
< ARRAY_SIZE(trans_pcie
->txq
); i
++) {
1147 if (!trans_pcie
->txq
[i
])
1150 iwl_pcie_gen2_txq_free(trans
, i
);
1154 int iwl_pcie_gen2_tx_init(struct iwl_trans
*trans
)
1156 struct iwl_trans_pcie
*trans_pcie
= IWL_TRANS_GET_PCIE_TRANS(trans
);
1157 struct iwl_txq
*cmd_queue
;
1158 int txq_id
= trans_pcie
->cmd_queue
, ret
;
1160 iwl_pcie_set_tx_cmd_queue_size(trans
);
1162 /* alloc and init the command queue */
1163 if (!trans_pcie
->txq
[txq_id
]) {
1164 cmd_queue
= kzalloc(sizeof(*cmd_queue
), GFP_KERNEL
);
1166 IWL_ERR(trans
, "Not enough memory for command queue\n");
1169 trans_pcie
->txq
[txq_id
] = cmd_queue
;
1170 ret
= iwl_pcie_txq_alloc(trans
, cmd_queue
,
1171 trans_pcie
->tx_cmd_queue_size
, true);
1173 IWL_ERR(trans
, "Tx %d queue init failed\n", txq_id
);
1177 cmd_queue
= trans_pcie
->txq
[txq_id
];
1180 ret
= iwl_pcie_txq_init(trans
, cmd_queue
,
1181 trans_pcie
->tx_cmd_queue_size
, true);
1183 IWL_ERR(trans
, "Tx %d queue alloc failed\n", txq_id
);
1186 trans_pcie
->txq
[txq_id
]->id
= txq_id
;
1187 set_bit(txq_id
, trans_pcie
->queue_used
);
1192 iwl_pcie_gen2_tx_free(trans
);