]>
Commit | Line | Data |
---|---|---|
1 | /****************************************************************************** | |
2 | * | |
3 | * This file is provided under a dual BSD/GPLv2 license. When using or | |
4 | * redistributing this file, you may do so under either license. | |
5 | * | |
6 | * GPL LICENSE SUMMARY | |
7 | * | |
8 | * Copyright(c) 2017 Intel Deutschland GmbH | |
9 | * | |
10 | * This program is free software; you can redistribute it and/or modify | |
11 | * it under the terms of version 2 of the GNU General Public License as | |
12 | * published by the Free Software Foundation. | |
13 | * | |
14 | * This program is distributed in the hope that it will be useful, but | |
15 | * WITHOUT ANY WARRANTY; without even the implied warranty of | |
16 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
17 | * General Public License for more details. | |
18 | * | |
19 | * BSD LICENSE | |
20 | * | |
21 | * Copyright(c) 2017 Intel Deutschland GmbH | |
22 | * All rights reserved. | |
23 | * | |
24 | * Redistribution and use in source and binary forms, with or without | |
25 | * modification, are permitted provided that the following conditions | |
26 | * are met: | |
27 | * | |
28 | * * Redistributions of source code must retain the above copyright | |
29 | * notice, this list of conditions and the following disclaimer. | |
30 | * * Redistributions in binary form must reproduce the above copyright | |
31 | * notice, this list of conditions and the following disclaimer in | |
32 | * the documentation and/or other materials provided with the | |
33 | * distribution. | |
34 | * * Neither the name Intel Corporation nor the names of its | |
35 | * contributors may be used to endorse or promote products derived | |
36 | * from this software without specific prior written permission. | |
37 | * | |
38 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS | |
39 | * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT | |
40 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR | |
41 | * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT | |
42 | * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, | |
43 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT | |
44 | * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, | |
45 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY | |
46 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT | |
47 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE | |
48 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | |
49 | * | |
50 | *****************************************************************************/ | |
51 | #include <linux/pm_runtime.h> | |
52 | #include <net/tso.h> | |
53 | ||
54 | #include "iwl-debug.h" | |
55 | #include "iwl-csr.h" | |
56 | #include "iwl-io.h" | |
57 | #include "internal.h" | |
58 | #include "fw/api/tx.h" | |
59 | ||
60 | /* | |
61 | * iwl_pcie_gen2_tx_stop - Stop all Tx DMA channels | |
62 | */ | |
63 | void iwl_pcie_gen2_tx_stop(struct iwl_trans *trans) | |
64 | { | |
65 | struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); | |
66 | int txq_id; | |
67 | ||
68 | /* | |
69 | * This function can be called before the op_mode disabled the | |
70 | * queues. This happens when we have an rfkill interrupt. | |
71 | * Since we stop Tx altogether - mark the queues as stopped. | |
72 | */ | |
73 | memset(trans_pcie->queue_stopped, 0, sizeof(trans_pcie->queue_stopped)); | |
74 | memset(trans_pcie->queue_used, 0, sizeof(trans_pcie->queue_used)); | |
75 | ||
76 | /* Unmap DMA from host system and free skb's */ | |
77 | for (txq_id = 0; txq_id < ARRAY_SIZE(trans_pcie->txq); txq_id++) { | |
78 | if (!trans_pcie->txq[txq_id]) | |
79 | continue; | |
80 | iwl_pcie_gen2_txq_unmap(trans, txq_id); | |
81 | } | |
82 | } | |
83 | ||
84 | /* | |
85 | * iwl_pcie_txq_update_byte_tbl - Set up entry in Tx byte-count array | |
86 | */ | |
87 | static void iwl_pcie_gen2_update_byte_tbl(struct iwl_txq *txq, u16 byte_cnt, | |
88 | int num_tbs) | |
89 | { | |
90 | struct iwlagn_scd_bc_tbl *scd_bc_tbl = txq->bc_tbl.addr; | |
91 | int idx = iwl_pcie_get_cmd_index(txq, txq->write_ptr); | |
92 | u8 filled_tfd_size, num_fetch_chunks; | |
93 | u16 len = byte_cnt; | |
94 | __le16 bc_ent; | |
95 | ||
96 | len = DIV_ROUND_UP(len, 4); | |
97 | ||
98 | if (WARN_ON(len > 0xFFF || idx >= txq->n_window)) | |
99 | return; | |
100 | ||
101 | filled_tfd_size = offsetof(struct iwl_tfh_tfd, tbs) + | |
102 | num_tbs * sizeof(struct iwl_tfh_tb); | |
103 | /* | |
104 | * filled_tfd_size contains the number of filled bytes in the TFD. | |
105 | * Dividing it by 64 will give the number of chunks to fetch | |
106 | * to SRAM- 0 for one chunk, 1 for 2 and so on. | |
107 | * If, for example, TFD contains only 3 TBs then 32 bytes | |
108 | * of the TFD are used, and only one chunk of 64 bytes should | |
109 | * be fetched | |
110 | */ | |
111 | num_fetch_chunks = DIV_ROUND_UP(filled_tfd_size, 64) - 1; | |
112 | ||
113 | bc_ent = cpu_to_le16(len | (num_fetch_chunks << 12)); | |
114 | scd_bc_tbl->tfd_offset[idx] = bc_ent; | |
115 | } | |
116 | ||
117 | /* | |
118 | * iwl_pcie_gen2_txq_inc_wr_ptr - Send new write index to hardware | |
119 | */ | |
120 | static void iwl_pcie_gen2_txq_inc_wr_ptr(struct iwl_trans *trans, | |
121 | struct iwl_txq *txq) | |
122 | { | |
123 | lockdep_assert_held(&txq->lock); | |
124 | ||
125 | IWL_DEBUG_TX(trans, "Q:%d WR: 0x%x\n", txq->id, txq->write_ptr); | |
126 | ||
127 | /* | |
128 | * if not in power-save mode, uCode will never sleep when we're | |
129 | * trying to tx (during RFKILL, we're not trying to tx). | |
130 | */ | |
131 | iwl_write32(trans, HBUS_TARG_WRPTR, txq->write_ptr | (txq->id << 16)); | |
132 | } | |
133 | ||
134 | static u8 iwl_pcie_gen2_get_num_tbs(struct iwl_trans *trans, | |
135 | struct iwl_tfh_tfd *tfd) | |
136 | { | |
137 | return le16_to_cpu(tfd->num_tbs) & 0x1f; | |
138 | } | |
139 | ||
140 | static void iwl_pcie_gen2_tfd_unmap(struct iwl_trans *trans, | |
141 | struct iwl_cmd_meta *meta, | |
142 | struct iwl_tfh_tfd *tfd) | |
143 | { | |
144 | struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); | |
145 | int i, num_tbs; | |
146 | ||
147 | /* Sanity check on number of chunks */ | |
148 | num_tbs = iwl_pcie_gen2_get_num_tbs(trans, tfd); | |
149 | ||
150 | if (num_tbs >= trans_pcie->max_tbs) { | |
151 | IWL_ERR(trans, "Too many chunks: %i\n", num_tbs); | |
152 | return; | |
153 | } | |
154 | ||
155 | /* first TB is never freed - it's the bidirectional DMA data */ | |
156 | for (i = 1; i < num_tbs; i++) { | |
157 | if (meta->tbs & BIT(i)) | |
158 | dma_unmap_page(trans->dev, | |
159 | le64_to_cpu(tfd->tbs[i].addr), | |
160 | le16_to_cpu(tfd->tbs[i].tb_len), | |
161 | DMA_TO_DEVICE); | |
162 | else | |
163 | dma_unmap_single(trans->dev, | |
164 | le64_to_cpu(tfd->tbs[i].addr), | |
165 | le16_to_cpu(tfd->tbs[i].tb_len), | |
166 | DMA_TO_DEVICE); | |
167 | } | |
168 | ||
169 | tfd->num_tbs = 0; | |
170 | } | |
171 | ||
172 | static void iwl_pcie_gen2_free_tfd(struct iwl_trans *trans, struct iwl_txq *txq) | |
173 | { | |
174 | /* rd_ptr is bounded by TFD_QUEUE_SIZE_MAX and | |
175 | * idx is bounded by n_window | |
176 | */ | |
177 | int idx = iwl_pcie_get_cmd_index(txq, txq->read_ptr); | |
178 | ||
179 | lockdep_assert_held(&txq->lock); | |
180 | ||
181 | iwl_pcie_gen2_tfd_unmap(trans, &txq->entries[idx].meta, | |
182 | iwl_pcie_get_tfd(trans, txq, idx)); | |
183 | ||
184 | /* free SKB */ | |
185 | if (txq->entries) { | |
186 | struct sk_buff *skb; | |
187 | ||
188 | skb = txq->entries[idx].skb; | |
189 | ||
190 | /* Can be called from irqs-disabled context | |
191 | * If skb is not NULL, it means that the whole queue is being | |
192 | * freed and that the queue is not empty - free the skb | |
193 | */ | |
194 | if (skb) { | |
195 | iwl_op_mode_free_skb(trans->op_mode, skb); | |
196 | txq->entries[idx].skb = NULL; | |
197 | } | |
198 | } | |
199 | } | |
200 | ||
201 | static int iwl_pcie_gen2_set_tb(struct iwl_trans *trans, | |
202 | struct iwl_tfh_tfd *tfd, dma_addr_t addr, | |
203 | u16 len) | |
204 | { | |
205 | struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); | |
206 | int idx = iwl_pcie_gen2_get_num_tbs(trans, tfd); | |
207 | struct iwl_tfh_tb *tb = &tfd->tbs[idx]; | |
208 | ||
209 | /* Each TFD can point to a maximum max_tbs Tx buffers */ | |
210 | if (le16_to_cpu(tfd->num_tbs) >= trans_pcie->max_tbs) { | |
211 | IWL_ERR(trans, "Error can not send more than %d chunks\n", | |
212 | trans_pcie->max_tbs); | |
213 | return -EINVAL; | |
214 | } | |
215 | ||
216 | put_unaligned_le64(addr, &tb->addr); | |
217 | tb->tb_len = cpu_to_le16(len); | |
218 | ||
219 | tfd->num_tbs = cpu_to_le16(idx + 1); | |
220 | ||
221 | return idx; | |
222 | } | |
223 | ||
224 | static int iwl_pcie_gen2_build_amsdu(struct iwl_trans *trans, | |
225 | struct sk_buff *skb, | |
226 | struct iwl_tfh_tfd *tfd, int start_len, | |
227 | u8 hdr_len, struct iwl_device_cmd *dev_cmd) | |
228 | { | |
229 | #ifdef CONFIG_INET | |
230 | struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); | |
231 | struct iwl_tx_cmd *tx_cmd = (void *)dev_cmd->payload; | |
232 | struct ieee80211_hdr *hdr = (void *)skb->data; | |
233 | unsigned int snap_ip_tcp_hdrlen, ip_hdrlen, total_len, hdr_room; | |
234 | unsigned int mss = skb_shinfo(skb)->gso_size; | |
235 | u16 length, iv_len, amsdu_pad; | |
236 | u8 *start_hdr; | |
237 | struct iwl_tso_hdr_page *hdr_page; | |
238 | struct page **page_ptr; | |
239 | struct tso_t tso; | |
240 | ||
241 | /* if the packet is protected, then it must be CCMP or GCMP */ | |
242 | iv_len = ieee80211_has_protected(hdr->frame_control) ? | |
243 | IEEE80211_CCMP_HDR_LEN : 0; | |
244 | ||
245 | trace_iwlwifi_dev_tx(trans->dev, skb, tfd, sizeof(*tfd), | |
246 | &dev_cmd->hdr, start_len, 0); | |
247 | ||
248 | ip_hdrlen = skb_transport_header(skb) - skb_network_header(skb); | |
249 | snap_ip_tcp_hdrlen = 8 + ip_hdrlen + tcp_hdrlen(skb); | |
250 | total_len = skb->len - snap_ip_tcp_hdrlen - hdr_len - iv_len; | |
251 | amsdu_pad = 0; | |
252 | ||
253 | /* total amount of header we may need for this A-MSDU */ | |
254 | hdr_room = DIV_ROUND_UP(total_len, mss) * | |
255 | (3 + snap_ip_tcp_hdrlen + sizeof(struct ethhdr)) + iv_len; | |
256 | ||
257 | /* Our device supports 9 segments at most, it will fit in 1 page */ | |
258 | hdr_page = get_page_hdr(trans, hdr_room); | |
259 | if (!hdr_page) | |
260 | return -ENOMEM; | |
261 | ||
262 | get_page(hdr_page->page); | |
263 | start_hdr = hdr_page->pos; | |
264 | page_ptr = (void *)((u8 *)skb->cb + trans_pcie->page_offs); | |
265 | *page_ptr = hdr_page->page; | |
266 | memcpy(hdr_page->pos, skb->data + hdr_len, iv_len); | |
267 | hdr_page->pos += iv_len; | |
268 | ||
269 | /* | |
270 | * Pull the ieee80211 header + IV to be able to use TSO core, | |
271 | * we will restore it for the tx_status flow. | |
272 | */ | |
273 | skb_pull(skb, hdr_len + iv_len); | |
274 | ||
275 | /* | |
276 | * Remove the length of all the headers that we don't actually | |
277 | * have in the MPDU by themselves, but that we duplicate into | |
278 | * all the different MSDUs inside the A-MSDU. | |
279 | */ | |
280 | le16_add_cpu(&tx_cmd->len, -snap_ip_tcp_hdrlen); | |
281 | ||
282 | tso_start(skb, &tso); | |
283 | ||
284 | while (total_len) { | |
285 | /* this is the data left for this subframe */ | |
286 | unsigned int data_left = min_t(unsigned int, mss, total_len); | |
287 | struct sk_buff *csum_skb = NULL; | |
288 | unsigned int tb_len; | |
289 | dma_addr_t tb_phys; | |
290 | u8 *subf_hdrs_start = hdr_page->pos; | |
291 | ||
292 | total_len -= data_left; | |
293 | ||
294 | memset(hdr_page->pos, 0, amsdu_pad); | |
295 | hdr_page->pos += amsdu_pad; | |
296 | amsdu_pad = (4 - (sizeof(struct ethhdr) + snap_ip_tcp_hdrlen + | |
297 | data_left)) & 0x3; | |
298 | ether_addr_copy(hdr_page->pos, ieee80211_get_DA(hdr)); | |
299 | hdr_page->pos += ETH_ALEN; | |
300 | ether_addr_copy(hdr_page->pos, ieee80211_get_SA(hdr)); | |
301 | hdr_page->pos += ETH_ALEN; | |
302 | ||
303 | length = snap_ip_tcp_hdrlen + data_left; | |
304 | *((__be16 *)hdr_page->pos) = cpu_to_be16(length); | |
305 | hdr_page->pos += sizeof(length); | |
306 | ||
307 | /* | |
308 | * This will copy the SNAP as well which will be considered | |
309 | * as MAC header. | |
310 | */ | |
311 | tso_build_hdr(skb, hdr_page->pos, &tso, data_left, !total_len); | |
312 | ||
313 | hdr_page->pos += snap_ip_tcp_hdrlen; | |
314 | ||
315 | tb_len = hdr_page->pos - start_hdr; | |
316 | tb_phys = dma_map_single(trans->dev, start_hdr, | |
317 | tb_len, DMA_TO_DEVICE); | |
318 | if (unlikely(dma_mapping_error(trans->dev, tb_phys))) { | |
319 | dev_kfree_skb(csum_skb); | |
320 | goto out_err; | |
321 | } | |
322 | iwl_pcie_gen2_set_tb(trans, tfd, tb_phys, tb_len); | |
323 | trace_iwlwifi_dev_tx_tso_chunk(trans->dev, start_hdr, tb_len); | |
324 | /* add this subframe's headers' length to the tx_cmd */ | |
325 | le16_add_cpu(&tx_cmd->len, hdr_page->pos - subf_hdrs_start); | |
326 | ||
327 | /* prepare the start_hdr for the next subframe */ | |
328 | start_hdr = hdr_page->pos; | |
329 | ||
330 | /* put the payload */ | |
331 | while (data_left) { | |
332 | tb_len = min_t(unsigned int, tso.size, data_left); | |
333 | tb_phys = dma_map_single(trans->dev, tso.data, | |
334 | tb_len, DMA_TO_DEVICE); | |
335 | if (unlikely(dma_mapping_error(trans->dev, tb_phys))) { | |
336 | dev_kfree_skb(csum_skb); | |
337 | goto out_err; | |
338 | } | |
339 | iwl_pcie_gen2_set_tb(trans, tfd, tb_phys, tb_len); | |
340 | trace_iwlwifi_dev_tx_tso_chunk(trans->dev, tso.data, | |
341 | tb_len); | |
342 | ||
343 | data_left -= tb_len; | |
344 | tso_build_data(skb, &tso, tb_len); | |
345 | } | |
346 | } | |
347 | ||
348 | /* re -add the WiFi header and IV */ | |
349 | skb_push(skb, hdr_len + iv_len); | |
350 | ||
351 | return 0; | |
352 | ||
353 | out_err: | |
354 | #endif | |
355 | return -EINVAL; | |
356 | } | |
357 | ||
358 | static | |
359 | struct iwl_tfh_tfd *iwl_pcie_gen2_build_tfd(struct iwl_trans *trans, | |
360 | struct iwl_txq *txq, | |
361 | struct iwl_device_cmd *dev_cmd, | |
362 | struct sk_buff *skb, | |
363 | struct iwl_cmd_meta *out_meta) | |
364 | { | |
365 | struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)skb->data; | |
366 | int idx = iwl_pcie_get_cmd_index(txq, txq->write_ptr); | |
367 | struct iwl_tfh_tfd *tfd = iwl_pcie_get_tfd(trans, txq, idx); | |
368 | dma_addr_t tb_phys; | |
369 | bool amsdu; | |
370 | int i, len, tb1_len, tb2_len, hdr_len; | |
371 | void *tb1_addr; | |
372 | ||
373 | memset(tfd, 0, sizeof(*tfd)); | |
374 | ||
375 | amsdu = ieee80211_is_data_qos(hdr->frame_control) && | |
376 | (*ieee80211_get_qos_ctl(hdr) & | |
377 | IEEE80211_QOS_CTL_A_MSDU_PRESENT); | |
378 | ||
379 | tb_phys = iwl_pcie_get_first_tb_dma(txq, idx); | |
380 | /* The first TB points to bi-directional DMA data */ | |
381 | if (!amsdu) | |
382 | memcpy(&txq->first_tb_bufs[idx], &dev_cmd->hdr, | |
383 | IWL_FIRST_TB_SIZE); | |
384 | ||
385 | iwl_pcie_gen2_set_tb(trans, tfd, tb_phys, IWL_FIRST_TB_SIZE); | |
386 | ||
387 | /* there must be data left over for TB1 or this code must be changed */ | |
388 | BUILD_BUG_ON(sizeof(struct iwl_tx_cmd_gen2) < IWL_FIRST_TB_SIZE); | |
389 | ||
390 | /* | |
391 | * The second TB (tb1) points to the remainder of the TX command | |
392 | * and the 802.11 header - dword aligned size | |
393 | * (This calculation modifies the TX command, so do it before the | |
394 | * setup of the first TB) | |
395 | */ | |
396 | len = sizeof(struct iwl_tx_cmd_gen2) + sizeof(struct iwl_cmd_header) + | |
397 | ieee80211_hdrlen(hdr->frame_control) - IWL_FIRST_TB_SIZE; | |
398 | ||
399 | /* do not align A-MSDU to dword as the subframe header aligns it */ | |
400 | if (amsdu) | |
401 | tb1_len = len; | |
402 | else | |
403 | tb1_len = ALIGN(len, 4); | |
404 | ||
405 | /* map the data for TB1 */ | |
406 | tb1_addr = ((u8 *)&dev_cmd->hdr) + IWL_FIRST_TB_SIZE; | |
407 | tb_phys = dma_map_single(trans->dev, tb1_addr, tb1_len, DMA_TO_DEVICE); | |
408 | if (unlikely(dma_mapping_error(trans->dev, tb_phys))) | |
409 | goto out_err; | |
410 | iwl_pcie_gen2_set_tb(trans, tfd, tb_phys, tb1_len); | |
411 | ||
412 | hdr_len = ieee80211_hdrlen(hdr->frame_control); | |
413 | ||
414 | if (amsdu) { | |
415 | if (iwl_pcie_gen2_build_amsdu(trans, skb, tfd, | |
416 | tb1_len + IWL_FIRST_TB_SIZE, | |
417 | hdr_len, dev_cmd)) | |
418 | goto out_err; | |
419 | ||
420 | /* | |
421 | * building the A-MSDU might have changed this data, so memcpy | |
422 | * it now | |
423 | */ | |
424 | memcpy(&txq->first_tb_bufs[idx], &dev_cmd->hdr, | |
425 | IWL_FIRST_TB_SIZE); | |
426 | return tfd; | |
427 | } | |
428 | ||
429 | /* set up TFD's third entry to point to remainder of skb's head */ | |
430 | tb2_len = skb_headlen(skb) - hdr_len; | |
431 | ||
432 | if (tb2_len > 0) { | |
433 | tb_phys = dma_map_single(trans->dev, skb->data + hdr_len, | |
434 | tb2_len, DMA_TO_DEVICE); | |
435 | if (unlikely(dma_mapping_error(trans->dev, tb_phys))) | |
436 | goto out_err; | |
437 | iwl_pcie_gen2_set_tb(trans, tfd, tb_phys, tb2_len); | |
438 | } | |
439 | ||
440 | /* set up the remaining entries to point to the data */ | |
441 | for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { | |
442 | const skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; | |
443 | int tb_idx; | |
444 | ||
445 | if (!skb_frag_size(frag)) | |
446 | continue; | |
447 | ||
448 | tb_phys = skb_frag_dma_map(trans->dev, frag, 0, | |
449 | skb_frag_size(frag), DMA_TO_DEVICE); | |
450 | ||
451 | if (unlikely(dma_mapping_error(trans->dev, tb_phys))) | |
452 | goto out_err; | |
453 | tb_idx = iwl_pcie_gen2_set_tb(trans, tfd, tb_phys, | |
454 | skb_frag_size(frag)); | |
455 | ||
456 | out_meta->tbs |= BIT(tb_idx); | |
457 | } | |
458 | ||
459 | trace_iwlwifi_dev_tx(trans->dev, skb, tfd, sizeof(*tfd), &dev_cmd->hdr, | |
460 | IWL_FIRST_TB_SIZE + tb1_len, hdr_len); | |
461 | trace_iwlwifi_dev_tx_data(trans->dev, skb, hdr_len); | |
462 | ||
463 | return tfd; | |
464 | ||
465 | out_err: | |
466 | iwl_pcie_gen2_tfd_unmap(trans, out_meta, tfd); | |
467 | return NULL; | |
468 | } | |
469 | ||
470 | int iwl_trans_pcie_gen2_tx(struct iwl_trans *trans, struct sk_buff *skb, | |
471 | struct iwl_device_cmd *dev_cmd, int txq_id) | |
472 | { | |
473 | struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); | |
474 | struct iwl_tx_cmd_gen2 *tx_cmd = (void *)dev_cmd->payload; | |
475 | struct iwl_cmd_meta *out_meta; | |
476 | struct iwl_txq *txq = trans_pcie->txq[txq_id]; | |
477 | int idx; | |
478 | void *tfd; | |
479 | ||
480 | if (WARN_ONCE(!test_bit(txq_id, trans_pcie->queue_used), | |
481 | "TX on unused queue %d\n", txq_id)) | |
482 | return -EINVAL; | |
483 | ||
484 | if (skb_is_nonlinear(skb) && | |
485 | skb_shinfo(skb)->nr_frags > IWL_PCIE_MAX_FRAGS(trans_pcie) && | |
486 | __skb_linearize(skb)) | |
487 | return -ENOMEM; | |
488 | ||
489 | spin_lock(&txq->lock); | |
490 | ||
491 | idx = iwl_pcie_get_cmd_index(txq, txq->write_ptr); | |
492 | ||
493 | /* Set up driver data for this TFD */ | |
494 | txq->entries[idx].skb = skb; | |
495 | txq->entries[idx].cmd = dev_cmd; | |
496 | ||
497 | dev_cmd->hdr.sequence = | |
498 | cpu_to_le16((u16)(QUEUE_TO_SEQ(txq_id) | | |
499 | INDEX_TO_SEQ(idx))); | |
500 | ||
501 | /* Set up first empty entry in queue's array of Tx/cmd buffers */ | |
502 | out_meta = &txq->entries[idx].meta; | |
503 | out_meta->flags = 0; | |
504 | ||
505 | tfd = iwl_pcie_gen2_build_tfd(trans, txq, dev_cmd, skb, out_meta); | |
506 | if (!tfd) { | |
507 | spin_unlock(&txq->lock); | |
508 | return -1; | |
509 | } | |
510 | ||
511 | /* Set up entry for this TFD in Tx byte-count array */ | |
512 | iwl_pcie_gen2_update_byte_tbl(txq, le16_to_cpu(tx_cmd->len), | |
513 | iwl_pcie_gen2_get_num_tbs(trans, tfd)); | |
514 | ||
515 | /* start timer if queue currently empty */ | |
516 | if (txq->read_ptr == txq->write_ptr) { | |
517 | if (txq->wd_timeout) | |
518 | mod_timer(&txq->stuck_timer, jiffies + txq->wd_timeout); | |
519 | IWL_DEBUG_RPM(trans, "Q: %d first tx - take ref\n", txq->id); | |
520 | iwl_trans_ref(trans); | |
521 | } | |
522 | ||
523 | /* Tell device the write index *just past* this latest filled TFD */ | |
524 | txq->write_ptr = iwl_queue_inc_wrap(txq->write_ptr); | |
525 | iwl_pcie_gen2_txq_inc_wr_ptr(trans, txq); | |
526 | if (iwl_queue_space(txq) < txq->high_mark) | |
527 | iwl_stop_queue(trans, txq); | |
528 | ||
529 | /* | |
530 | * At this point the frame is "transmitted" successfully | |
531 | * and we will get a TX status notification eventually. | |
532 | */ | |
533 | spin_unlock(&txq->lock); | |
534 | return 0; | |
535 | } | |
536 | ||
537 | /*************** HOST COMMAND QUEUE FUNCTIONS *****/ | |
538 | ||
539 | /* | |
540 | * iwl_pcie_gen2_enqueue_hcmd - enqueue a uCode command | |
541 | * @priv: device private data point | |
542 | * @cmd: a pointer to the ucode command structure | |
543 | * | |
544 | * The function returns < 0 values to indicate the operation | |
545 | * failed. On success, it returns the index (>= 0) of command in the | |
546 | * command queue. | |
547 | */ | |
548 | static int iwl_pcie_gen2_enqueue_hcmd(struct iwl_trans *trans, | |
549 | struct iwl_host_cmd *cmd) | |
550 | { | |
551 | struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); | |
552 | struct iwl_txq *txq = trans_pcie->txq[trans_pcie->cmd_queue]; | |
553 | struct iwl_device_cmd *out_cmd; | |
554 | struct iwl_cmd_meta *out_meta; | |
555 | unsigned long flags; | |
556 | void *dup_buf = NULL; | |
557 | dma_addr_t phys_addr; | |
558 | int i, cmd_pos, idx = iwl_pcie_get_cmd_index(txq, txq->write_ptr); | |
559 | u16 copy_size, cmd_size, tb0_size; | |
560 | bool had_nocopy = false; | |
561 | u8 group_id = iwl_cmd_groupid(cmd->id); | |
562 | const u8 *cmddata[IWL_MAX_CMD_TBS_PER_TFD]; | |
563 | u16 cmdlen[IWL_MAX_CMD_TBS_PER_TFD]; | |
564 | struct iwl_tfh_tfd *tfd = iwl_pcie_get_tfd(trans, txq, txq->write_ptr); | |
565 | ||
566 | memset(tfd, 0, sizeof(*tfd)); | |
567 | ||
568 | copy_size = sizeof(struct iwl_cmd_header_wide); | |
569 | cmd_size = sizeof(struct iwl_cmd_header_wide); | |
570 | ||
571 | for (i = 0; i < IWL_MAX_CMD_TBS_PER_TFD; i++) { | |
572 | cmddata[i] = cmd->data[i]; | |
573 | cmdlen[i] = cmd->len[i]; | |
574 | ||
575 | if (!cmd->len[i]) | |
576 | continue; | |
577 | ||
578 | /* need at least IWL_FIRST_TB_SIZE copied */ | |
579 | if (copy_size < IWL_FIRST_TB_SIZE) { | |
580 | int copy = IWL_FIRST_TB_SIZE - copy_size; | |
581 | ||
582 | if (copy > cmdlen[i]) | |
583 | copy = cmdlen[i]; | |
584 | cmdlen[i] -= copy; | |
585 | cmddata[i] += copy; | |
586 | copy_size += copy; | |
587 | } | |
588 | ||
589 | if (cmd->dataflags[i] & IWL_HCMD_DFL_NOCOPY) { | |
590 | had_nocopy = true; | |
591 | if (WARN_ON(cmd->dataflags[i] & IWL_HCMD_DFL_DUP)) { | |
592 | idx = -EINVAL; | |
593 | goto free_dup_buf; | |
594 | } | |
595 | } else if (cmd->dataflags[i] & IWL_HCMD_DFL_DUP) { | |
596 | /* | |
597 | * This is also a chunk that isn't copied | |
598 | * to the static buffer so set had_nocopy. | |
599 | */ | |
600 | had_nocopy = true; | |
601 | ||
602 | /* only allowed once */ | |
603 | if (WARN_ON(dup_buf)) { | |
604 | idx = -EINVAL; | |
605 | goto free_dup_buf; | |
606 | } | |
607 | ||
608 | dup_buf = kmemdup(cmddata[i], cmdlen[i], | |
609 | GFP_ATOMIC); | |
610 | if (!dup_buf) | |
611 | return -ENOMEM; | |
612 | } else { | |
613 | /* NOCOPY must not be followed by normal! */ | |
614 | if (WARN_ON(had_nocopy)) { | |
615 | idx = -EINVAL; | |
616 | goto free_dup_buf; | |
617 | } | |
618 | copy_size += cmdlen[i]; | |
619 | } | |
620 | cmd_size += cmd->len[i]; | |
621 | } | |
622 | ||
623 | /* | |
624 | * If any of the command structures end up being larger than the | |
625 | * TFD_MAX_PAYLOAD_SIZE and they aren't dynamically allocated into | |
626 | * separate TFDs, then we will need to increase the size of the buffers | |
627 | */ | |
628 | if (WARN(copy_size > TFD_MAX_PAYLOAD_SIZE, | |
629 | "Command %s (%#x) is too large (%d bytes)\n", | |
630 | iwl_get_cmd_string(trans, cmd->id), cmd->id, copy_size)) { | |
631 | idx = -EINVAL; | |
632 | goto free_dup_buf; | |
633 | } | |
634 | ||
635 | spin_lock_bh(&txq->lock); | |
636 | ||
637 | if (iwl_queue_space(txq) < ((cmd->flags & CMD_ASYNC) ? 2 : 1)) { | |
638 | spin_unlock_bh(&txq->lock); | |
639 | ||
640 | IWL_ERR(trans, "No space in command queue\n"); | |
641 | iwl_op_mode_cmd_queue_full(trans->op_mode); | |
642 | idx = -ENOSPC; | |
643 | goto free_dup_buf; | |
644 | } | |
645 | ||
646 | out_cmd = txq->entries[idx].cmd; | |
647 | out_meta = &txq->entries[idx].meta; | |
648 | ||
649 | /* re-initialize to NULL */ | |
650 | memset(out_meta, 0, sizeof(*out_meta)); | |
651 | if (cmd->flags & CMD_WANT_SKB) | |
652 | out_meta->source = cmd; | |
653 | ||
654 | /* set up the header */ | |
655 | out_cmd->hdr_wide.cmd = iwl_cmd_opcode(cmd->id); | |
656 | out_cmd->hdr_wide.group_id = group_id; | |
657 | out_cmd->hdr_wide.version = iwl_cmd_version(cmd->id); | |
658 | out_cmd->hdr_wide.length = | |
659 | cpu_to_le16(cmd_size - sizeof(struct iwl_cmd_header_wide)); | |
660 | out_cmd->hdr_wide.reserved = 0; | |
661 | out_cmd->hdr_wide.sequence = | |
662 | cpu_to_le16(QUEUE_TO_SEQ(trans_pcie->cmd_queue) | | |
663 | INDEX_TO_SEQ(txq->write_ptr)); | |
664 | ||
665 | cmd_pos = sizeof(struct iwl_cmd_header_wide); | |
666 | copy_size = sizeof(struct iwl_cmd_header_wide); | |
667 | ||
668 | /* and copy the data that needs to be copied */ | |
669 | for (i = 0; i < IWL_MAX_CMD_TBS_PER_TFD; i++) { | |
670 | int copy; | |
671 | ||
672 | if (!cmd->len[i]) | |
673 | continue; | |
674 | ||
675 | /* copy everything if not nocopy/dup */ | |
676 | if (!(cmd->dataflags[i] & (IWL_HCMD_DFL_NOCOPY | | |
677 | IWL_HCMD_DFL_DUP))) { | |
678 | copy = cmd->len[i]; | |
679 | ||
680 | memcpy((u8 *)out_cmd + cmd_pos, cmd->data[i], copy); | |
681 | cmd_pos += copy; | |
682 | copy_size += copy; | |
683 | continue; | |
684 | } | |
685 | ||
686 | /* | |
687 | * Otherwise we need at least IWL_FIRST_TB_SIZE copied | |
688 | * in total (for bi-directional DMA), but copy up to what | |
689 | * we can fit into the payload for debug dump purposes. | |
690 | */ | |
691 | copy = min_t(int, TFD_MAX_PAYLOAD_SIZE - cmd_pos, cmd->len[i]); | |
692 | ||
693 | memcpy((u8 *)out_cmd + cmd_pos, cmd->data[i], copy); | |
694 | cmd_pos += copy; | |
695 | ||
696 | /* However, treat copy_size the proper way, we need it below */ | |
697 | if (copy_size < IWL_FIRST_TB_SIZE) { | |
698 | copy = IWL_FIRST_TB_SIZE - copy_size; | |
699 | ||
700 | if (copy > cmd->len[i]) | |
701 | copy = cmd->len[i]; | |
702 | copy_size += copy; | |
703 | } | |
704 | } | |
705 | ||
706 | IWL_DEBUG_HC(trans, | |
707 | "Sending command %s (%.2x.%.2x), seq: 0x%04X, %d bytes at %d[%d]:%d\n", | |
708 | iwl_get_cmd_string(trans, cmd->id), group_id, | |
709 | out_cmd->hdr.cmd, le16_to_cpu(out_cmd->hdr.sequence), | |
710 | cmd_size, txq->write_ptr, idx, trans_pcie->cmd_queue); | |
711 | ||
712 | /* start the TFD with the minimum copy bytes */ | |
713 | tb0_size = min_t(int, copy_size, IWL_FIRST_TB_SIZE); | |
714 | memcpy(&txq->first_tb_bufs[idx], &out_cmd->hdr, tb0_size); | |
715 | iwl_pcie_gen2_set_tb(trans, tfd, iwl_pcie_get_first_tb_dma(txq, idx), | |
716 | tb0_size); | |
717 | ||
718 | /* map first command fragment, if any remains */ | |
719 | if (copy_size > tb0_size) { | |
720 | phys_addr = dma_map_single(trans->dev, | |
721 | ((u8 *)&out_cmd->hdr) + tb0_size, | |
722 | copy_size - tb0_size, | |
723 | DMA_TO_DEVICE); | |
724 | if (dma_mapping_error(trans->dev, phys_addr)) { | |
725 | idx = -ENOMEM; | |
726 | iwl_pcie_gen2_tfd_unmap(trans, out_meta, tfd); | |
727 | goto out; | |
728 | } | |
729 | iwl_pcie_gen2_set_tb(trans, tfd, phys_addr, | |
730 | copy_size - tb0_size); | |
731 | } | |
732 | ||
733 | /* map the remaining (adjusted) nocopy/dup fragments */ | |
734 | for (i = 0; i < IWL_MAX_CMD_TBS_PER_TFD; i++) { | |
735 | const void *data = cmddata[i]; | |
736 | ||
737 | if (!cmdlen[i]) | |
738 | continue; | |
739 | if (!(cmd->dataflags[i] & (IWL_HCMD_DFL_NOCOPY | | |
740 | IWL_HCMD_DFL_DUP))) | |
741 | continue; | |
742 | if (cmd->dataflags[i] & IWL_HCMD_DFL_DUP) | |
743 | data = dup_buf; | |
744 | phys_addr = dma_map_single(trans->dev, (void *)data, | |
745 | cmdlen[i], DMA_TO_DEVICE); | |
746 | if (dma_mapping_error(trans->dev, phys_addr)) { | |
747 | idx = -ENOMEM; | |
748 | iwl_pcie_gen2_tfd_unmap(trans, out_meta, tfd); | |
749 | goto out; | |
750 | } | |
751 | iwl_pcie_gen2_set_tb(trans, tfd, phys_addr, cmdlen[i]); | |
752 | } | |
753 | ||
754 | BUILD_BUG_ON(IWL_TFH_NUM_TBS > sizeof(out_meta->tbs) * BITS_PER_BYTE); | |
755 | out_meta->flags = cmd->flags; | |
756 | if (WARN_ON_ONCE(txq->entries[idx].free_buf)) | |
757 | kzfree(txq->entries[idx].free_buf); | |
758 | txq->entries[idx].free_buf = dup_buf; | |
759 | ||
760 | trace_iwlwifi_dev_hcmd(trans->dev, cmd, cmd_size, &out_cmd->hdr_wide); | |
761 | ||
762 | /* start timer if queue currently empty */ | |
763 | if (txq->read_ptr == txq->write_ptr && txq->wd_timeout) | |
764 | mod_timer(&txq->stuck_timer, jiffies + txq->wd_timeout); | |
765 | ||
766 | spin_lock_irqsave(&trans_pcie->reg_lock, flags); | |
767 | if (!(cmd->flags & CMD_SEND_IN_IDLE) && | |
768 | !trans_pcie->ref_cmd_in_flight) { | |
769 | trans_pcie->ref_cmd_in_flight = true; | |
770 | IWL_DEBUG_RPM(trans, "set ref_cmd_in_flight - ref\n"); | |
771 | iwl_trans_ref(trans); | |
772 | } | |
773 | /* Increment and update queue's write index */ | |
774 | txq->write_ptr = iwl_queue_inc_wrap(txq->write_ptr); | |
775 | iwl_pcie_gen2_txq_inc_wr_ptr(trans, txq); | |
776 | spin_unlock_irqrestore(&trans_pcie->reg_lock, flags); | |
777 | ||
778 | out: | |
779 | spin_unlock_bh(&txq->lock); | |
780 | free_dup_buf: | |
781 | if (idx < 0) | |
782 | kfree(dup_buf); | |
783 | return idx; | |
784 | } | |
785 | ||
786 | #define HOST_COMPLETE_TIMEOUT (2 * HZ) | |
787 | ||
788 | static int iwl_pcie_gen2_send_hcmd_sync(struct iwl_trans *trans, | |
789 | struct iwl_host_cmd *cmd) | |
790 | { | |
791 | struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); | |
792 | const char *cmd_str = iwl_get_cmd_string(trans, cmd->id); | |
793 | struct iwl_txq *txq = trans_pcie->txq[trans_pcie->cmd_queue]; | |
794 | int cmd_idx; | |
795 | int ret; | |
796 | ||
797 | IWL_DEBUG_INFO(trans, "Attempting to send sync command %s\n", cmd_str); | |
798 | ||
799 | if (WARN(test_and_set_bit(STATUS_SYNC_HCMD_ACTIVE, | |
800 | &trans->status), | |
801 | "Command %s: a command is already active!\n", cmd_str)) | |
802 | return -EIO; | |
803 | ||
804 | IWL_DEBUG_INFO(trans, "Setting HCMD_ACTIVE for command %s\n", cmd_str); | |
805 | ||
806 | if (pm_runtime_suspended(&trans_pcie->pci_dev->dev)) { | |
807 | ret = wait_event_timeout(trans_pcie->d0i3_waitq, | |
808 | pm_runtime_active(&trans_pcie->pci_dev->dev), | |
809 | msecs_to_jiffies(IWL_TRANS_IDLE_TIMEOUT)); | |
810 | if (!ret) { | |
811 | IWL_ERR(trans, "Timeout exiting D0i3 before hcmd\n"); | |
812 | return -ETIMEDOUT; | |
813 | } | |
814 | } | |
815 | ||
816 | cmd_idx = iwl_pcie_gen2_enqueue_hcmd(trans, cmd); | |
817 | if (cmd_idx < 0) { | |
818 | ret = cmd_idx; | |
819 | clear_bit(STATUS_SYNC_HCMD_ACTIVE, &trans->status); | |
820 | IWL_ERR(trans, "Error sending %s: enqueue_hcmd failed: %d\n", | |
821 | cmd_str, ret); | |
822 | return ret; | |
823 | } | |
824 | ||
825 | ret = wait_event_timeout(trans_pcie->wait_command_queue, | |
826 | !test_bit(STATUS_SYNC_HCMD_ACTIVE, | |
827 | &trans->status), | |
828 | HOST_COMPLETE_TIMEOUT); | |
829 | if (!ret) { | |
830 | IWL_ERR(trans, "Error sending %s: time out after %dms.\n", | |
831 | cmd_str, jiffies_to_msecs(HOST_COMPLETE_TIMEOUT)); | |
832 | ||
833 | IWL_ERR(trans, "Current CMD queue read_ptr %d write_ptr %d\n", | |
834 | txq->read_ptr, txq->write_ptr); | |
835 | ||
836 | clear_bit(STATUS_SYNC_HCMD_ACTIVE, &trans->status); | |
837 | IWL_DEBUG_INFO(trans, "Clearing HCMD_ACTIVE for command %s\n", | |
838 | cmd_str); | |
839 | ret = -ETIMEDOUT; | |
840 | ||
841 | iwl_force_nmi(trans); | |
842 | iwl_trans_fw_error(trans); | |
843 | ||
844 | goto cancel; | |
845 | } | |
846 | ||
847 | if (test_bit(STATUS_FW_ERROR, &trans->status)) { | |
848 | IWL_ERR(trans, "FW error in SYNC CMD %s\n", cmd_str); | |
849 | dump_stack(); | |
850 | ret = -EIO; | |
851 | goto cancel; | |
852 | } | |
853 | ||
854 | if (!(cmd->flags & CMD_SEND_IN_RFKILL) && | |
855 | test_bit(STATUS_RFKILL_OPMODE, &trans->status)) { | |
856 | IWL_DEBUG_RF_KILL(trans, "RFKILL in SYNC CMD... no rsp\n"); | |
857 | ret = -ERFKILL; | |
858 | goto cancel; | |
859 | } | |
860 | ||
861 | if ((cmd->flags & CMD_WANT_SKB) && !cmd->resp_pkt) { | |
862 | IWL_ERR(trans, "Error: Response NULL in '%s'\n", cmd_str); | |
863 | ret = -EIO; | |
864 | goto cancel; | |
865 | } | |
866 | ||
867 | return 0; | |
868 | ||
869 | cancel: | |
870 | if (cmd->flags & CMD_WANT_SKB) { | |
871 | /* | |
872 | * Cancel the CMD_WANT_SKB flag for the cmd in the | |
873 | * TX cmd queue. Otherwise in case the cmd comes | |
874 | * in later, it will possibly set an invalid | |
875 | * address (cmd->meta.source). | |
876 | */ | |
877 | txq->entries[cmd_idx].meta.flags &= ~CMD_WANT_SKB; | |
878 | } | |
879 | ||
880 | if (cmd->resp_pkt) { | |
881 | iwl_free_resp(cmd); | |
882 | cmd->resp_pkt = NULL; | |
883 | } | |
884 | ||
885 | return ret; | |
886 | } | |
887 | ||
888 | int iwl_trans_pcie_gen2_send_hcmd(struct iwl_trans *trans, | |
889 | struct iwl_host_cmd *cmd) | |
890 | { | |
891 | if (!(cmd->flags & CMD_SEND_IN_RFKILL) && | |
892 | test_bit(STATUS_RFKILL_OPMODE, &trans->status)) { | |
893 | IWL_DEBUG_RF_KILL(trans, "Dropping CMD 0x%x: RF KILL\n", | |
894 | cmd->id); | |
895 | return -ERFKILL; | |
896 | } | |
897 | ||
898 | if (cmd->flags & CMD_ASYNC) { | |
899 | int ret; | |
900 | ||
901 | /* An asynchronous command can not expect an SKB to be set. */ | |
902 | if (WARN_ON(cmd->flags & CMD_WANT_SKB)) | |
903 | return -EINVAL; | |
904 | ||
905 | ret = iwl_pcie_gen2_enqueue_hcmd(trans, cmd); | |
906 | if (ret < 0) { | |
907 | IWL_ERR(trans, | |
908 | "Error sending %s: enqueue_hcmd failed: %d\n", | |
909 | iwl_get_cmd_string(trans, cmd->id), ret); | |
910 | return ret; | |
911 | } | |
912 | return 0; | |
913 | } | |
914 | ||
915 | return iwl_pcie_gen2_send_hcmd_sync(trans, cmd); | |
916 | } | |
917 | ||
918 | /* | |
919 | * iwl_pcie_gen2_txq_unmap - Unmap any remaining DMA mappings and free skb's | |
920 | */ | |
921 | void iwl_pcie_gen2_txq_unmap(struct iwl_trans *trans, int txq_id) | |
922 | { | |
923 | struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); | |
924 | struct iwl_txq *txq = trans_pcie->txq[txq_id]; | |
925 | ||
926 | spin_lock_bh(&txq->lock); | |
927 | while (txq->write_ptr != txq->read_ptr) { | |
928 | IWL_DEBUG_TX_REPLY(trans, "Q %d Free %d\n", | |
929 | txq_id, txq->read_ptr); | |
930 | ||
931 | if (txq_id != trans_pcie->cmd_queue) { | |
932 | int idx = iwl_pcie_get_cmd_index(txq, txq->read_ptr); | |
933 | struct sk_buff *skb = txq->entries[idx].skb; | |
934 | ||
935 | if (WARN_ON_ONCE(!skb)) | |
936 | continue; | |
937 | ||
938 | iwl_pcie_free_tso_page(trans_pcie, skb); | |
939 | } | |
940 | iwl_pcie_gen2_free_tfd(trans, txq); | |
941 | txq->read_ptr = iwl_queue_inc_wrap(txq->read_ptr); | |
942 | ||
943 | if (txq->read_ptr == txq->write_ptr) { | |
944 | unsigned long flags; | |
945 | ||
946 | spin_lock_irqsave(&trans_pcie->reg_lock, flags); | |
947 | if (txq_id != trans_pcie->cmd_queue) { | |
948 | IWL_DEBUG_RPM(trans, "Q %d - last tx freed\n", | |
949 | txq->id); | |
950 | iwl_trans_unref(trans); | |
951 | } else if (trans_pcie->ref_cmd_in_flight) { | |
952 | trans_pcie->ref_cmd_in_flight = false; | |
953 | IWL_DEBUG_RPM(trans, | |
954 | "clear ref_cmd_in_flight\n"); | |
955 | iwl_trans_unref(trans); | |
956 | } | |
957 | spin_unlock_irqrestore(&trans_pcie->reg_lock, flags); | |
958 | } | |
959 | } | |
960 | spin_unlock_bh(&txq->lock); | |
961 | ||
962 | /* just in case - this queue may have been stopped */ | |
963 | iwl_wake_queue(trans, txq); | |
964 | } | |
965 | ||
966 | static void iwl_pcie_gen2_txq_free_memory(struct iwl_trans *trans, | |
967 | struct iwl_txq *txq) | |
968 | { | |
969 | struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); | |
970 | struct device *dev = trans->dev; | |
971 | ||
972 | /* De-alloc circular buffer of TFDs */ | |
973 | if (txq->tfds) { | |
974 | dma_free_coherent(dev, | |
975 | trans_pcie->tfd_size * TFD_QUEUE_SIZE_MAX, | |
976 | txq->tfds, txq->dma_addr); | |
977 | dma_free_coherent(dev, | |
978 | sizeof(*txq->first_tb_bufs) * txq->n_window, | |
979 | txq->first_tb_bufs, txq->first_tb_dma); | |
980 | } | |
981 | ||
982 | kfree(txq->entries); | |
983 | iwl_pcie_free_dma_ptr(trans, &txq->bc_tbl); | |
984 | kfree(txq); | |
985 | } | |
986 | ||
987 | /* | |
988 | * iwl_pcie_txq_free - Deallocate DMA queue. | |
989 | * @txq: Transmit queue to deallocate. | |
990 | * | |
991 | * Empty queue by removing and destroying all BD's. | |
992 | * Free all buffers. | |
993 | * 0-fill, but do not free "txq" descriptor structure. | |
994 | */ | |
995 | static void iwl_pcie_gen2_txq_free(struct iwl_trans *trans, int txq_id) | |
996 | { | |
997 | struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); | |
998 | struct iwl_txq *txq = trans_pcie->txq[txq_id]; | |
999 | int i; | |
1000 | ||
1001 | if (WARN_ON(!txq)) | |
1002 | return; | |
1003 | ||
1004 | iwl_pcie_gen2_txq_unmap(trans, txq_id); | |
1005 | ||
1006 | /* De-alloc array of command/tx buffers */ | |
1007 | if (txq_id == trans_pcie->cmd_queue) | |
1008 | for (i = 0; i < txq->n_window; i++) { | |
1009 | kzfree(txq->entries[i].cmd); | |
1010 | kzfree(txq->entries[i].free_buf); | |
1011 | } | |
1012 | del_timer_sync(&txq->stuck_timer); | |
1013 | ||
1014 | iwl_pcie_gen2_txq_free_memory(trans, txq); | |
1015 | ||
1016 | trans_pcie->txq[txq_id] = NULL; | |
1017 | ||
1018 | clear_bit(txq_id, trans_pcie->queue_used); | |
1019 | } | |
1020 | ||
1021 | int iwl_trans_pcie_dyn_txq_alloc(struct iwl_trans *trans, | |
1022 | struct iwl_tx_queue_cfg_cmd *cmd, | |
1023 | int cmd_id, | |
1024 | unsigned int timeout) | |
1025 | { | |
1026 | struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); | |
1027 | struct iwl_tx_queue_cfg_rsp *rsp; | |
1028 | struct iwl_txq *txq; | |
1029 | struct iwl_host_cmd hcmd = { | |
1030 | .id = cmd_id, | |
1031 | .len = { sizeof(*cmd) }, | |
1032 | .data = { cmd, }, | |
1033 | .flags = CMD_WANT_SKB, | |
1034 | }; | |
1035 | int ret, qid; | |
1036 | u32 wr_ptr; | |
1037 | ||
1038 | txq = kzalloc(sizeof(*txq), GFP_KERNEL); | |
1039 | if (!txq) | |
1040 | return -ENOMEM; | |
1041 | ret = iwl_pcie_alloc_dma_ptr(trans, &txq->bc_tbl, | |
1042 | sizeof(struct iwlagn_scd_bc_tbl)); | |
1043 | if (ret) { | |
1044 | IWL_ERR(trans, "Scheduler BC Table allocation failed\n"); | |
1045 | kfree(txq); | |
1046 | return -ENOMEM; | |
1047 | } | |
1048 | ||
1049 | ret = iwl_pcie_txq_alloc(trans, txq, TFD_TX_CMD_SLOTS, false); | |
1050 | if (ret) { | |
1051 | IWL_ERR(trans, "Tx queue alloc failed\n"); | |
1052 | goto error; | |
1053 | } | |
1054 | ret = iwl_pcie_txq_init(trans, txq, TFD_TX_CMD_SLOTS, false); | |
1055 | if (ret) { | |
1056 | IWL_ERR(trans, "Tx queue init failed\n"); | |
1057 | goto error; | |
1058 | } | |
1059 | ||
1060 | txq->wd_timeout = msecs_to_jiffies(timeout); | |
1061 | ||
1062 | cmd->tfdq_addr = cpu_to_le64(txq->dma_addr); | |
1063 | cmd->byte_cnt_addr = cpu_to_le64(txq->bc_tbl.dma); | |
1064 | cmd->cb_size = cpu_to_le32(TFD_QUEUE_CB_SIZE(TFD_TX_CMD_SLOTS)); | |
1065 | ||
1066 | ret = iwl_trans_send_cmd(trans, &hcmd); | |
1067 | if (ret) | |
1068 | goto error; | |
1069 | ||
1070 | if (WARN_ON(iwl_rx_packet_payload_len(hcmd.resp_pkt) != sizeof(*rsp))) { | |
1071 | ret = -EINVAL; | |
1072 | goto error_free_resp; | |
1073 | } | |
1074 | ||
1075 | rsp = (void *)hcmd.resp_pkt->data; | |
1076 | qid = le16_to_cpu(rsp->queue_number); | |
1077 | wr_ptr = le16_to_cpu(rsp->write_pointer); | |
1078 | ||
1079 | if (qid >= ARRAY_SIZE(trans_pcie->txq)) { | |
1080 | WARN_ONCE(1, "queue index %d unsupported", qid); | |
1081 | ret = -EIO; | |
1082 | goto error_free_resp; | |
1083 | } | |
1084 | ||
1085 | if (test_and_set_bit(qid, trans_pcie->queue_used)) { | |
1086 | WARN_ONCE(1, "queue %d already used", qid); | |
1087 | ret = -EIO; | |
1088 | goto error_free_resp; | |
1089 | } | |
1090 | ||
1091 | txq->id = qid; | |
1092 | trans_pcie->txq[qid] = txq; | |
1093 | wr_ptr &= (TFD_QUEUE_SIZE_MAX - 1); | |
1094 | ||
1095 | /* Place first TFD at index corresponding to start sequence number */ | |
1096 | txq->read_ptr = wr_ptr; | |
1097 | txq->write_ptr = wr_ptr; | |
1098 | iwl_write_direct32(trans, HBUS_TARG_WRPTR, | |
1099 | (txq->write_ptr) | (qid << 16)); | |
1100 | IWL_DEBUG_TX_QUEUES(trans, "Activate queue %d\n", qid); | |
1101 | ||
1102 | iwl_free_resp(&hcmd); | |
1103 | return qid; | |
1104 | ||
1105 | error_free_resp: | |
1106 | iwl_free_resp(&hcmd); | |
1107 | error: | |
1108 | iwl_pcie_gen2_txq_free_memory(trans, txq); | |
1109 | return ret; | |
1110 | } | |
1111 | ||
1112 | void iwl_trans_pcie_dyn_txq_free(struct iwl_trans *trans, int queue) | |
1113 | { | |
1114 | struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); | |
1115 | ||
1116 | /* | |
1117 | * Upon HW Rfkill - we stop the device, and then stop the queues | |
1118 | * in the op_mode. Just for the sake of the simplicity of the op_mode, | |
1119 | * allow the op_mode to call txq_disable after it already called | |
1120 | * stop_device. | |
1121 | */ | |
1122 | if (!test_and_clear_bit(queue, trans_pcie->queue_used)) { | |
1123 | WARN_ONCE(test_bit(STATUS_DEVICE_ENABLED, &trans->status), | |
1124 | "queue %d not used", queue); | |
1125 | return; | |
1126 | } | |
1127 | ||
1128 | iwl_pcie_gen2_txq_unmap(trans, queue); | |
1129 | ||
1130 | IWL_DEBUG_TX_QUEUES(trans, "Deactivate queue %d\n", queue); | |
1131 | } | |
1132 | ||
1133 | void iwl_pcie_gen2_tx_free(struct iwl_trans *trans) | |
1134 | { | |
1135 | struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); | |
1136 | int i; | |
1137 | ||
1138 | memset(trans_pcie->queue_used, 0, sizeof(trans_pcie->queue_used)); | |
1139 | ||
1140 | /* Free all TX queues */ | |
1141 | for (i = 0; i < ARRAY_SIZE(trans_pcie->txq); i++) { | |
1142 | if (!trans_pcie->txq[i]) | |
1143 | continue; | |
1144 | ||
1145 | iwl_pcie_gen2_txq_free(trans, i); | |
1146 | } | |
1147 | } | |
1148 | ||
1149 | int iwl_pcie_gen2_tx_init(struct iwl_trans *trans) | |
1150 | { | |
1151 | struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); | |
1152 | struct iwl_txq *cmd_queue; | |
1153 | int txq_id = trans_pcie->cmd_queue, ret; | |
1154 | ||
1155 | iwl_pcie_set_tx_cmd_queue_size(trans); | |
1156 | ||
1157 | /* alloc and init the command queue */ | |
1158 | if (!trans_pcie->txq[txq_id]) { | |
1159 | cmd_queue = kzalloc(sizeof(*cmd_queue), GFP_KERNEL); | |
1160 | if (!cmd_queue) { | |
1161 | IWL_ERR(trans, "Not enough memory for command queue\n"); | |
1162 | return -ENOMEM; | |
1163 | } | |
1164 | trans_pcie->txq[txq_id] = cmd_queue; | |
1165 | ret = iwl_pcie_txq_alloc(trans, cmd_queue, | |
1166 | trans_pcie->tx_cmd_queue_size, true); | |
1167 | if (ret) { | |
1168 | IWL_ERR(trans, "Tx %d queue init failed\n", txq_id); | |
1169 | goto error; | |
1170 | } | |
1171 | } else { | |
1172 | cmd_queue = trans_pcie->txq[txq_id]; | |
1173 | } | |
1174 | ||
1175 | ret = iwl_pcie_txq_init(trans, cmd_queue, | |
1176 | trans_pcie->tx_cmd_queue_size, true); | |
1177 | if (ret) { | |
1178 | IWL_ERR(trans, "Tx %d queue alloc failed\n", txq_id); | |
1179 | goto error; | |
1180 | } | |
1181 | trans_pcie->txq[txq_id]->id = txq_id; | |
1182 | set_bit(txq_id, trans_pcie->queue_used); | |
1183 | ||
1184 | return 0; | |
1185 | ||
1186 | error: | |
1187 | iwl_pcie_gen2_tx_free(trans); | |
1188 | return ret; | |
1189 | } | |
1190 |