]>
Commit | Line | Data |
---|---|---|
a672a9dc TS |
1 | /* |
2 | This file is provided under a dual BSD/GPLv2 license. When using or | |
3 | redistributing this file, you may do so under either license. | |
4 | ||
5 | GPL LICENSE SUMMARY | |
6 | Copyright(c) 2014 Intel Corporation. | |
7 | This program is free software; you can redistribute it and/or modify | |
8 | it under the terms of version 2 of the GNU General Public License as | |
9 | published by the Free Software Foundation. | |
10 | ||
11 | This program is distributed in the hope that it will be useful, but | |
12 | WITHOUT ANY WARRANTY; without even the implied warranty of | |
13 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
14 | General Public License for more details. | |
15 | ||
16 | Contact Information: | |
17 | qat-linux@intel.com | |
18 | ||
19 | BSD LICENSE | |
20 | Copyright(c) 2014 Intel Corporation. | |
21 | Redistribution and use in source and binary forms, with or without | |
22 | modification, are permitted provided that the following conditions | |
23 | are met: | |
24 | ||
25 | * Redistributions of source code must retain the above copyright | |
26 | notice, this list of conditions and the following disclaimer. | |
27 | * Redistributions in binary form must reproduce the above copyright | |
28 | notice, this list of conditions and the following disclaimer in | |
29 | the documentation and/or other materials provided with the | |
30 | distribution. | |
31 | * Neither the name of Intel Corporation nor the names of its | |
32 | contributors may be used to endorse or promote products derived | |
33 | from this software without specific prior written permission. | |
34 | ||
35 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS | |
36 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT | |
37 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR | |
38 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT | |
39 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, | |
40 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT | |
41 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, | |
42 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY | |
43 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT | |
44 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE | |
45 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | |
46 | */ | |
47 | #include <linux/delay.h> | |
48 | #include "adf_accel_devices.h" | |
49 | #include "adf_transport_internal.h" | |
50 | #include "adf_transport_access_macros.h" | |
51 | #include "adf_cfg.h" | |
52 | #include "adf_common_drv.h" | |
53 | ||
54 | static inline uint32_t adf_modulo(uint32_t data, uint32_t shift) | |
55 | { | |
56 | uint32_t div = data >> shift; | |
57 | uint32_t mult = div << shift; | |
d65071ec | 58 | |
a672a9dc TS |
59 | return data - mult; |
60 | } | |
61 | ||
62 | static inline int adf_check_ring_alignment(uint64_t addr, uint64_t size) | |
63 | { | |
64 | if (((size - 1) & addr) != 0) | |
65 | return -EFAULT; | |
66 | return 0; | |
67 | } | |
68 | ||
69 | static int adf_verify_ring_size(uint32_t msg_size, uint32_t msg_num) | |
70 | { | |
71 | int i = ADF_MIN_RING_SIZE; | |
d65071ec | 72 | |
a672a9dc TS |
73 | for (; i <= ADF_MAX_RING_SIZE; i++) |
74 | if ((msg_size * msg_num) == ADF_SIZE_TO_RING_SIZE_IN_BYTES(i)) | |
75 | return i; | |
76 | ||
77 | return ADF_DEFAULT_RING_SIZE; | |
78 | } | |
79 | ||
80 | static int adf_reserve_ring(struct adf_etr_bank_data *bank, uint32_t ring) | |
81 | { | |
82 | spin_lock(&bank->lock); | |
83 | if (bank->ring_mask & (1 << ring)) { | |
84 | spin_unlock(&bank->lock); | |
85 | return -EFAULT; | |
86 | } | |
87 | bank->ring_mask |= (1 << ring); | |
88 | spin_unlock(&bank->lock); | |
89 | return 0; | |
90 | } | |
91 | ||
92 | static void adf_unreserve_ring(struct adf_etr_bank_data *bank, uint32_t ring) | |
93 | { | |
94 | spin_lock(&bank->lock); | |
95 | bank->ring_mask &= ~(1 << ring); | |
96 | spin_unlock(&bank->lock); | |
97 | } | |
98 | ||
99 | static void adf_enable_ring_irq(struct adf_etr_bank_data *bank, uint32_t ring) | |
100 | { | |
101 | spin_lock_bh(&bank->lock); | |
102 | bank->irq_mask |= (1 << ring); | |
103 | spin_unlock_bh(&bank->lock); | |
104 | WRITE_CSR_INT_COL_EN(bank->csr_addr, bank->bank_number, bank->irq_mask); | |
105 | WRITE_CSR_INT_COL_CTL(bank->csr_addr, bank->bank_number, | |
106 | bank->irq_coalesc_timer); | |
107 | } | |
108 | ||
109 | static void adf_disable_ring_irq(struct adf_etr_bank_data *bank, uint32_t ring) | |
110 | { | |
111 | spin_lock_bh(&bank->lock); | |
112 | bank->irq_mask &= ~(1 << ring); | |
113 | spin_unlock_bh(&bank->lock); | |
114 | WRITE_CSR_INT_COL_EN(bank->csr_addr, bank->bank_number, bank->irq_mask); | |
115 | } | |
116 | ||
117 | int adf_send_message(struct adf_etr_ring_data *ring, uint32_t *msg) | |
118 | { | |
119 | if (atomic_add_return(1, ring->inflights) > | |
120 | ADF_MAX_INFLIGHTS(ring->ring_size, ring->msg_size)) { | |
121 | atomic_dec(ring->inflights); | |
122 | return -EAGAIN; | |
123 | } | |
124 | spin_lock_bh(&ring->lock); | |
1a72d3a6 | 125 | memcpy((void *)((uintptr_t)ring->base_addr + ring->tail), msg, |
a672a9dc TS |
126 | ADF_MSG_SIZE_TO_BYTES(ring->msg_size)); |
127 | ||
128 | ring->tail = adf_modulo(ring->tail + | |
129 | ADF_MSG_SIZE_TO_BYTES(ring->msg_size), | |
130 | ADF_RING_SIZE_MODULO(ring->ring_size)); | |
131 | WRITE_CSR_RING_TAIL(ring->bank->csr_addr, ring->bank->bank_number, | |
132 | ring->ring_number, ring->tail); | |
133 | spin_unlock_bh(&ring->lock); | |
134 | return 0; | |
135 | } | |
136 | ||
137 | static int adf_handle_response(struct adf_etr_ring_data *ring) | |
138 | { | |
139 | uint32_t msg_counter = 0; | |
1a72d3a6 | 140 | uint32_t *msg = (uint32_t *)((uintptr_t)ring->base_addr + ring->head); |
a672a9dc TS |
141 | |
142 | while (*msg != ADF_RING_EMPTY_SIG) { | |
143 | ring->callback((uint32_t *)msg); | |
144 | *msg = ADF_RING_EMPTY_SIG; | |
145 | ring->head = adf_modulo(ring->head + | |
146 | ADF_MSG_SIZE_TO_BYTES(ring->msg_size), | |
147 | ADF_RING_SIZE_MODULO(ring->ring_size)); | |
148 | msg_counter++; | |
1a72d3a6 | 149 | msg = (uint32_t *)((uintptr_t)ring->base_addr + ring->head); |
a672a9dc TS |
150 | } |
151 | if (msg_counter > 0) { | |
152 | WRITE_CSR_RING_HEAD(ring->bank->csr_addr, | |
153 | ring->bank->bank_number, | |
154 | ring->ring_number, ring->head); | |
155 | atomic_sub(msg_counter, ring->inflights); | |
156 | } | |
157 | return 0; | |
158 | } | |
159 | ||
160 | static void adf_configure_tx_ring(struct adf_etr_ring_data *ring) | |
161 | { | |
162 | uint32_t ring_config = BUILD_RING_CONFIG(ring->ring_size); | |
163 | ||
164 | WRITE_CSR_RING_CONFIG(ring->bank->csr_addr, ring->bank->bank_number, | |
165 | ring->ring_number, ring_config); | |
166 | } | |
167 | ||
168 | static void adf_configure_rx_ring(struct adf_etr_ring_data *ring) | |
169 | { | |
170 | uint32_t ring_config = | |
171 | BUILD_RESP_RING_CONFIG(ring->ring_size, | |
172 | ADF_RING_NEAR_WATERMARK_512, | |
173 | ADF_RING_NEAR_WATERMARK_0); | |
174 | ||
175 | WRITE_CSR_RING_CONFIG(ring->bank->csr_addr, ring->bank->bank_number, | |
176 | ring->ring_number, ring_config); | |
177 | } | |
178 | ||
179 | static int adf_init_ring(struct adf_etr_ring_data *ring) | |
180 | { | |
181 | struct adf_etr_bank_data *bank = ring->bank; | |
182 | struct adf_accel_dev *accel_dev = bank->accel_dev; | |
183 | struct adf_hw_device_data *hw_data = accel_dev->hw_device; | |
184 | uint64_t ring_base; | |
185 | uint32_t ring_size_bytes = | |
186 | ADF_SIZE_TO_RING_SIZE_IN_BYTES(ring->ring_size); | |
187 | ||
188 | ring_size_bytes = ADF_RING_SIZE_BYTES_MIN(ring_size_bytes); | |
189 | ring->base_addr = dma_alloc_coherent(&GET_DEV(accel_dev), | |
190 | ring_size_bytes, &ring->dma_addr, | |
191 | GFP_KERNEL); | |
192 | if (!ring->base_addr) | |
193 | return -ENOMEM; | |
194 | ||
195 | memset(ring->base_addr, 0x7F, ring_size_bytes); | |
196 | /* The base_addr has to be aligned to the size of the buffer */ | |
197 | if (adf_check_ring_alignment(ring->dma_addr, ring_size_bytes)) { | |
66550304 | 198 | dev_err(&GET_DEV(accel_dev), "Ring address not aligned\n"); |
a672a9dc TS |
199 | dma_free_coherent(&GET_DEV(accel_dev), ring_size_bytes, |
200 | ring->base_addr, ring->dma_addr); | |
201 | return -EFAULT; | |
202 | } | |
203 | ||
204 | if (hw_data->tx_rings_mask & (1 << ring->ring_number)) | |
205 | adf_configure_tx_ring(ring); | |
206 | ||
207 | else | |
208 | adf_configure_rx_ring(ring); | |
209 | ||
210 | ring_base = BUILD_RING_BASE_ADDR(ring->dma_addr, ring->ring_size); | |
211 | WRITE_CSR_RING_BASE(ring->bank->csr_addr, ring->bank->bank_number, | |
212 | ring->ring_number, ring_base); | |
213 | spin_lock_init(&ring->lock); | |
214 | return 0; | |
215 | } | |
216 | ||
217 | static void adf_cleanup_ring(struct adf_etr_ring_data *ring) | |
218 | { | |
219 | uint32_t ring_size_bytes = | |
220 | ADF_SIZE_TO_RING_SIZE_IN_BYTES(ring->ring_size); | |
221 | ring_size_bytes = ADF_RING_SIZE_BYTES_MIN(ring_size_bytes); | |
222 | ||
223 | if (ring->base_addr) { | |
224 | memset(ring->base_addr, 0x7F, ring_size_bytes); | |
225 | dma_free_coherent(&GET_DEV(ring->bank->accel_dev), | |
226 | ring_size_bytes, ring->base_addr, | |
227 | ring->dma_addr); | |
228 | } | |
229 | } | |
230 | ||
231 | int adf_create_ring(struct adf_accel_dev *accel_dev, const char *section, | |
232 | uint32_t bank_num, uint32_t num_msgs, | |
233 | uint32_t msg_size, const char *ring_name, | |
234 | adf_callback_fn callback, int poll_mode, | |
235 | struct adf_etr_ring_data **ring_ptr) | |
236 | { | |
237 | struct adf_etr_data *transport_data = accel_dev->transport; | |
238 | struct adf_etr_bank_data *bank; | |
239 | struct adf_etr_ring_data *ring; | |
240 | char val[ADF_CFG_MAX_VAL_LEN_IN_BYTES]; | |
241 | uint32_t ring_num; | |
242 | int ret; | |
243 | ||
244 | if (bank_num >= GET_MAX_BANKS(accel_dev)) { | |
66550304 | 245 | dev_err(&GET_DEV(accel_dev), "Invalid bank number\n"); |
a672a9dc TS |
246 | return -EFAULT; |
247 | } | |
248 | if (msg_size > ADF_MSG_SIZE_TO_BYTES(ADF_MAX_MSG_SIZE)) { | |
66550304 | 249 | dev_err(&GET_DEV(accel_dev), "Invalid msg size\n"); |
a672a9dc TS |
250 | return -EFAULT; |
251 | } | |
252 | if (ADF_MAX_INFLIGHTS(adf_verify_ring_size(msg_size, num_msgs), | |
253 | ADF_BYTES_TO_MSG_SIZE(msg_size)) < 2) { | |
66550304 AB |
254 | dev_err(&GET_DEV(accel_dev), |
255 | "Invalid ring size for given msg size\n"); | |
a672a9dc TS |
256 | return -EFAULT; |
257 | } | |
258 | if (adf_cfg_get_param_value(accel_dev, section, ring_name, val)) { | |
66550304 AB |
259 | dev_err(&GET_DEV(accel_dev), "Section %s, no such entry : %s\n", |
260 | section, ring_name); | |
a672a9dc TS |
261 | return -EFAULT; |
262 | } | |
263 | if (kstrtouint(val, 10, &ring_num)) { | |
66550304 | 264 | dev_err(&GET_DEV(accel_dev), "Can't get ring number\n"); |
a672a9dc TS |
265 | return -EFAULT; |
266 | } | |
55e8dba1 TS |
267 | if (ring_num >= ADF_ETR_MAX_RINGS_PER_BANK) { |
268 | dev_err(&GET_DEV(accel_dev), "Invalid ring number\n"); | |
269 | return -EFAULT; | |
270 | } | |
a672a9dc TS |
271 | |
272 | bank = &transport_data->banks[bank_num]; | |
273 | if (adf_reserve_ring(bank, ring_num)) { | |
66550304 AB |
274 | dev_err(&GET_DEV(accel_dev), "Ring %d, %s already exists.\n", |
275 | ring_num, ring_name); | |
a672a9dc TS |
276 | return -EFAULT; |
277 | } | |
278 | ring = &bank->rings[ring_num]; | |
279 | ring->ring_number = ring_num; | |
280 | ring->bank = bank; | |
281 | ring->callback = callback; | |
282 | ring->msg_size = ADF_BYTES_TO_MSG_SIZE(msg_size); | |
283 | ring->ring_size = adf_verify_ring_size(msg_size, num_msgs); | |
284 | ring->head = 0; | |
285 | ring->tail = 0; | |
286 | atomic_set(ring->inflights, 0); | |
287 | ret = adf_init_ring(ring); | |
288 | if (ret) | |
289 | goto err; | |
290 | ||
291 | /* Enable HW arbitration for the given ring */ | |
a5733139 | 292 | adf_update_ring_arb(ring); |
a672a9dc TS |
293 | |
294 | if (adf_ring_debugfs_add(ring, ring_name)) { | |
66550304 AB |
295 | dev_err(&GET_DEV(accel_dev), |
296 | "Couldn't add ring debugfs entry\n"); | |
a672a9dc TS |
297 | ret = -EFAULT; |
298 | goto err; | |
299 | } | |
300 | ||
301 | /* Enable interrupts if needed */ | |
302 | if (callback && (!poll_mode)) | |
303 | adf_enable_ring_irq(bank, ring->ring_number); | |
304 | *ring_ptr = ring; | |
305 | return 0; | |
306 | err: | |
307 | adf_cleanup_ring(ring); | |
308 | adf_unreserve_ring(bank, ring_num); | |
a5733139 | 309 | adf_update_ring_arb(ring); |
a672a9dc TS |
310 | return ret; |
311 | } | |
312 | ||
313 | void adf_remove_ring(struct adf_etr_ring_data *ring) | |
314 | { | |
315 | struct adf_etr_bank_data *bank = ring->bank; | |
a672a9dc TS |
316 | |
317 | /* Disable interrupts for the given ring */ | |
318 | adf_disable_ring_irq(bank, ring->ring_number); | |
319 | ||
320 | /* Clear PCI config space */ | |
321 | WRITE_CSR_RING_CONFIG(bank->csr_addr, bank->bank_number, | |
322 | ring->ring_number, 0); | |
323 | WRITE_CSR_RING_BASE(bank->csr_addr, bank->bank_number, | |
324 | ring->ring_number, 0); | |
325 | adf_ring_debugfs_rm(ring); | |
326 | adf_unreserve_ring(bank, ring->ring_number); | |
327 | /* Disable HW arbitration for the given ring */ | |
a5733139 | 328 | adf_update_ring_arb(ring); |
a672a9dc TS |
329 | adf_cleanup_ring(ring); |
330 | } | |
331 | ||
332 | static void adf_ring_response_handler(struct adf_etr_bank_data *bank) | |
333 | { | |
334 | uint32_t empty_rings, i; | |
335 | ||
336 | empty_rings = READ_CSR_E_STAT(bank->csr_addr, bank->bank_number); | |
337 | empty_rings = ~empty_rings & bank->irq_mask; | |
338 | ||
339 | for (i = 0; i < ADF_ETR_MAX_RINGS_PER_BANK; ++i) { | |
340 | if (empty_rings & (1 << i)) | |
341 | adf_handle_response(&bank->rings[i]); | |
342 | } | |
343 | } | |
344 | ||
1a72d3a6 | 345 | void adf_response_handler(uintptr_t bank_addr) |
a672a9dc TS |
346 | { |
347 | struct adf_etr_bank_data *bank = (void *)bank_addr; | |
348 | ||
349 | /* Handle all the responses nad reenable IRQs */ | |
350 | adf_ring_response_handler(bank); | |
351 | WRITE_CSR_INT_FLAG_AND_COL(bank->csr_addr, bank->bank_number, | |
352 | bank->irq_mask); | |
353 | } | |
a672a9dc TS |
354 | |
355 | static inline int adf_get_cfg_int(struct adf_accel_dev *accel_dev, | |
356 | const char *section, const char *format, | |
357 | uint32_t key, uint32_t *value) | |
358 | { | |
359 | char key_buf[ADF_CFG_MAX_KEY_LEN_IN_BYTES]; | |
360 | char val_buf[ADF_CFG_MAX_VAL_LEN_IN_BYTES]; | |
361 | ||
362 | snprintf(key_buf, ADF_CFG_MAX_KEY_LEN_IN_BYTES, format, key); | |
363 | ||
364 | if (adf_cfg_get_param_value(accel_dev, section, key_buf, val_buf)) | |
365 | return -EFAULT; | |
366 | ||
367 | if (kstrtouint(val_buf, 10, value)) | |
368 | return -EFAULT; | |
369 | return 0; | |
370 | } | |
371 | ||
bc84b94a TS |
372 | static void adf_get_coalesc_timer(struct adf_etr_bank_data *bank, |
373 | const char *section, | |
374 | uint32_t bank_num_in_accel) | |
a672a9dc TS |
375 | { |
376 | if (adf_get_cfg_int(bank->accel_dev, section, | |
377 | ADF_ETRMGR_COALESCE_TIMER_FORMAT, | |
378 | bank_num_in_accel, &bank->irq_coalesc_timer)) | |
379 | bank->irq_coalesc_timer = ADF_COALESCING_DEF_TIME; | |
380 | ||
381 | if (ADF_COALESCING_MAX_TIME < bank->irq_coalesc_timer || | |
382 | ADF_COALESCING_MIN_TIME > bank->irq_coalesc_timer) | |
383 | bank->irq_coalesc_timer = ADF_COALESCING_DEF_TIME; | |
384 | } | |
385 | ||
386 | static int adf_init_bank(struct adf_accel_dev *accel_dev, | |
387 | struct adf_etr_bank_data *bank, | |
388 | uint32_t bank_num, void __iomem *csr_addr) | |
389 | { | |
390 | struct adf_hw_device_data *hw_data = accel_dev->hw_device; | |
391 | struct adf_etr_ring_data *ring; | |
392 | struct adf_etr_ring_data *tx_ring; | |
bc84b94a | 393 | uint32_t i, coalesc_enabled = 0; |
a672a9dc TS |
394 | |
395 | memset(bank, 0, sizeof(*bank)); | |
396 | bank->bank_number = bank_num; | |
397 | bank->csr_addr = csr_addr; | |
398 | bank->accel_dev = accel_dev; | |
399 | spin_lock_init(&bank->lock); | |
400 | ||
401 | /* Enable IRQ coalescing always. This will allow to use | |
402 | * the optimised flag and coalesc register. | |
403 | * If it is disabled in the config file just use min time value */ | |
bc84b94a TS |
404 | if ((adf_get_cfg_int(accel_dev, "Accelerator0", |
405 | ADF_ETRMGR_COALESCING_ENABLED_FORMAT, bank_num, | |
406 | &coalesc_enabled) == 0) && coalesc_enabled) | |
407 | adf_get_coalesc_timer(bank, "Accelerator0", bank_num); | |
a672a9dc TS |
408 | else |
409 | bank->irq_coalesc_timer = ADF_COALESCING_MIN_TIME; | |
410 | ||
411 | for (i = 0; i < ADF_ETR_MAX_RINGS_PER_BANK; i++) { | |
412 | WRITE_CSR_RING_CONFIG(csr_addr, bank_num, i, 0); | |
413 | WRITE_CSR_RING_BASE(csr_addr, bank_num, i, 0); | |
414 | ring = &bank->rings[i]; | |
415 | if (hw_data->tx_rings_mask & (1 << i)) { | |
09adc878 TS |
416 | ring->inflights = |
417 | kzalloc_node(sizeof(atomic_t), | |
418 | GFP_KERNEL, | |
419 | dev_to_node(&GET_DEV(accel_dev))); | |
a672a9dc TS |
420 | if (!ring->inflights) |
421 | goto err; | |
422 | } else { | |
423 | if (i < hw_data->tx_rx_gap) { | |
66550304 AB |
424 | dev_err(&GET_DEV(accel_dev), |
425 | "Invalid tx rings mask config\n"); | |
a672a9dc TS |
426 | goto err; |
427 | } | |
428 | tx_ring = &bank->rings[i - hw_data->tx_rx_gap]; | |
429 | ring->inflights = tx_ring->inflights; | |
430 | } | |
431 | } | |
432 | if (adf_bank_debugfs_add(bank)) { | |
66550304 AB |
433 | dev_err(&GET_DEV(accel_dev), |
434 | "Failed to add bank debugfs entry\n"); | |
a672a9dc TS |
435 | goto err; |
436 | } | |
437 | ||
438 | WRITE_CSR_INT_SRCSEL(csr_addr, bank_num); | |
439 | return 0; | |
440 | err: | |
441 | for (i = 0; i < ADF_ETR_MAX_RINGS_PER_BANK; i++) { | |
442 | ring = &bank->rings[i]; | |
12a4bd31 | 443 | if (hw_data->tx_rings_mask & (1 << i)) |
a672a9dc TS |
444 | kfree(ring->inflights); |
445 | } | |
446 | return -ENOMEM; | |
447 | } | |
448 | ||
449 | /** | |
450 | * adf_init_etr_data() - Initialize transport rings for acceleration device | |
451 | * @accel_dev: Pointer to acceleration device. | |
452 | * | |
453 | * Function is the initializes the communications channels (rings) to the | |
454 | * acceleration device accel_dev. | |
455 | * To be used by QAT device specific drivers. | |
456 | * | |
ec0d6fa3 | 457 | * Return: 0 on success, error code otherwise. |
a672a9dc TS |
458 | */ |
459 | int adf_init_etr_data(struct adf_accel_dev *accel_dev) | |
460 | { | |
461 | struct adf_etr_data *etr_data; | |
462 | struct adf_hw_device_data *hw_data = accel_dev->hw_device; | |
463 | void __iomem *csr_addr; | |
464 | uint32_t size; | |
465 | uint32_t num_banks = 0; | |
466 | int i, ret; | |
467 | ||
468 | etr_data = kzalloc_node(sizeof(*etr_data), GFP_KERNEL, | |
09adc878 | 469 | dev_to_node(&GET_DEV(accel_dev))); |
a672a9dc TS |
470 | if (!etr_data) |
471 | return -ENOMEM; | |
472 | ||
473 | num_banks = GET_MAX_BANKS(accel_dev); | |
474 | size = num_banks * sizeof(struct adf_etr_bank_data); | |
09adc878 TS |
475 | etr_data->banks = kzalloc_node(size, GFP_KERNEL, |
476 | dev_to_node(&GET_DEV(accel_dev))); | |
a672a9dc TS |
477 | if (!etr_data->banks) { |
478 | ret = -ENOMEM; | |
479 | goto err_bank; | |
480 | } | |
481 | ||
482 | accel_dev->transport = etr_data; | |
483 | i = hw_data->get_etr_bar_id(hw_data); | |
484 | csr_addr = accel_dev->accel_pci_dev.pci_bars[i].virt_addr; | |
485 | ||
486 | /* accel_dev->debugfs_dir should always be non-NULL here */ | |
487 | etr_data->debug = debugfs_create_dir("transport", | |
488 | accel_dev->debugfs_dir); | |
489 | if (!etr_data->debug) { | |
66550304 AB |
490 | dev_err(&GET_DEV(accel_dev), |
491 | "Unable to create transport debugfs entry\n"); | |
a672a9dc TS |
492 | ret = -ENOENT; |
493 | goto err_bank_debug; | |
494 | } | |
495 | ||
496 | for (i = 0; i < num_banks; i++) { | |
497 | ret = adf_init_bank(accel_dev, &etr_data->banks[i], i, | |
498 | csr_addr); | |
499 | if (ret) | |
500 | goto err_bank_all; | |
501 | } | |
502 | ||
503 | return 0; | |
504 | ||
505 | err_bank_all: | |
506 | debugfs_remove(etr_data->debug); | |
507 | err_bank_debug: | |
508 | kfree(etr_data->banks); | |
509 | err_bank: | |
510 | kfree(etr_data); | |
511 | accel_dev->transport = NULL; | |
512 | return ret; | |
513 | } | |
514 | EXPORT_SYMBOL_GPL(adf_init_etr_data); | |
515 | ||
516 | static void cleanup_bank(struct adf_etr_bank_data *bank) | |
517 | { | |
518 | uint32_t i; | |
519 | ||
520 | for (i = 0; i < ADF_ETR_MAX_RINGS_PER_BANK; i++) { | |
521 | struct adf_accel_dev *accel_dev = bank->accel_dev; | |
522 | struct adf_hw_device_data *hw_data = accel_dev->hw_device; | |
523 | struct adf_etr_ring_data *ring = &bank->rings[i]; | |
524 | ||
525 | if (bank->ring_mask & (1 << i)) | |
526 | adf_cleanup_ring(ring); | |
527 | ||
528 | if (hw_data->tx_rings_mask & (1 << i)) | |
529 | kfree(ring->inflights); | |
530 | } | |
531 | adf_bank_debugfs_rm(bank); | |
532 | memset(bank, 0, sizeof(*bank)); | |
533 | } | |
534 | ||
535 | static void adf_cleanup_etr_handles(struct adf_accel_dev *accel_dev) | |
536 | { | |
537 | struct adf_etr_data *etr_data = accel_dev->transport; | |
538 | uint32_t i, num_banks = GET_MAX_BANKS(accel_dev); | |
539 | ||
540 | for (i = 0; i < num_banks; i++) | |
541 | cleanup_bank(&etr_data->banks[i]); | |
542 | } | |
543 | ||
544 | /** | |
545 | * adf_cleanup_etr_data() - Clear transport rings for acceleration device | |
546 | * @accel_dev: Pointer to acceleration device. | |
547 | * | |
548 | * Function is the clears the communications channels (rings) of the | |
549 | * acceleration device accel_dev. | |
550 | * To be used by QAT device specific drivers. | |
551 | * | |
552 | * Return: void | |
553 | */ | |
554 | void adf_cleanup_etr_data(struct adf_accel_dev *accel_dev) | |
555 | { | |
556 | struct adf_etr_data *etr_data = accel_dev->transport; | |
557 | ||
558 | if (etr_data) { | |
559 | adf_cleanup_etr_handles(accel_dev); | |
560 | debugfs_remove(etr_data->debug); | |
561 | kfree(etr_data->banks); | |
562 | kfree(etr_data); | |
563 | accel_dev->transport = NULL; | |
564 | } | |
565 | } | |
566 | EXPORT_SYMBOL_GPL(adf_cleanup_etr_data); |