]> git.proxmox.com Git - mirror_ubuntu-zesty-kernel.git/blob - drivers/net/ethernet/amazon/ena/ena_com.c
projects / mirror_ubuntu-zesty-kernel.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
Merge branch 'for-4.9' of git://git.kernel.org/pub/scm/linux/kernel/git/tj/percpu
[mirror_ubuntu-zesty-kernel.git] / drivers / net / ethernet / amazon / ena / ena_com.c
1 /*
2 * Copyright 2015 Amazon.com, Inc. or its affiliates.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 */
32
33 #include "ena_com.h"
34
35 /*****************************************************************************/
36 /*****************************************************************************/
37
38 /* Timeout in micro-sec */
39 #define ADMIN_CMD_TIMEOUT_US (1000000)
40
41 #define ENA_ASYNC_QUEUE_DEPTH 4
42 #define ENA_ADMIN_QUEUE_DEPTH 32
43
44 #define MIN_ENA_VER (((ENA_COMMON_SPEC_VERSION_MAJOR) << \
45 ENA_REGS_VERSION_MAJOR_VERSION_SHIFT) \
46 | (ENA_COMMON_SPEC_VERSION_MINOR))
47
48 #define ENA_CTRL_MAJOR 0
49 #define ENA_CTRL_MINOR 0
50 #define ENA_CTRL_SUB_MINOR 1
51
52 #define MIN_ENA_CTRL_VER \
53 (((ENA_CTRL_MAJOR) << \
54 (ENA_REGS_CONTROLLER_VERSION_MAJOR_VERSION_SHIFT)) | \
55 ((ENA_CTRL_MINOR) << \
56 (ENA_REGS_CONTROLLER_VERSION_MINOR_VERSION_SHIFT)) | \
57 (ENA_CTRL_SUB_MINOR))
58
59 #define ENA_DMA_ADDR_TO_UINT32_LOW(x) ((u32)((u64)(x)))
60 #define ENA_DMA_ADDR_TO_UINT32_HIGH(x) ((u32)(((u64)(x)) >> 32))
61
62 #define ENA_MMIO_READ_TIMEOUT 0xFFFFFFFF
63
64 /*****************************************************************************/
65 /*****************************************************************************/
66 /*****************************************************************************/
67
68 enum ena_cmd_status {
69 ENA_CMD_SUBMITTED,
70 ENA_CMD_COMPLETED,
71 /* Abort - canceled by the driver */
72 ENA_CMD_ABORTED,
73 };
74
75 struct ena_comp_ctx {
76 struct completion wait_event;
77 struct ena_admin_acq_entry *user_cqe;
78 u32 comp_size;
79 enum ena_cmd_status status;
80 /* status from the device */
81 u8 comp_status;
82 u8 cmd_opcode;
83 bool occupied;
84 };
85
86 struct ena_com_stats_ctx {
87 struct ena_admin_aq_get_stats_cmd get_cmd;
88 struct ena_admin_acq_get_stats_resp get_resp;
89 };
90
91 static inline int ena_com_mem_addr_set(struct ena_com_dev *ena_dev,
92 struct ena_common_mem_addr *ena_addr,
93 dma_addr_t addr)
94 {
95 if ((addr & GENMASK_ULL(ena_dev->dma_addr_bits - 1, 0)) != addr) {
96 pr_err("dma address has more bits that the device supports\n");
97 return -EINVAL;
98 }
99
100 ena_addr->mem_addr_low = (u32)addr;
101 ena_addr->mem_addr_high = (u64)addr >> 32;
102
103 return 0;
104 }
105
106 static int ena_com_admin_init_sq(struct ena_com_admin_queue *queue)
107 {
108 struct ena_com_admin_sq *sq = &queue->sq;
109 u16 size = ADMIN_SQ_SIZE(queue->q_depth);
110
111 sq->entries = dma_zalloc_coherent(queue->q_dmadev, size, &sq->dma_addr,
112 GFP_KERNEL);
113
114 if (!sq->entries) {
115 pr_err("memory allocation failed");
116 return -ENOMEM;
117 }
118
119 sq->head = 0;
120 sq->tail = 0;
121 sq->phase = 1;
122
123 sq->db_addr = NULL;
124
125 return 0;
126 }
127
128 static int ena_com_admin_init_cq(struct ena_com_admin_queue *queue)
129 {
130 struct ena_com_admin_cq *cq = &queue->cq;
131 u16 size = ADMIN_CQ_SIZE(queue->q_depth);
132
133 cq->entries = dma_zalloc_coherent(queue->q_dmadev, size, &cq->dma_addr,
134 GFP_KERNEL);
135
136 if (!cq->entries) {
137 pr_err("memory allocation failed");
138 return -ENOMEM;
139 }
140
141 cq->head = 0;
142 cq->phase = 1;
143
144 return 0;
145 }
146
147 static int ena_com_admin_init_aenq(struct ena_com_dev *dev,
148 struct ena_aenq_handlers *aenq_handlers)
149 {
150 struct ena_com_aenq *aenq = &dev->aenq;
151 u32 addr_low, addr_high, aenq_caps;
152 u16 size;
153
154 dev->aenq.q_depth = ENA_ASYNC_QUEUE_DEPTH;
155 size = ADMIN_AENQ_SIZE(ENA_ASYNC_QUEUE_DEPTH);
156 aenq->entries = dma_zalloc_coherent(dev->dmadev, size, &aenq->dma_addr,
157 GFP_KERNEL);
158
159 if (!aenq->entries) {
160 pr_err("memory allocation failed");
161 return -ENOMEM;
162 }
163
164 aenq->head = aenq->q_depth;
165 aenq->phase = 1;
166
167 addr_low = ENA_DMA_ADDR_TO_UINT32_LOW(aenq->dma_addr);
168 addr_high = ENA_DMA_ADDR_TO_UINT32_HIGH(aenq->dma_addr);
169
170 writel(addr_low, dev->reg_bar + ENA_REGS_AENQ_BASE_LO_OFF);
171 writel(addr_high, dev->reg_bar + ENA_REGS_AENQ_BASE_HI_OFF);
172
173 aenq_caps = 0;
174 aenq_caps |= dev->aenq.q_depth & ENA_REGS_AENQ_CAPS_AENQ_DEPTH_MASK;
175 aenq_caps |= (sizeof(struct ena_admin_aenq_entry)
176 << ENA_REGS_AENQ_CAPS_AENQ_ENTRY_SIZE_SHIFT) &
177 ENA_REGS_AENQ_CAPS_AENQ_ENTRY_SIZE_MASK;
178 writel(aenq_caps, dev->reg_bar + ENA_REGS_AENQ_CAPS_OFF);
179
180 if (unlikely(!aenq_handlers)) {
181 pr_err("aenq handlers pointer is NULL\n");
182 return -EINVAL;
183 }
184
185 aenq->aenq_handlers = aenq_handlers;
186
187 return 0;
188 }
189
190 static inline void comp_ctxt_release(struct ena_com_admin_queue *queue,
191 struct ena_comp_ctx *comp_ctx)
192 {
193 comp_ctx->occupied = false;
194 atomic_dec(&queue->outstanding_cmds);
195 }
196
197 static struct ena_comp_ctx *get_comp_ctxt(struct ena_com_admin_queue *queue,
198 u16 command_id, bool capture)
199 {
200 if (unlikely(command_id >= queue->q_depth)) {
201 pr_err("command id is larger than the queue size. cmd_id: %u queue size %d\n",
202 command_id, queue->q_depth);
203 return NULL;
204 }
205
206 if (unlikely(queue->comp_ctx[command_id].occupied && capture)) {
207 pr_err("Completion context is occupied\n");
208 return NULL;
209 }
210
211 if (capture) {
212 atomic_inc(&queue->outstanding_cmds);
213 queue->comp_ctx[command_id].occupied = true;
214 }
215
216 return &queue->comp_ctx[command_id];
217 }
218
219 static struct ena_comp_ctx *__ena_com_submit_admin_cmd(struct ena_com_admin_queue *admin_queue,
220 struct ena_admin_aq_entry *cmd,
221 size_t cmd_size_in_bytes,
222 struct ena_admin_acq_entry *comp,
223 size_t comp_size_in_bytes)
224 {
225 struct ena_comp_ctx *comp_ctx;
226 u16 tail_masked, cmd_id;
227 u16 queue_size_mask;
228 u16 cnt;
229
230 queue_size_mask = admin_queue->q_depth - 1;
231
232 tail_masked = admin_queue->sq.tail & queue_size_mask;
233
234 /* In case of queue FULL */
235 cnt = admin_queue->sq.tail - admin_queue->sq.head;
236 if (cnt >= admin_queue->q_depth) {
237 pr_debug("admin queue is FULL (tail %d head %d depth: %d)\n",
238 admin_queue->sq.tail, admin_queue->sq.head,
239 admin_queue->q_depth);
240 admin_queue->stats.out_of_space++;
241 return ERR_PTR(-ENOSPC);
242 }
243
244 cmd_id = admin_queue->curr_cmd_id;
245
246 cmd->aq_common_descriptor.flags |= admin_queue->sq.phase &
247 ENA_ADMIN_AQ_COMMON_DESC_PHASE_MASK;
248
249 cmd->aq_common_descriptor.command_id |= cmd_id &
250 ENA_ADMIN_AQ_COMMON_DESC_COMMAND_ID_MASK;
251
252 comp_ctx = get_comp_ctxt(admin_queue, cmd_id, true);
253 if (unlikely(!comp_ctx))
254 return ERR_PTR(-EINVAL);
255
256 comp_ctx->status = ENA_CMD_SUBMITTED;
257 comp_ctx->comp_size = (u32)comp_size_in_bytes;
258 comp_ctx->user_cqe = comp;
259 comp_ctx->cmd_opcode = cmd->aq_common_descriptor.opcode;
260
261 reinit_completion(&comp_ctx->wait_event);
262
263 memcpy(&admin_queue->sq.entries[tail_masked], cmd, cmd_size_in_bytes);
264
265 admin_queue->curr_cmd_id = (admin_queue->curr_cmd_id + 1) &
266 queue_size_mask;
267
268 admin_queue->sq.tail++;
269 admin_queue->stats.submitted_cmd++;
270
271 if (unlikely((admin_queue->sq.tail & queue_size_mask) == 0))
272 admin_queue->sq.phase = !admin_queue->sq.phase;
273
274 writel(admin_queue->sq.tail, admin_queue->sq.db_addr);
275
276 return comp_ctx;
277 }
278
279 static inline int ena_com_init_comp_ctxt(struct ena_com_admin_queue *queue)
280 {
281 size_t size = queue->q_depth * sizeof(struct ena_comp_ctx);
282 struct ena_comp_ctx *comp_ctx;
283 u16 i;
284
285 queue->comp_ctx = devm_kzalloc(queue->q_dmadev, size, GFP_KERNEL);
286 if (unlikely(!queue->comp_ctx)) {
287 pr_err("memory allocation failed");
288 return -ENOMEM;
289 }
290
291 for (i = 0; i < queue->q_depth; i++) {
292 comp_ctx = get_comp_ctxt(queue, i, false);
293 if (comp_ctx)
294 init_completion(&comp_ctx->wait_event);
295 }
296
297 return 0;
298 }
299
300 static struct ena_comp_ctx *ena_com_submit_admin_cmd(struct ena_com_admin_queue *admin_queue,
301 struct ena_admin_aq_entry *cmd,
302 size_t cmd_size_in_bytes,
303 struct ena_admin_acq_entry *comp,
304 size_t comp_size_in_bytes)
305 {
306 unsigned long flags;
307 struct ena_comp_ctx *comp_ctx;
308
309 spin_lock_irqsave(&admin_queue->q_lock, flags);
310 if (unlikely(!admin_queue->running_state)) {
311 spin_unlock_irqrestore(&admin_queue->q_lock, flags);
312 return ERR_PTR(-ENODEV);
313 }
314 comp_ctx = __ena_com_submit_admin_cmd(admin_queue, cmd,
315 cmd_size_in_bytes,
316 comp,
317 comp_size_in_bytes);
318 if (unlikely(IS_ERR(comp_ctx)))
319 admin_queue->running_state = false;
320 spin_unlock_irqrestore(&admin_queue->q_lock, flags);
321
322 return comp_ctx;
323 }
324
325 static int ena_com_init_io_sq(struct ena_com_dev *ena_dev,
326 struct ena_com_create_io_ctx *ctx,
327 struct ena_com_io_sq *io_sq)
328 {
329 size_t size;
330 int dev_node = 0;
331
332 memset(&io_sq->desc_addr, 0x0, sizeof(struct ena_com_io_desc_addr));
333
334 io_sq->desc_entry_size =
335 (io_sq->direction == ENA_COM_IO_QUEUE_DIRECTION_TX) ?
336 sizeof(struct ena_eth_io_tx_desc) :
337 sizeof(struct ena_eth_io_rx_desc);
338
339 size = io_sq->desc_entry_size * io_sq->q_depth;
340
341 if (io_sq->mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_HOST) {
342 dev_node = dev_to_node(ena_dev->dmadev);
343 set_dev_node(ena_dev->dmadev, ctx->numa_node);
344 io_sq->desc_addr.virt_addr =
345 dma_zalloc_coherent(ena_dev->dmadev, size,
346 &io_sq->desc_addr.phys_addr,
347 GFP_KERNEL);
348 set_dev_node(ena_dev->dmadev, dev_node);
349 if (!io_sq->desc_addr.virt_addr) {
350 io_sq->desc_addr.virt_addr =
351 dma_zalloc_coherent(ena_dev->dmadev, size,
352 &io_sq->desc_addr.phys_addr,
353 GFP_KERNEL);
354 }
355 } else {
356 dev_node = dev_to_node(ena_dev->dmadev);
357 set_dev_node(ena_dev->dmadev, ctx->numa_node);
358 io_sq->desc_addr.virt_addr =
359 devm_kzalloc(ena_dev->dmadev, size, GFP_KERNEL);
360 set_dev_node(ena_dev->dmadev, dev_node);
361 if (!io_sq->desc_addr.virt_addr) {
362 io_sq->desc_addr.virt_addr =
363 devm_kzalloc(ena_dev->dmadev, size, GFP_KERNEL);
364 }
365 }
366
367 if (!io_sq->desc_addr.virt_addr) {
368 pr_err("memory allocation failed");
369 return -ENOMEM;
370 }
371
372 io_sq->tail = 0;
373 io_sq->next_to_comp = 0;
374 io_sq->phase = 1;
375
376 return 0;
377 }
378
379 static int ena_com_init_io_cq(struct ena_com_dev *ena_dev,
380 struct ena_com_create_io_ctx *ctx,
381 struct ena_com_io_cq *io_cq)
382 {
383 size_t size;
384 int prev_node = 0;
385
386 memset(&io_cq->cdesc_addr, 0x0, sizeof(struct ena_com_io_desc_addr));
387
388 /* Use the basic completion descriptor for Rx */
389 io_cq->cdesc_entry_size_in_bytes =
390 (io_cq->direction == ENA_COM_IO_QUEUE_DIRECTION_TX) ?
391 sizeof(struct ena_eth_io_tx_cdesc) :
392 sizeof(struct ena_eth_io_rx_cdesc_base);
393
394 size = io_cq->cdesc_entry_size_in_bytes * io_cq->q_depth;
395
396 prev_node = dev_to_node(ena_dev->dmadev);
397 set_dev_node(ena_dev->dmadev, ctx->numa_node);
398 io_cq->cdesc_addr.virt_addr =
399 dma_zalloc_coherent(ena_dev->dmadev, size,
400 &io_cq->cdesc_addr.phys_addr, GFP_KERNEL);
401 set_dev_node(ena_dev->dmadev, prev_node);
402 if (!io_cq->cdesc_addr.virt_addr) {
403 io_cq->cdesc_addr.virt_addr =
404 dma_zalloc_coherent(ena_dev->dmadev, size,
405 &io_cq->cdesc_addr.phys_addr,
406 GFP_KERNEL);
407 }
408
409 if (!io_cq->cdesc_addr.virt_addr) {
410 pr_err("memory allocation failed");
411 return -ENOMEM;
412 }
413
414 io_cq->phase = 1;
415 io_cq->head = 0;
416
417 return 0;
418 }
419
420 static void ena_com_handle_single_admin_completion(struct ena_com_admin_queue *admin_queue,
421 struct ena_admin_acq_entry *cqe)
422 {
423 struct ena_comp_ctx *comp_ctx;
424 u16 cmd_id;
425
426 cmd_id = cqe->acq_common_descriptor.command &
427 ENA_ADMIN_ACQ_COMMON_DESC_COMMAND_ID_MASK;
428
429 comp_ctx = get_comp_ctxt(admin_queue, cmd_id, false);
430 if (unlikely(!comp_ctx)) {
431 pr_err("comp_ctx is NULL. Changing the admin queue running state\n");
432 admin_queue->running_state = false;
433 return;
434 }
435
436 comp_ctx->status = ENA_CMD_COMPLETED;
437 comp_ctx->comp_status = cqe->acq_common_descriptor.status;
438
439 if (comp_ctx->user_cqe)
440 memcpy(comp_ctx->user_cqe, (void *)cqe, comp_ctx->comp_size);
441
442 if (!admin_queue->polling)
443 complete(&comp_ctx->wait_event);
444 }
445
446 static void ena_com_handle_admin_completion(struct ena_com_admin_queue *admin_queue)
447 {
448 struct ena_admin_acq_entry *cqe = NULL;
449 u16 comp_num = 0;
450 u16 head_masked;
451 u8 phase;
452
453 head_masked = admin_queue->cq.head & (admin_queue->q_depth - 1);
454 phase = admin_queue->cq.phase;
455
456 cqe = &admin_queue->cq.entries[head_masked];
457
458 /* Go over all the completions */
459 while ((cqe->acq_common_descriptor.flags &
460 ENA_ADMIN_ACQ_COMMON_DESC_PHASE_MASK) == phase) {
461 /* Do not read the rest of the completion entry before the
462 * phase bit was validated
463 */
464 rmb();
465 ena_com_handle_single_admin_completion(admin_queue, cqe);
466
467 head_masked++;
468 comp_num++;
469 if (unlikely(head_masked == admin_queue->q_depth)) {
470 head_masked = 0;
471 phase = !phase;
472 }
473
474 cqe = &admin_queue->cq.entries[head_masked];
475 }
476
477 admin_queue->cq.head += comp_num;
478 admin_queue->cq.phase = phase;
479 admin_queue->sq.head += comp_num;
480 admin_queue->stats.completed_cmd += comp_num;
481 }
482
483 static int ena_com_comp_status_to_errno(u8 comp_status)
484 {
485 if (unlikely(comp_status != 0))
486 pr_err("admin command failed[%u]\n", comp_status);
487
488 if (unlikely(comp_status > ENA_ADMIN_UNKNOWN_ERROR))
489 return -EINVAL;
490
491 switch (comp_status) {
492 case ENA_ADMIN_SUCCESS:
493 return 0;
494 case ENA_ADMIN_RESOURCE_ALLOCATION_FAILURE:
495 return -ENOMEM;
496 case ENA_ADMIN_UNSUPPORTED_OPCODE:
497 return -EPERM;
498 case ENA_ADMIN_BAD_OPCODE:
499 case ENA_ADMIN_MALFORMED_REQUEST:
500 case ENA_ADMIN_ILLEGAL_PARAMETER:
501 case ENA_ADMIN_UNKNOWN_ERROR:
502 return -EINVAL;
503 }
504
505 return 0;
506 }
507
508 static int ena_com_wait_and_process_admin_cq_polling(struct ena_comp_ctx *comp_ctx,
509 struct ena_com_admin_queue *admin_queue)
510 {
511 unsigned long flags;
512 u32 start_time;
513 int ret;
514
515 start_time = ((u32)jiffies_to_usecs(jiffies));
516
517 while (comp_ctx->status == ENA_CMD_SUBMITTED) {
518 if ((((u32)jiffies_to_usecs(jiffies)) - start_time) >
519 ADMIN_CMD_TIMEOUT_US) {
520 pr_err("Wait for completion (polling) timeout\n");
521 /* ENA didn't have any completion */
522 spin_lock_irqsave(&admin_queue->q_lock, flags);
523 admin_queue->stats.no_completion++;
524 admin_queue->running_state = false;
525 spin_unlock_irqrestore(&admin_queue->q_lock, flags);
526
527 ret = -ETIME;
528 goto err;
529 }
530
531 spin_lock_irqsave(&admin_queue->q_lock, flags);
532 ena_com_handle_admin_completion(admin_queue);
533 spin_unlock_irqrestore(&admin_queue->q_lock, flags);
534
535 msleep(100);
536 }
537
538 if (unlikely(comp_ctx->status == ENA_CMD_ABORTED)) {
539 pr_err("Command was aborted\n");
540 spin_lock_irqsave(&admin_queue->q_lock, flags);
541 admin_queue->stats.aborted_cmd++;
542 spin_unlock_irqrestore(&admin_queue->q_lock, flags);
543 ret = -ENODEV;
544 goto err;
545 }
546
547 WARN(comp_ctx->status != ENA_CMD_COMPLETED, "Invalid comp status %d\n",
548 comp_ctx->status);
549
550 ret = ena_com_comp_status_to_errno(comp_ctx->comp_status);
551 err:
552 comp_ctxt_release(admin_queue, comp_ctx);
553 return ret;
554 }
555
556 static int ena_com_wait_and_process_admin_cq_interrupts(struct ena_comp_ctx *comp_ctx,
557 struct ena_com_admin_queue *admin_queue)
558 {
559 unsigned long flags;
560 int ret;
561
562 wait_for_completion_timeout(&comp_ctx->wait_event,
563 usecs_to_jiffies(ADMIN_CMD_TIMEOUT_US));
564
565 /* In case the command wasn't completed find out the root cause.
566 * There might be 2 kinds of errors
567 * 1) No completion (timeout reached)
568 * 2) There is completion but the device didn't get any msi-x interrupt.
569 */
570 if (unlikely(comp_ctx->status == ENA_CMD_SUBMITTED)) {
571 spin_lock_irqsave(&admin_queue->q_lock, flags);
572 ena_com_handle_admin_completion(admin_queue);
573 admin_queue->stats.no_completion++;
574 spin_unlock_irqrestore(&admin_queue->q_lock, flags);
575
576 if (comp_ctx->status == ENA_CMD_COMPLETED)
577 pr_err("The ena device have completion but the driver didn't receive any MSI-X interrupt (cmd %d)\n",
578 comp_ctx->cmd_opcode);
579 else
580 pr_err("The ena device doesn't send any completion for the admin cmd %d status %d\n",
581 comp_ctx->cmd_opcode, comp_ctx->status);
582
583 admin_queue->running_state = false;
584 ret = -ETIME;
585 goto err;
586 }
587
588 ret = ena_com_comp_status_to_errno(comp_ctx->comp_status);
589 err:
590 comp_ctxt_release(admin_queue, comp_ctx);
591 return ret;
592 }
593
594 /* This method read the hardware device register through posting writes
595 * and waiting for response
596 * On timeout the function will return ENA_MMIO_READ_TIMEOUT
597 */
598 static u32 ena_com_reg_bar_read32(struct ena_com_dev *ena_dev, u16 offset)
599 {
600 struct ena_com_mmio_read *mmio_read = &ena_dev->mmio_read;
601 volatile struct ena_admin_ena_mmio_req_read_less_resp *read_resp =
602 mmio_read->read_resp;
603 u32 mmio_read_reg, ret;
604 unsigned long flags;
605 int i;
606
607 might_sleep();
608
609 /* If readless is disabled, perform regular read */
610 if (!mmio_read->readless_supported)
611 return readl(ena_dev->reg_bar + offset);
612
613 spin_lock_irqsave(&mmio_read->lock, flags);
614 mmio_read->seq_num++;
615
616 read_resp->req_id = mmio_read->seq_num + 0xDEAD;
617 mmio_read_reg = (offset << ENA_REGS_MMIO_REG_READ_REG_OFF_SHIFT) &
618 ENA_REGS_MMIO_REG_READ_REG_OFF_MASK;
619 mmio_read_reg |= mmio_read->seq_num &
620 ENA_REGS_MMIO_REG_READ_REQ_ID_MASK;
621
622 /* make sure read_resp->req_id get updated before the hw can write
623 * there
624 */
625 wmb();
626
627 writel(mmio_read_reg, ena_dev->reg_bar + ENA_REGS_MMIO_REG_READ_OFF);
628
629 for (i = 0; i < ENA_REG_READ_TIMEOUT; i++) {
630 if (read_resp->req_id == mmio_read->seq_num)
631 break;
632
633 udelay(1);
634 }
635
636 if (unlikely(i == ENA_REG_READ_TIMEOUT)) {
637 pr_err("reading reg failed for timeout. expected: req id[%hu] offset[%hu] actual: req id[%hu] offset[%hu]\n",
638 mmio_read->seq_num, offset, read_resp->req_id,
639 read_resp->reg_off);
640 ret = ENA_MMIO_READ_TIMEOUT;
641 goto err;
642 }
643
644 if (read_resp->reg_off != offset) {
645 pr_err("Read failure: wrong offset provided");
646 ret = ENA_MMIO_READ_TIMEOUT;
647 } else {
648 ret = read_resp->reg_val;
649 }
650 err:
651 spin_unlock_irqrestore(&mmio_read->lock, flags);
652
653 return ret;
654 }
655
656 /* There are two types to wait for completion.
657 * Polling mode - wait until the completion is available.
658 * Async mode - wait on wait queue until the completion is ready
659 * (or the timeout expired).
660 * It is expected that the IRQ called ena_com_handle_admin_completion
661 * to mark the completions.
662 */
663 static int ena_com_wait_and_process_admin_cq(struct ena_comp_ctx *comp_ctx,
664 struct ena_com_admin_queue *admin_queue)
665 {
666 if (admin_queue->polling)
667 return ena_com_wait_and_process_admin_cq_polling(comp_ctx,
668 admin_queue);
669
670 return ena_com_wait_and_process_admin_cq_interrupts(comp_ctx,
671 admin_queue);
672 }
673
674 static int ena_com_destroy_io_sq(struct ena_com_dev *ena_dev,
675 struct ena_com_io_sq *io_sq)
676 {
677 struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;
678 struct ena_admin_aq_destroy_sq_cmd destroy_cmd;
679 struct ena_admin_acq_destroy_sq_resp_desc destroy_resp;
680 u8 direction;
681 int ret;
682
683 memset(&destroy_cmd, 0x0, sizeof(struct ena_admin_aq_destroy_sq_cmd));
684
685 if (io_sq->direction == ENA_COM_IO_QUEUE_DIRECTION_TX)
686 direction = ENA_ADMIN_SQ_DIRECTION_TX;
687 else
688 direction = ENA_ADMIN_SQ_DIRECTION_RX;
689
690 destroy_cmd.sq.sq_identity |= (direction <<
691 ENA_ADMIN_SQ_SQ_DIRECTION_SHIFT) &
692 ENA_ADMIN_SQ_SQ_DIRECTION_MASK;
693
694 destroy_cmd.sq.sq_idx = io_sq->idx;
695 destroy_cmd.aq_common_descriptor.opcode = ENA_ADMIN_DESTROY_SQ;
696
697 ret = ena_com_execute_admin_command(admin_queue,
698 (struct ena_admin_aq_entry *)&destroy_cmd,
699 sizeof(destroy_cmd),
700 (struct ena_admin_acq_entry *)&destroy_resp,
701 sizeof(destroy_resp));
702
703 if (unlikely(ret && (ret != -ENODEV)))
704 pr_err("failed to destroy io sq error: %d\n", ret);
705
706 return ret;
707 }
708
709 static void ena_com_io_queue_free(struct ena_com_dev *ena_dev,
710 struct ena_com_io_sq *io_sq,
711 struct ena_com_io_cq *io_cq)
712 {
713 size_t size;
714
715 if (io_cq->cdesc_addr.virt_addr) {
716 size = io_cq->cdesc_entry_size_in_bytes * io_cq->q_depth;
717
718 dma_free_coherent(ena_dev->dmadev, size,
719 io_cq->cdesc_addr.virt_addr,
720 io_cq->cdesc_addr.phys_addr);
721
722 io_cq->cdesc_addr.virt_addr = NULL;
723 }
724
725 if (io_sq->desc_addr.virt_addr) {
726 size = io_sq->desc_entry_size * io_sq->q_depth;
727
728 if (io_sq->mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_HOST)
729 dma_free_coherent(ena_dev->dmadev, size,
730 io_sq->desc_addr.virt_addr,
731 io_sq->desc_addr.phys_addr);
732 else
733 devm_kfree(ena_dev->dmadev, io_sq->desc_addr.virt_addr);
734
735 io_sq->desc_addr.virt_addr = NULL;
736 }
737 }
738
739 static int wait_for_reset_state(struct ena_com_dev *ena_dev, u32 timeout,
740 u16 exp_state)
741 {
742 u32 val, i;
743
744 for (i = 0; i < timeout; i++) {
745 val = ena_com_reg_bar_read32(ena_dev, ENA_REGS_DEV_STS_OFF);
746
747 if (unlikely(val == ENA_MMIO_READ_TIMEOUT)) {
748 pr_err("Reg read timeout occurred\n");
749 return -ETIME;
750 }
751
752 if ((val & ENA_REGS_DEV_STS_RESET_IN_PROGRESS_MASK) ==
753 exp_state)
754 return 0;
755
756 /* The resolution of the timeout is 100ms */
757 msleep(100);
758 }
759
760 return -ETIME;
761 }
762
763 static bool ena_com_check_supported_feature_id(struct ena_com_dev *ena_dev,
764 enum ena_admin_aq_feature_id feature_id)
765 {
766 u32 feature_mask = 1 << feature_id;
767
768 /* Device attributes is always supported */
769 if ((feature_id != ENA_ADMIN_DEVICE_ATTRIBUTES) &&
770 !(ena_dev->supported_features & feature_mask))
771 return false;
772
773 return true;
774 }
775
776 static int ena_com_get_feature_ex(struct ena_com_dev *ena_dev,
777 struct ena_admin_get_feat_resp *get_resp,
778 enum ena_admin_aq_feature_id feature_id,
779 dma_addr_t control_buf_dma_addr,
780 u32 control_buff_size)
781 {
782 struct ena_com_admin_queue *admin_queue;
783 struct ena_admin_get_feat_cmd get_cmd;
784 int ret;
785
786 if (!ena_com_check_supported_feature_id(ena_dev, feature_id)) {
787 pr_info("Feature %d isn't supported\n", feature_id);
788 return -EPERM;
789 }
790
791 memset(&get_cmd, 0x0, sizeof(get_cmd));
792 admin_queue = &ena_dev->admin_queue;
793
794 get_cmd.aq_common_descriptor.opcode = ENA_ADMIN_GET_FEATURE;
795
796 if (control_buff_size)
797 get_cmd.aq_common_descriptor.flags =
798 ENA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_INDIRECT_MASK;
799 else
800 get_cmd.aq_common_descriptor.flags = 0;
801
802 ret = ena_com_mem_addr_set(ena_dev,
803 &get_cmd.control_buffer.address,
804 control_buf_dma_addr);
805 if (unlikely(ret)) {
806 pr_err("memory address set failed\n");
807 return ret;
808 }
809
810 get_cmd.control_buffer.length = control_buff_size;
811
812 get_cmd.feat_common.feature_id = feature_id;
813
814 ret = ena_com_execute_admin_command(admin_queue,
815 (struct ena_admin_aq_entry *)
816 &get_cmd,
817 sizeof(get_cmd),
818 (struct ena_admin_acq_entry *)
819 get_resp,
820 sizeof(*get_resp));
821
822 if (unlikely(ret))
823 pr_err("Failed to submit get_feature command %d error: %d\n",
824 feature_id, ret);
825
826 return ret;
827 }
828
829 static int ena_com_get_feature(struct ena_com_dev *ena_dev,
830 struct ena_admin_get_feat_resp *get_resp,
831 enum ena_admin_aq_feature_id feature_id)
832 {
833 return ena_com_get_feature_ex(ena_dev,
834 get_resp,
835 feature_id,
836 0,
837 0);
838 }
839
840 static int ena_com_hash_key_allocate(struct ena_com_dev *ena_dev)
841 {
842 struct ena_rss *rss = &ena_dev->rss;
843
844 rss->hash_key =
845 dma_zalloc_coherent(ena_dev->dmadev, sizeof(*rss->hash_key),
846 &rss->hash_key_dma_addr, GFP_KERNEL);
847
848 if (unlikely(!rss->hash_key))
849 return -ENOMEM;
850
851 return 0;
852 }
853
854 static void ena_com_hash_key_destroy(struct ena_com_dev *ena_dev)
855 {
856 struct ena_rss *rss = &ena_dev->rss;
857
858 if (rss->hash_key)
859 dma_free_coherent(ena_dev->dmadev, sizeof(*rss->hash_key),
860 rss->hash_key, rss->hash_key_dma_addr);
861 rss->hash_key = NULL;
862 }
863
864 static int ena_com_hash_ctrl_init(struct ena_com_dev *ena_dev)
865 {
866 struct ena_rss *rss = &ena_dev->rss;
867
868 rss->hash_ctrl =
869 dma_zalloc_coherent(ena_dev->dmadev, sizeof(*rss->hash_ctrl),
870 &rss->hash_ctrl_dma_addr, GFP_KERNEL);
871
872 if (unlikely(!rss->hash_ctrl))
873 return -ENOMEM;
874
875 return 0;
876 }
877
878 static void ena_com_hash_ctrl_destroy(struct ena_com_dev *ena_dev)
879 {
880 struct ena_rss *rss = &ena_dev->rss;
881
882 if (rss->hash_ctrl)
883 dma_free_coherent(ena_dev->dmadev, sizeof(*rss->hash_ctrl),
884 rss->hash_ctrl, rss->hash_ctrl_dma_addr);
885 rss->hash_ctrl = NULL;
886 }
887
888 static int ena_com_indirect_table_allocate(struct ena_com_dev *ena_dev,
889 u16 log_size)
890 {
891 struct ena_rss *rss = &ena_dev->rss;
892 struct ena_admin_get_feat_resp get_resp;
893 size_t tbl_size;
894 int ret;
895
896 ret = ena_com_get_feature(ena_dev, &get_resp,
897 ENA_ADMIN_RSS_REDIRECTION_TABLE_CONFIG);
898 if (unlikely(ret))
899 return ret;
900
901 if ((get_resp.u.ind_table.min_size > log_size) ||
902 (get_resp.u.ind_table.max_size < log_size)) {
903 pr_err("indirect table size doesn't fit. requested size: %d while min is:%d and max %d\n",
904 1 << log_size, 1 << get_resp.u.ind_table.min_size,
905 1 << get_resp.u.ind_table.max_size);
906 return -EINVAL;
907 }
908
909 tbl_size = (1ULL << log_size) *
910 sizeof(struct ena_admin_rss_ind_table_entry);
911
912 rss->rss_ind_tbl =
913 dma_zalloc_coherent(ena_dev->dmadev, tbl_size,
914 &rss->rss_ind_tbl_dma_addr, GFP_KERNEL);
915 if (unlikely(!rss->rss_ind_tbl))
916 goto mem_err1;
917
918 tbl_size = (1ULL << log_size) * sizeof(u16);
919 rss->host_rss_ind_tbl =
920 devm_kzalloc(ena_dev->dmadev, tbl_size, GFP_KERNEL);
921 if (unlikely(!rss->host_rss_ind_tbl))
922 goto mem_err2;
923
924 rss->tbl_log_size = log_size;
925
926 return 0;
927
928 mem_err2:
929 tbl_size = (1ULL << log_size) *
930 sizeof(struct ena_admin_rss_ind_table_entry);
931
932 dma_free_coherent(ena_dev->dmadev, tbl_size, rss->rss_ind_tbl,
933 rss->rss_ind_tbl_dma_addr);
934 rss->rss_ind_tbl = NULL;
935 mem_err1:
936 rss->tbl_log_size = 0;
937 return -ENOMEM;
938 }
939
940 static void ena_com_indirect_table_destroy(struct ena_com_dev *ena_dev)
941 {
942 struct ena_rss *rss = &ena_dev->rss;
943 size_t tbl_size = (1ULL << rss->tbl_log_size) *
944 sizeof(struct ena_admin_rss_ind_table_entry);
945
946 if (rss->rss_ind_tbl)
947 dma_free_coherent(ena_dev->dmadev, tbl_size, rss->rss_ind_tbl,
948 rss->rss_ind_tbl_dma_addr);
949 rss->rss_ind_tbl = NULL;
950
951 if (rss->host_rss_ind_tbl)
952 devm_kfree(ena_dev->dmadev, rss->host_rss_ind_tbl);
953 rss->host_rss_ind_tbl = NULL;
954 }
955
956 static int ena_com_create_io_sq(struct ena_com_dev *ena_dev,
957 struct ena_com_io_sq *io_sq, u16 cq_idx)
958 {
959 struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;
960 struct ena_admin_aq_create_sq_cmd create_cmd;
961 struct ena_admin_acq_create_sq_resp_desc cmd_completion;
962 u8 direction;
963 int ret;
964
965 memset(&create_cmd, 0x0, sizeof(struct ena_admin_aq_create_sq_cmd));
966
967 create_cmd.aq_common_descriptor.opcode = ENA_ADMIN_CREATE_SQ;
968
969 if (io_sq->direction == ENA_COM_IO_QUEUE_DIRECTION_TX)
970 direction = ENA_ADMIN_SQ_DIRECTION_TX;
971 else
972 direction = ENA_ADMIN_SQ_DIRECTION_RX;
973
974 create_cmd.sq_identity |= (direction <<
975 ENA_ADMIN_AQ_CREATE_SQ_CMD_SQ_DIRECTION_SHIFT) &
976 ENA_ADMIN_AQ_CREATE_SQ_CMD_SQ_DIRECTION_MASK;
977
978 create_cmd.sq_caps_2 |= io_sq->mem_queue_type &
979 ENA_ADMIN_AQ_CREATE_SQ_CMD_PLACEMENT_POLICY_MASK;
980
981 create_cmd.sq_caps_2 |= (ENA_ADMIN_COMPLETION_POLICY_DESC <<
982 ENA_ADMIN_AQ_CREATE_SQ_CMD_COMPLETION_POLICY_SHIFT) &
983 ENA_ADMIN_AQ_CREATE_SQ_CMD_COMPLETION_POLICY_MASK;
984
985 create_cmd.sq_caps_3 |=
986 ENA_ADMIN_AQ_CREATE_SQ_CMD_IS_PHYSICALLY_CONTIGUOUS_MASK;
987
988 create_cmd.cq_idx = cq_idx;
989 create_cmd.sq_depth = io_sq->q_depth;
990
991 if (io_sq->mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_HOST) {
992 ret = ena_com_mem_addr_set(ena_dev,
993 &create_cmd.sq_ba,
994 io_sq->desc_addr.phys_addr);
995 if (unlikely(ret)) {
996 pr_err("memory address set failed\n");
997 return ret;
998 }
999 }
1000
1001 ret = ena_com_execute_admin_command(admin_queue,
1002 (struct ena_admin_aq_entry *)&create_cmd,
1003 sizeof(create_cmd),
1004 (struct ena_admin_acq_entry *)&cmd_completion,
1005 sizeof(cmd_completion));
1006 if (unlikely(ret)) {
1007 pr_err("Failed to create IO SQ. error: %d\n", ret);
1008 return ret;
1009 }
1010
1011 io_sq->idx = cmd_completion.sq_idx;
1012
1013 io_sq->db_addr = (u32 __iomem *)((uintptr_t)ena_dev->reg_bar +
1014 (uintptr_t)cmd_completion.sq_doorbell_offset);
1015
1016 if (io_sq->mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) {
1017 io_sq->header_addr = (u8 __iomem *)((uintptr_t)ena_dev->mem_bar
1018 + cmd_completion.llq_headers_offset);
1019
1020 io_sq->desc_addr.pbuf_dev_addr =
1021 (u8 __iomem *)((uintptr_t)ena_dev->mem_bar +
1022 cmd_completion.llq_descriptors_offset);
1023 }
1024
1025 pr_debug("created sq[%u], depth[%u]\n", io_sq->idx, io_sq->q_depth);
1026
1027 return ret;
1028 }
1029
1030 static int ena_com_ind_tbl_convert_to_device(struct ena_com_dev *ena_dev)
1031 {
1032 struct ena_rss *rss = &ena_dev->rss;
1033 struct ena_com_io_sq *io_sq;
1034 u16 qid;
1035 int i;
1036
1037 for (i = 0; i < 1 << rss->tbl_log_size; i++) {
1038 qid = rss->host_rss_ind_tbl[i];
1039 if (qid >= ENA_TOTAL_NUM_QUEUES)
1040 return -EINVAL;
1041
1042 io_sq = &ena_dev->io_sq_queues[qid];
1043
1044 if (io_sq->direction != ENA_COM_IO_QUEUE_DIRECTION_RX)
1045 return -EINVAL;
1046
1047 rss->rss_ind_tbl[i].cq_idx = io_sq->idx;
1048 }
1049
1050 return 0;
1051 }
1052
1053 static int ena_com_ind_tbl_convert_from_device(struct ena_com_dev *ena_dev)
1054 {
1055 u16 dev_idx_to_host_tbl[ENA_TOTAL_NUM_QUEUES] = { (u16)-1 };
1056 struct ena_rss *rss = &ena_dev->rss;
1057 u8 idx;
1058 u16 i;
1059
1060 for (i = 0; i < ENA_TOTAL_NUM_QUEUES; i++)
1061 dev_idx_to_host_tbl[ena_dev->io_sq_queues[i].idx] = i;
1062
1063 for (i = 0; i < 1 << rss->tbl_log_size; i++) {
1064 if (rss->rss_ind_tbl[i].cq_idx > ENA_TOTAL_NUM_QUEUES)
1065 return -EINVAL;
1066 idx = (u8)rss->rss_ind_tbl[i].cq_idx;
1067
1068 if (dev_idx_to_host_tbl[idx] > ENA_TOTAL_NUM_QUEUES)
1069 return -EINVAL;
1070
1071 rss->host_rss_ind_tbl[i] = dev_idx_to_host_tbl[idx];
1072 }
1073
1074 return 0;
1075 }
1076
1077 static int ena_com_init_interrupt_moderation_table(struct ena_com_dev *ena_dev)
1078 {
1079 size_t size;
1080
1081 size = sizeof(struct ena_intr_moder_entry) * ENA_INTR_MAX_NUM_OF_LEVELS;
1082
1083 ena_dev->intr_moder_tbl =
1084 devm_kzalloc(ena_dev->dmadev, size, GFP_KERNEL);
1085 if (!ena_dev->intr_moder_tbl)
1086 return -ENOMEM;
1087
1088 ena_com_config_default_interrupt_moderation_table(ena_dev);
1089
1090 return 0;
1091 }
1092
1093 static void ena_com_update_intr_delay_resolution(struct ena_com_dev *ena_dev,
1094 u16 intr_delay_resolution)
1095 {
1096 struct ena_intr_moder_entry *intr_moder_tbl = ena_dev->intr_moder_tbl;
1097 unsigned int i;
1098
1099 if (!intr_delay_resolution) {
1100 pr_err("Illegal intr_delay_resolution provided. Going to use default 1 usec resolution\n");
1101 intr_delay_resolution = 1;
1102 }
1103 ena_dev->intr_delay_resolution = intr_delay_resolution;
1104
1105 /* update Rx */
1106 for (i = 0; i < ENA_INTR_MAX_NUM_OF_LEVELS; i++)
1107 intr_moder_tbl[i].intr_moder_interval /= intr_delay_resolution;
1108
1109 /* update Tx */
1110 ena_dev->intr_moder_tx_interval /= intr_delay_resolution;
1111 }
1112
1113 /*****************************************************************************/
1114 /******************************* API ******************************/
1115 /*****************************************************************************/
1116
1117 int ena_com_execute_admin_command(struct ena_com_admin_queue *admin_queue,
1118 struct ena_admin_aq_entry *cmd,
1119 size_t cmd_size,
1120 struct ena_admin_acq_entry *comp,
1121 size_t comp_size)
1122 {
1123 struct ena_comp_ctx *comp_ctx;
1124 int ret;
1125
1126 comp_ctx = ena_com_submit_admin_cmd(admin_queue, cmd, cmd_size,
1127 comp, comp_size);
1128 if (unlikely(IS_ERR(comp_ctx))) {
1129 pr_err("Failed to submit command [%ld]\n", PTR_ERR(comp_ctx));
1130 return PTR_ERR(comp_ctx);
1131 }
1132
1133 ret = ena_com_wait_and_process_admin_cq(comp_ctx, admin_queue);
1134 if (unlikely(ret)) {
1135 if (admin_queue->running_state)
1136 pr_err("Failed to process command. ret = %d\n", ret);
1137 else
1138 pr_debug("Failed to process command. ret = %d\n", ret);
1139 }
1140 return ret;
1141 }
1142
1143 int ena_com_create_io_cq(struct ena_com_dev *ena_dev,
1144 struct ena_com_io_cq *io_cq)
1145 {
1146 struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;
1147 struct ena_admin_aq_create_cq_cmd create_cmd;
1148 struct ena_admin_acq_create_cq_resp_desc cmd_completion;
1149 int ret;
1150
1151 memset(&create_cmd, 0x0, sizeof(struct ena_admin_aq_create_cq_cmd));
1152
1153 create_cmd.aq_common_descriptor.opcode = ENA_ADMIN_CREATE_CQ;
1154
1155 create_cmd.cq_caps_2 |= (io_cq->cdesc_entry_size_in_bytes / 4) &
1156 ENA_ADMIN_AQ_CREATE_CQ_CMD_CQ_ENTRY_SIZE_WORDS_MASK;
1157 create_cmd.cq_caps_1 |=
1158 ENA_ADMIN_AQ_CREATE_CQ_CMD_INTERRUPT_MODE_ENABLED_MASK;
1159
1160 create_cmd.msix_vector = io_cq->msix_vector;
1161 create_cmd.cq_depth = io_cq->q_depth;
1162
1163 ret = ena_com_mem_addr_set(ena_dev,
1164 &create_cmd.cq_ba,
1165 io_cq->cdesc_addr.phys_addr);
1166 if (unlikely(ret)) {
1167 pr_err("memory address set failed\n");
1168 return ret;
1169 }
1170
1171 ret = ena_com_execute_admin_command(admin_queue,
1172 (struct ena_admin_aq_entry *)&create_cmd,
1173 sizeof(create_cmd),
1174 (struct ena_admin_acq_entry *)&cmd_completion,
1175 sizeof(cmd_completion));
1176 if (unlikely(ret)) {
1177 pr_err("Failed to create IO CQ. error: %d\n", ret);
1178 return ret;
1179 }
1180
1181 io_cq->idx = cmd_completion.cq_idx;
1182
1183 io_cq->unmask_reg = (u32 __iomem *)((uintptr_t)ena_dev->reg_bar +
1184 cmd_completion.cq_interrupt_unmask_register_offset);
1185
1186 if (cmd_completion.cq_head_db_register_offset)
1187 io_cq->cq_head_db_reg =
1188 (u32 __iomem *)((uintptr_t)ena_dev->reg_bar +
1189 cmd_completion.cq_head_db_register_offset);
1190
1191 if (cmd_completion.numa_node_register_offset)
1192 io_cq->numa_node_cfg_reg =
1193 (u32 __iomem *)((uintptr_t)ena_dev->reg_bar +
1194 cmd_completion.numa_node_register_offset);
1195
1196 pr_debug("created cq[%u], depth[%u]\n", io_cq->idx, io_cq->q_depth);
1197
1198 return ret;
1199 }
1200
1201 int ena_com_get_io_handlers(struct ena_com_dev *ena_dev, u16 qid,
1202 struct ena_com_io_sq **io_sq,
1203 struct ena_com_io_cq **io_cq)
1204 {
1205 if (qid >= ENA_TOTAL_NUM_QUEUES) {
1206 pr_err("Invalid queue number %d but the max is %d\n", qid,
1207 ENA_TOTAL_NUM_QUEUES);
1208 return -EINVAL;
1209 }
1210
1211 *io_sq = &ena_dev->io_sq_queues[qid];
1212 *io_cq = &ena_dev->io_cq_queues[qid];
1213
1214 return 0;
1215 }
1216
1217 void ena_com_abort_admin_commands(struct ena_com_dev *ena_dev)
1218 {
1219 struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;
1220 struct ena_comp_ctx *comp_ctx;
1221 u16 i;
1222
1223 if (!admin_queue->comp_ctx)
1224 return;
1225
1226 for (i = 0; i < admin_queue->q_depth; i++) {
1227 comp_ctx = get_comp_ctxt(admin_queue, i, false);
1228 if (unlikely(!comp_ctx))
1229 break;
1230
1231 comp_ctx->status = ENA_CMD_ABORTED;
1232
1233 complete(&comp_ctx->wait_event);
1234 }
1235 }
1236
1237 void ena_com_wait_for_abort_completion(struct ena_com_dev *ena_dev)
1238 {
1239 struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;
1240 unsigned long flags;
1241
1242 spin_lock_irqsave(&admin_queue->q_lock, flags);
1243 while (atomic_read(&admin_queue->outstanding_cmds) != 0) {
1244 spin_unlock_irqrestore(&admin_queue->q_lock, flags);
1245 msleep(20);
1246 spin_lock_irqsave(&admin_queue->q_lock, flags);
1247 }
1248 spin_unlock_irqrestore(&admin_queue->q_lock, flags);
1249 }
1250
1251 int ena_com_destroy_io_cq(struct ena_com_dev *ena_dev,
1252 struct ena_com_io_cq *io_cq)
1253 {
1254 struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;
1255 struct ena_admin_aq_destroy_cq_cmd destroy_cmd;
1256 struct ena_admin_acq_destroy_cq_resp_desc destroy_resp;
1257 int ret;
1258
1259 memset(&destroy_cmd, 0x0, sizeof(struct ena_admin_aq_destroy_sq_cmd));
1260
1261 destroy_cmd.cq_idx = io_cq->idx;
1262 destroy_cmd.aq_common_descriptor.opcode = ENA_ADMIN_DESTROY_CQ;
1263
1264 ret = ena_com_execute_admin_command(admin_queue,
1265 (struct ena_admin_aq_entry *)&destroy_cmd,
1266 sizeof(destroy_cmd),
1267 (struct ena_admin_acq_entry *)&destroy_resp,
1268 sizeof(destroy_resp));
1269
1270 if (unlikely(ret && (ret != -ENODEV)))
1271 pr_err("Failed to destroy IO CQ. error: %d\n", ret);
1272
1273 return ret;
1274 }
1275
1276 bool ena_com_get_admin_running_state(struct ena_com_dev *ena_dev)
1277 {
1278 return ena_dev->admin_queue.running_state;
1279 }
1280
1281 void ena_com_set_admin_running_state(struct ena_com_dev *ena_dev, bool state)
1282 {
1283 struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;
1284 unsigned long flags;
1285
1286 spin_lock_irqsave(&admin_queue->q_lock, flags);
1287 ena_dev->admin_queue.running_state = state;
1288 spin_unlock_irqrestore(&admin_queue->q_lock, flags);
1289 }
1290
1291 void ena_com_admin_aenq_enable(struct ena_com_dev *ena_dev)
1292 {
1293 u16 depth = ena_dev->aenq.q_depth;
1294
1295 WARN(ena_dev->aenq.head != depth, "Invalid AENQ state\n");
1296
1297 /* Init head_db to mark that all entries in the queue
1298 * are initially available
1299 */
1300 writel(depth, ena_dev->reg_bar + ENA_REGS_AENQ_HEAD_DB_OFF);
1301 }
1302
1303 int ena_com_set_aenq_config(struct ena_com_dev *ena_dev, u32 groups_flag)
1304 {
1305 struct ena_com_admin_queue *admin_queue;
1306 struct ena_admin_set_feat_cmd cmd;
1307 struct ena_admin_set_feat_resp resp;
1308 struct ena_admin_get_feat_resp get_resp;
1309 int ret;
1310
1311 ret = ena_com_get_feature(ena_dev, &get_resp, ENA_ADMIN_AENQ_CONFIG);
1312 if (ret) {
1313 pr_info("Can't get aenq configuration\n");
1314 return ret;
1315 }
1316
1317 if ((get_resp.u.aenq.supported_groups & groups_flag) != groups_flag) {
1318 pr_warn("Trying to set unsupported aenq events. supported flag: %x asked flag: %x\n",
1319 get_resp.u.aenq.supported_groups, groups_flag);
1320 return -EPERM;
1321 }
1322
1323 memset(&cmd, 0x0, sizeof(cmd));
1324 admin_queue = &ena_dev->admin_queue;
1325
1326 cmd.aq_common_descriptor.opcode = ENA_ADMIN_SET_FEATURE;
1327 cmd.aq_common_descriptor.flags = 0;
1328 cmd.feat_common.feature_id = ENA_ADMIN_AENQ_CONFIG;
1329 cmd.u.aenq.enabled_groups = groups_flag;
1330
1331 ret = ena_com_execute_admin_command(admin_queue,
1332 (struct ena_admin_aq_entry *)&cmd,
1333 sizeof(cmd),
1334 (struct ena_admin_acq_entry *)&resp,
1335 sizeof(resp));
1336
1337 if (unlikely(ret))
1338 pr_err("Failed to config AENQ ret: %d\n", ret);
1339
1340 return ret;
1341 }
1342
1343 int ena_com_get_dma_width(struct ena_com_dev *ena_dev)
1344 {
1345 u32 caps = ena_com_reg_bar_read32(ena_dev, ENA_REGS_CAPS_OFF);
1346 int width;
1347
1348 if (unlikely(caps == ENA_MMIO_READ_TIMEOUT)) {
1349 pr_err("Reg read timeout occurred\n");
1350 return -ETIME;
1351 }
1352
1353 width = (caps & ENA_REGS_CAPS_DMA_ADDR_WIDTH_MASK) >>
1354 ENA_REGS_CAPS_DMA_ADDR_WIDTH_SHIFT;
1355
1356 pr_debug("ENA dma width: %d\n", width);
1357
1358 if ((width < 32) || width > ENA_MAX_PHYS_ADDR_SIZE_BITS) {
1359 pr_err("DMA width illegal value: %d\n", width);
1360 return -EINVAL;
1361 }
1362
1363 ena_dev->dma_addr_bits = width;
1364
1365 return width;
1366 }
1367
1368 int ena_com_validate_version(struct ena_com_dev *ena_dev)
1369 {
1370 u32 ver;
1371 u32 ctrl_ver;
1372 u32 ctrl_ver_masked;
1373
1374 /* Make sure the ENA version and the controller version are at least
1375 * as the driver expects
1376 */
1377 ver = ena_com_reg_bar_read32(ena_dev, ENA_REGS_VERSION_OFF);
1378 ctrl_ver = ena_com_reg_bar_read32(ena_dev,
1379 ENA_REGS_CONTROLLER_VERSION_OFF);
1380
1381 if (unlikely((ver == ENA_MMIO_READ_TIMEOUT) ||
1382 (ctrl_ver == ENA_MMIO_READ_TIMEOUT))) {
1383 pr_err("Reg read timeout occurred\n");
1384 return -ETIME;
1385 }
1386
1387 pr_info("ena device version: %d.%d\n",
1388 (ver & ENA_REGS_VERSION_MAJOR_VERSION_MASK) >>
1389 ENA_REGS_VERSION_MAJOR_VERSION_SHIFT,
1390 ver & ENA_REGS_VERSION_MINOR_VERSION_MASK);
1391
1392 if (ver < MIN_ENA_VER) {
1393 pr_err("ENA version is lower than the minimal version the driver supports\n");
1394 return -1;
1395 }
1396
1397 pr_info("ena controller version: %d.%d.%d implementation version %d\n",
1398 (ctrl_ver & ENA_REGS_CONTROLLER_VERSION_MAJOR_VERSION_MASK) >>
1399 ENA_REGS_CONTROLLER_VERSION_MAJOR_VERSION_SHIFT,
1400 (ctrl_ver & ENA_REGS_CONTROLLER_VERSION_MINOR_VERSION_MASK) >>
1401 ENA_REGS_CONTROLLER_VERSION_MINOR_VERSION_SHIFT,
1402 (ctrl_ver & ENA_REGS_CONTROLLER_VERSION_SUBMINOR_VERSION_MASK),
1403 (ctrl_ver & ENA_REGS_CONTROLLER_VERSION_IMPL_ID_MASK) >>
1404 ENA_REGS_CONTROLLER_VERSION_IMPL_ID_SHIFT);
1405
1406 ctrl_ver_masked =
1407 (ctrl_ver & ENA_REGS_CONTROLLER_VERSION_MAJOR_VERSION_MASK) |
1408 (ctrl_ver & ENA_REGS_CONTROLLER_VERSION_MINOR_VERSION_MASK) |
1409 (ctrl_ver & ENA_REGS_CONTROLLER_VERSION_SUBMINOR_VERSION_MASK);
1410
1411 /* Validate the ctrl version without the implementation ID */
1412 if (ctrl_ver_masked < MIN_ENA_CTRL_VER) {
1413 pr_err("ENA ctrl version is lower than the minimal ctrl version the driver supports\n");
1414 return -1;
1415 }
1416
1417 return 0;
1418 }
1419
1420 void ena_com_admin_destroy(struct ena_com_dev *ena_dev)
1421 {
1422 struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;
1423 struct ena_com_admin_cq *cq = &admin_queue->cq;
1424 struct ena_com_admin_sq *sq = &admin_queue->sq;
1425 struct ena_com_aenq *aenq = &ena_dev->aenq;
1426 u16 size;
1427
1428 if (admin_queue->comp_ctx)
1429 devm_kfree(ena_dev->dmadev, admin_queue->comp_ctx);
1430 admin_queue->comp_ctx = NULL;
1431 size = ADMIN_SQ_SIZE(admin_queue->q_depth);
1432 if (sq->entries)
1433 dma_free_coherent(ena_dev->dmadev, size, sq->entries,
1434 sq->dma_addr);
1435 sq->entries = NULL;
1436
1437 size = ADMIN_CQ_SIZE(admin_queue->q_depth);
1438 if (cq->entries)
1439 dma_free_coherent(ena_dev->dmadev, size, cq->entries,
1440 cq->dma_addr);
1441 cq->entries = NULL;
1442
1443 size = ADMIN_AENQ_SIZE(aenq->q_depth);
1444 if (ena_dev->aenq.entries)
1445 dma_free_coherent(ena_dev->dmadev, size, aenq->entries,
1446 aenq->dma_addr);
1447 aenq->entries = NULL;
1448 }
1449
1450 void ena_com_set_admin_polling_mode(struct ena_com_dev *ena_dev, bool polling)
1451 {
1452 ena_dev->admin_queue.polling = polling;
1453 }
1454
1455 int ena_com_mmio_reg_read_request_init(struct ena_com_dev *ena_dev)
1456 {
1457 struct ena_com_mmio_read *mmio_read = &ena_dev->mmio_read;
1458
1459 spin_lock_init(&mmio_read->lock);
1460 mmio_read->read_resp =
1461 dma_zalloc_coherent(ena_dev->dmadev,
1462 sizeof(*mmio_read->read_resp),
1463 &mmio_read->read_resp_dma_addr, GFP_KERNEL);
1464 if (unlikely(!mmio_read->read_resp))
1465 return -ENOMEM;
1466
1467 ena_com_mmio_reg_read_request_write_dev_addr(ena_dev);
1468
1469 mmio_read->read_resp->req_id = 0x0;
1470 mmio_read->seq_num = 0x0;
1471 mmio_read->readless_supported = true;
1472
1473 return 0;
1474 }
1475
1476 void ena_com_set_mmio_read_mode(struct ena_com_dev *ena_dev, bool readless_supported)
1477 {
1478 struct ena_com_mmio_read *mmio_read = &ena_dev->mmio_read;
1479
1480 mmio_read->readless_supported = readless_supported;
1481 }
1482
1483 void ena_com_mmio_reg_read_request_destroy(struct ena_com_dev *ena_dev)
1484 {
1485 struct ena_com_mmio_read *mmio_read = &ena_dev->mmio_read;
1486
1487 writel(0x0, ena_dev->reg_bar + ENA_REGS_MMIO_RESP_LO_OFF);
1488 writel(0x0, ena_dev->reg_bar + ENA_REGS_MMIO_RESP_HI_OFF);
1489
1490 dma_free_coherent(ena_dev->dmadev, sizeof(*mmio_read->read_resp),
1491 mmio_read->read_resp, mmio_read->read_resp_dma_addr);
1492
1493 mmio_read->read_resp = NULL;
1494 }
1495
1496 void ena_com_mmio_reg_read_request_write_dev_addr(struct ena_com_dev *ena_dev)
1497 {
1498 struct ena_com_mmio_read *mmio_read = &ena_dev->mmio_read;
1499 u32 addr_low, addr_high;
1500
1501 addr_low = ENA_DMA_ADDR_TO_UINT32_LOW(mmio_read->read_resp_dma_addr);
1502 addr_high = ENA_DMA_ADDR_TO_UINT32_HIGH(mmio_read->read_resp_dma_addr);
1503
1504 writel(addr_low, ena_dev->reg_bar + ENA_REGS_MMIO_RESP_LO_OFF);
1505 writel(addr_high, ena_dev->reg_bar + ENA_REGS_MMIO_RESP_HI_OFF);
1506 }
1507
1508 int ena_com_admin_init(struct ena_com_dev *ena_dev,
1509 struct ena_aenq_handlers *aenq_handlers,
1510 bool init_spinlock)
1511 {
1512 struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;
1513 u32 aq_caps, acq_caps, dev_sts, addr_low, addr_high;
1514 int ret;
1515
1516 dev_sts = ena_com_reg_bar_read32(ena_dev, ENA_REGS_DEV_STS_OFF);
1517
1518 if (unlikely(dev_sts == ENA_MMIO_READ_TIMEOUT)) {
1519 pr_err("Reg read timeout occurred\n");
1520 return -ETIME;
1521 }
1522
1523 if (!(dev_sts & ENA_REGS_DEV_STS_READY_MASK)) {
1524 pr_err("Device isn't ready, abort com init\n");
1525 return -ENODEV;
1526 }
1527
1528 admin_queue->q_depth = ENA_ADMIN_QUEUE_DEPTH;
1529
1530 admin_queue->q_dmadev = ena_dev->dmadev;
1531 admin_queue->polling = false;
1532 admin_queue->curr_cmd_id = 0;
1533
1534 atomic_set(&admin_queue->outstanding_cmds, 0);
1535
1536 if (init_spinlock)
1537 spin_lock_init(&admin_queue->q_lock);
1538
1539 ret = ena_com_init_comp_ctxt(admin_queue);
1540 if (ret)
1541 goto error;
1542
1543 ret = ena_com_admin_init_sq(admin_queue);
1544 if (ret)
1545 goto error;
1546
1547 ret = ena_com_admin_init_cq(admin_queue);
1548 if (ret)
1549 goto error;
1550
1551 admin_queue->sq.db_addr = (u32 __iomem *)((uintptr_t)ena_dev->reg_bar +
1552 ENA_REGS_AQ_DB_OFF);
1553
1554 addr_low = ENA_DMA_ADDR_TO_UINT32_LOW(admin_queue->sq.dma_addr);
1555 addr_high = ENA_DMA_ADDR_TO_UINT32_HIGH(admin_queue->sq.dma_addr);
1556
1557 writel(addr_low, ena_dev->reg_bar + ENA_REGS_AQ_BASE_LO_OFF);
1558 writel(addr_high, ena_dev->reg_bar + ENA_REGS_AQ_BASE_HI_OFF);
1559
1560 addr_low = ENA_DMA_ADDR_TO_UINT32_LOW(admin_queue->cq.dma_addr);
1561 addr_high = ENA_DMA_ADDR_TO_UINT32_HIGH(admin_queue->cq.dma_addr);
1562
1563 writel(addr_low, ena_dev->reg_bar + ENA_REGS_ACQ_BASE_LO_OFF);
1564 writel(addr_high, ena_dev->reg_bar + ENA_REGS_ACQ_BASE_HI_OFF);
1565
1566 aq_caps = 0;
1567 aq_caps |= admin_queue->q_depth & ENA_REGS_AQ_CAPS_AQ_DEPTH_MASK;
1568 aq_caps |= (sizeof(struct ena_admin_aq_entry) <<
1569 ENA_REGS_AQ_CAPS_AQ_ENTRY_SIZE_SHIFT) &
1570 ENA_REGS_AQ_CAPS_AQ_ENTRY_SIZE_MASK;
1571
1572 acq_caps = 0;
1573 acq_caps |= admin_queue->q_depth & ENA_REGS_ACQ_CAPS_ACQ_DEPTH_MASK;
1574 acq_caps |= (sizeof(struct ena_admin_acq_entry) <<
1575 ENA_REGS_ACQ_CAPS_ACQ_ENTRY_SIZE_SHIFT) &
1576 ENA_REGS_ACQ_CAPS_ACQ_ENTRY_SIZE_MASK;
1577
1578 writel(aq_caps, ena_dev->reg_bar + ENA_REGS_AQ_CAPS_OFF);
1579 writel(acq_caps, ena_dev->reg_bar + ENA_REGS_ACQ_CAPS_OFF);
1580 ret = ena_com_admin_init_aenq(ena_dev, aenq_handlers);
1581 if (ret)
1582 goto error;
1583
1584 admin_queue->running_state = true;
1585
1586 return 0;
1587 error:
1588 ena_com_admin_destroy(ena_dev);
1589
1590 return ret;
1591 }
1592
1593 int ena_com_create_io_queue(struct ena_com_dev *ena_dev,
1594 struct ena_com_create_io_ctx *ctx)
1595 {
1596 struct ena_com_io_sq *io_sq;
1597 struct ena_com_io_cq *io_cq;
1598 int ret;
1599
1600 if (ctx->qid >= ENA_TOTAL_NUM_QUEUES) {
1601 pr_err("Qid (%d) is bigger than max num of queues (%d)\n",
1602 ctx->qid, ENA_TOTAL_NUM_QUEUES);
1603 return -EINVAL;
1604 }
1605
1606 io_sq = &ena_dev->io_sq_queues[ctx->qid];
1607 io_cq = &ena_dev->io_cq_queues[ctx->qid];
1608
1609 memset(io_sq, 0x0, sizeof(struct ena_com_io_sq));
1610 memset(io_cq, 0x0, sizeof(struct ena_com_io_cq));
1611
1612 /* Init CQ */
1613 io_cq->q_depth = ctx->queue_size;
1614 io_cq->direction = ctx->direction;
1615 io_cq->qid = ctx->qid;
1616
1617 io_cq->msix_vector = ctx->msix_vector;
1618
1619 io_sq->q_depth = ctx->queue_size;
1620 io_sq->direction = ctx->direction;
1621 io_sq->qid = ctx->qid;
1622
1623 io_sq->mem_queue_type = ctx->mem_queue_type;
1624
1625 if (ctx->direction == ENA_COM_IO_QUEUE_DIRECTION_TX)
1626 /* header length is limited to 8 bits */
1627 io_sq->tx_max_header_size =
1628 min_t(u32, ena_dev->tx_max_header_size, SZ_256);
1629
1630 ret = ena_com_init_io_sq(ena_dev, ctx, io_sq);
1631 if (ret)
1632 goto error;
1633 ret = ena_com_init_io_cq(ena_dev, ctx, io_cq);
1634 if (ret)
1635 goto error;
1636
1637 ret = ena_com_create_io_cq(ena_dev, io_cq);
1638 if (ret)
1639 goto error;
1640
1641 ret = ena_com_create_io_sq(ena_dev, io_sq, io_cq->idx);
1642 if (ret)
1643 goto destroy_io_cq;
1644
1645 return 0;
1646
1647 destroy_io_cq:
1648 ena_com_destroy_io_cq(ena_dev, io_cq);
1649 error:
1650 ena_com_io_queue_free(ena_dev, io_sq, io_cq);
1651 return ret;
1652 }
1653
1654 void ena_com_destroy_io_queue(struct ena_com_dev *ena_dev, u16 qid)
1655 {
1656 struct ena_com_io_sq *io_sq;
1657 struct ena_com_io_cq *io_cq;
1658
1659 if (qid >= ENA_TOTAL_NUM_QUEUES) {
1660 pr_err("Qid (%d) is bigger than max num of queues (%d)\n", qid,
1661 ENA_TOTAL_NUM_QUEUES);
1662 return;
1663 }
1664
1665 io_sq = &ena_dev->io_sq_queues[qid];
1666 io_cq = &ena_dev->io_cq_queues[qid];
1667
1668 ena_com_destroy_io_sq(ena_dev, io_sq);
1669 ena_com_destroy_io_cq(ena_dev, io_cq);
1670
1671 ena_com_io_queue_free(ena_dev, io_sq, io_cq);
1672 }
1673
1674 int ena_com_get_link_params(struct ena_com_dev *ena_dev,
1675 struct ena_admin_get_feat_resp *resp)
1676 {
1677 return ena_com_get_feature(ena_dev, resp, ENA_ADMIN_LINK_CONFIG);
1678 }
1679
1680 int ena_com_get_dev_attr_feat(struct ena_com_dev *ena_dev,
1681 struct ena_com_dev_get_features_ctx *get_feat_ctx)
1682 {
1683 struct ena_admin_get_feat_resp get_resp;
1684 int rc;
1685
1686 rc = ena_com_get_feature(ena_dev, &get_resp,
1687 ENA_ADMIN_DEVICE_ATTRIBUTES);
1688 if (rc)
1689 return rc;
1690
1691 memcpy(&get_feat_ctx->dev_attr, &get_resp.u.dev_attr,
1692 sizeof(get_resp.u.dev_attr));
1693 ena_dev->supported_features = get_resp.u.dev_attr.supported_features;
1694
1695 rc = ena_com_get_feature(ena_dev, &get_resp,
1696 ENA_ADMIN_MAX_QUEUES_NUM);
1697 if (rc)
1698 return rc;
1699
1700 memcpy(&get_feat_ctx->max_queues, &get_resp.u.max_queue,
1701 sizeof(get_resp.u.max_queue));
1702 ena_dev->tx_max_header_size = get_resp.u.max_queue.max_header_size;
1703
1704 rc = ena_com_get_feature(ena_dev, &get_resp,
1705 ENA_ADMIN_AENQ_CONFIG);
1706 if (rc)
1707 return rc;
1708
1709 memcpy(&get_feat_ctx->aenq, &get_resp.u.aenq,
1710 sizeof(get_resp.u.aenq));
1711
1712 rc = ena_com_get_feature(ena_dev, &get_resp,
1713 ENA_ADMIN_STATELESS_OFFLOAD_CONFIG);
1714 if (rc)
1715 return rc;
1716
1717 memcpy(&get_feat_ctx->offload, &get_resp.u.offload,
1718 sizeof(get_resp.u.offload));
1719
1720 return 0;
1721 }
1722
1723 void ena_com_admin_q_comp_intr_handler(struct ena_com_dev *ena_dev)
1724 {
1725 ena_com_handle_admin_completion(&ena_dev->admin_queue);
1726 }
1727
1728 /* ena_handle_specific_aenq_event:
1729 * return the handler that is relevant to the specific event group
1730 */
1731 static ena_aenq_handler ena_com_get_specific_aenq_cb(struct ena_com_dev *dev,
1732 u16 group)
1733 {
1734 struct ena_aenq_handlers *aenq_handlers = dev->aenq.aenq_handlers;
1735
1736 if ((group < ENA_MAX_HANDLERS) && aenq_handlers->handlers[group])
1737 return aenq_handlers->handlers[group];
1738
1739 return aenq_handlers->unimplemented_handler;
1740 }
1741
1742 /* ena_aenq_intr_handler:
1743 * handles the aenq incoming events.
1744 * pop events from the queue and apply the specific handler
1745 */
1746 void ena_com_aenq_intr_handler(struct ena_com_dev *dev, void *data)
1747 {
1748 struct ena_admin_aenq_entry *aenq_e;
1749 struct ena_admin_aenq_common_desc *aenq_common;
1750 struct ena_com_aenq *aenq = &dev->aenq;
1751 ena_aenq_handler handler_cb;
1752 u16 masked_head, processed = 0;
1753 u8 phase;
1754
1755 masked_head = aenq->head & (aenq->q_depth - 1);
1756 phase = aenq->phase;
1757 aenq_e = &aenq->entries[masked_head]; /* Get first entry */
1758 aenq_common = &aenq_e->aenq_common_desc;
1759
1760 /* Go over all the events */
1761 while ((aenq_common->flags & ENA_ADMIN_AENQ_COMMON_DESC_PHASE_MASK) ==
1762 phase) {
1763 pr_debug("AENQ! Group[%x] Syndrom[%x] timestamp: [%llus]\n",
1764 aenq_common->group, aenq_common->syndrom,
1765 (u64)aenq_common->timestamp_low +
1766 ((u64)aenq_common->timestamp_high << 32));
1767
1768 /* Handle specific event*/
1769 handler_cb = ena_com_get_specific_aenq_cb(dev,
1770 aenq_common->group);
1771 handler_cb(data, aenq_e); /* call the actual event handler*/
1772
1773 /* Get next event entry */
1774 masked_head++;
1775 processed++;
1776
1777 if (unlikely(masked_head == aenq->q_depth)) {
1778 masked_head = 0;
1779 phase = !phase;
1780 }
1781 aenq_e = &aenq->entries[masked_head];
1782 aenq_common = &aenq_e->aenq_common_desc;
1783 }
1784
1785 aenq->head += processed;
1786 aenq->phase = phase;
1787
1788 /* Don't update aenq doorbell if there weren't any processed events */
1789 if (!processed)
1790 return;
1791
1792 /* write the aenq doorbell after all AENQ descriptors were read */
1793 mb();
1794 writel((u32)aenq->head, dev->reg_bar + ENA_REGS_AENQ_HEAD_DB_OFF);
1795 }
1796
1797 int ena_com_dev_reset(struct ena_com_dev *ena_dev)
1798 {
1799 u32 stat, timeout, cap, reset_val;
1800 int rc;
1801
1802 stat = ena_com_reg_bar_read32(ena_dev, ENA_REGS_DEV_STS_OFF);
1803 cap = ena_com_reg_bar_read32(ena_dev, ENA_REGS_CAPS_OFF);
1804
1805 if (unlikely((stat == ENA_MMIO_READ_TIMEOUT) ||
1806 (cap == ENA_MMIO_READ_TIMEOUT))) {
1807 pr_err("Reg read32 timeout occurred\n");
1808 return -ETIME;
1809 }
1810
1811 if ((stat & ENA_REGS_DEV_STS_READY_MASK) == 0) {
1812 pr_err("Device isn't ready, can't reset device\n");
1813 return -EINVAL;
1814 }
1815
1816 timeout = (cap & ENA_REGS_CAPS_RESET_TIMEOUT_MASK) >>
1817 ENA_REGS_CAPS_RESET_TIMEOUT_SHIFT;
1818 if (timeout == 0) {
1819 pr_err("Invalid timeout value\n");
1820 return -EINVAL;
1821 }
1822
1823 /* start reset */
1824 reset_val = ENA_REGS_DEV_CTL_DEV_RESET_MASK;
1825 writel(reset_val, ena_dev->reg_bar + ENA_REGS_DEV_CTL_OFF);
1826
1827 /* Write again the MMIO read request address */
1828 ena_com_mmio_reg_read_request_write_dev_addr(ena_dev);
1829
1830 rc = wait_for_reset_state(ena_dev, timeout,
1831 ENA_REGS_DEV_STS_RESET_IN_PROGRESS_MASK);
1832 if (rc != 0) {
1833 pr_err("Reset indication didn't turn on\n");
1834 return rc;
1835 }
1836
1837 /* reset done */
1838 writel(0, ena_dev->reg_bar + ENA_REGS_DEV_CTL_OFF);
1839 rc = wait_for_reset_state(ena_dev, timeout, 0);
1840 if (rc != 0) {
1841 pr_err("Reset indication didn't turn off\n");
1842 return rc;
1843 }
1844
1845 return 0;
1846 }
1847
1848 static int ena_get_dev_stats(struct ena_com_dev *ena_dev,
1849 struct ena_com_stats_ctx *ctx,
1850 enum ena_admin_get_stats_type type)
1851 {
1852 struct ena_admin_aq_get_stats_cmd *get_cmd = &ctx->get_cmd;
1853 struct ena_admin_acq_get_stats_resp *get_resp = &ctx->get_resp;
1854 struct ena_com_admin_queue *admin_queue;
1855 int ret;
1856
1857 admin_queue = &ena_dev->admin_queue;
1858
1859 get_cmd->aq_common_descriptor.opcode = ENA_ADMIN_GET_STATS;
1860 get_cmd->aq_common_descriptor.flags = 0;
1861 get_cmd->type = type;
1862
1863 ret = ena_com_execute_admin_command(admin_queue,
1864 (struct ena_admin_aq_entry *)get_cmd,
1865 sizeof(*get_cmd),
1866 (struct ena_admin_acq_entry *)get_resp,
1867 sizeof(*get_resp));
1868
1869 if (unlikely(ret))
1870 pr_err("Failed to get stats. error: %d\n", ret);
1871
1872 return ret;
1873 }
1874
1875 int ena_com_get_dev_basic_stats(struct ena_com_dev *ena_dev,
1876 struct ena_admin_basic_stats *stats)
1877 {
1878 struct ena_com_stats_ctx ctx;
1879 int ret;
1880
1881 memset(&ctx, 0x0, sizeof(ctx));
1882 ret = ena_get_dev_stats(ena_dev, &ctx, ENA_ADMIN_GET_STATS_TYPE_BASIC);
1883 if (likely(ret == 0))
1884 memcpy(stats, &ctx.get_resp.basic_stats,
1885 sizeof(ctx.get_resp.basic_stats));
1886
1887 return ret;
1888 }
1889
1890 int ena_com_set_dev_mtu(struct ena_com_dev *ena_dev, int mtu)
1891 {
1892 struct ena_com_admin_queue *admin_queue;
1893 struct ena_admin_set_feat_cmd cmd;
1894 struct ena_admin_set_feat_resp resp;
1895 int ret;
1896
1897 if (!ena_com_check_supported_feature_id(ena_dev, ENA_ADMIN_MTU)) {
1898 pr_info("Feature %d isn't supported\n", ENA_ADMIN_MTU);
1899 return -EPERM;
1900 }
1901
1902 memset(&cmd, 0x0, sizeof(cmd));
1903 admin_queue = &ena_dev->admin_queue;
1904
1905 cmd.aq_common_descriptor.opcode = ENA_ADMIN_SET_FEATURE;
1906 cmd.aq_common_descriptor.flags = 0;
1907 cmd.feat_common.feature_id = ENA_ADMIN_MTU;
1908 cmd.u.mtu.mtu = mtu;
1909
1910 ret = ena_com_execute_admin_command(admin_queue,
1911 (struct ena_admin_aq_entry *)&cmd,
1912 sizeof(cmd),
1913 (struct ena_admin_acq_entry *)&resp,
1914 sizeof(resp));
1915
1916 if (unlikely(ret))
1917 pr_err("Failed to set mtu %d. error: %d\n", mtu, ret);
1918
1919 return ret;
1920 }
1921
1922 int ena_com_get_offload_settings(struct ena_com_dev *ena_dev,
1923 struct ena_admin_feature_offload_desc *offload)
1924 {
1925 int ret;
1926 struct ena_admin_get_feat_resp resp;
1927
1928 ret = ena_com_get_feature(ena_dev, &resp,
1929 ENA_ADMIN_STATELESS_OFFLOAD_CONFIG);
1930 if (unlikely(ret)) {
1931 pr_err("Failed to get offload capabilities %d\n", ret);
1932 return ret;
1933 }
1934
1935 memcpy(offload, &resp.u.offload, sizeof(resp.u.offload));
1936
1937 return 0;
1938 }
1939
1940 int ena_com_set_hash_function(struct ena_com_dev *ena_dev)
1941 {
1942 struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;
1943 struct ena_rss *rss = &ena_dev->rss;
1944 struct ena_admin_set_feat_cmd cmd;
1945 struct ena_admin_set_feat_resp resp;
1946 struct ena_admin_get_feat_resp get_resp;
1947 int ret;
1948
1949 if (!ena_com_check_supported_feature_id(ena_dev,
1950 ENA_ADMIN_RSS_HASH_FUNCTION)) {
1951 pr_info("Feature %d isn't supported\n",
1952 ENA_ADMIN_RSS_HASH_FUNCTION);
1953 return -EPERM;
1954 }
1955
1956 /* Validate hash function is supported */
1957 ret = ena_com_get_feature(ena_dev, &get_resp,
1958 ENA_ADMIN_RSS_HASH_FUNCTION);
1959 if (unlikely(ret))
1960 return ret;
1961
1962 if (get_resp.u.flow_hash_func.supported_func & (1 << rss->hash_func)) {
1963 pr_err("Func hash %d isn't supported by device, abort\n",
1964 rss->hash_func);
1965 return -EPERM;
1966 }
1967
1968 memset(&cmd, 0x0, sizeof(cmd));
1969
1970 cmd.aq_common_descriptor.opcode = ENA_ADMIN_SET_FEATURE;
1971 cmd.aq_common_descriptor.flags =
1972 ENA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_INDIRECT_MASK;
1973 cmd.feat_common.feature_id = ENA_ADMIN_RSS_HASH_FUNCTION;
1974 cmd.u.flow_hash_func.init_val = rss->hash_init_val;
1975 cmd.u.flow_hash_func.selected_func = 1 << rss->hash_func;
1976
1977 ret = ena_com_mem_addr_set(ena_dev,
1978 &cmd.control_buffer.address,
1979 rss->hash_key_dma_addr);
1980 if (unlikely(ret)) {
1981 pr_err("memory address set failed\n");
1982 return ret;
1983 }
1984
1985 cmd.control_buffer.length = sizeof(*rss->hash_key);
1986
1987 ret = ena_com_execute_admin_command(admin_queue,
1988 (struct ena_admin_aq_entry *)&cmd,
1989 sizeof(cmd),
1990 (struct ena_admin_acq_entry *)&resp,
1991 sizeof(resp));
1992 if (unlikely(ret)) {
1993 pr_err("Failed to set hash function %d. error: %d\n",
1994 rss->hash_func, ret);
1995 return -EINVAL;
1996 }
1997
1998 return 0;
1999 }
2000
2001 int ena_com_fill_hash_function(struct ena_com_dev *ena_dev,
2002 enum ena_admin_hash_functions func,
2003 const u8 *key, u16 key_len, u32 init_val)
2004 {
2005 struct ena_rss *rss = &ena_dev->rss;
2006 struct ena_admin_get_feat_resp get_resp;
2007 struct ena_admin_feature_rss_flow_hash_control *hash_key =
2008 rss->hash_key;
2009 int rc;
2010
2011 /* Make sure size is a mult of DWs */
2012 if (unlikely(key_len & 0x3))
2013 return -EINVAL;
2014
2015 rc = ena_com_get_feature_ex(ena_dev, &get_resp,
2016 ENA_ADMIN_RSS_HASH_FUNCTION,
2017 rss->hash_key_dma_addr,
2018 sizeof(*rss->hash_key));
2019 if (unlikely(rc))
2020 return rc;
2021
2022 if (!((1 << func) & get_resp.u.flow_hash_func.supported_func)) {
2023 pr_err("Flow hash function %d isn't supported\n", func);
2024 return -EPERM;
2025 }
2026
2027 switch (func) {
2028 case ENA_ADMIN_TOEPLITZ:
2029 if (key_len > sizeof(hash_key->key)) {
2030 pr_err("key len (%hu) is bigger than the max supported (%zu)\n",
2031 key_len, sizeof(hash_key->key));
2032 return -EINVAL;
2033 }
2034
2035 memcpy(hash_key->key, key, key_len);
2036 rss->hash_init_val = init_val;
2037 hash_key->keys_num = key_len >> 2;
2038 break;
2039 case ENA_ADMIN_CRC32:
2040 rss->hash_init_val = init_val;
2041 break;
2042 default:
2043 pr_err("Invalid hash function (%d)\n", func);
2044 return -EINVAL;
2045 }
2046
2047 rc = ena_com_set_hash_function(ena_dev);
2048
2049 /* Restore the old function */
2050 if (unlikely(rc))
2051 ena_com_get_hash_function(ena_dev, NULL, NULL);
2052
2053 return rc;
2054 }
2055
2056 int ena_com_get_hash_function(struct ena_com_dev *ena_dev,
2057 enum ena_admin_hash_functions *func,
2058 u8 *key)
2059 {
2060 struct ena_rss *rss = &ena_dev->rss;
2061 struct ena_admin_get_feat_resp get_resp;
2062 struct ena_admin_feature_rss_flow_hash_control *hash_key =
2063 rss->hash_key;
2064 int rc;
2065
2066 rc = ena_com_get_feature_ex(ena_dev, &get_resp,
2067 ENA_ADMIN_RSS_HASH_FUNCTION,
2068 rss->hash_key_dma_addr,
2069 sizeof(*rss->hash_key));
2070 if (unlikely(rc))
2071 return rc;
2072
2073 rss->hash_func = get_resp.u.flow_hash_func.selected_func;
2074 if (func)
2075 *func = rss->hash_func;
2076
2077 if (key)
2078 memcpy(key, hash_key->key, (size_t)(hash_key->keys_num) << 2);
2079
2080 return 0;
2081 }
2082
2083 int ena_com_get_hash_ctrl(struct ena_com_dev *ena_dev,
2084 enum ena_admin_flow_hash_proto proto,
2085 u16 *fields)
2086 {
2087 struct ena_rss *rss = &ena_dev->rss;
2088 struct ena_admin_get_feat_resp get_resp;
2089 int rc;
2090
2091 rc = ena_com_get_feature_ex(ena_dev, &get_resp,
2092 ENA_ADMIN_RSS_HASH_INPUT,
2093 rss->hash_ctrl_dma_addr,
2094 sizeof(*rss->hash_ctrl));
2095 if (unlikely(rc))
2096 return rc;
2097
2098 if (fields)
2099 *fields = rss->hash_ctrl->selected_fields[proto].fields;
2100
2101 return 0;
2102 }
2103
2104 int ena_com_set_hash_ctrl(struct ena_com_dev *ena_dev)
2105 {
2106 struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;
2107 struct ena_rss *rss = &ena_dev->rss;
2108 struct ena_admin_feature_rss_hash_control *hash_ctrl = rss->hash_ctrl;
2109 struct ena_admin_set_feat_cmd cmd;
2110 struct ena_admin_set_feat_resp resp;
2111 int ret;
2112
2113 if (!ena_com_check_supported_feature_id(ena_dev,
2114 ENA_ADMIN_RSS_HASH_INPUT)) {
2115 pr_info("Feature %d isn't supported\n", ENA_ADMIN_RSS_HASH_INPUT);
2116 return -EPERM;
2117 }
2118
2119 memset(&cmd, 0x0, sizeof(cmd));
2120
2121 cmd.aq_common_descriptor.opcode = ENA_ADMIN_SET_FEATURE;
2122 cmd.aq_common_descriptor.flags =
2123 ENA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_INDIRECT_MASK;
2124 cmd.feat_common.feature_id = ENA_ADMIN_RSS_HASH_INPUT;
2125 cmd.u.flow_hash_input.enabled_input_sort =
2126 ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_L3_SORT_MASK |
2127 ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_L4_SORT_MASK;
2128
2129 ret = ena_com_mem_addr_set(ena_dev,
2130 &cmd.control_buffer.address,
2131 rss->hash_ctrl_dma_addr);
2132 if (unlikely(ret)) {
2133 pr_err("memory address set failed\n");
2134 return ret;
2135 }
2136 cmd.control_buffer.length = sizeof(*hash_ctrl);
2137
2138 ret = ena_com_execute_admin_command(admin_queue,
2139 (struct ena_admin_aq_entry *)&cmd,
2140 sizeof(cmd),
2141 (struct ena_admin_acq_entry *)&resp,
2142 sizeof(resp));
2143 if (unlikely(ret))
2144 pr_err("Failed to set hash input. error: %d\n", ret);
2145
2146 return ret;
2147 }
2148
2149 int ena_com_set_default_hash_ctrl(struct ena_com_dev *ena_dev)
2150 {
2151 struct ena_rss *rss = &ena_dev->rss;
2152 struct ena_admin_feature_rss_hash_control *hash_ctrl =
2153 rss->hash_ctrl;
2154 u16 available_fields = 0;
2155 int rc, i;
2156
2157 /* Get the supported hash input */
2158 rc = ena_com_get_hash_ctrl(ena_dev, 0, NULL);
2159 if (unlikely(rc))
2160 return rc;
2161
2162 hash_ctrl->selected_fields[ENA_ADMIN_RSS_TCP4].fields =
2163 ENA_ADMIN_RSS_L3_SA | ENA_ADMIN_RSS_L3_DA |
2164 ENA_ADMIN_RSS_L4_DP | ENA_ADMIN_RSS_L4_SP;
2165
2166 hash_ctrl->selected_fields[ENA_ADMIN_RSS_UDP4].fields =
2167 ENA_ADMIN_RSS_L3_SA | ENA_ADMIN_RSS_L3_DA |
2168 ENA_ADMIN_RSS_L4_DP | ENA_ADMIN_RSS_L4_SP;
2169
2170 hash_ctrl->selected_fields[ENA_ADMIN_RSS_TCP6].fields =
2171 ENA_ADMIN_RSS_L3_SA | ENA_ADMIN_RSS_L3_DA |
2172 ENA_ADMIN_RSS_L4_DP | ENA_ADMIN_RSS_L4_SP;
2173
2174 hash_ctrl->selected_fields[ENA_ADMIN_RSS_UDP6].fields =
2175 ENA_ADMIN_RSS_L3_SA | ENA_ADMIN_RSS_L3_DA |
2176 ENA_ADMIN_RSS_L4_DP | ENA_ADMIN_RSS_L4_SP;
2177
2178 hash_ctrl->selected_fields[ENA_ADMIN_RSS_IP4].fields =
2179 ENA_ADMIN_RSS_L3_SA | ENA_ADMIN_RSS_L3_DA;
2180
2181 hash_ctrl->selected_fields[ENA_ADMIN_RSS_IP6].fields =
2182 ENA_ADMIN_RSS_L3_SA | ENA_ADMIN_RSS_L3_DA;
2183
2184 hash_ctrl->selected_fields[ENA_ADMIN_RSS_IP4_FRAG].fields =
2185 ENA_ADMIN_RSS_L3_SA | ENA_ADMIN_RSS_L3_DA;
2186
2187 hash_ctrl->selected_fields[ENA_ADMIN_RSS_IP4_FRAG].fields =
2188 ENA_ADMIN_RSS_L2_DA | ENA_ADMIN_RSS_L2_SA;
2189
2190 for (i = 0; i < ENA_ADMIN_RSS_PROTO_NUM; i++) {
2191 available_fields = hash_ctrl->selected_fields[i].fields &
2192 hash_ctrl->supported_fields[i].fields;
2193 if (available_fields != hash_ctrl->selected_fields[i].fields) {
2194 pr_err("hash control doesn't support all the desire configuration. proto %x supported %x selected %x\n",
2195 i, hash_ctrl->supported_fields[i].fields,
2196 hash_ctrl->selected_fields[i].fields);
2197 return -EPERM;
2198 }
2199 }
2200
2201 rc = ena_com_set_hash_ctrl(ena_dev);
2202
2203 /* In case of failure, restore the old hash ctrl */
2204 if (unlikely(rc))
2205 ena_com_get_hash_ctrl(ena_dev, 0, NULL);
2206
2207 return rc;
2208 }
2209
2210 int ena_com_fill_hash_ctrl(struct ena_com_dev *ena_dev,
2211 enum ena_admin_flow_hash_proto proto,
2212 u16 hash_fields)
2213 {
2214 struct ena_rss *rss = &ena_dev->rss;
2215 struct ena_admin_feature_rss_hash_control *hash_ctrl = rss->hash_ctrl;
2216 u16 supported_fields;
2217 int rc;
2218
2219 if (proto >= ENA_ADMIN_RSS_PROTO_NUM) {
2220 pr_err("Invalid proto num (%u)\n", proto);
2221 return -EINVAL;
2222 }
2223
2224 /* Get the ctrl table */
2225 rc = ena_com_get_hash_ctrl(ena_dev, proto, NULL);
2226 if (unlikely(rc))
2227 return rc;
2228
2229 /* Make sure all the fields are supported */
2230 supported_fields = hash_ctrl->supported_fields[proto].fields;
2231 if ((hash_fields & supported_fields) != hash_fields) {
2232 pr_err("proto %d doesn't support the required fields %x. supports only: %x\n",
2233 proto, hash_fields, supported_fields);
2234 }
2235
2236 hash_ctrl->selected_fields[proto].fields = hash_fields;
2237
2238 rc = ena_com_set_hash_ctrl(ena_dev);
2239
2240 /* In case of failure, restore the old hash ctrl */
2241 if (unlikely(rc))
2242 ena_com_get_hash_ctrl(ena_dev, 0, NULL);
2243
2244 return 0;
2245 }
2246
2247 int ena_com_indirect_table_fill_entry(struct ena_com_dev *ena_dev,
2248 u16 entry_idx, u16 entry_value)
2249 {
2250 struct ena_rss *rss = &ena_dev->rss;
2251
2252 if (unlikely(entry_idx >= (1 << rss->tbl_log_size)))
2253 return -EINVAL;
2254
2255 if (unlikely((entry_value > ENA_TOTAL_NUM_QUEUES)))
2256 return -EINVAL;
2257
2258 rss->host_rss_ind_tbl[entry_idx] = entry_value;
2259
2260 return 0;
2261 }
2262
2263 int ena_com_indirect_table_set(struct ena_com_dev *ena_dev)
2264 {
2265 struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;
2266 struct ena_rss *rss = &ena_dev->rss;
2267 struct ena_admin_set_feat_cmd cmd;
2268 struct ena_admin_set_feat_resp resp;
2269 int ret;
2270
2271 if (!ena_com_check_supported_feature_id(
2272 ena_dev, ENA_ADMIN_RSS_REDIRECTION_TABLE_CONFIG)) {
2273 pr_info("Feature %d isn't supported\n",
2274 ENA_ADMIN_RSS_REDIRECTION_TABLE_CONFIG);
2275 return -EPERM;
2276 }
2277
2278 ret = ena_com_ind_tbl_convert_to_device(ena_dev);
2279 if (ret) {
2280 pr_err("Failed to convert host indirection table to device table\n");
2281 return ret;
2282 }
2283
2284 memset(&cmd, 0x0, sizeof(cmd));
2285
2286 cmd.aq_common_descriptor.opcode = ENA_ADMIN_SET_FEATURE;
2287 cmd.aq_common_descriptor.flags =
2288 ENA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_INDIRECT_MASK;
2289 cmd.feat_common.feature_id = ENA_ADMIN_RSS_REDIRECTION_TABLE_CONFIG;
2290 cmd.u.ind_table.size = rss->tbl_log_size;
2291 cmd.u.ind_table.inline_index = 0xFFFFFFFF;
2292
2293 ret = ena_com_mem_addr_set(ena_dev,
2294 &cmd.control_buffer.address,
2295 rss->rss_ind_tbl_dma_addr);
2296 if (unlikely(ret)) {
2297 pr_err("memory address set failed\n");
2298 return ret;
2299 }
2300
2301 cmd.control_buffer.length = (1ULL << rss->tbl_log_size) *
2302 sizeof(struct ena_admin_rss_ind_table_entry);
2303
2304 ret = ena_com_execute_admin_command(admin_queue,
2305 (struct ena_admin_aq_entry *)&cmd,
2306 sizeof(cmd),
2307 (struct ena_admin_acq_entry *)&resp,
2308 sizeof(resp));
2309
2310 if (unlikely(ret))
2311 pr_err("Failed to set indirect table. error: %d\n", ret);
2312
2313 return ret;
2314 }
2315
2316 int ena_com_indirect_table_get(struct ena_com_dev *ena_dev, u32 *ind_tbl)
2317 {
2318 struct ena_rss *rss = &ena_dev->rss;
2319 struct ena_admin_get_feat_resp get_resp;
2320 u32 tbl_size;
2321 int i, rc;
2322
2323 tbl_size = (1ULL << rss->tbl_log_size) *
2324 sizeof(struct ena_admin_rss_ind_table_entry);
2325
2326 rc = ena_com_get_feature_ex(ena_dev, &get_resp,
2327 ENA_ADMIN_RSS_REDIRECTION_TABLE_CONFIG,
2328 rss->rss_ind_tbl_dma_addr,
2329 tbl_size);
2330 if (unlikely(rc))
2331 return rc;
2332
2333 if (!ind_tbl)
2334 return 0;
2335
2336 rc = ena_com_ind_tbl_convert_from_device(ena_dev);
2337 if (unlikely(rc))
2338 return rc;
2339
2340 for (i = 0; i < (1 << rss->tbl_log_size); i++)
2341 ind_tbl[i] = rss->host_rss_ind_tbl[i];
2342
2343 return 0;
2344 }
2345
2346 int ena_com_rss_init(struct ena_com_dev *ena_dev, u16 indr_tbl_log_size)
2347 {
2348 int rc;
2349
2350 memset(&ena_dev->rss, 0x0, sizeof(ena_dev->rss));
2351
2352 rc = ena_com_indirect_table_allocate(ena_dev, indr_tbl_log_size);
2353 if (unlikely(rc))
2354 goto err_indr_tbl;
2355
2356 rc = ena_com_hash_key_allocate(ena_dev);
2357 if (unlikely(rc))
2358 goto err_hash_key;
2359
2360 rc = ena_com_hash_ctrl_init(ena_dev);
2361 if (unlikely(rc))
2362 goto err_hash_ctrl;
2363
2364 return 0;
2365
2366 err_hash_ctrl:
2367 ena_com_hash_key_destroy(ena_dev);
2368 err_hash_key:
2369 ena_com_indirect_table_destroy(ena_dev);
2370 err_indr_tbl:
2371
2372 return rc;
2373 }
2374
2375 void ena_com_rss_destroy(struct ena_com_dev *ena_dev)
2376 {
2377 ena_com_indirect_table_destroy(ena_dev);
2378 ena_com_hash_key_destroy(ena_dev);
2379 ena_com_hash_ctrl_destroy(ena_dev);
2380
2381 memset(&ena_dev->rss, 0x0, sizeof(ena_dev->rss));
2382 }
2383
2384 int ena_com_allocate_host_info(struct ena_com_dev *ena_dev)
2385 {
2386 struct ena_host_attribute *host_attr = &ena_dev->host_attr;
2387
2388 host_attr->host_info =
2389 dma_zalloc_coherent(ena_dev->dmadev, SZ_4K,
2390 &host_attr->host_info_dma_addr, GFP_KERNEL);
2391 if (unlikely(!host_attr->host_info))
2392 return -ENOMEM;
2393
2394 return 0;
2395 }
2396
2397 int ena_com_allocate_debug_area(struct ena_com_dev *ena_dev,
2398 u32 debug_area_size)
2399 {
2400 struct ena_host_attribute *host_attr = &ena_dev->host_attr;
2401
2402 host_attr->debug_area_virt_addr =
2403 dma_zalloc_coherent(ena_dev->dmadev, debug_area_size,
2404 &host_attr->debug_area_dma_addr, GFP_KERNEL);
2405 if (unlikely(!host_attr->debug_area_virt_addr)) {
2406 host_attr->debug_area_size = 0;
2407 return -ENOMEM;
2408 }
2409
2410 host_attr->debug_area_size = debug_area_size;
2411
2412 return 0;
2413 }
2414
2415 void ena_com_delete_host_info(struct ena_com_dev *ena_dev)
2416 {
2417 struct ena_host_attribute *host_attr = &ena_dev->host_attr;
2418
2419 if (host_attr->host_info) {
2420 dma_free_coherent(ena_dev->dmadev, SZ_4K, host_attr->host_info,
2421 host_attr->host_info_dma_addr);
2422 host_attr->host_info = NULL;
2423 }
2424 }
2425
2426 void ena_com_delete_debug_area(struct ena_com_dev *ena_dev)
2427 {
2428 struct ena_host_attribute *host_attr = &ena_dev->host_attr;
2429
2430 if (host_attr->debug_area_virt_addr) {
2431 dma_free_coherent(ena_dev->dmadev, host_attr->debug_area_size,
2432 host_attr->debug_area_virt_addr,
2433 host_attr->debug_area_dma_addr);
2434 host_attr->debug_area_virt_addr = NULL;
2435 }
2436 }
2437
2438 int ena_com_set_host_attributes(struct ena_com_dev *ena_dev)
2439 {
2440 struct ena_host_attribute *host_attr = &ena_dev->host_attr;
2441 struct ena_com_admin_queue *admin_queue;
2442 struct ena_admin_set_feat_cmd cmd;
2443 struct ena_admin_set_feat_resp resp;
2444
2445 int ret;
2446
2447 if (!ena_com_check_supported_feature_id(ena_dev,
2448 ENA_ADMIN_HOST_ATTR_CONFIG)) {
2449 pr_warn("Set host attribute isn't supported\n");
2450 return -EPERM;
2451 }
2452
2453 memset(&cmd, 0x0, sizeof(cmd));
2454 admin_queue = &ena_dev->admin_queue;
2455
2456 cmd.aq_common_descriptor.opcode = ENA_ADMIN_SET_FEATURE;
2457 cmd.feat_common.feature_id = ENA_ADMIN_HOST_ATTR_CONFIG;
2458
2459 ret = ena_com_mem_addr_set(ena_dev,
2460 &cmd.u.host_attr.debug_ba,
2461 host_attr->debug_area_dma_addr);
2462 if (unlikely(ret)) {
2463 pr_err("memory address set failed\n");
2464 return ret;
2465 }
2466
2467 ret = ena_com_mem_addr_set(ena_dev,
2468 &cmd.u.host_attr.os_info_ba,
2469 host_attr->host_info_dma_addr);
2470 if (unlikely(ret)) {
2471 pr_err("memory address set failed\n");
2472 return ret;
2473 }
2474
2475 cmd.u.host_attr.debug_area_size = host_attr->debug_area_size;
2476
2477 ret = ena_com_execute_admin_command(admin_queue,
2478 (struct ena_admin_aq_entry *)&cmd,
2479 sizeof(cmd),
2480 (struct ena_admin_acq_entry *)&resp,
2481 sizeof(resp));
2482
2483 if (unlikely(ret))
2484 pr_err("Failed to set host attributes: %d\n", ret);
2485
2486 return ret;
2487 }
2488
2489 /* Interrupt moderation */
2490 bool ena_com_interrupt_moderation_supported(struct ena_com_dev *ena_dev)
2491 {
2492 return ena_com_check_supported_feature_id(ena_dev,
2493 ENA_ADMIN_INTERRUPT_MODERATION);
2494 }
2495
2496 int ena_com_update_nonadaptive_moderation_interval_tx(struct ena_com_dev *ena_dev,
2497 u32 tx_coalesce_usecs)
2498 {
2499 if (!ena_dev->intr_delay_resolution) {
2500 pr_err("Illegal interrupt delay granularity value\n");
2501 return -EFAULT;
2502 }
2503
2504 ena_dev->intr_moder_tx_interval = tx_coalesce_usecs /
2505 ena_dev->intr_delay_resolution;
2506
2507 return 0;
2508 }
2509
2510 int ena_com_update_nonadaptive_moderation_interval_rx(struct ena_com_dev *ena_dev,
2511 u32 rx_coalesce_usecs)
2512 {
2513 if (!ena_dev->intr_delay_resolution) {
2514 pr_err("Illegal interrupt delay granularity value\n");
2515 return -EFAULT;
2516 }
2517
2518 /* We use LOWEST entry of moderation table for storing
2519 * nonadaptive interrupt coalescing values
2520 */
2521 ena_dev->intr_moder_tbl[ENA_INTR_MODER_LOWEST].intr_moder_interval =
2522 rx_coalesce_usecs / ena_dev->intr_delay_resolution;
2523
2524 return 0;
2525 }
2526
2527 void ena_com_destroy_interrupt_moderation(struct ena_com_dev *ena_dev)
2528 {
2529 if (ena_dev->intr_moder_tbl)
2530 devm_kfree(ena_dev->dmadev, ena_dev->intr_moder_tbl);
2531 ena_dev->intr_moder_tbl = NULL;
2532 }
2533
2534 int ena_com_init_interrupt_moderation(struct ena_com_dev *ena_dev)
2535 {
2536 struct ena_admin_get_feat_resp get_resp;
2537 u16 delay_resolution;
2538 int rc;
2539
2540 rc = ena_com_get_feature(ena_dev, &get_resp,
2541 ENA_ADMIN_INTERRUPT_MODERATION);
2542
2543 if (rc) {
2544 if (rc == -EPERM) {
2545 pr_info("Feature %d isn't supported\n",
2546 ENA_ADMIN_INTERRUPT_MODERATION);
2547 rc = 0;
2548 } else {
2549 pr_err("Failed to get interrupt moderation admin cmd. rc: %d\n",
2550 rc);
2551 }
2552
2553 /* no moderation supported, disable adaptive support */
2554 ena_com_disable_adaptive_moderation(ena_dev);
2555 return rc;
2556 }
2557
2558 rc = ena_com_init_interrupt_moderation_table(ena_dev);
2559 if (rc)
2560 goto err;
2561
2562 /* if moderation is supported by device we set adaptive moderation */
2563 delay_resolution = get_resp.u.intr_moderation.intr_delay_resolution;
2564 ena_com_update_intr_delay_resolution(ena_dev, delay_resolution);
2565 ena_com_enable_adaptive_moderation(ena_dev);
2566
2567 return 0;
2568 err:
2569 ena_com_destroy_interrupt_moderation(ena_dev);
2570 return rc;
2571 }
2572
2573 void ena_com_config_default_interrupt_moderation_table(struct ena_com_dev *ena_dev)
2574 {
2575 struct ena_intr_moder_entry *intr_moder_tbl = ena_dev->intr_moder_tbl;
2576
2577 if (!intr_moder_tbl)
2578 return;
2579
2580 intr_moder_tbl[ENA_INTR_MODER_LOWEST].intr_moder_interval =
2581 ENA_INTR_LOWEST_USECS;
2582 intr_moder_tbl[ENA_INTR_MODER_LOWEST].pkts_per_interval =
2583 ENA_INTR_LOWEST_PKTS;
2584 intr_moder_tbl[ENA_INTR_MODER_LOWEST].bytes_per_interval =
2585 ENA_INTR_LOWEST_BYTES;
2586
2587 intr_moder_tbl[ENA_INTR_MODER_LOW].intr_moder_interval =
2588 ENA_INTR_LOW_USECS;
2589 intr_moder_tbl[ENA_INTR_MODER_LOW].pkts_per_interval =
2590 ENA_INTR_LOW_PKTS;
2591 intr_moder_tbl[ENA_INTR_MODER_LOW].bytes_per_interval =
2592 ENA_INTR_LOW_BYTES;
2593
2594 intr_moder_tbl[ENA_INTR_MODER_MID].intr_moder_interval =
2595 ENA_INTR_MID_USECS;
2596 intr_moder_tbl[ENA_INTR_MODER_MID].pkts_per_interval =
2597 ENA_INTR_MID_PKTS;
2598 intr_moder_tbl[ENA_INTR_MODER_MID].bytes_per_interval =
2599 ENA_INTR_MID_BYTES;
2600
2601 intr_moder_tbl[ENA_INTR_MODER_HIGH].intr_moder_interval =
2602 ENA_INTR_HIGH_USECS;
2603 intr_moder_tbl[ENA_INTR_MODER_HIGH].pkts_per_interval =
2604 ENA_INTR_HIGH_PKTS;
2605 intr_moder_tbl[ENA_INTR_MODER_HIGH].bytes_per_interval =
2606 ENA_INTR_HIGH_BYTES;
2607
2608 intr_moder_tbl[ENA_INTR_MODER_HIGHEST].intr_moder_interval =
2609 ENA_INTR_HIGHEST_USECS;
2610 intr_moder_tbl[ENA_INTR_MODER_HIGHEST].pkts_per_interval =
2611 ENA_INTR_HIGHEST_PKTS;
2612 intr_moder_tbl[ENA_INTR_MODER_HIGHEST].bytes_per_interval =
2613 ENA_INTR_HIGHEST_BYTES;
2614 }
2615
2616 unsigned int ena_com_get_nonadaptive_moderation_interval_tx(struct ena_com_dev *ena_dev)
2617 {
2618 return ena_dev->intr_moder_tx_interval;
2619 }
2620
2621 unsigned int ena_com_get_nonadaptive_moderation_interval_rx(struct ena_com_dev *ena_dev)
2622 {
2623 struct ena_intr_moder_entry *intr_moder_tbl = ena_dev->intr_moder_tbl;
2624
2625 if (intr_moder_tbl)
2626 return intr_moder_tbl[ENA_INTR_MODER_LOWEST].intr_moder_interval;
2627
2628 return 0;
2629 }
2630
2631 void ena_com_init_intr_moderation_entry(struct ena_com_dev *ena_dev,
2632 enum ena_intr_moder_level level,
2633 struct ena_intr_moder_entry *entry)
2634 {
2635 struct ena_intr_moder_entry *intr_moder_tbl = ena_dev->intr_moder_tbl;
2636
2637 if (level >= ENA_INTR_MAX_NUM_OF_LEVELS)
2638 return;
2639
2640 intr_moder_tbl[level].intr_moder_interval = entry->intr_moder_interval;
2641 if (ena_dev->intr_delay_resolution)
2642 intr_moder_tbl[level].intr_moder_interval /=
2643 ena_dev->intr_delay_resolution;
2644 intr_moder_tbl[level].pkts_per_interval = entry->pkts_per_interval;
2645
2646 /* use hardcoded value until ethtool supports bytecount parameter */
2647 if (entry->bytes_per_interval != ENA_INTR_BYTE_COUNT_NOT_SUPPORTED)
2648 intr_moder_tbl[level].bytes_per_interval = entry->bytes_per_interval;
2649 }
2650
2651 void ena_com_get_intr_moderation_entry(struct ena_com_dev *ena_dev,
2652 enum ena_intr_moder_level level,
2653 struct ena_intr_moder_entry *entry)
2654 {
2655 struct ena_intr_moder_entry *intr_moder_tbl = ena_dev->intr_moder_tbl;
2656
2657 if (level >= ENA_INTR_MAX_NUM_OF_LEVELS)
2658 return;
2659
2660 entry->intr_moder_interval = intr_moder_tbl[level].intr_moder_interval;
2661 if (ena_dev->intr_delay_resolution)
2662 entry->intr_moder_interval *= ena_dev->intr_delay_resolution;
2663 entry->pkts_per_interval =
2664 intr_moder_tbl[level].pkts_per_interval;
2665 entry->bytes_per_interval = intr_moder_tbl[level].bytes_per_interval;
2666 }
ubuntu-zesty-kernel mirror