import 15.2.0 Octopus source

[ceph.git] / ceph / src / seastar / dpdk / drivers / net / ena / base / ena_eth_com.h

diff --git a/ceph/src/seastar/dpdk/drivers/net/ena/base/ena_eth_com.h b/ceph/src/seastar/dpdk/drivers/net/ena/base/ena_eth_com.h
index 71a880c0f16a3ed76e69a88b416c0c17cab5de4a..79acbf31d041e5d32ab276573d9bce1966ec8440 100644 (file)
--- a/ceph/src/seastar/dpdk/drivers/net/ena/base/ena_eth_com.h
+++ b/ceph/src/seastar/dpdk/drivers/net/ena/base/ena_eth_com.h
@@ -71,6 +71,7 @@ struct ena_com_rx_ctx {
        enum ena_eth_io_l4_proto_index l4_proto;
        bool l3_csum_err;
        bool l4_csum_err;
+       bool l4_csum_checked;
        /* fragmented packet */
        bool frag;
        u32 hash;
@@ -78,6 +79,9 @@ struct ena_com_rx_ctx {
        int max_bufs;
 };
 
+bool ena_com_is_doorbell_needed(struct ena_com_io_sq *io_sq,
+                               struct ena_com_tx_ctx *ena_tx_ctx);
+
 int ena_com_prepare_tx(struct ena_com_io_sq *io_sq,
                       struct ena_com_tx_ctx *ena_tx_ctx,
                       int *nb_hw_desc);
@@ -90,15 +94,15 @@ int ena_com_add_single_rx_desc(struct ena_com_io_sq *io_sq,
                               struct ena_com_buf *ena_buf,
                               u16 req_id);
 
-int ena_com_tx_comp_req_id_get(struct ena_com_io_cq *io_cq, u16 *req_id);
+bool ena_com_cq_empty(struct ena_com_io_cq *io_cq);
 
 static inline void ena_com_unmask_intr(struct ena_com_io_cq *io_cq,
                                       struct ena_eth_io_intr_reg *intr_reg)
 {
-       ENA_REG_WRITE32(intr_reg->intr_control, io_cq->unmask_reg);
+       ENA_REG_WRITE32(io_cq->bus, intr_reg->intr_control, io_cq->unmask_reg);
 }
 
-static inline int ena_com_sq_empty_space(struct ena_com_io_sq *io_sq)
+static inline int ena_com_free_desc(struct ena_com_io_sq *io_sq)
 {
        u16 tail, next_to_comp, cnt;
 
@@ -109,16 +113,46 @@ static inline int ena_com_sq_empty_space(struct ena_com_io_sq *io_sq)
        return io_sq->q_depth - 1 - cnt;
 }
 
-static inline int ena_com_write_sq_doorbell(struct ena_com_io_sq *io_sq)
+/* Check if the submission queue has enough space to hold required_buffers */
+static inline bool ena_com_sq_have_enough_space(struct ena_com_io_sq *io_sq,
+                                               u16 required_buffers)
 {
-       u16 tail;
+       int temp;
 
-       tail = io_sq->tail;
+       if (io_sq->mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_HOST)
+               return ena_com_free_desc(io_sq) >= required_buffers;
+
+       /* This calculation doesn't need to be 100% accurate. So to reduce
+        * the calculation overhead just Subtract 2 lines from the free descs
+        * (one for the header line and one to compensate the devision
+        * down calculation.
+        */
+       temp = required_buffers / io_sq->llq_info.descs_per_entry + 2;
+
+       return ena_com_free_desc(io_sq) > temp;
+}
+
+static inline bool is_llq_max_tx_burst_exists(struct ena_com_io_sq *io_sq)
+{
+       return (io_sq->mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) &&
+              io_sq->llq_info.max_entries_in_tx_burst > 0;
+}
+
+static inline int ena_com_write_sq_doorbell(struct ena_com_io_sq *io_sq)
+{
+       u16 tail = io_sq->tail;
+       u16 max_entries_in_tx_burst = io_sq->llq_info.max_entries_in_tx_burst;
 
        ena_trc_dbg("write submission queue doorbell for queue: %d tail: %d\n",
                    io_sq->qid, tail);
 
-       ENA_REG_WRITE32(tail, io_sq->db_addr);
+       ENA_REG_WRITE32(io_sq->bus, tail, io_sq->db_addr);
+
+       if (is_llq_max_tx_burst_exists(io_sq)) {
+               ena_trc_dbg("reset available entries in tx burst for queue %d to %d\n",
+                            io_sq->qid, max_entries_in_tx_burst);
+               io_sq->entries_in_tx_burst_left = max_entries_in_tx_burst;
+       }
 
        return 0;
 }
@@ -135,7 +169,7 @@ static inline int ena_com_update_dev_comp_head(struct ena_com_io_cq *io_cq)
        if (io_cq->cq_head_db_reg && need_update) {
                ena_trc_dbg("Write completion queue doorbell for queue %d: head: %d\n",
                            io_cq->qid, head);
-               ENA_REG_WRITE32(head, io_cq->cq_head_db_reg);
+               ENA_REG_WRITE32(io_cq->bus, head, io_cq->cq_head_db_reg);
                io_cq->last_head_update = head;
        }
 
@@ -153,7 +187,7 @@ static inline void ena_com_update_numa_node(struct ena_com_io_cq *io_cq,
        numa_cfg.numa_cfg = (numa_node & ENA_ETH_IO_NUMA_NODE_CFG_REG_NUMA_MASK)
                | ENA_ETH_IO_NUMA_NODE_CFG_REG_ENABLED_MASK;
 
-       ENA_REG_WRITE32(numa_cfg.numa_cfg, io_cq->numa_node_cfg_reg);
+       ENA_REG_WRITE32(io_cq->bus, numa_cfg.numa_cfg, io_cq->numa_node_cfg_reg);
 }
 
 static inline void ena_com_comp_ack(struct ena_com_io_sq *io_sq, u16 elem)
@@ -161,6 +195,49 @@ static inline void ena_com_comp_ack(struct ena_com_io_sq *io_sq, u16 elem)
        io_sq->next_to_comp += elem;
 }
 
+static inline void ena_com_cq_inc_head(struct ena_com_io_cq *io_cq)
+{
+       io_cq->head++;
+
+       /* Switch phase bit in case of wrap around */
+       if (unlikely((io_cq->head & (io_cq->q_depth - 1)) == 0))
+               io_cq->phase ^= 1;
+}
+
+static inline int ena_com_tx_comp_req_id_get(struct ena_com_io_cq *io_cq, u16 *req_id)
+{
+       u8 expected_phase, cdesc_phase;
+       struct ena_eth_io_tx_cdesc *cdesc;
+       u16 masked_head;
+
+       masked_head = io_cq->head & (io_cq->q_depth - 1);
+       expected_phase = io_cq->phase;
+
+       cdesc = (struct ena_eth_io_tx_cdesc *)
+               ((uintptr_t)io_cq->cdesc_addr.virt_addr +
+               (masked_head * io_cq->cdesc_entry_size_in_bytes));
+
+       /* When the current completion descriptor phase isn't the same as the
+        * expected, it mean that the device still didn't update
+        * this completion.
+        */
+       cdesc_phase = READ_ONCE16(cdesc->flags) & ENA_ETH_IO_TX_CDESC_PHASE_MASK;
+       if (cdesc_phase != expected_phase)
+               return ENA_COM_TRY_AGAIN;
+
+       dma_rmb();
+
+       *req_id = READ_ONCE16(cdesc->req_id);
+       if (unlikely(*req_id >= io_cq->q_depth)) {
+               ena_trc_err("Invalid req id %d\n", cdesc->req_id);
+               return ENA_COM_INVAL;
+       }
+
+       ena_com_cq_inc_head(io_cq);
+
+       return 0;
+}
+
 #if defined(__cplusplus)
 }
 #endif