1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2017 Cavium, Inc
11 #include <rte_spinlock.h>
12 #include <rte_memory.h>
14 #include "lio_struct.h"
17 #define ROUNDUP4(val) (((val) + 3) & 0xfffffffc)
20 #define LIO_STQUEUE_FIRST_ENTRY(ptr, type, elem) \
21 (type *)((char *)((ptr)->stqh_first) - offsetof(type, elem))
23 #define lio_check_timeout(cur_time, chk_time) ((cur_time) > (chk_time))
26 (size_t)(rte_get_timer_cycles() / rte_get_timer_hz())
28 /** Descriptor format.
29 * The descriptor ring is made of descriptors which have 2 64-bit values:
30 * -# Physical (bus) address of the data buffer.
31 * -# Physical (bus) address of a lio_droq_info structure.
32 * The device DMA's incoming packets and its information at the address
33 * given by these descriptor fields.
35 struct lio_droq_desc
{
36 /** The buffer pointer */
39 /** The Info pointer */
43 #define LIO_DROQ_DESC_SIZE (sizeof(struct lio_droq_desc))
45 /** Information about packet DMA'ed by Octeon.
46 * The format of the information available at Info Pointer after Octeon
47 * has posted a packet. Not all descriptors have valid information. Only
48 * the Info field of the first descriptor for a packet has information
51 struct lio_droq_info
{
52 /** The Output Receive Header. */
55 /** The Length of the packet. */
59 #define LIO_DROQ_INFO_SIZE (sizeof(struct lio_droq_info))
61 /** Pointer to data buffer.
62 * Driver keeps a pointer to the data buffer that it made available to
63 * the Octeon device. Since the descriptor ring keeps physical (bus)
64 * addresses, this field is required for the driver to keep track of
65 * the virtual address pointers.
67 struct lio_recv_buffer
{
68 /** Packet buffer, including meta data. */
71 /** Data in the packet buffer. */
76 #define LIO_DROQ_RECVBUF_SIZE (sizeof(struct lio_recv_buffer))
78 #define LIO_DROQ_SIZE (sizeof(struct lio_droq))
80 #define LIO_IQ_SEND_OK 0
81 #define LIO_IQ_SEND_STOP 1
82 #define LIO_IQ_SEND_FAILED -1
85 #define LIO_REQTYPE_NONE 0
86 #define LIO_REQTYPE_NORESP_NET 1
87 #define LIO_REQTYPE_NORESP_NET_SG 2
88 #define LIO_REQTYPE_SOFT_COMMAND 3
90 struct lio_request_list
{
95 /*---------------------- INSTRUCTION FORMAT ----------------------------*/
97 struct lio_instr3_64B
{
98 /** Pointer where the input data is available. */
101 /** Instruction Header. */
104 /** Instruction Header. */
107 /** Input Request Header. */
110 /** opcode/subcode specific parameters */
113 /** Return Data Parameters */
116 /** Pointer where the response for a RAW mode packet will be written
123 union lio_instr_64B
{
124 struct lio_instr3_64B cmd3
;
127 /** The size of each buffer in soft command buffer pool */
128 #define LIO_SOFT_COMMAND_BUFFER_SIZE 1536
130 /** Maximum number of buffers to allocate into soft command buffer pool */
131 #define LIO_MAX_SOFT_COMMAND_BUFFERS 255
133 struct lio_soft_command
{
134 /** Soft command buffer info. */
135 struct lio_stailq_node node
;
139 /** Command and return status */
140 union lio_instr_64B cmd
;
142 #define LIO_COMPLETION_WORD_INIT 0xffffffffffffffffULL
143 uint64_t *status_word
;
145 /** Data buffer info */
150 /** Return buffer info */
155 /** Context buffer info */
159 /** Time out and callback */
163 void (*callback
)(uint32_t, void *);
165 struct rte_mbuf
*mbuf
;
168 struct lio_iq_post_status
{
194 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
197 uint64_t more
: 6; /* How many udd words follow the command */
199 uint64_t reserved
: 29;
201 uint64_t param1
: 16;
205 #elif RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
209 uint64_t param1
: 16;
211 uint64_t reserved
: 29;
221 #define OCTEON_CMD_SIZE (sizeof(union octeon_cmd))
223 /* Maximum number of 8-byte words can be
224 * sent in a NIC control message.
226 #define LIO_MAX_NCTRL_UDD 32
228 /* Structure of control information passed by driver to the BASE
229 * layer when sending control commands to Octeon device software.
231 struct lio_ctrl_pkt
{
232 /** Command to be passed to the Octeon device software. */
233 union octeon_cmd ncmd
;
239 /** Response buffer */
243 /** Additional data that may be needed by some commands. */
244 uint64_t udd
[LIO_MAX_NCTRL_UDD
];
246 /** Input queue to use to send this command. */
249 /** Time to wait for Octeon software to respond to this control command.
250 * If wait_time is 0, BASE assumes no response is expected.
254 struct lio_dev_ctrl_cmd
*ctrl_cmd
;
257 /** Structure of data information passed by driver to the BASE
258 * layer when forwarding data to Octeon device software.
260 struct lio_data_pkt
{
261 /** Pointer to information maintained by NIC module for this packet. The
262 * BASE layer passes this as-is to the driver.
266 /** Type of buffer passed in "buf" above. */
269 /** Total data bytes to be transferred in this command. */
272 /** Command to be passed to the Octeon device software. */
273 union lio_instr_64B cmd
;
275 /** Input queue to use to send this command. */
279 /** Structure passed by driver to BASE layer to prepare a command to send
280 * network data to Octeon.
282 union lio_cmd_setup
{
286 uint32_t timestamp
: 1;
287 uint32_t ip_csum
: 1;
288 uint32_t transport_csum
: 1;
289 uint32_t tnl_csum
: 1;
298 uint64_t cmd_setup64
;
301 /* Instruction Header */
302 struct octeon_instr_ih3
{
303 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
306 uint64_t reserved3
: 1;
308 /** Gather indicator 1=gather*/
311 /** Data length OR no. of entries in gather list */
312 uint64_t dlengsz
: 14;
314 /** Front Data size */
318 uint64_t reserved2
: 4;
320 /** PKI port kind - PKIND */
324 uint64_t reserved1
: 32;
326 #elif RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
328 uint64_t reserved1
: 32;
330 /** PKI port kind - PKIND */
334 uint64_t reserved2
: 4;
336 /** Front Data size */
339 /** Data length OR no. of entries in gather list */
340 uint64_t dlengsz
: 14;
342 /** Gather indicator 1=gather*/
346 uint64_t reserved3
: 1;
351 /* PKI Instruction Header(PKI IH) */
352 struct octeon_instr_pki_ih3
{
353 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
358 /** Raw mode indicator 1 = RAW */
368 uint64_t reserved2
: 1;
380 uint64_t tagtype
: 2;
383 uint64_t reserved1
: 2;
391 #elif RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
400 uint64_t reserved1
: 2;
403 uint64_t tagtype
: 2;
415 uint64_t reserved2
: 1;
423 /** Raw mode indicator 1 = RAW */
431 /** Input Request Header */
432 struct octeon_instr_irh
{
433 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
436 uint64_t subcode
: 7;
438 uint64_t priority
: 3;
439 uint64_t reserved
: 5;
440 uint64_t ossp
: 32; /* opcode/subcode specific parameters */
441 #elif RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
442 uint64_t ossp
: 32; /* opcode/subcode specific parameters */
443 uint64_t reserved
: 5;
444 uint64_t priority
: 3;
446 uint64_t subcode
: 7;
452 /* pkiih3 + irh + ossp[0] + ossp[1] + rdp + rptr = 40 bytes */
453 #define OCTEON_SOFT_CMD_RESP_IH3 (40 + 8)
454 /* pki_h3 + irh + ossp[0] + ossp[1] = 32 bytes */
455 #define OCTEON_PCI_CMD_O3 (24 + 8)
457 /** Return Data Parameters */
458 struct octeon_instr_rdp
{
459 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
460 uint64_t reserved
: 49;
461 uint64_t pcie_port
: 3;
463 #elif RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
465 uint64_t pcie_port
: 3;
466 uint64_t reserved
: 49;
470 union octeon_packet_params
{
471 uint32_t pkt_params32
;
473 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
474 uint32_t reserved
: 24;
475 uint32_t ip_csum
: 1; /* Perform IP header checksum(s) */
476 /* Perform Outer transport header checksum */
477 uint32_t transport_csum
: 1;
478 /* Find tunnel, and perform transport csum. */
479 uint32_t tnl_csum
: 1;
480 uint32_t tsflag
: 1; /* Timestamp this packet */
481 uint32_t ipsec_ops
: 4; /* IPsec operation */
483 uint32_t ipsec_ops
: 4;
485 uint32_t tnl_csum
: 1;
486 uint32_t transport_csum
: 1;
487 uint32_t ip_csum
: 1;
488 uint32_t reserved
: 7;
493 /** Utility function to prepare a 64B NIC instruction based on a setup command
494 * @param cmd - pointer to instruction to be filled in.
495 * @param setup - pointer to the setup structure
496 * @param q_no - which queue for back pressure
498 * Assumes the cmd instruction is pre-allocated, but no fields are filled in.
501 lio_prepare_pci_cmd(struct lio_device
*lio_dev
,
502 union lio_instr_64B
*cmd
,
503 union lio_cmd_setup
*setup
,
506 union octeon_packet_params packet_params
;
507 struct octeon_instr_pki_ih3
*pki_ih3
;
508 struct octeon_instr_irh
*irh
;
509 struct octeon_instr_ih3
*ih3
;
512 memset(cmd
, 0, sizeof(union lio_instr_64B
));
514 ih3
= (struct octeon_instr_ih3
*)&cmd
->cmd3
.ih3
;
515 pki_ih3
= (struct octeon_instr_pki_ih3
*)&cmd
->cmd3
.pki_ih3
;
517 /* assume that rflag is cleared so therefore front data will only have
518 * irh and ossp[1] and ossp[2] for a total of 24 bytes
520 ih3
->pkind
= lio_dev
->instr_queue
[setup
->s
.iq_no
]->txpciq
.s
.pkind
;
522 ih3
->fsz
= OCTEON_PCI_CMD_O3
;
524 if (!setup
->s
.gather
) {
525 ih3
->dlengsz
= setup
->s
.u
.datasize
;
528 ih3
->dlengsz
= setup
->s
.u
.gatherptrs
;
535 pki_ih3
->uqpg
= lio_dev
->instr_queue
[setup
->s
.iq_no
]->txpciq
.s
.use_qpg
;
537 port
= (int)lio_dev
->instr_queue
[setup
->s
.iq_no
]->txpciq
.s
.port
;
542 pki_ih3
->tag
= LIO_DATA(port
);
544 pki_ih3
->tagtype
= OCTEON_ORDERED_TAG
;
545 pki_ih3
->qpg
= lio_dev
->instr_queue
[setup
->s
.iq_no
]->txpciq
.s
.qpg
;
546 pki_ih3
->pm
= 0x0; /* parse from L2 */
547 pki_ih3
->sl
= 32; /* sl will be sizeof(pki_ih3) + irh + ossp0 + ossp1*/
549 irh
= (struct octeon_instr_irh
*)&cmd
->cmd3
.irh
;
551 irh
->opcode
= LIO_OPCODE
;
552 irh
->subcode
= LIO_OPCODE_NW_DATA
;
554 packet_params
.pkt_params32
= 0;
555 packet_params
.s
.ip_csum
= setup
->s
.ip_csum
;
556 packet_params
.s
.transport_csum
= setup
->s
.transport_csum
;
557 packet_params
.s
.tnl_csum
= setup
->s
.tnl_csum
;
558 packet_params
.s
.tsflag
= setup
->s
.timestamp
;
560 irh
->ossp
= packet_params
.pkt_params32
;
563 int lio_setup_sc_buffer_pool(struct lio_device
*lio_dev
);
564 void lio_free_sc_buffer_pool(struct lio_device
*lio_dev
);
566 struct lio_soft_command
*
567 lio_alloc_soft_command(struct lio_device
*lio_dev
,
568 uint32_t datasize
, uint32_t rdatasize
,
570 void lio_prepare_soft_command(struct lio_device
*lio_dev
,
571 struct lio_soft_command
*sc
,
572 uint8_t opcode
, uint8_t subcode
,
573 uint32_t irh_ossp
, uint64_t ossp0
,
575 int lio_send_soft_command(struct lio_device
*lio_dev
,
576 struct lio_soft_command
*sc
);
577 void lio_free_soft_command(struct lio_soft_command
*sc
);
579 /** Send control packet to the device
580 * @param lio_dev - lio device pointer
581 * @param nctrl - control structure with command, timeout, and callback info
583 * @returns IQ_FAILED if it failed to add to the input queue. IQ_STOP if it the
584 * queue should be stopped, and LIO_IQ_SEND_OK if it sent okay.
586 int lio_send_ctrl_pkt(struct lio_device
*lio_dev
,
587 struct lio_ctrl_pkt
*ctrl_pkt
);
589 /** Maximum ordered requests to process in every invocation of
590 * lio_process_ordered_list(). The function will continue to process requests
591 * as long as it can find one that has finished processing. If it keeps
592 * finding requests that have completed, the function can run for ever. The
593 * value defined here sets an upper limit on the number of requests it can
594 * process before it returns control to the poll thread.
596 #define LIO_MAX_ORD_REQS_TO_PROCESS 4096
598 /** Error codes used in Octeon Host-Core communication.
601 * ----------------------------
603 * ----------------------------
604 * Error codes are 32-bit wide. The upper 16-bits, called Major Error Number,
605 * are reserved to identify the group to which the error code belongs. The
606 * lower 16-bits, called Minor Error Number, carry the actual code.
608 * So error codes are (MAJOR NUMBER << 16)| MINOR_NUMBER.
610 /** Status for a request.
611 * If the request is successfully queued, the driver will return
612 * a LIO_REQUEST_PENDING status. LIO_REQUEST_TIMEOUT is only returned by
613 * the driver if the response for request failed to arrive before a
614 * time-out period or if the request processing * got interrupted due to
615 * a signal respectively.
618 /** A value of 0x00000000 indicates no error i.e. success */
619 LIO_REQUEST_DONE
= 0x00000000,
620 /** (Major number: 0x0000; Minor Number: 0x0001) */
621 LIO_REQUEST_PENDING
= 0x00000001,
622 LIO_REQUEST_TIMEOUT
= 0x00000003,
626 /*------ Error codes used by firmware (bits 15..0 set by firmware */
627 #define LIO_FIRMWARE_MAJOR_ERROR_CODE 0x0001
628 #define LIO_FIRMWARE_STATUS_CODE(status) \
629 ((LIO_FIRMWARE_MAJOR_ERROR_CODE << 16) | (status))
631 /** Initialize the response lists. The number of response lists to create is
633 * @param lio_dev - the lio device structure.
635 void lio_setup_response_list(struct lio_device
*lio_dev
);
637 /** Check the status of first entry in the ordered list. If the instruction at
638 * that entry finished processing or has timed-out, the entry is cleaned.
639 * @param lio_dev - the lio device structure.
640 * @return 1 if the ordered list is empty, 0 otherwise.
642 int lio_process_ordered_list(struct lio_device
*lio_dev
);
644 #define LIO_INCR_INSTRQUEUE_PKT_COUNT(lio_dev, iq_no, field, count) \
645 (((lio_dev)->instr_queue[iq_no]->stats.field) += count)
648 lio_swap_8B_data(uint64_t *data
, uint32_t blocks
)
651 *data
= rte_cpu_to_be_64(*data
);
657 static inline uint64_t
658 lio_map_ring(void *buf
)
662 dma_addr
= rte_mbuf_data_iova_default(((struct rte_mbuf
*)buf
));
664 return (uint64_t)dma_addr
;
667 static inline uint64_t
668 lio_map_ring_info(struct lio_droq
*droq
, uint32_t i
)
672 dma_addr
= droq
->info_list_dma
+ (i
* LIO_DROQ_INFO_SIZE
);
674 return (uint64_t)dma_addr
;
678 lio_opcode_slow_path(union octeon_rh
*rh
)
680 uint16_t subcode1
, subcode2
;
682 subcode1
= LIO_OPCODE_SUBCODE(rh
->r
.opcode
, rh
->r
.subcode
);
683 subcode2
= LIO_OPCODE_SUBCODE(LIO_OPCODE
, LIO_OPCODE_NW_DATA
);
685 return subcode2
!= subcode1
;
689 lio_add_sg_size(struct lio_sg_entry
*sg_entry
,
690 uint16_t size
, uint32_t pos
)
692 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
693 sg_entry
->u
.size
[pos
] = size
;
694 #elif RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
695 sg_entry
->u
.size
[3 - pos
] = size
;
699 /* Macro to increment index.
700 * Index is incremented by count; if the sum exceeds
701 * max, index is wrapped-around to the start.
703 static inline uint32_t
704 lio_incr_index(uint32_t index
, uint32_t count
, uint32_t max
)
706 if ((index
+ count
) >= max
)
707 index
= index
+ count
- max
;
714 int lio_setup_droq(struct lio_device
*lio_dev
, int q_no
, int num_descs
,
715 int desc_size
, struct rte_mempool
*mpool
,
716 unsigned int socket_id
);
717 uint16_t lio_dev_recv_pkts(void *rx_queue
, struct rte_mbuf
**rx_pkts
,
719 void lio_delete_droq_queue(struct lio_device
*lio_dev
, int oq_no
);
721 void lio_delete_sglist(struct lio_instr_queue
*txq
);
722 int lio_setup_sglists(struct lio_device
*lio_dev
, int iq_no
,
723 int fw_mapped_iq
, int num_descs
, unsigned int socket_id
);
724 uint16_t lio_dev_xmit_pkts(void *tx_queue
, struct rte_mbuf
**pkts
,
726 int lio_wait_for_instr_fetch(struct lio_device
*lio_dev
);
727 int lio_setup_iq(struct lio_device
*lio_dev
, int q_index
,
728 union octeon_txpciq iq_no
, uint32_t num_descs
, void *app_ctx
,
729 unsigned int socket_id
);
730 int lio_flush_iq(struct lio_device
*lio_dev
, struct lio_instr_queue
*iq
);
731 void lio_delete_instruction_queue(struct lio_device
*lio_dev
, int iq_no
);
732 /** Setup instruction queue zero for the device
733 * @param lio_dev which lio device to setup
735 * @return 0 if success. -1 if fails
737 int lio_setup_instr_queue0(struct lio_device
*lio_dev
);
738 void lio_free_instr_queue0(struct lio_device
*lio_dev
);
739 void lio_dev_clear_queues(struct rte_eth_dev
*eth_dev
);
740 #endif /* _LIO_RXTX_H_ */