4 * Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
11 * * Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * * Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
17 * * Neither the name of Intel Corporation nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific prior written permission.
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
34 #ifndef RTE_EXEC_ENV_LINUXAPP
35 #error "KNI is not supported"
41 #include <sys/ioctl.h>
43 #include <rte_spinlock.h>
44 #include <rte_string_fns.h>
45 #include <rte_ethdev.h>
46 #include <rte_malloc.h>
49 #include <rte_memzone.h>
50 #include <exec-env/rte_kni_common.h>
51 #include "rte_kni_fifo.h"
53 #define MAX_MBUF_BURST_NUM 32
55 /* Maximum number of ring entries */
56 #define KNI_FIFO_COUNT_MAX 1024
57 #define KNI_FIFO_SIZE (KNI_FIFO_COUNT_MAX * sizeof(void *) + \
58 sizeof(struct rte_kni_fifo))
60 #define KNI_REQUEST_MBUF_NUM_MAX 32
62 #define KNI_MEM_CHECK(cond) do { if (cond) goto kni_fail; } while (0)
68 char name
[RTE_KNI_NAMESIZE
]; /**< KNI interface name */
69 uint16_t group_id
; /**< Group ID of KNI devices */
70 uint32_t slot_id
; /**< KNI pool slot ID */
71 struct rte_mempool
*pktmbuf_pool
; /**< pkt mbuf mempool */
72 unsigned mbuf_size
; /**< mbuf size */
74 struct rte_kni_fifo
*tx_q
; /**< TX queue */
75 struct rte_kni_fifo
*rx_q
; /**< RX queue */
76 struct rte_kni_fifo
*alloc_q
; /**< Allocated mbufs queue */
77 struct rte_kni_fifo
*free_q
; /**< To be freed mbufs queue */
79 /* For request & response */
80 struct rte_kni_fifo
*req_q
; /**< Request queue */
81 struct rte_kni_fifo
*resp_q
; /**< Response queue */
82 void * sync_addr
; /**< Req/Resp Mem address */
84 struct rte_kni_ops ops
; /**< operations for request */
85 uint8_t in_use
: 1; /**< kni in use */
89 KNI_REQ_NO_REGISTER
= 0,
94 * KNI memzone pool slot
96 struct rte_kni_memzone_slot
{
98 uint8_t in_use
: 1; /**< slot in use */
101 const struct rte_memzone
*m_ctx
; /**< KNI ctx */
102 const struct rte_memzone
*m_tx_q
; /**< TX queue */
103 const struct rte_memzone
*m_rx_q
; /**< RX queue */
104 const struct rte_memzone
*m_alloc_q
; /**< Allocated mbufs queue */
105 const struct rte_memzone
*m_free_q
; /**< To be freed mbufs queue */
106 const struct rte_memzone
*m_req_q
; /**< Request queue */
107 const struct rte_memzone
*m_resp_q
; /**< Response queue */
108 const struct rte_memzone
*m_sync_addr
;
110 /* Free linked list */
111 struct rte_kni_memzone_slot
*next
; /**< Next slot link.list */
117 struct rte_kni_memzone_pool
{
118 uint8_t initialized
: 1; /**< Global KNI pool init flag */
120 uint32_t max_ifaces
; /**< Max. num of KNI ifaces */
121 struct rte_kni_memzone_slot
*slots
; /**< Pool slots */
122 rte_spinlock_t mutex
; /**< alloc/relase mutex */
124 /* Free memzone slots linked-list */
125 struct rte_kni_memzone_slot
*free
; /**< First empty slot */
126 struct rte_kni_memzone_slot
*free_tail
; /**< Last empty slot */
130 static void kni_free_mbufs(struct rte_kni
*kni
);
131 static void kni_allocate_mbufs(struct rte_kni
*kni
);
133 static volatile int kni_fd
= -1;
134 static struct rte_kni_memzone_pool kni_memzone_pool
= {
138 static const struct rte_memzone
*
139 kni_memzone_reserve(const char *name
, size_t len
, int socket_id
,
142 const struct rte_memzone
*mz
= rte_memzone_lookup(name
);
145 mz
= rte_memzone_reserve(name
, len
, socket_id
, flags
);
151 static struct rte_kni_memzone_slot
*
152 kni_memzone_pool_alloc(void)
154 struct rte_kni_memzone_slot
*slot
;
156 rte_spinlock_lock(&kni_memzone_pool
.mutex
);
158 if (!kni_memzone_pool
.free
) {
159 rte_spinlock_unlock(&kni_memzone_pool
.mutex
);
163 slot
= kni_memzone_pool
.free
;
164 kni_memzone_pool
.free
= slot
->next
;
167 if (!kni_memzone_pool
.free
)
168 kni_memzone_pool
.free_tail
= NULL
;
170 rte_spinlock_unlock(&kni_memzone_pool
.mutex
);
176 kni_memzone_pool_release(struct rte_kni_memzone_slot
*slot
)
178 rte_spinlock_lock(&kni_memzone_pool
.mutex
);
180 if (kni_memzone_pool
.free
)
181 kni_memzone_pool
.free_tail
->next
= slot
;
183 kni_memzone_pool
.free
= slot
;
185 kni_memzone_pool
.free_tail
= slot
;
189 rte_spinlock_unlock(&kni_memzone_pool
.mutex
);
193 /* Shall be called before any allocation happens */
195 rte_kni_init(unsigned int max_kni_ifaces
)
198 struct rte_kni_memzone_slot
*it
;
199 const struct rte_memzone
*mz
;
201 char obj_name
[OBJNAMSIZ
];
202 char mz_name
[RTE_MEMZONE_NAMESIZE
];
204 /* Immediately return if KNI is already initialized */
205 if (kni_memzone_pool
.initialized
) {
206 RTE_LOG(WARNING
, KNI
, "Double call to rte_kni_init()");
210 if (max_kni_ifaces
== 0) {
211 RTE_LOG(ERR
, KNI
, "Invalid number of max_kni_ifaces %d\n",
213 RTE_LOG(ERR
, KNI
, "Unable to initialize KNI\n");
217 /* Check FD and open */
219 kni_fd
= open("/dev/" KNI_DEVICE
, O_RDWR
);
222 "Can not open /dev/%s\n", KNI_DEVICE
);
227 /* Allocate slot objects */
228 kni_memzone_pool
.slots
= (struct rte_kni_memzone_slot
*)
230 sizeof(struct rte_kni_memzone_slot
) *
233 KNI_MEM_CHECK(kni_memzone_pool
.slots
== NULL
);
235 /* Initialize general pool variables */
236 kni_memzone_pool
.initialized
= 1;
237 kni_memzone_pool
.max_ifaces
= max_kni_ifaces
;
238 kni_memzone_pool
.free
= &kni_memzone_pool
.slots
[0];
239 rte_spinlock_init(&kni_memzone_pool
.mutex
);
241 /* Pre-allocate all memzones of all the slots; panic on error */
242 for (i
= 0; i
< max_kni_ifaces
; i
++) {
244 /* Recover current slot */
245 it
= &kni_memzone_pool
.slots
[i
];
248 /* Allocate KNI context */
249 snprintf(mz_name
, RTE_MEMZONE_NAMESIZE
, "KNI_INFO_%d", i
);
250 mz
= kni_memzone_reserve(mz_name
, sizeof(struct rte_kni
),
252 KNI_MEM_CHECK(mz
== NULL
);
256 snprintf(obj_name
, OBJNAMSIZ
, "kni_tx_%d", i
);
257 mz
= kni_memzone_reserve(obj_name
, KNI_FIFO_SIZE
,
259 KNI_MEM_CHECK(mz
== NULL
);
263 snprintf(obj_name
, OBJNAMSIZ
, "kni_rx_%d", i
);
264 mz
= kni_memzone_reserve(obj_name
, KNI_FIFO_SIZE
,
266 KNI_MEM_CHECK(mz
== NULL
);
270 snprintf(obj_name
, OBJNAMSIZ
, "kni_alloc_%d", i
);
271 mz
= kni_memzone_reserve(obj_name
, KNI_FIFO_SIZE
,
273 KNI_MEM_CHECK(mz
== NULL
);
277 snprintf(obj_name
, OBJNAMSIZ
, "kni_free_%d", i
);
278 mz
= kni_memzone_reserve(obj_name
, KNI_FIFO_SIZE
,
280 KNI_MEM_CHECK(mz
== NULL
);
284 snprintf(obj_name
, OBJNAMSIZ
, "kni_req_%d", i
);
285 mz
= kni_memzone_reserve(obj_name
, KNI_FIFO_SIZE
,
287 KNI_MEM_CHECK(mz
== NULL
);
291 snprintf(obj_name
, OBJNAMSIZ
, "kni_resp_%d", i
);
292 mz
= kni_memzone_reserve(obj_name
, KNI_FIFO_SIZE
,
294 KNI_MEM_CHECK(mz
== NULL
);
297 /* Req/Resp sync mem area */
298 snprintf(obj_name
, OBJNAMSIZ
, "kni_sync_%d", i
);
299 mz
= kni_memzone_reserve(obj_name
, KNI_FIFO_SIZE
,
301 KNI_MEM_CHECK(mz
== NULL
);
302 it
->m_sync_addr
= mz
;
304 if ((i
+1) == max_kni_ifaces
) {
306 kni_memzone_pool
.free_tail
= it
;
308 it
->next
= &kni_memzone_pool
.slots
[i
+1];
314 RTE_LOG(ERR
, KNI
, "Unable to allocate memory for max_kni_ifaces:%d."
315 "Increase the amount of hugepages memory\n", max_kni_ifaces
);
320 rte_kni_alloc(struct rte_mempool
*pktmbuf_pool
,
321 const struct rte_kni_conf
*conf
,
322 struct rte_kni_ops
*ops
)
325 struct rte_kni_device_info dev_info
;
327 char intf_name
[RTE_KNI_NAMESIZE
];
328 const struct rte_memzone
*mz
;
329 struct rte_kni_memzone_slot
*slot
= NULL
;
331 if (!pktmbuf_pool
|| !conf
|| !conf
->name
[0])
334 /* Check if KNI subsystem has been initialized */
335 if (kni_memzone_pool
.initialized
!= 1) {
336 RTE_LOG(ERR
, KNI
, "KNI subsystem has not been initialized. Invoke rte_kni_init() first\n");
340 /* Get an available slot from the pool */
341 slot
= kni_memzone_pool_alloc();
343 RTE_LOG(ERR
, KNI
, "Cannot allocate more KNI interfaces; increase the number of max_kni_ifaces(current %d) or release unusued ones.\n",
344 kni_memzone_pool
.max_ifaces
);
349 ctx
= slot
->m_ctx
->addr
;
350 snprintf(intf_name
, RTE_KNI_NAMESIZE
, "%s", conf
->name
);
353 RTE_LOG(ERR
, KNI
, "KNI %s is in use\n", ctx
->name
);
356 memset(ctx
, 0, sizeof(struct rte_kni
));
358 memcpy(&ctx
->ops
, ops
, sizeof(struct rte_kni_ops
));
360 memset(&dev_info
, 0, sizeof(dev_info
));
361 dev_info
.bus
= conf
->addr
.bus
;
362 dev_info
.devid
= conf
->addr
.devid
;
363 dev_info
.function
= conf
->addr
.function
;
364 dev_info
.vendor_id
= conf
->id
.vendor_id
;
365 dev_info
.device_id
= conf
->id
.device_id
;
366 dev_info
.core_id
= conf
->core_id
;
367 dev_info
.force_bind
= conf
->force_bind
;
368 dev_info
.group_id
= conf
->group_id
;
369 dev_info
.mbuf_size
= conf
->mbuf_size
;
371 snprintf(ctx
->name
, RTE_KNI_NAMESIZE
, "%s", intf_name
);
372 snprintf(dev_info
.name
, RTE_KNI_NAMESIZE
, "%s", intf_name
);
374 RTE_LOG(INFO
, KNI
, "pci: %02x:%02x:%02x \t %02x:%02x\n",
375 dev_info
.bus
, dev_info
.devid
, dev_info
.function
,
376 dev_info
.vendor_id
, dev_info
.device_id
);
379 ctx
->tx_q
= mz
->addr
;
380 kni_fifo_init(ctx
->tx_q
, KNI_FIFO_COUNT_MAX
);
381 dev_info
.tx_phys
= mz
->phys_addr
;
385 ctx
->rx_q
= mz
->addr
;
386 kni_fifo_init(ctx
->rx_q
, KNI_FIFO_COUNT_MAX
);
387 dev_info
.rx_phys
= mz
->phys_addr
;
390 mz
= slot
->m_alloc_q
;
391 ctx
->alloc_q
= mz
->addr
;
392 kni_fifo_init(ctx
->alloc_q
, KNI_FIFO_COUNT_MAX
);
393 dev_info
.alloc_phys
= mz
->phys_addr
;
397 ctx
->free_q
= mz
->addr
;
398 kni_fifo_init(ctx
->free_q
, KNI_FIFO_COUNT_MAX
);
399 dev_info
.free_phys
= mz
->phys_addr
;
403 ctx
->req_q
= mz
->addr
;
404 kni_fifo_init(ctx
->req_q
, KNI_FIFO_COUNT_MAX
);
405 dev_info
.req_phys
= mz
->phys_addr
;
409 ctx
->resp_q
= mz
->addr
;
410 kni_fifo_init(ctx
->resp_q
, KNI_FIFO_COUNT_MAX
);
411 dev_info
.resp_phys
= mz
->phys_addr
;
413 /* Req/Resp sync mem area */
414 mz
= slot
->m_sync_addr
;
415 ctx
->sync_addr
= mz
->addr
;
416 dev_info
.sync_va
= mz
->addr
;
417 dev_info
.sync_phys
= mz
->phys_addr
;
419 ctx
->pktmbuf_pool
= pktmbuf_pool
;
420 ctx
->group_id
= conf
->group_id
;
421 ctx
->slot_id
= slot
->id
;
422 ctx
->mbuf_size
= conf
->mbuf_size
;
424 ret
= ioctl(kni_fd
, RTE_KNI_IOCTL_CREATE
, &dev_info
);
425 KNI_MEM_CHECK(ret
< 0);
429 /* Allocate mbufs and then put them into alloc_q */
430 kni_allocate_mbufs(ctx
);
436 kni_memzone_pool_release(&kni_memzone_pool
.slots
[slot
->id
]);
442 kni_free_fifo(struct rte_kni_fifo
*fifo
)
445 struct rte_mbuf
*pkt
;
448 ret
= kni_fifo_get(fifo
, (void **)&pkt
, 1);
450 rte_pktmbuf_free(pkt
);
455 kni_free_fifo_phy(struct rte_kni_fifo
*fifo
)
461 ret
= kni_fifo_get(fifo
, &mbuf_phys
, 1);
469 rte_kni_release(struct rte_kni
*kni
)
471 struct rte_kni_device_info dev_info
;
474 if (!kni
|| !kni
->in_use
)
477 snprintf(dev_info
.name
, sizeof(dev_info
.name
), "%s", kni
->name
);
478 if (ioctl(kni_fd
, RTE_KNI_IOCTL_RELEASE
, &dev_info
) < 0) {
479 RTE_LOG(ERR
, KNI
, "Fail to release kni device\n");
483 /* mbufs in all fifo should be released, except request/response */
484 kni_free_fifo(kni
->tx_q
);
485 kni_free_fifo_phy(kni
->rx_q
);
486 kni_free_fifo_phy(kni
->alloc_q
);
487 kni_free_fifo(kni
->free_q
);
489 slot_id
= kni
->slot_id
;
491 /* Memset the KNI struct */
492 memset(kni
, 0, sizeof(struct rte_kni
));
494 /* Release memzone */
495 if (slot_id
> kni_memzone_pool
.max_ifaces
) {
496 RTE_LOG(ERR
, KNI
, "KNI pool: corrupted slot ID: %d, max: %d\n",
497 slot_id
, kni_memzone_pool
.max_ifaces
);
500 kni_memzone_pool_release(&kni_memzone_pool
.slots
[slot_id
]);
506 rte_kni_handle_request(struct rte_kni
*kni
)
509 struct rte_kni_request
*req
;
514 /* Get request mbuf */
515 ret
= kni_fifo_get(kni
->req_q
, (void **)&req
, 1);
517 return 0; /* It is OK of can not getting the request mbuf */
519 if (req
!= kni
->sync_addr
) {
520 RTE_LOG(ERR
, KNI
, "Wrong req pointer %p\n", req
);
524 /* Analyze the request and call the relevant actions for it */
525 switch (req
->req_id
) {
526 case RTE_KNI_REQ_CHANGE_MTU
: /* Change MTU */
527 if (kni
->ops
.change_mtu
)
528 req
->result
= kni
->ops
.change_mtu(kni
->ops
.port_id
,
531 case RTE_KNI_REQ_CFG_NETWORK_IF
: /* Set network interface up/down */
532 if (kni
->ops
.config_network_if
)
533 req
->result
= kni
->ops
.config_network_if(\
534 kni
->ops
.port_id
, req
->if_up
);
537 RTE_LOG(ERR
, KNI
, "Unknown request id %u\n", req
->req_id
);
538 req
->result
= -EINVAL
;
542 /* Construct response mbuf and put it back to resp_q */
543 ret
= kni_fifo_put(kni
->resp_q
, (void **)&req
, 1);
545 RTE_LOG(ERR
, KNI
, "Fail to put the muf back to resp_q\n");
546 return -1; /* It is an error of can't putting the mbuf back */
553 va2pa(struct rte_mbuf
*m
)
555 return (void *)((unsigned long)m
-
556 ((unsigned long)m
->buf_addr
-
557 (unsigned long)m
->buf_physaddr
));
561 rte_kni_tx_burst(struct rte_kni
*kni
, struct rte_mbuf
**mbufs
, unsigned num
)
563 void *phy_mbufs
[num
];
567 for (i
= 0; i
< num
; i
++)
568 phy_mbufs
[i
] = va2pa(mbufs
[i
]);
570 ret
= kni_fifo_put(kni
->rx_q
, phy_mbufs
, num
);
572 /* Get mbufs from free_q and then free them */
579 rte_kni_rx_burst(struct rte_kni
*kni
, struct rte_mbuf
**mbufs
, unsigned num
)
581 unsigned ret
= kni_fifo_get(kni
->tx_q
, (void **)mbufs
, num
);
583 /* If buffers removed, allocate mbufs and then put them into alloc_q */
585 kni_allocate_mbufs(kni
);
591 kni_free_mbufs(struct rte_kni
*kni
)
594 struct rte_mbuf
*pkts
[MAX_MBUF_BURST_NUM
];
596 ret
= kni_fifo_get(kni
->free_q
, (void **)pkts
, MAX_MBUF_BURST_NUM
);
597 if (likely(ret
> 0)) {
598 for (i
= 0; i
< ret
; i
++)
599 rte_pktmbuf_free(pkts
[i
]);
604 kni_allocate_mbufs(struct rte_kni
*kni
)
607 struct rte_mbuf
*pkts
[MAX_MBUF_BURST_NUM
];
608 void *phys
[MAX_MBUF_BURST_NUM
];
610 RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf
, pool
) !=
611 offsetof(struct rte_kni_mbuf
, pool
));
612 RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf
, buf_addr
) !=
613 offsetof(struct rte_kni_mbuf
, buf_addr
));
614 RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf
, next
) !=
615 offsetof(struct rte_kni_mbuf
, next
));
616 RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf
, data_off
) !=
617 offsetof(struct rte_kni_mbuf
, data_off
));
618 RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf
, data_len
) !=
619 offsetof(struct rte_kni_mbuf
, data_len
));
620 RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf
, pkt_len
) !=
621 offsetof(struct rte_kni_mbuf
, pkt_len
));
622 RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf
, ol_flags
) !=
623 offsetof(struct rte_kni_mbuf
, ol_flags
));
625 /* Check if pktmbuf pool has been configured */
626 if (kni
->pktmbuf_pool
== NULL
) {
627 RTE_LOG(ERR
, KNI
, "No valid mempool for allocating mbufs\n");
631 for (i
= 0; i
< MAX_MBUF_BURST_NUM
; i
++) {
632 pkts
[i
] = rte_pktmbuf_alloc(kni
->pktmbuf_pool
);
633 if (unlikely(pkts
[i
] == NULL
)) {
635 RTE_LOG(ERR
, KNI
, "Out of memory\n");
638 phys
[i
] = va2pa(pkts
[i
]);
641 /* No pkt mbuf alocated */
645 ret
= kni_fifo_put(kni
->alloc_q
, phys
, i
);
647 /* Check if any mbufs not put into alloc_q, and then free them */
648 if (ret
>= 0 && ret
< i
&& ret
< MAX_MBUF_BURST_NUM
) {
651 for (j
= ret
; j
< i
; j
++)
652 rte_pktmbuf_free(pkts
[j
]);
657 rte_kni_get(const char *name
)
660 struct rte_kni_memzone_slot
*it
;
663 /* Note: could be improved perf-wise if necessary */
664 for (i
= 0; i
< kni_memzone_pool
.max_ifaces
; i
++) {
665 it
= &kni_memzone_pool
.slots
[i
];
668 kni
= it
->m_ctx
->addr
;
669 if (strncmp(kni
->name
, name
, RTE_KNI_NAMESIZE
) == 0)
677 rte_kni_get_name(const struct rte_kni
*kni
)
682 static enum kni_ops_status
683 kni_check_request_register(struct rte_kni_ops
*ops
)
685 /* check if KNI request ops has been registered*/
687 return KNI_REQ_NO_REGISTER
;
689 if((NULL
== ops
->change_mtu
) && (NULL
== ops
->config_network_if
))
690 return KNI_REQ_NO_REGISTER
;
692 return KNI_REQ_REGISTERED
;
696 rte_kni_register_handlers(struct rte_kni
*kni
,struct rte_kni_ops
*ops
)
698 enum kni_ops_status req_status
;
701 RTE_LOG(ERR
, KNI
, "Invalid KNI request operation.\n");
706 RTE_LOG(ERR
, KNI
, "Invalid kni info.\n");
710 req_status
= kni_check_request_register(&kni
->ops
);
711 if ( KNI_REQ_REGISTERED
== req_status
) {
712 RTE_LOG(ERR
, KNI
, "The KNI request operation has already registered.\n");
716 memcpy(&kni
->ops
, ops
, sizeof(struct rte_kni_ops
));
721 rte_kni_unregister_handlers(struct rte_kni
*kni
)
724 RTE_LOG(ERR
, KNI
, "Invalid kni info.\n");
728 kni
->ops
.change_mtu
= NULL
;
729 kni
->ops
.config_network_if
= NULL
;