1 /**********************************************************************
4 * Contact: support@cavium.com
5 * Please include "LiquidIO" in the subject.
7 * Copyright (c) 2003-2015 Cavium, Inc.
9 * This file is free software; you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License, Version 2, as
11 * published by the Free Software Foundation.
13 * This file is distributed in the hope that it will be useful, but
14 * AS-IS and WITHOUT ANY WARRANTY; without even the implied warranty
15 * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, TITLE, or
16 * NONINFRINGEMENT. See the GNU General Public License for more
19 * This file may also be available under a different license from Cavium.
20 * Contact Cavium, Inc. for more information
21 **********************************************************************/
22 #include <linux/version.h>
23 #include <linux/pci.h>
24 #include <linux/firmware.h>
25 #include <linux/ptp_clock_kernel.h>
26 #include <net/vxlan.h>
27 #include "liquidio_common.h"
28 #include "octeon_droq.h"
29 #include "octeon_iq.h"
30 #include "response_manager.h"
31 #include "octeon_device.h"
32 #include "octeon_nic.h"
33 #include "octeon_main.h"
34 #include "octeon_network.h"
35 #include "cn66xx_regs.h"
36 #include "cn66xx_device.h"
37 #include "cn68xx_device.h"
38 #include "cn23xx_pf_device.h"
39 #include "liquidio_image.h"
41 MODULE_AUTHOR("Cavium Networks, <support@cavium.com>");
42 MODULE_DESCRIPTION("Cavium LiquidIO Intelligent Server Adapter Driver");
43 MODULE_LICENSE("GPL");
44 MODULE_VERSION(LIQUIDIO_VERSION
);
45 MODULE_FIRMWARE(LIO_FW_DIR LIO_FW_BASE_NAME LIO_210SV_NAME LIO_FW_NAME_SUFFIX
);
46 MODULE_FIRMWARE(LIO_FW_DIR LIO_FW_BASE_NAME LIO_210NV_NAME LIO_FW_NAME_SUFFIX
);
47 MODULE_FIRMWARE(LIO_FW_DIR LIO_FW_BASE_NAME LIO_410NV_NAME LIO_FW_NAME_SUFFIX
);
49 static int ddr_timeout
= 10000;
50 module_param(ddr_timeout
, int, 0644);
51 MODULE_PARM_DESC(ddr_timeout
,
52 "Number of milliseconds to wait for DDR initialization. 0 waits for ddr_timeout to be set to non-zero value before starting to check");
54 #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK)
56 #define INCR_INSTRQUEUE_PKT_COUNT(octeon_dev_ptr, iq_no, field, count) \
57 (octeon_dev_ptr->instr_queue[iq_no]->stats.field += count)
59 static int debug
= -1;
60 module_param(debug
, int, 0644);
61 MODULE_PARM_DESC(debug
, "NETIF_MSG debug bits");
63 static char fw_type
[LIO_MAX_FW_TYPE_LEN
];
64 module_param_string(fw_type
, fw_type
, sizeof(fw_type
), 0000);
65 MODULE_PARM_DESC(fw_type
, "Type of firmware to be loaded. Default \"nic\"");
68 module_param(conf_type
, int, 0);
69 MODULE_PARM_DESC(conf_type
, "select octeon configuration 0 default 1 ovs");
71 static int ptp_enable
= 1;
73 /* Bit mask values for lio->ifstate */
74 #define LIO_IFSTATE_DROQ_OPS 0x01
75 #define LIO_IFSTATE_REGISTERED 0x02
76 #define LIO_IFSTATE_RUNNING 0x04
77 #define LIO_IFSTATE_RX_TIMESTAMP_ENABLED 0x08
79 /* Polling interval for determining when NIC application is alive */
80 #define LIQUIDIO_STARTER_POLL_INTERVAL_MS 100
82 /* runtime link query interval */
83 #define LIQUIDIO_LINK_QUERY_INTERVAL_MS 1000
85 struct liquidio_if_cfg_context
{
93 struct liquidio_if_cfg_resp
{
95 struct liquidio_if_cfg_info cfg_info
;
99 struct oct_link_status_resp
{
101 struct oct_link_info link_info
;
105 struct oct_timestamp_resp
{
111 #define OCT_TIMESTAMP_RESP_SIZE (sizeof(struct oct_timestamp_resp))
116 #ifdef __BIG_ENDIAN_BITFIELD
128 /** Octeon device properties to be used by the NIC module.
129 * Each octeon device in the system will be represented
130 * by this structure in the NIC module.
133 #define OCTNIC_MAX_SG (MAX_SKB_FRAGS)
135 #define OCTNIC_GSO_MAX_HEADER_SIZE 128
136 #define OCTNIC_GSO_MAX_SIZE \
137 (CN23XX_DEFAULT_INPUT_JABBER - OCTNIC_GSO_MAX_HEADER_SIZE)
139 /** Structure of a node in list of gather components maintained by
140 * NIC driver for each network device.
142 struct octnic_gather
{
143 /** List manipulation. Next and prev pointers. */
144 struct list_head list
;
146 /** Size of the gather component at sg in bytes. */
149 /** Number of bytes that sg was adjusted to make it 8B-aligned. */
152 /** Gather component that can accommodate max sized fragment list
153 * received from the IP layer.
155 struct octeon_sg_entry
*sg
;
161 struct completion init
;
162 struct completion started
;
163 struct pci_dev
*pci_dev
;
168 struct octeon_device_priv
{
169 /** Tasklet structures for this device. */
170 struct tasklet_struct droq_tasklet
;
171 unsigned long napi_mask
;
174 static int octeon_device_init(struct octeon_device
*);
175 static int liquidio_stop(struct net_device
*netdev
);
176 static void liquidio_remove(struct pci_dev
*pdev
);
177 static int liquidio_probe(struct pci_dev
*pdev
,
178 const struct pci_device_id
*ent
);
180 static struct handshake handshake
[MAX_OCTEON_DEVICES
];
181 static struct completion first_stage
;
183 static void octeon_droq_bh(unsigned long pdev
)
187 struct octeon_device
*oct
= (struct octeon_device
*)pdev
;
188 struct octeon_device_priv
*oct_priv
=
189 (struct octeon_device_priv
*)oct
->priv
;
191 /* for (q_no = 0; q_no < oct->num_oqs; q_no++) { */
192 for (q_no
= 0; q_no
< MAX_OCTEON_OUTPUT_QUEUES(oct
); q_no
++) {
193 if (!(oct
->io_qmask
.oq
& (1ULL << q_no
)))
195 reschedule
|= octeon_droq_process_packets(oct
, oct
->droq
[q_no
],
197 lio_enable_irq(oct
->droq
[q_no
], NULL
);
201 tasklet_schedule(&oct_priv
->droq_tasklet
);
204 static int lio_wait_for_oq_pkts(struct octeon_device
*oct
)
206 struct octeon_device_priv
*oct_priv
=
207 (struct octeon_device_priv
*)oct
->priv
;
208 int retry
= 100, pkt_cnt
= 0, pending_pkts
= 0;
214 for (i
= 0; i
< MAX_OCTEON_OUTPUT_QUEUES(oct
); i
++) {
215 if (!(oct
->io_qmask
.oq
& (1ULL << i
)))
217 pkt_cnt
+= octeon_droq_check_hw_for_pkts(oct
->droq
[i
]);
220 pending_pkts
+= pkt_cnt
;
221 tasklet_schedule(&oct_priv
->droq_tasklet
);
224 schedule_timeout_uninterruptible(1);
226 } while (retry
-- && pending_pkts
);
232 * \brief Forces all IO queues off on a given device
233 * @param oct Pointer to Octeon device
235 static void force_io_queues_off(struct octeon_device
*oct
)
237 if ((oct
->chip_id
== OCTEON_CN66XX
) ||
238 (oct
->chip_id
== OCTEON_CN68XX
)) {
239 /* Reset the Enable bits for Input Queues. */
240 octeon_write_csr(oct
, CN6XXX_SLI_PKT_INSTR_ENB
, 0);
242 /* Reset the Enable bits for Output Queues. */
243 octeon_write_csr(oct
, CN6XXX_SLI_PKT_OUT_ENB
, 0);
248 * \brief wait for all pending requests to complete
249 * @param oct Pointer to Octeon device
251 * Called during shutdown sequence
253 static int wait_for_pending_requests(struct octeon_device
*oct
)
257 for (i
= 0; i
< 100; i
++) {
259 atomic_read(&oct
->response_list
260 [OCTEON_ORDERED_SC_LIST
].pending_req_count
);
262 schedule_timeout_uninterruptible(HZ
/ 10);
274 * \brief Cause device to go quiet so it can be safely removed/reset/etc
275 * @param oct Pointer to Octeon device
277 static inline void pcierror_quiesce_device(struct octeon_device
*oct
)
281 /* Disable the input and output queues now. No more packets will
282 * arrive from Octeon, but we should wait for all packet processing
285 force_io_queues_off(oct
);
287 /* To allow for in-flight requests */
288 schedule_timeout_uninterruptible(100);
290 if (wait_for_pending_requests(oct
))
291 dev_err(&oct
->pci_dev
->dev
, "There were pending requests\n");
293 /* Force all requests waiting to be fetched by OCTEON to complete. */
294 for (i
= 0; i
< MAX_OCTEON_INSTR_QUEUES(oct
); i
++) {
295 struct octeon_instr_queue
*iq
;
297 if (!(oct
->io_qmask
.iq
& (1ULL << i
)))
299 iq
= oct
->instr_queue
[i
];
301 if (atomic_read(&iq
->instr_pending
)) {
302 spin_lock_bh(&iq
->lock
);
304 iq
->octeon_read_index
= iq
->host_write_index
;
305 iq
->stats
.instr_processed
+=
306 atomic_read(&iq
->instr_pending
);
307 lio_process_iq_request_list(oct
, iq
, 0);
308 spin_unlock_bh(&iq
->lock
);
312 /* Force all pending ordered list requests to time out. */
313 lio_process_ordered_list(oct
, 1);
315 /* We do not need to wait for output queue packets to be processed. */
319 * \brief Cleanup PCI AER uncorrectable error status
320 * @param dev Pointer to PCI device
322 static void cleanup_aer_uncorrect_error_status(struct pci_dev
*dev
)
327 pr_info("%s :\n", __func__
);
329 pci_read_config_dword(dev
, pos
+ PCI_ERR_UNCOR_STATUS
, &status
);
330 pci_read_config_dword(dev
, pos
+ PCI_ERR_UNCOR_SEVER
, &mask
);
331 if (dev
->error_state
== pci_channel_io_normal
)
332 status
&= ~mask
; /* Clear corresponding nonfatal bits */
334 status
&= mask
; /* Clear corresponding fatal bits */
335 pci_write_config_dword(dev
, pos
+ PCI_ERR_UNCOR_STATUS
, status
);
339 * \brief Stop all PCI IO to a given device
340 * @param dev Pointer to Octeon device
342 static void stop_pci_io(struct octeon_device
*oct
)
344 /* No more instructions will be forwarded. */
345 atomic_set(&oct
->status
, OCT_DEV_IN_RESET
);
347 pci_disable_device(oct
->pci_dev
);
349 /* Disable interrupts */
350 oct
->fn_list
.disable_interrupt(oct
->chip
);
352 pcierror_quiesce_device(oct
);
354 /* Release the interrupt line */
355 free_irq(oct
->pci_dev
->irq
, oct
);
357 if (oct
->flags
& LIO_FLAG_MSI_ENABLED
)
358 pci_disable_msi(oct
->pci_dev
);
360 dev_dbg(&oct
->pci_dev
->dev
, "Device state is now %s\n",
361 lio_get_state_string(&oct
->status
));
363 /* cn63xx_cleanup_aer_uncorrect_error_status(oct->pci_dev); */
364 /* making it a common function for all OCTEON models */
365 cleanup_aer_uncorrect_error_status(oct
->pci_dev
);
369 * \brief called when PCI error is detected
370 * @param pdev Pointer to PCI device
371 * @param state The current pci connection state
373 * This function is called after a PCI bus error affecting
374 * this device has been detected.
376 static pci_ers_result_t
liquidio_pcie_error_detected(struct pci_dev
*pdev
,
377 pci_channel_state_t state
)
379 struct octeon_device
*oct
= pci_get_drvdata(pdev
);
381 /* Non-correctable Non-fatal errors */
382 if (state
== pci_channel_io_normal
) {
383 dev_err(&oct
->pci_dev
->dev
, "Non-correctable non-fatal error reported:\n");
384 cleanup_aer_uncorrect_error_status(oct
->pci_dev
);
385 return PCI_ERS_RESULT_CAN_RECOVER
;
388 /* Non-correctable Fatal errors */
389 dev_err(&oct
->pci_dev
->dev
, "Non-correctable FATAL reported by PCI AER driver\n");
392 /* Always return a DISCONNECT. There is no support for recovery but only
393 * for a clean shutdown.
395 return PCI_ERS_RESULT_DISCONNECT
;
399 * \brief mmio handler
400 * @param pdev Pointer to PCI device
402 static pci_ers_result_t
liquidio_pcie_mmio_enabled(
403 struct pci_dev
*pdev
__attribute__((unused
)))
405 /* We should never hit this since we never ask for a reset for a Fatal
406 * Error. We always return DISCONNECT in io_error above.
407 * But play safe and return RECOVERED for now.
409 return PCI_ERS_RESULT_RECOVERED
;
413 * \brief called after the pci bus has been reset.
414 * @param pdev Pointer to PCI device
416 * Restart the card from scratch, as if from a cold-boot. Implementation
417 * resembles the first-half of the octeon_resume routine.
419 static pci_ers_result_t
liquidio_pcie_slot_reset(
420 struct pci_dev
*pdev
__attribute__((unused
)))
422 /* We should never hit this since we never ask for a reset for a Fatal
423 * Error. We always return DISCONNECT in io_error above.
424 * But play safe and return RECOVERED for now.
426 return PCI_ERS_RESULT_RECOVERED
;
430 * \brief called when traffic can start flowing again.
431 * @param pdev Pointer to PCI device
433 * This callback is called when the error recovery driver tells us that
434 * its OK to resume normal operation. Implementation resembles the
435 * second-half of the octeon_resume routine.
437 static void liquidio_pcie_resume(struct pci_dev
*pdev
__attribute__((unused
)))
439 /* Nothing to be done here. */
444 * \brief called when suspending
445 * @param pdev Pointer to PCI device
446 * @param state state to suspend to
448 static int liquidio_suspend(struct pci_dev
*pdev
__attribute__((unused
)),
449 pm_message_t state
__attribute__((unused
)))
455 * \brief called when resuming
456 * @param pdev Pointer to PCI device
458 static int liquidio_resume(struct pci_dev
*pdev
__attribute__((unused
)))
464 /* For PCI-E Advanced Error Recovery (AER) Interface */
465 static const struct pci_error_handlers liquidio_err_handler
= {
466 .error_detected
= liquidio_pcie_error_detected
,
467 .mmio_enabled
= liquidio_pcie_mmio_enabled
,
468 .slot_reset
= liquidio_pcie_slot_reset
,
469 .resume
= liquidio_pcie_resume
,
472 static const struct pci_device_id liquidio_pci_tbl
[] = {
474 PCI_VENDOR_ID_CAVIUM
, 0x91, PCI_ANY_ID
, PCI_ANY_ID
, 0, 0, 0
477 PCI_VENDOR_ID_CAVIUM
, 0x92, PCI_ANY_ID
, PCI_ANY_ID
, 0, 0, 0
480 PCI_VENDOR_ID_CAVIUM
, 0x9702, PCI_ANY_ID
, PCI_ANY_ID
, 0, 0, 0
486 MODULE_DEVICE_TABLE(pci
, liquidio_pci_tbl
);
488 static struct pci_driver liquidio_pci_driver
= {
490 .id_table
= liquidio_pci_tbl
,
491 .probe
= liquidio_probe
,
492 .remove
= liquidio_remove
,
493 .err_handler
= &liquidio_err_handler
, /* For AER */
496 .suspend
= liquidio_suspend
,
497 .resume
= liquidio_resume
,
502 * \brief register PCI driver
504 static int liquidio_init_pci(void)
506 return pci_register_driver(&liquidio_pci_driver
);
510 * \brief unregister PCI driver
512 static void liquidio_deinit_pci(void)
514 pci_unregister_driver(&liquidio_pci_driver
);
518 * \brief check interface state
519 * @param lio per-network private data
520 * @param state_flag flag state to check
522 static inline int ifstate_check(struct lio
*lio
, int state_flag
)
524 return atomic_read(&lio
->ifstate
) & state_flag
;
528 * \brief set interface state
529 * @param lio per-network private data
530 * @param state_flag flag state to set
532 static inline void ifstate_set(struct lio
*lio
, int state_flag
)
534 atomic_set(&lio
->ifstate
, (atomic_read(&lio
->ifstate
) | state_flag
));
538 * \brief clear interface state
539 * @param lio per-network private data
540 * @param state_flag flag state to clear
542 static inline void ifstate_reset(struct lio
*lio
, int state_flag
)
544 atomic_set(&lio
->ifstate
, (atomic_read(&lio
->ifstate
) & ~(state_flag
)));
548 * \brief Stop Tx queues
549 * @param netdev network device
551 static inline void txqs_stop(struct net_device
*netdev
)
553 if (netif_is_multiqueue(netdev
)) {
556 for (i
= 0; i
< netdev
->num_tx_queues
; i
++)
557 netif_stop_subqueue(netdev
, i
);
559 netif_stop_queue(netdev
);
564 * \brief Start Tx queues
565 * @param netdev network device
567 static inline void txqs_start(struct net_device
*netdev
)
569 if (netif_is_multiqueue(netdev
)) {
572 for (i
= 0; i
< netdev
->num_tx_queues
; i
++)
573 netif_start_subqueue(netdev
, i
);
575 netif_start_queue(netdev
);
580 * \brief Wake Tx queues
581 * @param netdev network device
583 static inline void txqs_wake(struct net_device
*netdev
)
585 struct lio
*lio
= GET_LIO(netdev
);
587 if (netif_is_multiqueue(netdev
)) {
590 for (i
= 0; i
< netdev
->num_tx_queues
; i
++) {
591 int qno
= lio
->linfo
.txpciq
[i
%
592 (lio
->linfo
.num_txpciq
)].s
.q_no
;
594 if (__netif_subqueue_stopped(netdev
, i
)) {
595 INCR_INSTRQUEUE_PKT_COUNT(lio
->oct_dev
, qno
,
597 netif_wake_subqueue(netdev
, i
);
601 INCR_INSTRQUEUE_PKT_COUNT(lio
->oct_dev
, lio
->txq
,
603 netif_wake_queue(netdev
);
608 * \brief Stop Tx queue
609 * @param netdev network device
611 static void stop_txq(struct net_device
*netdev
)
617 * \brief Start Tx queue
618 * @param netdev network device
620 static void start_txq(struct net_device
*netdev
)
622 struct lio
*lio
= GET_LIO(netdev
);
624 if (lio
->linfo
.link
.s
.link_up
) {
631 * \brief Wake a queue
632 * @param netdev network device
633 * @param q which queue to wake
635 static inline void wake_q(struct net_device
*netdev
, int q
)
637 if (netif_is_multiqueue(netdev
))
638 netif_wake_subqueue(netdev
, q
);
640 netif_wake_queue(netdev
);
644 * \brief Stop a queue
645 * @param netdev network device
646 * @param q which queue to stop
648 static inline void stop_q(struct net_device
*netdev
, int q
)
650 if (netif_is_multiqueue(netdev
))
651 netif_stop_subqueue(netdev
, q
);
653 netif_stop_queue(netdev
);
657 * \brief Check Tx queue status, and take appropriate action
658 * @param lio per-network private data
659 * @returns 0 if full, number of queues woken up otherwise
661 static inline int check_txq_status(struct lio
*lio
)
665 if (netif_is_multiqueue(lio
->netdev
)) {
666 int numqs
= lio
->netdev
->num_tx_queues
;
669 /* check each sub-queue state */
670 for (q
= 0; q
< numqs
; q
++) {
671 iq
= lio
->linfo
.txpciq
[q
%
672 (lio
->linfo
.num_txpciq
)].s
.q_no
;
673 if (octnet_iq_is_full(lio
->oct_dev
, iq
))
675 if (__netif_subqueue_stopped(lio
->netdev
, q
)) {
676 wake_q(lio
->netdev
, q
);
677 INCR_INSTRQUEUE_PKT_COUNT(lio
->oct_dev
, iq
,
683 if (octnet_iq_is_full(lio
->oct_dev
, lio
->txq
))
685 wake_q(lio
->netdev
, lio
->txq
);
686 INCR_INSTRQUEUE_PKT_COUNT(lio
->oct_dev
, lio
->txq
,
694 * Remove the node at the head of the list. The list would be empty at
695 * the end of this call if there are no more nodes in the list.
697 static inline struct list_head
*list_delete_head(struct list_head
*root
)
699 struct list_head
*node
;
701 if ((root
->prev
== root
) && (root
->next
== root
))
713 * \brief Delete gather lists
714 * @param lio per-network private data
716 static void delete_glists(struct lio
*lio
)
718 struct octnic_gather
*g
;
724 for (i
= 0; i
< lio
->linfo
.num_txpciq
; i
++) {
726 g
= (struct octnic_gather
*)
727 list_delete_head(&lio
->glist
[i
]);
730 dma_unmap_single(&lio
->oct_dev
->
735 kfree((void *)((unsigned long)g
->sg
-
743 kfree((void *)lio
->glist
);
747 * \brief Setup gather lists
748 * @param lio per-network private data
750 static int setup_glists(struct octeon_device
*oct
, struct lio
*lio
, int num_iqs
)
753 struct octnic_gather
*g
;
755 lio
->glist_lock
= kcalloc(num_iqs
, sizeof(*lio
->glist_lock
),
757 if (!lio
->glist_lock
)
760 lio
->glist
= kcalloc(num_iqs
, sizeof(*lio
->glist
),
763 kfree((void *)lio
->glist_lock
);
767 for (i
= 0; i
< num_iqs
; i
++) {
768 int numa_node
= cpu_to_node(i
% num_online_cpus());
770 spin_lock_init(&lio
->glist_lock
[i
]);
772 INIT_LIST_HEAD(&lio
->glist
[i
]);
774 for (j
= 0; j
< lio
->tx_qsize
; j
++) {
775 g
= kzalloc_node(sizeof(*g
), GFP_KERNEL
,
778 g
= kzalloc(sizeof(*g
), GFP_KERNEL
);
782 g
->sg_size
= ((ROUNDUP4(OCTNIC_MAX_SG
) >> 2) *
785 g
->sg
= kmalloc_node(g
->sg_size
+ 8,
786 GFP_KERNEL
, numa_node
);
788 g
->sg
= kmalloc(g
->sg_size
+ 8, GFP_KERNEL
);
794 /* The gather component should be aligned on 64-bit
797 if (((unsigned long)g
->sg
) & 7) {
798 g
->adjust
= 8 - (((unsigned long)g
->sg
) & 7);
799 g
->sg
= (struct octeon_sg_entry
*)
800 ((unsigned long)g
->sg
+ g
->adjust
);
802 g
->sg_dma_ptr
= dma_map_single(&oct
->pci_dev
->dev
,
805 if (dma_mapping_error(&oct
->pci_dev
->dev
,
807 kfree((void *)((unsigned long)g
->sg
-
813 list_add_tail(&g
->list
, &lio
->glist
[i
]);
816 if (j
!= lio
->tx_qsize
) {
826 * \brief Print link information
827 * @param netdev network device
829 static void print_link_info(struct net_device
*netdev
)
831 struct lio
*lio
= GET_LIO(netdev
);
833 if (atomic_read(&lio
->ifstate
) & LIO_IFSTATE_REGISTERED
) {
834 struct oct_link_info
*linfo
= &lio
->linfo
;
836 if (linfo
->link
.s
.link_up
) {
837 netif_info(lio
, link
, lio
->netdev
, "%d Mbps %s Duplex UP\n",
839 (linfo
->link
.s
.duplex
) ? "Full" : "Half");
841 netif_info(lio
, link
, lio
->netdev
, "Link Down\n");
847 * \brief Update link status
848 * @param netdev network device
849 * @param ls link status structure
851 * Called on receipt of a link status response from the core application to
852 * update each interface's link status.
854 static inline void update_link_status(struct net_device
*netdev
,
855 union oct_link_status
*ls
)
857 struct lio
*lio
= GET_LIO(netdev
);
858 int changed
= (lio
->linfo
.link
.u64
!= ls
->u64
);
860 lio
->linfo
.link
.u64
= ls
->u64
;
862 if ((lio
->intf_open
) && (changed
)) {
863 print_link_info(netdev
);
866 if (lio
->linfo
.link
.s
.link_up
) {
867 netif_carrier_on(netdev
);
868 /* start_txq(netdev); */
871 netif_carrier_off(netdev
);
877 /* Runs in interrupt context. */
878 static void update_txq_status(struct octeon_device
*oct
, int iq_num
)
880 struct net_device
*netdev
;
882 struct octeon_instr_queue
*iq
= oct
->instr_queue
[iq_num
];
884 /*octeon_update_iq_read_idx(oct, iq);*/
886 netdev
= oct
->props
[iq
->ifidx
].netdev
;
888 /* This is needed because the first IQ does not have
889 * a netdev associated with it.
894 lio
= GET_LIO(netdev
);
895 if (netif_is_multiqueue(netdev
)) {
896 if (__netif_subqueue_stopped(netdev
, iq
->q_index
) &&
897 lio
->linfo
.link
.s
.link_up
&&
898 (!octnet_iq_is_full(oct
, iq_num
))) {
899 INCR_INSTRQUEUE_PKT_COUNT(lio
->oct_dev
, iq_num
,
901 netif_wake_subqueue(netdev
, iq
->q_index
);
903 if (!octnet_iq_is_full(oct
, lio
->txq
)) {
904 INCR_INSTRQUEUE_PKT_COUNT(lio
->oct_dev
,
907 wake_q(netdev
, lio
->txq
);
914 * \brief Droq packet processor sceduler
915 * @param oct octeon device
918 void liquidio_schedule_droq_pkt_handlers(struct octeon_device
*oct
)
920 struct octeon_device_priv
*oct_priv
=
921 (struct octeon_device_priv
*)oct
->priv
;
923 struct octeon_droq
*droq
;
925 if (oct
->int_status
& OCT_DEV_INTR_PKT_DATA
) {
926 for (oq_no
= 0; oq_no
< MAX_OCTEON_OUTPUT_QUEUES(oct
);
928 if (!(oct
->droq_intr
& (1ULL << oq_no
)))
931 droq
= oct
->droq
[oq_no
];
933 if (droq
->ops
.poll_mode
) {
934 droq
->ops
.napi_fn(droq
);
935 oct_priv
->napi_mask
|= (1 << oq_no
);
937 tasklet_schedule(&oct_priv
->droq_tasklet
);
944 * \brief Interrupt handler for octeon
946 * @param dev octeon device
949 irqreturn_t
liquidio_intr_handler(int irq
__attribute__((unused
)), void *dev
)
951 struct octeon_device
*oct
= (struct octeon_device
*)dev
;
954 /* Disable our interrupts for the duration of ISR */
955 oct
->fn_list
.disable_interrupt(oct
->chip
);
957 ret
= oct
->fn_list
.process_interrupt_regs(oct
);
959 if (ret
== IRQ_HANDLED
)
960 liquidio_schedule_droq_pkt_handlers(oct
);
962 /* Re-enable our interrupts */
963 if (!(atomic_read(&oct
->status
) == OCT_DEV_IN_RESET
))
964 oct
->fn_list
.enable_interrupt(oct
->chip
);
970 * \brief Setup interrupt for octeon device
971 * @param oct octeon device
973 * Enable interrupt in Octeon device as given in the PCI interrupt mask.
975 static int octeon_setup_interrupt(struct octeon_device
*oct
)
979 err
= pci_enable_msi(oct
->pci_dev
);
981 dev_warn(&oct
->pci_dev
->dev
, "Reverting to legacy interrupts. Error: %d\n",
984 oct
->flags
|= LIO_FLAG_MSI_ENABLED
;
986 irqret
= request_irq(oct
->pci_dev
->irq
, liquidio_intr_handler
,
987 IRQF_SHARED
, "octeon", oct
);
989 if (oct
->flags
& LIO_FLAG_MSI_ENABLED
)
990 pci_disable_msi(oct
->pci_dev
);
991 dev_err(&oct
->pci_dev
->dev
, "Request IRQ failed with code: %d\n",
1000 * \brief PCI probe handler
1001 * @param pdev PCI device structure
1005 liquidio_probe(struct pci_dev
*pdev
,
1006 const struct pci_device_id
*ent
__attribute__((unused
)))
1008 struct octeon_device
*oct_dev
= NULL
;
1009 struct handshake
*hs
;
1011 oct_dev
= octeon_allocate_device(pdev
->device
,
1012 sizeof(struct octeon_device_priv
));
1014 dev_err(&pdev
->dev
, "Unable to allocate device\n");
1018 dev_info(&pdev
->dev
, "Initializing device %x:%x.\n",
1019 (u32
)pdev
->vendor
, (u32
)pdev
->device
);
1021 /* Assign octeon_device for this device to the private data area. */
1022 pci_set_drvdata(pdev
, oct_dev
);
1024 /* set linux specific device pointer */
1025 oct_dev
->pci_dev
= (void *)pdev
;
1027 hs
= &handshake
[oct_dev
->octeon_id
];
1028 init_completion(&hs
->init
);
1029 init_completion(&hs
->started
);
1032 if (oct_dev
->octeon_id
== 0)
1033 /* first LiquidIO NIC is detected */
1034 complete(&first_stage
);
1036 if (octeon_device_init(oct_dev
)) {
1037 liquidio_remove(pdev
);
1041 oct_dev
->rx_pause
= 1;
1042 oct_dev
->tx_pause
= 1;
1044 dev_dbg(&oct_dev
->pci_dev
->dev
, "Device is ready\n");
1050 *\brief Destroy resources associated with octeon device
1051 * @param pdev PCI device structure
1054 static void octeon_destroy_resources(struct octeon_device
*oct
)
1057 struct octeon_device_priv
*oct_priv
=
1058 (struct octeon_device_priv
*)oct
->priv
;
1060 struct handshake
*hs
;
1062 switch (atomic_read(&oct
->status
)) {
1063 case OCT_DEV_RUNNING
:
1064 case OCT_DEV_CORE_OK
:
1066 /* No more instructions will be forwarded. */
1067 atomic_set(&oct
->status
, OCT_DEV_IN_RESET
);
1069 oct
->app_mode
= CVM_DRV_INVALID_APP
;
1070 dev_dbg(&oct
->pci_dev
->dev
, "Device state is now %s\n",
1071 lio_get_state_string(&oct
->status
));
1073 schedule_timeout_uninterruptible(HZ
/ 10);
1076 case OCT_DEV_HOST_OK
:
1079 case OCT_DEV_CONSOLE_INIT_DONE
:
1080 /* Remove any consoles */
1081 octeon_remove_consoles(oct
);
1084 case OCT_DEV_IO_QUEUES_DONE
:
1085 if (wait_for_pending_requests(oct
))
1086 dev_err(&oct
->pci_dev
->dev
, "There were pending requests\n");
1088 if (lio_wait_for_instr_fetch(oct
))
1089 dev_err(&oct
->pci_dev
->dev
, "IQ had pending instructions\n");
1091 /* Disable the input and output queues now. No more packets will
1092 * arrive from Octeon, but we should wait for all packet
1093 * processing to finish.
1095 oct
->fn_list
.disable_io_queues(oct
);
1097 if (lio_wait_for_oq_pkts(oct
))
1098 dev_err(&oct
->pci_dev
->dev
, "OQ had pending packets\n");
1100 /* Disable interrupts */
1101 oct
->fn_list
.disable_interrupt(oct
->chip
);
1103 /* Release the interrupt line */
1104 free_irq(oct
->pci_dev
->irq
, oct
);
1106 if (oct
->flags
& LIO_FLAG_MSI_ENABLED
)
1107 pci_disable_msi(oct
->pci_dev
);
1110 case OCT_DEV_IN_RESET
:
1111 case OCT_DEV_DROQ_INIT_DONE
:
1112 /*atomic_set(&oct->status, OCT_DEV_DROQ_INIT_DONE);*/
1114 for (i
= 0; i
< MAX_OCTEON_OUTPUT_QUEUES(oct
); i
++) {
1115 if (!(oct
->io_qmask
.oq
& (1ULL << i
)))
1117 octeon_delete_droq(oct
, i
);
1120 /* Force any pending handshakes to complete */
1121 for (i
= 0; i
< MAX_OCTEON_DEVICES
; i
++) {
1125 handshake
[oct
->octeon_id
].init_ok
= 0;
1126 complete(&handshake
[oct
->octeon_id
].init
);
1127 handshake
[oct
->octeon_id
].started_ok
= 0;
1128 complete(&handshake
[oct
->octeon_id
].started
);
1133 case OCT_DEV_RESP_LIST_INIT_DONE
:
1134 octeon_delete_response_list(oct
);
1137 case OCT_DEV_SC_BUFF_POOL_INIT_DONE
:
1138 octeon_free_sc_buffer_pool(oct
);
1141 case OCT_DEV_INSTR_QUEUE_INIT_DONE
:
1142 for (i
= 0; i
< MAX_OCTEON_INSTR_QUEUES(oct
); i
++) {
1143 if (!(oct
->io_qmask
.iq
& (1ULL << i
)))
1145 octeon_delete_instr_queue(oct
, i
);
1149 case OCT_DEV_DISPATCH_INIT_DONE
:
1150 octeon_delete_dispatch_list(oct
);
1151 cancel_delayed_work_sync(&oct
->nic_poll_work
.work
);
1154 case OCT_DEV_PCI_MAP_DONE
:
1156 /* Soft reset the octeon device before exiting */
1157 oct
->fn_list
.soft_reset(oct
);
1159 octeon_unmap_pci_barx(oct
, 0);
1160 octeon_unmap_pci_barx(oct
, 1);
1163 case OCT_DEV_BEGIN_STATE
:
1164 /* Disable the device, releasing the PCI INT */
1165 pci_disable_device(oct
->pci_dev
);
1167 /* Nothing to be done here either */
1169 } /* end switch (oct->status) */
1171 tasklet_kill(&oct_priv
->droq_tasklet
);
1175 * \brief Send Rx control command
1176 * @param lio per-network private data
1177 * @param start_stop whether to start or stop
1179 static void send_rx_ctrl_cmd(struct lio
*lio
, int start_stop
)
1181 struct octnic_ctrl_pkt nctrl
;
1183 memset(&nctrl
, 0, sizeof(struct octnic_ctrl_pkt
));
1185 nctrl
.ncmd
.s
.cmd
= OCTNET_CMD_RX_CTL
;
1186 nctrl
.ncmd
.s
.param1
= start_stop
;
1187 nctrl
.iq_no
= lio
->linfo
.txpciq
[0].s
.q_no
;
1188 nctrl
.netpndev
= (u64
)lio
->netdev
;
1190 if (octnet_send_nic_ctrl_pkt(lio
->oct_dev
, &nctrl
) < 0)
1191 netif_info(lio
, rx_err
, lio
->netdev
, "Failed to send RX Control message\n");
1195 * \brief Destroy NIC device interface
1196 * @param oct octeon device
1197 * @param ifidx which interface to destroy
1199 * Cleanup associated with each interface for an Octeon device when NIC
1200 * module is being unloaded or if initialization fails during load.
1202 static void liquidio_destroy_nic_device(struct octeon_device
*oct
, int ifidx
)
1204 struct net_device
*netdev
= oct
->props
[ifidx
].netdev
;
1206 struct napi_struct
*napi
, *n
;
1209 dev_err(&oct
->pci_dev
->dev
, "%s No netdevice ptr for index %d\n",
1214 lio
= GET_LIO(netdev
);
1216 dev_dbg(&oct
->pci_dev
->dev
, "NIC device cleanup\n");
1218 send_rx_ctrl_cmd(lio
, 0);
1220 if (atomic_read(&lio
->ifstate
) & LIO_IFSTATE_RUNNING
)
1223 if (oct
->props
[lio
->ifidx
].napi_enabled
== 1) {
1224 list_for_each_entry_safe(napi
, n
, &netdev
->napi_list
, dev_list
)
1227 oct
->props
[lio
->ifidx
].napi_enabled
= 0;
1230 if (atomic_read(&lio
->ifstate
) & LIO_IFSTATE_REGISTERED
)
1231 unregister_netdev(netdev
);
1235 free_netdev(netdev
);
1237 oct
->props
[ifidx
].gmxport
= -1;
1239 oct
->props
[ifidx
].netdev
= NULL
;
1243 * \brief Stop complete NIC functionality
1244 * @param oct octeon device
1246 static int liquidio_stop_nic_module(struct octeon_device
*oct
)
1251 dev_dbg(&oct
->pci_dev
->dev
, "Stopping network interfaces\n");
1252 if (!oct
->ifcount
) {
1253 dev_err(&oct
->pci_dev
->dev
, "Init for Octeon was not completed\n");
1257 spin_lock_bh(&oct
->cmd_resp_wqlock
);
1258 oct
->cmd_resp_state
= OCT_DRV_OFFLINE
;
1259 spin_unlock_bh(&oct
->cmd_resp_wqlock
);
1261 for (i
= 0; i
< oct
->ifcount
; i
++) {
1262 lio
= GET_LIO(oct
->props
[i
].netdev
);
1263 for (j
= 0; j
< lio
->linfo
.num_rxpciq
; j
++)
1264 octeon_unregister_droq_ops(oct
,
1265 lio
->linfo
.rxpciq
[j
].s
.q_no
);
1268 for (i
= 0; i
< oct
->ifcount
; i
++)
1269 liquidio_destroy_nic_device(oct
, i
);
1271 dev_dbg(&oct
->pci_dev
->dev
, "Network interfaces stopped\n");
1276 * \brief Cleans up resources at unload time
1277 * @param pdev PCI device structure
1279 static void liquidio_remove(struct pci_dev
*pdev
)
1281 struct octeon_device
*oct_dev
= pci_get_drvdata(pdev
);
1283 dev_dbg(&oct_dev
->pci_dev
->dev
, "Stopping device\n");
1285 if (oct_dev
->app_mode
&& (oct_dev
->app_mode
== CVM_DRV_NIC_APP
))
1286 liquidio_stop_nic_module(oct_dev
);
1288 /* Reset the octeon device and cleanup all memory allocated for
1289 * the octeon device by driver.
1291 octeon_destroy_resources(oct_dev
);
1293 dev_info(&oct_dev
->pci_dev
->dev
, "Device removed\n");
1295 /* This octeon device has been removed. Update the global
1296 * data structure to reflect this. Free the device structure.
1298 octeon_free_device_mem(oct_dev
);
1302 * \brief Identify the Octeon device and to map the BAR address space
1303 * @param oct octeon device
1305 static int octeon_chip_specific_setup(struct octeon_device
*oct
)
1311 pci_read_config_dword(oct
->pci_dev
, 0, &dev_id
);
1312 pci_read_config_dword(oct
->pci_dev
, 8, &rev_id
);
1313 oct
->rev_id
= rev_id
& 0xff;
1316 case OCTEON_CN68XX_PCIID
:
1317 oct
->chip_id
= OCTEON_CN68XX
;
1318 ret
= lio_setup_cn68xx_octeon_device(oct
);
1322 case OCTEON_CN66XX_PCIID
:
1323 oct
->chip_id
= OCTEON_CN66XX
;
1324 ret
= lio_setup_cn66xx_octeon_device(oct
);
1328 case OCTEON_CN23XX_PCIID_PF
:
1329 oct
->chip_id
= OCTEON_CN23XX_PF_VID
;
1330 ret
= setup_cn23xx_octeon_pf_device(oct
);
1336 dev_err(&oct
->pci_dev
->dev
, "Unknown device found (dev_id: %x)\n",
1341 dev_info(&oct
->pci_dev
->dev
, "%s PASS%d.%d %s Version: %s\n", s
,
1342 OCTEON_MAJOR_REV(oct
),
1343 OCTEON_MINOR_REV(oct
),
1344 octeon_get_conf(oct
)->card_name
,
1351 * \brief PCI initialization for each Octeon device.
1352 * @param oct octeon device
1354 static int octeon_pci_os_setup(struct octeon_device
*oct
)
1356 /* setup PCI stuff first */
1357 if (pci_enable_device(oct
->pci_dev
)) {
1358 dev_err(&oct
->pci_dev
->dev
, "pci_enable_device failed\n");
1362 if (dma_set_mask_and_coherent(&oct
->pci_dev
->dev
, DMA_BIT_MASK(64))) {
1363 dev_err(&oct
->pci_dev
->dev
, "Unexpected DMA device capability\n");
1367 /* Enable PCI DMA Master. */
1368 pci_set_master(oct
->pci_dev
);
1373 static inline int skb_iq(struct lio
*lio
, struct sk_buff
*skb
)
1377 if (netif_is_multiqueue(lio
->netdev
))
1378 q
= skb
->queue_mapping
% lio
->linfo
.num_txpciq
;
1384 * \brief Check Tx queue state for a given network buffer
1385 * @param lio per-network private data
1386 * @param skb network buffer
1388 static inline int check_txq_state(struct lio
*lio
, struct sk_buff
*skb
)
1392 if (netif_is_multiqueue(lio
->netdev
)) {
1393 q
= skb
->queue_mapping
;
1394 iq
= lio
->linfo
.txpciq
[(q
% (lio
->linfo
.num_txpciq
))].s
.q_no
;
1400 if (octnet_iq_is_full(lio
->oct_dev
, iq
))
1403 if (__netif_subqueue_stopped(lio
->netdev
, q
)) {
1404 INCR_INSTRQUEUE_PKT_COUNT(lio
->oct_dev
, iq
, tx_restart
, 1);
1405 wake_q(lio
->netdev
, q
);
1411 * \brief Unmap and free network buffer
1414 static void free_netbuf(void *buf
)
1416 struct sk_buff
*skb
;
1417 struct octnet_buf_free_info
*finfo
;
1420 finfo
= (struct octnet_buf_free_info
*)buf
;
1424 dma_unmap_single(&lio
->oct_dev
->pci_dev
->dev
, finfo
->dptr
, skb
->len
,
1427 check_txq_state(lio
, skb
);
1429 tx_buffer_free(skb
);
1433 * \brief Unmap and free gather buffer
1436 static void free_netsgbuf(void *buf
)
1438 struct octnet_buf_free_info
*finfo
;
1439 struct sk_buff
*skb
;
1441 struct octnic_gather
*g
;
1444 finfo
= (struct octnet_buf_free_info
*)buf
;
1448 frags
= skb_shinfo(skb
)->nr_frags
;
1450 dma_unmap_single(&lio
->oct_dev
->pci_dev
->dev
,
1451 g
->sg
[0].ptr
[0], (skb
->len
- skb
->data_len
),
1456 struct skb_frag_struct
*frag
= &skb_shinfo(skb
)->frags
[i
- 1];
1458 pci_unmap_page((lio
->oct_dev
)->pci_dev
,
1459 g
->sg
[(i
>> 2)].ptr
[(i
& 3)],
1460 frag
->size
, DMA_TO_DEVICE
);
1464 dma_sync_single_for_cpu(&lio
->oct_dev
->pci_dev
->dev
,
1465 g
->sg_dma_ptr
, g
->sg_size
, DMA_TO_DEVICE
);
1467 iq
= skb_iq(lio
, skb
);
1468 spin_lock(&lio
->glist_lock
[iq
]);
1469 list_add_tail(&g
->list
, &lio
->glist
[iq
]);
1470 spin_unlock(&lio
->glist_lock
[iq
]);
1472 check_txq_state(lio
, skb
); /* mq support: sub-queue state check */
1474 tx_buffer_free(skb
);
1478 * \brief Unmap and free gather buffer with response
1481 static void free_netsgbuf_with_resp(void *buf
)
1483 struct octeon_soft_command
*sc
;
1484 struct octnet_buf_free_info
*finfo
;
1485 struct sk_buff
*skb
;
1487 struct octnic_gather
*g
;
1490 sc
= (struct octeon_soft_command
*)buf
;
1491 skb
= (struct sk_buff
*)sc
->callback_arg
;
1492 finfo
= (struct octnet_buf_free_info
*)&skb
->cb
;
1496 frags
= skb_shinfo(skb
)->nr_frags
;
1498 dma_unmap_single(&lio
->oct_dev
->pci_dev
->dev
,
1499 g
->sg
[0].ptr
[0], (skb
->len
- skb
->data_len
),
1504 struct skb_frag_struct
*frag
= &skb_shinfo(skb
)->frags
[i
- 1];
1506 pci_unmap_page((lio
->oct_dev
)->pci_dev
,
1507 g
->sg
[(i
>> 2)].ptr
[(i
& 3)],
1508 frag
->size
, DMA_TO_DEVICE
);
1512 dma_sync_single_for_cpu(&lio
->oct_dev
->pci_dev
->dev
,
1513 g
->sg_dma_ptr
, g
->sg_size
, DMA_TO_DEVICE
);
1515 iq
= skb_iq(lio
, skb
);
1517 spin_lock(&lio
->glist_lock
[iq
]);
1518 list_add_tail(&g
->list
, &lio
->glist
[iq
]);
1519 spin_unlock(&lio
->glist_lock
[iq
]);
1521 /* Don't free the skb yet */
1523 check_txq_state(lio
, skb
);
1527 * \brief Adjust ptp frequency
1528 * @param ptp PTP clock info
1529 * @param ppb how much to adjust by, in parts-per-billion
1531 static int liquidio_ptp_adjfreq(struct ptp_clock_info
*ptp
, s32 ppb
)
1533 struct lio
*lio
= container_of(ptp
, struct lio
, ptp_info
);
1534 struct octeon_device
*oct
= (struct octeon_device
*)lio
->oct_dev
;
1536 unsigned long flags
;
1537 bool neg_adj
= false;
1544 /* The hardware adds the clock compensation value to the
1545 * PTP clock on every coprocessor clock cycle, so we
1546 * compute the delta in terms of coprocessor clocks.
1548 delta
= (u64
)ppb
<< 32;
1549 do_div(delta
, oct
->coproc_clock_rate
);
1551 spin_lock_irqsave(&lio
->ptp_lock
, flags
);
1552 comp
= lio_pci_readq(oct
, CN6XXX_MIO_PTP_CLOCK_COMP
);
1557 lio_pci_writeq(oct
, comp
, CN6XXX_MIO_PTP_CLOCK_COMP
);
1558 spin_unlock_irqrestore(&lio
->ptp_lock
, flags
);
1564 * \brief Adjust ptp time
1565 * @param ptp PTP clock info
1566 * @param delta how much to adjust by, in nanosecs
1568 static int liquidio_ptp_adjtime(struct ptp_clock_info
*ptp
, s64 delta
)
1570 unsigned long flags
;
1571 struct lio
*lio
= container_of(ptp
, struct lio
, ptp_info
);
1573 spin_lock_irqsave(&lio
->ptp_lock
, flags
);
1574 lio
->ptp_adjust
+= delta
;
1575 spin_unlock_irqrestore(&lio
->ptp_lock
, flags
);
1581 * \brief Get hardware clock time, including any adjustment
1582 * @param ptp PTP clock info
1583 * @param ts timespec
1585 static int liquidio_ptp_gettime(struct ptp_clock_info
*ptp
,
1586 struct timespec64
*ts
)
1589 unsigned long flags
;
1590 struct lio
*lio
= container_of(ptp
, struct lio
, ptp_info
);
1591 struct octeon_device
*oct
= (struct octeon_device
*)lio
->oct_dev
;
1593 spin_lock_irqsave(&lio
->ptp_lock
, flags
);
1594 ns
= lio_pci_readq(oct
, CN6XXX_MIO_PTP_CLOCK_HI
);
1595 ns
+= lio
->ptp_adjust
;
1596 spin_unlock_irqrestore(&lio
->ptp_lock
, flags
);
1598 *ts
= ns_to_timespec64(ns
);
1604 * \brief Set hardware clock time. Reset adjustment
1605 * @param ptp PTP clock info
1606 * @param ts timespec
1608 static int liquidio_ptp_settime(struct ptp_clock_info
*ptp
,
1609 const struct timespec64
*ts
)
1612 unsigned long flags
;
1613 struct lio
*lio
= container_of(ptp
, struct lio
, ptp_info
);
1614 struct octeon_device
*oct
= (struct octeon_device
*)lio
->oct_dev
;
1616 ns
= timespec_to_ns(ts
);
1618 spin_lock_irqsave(&lio
->ptp_lock
, flags
);
1619 lio_pci_writeq(oct
, ns
, CN6XXX_MIO_PTP_CLOCK_HI
);
1620 lio
->ptp_adjust
= 0;
1621 spin_unlock_irqrestore(&lio
->ptp_lock
, flags
);
1627 * \brief Check if PTP is enabled
1628 * @param ptp PTP clock info
1630 * @param on is it on
1633 liquidio_ptp_enable(struct ptp_clock_info
*ptp
__attribute__((unused
)),
1634 struct ptp_clock_request
*rq
__attribute__((unused
)),
1635 int on
__attribute__((unused
)))
1641 * \brief Open PTP clock source
1642 * @param netdev network device
1644 static void oct_ptp_open(struct net_device
*netdev
)
1646 struct lio
*lio
= GET_LIO(netdev
);
1647 struct octeon_device
*oct
= (struct octeon_device
*)lio
->oct_dev
;
1649 spin_lock_init(&lio
->ptp_lock
);
1651 snprintf(lio
->ptp_info
.name
, 16, "%s", netdev
->name
);
1652 lio
->ptp_info
.owner
= THIS_MODULE
;
1653 lio
->ptp_info
.max_adj
= 250000000;
1654 lio
->ptp_info
.n_alarm
= 0;
1655 lio
->ptp_info
.n_ext_ts
= 0;
1656 lio
->ptp_info
.n_per_out
= 0;
1657 lio
->ptp_info
.pps
= 0;
1658 lio
->ptp_info
.adjfreq
= liquidio_ptp_adjfreq
;
1659 lio
->ptp_info
.adjtime
= liquidio_ptp_adjtime
;
1660 lio
->ptp_info
.gettime64
= liquidio_ptp_gettime
;
1661 lio
->ptp_info
.settime64
= liquidio_ptp_settime
;
1662 lio
->ptp_info
.enable
= liquidio_ptp_enable
;
1664 lio
->ptp_adjust
= 0;
1666 lio
->ptp_clock
= ptp_clock_register(&lio
->ptp_info
,
1667 &oct
->pci_dev
->dev
);
1669 if (IS_ERR(lio
->ptp_clock
))
1670 lio
->ptp_clock
= NULL
;
1674 * \brief Init PTP clock
1675 * @param oct octeon device
1677 static void liquidio_ptp_init(struct octeon_device
*oct
)
1679 u64 clock_comp
, cfg
;
1681 clock_comp
= (u64
)NSEC_PER_SEC
<< 32;
1682 do_div(clock_comp
, oct
->coproc_clock_rate
);
1683 lio_pci_writeq(oct
, clock_comp
, CN6XXX_MIO_PTP_CLOCK_COMP
);
1686 cfg
= lio_pci_readq(oct
, CN6XXX_MIO_PTP_CLOCK_CFG
);
1687 lio_pci_writeq(oct
, cfg
| 0x01, CN6XXX_MIO_PTP_CLOCK_CFG
);
1691 * \brief Load firmware to device
1692 * @param oct octeon device
1694 * Maps device to firmware filename, requests firmware, and downloads it
1696 static int load_firmware(struct octeon_device
*oct
)
1699 const struct firmware
*fw
;
1700 char fw_name
[LIO_MAX_FW_FILENAME_LEN
];
1703 if (strncmp(fw_type
, LIO_FW_NAME_TYPE_NONE
,
1704 sizeof(LIO_FW_NAME_TYPE_NONE
)) == 0) {
1705 dev_info(&oct
->pci_dev
->dev
, "Skipping firmware load\n");
1709 if (fw_type
[0] == '\0')
1710 tmp_fw_type
= LIO_FW_NAME_TYPE_NIC
;
1712 tmp_fw_type
= fw_type
;
1714 sprintf(fw_name
, "%s%s%s_%s%s", LIO_FW_DIR
, LIO_FW_BASE_NAME
,
1715 octeon_get_conf(oct
)->card_name
, tmp_fw_type
,
1716 LIO_FW_NAME_SUFFIX
);
1718 ret
= request_firmware(&fw
, fw_name
, &oct
->pci_dev
->dev
);
1720 dev_err(&oct
->pci_dev
->dev
, "Request firmware failed. Could not find file %s.\n.",
1722 release_firmware(fw
);
1726 ret
= octeon_download_firmware(oct
, fw
->data
, fw
->size
);
1728 release_firmware(fw
);
1734 * \brief Setup output queue
1735 * @param oct octeon device
1736 * @param q_no which queue
1737 * @param num_descs how many descriptors
1738 * @param desc_size size of each descriptor
1739 * @param app_ctx application context
1741 static int octeon_setup_droq(struct octeon_device
*oct
, int q_no
, int num_descs
,
1742 int desc_size
, void *app_ctx
)
1746 dev_dbg(&oct
->pci_dev
->dev
, "Creating Droq: %d\n", q_no
);
1747 /* droq creation and local register settings. */
1748 ret_val
= octeon_create_droq(oct
, q_no
, num_descs
, desc_size
, app_ctx
);
1753 dev_dbg(&oct
->pci_dev
->dev
, "Using default droq %d\n", q_no
);
1756 /* tasklet creation for the droq */
1758 /* Enable the droq queues */
1759 octeon_set_droq_pkt_op(oct
, q_no
, 1);
1761 /* Send Credit for Octeon Output queues. Credits are always
1762 * sent after the output queue is enabled.
1764 writel(oct
->droq
[q_no
]->max_count
,
1765 oct
->droq
[q_no
]->pkts_credit_reg
);
1771 * \brief Callback for getting interface configuration
1772 * @param status status of request
1773 * @param buf pointer to resp structure
1775 static void if_cfg_callback(struct octeon_device
*oct
,
1776 u32 status
__attribute__((unused
)),
1779 struct octeon_soft_command
*sc
= (struct octeon_soft_command
*)buf
;
1780 struct liquidio_if_cfg_resp
*resp
;
1781 struct liquidio_if_cfg_context
*ctx
;
1783 resp
= (struct liquidio_if_cfg_resp
*)sc
->virtrptr
;
1784 ctx
= (struct liquidio_if_cfg_context
*)sc
->ctxptr
;
1786 oct
= lio_get_device(ctx
->octeon_id
);
1788 dev_err(&oct
->pci_dev
->dev
, "nic if cfg instruction failed. Status: %llx\n",
1789 CVM_CAST64(resp
->status
));
1790 WRITE_ONCE(ctx
->cond
, 1);
1792 snprintf(oct
->fw_info
.liquidio_firmware_version
, 32, "%s",
1793 resp
->cfg_info
.liquidio_firmware_version
);
1795 /* This barrier is required to be sure that the response has been
1796 * written fully before waking up the handler
1800 wake_up_interruptible(&ctx
->wc
);
1804 * \brief Select queue based on hash
1805 * @param dev Net device
1806 * @param skb sk_buff structure
1807 * @returns selected queue number
1809 static u16
select_q(struct net_device
*dev
, struct sk_buff
*skb
,
1810 void *accel_priv
__attribute__((unused
)),
1811 select_queue_fallback_t fallback
__attribute__((unused
)))
1817 qindex
= skb_tx_hash(dev
, skb
);
1819 return (u16
)(qindex
% (lio
->linfo
.num_txpciq
));
1822 /** Routine to push packets arriving on Octeon interface upto network layer.
1823 * @param oct_id - octeon device id.
1824 * @param skbuff - skbuff struct to be passed to network layer.
1825 * @param len - size of total data received.
1826 * @param rh - Control header associated with the packet
1827 * @param param - additional control data with the packet
1828 * @param arg - farg registered in droq_ops
1831 liquidio_push_packet(u32 octeon_id
__attribute__((unused
)),
1834 union octeon_rh
*rh
,
1838 struct napi_struct
*napi
= param
;
1839 struct sk_buff
*skb
= (struct sk_buff
*)skbuff
;
1840 struct skb_shared_hwtstamps
*shhwtstamps
;
1843 struct net_device
*netdev
= (struct net_device
*)arg
;
1844 struct octeon_droq
*droq
= container_of(param
, struct octeon_droq
,
1847 int packet_was_received
;
1848 struct lio
*lio
= GET_LIO(netdev
);
1849 struct octeon_device
*oct
= lio
->oct_dev
;
1851 /* Do not proceed if the interface is not in RUNNING state. */
1852 if (!ifstate_check(lio
, LIO_IFSTATE_RUNNING
)) {
1853 recv_buffer_free(skb
);
1854 droq
->stats
.rx_dropped
++;
1860 skb_record_rx_queue(skb
, droq
->q_no
);
1861 if (likely(len
> MIN_SKB_SIZE
)) {
1862 struct octeon_skb_page_info
*pg_info
;
1865 pg_info
= ((struct octeon_skb_page_info
*)(skb
->cb
));
1866 if (pg_info
->page
) {
1867 /* For Paged allocation use the frags */
1868 va
= page_address(pg_info
->page
) +
1869 pg_info
->page_offset
;
1870 memcpy(skb
->data
, va
, MIN_SKB_SIZE
);
1871 skb_put(skb
, MIN_SKB_SIZE
);
1872 skb_add_rx_frag(skb
, skb_shinfo(skb
)->nr_frags
,
1874 pg_info
->page_offset
+
1880 struct octeon_skb_page_info
*pg_info
=
1881 ((struct octeon_skb_page_info
*)(skb
->cb
));
1882 skb_copy_to_linear_data(skb
, page_address(pg_info
->page
)
1883 + pg_info
->page_offset
, len
);
1885 put_page(pg_info
->page
);
1888 if (((oct
->chip_id
== OCTEON_CN66XX
) ||
1889 (oct
->chip_id
== OCTEON_CN68XX
)) &&
1891 if (rh
->r_dh
.has_hwtstamp
) {
1892 /* timestamp is included from the hardware at
1893 * the beginning of the packet.
1896 (lio
, LIO_IFSTATE_RX_TIMESTAMP_ENABLED
)) {
1897 /* Nanoseconds are in the first 64-bits
1900 memcpy(&ns
, (skb
->data
), sizeof(ns
));
1901 shhwtstamps
= skb_hwtstamps(skb
);
1902 shhwtstamps
->hwtstamp
=
1906 skb_pull(skb
, sizeof(ns
));
1910 skb
->protocol
= eth_type_trans(skb
, skb
->dev
);
1911 if ((netdev
->features
& NETIF_F_RXCSUM
) &&
1912 (((rh
->r_dh
.encap_on
) &&
1913 (rh
->r_dh
.csum_verified
& CNNIC_TUN_CSUM_VERIFIED
)) ||
1914 (!(rh
->r_dh
.encap_on
) &&
1915 (rh
->r_dh
.csum_verified
& CNNIC_CSUM_VERIFIED
))))
1916 /* checksum has already been verified */
1917 skb
->ip_summed
= CHECKSUM_UNNECESSARY
;
1919 skb
->ip_summed
= CHECKSUM_NONE
;
1921 /* Setting Encapsulation field on basis of status received
1924 if (rh
->r_dh
.encap_on
) {
1925 skb
->encapsulation
= 1;
1926 skb
->csum_level
= 1;
1927 droq
->stats
.rx_vxlan
++;
1930 /* inbound VLAN tag */
1931 if ((netdev
->features
& NETIF_F_HW_VLAN_CTAG_RX
) &&
1932 (rh
->r_dh
.vlan
!= 0)) {
1933 u16 vid
= rh
->r_dh
.vlan
;
1934 u16 priority
= rh
->r_dh
.priority
;
1936 vtag
= priority
<< 13 | vid
;
1937 __vlan_hwaccel_put_tag(skb
, htons(ETH_P_8021Q
), vtag
);
1940 packet_was_received
= napi_gro_receive(napi
, skb
) != GRO_DROP
;
1942 if (packet_was_received
) {
1943 droq
->stats
.rx_bytes_received
+= len
;
1944 droq
->stats
.rx_pkts_received
++;
1945 netdev
->last_rx
= jiffies
;
1947 droq
->stats
.rx_dropped
++;
1948 netif_info(lio
, rx_err
, lio
->netdev
,
1949 "droq:%d error rx_dropped:%llu\n",
1950 droq
->q_no
, droq
->stats
.rx_dropped
);
1954 recv_buffer_free(skb
);
1959 * \brief wrapper for calling napi_schedule
1960 * @param param parameters to pass to napi_schedule
1962 * Used when scheduling on different CPUs
1964 static void napi_schedule_wrapper(void *param
)
1966 struct napi_struct
*napi
= param
;
1968 napi_schedule(napi
);
1972 * \brief callback when receive interrupt occurs and we are in NAPI mode
1973 * @param arg pointer to octeon output queue
1975 static void liquidio_napi_drv_callback(void *arg
)
1977 struct octeon_droq
*droq
= arg
;
1978 int this_cpu
= smp_processor_id();
1980 if (droq
->cpu_id
== this_cpu
) {
1981 napi_schedule(&droq
->napi
);
1983 struct call_single_data
*csd
= &droq
->csd
;
1985 csd
->func
= napi_schedule_wrapper
;
1986 csd
->info
= &droq
->napi
;
1989 smp_call_function_single_async(droq
->cpu_id
, csd
);
1994 * \brief Entry point for NAPI polling
1995 * @param napi NAPI structure
1996 * @param budget maximum number of items to process
1998 static int liquidio_napi_poll(struct napi_struct
*napi
, int budget
)
2000 struct octeon_droq
*droq
;
2002 int tx_done
= 0, iq_no
;
2003 struct octeon_instr_queue
*iq
;
2004 struct octeon_device
*oct
;
2006 droq
= container_of(napi
, struct octeon_droq
, napi
);
2007 oct
= droq
->oct_dev
;
2009 /* Handle Droq descriptors */
2010 work_done
= octeon_process_droq_poll_cmd(oct
, droq
->q_no
,
2011 POLL_EVENT_PROCESS_PKTS
,
2014 /* Flush the instruction queue */
2015 iq
= oct
->instr_queue
[iq_no
];
2017 /* Process iq buffers with in the budget limits */
2018 tx_done
= octeon_flush_iq(oct
, iq
, 1, budget
);
2019 /* Update iq read-index rather than waiting for next interrupt.
2020 * Return back if tx_done is false.
2022 update_txq_status(oct
, iq_no
);
2023 /*tx_done = (iq->flush_index == iq->octeon_read_index);*/
2025 dev_err(&oct
->pci_dev
->dev
, "%s: iq (%d) num invalid\n",
2029 if ((work_done
< budget
) && (tx_done
)) {
2030 napi_complete(napi
);
2031 octeon_process_droq_poll_cmd(droq
->oct_dev
, droq
->q_no
,
2032 POLL_EVENT_ENABLE_INTR
, 0);
2036 return (!tx_done
) ? (budget
) : (work_done
);
2040 * \brief Setup input and output queues
2041 * @param octeon_dev octeon device
2042 * @param ifidx Interface Index
2044 * Note: Queues are with respect to the octeon device. Thus
2045 * an input queue is for egress packets, and output queues
2046 * are for ingress packets.
2048 static inline int setup_io_queues(struct octeon_device
*octeon_dev
,
2051 struct octeon_droq_ops droq_ops
;
2052 struct net_device
*netdev
;
2054 static int cpu_id_modulus
;
2055 struct octeon_droq
*droq
;
2056 struct napi_struct
*napi
;
2057 int q
, q_no
, retval
= 0;
2061 netdev
= octeon_dev
->props
[ifidx
].netdev
;
2063 lio
= GET_LIO(netdev
);
2065 memset(&droq_ops
, 0, sizeof(struct octeon_droq_ops
));
2067 droq_ops
.fptr
= liquidio_push_packet
;
2068 droq_ops
.farg
= (void *)netdev
;
2070 droq_ops
.poll_mode
= 1;
2071 droq_ops
.napi_fn
= liquidio_napi_drv_callback
;
2073 cpu_id_modulus
= num_present_cpus();
2076 for (q
= 0; q
< lio
->linfo
.num_rxpciq
; q
++) {
2077 q_no
= lio
->linfo
.rxpciq
[q
].s
.q_no
;
2078 dev_dbg(&octeon_dev
->pci_dev
->dev
,
2079 "setup_io_queues index:%d linfo.rxpciq.s.q_no:%d\n",
2081 retval
= octeon_setup_droq(octeon_dev
, q_no
,
2082 CFG_GET_NUM_RX_DESCS_NIC_IF
2083 (octeon_get_conf(octeon_dev
),
2085 CFG_GET_NUM_RX_BUF_SIZE_NIC_IF
2086 (octeon_get_conf(octeon_dev
),
2089 dev_err(&octeon_dev
->pci_dev
->dev
,
2090 "%s : Runtime DROQ(RxQ) creation failed.\n",
2095 droq
= octeon_dev
->droq
[q_no
];
2097 dev_dbg(&octeon_dev
->pci_dev
->dev
,
2098 "netif_napi_add netdev:%llx oct:%llx\n",
2101 netif_napi_add(netdev
, napi
, liquidio_napi_poll
, 64);
2103 /* designate a CPU for this droq */
2104 droq
->cpu_id
= cpu_id
;
2106 if (cpu_id
>= cpu_id_modulus
)
2109 octeon_register_droq_ops(octeon_dev
, q_no
, &droq_ops
);
2113 for (q
= 0; q
< lio
->linfo
.num_txpciq
; q
++) {
2114 num_tx_descs
= CFG_GET_NUM_TX_DESCS_NIC_IF(octeon_get_conf
2117 retval
= octeon_setup_iq(octeon_dev
, ifidx
, q
,
2118 lio
->linfo
.txpciq
[q
], num_tx_descs
,
2119 netdev_get_tx_queue(netdev
, q
));
2121 dev_err(&octeon_dev
->pci_dev
->dev
,
2122 " %s : Runtime IQ(TxQ) creation failed.\n",
2132 * \brief Poll routine for checking transmit queue status
2133 * @param work work_struct data structure
2135 static void octnet_poll_check_txq_status(struct work_struct
*work
)
2137 struct cavium_wk
*wk
= (struct cavium_wk
*)work
;
2138 struct lio
*lio
= (struct lio
*)wk
->ctxptr
;
2140 if (!ifstate_check(lio
, LIO_IFSTATE_RUNNING
))
2143 check_txq_status(lio
);
2144 queue_delayed_work(lio
->txq_status_wq
.wq
,
2145 &lio
->txq_status_wq
.wk
.work
, msecs_to_jiffies(1));
2149 * \brief Sets up the txq poll check
2150 * @param netdev network device
2152 static inline void setup_tx_poll_fn(struct net_device
*netdev
)
2154 struct lio
*lio
= GET_LIO(netdev
);
2155 struct octeon_device
*oct
= lio
->oct_dev
;
2157 lio
->txq_status_wq
.wq
= alloc_workqueue("txq-status",
2159 if (!lio
->txq_status_wq
.wq
) {
2160 dev_err(&oct
->pci_dev
->dev
, "unable to create cavium txq status wq\n");
2163 INIT_DELAYED_WORK(&lio
->txq_status_wq
.wk
.work
,
2164 octnet_poll_check_txq_status
);
2165 lio
->txq_status_wq
.wk
.ctxptr
= lio
;
2166 queue_delayed_work(lio
->txq_status_wq
.wq
,
2167 &lio
->txq_status_wq
.wk
.work
, msecs_to_jiffies(1));
2170 static inline void cleanup_tx_poll_fn(struct net_device
*netdev
)
2172 struct lio
*lio
= GET_LIO(netdev
);
2174 cancel_delayed_work_sync(&lio
->txq_status_wq
.wk
.work
);
2175 destroy_workqueue(lio
->txq_status_wq
.wq
);
2179 * \brief Net device open for LiquidIO
2180 * @param netdev network device
2182 static int liquidio_open(struct net_device
*netdev
)
2184 struct lio
*lio
= GET_LIO(netdev
);
2185 struct octeon_device
*oct
= lio
->oct_dev
;
2186 struct napi_struct
*napi
, *n
;
2188 if (oct
->props
[lio
->ifidx
].napi_enabled
== 0) {
2189 list_for_each_entry_safe(napi
, n
, &netdev
->napi_list
, dev_list
)
2192 oct
->props
[lio
->ifidx
].napi_enabled
= 1;
2195 oct_ptp_open(netdev
);
2197 ifstate_set(lio
, LIO_IFSTATE_RUNNING
);
2199 setup_tx_poll_fn(netdev
);
2203 netif_info(lio
, ifup
, lio
->netdev
, "Interface Open, ready for traffic\n");
2205 /* tell Octeon to start forwarding packets to host */
2206 send_rx_ctrl_cmd(lio
, 1);
2208 /* Ready for link status updates */
2211 dev_info(&oct
->pci_dev
->dev
, "%s interface is opened\n",
2218 * \brief Net device stop for LiquidIO
2219 * @param netdev network device
2221 static int liquidio_stop(struct net_device
*netdev
)
2223 struct lio
*lio
= GET_LIO(netdev
);
2224 struct octeon_device
*oct
= lio
->oct_dev
;
2226 ifstate_reset(lio
, LIO_IFSTATE_RUNNING
);
2228 netif_tx_disable(netdev
);
2230 /* Inform that netif carrier is down */
2231 netif_carrier_off(netdev
);
2233 lio
->linfo
.link
.s
.link_up
= 0;
2234 lio
->link_changes
++;
2236 /* Pause for a moment and wait for Octeon to flush out (to the wire) any
2237 * egress packets that are in-flight.
2239 set_current_state(TASK_INTERRUPTIBLE
);
2240 schedule_timeout(msecs_to_jiffies(100));
2242 /* Now it should be safe to tell Octeon that nic interface is down. */
2243 send_rx_ctrl_cmd(lio
, 0);
2245 cleanup_tx_poll_fn(netdev
);
2247 if (lio
->ptp_clock
) {
2248 ptp_clock_unregister(lio
->ptp_clock
);
2249 lio
->ptp_clock
= NULL
;
2252 dev_info(&oct
->pci_dev
->dev
, "%s interface is stopped\n", netdev
->name
);
2258 * \brief Converts a mask based on net device flags
2259 * @param netdev network device
2261 * This routine generates a octnet_ifflags mask from the net device flags
2262 * received from the OS.
2264 static inline enum octnet_ifflags
get_new_flags(struct net_device
*netdev
)
2266 enum octnet_ifflags f
= OCTNET_IFFLAG_UNICAST
;
2268 if (netdev
->flags
& IFF_PROMISC
)
2269 f
|= OCTNET_IFFLAG_PROMISC
;
2271 if (netdev
->flags
& IFF_ALLMULTI
)
2272 f
|= OCTNET_IFFLAG_ALLMULTI
;
2274 if (netdev
->flags
& IFF_MULTICAST
) {
2275 f
|= OCTNET_IFFLAG_MULTICAST
;
2277 /* Accept all multicast addresses if there are more than we
2280 if (netdev_mc_count(netdev
) > MAX_OCTEON_MULTICAST_ADDR
)
2281 f
|= OCTNET_IFFLAG_ALLMULTI
;
2284 if (netdev
->flags
& IFF_BROADCAST
)
2285 f
|= OCTNET_IFFLAG_BROADCAST
;
2291 * \brief Net device set_multicast_list
2292 * @param netdev network device
2294 static void liquidio_set_mcast_list(struct net_device
*netdev
)
2296 struct lio
*lio
= GET_LIO(netdev
);
2297 struct octeon_device
*oct
= lio
->oct_dev
;
2298 struct octnic_ctrl_pkt nctrl
;
2299 struct netdev_hw_addr
*ha
;
2302 int mc_count
= min(netdev_mc_count(netdev
), MAX_OCTEON_MULTICAST_ADDR
);
2304 memset(&nctrl
, 0, sizeof(struct octnic_ctrl_pkt
));
2306 /* Create a ctrl pkt command to be sent to core app. */
2308 nctrl
.ncmd
.s
.cmd
= OCTNET_CMD_SET_MULTI_LIST
;
2309 nctrl
.ncmd
.s
.param1
= get_new_flags(netdev
);
2310 nctrl
.ncmd
.s
.param2
= mc_count
;
2311 nctrl
.ncmd
.s
.more
= mc_count
;
2312 nctrl
.iq_no
= lio
->linfo
.txpciq
[0].s
.q_no
;
2313 nctrl
.netpndev
= (u64
)netdev
;
2314 nctrl
.cb_fn
= liquidio_link_ctrl_cmd_completion
;
2316 /* copy all the addresses into the udd */
2318 netdev_for_each_mc_addr(ha
, netdev
) {
2320 memcpy(((u8
*)mc
) + 2, ha
->addr
, ETH_ALEN
);
2321 /* no need to swap bytes */
2323 if (++mc
> &nctrl
.udd
[mc_count
])
2327 /* Apparently, any activity in this call from the kernel has to
2328 * be atomic. So we won't wait for response.
2330 nctrl
.wait_time
= 0;
2332 ret
= octnet_send_nic_ctrl_pkt(lio
->oct_dev
, &nctrl
);
2334 dev_err(&oct
->pci_dev
->dev
, "DEVFLAGS change failed in core (ret: 0x%x)\n",
2340 * \brief Net device set_mac_address
2341 * @param netdev network device
2343 static int liquidio_set_mac(struct net_device
*netdev
, void *p
)
2346 struct lio
*lio
= GET_LIO(netdev
);
2347 struct octeon_device
*oct
= lio
->oct_dev
;
2348 struct sockaddr
*addr
= (struct sockaddr
*)p
;
2349 struct octnic_ctrl_pkt nctrl
;
2351 if (!is_valid_ether_addr(addr
->sa_data
))
2352 return -EADDRNOTAVAIL
;
2354 memset(&nctrl
, 0, sizeof(struct octnic_ctrl_pkt
));
2357 nctrl
.ncmd
.s
.cmd
= OCTNET_CMD_CHANGE_MACADDR
;
2358 nctrl
.ncmd
.s
.param1
= 0;
2359 nctrl
.ncmd
.s
.more
= 1;
2360 nctrl
.iq_no
= lio
->linfo
.txpciq
[0].s
.q_no
;
2361 nctrl
.netpndev
= (u64
)netdev
;
2362 nctrl
.cb_fn
= liquidio_link_ctrl_cmd_completion
;
2363 nctrl
.wait_time
= 100;
2366 /* The MAC Address is presented in network byte order. */
2367 memcpy((u8
*)&nctrl
.udd
[0] + 2, addr
->sa_data
, ETH_ALEN
);
2369 ret
= octnet_send_nic_ctrl_pkt(lio
->oct_dev
, &nctrl
);
2371 dev_err(&oct
->pci_dev
->dev
, "MAC Address change failed\n");
2374 memcpy(netdev
->dev_addr
, addr
->sa_data
, netdev
->addr_len
);
2375 memcpy(((u8
*)&lio
->linfo
.hw_addr
) + 2, addr
->sa_data
, ETH_ALEN
);
2381 * \brief Net device get_stats
2382 * @param netdev network device
2384 static struct net_device_stats
*liquidio_get_stats(struct net_device
*netdev
)
2386 struct lio
*lio
= GET_LIO(netdev
);
2387 struct net_device_stats
*stats
= &netdev
->stats
;
2388 struct octeon_device
*oct
;
2389 u64 pkts
= 0, drop
= 0, bytes
= 0;
2390 struct oct_droq_stats
*oq_stats
;
2391 struct oct_iq_stats
*iq_stats
;
2392 int i
, iq_no
, oq_no
;
2396 for (i
= 0; i
< lio
->linfo
.num_txpciq
; i
++) {
2397 iq_no
= lio
->linfo
.txpciq
[i
].s
.q_no
;
2398 iq_stats
= &oct
->instr_queue
[iq_no
]->stats
;
2399 pkts
+= iq_stats
->tx_done
;
2400 drop
+= iq_stats
->tx_dropped
;
2401 bytes
+= iq_stats
->tx_tot_bytes
;
2404 stats
->tx_packets
= pkts
;
2405 stats
->tx_bytes
= bytes
;
2406 stats
->tx_dropped
= drop
;
2412 for (i
= 0; i
< lio
->linfo
.num_rxpciq
; i
++) {
2413 oq_no
= lio
->linfo
.rxpciq
[i
].s
.q_no
;
2414 oq_stats
= &oct
->droq
[oq_no
]->stats
;
2415 pkts
+= oq_stats
->rx_pkts_received
;
2416 drop
+= (oq_stats
->rx_dropped
+
2417 oq_stats
->dropped_nodispatch
+
2418 oq_stats
->dropped_toomany
+
2419 oq_stats
->dropped_nomem
);
2420 bytes
+= oq_stats
->rx_bytes_received
;
2423 stats
->rx_bytes
= bytes
;
2424 stats
->rx_packets
= pkts
;
2425 stats
->rx_dropped
= drop
;
2431 * \brief Net device change_mtu
2432 * @param netdev network device
2434 static int liquidio_change_mtu(struct net_device
*netdev
, int new_mtu
)
2436 struct lio
*lio
= GET_LIO(netdev
);
2437 struct octeon_device
*oct
= lio
->oct_dev
;
2438 struct octnic_ctrl_pkt nctrl
;
2441 /* Limit the MTU to make sure the ethernet packets are between 68 bytes
2444 if ((new_mtu
< LIO_MIN_MTU_SIZE
) ||
2445 (new_mtu
> LIO_MAX_MTU_SIZE
)) {
2446 dev_err(&oct
->pci_dev
->dev
, "Invalid MTU: %d\n", new_mtu
);
2447 dev_err(&oct
->pci_dev
->dev
, "Valid range %d and %d\n",
2448 LIO_MIN_MTU_SIZE
, LIO_MAX_MTU_SIZE
);
2452 memset(&nctrl
, 0, sizeof(struct octnic_ctrl_pkt
));
2455 nctrl
.ncmd
.s
.cmd
= OCTNET_CMD_CHANGE_MTU
;
2456 nctrl
.ncmd
.s
.param1
= new_mtu
;
2457 nctrl
.iq_no
= lio
->linfo
.txpciq
[0].s
.q_no
;
2458 nctrl
.wait_time
= 100;
2459 nctrl
.netpndev
= (u64
)netdev
;
2460 nctrl
.cb_fn
= liquidio_link_ctrl_cmd_completion
;
2462 ret
= octnet_send_nic_ctrl_pkt(lio
->oct_dev
, &nctrl
);
2464 dev_err(&oct
->pci_dev
->dev
, "Failed to set MTU\n");
2474 * \brief Handler for SIOCSHWTSTAMP ioctl
2475 * @param netdev network device
2476 * @param ifr interface request
2477 * @param cmd command
2479 static int hwtstamp_ioctl(struct net_device
*netdev
, struct ifreq
*ifr
)
2481 struct hwtstamp_config conf
;
2482 struct lio
*lio
= GET_LIO(netdev
);
2484 if (copy_from_user(&conf
, ifr
->ifr_data
, sizeof(conf
)))
2490 switch (conf
.tx_type
) {
2491 case HWTSTAMP_TX_ON
:
2492 case HWTSTAMP_TX_OFF
:
2498 switch (conf
.rx_filter
) {
2499 case HWTSTAMP_FILTER_NONE
:
2501 case HWTSTAMP_FILTER_ALL
:
2502 case HWTSTAMP_FILTER_SOME
:
2503 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT
:
2504 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC
:
2505 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ
:
2506 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT
:
2507 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC
:
2508 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ
:
2509 case HWTSTAMP_FILTER_PTP_V2_L2_EVENT
:
2510 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC
:
2511 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ
:
2512 case HWTSTAMP_FILTER_PTP_V2_EVENT
:
2513 case HWTSTAMP_FILTER_PTP_V2_SYNC
:
2514 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ
:
2515 conf
.rx_filter
= HWTSTAMP_FILTER_ALL
;
2521 if (conf
.rx_filter
== HWTSTAMP_FILTER_ALL
)
2522 ifstate_set(lio
, LIO_IFSTATE_RX_TIMESTAMP_ENABLED
);
2525 ifstate_reset(lio
, LIO_IFSTATE_RX_TIMESTAMP_ENABLED
);
2527 return copy_to_user(ifr
->ifr_data
, &conf
, sizeof(conf
)) ? -EFAULT
: 0;
2531 * \brief ioctl handler
2532 * @param netdev network device
2533 * @param ifr interface request
2534 * @param cmd command
2536 static int liquidio_ioctl(struct net_device
*netdev
, struct ifreq
*ifr
, int cmd
)
2540 return hwtstamp_ioctl(netdev
, ifr
);
2547 * \brief handle a Tx timestamp response
2548 * @param status response status
2549 * @param buf pointer to skb
2551 static void handle_timestamp(struct octeon_device
*oct
,
2555 struct octnet_buf_free_info
*finfo
;
2556 struct octeon_soft_command
*sc
;
2557 struct oct_timestamp_resp
*resp
;
2559 struct sk_buff
*skb
= (struct sk_buff
*)buf
;
2561 finfo
= (struct octnet_buf_free_info
*)skb
->cb
;
2565 resp
= (struct oct_timestamp_resp
*)sc
->virtrptr
;
2567 if (status
!= OCTEON_REQUEST_DONE
) {
2568 dev_err(&oct
->pci_dev
->dev
, "Tx timestamp instruction failed. Status: %llx\n",
2569 CVM_CAST64(status
));
2570 resp
->timestamp
= 0;
2573 octeon_swap_8B_data(&resp
->timestamp
, 1);
2575 if (unlikely((skb_shinfo(skb
)->tx_flags
& SKBTX_IN_PROGRESS
) != 0)) {
2576 struct skb_shared_hwtstamps ts
;
2577 u64 ns
= resp
->timestamp
;
2579 netif_info(lio
, tx_done
, lio
->netdev
,
2580 "Got resulting SKBTX_HW_TSTAMP skb=%p ns=%016llu\n",
2581 skb
, (unsigned long long)ns
);
2582 ts
.hwtstamp
= ns_to_ktime(ns
+ lio
->ptp_adjust
);
2583 skb_tstamp_tx(skb
, &ts
);
2586 octeon_free_soft_command(oct
, sc
);
2587 tx_buffer_free(skb
);
2590 /* \brief Send a data packet that will be timestamped
2591 * @param oct octeon device
2592 * @param ndata pointer to network data
2593 * @param finfo pointer to private network data
2595 static inline int send_nic_timestamp_pkt(struct octeon_device
*oct
,
2596 struct octnic_data_pkt
*ndata
,
2597 struct octnet_buf_free_info
*finfo
)
2600 struct octeon_soft_command
*sc
;
2607 sc
= octeon_alloc_soft_command_resp(oct
, &ndata
->cmd
,
2608 sizeof(struct oct_timestamp_resp
));
2612 dev_err(&oct
->pci_dev
->dev
, "No memory for timestamped data packet\n");
2613 return IQ_SEND_FAILED
;
2616 if (ndata
->reqtype
== REQTYPE_NORESP_NET
)
2617 ndata
->reqtype
= REQTYPE_RESP_NET
;
2618 else if (ndata
->reqtype
== REQTYPE_NORESP_NET_SG
)
2619 ndata
->reqtype
= REQTYPE_RESP_NET_SG
;
2621 sc
->callback
= handle_timestamp
;
2622 sc
->callback_arg
= finfo
->skb
;
2623 sc
->iq_no
= ndata
->q_no
;
2625 len
= (u32
)((struct octeon_instr_ih2
*)(&sc
->cmd
.cmd2
.ih2
))->dlengsz
;
2628 retval
= octeon_send_command(oct
, sc
->iq_no
, ring_doorbell
, &sc
->cmd
,
2629 sc
, len
, ndata
->reqtype
);
2631 if (retval
== IQ_SEND_FAILED
) {
2632 dev_err(&oct
->pci_dev
->dev
, "timestamp data packet failed status: %x\n",
2634 octeon_free_soft_command(oct
, sc
);
2636 netif_info(lio
, tx_queued
, lio
->netdev
, "Queued timestamp packet\n");
2642 /** \brief Transmit networks packets to the Octeon interface
2643 * @param skbuff skbuff struct to be passed to network layer.
2644 * @param netdev pointer to network device
2645 * @returns whether the packet was transmitted to the device okay or not
2646 * (NETDEV_TX_OK or NETDEV_TX_BUSY)
2648 static int liquidio_xmit(struct sk_buff
*skb
, struct net_device
*netdev
)
2651 struct octnet_buf_free_info
*finfo
;
2652 union octnic_cmd_setup cmdsetup
;
2653 struct octnic_data_pkt ndata
;
2654 struct octeon_device
*oct
;
2655 struct oct_iq_stats
*stats
;
2656 struct octeon_instr_irh
*irh
;
2657 union tx_info
*tx_info
;
2659 int q_idx
= 0, iq_no
= 0;
2664 lio
= GET_LIO(netdev
);
2667 if (netif_is_multiqueue(netdev
)) {
2668 q_idx
= skb
->queue_mapping
;
2669 q_idx
= (q_idx
% (lio
->linfo
.num_txpciq
));
2671 iq_no
= lio
->linfo
.txpciq
[q_idx
].s
.q_no
;
2676 stats
= &oct
->instr_queue
[iq_no
]->stats
;
2678 /* Check for all conditions in which the current packet cannot be
2681 if (!(atomic_read(&lio
->ifstate
) & LIO_IFSTATE_RUNNING
) ||
2682 (!lio
->linfo
.link
.s
.link_up
) ||
2684 netif_info(lio
, tx_err
, lio
->netdev
,
2685 "Transmit failed link_status : %d\n",
2686 lio
->linfo
.link
.s
.link_up
);
2687 goto lio_xmit_failed
;
2690 /* Use space in skb->cb to store info used to unmap and
2693 finfo
= (struct octnet_buf_free_info
*)skb
->cb
;
2698 /* Prepare the attributes for the data to be passed to OSI. */
2699 memset(&ndata
, 0, sizeof(struct octnic_data_pkt
));
2701 ndata
.buf
= (void *)finfo
;
2705 if (netif_is_multiqueue(netdev
)) {
2706 if (octnet_iq_is_full(oct
, ndata
.q_no
)) {
2707 /* defer sending if queue is full */
2708 netif_info(lio
, tx_err
, lio
->netdev
, "Transmit failed iq:%d full\n",
2710 stats
->tx_iq_busy
++;
2711 return NETDEV_TX_BUSY
;
2714 if (octnet_iq_is_full(oct
, lio
->txq
)) {
2715 /* defer sending if queue is full */
2716 stats
->tx_iq_busy
++;
2717 netif_info(lio
, tx_err
, lio
->netdev
, "Transmit failed iq:%d full\n",
2719 return NETDEV_TX_BUSY
;
2722 /* pr_info(" XMIT - valid Qs: %d, 1st Q no: %d, cpu: %d, q_no:%d\n",
2723 * lio->linfo.num_txpciq, lio->txq, cpu, ndata.q_no);
2726 ndata
.datasize
= skb
->len
;
2729 cmdsetup
.s
.iq_no
= iq_no
;
2731 if (skb
->ip_summed
== CHECKSUM_PARTIAL
) {
2732 if (skb
->encapsulation
) {
2733 cmdsetup
.s
.tnl_csum
= 1;
2736 cmdsetup
.s
.transport_csum
= 1;
2739 if (unlikely(skb_shinfo(skb
)->tx_flags
& SKBTX_HW_TSTAMP
)) {
2740 skb_shinfo(skb
)->tx_flags
|= SKBTX_IN_PROGRESS
;
2741 cmdsetup
.s
.timestamp
= 1;
2744 if (skb_shinfo(skb
)->nr_frags
== 0) {
2745 cmdsetup
.s
.u
.datasize
= skb
->len
;
2746 octnet_prepare_pci_cmd(oct
, &ndata
.cmd
, &cmdsetup
, tag
);
2748 /* Offload checksum calculation for TCP/UDP packets */
2749 dptr
= dma_map_single(&oct
->pci_dev
->dev
,
2753 if (dma_mapping_error(&oct
->pci_dev
->dev
, dptr
)) {
2754 dev_err(&oct
->pci_dev
->dev
, "%s DMA mapping error 1\n",
2756 return NETDEV_TX_BUSY
;
2759 ndata
.cmd
.cmd2
.dptr
= dptr
;
2761 ndata
.reqtype
= REQTYPE_NORESP_NET
;
2765 struct skb_frag_struct
*frag
;
2766 struct octnic_gather
*g
;
2768 spin_lock(&lio
->glist_lock
[q_idx
]);
2769 g
= (struct octnic_gather
*)
2770 list_delete_head(&lio
->glist
[q_idx
]);
2771 spin_unlock(&lio
->glist_lock
[q_idx
]);
2774 netif_info(lio
, tx_err
, lio
->netdev
,
2775 "Transmit scatter gather: glist null!\n");
2776 goto lio_xmit_failed
;
2779 cmdsetup
.s
.gather
= 1;
2780 cmdsetup
.s
.u
.gatherptrs
= (skb_shinfo(skb
)->nr_frags
+ 1);
2781 octnet_prepare_pci_cmd(oct
, &ndata
.cmd
, &cmdsetup
, tag
);
2783 memset(g
->sg
, 0, g
->sg_size
);
2785 g
->sg
[0].ptr
[0] = dma_map_single(&oct
->pci_dev
->dev
,
2787 (skb
->len
- skb
->data_len
),
2789 if (dma_mapping_error(&oct
->pci_dev
->dev
, g
->sg
[0].ptr
[0])) {
2790 dev_err(&oct
->pci_dev
->dev
, "%s DMA mapping error 2\n",
2792 return NETDEV_TX_BUSY
;
2794 add_sg_size(&g
->sg
[0], (skb
->len
- skb
->data_len
), 0);
2796 frags
= skb_shinfo(skb
)->nr_frags
;
2799 frag
= &skb_shinfo(skb
)->frags
[i
- 1];
2801 g
->sg
[(i
>> 2)].ptr
[(i
& 3)] =
2802 dma_map_page(&oct
->pci_dev
->dev
,
2808 if (dma_mapping_error(&oct
->pci_dev
->dev
,
2809 g
->sg
[i
>> 2].ptr
[i
& 3])) {
2810 dma_unmap_single(&oct
->pci_dev
->dev
,
2812 skb
->len
- skb
->data_len
,
2814 for (j
= 1; j
< i
; j
++) {
2815 frag
= &skb_shinfo(skb
)->frags
[j
- 1];
2816 dma_unmap_page(&oct
->pci_dev
->dev
,
2817 g
->sg
[j
>> 2].ptr
[j
& 3],
2821 dev_err(&oct
->pci_dev
->dev
, "%s DMA mapping error 3\n",
2823 return NETDEV_TX_BUSY
;
2826 add_sg_size(&g
->sg
[(i
>> 2)], frag
->size
, (i
& 3));
2830 dma_sync_single_for_device(&oct
->pci_dev
->dev
, g
->sg_dma_ptr
,
2831 g
->sg_size
, DMA_TO_DEVICE
);
2832 dptr
= g
->sg_dma_ptr
;
2834 ndata
.cmd
.cmd2
.dptr
= dptr
;
2838 ndata
.reqtype
= REQTYPE_NORESP_NET_SG
;
2841 irh
= (struct octeon_instr_irh
*)&ndata
.cmd
.cmd2
.irh
;
2842 tx_info
= (union tx_info
*)&ndata
.cmd
.cmd2
.ossp
[0];
2844 if (skb_shinfo(skb
)->gso_size
) {
2845 tx_info
->s
.gso_size
= skb_shinfo(skb
)->gso_size
;
2846 tx_info
->s
.gso_segs
= skb_shinfo(skb
)->gso_segs
;
2850 /* HW insert VLAN tag */
2851 if (skb_vlan_tag_present(skb
)) {
2852 irh
->priority
= skb_vlan_tag_get(skb
) >> 13;
2853 irh
->vlan
= skb_vlan_tag_get(skb
) & 0xfff;
2856 if (unlikely(cmdsetup
.s
.timestamp
))
2857 status
= send_nic_timestamp_pkt(oct
, &ndata
, finfo
);
2859 status
= octnet_send_nic_data_pkt(oct
, &ndata
);
2860 if (status
== IQ_SEND_FAILED
)
2861 goto lio_xmit_failed
;
2863 netif_info(lio
, tx_queued
, lio
->netdev
, "Transmit queued successfully\n");
2865 if (status
== IQ_SEND_STOP
)
2866 stop_q(lio
->netdev
, q_idx
);
2868 netif_trans_update(netdev
);
2870 if (skb_shinfo(skb
)->gso_size
)
2871 stats
->tx_done
+= skb_shinfo(skb
)->gso_segs
;
2874 stats
->tx_tot_bytes
+= skb
->len
;
2876 return NETDEV_TX_OK
;
2879 stats
->tx_dropped
++;
2880 netif_info(lio
, tx_err
, lio
->netdev
, "IQ%d Transmit dropped:%llu\n",
2881 iq_no
, stats
->tx_dropped
);
2883 dma_unmap_single(&oct
->pci_dev
->dev
, dptr
,
2884 ndata
.datasize
, DMA_TO_DEVICE
);
2885 tx_buffer_free(skb
);
2886 return NETDEV_TX_OK
;
2889 /** \brief Network device Tx timeout
2890 * @param netdev pointer to network device
2892 static void liquidio_tx_timeout(struct net_device
*netdev
)
2896 lio
= GET_LIO(netdev
);
2898 netif_info(lio
, tx_err
, lio
->netdev
,
2899 "Transmit timeout tx_dropped:%ld, waking up queues now!!\n",
2900 netdev
->stats
.tx_dropped
);
2901 netif_trans_update(netdev
);
2905 static int liquidio_vlan_rx_add_vid(struct net_device
*netdev
,
2906 __be16 proto
__attribute__((unused
)),
2909 struct lio
*lio
= GET_LIO(netdev
);
2910 struct octeon_device
*oct
= lio
->oct_dev
;
2911 struct octnic_ctrl_pkt nctrl
;
2914 memset(&nctrl
, 0, sizeof(struct octnic_ctrl_pkt
));
2917 nctrl
.ncmd
.s
.cmd
= OCTNET_CMD_ADD_VLAN_FILTER
;
2918 nctrl
.ncmd
.s
.param1
= vid
;
2919 nctrl
.iq_no
= lio
->linfo
.txpciq
[0].s
.q_no
;
2920 nctrl
.wait_time
= 100;
2921 nctrl
.netpndev
= (u64
)netdev
;
2922 nctrl
.cb_fn
= liquidio_link_ctrl_cmd_completion
;
2924 ret
= octnet_send_nic_ctrl_pkt(lio
->oct_dev
, &nctrl
);
2926 dev_err(&oct
->pci_dev
->dev
, "Add VLAN filter failed in core (ret: 0x%x)\n",
2933 static int liquidio_vlan_rx_kill_vid(struct net_device
*netdev
,
2934 __be16 proto
__attribute__((unused
)),
2937 struct lio
*lio
= GET_LIO(netdev
);
2938 struct octeon_device
*oct
= lio
->oct_dev
;
2939 struct octnic_ctrl_pkt nctrl
;
2942 memset(&nctrl
, 0, sizeof(struct octnic_ctrl_pkt
));
2945 nctrl
.ncmd
.s
.cmd
= OCTNET_CMD_DEL_VLAN_FILTER
;
2946 nctrl
.ncmd
.s
.param1
= vid
;
2947 nctrl
.iq_no
= lio
->linfo
.txpciq
[0].s
.q_no
;
2948 nctrl
.wait_time
= 100;
2949 nctrl
.netpndev
= (u64
)netdev
;
2950 nctrl
.cb_fn
= liquidio_link_ctrl_cmd_completion
;
2952 ret
= octnet_send_nic_ctrl_pkt(lio
->oct_dev
, &nctrl
);
2954 dev_err(&oct
->pci_dev
->dev
, "Add VLAN filter failed in core (ret: 0x%x)\n",
2960 /** Sending command to enable/disable RX checksum offload
2961 * @param netdev pointer to network device
2962 * @param command OCTNET_CMD_TNL_RX_CSUM_CTL
2963 * @param rx_cmd_bit OCTNET_CMD_RXCSUM_ENABLE/
2964 * OCTNET_CMD_RXCSUM_DISABLE
2965 * @returns SUCCESS or FAILURE
2967 static int liquidio_set_rxcsum_command(struct net_device
*netdev
, int command
,
2970 struct lio
*lio
= GET_LIO(netdev
);
2971 struct octeon_device
*oct
= lio
->oct_dev
;
2972 struct octnic_ctrl_pkt nctrl
;
2976 nctrl
.ncmd
.s
.cmd
= command
;
2977 nctrl
.ncmd
.s
.param1
= rx_cmd
;
2978 nctrl
.iq_no
= lio
->linfo
.txpciq
[0].s
.q_no
;
2979 nctrl
.wait_time
= 100;
2980 nctrl
.netpndev
= (u64
)netdev
;
2981 nctrl
.cb_fn
= liquidio_link_ctrl_cmd_completion
;
2983 ret
= octnet_send_nic_ctrl_pkt(lio
->oct_dev
, &nctrl
);
2985 dev_err(&oct
->pci_dev
->dev
,
2986 "DEVFLAGS RXCSUM change failed in core(ret:0x%x)\n",
2992 /** Sending command to add/delete VxLAN UDP port to firmware
2993 * @param netdev pointer to network device
2994 * @param command OCTNET_CMD_VXLAN_PORT_CONFIG
2995 * @param vxlan_port VxLAN port to be added or deleted
2996 * @param vxlan_cmd_bit OCTNET_CMD_VXLAN_PORT_ADD,
2997 * OCTNET_CMD_VXLAN_PORT_DEL
2998 * @returns SUCCESS or FAILURE
3000 static int liquidio_vxlan_port_command(struct net_device
*netdev
, int command
,
3001 u16 vxlan_port
, u8 vxlan_cmd_bit
)
3003 struct lio
*lio
= GET_LIO(netdev
);
3004 struct octeon_device
*oct
= lio
->oct_dev
;
3005 struct octnic_ctrl_pkt nctrl
;
3009 nctrl
.ncmd
.s
.cmd
= command
;
3010 nctrl
.ncmd
.s
.more
= vxlan_cmd_bit
;
3011 nctrl
.ncmd
.s
.param1
= vxlan_port
;
3012 nctrl
.iq_no
= lio
->linfo
.txpciq
[0].s
.q_no
;
3013 nctrl
.wait_time
= 100;
3014 nctrl
.netpndev
= (u64
)netdev
;
3015 nctrl
.cb_fn
= liquidio_link_ctrl_cmd_completion
;
3017 ret
= octnet_send_nic_ctrl_pkt(lio
->oct_dev
, &nctrl
);
3019 dev_err(&oct
->pci_dev
->dev
,
3020 "VxLAN port add/delete failed in core (ret:0x%x)\n",
3026 /** \brief Net device fix features
3027 * @param netdev pointer to network device
3028 * @param request features requested
3029 * @returns updated features list
3031 static netdev_features_t
liquidio_fix_features(struct net_device
*netdev
,
3032 netdev_features_t request
)
3034 struct lio
*lio
= netdev_priv(netdev
);
3036 if ((request
& NETIF_F_RXCSUM
) &&
3037 !(lio
->dev_capability
& NETIF_F_RXCSUM
))
3038 request
&= ~NETIF_F_RXCSUM
;
3040 if ((request
& NETIF_F_HW_CSUM
) &&
3041 !(lio
->dev_capability
& NETIF_F_HW_CSUM
))
3042 request
&= ~NETIF_F_HW_CSUM
;
3044 if ((request
& NETIF_F_TSO
) && !(lio
->dev_capability
& NETIF_F_TSO
))
3045 request
&= ~NETIF_F_TSO
;
3047 if ((request
& NETIF_F_TSO6
) && !(lio
->dev_capability
& NETIF_F_TSO6
))
3048 request
&= ~NETIF_F_TSO6
;
3050 if ((request
& NETIF_F_LRO
) && !(lio
->dev_capability
& NETIF_F_LRO
))
3051 request
&= ~NETIF_F_LRO
;
3053 /*Disable LRO if RXCSUM is off */
3054 if (!(request
& NETIF_F_RXCSUM
) && (netdev
->features
& NETIF_F_LRO
) &&
3055 (lio
->dev_capability
& NETIF_F_LRO
))
3056 request
&= ~NETIF_F_LRO
;
3061 /** \brief Net device set features
3062 * @param netdev pointer to network device
3063 * @param features features to enable/disable
3065 static int liquidio_set_features(struct net_device
*netdev
,
3066 netdev_features_t features
)
3068 struct lio
*lio
= netdev_priv(netdev
);
3070 if (!((netdev
->features
^ features
) & NETIF_F_LRO
))
3073 if ((features
& NETIF_F_LRO
) && (lio
->dev_capability
& NETIF_F_LRO
))
3074 liquidio_set_feature(netdev
, OCTNET_CMD_LRO_ENABLE
,
3075 OCTNIC_LROIPV4
| OCTNIC_LROIPV6
);
3076 else if (!(features
& NETIF_F_LRO
) &&
3077 (lio
->dev_capability
& NETIF_F_LRO
))
3078 liquidio_set_feature(netdev
, OCTNET_CMD_LRO_DISABLE
,
3079 OCTNIC_LROIPV4
| OCTNIC_LROIPV6
);
3081 /* Sending command to firmware to enable/disable RX checksum
3082 * offload settings using ethtool
3084 if (!(netdev
->features
& NETIF_F_RXCSUM
) &&
3085 (lio
->enc_dev_capability
& NETIF_F_RXCSUM
) &&
3086 (features
& NETIF_F_RXCSUM
))
3087 liquidio_set_rxcsum_command(netdev
,
3088 OCTNET_CMD_TNL_RX_CSUM_CTL
,
3089 OCTNET_CMD_RXCSUM_ENABLE
);
3090 else if ((netdev
->features
& NETIF_F_RXCSUM
) &&
3091 (lio
->enc_dev_capability
& NETIF_F_RXCSUM
) &&
3092 !(features
& NETIF_F_RXCSUM
))
3093 liquidio_set_rxcsum_command(netdev
, OCTNET_CMD_TNL_RX_CSUM_CTL
,
3094 OCTNET_CMD_RXCSUM_DISABLE
);
3099 static void liquidio_add_vxlan_port(struct net_device
*netdev
,
3100 struct udp_tunnel_info
*ti
)
3102 if (ti
->type
!= UDP_TUNNEL_TYPE_VXLAN
)
3105 liquidio_vxlan_port_command(netdev
,
3106 OCTNET_CMD_VXLAN_PORT_CONFIG
,
3108 OCTNET_CMD_VXLAN_PORT_ADD
);
3111 static void liquidio_del_vxlan_port(struct net_device
*netdev
,
3112 struct udp_tunnel_info
*ti
)
3114 if (ti
->type
!= UDP_TUNNEL_TYPE_VXLAN
)
3117 liquidio_vxlan_port_command(netdev
,
3118 OCTNET_CMD_VXLAN_PORT_CONFIG
,
3120 OCTNET_CMD_VXLAN_PORT_DEL
);
3123 static struct net_device_ops lionetdevops
= {
3124 .ndo_open
= liquidio_open
,
3125 .ndo_stop
= liquidio_stop
,
3126 .ndo_start_xmit
= liquidio_xmit
,
3127 .ndo_get_stats
= liquidio_get_stats
,
3128 .ndo_set_mac_address
= liquidio_set_mac
,
3129 .ndo_set_rx_mode
= liquidio_set_mcast_list
,
3130 .ndo_tx_timeout
= liquidio_tx_timeout
,
3132 .ndo_vlan_rx_add_vid
= liquidio_vlan_rx_add_vid
,
3133 .ndo_vlan_rx_kill_vid
= liquidio_vlan_rx_kill_vid
,
3134 .ndo_change_mtu
= liquidio_change_mtu
,
3135 .ndo_do_ioctl
= liquidio_ioctl
,
3136 .ndo_fix_features
= liquidio_fix_features
,
3137 .ndo_set_features
= liquidio_set_features
,
3138 .ndo_udp_tunnel_add
= liquidio_add_vxlan_port
,
3139 .ndo_udp_tunnel_del
= liquidio_del_vxlan_port
,
3142 /** \brief Entry point for the liquidio module
3144 static int __init
liquidio_init(void)
3147 struct handshake
*hs
;
3149 init_completion(&first_stage
);
3151 octeon_init_device_list(conf_type
);
3153 if (liquidio_init_pci())
3156 wait_for_completion_timeout(&first_stage
, msecs_to_jiffies(1000));
3158 for (i
= 0; i
< MAX_OCTEON_DEVICES
; i
++) {
3161 wait_for_completion(&hs
->init
);
3163 /* init handshake failed */
3164 dev_err(&hs
->pci_dev
->dev
,
3165 "Failed to init device\n");
3166 liquidio_deinit_pci();
3172 for (i
= 0; i
< MAX_OCTEON_DEVICES
; i
++) {
3175 wait_for_completion_timeout(&hs
->started
,
3176 msecs_to_jiffies(30000));
3177 if (!hs
->started_ok
) {
3178 /* starter handshake failed */
3179 dev_err(&hs
->pci_dev
->dev
,
3180 "Firmware failed to start\n");
3181 liquidio_deinit_pci();
3190 static int lio_nic_info(struct octeon_recv_info
*recv_info
, void *buf
)
3192 struct octeon_device
*oct
= (struct octeon_device
*)buf
;
3193 struct octeon_recv_pkt
*recv_pkt
= recv_info
->recv_pkt
;
3195 union oct_link_status
*ls
;
3198 if (recv_pkt
->buffer_size
[0] != sizeof(*ls
)) {
3199 dev_err(&oct
->pci_dev
->dev
, "Malformed NIC_INFO, len=%d, ifidx=%d\n",
3200 recv_pkt
->buffer_size
[0],
3201 recv_pkt
->rh
.r_nic_info
.gmxport
);
3205 gmxport
= recv_pkt
->rh
.r_nic_info
.gmxport
;
3206 ls
= (union oct_link_status
*)get_rbd(recv_pkt
->buffer_ptr
[0]);
3208 octeon_swap_8B_data((u64
*)ls
, (sizeof(union oct_link_status
)) >> 3);
3209 for (i
= 0; i
< oct
->ifcount
; i
++) {
3210 if (oct
->props
[i
].gmxport
== gmxport
) {
3211 update_link_status(oct
->props
[i
].netdev
, ls
);
3217 for (i
= 0; i
< recv_pkt
->buffer_count
; i
++)
3218 recv_buffer_free(recv_pkt
->buffer_ptr
[i
]);
3219 octeon_free_recv_info(recv_info
);
3224 * \brief Setup network interfaces
3225 * @param octeon_dev octeon device
3227 * Called during init time for each device. It assumes the NIC
3228 * is already up and running. The link information for each
3229 * interface is passed in link_info.
3231 static int setup_nic_devices(struct octeon_device
*octeon_dev
)
3233 struct lio
*lio
= NULL
;
3234 struct net_device
*netdev
;
3236 struct octeon_soft_command
*sc
;
3237 struct liquidio_if_cfg_context
*ctx
;
3238 struct liquidio_if_cfg_resp
*resp
;
3239 struct octdev_props
*props
;
3240 int retval
, num_iqueues
, num_oqueues
;
3241 union oct_nic_if_cfg if_cfg
;
3242 unsigned int base_queue
;
3243 unsigned int gmx_port_id
;
3244 u32 resp_size
, ctx_size
, data_size
;
3246 struct lio_version
*vdata
;
3248 /* This is to handle link status changes */
3249 octeon_register_dispatch_fn(octeon_dev
, OPCODE_NIC
,
3251 lio_nic_info
, octeon_dev
);
3253 /* REQTYPE_RESP_NET and REQTYPE_SOFT_COMMAND do not have free functions.
3254 * They are handled directly.
3256 octeon_register_reqtype_free_fn(octeon_dev
, REQTYPE_NORESP_NET
,
3259 octeon_register_reqtype_free_fn(octeon_dev
, REQTYPE_NORESP_NET_SG
,
3262 octeon_register_reqtype_free_fn(octeon_dev
, REQTYPE_RESP_NET_SG
,
3263 free_netsgbuf_with_resp
);
3265 for (i
= 0; i
< octeon_dev
->ifcount
; i
++) {
3266 resp_size
= sizeof(struct liquidio_if_cfg_resp
);
3267 ctx_size
= sizeof(struct liquidio_if_cfg_context
);
3268 data_size
= sizeof(struct lio_version
);
3269 sc
= (struct octeon_soft_command
*)
3270 octeon_alloc_soft_command(octeon_dev
, data_size
,
3271 resp_size
, ctx_size
);
3272 resp
= (struct liquidio_if_cfg_resp
*)sc
->virtrptr
;
3273 ctx
= (struct liquidio_if_cfg_context
*)sc
->ctxptr
;
3274 vdata
= (struct lio_version
*)sc
->virtdptr
;
3276 *((u64
*)vdata
) = 0;
3277 vdata
->major
= cpu_to_be16(LIQUIDIO_BASE_MAJOR_VERSION
);
3278 vdata
->minor
= cpu_to_be16(LIQUIDIO_BASE_MINOR_VERSION
);
3279 vdata
->micro
= cpu_to_be16(LIQUIDIO_BASE_MICRO_VERSION
);
3281 if (OCTEON_CN23XX_PF(octeon_dev
)) {
3282 num_iqueues
= octeon_dev
->sriov_info
.num_pf_rings
;
3283 num_oqueues
= octeon_dev
->sriov_info
.num_pf_rings
;
3284 base_queue
= octeon_dev
->sriov_info
.pf_srn
;
3286 gmx_port_id
= octeon_dev
->pf_num
;
3287 ifidx_or_pfnum
= octeon_dev
->pf_num
;
3289 num_iqueues
= CFG_GET_NUM_TXQS_NIC_IF(
3290 octeon_get_conf(octeon_dev
), i
);
3291 num_oqueues
= CFG_GET_NUM_RXQS_NIC_IF(
3292 octeon_get_conf(octeon_dev
), i
);
3293 base_queue
= CFG_GET_BASE_QUE_NIC_IF(
3294 octeon_get_conf(octeon_dev
), i
);
3295 gmx_port_id
= CFG_GET_GMXID_NIC_IF(
3296 octeon_get_conf(octeon_dev
), i
);
3300 dev_dbg(&octeon_dev
->pci_dev
->dev
,
3301 "requesting config for interface %d, iqs %d, oqs %d\n",
3302 ifidx_or_pfnum
, num_iqueues
, num_oqueues
);
3303 WRITE_ONCE(ctx
->cond
, 0);
3304 ctx
->octeon_id
= lio_get_device_id(octeon_dev
);
3305 init_waitqueue_head(&ctx
->wc
);
3308 if_cfg
.s
.num_iqueues
= num_iqueues
;
3309 if_cfg
.s
.num_oqueues
= num_oqueues
;
3310 if_cfg
.s
.base_queue
= base_queue
;
3311 if_cfg
.s
.gmx_port_id
= gmx_port_id
;
3315 octeon_prepare_soft_command(octeon_dev
, sc
, OPCODE_NIC
,
3316 OPCODE_NIC_IF_CFG
, 0,
3319 sc
->callback
= if_cfg_callback
;
3320 sc
->callback_arg
= sc
;
3321 sc
->wait_time
= 3000;
3323 retval
= octeon_send_soft_command(octeon_dev
, sc
);
3324 if (retval
== IQ_SEND_FAILED
) {
3325 dev_err(&octeon_dev
->pci_dev
->dev
,
3326 "iq/oq config failed status: %x\n",
3328 /* Soft instr is freed by driver in case of failure. */
3329 goto setup_nic_dev_fail
;
3332 /* Sleep on a wait queue till the cond flag indicates that the
3333 * response arrived or timed-out.
3335 sleep_cond(&ctx
->wc
, &ctx
->cond
);
3336 retval
= resp
->status
;
3338 dev_err(&octeon_dev
->pci_dev
->dev
, "iq/oq config failed\n");
3339 goto setup_nic_dev_fail
;
3342 octeon_swap_8B_data((u64
*)(&resp
->cfg_info
),
3343 (sizeof(struct liquidio_if_cfg_info
)) >> 3);
3345 num_iqueues
= hweight64(resp
->cfg_info
.iqmask
);
3346 num_oqueues
= hweight64(resp
->cfg_info
.oqmask
);
3348 if (!(num_iqueues
) || !(num_oqueues
)) {
3349 dev_err(&octeon_dev
->pci_dev
->dev
,
3350 "Got bad iqueues (%016llx) or oqueues (%016llx) from firmware.\n",
3351 resp
->cfg_info
.iqmask
,
3352 resp
->cfg_info
.oqmask
);
3353 goto setup_nic_dev_fail
;
3355 dev_dbg(&octeon_dev
->pci_dev
->dev
,
3356 "interface %d, iqmask %016llx, oqmask %016llx, numiqueues %d, numoqueues %d\n",
3357 i
, resp
->cfg_info
.iqmask
, resp
->cfg_info
.oqmask
,
3358 num_iqueues
, num_oqueues
);
3359 netdev
= alloc_etherdev_mq(LIO_SIZE
, num_iqueues
);
3362 dev_err(&octeon_dev
->pci_dev
->dev
, "Device allocation failed\n");
3363 goto setup_nic_dev_fail
;
3366 SET_NETDEV_DEV(netdev
, &octeon_dev
->pci_dev
->dev
);
3368 if (num_iqueues
> 1)
3369 lionetdevops
.ndo_select_queue
= select_q
;
3371 /* Associate the routines that will handle different
3374 netdev
->netdev_ops
= &lionetdevops
;
3376 lio
= GET_LIO(netdev
);
3378 memset(lio
, 0, sizeof(struct lio
));
3380 lio
->ifidx
= ifidx_or_pfnum
;
3382 props
= &octeon_dev
->props
[i
];
3383 props
->gmxport
= resp
->cfg_info
.linfo
.gmxport
;
3384 props
->netdev
= netdev
;
3386 lio
->linfo
.num_rxpciq
= num_oqueues
;
3387 lio
->linfo
.num_txpciq
= num_iqueues
;
3388 for (j
= 0; j
< num_oqueues
; j
++) {
3389 lio
->linfo
.rxpciq
[j
].u64
=
3390 resp
->cfg_info
.linfo
.rxpciq
[j
].u64
;
3392 for (j
= 0; j
< num_iqueues
; j
++) {
3393 lio
->linfo
.txpciq
[j
].u64
=
3394 resp
->cfg_info
.linfo
.txpciq
[j
].u64
;
3396 lio
->linfo
.hw_addr
= resp
->cfg_info
.linfo
.hw_addr
;
3397 lio
->linfo
.gmxport
= resp
->cfg_info
.linfo
.gmxport
;
3398 lio
->linfo
.link
.u64
= resp
->cfg_info
.linfo
.link
.u64
;
3400 lio
->msg_enable
= netif_msg_init(debug
, DEFAULT_MSG_ENABLE
);
3402 if (OCTEON_CN23XX_PF(octeon_dev
) ||
3403 OCTEON_CN6XXX(octeon_dev
)) {
3404 lio
->dev_capability
= NETIF_F_HIGHDMA
3407 | NETIF_F_SG
| NETIF_F_RXCSUM
3409 | NETIF_F_TSO
| NETIF_F_TSO6
3412 netif_set_gso_max_size(netdev
, OCTNIC_GSO_MAX_SIZE
);
3414 /* Copy of transmit encapsulation capabilities:
3415 * TSO, TSO6, Checksums for this device
3417 lio
->enc_dev_capability
= NETIF_F_IP_CSUM
3419 | NETIF_F_GSO_UDP_TUNNEL
3420 | NETIF_F_HW_CSUM
| NETIF_F_SG
3422 | NETIF_F_TSO
| NETIF_F_TSO6
3425 netdev
->hw_enc_features
= (lio
->enc_dev_capability
&
3428 lio
->dev_capability
|= NETIF_F_GSO_UDP_TUNNEL
;
3430 netdev
->vlan_features
= lio
->dev_capability
;
3431 /* Add any unchangeable hw features */
3432 lio
->dev_capability
|= NETIF_F_HW_VLAN_CTAG_FILTER
|
3433 NETIF_F_HW_VLAN_CTAG_RX
|
3434 NETIF_F_HW_VLAN_CTAG_TX
;
3436 netdev
->features
= (lio
->dev_capability
& ~NETIF_F_LRO
);
3438 netdev
->hw_features
= lio
->dev_capability
;
3439 /*HW_VLAN_RX and HW_VLAN_FILTER is always on*/
3440 netdev
->hw_features
= netdev
->hw_features
&
3441 ~NETIF_F_HW_VLAN_CTAG_RX
;
3443 /* Point to the properties for octeon device to which this
3444 * interface belongs.
3446 lio
->oct_dev
= octeon_dev
;
3447 lio
->octprops
= props
;
3448 lio
->netdev
= netdev
;
3450 dev_dbg(&octeon_dev
->pci_dev
->dev
,
3451 "if%d gmx: %d hw_addr: 0x%llx\n", i
,
3452 lio
->linfo
.gmxport
, CVM_CAST64(lio
->linfo
.hw_addr
));
3454 /* 64-bit swap required on LE machines */
3455 octeon_swap_8B_data(&lio
->linfo
.hw_addr
, 1);
3456 for (j
= 0; j
< 6; j
++)
3457 mac
[j
] = *((u8
*)(((u8
*)&lio
->linfo
.hw_addr
) + 2 + j
));
3459 /* Copy MAC Address to OS network device structure */
3461 ether_addr_copy(netdev
->dev_addr
, mac
);
3463 /* By default all interfaces on a single Octeon uses the same
3466 lio
->txq
= lio
->linfo
.txpciq
[0].s
.q_no
;
3467 lio
->rxq
= lio
->linfo
.rxpciq
[0].s
.q_no
;
3468 if (setup_io_queues(octeon_dev
, i
)) {
3469 dev_err(&octeon_dev
->pci_dev
->dev
, "I/O queues creation failed\n");
3470 goto setup_nic_dev_fail
;
3473 ifstate_set(lio
, LIO_IFSTATE_DROQ_OPS
);
3475 lio
->tx_qsize
= octeon_get_tx_qsize(octeon_dev
, lio
->txq
);
3476 lio
->rx_qsize
= octeon_get_rx_qsize(octeon_dev
, lio
->rxq
);
3478 if (setup_glists(octeon_dev
, lio
, num_iqueues
)) {
3479 dev_err(&octeon_dev
->pci_dev
->dev
,
3480 "Gather list allocation failed\n");
3481 goto setup_nic_dev_fail
;
3484 /* Register ethtool support */
3485 liquidio_set_ethtool_ops(netdev
);
3486 octeon_dev
->priv_flags
= 0x0;
3488 if (netdev
->features
& NETIF_F_LRO
)
3489 liquidio_set_feature(netdev
, OCTNET_CMD_LRO_ENABLE
,
3490 OCTNIC_LROIPV4
| OCTNIC_LROIPV6
);
3492 liquidio_set_feature(netdev
, OCTNET_CMD_ENABLE_VLAN_FILTER
, 0);
3494 if ((debug
!= -1) && (debug
& NETIF_MSG_HW
))
3495 liquidio_set_feature(netdev
,
3496 OCTNET_CMD_VERBOSE_ENABLE
, 0);
3498 /* Register the network device with the OS */
3499 if (register_netdev(netdev
)) {
3500 dev_err(&octeon_dev
->pci_dev
->dev
, "Device registration failed\n");
3501 goto setup_nic_dev_fail
;
3504 dev_dbg(&octeon_dev
->pci_dev
->dev
,
3505 "Setup NIC ifidx:%d mac:%02x%02x%02x%02x%02x%02x\n",
3506 i
, mac
[0], mac
[1], mac
[2], mac
[3], mac
[4], mac
[5]);
3507 netif_carrier_off(netdev
);
3508 lio
->link_changes
++;
3510 ifstate_set(lio
, LIO_IFSTATE_REGISTERED
);
3512 /* Sending command to firmware to enable Rx checksum offload
3513 * by default at the time of setup of Liquidio driver for
3516 liquidio_set_rxcsum_command(netdev
, OCTNET_CMD_TNL_RX_CSUM_CTL
,
3517 OCTNET_CMD_RXCSUM_ENABLE
);
3518 liquidio_set_feature(netdev
, OCTNET_CMD_TNL_TX_CSUM_CTL
,
3519 OCTNET_CMD_TXCSUM_ENABLE
);
3521 dev_dbg(&octeon_dev
->pci_dev
->dev
,
3522 "NIC ifidx:%d Setup successful\n", i
);
3524 octeon_free_soft_command(octeon_dev
, sc
);
3531 octeon_free_soft_command(octeon_dev
, sc
);
3534 dev_err(&octeon_dev
->pci_dev
->dev
,
3535 "NIC ifidx:%d Setup failed\n", i
);
3536 liquidio_destroy_nic_device(octeon_dev
, i
);
3542 * \brief initialize the NIC
3543 * @param oct octeon device
3545 * This initialization routine is called once the Octeon device application is
3548 static int liquidio_init_nic_module(struct octeon_device
*oct
)
3550 struct oct_intrmod_cfg
*intrmod_cfg
;
3552 int num_nic_ports
= CFG_GET_NUM_NIC_PORTS(octeon_get_conf(oct
));
3554 dev_dbg(&oct
->pci_dev
->dev
, "Initializing network interfaces\n");
3556 /* only default iq and oq were initialized
3557 * initialize the rest as well
3559 /* run port_config command for each port */
3560 oct
->ifcount
= num_nic_ports
;
3562 memset(oct
->props
, 0,
3563 sizeof(struct octdev_props
) * num_nic_ports
);
3565 for (i
= 0; i
< MAX_OCTEON_LINKS
; i
++)
3566 oct
->props
[i
].gmxport
= -1;
3568 retval
= setup_nic_devices(oct
);
3570 dev_err(&oct
->pci_dev
->dev
, "Setup NIC devices failed\n");
3571 goto octnet_init_failure
;
3574 liquidio_ptp_init(oct
);
3576 /* Initialize interrupt moderation params */
3577 intrmod_cfg
= &((struct octeon_device
*)oct
)->intrmod
;
3578 intrmod_cfg
->rx_enable
= 1;
3579 intrmod_cfg
->check_intrvl
= LIO_INTRMOD_CHECK_INTERVAL
;
3580 intrmod_cfg
->maxpkt_ratethr
= LIO_INTRMOD_MAXPKT_RATETHR
;
3581 intrmod_cfg
->minpkt_ratethr
= LIO_INTRMOD_MINPKT_RATETHR
;
3582 intrmod_cfg
->rx_maxcnt_trigger
= LIO_INTRMOD_RXMAXCNT_TRIGGER
;
3583 intrmod_cfg
->rx_maxtmr_trigger
= LIO_INTRMOD_RXMAXTMR_TRIGGER
;
3584 intrmod_cfg
->rx_mintmr_trigger
= LIO_INTRMOD_RXMINTMR_TRIGGER
;
3585 intrmod_cfg
->rx_mincnt_trigger
= LIO_INTRMOD_RXMINCNT_TRIGGER
;
3586 intrmod_cfg
->tx_enable
= 1;
3587 intrmod_cfg
->tx_maxcnt_trigger
= LIO_INTRMOD_TXMAXCNT_TRIGGER
;
3588 intrmod_cfg
->tx_mincnt_trigger
= LIO_INTRMOD_TXMINCNT_TRIGGER
;
3589 intrmod_cfg
->rx_frames
= CFG_GET_OQ_INTR_PKT(octeon_get_conf(oct
));
3590 intrmod_cfg
->rx_usecs
= CFG_GET_OQ_INTR_TIME(octeon_get_conf(oct
));
3591 dev_dbg(&oct
->pci_dev
->dev
, "Network interfaces ready\n");
3595 octnet_init_failure
:
3603 * \brief starter callback that invokes the remaining initialization work after
3604 * the NIC is up and running.
3605 * @param octptr work struct work_struct
3607 static void nic_starter(struct work_struct
*work
)
3609 struct octeon_device
*oct
;
3610 struct cavium_wk
*wk
= (struct cavium_wk
*)work
;
3612 oct
= (struct octeon_device
*)wk
->ctxptr
;
3614 if (atomic_read(&oct
->status
) == OCT_DEV_RUNNING
)
3617 /* If the status of the device is CORE_OK, the core
3618 * application has reported its application type. Call
3619 * any registered handlers now and move to the RUNNING
3622 if (atomic_read(&oct
->status
) != OCT_DEV_CORE_OK
) {
3623 schedule_delayed_work(&oct
->nic_poll_work
.work
,
3624 LIQUIDIO_STARTER_POLL_INTERVAL_MS
);
3628 atomic_set(&oct
->status
, OCT_DEV_RUNNING
);
3630 if (oct
->app_mode
&& oct
->app_mode
== CVM_DRV_NIC_APP
) {
3631 dev_dbg(&oct
->pci_dev
->dev
, "Starting NIC module\n");
3633 if (liquidio_init_nic_module(oct
))
3634 dev_err(&oct
->pci_dev
->dev
, "NIC initialization failed\n");
3636 handshake
[oct
->octeon_id
].started_ok
= 1;
3638 dev_err(&oct
->pci_dev
->dev
,
3639 "Unexpected application running on NIC (%d). Check firmware.\n",
3643 complete(&handshake
[oct
->octeon_id
].started
);
3647 * \brief Device initialization for each Octeon device that is probed
3648 * @param octeon_dev octeon device
3650 static int octeon_device_init(struct octeon_device
*octeon_dev
)
3653 char bootcmd
[] = "\n";
3654 struct octeon_device_priv
*oct_priv
=
3655 (struct octeon_device_priv
*)octeon_dev
->priv
;
3656 atomic_set(&octeon_dev
->status
, OCT_DEV_BEGIN_STATE
);
3658 /* Enable access to the octeon device and make its DMA capability
3661 if (octeon_pci_os_setup(octeon_dev
))
3664 /* Identify the Octeon type and map the BAR address space. */
3665 if (octeon_chip_specific_setup(octeon_dev
)) {
3666 dev_err(&octeon_dev
->pci_dev
->dev
, "Chip specific setup failed\n");
3670 atomic_set(&octeon_dev
->status
, OCT_DEV_PCI_MAP_DONE
);
3672 octeon_dev
->app_mode
= CVM_DRV_INVALID_APP
;
3674 /* Do a soft reset of the Octeon device. */
3675 if (octeon_dev
->fn_list
.soft_reset(octeon_dev
))
3678 /* Initialize the dispatch mechanism used to push packets arriving on
3679 * Octeon Output queues.
3681 if (octeon_init_dispatch_list(octeon_dev
))
3684 octeon_register_dispatch_fn(octeon_dev
, OPCODE_NIC
,
3685 OPCODE_NIC_CORE_DRV_ACTIVE
,
3686 octeon_core_drv_init
,
3689 INIT_DELAYED_WORK(&octeon_dev
->nic_poll_work
.work
, nic_starter
);
3690 octeon_dev
->nic_poll_work
.ctxptr
= (void *)octeon_dev
;
3691 schedule_delayed_work(&octeon_dev
->nic_poll_work
.work
,
3692 LIQUIDIO_STARTER_POLL_INTERVAL_MS
);
3694 atomic_set(&octeon_dev
->status
, OCT_DEV_DISPATCH_INIT_DONE
);
3696 octeon_set_io_queues_off(octeon_dev
);
3698 if (OCTEON_CN23XX_PF(octeon_dev
)) {
3699 ret
= octeon_dev
->fn_list
.setup_device_regs(octeon_dev
);
3701 dev_err(&octeon_dev
->pci_dev
->dev
, "OCTEON: Failed to configure device registers\n");
3706 /* Initialize soft command buffer pool
3708 if (octeon_setup_sc_buffer_pool(octeon_dev
)) {
3709 dev_err(&octeon_dev
->pci_dev
->dev
, "sc buffer pool allocation failed\n");
3712 atomic_set(&octeon_dev
->status
, OCT_DEV_SC_BUFF_POOL_INIT_DONE
);
3714 /* Setup the data structures that manage this Octeon's Input queues. */
3715 if (octeon_setup_instr_queues(octeon_dev
)) {
3716 dev_err(&octeon_dev
->pci_dev
->dev
,
3717 "instruction queue initialization failed\n");
3718 /* On error, release any previously allocated queues */
3719 for (j
= 0; j
< octeon_dev
->num_iqs
; j
++)
3720 octeon_delete_instr_queue(octeon_dev
, j
);
3723 atomic_set(&octeon_dev
->status
, OCT_DEV_INSTR_QUEUE_INIT_DONE
);
3725 /* Initialize lists to manage the requests of different types that
3726 * arrive from user & kernel applications for this octeon device.
3728 if (octeon_setup_response_list(octeon_dev
)) {
3729 dev_err(&octeon_dev
->pci_dev
->dev
, "Response list allocation failed\n");
3732 atomic_set(&octeon_dev
->status
, OCT_DEV_RESP_LIST_INIT_DONE
);
3734 if (octeon_setup_output_queues(octeon_dev
)) {
3735 dev_err(&octeon_dev
->pci_dev
->dev
, "Output queue initialization failed\n");
3736 /* Release any previously allocated queues */
3737 for (j
= 0; j
< octeon_dev
->num_oqs
; j
++)
3738 octeon_delete_droq(octeon_dev
, j
);
3742 atomic_set(&octeon_dev
->status
, OCT_DEV_DROQ_INIT_DONE
);
3744 /* The input and output queue registers were setup earlier (the queues
3745 * were not enabled). Any additional registers that need to be
3746 * programmed should be done now.
3748 ret
= octeon_dev
->fn_list
.setup_device_regs(octeon_dev
);
3750 dev_err(&octeon_dev
->pci_dev
->dev
,
3751 "Failed to configure device registers\n");
3755 /* Initialize the tasklet that handles output queue packet processing.*/
3756 dev_dbg(&octeon_dev
->pci_dev
->dev
, "Initializing droq tasklet\n");
3757 tasklet_init(&oct_priv
->droq_tasklet
, octeon_droq_bh
,
3758 (unsigned long)octeon_dev
);
3760 /* Setup the interrupt handler and record the INT SUM register address
3762 if (octeon_setup_interrupt(octeon_dev
))
3765 /* Enable Octeon device interrupts */
3766 octeon_dev
->fn_list
.enable_interrupt(octeon_dev
->chip
);
3768 /* Enable the input and output queues for this Octeon device */
3769 octeon_dev
->fn_list
.enable_io_queues(octeon_dev
);
3771 atomic_set(&octeon_dev
->status
, OCT_DEV_IO_QUEUES_DONE
);
3773 dev_dbg(&octeon_dev
->pci_dev
->dev
, "Waiting for DDR initialization...\n");
3775 if (ddr_timeout
== 0)
3776 dev_info(&octeon_dev
->pci_dev
->dev
, "WAITING. Set ddr_timeout to non-zero value to proceed with initialization.\n");
3778 schedule_timeout_uninterruptible(HZ
* LIO_RESET_SECS
);
3780 /* Wait for the octeon to initialize DDR after the soft-reset. */
3781 while (ddr_timeout
== 0) {
3782 set_current_state(TASK_INTERRUPTIBLE
);
3783 if (schedule_timeout(HZ
/ 10)) {
3784 /* user probably pressed Control-C */
3788 ret
= octeon_wait_for_ddr_init(octeon_dev
, &ddr_timeout
);
3790 dev_err(&octeon_dev
->pci_dev
->dev
,
3791 "DDR not initialized. Please confirm that board is configured to boot from Flash, ret: %d\n",
3796 if (octeon_wait_for_bootloader(octeon_dev
, 1000) != 0) {
3797 dev_err(&octeon_dev
->pci_dev
->dev
, "Board not responding\n");
3801 /* Divert uboot to take commands from host instead. */
3802 ret
= octeon_console_send_cmd(octeon_dev
, bootcmd
, 50);
3804 dev_dbg(&octeon_dev
->pci_dev
->dev
, "Initializing consoles\n");
3805 ret
= octeon_init_consoles(octeon_dev
);
3807 dev_err(&octeon_dev
->pci_dev
->dev
, "Could not access board consoles\n");
3810 ret
= octeon_add_console(octeon_dev
, 0);
3812 dev_err(&octeon_dev
->pci_dev
->dev
, "Could not access board console\n");
3816 atomic_set(&octeon_dev
->status
, OCT_DEV_CONSOLE_INIT_DONE
);
3818 dev_dbg(&octeon_dev
->pci_dev
->dev
, "Loading firmware\n");
3819 ret
= load_firmware(octeon_dev
);
3821 dev_err(&octeon_dev
->pci_dev
->dev
, "Could not load firmware to board\n");
3825 handshake
[octeon_dev
->octeon_id
].init_ok
= 1;
3826 complete(&handshake
[octeon_dev
->octeon_id
].init
);
3828 atomic_set(&octeon_dev
->status
, OCT_DEV_HOST_OK
);
3830 /* Send Credit for Octeon Output queues. Credits are always sent after
3831 * the output queue is enabled.
3833 for (j
= 0; j
< octeon_dev
->num_oqs
; j
++)
3834 writel(octeon_dev
->droq
[j
]->max_count
,
3835 octeon_dev
->droq
[j
]->pkts_credit_reg
);
3837 /* Packets can start arriving on the output queues from this point. */
3843 * \brief Exits the module
3845 static void __exit
liquidio_exit(void)
3847 liquidio_deinit_pci();
3849 pr_info("LiquidIO network module is now unloaded\n");
3852 module_init(liquidio_init
);
3853 module_exit(liquidio_exit
);