{
struct ena_adapter *adapter = netdev_priv(dev);
+ /* Change the state of the device to trigger reset
+ * Check that we are not in the middle or a trigger already
+ */
+
+ if (test_and_set_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags))
+ return;
+
+ adapter->reset_reason = ENA_REGS_RESET_OS_NETDEV_WD;
u64_stats_update_begin(&adapter->syncp);
adapter->dev_stats.tx_timeout++;
u64_stats_update_end(&adapter->syncp);
netif_err(adapter, tx_err, dev, "Transmit time out\n");
-
- /* Change the state of the device to trigger reset */
- set_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags);
}
static void update_rx_ring_mtu(struct ena_adapter *adapter, int mtu)
int irq_idx = ENA_IO_IRQ_IDX(i);
rc = irq_cpu_rmap_add(adapter->netdev->rx_cpu_rmap,
- adapter->msix_entries[irq_idx].vector);
+ pci_irq_vector(adapter->pdev, irq_idx));
if (rc) {
free_irq_cpu_rmap(adapter->netdev->rx_cpu_rmap);
adapter->netdev->rx_cpu_rmap = NULL;
rxr->sgl_size = adapter->max_rx_sgl_size;
rxr->smoothed_interval =
ena_com_get_nonadaptive_moderation_interval_rx(ena_dev);
+ rxr->empty_rx_queue = 0;
}
}
ena_free_tx_resources(adapter, i);
}
+static inline int validate_rx_req_id(struct ena_ring *rx_ring, u16 req_id)
+{
+ if (likely(req_id < rx_ring->ring_size))
+ return 0;
+
+ netif_err(rx_ring->adapter, rx_err, rx_ring->netdev,
+ "Invalid rx req_id: %hu\n", req_id);
+
+ u64_stats_update_begin(&rx_ring->syncp);
+ rx_ring->rx_stats.bad_req_id++;
+ u64_stats_update_end(&rx_ring->syncp);
+
+ /* Trigger device reset */
+ rx_ring->adapter->reset_reason = ENA_REGS_RESET_INV_RX_REQ_ID;
+ set_bit(ENA_FLAG_TRIGGER_RESET, &rx_ring->adapter->flags);
+ return -EFAULT;
+}
+
/* ena_setup_rx_resources - allocate I/O Rx resources (Descriptors)
* @adapter: network interface device structure
* @qid: queue index
{
struct ena_ring *rx_ring = &adapter->rx_ring[qid];
struct ena_irq *ena_irq = &adapter->irq_tbl[ENA_IO_IRQ_IDX(qid)];
- int size, node;
+ int size, node, i;
if (rx_ring->rx_buffer_info) {
netif_err(adapter, ifup, adapter->netdev,
return -ENOMEM;
}
+ size = sizeof(u16) * rx_ring->ring_size;
+ rx_ring->free_rx_ids = vzalloc_node(size, node);
+ if (!rx_ring->free_rx_ids) {
+ rx_ring->free_rx_ids = vzalloc(size);
+ if (!rx_ring->free_rx_ids) {
+ vfree(rx_ring->rx_buffer_info);
+ return -ENOMEM;
+ }
+ }
+
+ /* Req id ring for receiving RX pkts out of order */
+ for (i = 0; i < rx_ring->ring_size; i++)
+ rx_ring->free_rx_ids[i] = i;
+
/* Reset rx statistics */
memset(&rx_ring->rx_stats, 0x0, sizeof(rx_ring->rx_stats));
vfree(rx_ring->rx_buffer_info);
rx_ring->rx_buffer_info = NULL;
+
+ vfree(rx_ring->free_rx_ids);
+ rx_ring->free_rx_ids = NULL;
}
/* ena_setup_all_rx_resources - allocate I/O Rx queues resources for all queues
static int ena_refill_rx_bufs(struct ena_ring *rx_ring, u32 num)
{
- u16 next_to_use;
+ u16 next_to_use, req_id;
u32 i;
int rc;
next_to_use = rx_ring->next_to_use;
for (i = 0; i < num; i++) {
- struct ena_rx_buffer *rx_info =
- &rx_ring->rx_buffer_info[next_to_use];
+ struct ena_rx_buffer *rx_info;
+
+ req_id = rx_ring->free_rx_ids[next_to_use];
+ rc = validate_rx_req_id(rx_ring, req_id);
+ if (unlikely(rc < 0))
+ break;
+
+ rx_info = &rx_ring->rx_buffer_info[req_id];
+
rc = ena_alloc_rx_page(rx_ring, rx_info,
__GFP_COLD | GFP_ATOMIC | __GFP_COMP);
}
rc = ena_com_add_single_rx_desc(rx_ring->ena_com_io_sq,
&rx_info->ena_buf,
- next_to_use);
+ req_id);
if (unlikely(rc)) {
netif_warn(rx_ring->adapter, rx_status, rx_ring->netdev,
"failed to add buffer for rx queue %d\n",
*/
static void ena_free_tx_bufs(struct ena_ring *tx_ring)
{
+ bool print_once = true;
u32 i;
for (i = 0; i < tx_ring->ring_size; i++) {
if (!tx_info->skb)
continue;
- netdev_notice(tx_ring->netdev,
- "free uncompleted tx skb qid %d idx 0x%x\n",
- tx_ring->qid, i);
+ if (print_once) {
+ netdev_notice(tx_ring->netdev,
+ "free uncompleted tx skb qid %d idx 0x%x\n",
+ tx_ring->qid, i);
+ print_once = false;
+ } else {
+ netdev_dbg(tx_ring->netdev,
+ "free uncompleted tx skb qid %d idx 0x%x\n",
+ tx_ring->qid, i);
+ }
ena_buf = tx_info->bufs;
dma_unmap_single(tx_ring->dev,
u64_stats_update_end(&tx_ring->syncp);
/* Trigger device reset */
+ tx_ring->adapter->reset_reason = ENA_REGS_RESET_INV_TX_REQ_ID;
set_bit(ENA_FLAG_TRIGGER_RESET, &tx_ring->adapter->flags);
return -EFAULT;
}
return tx_pkts;
}
+static struct sk_buff *ena_alloc_skb(struct ena_ring *rx_ring, bool frags)
+{
+ struct sk_buff *skb;
+
+ if (frags)
+ skb = napi_get_frags(rx_ring->napi);
+ else
+ skb = netdev_alloc_skb_ip_align(rx_ring->netdev,
+ rx_ring->rx_copybreak);
+
+ if (unlikely(!skb)) {
+ u64_stats_update_begin(&rx_ring->syncp);
+ rx_ring->rx_stats.skb_alloc_fail++;
+ u64_stats_update_end(&rx_ring->syncp);
+ netif_dbg(rx_ring->adapter, rx_err, rx_ring->netdev,
+ "Failed to allocate skb. frags: %d\n", frags);
+ return NULL;
+ }
+
+ return skb;
+}
+
static struct sk_buff *ena_rx_skb(struct ena_ring *rx_ring,
struct ena_com_rx_buf_info *ena_bufs,
u32 descs,
u16 *next_to_clean)
{
struct sk_buff *skb;
- struct ena_rx_buffer *rx_info =
- &rx_ring->rx_buffer_info[*next_to_clean];
- u32 len;
- u32 buf = 0;
+ struct ena_rx_buffer *rx_info;
+ u16 len, req_id, buf = 0;
void *va;
- len = ena_bufs[0].len;
+ len = ena_bufs[buf].len;
+ req_id = ena_bufs[buf].req_id;
+ rx_info = &rx_ring->rx_buffer_info[req_id];
+
if (unlikely(!rx_info->page)) {
netif_err(rx_ring->adapter, rx_err, rx_ring->netdev,
"Page is NULL\n");
prefetch(va + NET_IP_ALIGN);
if (len <= rx_ring->rx_copybreak) {
- skb = netdev_alloc_skb_ip_align(rx_ring->netdev,
- rx_ring->rx_copybreak);
- if (unlikely(!skb)) {
- u64_stats_update_begin(&rx_ring->syncp);
- rx_ring->rx_stats.skb_alloc_fail++;
- u64_stats_update_end(&rx_ring->syncp);
- netif_err(rx_ring->adapter, rx_err, rx_ring->netdev,
- "Failed to allocate skb\n");
+ skb = ena_alloc_skb(rx_ring, false);
+ if (unlikely(!skb))
return NULL;
- }
netif_dbg(rx_ring->adapter, rx_status, rx_ring->netdev,
"rx allocated small packet. len %d. data_len %d\n",
skb_put(skb, len);
skb->protocol = eth_type_trans(skb, rx_ring->netdev);
+ rx_ring->free_rx_ids[*next_to_clean] = req_id;
*next_to_clean = ENA_RX_RING_IDX_ADD(*next_to_clean, descs,
rx_ring->ring_size);
return skb;
}
- skb = napi_get_frags(rx_ring->napi);
- if (unlikely(!skb)) {
- netif_dbg(rx_ring->adapter, rx_status, rx_ring->netdev,
- "Failed allocating skb\n");
- u64_stats_update_begin(&rx_ring->syncp);
- rx_ring->rx_stats.skb_alloc_fail++;
- u64_stats_update_end(&rx_ring->syncp);
+ skb = ena_alloc_skb(rx_ring, true);
+ if (unlikely(!skb))
return NULL;
- }
do {
dma_unmap_page(rx_ring->dev,
skb->len, skb->data_len);
rx_info->page = NULL;
+
+ rx_ring->free_rx_ids[*next_to_clean] = req_id;
*next_to_clean =
ENA_RX_RING_IDX_NEXT(*next_to_clean,
rx_ring->ring_size);
if (likely(--descs == 0))
break;
- rx_info = &rx_ring->rx_buffer_info[*next_to_clean];
- len = ena_bufs[++buf].len;
+
+ buf++;
+ len = ena_bufs[buf].len;
+ req_id = ena_bufs[buf].req_id;
+ rx_info = &rx_ring->rx_buffer_info[req_id];
} while (1);
return skb;
int rc = 0;
int total_len = 0;
int rx_copybreak_pkt = 0;
+ int i;
netif_dbg(rx_ring->adapter, rx_status, rx_ring->netdev,
"%s qid %d\n", __func__, rx_ring->qid);
/* exit if we failed to retrieve a buffer */
if (unlikely(!skb)) {
- next_to_clean = ENA_RX_RING_IDX_ADD(next_to_clean,
- ena_rx_ctx.descs,
- rx_ring->ring_size);
+ for (i = 0; i < ena_rx_ctx.descs; i++) {
+ rx_ring->free_tx_ids[next_to_clean] =
+ rx_ring->ena_bufs[i].req_id;
+ next_to_clean =
+ ENA_RX_RING_IDX_NEXT(next_to_clean,
+ rx_ring->ring_size);
+ }
break;
}
u64_stats_update_end(&rx_ring->syncp);
/* Too many desc from the device. Trigger reset */
+ adapter->reset_reason = ENA_REGS_RESET_TOO_MANY_RX_DESCS;
set_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags);
return 0;
rx_ring->per_napi_bytes = 0;
}
+static inline void ena_unmask_interrupt(struct ena_ring *tx_ring,
+ struct ena_ring *rx_ring)
+{
+ struct ena_eth_io_intr_reg intr_reg;
+
+ /* Update intr register: rx intr delay,
+ * tx intr delay and interrupt unmask
+ */
+ ena_com_update_intr_reg(&intr_reg,
+ rx_ring->smoothed_interval,
+ tx_ring->smoothed_interval,
+ true);
+
+ /* It is a shared MSI-X.
+ * Tx and Rx CQ have pointer to it.
+ * So we use one of them to reach the intr reg
+ */
+ ena_com_unmask_intr(rx_ring->ena_com_io_cq, &intr_reg);
+}
+
static inline void ena_update_ring_numa_node(struct ena_ring *tx_ring,
struct ena_ring *rx_ring)
{
{
struct ena_napi *ena_napi = container_of(napi, struct ena_napi, napi);
struct ena_ring *tx_ring, *rx_ring;
- struct ena_eth_io_intr_reg intr_reg;
u32 tx_work_done;
u32 rx_work_done;
tx_budget = tx_ring->ring_size / ENA_TX_POLL_BUDGET_DIVIDER;
- if (!test_bit(ENA_FLAG_DEV_UP, &tx_ring->adapter->flags)) {
+ if (!test_bit(ENA_FLAG_DEV_UP, &tx_ring->adapter->flags) ||
+ test_bit(ENA_FLAG_TRIGGER_RESET, &tx_ring->adapter->flags)) {
napi_complete_done(napi, 0);
return 0;
}
tx_work_done = ena_clean_tx_irq(tx_ring, tx_budget);
rx_work_done = ena_clean_rx_irq(rx_ring, napi, budget);
- if ((budget > rx_work_done) && (tx_budget > tx_work_done)) {
- napi_complete_done(napi, rx_work_done);
+ /* If the device is about to reset or down, avoid unmask
+ * the interrupt and return 0 so NAPI won't reschedule
+ */
+ if (unlikely(!test_bit(ENA_FLAG_DEV_UP, &tx_ring->adapter->flags) ||
+ test_bit(ENA_FLAG_TRIGGER_RESET, &tx_ring->adapter->flags))) {
+ napi_complete_done(napi, 0);
+ ret = 0;
+ } else if ((budget > rx_work_done) && (tx_budget > tx_work_done)) {
napi_comp_call = 1;
- /* Tx and Rx share the same interrupt vector */
- if (ena_com_get_adaptive_moderation_enabled(rx_ring->ena_dev))
- ena_adjust_intr_moderation(rx_ring, tx_ring);
- /* Update intr register: rx intr delay, tx intr delay and
- * interrupt unmask
+ /* Update numa and unmask the interrupt only when schedule
+ * from the interrupt context (vs from sk_busy_loop)
*/
- ena_com_update_intr_reg(&intr_reg,
- rx_ring->smoothed_interval,
- tx_ring->smoothed_interval,
- true);
+ if (napi_complete_done(napi, rx_work_done)) {
+ /* Tx and Rx share the same interrupt vector */
+ if (ena_com_get_adaptive_moderation_enabled(rx_ring->ena_dev))
+ ena_adjust_intr_moderation(rx_ring, tx_ring);
- /* It is a shared MSI-X. Tx and Rx CQ have pointer to it.
- * So we use one of them to reach the intr reg
- */
- ena_com_unmask_intr(rx_ring->ena_com_io_cq, &intr_reg);
+ ena_unmask_interrupt(tx_ring, rx_ring);
+ }
ena_update_ring_numa_node(tx_ring, rx_ring);
{
struct ena_napi *ena_napi = data;
- napi_schedule(&ena_napi->napi);
+ napi_schedule_irqoff(&ena_napi->napi);
return IRQ_HANDLED;
}
+/* Reserve a single MSI-X vector for management (admin + aenq).
+ * plus reserve one vector for each potential io queue.
+ * the number of potential io queues is the minimum of what the device
+ * supports and the number of vCPUs.
+ */
static int ena_enable_msix(struct ena_adapter *adapter, int num_queues)
{
- int i, msix_vecs, rc;
+ int msix_vecs, irq_cnt;
if (test_bit(ENA_FLAG_MSIX_ENABLED, &adapter->flags)) {
netif_err(adapter, probe, adapter->netdev,
netif_dbg(adapter, probe, adapter->netdev,
"trying to enable MSI-X, vectors %d\n", msix_vecs);
- adapter->msix_entries = vzalloc(msix_vecs * sizeof(struct msix_entry));
+ irq_cnt = pci_alloc_irq_vectors(adapter->pdev, ENA_MIN_MSIX_VEC,
+ msix_vecs, PCI_IRQ_MSIX);
- if (!adapter->msix_entries)
- return -ENOMEM;
-
- for (i = 0; i < msix_vecs; i++)
- adapter->msix_entries[i].entry = i;
-
- rc = pci_enable_msix(adapter->pdev, adapter->msix_entries, msix_vecs);
- if (rc != 0) {
+ if (irq_cnt < 0) {
netif_err(adapter, probe, adapter->netdev,
- "Failed to enable MSI-X, vectors %d rc %d\n",
- msix_vecs, rc);
+ "Failed to enable MSI-X. irq_cnt %d\n", irq_cnt);
return -ENOSPC;
}
- netif_dbg(adapter, probe, adapter->netdev, "enable MSI-X, vectors %d\n",
- msix_vecs);
-
- if (msix_vecs >= 1) {
- if (ena_init_rx_cpu_rmap(adapter))
- netif_warn(adapter, probe, adapter->netdev,
- "Failed to map IRQs to CPUs\n");
+ if (irq_cnt != msix_vecs) {
+ netif_notice(adapter, probe, adapter->netdev,
+ "enable only %d MSI-X (out of %d), reduce the number of queues\n",
+ irq_cnt, msix_vecs);
+ adapter->num_queues = irq_cnt - ENA_ADMIN_MSIX_VEC;
}
- adapter->msix_vecs = msix_vecs;
+ if (ena_init_rx_cpu_rmap(adapter))
+ netif_warn(adapter, probe, adapter->netdev,
+ "Failed to map IRQs to CPUs\n");
+
+ adapter->msix_vecs = irq_cnt;
set_bit(ENA_FLAG_MSIX_ENABLED, &adapter->flags);
return 0;
ena_intr_msix_mgmnt;
adapter->irq_tbl[ENA_MGMNT_IRQ_IDX].data = adapter;
adapter->irq_tbl[ENA_MGMNT_IRQ_IDX].vector =
- adapter->msix_entries[ENA_MGMNT_IRQ_IDX].vector;
+ pci_irq_vector(adapter->pdev, ENA_MGMNT_IRQ_IDX);
cpu = cpumask_first(cpu_online_mask);
adapter->irq_tbl[ENA_MGMNT_IRQ_IDX].cpu = cpu;
cpumask_set_cpu(cpu,
adapter->irq_tbl[irq_idx].handler = ena_intr_msix_io;
adapter->irq_tbl[irq_idx].data = &adapter->ena_napi[i];
adapter->irq_tbl[irq_idx].vector =
- adapter->msix_entries[irq_idx].vector;
+ pci_irq_vector(adapter->pdev, irq_idx);
adapter->irq_tbl[irq_idx].cpu = cpu;
cpumask_set_cpu(cpu,
static void ena_disable_msix(struct ena_adapter *adapter)
{
if (test_and_clear_bit(ENA_FLAG_MSIX_ENABLED, &adapter->flags))
- pci_disable_msix(adapter->pdev);
-
- if (adapter->msix_entries)
- vfree(adapter->msix_entries);
- adapter->msix_entries = NULL;
+ pci_free_irq_vectors(adapter->pdev);
}
static void ena_disable_io_intr_sync(struct ena_adapter *adapter)
/* In case the RSS table wasn't initialized by probe */
if (!ena_dev->rss.tbl_log_size) {
rc = ena_rss_init_default(adapter);
- if (rc && (rc != -EPERM)) {
+ if (rc && (rc != -EOPNOTSUPP)) {
netif_err(adapter, ifup, adapter->netdev,
"Failed to init RSS rc: %d\n", rc);
return rc;
/* Set indirect table */
rc = ena_com_indirect_table_set(ena_dev);
- if (unlikely(rc && rc != -EPERM))
+ if (unlikely(rc && rc != -EOPNOTSUPP))
return rc;
/* Configure hash function (if supported) */
rc = ena_com_set_hash_function(ena_dev);
- if (unlikely(rc && (rc != -EPERM)))
+ if (unlikely(rc && (rc != -EOPNOTSUPP)))
return rc;
/* Configure hash inputs (if supported) */
rc = ena_com_set_hash_ctrl(ena_dev);
- if (unlikely(rc && (rc != -EPERM)))
+ if (unlikely(rc && (rc != -EOPNOTSUPP)))
return rc;
return 0;
ena_napi_enable_all(adapter);
+ /* Enable completion queues interrupt */
+ for (i = 0; i < adapter->num_queues; i++)
+ ena_unmask_interrupt(&adapter->tx_ring[i],
+ &adapter->rx_ring[i]);
+
/* schedule napi in case we had pending packets
* from the last time we disable napi
*/
"Failed to get TX queue handlers. TX queue num %d rc: %d\n",
qid, rc);
ena_com_destroy_io_queue(ena_dev, ena_qid);
+ return rc;
}
ena_com_update_numa_node(tx_ring->ena_com_io_cq, ctx.numa_node);
"Failed to get RX queue handlers. RX queue num %d rc: %d\n",
qid, rc);
ena_com_destroy_io_queue(ena_dev, ena_qid);
+ return rc;
}
ena_com_update_numa_node(rx_ring->ena_com_io_cq, ctx.numa_node);
adapter->dev_stats.interface_down++;
u64_stats_update_end(&adapter->syncp);
- /* After this point the napi handler won't enable the tx queue */
- ena_napi_disable_all(adapter);
netif_carrier_off(adapter->netdev);
netif_tx_disable(adapter->netdev);
+ /* After this point the napi handler won't enable the tx queue */
+ ena_napi_disable_all(adapter);
+
/* After destroy the queue there won't be any new interrupts */
+
+ if (test_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags)) {
+ int rc;
+
+ rc = ena_com_dev_reset(adapter->ena_dev, adapter->reset_reason);
+ if (rc)
+ dev_err(&adapter->pdev->dev, "Device reset failed\n");
+ }
+
ena_destroy_all_io_queues(adapter);
ena_disable_io_intr_sync(adapter);
tx_info->tx_descs = nb_hw_desc;
tx_info->last_jiffies = jiffies;
+ tx_info->print_once = 0;
tx_ring->next_to_use = ENA_TX_RING_IDX_NEXT(next_to_use,
tx_ring->ring_size);
struct ena_adapter *adapter = netdev_priv(netdev);
int i;
+ /* Dont schedule NAPI if the driver is in the middle of reset
+ * or netdev is down.
+ */
+
+ if (!test_bit(ENA_FLAG_DEV_UP, &adapter->flags) ||
+ test_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags))
+ return;
+
for (i = 0; i < adapter->num_queues; i++)
napi_schedule(&adapter->ena_napi[i].napi);
}
rc = ena_com_set_host_attributes(ena_dev);
if (rc) {
- if (rc == -EPERM)
+ if (rc == -EOPNOTSUPP)
pr_warn("Cannot set host attributes\n");
else
pr_err("Cannot set host attributes\n");
rc = ena_com_set_host_attributes(adapter->ena_dev);
if (rc) {
- if (rc == -EPERM)
+ if (rc == -EOPNOTSUPP)
netif_warn(adapter, drv, adapter->netdev,
"Cannot set host attributes\n");
else
struct rtnl_link_stats64 *stats)
{
struct ena_adapter *adapter = netdev_priv(netdev);
- struct ena_admin_basic_stats ena_stats;
- int rc;
+ struct ena_ring *rx_ring, *tx_ring;
+ unsigned int start;
+ u64 rx_drops;
+ int i;
if (!test_bit(ENA_FLAG_DEV_UP, &adapter->flags))
return NULL;
- rc = ena_com_get_dev_basic_stats(adapter->ena_dev, &ena_stats);
- if (rc)
- return NULL;
+ for (i = 0; i < adapter->num_queues; i++) {
+ u64 bytes, packets;
+
+ tx_ring = &adapter->tx_ring[i];
+
+ do {
+ start = u64_stats_fetch_begin_irq(&tx_ring->syncp);
+ packets = tx_ring->tx_stats.cnt;
+ bytes = tx_ring->tx_stats.bytes;
+ } while (u64_stats_fetch_retry_irq(&tx_ring->syncp, start));
- stats->tx_bytes = ((u64)ena_stats.tx_bytes_high << 32) |
- ena_stats.tx_bytes_low;
- stats->rx_bytes = ((u64)ena_stats.rx_bytes_high << 32) |
- ena_stats.rx_bytes_low;
+ stats->tx_packets += packets;
+ stats->tx_bytes += bytes;
- stats->rx_packets = ((u64)ena_stats.rx_pkts_high << 32) |
- ena_stats.rx_pkts_low;
- stats->tx_packets = ((u64)ena_stats.tx_pkts_high << 32) |
- ena_stats.tx_pkts_low;
+ rx_ring = &adapter->rx_ring[i];
+
+ do {
+ start = u64_stats_fetch_begin_irq(&rx_ring->syncp);
+ packets = rx_ring->rx_stats.cnt;
+ bytes = rx_ring->rx_stats.bytes;
+ } while (u64_stats_fetch_retry_irq(&rx_ring->syncp, start));
+
+ stats->rx_packets += packets;
+ stats->rx_bytes += bytes;
+ }
+
+ do {
+ start = u64_stats_fetch_begin_irq(&adapter->syncp);
+ rx_drops = adapter->dev_stats.rx_drops;
+ } while (u64_stats_fetch_retry_irq(&adapter->syncp, start));
- stats->rx_dropped = ((u64)ena_stats.rx_drops_high << 32) |
- ena_stats.rx_drops_low;
+ stats->rx_dropped = rx_drops;
stats->multicast = 0;
stats->collisions = 0;
readless_supported = !(pdev->revision & ENA_MMIO_DISABLE_REG_READ);
ena_com_set_mmio_read_mode(ena_dev, readless_supported);
- rc = ena_com_dev_reset(ena_dev);
+ rc = ena_com_dev_reset(ena_dev, ENA_REGS_RESET_NORMAL);
if (rc) {
dev_err(dev, "Can not reset device\n");
goto err_mmio_read_less;
*/
ena_com_set_admin_polling_mode(ena_dev, true);
+ ena_config_host_info(ena_dev);
+
/* Get Device Attributes*/
rc = ena_com_get_dev_attr_feat(ena_dev, get_feat_ctx);
if (rc) {
*wd_state = !!(aenq_groups & BIT(ENA_ADMIN_KEEP_ALIVE));
- ena_config_host_info(ena_dev);
-
return 0;
err_admin_init:
+ ena_com_delete_host_info(ena_dev);
ena_com_admin_destroy(ena_dev);
err_mmio_read_less:
ena_com_mmio_reg_read_request_destroy(ena_dev);
bool dev_up, wd_state;
int rc;
+ if (unlikely(!test_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags))) {
+ dev_err(&pdev->dev,
+ "device reset schedule while reset bit is off\n");
+ return;
+ }
+
+ netif_carrier_off(netdev);
+
del_timer_sync(&adapter->timer_service);
rtnl_lock();
*/
ena_close(netdev);
- rc = ena_com_dev_reset(ena_dev);
- if (rc) {
- dev_err(&pdev->dev, "Device reset failed\n");
- goto err;
- }
-
ena_free_mgmnt_irq(adapter);
ena_disable_msix(adapter);
ena_com_mmio_reg_read_request_destroy(ena_dev);
+ adapter->reset_reason = ENA_REGS_RESET_NORMAL;
+ clear_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags);
+
/* Finish with the destroy part. Start the init part */
rc = ena_device_init(ena_dev, adapter->pdev, &get_feat_ctx, &wd_state);
err:
rtnl_unlock();
+ clear_bit(ENA_FLAG_DEVICE_RUNNING, &adapter->flags);
+
dev_err(&pdev->dev,
"Reset attempt failed. Can not reset the device\n");
}
-static void check_for_missing_tx_completions(struct ena_adapter *adapter)
+static int check_missing_comp_in_queue(struct ena_adapter *adapter,
+ struct ena_ring *tx_ring)
{
struct ena_tx_buffer *tx_buf;
unsigned long last_jiffies;
+ u32 missed_tx = 0;
+ int i;
+
+ for (i = 0; i < tx_ring->ring_size; i++) {
+ tx_buf = &tx_ring->tx_buffer_info[i];
+ last_jiffies = tx_buf->last_jiffies;
+ if (unlikely(last_jiffies &&
+ time_is_before_jiffies(last_jiffies + adapter->missing_tx_completion_to))) {
+ if (!tx_buf->print_once)
+ netif_notice(adapter, tx_err, adapter->netdev,
+ "Found a Tx that wasn't completed on time, qid %d, index %d.\n",
+ tx_ring->qid, i);
+
+ tx_buf->print_once = 1;
+ missed_tx++;
+
+ if (unlikely(missed_tx > adapter->missing_tx_completion_threshold)) {
+ netif_err(adapter, tx_err, adapter->netdev,
+ "The number of lost tx completions is above the threshold (%d > %d). Reset the device\n",
+ missed_tx,
+ adapter->missing_tx_completion_threshold);
+ adapter->reset_reason =
+ ENA_REGS_RESET_MISS_TX_CMPL;
+ set_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags);
+ return -EIO;
+ }
+ }
+ }
+
+ return 0;
+}
+
+static void check_for_missing_tx_completions(struct ena_adapter *adapter)
+{
struct ena_ring *tx_ring;
- int i, j, budget;
- u32 missed_tx;
+ int i, budget, rc;
/* Make sure the driver doesn't turn the device in other process */
smp_rmb();
if (!test_bit(ENA_FLAG_DEV_UP, &adapter->flags))
return;
+ if (test_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags))
+ return;
+
+ if (adapter->missing_tx_completion_to == ENA_HW_HINTS_NO_TIMEOUT)
+ return;
+
budget = ENA_MONITORED_TX_QUEUES;
for (i = adapter->last_monitored_tx_qid; i < adapter->num_queues; i++) {
tx_ring = &adapter->tx_ring[i];
- for (j = 0; j < tx_ring->ring_size; j++) {
- tx_buf = &tx_ring->tx_buffer_info[j];
- last_jiffies = tx_buf->last_jiffies;
- if (unlikely(last_jiffies && time_is_before_jiffies(last_jiffies + TX_TIMEOUT))) {
- netif_notice(adapter, tx_err, adapter->netdev,
- "Found a Tx that wasn't completed on time, qid %d, index %d.\n",
- tx_ring->qid, j);
-
- u64_stats_update_begin(&tx_ring->syncp);
- missed_tx = tx_ring->tx_stats.missing_tx_comp++;
- u64_stats_update_end(&tx_ring->syncp);
-
- /* Clear last jiffies so the lost buffer won't
- * be counted twice.
- */
- tx_buf->last_jiffies = 0;
-
- if (unlikely(missed_tx > MAX_NUM_OF_TIMEOUTED_PACKETS)) {
- netif_err(adapter, tx_err, adapter->netdev,
- "The number of lost tx completion is above the threshold (%d > %d). Reset the device\n",
- missed_tx, MAX_NUM_OF_TIMEOUTED_PACKETS);
- set_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags);
- }
- }
- }
+ rc = check_missing_comp_in_queue(adapter, tx_ring);
+ if (unlikely(rc))
+ return;
budget--;
if (!budget)
adapter->last_monitored_tx_qid = i % adapter->num_queues;
}
+/* trigger napi schedule after 2 consecutive detections */
+#define EMPTY_RX_REFILL 2
+/* For the rare case where the device runs out of Rx descriptors and the
+ * napi handler failed to refill new Rx descriptors (due to a lack of memory
+ * for example).
+ * This case will lead to a deadlock:
+ * The device won't send interrupts since all the new Rx packets will be dropped
+ * The napi handler won't allocate new Rx descriptors so the device will be
+ * able to send new packets.
+ *
+ * This scenario can happen when the kernel's vm.min_free_kbytes is too small.
+ * It is recommended to have at least 512MB, with a minimum of 128MB for
+ * constrained environment).
+ *
+ * When such a situation is detected - Reschedule napi
+ */
+static void check_for_empty_rx_ring(struct ena_adapter *adapter)
+{
+ struct ena_ring *rx_ring;
+ int i, refill_required;
+
+ if (!test_bit(ENA_FLAG_DEV_UP, &adapter->flags))
+ return;
+
+ if (test_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags))
+ return;
+
+ for (i = 0; i < adapter->num_queues; i++) {
+ rx_ring = &adapter->rx_ring[i];
+
+ refill_required =
+ ena_com_sq_empty_space(rx_ring->ena_com_io_sq);
+ if (unlikely(refill_required == (rx_ring->ring_size - 1))) {
+ rx_ring->empty_rx_queue++;
+
+ if (rx_ring->empty_rx_queue >= EMPTY_RX_REFILL) {
+ u64_stats_update_begin(&rx_ring->syncp);
+ rx_ring->rx_stats.empty_rx_ring++;
+ u64_stats_update_end(&rx_ring->syncp);
+
+ netif_err(adapter, drv, adapter->netdev,
+ "trigger refill for ring %d\n", i);
+
+ napi_schedule(rx_ring->napi);
+ rx_ring->empty_rx_queue = 0;
+ }
+ } else {
+ rx_ring->empty_rx_queue = 0;
+ }
+ }
+}
+
/* Check for keep alive expiration */
static void check_for_missing_keep_alive(struct ena_adapter *adapter)
{
if (!adapter->wd_state)
return;
- keep_alive_expired = round_jiffies(adapter->last_keep_alive_jiffies
- + ENA_DEVICE_KALIVE_TIMEOUT);
+ if (adapter->keep_alive_timeout == ENA_HW_HINTS_NO_TIMEOUT)
+ return;
+
+ keep_alive_expired = round_jiffies(adapter->last_keep_alive_jiffies +
+ adapter->keep_alive_timeout);
if (unlikely(time_is_before_jiffies(keep_alive_expired))) {
netif_err(adapter, drv, adapter->netdev,
"Keep alive watchdog timeout.\n");
u64_stats_update_begin(&adapter->syncp);
adapter->dev_stats.wd_expired++;
u64_stats_update_end(&adapter->syncp);
+ adapter->reset_reason = ENA_REGS_RESET_KEEP_ALIVE_TO;
set_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags);
}
}
u64_stats_update_begin(&adapter->syncp);
adapter->dev_stats.admin_q_pause++;
u64_stats_update_end(&adapter->syncp);
+ adapter->reset_reason = ENA_REGS_RESET_ADMIN_TO;
set_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags);
}
}
+static void ena_update_hints(struct ena_adapter *adapter,
+ struct ena_admin_ena_hw_hints *hints)
+{
+ struct net_device *netdev = adapter->netdev;
+
+ if (hints->admin_completion_tx_timeout)
+ adapter->ena_dev->admin_queue.completion_timeout =
+ hints->admin_completion_tx_timeout * 1000;
+
+ if (hints->mmio_read_timeout)
+ /* convert to usec */
+ adapter->ena_dev->mmio_read.reg_read_to =
+ hints->mmio_read_timeout * 1000;
+
+ if (hints->missed_tx_completion_count_threshold_to_reset)
+ adapter->missing_tx_completion_threshold =
+ hints->missed_tx_completion_count_threshold_to_reset;
+
+ if (hints->missing_tx_completion_timeout) {
+ if (hints->missing_tx_completion_timeout == ENA_HW_HINTS_NO_TIMEOUT)
+ adapter->missing_tx_completion_to = ENA_HW_HINTS_NO_TIMEOUT;
+ else
+ adapter->missing_tx_completion_to =
+ msecs_to_jiffies(hints->missing_tx_completion_timeout);
+ }
+
+ if (hints->netdev_wd_timeout)
+ netdev->watchdog_timeo = msecs_to_jiffies(hints->netdev_wd_timeout);
+
+ if (hints->driver_watchdog_timeout) {
+ if (hints->driver_watchdog_timeout == ENA_HW_HINTS_NO_TIMEOUT)
+ adapter->keep_alive_timeout = ENA_HW_HINTS_NO_TIMEOUT;
+ else
+ adapter->keep_alive_timeout =
+ msecs_to_jiffies(hints->driver_watchdog_timeout);
+ }
+}
+
static void ena_update_host_info(struct ena_admin_host_info *host_info,
struct net_device *netdev)
{
check_for_missing_tx_completions(adapter);
+ check_for_empty_rx_ring(adapter);
+
if (debug_area)
ena_dump_stats_to_buf(adapter, debug_area);
if (host_info)
ena_update_host_info(host_info, adapter->netdev);
- if (unlikely(test_and_clear_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags))) {
+ if (unlikely(test_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags))) {
netif_err(adapter, drv, adapter->netdev,
"Trigger reset is on\n");
ena_dump_stats_to_dmesg(adapter);
io_sq_num = get_feat_ctx->max_queues.max_sq_num;
}
- io_queue_num = min_t(int, num_possible_cpus(), ENA_MAX_NUM_IO_QUEUES);
+ io_queue_num = min_t(int, num_online_cpus(), ENA_MAX_NUM_IO_QUEUES);
io_queue_num = min_t(int, io_queue_num, io_sq_num);
io_queue_num = min_t(int, io_queue_num,
get_feat_ctx->max_queues.max_cq_num);
netdev->features =
dev_features |
NETIF_F_SG |
- NETIF_F_NTUPLE |
NETIF_F_RXHASH |
NETIF_F_HIGHDMA;
val = ethtool_rxfh_indir_default(i, adapter->num_queues);
rc = ena_com_indirect_table_fill_entry(ena_dev, i,
ENA_IO_RXQ_IDX(val));
- if (unlikely(rc && (rc != -EPERM))) {
+ if (unlikely(rc && (rc != -EOPNOTSUPP))) {
dev_err(dev, "Cannot fill indirect table\n");
goto err_fill_indir;
}
rc = ena_com_fill_hash_function(ena_dev, ENA_ADMIN_CRC32, NULL,
ENA_HASH_KEY_SIZE, 0xFFFFFFFF);
- if (unlikely(rc && (rc != -EPERM))) {
+ if (unlikely(rc && (rc != -EOPNOTSUPP))) {
dev_err(dev, "Cannot fill hash function\n");
goto err_fill_indir;
}
rc = ena_com_set_default_hash_ctrl(ena_dev);
- if (unlikely(rc && (rc != -EPERM))) {
+ if (unlikely(rc && (rc != -EOPNOTSUPP))) {
dev_err(dev, "Cannot fill hash control\n");
goto err_fill_indir;
}
{
int release_bars;
+ if (ena_dev->mem_bar)
+ devm_iounmap(&pdev->dev, ena_dev->mem_bar);
+
+ devm_iounmap(&pdev->dev, ena_dev->reg_bar);
+
release_bars = pci_select_bars(pdev, IORESOURCE_MEM) & ENA_BAR_MASK;
pci_release_selected_regions(pdev, release_bars);
}
goto err_free_ena_dev;
}
- ena_dev->reg_bar = ioremap(pci_resource_start(pdev, ENA_REG_BAR),
- pci_resource_len(pdev, ENA_REG_BAR));
+ ena_dev->reg_bar = devm_ioremap(&pdev->dev,
+ pci_resource_start(pdev, ENA_REG_BAR),
+ pci_resource_len(pdev, ENA_REG_BAR));
if (!ena_dev->reg_bar) {
dev_err(&pdev->dev, "failed to remap regs bar\n");
rc = -EFAULT;
ena_set_push_mode(pdev, ena_dev, &get_feat_ctx);
if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) {
- ena_dev->mem_bar = ioremap_wc(pci_resource_start(pdev, ENA_MEM_BAR),
- pci_resource_len(pdev, ENA_MEM_BAR));
+ ena_dev->mem_bar = devm_ioremap_wc(&pdev->dev,
+ pci_resource_start(pdev, ENA_MEM_BAR),
+ pci_resource_len(pdev, ENA_MEM_BAR));
if (!ena_dev->mem_bar) {
rc = -EFAULT;
goto err_device_destroy;
ena_set_conf_feat_params(adapter, &get_feat_ctx);
adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE);
+ adapter->reset_reason = ENA_REGS_RESET_NORMAL;
adapter->tx_ring_size = queue_size;
adapter->rx_ring_size = queue_size;
goto err_worker_destroy;
}
rc = ena_rss_init_default(adapter);
- if (rc && (rc != -EPERM)) {
+ if (rc && (rc != -EOPNOTSUPP)) {
dev_err(&pdev->dev, "Cannot init RSS rc: %d\n", rc);
goto err_free_msix;
}
INIT_WORK(&adapter->reset_task, ena_fw_reset_device);
adapter->last_keep_alive_jiffies = jiffies;
+ adapter->keep_alive_timeout = ENA_DEVICE_KALIVE_TIMEOUT;
+ adapter->missing_tx_completion_to = TX_TIMEOUT;
+ adapter->missing_tx_completion_threshold = MAX_NUM_OF_TIMEOUTED_PACKETS;
+
+ ena_update_hints(adapter, &get_feat_ctx.hw_hints);
setup_timer(&adapter->timer_service, ena_timer_service,
(unsigned long)adapter);
ena_com_delete_debug_area(ena_dev);
ena_com_rss_destroy(ena_dev);
err_free_msix:
- ena_com_dev_reset(ena_dev);
+ ena_com_dev_reset(ena_dev, ENA_REGS_RESET_INIT_ERR);
ena_free_mgmnt_irq(adapter);
ena_disable_msix(adapter);
err_worker_destroy:
struct ena_com_dev *ena_dev;
struct net_device *netdev;
- if (!adapter)
- /* This device didn't load properly and it's resources
- * already released, nothing to do
- */
- return;
-
ena_dev = adapter->ena_dev;
netdev = adapter->netdev;
cancel_work_sync(&adapter->resume_io_task);
- ena_com_dev_reset(ena_dev);
+ /* Reset the device only if the device is running. */
+ if (test_bit(ENA_FLAG_DEVICE_RUNNING, &adapter->flags))
+ ena_com_dev_reset(ena_dev, adapter->reset_reason);
ena_free_mgmnt_irq(adapter);
struct ena_admin_aenq_entry *aenq_e)
{
struct ena_adapter *adapter = (struct ena_adapter *)adapter_data;
+ struct ena_admin_ena_hw_hints *hints;
WARN(aenq_e->aenq_common_desc.group != ENA_ADMIN_NOTIFICATION,
"Invalid group(%x) expected %x\n",
case ENA_ADMIN_RESUME:
queue_work(ena_wq, &adapter->resume_io_task);
break;
+ case ENA_ADMIN_UPDATE_HINTS:
+ hints = (struct ena_admin_ena_hw_hints *)
+ (&aenq_e->inline_data_w4);
+ ena_update_hints(adapter, hints);
+ break;
default:
netif_err(adapter, drv, adapter->netdev,
"Invalid aenq notification link state %d\n",
projects / mirror_ubuntu-zesty-kernel.git / blobdiff