net/mlx5e: Report and recover from CQE error on ICOSQ

Add support for report and recovery from error on completion on ICOSQ.
Deactivate RQ and flush, then deactivate ICOSQ. Set the queue back to
ready state (firmware) and reset the ICOSQ and the RQ (software
resources). Finally, activate the ICOSQ and the RQ.

Signed-off-by: Aya Levin <ayal@mellanox.com>
Reviewed-by: Tariq Toukan <tariqt@mellanox.com>
Acked-by: Jiri Pirko <jiri@mellanox.com>
Signed-off-by: Saeed Mahameed <saeedm@mellanox.com>

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en.h b/drivers/net/ethernet/mellanox/mlx5/core/en.h
index 7381d57b4b829c3ee1f851a607dcfbf3723fadf0..842adf3719ff1aa7fbbaf04ba33cf8b23843f5ec 100644 (file)
--- a/drivers/net/ethernet/mellanox/mlx5/core/en.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en.h
@@ -551,6 +551,8 @@ struct mlx5e_icosq {
        /* control path */
        struct mlx5_wq_ctrl        wq_ctrl;
        struct mlx5e_channel      *channel;
+
+       struct work_struct         recover_work;
 } ____cacheline_aligned_in_smp;
 
 struct mlx5e_wqe_frag_info {
@@ -1026,6 +1028,12 @@ void mlx5e_set_rx_cq_mode_params(struct mlx5e_params *params,
 void mlx5e_set_rq_type(struct mlx5_core_dev *mdev, struct mlx5e_params *params);
 void mlx5e_init_rq_type_params(struct mlx5_core_dev *mdev,
                               struct mlx5e_params *params);
+int mlx5e_modify_rq_state(struct mlx5e_rq *rq, int curr_state, int next_state);
+void mlx5e_activate_rq(struct mlx5e_rq *rq);
+void mlx5e_deactivate_rq(struct mlx5e_rq *rq);
+void mlx5e_free_rx_descs(struct mlx5e_rq *rq);
+void mlx5e_activate_icosq(struct mlx5e_icosq *icosq);
+void mlx5e_deactivate_icosq(struct mlx5e_icosq *icosq);
 
 int mlx5e_modify_sq(struct mlx5_core_dev *mdev, u32 sqn,
                    struct mlx5e_modify_sq_param *p);

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/health.h b/drivers/net/ethernet/mellanox/mlx5/core/en/health.h
index a751c5316baf6c4391ec760870018341956b4e7d..8acd9dc520cf8bc96249fa3d0d58e16dbc57c540 100644 (file)
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/health.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/health.h
@@ -18,6 +18,7 @@ int mlx5e_reporter_named_obj_nest_end(struct devlink_fmsg *fmsg);
 
 int mlx5e_reporter_rx_create(struct mlx5e_priv *priv);
 void mlx5e_reporter_rx_destroy(struct mlx5e_priv *priv);
+void mlx5e_reporter_icosq_cqe_err(struct mlx5e_icosq *icosq);
 
 #define MLX5E_REPORTER_PER_Q_MAX_LEN 256
 

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_rx.c b/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_rx.c
index 24b9c819b2081a5978b43e372c465256b3ff83c2..ff49853b65d2fb178f45744b7dd11ce1cc48fe38 100644 (file)
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_rx.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_rx.c
@@ -27,6 +27,109 @@ out:
        return err;
 }
 
+static int mlx5e_wait_for_icosq_flush(struct mlx5e_icosq *icosq)
+{
+       unsigned long exp_time = jiffies + msecs_to_jiffies(2000);
+
+       while (time_before(jiffies, exp_time)) {
+               if (icosq->cc == icosq->pc)
+                       return 0;
+
+               msleep(20);
+       }
+
+       netdev_err(icosq->channel->netdev,
+                  "Wait for ICOSQ 0x%x flush timeout (cc = 0x%x, pc = 0x%x)\n",
+                  icosq->sqn, icosq->cc, icosq->pc);
+
+       return -ETIMEDOUT;
+}
+
+static void mlx5e_reset_icosq_cc_pc(struct mlx5e_icosq *icosq)
+{
+       WARN_ONCE(icosq->cc != icosq->pc, "ICOSQ 0x%x: cc (0x%x) != pc (0x%x)\n",
+                 icosq->sqn, icosq->cc, icosq->pc);
+       icosq->cc = 0;
+       icosq->pc = 0;
+}
+
+static int mlx5e_rx_reporter_err_icosq_cqe_recover(void *ctx)
+{
+       struct mlx5_core_dev *mdev;
+       struct mlx5e_icosq *icosq;
+       struct net_device *dev;
+       struct mlx5e_rq *rq;
+       u8 state;
+       int err;
+
+       icosq = ctx;
+       rq = &icosq->channel->rq;
+       mdev = icosq->channel->mdev;
+       dev = icosq->channel->netdev;
+       err = mlx5_core_query_sq_state(mdev, icosq->sqn, &state);
+       if (err) {
+               netdev_err(dev, "Failed to query ICOSQ 0x%x state. err = %d\n",
+                          icosq->sqn, err);
+               goto out;
+       }
+
+       if (state != MLX5_SQC_STATE_ERR)
+               goto out;
+
+       mlx5e_deactivate_rq(rq);
+       err = mlx5e_wait_for_icosq_flush(icosq);
+       if (err)
+               goto out;
+
+       mlx5e_deactivate_icosq(icosq);
+
+       /* At this point, both the rq and the icosq are disabled */
+
+       err = mlx5e_health_sq_to_ready(icosq->channel, icosq->sqn);
+       if (err)
+               goto out;
+
+       mlx5e_reset_icosq_cc_pc(icosq);
+       mlx5e_free_rx_descs(rq);
+       clear_bit(MLX5E_SQ_STATE_RECOVERING, &icosq->state);
+       mlx5e_activate_icosq(icosq);
+       mlx5e_activate_rq(rq);
+
+       rq->stats->recover++;
+       return 0;
+out:
+       clear_bit(MLX5E_SQ_STATE_RECOVERING, &icosq->state);
+       return err;
+}
+
+void mlx5e_reporter_icosq_cqe_err(struct mlx5e_icosq *icosq)
+{
+       struct mlx5e_priv *priv = icosq->channel->priv;
+       char err_str[MLX5E_REPORTER_PER_Q_MAX_LEN];
+       struct mlx5e_err_ctx err_ctx = {};
+
+       err_ctx.ctx = icosq;
+       err_ctx.recover = mlx5e_rx_reporter_err_icosq_cqe_recover;
+       sprintf(err_str, "ERR CQE on ICOSQ: 0x%x", icosq->sqn);
+
+       mlx5e_health_report(priv, priv->rx_reporter, err_str, &err_ctx);
+}
+
+static int mlx5e_rx_reporter_recover_from_ctx(struct mlx5e_err_ctx *err_ctx)
+{
+       return err_ctx->recover(err_ctx->ctx);
+}
+
+static int mlx5e_rx_reporter_recover(struct devlink_health_reporter *reporter,
+                                    void *context)
+{
+       struct mlx5e_priv *priv = devlink_health_reporter_priv(reporter);
+       struct mlx5e_err_ctx *err_ctx = context;
+
+       return err_ctx ? mlx5e_rx_reporter_recover_from_ctx(err_ctx) :
+                        mlx5e_health_recover_channels(priv);
+}
+
 static int mlx5e_rx_reporter_build_diagnose_output(struct mlx5e_rq *rq,
                                                   struct devlink_fmsg *fmsg)
 {
@@ -167,9 +270,12 @@ unlock:
 
 static const struct devlink_health_reporter_ops mlx5_rx_reporter_ops = {
        .name = "rx",
+       .recover = mlx5e_rx_reporter_recover,
        .diagnose = mlx5e_rx_reporter_diagnose,
 };
 
+#define MLX5E_REPORTER_RX_GRACEFUL_PERIOD 500
+
 int mlx5e_reporter_rx_create(struct mlx5e_priv *priv)
 {
        struct devlink *devlink = priv_to_devlink(priv->mdev);
@@ -177,7 +283,8 @@ int mlx5e_reporter_rx_create(struct mlx5e_priv *priv)
 
        reporter = devlink_health_reporter_create(devlink,
                                                  &mlx5_rx_reporter_ops,
-                                                 0, false, priv);
+                                                 MLX5E_REPORTER_RX_GRACEFUL_PERIOD,
+                                                 true, priv);
        if (IS_ERR(reporter)) {
                netdev_warn(priv->netdev, "Failed to create rx reporter, err = %ld\n",
                            PTR_ERR(reporter));

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
index 2c4e1cb54402cd4499c403d943109f1bc861173c..0e43ea1c27b0bebda7c6134a20ed8a82c07495e7 100644 (file)
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
@@ -700,8 +700,7 @@ static int mlx5e_create_rq(struct mlx5e_rq *rq,
        return err;
 }
 
-static int mlx5e_modify_rq_state(struct mlx5e_rq *rq, int curr_state,
-                                int next_state)
+int mlx5e_modify_rq_state(struct mlx5e_rq *rq, int curr_state, int next_state)
 {
        struct mlx5_core_dev *mdev = rq->mdev;
 
@@ -812,7 +811,7 @@ int mlx5e_wait_for_min_rx_wqes(struct mlx5e_rq *rq, int wait_time)
        return -ETIMEDOUT;
 }
 
-static void mlx5e_free_rx_descs(struct mlx5e_rq *rq)
+void mlx5e_free_rx_descs(struct mlx5e_rq *rq)
 {
        __be16 wqe_ix_be;
        u16 wqe_ix;
@@ -891,7 +890,7 @@ err_free_rq:
        return err;
 }
 
-static void mlx5e_activate_rq(struct mlx5e_rq *rq)
+void mlx5e_activate_rq(struct mlx5e_rq *rq)
 {
        set_bit(MLX5E_RQ_STATE_ENABLED, &rq->state);
        mlx5e_trigger_irq(&rq->channel->icosq);
@@ -906,6 +905,7 @@ void mlx5e_deactivate_rq(struct mlx5e_rq *rq)
 void mlx5e_close_rq(struct mlx5e_rq *rq)
 {
        cancel_work_sync(&rq->dim.work);
+       cancel_work_sync(&rq->channel->icosq.recover_work);
        mlx5e_destroy_rq(rq);
        mlx5e_free_rx_descs(rq);
        mlx5e_free_rq(rq);
@@ -1022,6 +1022,14 @@ static int mlx5e_alloc_icosq_db(struct mlx5e_icosq *sq, int numa)
        return 0;
 }
 
+static void mlx5e_icosq_err_cqe_work(struct work_struct *recover_work)
+{
+       struct mlx5e_icosq *sq = container_of(recover_work, struct mlx5e_icosq,
+                                             recover_work);
+
+       mlx5e_reporter_icosq_cqe_err(sq);
+}
+
 static int mlx5e_alloc_icosq(struct mlx5e_channel *c,
                             struct mlx5e_sq_param *param,
                             struct mlx5e_icosq *sq)
@@ -1044,6 +1052,8 @@ static int mlx5e_alloc_icosq(struct mlx5e_channel *c,
        if (err)
                goto err_sq_wq_destroy;
 
+       INIT_WORK(&sq->recover_work, mlx5e_icosq_err_cqe_work);
+
        return 0;
 
 err_sq_wq_destroy:
@@ -1388,12 +1398,12 @@ err_free_icosq:
        return err;
 }
 
-static void mlx5e_activate_icosq(struct mlx5e_icosq *icosq)
+void mlx5e_activate_icosq(struct mlx5e_icosq *icosq)
 {
        set_bit(MLX5E_SQ_STATE_ENABLED, &icosq->state);
 }
 
-static void mlx5e_deactivate_icosq(struct mlx5e_icosq *icosq)
+void mlx5e_deactivate_icosq(struct mlx5e_icosq *icosq)
 {
        struct mlx5e_channel *c = icosq->channel;
 

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c
index 60570b442fffe6d0ac29fa56a7f457f9475ff78e..0f6033ea475d941fed26ad5f1d4279b243512988 100644 (file)
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c
@@ -615,6 +615,8 @@ void mlx5e_poll_ico_cq(struct mlx5e_cq *cq)
                if (unlikely(get_cqe_opcode(cqe) != MLX5_CQE_REQ)) {
                        netdev_WARN_ONCE(cq->channel->netdev,
                                         "Bad OP in ICOSQ CQE: 0x%x\n", get_cqe_opcode(cqe));
+                       if (!test_and_set_bit(MLX5E_SQ_STATE_RECOVERING, &sq->state))
+                               queue_work(cq->channel->priv->wq, &sq->recover_work);
                        break;
                }
                do {

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c
index 94a32c76c182c01af25a2abf7132a24786a029fd..18e4c162256a27bffb9d1a4c9a446dfd57a45977 100644 (file)
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c
@@ -109,6 +109,7 @@ static const struct counter_desc sw_stats_desc[] = {
        { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_cache_waive) },
        { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_congst_umr) },
        { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_arfs_err) },
+       { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_recover) },
        { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, ch_events) },
        { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, ch_poll) },
        { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, ch_arm) },
@@ -220,6 +221,7 @@ static void mlx5e_grp_sw_update_stats(struct mlx5e_priv *priv)
                s->rx_cache_waive += rq_stats->cache_waive;
                s->rx_congst_umr  += rq_stats->congst_umr;
                s->rx_arfs_err    += rq_stats->arfs_err;
+               s->rx_recover     += rq_stats->recover;
                s->ch_events      += ch_stats->events;
                s->ch_poll        += ch_stats->poll;
                s->ch_arm         += ch_stats->arm;
@@ -1298,6 +1300,7 @@ static const struct counter_desc rq_stats_desc[] = {
        { MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, cache_waive) },
        { MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, congst_umr) },
        { MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, arfs_err) },
+       { MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, recover) },
 };
 
 static const struct counter_desc sq_stats_desc[] = {

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.h b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.h
index bf645d42c833d1e6566b457335d0accae4198b23..c281e567711d94b387b9968eea134a4995ecab50 100644 (file)
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.h
@@ -116,6 +116,7 @@ struct mlx5e_sw_stats {
        u64 rx_cache_waive;
        u64 rx_congst_umr;
        u64 rx_arfs_err;
+       u64 rx_recover;
        u64 ch_events;
        u64 ch_poll;
        u64 ch_arm;
@@ -249,6 +250,7 @@ struct mlx5e_rq_stats {
        u64 cache_waive;
        u64 congst_umr;
        u64 arfs_err;
+       u64 recover;
 };
 
 struct mlx5e_sq_stats {