drivers/net/ethernet/huawei/hinic/hinic_rx.c

   1 /*
   2  * Huawei HiNIC PCI Express Linux driver
   3  * Copyright(c) 2017 Huawei Technologies Co., Ltd
   4  *
   5  * This program is free software; you can redistribute it and/or modify it
   6  * under the terms and conditions of the GNU General Public License,
   7  * version 2, as published by the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * for more details.
  13  *
  14  */
  15
  16 #include <linux/kernel.h>
  17 #include <linux/types.h>
  18 #include <linux/errno.h>
  19 #include <linux/pci.h>
  20 #include <linux/device.h>
  21 #include <linux/netdevice.h>
  22 #include <linux/etherdevice.h>
  23 #include <linux/u64_stats_sync.h>
  24 #include <linux/slab.h>
  25 #include <linux/interrupt.h>
  26 #include <linux/skbuff.h>
  27 #include <linux/dma-mapping.h>
  28 #include <linux/prefetch.h>
  29 #include <linux/cpumask.h>
  30 #include <asm/barrier.h>
  31
  32 #include "hinic_common.h"
  33 #include "hinic_hw_if.h"
  34 #include "hinic_hw_wqe.h"
  35 #include "hinic_hw_wq.h"
  36 #include "hinic_hw_qp.h"
  37 #include "hinic_hw_dev.h"
  38 #include "hinic_rx.h"
  39 #include "hinic_dev.h"
  40
  41 #define RX_IRQ_NO_PENDING               0
  42 #define RX_IRQ_NO_COALESC               0
  43 #define RX_IRQ_NO_LLI_TIMER             0
  44 #define RX_IRQ_NO_CREDIT                0
  45 #define RX_IRQ_NO_RESEND_TIMER          0
  46 #define HINIC_RX_BUFFER_WRITE           16
  47
  48 /**
  49  * hinic_rxq_clean_stats - Clean the statistics of specific queue
  50  * @rxq: Logical Rx Queue
  51  **/
  52 void hinic_rxq_clean_stats(struct hinic_rxq *rxq)
  53 {
  54         struct hinic_rxq_stats *rxq_stats = &rxq->rxq_stats;
  55
  56         u64_stats_update_begin(&rxq_stats->syncp);
  57         rxq_stats->pkts  = 0;
  58         rxq_stats->bytes = 0;
  59         u64_stats_update_end(&rxq_stats->syncp);
  60 }
  61
  62 /**
  63  * hinic_rxq_get_stats - get statistics of Rx Queue
  64  * @rxq: Logical Rx Queue
  65  * @stats: return updated stats here
  66  **/
  67 void hinic_rxq_get_stats(struct hinic_rxq *rxq, struct hinic_rxq_stats *stats)
  68 {
  69         struct hinic_rxq_stats *rxq_stats = &rxq->rxq_stats;
  70         unsigned int start;
  71
  72         u64_stats_update_begin(&stats->syncp);
  73         do {
  74                 start = u64_stats_fetch_begin(&rxq_stats->syncp);
  75                 stats->pkts = rxq_stats->pkts;
  76                 stats->bytes = rxq_stats->bytes;
  77         } while (u64_stats_fetch_retry(&rxq_stats->syncp, start));
  78         u64_stats_update_end(&stats->syncp);
  79 }
  80
  81 /**
  82  * rxq_stats_init - Initialize the statistics of specific queue
  83  * @rxq: Logical Rx Queue
  84  **/
  85 static void rxq_stats_init(struct hinic_rxq *rxq)
  86 {
  87         struct hinic_rxq_stats *rxq_stats = &rxq->rxq_stats;
  88
  89         u64_stats_init(&rxq_stats->syncp);
  90         hinic_rxq_clean_stats(rxq);
  91 }
  92
  93 static void rx_csum(struct hinic_rxq *rxq, u16 cons_idx,
  94                     struct sk_buff *skb)
  95 {
  96         struct net_device *netdev = rxq->netdev;
  97         struct hinic_rq_cqe *cqe;
  98         struct hinic_rq *rq;
  99         u32 csum_err;
 100         u32 status;
 101
 102         rq = rxq->rq;
 103         cqe = rq->cqe[cons_idx];
 104         status = be32_to_cpu(cqe->status);
 105         csum_err = HINIC_RQ_CQE_STATUS_GET(status, CSUM_ERR);
 106
 107         if (!(netdev->features & NETIF_F_RXCSUM))
 108                 return;
 109
 110         if (!csum_err)
 111                 skb->ip_summed = CHECKSUM_UNNECESSARY;
 112         else
 113                 skb->ip_summed = CHECKSUM_NONE;
 114 }
 115 /**
 116  * rx_alloc_skb - allocate skb and map it to dma address
 117  * @rxq: rx queue
 118  * @dma_addr: returned dma address for the skb
 119  *
 120  * Return skb
 121  **/
 122 static struct sk_buff *rx_alloc_skb(struct hinic_rxq *rxq,
 123                                     dma_addr_t *dma_addr)
 124 {
 125         struct hinic_dev *nic_dev = netdev_priv(rxq->netdev);
 126         struct hinic_hwdev *hwdev = nic_dev->hwdev;
 127         struct hinic_hwif *hwif = hwdev->hwif;
 128         struct pci_dev *pdev = hwif->pdev;
 129         struct sk_buff *skb;
 130         dma_addr_t addr;
 131         int err;
 132
 133         skb = netdev_alloc_skb_ip_align(rxq->netdev, rxq->rq->buf_sz);
 134         if (!skb) {
 135                 netdev_err(rxq->netdev, "Failed to allocate Rx SKB\n");
 136                 return NULL;
 137         }
 138
 139         addr = dma_map_single(&pdev->dev, skb->data, rxq->rq->buf_sz,
 140                               DMA_FROM_DEVICE);
 141         err = dma_mapping_error(&pdev->dev, addr);
 142         if (err) {
 143                 dev_err(&pdev->dev, "Failed to map Rx DMA, err = %d\n", err);
 144                 goto err_rx_map;
 145         }
 146
 147         *dma_addr = addr;
 148         return skb;
 149
 150 err_rx_map:
 151         dev_kfree_skb_any(skb);
 152         return NULL;
 153 }
 154
 155 /**
 156  * rx_unmap_skb - unmap the dma address of the skb
 157  * @rxq: rx queue
 158  * @dma_addr: dma address of the skb
 159  **/
 160 static void rx_unmap_skb(struct hinic_rxq *rxq, dma_addr_t dma_addr)
 161 {
 162         struct hinic_dev *nic_dev = netdev_priv(rxq->netdev);
 163         struct hinic_hwdev *hwdev = nic_dev->hwdev;
 164         struct hinic_hwif *hwif = hwdev->hwif;
 165         struct pci_dev *pdev = hwif->pdev;
 166
 167         dma_unmap_single(&pdev->dev, dma_addr, rxq->rq->buf_sz,
 168                          DMA_FROM_DEVICE);
 169 }
 170
 171 /**
 172  * rx_free_skb - unmap and free skb
 173  * @rxq: rx queue
 174  * @skb: skb to free
 175  * @dma_addr: dma address of the skb
 176  **/
 177 static void rx_free_skb(struct hinic_rxq *rxq, struct sk_buff *skb,
 178                         dma_addr_t dma_addr)
 179 {
 180         rx_unmap_skb(rxq, dma_addr);
 181         dev_kfree_skb_any(skb);
 182 }
 183
 184 /**
 185  * rx_alloc_pkts - allocate pkts in rx queue
 186  * @rxq: rx queue
 187  *
 188  * Return number of skbs allocated
 189  **/
 190 static int rx_alloc_pkts(struct hinic_rxq *rxq)
 191 {
 192         struct hinic_dev *nic_dev = netdev_priv(rxq->netdev);
 193         struct hinic_rq_wqe *rq_wqe;
 194         unsigned int free_wqebbs;
 195         struct hinic_sge sge;
 196         dma_addr_t dma_addr;
 197         struct sk_buff *skb;
 198         u16 prod_idx;
 199         int i;
 200
 201         free_wqebbs = hinic_get_rq_free_wqebbs(rxq->rq);
 202
 203         /* Limit the allocation chunks */
 204         if (free_wqebbs > nic_dev->rx_weight)
 205                 free_wqebbs = nic_dev->rx_weight;
 206
 207         for (i = 0; i < free_wqebbs; i++) {
 208                 skb = rx_alloc_skb(rxq, &dma_addr);
 209                 if (!skb) {
 210                         netdev_err(rxq->netdev, "Failed to alloc Rx skb\n");
 211                         goto skb_out;
 212                 }
 213
 214                 hinic_set_sge(&sge, dma_addr, skb->len);
 215
 216                 rq_wqe = hinic_rq_get_wqe(rxq->rq, HINIC_RQ_WQE_SIZE,
 217                                           &prod_idx);
 218                 if (!rq_wqe) {
 219                         rx_free_skb(rxq, skb, dma_addr);
 220                         goto skb_out;
 221                 }
 222
 223                 hinic_rq_prepare_wqe(rxq->rq, prod_idx, rq_wqe, &sge);
 224
 225                 hinic_rq_write_wqe(rxq->rq, prod_idx, rq_wqe, skb);
 226         }
 227
 228 skb_out:
 229         if (i) {
 230                 wmb();  /* write all the wqes before update PI */
 231
 232                 hinic_rq_update(rxq->rq, prod_idx);
 233         }
 234
 235         return i;
 236 }
 237
 238 /**
 239  * free_all_rx_skbs - free all skbs in rx queue
 240  * @rxq: rx queue
 241  **/
 242 static void free_all_rx_skbs(struct hinic_rxq *rxq)
 243 {
 244         struct hinic_rq *rq = rxq->rq;
 245         struct hinic_hw_wqe *hw_wqe;
 246         struct hinic_sge sge;
 247         u16 ci;
 248
 249         while ((hw_wqe = hinic_read_wqe(rq->wq, HINIC_RQ_WQE_SIZE, &ci))) {
 250                 if (IS_ERR(hw_wqe))
 251                         break;
 252
 253                 hinic_rq_get_sge(rq, &hw_wqe->rq_wqe, ci, &sge);
 254
 255                 hinic_put_wqe(rq->wq, HINIC_RQ_WQE_SIZE);
 256
 257                 rx_free_skb(rxq, rq->saved_skb[ci], hinic_sge_to_dma(&sge));
 258         }
 259 }
 260
 261 /**
 262  * rx_recv_jumbo_pkt - Rx handler for jumbo pkt
 263  * @rxq: rx queue
 264  * @head_skb: the first skb in the list
 265  * @left_pkt_len: left size of the pkt exclude head skb
 266  * @ci: consumer index
 267  *
 268  * Return number of wqes that used for the left of the pkt
 269  **/
 270 static int rx_recv_jumbo_pkt(struct hinic_rxq *rxq, struct sk_buff *head_skb,
 271                              unsigned int left_pkt_len, u16 ci)
 272 {
 273         struct sk_buff *skb, *curr_skb = head_skb;
 274         struct hinic_rq_wqe *rq_wqe;
 275         unsigned int curr_len;
 276         struct hinic_sge sge;
 277         int num_wqes = 0;
 278
 279         while (left_pkt_len > 0) {
 280                 rq_wqe = hinic_rq_read_next_wqe(rxq->rq, HINIC_RQ_WQE_SIZE,
 281                                                 &skb, &ci);
 282
 283                 num_wqes++;
 284
 285                 hinic_rq_get_sge(rxq->rq, rq_wqe, ci, &sge);
 286
 287                 rx_unmap_skb(rxq, hinic_sge_to_dma(&sge));
 288
 289                 prefetch(skb->data);
 290
 291                 curr_len = (left_pkt_len > HINIC_RX_BUF_SZ) ? HINIC_RX_BUF_SZ :
 292                             left_pkt_len;
 293
 294                 left_pkt_len -= curr_len;
 295
 296                 __skb_put(skb, curr_len);
 297
 298                 if (curr_skb == head_skb)
 299                         skb_shinfo(head_skb)->frag_list = skb;
 300                 else
 301                         curr_skb->next = skb;
 302
 303                 head_skb->len += skb->len;
 304                 head_skb->data_len += skb->len;
 305                 head_skb->truesize += skb->truesize;
 306
 307                 curr_skb = skb;
 308         }
 309
 310         return num_wqes;
 311 }
 312
 313 /**
 314  * rxq_recv - Rx handler
 315  * @rxq: rx queue
 316  * @budget: maximum pkts to process
 317  *
 318  * Return number of pkts received
 319  **/
 320 static int rxq_recv(struct hinic_rxq *rxq, int budget)
 321 {
 322         struct hinic_qp *qp = container_of(rxq->rq, struct hinic_qp, rq);
 323         u64 pkt_len = 0, rx_bytes = 0;
 324         struct hinic_rq_wqe *rq_wqe;
 325         unsigned int free_wqebbs;
 326         int num_wqes, pkts = 0;
 327         struct hinic_sge sge;
 328         struct sk_buff *skb;
 329         u16 ci;
 330
 331         while (pkts < budget) {
 332                 num_wqes = 0;
 333
 334                 rq_wqe = hinic_rq_read_wqe(rxq->rq, HINIC_RQ_WQE_SIZE, &skb,
 335                                            &ci);
 336                 if (!rq_wqe)
 337                         break;
 338
 339                 hinic_rq_get_sge(rxq->rq, rq_wqe, ci, &sge);
 340
 341                 rx_unmap_skb(rxq, hinic_sge_to_dma(&sge));
 342
 343                 rx_csum(rxq, ci, skb);
 344
 345                 prefetch(skb->data);
 346
 347                 pkt_len = sge.len;
 348
 349                 if (pkt_len <= HINIC_RX_BUF_SZ) {
 350                         __skb_put(skb, pkt_len);
 351                 } else {
 352                         __skb_put(skb, HINIC_RX_BUF_SZ);
 353                         num_wqes = rx_recv_jumbo_pkt(rxq, skb, pkt_len -
 354                                                      HINIC_RX_BUF_SZ, ci);
 355                 }
 356
 357                 hinic_rq_put_wqe(rxq->rq, ci,
 358                                  (num_wqes + 1) * HINIC_RQ_WQE_SIZE);
 359
 360                 skb_record_rx_queue(skb, qp->q_id);
 361                 skb->protocol = eth_type_trans(skb, rxq->netdev);
 362
 363                 napi_gro_receive(&rxq->napi, skb);
 364
 365                 pkts++;
 366                 rx_bytes += pkt_len;
 367         }
 368
 369         free_wqebbs = hinic_get_rq_free_wqebbs(rxq->rq);
 370         if (free_wqebbs > HINIC_RX_BUFFER_WRITE)
 371                 rx_alloc_pkts(rxq);
 372
 373         u64_stats_update_begin(&rxq->rxq_stats.syncp);
 374         rxq->rxq_stats.pkts += pkts;
 375         rxq->rxq_stats.bytes += rx_bytes;
 376         u64_stats_update_end(&rxq->rxq_stats.syncp);
 377
 378         return pkts;
 379 }
 380
 381 static int rx_poll(struct napi_struct *napi, int budget)
 382 {
 383         struct hinic_rxq *rxq = container_of(napi, struct hinic_rxq, napi);
 384         struct hinic_dev *nic_dev = netdev_priv(rxq->netdev);
 385         struct hinic_rq *rq = rxq->rq;
 386         int pkts;
 387
 388         pkts = rxq_recv(rxq, budget);
 389         if (pkts >= budget)
 390                 return budget;
 391
 392         napi_complete(napi);
 393         hinic_hwdev_set_msix_state(nic_dev->hwdev,
 394                                    rq->msix_entry,
 395                                    HINIC_MSIX_ENABLE);
 396
 397         return pkts;
 398 }
 399
 400 static void rx_add_napi(struct hinic_rxq *rxq)
 401 {
 402         struct hinic_dev *nic_dev = netdev_priv(rxq->netdev);
 403
 404         netif_napi_add(rxq->netdev, &rxq->napi, rx_poll, nic_dev->rx_weight);
 405         napi_enable(&rxq->napi);
 406 }
 407
 408 static void rx_del_napi(struct hinic_rxq *rxq)
 409 {
 410         napi_disable(&rxq->napi);
 411         netif_napi_del(&rxq->napi);
 412 }
 413
 414 static irqreturn_t rx_irq(int irq, void *data)
 415 {
 416         struct hinic_rxq *rxq = (struct hinic_rxq *)data;
 417         struct hinic_rq *rq = rxq->rq;
 418         struct hinic_dev *nic_dev;
 419
 420         /* Disable the interrupt until napi will be completed */
 421         nic_dev = netdev_priv(rxq->netdev);
 422         hinic_hwdev_set_msix_state(nic_dev->hwdev,
 423                                    rq->msix_entry,
 424                                    HINIC_MSIX_DISABLE);
 425
 426         nic_dev = netdev_priv(rxq->netdev);
 427         hinic_hwdev_msix_cnt_set(nic_dev->hwdev, rq->msix_entry);
 428
 429         napi_schedule(&rxq->napi);
 430         return IRQ_HANDLED;
 431 }
 432
 433 static int rx_request_irq(struct hinic_rxq *rxq)
 434 {
 435         struct hinic_dev *nic_dev = netdev_priv(rxq->netdev);
 436         struct hinic_hwdev *hwdev = nic_dev->hwdev;
 437         struct hinic_rq *rq = rxq->rq;
 438         struct hinic_qp *qp;
 439         struct cpumask mask;
 440         int err;
 441
 442         rx_add_napi(rxq);
 443
 444         hinic_hwdev_msix_set(hwdev, rq->msix_entry,
 445                              RX_IRQ_NO_PENDING, RX_IRQ_NO_COALESC,
 446                              RX_IRQ_NO_LLI_TIMER, RX_IRQ_NO_CREDIT,
 447                              RX_IRQ_NO_RESEND_TIMER);
 448
 449         err = request_irq(rq->irq, rx_irq, 0, rxq->irq_name, rxq);
 450         if (err) {
 451                 rx_del_napi(rxq);
 452                 return err;
 453         }
 454
 455         qp = container_of(rq, struct hinic_qp, rq);
 456         cpumask_set_cpu(qp->q_id % num_online_cpus(), &mask);
 457         return irq_set_affinity_hint(rq->irq, &mask);
 458 }
 459
 460 static void rx_free_irq(struct hinic_rxq *rxq)
 461 {
 462         struct hinic_rq *rq = rxq->rq;
 463
 464         irq_set_affinity_hint(rq->irq, NULL);
 465         free_irq(rq->irq, rxq);
 466         rx_del_napi(rxq);
 467 }
 468
 469 /**
 470  * hinic_init_rxq - Initialize the Rx Queue
 471  * @rxq: Logical Rx Queue
 472  * @rq: Hardware Rx Queue to connect the Logical queue with
 473  * @netdev: network device to connect the Logical queue with
 474  *
 475  * Return 0 - Success, negative - Failure
 476  **/
 477 int hinic_init_rxq(struct hinic_rxq *rxq, struct hinic_rq *rq,
 478                    struct net_device *netdev)
 479 {
 480         struct hinic_qp *qp = container_of(rq, struct hinic_qp, rq);
 481         int err, pkts, irqname_len;
 482
 483         rxq->netdev = netdev;
 484         rxq->rq = rq;
 485
 486         rxq_stats_init(rxq);
 487
 488         irqname_len = snprintf(NULL, 0, "hinic_rxq%d", qp->q_id) + 1;
 489         rxq->irq_name = devm_kzalloc(&netdev->dev, irqname_len, GFP_KERNEL);
 490         if (!rxq->irq_name)
 491                 return -ENOMEM;
 492
 493         sprintf(rxq->irq_name, "hinic_rxq%d", qp->q_id);
 494
 495         pkts = rx_alloc_pkts(rxq);
 496         if (!pkts) {
 497                 err = -ENOMEM;
 498                 goto err_rx_pkts;
 499         }
 500
 501         err = rx_request_irq(rxq);
 502         if (err) {
 503                 netdev_err(netdev, "Failed to request Rx irq\n");
 504                 goto err_req_rx_irq;
 505         }
 506
 507         return 0;
 508
 509 err_req_rx_irq:
 510 err_rx_pkts:
 511         free_all_rx_skbs(rxq);
 512         devm_kfree(&netdev->dev, rxq->irq_name);
 513         return err;
 514 }
 515
 516 /**
 517  * hinic_clean_rxq - Clean the Rx Queue
 518  * @rxq: Logical Rx Queue
 519  **/
 520 void hinic_clean_rxq(struct hinic_rxq *rxq)
 521 {
 522         struct net_device *netdev = rxq->netdev;
 523
 524         rx_free_irq(rxq);
 525
 526         free_all_rx_skbs(rxq);
 527         devm_kfree(&netdev->dev, rxq->irq_name);
 528 }