drivers/infiniband/hw/hfi1/ipoib_tx.c

   1 // SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)
   2 /*
   3  * Copyright(c) 2020 Intel Corporation.
   4  *
   5  */
   6
   7 /*
   8  * This file contains HFI1 support for IPOIB SDMA functionality
   9  */
  10
  11 #include <linux/log2.h>
  12 #include <linux/circ_buf.h>
  13
  14 #include "sdma.h"
  15 #include "verbs.h"
  16 #include "trace_ibhdrs.h"
  17 #include "ipoib.h"
  18
  19 /* Add a convenience helper */
  20 #define CIRC_ADD(val, add, size) (((val) + (add)) & ((size) - 1))
  21 #define CIRC_NEXT(val, size) CIRC_ADD(val, 1, size)
  22 #define CIRC_PREV(val, size) CIRC_ADD(val, -1, size)
  23
  24 /**
  25  * struct ipoib_txreq - IPOIB transmit descriptor
  26  * @txreq: sdma transmit request
  27  * @sdma_hdr: 9b ib headers
  28  * @sdma_status: status returned by sdma engine
  29  * @priv: ipoib netdev private data
  30  * @txq: txq on which skb was output
  31  * @skb: skb to send
  32  */
  33 struct ipoib_txreq {
  34         struct sdma_txreq           txreq;
  35         struct hfi1_sdma_header     sdma_hdr;
  36         int                         sdma_status;
  37         struct hfi1_ipoib_dev_priv *priv;
  38         struct hfi1_ipoib_txq      *txq;
  39         struct sk_buff             *skb;
  40 };
  41
  42 struct ipoib_txparms {
  43         struct hfi1_devdata        *dd;
  44         struct rdma_ah_attr        *ah_attr;
  45         struct hfi1_ibport         *ibp;
  46         struct hfi1_ipoib_txq      *txq;
  47         union hfi1_ipoib_flow       flow;
  48         u32                         dqpn;
  49         u8                          hdr_dwords;
  50         u8                          entropy;
  51 };
  52
  53 static u64 hfi1_ipoib_txreqs(const u64 sent, const u64 completed)
  54 {
  55         return sent - completed;
  56 }
  57
  58 static u64 hfi1_ipoib_used(struct hfi1_ipoib_txq *txq)
  59 {
  60         return hfi1_ipoib_txreqs(txq->sent_txreqs,
  61                                  atomic64_read(&txq->complete_txreqs));
  62 }
  63
  64 static void hfi1_ipoib_stop_txq(struct hfi1_ipoib_txq *txq)
  65 {
  66         if (atomic_inc_return(&txq->stops) == 1)
  67                 netif_stop_subqueue(txq->priv->netdev, txq->q_idx);
  68 }
  69
  70 static void hfi1_ipoib_wake_txq(struct hfi1_ipoib_txq *txq)
  71 {
  72         if (atomic_dec_and_test(&txq->stops))
  73                 netif_wake_subqueue(txq->priv->netdev, txq->q_idx);
  74 }
  75
  76 static uint hfi1_ipoib_ring_hwat(struct hfi1_ipoib_txq *txq)
  77 {
  78         return min_t(uint, txq->priv->netdev->tx_queue_len,
  79                      txq->tx_ring.max_items - 1);
  80 }
  81
  82 static uint hfi1_ipoib_ring_lwat(struct hfi1_ipoib_txq *txq)
  83 {
  84         return min_t(uint, txq->priv->netdev->tx_queue_len,
  85                      txq->tx_ring.max_items) >> 1;
  86 }
  87
  88 static void hfi1_ipoib_check_queue_depth(struct hfi1_ipoib_txq *txq)
  89 {
  90         ++txq->sent_txreqs;
  91         if (hfi1_ipoib_used(txq) >= hfi1_ipoib_ring_hwat(txq) &&
  92             !atomic_xchg(&txq->ring_full, 1))
  93                 hfi1_ipoib_stop_txq(txq);
  94 }
  95
  96 static void hfi1_ipoib_check_queue_stopped(struct hfi1_ipoib_txq *txq)
  97 {
  98         struct net_device *dev = txq->priv->netdev;
  99
 100         /* If shutting down just return as queue state is irrelevant */
 101         if (unlikely(dev->reg_state != NETREG_REGISTERED))
 102                 return;
 103
 104         /*
 105          * When the queue has been drained to less than half full it will be
 106          * restarted.
 107          * The size of the txreq ring is fixed at initialization.
 108          * The tx queue len can be adjusted upward while the interface is
 109          * running.
 110          * The tx queue len can be large enough to overflow the txreq_ring.
 111          * Use the minimum of the current tx_queue_len or the rings max txreqs
 112          * to protect against ring overflow.
 113          */
 114         if (hfi1_ipoib_used(txq) < hfi1_ipoib_ring_lwat(txq) &&
 115             atomic_xchg(&txq->ring_full, 0))
 116                 hfi1_ipoib_wake_txq(txq);
 117 }
 118
 119 static void hfi1_ipoib_free_tx(struct ipoib_txreq *tx, int budget)
 120 {
 121         struct hfi1_ipoib_dev_priv *priv = tx->priv;
 122
 123         if (likely(!tx->sdma_status)) {
 124                 hfi1_ipoib_update_tx_netstats(priv, 1, tx->skb->len);
 125         } else {
 126                 ++priv->netdev->stats.tx_errors;
 127                 dd_dev_warn(priv->dd,
 128                             "%s: Status = 0x%x pbc 0x%llx txq = %d sde = %d\n",
 129                             __func__, tx->sdma_status,
 130                             le64_to_cpu(tx->sdma_hdr.pbc), tx->txq->q_idx,
 131                             tx->txq->sde->this_idx);
 132         }
 133
 134         napi_consume_skb(tx->skb, budget);
 135         sdma_txclean(priv->dd, &tx->txreq);
 136         kmem_cache_free(priv->txreq_cache, tx);
 137 }
 138
 139 static int hfi1_ipoib_drain_tx_ring(struct hfi1_ipoib_txq *txq, int budget)
 140 {
 141         struct hfi1_ipoib_circ_buf *tx_ring = &txq->tx_ring;
 142         unsigned long head;
 143         unsigned long tail;
 144         unsigned int max_tx;
 145         int work_done;
 146         int tx_count;
 147
 148         spin_lock_bh(&tx_ring->consumer_lock);
 149
 150         /* Read index before reading contents at that index. */
 151         head = smp_load_acquire(&tx_ring->head);
 152         tail = tx_ring->tail;
 153         max_tx = tx_ring->max_items;
 154
 155         work_done = min_t(int, CIRC_CNT(head, tail, max_tx), budget);
 156
 157         for (tx_count = work_done; tx_count; tx_count--) {
 158                 hfi1_ipoib_free_tx(tx_ring->items[tail], budget);
 159                 tail = CIRC_NEXT(tail, max_tx);
 160         }
 161
 162         atomic64_add(work_done, &txq->complete_txreqs);
 163
 164         /* Finished freeing tx items so store the tail value. */
 165         smp_store_release(&tx_ring->tail, tail);
 166
 167         spin_unlock_bh(&tx_ring->consumer_lock);
 168
 169         hfi1_ipoib_check_queue_stopped(txq);
 170
 171         return work_done;
 172 }
 173
 174 static int hfi1_ipoib_process_tx_ring(struct napi_struct *napi, int budget)
 175 {
 176         struct hfi1_ipoib_dev_priv *priv = hfi1_ipoib_priv(napi->dev);
 177         struct hfi1_ipoib_txq *txq = &priv->txqs[napi - priv->tx_napis];
 178
 179         int work_done = hfi1_ipoib_drain_tx_ring(txq, budget);
 180
 181         if (work_done < budget)
 182                 napi_complete_done(napi, work_done);
 183
 184         return work_done;
 185 }
 186
 187 static void hfi1_ipoib_add_tx(struct ipoib_txreq *tx)
 188 {
 189         struct hfi1_ipoib_circ_buf *tx_ring = &tx->txq->tx_ring;
 190         unsigned long head;
 191         unsigned long tail;
 192         size_t max_tx;
 193
 194         spin_lock(&tx_ring->producer_lock);
 195
 196         head = tx_ring->head;
 197         tail = READ_ONCE(tx_ring->tail);
 198         max_tx = tx_ring->max_items;
 199
 200         if (likely(CIRC_SPACE(head, tail, max_tx))) {
 201                 tx_ring->items[head] = tx;
 202
 203                 /* Finish storing txreq before incrementing head. */
 204                 smp_store_release(&tx_ring->head, CIRC_ADD(head, 1, max_tx));
 205                 napi_schedule(tx->txq->napi);
 206         } else {
 207                 struct hfi1_ipoib_txq *txq = tx->txq;
 208                 struct hfi1_ipoib_dev_priv *priv = tx->priv;
 209
 210                 /* Ring was full */
 211                 hfi1_ipoib_free_tx(tx, 0);
 212                 atomic64_inc(&txq->complete_txreqs);
 213                 dd_dev_dbg(priv->dd, "txq %d full.\n", txq->q_idx);
 214         }
 215
 216         spin_unlock(&tx_ring->producer_lock);
 217 }
 218
 219 static void hfi1_ipoib_sdma_complete(struct sdma_txreq *txreq, int status)
 220 {
 221         struct ipoib_txreq *tx = container_of(txreq, struct ipoib_txreq, txreq);
 222
 223         tx->sdma_status = status;
 224
 225         hfi1_ipoib_add_tx(tx);
 226 }
 227
 228 static int hfi1_ipoib_build_ulp_payload(struct ipoib_txreq *tx,
 229                                         struct ipoib_txparms *txp)
 230 {
 231         struct hfi1_devdata *dd = txp->dd;
 232         struct sdma_txreq *txreq = &tx->txreq;
 233         struct sk_buff *skb = tx->skb;
 234         int ret = 0;
 235         int i;
 236
 237         if (skb_headlen(skb)) {
 238                 ret = sdma_txadd_kvaddr(dd, txreq, skb->data, skb_headlen(skb));
 239                 if (unlikely(ret))
 240                         return ret;
 241         }
 242
 243         for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
 244                 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
 245
 246                 ret = sdma_txadd_page(dd,
 247                                       txreq,
 248                                       skb_frag_page(frag),
 249                                       frag->bv_offset,
 250                                       skb_frag_size(frag));
 251                 if (unlikely(ret))
 252                         break;
 253         }
 254
 255         return ret;
 256 }
 257
 258 static int hfi1_ipoib_build_tx_desc(struct ipoib_txreq *tx,
 259                                     struct ipoib_txparms *txp)
 260 {
 261         struct hfi1_devdata *dd = txp->dd;
 262         struct sdma_txreq *txreq = &tx->txreq;
 263         struct hfi1_sdma_header *sdma_hdr = &tx->sdma_hdr;
 264         u16 pkt_bytes =
 265                 sizeof(sdma_hdr->pbc) + (txp->hdr_dwords << 2) + tx->skb->len;
 266         int ret;
 267
 268         ret = sdma_txinit(txreq, 0, pkt_bytes, hfi1_ipoib_sdma_complete);
 269         if (unlikely(ret))
 270                 return ret;
 271
 272         /* add pbc + headers */
 273         ret = sdma_txadd_kvaddr(dd,
 274                                 txreq,
 275                                 sdma_hdr,
 276                                 sizeof(sdma_hdr->pbc) + (txp->hdr_dwords << 2));
 277         if (unlikely(ret))
 278                 return ret;
 279
 280         /* add the ulp payload */
 281         return hfi1_ipoib_build_ulp_payload(tx, txp);
 282 }
 283
 284 static void hfi1_ipoib_build_ib_tx_headers(struct ipoib_txreq *tx,
 285                                            struct ipoib_txparms *txp)
 286 {
 287         struct hfi1_ipoib_dev_priv *priv = tx->priv;
 288         struct hfi1_sdma_header *sdma_hdr = &tx->sdma_hdr;
 289         struct sk_buff *skb = tx->skb;
 290         struct hfi1_pportdata *ppd = ppd_from_ibp(txp->ibp);
 291         struct rdma_ah_attr *ah_attr = txp->ah_attr;
 292         struct ib_other_headers *ohdr;
 293         struct ib_grh *grh;
 294         u16 dwords;
 295         u16 slid;
 296         u16 dlid;
 297         u16 lrh0;
 298         u32 bth0;
 299         u32 sqpn = (u32)(priv->netdev->dev_addr[1] << 16 |
 300                          priv->netdev->dev_addr[2] << 8 |
 301                          priv->netdev->dev_addr[3]);
 302         u16 payload_dwords;
 303         u8 pad_cnt;
 304
 305         pad_cnt = -skb->len & 3;
 306
 307         /* Includes ICRC */
 308         payload_dwords = ((skb->len + pad_cnt) >> 2) + SIZE_OF_CRC;
 309
 310         /* header size in dwords LRH+BTH+DETH = (8+12+8)/4. */
 311         txp->hdr_dwords = 7;
 312
 313         if (rdma_ah_get_ah_flags(ah_attr) & IB_AH_GRH) {
 314                 grh = &sdma_hdr->hdr.ibh.u.l.grh;
 315                 txp->hdr_dwords +=
 316                         hfi1_make_grh(txp->ibp,
 317                                       grh,
 318                                       rdma_ah_read_grh(ah_attr),
 319                                       txp->hdr_dwords - LRH_9B_DWORDS,
 320                                       payload_dwords);
 321                 lrh0 = HFI1_LRH_GRH;
 322                 ohdr = &sdma_hdr->hdr.ibh.u.l.oth;
 323         } else {
 324                 lrh0 = HFI1_LRH_BTH;
 325                 ohdr = &sdma_hdr->hdr.ibh.u.oth;
 326         }
 327
 328         lrh0 |= (rdma_ah_get_sl(ah_attr) & 0xf) << 4;
 329         lrh0 |= (txp->flow.sc5 & 0xf) << 12;
 330
 331         dlid = opa_get_lid(rdma_ah_get_dlid(ah_attr), 9B);
 332         if (dlid == be16_to_cpu(IB_LID_PERMISSIVE)) {
 333                 slid = be16_to_cpu(IB_LID_PERMISSIVE);
 334         } else {
 335                 u16 lid = (u16)ppd->lid;
 336
 337                 if (lid) {
 338                         lid |= rdma_ah_get_path_bits(ah_attr) &
 339                                 ((1 << ppd->lmc) - 1);
 340                         slid = lid;
 341                 } else {
 342                         slid = be16_to_cpu(IB_LID_PERMISSIVE);
 343                 }
 344         }
 345
 346         /* Includes ICRC */
 347         dwords = txp->hdr_dwords + payload_dwords;
 348
 349         /* Build the lrh */
 350         sdma_hdr->hdr.hdr_type = HFI1_PKT_TYPE_9B;
 351         hfi1_make_ib_hdr(&sdma_hdr->hdr.ibh, lrh0, dwords, dlid, slid);
 352
 353         /* Build the bth */
 354         bth0 = (IB_OPCODE_UD_SEND_ONLY << 24) | (pad_cnt << 20) | priv->pkey;
 355
 356         ohdr->bth[0] = cpu_to_be32(bth0);
 357         ohdr->bth[1] = cpu_to_be32(txp->dqpn);
 358         ohdr->bth[2] = cpu_to_be32(mask_psn((u32)txp->txq->sent_txreqs));
 359
 360         /* Build the deth */
 361         ohdr->u.ud.deth[0] = cpu_to_be32(priv->qkey);
 362         ohdr->u.ud.deth[1] = cpu_to_be32((txp->entropy <<
 363                                           HFI1_IPOIB_ENTROPY_SHIFT) | sqpn);
 364
 365         /* Construct the pbc. */
 366         sdma_hdr->pbc =
 367                 cpu_to_le64(create_pbc(ppd,
 368                                        ib_is_sc5(txp->flow.sc5) <<
 369                                                               PBC_DC_INFO_SHIFT,
 370                                        0,
 371                                        sc_to_vlt(priv->dd, txp->flow.sc5),
 372                                        dwords - SIZE_OF_CRC +
 373                                                 (sizeof(sdma_hdr->pbc) >> 2)));
 374 }
 375
 376 static struct ipoib_txreq *hfi1_ipoib_send_dma_common(struct net_device *dev,
 377                                                       struct sk_buff *skb,
 378                                                       struct ipoib_txparms *txp)
 379 {
 380         struct hfi1_ipoib_dev_priv *priv = hfi1_ipoib_priv(dev);
 381         struct ipoib_txreq *tx;
 382         int ret;
 383
 384         tx = kmem_cache_alloc_node(priv->txreq_cache,
 385                                    GFP_ATOMIC,
 386                                    priv->dd->node);
 387         if (unlikely(!tx))
 388                 return ERR_PTR(-ENOMEM);
 389
 390         /* so that we can test if the sdma descriptors are there */
 391         tx->txreq.num_desc = 0;
 392         tx->priv = priv;
 393         tx->txq = txp->txq;
 394         tx->skb = skb;
 395         INIT_LIST_HEAD(&tx->txreq.list);
 396
 397         hfi1_ipoib_build_ib_tx_headers(tx, txp);
 398
 399         ret = hfi1_ipoib_build_tx_desc(tx, txp);
 400         if (likely(!ret)) {
 401                 if (txp->txq->flow.as_int != txp->flow.as_int) {
 402                         txp->txq->flow.tx_queue = txp->flow.tx_queue;
 403                         txp->txq->flow.sc5 = txp->flow.sc5;
 404                         txp->txq->sde =
 405                                 sdma_select_engine_sc(priv->dd,
 406                                                       txp->flow.tx_queue,
 407                                                       txp->flow.sc5);
 408                 }
 409
 410                 return tx;
 411         }
 412
 413         sdma_txclean(priv->dd, &tx->txreq);
 414         kmem_cache_free(priv->txreq_cache, tx);
 415
 416         return ERR_PTR(ret);
 417 }
 418
 419 static int hfi1_ipoib_submit_tx_list(struct net_device *dev,
 420                                      struct hfi1_ipoib_txq *txq)
 421 {
 422         int ret;
 423         u16 count_out;
 424
 425         ret = sdma_send_txlist(txq->sde,
 426                                iowait_get_ib_work(&txq->wait),
 427                                &txq->tx_list,
 428                                &count_out);
 429         if (likely(!ret) || ret == -EBUSY || ret == -ECOMM)
 430                 return ret;
 431
 432         dd_dev_warn(txq->priv->dd, "cannot send skb tx list, err %d.\n", ret);
 433
 434         return ret;
 435 }
 436
 437 static int hfi1_ipoib_flush_tx_list(struct net_device *dev,
 438                                     struct hfi1_ipoib_txq *txq)
 439 {
 440         int ret = 0;
 441
 442         if (!list_empty(&txq->tx_list)) {
 443                 /* Flush the current list */
 444                 ret = hfi1_ipoib_submit_tx_list(dev, txq);
 445
 446                 if (unlikely(ret))
 447                         if (ret != -EBUSY)
 448                                 ++dev->stats.tx_carrier_errors;
 449         }
 450
 451         return ret;
 452 }
 453
 454 static int hfi1_ipoib_submit_tx(struct hfi1_ipoib_txq *txq,
 455                                 struct ipoib_txreq *tx)
 456 {
 457         int ret;
 458
 459         ret = sdma_send_txreq(txq->sde,
 460                               iowait_get_ib_work(&txq->wait),
 461                               &tx->txreq,
 462                               txq->pkts_sent);
 463         if (likely(!ret)) {
 464                 txq->pkts_sent = true;
 465                 iowait_starve_clear(txq->pkts_sent, &txq->wait);
 466         }
 467
 468         return ret;
 469 }
 470
 471 static int hfi1_ipoib_send_dma_single(struct net_device *dev,
 472                                       struct sk_buff *skb,
 473                                       struct ipoib_txparms *txp)
 474 {
 475         struct hfi1_ipoib_dev_priv *priv = hfi1_ipoib_priv(dev);
 476         struct hfi1_ipoib_txq *txq = txp->txq;
 477         struct ipoib_txreq *tx;
 478         int ret;
 479
 480         tx = hfi1_ipoib_send_dma_common(dev, skb, txp);
 481         if (IS_ERR(tx)) {
 482                 int ret = PTR_ERR(tx);
 483
 484                 dev_kfree_skb_any(skb);
 485
 486                 if (ret == -ENOMEM)
 487                         ++dev->stats.tx_errors;
 488                 else
 489                         ++dev->stats.tx_carrier_errors;
 490
 491                 return NETDEV_TX_OK;
 492         }
 493
 494         ret = hfi1_ipoib_submit_tx(txq, tx);
 495         if (likely(!ret)) {
 496 tx_ok:
 497                 trace_sdma_output_ibhdr(tx->priv->dd,
 498                                         &tx->sdma_hdr.hdr,
 499                                         ib_is_sc5(txp->flow.sc5));
 500                 hfi1_ipoib_check_queue_depth(txq);
 501                 return NETDEV_TX_OK;
 502         }
 503
 504         txq->pkts_sent = false;
 505
 506         if (ret == -EBUSY || ret == -ECOMM)
 507                 goto tx_ok;
 508
 509         sdma_txclean(priv->dd, &tx->txreq);
 510         dev_kfree_skb_any(skb);
 511         kmem_cache_free(priv->txreq_cache, tx);
 512         ++dev->stats.tx_carrier_errors;
 513
 514         return NETDEV_TX_OK;
 515 }
 516
 517 static int hfi1_ipoib_send_dma_list(struct net_device *dev,
 518                                     struct sk_buff *skb,
 519                                     struct ipoib_txparms *txp)
 520 {
 521         struct hfi1_ipoib_txq *txq = txp->txq;
 522         struct ipoib_txreq *tx;
 523
 524         /* Has the flow change ? */
 525         if (txq->flow.as_int != txp->flow.as_int) {
 526                 int ret;
 527
 528                 ret = hfi1_ipoib_flush_tx_list(dev, txq);
 529                 if (unlikely(ret)) {
 530                         if (ret == -EBUSY)
 531                                 ++dev->stats.tx_dropped;
 532                         dev_kfree_skb_any(skb);
 533                         return NETDEV_TX_OK;
 534                 }
 535         }
 536         tx = hfi1_ipoib_send_dma_common(dev, skb, txp);
 537         if (IS_ERR(tx)) {
 538                 int ret = PTR_ERR(tx);
 539
 540                 dev_kfree_skb_any(skb);
 541
 542                 if (ret == -ENOMEM)
 543                         ++dev->stats.tx_errors;
 544                 else
 545                         ++dev->stats.tx_carrier_errors;
 546
 547                 return NETDEV_TX_OK;
 548         }
 549
 550         list_add_tail(&tx->txreq.list, &txq->tx_list);
 551
 552         hfi1_ipoib_check_queue_depth(txq);
 553
 554         trace_sdma_output_ibhdr(tx->priv->dd,
 555                                 &tx->sdma_hdr.hdr,
 556                                 ib_is_sc5(txp->flow.sc5));
 557
 558         if (!netdev_xmit_more())
 559                 (void)hfi1_ipoib_flush_tx_list(dev, txq);
 560
 561         return NETDEV_TX_OK;
 562 }
 563
 564 static u8 hfi1_ipoib_calc_entropy(struct sk_buff *skb)
 565 {
 566         if (skb_transport_header_was_set(skb)) {
 567                 u8 *hdr = (u8 *)skb_transport_header(skb);
 568
 569                 return (hdr[0] ^ hdr[1] ^ hdr[2] ^ hdr[3]);
 570         }
 571
 572         return (u8)skb_get_queue_mapping(skb);
 573 }
 574
 575 int hfi1_ipoib_send_dma(struct net_device *dev,
 576                         struct sk_buff *skb,
 577                         struct ib_ah *address,
 578                         u32 dqpn)
 579 {
 580         struct hfi1_ipoib_dev_priv *priv = hfi1_ipoib_priv(dev);
 581         struct ipoib_txparms txp;
 582         struct rdma_netdev *rn = netdev_priv(dev);
 583
 584         if (unlikely(skb->len > rn->mtu + HFI1_IPOIB_ENCAP_LEN)) {
 585                 dd_dev_warn(priv->dd, "packet len %d (> %d) too long to send, dropping\n",
 586                             skb->len,
 587                             rn->mtu + HFI1_IPOIB_ENCAP_LEN);
 588                 ++dev->stats.tx_dropped;
 589                 ++dev->stats.tx_errors;
 590                 dev_kfree_skb_any(skb);
 591                 return NETDEV_TX_OK;
 592         }
 593
 594         txp.dd = priv->dd;
 595         txp.ah_attr = &ibah_to_rvtah(address)->attr;
 596         txp.ibp = to_iport(priv->device, priv->port_num);
 597         txp.txq = &priv->txqs[skb_get_queue_mapping(skb)];
 598         txp.dqpn = dqpn;
 599         txp.flow.sc5 = txp.ibp->sl_to_sc[rdma_ah_get_sl(txp.ah_attr)];
 600         txp.flow.tx_queue = (u8)skb_get_queue_mapping(skb);
 601         txp.entropy = hfi1_ipoib_calc_entropy(skb);
 602
 603         if (netdev_xmit_more() || !list_empty(&txp.txq->tx_list))
 604                 return hfi1_ipoib_send_dma_list(dev, skb, &txp);
 605
 606         return hfi1_ipoib_send_dma_single(dev, skb,  &txp);
 607 }
 608
 609 /*
 610  * hfi1_ipoib_sdma_sleep - ipoib sdma sleep function
 611  *
 612  * This function gets called from sdma_send_txreq() when there are not enough
 613  * sdma descriptors available to send the packet. It adds Tx queue's wait
 614  * structure to sdma engine's dmawait list to be woken up when descriptors
 615  * become available.
 616  */
 617 static int hfi1_ipoib_sdma_sleep(struct sdma_engine *sde,
 618                                  struct iowait_work *wait,
 619                                  struct sdma_txreq *txreq,
 620                                  uint seq,
 621                                  bool pkts_sent)
 622 {
 623         struct hfi1_ipoib_txq *txq =
 624                 container_of(wait->iow, struct hfi1_ipoib_txq, wait);
 625
 626         write_seqlock(&sde->waitlock);
 627
 628         if (likely(txq->priv->netdev->reg_state == NETREG_REGISTERED)) {
 629                 if (sdma_progress(sde, seq, txreq)) {
 630                         write_sequnlock(&sde->waitlock);
 631                         return -EAGAIN;
 632                 }
 633
 634                 if (list_empty(&txreq->list))
 635                         /* came from non-list submit */
 636                         list_add_tail(&txreq->list, &txq->tx_list);
 637                 if (list_empty(&txq->wait.list)) {
 638                         if (!atomic_xchg(&txq->no_desc, 1))
 639                                 hfi1_ipoib_stop_txq(txq);
 640                         iowait_queue(pkts_sent, wait->iow, &sde->dmawait);
 641                 }
 642
 643                 write_sequnlock(&sde->waitlock);
 644                 return -EBUSY;
 645         }
 646
 647         write_sequnlock(&sde->waitlock);
 648         return -EINVAL;
 649 }
 650
 651 /*
 652  * hfi1_ipoib_sdma_wakeup - ipoib sdma wakeup function
 653  *
 654  * This function gets called when SDMA descriptors becomes available and Tx
 655  * queue's wait structure was previously added to sdma engine's dmawait list.
 656  */
 657 static void hfi1_ipoib_sdma_wakeup(struct iowait *wait, int reason)
 658 {
 659         struct hfi1_ipoib_txq *txq =
 660                 container_of(wait, struct hfi1_ipoib_txq, wait);
 661
 662         if (likely(txq->priv->netdev->reg_state == NETREG_REGISTERED))
 663                 iowait_schedule(wait, system_highpri_wq, WORK_CPU_UNBOUND);
 664 }
 665
 666 static void hfi1_ipoib_flush_txq(struct work_struct *work)
 667 {
 668         struct iowait_work *ioww =
 669                 container_of(work, struct iowait_work, iowork);
 670         struct iowait *wait = iowait_ioww_to_iow(ioww);
 671         struct hfi1_ipoib_txq *txq =
 672                 container_of(wait, struct hfi1_ipoib_txq, wait);
 673         struct net_device *dev = txq->priv->netdev;
 674
 675         if (likely(dev->reg_state == NETREG_REGISTERED) &&
 676             likely(!hfi1_ipoib_flush_tx_list(dev, txq)))
 677                 if (atomic_xchg(&txq->no_desc, 0))
 678                         hfi1_ipoib_wake_txq(txq);
 679 }
 680
 681 int hfi1_ipoib_txreq_init(struct hfi1_ipoib_dev_priv *priv)
 682 {
 683         struct net_device *dev = priv->netdev;
 684         char buf[HFI1_IPOIB_TXREQ_NAME_LEN];
 685         unsigned long tx_ring_size;
 686         int i;
 687
 688         /*
 689          * Ring holds 1 less than tx_ring_size
 690          * Round up to next power of 2 in order to hold at least tx_queue_len
 691          */
 692         tx_ring_size = roundup_pow_of_two((unsigned long)dev->tx_queue_len + 1);
 693
 694         snprintf(buf, sizeof(buf), "hfi1_%u_ipoib_txreq_cache", priv->dd->unit);
 695         priv->txreq_cache = kmem_cache_create(buf,
 696                                               sizeof(struct ipoib_txreq),
 697                                               0,
 698                                               0,
 699                                               NULL);
 700         if (!priv->txreq_cache)
 701                 return -ENOMEM;
 702
 703         priv->tx_napis = kcalloc_node(dev->num_tx_queues,
 704                                       sizeof(struct napi_struct),
 705                                       GFP_ATOMIC,
 706                                       priv->dd->node);
 707         if (!priv->tx_napis)
 708                 goto free_txreq_cache;
 709
 710         priv->txqs = kcalloc_node(dev->num_tx_queues,
 711                                   sizeof(struct hfi1_ipoib_txq),
 712                                   GFP_ATOMIC,
 713                                   priv->dd->node);
 714         if (!priv->txqs)
 715                 goto free_tx_napis;
 716
 717         for (i = 0; i < dev->num_tx_queues; i++) {
 718                 struct hfi1_ipoib_txq *txq = &priv->txqs[i];
 719
 720                 iowait_init(&txq->wait,
 721                             0,
 722                             hfi1_ipoib_flush_txq,
 723                             NULL,
 724                             hfi1_ipoib_sdma_sleep,
 725                             hfi1_ipoib_sdma_wakeup,
 726                             NULL,
 727                             NULL);
 728                 txq->priv = priv;
 729                 txq->sde = NULL;
 730                 INIT_LIST_HEAD(&txq->tx_list);
 731                 atomic64_set(&txq->complete_txreqs, 0);
 732                 atomic_set(&txq->stops, 0);
 733                 atomic_set(&txq->ring_full, 0);
 734                 atomic_set(&txq->no_desc, 0);
 735                 txq->q_idx = i;
 736                 txq->flow.tx_queue = 0xff;
 737                 txq->flow.sc5 = 0xff;
 738                 txq->pkts_sent = false;
 739
 740                 netdev_queue_numa_node_write(netdev_get_tx_queue(dev, i),
 741                                              priv->dd->node);
 742
 743                 txq->tx_ring.items =
 744                         vzalloc_node(array_size(tx_ring_size,
 745                                                 sizeof(struct ipoib_txreq)),
 746                                      priv->dd->node);
 747                 if (!txq->tx_ring.items)
 748                         goto free_txqs;
 749
 750                 spin_lock_init(&txq->tx_ring.producer_lock);
 751                 spin_lock_init(&txq->tx_ring.consumer_lock);
 752                 txq->tx_ring.max_items = tx_ring_size;
 753
 754                 txq->napi = &priv->tx_napis[i];
 755                 netif_tx_napi_add(dev, txq->napi,
 756                                   hfi1_ipoib_process_tx_ring,
 757                                   NAPI_POLL_WEIGHT);
 758         }
 759
 760         return 0;
 761
 762 free_txqs:
 763         for (i--; i >= 0; i--) {
 764                 struct hfi1_ipoib_txq *txq = &priv->txqs[i];
 765
 766                 netif_napi_del(txq->napi);
 767                 vfree(txq->tx_ring.items);
 768         }
 769
 770         kfree(priv->txqs);
 771         priv->txqs = NULL;
 772
 773 free_tx_napis:
 774         kfree(priv->tx_napis);
 775         priv->tx_napis = NULL;
 776
 777 free_txreq_cache:
 778         kmem_cache_destroy(priv->txreq_cache);
 779         priv->txreq_cache = NULL;
 780         return -ENOMEM;
 781 }
 782
 783 static void hfi1_ipoib_drain_tx_list(struct hfi1_ipoib_txq *txq)
 784 {
 785         struct sdma_txreq *txreq;
 786         struct sdma_txreq *txreq_tmp;
 787         atomic64_t *complete_txreqs = &txq->complete_txreqs;
 788
 789         list_for_each_entry_safe(txreq, txreq_tmp, &txq->tx_list, list) {
 790                 struct ipoib_txreq *tx =
 791                         container_of(txreq, struct ipoib_txreq, txreq);
 792
 793                 list_del(&txreq->list);
 794                 sdma_txclean(txq->priv->dd, &tx->txreq);
 795                 dev_kfree_skb_any(tx->skb);
 796                 kmem_cache_free(txq->priv->txreq_cache, tx);
 797                 atomic64_inc(complete_txreqs);
 798         }
 799
 800         if (hfi1_ipoib_used(txq))
 801                 dd_dev_warn(txq->priv->dd,
 802                             "txq %d not empty found %llu requests\n",
 803                             txq->q_idx,
 804                             hfi1_ipoib_txreqs(txq->sent_txreqs,
 805                                               atomic64_read(complete_txreqs)));
 806 }
 807
 808 void hfi1_ipoib_txreq_deinit(struct hfi1_ipoib_dev_priv *priv)
 809 {
 810         int i;
 811
 812         for (i = 0; i < priv->netdev->num_tx_queues; i++) {
 813                 struct hfi1_ipoib_txq *txq = &priv->txqs[i];
 814
 815                 iowait_cancel_work(&txq->wait);
 816                 iowait_sdma_drain(&txq->wait);
 817                 hfi1_ipoib_drain_tx_list(txq);
 818                 netif_napi_del(txq->napi);
 819                 (void)hfi1_ipoib_drain_tx_ring(txq, txq->tx_ring.max_items);
 820                 vfree(txq->tx_ring.items);
 821         }
 822
 823         kfree(priv->txqs);
 824         priv->txqs = NULL;
 825
 826         kfree(priv->tx_napis);
 827         priv->tx_napis = NULL;
 828
 829         kmem_cache_destroy(priv->txreq_cache);
 830         priv->txreq_cache = NULL;
 831 }
 832
 833 void hfi1_ipoib_napi_tx_enable(struct net_device *dev)
 834 {
 835         struct hfi1_ipoib_dev_priv *priv = hfi1_ipoib_priv(dev);
 836         int i;
 837
 838         for (i = 0; i < dev->num_tx_queues; i++) {
 839                 struct hfi1_ipoib_txq *txq = &priv->txqs[i];
 840
 841                 napi_enable(txq->napi);
 842         }
 843 }
 844
 845 void hfi1_ipoib_napi_tx_disable(struct net_device *dev)
 846 {
 847         struct hfi1_ipoib_dev_priv *priv = hfi1_ipoib_priv(dev);
 848         int i;
 849
 850         for (i = 0; i < dev->num_tx_queues; i++) {
 851                 struct hfi1_ipoib_txq *txq = &priv->txqs[i];
 852
 853                 napi_disable(txq->napi);
 854                 (void)hfi1_ipoib_drain_tx_ring(txq, txq->tx_ring.max_items);
 855         }
 856 }