drivers/net/wireless/mediatek/mt76/dma.c

   1 /*
   2  * Copyright (C) 2016 Felix Fietkau <nbd@nbd.name>
   3  *
   4  * Permission to use, copy, modify, and/or distribute this software for any
   5  * purpose with or without fee is hereby granted, provided that the above
   6  * copyright notice and this permission notice appear in all copies.
   7  *
   8  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
   9  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
  10  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
  11  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
  12  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
  13  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
  14  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
  15  */
  16
  17 #include <linux/dma-mapping.h>
  18 #include "mt76.h"
  19 #include "dma.h"
  20
  21 #define DMA_DUMMY_TXWI  ((void *) ~0)
  22
  23 static int
  24 mt76_dma_alloc_queue(struct mt76_dev *dev, struct mt76_queue *q)
  25 {
  26         int size;
  27         int i;
  28
  29         spin_lock_init(&q->lock);
  30         INIT_LIST_HEAD(&q->swq);
  31
  32         size = q->ndesc * sizeof(struct mt76_desc);
  33         q->desc = dmam_alloc_coherent(dev->dev, size, &q->desc_dma, GFP_KERNEL);
  34         if (!q->desc)
  35                 return -ENOMEM;
  36
  37         size = q->ndesc * sizeof(*q->entry);
  38         q->entry = devm_kzalloc(dev->dev, size, GFP_KERNEL);
  39         if (!q->entry)
  40                 return -ENOMEM;
  41
  42         /* clear descriptors */
  43         for (i = 0; i < q->ndesc; i++)
  44                 q->desc[i].ctrl = cpu_to_le32(MT_DMA_CTL_DMA_DONE);
  45
  46         iowrite32(q->desc_dma, &q->regs->desc_base);
  47         iowrite32(0, &q->regs->cpu_idx);
  48         iowrite32(0, &q->regs->dma_idx);
  49         iowrite32(q->ndesc, &q->regs->ring_size);
  50
  51         return 0;
  52 }
  53
  54 static int
  55 mt76_dma_add_buf(struct mt76_dev *dev, struct mt76_queue *q,
  56                  struct mt76_queue_buf *buf, int nbufs, u32 info,
  57                  struct sk_buff *skb, void *txwi)
  58 {
  59         struct mt76_desc *desc;
  60         u32 ctrl;
  61         int i, idx = -1;
  62
  63         if (txwi)
  64                 q->entry[q->head].txwi = DMA_DUMMY_TXWI;
  65
  66         for (i = 0; i < nbufs; i += 2, buf += 2) {
  67                 u32 buf0 = buf[0].addr, buf1 = 0;
  68
  69                 ctrl = FIELD_PREP(MT_DMA_CTL_SD_LEN0, buf[0].len);
  70                 if (i < nbufs - 1) {
  71                         buf1 = buf[1].addr;
  72                         ctrl |= FIELD_PREP(MT_DMA_CTL_SD_LEN1, buf[1].len);
  73                 }
  74
  75                 if (i == nbufs - 1)
  76                         ctrl |= MT_DMA_CTL_LAST_SEC0;
  77                 else if (i == nbufs - 2)
  78                         ctrl |= MT_DMA_CTL_LAST_SEC1;
  79
  80                 idx = q->head;
  81                 q->head = (q->head + 1) % q->ndesc;
  82
  83                 desc = &q->desc[idx];
  84
  85                 WRITE_ONCE(desc->buf0, cpu_to_le32(buf0));
  86                 WRITE_ONCE(desc->buf1, cpu_to_le32(buf1));
  87                 WRITE_ONCE(desc->info, cpu_to_le32(info));
  88                 WRITE_ONCE(desc->ctrl, cpu_to_le32(ctrl));
  89
  90                 q->queued++;
  91         }
  92
  93         q->entry[idx].txwi = txwi;
  94         q->entry[idx].skb = skb;
  95
  96         return idx;
  97 }
  98
  99 static void
 100 mt76_dma_tx_cleanup_idx(struct mt76_dev *dev, struct mt76_queue *q, int idx,
 101                         struct mt76_queue_entry *prev_e)
 102 {
 103         struct mt76_queue_entry *e = &q->entry[idx];
 104         __le32 __ctrl = READ_ONCE(q->desc[idx].ctrl);
 105         u32 ctrl = le32_to_cpu(__ctrl);
 106
 107         if (!e->txwi || !e->skb) {
 108                 __le32 addr = READ_ONCE(q->desc[idx].buf0);
 109                 u32 len = FIELD_GET(MT_DMA_CTL_SD_LEN0, ctrl);
 110
 111                 dma_unmap_single(dev->dev, le32_to_cpu(addr), len,
 112                                  DMA_TO_DEVICE);
 113         }
 114
 115         if (!(ctrl & MT_DMA_CTL_LAST_SEC0)) {
 116                 __le32 addr = READ_ONCE(q->desc[idx].buf1);
 117                 u32 len = FIELD_GET(MT_DMA_CTL_SD_LEN1, ctrl);
 118
 119                 dma_unmap_single(dev->dev, le32_to_cpu(addr), len,
 120                                  DMA_TO_DEVICE);
 121         }
 122
 123         if (e->txwi == DMA_DUMMY_TXWI)
 124                 e->txwi = NULL;
 125
 126         *prev_e = *e;
 127         memset(e, 0, sizeof(*e));
 128 }
 129
 130 static void
 131 mt76_dma_sync_idx(struct mt76_dev *dev, struct mt76_queue *q)
 132 {
 133         iowrite32(q->desc_dma, &q->regs->desc_base);
 134         iowrite32(q->ndesc, &q->regs->ring_size);
 135         q->head = ioread32(&q->regs->dma_idx);
 136         q->tail = q->head;
 137         iowrite32(q->head, &q->regs->cpu_idx);
 138 }
 139
 140 static void
 141 mt76_dma_tx_cleanup(struct mt76_dev *dev, enum mt76_txq_id qid, bool flush)
 142 {
 143         struct mt76_queue *q = &dev->q_tx[qid];
 144         struct mt76_queue_entry entry;
 145         bool wake = false;
 146         int last;
 147
 148         if (!q->ndesc)
 149                 return;
 150
 151         spin_lock_bh(&q->lock);
 152         if (flush)
 153                 last = -1;
 154         else
 155                 last = ioread32(&q->regs->dma_idx);
 156
 157         while (q->queued && q->tail != last) {
 158                 mt76_dma_tx_cleanup_idx(dev, q, q->tail, &entry);
 159                 if (entry.schedule)
 160                         q->swq_queued--;
 161
 162                 q->tail = (q->tail + 1) % q->ndesc;
 163                 q->queued--;
 164
 165                 if (entry.skb) {
 166                         spin_unlock_bh(&q->lock);
 167                         dev->drv->tx_complete_skb(dev, q, &entry, flush);
 168                         spin_lock_bh(&q->lock);
 169                 }
 170
 171                 if (entry.txwi) {
 172                         mt76_put_txwi(dev, entry.txwi);
 173                         wake = !flush;
 174                 }
 175
 176                 if (!flush && q->tail == last)
 177                         last = ioread32(&q->regs->dma_idx);
 178         }
 179
 180         if (!flush)
 181                 mt76_txq_schedule(dev, q);
 182         else
 183                 mt76_dma_sync_idx(dev, q);
 184
 185         wake = wake && q->stopped &&
 186                qid < IEEE80211_NUM_ACS && q->queued < q->ndesc - 8;
 187         if (wake)
 188                 q->stopped = false;
 189
 190         if (!q->queued)
 191                 wake_up(&dev->tx_wait);
 192
 193         spin_unlock_bh(&q->lock);
 194
 195         if (wake)
 196                 ieee80211_wake_queue(dev->hw, qid);
 197 }
 198
 199 static void *
 200 mt76_dma_get_buf(struct mt76_dev *dev, struct mt76_queue *q, int idx,
 201                  int *len, u32 *info, bool *more)
 202 {
 203         struct mt76_queue_entry *e = &q->entry[idx];
 204         struct mt76_desc *desc = &q->desc[idx];
 205         dma_addr_t buf_addr;
 206         void *buf = e->buf;
 207         int buf_len = SKB_WITH_OVERHEAD(q->buf_size);
 208
 209         buf_addr = le32_to_cpu(READ_ONCE(desc->buf0));
 210         if (len) {
 211                 u32 ctl = le32_to_cpu(READ_ONCE(desc->ctrl));
 212                 *len = FIELD_GET(MT_DMA_CTL_SD_LEN0, ctl);
 213                 *more = !(ctl & MT_DMA_CTL_LAST_SEC0);
 214         }
 215
 216         if (info)
 217                 *info = le32_to_cpu(desc->info);
 218
 219         dma_unmap_single(dev->dev, buf_addr, buf_len, DMA_FROM_DEVICE);
 220         e->buf = NULL;
 221
 222         return buf;
 223 }
 224
 225 static void *
 226 mt76_dma_dequeue(struct mt76_dev *dev, struct mt76_queue *q, bool flush,
 227                  int *len, u32 *info, bool *more)
 228 {
 229         int idx = q->tail;
 230
 231         *more = false;
 232         if (!q->queued)
 233                 return NULL;
 234
 235         if (!flush && !(q->desc[idx].ctrl & cpu_to_le32(MT_DMA_CTL_DMA_DONE)))
 236                 return NULL;
 237
 238         q->tail = (q->tail + 1) % q->ndesc;
 239         q->queued--;
 240
 241         return mt76_dma_get_buf(dev, q, idx, len, info, more);
 242 }
 243
 244 static void
 245 mt76_dma_kick_queue(struct mt76_dev *dev, struct mt76_queue *q)
 246 {
 247         iowrite32(q->head, &q->regs->cpu_idx);
 248 }
 249
 250 static int
 251 mt76_dma_tx_queue_skb_raw(struct mt76_dev *dev, enum mt76_txq_id qid,
 252                           struct sk_buff *skb, u32 tx_info)
 253 {
 254         struct mt76_queue *q = &dev->q_tx[qid];
 255         struct mt76_queue_buf buf;
 256         dma_addr_t addr;
 257
 258         addr = dma_map_single(dev->dev, skb->data, skb->len,
 259                               DMA_TO_DEVICE);
 260         if (dma_mapping_error(dev->dev, addr))
 261                 return -ENOMEM;
 262
 263         buf.addr = addr;
 264         buf.len = skb->len;
 265
 266         spin_lock_bh(&q->lock);
 267         mt76_dma_add_buf(dev, q, &buf, 1, tx_info, skb, NULL);
 268         mt76_dma_kick_queue(dev, q);
 269         spin_unlock_bh(&q->lock);
 270
 271         return 0;
 272 }
 273
 274 int mt76_dma_tx_queue_skb(struct mt76_dev *dev, struct mt76_queue *q,
 275                           struct sk_buff *skb, struct mt76_wcid *wcid,
 276                           struct ieee80211_sta *sta)
 277 {
 278         struct mt76_queue_entry e;
 279         struct mt76_txwi_cache *t;
 280         struct mt76_queue_buf buf[32];
 281         struct sk_buff *iter;
 282         dma_addr_t addr;
 283         int len;
 284         u32 tx_info = 0;
 285         int n, ret;
 286
 287         t = mt76_get_txwi(dev);
 288         if (!t) {
 289                 ieee80211_free_txskb(dev->hw, skb);
 290                 return -ENOMEM;
 291         }
 292
 293         skb->prev = skb->next = NULL;
 294         dma_sync_single_for_cpu(dev->dev, t->dma_addr, sizeof(t->txwi),
 295                                 DMA_TO_DEVICE);
 296         ret = dev->drv->tx_prepare_skb(dev, &t->txwi, skb, q, wcid, sta,
 297                                        &tx_info);
 298         dma_sync_single_for_device(dev->dev, t->dma_addr, sizeof(t->txwi),
 299                                    DMA_TO_DEVICE);
 300         if (ret < 0)
 301                 goto free;
 302
 303         len = skb->len - skb->data_len;
 304         addr = dma_map_single(dev->dev, skb->data, len, DMA_TO_DEVICE);
 305         if (dma_mapping_error(dev->dev, addr)) {
 306                 ret = -ENOMEM;
 307                 goto free;
 308         }
 309
 310         n = 0;
 311         buf[n].addr = t->dma_addr;
 312         buf[n++].len = dev->drv->txwi_size;
 313         buf[n].addr = addr;
 314         buf[n++].len = len;
 315
 316         skb_walk_frags(skb, iter) {
 317                 if (n == ARRAY_SIZE(buf))
 318                         goto unmap;
 319
 320                 addr = dma_map_single(dev->dev, iter->data, iter->len,
 321                                       DMA_TO_DEVICE);
 322                 if (dma_mapping_error(dev->dev, addr))
 323                         goto unmap;
 324
 325                 buf[n].addr = addr;
 326                 buf[n++].len = iter->len;
 327         }
 328
 329         if (q->queued + (n + 1) / 2 >= q->ndesc - 1)
 330                 goto unmap;
 331
 332         return mt76_dma_add_buf(dev, q, buf, n, tx_info, skb, t);
 333
 334 unmap:
 335         ret = -ENOMEM;
 336         for (n--; n > 0; n--)
 337                 dma_unmap_single(dev->dev, buf[n].addr, buf[n].len,
 338                                  DMA_TO_DEVICE);
 339
 340 free:
 341         e.skb = skb;
 342         e.txwi = t;
 343         dev->drv->tx_complete_skb(dev, q, &e, true);
 344         mt76_put_txwi(dev, t);
 345         return ret;
 346 }
 347 EXPORT_SYMBOL_GPL(mt76_dma_tx_queue_skb);
 348
 349 static int
 350 mt76_dma_rx_fill(struct mt76_dev *dev, struct mt76_queue *q)
 351 {
 352         dma_addr_t addr;
 353         void *buf;
 354         int frames = 0;
 355         int len = SKB_WITH_OVERHEAD(q->buf_size);
 356         int offset = q->buf_offset;
 357         int idx;
 358
 359         spin_lock_bh(&q->lock);
 360
 361         while (q->queued < q->ndesc - 1) {
 362                 struct mt76_queue_buf qbuf;
 363
 364                 buf = page_frag_alloc(&q->rx_page, q->buf_size, GFP_ATOMIC);
 365                 if (!buf)
 366                         break;
 367
 368                 addr = dma_map_single(dev->dev, buf, len, DMA_FROM_DEVICE);
 369                 if (dma_mapping_error(dev->dev, addr)) {
 370                         skb_free_frag(buf);
 371                         break;
 372                 }
 373
 374                 qbuf.addr = addr + offset;
 375                 qbuf.len = len - offset;
 376                 idx = mt76_dma_add_buf(dev, q, &qbuf, 1, 0, buf, NULL);
 377                 frames++;
 378         }
 379
 380         if (frames)
 381                 mt76_dma_kick_queue(dev, q);
 382
 383         spin_unlock_bh(&q->lock);
 384
 385         return frames;
 386 }
 387
 388 static void
 389 mt76_dma_rx_cleanup(struct mt76_dev *dev, struct mt76_queue *q)
 390 {
 391         struct page *page;
 392         void *buf;
 393         bool more;
 394
 395         spin_lock_bh(&q->lock);
 396         do {
 397                 buf = mt76_dma_dequeue(dev, q, true, NULL, NULL, &more);
 398                 if (!buf)
 399                         break;
 400
 401                 skb_free_frag(buf);
 402         } while (1);
 403         spin_unlock_bh(&q->lock);
 404
 405         if (!q->rx_page.va)
 406                 return;
 407
 408         page = virt_to_page(q->rx_page.va);
 409         __page_frag_cache_drain(page, q->rx_page.pagecnt_bias);
 410         memset(&q->rx_page, 0, sizeof(q->rx_page));
 411 }
 412
 413 static void
 414 mt76_dma_rx_reset(struct mt76_dev *dev, enum mt76_rxq_id qid)
 415 {
 416         struct mt76_queue *q = &dev->q_rx[qid];
 417         int i;
 418
 419         for (i = 0; i < q->ndesc; i++)
 420                 q->desc[i].ctrl &= ~cpu_to_le32(MT_DMA_CTL_DMA_DONE);
 421
 422         mt76_dma_rx_cleanup(dev, q);
 423         mt76_dma_sync_idx(dev, q);
 424         mt76_dma_rx_fill(dev, q);
 425 }
 426
 427 static void
 428 mt76_add_fragment(struct mt76_dev *dev, struct mt76_queue *q, void *data,
 429                   int len, bool more)
 430 {
 431         struct page *page = virt_to_head_page(data);
 432         int offset = data - page_address(page);
 433         struct sk_buff *skb = q->rx_head;
 434
 435         offset += q->buf_offset;
 436         skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page, offset, len,
 437                         q->buf_size);
 438
 439         if (more)
 440                 return;
 441
 442         q->rx_head = NULL;
 443         dev->drv->rx_skb(dev, q - dev->q_rx, skb);
 444 }
 445
 446 static int
 447 mt76_dma_rx_process(struct mt76_dev *dev, struct mt76_queue *q, int budget)
 448 {
 449         int len, data_len, done = 0;
 450         struct sk_buff *skb;
 451         unsigned char *data;
 452         bool more;
 453
 454         while (done < budget) {
 455                 u32 info;
 456
 457                 data = mt76_dma_dequeue(dev, q, false, &len, &info, &more);
 458                 if (!data)
 459                         break;
 460
 461                 if (q->rx_head)
 462                         data_len = q->buf_size;
 463                 else
 464                         data_len = SKB_WITH_OVERHEAD(q->buf_size);
 465
 466                 if (data_len < len + q->buf_offset) {
 467                         dev_kfree_skb(q->rx_head);
 468                         q->rx_head = NULL;
 469
 470                         skb_free_frag(data);
 471                         continue;
 472                 }
 473
 474                 if (q->rx_head) {
 475                         mt76_add_fragment(dev, q, data, len, more);
 476                         continue;
 477                 }
 478
 479                 skb = build_skb(data, q->buf_size);
 480                 if (!skb) {
 481                         skb_free_frag(data);
 482                         continue;
 483                 }
 484                 skb_reserve(skb, q->buf_offset);
 485
 486                 if (q == &dev->q_rx[MT_RXQ_MCU]) {
 487                         u32 *rxfce = (u32 *) skb->cb;
 488                         *rxfce = info;
 489                 }
 490
 491                 __skb_put(skb, len);
 492                 done++;
 493
 494                 if (more) {
 495                         q->rx_head = skb;
 496                         continue;
 497                 }
 498
 499                 dev->drv->rx_skb(dev, q - dev->q_rx, skb);
 500         }
 501
 502         mt76_dma_rx_fill(dev, q);
 503         return done;
 504 }
 505
 506 static int
 507 mt76_dma_rx_poll(struct napi_struct *napi, int budget)
 508 {
 509         struct mt76_dev *dev;
 510         int qid, done = 0, cur;
 511
 512         dev = container_of(napi->dev, struct mt76_dev, napi_dev);
 513         qid = napi - dev->napi;
 514
 515         rcu_read_lock();
 516
 517         do {
 518                 cur = mt76_dma_rx_process(dev, &dev->q_rx[qid], budget - done);
 519                 mt76_rx_poll_complete(dev, qid, napi);
 520                 done += cur;
 521         } while (cur && done < budget);
 522
 523         rcu_read_unlock();
 524
 525         if (done < budget) {
 526                 napi_complete(napi);
 527                 dev->drv->rx_poll_complete(dev, qid);
 528         }
 529
 530         return done;
 531 }
 532
 533 static int
 534 mt76_dma_init(struct mt76_dev *dev)
 535 {
 536         int i;
 537
 538         init_dummy_netdev(&dev->napi_dev);
 539
 540         for (i = 0; i < ARRAY_SIZE(dev->q_rx); i++) {
 541                 netif_napi_add(&dev->napi_dev, &dev->napi[i], mt76_dma_rx_poll,
 542                                64);
 543                 mt76_dma_rx_fill(dev, &dev->q_rx[i]);
 544                 skb_queue_head_init(&dev->rx_skb[i]);
 545                 napi_enable(&dev->napi[i]);
 546         }
 547
 548         return 0;
 549 }
 550
 551 static const struct mt76_queue_ops mt76_dma_ops = {
 552         .init = mt76_dma_init,
 553         .alloc = mt76_dma_alloc_queue,
 554         .tx_queue_skb_raw = mt76_dma_tx_queue_skb_raw,
 555         .tx_queue_skb = mt76_dma_tx_queue_skb,
 556         .tx_cleanup = mt76_dma_tx_cleanup,
 557         .rx_reset = mt76_dma_rx_reset,
 558         .kick = mt76_dma_kick_queue,
 559 };
 560
 561 void mt76_dma_attach(struct mt76_dev *dev)
 562 {
 563         dev->queue_ops = &mt76_dma_ops;
 564 }
 565 EXPORT_SYMBOL_GPL(mt76_dma_attach);
 566
 567 void mt76_dma_cleanup(struct mt76_dev *dev)
 568 {
 569         int i;
 570
 571         for (i = 0; i < ARRAY_SIZE(dev->q_tx); i++)
 572                 mt76_dma_tx_cleanup(dev, i, true);
 573
 574         for (i = 0; i < ARRAY_SIZE(dev->q_rx); i++) {
 575                 netif_napi_del(&dev->napi[i]);
 576                 mt76_dma_rx_cleanup(dev, &dev->q_rx[i]);
 577         }
 578 }
 579 EXPORT_SYMBOL_GPL(mt76_dma_cleanup);