drivers/gpu/host1x/job.c

   1 /*
   2  * Tegra host1x Job
   3  *
   4  * Copyright (c) 2010-2015, NVIDIA Corporation.
   5  *
   6  * This program is free software; you can redistribute it and/or modify it
   7  * under the terms and conditions of the GNU General Public License,
   8  * version 2, as published by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope it will be useful, but WITHOUT
  11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  13  * more details.
  14  *
  15  * You should have received a copy of the GNU General Public License
  16  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  17  */
  18
  19 #include <linux/dma-mapping.h>
  20 #include <linux/err.h>
  21 #include <linux/host1x.h>
  22 #include <linux/kref.h>
  23 #include <linux/module.h>
  24 #include <linux/scatterlist.h>
  25 #include <linux/slab.h>
  26 #include <linux/vmalloc.h>
  27 #include <trace/events/host1x.h>
  28
  29 #include "channel.h"
  30 #include "dev.h"
  31 #include "job.h"
  32 #include "syncpt.h"
  33
  34 struct host1x_job *host1x_job_alloc(struct host1x_channel *ch,
  35                                     u32 num_cmdbufs, u32 num_relocs,
  36                                     u32 num_waitchks)
  37 {
  38         struct host1x_job *job = NULL;
  39         unsigned int num_unpins = num_cmdbufs + num_relocs;
  40         u64 total;
  41         void *mem;
  42
  43         /* Check that we're not going to overflow */
  44         total = sizeof(struct host1x_job) +
  45                 (u64)num_relocs * sizeof(struct host1x_reloc) +
  46                 (u64)num_unpins * sizeof(struct host1x_job_unpin_data) +
  47                 (u64)num_waitchks * sizeof(struct host1x_waitchk) +
  48                 (u64)num_cmdbufs * sizeof(struct host1x_job_gather) +
  49                 (u64)num_unpins * sizeof(dma_addr_t) +
  50                 (u64)num_unpins * sizeof(u32 *);
  51         if (total > ULONG_MAX)
  52                 return NULL;
  53
  54         mem = job = kzalloc(total, GFP_KERNEL);
  55         if (!job)
  56                 return NULL;
  57
  58         kref_init(&job->ref);
  59         job->channel = ch;
  60
  61         /* Redistribute memory to the structs  */
  62         mem += sizeof(struct host1x_job);
  63         job->relocarray = num_relocs ? mem : NULL;
  64         mem += num_relocs * sizeof(struct host1x_reloc);
  65         job->unpins = num_unpins ? mem : NULL;
  66         mem += num_unpins * sizeof(struct host1x_job_unpin_data);
  67         job->waitchk = num_waitchks ? mem : NULL;
  68         mem += num_waitchks * sizeof(struct host1x_waitchk);
  69         job->gathers = num_cmdbufs ? mem : NULL;
  70         mem += num_cmdbufs * sizeof(struct host1x_job_gather);
  71         job->addr_phys = num_unpins ? mem : NULL;
  72
  73         job->reloc_addr_phys = job->addr_phys;
  74         job->gather_addr_phys = &job->addr_phys[num_relocs];
  75
  76         return job;
  77 }
  78 EXPORT_SYMBOL(host1x_job_alloc);
  79
  80 struct host1x_job *host1x_job_get(struct host1x_job *job)
  81 {
  82         kref_get(&job->ref);
  83         return job;
  84 }
  85 EXPORT_SYMBOL(host1x_job_get);
  86
  87 static void job_free(struct kref *ref)
  88 {
  89         struct host1x_job *job = container_of(ref, struct host1x_job, ref);
  90
  91         kfree(job);
  92 }
  93
  94 void host1x_job_put(struct host1x_job *job)
  95 {
  96         kref_put(&job->ref, job_free);
  97 }
  98 EXPORT_SYMBOL(host1x_job_put);
  99
 100 void host1x_job_add_gather(struct host1x_job *job, struct host1x_bo *bo,
 101                            u32 words, u32 offset)
 102 {
 103         struct host1x_job_gather *cur_gather = &job->gathers[job->num_gathers];
 104
 105         cur_gather->words = words;
 106         cur_gather->bo = bo;
 107         cur_gather->offset = offset;
 108         job->num_gathers++;
 109 }
 110 EXPORT_SYMBOL(host1x_job_add_gather);
 111
 112 /*
 113  * NULL an already satisfied WAIT_SYNCPT host method, by patching its
 114  * args in the command stream. The method data is changed to reference
 115  * a reserved (never given out or incr) HOST1X_SYNCPT_RESERVED syncpt
 116  * with a matching threshold value of 0, so is guaranteed to be popped
 117  * by the host HW.
 118  */
 119 static void host1x_syncpt_patch_offset(struct host1x_syncpt *sp,
 120                                        struct host1x_bo *h, u32 offset)
 121 {
 122         void *patch_addr = NULL;
 123
 124         /* patch the wait */
 125         patch_addr = host1x_bo_kmap(h, offset >> PAGE_SHIFT);
 126         if (patch_addr) {
 127                 host1x_syncpt_patch_wait(sp,
 128                                          patch_addr + (offset & ~PAGE_MASK));
 129                 host1x_bo_kunmap(h, offset >> PAGE_SHIFT, patch_addr);
 130         } else
 131                 pr_err("Could not map cmdbuf for wait check\n");
 132 }
 133
 134 /*
 135  * Check driver supplied waitchk structs for syncpt thresholds
 136  * that have already been satisfied and NULL the comparison (to
 137  * avoid a wrap condition in the HW).
 138  */
 139 static int do_waitchks(struct host1x_job *job, struct host1x *host,
 140                        struct host1x_bo *patch)
 141 {
 142         int i;
 143
 144         /* compare syncpt vs wait threshold */
 145         for (i = 0; i < job->num_waitchk; i++) {
 146                 struct host1x_waitchk *wait = &job->waitchk[i];
 147                 struct host1x_syncpt *sp =
 148                         host1x_syncpt_get(host, wait->syncpt_id);
 149
 150                 /* validate syncpt id */
 151                 if (wait->syncpt_id > host1x_syncpt_nb_pts(host))
 152                         continue;
 153
 154                 /* skip all other gathers */
 155                 if (patch != wait->bo)
 156                         continue;
 157
 158                 trace_host1x_syncpt_wait_check(wait->bo, wait->offset,
 159                                                wait->syncpt_id, wait->thresh,
 160                                                host1x_syncpt_read_min(sp));
 161
 162                 if (host1x_syncpt_is_expired(sp, wait->thresh)) {
 163                         dev_dbg(host->dev,
 164                                 "drop WAIT id %u (%s) thresh 0x%x, min 0x%x\n",
 165                                 wait->syncpt_id, sp->name, wait->thresh,
 166                                 host1x_syncpt_read_min(sp));
 167
 168                         host1x_syncpt_patch_offset(sp, patch, wait->offset);
 169                 }
 170
 171                 wait->bo = NULL;
 172         }
 173
 174         return 0;
 175 }
 176
 177 static unsigned int pin_job(struct host1x *host, struct host1x_job *job)
 178 {
 179         unsigned int i;
 180         int err;
 181
 182         job->num_unpins = 0;
 183
 184         for (i = 0; i < job->num_relocs; i++) {
 185                 struct host1x_reloc *reloc = &job->relocarray[i];
 186                 struct sg_table *sgt;
 187                 dma_addr_t phys_addr;
 188
 189                 reloc->target.bo = host1x_bo_get(reloc->target.bo);
 190                 if (!reloc->target.bo) {
 191                         err = -EINVAL;
 192                         goto unpin;
 193                 }
 194
 195                 phys_addr = host1x_bo_pin(reloc->target.bo, &sgt);
 196                 if (!phys_addr) {
 197                         err = -EINVAL;
 198                         goto unpin;
 199                 }
 200
 201                 job->addr_phys[job->num_unpins] = phys_addr;
 202                 job->unpins[job->num_unpins].bo = reloc->target.bo;
 203                 job->unpins[job->num_unpins].sgt = sgt;
 204                 job->num_unpins++;
 205         }
 206
 207         for (i = 0; i < job->num_gathers; i++) {
 208                 struct host1x_job_gather *g = &job->gathers[i];
 209                 size_t gather_size = 0;
 210                 struct scatterlist *sg;
 211                 struct sg_table *sgt;
 212                 dma_addr_t phys_addr;
 213                 unsigned long shift;
 214                 struct iova *alloc;
 215                 unsigned int j;
 216
 217                 g->bo = host1x_bo_get(g->bo);
 218                 if (!g->bo) {
 219                         err = -EINVAL;
 220                         goto unpin;
 221                 }
 222
 223                 phys_addr = host1x_bo_pin(g->bo, &sgt);
 224                 if (!phys_addr) {
 225                         err = -EINVAL;
 226                         goto unpin;
 227                 }
 228
 229                 if (!IS_ENABLED(CONFIG_TEGRA_HOST1X_FIREWALL) && host->domain) {
 230                         for_each_sg(sgt->sgl, sg, sgt->nents, j)
 231                                 gather_size += sg->length;
 232                         gather_size = iova_align(&host->iova, gather_size);
 233
 234                         shift = iova_shift(&host->iova);
 235                         alloc = alloc_iova(&host->iova, gather_size >> shift,
 236                                            host->iova_end >> shift, true);
 237                         if (!alloc) {
 238                                 err = -ENOMEM;
 239                                 goto unpin;
 240                         }
 241
 242                         err = iommu_map_sg(host->domain,
 243                                         iova_dma_addr(&host->iova, alloc),
 244                                         sgt->sgl, sgt->nents, IOMMU_READ);
 245                         if (err == 0) {
 246                                 __free_iova(&host->iova, alloc);
 247                                 err = -EINVAL;
 248                                 goto unpin;
 249                         }
 250
 251                         job->addr_phys[job->num_unpins] =
 252                                 iova_dma_addr(&host->iova, alloc);
 253                         job->unpins[job->num_unpins].size = gather_size;
 254                 } else {
 255                         job->addr_phys[job->num_unpins] = phys_addr;
 256                 }
 257
 258                 job->gather_addr_phys[i] = job->addr_phys[job->num_unpins];
 259
 260                 job->unpins[job->num_unpins].bo = g->bo;
 261                 job->unpins[job->num_unpins].sgt = sgt;
 262                 job->num_unpins++;
 263         }
 264
 265         return 0;
 266
 267 unpin:
 268         host1x_job_unpin(job);
 269         return err;
 270 }
 271
 272 static int do_relocs(struct host1x_job *job, struct host1x_bo *cmdbuf)
 273 {
 274         int i = 0;
 275         u32 last_page = ~0;
 276         void *cmdbuf_page_addr = NULL;
 277
 278         /* pin & patch the relocs for one gather */
 279         for (i = 0; i < job->num_relocs; i++) {
 280                 struct host1x_reloc *reloc = &job->relocarray[i];
 281                 u32 reloc_addr = (job->reloc_addr_phys[i] +
 282                                   reloc->target.offset) >> reloc->shift;
 283                 u32 *target;
 284
 285                 /* skip all other gathers */
 286                 if (cmdbuf != reloc->cmdbuf.bo)
 287                         continue;
 288
 289                 if (last_page != reloc->cmdbuf.offset >> PAGE_SHIFT) {
 290                         if (cmdbuf_page_addr)
 291                                 host1x_bo_kunmap(cmdbuf, last_page,
 292                                                  cmdbuf_page_addr);
 293
 294                         cmdbuf_page_addr = host1x_bo_kmap(cmdbuf,
 295                                         reloc->cmdbuf.offset >> PAGE_SHIFT);
 296                         last_page = reloc->cmdbuf.offset >> PAGE_SHIFT;
 297
 298                         if (unlikely(!cmdbuf_page_addr)) {
 299                                 pr_err("Could not map cmdbuf for relocation\n");
 300                                 return -ENOMEM;
 301                         }
 302                 }
 303
 304                 target = cmdbuf_page_addr + (reloc->cmdbuf.offset & ~PAGE_MASK);
 305                 *target = reloc_addr;
 306         }
 307
 308         if (cmdbuf_page_addr)
 309                 host1x_bo_kunmap(cmdbuf, last_page, cmdbuf_page_addr);
 310
 311         return 0;
 312 }
 313
 314 static bool check_reloc(struct host1x_reloc *reloc, struct host1x_bo *cmdbuf,
 315                         unsigned int offset)
 316 {
 317         offset *= sizeof(u32);
 318
 319         if (reloc->cmdbuf.bo != cmdbuf || reloc->cmdbuf.offset != offset)
 320                 return false;
 321
 322         return true;
 323 }
 324
 325 struct host1x_firewall {
 326         struct host1x_job *job;
 327         struct device *dev;
 328
 329         unsigned int num_relocs;
 330         struct host1x_reloc *reloc;
 331
 332         struct host1x_bo *cmdbuf;
 333         unsigned int offset;
 334
 335         u32 words;
 336         u32 class;
 337         u32 reg;
 338         u32 mask;
 339         u32 count;
 340 };
 341
 342 static int check_register(struct host1x_firewall *fw, unsigned long offset)
 343 {
 344         if (fw->job->is_addr_reg(fw->dev, fw->class, offset)) {
 345                 if (!fw->num_relocs)
 346                         return -EINVAL;
 347
 348                 if (!check_reloc(fw->reloc, fw->cmdbuf, fw->offset))
 349                         return -EINVAL;
 350
 351                 fw->num_relocs--;
 352                 fw->reloc++;
 353         }
 354
 355         return 0;
 356 }
 357
 358 static int check_mask(struct host1x_firewall *fw)
 359 {
 360         u32 mask = fw->mask;
 361         u32 reg = fw->reg;
 362         int ret;
 363
 364         while (mask) {
 365                 if (fw->words == 0)
 366                         return -EINVAL;
 367
 368                 if (mask & 1) {
 369                         ret = check_register(fw, reg);
 370                         if (ret < 0)
 371                                 return ret;
 372
 373                         fw->words--;
 374                         fw->offset++;
 375                 }
 376                 mask >>= 1;
 377                 reg++;
 378         }
 379
 380         return 0;
 381 }
 382
 383 static int check_incr(struct host1x_firewall *fw)
 384 {
 385         u32 count = fw->count;
 386         u32 reg = fw->reg;
 387         int ret;
 388
 389         while (count) {
 390                 if (fw->words == 0)
 391                         return -EINVAL;
 392
 393                 ret = check_register(fw, reg);
 394                 if (ret < 0)
 395                         return ret;
 396
 397                 reg++;
 398                 fw->words--;
 399                 fw->offset++;
 400                 count--;
 401         }
 402
 403         return 0;
 404 }
 405
 406 static int check_nonincr(struct host1x_firewall *fw)
 407 {
 408         u32 count = fw->count;
 409         int ret;
 410
 411         while (count) {
 412                 if (fw->words == 0)
 413                         return -EINVAL;
 414
 415                 ret = check_register(fw, fw->reg);
 416                 if (ret < 0)
 417                         return ret;
 418
 419                 fw->words--;
 420                 fw->offset++;
 421                 count--;
 422         }
 423
 424         return 0;
 425 }
 426
 427 static int validate(struct host1x_firewall *fw, struct host1x_job_gather *g)
 428 {
 429         u32 *cmdbuf_base = (u32 *)fw->job->gather_copy_mapped +
 430                 (g->offset / sizeof(u32));
 431         int err = 0;
 432
 433         if (!fw->job->is_addr_reg)
 434                 return 0;
 435
 436         fw->words = g->words;
 437         fw->cmdbuf = g->bo;
 438         fw->offset = 0;
 439
 440         while (fw->words && !err) {
 441                 u32 word = cmdbuf_base[fw->offset];
 442                 u32 opcode = (word & 0xf0000000) >> 28;
 443
 444                 fw->mask = 0;
 445                 fw->reg = 0;
 446                 fw->count = 0;
 447                 fw->words--;
 448                 fw->offset++;
 449
 450                 switch (opcode) {
 451                 case 0:
 452                         fw->class = word >> 6 & 0x3ff;
 453                         fw->mask = word & 0x3f;
 454                         fw->reg = word >> 16 & 0xfff;
 455                         err = check_mask(fw);
 456                         if (err)
 457                                 goto out;
 458                         break;
 459                 case 1:
 460                         fw->reg = word >> 16 & 0xfff;
 461                         fw->count = word & 0xffff;
 462                         err = check_incr(fw);
 463                         if (err)
 464                                 goto out;
 465                         break;
 466
 467                 case 2:
 468                         fw->reg = word >> 16 & 0xfff;
 469                         fw->count = word & 0xffff;
 470                         err = check_nonincr(fw);
 471                         if (err)
 472                                 goto out;
 473                         break;
 474
 475                 case 3:
 476                         fw->mask = word & 0xffff;
 477                         fw->reg = word >> 16 & 0xfff;
 478                         err = check_mask(fw);
 479                         if (err)
 480                                 goto out;
 481                         break;
 482                 case 4:
 483                 case 5:
 484                 case 14:
 485                         break;
 486                 default:
 487                         err = -EINVAL;
 488                         break;
 489                 }
 490         }
 491
 492 out:
 493         return err;
 494 }
 495
 496 static inline int copy_gathers(struct host1x_job *job, struct device *dev)
 497 {
 498         struct host1x_firewall fw;
 499         size_t size = 0;
 500         size_t offset = 0;
 501         int i;
 502
 503         fw.job = job;
 504         fw.dev = dev;
 505         fw.reloc = job->relocarray;
 506         fw.num_relocs = job->num_relocs;
 507         fw.class = 0;
 508
 509         for (i = 0; i < job->num_gathers; i++) {
 510                 struct host1x_job_gather *g = &job->gathers[i];
 511
 512                 size += g->words * sizeof(u32);
 513         }
 514
 515         job->gather_copy_mapped = dma_alloc_wc(dev, size, &job->gather_copy,
 516                                                GFP_KERNEL);
 517         if (!job->gather_copy_mapped) {
 518                 job->gather_copy_mapped = NULL;
 519                 return -ENOMEM;
 520         }
 521
 522         job->gather_copy_size = size;
 523
 524         for (i = 0; i < job->num_gathers; i++) {
 525                 struct host1x_job_gather *g = &job->gathers[i];
 526                 void *gather;
 527
 528                 /* Copy the gather */
 529                 gather = host1x_bo_mmap(g->bo);
 530                 memcpy(job->gather_copy_mapped + offset, gather + g->offset,
 531                        g->words * sizeof(u32));
 532                 host1x_bo_munmap(g->bo, gather);
 533
 534                 /* Store the location in the buffer */
 535                 g->base = job->gather_copy;
 536                 g->offset = offset;
 537
 538                 /* Validate the job */
 539                 if (validate(&fw, g))
 540                         return -EINVAL;
 541
 542                 offset += g->words * sizeof(u32);
 543         }
 544
 545         /* No relocs should remain at this point */
 546         if (fw.num_relocs)
 547                 return -EINVAL;
 548
 549         return 0;
 550 }
 551
 552 int host1x_job_pin(struct host1x_job *job, struct device *dev)
 553 {
 554         int err;
 555         unsigned int i, j;
 556         struct host1x *host = dev_get_drvdata(dev->parent);
 557         DECLARE_BITMAP(waitchk_mask, host1x_syncpt_nb_pts(host));
 558
 559         bitmap_zero(waitchk_mask, host1x_syncpt_nb_pts(host));
 560         for (i = 0; i < job->num_waitchk; i++) {
 561                 u32 syncpt_id = job->waitchk[i].syncpt_id;
 562
 563                 if (syncpt_id < host1x_syncpt_nb_pts(host))
 564                         set_bit(syncpt_id, waitchk_mask);
 565         }
 566
 567         /* get current syncpt values for waitchk */
 568         for_each_set_bit(i, waitchk_mask, host1x_syncpt_nb_pts(host))
 569                 host1x_syncpt_load(host->syncpt + i);
 570
 571         /* pin memory */
 572         err = pin_job(host, job);
 573         if (err)
 574                 goto out;
 575
 576         /* patch gathers */
 577         for (i = 0; i < job->num_gathers; i++) {
 578                 struct host1x_job_gather *g = &job->gathers[i];
 579
 580                 /* process each gather mem only once */
 581                 if (g->handled)
 582                         continue;
 583
 584                 g->base = job->gather_addr_phys[i];
 585
 586                 for (j = i + 1; j < job->num_gathers; j++) {
 587                         if (job->gathers[j].bo == g->bo) {
 588                                 job->gathers[j].handled = true;
 589                                 job->gathers[j].base = g->base;
 590                         }
 591                 }
 592
 593                 err = do_relocs(job, g->bo);
 594                 if (err)
 595                         break;
 596
 597                 err = do_waitchks(job, host, g->bo);
 598                 if (err)
 599                         break;
 600         }
 601
 602         if (IS_ENABLED(CONFIG_TEGRA_HOST1X_FIREWALL) && !err) {
 603                 err = copy_gathers(job, dev);
 604                 if (err) {
 605                         host1x_job_unpin(job);
 606                         return err;
 607                 }
 608         }
 609
 610 out:
 611         wmb();
 612
 613         return err;
 614 }
 615 EXPORT_SYMBOL(host1x_job_pin);
 616
 617 void host1x_job_unpin(struct host1x_job *job)
 618 {
 619         struct host1x *host = dev_get_drvdata(job->channel->dev->parent);
 620         unsigned int i;
 621
 622         for (i = 0; i < job->num_unpins; i++) {
 623                 struct host1x_job_unpin_data *unpin = &job->unpins[i];
 624
 625                 if (!IS_ENABLED(CONFIG_TEGRA_HOST1X_FIREWALL) && host->domain) {
 626                         iommu_unmap(host->domain, job->addr_phys[i],
 627                                     unpin->size);
 628                         free_iova(&host->iova,
 629                                 iova_pfn(&host->iova, job->addr_phys[i]));
 630                 }
 631
 632                 host1x_bo_unpin(unpin->bo, unpin->sgt);
 633                 host1x_bo_put(unpin->bo);
 634         }
 635
 636         job->num_unpins = 0;
 637
 638         if (job->gather_copy_size)
 639                 dma_free_wc(job->channel->dev, job->gather_copy_size,
 640                             job->gather_copy_mapped, job->gather_copy);
 641 }
 642 EXPORT_SYMBOL(host1x_job_unpin);
 643
 644 /*
 645  * Debug routine used to dump job entries
 646  */
 647 void host1x_job_dump(struct device *dev, struct host1x_job *job)
 648 {
 649         dev_dbg(dev, "    SYNCPT_ID   %d\n", job->syncpt_id);
 650         dev_dbg(dev, "    SYNCPT_VAL  %d\n", job->syncpt_end);
 651         dev_dbg(dev, "    FIRST_GET   0x%x\n", job->first_get);
 652         dev_dbg(dev, "    TIMEOUT     %d\n", job->timeout);
 653         dev_dbg(dev, "    NUM_SLOTS   %d\n", job->num_slots);
 654         dev_dbg(dev, "    NUM_HANDLES %d\n", job->num_unpins);
 655 }