drivers/misc/pci_endpoint_test.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /**
   3  * Host side test driver to test endpoint functionality
   4  *
   5  * Copyright (C) 2017 Texas Instruments
   6  * Author: Kishon Vijay Abraham I <kishon@ti.com>
   7  */
   8
   9 #include <linux/crc32.h>
  10 #include <linux/delay.h>
  11 #include <linux/fs.h>
  12 #include <linux/io.h>
  13 #include <linux/interrupt.h>
  14 #include <linux/irq.h>
  15 #include <linux/miscdevice.h>
  16 #include <linux/module.h>
  17 #include <linux/mutex.h>
  18 #include <linux/random.h>
  19 #include <linux/slab.h>
  20 #include <linux/uaccess.h>
  21 #include <linux/pci.h>
  22 #include <linux/pci_ids.h>
  23
  24 #include <linux/pci_regs.h>
  25
  26 #include <uapi/linux/pcitest.h>
  27
  28 #define DRV_MODULE_NAME                         "pci-endpoint-test"
  29
  30 #define IRQ_TYPE_UNDEFINED                      -1
  31 #define IRQ_TYPE_LEGACY                         0
  32 #define IRQ_TYPE_MSI                            1
  33 #define IRQ_TYPE_MSIX                           2
  34
  35 #define PCI_ENDPOINT_TEST_MAGIC                 0x0
  36
  37 #define PCI_ENDPOINT_TEST_COMMAND               0x4
  38 #define COMMAND_RAISE_LEGACY_IRQ                BIT(0)
  39 #define COMMAND_RAISE_MSI_IRQ                   BIT(1)
  40 #define COMMAND_RAISE_MSIX_IRQ                  BIT(2)
  41 #define COMMAND_READ                            BIT(3)
  42 #define COMMAND_WRITE                           BIT(4)
  43 #define COMMAND_COPY                            BIT(5)
  44
  45 #define PCI_ENDPOINT_TEST_STATUS                0x8
  46 #define STATUS_READ_SUCCESS                     BIT(0)
  47 #define STATUS_READ_FAIL                        BIT(1)
  48 #define STATUS_WRITE_SUCCESS                    BIT(2)
  49 #define STATUS_WRITE_FAIL                       BIT(3)
  50 #define STATUS_COPY_SUCCESS                     BIT(4)
  51 #define STATUS_COPY_FAIL                        BIT(5)
  52 #define STATUS_IRQ_RAISED                       BIT(6)
  53 #define STATUS_SRC_ADDR_INVALID                 BIT(7)
  54 #define STATUS_DST_ADDR_INVALID                 BIT(8)
  55
  56 #define PCI_ENDPOINT_TEST_LOWER_SRC_ADDR        0x0c
  57 #define PCI_ENDPOINT_TEST_UPPER_SRC_ADDR        0x10
  58
  59 #define PCI_ENDPOINT_TEST_LOWER_DST_ADDR        0x14
  60 #define PCI_ENDPOINT_TEST_UPPER_DST_ADDR        0x18
  61
  62 #define PCI_ENDPOINT_TEST_SIZE                  0x1c
  63 #define PCI_ENDPOINT_TEST_CHECKSUM              0x20
  64
  65 #define PCI_ENDPOINT_TEST_IRQ_TYPE              0x24
  66 #define PCI_ENDPOINT_TEST_IRQ_NUMBER            0x28
  67
  68 #define PCI_ENDPOINT_TEST_FLAGS                 0x2c
  69 #define FLAG_USE_DMA                            BIT(0)
  70
  71 #define PCI_DEVICE_ID_TI_AM654                  0xb00c
  72 #define PCI_DEVICE_ID_LS1088A                   0x80c0
  73
  74 #define is_am654_pci_dev(pdev)          \
  75                 ((pdev)->device == PCI_DEVICE_ID_TI_AM654)
  76
  77 #define PCI_DEVICE_ID_RENESAS_R8A774A1          0x0028
  78 #define PCI_DEVICE_ID_RENESAS_R8A774B1          0x002b
  79 #define PCI_DEVICE_ID_RENESAS_R8A774C0          0x002d
  80 #define PCI_DEVICE_ID_RENESAS_R8A774E1          0x0025
  81
  82 static DEFINE_IDA(pci_endpoint_test_ida);
  83
  84 #define to_endpoint_test(priv) container_of((priv), struct pci_endpoint_test, \
  85                                             miscdev)
  86
  87 static bool no_msi;
  88 module_param(no_msi, bool, 0444);
  89 MODULE_PARM_DESC(no_msi, "Disable MSI interrupt in pci_endpoint_test");
  90
  91 static int irq_type = IRQ_TYPE_MSI;
  92 module_param(irq_type, int, 0444);
  93 MODULE_PARM_DESC(irq_type, "IRQ mode selection in pci_endpoint_test (0 - Legacy, 1 - MSI, 2 - MSI-X)");
  94
  95 enum pci_barno {
  96         BAR_0,
  97         BAR_1,
  98         BAR_2,
  99         BAR_3,
 100         BAR_4,
 101         BAR_5,
 102 };
 103
 104 struct pci_endpoint_test {
 105         struct pci_dev  *pdev;
 106         void __iomem    *base;
 107         void __iomem    *bar[PCI_STD_NUM_BARS];
 108         struct completion irq_raised;
 109         int             last_irq;
 110         int             num_irqs;
 111         int             irq_type;
 112         /* mutex to protect the ioctls */
 113         struct mutex    mutex;
 114         struct miscdevice miscdev;
 115         enum pci_barno test_reg_bar;
 116         size_t alignment;
 117         const char *name;
 118 };
 119
 120 struct pci_endpoint_test_data {
 121         enum pci_barno test_reg_bar;
 122         size_t alignment;
 123         int irq_type;
 124 };
 125
 126 static inline u32 pci_endpoint_test_readl(struct pci_endpoint_test *test,
 127                                           u32 offset)
 128 {
 129         return readl(test->base + offset);
 130 }
 131
 132 static inline void pci_endpoint_test_writel(struct pci_endpoint_test *test,
 133                                             u32 offset, u32 value)
 134 {
 135         writel(value, test->base + offset);
 136 }
 137
 138 static inline u32 pci_endpoint_test_bar_readl(struct pci_endpoint_test *test,
 139                                               int bar, int offset)
 140 {
 141         return readl(test->bar[bar] + offset);
 142 }
 143
 144 static inline void pci_endpoint_test_bar_writel(struct pci_endpoint_test *test,
 145                                                 int bar, u32 offset, u32 value)
 146 {
 147         writel(value, test->bar[bar] + offset);
 148 }
 149
 150 static irqreturn_t pci_endpoint_test_irqhandler(int irq, void *dev_id)
 151 {
 152         struct pci_endpoint_test *test = dev_id;
 153         u32 reg;
 154
 155         reg = pci_endpoint_test_readl(test, PCI_ENDPOINT_TEST_STATUS);
 156         if (reg & STATUS_IRQ_RAISED) {
 157                 test->last_irq = irq;
 158                 complete(&test->irq_raised);
 159                 reg &= ~STATUS_IRQ_RAISED;
 160         }
 161         pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_STATUS,
 162                                  reg);
 163
 164         return IRQ_HANDLED;
 165 }
 166
 167 static void pci_endpoint_test_free_irq_vectors(struct pci_endpoint_test *test)
 168 {
 169         struct pci_dev *pdev = test->pdev;
 170
 171         pci_free_irq_vectors(pdev);
 172         test->irq_type = IRQ_TYPE_UNDEFINED;
 173 }
 174
 175 static bool pci_endpoint_test_alloc_irq_vectors(struct pci_endpoint_test *test,
 176                                                 int type)
 177 {
 178         int irq = -1;
 179         struct pci_dev *pdev = test->pdev;
 180         struct device *dev = &pdev->dev;
 181         bool res = true;
 182
 183         switch (type) {
 184         case IRQ_TYPE_LEGACY:
 185                 irq = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_LEGACY);
 186                 if (irq < 0)
 187                         dev_err(dev, "Failed to get Legacy interrupt\n");
 188                 break;
 189         case IRQ_TYPE_MSI:
 190                 irq = pci_alloc_irq_vectors(pdev, 1, 32, PCI_IRQ_MSI);
 191                 if (irq < 0)
 192                         dev_err(dev, "Failed to get MSI interrupts\n");
 193                 break;
 194         case IRQ_TYPE_MSIX:
 195                 irq = pci_alloc_irq_vectors(pdev, 1, 2048, PCI_IRQ_MSIX);
 196                 if (irq < 0)
 197                         dev_err(dev, "Failed to get MSI-X interrupts\n");
 198                 break;
 199         default:
 200                 dev_err(dev, "Invalid IRQ type selected\n");
 201         }
 202
 203         if (irq < 0) {
 204                 irq = 0;
 205                 res = false;
 206         }
 207
 208         test->irq_type = type;
 209         test->num_irqs = irq;
 210
 211         return res;
 212 }
 213
 214 static void pci_endpoint_test_release_irq(struct pci_endpoint_test *test)
 215 {
 216         int i;
 217         struct pci_dev *pdev = test->pdev;
 218         struct device *dev = &pdev->dev;
 219
 220         for (i = 0; i < test->num_irqs; i++)
 221                 devm_free_irq(dev, pci_irq_vector(pdev, i), test);
 222
 223         test->num_irqs = 0;
 224 }
 225
 226 static bool pci_endpoint_test_request_irq(struct pci_endpoint_test *test)
 227 {
 228         int i;
 229         int err;
 230         struct pci_dev *pdev = test->pdev;
 231         struct device *dev = &pdev->dev;
 232
 233         for (i = 0; i < test->num_irqs; i++) {
 234                 err = devm_request_irq(dev, pci_irq_vector(pdev, i),
 235                                        pci_endpoint_test_irqhandler,
 236                                        IRQF_SHARED, test->name, test);
 237                 if (err)
 238                         goto fail;
 239         }
 240
 241         return true;
 242
 243 fail:
 244         switch (irq_type) {
 245         case IRQ_TYPE_LEGACY:
 246                 dev_err(dev, "Failed to request IRQ %d for Legacy\n",
 247                         pci_irq_vector(pdev, i));
 248                 break;
 249         case IRQ_TYPE_MSI:
 250                 dev_err(dev, "Failed to request IRQ %d for MSI %d\n",
 251                         pci_irq_vector(pdev, i),
 252                         i + 1);
 253                 break;
 254         case IRQ_TYPE_MSIX:
 255                 dev_err(dev, "Failed to request IRQ %d for MSI-X %d\n",
 256                         pci_irq_vector(pdev, i),
 257                         i + 1);
 258                 break;
 259         }
 260
 261         return false;
 262 }
 263
 264 static bool pci_endpoint_test_bar(struct pci_endpoint_test *test,
 265                                   enum pci_barno barno)
 266 {
 267         int j;
 268         u32 val;
 269         int size;
 270         struct pci_dev *pdev = test->pdev;
 271
 272         if (!test->bar[barno])
 273                 return false;
 274
 275         size = pci_resource_len(pdev, barno);
 276
 277         if (barno == test->test_reg_bar)
 278                 size = 0x4;
 279
 280         for (j = 0; j < size; j += 4)
 281                 pci_endpoint_test_bar_writel(test, barno, j, 0xA0A0A0A0);
 282
 283         for (j = 0; j < size; j += 4) {
 284                 val = pci_endpoint_test_bar_readl(test, barno, j);
 285                 if (val != 0xA0A0A0A0)
 286                         return false;
 287         }
 288
 289         return true;
 290 }
 291
 292 static bool pci_endpoint_test_legacy_irq(struct pci_endpoint_test *test)
 293 {
 294         u32 val;
 295
 296         pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_IRQ_TYPE,
 297                                  IRQ_TYPE_LEGACY);
 298         pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_IRQ_NUMBER, 0);
 299         pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_COMMAND,
 300                                  COMMAND_RAISE_LEGACY_IRQ);
 301         val = wait_for_completion_timeout(&test->irq_raised,
 302                                           msecs_to_jiffies(1000));
 303         if (!val)
 304                 return false;
 305
 306         return true;
 307 }
 308
 309 static bool pci_endpoint_test_msi_irq(struct pci_endpoint_test *test,
 310                                        u16 msi_num, bool msix)
 311 {
 312         u32 val;
 313         struct pci_dev *pdev = test->pdev;
 314
 315         pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_IRQ_TYPE,
 316                                  msix == false ? IRQ_TYPE_MSI :
 317                                  IRQ_TYPE_MSIX);
 318         pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_IRQ_NUMBER, msi_num);
 319         pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_COMMAND,
 320                                  msix == false ? COMMAND_RAISE_MSI_IRQ :
 321                                  COMMAND_RAISE_MSIX_IRQ);
 322         val = wait_for_completion_timeout(&test->irq_raised,
 323                                           msecs_to_jiffies(1000));
 324         if (!val)
 325                 return false;
 326
 327         if (pci_irq_vector(pdev, msi_num - 1) == test->last_irq)
 328                 return true;
 329
 330         return false;
 331 }
 332
 333 static bool pci_endpoint_test_copy(struct pci_endpoint_test *test,
 334                                    unsigned long arg)
 335 {
 336         struct pci_endpoint_test_xfer_param param;
 337         bool ret = false;
 338         void *src_addr;
 339         void *dst_addr;
 340         u32 flags = 0;
 341         bool use_dma;
 342         size_t size;
 343         dma_addr_t src_phys_addr;
 344         dma_addr_t dst_phys_addr;
 345         struct pci_dev *pdev = test->pdev;
 346         struct device *dev = &pdev->dev;
 347         void *orig_src_addr;
 348         dma_addr_t orig_src_phys_addr;
 349         void *orig_dst_addr;
 350         dma_addr_t orig_dst_phys_addr;
 351         size_t offset;
 352         size_t alignment = test->alignment;
 353         int irq_type = test->irq_type;
 354         u32 src_crc32;
 355         u32 dst_crc32;
 356         int err;
 357
 358         err = copy_from_user(&param, (void __user *)arg, sizeof(param));
 359         if (err) {
 360                 dev_err(dev, "Failed to get transfer param\n");
 361                 return false;
 362         }
 363
 364         size = param.size;
 365         if (size > SIZE_MAX - alignment)
 366                 goto err;
 367
 368         use_dma = !!(param.flags & PCITEST_FLAGS_USE_DMA);
 369         if (use_dma)
 370                 flags |= FLAG_USE_DMA;
 371
 372         if (irq_type < IRQ_TYPE_LEGACY || irq_type > IRQ_TYPE_MSIX) {
 373                 dev_err(dev, "Invalid IRQ type option\n");
 374                 goto err;
 375         }
 376
 377         orig_src_addr = kzalloc(size + alignment, GFP_KERNEL);
 378         if (!orig_src_addr) {
 379                 dev_err(dev, "Failed to allocate source buffer\n");
 380                 ret = false;
 381                 goto err;
 382         }
 383
 384         get_random_bytes(orig_src_addr, size + alignment);
 385         orig_src_phys_addr = dma_map_single(dev, orig_src_addr,
 386                                             size + alignment, DMA_TO_DEVICE);
 387         if (dma_mapping_error(dev, orig_src_phys_addr)) {
 388                 dev_err(dev, "failed to map source buffer address\n");
 389                 ret = false;
 390                 goto err_src_phys_addr;
 391         }
 392
 393         if (alignment && !IS_ALIGNED(orig_src_phys_addr, alignment)) {
 394                 src_phys_addr = PTR_ALIGN(orig_src_phys_addr, alignment);
 395                 offset = src_phys_addr - orig_src_phys_addr;
 396                 src_addr = orig_src_addr + offset;
 397         } else {
 398                 src_phys_addr = orig_src_phys_addr;
 399                 src_addr = orig_src_addr;
 400         }
 401
 402         pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_LOWER_SRC_ADDR,
 403                                  lower_32_bits(src_phys_addr));
 404
 405         pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_UPPER_SRC_ADDR,
 406                                  upper_32_bits(src_phys_addr));
 407
 408         src_crc32 = crc32_le(~0, src_addr, size);
 409
 410         orig_dst_addr = kzalloc(size + alignment, GFP_KERNEL);
 411         if (!orig_dst_addr) {
 412                 dev_err(dev, "Failed to allocate destination address\n");
 413                 ret = false;
 414                 goto err_dst_addr;
 415         }
 416
 417         orig_dst_phys_addr = dma_map_single(dev, orig_dst_addr,
 418                                             size + alignment, DMA_FROM_DEVICE);
 419         if (dma_mapping_error(dev, orig_dst_phys_addr)) {
 420                 dev_err(dev, "failed to map destination buffer address\n");
 421                 ret = false;
 422                 goto err_dst_phys_addr;
 423         }
 424
 425         if (alignment && !IS_ALIGNED(orig_dst_phys_addr, alignment)) {
 426                 dst_phys_addr = PTR_ALIGN(orig_dst_phys_addr, alignment);
 427                 offset = dst_phys_addr - orig_dst_phys_addr;
 428                 dst_addr = orig_dst_addr + offset;
 429         } else {
 430                 dst_phys_addr = orig_dst_phys_addr;
 431                 dst_addr = orig_dst_addr;
 432         }
 433
 434         pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_LOWER_DST_ADDR,
 435                                  lower_32_bits(dst_phys_addr));
 436         pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_UPPER_DST_ADDR,
 437                                  upper_32_bits(dst_phys_addr));
 438
 439         pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_SIZE,
 440                                  size);
 441
 442         pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_FLAGS, flags);
 443         pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_IRQ_TYPE, irq_type);
 444         pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_IRQ_NUMBER, 1);
 445         pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_COMMAND,
 446                                  COMMAND_COPY);
 447
 448         wait_for_completion(&test->irq_raised);
 449
 450         dma_unmap_single(dev, orig_dst_phys_addr, size + alignment,
 451                          DMA_FROM_DEVICE);
 452
 453         dst_crc32 = crc32_le(~0, dst_addr, size);
 454         if (dst_crc32 == src_crc32)
 455                 ret = true;
 456
 457 err_dst_phys_addr:
 458         kfree(orig_dst_addr);
 459
 460 err_dst_addr:
 461         dma_unmap_single(dev, orig_src_phys_addr, size + alignment,
 462                          DMA_TO_DEVICE);
 463
 464 err_src_phys_addr:
 465         kfree(orig_src_addr);
 466
 467 err:
 468         return ret;
 469 }
 470
 471 static bool pci_endpoint_test_write(struct pci_endpoint_test *test,
 472                                     unsigned long arg)
 473 {
 474         struct pci_endpoint_test_xfer_param param;
 475         bool ret = false;
 476         u32 flags = 0;
 477         bool use_dma;
 478         u32 reg;
 479         void *addr;
 480         dma_addr_t phys_addr;
 481         struct pci_dev *pdev = test->pdev;
 482         struct device *dev = &pdev->dev;
 483         void *orig_addr;
 484         dma_addr_t orig_phys_addr;
 485         size_t offset;
 486         size_t alignment = test->alignment;
 487         int irq_type = test->irq_type;
 488         size_t size;
 489         u32 crc32;
 490         int err;
 491
 492         err = copy_from_user(&param, (void __user *)arg, sizeof(param));
 493         if (err != 0) {
 494                 dev_err(dev, "Failed to get transfer param\n");
 495                 return false;
 496         }
 497
 498         size = param.size;
 499         if (size > SIZE_MAX - alignment)
 500                 goto err;
 501
 502         use_dma = !!(param.flags & PCITEST_FLAGS_USE_DMA);
 503         if (use_dma)
 504                 flags |= FLAG_USE_DMA;
 505
 506         if (irq_type < IRQ_TYPE_LEGACY || irq_type > IRQ_TYPE_MSIX) {
 507                 dev_err(dev, "Invalid IRQ type option\n");
 508                 goto err;
 509         }
 510
 511         orig_addr = kzalloc(size + alignment, GFP_KERNEL);
 512         if (!orig_addr) {
 513                 dev_err(dev, "Failed to allocate address\n");
 514                 ret = false;
 515                 goto err;
 516         }
 517
 518         get_random_bytes(orig_addr, size + alignment);
 519
 520         orig_phys_addr = dma_map_single(dev, orig_addr, size + alignment,
 521                                         DMA_TO_DEVICE);
 522         if (dma_mapping_error(dev, orig_phys_addr)) {
 523                 dev_err(dev, "failed to map source buffer address\n");
 524                 ret = false;
 525                 goto err_phys_addr;
 526         }
 527
 528         if (alignment && !IS_ALIGNED(orig_phys_addr, alignment)) {
 529                 phys_addr =  PTR_ALIGN(orig_phys_addr, alignment);
 530                 offset = phys_addr - orig_phys_addr;
 531                 addr = orig_addr + offset;
 532         } else {
 533                 phys_addr = orig_phys_addr;
 534                 addr = orig_addr;
 535         }
 536
 537         crc32 = crc32_le(~0, addr, size);
 538         pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_CHECKSUM,
 539                                  crc32);
 540
 541         pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_LOWER_SRC_ADDR,
 542                                  lower_32_bits(phys_addr));
 543         pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_UPPER_SRC_ADDR,
 544                                  upper_32_bits(phys_addr));
 545
 546         pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_SIZE, size);
 547
 548         pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_FLAGS, flags);
 549         pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_IRQ_TYPE, irq_type);
 550         pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_IRQ_NUMBER, 1);
 551         pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_COMMAND,
 552                                  COMMAND_READ);
 553
 554         wait_for_completion(&test->irq_raised);
 555
 556         reg = pci_endpoint_test_readl(test, PCI_ENDPOINT_TEST_STATUS);
 557         if (reg & STATUS_READ_SUCCESS)
 558                 ret = true;
 559
 560         dma_unmap_single(dev, orig_phys_addr, size + alignment,
 561                          DMA_TO_DEVICE);
 562
 563 err_phys_addr:
 564         kfree(orig_addr);
 565
 566 err:
 567         return ret;
 568 }
 569
 570 static bool pci_endpoint_test_read(struct pci_endpoint_test *test,
 571                                    unsigned long arg)
 572 {
 573         struct pci_endpoint_test_xfer_param param;
 574         bool ret = false;
 575         u32 flags = 0;
 576         bool use_dma;
 577         size_t size;
 578         void *addr;
 579         dma_addr_t phys_addr;
 580         struct pci_dev *pdev = test->pdev;
 581         struct device *dev = &pdev->dev;
 582         void *orig_addr;
 583         dma_addr_t orig_phys_addr;
 584         size_t offset;
 585         size_t alignment = test->alignment;
 586         int irq_type = test->irq_type;
 587         u32 crc32;
 588         int err;
 589
 590         err = copy_from_user(&param, (void __user *)arg, sizeof(param));
 591         if (err) {
 592                 dev_err(dev, "Failed to get transfer param\n");
 593                 return false;
 594         }
 595
 596         size = param.size;
 597         if (size > SIZE_MAX - alignment)
 598                 goto err;
 599
 600         use_dma = !!(param.flags & PCITEST_FLAGS_USE_DMA);
 601         if (use_dma)
 602                 flags |= FLAG_USE_DMA;
 603
 604         if (irq_type < IRQ_TYPE_LEGACY || irq_type > IRQ_TYPE_MSIX) {
 605                 dev_err(dev, "Invalid IRQ type option\n");
 606                 goto err;
 607         }
 608
 609         orig_addr = kzalloc(size + alignment, GFP_KERNEL);
 610         if (!orig_addr) {
 611                 dev_err(dev, "Failed to allocate destination address\n");
 612                 ret = false;
 613                 goto err;
 614         }
 615
 616         orig_phys_addr = dma_map_single(dev, orig_addr, size + alignment,
 617                                         DMA_FROM_DEVICE);
 618         if (dma_mapping_error(dev, orig_phys_addr)) {
 619                 dev_err(dev, "failed to map source buffer address\n");
 620                 ret = false;
 621                 goto err_phys_addr;
 622         }
 623
 624         if (alignment && !IS_ALIGNED(orig_phys_addr, alignment)) {
 625                 phys_addr = PTR_ALIGN(orig_phys_addr, alignment);
 626                 offset = phys_addr - orig_phys_addr;
 627                 addr = orig_addr + offset;
 628         } else {
 629                 phys_addr = orig_phys_addr;
 630                 addr = orig_addr;
 631         }
 632
 633         pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_LOWER_DST_ADDR,
 634                                  lower_32_bits(phys_addr));
 635         pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_UPPER_DST_ADDR,
 636                                  upper_32_bits(phys_addr));
 637
 638         pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_SIZE, size);
 639
 640         pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_FLAGS, flags);
 641         pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_IRQ_TYPE, irq_type);
 642         pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_IRQ_NUMBER, 1);
 643         pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_COMMAND,
 644                                  COMMAND_WRITE);
 645
 646         wait_for_completion(&test->irq_raised);
 647
 648         dma_unmap_single(dev, orig_phys_addr, size + alignment,
 649                          DMA_FROM_DEVICE);
 650
 651         crc32 = crc32_le(~0, addr, size);
 652         if (crc32 == pci_endpoint_test_readl(test, PCI_ENDPOINT_TEST_CHECKSUM))
 653                 ret = true;
 654
 655 err_phys_addr:
 656         kfree(orig_addr);
 657 err:
 658         return ret;
 659 }
 660
 661 static bool pci_endpoint_test_clear_irq(struct pci_endpoint_test *test)
 662 {
 663         pci_endpoint_test_release_irq(test);
 664         pci_endpoint_test_free_irq_vectors(test);
 665         return true;
 666 }
 667
 668 static bool pci_endpoint_test_set_irq(struct pci_endpoint_test *test,
 669                                       int req_irq_type)
 670 {
 671         struct pci_dev *pdev = test->pdev;
 672         struct device *dev = &pdev->dev;
 673
 674         if (req_irq_type < IRQ_TYPE_LEGACY || req_irq_type > IRQ_TYPE_MSIX) {
 675                 dev_err(dev, "Invalid IRQ type option\n");
 676                 return false;
 677         }
 678
 679         if (test->irq_type == req_irq_type)
 680                 return true;
 681
 682         pci_endpoint_test_release_irq(test);
 683         pci_endpoint_test_free_irq_vectors(test);
 684
 685         if (!pci_endpoint_test_alloc_irq_vectors(test, req_irq_type))
 686                 goto err;
 687
 688         if (!pci_endpoint_test_request_irq(test))
 689                 goto err;
 690
 691         return true;
 692
 693 err:
 694         pci_endpoint_test_free_irq_vectors(test);
 695         return false;
 696 }
 697
 698 static long pci_endpoint_test_ioctl(struct file *file, unsigned int cmd,
 699                                     unsigned long arg)
 700 {
 701         int ret = -EINVAL;
 702         enum pci_barno bar;
 703         struct pci_endpoint_test *test = to_endpoint_test(file->private_data);
 704         struct pci_dev *pdev = test->pdev;
 705
 706         mutex_lock(&test->mutex);
 707         switch (cmd) {
 708         case PCITEST_BAR:
 709                 bar = arg;
 710                 if (bar > BAR_5)
 711                         goto ret;
 712                 if (is_am654_pci_dev(pdev) && bar == BAR_0)
 713                         goto ret;
 714                 ret = pci_endpoint_test_bar(test, bar);
 715                 break;
 716         case PCITEST_LEGACY_IRQ:
 717                 ret = pci_endpoint_test_legacy_irq(test);
 718                 break;
 719         case PCITEST_MSI:
 720         case PCITEST_MSIX:
 721                 ret = pci_endpoint_test_msi_irq(test, arg, cmd == PCITEST_MSIX);
 722                 break;
 723         case PCITEST_WRITE:
 724                 ret = pci_endpoint_test_write(test, arg);
 725                 break;
 726         case PCITEST_READ:
 727                 ret = pci_endpoint_test_read(test, arg);
 728                 break;
 729         case PCITEST_COPY:
 730                 ret = pci_endpoint_test_copy(test, arg);
 731                 break;
 732         case PCITEST_SET_IRQTYPE:
 733                 ret = pci_endpoint_test_set_irq(test, arg);
 734                 break;
 735         case PCITEST_GET_IRQTYPE:
 736                 ret = irq_type;
 737                 break;
 738         case PCITEST_CLEAR_IRQ:
 739                 ret = pci_endpoint_test_clear_irq(test);
 740                 break;
 741         }
 742
 743 ret:
 744         mutex_unlock(&test->mutex);
 745         return ret;
 746 }
 747
 748 static const struct file_operations pci_endpoint_test_fops = {
 749         .owner = THIS_MODULE,
 750         .unlocked_ioctl = pci_endpoint_test_ioctl,
 751 };
 752
 753 static int pci_endpoint_test_probe(struct pci_dev *pdev,
 754                                    const struct pci_device_id *ent)
 755 {
 756         int err;
 757         int id;
 758         char name[24];
 759         enum pci_barno bar;
 760         void __iomem *base;
 761         struct device *dev = &pdev->dev;
 762         struct pci_endpoint_test *test;
 763         struct pci_endpoint_test_data *data;
 764         enum pci_barno test_reg_bar = BAR_0;
 765         struct miscdevice *misc_device;
 766
 767         if (pci_is_bridge(pdev))
 768                 return -ENODEV;
 769
 770         test = devm_kzalloc(dev, sizeof(*test), GFP_KERNEL);
 771         if (!test)
 772                 return -ENOMEM;
 773
 774         test->test_reg_bar = 0;
 775         test->alignment = 0;
 776         test->pdev = pdev;
 777         test->irq_type = IRQ_TYPE_UNDEFINED;
 778
 779         if (no_msi)
 780                 irq_type = IRQ_TYPE_LEGACY;
 781
 782         data = (struct pci_endpoint_test_data *)ent->driver_data;
 783         if (data) {
 784                 test_reg_bar = data->test_reg_bar;
 785                 test->test_reg_bar = test_reg_bar;
 786                 test->alignment = data->alignment;
 787                 irq_type = data->irq_type;
 788         }
 789
 790         init_completion(&test->irq_raised);
 791         mutex_init(&test->mutex);
 792
 793         if ((dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(48)) != 0) &&
 794             dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)) != 0) {
 795                 dev_err(dev, "Cannot set DMA mask\n");
 796                 return -EINVAL;
 797         }
 798
 799         err = pci_enable_device(pdev);
 800         if (err) {
 801                 dev_err(dev, "Cannot enable PCI device\n");
 802                 return err;
 803         }
 804
 805         err = pci_request_regions(pdev, DRV_MODULE_NAME);
 806         if (err) {
 807                 dev_err(dev, "Cannot obtain PCI resources\n");
 808                 goto err_disable_pdev;
 809         }
 810
 811         pci_set_master(pdev);
 812
 813         if (!pci_endpoint_test_alloc_irq_vectors(test, irq_type)) {
 814                 err = -EINVAL;
 815                 goto err_disable_irq;
 816         }
 817
 818         for (bar = 0; bar < PCI_STD_NUM_BARS; bar++) {
 819                 if (pci_resource_flags(pdev, bar) & IORESOURCE_MEM) {
 820                         base = pci_ioremap_bar(pdev, bar);
 821                         if (!base) {
 822                                 dev_err(dev, "Failed to read BAR%d\n", bar);
 823                                 WARN_ON(bar == test_reg_bar);
 824                         }
 825                         test->bar[bar] = base;
 826                 }
 827         }
 828
 829         test->base = test->bar[test_reg_bar];
 830         if (!test->base) {
 831                 err = -ENOMEM;
 832                 dev_err(dev, "Cannot perform PCI test without BAR%d\n",
 833                         test_reg_bar);
 834                 goto err_iounmap;
 835         }
 836
 837         pci_set_drvdata(pdev, test);
 838
 839         id = ida_simple_get(&pci_endpoint_test_ida, 0, 0, GFP_KERNEL);
 840         if (id < 0) {
 841                 err = id;
 842                 dev_err(dev, "Unable to get id\n");
 843                 goto err_iounmap;
 844         }
 845
 846         snprintf(name, sizeof(name), DRV_MODULE_NAME ".%d", id);
 847         test->name = kstrdup(name, GFP_KERNEL);
 848         if (!test->name) {
 849                 err = -ENOMEM;
 850                 goto err_ida_remove;
 851         }
 852
 853         if (!pci_endpoint_test_request_irq(test)) {
 854                 err = -EINVAL;
 855                 goto err_kfree_test_name;
 856         }
 857
 858         misc_device = &test->miscdev;
 859         misc_device->minor = MISC_DYNAMIC_MINOR;
 860         misc_device->name = kstrdup(name, GFP_KERNEL);
 861         if (!misc_device->name) {
 862                 err = -ENOMEM;
 863                 goto err_release_irq;
 864         }
 865         misc_device->fops = &pci_endpoint_test_fops,
 866
 867         err = misc_register(misc_device);
 868         if (err) {
 869                 dev_err(dev, "Failed to register device\n");
 870                 goto err_kfree_name;
 871         }
 872
 873         return 0;
 874
 875 err_kfree_name:
 876         kfree(misc_device->name);
 877
 878 err_release_irq:
 879         pci_endpoint_test_release_irq(test);
 880
 881 err_kfree_test_name:
 882         kfree(test->name);
 883
 884 err_ida_remove:
 885         ida_simple_remove(&pci_endpoint_test_ida, id);
 886
 887 err_iounmap:
 888         for (bar = 0; bar < PCI_STD_NUM_BARS; bar++) {
 889                 if (test->bar[bar])
 890                         pci_iounmap(pdev, test->bar[bar]);
 891         }
 892
 893 err_disable_irq:
 894         pci_endpoint_test_free_irq_vectors(test);
 895         pci_release_regions(pdev);
 896
 897 err_disable_pdev:
 898         pci_disable_device(pdev);
 899
 900         return err;
 901 }
 902
 903 static void pci_endpoint_test_remove(struct pci_dev *pdev)
 904 {
 905         int id;
 906         enum pci_barno bar;
 907         struct pci_endpoint_test *test = pci_get_drvdata(pdev);
 908         struct miscdevice *misc_device = &test->miscdev;
 909
 910         if (sscanf(misc_device->name, DRV_MODULE_NAME ".%d", &id) != 1)
 911                 return;
 912         if (id < 0)
 913                 return;
 914
 915         misc_deregister(&test->miscdev);
 916         kfree(misc_device->name);
 917         kfree(test->name);
 918         ida_simple_remove(&pci_endpoint_test_ida, id);
 919         for (bar = 0; bar < PCI_STD_NUM_BARS; bar++) {
 920                 if (test->bar[bar])
 921                         pci_iounmap(pdev, test->bar[bar]);
 922         }
 923
 924         pci_endpoint_test_release_irq(test);
 925         pci_endpoint_test_free_irq_vectors(test);
 926
 927         pci_release_regions(pdev);
 928         pci_disable_device(pdev);
 929 }
 930
 931 static const struct pci_endpoint_test_data default_data = {
 932         .test_reg_bar = BAR_0,
 933         .alignment = SZ_4K,
 934         .irq_type = IRQ_TYPE_MSI,
 935 };
 936
 937 static const struct pci_endpoint_test_data am654_data = {
 938         .test_reg_bar = BAR_2,
 939         .alignment = SZ_64K,
 940         .irq_type = IRQ_TYPE_MSI,
 941 };
 942
 943 static const struct pci_endpoint_test_data j721e_data = {
 944         .alignment = 256,
 945         .irq_type = IRQ_TYPE_MSI,
 946 };
 947
 948 static const struct pci_device_id pci_endpoint_test_tbl[] = {
 949         { PCI_DEVICE(PCI_VENDOR_ID_TI, PCI_DEVICE_ID_TI_DRA74x),
 950           .driver_data = (kernel_ulong_t)&default_data,
 951         },
 952         { PCI_DEVICE(PCI_VENDOR_ID_TI, PCI_DEVICE_ID_TI_DRA72x),
 953           .driver_data = (kernel_ulong_t)&default_data,
 954         },
 955         { PCI_DEVICE(PCI_VENDOR_ID_FREESCALE, 0x81c0),
 956           .driver_data = (kernel_ulong_t)&default_data,
 957         },
 958         { PCI_DEVICE(PCI_VENDOR_ID_FREESCALE, PCI_DEVICE_ID_LS1088A),
 959           .driver_data = (kernel_ulong_t)&default_data,
 960         },
 961         { PCI_DEVICE_DATA(SYNOPSYS, EDDA, NULL) },
 962         { PCI_DEVICE(PCI_VENDOR_ID_TI, PCI_DEVICE_ID_TI_AM654),
 963           .driver_data = (kernel_ulong_t)&am654_data
 964         },
 965         { PCI_DEVICE(PCI_VENDOR_ID_RENESAS, PCI_DEVICE_ID_RENESAS_R8A774A1),},
 966         { PCI_DEVICE(PCI_VENDOR_ID_RENESAS, PCI_DEVICE_ID_RENESAS_R8A774B1),},
 967         { PCI_DEVICE(PCI_VENDOR_ID_RENESAS, PCI_DEVICE_ID_RENESAS_R8A774C0),},
 968         { PCI_DEVICE(PCI_VENDOR_ID_RENESAS, PCI_DEVICE_ID_RENESAS_R8A774E1),},
 969         { PCI_DEVICE(PCI_VENDOR_ID_TI, PCI_DEVICE_ID_TI_J721E),
 970           .driver_data = (kernel_ulong_t)&j721e_data,
 971         },
 972         { }
 973 };
 974 MODULE_DEVICE_TABLE(pci, pci_endpoint_test_tbl);
 975
 976 static struct pci_driver pci_endpoint_test_driver = {
 977         .name           = DRV_MODULE_NAME,
 978         .id_table       = pci_endpoint_test_tbl,
 979         .probe          = pci_endpoint_test_probe,
 980         .remove         = pci_endpoint_test_remove,
 981 };
 982 module_pci_driver(pci_endpoint_test_driver);
 983
 984 MODULE_DESCRIPTION("PCI ENDPOINT TEST HOST DRIVER");
 985 MODULE_AUTHOR("Kishon Vijay Abraham I <kishon@ti.com>");
 986 MODULE_LICENSE("GPL v2");