]> git.proxmox.com Git - mirror_ubuntu-artful-kernel.git/blob - drivers/ntb/hw/intel/ntb_hw_intel.c
projects / mirror_ubuntu-artful-kernel.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
UBUNTU: Ubuntu-4.13.0-45.50
[mirror_ubuntu-artful-kernel.git] / drivers / ntb / hw / intel / ntb_hw_intel.c
1 /*
2 * This file is provided under a dual BSD/GPLv2 license. When using or
3 * redistributing this file, you may do so under either license.
4 *
5 * GPL LICENSE SUMMARY
6 *
7 * Copyright(c) 2012 Intel Corporation. All rights reserved.
8 * Copyright (C) 2015 EMC Corporation. All Rights Reserved.
9 * Copyright (C) 2016 T-Platforms. All Rights Reserved.
10 *
11 * This program is free software; you can redistribute it and/or modify
12 * it under the terms of version 2 of the GNU General Public License as
13 * published by the Free Software Foundation.
14 *
15 * BSD LICENSE
16 *
17 * Copyright(c) 2012 Intel Corporation. All rights reserved.
18 * Copyright (C) 2015 EMC Corporation. All Rights Reserved.
19 * Copyright (C) 2016 T-Platforms. All Rights Reserved.
20 *
21 * Redistribution and use in source and binary forms, with or without
22 * modification, are permitted provided that the following conditions
23 * are met:
24 *
25 * * Redistributions of source code must retain the above copyright
26 * notice, this list of conditions and the following disclaimer.
27 * * Redistributions in binary form must reproduce the above copy
28 * notice, this list of conditions and the following disclaimer in
29 * the documentation and/or other materials provided with the
30 * distribution.
31 * * Neither the name of Intel Corporation nor the names of its
32 * contributors may be used to endorse or promote products derived
33 * from this software without specific prior written permission.
34 *
35 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
36 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
37 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
38 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
39 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
40 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
41 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
42 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
43 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
44 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
45 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
46 *
47 * Intel PCIe NTB Linux driver
48 *
49 * Contact Information:
50 * Jon Mason <jon.mason@intel.com>
51 */
52
53 #include <linux/debugfs.h>
54 #include <linux/delay.h>
55 #include <linux/init.h>
56 #include <linux/interrupt.h>
57 #include <linux/module.h>
58 #include <linux/pci.h>
59 #include <linux/random.h>
60 #include <linux/slab.h>
61 #include <linux/ntb.h>
62
63 #include "ntb_hw_intel.h"
64
65 #define NTB_NAME "ntb_hw_intel"
66 #define NTB_DESC "Intel(R) PCI-E Non-Transparent Bridge Driver"
67 #define NTB_VER "2.0"
68
69 MODULE_DESCRIPTION(NTB_DESC);
70 MODULE_VERSION(NTB_VER);
71 MODULE_LICENSE("Dual BSD/GPL");
72 MODULE_AUTHOR("Intel Corporation");
73
74 #define bar0_off(base, bar) ((base) + ((bar) << 2))
75 #define bar2_off(base, bar) bar0_off(base, (bar) - 2)
76
77 static const struct intel_ntb_reg atom_reg;
78 static const struct intel_ntb_alt_reg atom_pri_reg;
79 static const struct intel_ntb_alt_reg atom_sec_reg;
80 static const struct intel_ntb_alt_reg atom_b2b_reg;
81 static const struct intel_ntb_xlat_reg atom_pri_xlat;
82 static const struct intel_ntb_xlat_reg atom_sec_xlat;
83 static const struct intel_ntb_reg xeon_reg;
84 static const struct intel_ntb_alt_reg xeon_pri_reg;
85 static const struct intel_ntb_alt_reg xeon_sec_reg;
86 static const struct intel_ntb_alt_reg xeon_b2b_reg;
87 static const struct intel_ntb_xlat_reg xeon_pri_xlat;
88 static const struct intel_ntb_xlat_reg xeon_sec_xlat;
89 static struct intel_b2b_addr xeon_b2b_usd_addr;
90 static struct intel_b2b_addr xeon_b2b_dsd_addr;
91 static const struct intel_ntb_reg skx_reg;
92 static const struct intel_ntb_alt_reg skx_pri_reg;
93 static const struct intel_ntb_alt_reg skx_b2b_reg;
94 static const struct intel_ntb_xlat_reg skx_sec_xlat;
95 static const struct ntb_dev_ops intel_ntb_ops;
96 static const struct ntb_dev_ops intel_ntb3_ops;
97
98 static const struct file_operations intel_ntb_debugfs_info;
99 static struct dentry *debugfs_dir;
100
101 static int b2b_mw_idx = -1;
102 module_param(b2b_mw_idx, int, 0644);
103 MODULE_PARM_DESC(b2b_mw_idx, "Use this mw idx to access the peer ntb. A "
104 "value of zero or positive starts from first mw idx, and a "
105 "negative value starts from last mw idx. Both sides MUST "
106 "set the same value here!");
107
108 static unsigned int b2b_mw_share;
109 module_param(b2b_mw_share, uint, 0644);
110 MODULE_PARM_DESC(b2b_mw_share, "If the b2b mw is large enough, configure the "
111 "ntb so that the peer ntb only occupies the first half of "
112 "the mw, so the second half can still be used as a mw. Both "
113 "sides MUST set the same value here!");
114
115 module_param_named(xeon_b2b_usd_bar2_addr64,
116 xeon_b2b_usd_addr.bar2_addr64, ullong, 0644);
117 MODULE_PARM_DESC(xeon_b2b_usd_bar2_addr64,
118 "XEON B2B USD BAR 2 64-bit address");
119
120 module_param_named(xeon_b2b_usd_bar4_addr64,
121 xeon_b2b_usd_addr.bar4_addr64, ullong, 0644);
122 MODULE_PARM_DESC(xeon_b2b_usd_bar4_addr64,
123 "XEON B2B USD BAR 4 64-bit address");
124
125 module_param_named(xeon_b2b_usd_bar4_addr32,
126 xeon_b2b_usd_addr.bar4_addr32, ullong, 0644);
127 MODULE_PARM_DESC(xeon_b2b_usd_bar4_addr32,
128 "XEON B2B USD split-BAR 4 32-bit address");
129
130 module_param_named(xeon_b2b_usd_bar5_addr32,
131 xeon_b2b_usd_addr.bar5_addr32, ullong, 0644);
132 MODULE_PARM_DESC(xeon_b2b_usd_bar5_addr32,
133 "XEON B2B USD split-BAR 5 32-bit address");
134
135 module_param_named(xeon_b2b_dsd_bar2_addr64,
136 xeon_b2b_dsd_addr.bar2_addr64, ullong, 0644);
137 MODULE_PARM_DESC(xeon_b2b_dsd_bar2_addr64,
138 "XEON B2B DSD BAR 2 64-bit address");
139
140 module_param_named(xeon_b2b_dsd_bar4_addr64,
141 xeon_b2b_dsd_addr.bar4_addr64, ullong, 0644);
142 MODULE_PARM_DESC(xeon_b2b_dsd_bar4_addr64,
143 "XEON B2B DSD BAR 4 64-bit address");
144
145 module_param_named(xeon_b2b_dsd_bar4_addr32,
146 xeon_b2b_dsd_addr.bar4_addr32, ullong, 0644);
147 MODULE_PARM_DESC(xeon_b2b_dsd_bar4_addr32,
148 "XEON B2B DSD split-BAR 4 32-bit address");
149
150 module_param_named(xeon_b2b_dsd_bar5_addr32,
151 xeon_b2b_dsd_addr.bar5_addr32, ullong, 0644);
152 MODULE_PARM_DESC(xeon_b2b_dsd_bar5_addr32,
153 "XEON B2B DSD split-BAR 5 32-bit address");
154
155 static inline enum ntb_topo xeon_ppd_topo(struct intel_ntb_dev *ndev, u8 ppd);
156 static int xeon_init_isr(struct intel_ntb_dev *ndev);
157
158 #ifndef ioread64
159 #ifdef readq
160 #define ioread64 readq
161 #else
162 #define ioread64 _ioread64
163 static inline u64 _ioread64(void __iomem *mmio)
164 {
165 u64 low, high;
166
167 low = ioread32(mmio);
168 high = ioread32(mmio + sizeof(u32));
169 return low | (high << 32);
170 }
171 #endif
172 #endif
173
174 #ifndef iowrite64
175 #ifdef writeq
176 #define iowrite64 writeq
177 #else
178 #define iowrite64 _iowrite64
179 static inline void _iowrite64(u64 val, void __iomem *mmio)
180 {
181 iowrite32(val, mmio);
182 iowrite32(val >> 32, mmio + sizeof(u32));
183 }
184 #endif
185 #endif
186
187 static inline int pdev_is_atom(struct pci_dev *pdev)
188 {
189 switch (pdev->device) {
190 case PCI_DEVICE_ID_INTEL_NTB_B2B_BWD:
191 return 1;
192 }
193 return 0;
194 }
195
196 static inline int pdev_is_xeon(struct pci_dev *pdev)
197 {
198 switch (pdev->device) {
199 case PCI_DEVICE_ID_INTEL_NTB_SS_JSF:
200 case PCI_DEVICE_ID_INTEL_NTB_SS_SNB:
201 case PCI_DEVICE_ID_INTEL_NTB_SS_IVT:
202 case PCI_DEVICE_ID_INTEL_NTB_SS_HSX:
203 case PCI_DEVICE_ID_INTEL_NTB_SS_BDX:
204 case PCI_DEVICE_ID_INTEL_NTB_PS_JSF:
205 case PCI_DEVICE_ID_INTEL_NTB_PS_SNB:
206 case PCI_DEVICE_ID_INTEL_NTB_PS_IVT:
207 case PCI_DEVICE_ID_INTEL_NTB_PS_HSX:
208 case PCI_DEVICE_ID_INTEL_NTB_PS_BDX:
209 case PCI_DEVICE_ID_INTEL_NTB_B2B_JSF:
210 case PCI_DEVICE_ID_INTEL_NTB_B2B_SNB:
211 case PCI_DEVICE_ID_INTEL_NTB_B2B_IVT:
212 case PCI_DEVICE_ID_INTEL_NTB_B2B_HSX:
213 case PCI_DEVICE_ID_INTEL_NTB_B2B_BDX:
214 return 1;
215 }
216 return 0;
217 }
218
219 static inline int pdev_is_skx_xeon(struct pci_dev *pdev)
220 {
221 if (pdev->device == PCI_DEVICE_ID_INTEL_NTB_B2B_SKX)
222 return 1;
223
224 return 0;
225 }
226
227 static inline void ndev_reset_unsafe_flags(struct intel_ntb_dev *ndev)
228 {
229 ndev->unsafe_flags = 0;
230 ndev->unsafe_flags_ignore = 0;
231
232 /* Only B2B has a workaround to avoid SDOORBELL */
233 if (ndev->hwerr_flags & NTB_HWERR_SDOORBELL_LOCKUP)
234 if (!ntb_topo_is_b2b(ndev->ntb.topo))
235 ndev->unsafe_flags |= NTB_UNSAFE_DB;
236
237 /* No low level workaround to avoid SB01BASE */
238 if (ndev->hwerr_flags & NTB_HWERR_SB01BASE_LOCKUP) {
239 ndev->unsafe_flags |= NTB_UNSAFE_DB;
240 ndev->unsafe_flags |= NTB_UNSAFE_SPAD;
241 }
242 }
243
244 static inline int ndev_is_unsafe(struct intel_ntb_dev *ndev,
245 unsigned long flag)
246 {
247 return !!(flag & ndev->unsafe_flags & ~ndev->unsafe_flags_ignore);
248 }
249
250 static inline int ndev_ignore_unsafe(struct intel_ntb_dev *ndev,
251 unsigned long flag)
252 {
253 flag &= ndev->unsafe_flags;
254 ndev->unsafe_flags_ignore |= flag;
255
256 return !!flag;
257 }
258
259 static int ndev_mw_to_bar(struct intel_ntb_dev *ndev, int idx)
260 {
261 if (idx < 0 || idx >= ndev->mw_count)
262 return -EINVAL;
263 return ndev->reg->mw_bar[idx];
264 }
265
266 static inline int ndev_db_addr(struct intel_ntb_dev *ndev,
267 phys_addr_t *db_addr, resource_size_t *db_size,
268 phys_addr_t reg_addr, unsigned long reg)
269 {
270 if (ndev_is_unsafe(ndev, NTB_UNSAFE_DB))
271 pr_warn_once("%s: NTB unsafe doorbell access", __func__);
272
273 if (db_addr) {
274 *db_addr = reg_addr + reg;
275 dev_dbg(&ndev->ntb.pdev->dev, "Peer db addr %llx\n", *db_addr);
276 }
277
278 if (db_size) {
279 *db_size = ndev->reg->db_size;
280 dev_dbg(&ndev->ntb.pdev->dev, "Peer db size %llx\n", *db_size);
281 }
282
283 return 0;
284 }
285
286 static inline u64 ndev_db_read(struct intel_ntb_dev *ndev,
287 void __iomem *mmio)
288 {
289 if (ndev_is_unsafe(ndev, NTB_UNSAFE_DB))
290 pr_warn_once("%s: NTB unsafe doorbell access", __func__);
291
292 return ndev->reg->db_ioread(mmio);
293 }
294
295 static inline int ndev_db_write(struct intel_ntb_dev *ndev, u64 db_bits,
296 void __iomem *mmio)
297 {
298 if (ndev_is_unsafe(ndev, NTB_UNSAFE_DB))
299 pr_warn_once("%s: NTB unsafe doorbell access", __func__);
300
301 if (db_bits & ~ndev->db_valid_mask)
302 return -EINVAL;
303
304 ndev->reg->db_iowrite(db_bits, mmio);
305
306 return 0;
307 }
308
309 static inline int ndev_db_set_mask(struct intel_ntb_dev *ndev, u64 db_bits,
310 void __iomem *mmio)
311 {
312 unsigned long irqflags;
313
314 if (ndev_is_unsafe(ndev, NTB_UNSAFE_DB))
315 pr_warn_once("%s: NTB unsafe doorbell access", __func__);
316
317 if (db_bits & ~ndev->db_valid_mask)
318 return -EINVAL;
319
320 spin_lock_irqsave(&ndev->db_mask_lock, irqflags);
321 {
322 ndev->db_mask |= db_bits;
323 ndev->reg->db_iowrite(ndev->db_mask, mmio);
324 }
325 spin_unlock_irqrestore(&ndev->db_mask_lock, irqflags);
326
327 return 0;
328 }
329
330 static inline int ndev_db_clear_mask(struct intel_ntb_dev *ndev, u64 db_bits,
331 void __iomem *mmio)
332 {
333 unsigned long irqflags;
334
335 if (ndev_is_unsafe(ndev, NTB_UNSAFE_DB))
336 pr_warn_once("%s: NTB unsafe doorbell access", __func__);
337
338 if (db_bits & ~ndev->db_valid_mask)
339 return -EINVAL;
340
341 spin_lock_irqsave(&ndev->db_mask_lock, irqflags);
342 {
343 ndev->db_mask &= ~db_bits;
344 ndev->reg->db_iowrite(ndev->db_mask, mmio);
345 }
346 spin_unlock_irqrestore(&ndev->db_mask_lock, irqflags);
347
348 return 0;
349 }
350
351 static inline int ndev_vec_mask(struct intel_ntb_dev *ndev, int db_vector)
352 {
353 u64 shift, mask;
354
355 shift = ndev->db_vec_shift;
356 mask = BIT_ULL(shift) - 1;
357
358 return mask << (shift * db_vector);
359 }
360
361 static inline int ndev_spad_addr(struct intel_ntb_dev *ndev, int idx,
362 phys_addr_t *spad_addr, phys_addr_t reg_addr,
363 unsigned long reg)
364 {
365 if (ndev_is_unsafe(ndev, NTB_UNSAFE_SPAD))
366 pr_warn_once("%s: NTB unsafe scratchpad access", __func__);
367
368 if (idx < 0 || idx >= ndev->spad_count)
369 return -EINVAL;
370
371 if (spad_addr) {
372 *spad_addr = reg_addr + reg + (idx << 2);
373 dev_dbg(&ndev->ntb.pdev->dev, "Peer spad addr %llx\n",
374 *spad_addr);
375 }
376
377 return 0;
378 }
379
380 static inline u32 ndev_spad_read(struct intel_ntb_dev *ndev, int idx,
381 void __iomem *mmio)
382 {
383 if (ndev_is_unsafe(ndev, NTB_UNSAFE_SPAD))
384 pr_warn_once("%s: NTB unsafe scratchpad access", __func__);
385
386 if (idx < 0 || idx >= ndev->spad_count)
387 return 0;
388
389 return ioread32(mmio + (idx << 2));
390 }
391
392 static inline int ndev_spad_write(struct intel_ntb_dev *ndev, int idx, u32 val,
393 void __iomem *mmio)
394 {
395 if (ndev_is_unsafe(ndev, NTB_UNSAFE_SPAD))
396 pr_warn_once("%s: NTB unsafe scratchpad access", __func__);
397
398 if (idx < 0 || idx >= ndev->spad_count)
399 return -EINVAL;
400
401 iowrite32(val, mmio + (idx << 2));
402
403 return 0;
404 }
405
406 static irqreturn_t ndev_interrupt(struct intel_ntb_dev *ndev, int vec)
407 {
408 u64 vec_mask;
409
410 vec_mask = ndev_vec_mask(ndev, vec);
411
412 if ((ndev->hwerr_flags & NTB_HWERR_MSIX_VECTOR32_BAD) && (vec == 31))
413 vec_mask |= ndev->db_link_mask;
414
415 dev_dbg(&ndev->ntb.pdev->dev, "vec %d vec_mask %llx\n", vec, vec_mask);
416
417 ndev->last_ts = jiffies;
418
419 if (vec_mask & ndev->db_link_mask) {
420 if (ndev->reg->poll_link(ndev))
421 ntb_link_event(&ndev->ntb);
422 }
423
424 if (vec_mask & ndev->db_valid_mask)
425 ntb_db_event(&ndev->ntb, vec);
426
427 return IRQ_HANDLED;
428 }
429
430 static irqreturn_t ndev_vec_isr(int irq, void *dev)
431 {
432 struct intel_ntb_vec *nvec = dev;
433
434 dev_dbg(&nvec->ndev->ntb.pdev->dev, "irq: %d nvec->num: %d\n",
435 irq, nvec->num);
436
437 return ndev_interrupt(nvec->ndev, nvec->num);
438 }
439
440 static irqreturn_t ndev_irq_isr(int irq, void *dev)
441 {
442 struct intel_ntb_dev *ndev = dev;
443
444 return ndev_interrupt(ndev, irq - ndev->ntb.pdev->irq);
445 }
446
447 static int ndev_init_isr(struct intel_ntb_dev *ndev,
448 int msix_min, int msix_max,
449 int msix_shift, int total_shift)
450 {
451 struct pci_dev *pdev;
452 int rc, i, msix_count, node;
453
454 pdev = ndev->ntb.pdev;
455
456 node = dev_to_node(&pdev->dev);
457
458 /* Mask all doorbell interrupts */
459 ndev->db_mask = ndev->db_valid_mask;
460 ndev->reg->db_iowrite(ndev->db_mask,
461 ndev->self_mmio +
462 ndev->self_reg->db_mask);
463
464 /* Try to set up msix irq */
465
466 ndev->vec = kzalloc_node(msix_max * sizeof(*ndev->vec),
467 GFP_KERNEL, node);
468 if (!ndev->vec)
469 goto err_msix_vec_alloc;
470
471 ndev->msix = kzalloc_node(msix_max * sizeof(*ndev->msix),
472 GFP_KERNEL, node);
473 if (!ndev->msix)
474 goto err_msix_alloc;
475
476 for (i = 0; i < msix_max; ++i)
477 ndev->msix[i].entry = i;
478
479 msix_count = pci_enable_msix_range(pdev, ndev->msix,
480 msix_min, msix_max);
481 if (msix_count < 0)
482 goto err_msix_enable;
483
484 for (i = 0; i < msix_count; ++i) {
485 ndev->vec[i].ndev = ndev;
486 ndev->vec[i].num = i;
487 rc = request_irq(ndev->msix[i].vector, ndev_vec_isr, 0,
488 "ndev_vec_isr", &ndev->vec[i]);
489 if (rc)
490 goto err_msix_request;
491 }
492
493 dev_dbg(&pdev->dev, "Using %d msix interrupts\n", msix_count);
494 ndev->db_vec_count = msix_count;
495 ndev->db_vec_shift = msix_shift;
496 return 0;
497
498 err_msix_request:
499 while (i-- > 0)
500 free_irq(ndev->msix[i].vector, &ndev->vec[i]);
501 pci_disable_msix(pdev);
502 err_msix_enable:
503 kfree(ndev->msix);
504 err_msix_alloc:
505 kfree(ndev->vec);
506 err_msix_vec_alloc:
507 ndev->msix = NULL;
508 ndev->vec = NULL;
509
510 /* Try to set up msi irq */
511
512 rc = pci_enable_msi(pdev);
513 if (rc)
514 goto err_msi_enable;
515
516 rc = request_irq(pdev->irq, ndev_irq_isr, 0,
517 "ndev_irq_isr", ndev);
518 if (rc)
519 goto err_msi_request;
520
521 dev_dbg(&pdev->dev, "Using msi interrupts\n");
522 ndev->db_vec_count = 1;
523 ndev->db_vec_shift = total_shift;
524 return 0;
525
526 err_msi_request:
527 pci_disable_msi(pdev);
528 err_msi_enable:
529
530 /* Try to set up intx irq */
531
532 pci_intx(pdev, 1);
533
534 rc = request_irq(pdev->irq, ndev_irq_isr, IRQF_SHARED,
535 "ndev_irq_isr", ndev);
536 if (rc)
537 goto err_intx_request;
538
539 dev_dbg(&pdev->dev, "Using intx interrupts\n");
540 ndev->db_vec_count = 1;
541 ndev->db_vec_shift = total_shift;
542 return 0;
543
544 err_intx_request:
545 return rc;
546 }
547
548 static void ndev_deinit_isr(struct intel_ntb_dev *ndev)
549 {
550 struct pci_dev *pdev;
551 int i;
552
553 pdev = ndev->ntb.pdev;
554
555 /* Mask all doorbell interrupts */
556 ndev->db_mask = ndev->db_valid_mask;
557 ndev->reg->db_iowrite(ndev->db_mask,
558 ndev->self_mmio +
559 ndev->self_reg->db_mask);
560
561 if (ndev->msix) {
562 i = ndev->db_vec_count;
563 while (i--)
564 free_irq(ndev->msix[i].vector, &ndev->vec[i]);
565 pci_disable_msix(pdev);
566 kfree(ndev->msix);
567 kfree(ndev->vec);
568 } else {
569 free_irq(pdev->irq, ndev);
570 if (pci_dev_msi_enabled(pdev))
571 pci_disable_msi(pdev);
572 }
573 }
574
575 static ssize_t ndev_ntb3_debugfs_read(struct file *filp, char __user *ubuf,
576 size_t count, loff_t *offp)
577 {
578 struct intel_ntb_dev *ndev;
579 void __iomem *mmio;
580 char *buf;
581 size_t buf_size;
582 ssize_t ret, off;
583 union { u64 v64; u32 v32; u16 v16; } u;
584
585 ndev = filp->private_data;
586 mmio = ndev->self_mmio;
587
588 buf_size = min(count, 0x800ul);
589
590 buf = kmalloc(buf_size, GFP_KERNEL);
591 if (!buf)
592 return -ENOMEM;
593
594 off = 0;
595
596 off += scnprintf(buf + off, buf_size - off,
597 "NTB Device Information:\n");
598
599 off += scnprintf(buf + off, buf_size - off,
600 "Connection Topology -\t%s\n",
601 ntb_topo_string(ndev->ntb.topo));
602
603 off += scnprintf(buf + off, buf_size - off,
604 "NTB CTL -\t\t%#06x\n", ndev->ntb_ctl);
605 off += scnprintf(buf + off, buf_size - off,
606 "LNK STA -\t\t%#06x\n", ndev->lnk_sta);
607
608 if (!ndev->reg->link_is_up(ndev))
609 off += scnprintf(buf + off, buf_size - off,
610 "Link Status -\t\tDown\n");
611 else {
612 off += scnprintf(buf + off, buf_size - off,
613 "Link Status -\t\tUp\n");
614 off += scnprintf(buf + off, buf_size - off,
615 "Link Speed -\t\tPCI-E Gen %u\n",
616 NTB_LNK_STA_SPEED(ndev->lnk_sta));
617 off += scnprintf(buf + off, buf_size - off,
618 "Link Width -\t\tx%u\n",
619 NTB_LNK_STA_WIDTH(ndev->lnk_sta));
620 }
621
622 off += scnprintf(buf + off, buf_size - off,
623 "Memory Window Count -\t%u\n", ndev->mw_count);
624 off += scnprintf(buf + off, buf_size - off,
625 "Scratchpad Count -\t%u\n", ndev->spad_count);
626 off += scnprintf(buf + off, buf_size - off,
627 "Doorbell Count -\t%u\n", ndev->db_count);
628 off += scnprintf(buf + off, buf_size - off,
629 "Doorbell Vector Count -\t%u\n", ndev->db_vec_count);
630 off += scnprintf(buf + off, buf_size - off,
631 "Doorbell Vector Shift -\t%u\n", ndev->db_vec_shift);
632
633 off += scnprintf(buf + off, buf_size - off,
634 "Doorbell Valid Mask -\t%#llx\n", ndev->db_valid_mask);
635 off += scnprintf(buf + off, buf_size - off,
636 "Doorbell Link Mask -\t%#llx\n", ndev->db_link_mask);
637 off += scnprintf(buf + off, buf_size - off,
638 "Doorbell Mask Cached -\t%#llx\n", ndev->db_mask);
639
640 u.v64 = ndev_db_read(ndev, mmio + ndev->self_reg->db_mask);
641 off += scnprintf(buf + off, buf_size - off,
642 "Doorbell Mask -\t\t%#llx\n", u.v64);
643
644 u.v64 = ndev_db_read(ndev, mmio + ndev->self_reg->db_bell);
645 off += scnprintf(buf + off, buf_size - off,
646 "Doorbell Bell -\t\t%#llx\n", u.v64);
647
648 off += scnprintf(buf + off, buf_size - off,
649 "\nNTB Incoming XLAT:\n");
650
651 u.v64 = ioread64(mmio + SKX_IMBAR1XBASE_OFFSET);
652 off += scnprintf(buf + off, buf_size - off,
653 "IMBAR1XBASE -\t\t%#018llx\n", u.v64);
654
655 u.v64 = ioread64(mmio + SKX_IMBAR2XBASE_OFFSET);
656 off += scnprintf(buf + off, buf_size - off,
657 "IMBAR2XBASE -\t\t%#018llx\n", u.v64);
658
659 u.v64 = ioread64(mmio + SKX_IMBAR1XLMT_OFFSET);
660 off += scnprintf(buf + off, buf_size - off,
661 "IMBAR1XLMT -\t\t\t%#018llx\n", u.v64);
662
663 u.v64 = ioread64(mmio + SKX_IMBAR2XLMT_OFFSET);
664 off += scnprintf(buf + off, buf_size - off,
665 "IMBAR2XLMT -\t\t\t%#018llx\n", u.v64);
666
667 if (ntb_topo_is_b2b(ndev->ntb.topo)) {
668 off += scnprintf(buf + off, buf_size - off,
669 "\nNTB Outgoing B2B XLAT:\n");
670
671 u.v64 = ioread64(mmio + SKX_EMBAR1XBASE_OFFSET);
672 off += scnprintf(buf + off, buf_size - off,
673 "EMBAR1XBASE -\t\t%#018llx\n", u.v64);
674
675 u.v64 = ioread64(mmio + SKX_EMBAR2XBASE_OFFSET);
676 off += scnprintf(buf + off, buf_size - off,
677 "EMBAR2XBASE -\t\t%#018llx\n", u.v64);
678
679 u.v64 = ioread64(mmio + SKX_EMBAR1XLMT_OFFSET);
680 off += scnprintf(buf + off, buf_size - off,
681 "EMBAR1XLMT -\t\t%#018llx\n", u.v64);
682
683 u.v64 = ioread64(mmio + SKX_EMBAR2XLMT_OFFSET);
684 off += scnprintf(buf + off, buf_size - off,
685 "EMBAR2XLMT -\t\t%#018llx\n", u.v64);
686
687 off += scnprintf(buf + off, buf_size - off,
688 "\nNTB Secondary BAR:\n");
689
690 u.v64 = ioread64(mmio + SKX_EMBAR0_OFFSET);
691 off += scnprintf(buf + off, buf_size - off,
692 "EMBAR0 -\t\t%#018llx\n", u.v64);
693
694 u.v64 = ioread64(mmio + SKX_EMBAR1_OFFSET);
695 off += scnprintf(buf + off, buf_size - off,
696 "EMBAR1 -\t\t%#018llx\n", u.v64);
697
698 u.v64 = ioread64(mmio + SKX_EMBAR2_OFFSET);
699 off += scnprintf(buf + off, buf_size - off,
700 "EMBAR2 -\t\t%#018llx\n", u.v64);
701 }
702
703 off += scnprintf(buf + off, buf_size - off,
704 "\nNTB Statistics:\n");
705
706 u.v16 = ioread16(mmio + SKX_USMEMMISS_OFFSET);
707 off += scnprintf(buf + off, buf_size - off,
708 "Upstream Memory Miss -\t%u\n", u.v16);
709
710 off += scnprintf(buf + off, buf_size - off,
711 "\nNTB Hardware Errors:\n");
712
713 if (!pci_read_config_word(ndev->ntb.pdev,
714 SKX_DEVSTS_OFFSET, &u.v16))
715 off += scnprintf(buf + off, buf_size - off,
716 "DEVSTS -\t\t%#06x\n", u.v16);
717
718 if (!pci_read_config_word(ndev->ntb.pdev,
719 SKX_LINK_STATUS_OFFSET, &u.v16))
720 off += scnprintf(buf + off, buf_size - off,
721 "LNKSTS -\t\t%#06x\n", u.v16);
722
723 if (!pci_read_config_dword(ndev->ntb.pdev,
724 SKX_UNCERRSTS_OFFSET, &u.v32))
725 off += scnprintf(buf + off, buf_size - off,
726 "UNCERRSTS -\t\t%#06x\n", u.v32);
727
728 if (!pci_read_config_dword(ndev->ntb.pdev,
729 SKX_CORERRSTS_OFFSET, &u.v32))
730 off += scnprintf(buf + off, buf_size - off,
731 "CORERRSTS -\t\t%#06x\n", u.v32);
732
733 ret = simple_read_from_buffer(ubuf, count, offp, buf, off);
734 kfree(buf);
735 return ret;
736 }
737
738 static ssize_t ndev_ntb_debugfs_read(struct file *filp, char __user *ubuf,
739 size_t count, loff_t *offp)
740 {
741 struct intel_ntb_dev *ndev;
742 struct pci_dev *pdev;
743 void __iomem *mmio;
744 char *buf;
745 size_t buf_size;
746 ssize_t ret, off;
747 union { u64 v64; u32 v32; u16 v16; u8 v8; } u;
748
749 ndev = filp->private_data;
750 pdev = ndev->ntb.pdev;
751 mmio = ndev->self_mmio;
752
753 buf_size = min(count, 0x800ul);
754
755 buf = kmalloc(buf_size, GFP_KERNEL);
756 if (!buf)
757 return -ENOMEM;
758
759 off = 0;
760
761 off += scnprintf(buf + off, buf_size - off,
762 "NTB Device Information:\n");
763
764 off += scnprintf(buf + off, buf_size - off,
765 "Connection Topology -\t%s\n",
766 ntb_topo_string(ndev->ntb.topo));
767
768 if (ndev->b2b_idx != UINT_MAX) {
769 off += scnprintf(buf + off, buf_size - off,
770 "B2B MW Idx -\t\t%u\n", ndev->b2b_idx);
771 off += scnprintf(buf + off, buf_size - off,
772 "B2B Offset -\t\t%#lx\n", ndev->b2b_off);
773 }
774
775 off += scnprintf(buf + off, buf_size - off,
776 "BAR4 Split -\t\t%s\n",
777 ndev->bar4_split ? "yes" : "no");
778
779 off += scnprintf(buf + off, buf_size - off,
780 "NTB CTL -\t\t%#06x\n", ndev->ntb_ctl);
781 off += scnprintf(buf + off, buf_size - off,
782 "LNK STA -\t\t%#06x\n", ndev->lnk_sta);
783
784 if (!ndev->reg->link_is_up(ndev)) {
785 off += scnprintf(buf + off, buf_size - off,
786 "Link Status -\t\tDown\n");
787 } else {
788 off += scnprintf(buf + off, buf_size - off,
789 "Link Status -\t\tUp\n");
790 off += scnprintf(buf + off, buf_size - off,
791 "Link Speed -\t\tPCI-E Gen %u\n",
792 NTB_LNK_STA_SPEED(ndev->lnk_sta));
793 off += scnprintf(buf + off, buf_size - off,
794 "Link Width -\t\tx%u\n",
795 NTB_LNK_STA_WIDTH(ndev->lnk_sta));
796 }
797
798 off += scnprintf(buf + off, buf_size - off,
799 "Memory Window Count -\t%u\n", ndev->mw_count);
800 off += scnprintf(buf + off, buf_size - off,
801 "Scratchpad Count -\t%u\n", ndev->spad_count);
802 off += scnprintf(buf + off, buf_size - off,
803 "Doorbell Count -\t%u\n", ndev->db_count);
804 off += scnprintf(buf + off, buf_size - off,
805 "Doorbell Vector Count -\t%u\n", ndev->db_vec_count);
806 off += scnprintf(buf + off, buf_size - off,
807 "Doorbell Vector Shift -\t%u\n", ndev->db_vec_shift);
808
809 off += scnprintf(buf + off, buf_size - off,
810 "Doorbell Valid Mask -\t%#llx\n", ndev->db_valid_mask);
811 off += scnprintf(buf + off, buf_size - off,
812 "Doorbell Link Mask -\t%#llx\n", ndev->db_link_mask);
813 off += scnprintf(buf + off, buf_size - off,
814 "Doorbell Mask Cached -\t%#llx\n", ndev->db_mask);
815
816 u.v64 = ndev_db_read(ndev, mmio + ndev->self_reg->db_mask);
817 off += scnprintf(buf + off, buf_size - off,
818 "Doorbell Mask -\t\t%#llx\n", u.v64);
819
820 u.v64 = ndev_db_read(ndev, mmio + ndev->self_reg->db_bell);
821 off += scnprintf(buf + off, buf_size - off,
822 "Doorbell Bell -\t\t%#llx\n", u.v64);
823
824 off += scnprintf(buf + off, buf_size - off,
825 "\nNTB Window Size:\n");
826
827 pci_read_config_byte(pdev, XEON_PBAR23SZ_OFFSET, &u.v8);
828 off += scnprintf(buf + off, buf_size - off,
829 "PBAR23SZ %hhu\n", u.v8);
830 if (!ndev->bar4_split) {
831 pci_read_config_byte(pdev, XEON_PBAR45SZ_OFFSET, &u.v8);
832 off += scnprintf(buf + off, buf_size - off,
833 "PBAR45SZ %hhu\n", u.v8);
834 } else {
835 pci_read_config_byte(pdev, XEON_PBAR4SZ_OFFSET, &u.v8);
836 off += scnprintf(buf + off, buf_size - off,
837 "PBAR4SZ %hhu\n", u.v8);
838 pci_read_config_byte(pdev, XEON_PBAR5SZ_OFFSET, &u.v8);
839 off += scnprintf(buf + off, buf_size - off,
840 "PBAR5SZ %hhu\n", u.v8);
841 }
842
843 pci_read_config_byte(pdev, XEON_SBAR23SZ_OFFSET, &u.v8);
844 off += scnprintf(buf + off, buf_size - off,
845 "SBAR23SZ %hhu\n", u.v8);
846 if (!ndev->bar4_split) {
847 pci_read_config_byte(pdev, XEON_SBAR45SZ_OFFSET, &u.v8);
848 off += scnprintf(buf + off, buf_size - off,
849 "SBAR45SZ %hhu\n", u.v8);
850 } else {
851 pci_read_config_byte(pdev, XEON_SBAR4SZ_OFFSET, &u.v8);
852 off += scnprintf(buf + off, buf_size - off,
853 "SBAR4SZ %hhu\n", u.v8);
854 pci_read_config_byte(pdev, XEON_SBAR5SZ_OFFSET, &u.v8);
855 off += scnprintf(buf + off, buf_size - off,
856 "SBAR5SZ %hhu\n", u.v8);
857 }
858
859 off += scnprintf(buf + off, buf_size - off,
860 "\nNTB Incoming XLAT:\n");
861
862 u.v64 = ioread64(mmio + bar2_off(ndev->xlat_reg->bar2_xlat, 2));
863 off += scnprintf(buf + off, buf_size - off,
864 "XLAT23 -\t\t%#018llx\n", u.v64);
865
866 if (ndev->bar4_split) {
867 u.v32 = ioread32(mmio + bar2_off(ndev->xlat_reg->bar2_xlat, 4));
868 off += scnprintf(buf + off, buf_size - off,
869 "XLAT4 -\t\t\t%#06x\n", u.v32);
870
871 u.v32 = ioread32(mmio + bar2_off(ndev->xlat_reg->bar2_xlat, 5));
872 off += scnprintf(buf + off, buf_size - off,
873 "XLAT5 -\t\t\t%#06x\n", u.v32);
874 } else {
875 u.v64 = ioread64(mmio + bar2_off(ndev->xlat_reg->bar2_xlat, 4));
876 off += scnprintf(buf + off, buf_size - off,
877 "XLAT45 -\t\t%#018llx\n", u.v64);
878 }
879
880 u.v64 = ioread64(mmio + bar2_off(ndev->xlat_reg->bar2_limit, 2));
881 off += scnprintf(buf + off, buf_size - off,
882 "LMT23 -\t\t\t%#018llx\n", u.v64);
883
884 if (ndev->bar4_split) {
885 u.v32 = ioread32(mmio + bar2_off(ndev->xlat_reg->bar2_limit, 4));
886 off += scnprintf(buf + off, buf_size - off,
887 "LMT4 -\t\t\t%#06x\n", u.v32);
888 u.v32 = ioread32(mmio + bar2_off(ndev->xlat_reg->bar2_limit, 5));
889 off += scnprintf(buf + off, buf_size - off,
890 "LMT5 -\t\t\t%#06x\n", u.v32);
891 } else {
892 u.v64 = ioread64(mmio + bar2_off(ndev->xlat_reg->bar2_limit, 4));
893 off += scnprintf(buf + off, buf_size - off,
894 "LMT45 -\t\t\t%#018llx\n", u.v64);
895 }
896
897 if (pdev_is_xeon(pdev)) {
898 if (ntb_topo_is_b2b(ndev->ntb.topo)) {
899 off += scnprintf(buf + off, buf_size - off,
900 "\nNTB Outgoing B2B XLAT:\n");
901
902 u.v64 = ioread64(mmio + XEON_PBAR23XLAT_OFFSET);
903 off += scnprintf(buf + off, buf_size - off,
904 "B2B XLAT23 -\t\t%#018llx\n", u.v64);
905
906 if (ndev->bar4_split) {
907 u.v32 = ioread32(mmio + XEON_PBAR4XLAT_OFFSET);
908 off += scnprintf(buf + off, buf_size - off,
909 "B2B XLAT4 -\t\t%#06x\n",
910 u.v32);
911 u.v32 = ioread32(mmio + XEON_PBAR5XLAT_OFFSET);
912 off += scnprintf(buf + off, buf_size - off,
913 "B2B XLAT5 -\t\t%#06x\n",
914 u.v32);
915 } else {
916 u.v64 = ioread64(mmio + XEON_PBAR45XLAT_OFFSET);
917 off += scnprintf(buf + off, buf_size - off,
918 "B2B XLAT45 -\t\t%#018llx\n",
919 u.v64);
920 }
921
922 u.v64 = ioread64(mmio + XEON_PBAR23LMT_OFFSET);
923 off += scnprintf(buf + off, buf_size - off,
924 "B2B LMT23 -\t\t%#018llx\n", u.v64);
925
926 if (ndev->bar4_split) {
927 u.v32 = ioread32(mmio + XEON_PBAR4LMT_OFFSET);
928 off += scnprintf(buf + off, buf_size - off,
929 "B2B LMT4 -\t\t%#06x\n",
930 u.v32);
931 u.v32 = ioread32(mmio + XEON_PBAR5LMT_OFFSET);
932 off += scnprintf(buf + off, buf_size - off,
933 "B2B LMT5 -\t\t%#06x\n",
934 u.v32);
935 } else {
936 u.v64 = ioread64(mmio + XEON_PBAR45LMT_OFFSET);
937 off += scnprintf(buf + off, buf_size - off,
938 "B2B LMT45 -\t\t%#018llx\n",
939 u.v64);
940 }
941
942 off += scnprintf(buf + off, buf_size - off,
943 "\nNTB Secondary BAR:\n");
944
945 u.v64 = ioread64(mmio + XEON_SBAR0BASE_OFFSET);
946 off += scnprintf(buf + off, buf_size - off,
947 "SBAR01 -\t\t%#018llx\n", u.v64);
948
949 u.v64 = ioread64(mmio + XEON_SBAR23BASE_OFFSET);
950 off += scnprintf(buf + off, buf_size - off,
951 "SBAR23 -\t\t%#018llx\n", u.v64);
952
953 if (ndev->bar4_split) {
954 u.v32 = ioread32(mmio + XEON_SBAR4BASE_OFFSET);
955 off += scnprintf(buf + off, buf_size - off,
956 "SBAR4 -\t\t\t%#06x\n", u.v32);
957 u.v32 = ioread32(mmio + XEON_SBAR5BASE_OFFSET);
958 off += scnprintf(buf + off, buf_size - off,
959 "SBAR5 -\t\t\t%#06x\n", u.v32);
960 } else {
961 u.v64 = ioread64(mmio + XEON_SBAR45BASE_OFFSET);
962 off += scnprintf(buf + off, buf_size - off,
963 "SBAR45 -\t\t%#018llx\n",
964 u.v64);
965 }
966 }
967
968 off += scnprintf(buf + off, buf_size - off,
969 "\nXEON NTB Statistics:\n");
970
971 u.v16 = ioread16(mmio + XEON_USMEMMISS_OFFSET);
972 off += scnprintf(buf + off, buf_size - off,
973 "Upstream Memory Miss -\t%u\n", u.v16);
974
975 off += scnprintf(buf + off, buf_size - off,
976 "\nXEON NTB Hardware Errors:\n");
977
978 if (!pci_read_config_word(pdev,
979 XEON_DEVSTS_OFFSET, &u.v16))
980 off += scnprintf(buf + off, buf_size - off,
981 "DEVSTS -\t\t%#06x\n", u.v16);
982
983 if (!pci_read_config_word(pdev,
984 XEON_LINK_STATUS_OFFSET, &u.v16))
985 off += scnprintf(buf + off, buf_size - off,
986 "LNKSTS -\t\t%#06x\n", u.v16);
987
988 if (!pci_read_config_dword(pdev,
989 XEON_UNCERRSTS_OFFSET, &u.v32))
990 off += scnprintf(buf + off, buf_size - off,
991 "UNCERRSTS -\t\t%#06x\n", u.v32);
992
993 if (!pci_read_config_dword(pdev,
994 XEON_CORERRSTS_OFFSET, &u.v32))
995 off += scnprintf(buf + off, buf_size - off,
996 "CORERRSTS -\t\t%#06x\n", u.v32);
997 }
998
999 ret = simple_read_from_buffer(ubuf, count, offp, buf, off);
1000 kfree(buf);
1001 return ret;
1002 }
1003
1004 static ssize_t ndev_debugfs_read(struct file *filp, char __user *ubuf,
1005 size_t count, loff_t *offp)
1006 {
1007 struct intel_ntb_dev *ndev = filp->private_data;
1008
1009 if (pdev_is_xeon(ndev->ntb.pdev) ||
1010 pdev_is_atom(ndev->ntb.pdev))
1011 return ndev_ntb_debugfs_read(filp, ubuf, count, offp);
1012 else if (pdev_is_skx_xeon(ndev->ntb.pdev))
1013 return ndev_ntb3_debugfs_read(filp, ubuf, count, offp);
1014
1015 return -ENXIO;
1016 }
1017
1018 static void ndev_init_debugfs(struct intel_ntb_dev *ndev)
1019 {
1020 if (!debugfs_dir) {
1021 ndev->debugfs_dir = NULL;
1022 ndev->debugfs_info = NULL;
1023 } else {
1024 ndev->debugfs_dir =
1025 debugfs_create_dir(pci_name(ndev->ntb.pdev),
1026 debugfs_dir);
1027 if (!ndev->debugfs_dir)
1028 ndev->debugfs_info = NULL;
1029 else
1030 ndev->debugfs_info =
1031 debugfs_create_file("info", S_IRUSR,
1032 ndev->debugfs_dir, ndev,
1033 &intel_ntb_debugfs_info);
1034 }
1035 }
1036
1037 static void ndev_deinit_debugfs(struct intel_ntb_dev *ndev)
1038 {
1039 debugfs_remove_recursive(ndev->debugfs_dir);
1040 }
1041
1042 static int intel_ntb_mw_count(struct ntb_dev *ntb, int pidx)
1043 {
1044 if (pidx != NTB_DEF_PEER_IDX)
1045 return -EINVAL;
1046
1047 return ntb_ndev(ntb)->mw_count;
1048 }
1049
1050 static int intel_ntb_mw_get_align(struct ntb_dev *ntb, int pidx, int idx,
1051 resource_size_t *addr_align,
1052 resource_size_t *size_align,
1053 resource_size_t *size_max)
1054 {
1055 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1056 resource_size_t bar_size, mw_size;
1057 int bar;
1058
1059 if (pidx != NTB_DEF_PEER_IDX)
1060 return -EINVAL;
1061
1062 if (idx >= ndev->b2b_idx && !ndev->b2b_off)
1063 idx += 1;
1064
1065 bar = ndev_mw_to_bar(ndev, idx);
1066 if (bar < 0)
1067 return bar;
1068
1069 bar_size = pci_resource_len(ndev->ntb.pdev, bar);
1070
1071 if (idx == ndev->b2b_idx)
1072 mw_size = bar_size - ndev->b2b_off;
1073 else
1074 mw_size = bar_size;
1075
1076 if (addr_align)
1077 *addr_align = pci_resource_len(ndev->ntb.pdev, bar);
1078
1079 if (size_align)
1080 *size_align = 1;
1081
1082 if (size_max)
1083 *size_max = mw_size;
1084
1085 return 0;
1086 }
1087
1088 static int intel_ntb_mw_set_trans(struct ntb_dev *ntb, int pidx, int idx,
1089 dma_addr_t addr, resource_size_t size)
1090 {
1091 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1092 unsigned long base_reg, xlat_reg, limit_reg;
1093 resource_size_t bar_size, mw_size;
1094 void __iomem *mmio;
1095 u64 base, limit, reg_val;
1096 int bar;
1097
1098 if (pidx != NTB_DEF_PEER_IDX)
1099 return -EINVAL;
1100
1101 if (idx >= ndev->b2b_idx && !ndev->b2b_off)
1102 idx += 1;
1103
1104 bar = ndev_mw_to_bar(ndev, idx);
1105 if (bar < 0)
1106 return bar;
1107
1108 bar_size = pci_resource_len(ndev->ntb.pdev, bar);
1109
1110 if (idx == ndev->b2b_idx)
1111 mw_size = bar_size - ndev->b2b_off;
1112 else
1113 mw_size = bar_size;
1114
1115 /* hardware requires that addr is aligned to bar size */
1116 if (addr & (bar_size - 1))
1117 return -EINVAL;
1118
1119 /* make sure the range fits in the usable mw size */
1120 if (size > mw_size)
1121 return -EINVAL;
1122
1123 mmio = ndev->self_mmio;
1124 base_reg = bar0_off(ndev->xlat_reg->bar0_base, bar);
1125 xlat_reg = bar2_off(ndev->xlat_reg->bar2_xlat, bar);
1126 limit_reg = bar2_off(ndev->xlat_reg->bar2_limit, bar);
1127
1128 if (bar < 4 || !ndev->bar4_split) {
1129 base = ioread64(mmio + base_reg) & NTB_BAR_MASK_64;
1130
1131 /* Set the limit if supported, if size is not mw_size */
1132 if (limit_reg && size != mw_size)
1133 limit = base + size;
1134 else
1135 limit = 0;
1136
1137 /* set and verify setting the translation address */
1138 iowrite64(addr, mmio + xlat_reg);
1139 reg_val = ioread64(mmio + xlat_reg);
1140 if (reg_val != addr) {
1141 iowrite64(0, mmio + xlat_reg);
1142 return -EIO;
1143 }
1144
1145 /* set and verify setting the limit */
1146 iowrite64(limit, mmio + limit_reg);
1147 reg_val = ioread64(mmio + limit_reg);
1148 if (reg_val != limit) {
1149 iowrite64(base, mmio + limit_reg);
1150 iowrite64(0, mmio + xlat_reg);
1151 return -EIO;
1152 }
1153 } else {
1154 /* split bar addr range must all be 32 bit */
1155 if (addr & (~0ull << 32))
1156 return -EINVAL;
1157 if ((addr + size) & (~0ull << 32))
1158 return -EINVAL;
1159
1160 base = ioread32(mmio + base_reg) & NTB_BAR_MASK_32;
1161
1162 /* Set the limit if supported, if size is not mw_size */
1163 if (limit_reg && size != mw_size)
1164 limit = base + size;
1165 else
1166 limit = 0;
1167
1168 /* set and verify setting the translation address */
1169 iowrite32(addr, mmio + xlat_reg);
1170 reg_val = ioread32(mmio + xlat_reg);
1171 if (reg_val != addr) {
1172 iowrite32(0, mmio + xlat_reg);
1173 return -EIO;
1174 }
1175
1176 /* set and verify setting the limit */
1177 iowrite32(limit, mmio + limit_reg);
1178 reg_val = ioread32(mmio + limit_reg);
1179 if (reg_val != limit) {
1180 iowrite32(base, mmio + limit_reg);
1181 iowrite32(0, mmio + xlat_reg);
1182 return -EIO;
1183 }
1184 }
1185
1186 return 0;
1187 }
1188
1189 static u64 intel_ntb_link_is_up(struct ntb_dev *ntb,
1190 enum ntb_speed *speed,
1191 enum ntb_width *width)
1192 {
1193 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1194
1195 if (ndev->reg->link_is_up(ndev)) {
1196 if (speed)
1197 *speed = NTB_LNK_STA_SPEED(ndev->lnk_sta);
1198 if (width)
1199 *width = NTB_LNK_STA_WIDTH(ndev->lnk_sta);
1200 return 1;
1201 } else {
1202 /* TODO MAYBE: is it possible to observe the link speed and
1203 * width while link is training? */
1204 if (speed)
1205 *speed = NTB_SPEED_NONE;
1206 if (width)
1207 *width = NTB_WIDTH_NONE;
1208 return 0;
1209 }
1210 }
1211
1212 static int intel_ntb_link_enable(struct ntb_dev *ntb,
1213 enum ntb_speed max_speed,
1214 enum ntb_width max_width)
1215 {
1216 struct intel_ntb_dev *ndev;
1217 u32 ntb_ctl;
1218
1219 ndev = container_of(ntb, struct intel_ntb_dev, ntb);
1220
1221 if (ndev->ntb.topo == NTB_TOPO_SEC)
1222 return -EINVAL;
1223
1224 dev_dbg(&ntb->pdev->dev,
1225 "Enabling link with max_speed %d max_width %d\n",
1226 max_speed, max_width);
1227 if (max_speed != NTB_SPEED_AUTO)
1228 dev_dbg(&ntb->pdev->dev, "ignoring max_speed %d\n", max_speed);
1229 if (max_width != NTB_WIDTH_AUTO)
1230 dev_dbg(&ntb->pdev->dev, "ignoring max_width %d\n", max_width);
1231
1232 ntb_ctl = ioread32(ndev->self_mmio + ndev->reg->ntb_ctl);
1233 ntb_ctl &= ~(NTB_CTL_DISABLE | NTB_CTL_CFG_LOCK);
1234 ntb_ctl |= NTB_CTL_P2S_BAR2_SNOOP | NTB_CTL_S2P_BAR2_SNOOP;
1235 ntb_ctl |= NTB_CTL_P2S_BAR4_SNOOP | NTB_CTL_S2P_BAR4_SNOOP;
1236 if (ndev->bar4_split)
1237 ntb_ctl |= NTB_CTL_P2S_BAR5_SNOOP | NTB_CTL_S2P_BAR5_SNOOP;
1238 iowrite32(ntb_ctl, ndev->self_mmio + ndev->reg->ntb_ctl);
1239
1240 return 0;
1241 }
1242
1243 static int intel_ntb_link_disable(struct ntb_dev *ntb)
1244 {
1245 struct intel_ntb_dev *ndev;
1246 u32 ntb_cntl;
1247
1248 ndev = container_of(ntb, struct intel_ntb_dev, ntb);
1249
1250 if (ndev->ntb.topo == NTB_TOPO_SEC)
1251 return -EINVAL;
1252
1253 dev_dbg(&ntb->pdev->dev, "Disabling link\n");
1254
1255 /* Bring NTB link down */
1256 ntb_cntl = ioread32(ndev->self_mmio + ndev->reg->ntb_ctl);
1257 ntb_cntl &= ~(NTB_CTL_P2S_BAR2_SNOOP | NTB_CTL_S2P_BAR2_SNOOP);
1258 ntb_cntl &= ~(NTB_CTL_P2S_BAR4_SNOOP | NTB_CTL_S2P_BAR4_SNOOP);
1259 if (ndev->bar4_split)
1260 ntb_cntl &= ~(NTB_CTL_P2S_BAR5_SNOOP | NTB_CTL_S2P_BAR5_SNOOP);
1261 ntb_cntl |= NTB_CTL_DISABLE | NTB_CTL_CFG_LOCK;
1262 iowrite32(ntb_cntl, ndev->self_mmio + ndev->reg->ntb_ctl);
1263
1264 return 0;
1265 }
1266
1267 static int intel_ntb_peer_mw_count(struct ntb_dev *ntb)
1268 {
1269 /* Numbers of inbound and outbound memory windows match */
1270 return ntb_ndev(ntb)->mw_count;
1271 }
1272
1273 static int intel_ntb_peer_mw_get_addr(struct ntb_dev *ntb, int idx,
1274 phys_addr_t *base, resource_size_t *size)
1275 {
1276 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1277 int bar;
1278
1279 if (idx >= ndev->b2b_idx && !ndev->b2b_off)
1280 idx += 1;
1281
1282 bar = ndev_mw_to_bar(ndev, idx);
1283 if (bar < 0)
1284 return bar;
1285
1286 if (base)
1287 *base = pci_resource_start(ndev->ntb.pdev, bar) +
1288 (idx == ndev->b2b_idx ? ndev->b2b_off : 0);
1289
1290 if (size)
1291 *size = pci_resource_len(ndev->ntb.pdev, bar) -
1292 (idx == ndev->b2b_idx ? ndev->b2b_off : 0);
1293
1294 return 0;
1295 }
1296
1297 static int intel_ntb_db_is_unsafe(struct ntb_dev *ntb)
1298 {
1299 return ndev_ignore_unsafe(ntb_ndev(ntb), NTB_UNSAFE_DB);
1300 }
1301
1302 static u64 intel_ntb_db_valid_mask(struct ntb_dev *ntb)
1303 {
1304 return ntb_ndev(ntb)->db_valid_mask;
1305 }
1306
1307 static int intel_ntb_db_vector_count(struct ntb_dev *ntb)
1308 {
1309 struct intel_ntb_dev *ndev;
1310
1311 ndev = container_of(ntb, struct intel_ntb_dev, ntb);
1312
1313 return ndev->db_vec_count;
1314 }
1315
1316 static u64 intel_ntb_db_vector_mask(struct ntb_dev *ntb, int db_vector)
1317 {
1318 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1319
1320 if (db_vector < 0 || db_vector > ndev->db_vec_count)
1321 return 0;
1322
1323 return ndev->db_valid_mask & ndev_vec_mask(ndev, db_vector);
1324 }
1325
1326 static u64 intel_ntb_db_read(struct ntb_dev *ntb)
1327 {
1328 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1329
1330 return ndev_db_read(ndev,
1331 ndev->self_mmio +
1332 ndev->self_reg->db_bell);
1333 }
1334
1335 static int intel_ntb_db_clear(struct ntb_dev *ntb, u64 db_bits)
1336 {
1337 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1338
1339 return ndev_db_write(ndev, db_bits,
1340 ndev->self_mmio +
1341 ndev->self_reg->db_bell);
1342 }
1343
1344 static int intel_ntb_db_set_mask(struct ntb_dev *ntb, u64 db_bits)
1345 {
1346 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1347
1348 return ndev_db_set_mask(ndev, db_bits,
1349 ndev->self_mmio +
1350 ndev->self_reg->db_mask);
1351 }
1352
1353 static int intel_ntb_db_clear_mask(struct ntb_dev *ntb, u64 db_bits)
1354 {
1355 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1356
1357 return ndev_db_clear_mask(ndev, db_bits,
1358 ndev->self_mmio +
1359 ndev->self_reg->db_mask);
1360 }
1361
1362 static int intel_ntb_peer_db_addr(struct ntb_dev *ntb,
1363 phys_addr_t *db_addr,
1364 resource_size_t *db_size)
1365 {
1366 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1367
1368 return ndev_db_addr(ndev, db_addr, db_size, ndev->peer_addr,
1369 ndev->peer_reg->db_bell);
1370 }
1371
1372 static int intel_ntb_peer_db_set(struct ntb_dev *ntb, u64 db_bits)
1373 {
1374 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1375
1376 return ndev_db_write(ndev, db_bits,
1377 ndev->peer_mmio +
1378 ndev->peer_reg->db_bell);
1379 }
1380
1381 static int intel_ntb_spad_is_unsafe(struct ntb_dev *ntb)
1382 {
1383 return ndev_ignore_unsafe(ntb_ndev(ntb), NTB_UNSAFE_SPAD);
1384 }
1385
1386 static int intel_ntb_spad_count(struct ntb_dev *ntb)
1387 {
1388 struct intel_ntb_dev *ndev;
1389
1390 ndev = container_of(ntb, struct intel_ntb_dev, ntb);
1391
1392 return ndev->spad_count;
1393 }
1394
1395 static u32 intel_ntb_spad_read(struct ntb_dev *ntb, int idx)
1396 {
1397 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1398
1399 return ndev_spad_read(ndev, idx,
1400 ndev->self_mmio +
1401 ndev->self_reg->spad);
1402 }
1403
1404 static int intel_ntb_spad_write(struct ntb_dev *ntb,
1405 int idx, u32 val)
1406 {
1407 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1408
1409 return ndev_spad_write(ndev, idx, val,
1410 ndev->self_mmio +
1411 ndev->self_reg->spad);
1412 }
1413
1414 static int intel_ntb_peer_spad_addr(struct ntb_dev *ntb, int pidx, int sidx,
1415 phys_addr_t *spad_addr)
1416 {
1417 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1418
1419 return ndev_spad_addr(ndev, sidx, spad_addr, ndev->peer_addr,
1420 ndev->peer_reg->spad);
1421 }
1422
1423 static u32 intel_ntb_peer_spad_read(struct ntb_dev *ntb, int pidx, int sidx)
1424 {
1425 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1426
1427 return ndev_spad_read(ndev, sidx,
1428 ndev->peer_mmio +
1429 ndev->peer_reg->spad);
1430 }
1431
1432 static int intel_ntb_peer_spad_write(struct ntb_dev *ntb, int pidx,
1433 int sidx, u32 val)
1434 {
1435 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1436
1437 return ndev_spad_write(ndev, sidx, val,
1438 ndev->peer_mmio +
1439 ndev->peer_reg->spad);
1440 }
1441
1442 /* ATOM */
1443
1444 static u64 atom_db_ioread(void __iomem *mmio)
1445 {
1446 return ioread64(mmio);
1447 }
1448
1449 static void atom_db_iowrite(u64 bits, void __iomem *mmio)
1450 {
1451 iowrite64(bits, mmio);
1452 }
1453
1454 static int atom_poll_link(struct intel_ntb_dev *ndev)
1455 {
1456 u32 ntb_ctl;
1457
1458 ntb_ctl = ioread32(ndev->self_mmio + ATOM_NTBCNTL_OFFSET);
1459
1460 if (ntb_ctl == ndev->ntb_ctl)
1461 return 0;
1462
1463 ndev->ntb_ctl = ntb_ctl;
1464
1465 ndev->lnk_sta = ioread32(ndev->self_mmio + ATOM_LINK_STATUS_OFFSET);
1466
1467 return 1;
1468 }
1469
1470 static int atom_link_is_up(struct intel_ntb_dev *ndev)
1471 {
1472 return ATOM_NTB_CTL_ACTIVE(ndev->ntb_ctl);
1473 }
1474
1475 static int atom_link_is_err(struct intel_ntb_dev *ndev)
1476 {
1477 if (ioread32(ndev->self_mmio + ATOM_LTSSMSTATEJMP_OFFSET)
1478 & ATOM_LTSSMSTATEJMP_FORCEDETECT)
1479 return 1;
1480
1481 if (ioread32(ndev->self_mmio + ATOM_IBSTERRRCRVSTS0_OFFSET)
1482 & ATOM_IBIST_ERR_OFLOW)
1483 return 1;
1484
1485 return 0;
1486 }
1487
1488 static inline enum ntb_topo atom_ppd_topo(struct intel_ntb_dev *ndev, u32 ppd)
1489 {
1490 struct device *dev = &ndev->ntb.pdev->dev;
1491
1492 switch (ppd & ATOM_PPD_TOPO_MASK) {
1493 case ATOM_PPD_TOPO_B2B_USD:
1494 dev_dbg(dev, "PPD %d B2B USD\n", ppd);
1495 return NTB_TOPO_B2B_USD;
1496
1497 case ATOM_PPD_TOPO_B2B_DSD:
1498 dev_dbg(dev, "PPD %d B2B DSD\n", ppd);
1499 return NTB_TOPO_B2B_DSD;
1500
1501 case ATOM_PPD_TOPO_PRI_USD:
1502 case ATOM_PPD_TOPO_PRI_DSD: /* accept bogus PRI_DSD */
1503 case ATOM_PPD_TOPO_SEC_USD:
1504 case ATOM_PPD_TOPO_SEC_DSD: /* accept bogus SEC_DSD */
1505 dev_dbg(dev, "PPD %d non B2B disabled\n", ppd);
1506 return NTB_TOPO_NONE;
1507 }
1508
1509 dev_dbg(dev, "PPD %d invalid\n", ppd);
1510 return NTB_TOPO_NONE;
1511 }
1512
1513 static void atom_link_hb(struct work_struct *work)
1514 {
1515 struct intel_ntb_dev *ndev = hb_ndev(work);
1516 struct device *dev = &ndev->ntb.pdev->dev;
1517 unsigned long poll_ts;
1518 void __iomem *mmio;
1519 u32 status32;
1520
1521 poll_ts = ndev->last_ts + ATOM_LINK_HB_TIMEOUT;
1522
1523 /* Delay polling the link status if an interrupt was received,
1524 * unless the cached link status says the link is down.
1525 */
1526 if (time_after(poll_ts, jiffies) && atom_link_is_up(ndev)) {
1527 schedule_delayed_work(&ndev->hb_timer, poll_ts - jiffies);
1528 return;
1529 }
1530
1531 if (atom_poll_link(ndev))
1532 ntb_link_event(&ndev->ntb);
1533
1534 if (atom_link_is_up(ndev) || !atom_link_is_err(ndev)) {
1535 schedule_delayed_work(&ndev->hb_timer, ATOM_LINK_HB_TIMEOUT);
1536 return;
1537 }
1538
1539 /* Link is down with error: recover the link! */
1540
1541 mmio = ndev->self_mmio;
1542
1543 /* Driver resets the NTB ModPhy lanes - magic! */
1544 iowrite8(0xe0, mmio + ATOM_MODPHY_PCSREG6);
1545 iowrite8(0x40, mmio + ATOM_MODPHY_PCSREG4);
1546 iowrite8(0x60, mmio + ATOM_MODPHY_PCSREG4);
1547 iowrite8(0x60, mmio + ATOM_MODPHY_PCSREG6);
1548
1549 /* Driver waits 100ms to allow the NTB ModPhy to settle */
1550 msleep(100);
1551
1552 /* Clear AER Errors, write to clear */
1553 status32 = ioread32(mmio + ATOM_ERRCORSTS_OFFSET);
1554 dev_dbg(dev, "ERRCORSTS = %x\n", status32);
1555 status32 &= PCI_ERR_COR_REP_ROLL;
1556 iowrite32(status32, mmio + ATOM_ERRCORSTS_OFFSET);
1557
1558 /* Clear unexpected electrical idle event in LTSSM, write to clear */
1559 status32 = ioread32(mmio + ATOM_LTSSMERRSTS0_OFFSET);
1560 dev_dbg(dev, "LTSSMERRSTS0 = %x\n", status32);
1561 status32 |= ATOM_LTSSMERRSTS0_UNEXPECTEDEI;
1562 iowrite32(status32, mmio + ATOM_LTSSMERRSTS0_OFFSET);
1563
1564 /* Clear DeSkew Buffer error, write to clear */
1565 status32 = ioread32(mmio + ATOM_DESKEWSTS_OFFSET);
1566 dev_dbg(dev, "DESKEWSTS = %x\n", status32);
1567 status32 |= ATOM_DESKEWSTS_DBERR;
1568 iowrite32(status32, mmio + ATOM_DESKEWSTS_OFFSET);
1569
1570 status32 = ioread32(mmio + ATOM_IBSTERRRCRVSTS0_OFFSET);
1571 dev_dbg(dev, "IBSTERRRCRVSTS0 = %x\n", status32);
1572 status32 &= ATOM_IBIST_ERR_OFLOW;
1573 iowrite32(status32, mmio + ATOM_IBSTERRRCRVSTS0_OFFSET);
1574
1575 /* Releases the NTB state machine to allow the link to retrain */
1576 status32 = ioread32(mmio + ATOM_LTSSMSTATEJMP_OFFSET);
1577 dev_dbg(dev, "LTSSMSTATEJMP = %x\n", status32);
1578 status32 &= ~ATOM_LTSSMSTATEJMP_FORCEDETECT;
1579 iowrite32(status32, mmio + ATOM_LTSSMSTATEJMP_OFFSET);
1580
1581 /* There is a potential race between the 2 NTB devices recovering at the
1582 * same time. If the times are the same, the link will not recover and
1583 * the driver will be stuck in this loop forever. Add a random interval
1584 * to the recovery time to prevent this race.
1585 */
1586 schedule_delayed_work(&ndev->hb_timer, ATOM_LINK_RECOVERY_TIME
1587 + prandom_u32() % ATOM_LINK_RECOVERY_TIME);
1588 }
1589
1590 static int atom_init_isr(struct intel_ntb_dev *ndev)
1591 {
1592 int rc;
1593
1594 rc = ndev_init_isr(ndev, 1, ATOM_DB_MSIX_VECTOR_COUNT,
1595 ATOM_DB_MSIX_VECTOR_SHIFT, ATOM_DB_TOTAL_SHIFT);
1596 if (rc)
1597 return rc;
1598
1599 /* ATOM doesn't have link status interrupt, poll on that platform */
1600 ndev->last_ts = jiffies;
1601 INIT_DELAYED_WORK(&ndev->hb_timer, atom_link_hb);
1602 schedule_delayed_work(&ndev->hb_timer, ATOM_LINK_HB_TIMEOUT);
1603
1604 return 0;
1605 }
1606
1607 static void atom_deinit_isr(struct intel_ntb_dev *ndev)
1608 {
1609 cancel_delayed_work_sync(&ndev->hb_timer);
1610 ndev_deinit_isr(ndev);
1611 }
1612
1613 static int atom_init_ntb(struct intel_ntb_dev *ndev)
1614 {
1615 ndev->mw_count = ATOM_MW_COUNT;
1616 ndev->spad_count = ATOM_SPAD_COUNT;
1617 ndev->db_count = ATOM_DB_COUNT;
1618
1619 switch (ndev->ntb.topo) {
1620 case NTB_TOPO_B2B_USD:
1621 case NTB_TOPO_B2B_DSD:
1622 ndev->self_reg = &atom_pri_reg;
1623 ndev->peer_reg = &atom_b2b_reg;
1624 ndev->xlat_reg = &atom_sec_xlat;
1625
1626 /* Enable Bus Master and Memory Space on the secondary side */
1627 iowrite16(PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER,
1628 ndev->self_mmio + ATOM_SPCICMD_OFFSET);
1629
1630 break;
1631
1632 default:
1633 return -EINVAL;
1634 }
1635
1636 ndev->db_valid_mask = BIT_ULL(ndev->db_count) - 1;
1637
1638 return 0;
1639 }
1640
1641 static int atom_init_dev(struct intel_ntb_dev *ndev)
1642 {
1643 u32 ppd;
1644 int rc;
1645
1646 rc = pci_read_config_dword(ndev->ntb.pdev, ATOM_PPD_OFFSET, &ppd);
1647 if (rc)
1648 return -EIO;
1649
1650 ndev->ntb.topo = atom_ppd_topo(ndev, ppd);
1651 if (ndev->ntb.topo == NTB_TOPO_NONE)
1652 return -EINVAL;
1653
1654 rc = atom_init_ntb(ndev);
1655 if (rc)
1656 return rc;
1657
1658 rc = atom_init_isr(ndev);
1659 if (rc)
1660 return rc;
1661
1662 if (ndev->ntb.topo != NTB_TOPO_SEC) {
1663 /* Initiate PCI-E link training */
1664 rc = pci_write_config_dword(ndev->ntb.pdev, ATOM_PPD_OFFSET,
1665 ppd | ATOM_PPD_INIT_LINK);
1666 if (rc)
1667 return rc;
1668 }
1669
1670 return 0;
1671 }
1672
1673 static void atom_deinit_dev(struct intel_ntb_dev *ndev)
1674 {
1675 atom_deinit_isr(ndev);
1676 }
1677
1678 /* Skylake Xeon NTB */
1679
1680 static int skx_poll_link(struct intel_ntb_dev *ndev)
1681 {
1682 u16 reg_val;
1683 int rc;
1684
1685 ndev->reg->db_iowrite(ndev->db_link_mask,
1686 ndev->self_mmio +
1687 ndev->self_reg->db_clear);
1688
1689 rc = pci_read_config_word(ndev->ntb.pdev,
1690 SKX_LINK_STATUS_OFFSET, &reg_val);
1691 if (rc)
1692 return 0;
1693
1694 if (reg_val == ndev->lnk_sta)
1695 return 0;
1696
1697 ndev->lnk_sta = reg_val;
1698
1699 return 1;
1700 }
1701
1702 static u64 skx_db_ioread(void __iomem *mmio)
1703 {
1704 return ioread64(mmio);
1705 }
1706
1707 static void skx_db_iowrite(u64 bits, void __iomem *mmio)
1708 {
1709 iowrite64(bits, mmio);
1710 }
1711
1712 static int skx_init_isr(struct intel_ntb_dev *ndev)
1713 {
1714 int i;
1715
1716 /*
1717 * The MSIX vectors and the interrupt status bits are not lined up
1718 * on Skylake. By default the link status bit is bit 32, however it
1719 * is by default MSIX vector0. We need to fixup to line them up.
1720 * The vectors at reset is 1-32,0. We need to reprogram to 0-32.
1721 */
1722
1723 for (i = 0; i < SKX_DB_MSIX_VECTOR_COUNT; i++)
1724 iowrite8(i, ndev->self_mmio + SKX_INTVEC_OFFSET + i);
1725
1726 /* move link status down one as workaround */
1727 if (ndev->hwerr_flags & NTB_HWERR_MSIX_VECTOR32_BAD) {
1728 iowrite8(SKX_DB_MSIX_VECTOR_COUNT - 2,
1729 ndev->self_mmio + SKX_INTVEC_OFFSET +
1730 (SKX_DB_MSIX_VECTOR_COUNT - 1));
1731 }
1732
1733 return ndev_init_isr(ndev, SKX_DB_MSIX_VECTOR_COUNT,
1734 SKX_DB_MSIX_VECTOR_COUNT,
1735 SKX_DB_MSIX_VECTOR_SHIFT,
1736 SKX_DB_TOTAL_SHIFT);
1737 }
1738
1739 static int skx_setup_b2b_mw(struct intel_ntb_dev *ndev,
1740 const struct intel_b2b_addr *addr,
1741 const struct intel_b2b_addr *peer_addr)
1742 {
1743 struct pci_dev *pdev;
1744 void __iomem *mmio;
1745 resource_size_t bar_size;
1746 phys_addr_t bar_addr;
1747 int b2b_bar;
1748 u8 bar_sz;
1749
1750 pdev = ndev->ntb.pdev;
1751 mmio = ndev->self_mmio;
1752
1753 if (ndev->b2b_idx == UINT_MAX) {
1754 dev_dbg(&pdev->dev, "not using b2b mw\n");
1755 b2b_bar = 0;
1756 ndev->b2b_off = 0;
1757 } else {
1758 b2b_bar = ndev_mw_to_bar(ndev, ndev->b2b_idx);
1759 if (b2b_bar < 0)
1760 return -EIO;
1761
1762 dev_dbg(&pdev->dev, "using b2b mw bar %d\n", b2b_bar);
1763
1764 bar_size = pci_resource_len(ndev->ntb.pdev, b2b_bar);
1765
1766 dev_dbg(&pdev->dev, "b2b bar size %#llx\n", bar_size);
1767
1768 if (b2b_mw_share && ((bar_size >> 1) >= XEON_B2B_MIN_SIZE)) {
1769 dev_dbg(&pdev->dev, "b2b using first half of bar\n");
1770 ndev->b2b_off = bar_size >> 1;
1771 } else if (bar_size >= XEON_B2B_MIN_SIZE) {
1772 dev_dbg(&pdev->dev, "b2b using whole bar\n");
1773 ndev->b2b_off = 0;
1774 --ndev->mw_count;
1775 } else {
1776 dev_dbg(&pdev->dev, "b2b bar size is too small\n");
1777 return -EIO;
1778 }
1779 }
1780
1781 /*
1782 * Reset the secondary bar sizes to match the primary bar sizes,
1783 * except disable or halve the size of the b2b secondary bar.
1784 */
1785 pci_read_config_byte(pdev, SKX_IMBAR1SZ_OFFSET, &bar_sz);
1786 dev_dbg(&pdev->dev, "IMBAR1SZ %#x\n", bar_sz);
1787 if (b2b_bar == 1) {
1788 if (ndev->b2b_off)
1789 bar_sz -= 1;
1790 else
1791 bar_sz = 0;
1792 }
1793
1794 pci_write_config_byte(pdev, SKX_EMBAR1SZ_OFFSET, bar_sz);
1795 pci_read_config_byte(pdev, SKX_EMBAR1SZ_OFFSET, &bar_sz);
1796 dev_dbg(&pdev->dev, "EMBAR1SZ %#x\n", bar_sz);
1797
1798 pci_read_config_byte(pdev, SKX_IMBAR2SZ_OFFSET, &bar_sz);
1799 dev_dbg(&pdev->dev, "IMBAR2SZ %#x\n", bar_sz);
1800 if (b2b_bar == 2) {
1801 if (ndev->b2b_off)
1802 bar_sz -= 1;
1803 else
1804 bar_sz = 0;
1805 }
1806
1807 pci_write_config_byte(pdev, SKX_EMBAR2SZ_OFFSET, bar_sz);
1808 pci_read_config_byte(pdev, SKX_EMBAR2SZ_OFFSET, &bar_sz);
1809 dev_dbg(&pdev->dev, "EMBAR2SZ %#x\n", bar_sz);
1810
1811 /* SBAR01 hit by first part of the b2b bar */
1812 if (b2b_bar == 0)
1813 bar_addr = addr->bar0_addr;
1814 else if (b2b_bar == 1)
1815 bar_addr = addr->bar2_addr64;
1816 else if (b2b_bar == 2)
1817 bar_addr = addr->bar4_addr64;
1818 else
1819 return -EIO;
1820
1821 /* setup incoming bar limits == base addrs (zero length windows) */
1822 bar_addr = addr->bar2_addr64 + (b2b_bar == 1 ? ndev->b2b_off : 0);
1823 iowrite64(bar_addr, mmio + SKX_IMBAR1XLMT_OFFSET);
1824 bar_addr = ioread64(mmio + SKX_IMBAR1XLMT_OFFSET);
1825 dev_dbg(&pdev->dev, "IMBAR1XLMT %#018llx\n", bar_addr);
1826
1827 bar_addr = addr->bar4_addr64 + (b2b_bar == 2 ? ndev->b2b_off : 0);
1828 iowrite64(bar_addr, mmio + SKX_IMBAR2XLMT_OFFSET);
1829 bar_addr = ioread64(mmio + SKX_IMBAR2XLMT_OFFSET);
1830 dev_dbg(&pdev->dev, "IMBAR2XLMT %#018llx\n", bar_addr);
1831
1832 /* zero incoming translation addrs */
1833 iowrite64(0, mmio + SKX_IMBAR1XBASE_OFFSET);
1834 iowrite64(0, mmio + SKX_IMBAR2XBASE_OFFSET);
1835
1836 ndev->peer_mmio = ndev->self_mmio;
1837
1838 return 0;
1839 }
1840
1841 static int skx_init_ntb(struct intel_ntb_dev *ndev)
1842 {
1843 int rc;
1844
1845
1846 ndev->mw_count = XEON_MW_COUNT;
1847 ndev->spad_count = SKX_SPAD_COUNT;
1848 ndev->db_count = SKX_DB_COUNT;
1849 ndev->db_link_mask = SKX_DB_LINK_BIT;
1850
1851 /* DB fixup for using 31 right now */
1852 if (ndev->hwerr_flags & NTB_HWERR_MSIX_VECTOR32_BAD)
1853 ndev->db_link_mask |= BIT_ULL(31);
1854
1855 switch (ndev->ntb.topo) {
1856 case NTB_TOPO_B2B_USD:
1857 case NTB_TOPO_B2B_DSD:
1858 ndev->self_reg = &skx_pri_reg;
1859 ndev->peer_reg = &skx_b2b_reg;
1860 ndev->xlat_reg = &skx_sec_xlat;
1861
1862 if (ndev->ntb.topo == NTB_TOPO_B2B_USD) {
1863 rc = skx_setup_b2b_mw(ndev,
1864 &xeon_b2b_dsd_addr,
1865 &xeon_b2b_usd_addr);
1866 } else {
1867 rc = skx_setup_b2b_mw(ndev,
1868 &xeon_b2b_usd_addr,
1869 &xeon_b2b_dsd_addr);
1870 }
1871
1872 if (rc)
1873 return rc;
1874
1875 /* Enable Bus Master and Memory Space on the secondary side */
1876 iowrite16(PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER,
1877 ndev->self_mmio + SKX_SPCICMD_OFFSET);
1878
1879 break;
1880
1881 default:
1882 return -EINVAL;
1883 }
1884
1885 ndev->db_valid_mask = BIT_ULL(ndev->db_count) - 1;
1886
1887 ndev->reg->db_iowrite(ndev->db_valid_mask,
1888 ndev->self_mmio +
1889 ndev->self_reg->db_mask);
1890
1891 return 0;
1892 }
1893
1894 static int skx_init_dev(struct intel_ntb_dev *ndev)
1895 {
1896 struct pci_dev *pdev;
1897 u8 ppd;
1898 int rc;
1899
1900 pdev = ndev->ntb.pdev;
1901
1902 ndev->reg = &skx_reg;
1903
1904 rc = pci_read_config_byte(pdev, XEON_PPD_OFFSET, &ppd);
1905 if (rc)
1906 return -EIO;
1907
1908 ndev->ntb.topo = xeon_ppd_topo(ndev, ppd);
1909 dev_dbg(&pdev->dev, "ppd %#x topo %s\n", ppd,
1910 ntb_topo_string(ndev->ntb.topo));
1911 if (ndev->ntb.topo == NTB_TOPO_NONE)
1912 return -EINVAL;
1913
1914 if (pdev_is_skx_xeon(pdev))
1915 ndev->hwerr_flags |= NTB_HWERR_MSIX_VECTOR32_BAD;
1916
1917 rc = skx_init_ntb(ndev);
1918 if (rc)
1919 return rc;
1920
1921 return skx_init_isr(ndev);
1922 }
1923
1924 static int intel_ntb3_link_enable(struct ntb_dev *ntb,
1925 enum ntb_speed max_speed,
1926 enum ntb_width max_width)
1927 {
1928 struct intel_ntb_dev *ndev;
1929 u32 ntb_ctl;
1930
1931 ndev = container_of(ntb, struct intel_ntb_dev, ntb);
1932
1933 dev_dbg(&ntb->pdev->dev,
1934 "Enabling link with max_speed %d max_width %d\n",
1935 max_speed, max_width);
1936
1937 if (max_speed != NTB_SPEED_AUTO)
1938 dev_dbg(&ntb->pdev->dev, "ignoring max_speed %d\n", max_speed);
1939 if (max_width != NTB_WIDTH_AUTO)
1940 dev_dbg(&ntb->pdev->dev, "ignoring max_width %d\n", max_width);
1941
1942 ntb_ctl = ioread32(ndev->self_mmio + ndev->reg->ntb_ctl);
1943 ntb_ctl &= ~(NTB_CTL_DISABLE | NTB_CTL_CFG_LOCK);
1944 ntb_ctl |= NTB_CTL_P2S_BAR2_SNOOP | NTB_CTL_S2P_BAR2_SNOOP;
1945 ntb_ctl |= NTB_CTL_P2S_BAR4_SNOOP | NTB_CTL_S2P_BAR4_SNOOP;
1946 iowrite32(ntb_ctl, ndev->self_mmio + ndev->reg->ntb_ctl);
1947
1948 return 0;
1949 }
1950 static int intel_ntb3_mw_set_trans(struct ntb_dev *ntb, int pidx, int idx,
1951 dma_addr_t addr, resource_size_t size)
1952 {
1953 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1954 unsigned long xlat_reg, limit_reg;
1955 resource_size_t bar_size, mw_size;
1956 void __iomem *mmio;
1957 u64 base, limit, reg_val;
1958 int bar;
1959
1960 if (pidx != NTB_DEF_PEER_IDX)
1961 return -EINVAL;
1962
1963 if (idx >= ndev->b2b_idx && !ndev->b2b_off)
1964 idx += 1;
1965
1966 bar = ndev_mw_to_bar(ndev, idx);
1967 if (bar < 0)
1968 return bar;
1969
1970 bar_size = pci_resource_len(ndev->ntb.pdev, bar);
1971
1972 if (idx == ndev->b2b_idx)
1973 mw_size = bar_size - ndev->b2b_off;
1974 else
1975 mw_size = bar_size;
1976
1977 /* hardware requires that addr is aligned to bar size */
1978 if (addr & (bar_size - 1))
1979 return -EINVAL;
1980
1981 /* make sure the range fits in the usable mw size */
1982 if (size > mw_size)
1983 return -EINVAL;
1984
1985 mmio = ndev->self_mmio;
1986 xlat_reg = ndev->xlat_reg->bar2_xlat + (idx * 0x10);
1987 limit_reg = ndev->xlat_reg->bar2_limit + (idx * 0x10);
1988 base = pci_resource_start(ndev->ntb.pdev, bar);
1989
1990 /* Set the limit if supported, if size is not mw_size */
1991 if (limit_reg && size != mw_size)
1992 limit = base + size;
1993 else
1994 limit = base + mw_size;
1995
1996 /* set and verify setting the translation address */
1997 iowrite64(addr, mmio + xlat_reg);
1998 reg_val = ioread64(mmio + xlat_reg);
1999 if (reg_val != addr) {
2000 iowrite64(0, mmio + xlat_reg);
2001 return -EIO;
2002 }
2003
2004 dev_dbg(&ntb->pdev->dev, "BAR %d IMBARXBASE: %#Lx\n", bar, reg_val);
2005
2006 /* set and verify setting the limit */
2007 iowrite64(limit, mmio + limit_reg);
2008 reg_val = ioread64(mmio + limit_reg);
2009 if (reg_val != limit) {
2010 iowrite64(base, mmio + limit_reg);
2011 iowrite64(0, mmio + xlat_reg);
2012 return -EIO;
2013 }
2014
2015 dev_dbg(&ntb->pdev->dev, "BAR %d IMBARXLMT: %#Lx\n", bar, reg_val);
2016
2017 /* setup the EP */
2018 limit_reg = ndev->xlat_reg->bar2_limit + (idx * 0x10) + 0x4000;
2019 base = ioread64(mmio + SKX_EMBAR1_OFFSET + (8 * idx));
2020 base &= ~0xf;
2021
2022 if (limit_reg && size != mw_size)
2023 limit = base + size;
2024 else
2025 limit = base + mw_size;
2026
2027 /* set and verify setting the limit */
2028 iowrite64(limit, mmio + limit_reg);
2029 reg_val = ioread64(mmio + limit_reg);
2030 if (reg_val != limit) {
2031 iowrite64(base, mmio + limit_reg);
2032 iowrite64(0, mmio + xlat_reg);
2033 return -EIO;
2034 }
2035
2036 dev_dbg(&ntb->pdev->dev, "BAR %d EMBARXLMT: %#Lx\n", bar, reg_val);
2037
2038 return 0;
2039 }
2040
2041 static int intel_ntb3_peer_db_set(struct ntb_dev *ntb, u64 db_bits)
2042 {
2043 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
2044 int bit;
2045
2046 if (db_bits & ~ndev->db_valid_mask)
2047 return -EINVAL;
2048
2049 while (db_bits) {
2050 bit = __ffs(db_bits);
2051 iowrite32(1, ndev->peer_mmio +
2052 ndev->peer_reg->db_bell + (bit * 4));
2053 db_bits &= db_bits - 1;
2054 }
2055
2056 return 0;
2057 }
2058
2059 static u64 intel_ntb3_db_read(struct ntb_dev *ntb)
2060 {
2061 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
2062
2063 return ndev_db_read(ndev,
2064 ndev->self_mmio +
2065 ndev->self_reg->db_clear);
2066 }
2067
2068 static int intel_ntb3_db_clear(struct ntb_dev *ntb, u64 db_bits)
2069 {
2070 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
2071
2072 return ndev_db_write(ndev, db_bits,
2073 ndev->self_mmio +
2074 ndev->self_reg->db_clear);
2075 }
2076
2077 /* XEON */
2078
2079 static u64 xeon_db_ioread(void __iomem *mmio)
2080 {
2081 return (u64)ioread16(mmio);
2082 }
2083
2084 static void xeon_db_iowrite(u64 bits, void __iomem *mmio)
2085 {
2086 iowrite16((u16)bits, mmio);
2087 }
2088
2089 static int xeon_poll_link(struct intel_ntb_dev *ndev)
2090 {
2091 u16 reg_val;
2092 int rc;
2093
2094 ndev->reg->db_iowrite(ndev->db_link_mask,
2095 ndev->self_mmio +
2096 ndev->self_reg->db_bell);
2097
2098 rc = pci_read_config_word(ndev->ntb.pdev,
2099 XEON_LINK_STATUS_OFFSET, &reg_val);
2100 if (rc)
2101 return 0;
2102
2103 if (reg_val == ndev->lnk_sta)
2104 return 0;
2105
2106 ndev->lnk_sta = reg_val;
2107
2108 return 1;
2109 }
2110
2111 static int xeon_link_is_up(struct intel_ntb_dev *ndev)
2112 {
2113 if (ndev->ntb.topo == NTB_TOPO_SEC)
2114 return 1;
2115
2116 return NTB_LNK_STA_ACTIVE(ndev->lnk_sta);
2117 }
2118
2119 static inline enum ntb_topo xeon_ppd_topo(struct intel_ntb_dev *ndev, u8 ppd)
2120 {
2121 switch (ppd & XEON_PPD_TOPO_MASK) {
2122 case XEON_PPD_TOPO_B2B_USD:
2123 return NTB_TOPO_B2B_USD;
2124
2125 case XEON_PPD_TOPO_B2B_DSD:
2126 return NTB_TOPO_B2B_DSD;
2127
2128 case XEON_PPD_TOPO_PRI_USD:
2129 case XEON_PPD_TOPO_PRI_DSD: /* accept bogus PRI_DSD */
2130 return NTB_TOPO_PRI;
2131
2132 case XEON_PPD_TOPO_SEC_USD:
2133 case XEON_PPD_TOPO_SEC_DSD: /* accept bogus SEC_DSD */
2134 return NTB_TOPO_SEC;
2135 }
2136
2137 return NTB_TOPO_NONE;
2138 }
2139
2140 static inline int xeon_ppd_bar4_split(struct intel_ntb_dev *ndev, u8 ppd)
2141 {
2142 if (ppd & XEON_PPD_SPLIT_BAR_MASK) {
2143 dev_dbg(&ndev->ntb.pdev->dev, "PPD %d split bar\n", ppd);
2144 return 1;
2145 }
2146 return 0;
2147 }
2148
2149 static int xeon_init_isr(struct intel_ntb_dev *ndev)
2150 {
2151 return ndev_init_isr(ndev, XEON_DB_MSIX_VECTOR_COUNT,
2152 XEON_DB_MSIX_VECTOR_COUNT,
2153 XEON_DB_MSIX_VECTOR_SHIFT,
2154 XEON_DB_TOTAL_SHIFT);
2155 }
2156
2157 static void xeon_deinit_isr(struct intel_ntb_dev *ndev)
2158 {
2159 ndev_deinit_isr(ndev);
2160 }
2161
2162 static int xeon_setup_b2b_mw(struct intel_ntb_dev *ndev,
2163 const struct intel_b2b_addr *addr,
2164 const struct intel_b2b_addr *peer_addr)
2165 {
2166 struct pci_dev *pdev;
2167 void __iomem *mmio;
2168 resource_size_t bar_size;
2169 phys_addr_t bar_addr;
2170 int b2b_bar;
2171 u8 bar_sz;
2172
2173 pdev = ndev->ntb.pdev;
2174 mmio = ndev->self_mmio;
2175
2176 if (ndev->b2b_idx == UINT_MAX) {
2177 dev_dbg(&pdev->dev, "not using b2b mw\n");
2178 b2b_bar = 0;
2179 ndev->b2b_off = 0;
2180 } else {
2181 b2b_bar = ndev_mw_to_bar(ndev, ndev->b2b_idx);
2182 if (b2b_bar < 0)
2183 return -EIO;
2184
2185 dev_dbg(&pdev->dev, "using b2b mw bar %d\n", b2b_bar);
2186
2187 bar_size = pci_resource_len(ndev->ntb.pdev, b2b_bar);
2188
2189 dev_dbg(&pdev->dev, "b2b bar size %#llx\n", bar_size);
2190
2191 if (b2b_mw_share && XEON_B2B_MIN_SIZE <= bar_size >> 1) {
2192 dev_dbg(&pdev->dev, "b2b using first half of bar\n");
2193 ndev->b2b_off = bar_size >> 1;
2194 } else if (XEON_B2B_MIN_SIZE <= bar_size) {
2195 dev_dbg(&pdev->dev, "b2b using whole bar\n");
2196 ndev->b2b_off = 0;
2197 --ndev->mw_count;
2198 } else {
2199 dev_dbg(&pdev->dev, "b2b bar size is too small\n");
2200 return -EIO;
2201 }
2202 }
2203
2204 /* Reset the secondary bar sizes to match the primary bar sizes,
2205 * except disable or halve the size of the b2b secondary bar.
2206 *
2207 * Note: code for each specific bar size register, because the register
2208 * offsets are not in a consistent order (bar5sz comes after ppd, odd).
2209 */
2210 pci_read_config_byte(pdev, XEON_PBAR23SZ_OFFSET, &bar_sz);
2211 dev_dbg(&pdev->dev, "PBAR23SZ %#x\n", bar_sz);
2212 if (b2b_bar == 2) {
2213 if (ndev->b2b_off)
2214 bar_sz -= 1;
2215 else
2216 bar_sz = 0;
2217 }
2218 pci_write_config_byte(pdev, XEON_SBAR23SZ_OFFSET, bar_sz);
2219 pci_read_config_byte(pdev, XEON_SBAR23SZ_OFFSET, &bar_sz);
2220 dev_dbg(&pdev->dev, "SBAR23SZ %#x\n", bar_sz);
2221
2222 if (!ndev->bar4_split) {
2223 pci_read_config_byte(pdev, XEON_PBAR45SZ_OFFSET, &bar_sz);
2224 dev_dbg(&pdev->dev, "PBAR45SZ %#x\n", bar_sz);
2225 if (b2b_bar == 4) {
2226 if (ndev->b2b_off)
2227 bar_sz -= 1;
2228 else
2229 bar_sz = 0;
2230 }
2231 pci_write_config_byte(pdev, XEON_SBAR45SZ_OFFSET, bar_sz);
2232 pci_read_config_byte(pdev, XEON_SBAR45SZ_OFFSET, &bar_sz);
2233 dev_dbg(&pdev->dev, "SBAR45SZ %#x\n", bar_sz);
2234 } else {
2235 pci_read_config_byte(pdev, XEON_PBAR4SZ_OFFSET, &bar_sz);
2236 dev_dbg(&pdev->dev, "PBAR4SZ %#x\n", bar_sz);
2237 if (b2b_bar == 4) {
2238 if (ndev->b2b_off)
2239 bar_sz -= 1;
2240 else
2241 bar_sz = 0;
2242 }
2243 pci_write_config_byte(pdev, XEON_SBAR4SZ_OFFSET, bar_sz);
2244 pci_read_config_byte(pdev, XEON_SBAR4SZ_OFFSET, &bar_sz);
2245 dev_dbg(&pdev->dev, "SBAR4SZ %#x\n", bar_sz);
2246
2247 pci_read_config_byte(pdev, XEON_PBAR5SZ_OFFSET, &bar_sz);
2248 dev_dbg(&pdev->dev, "PBAR5SZ %#x\n", bar_sz);
2249 if (b2b_bar == 5) {
2250 if (ndev->b2b_off)
2251 bar_sz -= 1;
2252 else
2253 bar_sz = 0;
2254 }
2255 pci_write_config_byte(pdev, XEON_SBAR5SZ_OFFSET, bar_sz);
2256 pci_read_config_byte(pdev, XEON_SBAR5SZ_OFFSET, &bar_sz);
2257 dev_dbg(&pdev->dev, "SBAR5SZ %#x\n", bar_sz);
2258 }
2259
2260 /* SBAR01 hit by first part of the b2b bar */
2261 if (b2b_bar == 0)
2262 bar_addr = addr->bar0_addr;
2263 else if (b2b_bar == 2)
2264 bar_addr = addr->bar2_addr64;
2265 else if (b2b_bar == 4 && !ndev->bar4_split)
2266 bar_addr = addr->bar4_addr64;
2267 else if (b2b_bar == 4)
2268 bar_addr = addr->bar4_addr32;
2269 else if (b2b_bar == 5)
2270 bar_addr = addr->bar5_addr32;
2271 else
2272 return -EIO;
2273
2274 dev_dbg(&pdev->dev, "SBAR01 %#018llx\n", bar_addr);
2275 iowrite64(bar_addr, mmio + XEON_SBAR0BASE_OFFSET);
2276
2277 /* Other SBAR are normally hit by the PBAR xlat, except for b2b bar.
2278 * The b2b bar is either disabled above, or configured half-size, and
2279 * it starts at the PBAR xlat + offset.
2280 */
2281
2282 bar_addr = addr->bar2_addr64 + (b2b_bar == 2 ? ndev->b2b_off : 0);
2283 iowrite64(bar_addr, mmio + XEON_SBAR23BASE_OFFSET);
2284 bar_addr = ioread64(mmio + XEON_SBAR23BASE_OFFSET);
2285 dev_dbg(&pdev->dev, "SBAR23 %#018llx\n", bar_addr);
2286
2287 if (!ndev->bar4_split) {
2288 bar_addr = addr->bar4_addr64 +
2289 (b2b_bar == 4 ? ndev->b2b_off : 0);
2290 iowrite64(bar_addr, mmio + XEON_SBAR45BASE_OFFSET);
2291 bar_addr = ioread64(mmio + XEON_SBAR45BASE_OFFSET);
2292 dev_dbg(&pdev->dev, "SBAR45 %#018llx\n", bar_addr);
2293 } else {
2294 bar_addr = addr->bar4_addr32 +
2295 (b2b_bar == 4 ? ndev->b2b_off : 0);
2296 iowrite32(bar_addr, mmio + XEON_SBAR4BASE_OFFSET);
2297 bar_addr = ioread32(mmio + XEON_SBAR4BASE_OFFSET);
2298 dev_dbg(&pdev->dev, "SBAR4 %#010llx\n", bar_addr);
2299
2300 bar_addr = addr->bar5_addr32 +
2301 (b2b_bar == 5 ? ndev->b2b_off : 0);
2302 iowrite32(bar_addr, mmio + XEON_SBAR5BASE_OFFSET);
2303 bar_addr = ioread32(mmio + XEON_SBAR5BASE_OFFSET);
2304 dev_dbg(&pdev->dev, "SBAR5 %#010llx\n", bar_addr);
2305 }
2306
2307 /* setup incoming bar limits == base addrs (zero length windows) */
2308
2309 bar_addr = addr->bar2_addr64 + (b2b_bar == 2 ? ndev->b2b_off : 0);
2310 iowrite64(bar_addr, mmio + XEON_SBAR23LMT_OFFSET);
2311 bar_addr = ioread64(mmio + XEON_SBAR23LMT_OFFSET);
2312 dev_dbg(&pdev->dev, "SBAR23LMT %#018llx\n", bar_addr);
2313
2314 if (!ndev->bar4_split) {
2315 bar_addr = addr->bar4_addr64 +
2316 (b2b_bar == 4 ? ndev->b2b_off : 0);
2317 iowrite64(bar_addr, mmio + XEON_SBAR45LMT_OFFSET);
2318 bar_addr = ioread64(mmio + XEON_SBAR45LMT_OFFSET);
2319 dev_dbg(&pdev->dev, "SBAR45LMT %#018llx\n", bar_addr);
2320 } else {
2321 bar_addr = addr->bar4_addr32 +
2322 (b2b_bar == 4 ? ndev->b2b_off : 0);
2323 iowrite32(bar_addr, mmio + XEON_SBAR4LMT_OFFSET);
2324 bar_addr = ioread32(mmio + XEON_SBAR4LMT_OFFSET);
2325 dev_dbg(&pdev->dev, "SBAR4LMT %#010llx\n", bar_addr);
2326
2327 bar_addr = addr->bar5_addr32 +
2328 (b2b_bar == 5 ? ndev->b2b_off : 0);
2329 iowrite32(bar_addr, mmio + XEON_SBAR5LMT_OFFSET);
2330 bar_addr = ioread32(mmio + XEON_SBAR5LMT_OFFSET);
2331 dev_dbg(&pdev->dev, "SBAR5LMT %#05llx\n", bar_addr);
2332 }
2333
2334 /* zero incoming translation addrs */
2335 iowrite64(0, mmio + XEON_SBAR23XLAT_OFFSET);
2336
2337 if (!ndev->bar4_split) {
2338 iowrite64(0, mmio + XEON_SBAR45XLAT_OFFSET);
2339 } else {
2340 iowrite32(0, mmio + XEON_SBAR4XLAT_OFFSET);
2341 iowrite32(0, mmio + XEON_SBAR5XLAT_OFFSET);
2342 }
2343
2344 /* zero outgoing translation limits (whole bar size windows) */
2345 iowrite64(0, mmio + XEON_PBAR23LMT_OFFSET);
2346 if (!ndev->bar4_split) {
2347 iowrite64(0, mmio + XEON_PBAR45LMT_OFFSET);
2348 } else {
2349 iowrite32(0, mmio + XEON_PBAR4LMT_OFFSET);
2350 iowrite32(0, mmio + XEON_PBAR5LMT_OFFSET);
2351 }
2352
2353 /* set outgoing translation offsets */
2354 bar_addr = peer_addr->bar2_addr64;
2355 iowrite64(bar_addr, mmio + XEON_PBAR23XLAT_OFFSET);
2356 bar_addr = ioread64(mmio + XEON_PBAR23XLAT_OFFSET);
2357 dev_dbg(&pdev->dev, "PBAR23XLAT %#018llx\n", bar_addr);
2358
2359 if (!ndev->bar4_split) {
2360 bar_addr = peer_addr->bar4_addr64;
2361 iowrite64(bar_addr, mmio + XEON_PBAR45XLAT_OFFSET);
2362 bar_addr = ioread64(mmio + XEON_PBAR45XLAT_OFFSET);
2363 dev_dbg(&pdev->dev, "PBAR45XLAT %#018llx\n", bar_addr);
2364 } else {
2365 bar_addr = peer_addr->bar4_addr32;
2366 iowrite32(bar_addr, mmio + XEON_PBAR4XLAT_OFFSET);
2367 bar_addr = ioread32(mmio + XEON_PBAR4XLAT_OFFSET);
2368 dev_dbg(&pdev->dev, "PBAR4XLAT %#010llx\n", bar_addr);
2369
2370 bar_addr = peer_addr->bar5_addr32;
2371 iowrite32(bar_addr, mmio + XEON_PBAR5XLAT_OFFSET);
2372 bar_addr = ioread32(mmio + XEON_PBAR5XLAT_OFFSET);
2373 dev_dbg(&pdev->dev, "PBAR5XLAT %#010llx\n", bar_addr);
2374 }
2375
2376 /* set the translation offset for b2b registers */
2377 if (b2b_bar == 0)
2378 bar_addr = peer_addr->bar0_addr;
2379 else if (b2b_bar == 2)
2380 bar_addr = peer_addr->bar2_addr64;
2381 else if (b2b_bar == 4 && !ndev->bar4_split)
2382 bar_addr = peer_addr->bar4_addr64;
2383 else if (b2b_bar == 4)
2384 bar_addr = peer_addr->bar4_addr32;
2385 else if (b2b_bar == 5)
2386 bar_addr = peer_addr->bar5_addr32;
2387 else
2388 return -EIO;
2389
2390 /* B2B_XLAT_OFFSET is 64bit, but can only take 32bit writes */
2391 dev_dbg(&pdev->dev, "B2BXLAT %#018llx\n", bar_addr);
2392 iowrite32(bar_addr, mmio + XEON_B2B_XLAT_OFFSETL);
2393 iowrite32(bar_addr >> 32, mmio + XEON_B2B_XLAT_OFFSETU);
2394
2395 if (b2b_bar) {
2396 /* map peer ntb mmio config space registers */
2397 ndev->peer_mmio = pci_iomap(pdev, b2b_bar,
2398 XEON_B2B_MIN_SIZE);
2399 if (!ndev->peer_mmio)
2400 return -EIO;
2401
2402 ndev->peer_addr = pci_resource_start(pdev, b2b_bar);
2403 }
2404
2405 return 0;
2406 }
2407
2408 static int xeon_init_ntb(struct intel_ntb_dev *ndev)
2409 {
2410 struct device *dev = &ndev->ntb.pdev->dev;
2411 int rc;
2412 u32 ntb_ctl;
2413
2414 if (ndev->bar4_split)
2415 ndev->mw_count = HSX_SPLIT_BAR_MW_COUNT;
2416 else
2417 ndev->mw_count = XEON_MW_COUNT;
2418
2419 ndev->spad_count = XEON_SPAD_COUNT;
2420 ndev->db_count = XEON_DB_COUNT;
2421 ndev->db_link_mask = XEON_DB_LINK_BIT;
2422
2423 switch (ndev->ntb.topo) {
2424 case NTB_TOPO_PRI:
2425 if (ndev->hwerr_flags & NTB_HWERR_SDOORBELL_LOCKUP) {
2426 dev_err(dev, "NTB Primary config disabled\n");
2427 return -EINVAL;
2428 }
2429
2430 /* enable link to allow secondary side device to appear */
2431 ntb_ctl = ioread32(ndev->self_mmio + ndev->reg->ntb_ctl);
2432 ntb_ctl &= ~NTB_CTL_DISABLE;
2433 iowrite32(ntb_ctl, ndev->self_mmio + ndev->reg->ntb_ctl);
2434
2435 /* use half the spads for the peer */
2436 ndev->spad_count >>= 1;
2437 ndev->self_reg = &xeon_pri_reg;
2438 ndev->peer_reg = &xeon_sec_reg;
2439 ndev->xlat_reg = &xeon_sec_xlat;
2440 break;
2441
2442 case NTB_TOPO_SEC:
2443 if (ndev->hwerr_flags & NTB_HWERR_SDOORBELL_LOCKUP) {
2444 dev_err(dev, "NTB Secondary config disabled\n");
2445 return -EINVAL;
2446 }
2447 /* use half the spads for the peer */
2448 ndev->spad_count >>= 1;
2449 ndev->self_reg = &xeon_sec_reg;
2450 ndev->peer_reg = &xeon_pri_reg;
2451 ndev->xlat_reg = &xeon_pri_xlat;
2452 break;
2453
2454 case NTB_TOPO_B2B_USD:
2455 case NTB_TOPO_B2B_DSD:
2456 ndev->self_reg = &xeon_pri_reg;
2457 ndev->peer_reg = &xeon_b2b_reg;
2458 ndev->xlat_reg = &xeon_sec_xlat;
2459
2460 if (ndev->hwerr_flags & NTB_HWERR_SDOORBELL_LOCKUP) {
2461 ndev->peer_reg = &xeon_pri_reg;
2462
2463 if (b2b_mw_idx < 0)
2464 ndev->b2b_idx = b2b_mw_idx + ndev->mw_count;
2465 else
2466 ndev->b2b_idx = b2b_mw_idx;
2467
2468 if (ndev->b2b_idx >= ndev->mw_count) {
2469 dev_dbg(dev,
2470 "b2b_mw_idx %d invalid for mw_count %u\n",
2471 b2b_mw_idx, ndev->mw_count);
2472 return -EINVAL;
2473 }
2474
2475 dev_dbg(dev, "setting up b2b mw idx %d means %d\n",
2476 b2b_mw_idx, ndev->b2b_idx);
2477
2478 } else if (ndev->hwerr_flags & NTB_HWERR_B2BDOORBELL_BIT14) {
2479 dev_warn(dev, "Reduce doorbell count by 1\n");
2480 ndev->db_count -= 1;
2481 }
2482
2483 if (ndev->ntb.topo == NTB_TOPO_B2B_USD) {
2484 rc = xeon_setup_b2b_mw(ndev,
2485 &xeon_b2b_dsd_addr,
2486 &xeon_b2b_usd_addr);
2487 } else {
2488 rc = xeon_setup_b2b_mw(ndev,
2489 &xeon_b2b_usd_addr,
2490 &xeon_b2b_dsd_addr);
2491 }
2492 if (rc)
2493 return rc;
2494
2495 /* Enable Bus Master and Memory Space on the secondary side */
2496 iowrite16(PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER,
2497 ndev->self_mmio + XEON_SPCICMD_OFFSET);
2498
2499 break;
2500
2501 default:
2502 return -EINVAL;
2503 }
2504
2505 ndev->db_valid_mask = BIT_ULL(ndev->db_count) - 1;
2506
2507 ndev->reg->db_iowrite(ndev->db_valid_mask,
2508 ndev->self_mmio +
2509 ndev->self_reg->db_mask);
2510
2511 return 0;
2512 }
2513
2514 static int xeon_init_dev(struct intel_ntb_dev *ndev)
2515 {
2516 struct pci_dev *pdev;
2517 u8 ppd;
2518 int rc, mem;
2519
2520 pdev = ndev->ntb.pdev;
2521
2522 switch (pdev->device) {
2523 /* There is a Xeon hardware errata related to writes to SDOORBELL or
2524 * B2BDOORBELL in conjunction with inbound access to NTB MMIO Space,
2525 * which may hang the system. To workaround this use the second memory
2526 * window to access the interrupt and scratch pad registers on the
2527 * remote system.
2528 */
2529 case PCI_DEVICE_ID_INTEL_NTB_SS_JSF:
2530 case PCI_DEVICE_ID_INTEL_NTB_PS_JSF:
2531 case PCI_DEVICE_ID_INTEL_NTB_B2B_JSF:
2532 case PCI_DEVICE_ID_INTEL_NTB_SS_SNB:
2533 case PCI_DEVICE_ID_INTEL_NTB_PS_SNB:
2534 case PCI_DEVICE_ID_INTEL_NTB_B2B_SNB:
2535 case PCI_DEVICE_ID_INTEL_NTB_SS_IVT:
2536 case PCI_DEVICE_ID_INTEL_NTB_PS_IVT:
2537 case PCI_DEVICE_ID_INTEL_NTB_B2B_IVT:
2538 case PCI_DEVICE_ID_INTEL_NTB_SS_HSX:
2539 case PCI_DEVICE_ID_INTEL_NTB_PS_HSX:
2540 case PCI_DEVICE_ID_INTEL_NTB_B2B_HSX:
2541 case PCI_DEVICE_ID_INTEL_NTB_SS_BDX:
2542 case PCI_DEVICE_ID_INTEL_NTB_PS_BDX:
2543 case PCI_DEVICE_ID_INTEL_NTB_B2B_BDX:
2544 ndev->hwerr_flags |= NTB_HWERR_SDOORBELL_LOCKUP;
2545 break;
2546 }
2547
2548 switch (pdev->device) {
2549 /* There is a hardware errata related to accessing any register in
2550 * SB01BASE in the presence of bidirectional traffic crossing the NTB.
2551 */
2552 case PCI_DEVICE_ID_INTEL_NTB_SS_IVT:
2553 case PCI_DEVICE_ID_INTEL_NTB_PS_IVT:
2554 case PCI_DEVICE_ID_INTEL_NTB_B2B_IVT:
2555 case PCI_DEVICE_ID_INTEL_NTB_SS_HSX:
2556 case PCI_DEVICE_ID_INTEL_NTB_PS_HSX:
2557 case PCI_DEVICE_ID_INTEL_NTB_B2B_HSX:
2558 case PCI_DEVICE_ID_INTEL_NTB_SS_BDX:
2559 case PCI_DEVICE_ID_INTEL_NTB_PS_BDX:
2560 case PCI_DEVICE_ID_INTEL_NTB_B2B_BDX:
2561 ndev->hwerr_flags |= NTB_HWERR_SB01BASE_LOCKUP;
2562 break;
2563 }
2564
2565 switch (pdev->device) {
2566 /* HW Errata on bit 14 of b2bdoorbell register. Writes will not be
2567 * mirrored to the remote system. Shrink the number of bits by one,
2568 * since bit 14 is the last bit.
2569 */
2570 case PCI_DEVICE_ID_INTEL_NTB_SS_JSF:
2571 case PCI_DEVICE_ID_INTEL_NTB_PS_JSF:
2572 case PCI_DEVICE_ID_INTEL_NTB_B2B_JSF:
2573 case PCI_DEVICE_ID_INTEL_NTB_SS_SNB:
2574 case PCI_DEVICE_ID_INTEL_NTB_PS_SNB:
2575 case PCI_DEVICE_ID_INTEL_NTB_B2B_SNB:
2576 case PCI_DEVICE_ID_INTEL_NTB_SS_IVT:
2577 case PCI_DEVICE_ID_INTEL_NTB_PS_IVT:
2578 case PCI_DEVICE_ID_INTEL_NTB_B2B_IVT:
2579 case PCI_DEVICE_ID_INTEL_NTB_SS_HSX:
2580 case PCI_DEVICE_ID_INTEL_NTB_PS_HSX:
2581 case PCI_DEVICE_ID_INTEL_NTB_B2B_HSX:
2582 case PCI_DEVICE_ID_INTEL_NTB_SS_BDX:
2583 case PCI_DEVICE_ID_INTEL_NTB_PS_BDX:
2584 case PCI_DEVICE_ID_INTEL_NTB_B2B_BDX:
2585 ndev->hwerr_flags |= NTB_HWERR_B2BDOORBELL_BIT14;
2586 break;
2587 }
2588
2589 ndev->reg = &xeon_reg;
2590
2591 rc = pci_read_config_byte(pdev, XEON_PPD_OFFSET, &ppd);
2592 if (rc)
2593 return -EIO;
2594
2595 ndev->ntb.topo = xeon_ppd_topo(ndev, ppd);
2596 dev_dbg(&pdev->dev, "ppd %#x topo %s\n", ppd,
2597 ntb_topo_string(ndev->ntb.topo));
2598 if (ndev->ntb.topo == NTB_TOPO_NONE)
2599 return -EINVAL;
2600
2601 if (ndev->ntb.topo != NTB_TOPO_SEC) {
2602 ndev->bar4_split = xeon_ppd_bar4_split(ndev, ppd);
2603 dev_dbg(&pdev->dev, "ppd %#x bar4_split %d\n",
2604 ppd, ndev->bar4_split);
2605 } else {
2606 /* This is a way for transparent BAR to figure out if we are
2607 * doing split BAR or not. There is no way for the hw on the
2608 * transparent side to know and set the PPD.
2609 */
2610 mem = pci_select_bars(pdev, IORESOURCE_MEM);
2611 ndev->bar4_split = hweight32(mem) ==
2612 HSX_SPLIT_BAR_MW_COUNT + 1;
2613 dev_dbg(&pdev->dev, "mem %#x bar4_split %d\n",
2614 mem, ndev->bar4_split);
2615 }
2616
2617 rc = xeon_init_ntb(ndev);
2618 if (rc)
2619 return rc;
2620
2621 return xeon_init_isr(ndev);
2622 }
2623
2624 static void xeon_deinit_dev(struct intel_ntb_dev *ndev)
2625 {
2626 xeon_deinit_isr(ndev);
2627 }
2628
2629 static int intel_ntb_init_pci(struct intel_ntb_dev *ndev, struct pci_dev *pdev)
2630 {
2631 int rc;
2632
2633 pci_set_drvdata(pdev, ndev);
2634
2635 rc = pci_enable_device(pdev);
2636 if (rc)
2637 goto err_pci_enable;
2638
2639 rc = pci_request_regions(pdev, NTB_NAME);
2640 if (rc)
2641 goto err_pci_regions;
2642
2643 pci_set_master(pdev);
2644
2645 rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
2646 if (rc) {
2647 rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
2648 if (rc)
2649 goto err_dma_mask;
2650 dev_warn(&pdev->dev, "Cannot DMA highmem\n");
2651 }
2652
2653 rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
2654 if (rc) {
2655 rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
2656 if (rc)
2657 goto err_dma_mask;
2658 dev_warn(&pdev->dev, "Cannot DMA consistent highmem\n");
2659 }
2660
2661 ndev->self_mmio = pci_iomap(pdev, 0, 0);
2662 if (!ndev->self_mmio) {
2663 rc = -EIO;
2664 goto err_mmio;
2665 }
2666 ndev->peer_mmio = ndev->self_mmio;
2667 ndev->peer_addr = pci_resource_start(pdev, 0);
2668
2669 return 0;
2670
2671 err_mmio:
2672 err_dma_mask:
2673 pci_clear_master(pdev);
2674 pci_release_regions(pdev);
2675 err_pci_regions:
2676 pci_disable_device(pdev);
2677 err_pci_enable:
2678 pci_set_drvdata(pdev, NULL);
2679 return rc;
2680 }
2681
2682 static void intel_ntb_deinit_pci(struct intel_ntb_dev *ndev)
2683 {
2684 struct pci_dev *pdev = ndev->ntb.pdev;
2685
2686 if (ndev->peer_mmio && ndev->peer_mmio != ndev->self_mmio)
2687 pci_iounmap(pdev, ndev->peer_mmio);
2688 pci_iounmap(pdev, ndev->self_mmio);
2689
2690 pci_clear_master(pdev);
2691 pci_release_regions(pdev);
2692 pci_disable_device(pdev);
2693 pci_set_drvdata(pdev, NULL);
2694 }
2695
2696 static inline void ndev_init_struct(struct intel_ntb_dev *ndev,
2697 struct pci_dev *pdev)
2698 {
2699 ndev->ntb.pdev = pdev;
2700 ndev->ntb.topo = NTB_TOPO_NONE;
2701 ndev->ntb.ops = &intel_ntb_ops;
2702
2703 ndev->b2b_off = 0;
2704 ndev->b2b_idx = UINT_MAX;
2705
2706 ndev->bar4_split = 0;
2707
2708 ndev->mw_count = 0;
2709 ndev->spad_count = 0;
2710 ndev->db_count = 0;
2711 ndev->db_vec_count = 0;
2712 ndev->db_vec_shift = 0;
2713
2714 ndev->ntb_ctl = 0;
2715 ndev->lnk_sta = 0;
2716
2717 ndev->db_valid_mask = 0;
2718 ndev->db_link_mask = 0;
2719 ndev->db_mask = 0;
2720
2721 spin_lock_init(&ndev->db_mask_lock);
2722 }
2723
2724 static int intel_ntb_pci_probe(struct pci_dev *pdev,
2725 const struct pci_device_id *id)
2726 {
2727 struct intel_ntb_dev *ndev;
2728 int rc, node;
2729
2730 node = dev_to_node(&pdev->dev);
2731
2732 if (pdev_is_atom(pdev)) {
2733 ndev = kzalloc_node(sizeof(*ndev), GFP_KERNEL, node);
2734 if (!ndev) {
2735 rc = -ENOMEM;
2736 goto err_ndev;
2737 }
2738
2739 ndev_init_struct(ndev, pdev);
2740
2741 rc = intel_ntb_init_pci(ndev, pdev);
2742 if (rc)
2743 goto err_init_pci;
2744
2745 rc = atom_init_dev(ndev);
2746 if (rc)
2747 goto err_init_dev;
2748
2749 } else if (pdev_is_xeon(pdev)) {
2750 ndev = kzalloc_node(sizeof(*ndev), GFP_KERNEL, node);
2751 if (!ndev) {
2752 rc = -ENOMEM;
2753 goto err_ndev;
2754 }
2755
2756 ndev_init_struct(ndev, pdev);
2757
2758 rc = intel_ntb_init_pci(ndev, pdev);
2759 if (rc)
2760 goto err_init_pci;
2761
2762 rc = xeon_init_dev(ndev);
2763 if (rc)
2764 goto err_init_dev;
2765
2766 } else if (pdev_is_skx_xeon(pdev)) {
2767 ndev = kzalloc_node(sizeof(*ndev), GFP_KERNEL, node);
2768 if (!ndev) {
2769 rc = -ENOMEM;
2770 goto err_ndev;
2771 }
2772
2773 ndev_init_struct(ndev, pdev);
2774 ndev->ntb.ops = &intel_ntb3_ops;
2775
2776 rc = intel_ntb_init_pci(ndev, pdev);
2777 if (rc)
2778 goto err_init_pci;
2779
2780 rc = skx_init_dev(ndev);
2781 if (rc)
2782 goto err_init_dev;
2783
2784 } else {
2785 rc = -EINVAL;
2786 goto err_ndev;
2787 }
2788
2789 ndev_reset_unsafe_flags(ndev);
2790
2791 ndev->reg->poll_link(ndev);
2792
2793 ndev_init_debugfs(ndev);
2794
2795 rc = ntb_register_device(&ndev->ntb);
2796 if (rc)
2797 goto err_register;
2798
2799 dev_info(&pdev->dev, "NTB device registered.\n");
2800
2801 return 0;
2802
2803 err_register:
2804 ndev_deinit_debugfs(ndev);
2805 if (pdev_is_atom(pdev))
2806 atom_deinit_dev(ndev);
2807 else if (pdev_is_xeon(pdev) || pdev_is_skx_xeon(pdev))
2808 xeon_deinit_dev(ndev);
2809 err_init_dev:
2810 intel_ntb_deinit_pci(ndev);
2811 err_init_pci:
2812 kfree(ndev);
2813 err_ndev:
2814 return rc;
2815 }
2816
2817 static void intel_ntb_pci_remove(struct pci_dev *pdev)
2818 {
2819 struct intel_ntb_dev *ndev = pci_get_drvdata(pdev);
2820
2821 ntb_unregister_device(&ndev->ntb);
2822 ndev_deinit_debugfs(ndev);
2823 if (pdev_is_atom(pdev))
2824 atom_deinit_dev(ndev);
2825 else if (pdev_is_xeon(pdev) || pdev_is_skx_xeon(pdev))
2826 xeon_deinit_dev(ndev);
2827 intel_ntb_deinit_pci(ndev);
2828 kfree(ndev);
2829 }
2830
2831 static const struct intel_ntb_reg atom_reg = {
2832 .poll_link = atom_poll_link,
2833 .link_is_up = atom_link_is_up,
2834 .db_ioread = atom_db_ioread,
2835 .db_iowrite = atom_db_iowrite,
2836 .db_size = sizeof(u64),
2837 .ntb_ctl = ATOM_NTBCNTL_OFFSET,
2838 .mw_bar = {2, 4},
2839 };
2840
2841 static const struct intel_ntb_alt_reg atom_pri_reg = {
2842 .db_bell = ATOM_PDOORBELL_OFFSET,
2843 .db_mask = ATOM_PDBMSK_OFFSET,
2844 .spad = ATOM_SPAD_OFFSET,
2845 };
2846
2847 static const struct intel_ntb_alt_reg atom_b2b_reg = {
2848 .db_bell = ATOM_B2B_DOORBELL_OFFSET,
2849 .spad = ATOM_B2B_SPAD_OFFSET,
2850 };
2851
2852 static const struct intel_ntb_xlat_reg atom_sec_xlat = {
2853 /* FIXME : .bar0_base = ATOM_SBAR0BASE_OFFSET, */
2854 /* FIXME : .bar2_limit = ATOM_SBAR2LMT_OFFSET, */
2855 .bar2_xlat = ATOM_SBAR2XLAT_OFFSET,
2856 };
2857
2858 static const struct intel_ntb_reg xeon_reg = {
2859 .poll_link = xeon_poll_link,
2860 .link_is_up = xeon_link_is_up,
2861 .db_ioread = xeon_db_ioread,
2862 .db_iowrite = xeon_db_iowrite,
2863 .db_size = sizeof(u32),
2864 .ntb_ctl = XEON_NTBCNTL_OFFSET,
2865 .mw_bar = {2, 4, 5},
2866 };
2867
2868 static const struct intel_ntb_alt_reg xeon_pri_reg = {
2869 .db_bell = XEON_PDOORBELL_OFFSET,
2870 .db_mask = XEON_PDBMSK_OFFSET,
2871 .spad = XEON_SPAD_OFFSET,
2872 };
2873
2874 static const struct intel_ntb_alt_reg xeon_sec_reg = {
2875 .db_bell = XEON_SDOORBELL_OFFSET,
2876 .db_mask = XEON_SDBMSK_OFFSET,
2877 /* second half of the scratchpads */
2878 .spad = XEON_SPAD_OFFSET + (XEON_SPAD_COUNT << 1),
2879 };
2880
2881 static const struct intel_ntb_alt_reg xeon_b2b_reg = {
2882 .db_bell = XEON_B2B_DOORBELL_OFFSET,
2883 .spad = XEON_B2B_SPAD_OFFSET,
2884 };
2885
2886 static const struct intel_ntb_xlat_reg xeon_pri_xlat = {
2887 /* Note: no primary .bar0_base visible to the secondary side.
2888 *
2889 * The secondary side cannot get the base address stored in primary
2890 * bars. The base address is necessary to set the limit register to
2891 * any value other than zero, or unlimited.
2892 *
2893 * WITHOUT THE BASE ADDRESS, THE SECONDARY SIDE CANNOT DISABLE the
2894 * window by setting the limit equal to base, nor can it limit the size
2895 * of the memory window by setting the limit to base + size.
2896 */
2897 .bar2_limit = XEON_PBAR23LMT_OFFSET,
2898 .bar2_xlat = XEON_PBAR23XLAT_OFFSET,
2899 };
2900
2901 static const struct intel_ntb_xlat_reg xeon_sec_xlat = {
2902 .bar0_base = XEON_SBAR0BASE_OFFSET,
2903 .bar2_limit = XEON_SBAR23LMT_OFFSET,
2904 .bar2_xlat = XEON_SBAR23XLAT_OFFSET,
2905 };
2906
2907 static struct intel_b2b_addr xeon_b2b_usd_addr = {
2908 .bar2_addr64 = XEON_B2B_BAR2_ADDR64,
2909 .bar4_addr64 = XEON_B2B_BAR4_ADDR64,
2910 .bar4_addr32 = XEON_B2B_BAR4_ADDR32,
2911 .bar5_addr32 = XEON_B2B_BAR5_ADDR32,
2912 };
2913
2914 static struct intel_b2b_addr xeon_b2b_dsd_addr = {
2915 .bar2_addr64 = XEON_B2B_BAR2_ADDR64,
2916 .bar4_addr64 = XEON_B2B_BAR4_ADDR64,
2917 .bar4_addr32 = XEON_B2B_BAR4_ADDR32,
2918 .bar5_addr32 = XEON_B2B_BAR5_ADDR32,
2919 };
2920
2921 static const struct intel_ntb_reg skx_reg = {
2922 .poll_link = skx_poll_link,
2923 .link_is_up = xeon_link_is_up,
2924 .db_ioread = skx_db_ioread,
2925 .db_iowrite = skx_db_iowrite,
2926 .db_size = sizeof(u32),
2927 .ntb_ctl = SKX_NTBCNTL_OFFSET,
2928 .mw_bar = {2, 4},
2929 };
2930
2931 static const struct intel_ntb_alt_reg skx_pri_reg = {
2932 .db_bell = SKX_EM_DOORBELL_OFFSET,
2933 .db_clear = SKX_IM_INT_STATUS_OFFSET,
2934 .db_mask = SKX_IM_INT_DISABLE_OFFSET,
2935 .spad = SKX_IM_SPAD_OFFSET,
2936 };
2937
2938 static const struct intel_ntb_alt_reg skx_b2b_reg = {
2939 .db_bell = SKX_IM_DOORBELL_OFFSET,
2940 .db_clear = SKX_EM_INT_STATUS_OFFSET,
2941 .db_mask = SKX_EM_INT_DISABLE_OFFSET,
2942 .spad = SKX_B2B_SPAD_OFFSET,
2943 };
2944
2945 static const struct intel_ntb_xlat_reg skx_sec_xlat = {
2946 /* .bar0_base = SKX_EMBAR0_OFFSET, */
2947 .bar2_limit = SKX_IMBAR1XLMT_OFFSET,
2948 .bar2_xlat = SKX_IMBAR1XBASE_OFFSET,
2949 };
2950
2951 /* operations for primary side of local ntb */
2952 static const struct ntb_dev_ops intel_ntb_ops = {
2953 .mw_count = intel_ntb_mw_count,
2954 .mw_get_align = intel_ntb_mw_get_align,
2955 .mw_set_trans = intel_ntb_mw_set_trans,
2956 .peer_mw_count = intel_ntb_peer_mw_count,
2957 .peer_mw_get_addr = intel_ntb_peer_mw_get_addr,
2958 .link_is_up = intel_ntb_link_is_up,
2959 .link_enable = intel_ntb_link_enable,
2960 .link_disable = intel_ntb_link_disable,
2961 .db_is_unsafe = intel_ntb_db_is_unsafe,
2962 .db_valid_mask = intel_ntb_db_valid_mask,
2963 .db_vector_count = intel_ntb_db_vector_count,
2964 .db_vector_mask = intel_ntb_db_vector_mask,
2965 .db_read = intel_ntb_db_read,
2966 .db_clear = intel_ntb_db_clear,
2967 .db_set_mask = intel_ntb_db_set_mask,
2968 .db_clear_mask = intel_ntb_db_clear_mask,
2969 .peer_db_addr = intel_ntb_peer_db_addr,
2970 .peer_db_set = intel_ntb_peer_db_set,
2971 .spad_is_unsafe = intel_ntb_spad_is_unsafe,
2972 .spad_count = intel_ntb_spad_count,
2973 .spad_read = intel_ntb_spad_read,
2974 .spad_write = intel_ntb_spad_write,
2975 .peer_spad_addr = intel_ntb_peer_spad_addr,
2976 .peer_spad_read = intel_ntb_peer_spad_read,
2977 .peer_spad_write = intel_ntb_peer_spad_write,
2978 };
2979
2980 static const struct ntb_dev_ops intel_ntb3_ops = {
2981 .mw_count = intel_ntb_mw_count,
2982 .mw_get_align = intel_ntb_mw_get_align,
2983 .mw_set_trans = intel_ntb3_mw_set_trans,
2984 .peer_mw_count = intel_ntb_peer_mw_count,
2985 .peer_mw_get_addr = intel_ntb_peer_mw_get_addr,
2986 .link_is_up = intel_ntb_link_is_up,
2987 .link_enable = intel_ntb3_link_enable,
2988 .link_disable = intel_ntb_link_disable,
2989 .db_valid_mask = intel_ntb_db_valid_mask,
2990 .db_vector_count = intel_ntb_db_vector_count,
2991 .db_vector_mask = intel_ntb_db_vector_mask,
2992 .db_read = intel_ntb3_db_read,
2993 .db_clear = intel_ntb3_db_clear,
2994 .db_set_mask = intel_ntb_db_set_mask,
2995 .db_clear_mask = intel_ntb_db_clear_mask,
2996 .peer_db_addr = intel_ntb_peer_db_addr,
2997 .peer_db_set = intel_ntb3_peer_db_set,
2998 .spad_is_unsafe = intel_ntb_spad_is_unsafe,
2999 .spad_count = intel_ntb_spad_count,
3000 .spad_read = intel_ntb_spad_read,
3001 .spad_write = intel_ntb_spad_write,
3002 .peer_spad_addr = intel_ntb_peer_spad_addr,
3003 .peer_spad_read = intel_ntb_peer_spad_read,
3004 .peer_spad_write = intel_ntb_peer_spad_write,
3005 };
3006
3007 static const struct file_operations intel_ntb_debugfs_info = {
3008 .owner = THIS_MODULE,
3009 .open = simple_open,
3010 .read = ndev_debugfs_read,
3011 };
3012
3013 static const struct pci_device_id intel_ntb_pci_tbl[] = {
3014 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_BWD)},
3015 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_JSF)},
3016 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_SNB)},
3017 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_IVT)},
3018 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_HSX)},
3019 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_BDX)},
3020 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_JSF)},
3021 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_SNB)},
3022 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_IVT)},
3023 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_HSX)},
3024 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_BDX)},
3025 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_JSF)},
3026 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_SNB)},
3027 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_IVT)},
3028 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_HSX)},
3029 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_BDX)},
3030 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_SKX)},
3031 {0}
3032 };
3033 MODULE_DEVICE_TABLE(pci, intel_ntb_pci_tbl);
3034
3035 static struct pci_driver intel_ntb_pci_driver = {
3036 .name = KBUILD_MODNAME,
3037 .id_table = intel_ntb_pci_tbl,
3038 .probe = intel_ntb_pci_probe,
3039 .remove = intel_ntb_pci_remove,
3040 };
3041
3042 static int __init intel_ntb_pci_driver_init(void)
3043 {
3044 pr_info("%s %s\n", NTB_DESC, NTB_VER);
3045
3046 if (debugfs_initialized())
3047 debugfs_dir = debugfs_create_dir(KBUILD_MODNAME, NULL);
3048
3049 return pci_register_driver(&intel_ntb_pci_driver);
3050 }
3051 module_init(intel_ntb_pci_driver_init);
3052
3053 static void __exit intel_ntb_pci_driver_exit(void)
3054 {
3055 pci_unregister_driver(&intel_ntb_pci_driver);
3056
3057 debugfs_remove_recursive(debugfs_dir);
3058 }
3059 module_exit(intel_ntb_pci_driver_exit);
Ubuntu Artful kernel mirror