2 * This file is provided under a dual BSD/GPLv2 license. When using or
3 * redistributing this file, you may do so under either license.
7 * Copyright(c) 2012 Intel Corporation. All rights reserved.
8 * Copyright (C) 2015 EMC Corporation. All Rights Reserved.
10 * This program is free software; you can redistribute it and/or modify
11 * it under the terms of version 2 of the GNU General Public License as
12 * published by the Free Software Foundation.
16 * Copyright(c) 2012 Intel Corporation. All rights reserved.
17 * Copyright (C) 2015 EMC Corporation. All Rights Reserved.
19 * Redistribution and use in source and binary forms, with or without
20 * modification, are permitted provided that the following conditions
23 * * Redistributions of source code must retain the above copyright
24 * notice, this list of conditions and the following disclaimer.
25 * * Redistributions in binary form must reproduce the above copy
26 * notice, this list of conditions and the following disclaimer in
27 * the documentation and/or other materials provided with the
29 * * Neither the name of Intel Corporation nor the names of its
30 * contributors may be used to endorse or promote products derived
31 * from this software without specific prior written permission.
33 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
34 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
35 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
36 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
37 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
38 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
39 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
40 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
41 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
42 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
43 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
45 * Intel PCIe NTB Linux driver
47 * Contact Information:
48 * Jon Mason <jon.mason@intel.com>
51 #include <linux/debugfs.h>
52 #include <linux/delay.h>
53 #include <linux/init.h>
54 #include <linux/interrupt.h>
55 #include <linux/module.h>
56 #include <linux/pci.h>
57 #include <linux/random.h>
58 #include <linux/slab.h>
59 #include <linux/ntb.h>
61 #include "ntb_hw_intel.h"
63 #define NTB_NAME "ntb_hw_intel"
64 #define NTB_DESC "Intel(R) PCI-E Non-Transparent Bridge Driver"
67 MODULE_DESCRIPTION(NTB_DESC
);
68 MODULE_VERSION(NTB_VER
);
69 MODULE_LICENSE("Dual BSD/GPL");
70 MODULE_AUTHOR("Intel Corporation");
72 #define bar0_off(base, bar) ((base) + ((bar) << 2))
73 #define bar2_off(base, bar) bar0_off(base, (bar) - 2)
75 static const struct intel_ntb_reg atom_reg
;
76 static const struct intel_ntb_alt_reg atom_pri_reg
;
77 static const struct intel_ntb_alt_reg atom_sec_reg
;
78 static const struct intel_ntb_alt_reg atom_b2b_reg
;
79 static const struct intel_ntb_xlat_reg atom_pri_xlat
;
80 static const struct intel_ntb_xlat_reg atom_sec_xlat
;
81 static const struct intel_ntb_reg xeon_reg
;
82 static const struct intel_ntb_alt_reg xeon_pri_reg
;
83 static const struct intel_ntb_alt_reg xeon_sec_reg
;
84 static const struct intel_ntb_alt_reg xeon_b2b_reg
;
85 static const struct intel_ntb_xlat_reg xeon_pri_xlat
;
86 static const struct intel_ntb_xlat_reg xeon_sec_xlat
;
87 static struct intel_b2b_addr xeon_b2b_usd_addr
;
88 static struct intel_b2b_addr xeon_b2b_dsd_addr
;
89 static const struct intel_ntb_reg skx_reg
;
90 static const struct intel_ntb_alt_reg skx_pri_reg
;
91 static const struct intel_ntb_alt_reg skx_b2b_reg
;
92 static const struct intel_ntb_xlat_reg skx_sec_xlat
;
93 static const struct ntb_dev_ops intel_ntb_ops
;
94 static const struct ntb_dev_ops intel_ntb3_ops
;
96 static const struct file_operations intel_ntb_debugfs_info
;
97 static struct dentry
*debugfs_dir
;
99 static int b2b_mw_idx
= -1;
100 module_param(b2b_mw_idx
, int, 0644);
101 MODULE_PARM_DESC(b2b_mw_idx
, "Use this mw idx to access the peer ntb. A "
102 "value of zero or positive starts from first mw idx, and a "
103 "negative value starts from last mw idx. Both sides MUST "
104 "set the same value here!");
106 static unsigned int b2b_mw_share
;
107 module_param(b2b_mw_share
, uint
, 0644);
108 MODULE_PARM_DESC(b2b_mw_share
, "If the b2b mw is large enough, configure the "
109 "ntb so that the peer ntb only occupies the first half of "
110 "the mw, so the second half can still be used as a mw. Both "
111 "sides MUST set the same value here!");
113 module_param_named(xeon_b2b_usd_bar2_addr64
,
114 xeon_b2b_usd_addr
.bar2_addr64
, ullong
, 0644);
115 MODULE_PARM_DESC(xeon_b2b_usd_bar2_addr64
,
116 "XEON B2B USD BAR 2 64-bit address");
118 module_param_named(xeon_b2b_usd_bar4_addr64
,
119 xeon_b2b_usd_addr
.bar4_addr64
, ullong
, 0644);
120 MODULE_PARM_DESC(xeon_b2b_usd_bar4_addr64
,
121 "XEON B2B USD BAR 4 64-bit address");
123 module_param_named(xeon_b2b_usd_bar4_addr32
,
124 xeon_b2b_usd_addr
.bar4_addr32
, ullong
, 0644);
125 MODULE_PARM_DESC(xeon_b2b_usd_bar4_addr32
,
126 "XEON B2B USD split-BAR 4 32-bit address");
128 module_param_named(xeon_b2b_usd_bar5_addr32
,
129 xeon_b2b_usd_addr
.bar5_addr32
, ullong
, 0644);
130 MODULE_PARM_DESC(xeon_b2b_usd_bar5_addr32
,
131 "XEON B2B USD split-BAR 5 32-bit address");
133 module_param_named(xeon_b2b_dsd_bar2_addr64
,
134 xeon_b2b_dsd_addr
.bar2_addr64
, ullong
, 0644);
135 MODULE_PARM_DESC(xeon_b2b_dsd_bar2_addr64
,
136 "XEON B2B DSD BAR 2 64-bit address");
138 module_param_named(xeon_b2b_dsd_bar4_addr64
,
139 xeon_b2b_dsd_addr
.bar4_addr64
, ullong
, 0644);
140 MODULE_PARM_DESC(xeon_b2b_dsd_bar4_addr64
,
141 "XEON B2B DSD BAR 4 64-bit address");
143 module_param_named(xeon_b2b_dsd_bar4_addr32
,
144 xeon_b2b_dsd_addr
.bar4_addr32
, ullong
, 0644);
145 MODULE_PARM_DESC(xeon_b2b_dsd_bar4_addr32
,
146 "XEON B2B DSD split-BAR 4 32-bit address");
148 module_param_named(xeon_b2b_dsd_bar5_addr32
,
149 xeon_b2b_dsd_addr
.bar5_addr32
, ullong
, 0644);
150 MODULE_PARM_DESC(xeon_b2b_dsd_bar5_addr32
,
151 "XEON B2B DSD split-BAR 5 32-bit address");
153 static inline enum ntb_topo
xeon_ppd_topo(struct intel_ntb_dev
*ndev
, u8 ppd
);
154 static int xeon_init_isr(struct intel_ntb_dev
*ndev
);
158 #define ioread64 readq
160 #define ioread64 _ioread64
161 static inline u64
_ioread64(void __iomem
*mmio
)
165 low
= ioread32(mmio
);
166 high
= ioread32(mmio
+ sizeof(u32
));
167 return low
| (high
<< 32);
174 #define iowrite64 writeq
176 #define iowrite64 _iowrite64
177 static inline void _iowrite64(u64 val
, void __iomem
*mmio
)
179 iowrite32(val
, mmio
);
180 iowrite32(val
>> 32, mmio
+ sizeof(u32
));
185 static inline int pdev_is_atom(struct pci_dev
*pdev
)
187 switch (pdev
->device
) {
188 case PCI_DEVICE_ID_INTEL_NTB_B2B_BWD
:
194 static inline int pdev_is_xeon(struct pci_dev
*pdev
)
196 switch (pdev
->device
) {
197 case PCI_DEVICE_ID_INTEL_NTB_SS_JSF
:
198 case PCI_DEVICE_ID_INTEL_NTB_SS_SNB
:
199 case PCI_DEVICE_ID_INTEL_NTB_SS_IVT
:
200 case PCI_DEVICE_ID_INTEL_NTB_SS_HSX
:
201 case PCI_DEVICE_ID_INTEL_NTB_SS_BDX
:
202 case PCI_DEVICE_ID_INTEL_NTB_PS_JSF
:
203 case PCI_DEVICE_ID_INTEL_NTB_PS_SNB
:
204 case PCI_DEVICE_ID_INTEL_NTB_PS_IVT
:
205 case PCI_DEVICE_ID_INTEL_NTB_PS_HSX
:
206 case PCI_DEVICE_ID_INTEL_NTB_PS_BDX
:
207 case PCI_DEVICE_ID_INTEL_NTB_B2B_JSF
:
208 case PCI_DEVICE_ID_INTEL_NTB_B2B_SNB
:
209 case PCI_DEVICE_ID_INTEL_NTB_B2B_IVT
:
210 case PCI_DEVICE_ID_INTEL_NTB_B2B_HSX
:
211 case PCI_DEVICE_ID_INTEL_NTB_B2B_BDX
:
217 static inline int pdev_is_skx_xeon(struct pci_dev
*pdev
)
219 if (pdev
->device
== PCI_DEVICE_ID_INTEL_NTB_B2B_SKX
)
225 static inline void ndev_reset_unsafe_flags(struct intel_ntb_dev
*ndev
)
227 ndev
->unsafe_flags
= 0;
228 ndev
->unsafe_flags_ignore
= 0;
230 /* Only B2B has a workaround to avoid SDOORBELL */
231 if (ndev
->hwerr_flags
& NTB_HWERR_SDOORBELL_LOCKUP
)
232 if (!ntb_topo_is_b2b(ndev
->ntb
.topo
))
233 ndev
->unsafe_flags
|= NTB_UNSAFE_DB
;
235 /* No low level workaround to avoid SB01BASE */
236 if (ndev
->hwerr_flags
& NTB_HWERR_SB01BASE_LOCKUP
) {
237 ndev
->unsafe_flags
|= NTB_UNSAFE_DB
;
238 ndev
->unsafe_flags
|= NTB_UNSAFE_SPAD
;
242 static inline int ndev_is_unsafe(struct intel_ntb_dev
*ndev
,
245 return !!(flag
& ndev
->unsafe_flags
& ~ndev
->unsafe_flags_ignore
);
248 static inline int ndev_ignore_unsafe(struct intel_ntb_dev
*ndev
,
251 flag
&= ndev
->unsafe_flags
;
252 ndev
->unsafe_flags_ignore
|= flag
;
257 static int ndev_mw_to_bar(struct intel_ntb_dev
*ndev
, int idx
)
259 if (idx
< 0 || idx
>= ndev
->mw_count
)
261 return ndev
->reg
->mw_bar
[idx
];
264 static inline int ndev_db_addr(struct intel_ntb_dev
*ndev
,
265 phys_addr_t
*db_addr
, resource_size_t
*db_size
,
266 phys_addr_t reg_addr
, unsigned long reg
)
268 if (ndev_is_unsafe(ndev
, NTB_UNSAFE_DB
))
269 pr_warn_once("%s: NTB unsafe doorbell access", __func__
);
272 *db_addr
= reg_addr
+ reg
;
273 dev_dbg(ndev_dev(ndev
), "Peer db addr %llx\n", *db_addr
);
277 *db_size
= ndev
->reg
->db_size
;
278 dev_dbg(ndev_dev(ndev
), "Peer db size %llx\n", *db_size
);
284 static inline u64
ndev_db_read(struct intel_ntb_dev
*ndev
,
287 if (ndev_is_unsafe(ndev
, NTB_UNSAFE_DB
))
288 pr_warn_once("%s: NTB unsafe doorbell access", __func__
);
290 return ndev
->reg
->db_ioread(mmio
);
293 static inline int ndev_db_write(struct intel_ntb_dev
*ndev
, u64 db_bits
,
296 if (ndev_is_unsafe(ndev
, NTB_UNSAFE_DB
))
297 pr_warn_once("%s: NTB unsafe doorbell access", __func__
);
299 if (db_bits
& ~ndev
->db_valid_mask
)
302 ndev
->reg
->db_iowrite(db_bits
, mmio
);
307 static inline int ndev_db_set_mask(struct intel_ntb_dev
*ndev
, u64 db_bits
,
310 unsigned long irqflags
;
312 if (ndev_is_unsafe(ndev
, NTB_UNSAFE_DB
))
313 pr_warn_once("%s: NTB unsafe doorbell access", __func__
);
315 if (db_bits
& ~ndev
->db_valid_mask
)
318 spin_lock_irqsave(&ndev
->db_mask_lock
, irqflags
);
320 ndev
->db_mask
|= db_bits
;
321 ndev
->reg
->db_iowrite(ndev
->db_mask
, mmio
);
323 spin_unlock_irqrestore(&ndev
->db_mask_lock
, irqflags
);
328 static inline int ndev_db_clear_mask(struct intel_ntb_dev
*ndev
, u64 db_bits
,
331 unsigned long irqflags
;
333 if (ndev_is_unsafe(ndev
, NTB_UNSAFE_DB
))
334 pr_warn_once("%s: NTB unsafe doorbell access", __func__
);
336 if (db_bits
& ~ndev
->db_valid_mask
)
339 spin_lock_irqsave(&ndev
->db_mask_lock
, irqflags
);
341 ndev
->db_mask
&= ~db_bits
;
342 ndev
->reg
->db_iowrite(ndev
->db_mask
, mmio
);
344 spin_unlock_irqrestore(&ndev
->db_mask_lock
, irqflags
);
349 static inline int ndev_vec_mask(struct intel_ntb_dev
*ndev
, int db_vector
)
353 shift
= ndev
->db_vec_shift
;
354 mask
= BIT_ULL(shift
) - 1;
356 return mask
<< (shift
* db_vector
);
359 static inline int ndev_spad_addr(struct intel_ntb_dev
*ndev
, int idx
,
360 phys_addr_t
*spad_addr
, phys_addr_t reg_addr
,
363 if (ndev_is_unsafe(ndev
, NTB_UNSAFE_SPAD
))
364 pr_warn_once("%s: NTB unsafe scratchpad access", __func__
);
366 if (idx
< 0 || idx
>= ndev
->spad_count
)
370 *spad_addr
= reg_addr
+ reg
+ (idx
<< 2);
371 dev_dbg(ndev_dev(ndev
), "Peer spad addr %llx\n", *spad_addr
);
377 static inline u32
ndev_spad_read(struct intel_ntb_dev
*ndev
, int idx
,
380 if (ndev_is_unsafe(ndev
, NTB_UNSAFE_SPAD
))
381 pr_warn_once("%s: NTB unsafe scratchpad access", __func__
);
383 if (idx
< 0 || idx
>= ndev
->spad_count
)
386 return ioread32(mmio
+ (idx
<< 2));
389 static inline int ndev_spad_write(struct intel_ntb_dev
*ndev
, int idx
, u32 val
,
392 if (ndev_is_unsafe(ndev
, NTB_UNSAFE_SPAD
))
393 pr_warn_once("%s: NTB unsafe scratchpad access", __func__
);
395 if (idx
< 0 || idx
>= ndev
->spad_count
)
398 iowrite32(val
, mmio
+ (idx
<< 2));
403 static irqreturn_t
ndev_interrupt(struct intel_ntb_dev
*ndev
, int vec
)
407 vec_mask
= ndev_vec_mask(ndev
, vec
);
409 if ((ndev
->hwerr_flags
& NTB_HWERR_MSIX_VECTOR32_BAD
) && (vec
== 31))
410 vec_mask
|= ndev
->db_link_mask
;
412 dev_dbg(ndev_dev(ndev
), "vec %d vec_mask %llx\n", vec
, vec_mask
);
414 ndev
->last_ts
= jiffies
;
416 if (vec_mask
& ndev
->db_link_mask
) {
417 if (ndev
->reg
->poll_link(ndev
))
418 ntb_link_event(&ndev
->ntb
);
421 if (vec_mask
& ndev
->db_valid_mask
)
422 ntb_db_event(&ndev
->ntb
, vec
);
427 static irqreturn_t
ndev_vec_isr(int irq
, void *dev
)
429 struct intel_ntb_vec
*nvec
= dev
;
431 dev_dbg(ndev_dev(nvec
->ndev
), "irq: %d nvec->num: %d\n",
434 return ndev_interrupt(nvec
->ndev
, nvec
->num
);
437 static irqreturn_t
ndev_irq_isr(int irq
, void *dev
)
439 struct intel_ntb_dev
*ndev
= dev
;
441 return ndev_interrupt(ndev
, irq
- ndev_pdev(ndev
)->irq
);
444 static int ndev_init_isr(struct intel_ntb_dev
*ndev
,
445 int msix_min
, int msix_max
,
446 int msix_shift
, int total_shift
)
448 struct pci_dev
*pdev
;
449 int rc
, i
, msix_count
, node
;
451 pdev
= ndev_pdev(ndev
);
453 node
= dev_to_node(&pdev
->dev
);
455 /* Mask all doorbell interrupts */
456 ndev
->db_mask
= ndev
->db_valid_mask
;
457 ndev
->reg
->db_iowrite(ndev
->db_mask
,
459 ndev
->self_reg
->db_mask
);
461 /* Try to set up msix irq */
463 ndev
->vec
= kzalloc_node(msix_max
* sizeof(*ndev
->vec
),
466 goto err_msix_vec_alloc
;
468 ndev
->msix
= kzalloc_node(msix_max
* sizeof(*ndev
->msix
),
473 for (i
= 0; i
< msix_max
; ++i
)
474 ndev
->msix
[i
].entry
= i
;
476 msix_count
= pci_enable_msix_range(pdev
, ndev
->msix
,
479 goto err_msix_enable
;
481 for (i
= 0; i
< msix_count
; ++i
) {
482 ndev
->vec
[i
].ndev
= ndev
;
483 ndev
->vec
[i
].num
= i
;
484 rc
= request_irq(ndev
->msix
[i
].vector
, ndev_vec_isr
, 0,
485 "ndev_vec_isr", &ndev
->vec
[i
]);
487 goto err_msix_request
;
490 dev_dbg(ndev_dev(ndev
), "Using %d msix interrupts\n", msix_count
);
491 ndev
->db_vec_count
= msix_count
;
492 ndev
->db_vec_shift
= msix_shift
;
497 free_irq(ndev
->msix
[i
].vector
, &ndev
->vec
[i
]);
498 pci_disable_msix(pdev
);
507 /* Try to set up msi irq */
509 rc
= pci_enable_msi(pdev
);
513 rc
= request_irq(pdev
->irq
, ndev_irq_isr
, 0,
514 "ndev_irq_isr", ndev
);
516 goto err_msi_request
;
518 dev_dbg(ndev_dev(ndev
), "Using msi interrupts\n");
519 ndev
->db_vec_count
= 1;
520 ndev
->db_vec_shift
= total_shift
;
524 pci_disable_msi(pdev
);
527 /* Try to set up intx irq */
531 rc
= request_irq(pdev
->irq
, ndev_irq_isr
, IRQF_SHARED
,
532 "ndev_irq_isr", ndev
);
534 goto err_intx_request
;
536 dev_dbg(ndev_dev(ndev
), "Using intx interrupts\n");
537 ndev
->db_vec_count
= 1;
538 ndev
->db_vec_shift
= total_shift
;
545 static void ndev_deinit_isr(struct intel_ntb_dev
*ndev
)
547 struct pci_dev
*pdev
;
550 pdev
= ndev_pdev(ndev
);
552 /* Mask all doorbell interrupts */
553 ndev
->db_mask
= ndev
->db_valid_mask
;
554 ndev
->reg
->db_iowrite(ndev
->db_mask
,
556 ndev
->self_reg
->db_mask
);
559 i
= ndev
->db_vec_count
;
561 free_irq(ndev
->msix
[i
].vector
, &ndev
->vec
[i
]);
562 pci_disable_msix(pdev
);
566 free_irq(pdev
->irq
, ndev
);
567 if (pci_dev_msi_enabled(pdev
))
568 pci_disable_msi(pdev
);
572 static ssize_t
ndev_ntb3_debugfs_read(struct file
*filp
, char __user
*ubuf
,
573 size_t count
, loff_t
*offp
)
575 struct intel_ntb_dev
*ndev
;
580 union { u64 v64
; u32 v32
; u16 v16
; } u
;
582 ndev
= filp
->private_data
;
583 mmio
= ndev
->self_mmio
;
585 buf_size
= min(count
, 0x800ul
);
587 buf
= kmalloc(buf_size
, GFP_KERNEL
);
593 off
+= scnprintf(buf
+ off
, buf_size
- off
,
594 "NTB Device Information:\n");
596 off
+= scnprintf(buf
+ off
, buf_size
- off
,
597 "Connection Topology -\t%s\n",
598 ntb_topo_string(ndev
->ntb
.topo
));
600 off
+= scnprintf(buf
+ off
, buf_size
- off
,
601 "NTB CTL -\t\t%#06x\n", ndev
->ntb_ctl
);
602 off
+= scnprintf(buf
+ off
, buf_size
- off
,
603 "LNK STA -\t\t%#06x\n", ndev
->lnk_sta
);
605 if (!ndev
->reg
->link_is_up(ndev
))
606 off
+= scnprintf(buf
+ off
, buf_size
- off
,
607 "Link Status -\t\tDown\n");
609 off
+= scnprintf(buf
+ off
, buf_size
- off
,
610 "Link Status -\t\tUp\n");
611 off
+= scnprintf(buf
+ off
, buf_size
- off
,
612 "Link Speed -\t\tPCI-E Gen %u\n",
613 NTB_LNK_STA_SPEED(ndev
->lnk_sta
));
614 off
+= scnprintf(buf
+ off
, buf_size
- off
,
615 "Link Width -\t\tx%u\n",
616 NTB_LNK_STA_WIDTH(ndev
->lnk_sta
));
619 off
+= scnprintf(buf
+ off
, buf_size
- off
,
620 "Memory Window Count -\t%u\n", ndev
->mw_count
);
621 off
+= scnprintf(buf
+ off
, buf_size
- off
,
622 "Scratchpad Count -\t%u\n", ndev
->spad_count
);
623 off
+= scnprintf(buf
+ off
, buf_size
- off
,
624 "Doorbell Count -\t%u\n", ndev
->db_count
);
625 off
+= scnprintf(buf
+ off
, buf_size
- off
,
626 "Doorbell Vector Count -\t%u\n", ndev
->db_vec_count
);
627 off
+= scnprintf(buf
+ off
, buf_size
- off
,
628 "Doorbell Vector Shift -\t%u\n", ndev
->db_vec_shift
);
630 off
+= scnprintf(buf
+ off
, buf_size
- off
,
631 "Doorbell Valid Mask -\t%#llx\n", ndev
->db_valid_mask
);
632 off
+= scnprintf(buf
+ off
, buf_size
- off
,
633 "Doorbell Link Mask -\t%#llx\n", ndev
->db_link_mask
);
634 off
+= scnprintf(buf
+ off
, buf_size
- off
,
635 "Doorbell Mask Cached -\t%#llx\n", ndev
->db_mask
);
637 u
.v64
= ndev_db_read(ndev
, mmio
+ ndev
->self_reg
->db_mask
);
638 off
+= scnprintf(buf
+ off
, buf_size
- off
,
639 "Doorbell Mask -\t\t%#llx\n", u
.v64
);
641 u
.v64
= ndev_db_read(ndev
, mmio
+ ndev
->self_reg
->db_bell
);
642 off
+= scnprintf(buf
+ off
, buf_size
- off
,
643 "Doorbell Bell -\t\t%#llx\n", u
.v64
);
645 off
+= scnprintf(buf
+ off
, buf_size
- off
,
646 "\nNTB Incoming XLAT:\n");
648 u
.v64
= ioread64(mmio
+ SKX_IMBAR1XBASE_OFFSET
);
649 off
+= scnprintf(buf
+ off
, buf_size
- off
,
650 "IMBAR1XBASE -\t\t%#018llx\n", u
.v64
);
652 u
.v64
= ioread64(mmio
+ SKX_IMBAR2XBASE_OFFSET
);
653 off
+= scnprintf(buf
+ off
, buf_size
- off
,
654 "IMBAR2XBASE -\t\t%#018llx\n", u
.v64
);
656 u
.v64
= ioread64(mmio
+ SKX_IMBAR1XLMT_OFFSET
);
657 off
+= scnprintf(buf
+ off
, buf_size
- off
,
658 "IMBAR1XLMT -\t\t\t%#018llx\n", u
.v64
);
660 u
.v64
= ioread64(mmio
+ SKX_IMBAR2XLMT_OFFSET
);
661 off
+= scnprintf(buf
+ off
, buf_size
- off
,
662 "IMBAR2XLMT -\t\t\t%#018llx\n", u
.v64
);
664 if (ntb_topo_is_b2b(ndev
->ntb
.topo
)) {
665 off
+= scnprintf(buf
+ off
, buf_size
- off
,
666 "\nNTB Outgoing B2B XLAT:\n");
668 u
.v64
= ioread64(mmio
+ SKX_EMBAR1XBASE_OFFSET
);
669 off
+= scnprintf(buf
+ off
, buf_size
- off
,
670 "EMBAR1XBASE -\t\t%#018llx\n", u
.v64
);
672 u
.v64
= ioread64(mmio
+ SKX_EMBAR2XBASE_OFFSET
);
673 off
+= scnprintf(buf
+ off
, buf_size
- off
,
674 "EMBAR2XBASE -\t\t%#018llx\n", u
.v64
);
676 u
.v64
= ioread64(mmio
+ SKX_EMBAR1XLMT_OFFSET
);
677 off
+= scnprintf(buf
+ off
, buf_size
- off
,
678 "EMBAR1XLMT -\t\t%#018llx\n", u
.v64
);
680 u
.v64
= ioread64(mmio
+ SKX_EMBAR2XLMT_OFFSET
);
681 off
+= scnprintf(buf
+ off
, buf_size
- off
,
682 "EMBAR2XLMT -\t\t%#018llx\n", u
.v64
);
684 off
+= scnprintf(buf
+ off
, buf_size
- off
,
685 "\nNTB Secondary BAR:\n");
687 u
.v64
= ioread64(mmio
+ SKX_EMBAR0_OFFSET
);
688 off
+= scnprintf(buf
+ off
, buf_size
- off
,
689 "EMBAR0 -\t\t%#018llx\n", u
.v64
);
691 u
.v64
= ioread64(mmio
+ SKX_EMBAR1_OFFSET
);
692 off
+= scnprintf(buf
+ off
, buf_size
- off
,
693 "EMBAR1 -\t\t%#018llx\n", u
.v64
);
695 u
.v64
= ioread64(mmio
+ SKX_EMBAR2_OFFSET
);
696 off
+= scnprintf(buf
+ off
, buf_size
- off
,
697 "EMBAR2 -\t\t%#018llx\n", u
.v64
);
700 off
+= scnprintf(buf
+ off
, buf_size
- off
,
701 "\nNTB Statistics:\n");
703 u
.v16
= ioread16(mmio
+ SKX_USMEMMISS_OFFSET
);
704 off
+= scnprintf(buf
+ off
, buf_size
- off
,
705 "Upstream Memory Miss -\t%u\n", u
.v16
);
707 off
+= scnprintf(buf
+ off
, buf_size
- off
,
708 "\nNTB Hardware Errors:\n");
710 if (!pci_read_config_word(ndev
->ntb
.pdev
,
711 SKX_DEVSTS_OFFSET
, &u
.v16
))
712 off
+= scnprintf(buf
+ off
, buf_size
- off
,
713 "DEVSTS -\t\t%#06x\n", u
.v16
);
715 if (!pci_read_config_word(ndev
->ntb
.pdev
,
716 SKX_LINK_STATUS_OFFSET
, &u
.v16
))
717 off
+= scnprintf(buf
+ off
, buf_size
- off
,
718 "LNKSTS -\t\t%#06x\n", u
.v16
);
720 if (!pci_read_config_dword(ndev
->ntb
.pdev
,
721 SKX_UNCERRSTS_OFFSET
, &u
.v32
))
722 off
+= scnprintf(buf
+ off
, buf_size
- off
,
723 "UNCERRSTS -\t\t%#06x\n", u
.v32
);
725 if (!pci_read_config_dword(ndev
->ntb
.pdev
,
726 SKX_CORERRSTS_OFFSET
, &u
.v32
))
727 off
+= scnprintf(buf
+ off
, buf_size
- off
,
728 "CORERRSTS -\t\t%#06x\n", u
.v32
);
730 ret
= simple_read_from_buffer(ubuf
, count
, offp
, buf
, off
);
735 static ssize_t
ndev_ntb_debugfs_read(struct file
*filp
, char __user
*ubuf
,
736 size_t count
, loff_t
*offp
)
738 struct intel_ntb_dev
*ndev
;
739 struct pci_dev
*pdev
;
744 union { u64 v64
; u32 v32
; u16 v16
; u8 v8
; } u
;
746 ndev
= filp
->private_data
;
747 pdev
= ndev_pdev(ndev
);
748 mmio
= ndev
->self_mmio
;
750 buf_size
= min(count
, 0x800ul
);
752 buf
= kmalloc(buf_size
, GFP_KERNEL
);
758 off
+= scnprintf(buf
+ off
, buf_size
- off
,
759 "NTB Device Information:\n");
761 off
+= scnprintf(buf
+ off
, buf_size
- off
,
762 "Connection Topology -\t%s\n",
763 ntb_topo_string(ndev
->ntb
.topo
));
765 if (ndev
->b2b_idx
!= UINT_MAX
) {
766 off
+= scnprintf(buf
+ off
, buf_size
- off
,
767 "B2B MW Idx -\t\t%u\n", ndev
->b2b_idx
);
768 off
+= scnprintf(buf
+ off
, buf_size
- off
,
769 "B2B Offset -\t\t%#lx\n", ndev
->b2b_off
);
772 off
+= scnprintf(buf
+ off
, buf_size
- off
,
773 "BAR4 Split -\t\t%s\n",
774 ndev
->bar4_split
? "yes" : "no");
776 off
+= scnprintf(buf
+ off
, buf_size
- off
,
777 "NTB CTL -\t\t%#06x\n", ndev
->ntb_ctl
);
778 off
+= scnprintf(buf
+ off
, buf_size
- off
,
779 "LNK STA -\t\t%#06x\n", ndev
->lnk_sta
);
781 if (!ndev
->reg
->link_is_up(ndev
)) {
782 off
+= scnprintf(buf
+ off
, buf_size
- off
,
783 "Link Status -\t\tDown\n");
785 off
+= scnprintf(buf
+ off
, buf_size
- off
,
786 "Link Status -\t\tUp\n");
787 off
+= scnprintf(buf
+ off
, buf_size
- off
,
788 "Link Speed -\t\tPCI-E Gen %u\n",
789 NTB_LNK_STA_SPEED(ndev
->lnk_sta
));
790 off
+= scnprintf(buf
+ off
, buf_size
- off
,
791 "Link Width -\t\tx%u\n",
792 NTB_LNK_STA_WIDTH(ndev
->lnk_sta
));
795 off
+= scnprintf(buf
+ off
, buf_size
- off
,
796 "Memory Window Count -\t%u\n", ndev
->mw_count
);
797 off
+= scnprintf(buf
+ off
, buf_size
- off
,
798 "Scratchpad Count -\t%u\n", ndev
->spad_count
);
799 off
+= scnprintf(buf
+ off
, buf_size
- off
,
800 "Doorbell Count -\t%u\n", ndev
->db_count
);
801 off
+= scnprintf(buf
+ off
, buf_size
- off
,
802 "Doorbell Vector Count -\t%u\n", ndev
->db_vec_count
);
803 off
+= scnprintf(buf
+ off
, buf_size
- off
,
804 "Doorbell Vector Shift -\t%u\n", ndev
->db_vec_shift
);
806 off
+= scnprintf(buf
+ off
, buf_size
- off
,
807 "Doorbell Valid Mask -\t%#llx\n", ndev
->db_valid_mask
);
808 off
+= scnprintf(buf
+ off
, buf_size
- off
,
809 "Doorbell Link Mask -\t%#llx\n", ndev
->db_link_mask
);
810 off
+= scnprintf(buf
+ off
, buf_size
- off
,
811 "Doorbell Mask Cached -\t%#llx\n", ndev
->db_mask
);
813 u
.v64
= ndev_db_read(ndev
, mmio
+ ndev
->self_reg
->db_mask
);
814 off
+= scnprintf(buf
+ off
, buf_size
- off
,
815 "Doorbell Mask -\t\t%#llx\n", u
.v64
);
817 u
.v64
= ndev_db_read(ndev
, mmio
+ ndev
->self_reg
->db_bell
);
818 off
+= scnprintf(buf
+ off
, buf_size
- off
,
819 "Doorbell Bell -\t\t%#llx\n", u
.v64
);
821 off
+= scnprintf(buf
+ off
, buf_size
- off
,
822 "\nNTB Window Size:\n");
824 pci_read_config_byte(pdev
, XEON_PBAR23SZ_OFFSET
, &u
.v8
);
825 off
+= scnprintf(buf
+ off
, buf_size
- off
,
826 "PBAR23SZ %hhu\n", u
.v8
);
827 if (!ndev
->bar4_split
) {
828 pci_read_config_byte(pdev
, XEON_PBAR45SZ_OFFSET
, &u
.v8
);
829 off
+= scnprintf(buf
+ off
, buf_size
- off
,
830 "PBAR45SZ %hhu\n", u
.v8
);
832 pci_read_config_byte(pdev
, XEON_PBAR4SZ_OFFSET
, &u
.v8
);
833 off
+= scnprintf(buf
+ off
, buf_size
- off
,
834 "PBAR4SZ %hhu\n", u
.v8
);
835 pci_read_config_byte(pdev
, XEON_PBAR5SZ_OFFSET
, &u
.v8
);
836 off
+= scnprintf(buf
+ off
, buf_size
- off
,
837 "PBAR5SZ %hhu\n", u
.v8
);
840 pci_read_config_byte(pdev
, XEON_SBAR23SZ_OFFSET
, &u
.v8
);
841 off
+= scnprintf(buf
+ off
, buf_size
- off
,
842 "SBAR23SZ %hhu\n", u
.v8
);
843 if (!ndev
->bar4_split
) {
844 pci_read_config_byte(pdev
, XEON_SBAR45SZ_OFFSET
, &u
.v8
);
845 off
+= scnprintf(buf
+ off
, buf_size
- off
,
846 "SBAR45SZ %hhu\n", u
.v8
);
848 pci_read_config_byte(pdev
, XEON_SBAR4SZ_OFFSET
, &u
.v8
);
849 off
+= scnprintf(buf
+ off
, buf_size
- off
,
850 "SBAR4SZ %hhu\n", u
.v8
);
851 pci_read_config_byte(pdev
, XEON_SBAR5SZ_OFFSET
, &u
.v8
);
852 off
+= scnprintf(buf
+ off
, buf_size
- off
,
853 "SBAR5SZ %hhu\n", u
.v8
);
856 off
+= scnprintf(buf
+ off
, buf_size
- off
,
857 "\nNTB Incoming XLAT:\n");
859 u
.v64
= ioread64(mmio
+ bar2_off(ndev
->xlat_reg
->bar2_xlat
, 2));
860 off
+= scnprintf(buf
+ off
, buf_size
- off
,
861 "XLAT23 -\t\t%#018llx\n", u
.v64
);
863 if (ndev
->bar4_split
) {
864 u
.v32
= ioread32(mmio
+ bar2_off(ndev
->xlat_reg
->bar2_xlat
, 4));
865 off
+= scnprintf(buf
+ off
, buf_size
- off
,
866 "XLAT4 -\t\t\t%#06x\n", u
.v32
);
868 u
.v32
= ioread32(mmio
+ bar2_off(ndev
->xlat_reg
->bar2_xlat
, 5));
869 off
+= scnprintf(buf
+ off
, buf_size
- off
,
870 "XLAT5 -\t\t\t%#06x\n", u
.v32
);
872 u
.v64
= ioread64(mmio
+ bar2_off(ndev
->xlat_reg
->bar2_xlat
, 4));
873 off
+= scnprintf(buf
+ off
, buf_size
- off
,
874 "XLAT45 -\t\t%#018llx\n", u
.v64
);
877 u
.v64
= ioread64(mmio
+ bar2_off(ndev
->xlat_reg
->bar2_limit
, 2));
878 off
+= scnprintf(buf
+ off
, buf_size
- off
,
879 "LMT23 -\t\t\t%#018llx\n", u
.v64
);
881 if (ndev
->bar4_split
) {
882 u
.v32
= ioread32(mmio
+ bar2_off(ndev
->xlat_reg
->bar2_limit
, 4));
883 off
+= scnprintf(buf
+ off
, buf_size
- off
,
884 "LMT4 -\t\t\t%#06x\n", u
.v32
);
885 u
.v32
= ioread32(mmio
+ bar2_off(ndev
->xlat_reg
->bar2_limit
, 5));
886 off
+= scnprintf(buf
+ off
, buf_size
- off
,
887 "LMT5 -\t\t\t%#06x\n", u
.v32
);
889 u
.v64
= ioread64(mmio
+ bar2_off(ndev
->xlat_reg
->bar2_limit
, 4));
890 off
+= scnprintf(buf
+ off
, buf_size
- off
,
891 "LMT45 -\t\t\t%#018llx\n", u
.v64
);
894 if (pdev_is_xeon(pdev
)) {
895 if (ntb_topo_is_b2b(ndev
->ntb
.topo
)) {
896 off
+= scnprintf(buf
+ off
, buf_size
- off
,
897 "\nNTB Outgoing B2B XLAT:\n");
899 u
.v64
= ioread64(mmio
+ XEON_PBAR23XLAT_OFFSET
);
900 off
+= scnprintf(buf
+ off
, buf_size
- off
,
901 "B2B XLAT23 -\t\t%#018llx\n", u
.v64
);
903 if (ndev
->bar4_split
) {
904 u
.v32
= ioread32(mmio
+ XEON_PBAR4XLAT_OFFSET
);
905 off
+= scnprintf(buf
+ off
, buf_size
- off
,
906 "B2B XLAT4 -\t\t%#06x\n",
908 u
.v32
= ioread32(mmio
+ XEON_PBAR5XLAT_OFFSET
);
909 off
+= scnprintf(buf
+ off
, buf_size
- off
,
910 "B2B XLAT5 -\t\t%#06x\n",
913 u
.v64
= ioread64(mmio
+ XEON_PBAR45XLAT_OFFSET
);
914 off
+= scnprintf(buf
+ off
, buf_size
- off
,
915 "B2B XLAT45 -\t\t%#018llx\n",
919 u
.v64
= ioread64(mmio
+ XEON_PBAR23LMT_OFFSET
);
920 off
+= scnprintf(buf
+ off
, buf_size
- off
,
921 "B2B LMT23 -\t\t%#018llx\n", u
.v64
);
923 if (ndev
->bar4_split
) {
924 u
.v32
= ioread32(mmio
+ XEON_PBAR4LMT_OFFSET
);
925 off
+= scnprintf(buf
+ off
, buf_size
- off
,
926 "B2B LMT4 -\t\t%#06x\n",
928 u
.v32
= ioread32(mmio
+ XEON_PBAR5LMT_OFFSET
);
929 off
+= scnprintf(buf
+ off
, buf_size
- off
,
930 "B2B LMT5 -\t\t%#06x\n",
933 u
.v64
= ioread64(mmio
+ XEON_PBAR45LMT_OFFSET
);
934 off
+= scnprintf(buf
+ off
, buf_size
- off
,
935 "B2B LMT45 -\t\t%#018llx\n",
939 off
+= scnprintf(buf
+ off
, buf_size
- off
,
940 "\nNTB Secondary BAR:\n");
942 u
.v64
= ioread64(mmio
+ XEON_SBAR0BASE_OFFSET
);
943 off
+= scnprintf(buf
+ off
, buf_size
- off
,
944 "SBAR01 -\t\t%#018llx\n", u
.v64
);
946 u
.v64
= ioread64(mmio
+ XEON_SBAR23BASE_OFFSET
);
947 off
+= scnprintf(buf
+ off
, buf_size
- off
,
948 "SBAR23 -\t\t%#018llx\n", u
.v64
);
950 if (ndev
->bar4_split
) {
951 u
.v32
= ioread32(mmio
+ XEON_SBAR4BASE_OFFSET
);
952 off
+= scnprintf(buf
+ off
, buf_size
- off
,
953 "SBAR4 -\t\t\t%#06x\n", u
.v32
);
954 u
.v32
= ioread32(mmio
+ XEON_SBAR5BASE_OFFSET
);
955 off
+= scnprintf(buf
+ off
, buf_size
- off
,
956 "SBAR5 -\t\t\t%#06x\n", u
.v32
);
958 u
.v64
= ioread64(mmio
+ XEON_SBAR45BASE_OFFSET
);
959 off
+= scnprintf(buf
+ off
, buf_size
- off
,
960 "SBAR45 -\t\t%#018llx\n",
965 off
+= scnprintf(buf
+ off
, buf_size
- off
,
966 "\nXEON NTB Statistics:\n");
968 u
.v16
= ioread16(mmio
+ XEON_USMEMMISS_OFFSET
);
969 off
+= scnprintf(buf
+ off
, buf_size
- off
,
970 "Upstream Memory Miss -\t%u\n", u
.v16
);
972 off
+= scnprintf(buf
+ off
, buf_size
- off
,
973 "\nXEON NTB Hardware Errors:\n");
975 if (!pci_read_config_word(pdev
,
976 XEON_DEVSTS_OFFSET
, &u
.v16
))
977 off
+= scnprintf(buf
+ off
, buf_size
- off
,
978 "DEVSTS -\t\t%#06x\n", u
.v16
);
980 if (!pci_read_config_word(pdev
,
981 XEON_LINK_STATUS_OFFSET
, &u
.v16
))
982 off
+= scnprintf(buf
+ off
, buf_size
- off
,
983 "LNKSTS -\t\t%#06x\n", u
.v16
);
985 if (!pci_read_config_dword(pdev
,
986 XEON_UNCERRSTS_OFFSET
, &u
.v32
))
987 off
+= scnprintf(buf
+ off
, buf_size
- off
,
988 "UNCERRSTS -\t\t%#06x\n", u
.v32
);
990 if (!pci_read_config_dword(pdev
,
991 XEON_CORERRSTS_OFFSET
, &u
.v32
))
992 off
+= scnprintf(buf
+ off
, buf_size
- off
,
993 "CORERRSTS -\t\t%#06x\n", u
.v32
);
996 ret
= simple_read_from_buffer(ubuf
, count
, offp
, buf
, off
);
1001 static ssize_t
ndev_debugfs_read(struct file
*filp
, char __user
*ubuf
,
1002 size_t count
, loff_t
*offp
)
1004 struct intel_ntb_dev
*ndev
= filp
->private_data
;
1006 if (pdev_is_xeon(ndev
->ntb
.pdev
) ||
1007 pdev_is_atom(ndev
->ntb
.pdev
))
1008 return ndev_ntb_debugfs_read(filp
, ubuf
, count
, offp
);
1009 else if (pdev_is_skx_xeon(ndev
->ntb
.pdev
))
1010 return ndev_ntb3_debugfs_read(filp
, ubuf
, count
, offp
);
1015 static void ndev_init_debugfs(struct intel_ntb_dev
*ndev
)
1018 ndev
->debugfs_dir
= NULL
;
1019 ndev
->debugfs_info
= NULL
;
1022 debugfs_create_dir(ndev_name(ndev
), debugfs_dir
);
1023 if (!ndev
->debugfs_dir
)
1024 ndev
->debugfs_info
= NULL
;
1026 ndev
->debugfs_info
=
1027 debugfs_create_file("info", S_IRUSR
,
1028 ndev
->debugfs_dir
, ndev
,
1029 &intel_ntb_debugfs_info
);
1033 static void ndev_deinit_debugfs(struct intel_ntb_dev
*ndev
)
1035 debugfs_remove_recursive(ndev
->debugfs_dir
);
1038 static int intel_ntb_mw_count(struct ntb_dev
*ntb
)
1040 return ntb_ndev(ntb
)->mw_count
;
1043 static int intel_ntb_mw_get_range(struct ntb_dev
*ntb
, int idx
,
1045 resource_size_t
*size
,
1046 resource_size_t
*align
,
1047 resource_size_t
*align_size
)
1049 struct intel_ntb_dev
*ndev
= ntb_ndev(ntb
);
1052 if (idx
>= ndev
->b2b_idx
&& !ndev
->b2b_off
)
1055 bar
= ndev_mw_to_bar(ndev
, idx
);
1060 *base
= pci_resource_start(ndev
->ntb
.pdev
, bar
) +
1061 (idx
== ndev
->b2b_idx
? ndev
->b2b_off
: 0);
1064 *size
= pci_resource_len(ndev
->ntb
.pdev
, bar
) -
1065 (idx
== ndev
->b2b_idx
? ndev
->b2b_off
: 0);
1068 *align
= pci_resource_len(ndev
->ntb
.pdev
, bar
);
1076 static int intel_ntb_mw_set_trans(struct ntb_dev
*ntb
, int idx
,
1077 dma_addr_t addr
, resource_size_t size
)
1079 struct intel_ntb_dev
*ndev
= ntb_ndev(ntb
);
1080 unsigned long base_reg
, xlat_reg
, limit_reg
;
1081 resource_size_t bar_size
, mw_size
;
1083 u64 base
, limit
, reg_val
;
1086 if (idx
>= ndev
->b2b_idx
&& !ndev
->b2b_off
)
1089 bar
= ndev_mw_to_bar(ndev
, idx
);
1093 bar_size
= pci_resource_len(ndev
->ntb
.pdev
, bar
);
1095 if (idx
== ndev
->b2b_idx
)
1096 mw_size
= bar_size
- ndev
->b2b_off
;
1100 /* hardware requires that addr is aligned to bar size */
1101 if (addr
& (bar_size
- 1))
1104 /* make sure the range fits in the usable mw size */
1108 mmio
= ndev
->self_mmio
;
1109 base_reg
= bar0_off(ndev
->xlat_reg
->bar0_base
, bar
);
1110 xlat_reg
= bar2_off(ndev
->xlat_reg
->bar2_xlat
, bar
);
1111 limit_reg
= bar2_off(ndev
->xlat_reg
->bar2_limit
, bar
);
1113 if (bar
< 4 || !ndev
->bar4_split
) {
1114 base
= ioread64(mmio
+ base_reg
) & NTB_BAR_MASK_64
;
1116 /* Set the limit if supported, if size is not mw_size */
1117 if (limit_reg
&& size
!= mw_size
)
1118 limit
= base
+ size
;
1122 /* set and verify setting the translation address */
1123 iowrite64(addr
, mmio
+ xlat_reg
);
1124 reg_val
= ioread64(mmio
+ xlat_reg
);
1125 if (reg_val
!= addr
) {
1126 iowrite64(0, mmio
+ xlat_reg
);
1130 /* set and verify setting the limit */
1131 iowrite64(limit
, mmio
+ limit_reg
);
1132 reg_val
= ioread64(mmio
+ limit_reg
);
1133 if (reg_val
!= limit
) {
1134 iowrite64(base
, mmio
+ limit_reg
);
1135 iowrite64(0, mmio
+ xlat_reg
);
1139 /* split bar addr range must all be 32 bit */
1140 if (addr
& (~0ull << 32))
1142 if ((addr
+ size
) & (~0ull << 32))
1145 base
= ioread32(mmio
+ base_reg
) & NTB_BAR_MASK_32
;
1147 /* Set the limit if supported, if size is not mw_size */
1148 if (limit_reg
&& size
!= mw_size
)
1149 limit
= base
+ size
;
1153 /* set and verify setting the translation address */
1154 iowrite32(addr
, mmio
+ xlat_reg
);
1155 reg_val
= ioread32(mmio
+ xlat_reg
);
1156 if (reg_val
!= addr
) {
1157 iowrite32(0, mmio
+ xlat_reg
);
1161 /* set and verify setting the limit */
1162 iowrite32(limit
, mmio
+ limit_reg
);
1163 reg_val
= ioread32(mmio
+ limit_reg
);
1164 if (reg_val
!= limit
) {
1165 iowrite32(base
, mmio
+ limit_reg
);
1166 iowrite32(0, mmio
+ xlat_reg
);
1174 static int intel_ntb_link_is_up(struct ntb_dev
*ntb
,
1175 enum ntb_speed
*speed
,
1176 enum ntb_width
*width
)
1178 struct intel_ntb_dev
*ndev
= ntb_ndev(ntb
);
1180 if (ndev
->reg
->link_is_up(ndev
)) {
1182 *speed
= NTB_LNK_STA_SPEED(ndev
->lnk_sta
);
1184 *width
= NTB_LNK_STA_WIDTH(ndev
->lnk_sta
);
1187 /* TODO MAYBE: is it possible to observe the link speed and
1188 * width while link is training? */
1190 *speed
= NTB_SPEED_NONE
;
1192 *width
= NTB_WIDTH_NONE
;
1197 static int intel_ntb_link_enable(struct ntb_dev
*ntb
,
1198 enum ntb_speed max_speed
,
1199 enum ntb_width max_width
)
1201 struct intel_ntb_dev
*ndev
;
1204 ndev
= container_of(ntb
, struct intel_ntb_dev
, ntb
);
1206 if (ndev
->ntb
.topo
== NTB_TOPO_SEC
)
1209 dev_dbg(ndev_dev(ndev
),
1210 "Enabling link with max_speed %d max_width %d\n",
1211 max_speed
, max_width
);
1212 if (max_speed
!= NTB_SPEED_AUTO
)
1213 dev_dbg(ndev_dev(ndev
), "ignoring max_speed %d\n", max_speed
);
1214 if (max_width
!= NTB_WIDTH_AUTO
)
1215 dev_dbg(ndev_dev(ndev
), "ignoring max_width %d\n", max_width
);
1217 ntb_ctl
= ioread32(ndev
->self_mmio
+ ndev
->reg
->ntb_ctl
);
1218 ntb_ctl
&= ~(NTB_CTL_DISABLE
| NTB_CTL_CFG_LOCK
);
1219 ntb_ctl
|= NTB_CTL_P2S_BAR2_SNOOP
| NTB_CTL_S2P_BAR2_SNOOP
;
1220 ntb_ctl
|= NTB_CTL_P2S_BAR4_SNOOP
| NTB_CTL_S2P_BAR4_SNOOP
;
1221 if (ndev
->bar4_split
)
1222 ntb_ctl
|= NTB_CTL_P2S_BAR5_SNOOP
| NTB_CTL_S2P_BAR5_SNOOP
;
1223 iowrite32(ntb_ctl
, ndev
->self_mmio
+ ndev
->reg
->ntb_ctl
);
1228 static int intel_ntb_link_disable(struct ntb_dev
*ntb
)
1230 struct intel_ntb_dev
*ndev
;
1233 ndev
= container_of(ntb
, struct intel_ntb_dev
, ntb
);
1235 if (ndev
->ntb
.topo
== NTB_TOPO_SEC
)
1238 dev_dbg(ndev_dev(ndev
), "Disabling link\n");
1240 /* Bring NTB link down */
1241 ntb_cntl
= ioread32(ndev
->self_mmio
+ ndev
->reg
->ntb_ctl
);
1242 ntb_cntl
&= ~(NTB_CTL_P2S_BAR2_SNOOP
| NTB_CTL_S2P_BAR2_SNOOP
);
1243 ntb_cntl
&= ~(NTB_CTL_P2S_BAR4_SNOOP
| NTB_CTL_S2P_BAR4_SNOOP
);
1244 if (ndev
->bar4_split
)
1245 ntb_cntl
&= ~(NTB_CTL_P2S_BAR5_SNOOP
| NTB_CTL_S2P_BAR5_SNOOP
);
1246 ntb_cntl
|= NTB_CTL_DISABLE
| NTB_CTL_CFG_LOCK
;
1247 iowrite32(ntb_cntl
, ndev
->self_mmio
+ ndev
->reg
->ntb_ctl
);
1252 static int intel_ntb_db_is_unsafe(struct ntb_dev
*ntb
)
1254 return ndev_ignore_unsafe(ntb_ndev(ntb
), NTB_UNSAFE_DB
);
1257 static u64
intel_ntb_db_valid_mask(struct ntb_dev
*ntb
)
1259 return ntb_ndev(ntb
)->db_valid_mask
;
1262 static int intel_ntb_db_vector_count(struct ntb_dev
*ntb
)
1264 struct intel_ntb_dev
*ndev
;
1266 ndev
= container_of(ntb
, struct intel_ntb_dev
, ntb
);
1268 return ndev
->db_vec_count
;
1271 static u64
intel_ntb_db_vector_mask(struct ntb_dev
*ntb
, int db_vector
)
1273 struct intel_ntb_dev
*ndev
= ntb_ndev(ntb
);
1275 if (db_vector
< 0 || db_vector
> ndev
->db_vec_count
)
1278 return ndev
->db_valid_mask
& ndev_vec_mask(ndev
, db_vector
);
1281 static u64
intel_ntb_db_read(struct ntb_dev
*ntb
)
1283 struct intel_ntb_dev
*ndev
= ntb_ndev(ntb
);
1285 return ndev_db_read(ndev
,
1287 ndev
->self_reg
->db_bell
);
1290 static int intel_ntb_db_clear(struct ntb_dev
*ntb
, u64 db_bits
)
1292 struct intel_ntb_dev
*ndev
= ntb_ndev(ntb
);
1294 return ndev_db_write(ndev
, db_bits
,
1296 ndev
->self_reg
->db_bell
);
1299 static int intel_ntb_db_set_mask(struct ntb_dev
*ntb
, u64 db_bits
)
1301 struct intel_ntb_dev
*ndev
= ntb_ndev(ntb
);
1303 return ndev_db_set_mask(ndev
, db_bits
,
1305 ndev
->self_reg
->db_mask
);
1308 static int intel_ntb_db_clear_mask(struct ntb_dev
*ntb
, u64 db_bits
)
1310 struct intel_ntb_dev
*ndev
= ntb_ndev(ntb
);
1312 return ndev_db_clear_mask(ndev
, db_bits
,
1314 ndev
->self_reg
->db_mask
);
1317 static int intel_ntb_peer_db_addr(struct ntb_dev
*ntb
,
1318 phys_addr_t
*db_addr
,
1319 resource_size_t
*db_size
)
1321 struct intel_ntb_dev
*ndev
= ntb_ndev(ntb
);
1323 return ndev_db_addr(ndev
, db_addr
, db_size
, ndev
->peer_addr
,
1324 ndev
->peer_reg
->db_bell
);
1327 static int intel_ntb_peer_db_set(struct ntb_dev
*ntb
, u64 db_bits
)
1329 struct intel_ntb_dev
*ndev
= ntb_ndev(ntb
);
1331 return ndev_db_write(ndev
, db_bits
,
1333 ndev
->peer_reg
->db_bell
);
1336 static int intel_ntb_spad_is_unsafe(struct ntb_dev
*ntb
)
1338 return ndev_ignore_unsafe(ntb_ndev(ntb
), NTB_UNSAFE_SPAD
);
1341 static int intel_ntb_spad_count(struct ntb_dev
*ntb
)
1343 struct intel_ntb_dev
*ndev
;
1345 ndev
= container_of(ntb
, struct intel_ntb_dev
, ntb
);
1347 return ndev
->spad_count
;
1350 static u32
intel_ntb_spad_read(struct ntb_dev
*ntb
, int idx
)
1352 struct intel_ntb_dev
*ndev
= ntb_ndev(ntb
);
1354 return ndev_spad_read(ndev
, idx
,
1356 ndev
->self_reg
->spad
);
1359 static int intel_ntb_spad_write(struct ntb_dev
*ntb
,
1362 struct intel_ntb_dev
*ndev
= ntb_ndev(ntb
);
1364 return ndev_spad_write(ndev
, idx
, val
,
1366 ndev
->self_reg
->spad
);
1369 static int intel_ntb_peer_spad_addr(struct ntb_dev
*ntb
, int idx
,
1370 phys_addr_t
*spad_addr
)
1372 struct intel_ntb_dev
*ndev
= ntb_ndev(ntb
);
1374 return ndev_spad_addr(ndev
, idx
, spad_addr
, ndev
->peer_addr
,
1375 ndev
->peer_reg
->spad
);
1378 static u32
intel_ntb_peer_spad_read(struct ntb_dev
*ntb
, int idx
)
1380 struct intel_ntb_dev
*ndev
= ntb_ndev(ntb
);
1382 return ndev_spad_read(ndev
, idx
,
1384 ndev
->peer_reg
->spad
);
1387 static int intel_ntb_peer_spad_write(struct ntb_dev
*ntb
,
1390 struct intel_ntb_dev
*ndev
= ntb_ndev(ntb
);
1392 return ndev_spad_write(ndev
, idx
, val
,
1394 ndev
->peer_reg
->spad
);
1399 static u64
atom_db_ioread(void __iomem
*mmio
)
1401 return ioread64(mmio
);
1404 static void atom_db_iowrite(u64 bits
, void __iomem
*mmio
)
1406 iowrite64(bits
, mmio
);
1409 static int atom_poll_link(struct intel_ntb_dev
*ndev
)
1413 ntb_ctl
= ioread32(ndev
->self_mmio
+ ATOM_NTBCNTL_OFFSET
);
1415 if (ntb_ctl
== ndev
->ntb_ctl
)
1418 ndev
->ntb_ctl
= ntb_ctl
;
1420 ndev
->lnk_sta
= ioread32(ndev
->self_mmio
+ ATOM_LINK_STATUS_OFFSET
);
1425 static int atom_link_is_up(struct intel_ntb_dev
*ndev
)
1427 return ATOM_NTB_CTL_ACTIVE(ndev
->ntb_ctl
);
1430 static int atom_link_is_err(struct intel_ntb_dev
*ndev
)
1432 if (ioread32(ndev
->self_mmio
+ ATOM_LTSSMSTATEJMP_OFFSET
)
1433 & ATOM_LTSSMSTATEJMP_FORCEDETECT
)
1436 if (ioread32(ndev
->self_mmio
+ ATOM_IBSTERRRCRVSTS0_OFFSET
)
1437 & ATOM_IBIST_ERR_OFLOW
)
1443 static inline enum ntb_topo
atom_ppd_topo(struct intel_ntb_dev
*ndev
, u32 ppd
)
1445 switch (ppd
& ATOM_PPD_TOPO_MASK
) {
1446 case ATOM_PPD_TOPO_B2B_USD
:
1447 dev_dbg(ndev_dev(ndev
), "PPD %d B2B USD\n", ppd
);
1448 return NTB_TOPO_B2B_USD
;
1450 case ATOM_PPD_TOPO_B2B_DSD
:
1451 dev_dbg(ndev_dev(ndev
), "PPD %d B2B DSD\n", ppd
);
1452 return NTB_TOPO_B2B_DSD
;
1454 case ATOM_PPD_TOPO_PRI_USD
:
1455 case ATOM_PPD_TOPO_PRI_DSD
: /* accept bogus PRI_DSD */
1456 case ATOM_PPD_TOPO_SEC_USD
:
1457 case ATOM_PPD_TOPO_SEC_DSD
: /* accept bogus SEC_DSD */
1458 dev_dbg(ndev_dev(ndev
), "PPD %d non B2B disabled\n", ppd
);
1459 return NTB_TOPO_NONE
;
1462 dev_dbg(ndev_dev(ndev
), "PPD %d invalid\n", ppd
);
1463 return NTB_TOPO_NONE
;
1466 static void atom_link_hb(struct work_struct
*work
)
1468 struct intel_ntb_dev
*ndev
= hb_ndev(work
);
1469 unsigned long poll_ts
;
1473 poll_ts
= ndev
->last_ts
+ ATOM_LINK_HB_TIMEOUT
;
1475 /* Delay polling the link status if an interrupt was received,
1476 * unless the cached link status says the link is down.
1478 if (time_after(poll_ts
, jiffies
) && atom_link_is_up(ndev
)) {
1479 schedule_delayed_work(&ndev
->hb_timer
, poll_ts
- jiffies
);
1483 if (atom_poll_link(ndev
))
1484 ntb_link_event(&ndev
->ntb
);
1486 if (atom_link_is_up(ndev
) || !atom_link_is_err(ndev
)) {
1487 schedule_delayed_work(&ndev
->hb_timer
, ATOM_LINK_HB_TIMEOUT
);
1491 /* Link is down with error: recover the link! */
1493 mmio
= ndev
->self_mmio
;
1495 /* Driver resets the NTB ModPhy lanes - magic! */
1496 iowrite8(0xe0, mmio
+ ATOM_MODPHY_PCSREG6
);
1497 iowrite8(0x40, mmio
+ ATOM_MODPHY_PCSREG4
);
1498 iowrite8(0x60, mmio
+ ATOM_MODPHY_PCSREG4
);
1499 iowrite8(0x60, mmio
+ ATOM_MODPHY_PCSREG6
);
1501 /* Driver waits 100ms to allow the NTB ModPhy to settle */
1504 /* Clear AER Errors, write to clear */
1505 status32
= ioread32(mmio
+ ATOM_ERRCORSTS_OFFSET
);
1506 dev_dbg(ndev_dev(ndev
), "ERRCORSTS = %x\n", status32
);
1507 status32
&= PCI_ERR_COR_REP_ROLL
;
1508 iowrite32(status32
, mmio
+ ATOM_ERRCORSTS_OFFSET
);
1510 /* Clear unexpected electrical idle event in LTSSM, write to clear */
1511 status32
= ioread32(mmio
+ ATOM_LTSSMERRSTS0_OFFSET
);
1512 dev_dbg(ndev_dev(ndev
), "LTSSMERRSTS0 = %x\n", status32
);
1513 status32
|= ATOM_LTSSMERRSTS0_UNEXPECTEDEI
;
1514 iowrite32(status32
, mmio
+ ATOM_LTSSMERRSTS0_OFFSET
);
1516 /* Clear DeSkew Buffer error, write to clear */
1517 status32
= ioread32(mmio
+ ATOM_DESKEWSTS_OFFSET
);
1518 dev_dbg(ndev_dev(ndev
), "DESKEWSTS = %x\n", status32
);
1519 status32
|= ATOM_DESKEWSTS_DBERR
;
1520 iowrite32(status32
, mmio
+ ATOM_DESKEWSTS_OFFSET
);
1522 status32
= ioread32(mmio
+ ATOM_IBSTERRRCRVSTS0_OFFSET
);
1523 dev_dbg(ndev_dev(ndev
), "IBSTERRRCRVSTS0 = %x\n", status32
);
1524 status32
&= ATOM_IBIST_ERR_OFLOW
;
1525 iowrite32(status32
, mmio
+ ATOM_IBSTERRRCRVSTS0_OFFSET
);
1527 /* Releases the NTB state machine to allow the link to retrain */
1528 status32
= ioread32(mmio
+ ATOM_LTSSMSTATEJMP_OFFSET
);
1529 dev_dbg(ndev_dev(ndev
), "LTSSMSTATEJMP = %x\n", status32
);
1530 status32
&= ~ATOM_LTSSMSTATEJMP_FORCEDETECT
;
1531 iowrite32(status32
, mmio
+ ATOM_LTSSMSTATEJMP_OFFSET
);
1533 /* There is a potential race between the 2 NTB devices recovering at the
1534 * same time. If the times are the same, the link will not recover and
1535 * the driver will be stuck in this loop forever. Add a random interval
1536 * to the recovery time to prevent this race.
1538 schedule_delayed_work(&ndev
->hb_timer
, ATOM_LINK_RECOVERY_TIME
1539 + prandom_u32() % ATOM_LINK_RECOVERY_TIME
);
1542 static int atom_init_isr(struct intel_ntb_dev
*ndev
)
1546 rc
= ndev_init_isr(ndev
, 1, ATOM_DB_MSIX_VECTOR_COUNT
,
1547 ATOM_DB_MSIX_VECTOR_SHIFT
, ATOM_DB_TOTAL_SHIFT
);
1551 /* ATOM doesn't have link status interrupt, poll on that platform */
1552 ndev
->last_ts
= jiffies
;
1553 INIT_DELAYED_WORK(&ndev
->hb_timer
, atom_link_hb
);
1554 schedule_delayed_work(&ndev
->hb_timer
, ATOM_LINK_HB_TIMEOUT
);
1559 static void atom_deinit_isr(struct intel_ntb_dev
*ndev
)
1561 cancel_delayed_work_sync(&ndev
->hb_timer
);
1562 ndev_deinit_isr(ndev
);
1565 static int atom_init_ntb(struct intel_ntb_dev
*ndev
)
1567 ndev
->mw_count
= ATOM_MW_COUNT
;
1568 ndev
->spad_count
= ATOM_SPAD_COUNT
;
1569 ndev
->db_count
= ATOM_DB_COUNT
;
1571 switch (ndev
->ntb
.topo
) {
1572 case NTB_TOPO_B2B_USD
:
1573 case NTB_TOPO_B2B_DSD
:
1574 ndev
->self_reg
= &atom_pri_reg
;
1575 ndev
->peer_reg
= &atom_b2b_reg
;
1576 ndev
->xlat_reg
= &atom_sec_xlat
;
1578 /* Enable Bus Master and Memory Space on the secondary side */
1579 iowrite16(PCI_COMMAND_MEMORY
| PCI_COMMAND_MASTER
,
1580 ndev
->self_mmio
+ ATOM_SPCICMD_OFFSET
);
1588 ndev
->db_valid_mask
= BIT_ULL(ndev
->db_count
) - 1;
1593 static int atom_init_dev(struct intel_ntb_dev
*ndev
)
1598 rc
= pci_read_config_dword(ndev
->ntb
.pdev
, ATOM_PPD_OFFSET
, &ppd
);
1602 ndev
->ntb
.topo
= atom_ppd_topo(ndev
, ppd
);
1603 if (ndev
->ntb
.topo
== NTB_TOPO_NONE
)
1606 rc
= atom_init_ntb(ndev
);
1610 rc
= atom_init_isr(ndev
);
1614 if (ndev
->ntb
.topo
!= NTB_TOPO_SEC
) {
1615 /* Initiate PCI-E link training */
1616 rc
= pci_write_config_dword(ndev
->ntb
.pdev
, ATOM_PPD_OFFSET
,
1617 ppd
| ATOM_PPD_INIT_LINK
);
1625 static void atom_deinit_dev(struct intel_ntb_dev
*ndev
)
1627 atom_deinit_isr(ndev
);
1630 /* Skylake Xeon NTB */
1632 static int skx_poll_link(struct intel_ntb_dev
*ndev
)
1637 ndev
->reg
->db_iowrite(ndev
->db_link_mask
,
1639 ndev
->self_reg
->db_clear
);
1641 rc
= pci_read_config_word(ndev
->ntb
.pdev
,
1642 SKX_LINK_STATUS_OFFSET
, ®_val
);
1646 if (reg_val
== ndev
->lnk_sta
)
1649 ndev
->lnk_sta
= reg_val
;
1654 static u64
skx_db_ioread(void __iomem
*mmio
)
1656 return ioread64(mmio
);
1659 static void skx_db_iowrite(u64 bits
, void __iomem
*mmio
)
1661 iowrite64(bits
, mmio
);
1664 static int skx_init_isr(struct intel_ntb_dev
*ndev
)
1669 * The MSIX vectors and the interrupt status bits are not lined up
1670 * on Skylake. By default the link status bit is bit 32, however it
1671 * is by default MSIX vector0. We need to fixup to line them up.
1672 * The vectors at reset is 1-32,0. We need to reprogram to 0-32.
1675 for (i
= 0; i
< SKX_DB_MSIX_VECTOR_COUNT
; i
++)
1676 iowrite8(i
, ndev
->self_mmio
+ SKX_INTVEC_OFFSET
+ i
);
1678 /* move link status down one as workaround */
1679 if (ndev
->hwerr_flags
& NTB_HWERR_MSIX_VECTOR32_BAD
) {
1680 iowrite8(SKX_DB_MSIX_VECTOR_COUNT
- 2,
1681 ndev
->self_mmio
+ SKX_INTVEC_OFFSET
+
1682 (SKX_DB_MSIX_VECTOR_COUNT
- 1));
1685 return ndev_init_isr(ndev
, SKX_DB_MSIX_VECTOR_COUNT
,
1686 SKX_DB_MSIX_VECTOR_COUNT
,
1687 SKX_DB_MSIX_VECTOR_SHIFT
,
1688 SKX_DB_TOTAL_SHIFT
);
1691 static int skx_setup_b2b_mw(struct intel_ntb_dev
*ndev
,
1692 const struct intel_b2b_addr
*addr
,
1693 const struct intel_b2b_addr
*peer_addr
)
1695 struct pci_dev
*pdev
;
1697 resource_size_t bar_size
;
1698 phys_addr_t bar_addr
;
1702 pdev
= ndev_pdev(ndev
);
1703 mmio
= ndev
->self_mmio
;
1705 if (ndev
->b2b_idx
== UINT_MAX
) {
1706 dev_dbg(ndev_dev(ndev
), "not using b2b mw\n");
1710 b2b_bar
= ndev_mw_to_bar(ndev
, ndev
->b2b_idx
);
1714 dev_dbg(ndev_dev(ndev
), "using b2b mw bar %d\n", b2b_bar
);
1716 bar_size
= pci_resource_len(ndev
->ntb
.pdev
, b2b_bar
);
1718 dev_dbg(ndev_dev(ndev
), "b2b bar size %#llx\n", bar_size
);
1720 if (b2b_mw_share
&& ((bar_size
>> 1) >= XEON_B2B_MIN_SIZE
)) {
1721 dev_dbg(ndev_dev(ndev
),
1722 "b2b using first half of bar\n");
1723 ndev
->b2b_off
= bar_size
>> 1;
1724 } else if (bar_size
>= XEON_B2B_MIN_SIZE
) {
1725 dev_dbg(ndev_dev(ndev
),
1726 "b2b using whole bar\n");
1730 dev_dbg(ndev_dev(ndev
),
1731 "b2b bar size is too small\n");
1737 * Reset the secondary bar sizes to match the primary bar sizes,
1738 * except disable or halve the size of the b2b secondary bar.
1740 pci_read_config_byte(pdev
, SKX_IMBAR1SZ_OFFSET
, &bar_sz
);
1741 dev_dbg(ndev_dev(ndev
), "IMBAR1SZ %#x\n", bar_sz
);
1749 pci_write_config_byte(pdev
, SKX_EMBAR1SZ_OFFSET
, bar_sz
);
1750 pci_read_config_byte(pdev
, SKX_EMBAR1SZ_OFFSET
, &bar_sz
);
1751 dev_dbg(ndev_dev(ndev
), "EMBAR1SZ %#x\n", bar_sz
);
1753 pci_read_config_byte(pdev
, SKX_IMBAR2SZ_OFFSET
, &bar_sz
);
1754 dev_dbg(ndev_dev(ndev
), "IMBAR2SZ %#x\n", bar_sz
);
1762 pci_write_config_byte(pdev
, SKX_EMBAR2SZ_OFFSET
, bar_sz
);
1763 pci_read_config_byte(pdev
, SKX_EMBAR2SZ_OFFSET
, &bar_sz
);
1764 dev_dbg(ndev_dev(ndev
), "EMBAR2SZ %#x\n", bar_sz
);
1766 /* SBAR01 hit by first part of the b2b bar */
1768 bar_addr
= addr
->bar0_addr
;
1769 else if (b2b_bar
== 1)
1770 bar_addr
= addr
->bar2_addr64
;
1771 else if (b2b_bar
== 2)
1772 bar_addr
= addr
->bar4_addr64
;
1776 /* setup incoming bar limits == base addrs (zero length windows) */
1777 bar_addr
= addr
->bar2_addr64
+ (b2b_bar
== 1 ? ndev
->b2b_off
: 0);
1778 iowrite64(bar_addr
, mmio
+ SKX_IMBAR1XLMT_OFFSET
);
1779 bar_addr
= ioread64(mmio
+ SKX_IMBAR1XLMT_OFFSET
);
1780 dev_dbg(ndev_dev(ndev
), "IMBAR1XLMT %#018llx\n", bar_addr
);
1782 bar_addr
= addr
->bar4_addr64
+ (b2b_bar
== 2 ? ndev
->b2b_off
: 0);
1783 iowrite64(bar_addr
, mmio
+ SKX_IMBAR2XLMT_OFFSET
);
1784 bar_addr
= ioread64(mmio
+ SKX_IMBAR2XLMT_OFFSET
);
1785 dev_dbg(ndev_dev(ndev
), "IMBAR2XLMT %#018llx\n", bar_addr
);
1787 /* zero incoming translation addrs */
1788 iowrite64(0, mmio
+ SKX_IMBAR1XBASE_OFFSET
);
1789 iowrite64(0, mmio
+ SKX_IMBAR2XBASE_OFFSET
);
1791 ndev
->peer_mmio
= ndev
->self_mmio
;
1796 static int skx_init_ntb(struct intel_ntb_dev
*ndev
)
1801 ndev
->mw_count
= XEON_MW_COUNT
;
1802 ndev
->spad_count
= SKX_SPAD_COUNT
;
1803 ndev
->db_count
= SKX_DB_COUNT
;
1804 ndev
->db_link_mask
= SKX_DB_LINK_BIT
;
1806 /* DB fixup for using 31 right now */
1807 if (ndev
->hwerr_flags
& NTB_HWERR_MSIX_VECTOR32_BAD
)
1808 ndev
->db_link_mask
|= BIT_ULL(31);
1810 switch (ndev
->ntb
.topo
) {
1811 case NTB_TOPO_B2B_USD
:
1812 case NTB_TOPO_B2B_DSD
:
1813 ndev
->self_reg
= &skx_pri_reg
;
1814 ndev
->peer_reg
= &skx_b2b_reg
;
1815 ndev
->xlat_reg
= &skx_sec_xlat
;
1817 if (ndev
->ntb
.topo
== NTB_TOPO_B2B_USD
) {
1818 rc
= skx_setup_b2b_mw(ndev
,
1820 &xeon_b2b_usd_addr
);
1822 rc
= skx_setup_b2b_mw(ndev
,
1824 &xeon_b2b_dsd_addr
);
1830 /* Enable Bus Master and Memory Space on the secondary side */
1831 iowrite16(PCI_COMMAND_MEMORY
| PCI_COMMAND_MASTER
,
1832 ndev
->self_mmio
+ SKX_SPCICMD_OFFSET
);
1840 ndev
->db_valid_mask
= BIT_ULL(ndev
->db_count
) - 1;
1842 ndev
->reg
->db_iowrite(ndev
->db_valid_mask
,
1844 ndev
->self_reg
->db_mask
);
1849 static int skx_init_dev(struct intel_ntb_dev
*ndev
)
1851 struct pci_dev
*pdev
;
1855 pdev
= ndev_pdev(ndev
);
1857 ndev
->reg
= &skx_reg
;
1859 rc
= pci_read_config_byte(pdev
, XEON_PPD_OFFSET
, &ppd
);
1863 ndev
->ntb
.topo
= xeon_ppd_topo(ndev
, ppd
);
1864 dev_dbg(ndev_dev(ndev
), "ppd %#x topo %s\n", ppd
,
1865 ntb_topo_string(ndev
->ntb
.topo
));
1866 if (ndev
->ntb
.topo
== NTB_TOPO_NONE
)
1869 if (pdev_is_skx_xeon(pdev
))
1870 ndev
->hwerr_flags
|= NTB_HWERR_MSIX_VECTOR32_BAD
;
1872 rc
= skx_init_ntb(ndev
);
1876 return skx_init_isr(ndev
);
1879 static int intel_ntb3_link_enable(struct ntb_dev
*ntb
,
1880 enum ntb_speed max_speed
,
1881 enum ntb_width max_width
)
1883 struct intel_ntb_dev
*ndev
;
1886 ndev
= container_of(ntb
, struct intel_ntb_dev
, ntb
);
1888 dev_dbg(ndev_dev(ndev
),
1889 "Enabling link with max_speed %d max_width %d\n",
1890 max_speed
, max_width
);
1892 if (max_speed
!= NTB_SPEED_AUTO
)
1893 dev_dbg(ndev_dev(ndev
), "ignoring max_speed %d\n", max_speed
);
1894 if (max_width
!= NTB_WIDTH_AUTO
)
1895 dev_dbg(ndev_dev(ndev
), "ignoring max_width %d\n", max_width
);
1897 ntb_ctl
= ioread32(ndev
->self_mmio
+ ndev
->reg
->ntb_ctl
);
1898 ntb_ctl
&= ~(NTB_CTL_DISABLE
| NTB_CTL_CFG_LOCK
);
1899 ntb_ctl
|= NTB_CTL_P2S_BAR2_SNOOP
| NTB_CTL_S2P_BAR2_SNOOP
;
1900 ntb_ctl
|= NTB_CTL_P2S_BAR4_SNOOP
| NTB_CTL_S2P_BAR4_SNOOP
;
1901 iowrite32(ntb_ctl
, ndev
->self_mmio
+ ndev
->reg
->ntb_ctl
);
1905 static int intel_ntb3_mw_set_trans(struct ntb_dev
*ntb
, int idx
,
1906 dma_addr_t addr
, resource_size_t size
)
1908 struct intel_ntb_dev
*ndev
= ntb_ndev(ntb
);
1909 unsigned long xlat_reg
, limit_reg
;
1910 resource_size_t bar_size
, mw_size
;
1912 u64 base
, limit
, reg_val
;
1915 if (idx
>= ndev
->b2b_idx
&& !ndev
->b2b_off
)
1918 bar
= ndev_mw_to_bar(ndev
, idx
);
1922 bar_size
= pci_resource_len(ndev
->ntb
.pdev
, bar
);
1924 if (idx
== ndev
->b2b_idx
)
1925 mw_size
= bar_size
- ndev
->b2b_off
;
1929 /* hardware requires that addr is aligned to bar size */
1930 if (addr
& (bar_size
- 1))
1933 /* make sure the range fits in the usable mw size */
1937 mmio
= ndev
->self_mmio
;
1938 xlat_reg
= ndev
->xlat_reg
->bar2_xlat
+ (idx
* 0x10);
1939 limit_reg
= ndev
->xlat_reg
->bar2_limit
+ (idx
* 0x10);
1940 base
= pci_resource_start(ndev
->ntb
.pdev
, bar
);
1942 /* Set the limit if supported, if size is not mw_size */
1943 if (limit_reg
&& size
!= mw_size
)
1944 limit
= base
+ size
;
1946 limit
= base
+ mw_size
;
1948 /* set and verify setting the translation address */
1949 iowrite64(addr
, mmio
+ xlat_reg
);
1950 reg_val
= ioread64(mmio
+ xlat_reg
);
1951 if (reg_val
!= addr
) {
1952 iowrite64(0, mmio
+ xlat_reg
);
1956 dev_dbg(ndev_dev(ndev
), "BAR %d IMBARXBASE: %#Lx\n", bar
, reg_val
);
1958 /* set and verify setting the limit */
1959 iowrite64(limit
, mmio
+ limit_reg
);
1960 reg_val
= ioread64(mmio
+ limit_reg
);
1961 if (reg_val
!= limit
) {
1962 iowrite64(base
, mmio
+ limit_reg
);
1963 iowrite64(0, mmio
+ xlat_reg
);
1967 dev_dbg(ndev_dev(ndev
), "BAR %d IMBARXLMT: %#Lx\n", bar
, reg_val
);
1970 limit_reg
= ndev
->xlat_reg
->bar2_limit
+ (idx
* 0x10) + 0x4000;
1971 base
= ioread64(mmio
+ SKX_EMBAR1_OFFSET
+ (8 * idx
));
1974 if (limit_reg
&& size
!= mw_size
)
1975 limit
= base
+ size
;
1977 limit
= base
+ mw_size
;
1979 /* set and verify setting the limit */
1980 iowrite64(limit
, mmio
+ limit_reg
);
1981 reg_val
= ioread64(mmio
+ limit_reg
);
1982 if (reg_val
!= limit
) {
1983 iowrite64(base
, mmio
+ limit_reg
);
1984 iowrite64(0, mmio
+ xlat_reg
);
1988 dev_dbg(ndev_dev(ndev
), "BAR %d EMBARXLMT: %#Lx\n", bar
, reg_val
);
1993 static int intel_ntb3_peer_db_set(struct ntb_dev
*ntb
, u64 db_bits
)
1995 struct intel_ntb_dev
*ndev
= ntb_ndev(ntb
);
1998 if (db_bits
& ~ndev
->db_valid_mask
)
2002 bit
= __ffs(db_bits
);
2003 iowrite32(1, ndev
->peer_mmio
+
2004 ndev
->peer_reg
->db_bell
+ (bit
* 4));
2005 db_bits
&= db_bits
- 1;
2011 static u64
intel_ntb3_db_read(struct ntb_dev
*ntb
)
2013 struct intel_ntb_dev
*ndev
= ntb_ndev(ntb
);
2015 return ndev_db_read(ndev
,
2017 ndev
->self_reg
->db_clear
);
2020 static int intel_ntb3_db_clear(struct ntb_dev
*ntb
, u64 db_bits
)
2022 struct intel_ntb_dev
*ndev
= ntb_ndev(ntb
);
2024 return ndev_db_write(ndev
, db_bits
,
2026 ndev
->self_reg
->db_clear
);
2031 static u64
xeon_db_ioread(void __iomem
*mmio
)
2033 return (u64
)ioread16(mmio
);
2036 static void xeon_db_iowrite(u64 bits
, void __iomem
*mmio
)
2038 iowrite16((u16
)bits
, mmio
);
2041 static int xeon_poll_link(struct intel_ntb_dev
*ndev
)
2046 ndev
->reg
->db_iowrite(ndev
->db_link_mask
,
2048 ndev
->self_reg
->db_bell
);
2050 rc
= pci_read_config_word(ndev
->ntb
.pdev
,
2051 XEON_LINK_STATUS_OFFSET
, ®_val
);
2055 if (reg_val
== ndev
->lnk_sta
)
2058 ndev
->lnk_sta
= reg_val
;
2063 static int xeon_link_is_up(struct intel_ntb_dev
*ndev
)
2065 if (ndev
->ntb
.topo
== NTB_TOPO_SEC
)
2068 return NTB_LNK_STA_ACTIVE(ndev
->lnk_sta
);
2071 static inline enum ntb_topo
xeon_ppd_topo(struct intel_ntb_dev
*ndev
, u8 ppd
)
2073 switch (ppd
& XEON_PPD_TOPO_MASK
) {
2074 case XEON_PPD_TOPO_B2B_USD
:
2075 return NTB_TOPO_B2B_USD
;
2077 case XEON_PPD_TOPO_B2B_DSD
:
2078 return NTB_TOPO_B2B_DSD
;
2080 case XEON_PPD_TOPO_PRI_USD
:
2081 case XEON_PPD_TOPO_PRI_DSD
: /* accept bogus PRI_DSD */
2082 return NTB_TOPO_PRI
;
2084 case XEON_PPD_TOPO_SEC_USD
:
2085 case XEON_PPD_TOPO_SEC_DSD
: /* accept bogus SEC_DSD */
2086 return NTB_TOPO_SEC
;
2089 return NTB_TOPO_NONE
;
2092 static inline int xeon_ppd_bar4_split(struct intel_ntb_dev
*ndev
, u8 ppd
)
2094 if (ppd
& XEON_PPD_SPLIT_BAR_MASK
) {
2095 dev_dbg(ndev_dev(ndev
), "PPD %d split bar\n", ppd
);
2101 static int xeon_init_isr(struct intel_ntb_dev
*ndev
)
2103 return ndev_init_isr(ndev
, XEON_DB_MSIX_VECTOR_COUNT
,
2104 XEON_DB_MSIX_VECTOR_COUNT
,
2105 XEON_DB_MSIX_VECTOR_SHIFT
,
2106 XEON_DB_TOTAL_SHIFT
);
2109 static void xeon_deinit_isr(struct intel_ntb_dev
*ndev
)
2111 ndev_deinit_isr(ndev
);
2114 static int xeon_setup_b2b_mw(struct intel_ntb_dev
*ndev
,
2115 const struct intel_b2b_addr
*addr
,
2116 const struct intel_b2b_addr
*peer_addr
)
2118 struct pci_dev
*pdev
;
2120 resource_size_t bar_size
;
2121 phys_addr_t bar_addr
;
2125 pdev
= ndev_pdev(ndev
);
2126 mmio
= ndev
->self_mmio
;
2128 if (ndev
->b2b_idx
== UINT_MAX
) {
2129 dev_dbg(ndev_dev(ndev
), "not using b2b mw\n");
2133 b2b_bar
= ndev_mw_to_bar(ndev
, ndev
->b2b_idx
);
2137 dev_dbg(ndev_dev(ndev
), "using b2b mw bar %d\n", b2b_bar
);
2139 bar_size
= pci_resource_len(ndev
->ntb
.pdev
, b2b_bar
);
2141 dev_dbg(ndev_dev(ndev
), "b2b bar size %#llx\n", bar_size
);
2143 if (b2b_mw_share
&& XEON_B2B_MIN_SIZE
<= bar_size
>> 1) {
2144 dev_dbg(ndev_dev(ndev
),
2145 "b2b using first half of bar\n");
2146 ndev
->b2b_off
= bar_size
>> 1;
2147 } else if (XEON_B2B_MIN_SIZE
<= bar_size
) {
2148 dev_dbg(ndev_dev(ndev
),
2149 "b2b using whole bar\n");
2153 dev_dbg(ndev_dev(ndev
),
2154 "b2b bar size is too small\n");
2159 /* Reset the secondary bar sizes to match the primary bar sizes,
2160 * except disable or halve the size of the b2b secondary bar.
2162 * Note: code for each specific bar size register, because the register
2163 * offsets are not in a consistent order (bar5sz comes after ppd, odd).
2165 pci_read_config_byte(pdev
, XEON_PBAR23SZ_OFFSET
, &bar_sz
);
2166 dev_dbg(ndev_dev(ndev
), "PBAR23SZ %#x\n", bar_sz
);
2173 pci_write_config_byte(pdev
, XEON_SBAR23SZ_OFFSET
, bar_sz
);
2174 pci_read_config_byte(pdev
, XEON_SBAR23SZ_OFFSET
, &bar_sz
);
2175 dev_dbg(ndev_dev(ndev
), "SBAR23SZ %#x\n", bar_sz
);
2177 if (!ndev
->bar4_split
) {
2178 pci_read_config_byte(pdev
, XEON_PBAR45SZ_OFFSET
, &bar_sz
);
2179 dev_dbg(ndev_dev(ndev
), "PBAR45SZ %#x\n", bar_sz
);
2186 pci_write_config_byte(pdev
, XEON_SBAR45SZ_OFFSET
, bar_sz
);
2187 pci_read_config_byte(pdev
, XEON_SBAR45SZ_OFFSET
, &bar_sz
);
2188 dev_dbg(ndev_dev(ndev
), "SBAR45SZ %#x\n", bar_sz
);
2190 pci_read_config_byte(pdev
, XEON_PBAR4SZ_OFFSET
, &bar_sz
);
2191 dev_dbg(ndev_dev(ndev
), "PBAR4SZ %#x\n", bar_sz
);
2198 pci_write_config_byte(pdev
, XEON_SBAR4SZ_OFFSET
, bar_sz
);
2199 pci_read_config_byte(pdev
, XEON_SBAR4SZ_OFFSET
, &bar_sz
);
2200 dev_dbg(ndev_dev(ndev
), "SBAR4SZ %#x\n", bar_sz
);
2202 pci_read_config_byte(pdev
, XEON_PBAR5SZ_OFFSET
, &bar_sz
);
2203 dev_dbg(ndev_dev(ndev
), "PBAR5SZ %#x\n", bar_sz
);
2210 pci_write_config_byte(pdev
, XEON_SBAR5SZ_OFFSET
, bar_sz
);
2211 pci_read_config_byte(pdev
, XEON_SBAR5SZ_OFFSET
, &bar_sz
);
2212 dev_dbg(ndev_dev(ndev
), "SBAR5SZ %#x\n", bar_sz
);
2215 /* SBAR01 hit by first part of the b2b bar */
2217 bar_addr
= addr
->bar0_addr
;
2218 else if (b2b_bar
== 2)
2219 bar_addr
= addr
->bar2_addr64
;
2220 else if (b2b_bar
== 4 && !ndev
->bar4_split
)
2221 bar_addr
= addr
->bar4_addr64
;
2222 else if (b2b_bar
== 4)
2223 bar_addr
= addr
->bar4_addr32
;
2224 else if (b2b_bar
== 5)
2225 bar_addr
= addr
->bar5_addr32
;
2229 dev_dbg(ndev_dev(ndev
), "SBAR01 %#018llx\n", bar_addr
);
2230 iowrite64(bar_addr
, mmio
+ XEON_SBAR0BASE_OFFSET
);
2232 /* Other SBAR are normally hit by the PBAR xlat, except for b2b bar.
2233 * The b2b bar is either disabled above, or configured half-size, and
2234 * it starts at the PBAR xlat + offset.
2237 bar_addr
= addr
->bar2_addr64
+ (b2b_bar
== 2 ? ndev
->b2b_off
: 0);
2238 iowrite64(bar_addr
, mmio
+ XEON_SBAR23BASE_OFFSET
);
2239 bar_addr
= ioread64(mmio
+ XEON_SBAR23BASE_OFFSET
);
2240 dev_dbg(ndev_dev(ndev
), "SBAR23 %#018llx\n", bar_addr
);
2242 if (!ndev
->bar4_split
) {
2243 bar_addr
= addr
->bar4_addr64
+
2244 (b2b_bar
== 4 ? ndev
->b2b_off
: 0);
2245 iowrite64(bar_addr
, mmio
+ XEON_SBAR45BASE_OFFSET
);
2246 bar_addr
= ioread64(mmio
+ XEON_SBAR45BASE_OFFSET
);
2247 dev_dbg(ndev_dev(ndev
), "SBAR45 %#018llx\n", bar_addr
);
2249 bar_addr
= addr
->bar4_addr32
+
2250 (b2b_bar
== 4 ? ndev
->b2b_off
: 0);
2251 iowrite32(bar_addr
, mmio
+ XEON_SBAR4BASE_OFFSET
);
2252 bar_addr
= ioread32(mmio
+ XEON_SBAR4BASE_OFFSET
);
2253 dev_dbg(ndev_dev(ndev
), "SBAR4 %#010llx\n", bar_addr
);
2255 bar_addr
= addr
->bar5_addr32
+
2256 (b2b_bar
== 5 ? ndev
->b2b_off
: 0);
2257 iowrite32(bar_addr
, mmio
+ XEON_SBAR5BASE_OFFSET
);
2258 bar_addr
= ioread32(mmio
+ XEON_SBAR5BASE_OFFSET
);
2259 dev_dbg(ndev_dev(ndev
), "SBAR5 %#010llx\n", bar_addr
);
2262 /* setup incoming bar limits == base addrs (zero length windows) */
2264 bar_addr
= addr
->bar2_addr64
+ (b2b_bar
== 2 ? ndev
->b2b_off
: 0);
2265 iowrite64(bar_addr
, mmio
+ XEON_SBAR23LMT_OFFSET
);
2266 bar_addr
= ioread64(mmio
+ XEON_SBAR23LMT_OFFSET
);
2267 dev_dbg(ndev_dev(ndev
), "SBAR23LMT %#018llx\n", bar_addr
);
2269 if (!ndev
->bar4_split
) {
2270 bar_addr
= addr
->bar4_addr64
+
2271 (b2b_bar
== 4 ? ndev
->b2b_off
: 0);
2272 iowrite64(bar_addr
, mmio
+ XEON_SBAR45LMT_OFFSET
);
2273 bar_addr
= ioread64(mmio
+ XEON_SBAR45LMT_OFFSET
);
2274 dev_dbg(ndev_dev(ndev
), "SBAR45LMT %#018llx\n", bar_addr
);
2276 bar_addr
= addr
->bar4_addr32
+
2277 (b2b_bar
== 4 ? ndev
->b2b_off
: 0);
2278 iowrite32(bar_addr
, mmio
+ XEON_SBAR4LMT_OFFSET
);
2279 bar_addr
= ioread32(mmio
+ XEON_SBAR4LMT_OFFSET
);
2280 dev_dbg(ndev_dev(ndev
), "SBAR4LMT %#010llx\n", bar_addr
);
2282 bar_addr
= addr
->bar5_addr32
+
2283 (b2b_bar
== 5 ? ndev
->b2b_off
: 0);
2284 iowrite32(bar_addr
, mmio
+ XEON_SBAR5LMT_OFFSET
);
2285 bar_addr
= ioread32(mmio
+ XEON_SBAR5LMT_OFFSET
);
2286 dev_dbg(ndev_dev(ndev
), "SBAR5LMT %#05llx\n", bar_addr
);
2289 /* zero incoming translation addrs */
2290 iowrite64(0, mmio
+ XEON_SBAR23XLAT_OFFSET
);
2292 if (!ndev
->bar4_split
) {
2293 iowrite64(0, mmio
+ XEON_SBAR45XLAT_OFFSET
);
2295 iowrite32(0, mmio
+ XEON_SBAR4XLAT_OFFSET
);
2296 iowrite32(0, mmio
+ XEON_SBAR5XLAT_OFFSET
);
2299 /* zero outgoing translation limits (whole bar size windows) */
2300 iowrite64(0, mmio
+ XEON_PBAR23LMT_OFFSET
);
2301 if (!ndev
->bar4_split
) {
2302 iowrite64(0, mmio
+ XEON_PBAR45LMT_OFFSET
);
2304 iowrite32(0, mmio
+ XEON_PBAR4LMT_OFFSET
);
2305 iowrite32(0, mmio
+ XEON_PBAR5LMT_OFFSET
);
2308 /* set outgoing translation offsets */
2309 bar_addr
= peer_addr
->bar2_addr64
;
2310 iowrite64(bar_addr
, mmio
+ XEON_PBAR23XLAT_OFFSET
);
2311 bar_addr
= ioread64(mmio
+ XEON_PBAR23XLAT_OFFSET
);
2312 dev_dbg(ndev_dev(ndev
), "PBAR23XLAT %#018llx\n", bar_addr
);
2314 if (!ndev
->bar4_split
) {
2315 bar_addr
= peer_addr
->bar4_addr64
;
2316 iowrite64(bar_addr
, mmio
+ XEON_PBAR45XLAT_OFFSET
);
2317 bar_addr
= ioread64(mmio
+ XEON_PBAR45XLAT_OFFSET
);
2318 dev_dbg(ndev_dev(ndev
), "PBAR45XLAT %#018llx\n", bar_addr
);
2320 bar_addr
= peer_addr
->bar4_addr32
;
2321 iowrite32(bar_addr
, mmio
+ XEON_PBAR4XLAT_OFFSET
);
2322 bar_addr
= ioread32(mmio
+ XEON_PBAR4XLAT_OFFSET
);
2323 dev_dbg(ndev_dev(ndev
), "PBAR4XLAT %#010llx\n", bar_addr
);
2325 bar_addr
= peer_addr
->bar5_addr32
;
2326 iowrite32(bar_addr
, mmio
+ XEON_PBAR5XLAT_OFFSET
);
2327 bar_addr
= ioread32(mmio
+ XEON_PBAR5XLAT_OFFSET
);
2328 dev_dbg(ndev_dev(ndev
), "PBAR5XLAT %#010llx\n", bar_addr
);
2331 /* set the translation offset for b2b registers */
2333 bar_addr
= peer_addr
->bar0_addr
;
2334 else if (b2b_bar
== 2)
2335 bar_addr
= peer_addr
->bar2_addr64
;
2336 else if (b2b_bar
== 4 && !ndev
->bar4_split
)
2337 bar_addr
= peer_addr
->bar4_addr64
;
2338 else if (b2b_bar
== 4)
2339 bar_addr
= peer_addr
->bar4_addr32
;
2340 else if (b2b_bar
== 5)
2341 bar_addr
= peer_addr
->bar5_addr32
;
2345 /* B2B_XLAT_OFFSET is 64bit, but can only take 32bit writes */
2346 dev_dbg(ndev_dev(ndev
), "B2BXLAT %#018llx\n", bar_addr
);
2347 iowrite32(bar_addr
, mmio
+ XEON_B2B_XLAT_OFFSETL
);
2348 iowrite32(bar_addr
>> 32, mmio
+ XEON_B2B_XLAT_OFFSETU
);
2351 /* map peer ntb mmio config space registers */
2352 ndev
->peer_mmio
= pci_iomap(pdev
, b2b_bar
,
2354 if (!ndev
->peer_mmio
)
2357 ndev
->peer_addr
= pci_resource_start(pdev
, b2b_bar
);
2363 static int xeon_init_ntb(struct intel_ntb_dev
*ndev
)
2368 if (ndev
->bar4_split
)
2369 ndev
->mw_count
= HSX_SPLIT_BAR_MW_COUNT
;
2371 ndev
->mw_count
= XEON_MW_COUNT
;
2373 ndev
->spad_count
= XEON_SPAD_COUNT
;
2374 ndev
->db_count
= XEON_DB_COUNT
;
2375 ndev
->db_link_mask
= XEON_DB_LINK_BIT
;
2377 switch (ndev
->ntb
.topo
) {
2379 if (ndev
->hwerr_flags
& NTB_HWERR_SDOORBELL_LOCKUP
) {
2380 dev_err(ndev_dev(ndev
), "NTB Primary config disabled\n");
2384 /* enable link to allow secondary side device to appear */
2385 ntb_ctl
= ioread32(ndev
->self_mmio
+ ndev
->reg
->ntb_ctl
);
2386 ntb_ctl
&= ~NTB_CTL_DISABLE
;
2387 iowrite32(ntb_ctl
, ndev
->self_mmio
+ ndev
->reg
->ntb_ctl
);
2389 /* use half the spads for the peer */
2390 ndev
->spad_count
>>= 1;
2391 ndev
->self_reg
= &xeon_pri_reg
;
2392 ndev
->peer_reg
= &xeon_sec_reg
;
2393 ndev
->xlat_reg
= &xeon_sec_xlat
;
2397 if (ndev
->hwerr_flags
& NTB_HWERR_SDOORBELL_LOCKUP
) {
2398 dev_err(ndev_dev(ndev
), "NTB Secondary config disabled\n");
2401 /* use half the spads for the peer */
2402 ndev
->spad_count
>>= 1;
2403 ndev
->self_reg
= &xeon_sec_reg
;
2404 ndev
->peer_reg
= &xeon_pri_reg
;
2405 ndev
->xlat_reg
= &xeon_pri_xlat
;
2408 case NTB_TOPO_B2B_USD
:
2409 case NTB_TOPO_B2B_DSD
:
2410 ndev
->self_reg
= &xeon_pri_reg
;
2411 ndev
->peer_reg
= &xeon_b2b_reg
;
2412 ndev
->xlat_reg
= &xeon_sec_xlat
;
2414 if (ndev
->hwerr_flags
& NTB_HWERR_SDOORBELL_LOCKUP
) {
2415 ndev
->peer_reg
= &xeon_pri_reg
;
2418 ndev
->b2b_idx
= b2b_mw_idx
+ ndev
->mw_count
;
2420 ndev
->b2b_idx
= b2b_mw_idx
;
2422 if (ndev
->b2b_idx
>= ndev
->mw_count
) {
2423 dev_dbg(ndev_dev(ndev
),
2424 "b2b_mw_idx %d invalid for mw_count %u\n",
2425 b2b_mw_idx
, ndev
->mw_count
);
2429 dev_dbg(ndev_dev(ndev
),
2430 "setting up b2b mw idx %d means %d\n",
2431 b2b_mw_idx
, ndev
->b2b_idx
);
2433 } else if (ndev
->hwerr_flags
& NTB_HWERR_B2BDOORBELL_BIT14
) {
2434 dev_warn(ndev_dev(ndev
), "Reduce doorbell count by 1\n");
2435 ndev
->db_count
-= 1;
2438 if (ndev
->ntb
.topo
== NTB_TOPO_B2B_USD
) {
2439 rc
= xeon_setup_b2b_mw(ndev
,
2441 &xeon_b2b_usd_addr
);
2443 rc
= xeon_setup_b2b_mw(ndev
,
2445 &xeon_b2b_dsd_addr
);
2450 /* Enable Bus Master and Memory Space on the secondary side */
2451 iowrite16(PCI_COMMAND_MEMORY
| PCI_COMMAND_MASTER
,
2452 ndev
->self_mmio
+ XEON_SPCICMD_OFFSET
);
2460 ndev
->db_valid_mask
= BIT_ULL(ndev
->db_count
) - 1;
2462 ndev
->reg
->db_iowrite(ndev
->db_valid_mask
,
2464 ndev
->self_reg
->db_mask
);
2469 static int xeon_init_dev(struct intel_ntb_dev
*ndev
)
2471 struct pci_dev
*pdev
;
2475 pdev
= ndev_pdev(ndev
);
2477 switch (pdev
->device
) {
2478 /* There is a Xeon hardware errata related to writes to SDOORBELL or
2479 * B2BDOORBELL in conjunction with inbound access to NTB MMIO Space,
2480 * which may hang the system. To workaround this use the second memory
2481 * window to access the interrupt and scratch pad registers on the
2484 case PCI_DEVICE_ID_INTEL_NTB_SS_JSF
:
2485 case PCI_DEVICE_ID_INTEL_NTB_PS_JSF
:
2486 case PCI_DEVICE_ID_INTEL_NTB_B2B_JSF
:
2487 case PCI_DEVICE_ID_INTEL_NTB_SS_SNB
:
2488 case PCI_DEVICE_ID_INTEL_NTB_PS_SNB
:
2489 case PCI_DEVICE_ID_INTEL_NTB_B2B_SNB
:
2490 case PCI_DEVICE_ID_INTEL_NTB_SS_IVT
:
2491 case PCI_DEVICE_ID_INTEL_NTB_PS_IVT
:
2492 case PCI_DEVICE_ID_INTEL_NTB_B2B_IVT
:
2493 case PCI_DEVICE_ID_INTEL_NTB_SS_HSX
:
2494 case PCI_DEVICE_ID_INTEL_NTB_PS_HSX
:
2495 case PCI_DEVICE_ID_INTEL_NTB_B2B_HSX
:
2496 case PCI_DEVICE_ID_INTEL_NTB_SS_BDX
:
2497 case PCI_DEVICE_ID_INTEL_NTB_PS_BDX
:
2498 case PCI_DEVICE_ID_INTEL_NTB_B2B_BDX
:
2499 ndev
->hwerr_flags
|= NTB_HWERR_SDOORBELL_LOCKUP
;
2503 switch (pdev
->device
) {
2504 /* There is a hardware errata related to accessing any register in
2505 * SB01BASE in the presence of bidirectional traffic crossing the NTB.
2507 case PCI_DEVICE_ID_INTEL_NTB_SS_IVT
:
2508 case PCI_DEVICE_ID_INTEL_NTB_PS_IVT
:
2509 case PCI_DEVICE_ID_INTEL_NTB_B2B_IVT
:
2510 case PCI_DEVICE_ID_INTEL_NTB_SS_HSX
:
2511 case PCI_DEVICE_ID_INTEL_NTB_PS_HSX
:
2512 case PCI_DEVICE_ID_INTEL_NTB_B2B_HSX
:
2513 case PCI_DEVICE_ID_INTEL_NTB_SS_BDX
:
2514 case PCI_DEVICE_ID_INTEL_NTB_PS_BDX
:
2515 case PCI_DEVICE_ID_INTEL_NTB_B2B_BDX
:
2516 ndev
->hwerr_flags
|= NTB_HWERR_SB01BASE_LOCKUP
;
2520 switch (pdev
->device
) {
2521 /* HW Errata on bit 14 of b2bdoorbell register. Writes will not be
2522 * mirrored to the remote system. Shrink the number of bits by one,
2523 * since bit 14 is the last bit.
2525 case PCI_DEVICE_ID_INTEL_NTB_SS_JSF
:
2526 case PCI_DEVICE_ID_INTEL_NTB_PS_JSF
:
2527 case PCI_DEVICE_ID_INTEL_NTB_B2B_JSF
:
2528 case PCI_DEVICE_ID_INTEL_NTB_SS_SNB
:
2529 case PCI_DEVICE_ID_INTEL_NTB_PS_SNB
:
2530 case PCI_DEVICE_ID_INTEL_NTB_B2B_SNB
:
2531 case PCI_DEVICE_ID_INTEL_NTB_SS_IVT
:
2532 case PCI_DEVICE_ID_INTEL_NTB_PS_IVT
:
2533 case PCI_DEVICE_ID_INTEL_NTB_B2B_IVT
:
2534 case PCI_DEVICE_ID_INTEL_NTB_SS_HSX
:
2535 case PCI_DEVICE_ID_INTEL_NTB_PS_HSX
:
2536 case PCI_DEVICE_ID_INTEL_NTB_B2B_HSX
:
2537 case PCI_DEVICE_ID_INTEL_NTB_SS_BDX
:
2538 case PCI_DEVICE_ID_INTEL_NTB_PS_BDX
:
2539 case PCI_DEVICE_ID_INTEL_NTB_B2B_BDX
:
2540 ndev
->hwerr_flags
|= NTB_HWERR_B2BDOORBELL_BIT14
;
2544 ndev
->reg
= &xeon_reg
;
2546 rc
= pci_read_config_byte(pdev
, XEON_PPD_OFFSET
, &ppd
);
2550 ndev
->ntb
.topo
= xeon_ppd_topo(ndev
, ppd
);
2551 dev_dbg(ndev_dev(ndev
), "ppd %#x topo %s\n", ppd
,
2552 ntb_topo_string(ndev
->ntb
.topo
));
2553 if (ndev
->ntb
.topo
== NTB_TOPO_NONE
)
2556 if (ndev
->ntb
.topo
!= NTB_TOPO_SEC
) {
2557 ndev
->bar4_split
= xeon_ppd_bar4_split(ndev
, ppd
);
2558 dev_dbg(ndev_dev(ndev
), "ppd %#x bar4_split %d\n",
2559 ppd
, ndev
->bar4_split
);
2561 /* This is a way for transparent BAR to figure out if we are
2562 * doing split BAR or not. There is no way for the hw on the
2563 * transparent side to know and set the PPD.
2565 mem
= pci_select_bars(pdev
, IORESOURCE_MEM
);
2566 ndev
->bar4_split
= hweight32(mem
) ==
2567 HSX_SPLIT_BAR_MW_COUNT
+ 1;
2568 dev_dbg(ndev_dev(ndev
), "mem %#x bar4_split %d\n",
2569 mem
, ndev
->bar4_split
);
2572 rc
= xeon_init_ntb(ndev
);
2576 return xeon_init_isr(ndev
);
2579 static void xeon_deinit_dev(struct intel_ntb_dev
*ndev
)
2581 xeon_deinit_isr(ndev
);
2584 static int intel_ntb_init_pci(struct intel_ntb_dev
*ndev
, struct pci_dev
*pdev
)
2588 pci_set_drvdata(pdev
, ndev
);
2590 rc
= pci_enable_device(pdev
);
2592 goto err_pci_enable
;
2594 rc
= pci_request_regions(pdev
, NTB_NAME
);
2596 goto err_pci_regions
;
2598 pci_set_master(pdev
);
2600 rc
= pci_set_dma_mask(pdev
, DMA_BIT_MASK(64));
2602 rc
= pci_set_dma_mask(pdev
, DMA_BIT_MASK(32));
2605 dev_warn(ndev_dev(ndev
), "Cannot DMA highmem\n");
2608 rc
= pci_set_consistent_dma_mask(pdev
, DMA_BIT_MASK(64));
2610 rc
= pci_set_consistent_dma_mask(pdev
, DMA_BIT_MASK(32));
2613 dev_warn(ndev_dev(ndev
), "Cannot DMA consistent highmem\n");
2616 ndev
->self_mmio
= pci_iomap(pdev
, 0, 0);
2617 if (!ndev
->self_mmio
) {
2621 ndev
->peer_mmio
= ndev
->self_mmio
;
2622 ndev
->peer_addr
= pci_resource_start(pdev
, 0);
2628 pci_clear_master(pdev
);
2629 pci_release_regions(pdev
);
2631 pci_disable_device(pdev
);
2633 pci_set_drvdata(pdev
, NULL
);
2637 static void intel_ntb_deinit_pci(struct intel_ntb_dev
*ndev
)
2639 struct pci_dev
*pdev
= ndev_pdev(ndev
);
2641 if (ndev
->peer_mmio
&& ndev
->peer_mmio
!= ndev
->self_mmio
)
2642 pci_iounmap(pdev
, ndev
->peer_mmio
);
2643 pci_iounmap(pdev
, ndev
->self_mmio
);
2645 pci_clear_master(pdev
);
2646 pci_release_regions(pdev
);
2647 pci_disable_device(pdev
);
2648 pci_set_drvdata(pdev
, NULL
);
2651 static inline void ndev_init_struct(struct intel_ntb_dev
*ndev
,
2652 struct pci_dev
*pdev
)
2654 ndev
->ntb
.pdev
= pdev
;
2655 ndev
->ntb
.topo
= NTB_TOPO_NONE
;
2656 ndev
->ntb
.ops
= &intel_ntb_ops
;
2659 ndev
->b2b_idx
= UINT_MAX
;
2661 ndev
->bar4_split
= 0;
2664 ndev
->spad_count
= 0;
2666 ndev
->db_vec_count
= 0;
2667 ndev
->db_vec_shift
= 0;
2672 ndev
->db_valid_mask
= 0;
2673 ndev
->db_link_mask
= 0;
2676 spin_lock_init(&ndev
->db_mask_lock
);
2679 static int intel_ntb_pci_probe(struct pci_dev
*pdev
,
2680 const struct pci_device_id
*id
)
2682 struct intel_ntb_dev
*ndev
;
2685 node
= dev_to_node(&pdev
->dev
);
2687 if (pdev_is_atom(pdev
)) {
2688 ndev
= kzalloc_node(sizeof(*ndev
), GFP_KERNEL
, node
);
2694 ndev_init_struct(ndev
, pdev
);
2696 rc
= intel_ntb_init_pci(ndev
, pdev
);
2700 rc
= atom_init_dev(ndev
);
2704 } else if (pdev_is_xeon(pdev
)) {
2705 ndev
= kzalloc_node(sizeof(*ndev
), GFP_KERNEL
, node
);
2711 ndev_init_struct(ndev
, pdev
);
2713 rc
= intel_ntb_init_pci(ndev
, pdev
);
2717 rc
= xeon_init_dev(ndev
);
2721 } else if (pdev_is_skx_xeon(pdev
)) {
2722 ndev
= kzalloc_node(sizeof(*ndev
), GFP_KERNEL
, node
);
2728 ndev_init_struct(ndev
, pdev
);
2729 ndev
->ntb
.ops
= &intel_ntb3_ops
;
2731 rc
= intel_ntb_init_pci(ndev
, pdev
);
2735 rc
= skx_init_dev(ndev
);
2744 ndev_reset_unsafe_flags(ndev
);
2746 ndev
->reg
->poll_link(ndev
);
2748 ndev_init_debugfs(ndev
);
2750 rc
= ntb_register_device(&ndev
->ntb
);
2754 dev_info(&pdev
->dev
, "NTB device registered.\n");
2759 ndev_deinit_debugfs(ndev
);
2760 if (pdev_is_atom(pdev
))
2761 atom_deinit_dev(ndev
);
2762 else if (pdev_is_xeon(pdev
) || pdev_is_skx_xeon(pdev
))
2763 xeon_deinit_dev(ndev
);
2765 intel_ntb_deinit_pci(ndev
);
2772 static void intel_ntb_pci_remove(struct pci_dev
*pdev
)
2774 struct intel_ntb_dev
*ndev
= pci_get_drvdata(pdev
);
2776 ntb_unregister_device(&ndev
->ntb
);
2777 ndev_deinit_debugfs(ndev
);
2778 if (pdev_is_atom(pdev
))
2779 atom_deinit_dev(ndev
);
2780 else if (pdev_is_xeon(pdev
) || pdev_is_skx_xeon(pdev
))
2781 xeon_deinit_dev(ndev
);
2782 intel_ntb_deinit_pci(ndev
);
2786 static const struct intel_ntb_reg atom_reg
= {
2787 .poll_link
= atom_poll_link
,
2788 .link_is_up
= atom_link_is_up
,
2789 .db_ioread
= atom_db_ioread
,
2790 .db_iowrite
= atom_db_iowrite
,
2791 .db_size
= sizeof(u64
),
2792 .ntb_ctl
= ATOM_NTBCNTL_OFFSET
,
2796 static const struct intel_ntb_alt_reg atom_pri_reg
= {
2797 .db_bell
= ATOM_PDOORBELL_OFFSET
,
2798 .db_mask
= ATOM_PDBMSK_OFFSET
,
2799 .spad
= ATOM_SPAD_OFFSET
,
2802 static const struct intel_ntb_alt_reg atom_b2b_reg
= {
2803 .db_bell
= ATOM_B2B_DOORBELL_OFFSET
,
2804 .spad
= ATOM_B2B_SPAD_OFFSET
,
2807 static const struct intel_ntb_xlat_reg atom_sec_xlat
= {
2808 /* FIXME : .bar0_base = ATOM_SBAR0BASE_OFFSET, */
2809 /* FIXME : .bar2_limit = ATOM_SBAR2LMT_OFFSET, */
2810 .bar2_xlat
= ATOM_SBAR2XLAT_OFFSET
,
2813 static const struct intel_ntb_reg xeon_reg
= {
2814 .poll_link
= xeon_poll_link
,
2815 .link_is_up
= xeon_link_is_up
,
2816 .db_ioread
= xeon_db_ioread
,
2817 .db_iowrite
= xeon_db_iowrite
,
2818 .db_size
= sizeof(u32
),
2819 .ntb_ctl
= XEON_NTBCNTL_OFFSET
,
2820 .mw_bar
= {2, 4, 5},
2823 static const struct intel_ntb_alt_reg xeon_pri_reg
= {
2824 .db_bell
= XEON_PDOORBELL_OFFSET
,
2825 .db_mask
= XEON_PDBMSK_OFFSET
,
2826 .spad
= XEON_SPAD_OFFSET
,
2829 static const struct intel_ntb_alt_reg xeon_sec_reg
= {
2830 .db_bell
= XEON_SDOORBELL_OFFSET
,
2831 .db_mask
= XEON_SDBMSK_OFFSET
,
2832 /* second half of the scratchpads */
2833 .spad
= XEON_SPAD_OFFSET
+ (XEON_SPAD_COUNT
<< 1),
2836 static const struct intel_ntb_alt_reg xeon_b2b_reg
= {
2837 .db_bell
= XEON_B2B_DOORBELL_OFFSET
,
2838 .spad
= XEON_B2B_SPAD_OFFSET
,
2841 static const struct intel_ntb_xlat_reg xeon_pri_xlat
= {
2842 /* Note: no primary .bar0_base visible to the secondary side.
2844 * The secondary side cannot get the base address stored in primary
2845 * bars. The base address is necessary to set the limit register to
2846 * any value other than zero, or unlimited.
2848 * WITHOUT THE BASE ADDRESS, THE SECONDARY SIDE CANNOT DISABLE the
2849 * window by setting the limit equal to base, nor can it limit the size
2850 * of the memory window by setting the limit to base + size.
2852 .bar2_limit
= XEON_PBAR23LMT_OFFSET
,
2853 .bar2_xlat
= XEON_PBAR23XLAT_OFFSET
,
2856 static const struct intel_ntb_xlat_reg xeon_sec_xlat
= {
2857 .bar0_base
= XEON_SBAR0BASE_OFFSET
,
2858 .bar2_limit
= XEON_SBAR23LMT_OFFSET
,
2859 .bar2_xlat
= XEON_SBAR23XLAT_OFFSET
,
2862 static struct intel_b2b_addr xeon_b2b_usd_addr
= {
2863 .bar2_addr64
= XEON_B2B_BAR2_ADDR64
,
2864 .bar4_addr64
= XEON_B2B_BAR4_ADDR64
,
2865 .bar4_addr32
= XEON_B2B_BAR4_ADDR32
,
2866 .bar5_addr32
= XEON_B2B_BAR5_ADDR32
,
2869 static struct intel_b2b_addr xeon_b2b_dsd_addr
= {
2870 .bar2_addr64
= XEON_B2B_BAR2_ADDR64
,
2871 .bar4_addr64
= XEON_B2B_BAR4_ADDR64
,
2872 .bar4_addr32
= XEON_B2B_BAR4_ADDR32
,
2873 .bar5_addr32
= XEON_B2B_BAR5_ADDR32
,
2876 static const struct intel_ntb_reg skx_reg
= {
2877 .poll_link
= skx_poll_link
,
2878 .link_is_up
= xeon_link_is_up
,
2879 .db_ioread
= skx_db_ioread
,
2880 .db_iowrite
= skx_db_iowrite
,
2881 .db_size
= sizeof(u64
),
2882 .ntb_ctl
= SKX_NTBCNTL_OFFSET
,
2886 static const struct intel_ntb_alt_reg skx_pri_reg
= {
2887 .db_bell
= SKX_EM_DOORBELL_OFFSET
,
2888 .db_clear
= SKX_IM_INT_STATUS_OFFSET
,
2889 .db_mask
= SKX_IM_INT_DISABLE_OFFSET
,
2890 .spad
= SKX_IM_SPAD_OFFSET
,
2893 static const struct intel_ntb_alt_reg skx_b2b_reg
= {
2894 .db_bell
= SKX_IM_DOORBELL_OFFSET
,
2895 .db_clear
= SKX_EM_INT_STATUS_OFFSET
,
2896 .db_mask
= SKX_EM_INT_DISABLE_OFFSET
,
2897 .spad
= SKX_B2B_SPAD_OFFSET
,
2900 static const struct intel_ntb_xlat_reg skx_sec_xlat
= {
2901 /* .bar0_base = SKX_EMBAR0_OFFSET, */
2902 .bar2_limit
= SKX_IMBAR1XLMT_OFFSET
,
2903 .bar2_xlat
= SKX_IMBAR1XBASE_OFFSET
,
2906 /* operations for primary side of local ntb */
2907 static const struct ntb_dev_ops intel_ntb_ops
= {
2908 .mw_count
= intel_ntb_mw_count
,
2909 .mw_get_range
= intel_ntb_mw_get_range
,
2910 .mw_set_trans
= intel_ntb_mw_set_trans
,
2911 .link_is_up
= intel_ntb_link_is_up
,
2912 .link_enable
= intel_ntb_link_enable
,
2913 .link_disable
= intel_ntb_link_disable
,
2914 .db_is_unsafe
= intel_ntb_db_is_unsafe
,
2915 .db_valid_mask
= intel_ntb_db_valid_mask
,
2916 .db_vector_count
= intel_ntb_db_vector_count
,
2917 .db_vector_mask
= intel_ntb_db_vector_mask
,
2918 .db_read
= intel_ntb_db_read
,
2919 .db_clear
= intel_ntb_db_clear
,
2920 .db_set_mask
= intel_ntb_db_set_mask
,
2921 .db_clear_mask
= intel_ntb_db_clear_mask
,
2922 .peer_db_addr
= intel_ntb_peer_db_addr
,
2923 .peer_db_set
= intel_ntb_peer_db_set
,
2924 .spad_is_unsafe
= intel_ntb_spad_is_unsafe
,
2925 .spad_count
= intel_ntb_spad_count
,
2926 .spad_read
= intel_ntb_spad_read
,
2927 .spad_write
= intel_ntb_spad_write
,
2928 .peer_spad_addr
= intel_ntb_peer_spad_addr
,
2929 .peer_spad_read
= intel_ntb_peer_spad_read
,
2930 .peer_spad_write
= intel_ntb_peer_spad_write
,
2933 static const struct ntb_dev_ops intel_ntb3_ops
= {
2934 .mw_count
= intel_ntb_mw_count
,
2935 .mw_get_range
= intel_ntb_mw_get_range
,
2936 .mw_set_trans
= intel_ntb3_mw_set_trans
,
2937 .link_is_up
= intel_ntb_link_is_up
,
2938 .link_enable
= intel_ntb3_link_enable
,
2939 .link_disable
= intel_ntb_link_disable
,
2940 .db_valid_mask
= intel_ntb_db_valid_mask
,
2941 .db_vector_count
= intel_ntb_db_vector_count
,
2942 .db_vector_mask
= intel_ntb_db_vector_mask
,
2943 .db_read
= intel_ntb3_db_read
,
2944 .db_clear
= intel_ntb3_db_clear
,
2945 .db_set_mask
= intel_ntb_db_set_mask
,
2946 .db_clear_mask
= intel_ntb_db_clear_mask
,
2947 .peer_db_addr
= intel_ntb_peer_db_addr
,
2948 .peer_db_set
= intel_ntb3_peer_db_set
,
2949 .spad_is_unsafe
= intel_ntb_spad_is_unsafe
,
2950 .spad_count
= intel_ntb_spad_count
,
2951 .spad_read
= intel_ntb_spad_read
,
2952 .spad_write
= intel_ntb_spad_write
,
2953 .peer_spad_addr
= intel_ntb_peer_spad_addr
,
2954 .peer_spad_read
= intel_ntb_peer_spad_read
,
2955 .peer_spad_write
= intel_ntb_peer_spad_write
,
2958 static const struct file_operations intel_ntb_debugfs_info
= {
2959 .owner
= THIS_MODULE
,
2960 .open
= simple_open
,
2961 .read
= ndev_debugfs_read
,
2964 static const struct pci_device_id intel_ntb_pci_tbl
[] = {
2965 {PCI_VDEVICE(INTEL
, PCI_DEVICE_ID_INTEL_NTB_B2B_BWD
)},
2966 {PCI_VDEVICE(INTEL
, PCI_DEVICE_ID_INTEL_NTB_B2B_JSF
)},
2967 {PCI_VDEVICE(INTEL
, PCI_DEVICE_ID_INTEL_NTB_B2B_SNB
)},
2968 {PCI_VDEVICE(INTEL
, PCI_DEVICE_ID_INTEL_NTB_B2B_IVT
)},
2969 {PCI_VDEVICE(INTEL
, PCI_DEVICE_ID_INTEL_NTB_B2B_HSX
)},
2970 {PCI_VDEVICE(INTEL
, PCI_DEVICE_ID_INTEL_NTB_B2B_BDX
)},
2971 {PCI_VDEVICE(INTEL
, PCI_DEVICE_ID_INTEL_NTB_PS_JSF
)},
2972 {PCI_VDEVICE(INTEL
, PCI_DEVICE_ID_INTEL_NTB_PS_SNB
)},
2973 {PCI_VDEVICE(INTEL
, PCI_DEVICE_ID_INTEL_NTB_PS_IVT
)},
2974 {PCI_VDEVICE(INTEL
, PCI_DEVICE_ID_INTEL_NTB_PS_HSX
)},
2975 {PCI_VDEVICE(INTEL
, PCI_DEVICE_ID_INTEL_NTB_PS_BDX
)},
2976 {PCI_VDEVICE(INTEL
, PCI_DEVICE_ID_INTEL_NTB_SS_JSF
)},
2977 {PCI_VDEVICE(INTEL
, PCI_DEVICE_ID_INTEL_NTB_SS_SNB
)},
2978 {PCI_VDEVICE(INTEL
, PCI_DEVICE_ID_INTEL_NTB_SS_IVT
)},
2979 {PCI_VDEVICE(INTEL
, PCI_DEVICE_ID_INTEL_NTB_SS_HSX
)},
2980 {PCI_VDEVICE(INTEL
, PCI_DEVICE_ID_INTEL_NTB_SS_BDX
)},
2981 {PCI_VDEVICE(INTEL
, PCI_DEVICE_ID_INTEL_NTB_B2B_SKX
)},
2984 MODULE_DEVICE_TABLE(pci
, intel_ntb_pci_tbl
);
2986 static struct pci_driver intel_ntb_pci_driver
= {
2987 .name
= KBUILD_MODNAME
,
2988 .id_table
= intel_ntb_pci_tbl
,
2989 .probe
= intel_ntb_pci_probe
,
2990 .remove
= intel_ntb_pci_remove
,
2993 static int __init
intel_ntb_pci_driver_init(void)
2995 pr_info("%s %s\n", NTB_DESC
, NTB_VER
);
2997 if (debugfs_initialized())
2998 debugfs_dir
= debugfs_create_dir(KBUILD_MODNAME
, NULL
);
3000 return pci_register_driver(&intel_ntb_pci_driver
);
3002 module_init(intel_ntb_pci_driver_init
);
3004 static void __exit
intel_ntb_pci_driver_exit(void)
3006 pci_unregister_driver(&intel_ntb_pci_driver
);
3008 debugfs_remove_recursive(debugfs_dir
);
3010 module_exit(intel_ntb_pci_driver_exit
);