* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
+#include "spdk/stdinc.h"
+
#include "ioat_internal.h"
#include "spdk/env.h"
#include "spdk_internal/log.h"
-#include <pthread.h>
-
struct ioat_driver {
pthread_mutex_t lock;
TAILQ_HEAD(, spdk_ioat_chan) attached_chans;
*hw_desc = &ioat->hw_ring[i];
}
-static uint64_t
-ioat_get_desc_phys_addr(struct spdk_ioat_chan *ioat, uint32_t index)
-{
- return ioat->hw_ring_phys_addr +
- ioat_get_ring_index(ioat, index) * sizeof(union spdk_ioat_hw_desc);
-}
-
static void
ioat_submit_single(struct spdk_ioat_chan *ioat)
{
int timeout;
uint64_t status;
uint32_t chanerr;
+ int rc;
status = ioat_get_chansts(ioat);
if (is_ioat_active(status) || is_ioat_idle(status)) {
chanerr = ioat->regs->chanerr;
ioat->regs->chanerr = chanerr;
+ if (ioat->regs->cbver < SPDK_IOAT_VER_3_3) {
+ rc = spdk_pci_device_cfg_read32(ioat->device, &chanerr,
+ SPDK_IOAT_PCI_CHANERR_INT_OFFSET);
+ if (rc) {
+ SPDK_ERRLOG("failed to read the internal channel error register\n");
+ return -1;
+ }
+
+ spdk_pci_device_cfg_write32(ioat->device, chanerr,
+ SPDK_IOAT_PCI_CHANERR_INT_OFFSET);
+ }
+
ioat_reset(ioat);
timeout = 20;
desc->callback_fn(desc->callback_arg);
}
- hw_desc_phys_addr = ioat_get_desc_phys_addr(ioat, ioat->tail);
+ hw_desc_phys_addr = desc->phys_addr;
ioat->tail++;
} while (hw_desc_phys_addr != completed_descriptor);
return 0;
}
-static int
+static void
ioat_channel_destruct(struct spdk_ioat_chan *ioat)
{
ioat_unmap_pci_bar(ioat);
}
if (ioat->hw_ring) {
- spdk_free(ioat->hw_ring);
+ spdk_dma_free(ioat->hw_ring);
}
if (ioat->comp_update) {
- spdk_free((void *)ioat->comp_update);
+ spdk_dma_free((void *)ioat->comp_update);
ioat->comp_update = NULL;
}
-
- return 0;
}
static int
uint64_t status;
int i, num_descriptors;
uint64_t comp_update_bus_addr = 0;
+ uint64_t phys_addr;
if (ioat_map_pci_bar(ioat) != 0) {
SPDK_ERRLOG("ioat_map_pci_bar() failed\n");
/* Always support DMA copy */
ioat->dma_capabilities = SPDK_IOAT_ENGINE_COPY_SUPPORTED;
- if (ioat->regs->dmacapability & SPDK_IOAT_DMACAP_BFILL)
+ if (ioat->regs->dmacapability & SPDK_IOAT_DMACAP_BFILL) {
ioat->dma_capabilities |= SPDK_IOAT_ENGINE_FILL_SUPPORTED;
+ }
xfercap = ioat->regs->xfercap;
/* Only bits [4:0] are valid. */
/* 0 means 4 GB max transfer size. */
ioat->max_xfer_size = 1ULL << 32;
} else if (xfercap < 12) {
- /* XFCERCAP must be at least 12 (4 KB) according to the spec. */
+ /* XFERCAP must be at least 12 (4 KB) according to the spec. */
SPDK_ERRLOG("invalid XFERCAP value %u\n", xfercap);
return -1;
} else {
ioat->max_xfer_size = 1U << xfercap;
}
- ioat->comp_update = spdk_zmalloc(sizeof(*ioat->comp_update), SPDK_IOAT_CHANCMP_ALIGN,
- &comp_update_bus_addr);
+ ioat->comp_update = spdk_dma_zmalloc(sizeof(*ioat->comp_update), SPDK_IOAT_CHANCMP_ALIGN,
+ &comp_update_bus_addr);
if (ioat->comp_update == NULL) {
return -1;
}
return -1;
}
- ioat->hw_ring = spdk_zmalloc(num_descriptors * sizeof(union spdk_ioat_hw_desc), 64,
- &ioat->hw_ring_phys_addr);
+ ioat->hw_ring = spdk_dma_zmalloc(num_descriptors * sizeof(union spdk_ioat_hw_desc), 64,
+ NULL);
if (!ioat->hw_ring) {
return -1;
}
for (i = 0; i < num_descriptors; i++) {
- ioat->hw_ring[i].generic.next = ioat_get_desc_phys_addr(ioat, i + 1);
+ phys_addr = spdk_vtophys(&ioat->hw_ring[i]);
+ if (phys_addr == SPDK_VTOPHYS_ERROR) {
+ SPDK_ERRLOG("Failed to translate descriptor %u to physical address\n", i);
+ return -1;
+ }
+
+ ioat->ring[i].phys_addr = phys_addr;
+ ioat->hw_ring[ioat_get_ring_index(ioat, i - 1)].generic.next = phys_addr;
}
ioat->head = 0;
ioat->regs->chanctrl = SPDK_IOAT_CHANCTRL_ANY_ERR_ABORT_EN;
ioat_write_chancmp(ioat, comp_update_bus_addr);
- ioat_write_chainaddr(ioat, ioat->hw_ring_phys_addr);
+ ioat_write_chainaddr(ioat, ioat->ring[0].phys_addr);
ioat_prep_null(ioat);
ioat_flush(ioat);
while (i-- > 0) {
spdk_delay_us(100);
status = ioat_get_chansts(ioat);
- if (is_ioat_idle(status))
+ if (is_ioat_idle(status)) {
break;
+ }
}
if (is_ioat_idle(status)) {
/* Caller must hold g_ioat_driver.lock */
static struct spdk_ioat_chan *
-ioat_attach(void *device)
+ioat_attach(struct spdk_pci_device *device)
{
- struct spdk_ioat_chan *ioat;
+ struct spdk_ioat_chan *ioat;
uint32_t cmd_reg;
ioat = calloc(1, sizeof(struct spdk_ioat_chan));
return rc;
}
-int
+void
spdk_ioat_detach(struct spdk_ioat_chan *ioat)
{
struct ioat_driver *driver = &g_ioat_driver;
ioat_channel_destruct(ioat);
free(ioat);
-
- return 0;
}
#define _2MB_PAGE(ptr) ((ptr) & ~(0x200000 - 1))
#define _2MB_OFFSET(ptr) ((ptr) & (0x200000 - 1))
-int64_t
+int
spdk_ioat_submit_copy(struct spdk_ioat_chan *ioat, void *cb_arg, spdk_ioat_req_cb cb_fn,
void *dst, const void *src, uint64_t nbytes)
{
uint32_t orig_head;
if (!ioat) {
- return -1;
+ return -EINVAL;
}
orig_head = ioat->head;
* in case we managed to fill out any descriptors.
*/
ioat->head = orig_head;
- return -1;
+ return -ENOMEM;
}
ioat_flush(ioat);
- return nbytes;
+ return 0;
}
-int64_t
+int
spdk_ioat_submit_fill(struct spdk_ioat_chan *ioat, void *cb_arg, spdk_ioat_req_cb cb_fn,
void *dst, uint64_t fill_pattern, uint64_t nbytes)
{
uint32_t orig_head;
if (!ioat) {
- return -1;
+ return -EINVAL;
}
if (!(ioat->dma_capabilities & SPDK_IOAT_ENGINE_FILL_SUPPORTED)) {
while (remaining) {
op_size = remaining;
+ op_size = spdk_min(op_size, (0x200000 - _2MB_OFFSET(vdst)));
op_size = spdk_min(op_size, ioat->max_xfer_size);
remaining -= op_size;
* in case we managed to fill out any descriptors.
*/
ioat->head = orig_head;
- return -1;
+ return -ENOMEM;
}
ioat_flush(ioat);
- return nbytes;
+ return 0;
}
uint32_t
return ioat_process_channel_events(ioat);
}
-SPDK_LOG_REGISTER_TRACE_FLAG("ioat", SPDK_TRACE_IOAT)
+SPDK_LOG_REGISTER_COMPONENT("ioat", SPDK_LOG_IOAT)
projects / ceph.git / blobdiff