1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2015-2018 Intel Corporation
5 #include <rte_bus_pci.h>
6 #include <rte_common.h>
8 #include <rte_malloc.h>
10 #include <rte_cryptodev_pmd.h>
14 #include "qat_sym_session.h"
15 #include "qat_sym_pmd.h"
17 uint8_t cryptodev_qat_driver_id
;
19 static const struct rte_cryptodev_capabilities qat_gen1_sym_capabilities
[] = {
20 QAT_BASE_GEN1_SYM_CAPABILITIES
,
21 RTE_CRYPTODEV_END_OF_CAPABILITIES_LIST()
24 static const struct rte_cryptodev_capabilities qat_gen2_sym_capabilities
[] = {
25 QAT_BASE_GEN1_SYM_CAPABILITIES
,
26 QAT_EXTRA_GEN2_SYM_CAPABILITIES
,
27 RTE_CRYPTODEV_END_OF_CAPABILITIES_LIST()
30 static int qat_sym_qp_release(struct rte_cryptodev
*dev
,
31 uint16_t queue_pair_id
);
33 static int qat_sym_dev_config(__rte_unused
struct rte_cryptodev
*dev
,
34 __rte_unused
struct rte_cryptodev_config
*config
)
39 static int qat_sym_dev_start(__rte_unused
struct rte_cryptodev
*dev
)
44 static void qat_sym_dev_stop(__rte_unused
struct rte_cryptodev
*dev
)
49 static int qat_sym_dev_close(struct rte_cryptodev
*dev
)
53 for (i
= 0; i
< dev
->data
->nb_queue_pairs
; i
++) {
54 ret
= qat_sym_qp_release(dev
, i
);
62 static void qat_sym_dev_info_get(struct rte_cryptodev
*dev
,
63 struct rte_cryptodev_info
*info
)
65 struct qat_sym_dev_private
*internals
= dev
->data
->dev_private
;
66 const struct qat_qp_hw_data
*sym_hw_qps
=
67 qat_gen_config
[internals
->qat_dev
->qat_dev_gen
]
68 .qp_hw_data
[QAT_SERVICE_SYMMETRIC
];
71 info
->max_nb_queue_pairs
=
72 qat_qps_per_service(sym_hw_qps
, QAT_SERVICE_SYMMETRIC
);
73 info
->feature_flags
= dev
->feature_flags
;
74 info
->capabilities
= internals
->qat_dev_capabilities
;
75 info
->driver_id
= cryptodev_qat_driver_id
;
76 /* No limit of number of sessions */
77 info
->sym
.max_nb_sessions
= 0;
81 static void qat_sym_stats_get(struct rte_cryptodev
*dev
,
82 struct rte_cryptodev_stats
*stats
)
84 struct qat_common_stats qat_stats
= {0};
85 struct qat_sym_dev_private
*qat_priv
;
87 if (stats
== NULL
|| dev
== NULL
) {
88 QAT_LOG(ERR
, "invalid ptr: stats %p, dev %p", stats
, dev
);
91 qat_priv
= dev
->data
->dev_private
;
93 qat_stats_get(qat_priv
->qat_dev
, &qat_stats
, QAT_SERVICE_SYMMETRIC
);
94 stats
->enqueued_count
= qat_stats
.enqueued_count
;
95 stats
->dequeued_count
= qat_stats
.dequeued_count
;
96 stats
->enqueue_err_count
= qat_stats
.enqueue_err_count
;
97 stats
->dequeue_err_count
= qat_stats
.dequeue_err_count
;
100 static void qat_sym_stats_reset(struct rte_cryptodev
*dev
)
102 struct qat_sym_dev_private
*qat_priv
;
105 QAT_LOG(ERR
, "invalid cryptodev ptr %p", dev
);
108 qat_priv
= dev
->data
->dev_private
;
110 qat_stats_reset(qat_priv
->qat_dev
, QAT_SERVICE_SYMMETRIC
);
114 static int qat_sym_qp_release(struct rte_cryptodev
*dev
, uint16_t queue_pair_id
)
116 struct qat_sym_dev_private
*qat_private
= dev
->data
->dev_private
;
118 QAT_LOG(DEBUG
, "Release sym qp %u on device %d",
119 queue_pair_id
, dev
->data
->dev_id
);
121 qat_private
->qat_dev
->qps_in_use
[QAT_SERVICE_SYMMETRIC
][queue_pair_id
]
124 return qat_qp_release((struct qat_qp
**)
125 &(dev
->data
->queue_pairs
[queue_pair_id
]));
128 static int qat_sym_qp_setup(struct rte_cryptodev
*dev
, uint16_t qp_id
,
129 const struct rte_cryptodev_qp_conf
*qp_conf
,
135 struct qat_qp_config qat_qp_conf
;
137 struct qat_qp
**qp_addr
=
138 (struct qat_qp
**)&(dev
->data
->queue_pairs
[qp_id
]);
139 struct qat_sym_dev_private
*qat_private
= dev
->data
->dev_private
;
140 const struct qat_qp_hw_data
*sym_hw_qps
=
141 qat_gen_config
[qat_private
->qat_dev
->qat_dev_gen
]
142 .qp_hw_data
[QAT_SERVICE_SYMMETRIC
];
143 const struct qat_qp_hw_data
*qp_hw_data
= sym_hw_qps
+ qp_id
;
145 /* If qp is already in use free ring memory and qp metadata. */
146 if (*qp_addr
!= NULL
) {
147 ret
= qat_sym_qp_release(dev
, qp_id
);
151 if (qp_id
>= qat_qps_per_service(sym_hw_qps
, QAT_SERVICE_SYMMETRIC
)) {
152 QAT_LOG(ERR
, "qp_id %u invalid for this device", qp_id
);
156 qat_qp_conf
.hw
= qp_hw_data
;
157 qat_qp_conf
.build_request
= qat_sym_build_request
;
158 qat_qp_conf
.cookie_size
= sizeof(struct qat_sym_op_cookie
);
159 qat_qp_conf
.nb_descriptors
= qp_conf
->nb_descriptors
;
160 qat_qp_conf
.socket_id
= socket_id
;
161 qat_qp_conf
.service_str
= "sym";
163 ret
= qat_qp_setup(qat_private
->qat_dev
, qp_addr
, qp_id
, &qat_qp_conf
);
167 /* store a link to the qp in the qat_pci_device */
168 qat_private
->qat_dev
->qps_in_use
[QAT_SERVICE_SYMMETRIC
][qp_id
]
171 qp
= (struct qat_qp
*)*qp_addr
;
173 for (i
= 0; i
< qp
->nb_descriptors
; i
++) {
175 struct qat_sym_op_cookie
*cookie
=
178 cookie
->qat_sgl_src_phys_addr
=
179 rte_mempool_virt2iova(cookie
) +
180 offsetof(struct qat_sym_op_cookie
,
183 cookie
->qat_sgl_dst_phys_addr
=
184 rte_mempool_virt2iova(cookie
) +
185 offsetof(struct qat_sym_op_cookie
,
192 static struct rte_cryptodev_ops crypto_qat_ops
= {
194 /* Device related operations */
195 .dev_configure
= qat_sym_dev_config
,
196 .dev_start
= qat_sym_dev_start
,
197 .dev_stop
= qat_sym_dev_stop
,
198 .dev_close
= qat_sym_dev_close
,
199 .dev_infos_get
= qat_sym_dev_info_get
,
201 .stats_get
= qat_sym_stats_get
,
202 .stats_reset
= qat_sym_stats_reset
,
203 .queue_pair_setup
= qat_sym_qp_setup
,
204 .queue_pair_release
= qat_sym_qp_release
,
205 .queue_pair_count
= NULL
,
207 /* Crypto related operations */
208 .sym_session_get_size
= qat_sym_session_get_private_size
,
209 .sym_session_configure
= qat_sym_session_configure
,
210 .sym_session_clear
= qat_sym_session_clear
214 qat_sym_pmd_enqueue_op_burst(void *qp
, struct rte_crypto_op
**ops
,
217 return qat_enqueue_op_burst(qp
, (void **)ops
, nb_ops
);
221 qat_sym_pmd_dequeue_op_burst(void *qp
, struct rte_crypto_op
**ops
,
224 return qat_dequeue_op_burst(qp
, (void **)ops
, nb_ops
);
227 /* An rte_driver is needed in the registration of both the device and the driver
229 * The actual qat pci's rte_driver can't be used as its name represents
230 * the whole pci device with all services. Think of this as a holder for a name
231 * for the crypto part of the pci device.
233 static const char qat_sym_drv_name
[] = RTE_STR(CRYPTODEV_NAME_QAT_SYM_PMD
);
234 static const struct rte_driver cryptodev_qat_sym_driver
= {
235 .name
= qat_sym_drv_name
,
236 .alias
= qat_sym_drv_name
240 qat_sym_dev_create(struct qat_pci_device
*qat_pci_dev
)
242 struct rte_cryptodev_pmd_init_params init_params
= {
244 .socket_id
= qat_pci_dev
->pci_dev
->device
.numa_node
,
245 .private_data_size
= sizeof(struct qat_sym_dev_private
)
247 char name
[RTE_CRYPTODEV_NAME_MAX_LEN
];
248 struct rte_cryptodev
*cryptodev
;
249 struct qat_sym_dev_private
*internals
;
251 snprintf(name
, RTE_CRYPTODEV_NAME_MAX_LEN
, "%s_%s",
252 qat_pci_dev
->name
, "sym");
253 QAT_LOG(DEBUG
, "Creating QAT SYM device %s", name
);
255 /* Populate subset device to use in cryptodev device creation */
256 qat_pci_dev
->sym_rte_dev
.driver
= &cryptodev_qat_sym_driver
;
257 qat_pci_dev
->sym_rte_dev
.numa_node
=
258 qat_pci_dev
->pci_dev
->device
.numa_node
;
259 qat_pci_dev
->sym_rte_dev
.devargs
= NULL
;
261 cryptodev
= rte_cryptodev_pmd_create(name
,
262 &(qat_pci_dev
->sym_rte_dev
), &init_params
);
264 if (cryptodev
== NULL
)
267 qat_pci_dev
->sym_rte_dev
.name
= cryptodev
->data
->name
;
268 cryptodev
->driver_id
= cryptodev_qat_driver_id
;
269 cryptodev
->dev_ops
= &crypto_qat_ops
;
271 cryptodev
->enqueue_burst
= qat_sym_pmd_enqueue_op_burst
;
272 cryptodev
->dequeue_burst
= qat_sym_pmd_dequeue_op_burst
;
274 cryptodev
->feature_flags
= RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO
|
275 RTE_CRYPTODEV_FF_HW_ACCELERATED
|
276 RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING
|
277 RTE_CRYPTODEV_FF_IN_PLACE_SGL
|
278 RTE_CRYPTODEV_FF_OOP_SGL_IN_SGL_OUT
|
279 RTE_CRYPTODEV_FF_OOP_SGL_IN_LB_OUT
|
280 RTE_CRYPTODEV_FF_OOP_LB_IN_SGL_OUT
|
281 RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT
;
283 internals
= cryptodev
->data
->dev_private
;
284 internals
->qat_dev
= qat_pci_dev
;
285 qat_pci_dev
->sym_dev
= internals
;
287 internals
->sym_dev_id
= cryptodev
->data
->dev_id
;
288 switch (qat_pci_dev
->qat_dev_gen
) {
290 internals
->qat_dev_capabilities
= qat_gen1_sym_capabilities
;
294 internals
->qat_dev_capabilities
= qat_gen2_sym_capabilities
;
297 internals
->qat_dev_capabilities
= qat_gen2_sym_capabilities
;
299 "QAT gen %d capabilities unknown, default to GEN2",
300 qat_pci_dev
->qat_dev_gen
);
304 QAT_LOG(DEBUG
, "Created QAT SYM device %s as cryptodev instance %d",
305 cryptodev
->data
->name
, internals
->sym_dev_id
);
310 qat_sym_dev_destroy(struct qat_pci_device
*qat_pci_dev
)
312 struct rte_cryptodev
*cryptodev
;
314 if (qat_pci_dev
== NULL
)
316 if (qat_pci_dev
->sym_dev
== NULL
)
319 /* free crypto device */
320 cryptodev
= rte_cryptodev_pmd_get_dev(qat_pci_dev
->sym_dev
->sym_dev_id
);
321 rte_cryptodev_pmd_destroy(cryptodev
);
322 qat_pci_dev
->sym_rte_dev
.name
= NULL
;
323 qat_pci_dev
->sym_dev
= NULL
;
328 static struct cryptodev_driver qat_crypto_drv
;
329 RTE_PMD_REGISTER_CRYPTO_DRIVER(qat_crypto_drv
,
330 cryptodev_qat_sym_driver
,
331 cryptodev_qat_driver_id
);