[ceph.git] / ceph / src / crypto / qat / qcccrypto.cc

#include "qcccrypto.h"
#include  <iostream>
#include "string.h"
#include <pthread.h>
#include "common/debug.h"
#include "include/scope_guard.h"
#include "common/dout.h"
#include "common/errno.h"

// -----------------------------------------------------------------------------
#define dout_context g_ceph_context
#define dout_subsys ceph_subsys_rgw
#undef dout_prefix
#define dout_prefix _prefix(_dout)

static std::ostream& _prefix(std::ostream* _dout)
{
  return *_dout << "QccCrypto: ";
}
// -----------------------------------------------------------------------------

/*
 * Poller thread & functions
*/
static std::mutex qcc_alloc_mutex;
static std::mutex qcc_eng_mutex;
static std::atomic<bool> init_called = { false };

void* QccCrypto::crypt_thread(void *args) {
  struct qcc_thread_args *thread_args = (struct qcc_thread_args *)args;
  thread_args->qccinstance->do_crypt(thread_args);
  return thread_args;
}

void QccCrypto::QccFreeInstance(int entry) {
  std::lock_guard<std::mutex> freeinst(qcc_alloc_mutex);
  open_instances.push(entry);
}

int QccCrypto::QccGetFreeInstance() {
  int ret = -1;
  std::lock_guard<std::mutex> getinst(qcc_alloc_mutex);
  if (!open_instances.empty()) {
    ret = open_instances.front();
    open_instances.pop();
  }
  return ret;
}

void QccCrypto::cleanup() {
  icp_sal_userStop();
  qaeMemDestroy();
  is_init = false;
  init_stat = stat;
  init_called = false;
  derr << "Failure during QAT init sequence. Quitting" << dendl;
}

/*
 * We initialize QAT instance and everything that is common for all ops
*/
bool QccCrypto::init()
{

  std::lock_guard<std::mutex> l(qcc_eng_mutex);

  if(init_called) {
    dout(10) << "Init sequence already called. Skipping duplicate call" << dendl;
    return true;
  }

  // First call to init
  dout(15) << "First init for QAT" << dendl;
  init_called = true;

  // Find if the usermode memory driver is available. We need to this to
  // create contiguous memory needed by QAT.
  stat = qaeMemInit();
  if(stat != CPA_STATUS_SUCCESS) {
    derr << "Unable to load memory driver" << dendl;
    this->cleanup();
    return false;
  }

  stat = icp_sal_userStart("CEPH");
  if(stat != CPA_STATUS_SUCCESS) {
    derr << "Unable to start qat device" << dendl;
    this->cleanup();
    return false;
  }

  qcc_os_mem_alloc((void **)&qcc_inst, sizeof(QCCINST));
  if(qcc_inst == NULL) {
    derr << "Unable to alloc mem for instance struct" << dendl;
    this->cleanup();
    return false;
  }

  // Initialize contents of qcc_inst
  qcc_inst->num_instances = 0;
  qcc_inst->cy_inst_handles = NULL;

  stat = cpaCyGetNumInstances(&(qcc_inst->num_instances));
  if ((stat != CPA_STATUS_SUCCESS) || (qcc_inst->num_instances <= 0)) {
    derr << "Unable to find available instances" << dendl;
    this->cleanup();
    return false;
  }

  qcc_os_mem_alloc((void **)&qcc_inst->cy_inst_handles,
      ((int)qcc_inst->num_instances * sizeof(CpaInstanceHandle)));
  if (qcc_inst->cy_inst_handles == NULL) {
    derr << "Unable to allocate instances array memory" << dendl;
    this->cleanup();
    return false;
  }

  stat = cpaCyGetInstances(qcc_inst->num_instances, qcc_inst->cy_inst_handles);
  if (stat != CPA_STATUS_SUCCESS) {
    derr << "Unable to get instances" << dendl;
    this->cleanup();
    return false;
  }

  int iter = 0;
  //Start Instances
  for(iter = 0; iter < qcc_inst->num_instances; iter++) {
    stat = cpaCyStartInstance(qcc_inst->cy_inst_handles[iter]);
    if(stat != CPA_STATUS_SUCCESS) {
      derr << "Unable to start instance" << dendl;
      this->cleanup();
      return false;
    }
  }

  qcc_os_mem_alloc((void **)&qcc_inst->is_polled,
      ((int)qcc_inst->num_instances * sizeof(CpaBoolean)));
  CpaInstanceInfo2 info;
  for(iter = 0; iter < qcc_inst->num_instances; iter++) {
    qcc_inst->is_polled[iter] = cpaCyInstanceGetInfo2(qcc_inst->cy_inst_handles[iter],
        &info) == CPA_STATUS_SUCCESS ? info.isPolled : CPA_FALSE;
  }

  // Allocate memory structures for all instances
  qcc_os_mem_alloc((void **)&qcc_sess,
      ((int)qcc_inst->num_instances * sizeof(QCCSESS)));
  if(qcc_sess == NULL) {
    derr << "Unable to allocate memory for session struct" << dendl;
    this->cleanup();
    return false;
  }

  qcc_os_mem_alloc((void **)&qcc_op_mem,
      ((int)qcc_inst->num_instances * sizeof(QCCOPMEM)));
  if(qcc_sess == NULL) {
    derr << "Unable to allocate memory for opmem struct" << dendl;
    this->cleanup();
    return false;
  }

  qcc_os_mem_alloc((void **)&cypollthreads,
      ((int)qcc_inst->num_instances * sizeof(pthread_t)));
  if(cypollthreads == NULL) {
    derr << "Unable to allocate memory for pthreads" << dendl;
    this->cleanup();
    return false;
  }

  //At this point we are only doing an user-space version.
  //To-Do: Maybe a kernel based one
  for(iter = 0; iter < qcc_inst->num_instances; iter++) {
    stat = cpaCySetAddressTranslation(qcc_inst->cy_inst_handles[iter],
                                       qaeVirtToPhysNUMA);
    if(stat == CPA_STATUS_SUCCESS) {
      // Start HW Polling Thread
      // To-Do: Enable epoll & interrupt based later?
      // QccCyStartPoll(iter);
      // Setup the session structures for crypto operation and populate
      // whatever we can now. Rest will be filled in when crypto operation
      // happens.
      qcc_sess[iter].sess_ctx_sz = 0;
      qcc_sess[iter].sess_ctx = NULL;
      qcc_sess[iter].sess_stp_data.sessionPriority = CPA_CY_PRIORITY_NORMAL;
      qcc_sess[iter].sess_stp_data.symOperation = CPA_CY_SYM_OP_CIPHER;
      open_instances.push(iter);
      qcc_op_mem[iter].is_mem_alloc = false;
      qcc_op_mem[iter].op_complete = false;
      qcc_op_mem[iter].op_result = CPA_STATUS_SUCCESS;
      qcc_op_mem[iter].sym_op_data = NULL;
      qcc_op_mem[iter].buff_meta_size = qcc_op_mem[iter].buff_size = 0;
      qcc_op_mem[iter].src_buff_meta = qcc_op_mem[iter].src_buff
        = qcc_op_mem[iter].iv_buff = NULL;
      qcc_op_mem[iter].src_buff_list = NULL;
      qcc_op_mem[iter].src_buff_flat = NULL;
      qcc_op_mem[iter].num_buffers = 1;
    } else {
      derr << "Unable to find address translations of instance " << iter << dendl;
      this->cleanup();
      return false;
    }
  }
  is_init = true;
  dout(10) << "Init complete" << dendl;
  return true;
}

bool QccCrypto::destroy() {
  if((!is_init) || (!init_called)) {
    dout(15) << "QAT not initialized here. Nothing to do" << dendl;
    return false;
  }

  unsigned int retry = 0;
  while(retry <= QCC_MAX_RETRIES) {
    if(open_instances.size() == qcc_inst->num_instances) {
      break;
    } else {
      retry++;
    }
    dout(5) << "QAT is still busy and cannot free resources yet" << dendl;
    return false;
  }

  dout(10) << "Destroying QAT crypto & related memory" << dendl;
  int iter = 0;

  // Free up op related memory
  for (iter =0; iter < qcc_inst->num_instances; iter++) {
    qcc_contig_mem_free((void **)&(qcc_op_mem[iter].src_buff));
    qcc_contig_mem_free((void **)&(qcc_op_mem[iter].iv_buff));
    qcc_os_mem_free((void **)&(qcc_op_mem[iter].src_buff_list));
    qcc_os_mem_free((void **)&(qcc_op_mem[iter].src_buff_flat));
    qcc_contig_mem_free((void **)&(qcc_op_mem[iter].sym_op_data));
  }

  // Free up Session memory
  for(iter = 0; iter < qcc_inst->num_instances; iter++) {
    cpaCySymRemoveSession(qcc_inst->cy_inst_handles[iter], qcc_sess[iter].sess_ctx);
    qcc_contig_mem_free((void **)&(qcc_sess[iter].sess_ctx));
  }

  // Stop QAT Instances
  for(iter = 0; iter < qcc_inst->num_instances; iter++) {
    cpaCyStopInstance(qcc_inst->cy_inst_handles[iter]);
  }

  // Free up the base structures we use
  qcc_os_mem_free((void **)&qcc_op_mem);
  qcc_os_mem_free((void **)&qcc_sess);
  qcc_os_mem_free((void **)&(qcc_inst->cy_inst_handles));
  qcc_os_mem_free((void **)&(qcc_inst->is_polled));
  qcc_os_mem_free((void **)&cypollthreads);
  qcc_os_mem_free((void **)&qcc_inst);

  //Un-init memory driver and QAT HW
  icp_sal_userStop();
  qaeMemDestroy();
  init_called = false;
  is_init = false;
  return true;
}

void QccCrypto::do_crypt(qcc_thread_args *thread_args) {
  auto entry = thread_args->entry;
  qcc_op_mem[entry].op_result = cpaCySymPerformOp(qcc_inst->cy_inst_handles[entry],
      NULL,
      qcc_op_mem[entry].sym_op_data,
      qcc_op_mem[entry].src_buff_list,
      qcc_op_mem[entry].src_buff_list,
      NULL);
  qcc_op_mem[entry].op_complete = true;
  free(thread_args);
}

bool QccCrypto::perform_op(unsigned char* out, const unsigned char* in,
    size_t size, uint8_t *iv, uint8_t *key, CpaCySymCipherDirection op_type)
{
  if (!init_called) {
    dout(10) << "QAT not intialized yet. Initializing now..." << dendl;
    if(!QccCrypto::init()) {
      derr << "QAT init failed" << dendl;
      return false;
    }
  }

  if(!is_init)
  {
    dout(10) << "QAT not initialized in this instance or init failed with possible error " << (int)init_stat << dendl;
    return is_init;
  }

  int avail_inst = -1;
  unsigned int retrycount = 0;
  while(retrycount <= QCC_MAX_RETRIES) {
    avail_inst = QccGetFreeInstance();
    if(avail_inst != -1) {
      break;
    } else {
      retrycount++;
      usleep(qcc_sleep_duration);
    }
  }

  if(avail_inst == -1) {
    derr << "Unable to get an QAT instance. Failing request" << dendl;
    return false;
  }

  dout(15) << "Using inst " << avail_inst << dendl;
  // Start polling threads for this instance
  //QccCyStartPoll(avail_inst);

  auto sg = make_scope_guard([=] {
      //free up the instance irrespective of the op status
      dout(15) << "Completed task under " << avail_inst << dendl;
      qcc_op_mem[avail_inst].op_complete = false;
      QccCrypto::QccFreeInstance(avail_inst);
      });

  /*
   * Allocate buffers for this version of the instance if not already done.
   * Hold onto to most of them until destructor is called.
  */
  if (qcc_op_mem[avail_inst].is_mem_alloc == false) {

    qcc_sess[avail_inst].sess_stp_data.cipherSetupData.cipherAlgorithm =
                                                     CPA_CY_SYM_CIPHER_AES_CBC;
    qcc_sess[avail_inst].sess_stp_data.cipherSetupData.cipherKeyLenInBytes =
                                                              AES_256_KEY_SIZE;

    // Allocate contig memory for buffers that are independent of the
    // input/output
    stat = cpaCyBufferListGetMetaSize(qcc_inst->cy_inst_handles[avail_inst],
        qcc_op_mem[avail_inst].num_buffers, &(qcc_op_mem[avail_inst].buff_meta_size));
    if(stat != CPA_STATUS_SUCCESS) {
      derr << "Unable to get buff meta size" << dendl;
      return false;
    }

    // Allocate Buffer List Private metadata
    stat = qcc_contig_mem_alloc((void **)&(qcc_op_mem[avail_inst].src_buff_meta),
        qcc_op_mem[avail_inst].buff_meta_size, 1);
    if(stat != CPA_STATUS_SUCCESS) {
      derr << "Unable to allocate private metadata memory" << dendl;
      return false;
    }

    // Allocate Buffer List Memory
    qcc_os_mem_alloc((void **)&(qcc_op_mem[avail_inst].src_buff_list), sizeof(CpaBufferList));
    qcc_os_mem_alloc((void **)&(qcc_op_mem[avail_inst].src_buff_flat),
              (qcc_op_mem[avail_inst].num_buffers * sizeof(CpaFlatBuffer)));
    if(qcc_op_mem[avail_inst].src_buff_list == NULL || qcc_op_mem[avail_inst].src_buff_flat == NULL) {
      derr << "Unable to allocate bufferlist memory" << dendl;
      return false;
    }

    // Allocate IV memory
    stat = qcc_contig_mem_alloc((void **)&(qcc_op_mem[avail_inst].iv_buff), AES_256_IV_LEN);
    if(stat != CPA_STATUS_SUCCESS) {
      derr << "Unable to allocate bufferlist memory" << dendl;
      return false;
    }

    //Assign src stuff for the operation
    (qcc_op_mem[avail_inst].src_buff_list)->pBuffers = qcc_op_mem[avail_inst].src_buff_flat;
    (qcc_op_mem[avail_inst].src_buff_list)->numBuffers = qcc_op_mem[avail_inst].num_buffers;
    (qcc_op_mem[avail_inst].src_buff_list)->pPrivateMetaData = qcc_op_mem[avail_inst].src_buff_meta;

    //Setup OpData
    stat = qcc_contig_mem_alloc((void **)&(qcc_op_mem[avail_inst].sym_op_data),
        sizeof(CpaCySymOpData));
    if(stat != CPA_STATUS_SUCCESS) {
      derr << "Unable to allocate opdata memory" << dendl;
      return false;
    }

    // Assuming op to be encryption for initiation. This will be reset when we
    // exit this block
    qcc_sess[avail_inst].sess_stp_data.cipherSetupData.cipherDirection =
                                            CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT;
    // Allocate Session memory
    stat = cpaCySymSessionCtxGetSize(qcc_inst->cy_inst_handles[avail_inst],
        &(qcc_sess[avail_inst].sess_stp_data), &(qcc_sess[avail_inst].sess_ctx_sz));
    if(stat != CPA_STATUS_SUCCESS) {
      derr << "Unable to find session size" << dendl;
      return false;
    }

    stat = qcc_contig_mem_alloc((void **)&(qcc_sess[avail_inst].sess_ctx),
        qcc_sess[avail_inst].sess_ctx_sz);
    if(stat != CPA_STATUS_SUCCESS) {
      derr << "Unable to allocate contig memory" << dendl;
      return false;
    }

    // Set memalloc flag so that we don't go through this exercise again.
    qcc_op_mem[avail_inst].is_mem_alloc = true;
    dout(15) << "Instantiation complete for " << avail_inst << dendl;
  }

  // Section that runs on every call
  // Identify the operation and assign to session
  qcc_sess[avail_inst].sess_stp_data.cipherSetupData.cipherDirection = op_type;
  qcc_sess[avail_inst].sess_stp_data.cipherSetupData.pCipherKey = (Cpa8U *)key;

  stat = cpaCySymInitSession(qcc_inst->cy_inst_handles[avail_inst],
      NULL,
      &(qcc_sess[avail_inst].sess_stp_data),
      qcc_sess[avail_inst].sess_ctx);
  if (stat != CPA_STATUS_SUCCESS) {
    derr << "Unable to init session" << dendl;
    return false;
  }

  // Allocate actual buffers that will hold data
  if (qcc_op_mem[avail_inst].buff_size != (Cpa32U)size) {
    qcc_contig_mem_free((void **)&(qcc_op_mem[avail_inst].src_buff));
    qcc_op_mem[avail_inst].buff_size = (Cpa32U)size;
    stat = qcc_contig_mem_alloc((void **)&(qcc_op_mem[avail_inst].src_buff),
              qcc_op_mem[avail_inst].buff_size);
    if(stat != CPA_STATUS_SUCCESS) {
      derr << "Unable to allocate contig memory" << dendl;
      return false;
    }
  }

  // Copy src & iv into the respective buffers
  memcpy(qcc_op_mem[avail_inst].src_buff, in, size);
  memcpy(qcc_op_mem[avail_inst].iv_buff, iv, AES_256_IV_LEN);

  //Assign the reminder of the stuff
  qcc_op_mem[avail_inst].src_buff_flat->dataLenInBytes = qcc_op_mem[avail_inst].buff_size;
  qcc_op_mem[avail_inst].src_buff_flat->pData = qcc_op_mem[avail_inst].src_buff;

  //OpData assignment
  qcc_op_mem[avail_inst].sym_op_data->sessionCtx = qcc_sess[avail_inst].sess_ctx;
  qcc_op_mem[avail_inst].sym_op_data->packetType = CPA_CY_SYM_PACKET_TYPE_FULL;
  qcc_op_mem[avail_inst].sym_op_data->pIv = qcc_op_mem[avail_inst].iv_buff;
  qcc_op_mem[avail_inst].sym_op_data->ivLenInBytes = AES_256_IV_LEN;
  qcc_op_mem[avail_inst].sym_op_data->cryptoStartSrcOffsetInBytes = 0;
  qcc_op_mem[avail_inst].sym_op_data->messageLenToCipherInBytes = qcc_op_mem[avail_inst].buff_size;

  // Perform cipher operation in a thread
  qcc_thread_args* thread_args = new qcc_thread_args();
  thread_args->qccinstance = this;
  thread_args->entry = avail_inst;

  if (pthread_create(&cypollthreads[avail_inst], NULL, crypt_thread, (void *)thread_args) != 0) {
    derr << "Unable to create thread for crypt operation" << dendl;
    return false;
  }
  if (qcc_inst->is_polled[avail_inst] == CPA_TRUE) {
    while (!qcc_op_mem[avail_inst].op_complete) {
      icp_sal_CyPollInstance(qcc_inst->cy_inst_handles[avail_inst], 0);
    }
  }
  pthread_join(cypollthreads[avail_inst], NULL);

  if(qcc_op_mem[avail_inst].op_result != CPA_STATUS_SUCCESS) {
    derr << "Unable to perform crypt operation" << dendl;
    return false;
  }

  //Copy data back to out buffer
  memcpy(out, qcc_op_mem[avail_inst].src_buff, size);
  //Always cleanup memory holding user-data at the end
  memset(qcc_op_mem[avail_inst].iv_buff, 0, AES_256_IV_LEN);
  memset(qcc_op_mem[avail_inst].src_buff, 0, qcc_op_mem[avail_inst].buff_size);

  return true;
}
Commit	Line	Data
11fdf7f2 TL	1	#include "qcccrypto.h"
	2	#include <iostream>
	3	#include "string.h"
	4	#include <pthread.h>
	5	#include "common/debug.h"
	6	#include "include/scope_guard.h"
1d09f67e TL	7	#include "common/dout.h"
1d09f67e TL	8	#include "common/errno.h"
11fdf7f2 TL	9
	10	// -----------------------------------------------------------------------------
	11	#define dout_context g_ceph_context
	12	#define dout_subsys ceph_subsys_rgw
	13	#undef dout_prefix
	14	#define dout_prefix _prefix(_dout)
	15
1d09f67e	16	static std::ostream& _prefix(std::ostream* _dout)
11fdf7f2 TL	17	{
	18	return *_dout << "QccCrypto: ";
	19	}
	20	// -----------------------------------------------------------------------------
	21
	22	/*
	23	* Poller thread & functions
	24	*/
	25	static std::mutex qcc_alloc_mutex;
	26	static std::mutex qcc_eng_mutex;
	27	static std::atomic<bool> init_called = { false };
	28
	29	void* QccCrypto::crypt_thread(void *args) {
	30	struct qcc_thread_args thread_args = (struct qcc_thread_args )args;
	31	thread_args->qccinstance->do_crypt(thread_args);
1d09f67e	32	return thread_args;
11fdf7f2 TL	33	}
	34
	35	void QccCrypto::QccFreeInstance(int entry) {
	36	std::lock_guard<std::mutex> freeinst(qcc_alloc_mutex);
	37	open_instances.push(entry);
	38	}
	39
	40	int QccCrypto::QccGetFreeInstance() {
	41	int ret = -1;
	42	std::lock_guard<std::mutex> getinst(qcc_alloc_mutex);
	43	if (!open_instances.empty()) {
	44	ret = open_instances.front();
	45	open_instances.pop();
	46	}
	47	return ret;
	48	}
	49
	50	void QccCrypto::cleanup() {
	51	icp_sal_userStop();
	52	qaeMemDestroy();
	53	is_init = false;
	54	init_stat = stat;
	55	init_called = false;
	56	derr << "Failure during QAT init sequence. Quitting" << dendl;
	57	}
	58
	59	/*
	60	* We initialize QAT instance and everything that is common for all ops
	61	*/
	62	bool QccCrypto::init()
	63	{
	64
	65	std::lock_guard<std::mutex> l(qcc_eng_mutex);
	66
	67	if(init_called) {
	68	dout(10) << "Init sequence already called. Skipping duplicate call" << dendl;
	69	return true;
	70	}
	71
	72	// First call to init
	73	dout(15) << "First init for QAT" << dendl;
	74	init_called = true;
	75
	76	// Find if the usermode memory driver is available. We need to this to
	77	// create contiguous memory needed by QAT.
	78	stat = qaeMemInit();
	79	if(stat != CPA_STATUS_SUCCESS) {
	80	derr << "Unable to load memory driver" << dendl;
	81	this->cleanup();
	82	return false;
	83	}
	84
	85	stat = icp_sal_userStart("CEPH");
	86	if(stat != CPA_STATUS_SUCCESS) {
	87	derr << "Unable to start qat device" << dendl;
	88	this->cleanup();
	89	return false;
	90	}
	91
	92	qcc_os_mem_alloc((void **)&qcc_inst, sizeof(QCCINST));
	93	if(qcc_inst == NULL) {
	94	derr << "Unable to alloc mem for instance struct" << dendl;
	95	this->cleanup();
	96	return false;
97	}
98
99	// Initialize contents of qcc_inst
100	qcc_inst->num_instances = 0;
101	qcc_inst->cy_inst_handles = NULL;
102
103	stat = cpaCyGetNumInstances(&(qcc_inst->num_instances));
104	if ((stat != CPA_STATUS_SUCCESS) \|\| (qcc_inst->num_instances <= 0)) {
105	derr << "Unable to find available instances" << dendl;
106	this->cleanup();
107	return false;
108	}
109
110	qcc_os_mem_alloc((void **)&qcc_inst->cy_inst_handles,
111	((int)qcc_inst->num_instances * sizeof(CpaInstanceHandle)));
112	if (qcc_inst->cy_inst_handles == NULL) {
113	derr << "Unable to allocate instances array memory" << dendl;
114	this->cleanup();
115	return false;
116	}
117
118	stat = cpaCyGetInstances(qcc_inst->num_instances, qcc_inst->cy_inst_handles);
119	if (stat != CPA_STATUS_SUCCESS) {
120	derr << "Unable to get instances" << dendl;
121	this->cleanup();
122	return false;
123	}
124
125	int iter = 0;
126	//Start Instances
127	for(iter = 0; iter < qcc_inst->num_instances; iter++) {
128	stat = cpaCyStartInstance(qcc_inst->cy_inst_handles[iter]);
129	if(stat != CPA_STATUS_SUCCESS) {
130	derr << "Unable to start instance" << dendl;
131	this->cleanup();
132	return false;
133	}
134	}
135
136	qcc_os_mem_alloc((void **)&qcc_inst->is_polled,
137	((int)qcc_inst->num_instances * sizeof(CpaBoolean)));
138	CpaInstanceInfo2 info;
139	for(iter = 0; iter < qcc_inst->num_instances; iter++) {
140	qcc_inst->is_polled[iter] = cpaCyInstanceGetInfo2(qcc_inst->cy_inst_handles[iter],
141	&info) == CPA_STATUS_SUCCESS ? info.isPolled : CPA_FALSE;
142	}
143
144	// Allocate memory structures for all instances
145	qcc_os_mem_alloc((void **)&qcc_sess,
146	((int)qcc_inst->num_instances * sizeof(QCCSESS)));
147	if(qcc_sess == NULL) {
148	derr << "Unable to allocate memory for session struct" << dendl;
149	this->cleanup();
150	return false;
151	}
152
153	qcc_os_mem_alloc((void **)&qcc_op_mem,
154	((int)qcc_inst->num_instances * sizeof(QCCOPMEM)));
155	if(qcc_sess == NULL) {
156	derr << "Unable to allocate memory for opmem struct" << dendl;
157	this->cleanup();
158	return false;
159	}
160
161	qcc_os_mem_alloc((void **)&cypollthreads,
162	((int)qcc_inst->num_instances * sizeof(pthread_t)));
163	if(cypollthreads == NULL) {
164	derr << "Unable to allocate memory for pthreads" << dendl;
165	this->cleanup();
166	return false;
167	}
168
169	//At this point we are only doing an user-space version.
170	//To-Do: Maybe a kernel based one
171	for(iter = 0; iter < qcc_inst->num_instances; iter++) {
172	stat = cpaCySetAddressTranslation(qcc_inst->cy_inst_handles[iter],
173	qaeVirtToPhysNUMA);
174	if(stat == CPA_STATUS_SUCCESS) {
175	// Start HW Polling Thread
176	// To-Do: Enable epoll & interrupt based later?
177	// QccCyStartPoll(iter);
178	// Setup the session structures for crypto operation and populate
179	// whatever we can now. Rest will be filled in when crypto operation
180	// happens.
181	qcc_sess[iter].sess_ctx_sz = 0;
182	qcc_sess[iter].sess_ctx = NULL;
183	qcc_sess[iter].sess_stp_data.sessionPriority = CPA_CY_PRIORITY_NORMAL;
184	qcc_sess[iter].sess_stp_data.symOperation = CPA_CY_SYM_OP_CIPHER;
185	open_instances.push(iter);
186	qcc_op_mem[iter].is_mem_alloc = false;
187	qcc_op_mem[iter].op_complete = false;
188	qcc_op_mem[iter].op_result = CPA_STATUS_SUCCESS;
189	qcc_op_mem[iter].sym_op_data = NULL;
190	qcc_op_mem[iter].buff_meta_size = qcc_op_mem[iter].buff_size = 0;
191	qcc_op_mem[iter].src_buff_meta = qcc_op_mem[iter].src_buff
192	= qcc_op_mem[iter].iv_buff = NULL;
193	qcc_op_mem[iter].src_buff_list = NULL;
194	qcc_op_mem[iter].src_buff_flat = NULL;
195	qcc_op_mem[iter].num_buffers = 1;
196	} else {
197	derr << "Unable to find address translations of instance " << iter << dendl;
198	this->cleanup();
199	return false;
200	}
201	}
202	is_init = true;
203	dout(10) << "Init complete" << dendl;
204	return true;
205	}
206
207	bool QccCrypto::destroy() {
208	if((!is_init) \|\| (!init_called)) {
209	dout(15) << "QAT not initialized here. Nothing to do" << dendl;
210	return false;
211	}
212
213	unsigned int retry = 0;
214	while(retry <= QCC_MAX_RETRIES) {
215	if(open_instances.size() == qcc_inst->num_instances) {
216	break;
217	} else {
218	retry++;
219	}
220	dout(5) << "QAT is still busy and cannot free resources yet" << dendl;
221	return false;
222	}
223
224	dout(10) << "Destroying QAT crypto & related memory" << dendl;
225	int iter = 0;
226
227	// Free up op related memory
228	for (iter =0; iter < qcc_inst->num_instances; iter++) {
229	qcc_contig_mem_free((void **)&(qcc_op_mem[iter].src_buff));
230	qcc_contig_mem_free((void **)&(qcc_op_mem[iter].iv_buff));
231	qcc_os_mem_free((void **)&(qcc_op_mem[iter].src_buff_list));
232	qcc_os_mem_free((void **)&(qcc_op_mem[iter].src_buff_flat));
233	qcc_contig_mem_free((void **)&(qcc_op_mem[iter].sym_op_data));
234	}
235
236	// Free up Session memory
237	for(iter = 0; iter < qcc_inst->num_instances; iter++) {
238	cpaCySymRemoveSession(qcc_inst->cy_inst_handles[iter], qcc_sess[iter].sess_ctx);
239	qcc_contig_mem_free((void **)&(qcc_sess[iter].sess_ctx));
240	}
241
242	// Stop QAT Instances
243	for(iter = 0; iter < qcc_inst->num_instances; iter++) {
244	cpaCyStopInstance(qcc_inst->cy_inst_handles[iter]);
245	}
246
247	// Free up the base structures we use
248	qcc_os_mem_free((void **)&qcc_op_mem);
249	qcc_os_mem_free((void **)&qcc_sess);
250	qcc_os_mem_free((void **)&(qcc_inst->cy_inst_handles));
251	qcc_os_mem_free((void **)&(qcc_inst->is_polled));
252	qcc_os_mem_free((void **)&cypollthreads);
253	qcc_os_mem_free((void **)&qcc_inst);
254
255	//Un-init memory driver and QAT HW
256	icp_sal_userStop();
257	qaeMemDestroy();
258	init_called = false;
259	is_init = false;
260	return true;
261	}
262
263	void QccCrypto::do_crypt(qcc_thread_args *thread_args) {
264	auto entry = thread_args->entry;
265	qcc_op_mem[entry].op_result = cpaCySymPerformOp(qcc_inst->cy_inst_handles[entry],
266	NULL,
267	qcc_op_mem[entry].sym_op_data,
268	qcc_op_mem[entry].src_buff_list,
269	qcc_op_mem[entry].src_buff_list,
270	NULL);
271	qcc_op_mem[entry].op_complete = true;
272	free(thread_args);
273	}
274
275	bool QccCrypto::perform_op(unsigned char* out, const unsigned char* in,
276	size_t size, uint8_t iv, uint8_t key, CpaCySymCipherDirection op_type)
277	{
278	if (!init_called) {
279	dout(10) << "QAT not intialized yet. Initializing now..." << dendl;
280	if(!QccCrypto::init()) {
281	derr << "QAT init failed" << dendl;
282	return false;
283	}
284	}
285
286	if(!is_init)
287	{
288	dout(10) << "QAT not initialized in this instance or init failed with possible error " << (int)init_stat << dendl;
289	return is_init;
290	}
291
292	int avail_inst = -1;
293	unsigned int retrycount = 0;
294	while(retrycount <= QCC_MAX_RETRIES) {
295	avail_inst = QccGetFreeInstance();
296	if(avail_inst != -1) {
297	break;
298	} else {
299	retrycount++;
300	usleep(qcc_sleep_duration);
301	}
302	}
303
304	if(avail_inst == -1) {
305	derr << "Unable to get an QAT instance. Failing request" << dendl;
306	return false;
307	}
308
309	dout(15) << "Using inst " << avail_inst << dendl;
310	// Start polling threads for this instance
311	//QccCyStartPoll(avail_inst);
312
313	auto sg = make_scope_guard([=] {
314	//free up the instance irrespective of the op status
315	dout(15) << "Completed task under " << avail_inst << dendl;
316	qcc_op_mem[avail_inst].op_complete = false;
317	QccCrypto::QccFreeInstance(avail_inst);
318	});
319
320	/*
321	* Allocate buffers for this version of the instance if not already done.
322	* Hold onto to most of them until destructor is called.
323	*/
324	if (qcc_op_mem[avail_inst].is_mem_alloc == false) {
325
326	qcc_sess[avail_inst].sess_stp_data.cipherSetupData.cipherAlgorithm =
327	CPA_CY_SYM_CIPHER_AES_CBC;
328	qcc_sess[avail_inst].sess_stp_data.cipherSetupData.cipherKeyLenInBytes =
329	AES_256_KEY_SIZE;
330
331	// Allocate contig memory for buffers that are independent of the
332	// input/output
333	stat = cpaCyBufferListGetMetaSize(qcc_inst->cy_inst_handles[avail_inst],
334	qcc_op_mem[avail_inst].num_buffers, &(qcc_op_mem[avail_inst].buff_meta_size));
335	if(stat != CPA_STATUS_SUCCESS) {
336	derr << "Unable to get buff meta size" << dendl;
337	return false;
338	}
339
340	// Allocate Buffer List Private metadata
341	stat = qcc_contig_mem_alloc((void **)&(qcc_op_mem[avail_inst].src_buff_meta),
342	qcc_op_mem[avail_inst].buff_meta_size, 1);
343	if(stat != CPA_STATUS_SUCCESS) {
344	derr << "Unable to allocate private metadata memory" << dendl;
345	return false;
346	}
347
348	// Allocate Buffer List Memory
349	qcc_os_mem_alloc((void **)&(qcc_op_mem[avail_inst].src_buff_list), sizeof(CpaBufferList));
350	qcc_os_mem_alloc((void **)&(qcc_op_mem[avail_inst].src_buff_flat),
351	(qcc_op_mem[avail_inst].num_buffers * sizeof(CpaFlatBuffer)));
352	if(qcc_op_mem[avail_inst].src_buff_list == NULL \|\| qcc_op_mem[avail_inst].src_buff_flat == NULL) {
353	derr << "Unable to allocate bufferlist memory" << dendl;
354	return false;
355	}
356
357	// Allocate IV memory
358	stat = qcc_contig_mem_alloc((void **)&(qcc_op_mem[avail_inst].iv_buff), AES_256_IV_LEN);
359	if(stat != CPA_STATUS_SUCCESS) {
360	derr << "Unable to allocate bufferlist memory" << dendl;
361	return false;
362	}
363
364	//Assign src stuff for the operation
365	(qcc_op_mem[avail_inst].src_buff_list)->pBuffers = qcc_op_mem[avail_inst].src_buff_flat;
366	(qcc_op_mem[avail_inst].src_buff_list)->numBuffers = qcc_op_mem[avail_inst].num_buffers;
367	(qcc_op_mem[avail_inst].src_buff_list)->pPrivateMetaData = qcc_op_mem[avail_inst].src_buff_meta;
368
369	//Setup OpData
370	stat = qcc_contig_mem_alloc((void **)&(qcc_op_mem[avail_inst].sym_op_data),
371	sizeof(CpaCySymOpData));
372	if(stat != CPA_STATUS_SUCCESS) {
373	derr << "Unable to allocate opdata memory" << dendl;
374	return false;
375	}
376
377	// Assuming op to be encryption for initiation. This will be reset when we
378	// exit this block
379	qcc_sess[avail_inst].sess_stp_data.cipherSetupData.cipherDirection =
380	CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT;
381	// Allocate Session memory
382	stat = cpaCySymSessionCtxGetSize(qcc_inst->cy_inst_handles[avail_inst],
383	&(qcc_sess[avail_inst].sess_stp_data), &(qcc_sess[avail_inst].sess_ctx_sz));
384	if(stat != CPA_STATUS_SUCCESS) {
385	derr << "Unable to find session size" << dendl;
386	return false;
387	}
388
389	stat = qcc_contig_mem_alloc((void **)&(qcc_sess[avail_inst].sess_ctx),
390	qcc_sess[avail_inst].sess_ctx_sz);
391	if(stat != CPA_STATUS_SUCCESS) {
392	derr << "Unable to allocate contig memory" << dendl;
393	return false;
394	}
395
396	// Set memalloc flag so that we don't go through this exercise again.
397	qcc_op_mem[avail_inst].is_mem_alloc = true;
398	dout(15) << "Instantiation complete for " << avail_inst << dendl;
399	}
400
401	// Section that runs on every call
402	// Identify the operation and assign to session
403	qcc_sess[avail_inst].sess_stp_data.cipherSetupData.cipherDirection = op_type;
404	qcc_sess[avail_inst].sess_stp_data.cipherSetupData.pCipherKey = (Cpa8U *)key;
405
406	stat = cpaCySymInitSession(qcc_inst->cy_inst_handles[avail_inst],
407	NULL,
408	&(qcc_sess[avail_inst].sess_stp_data),
409	qcc_sess[avail_inst].sess_ctx);
410	if (stat != CPA_STATUS_SUCCESS) {
411	derr << "Unable to init session" << dendl;
412	return false;
413	}
414
415	// Allocate actual buffers that will hold data
416	if (qcc_op_mem[avail_inst].buff_size != (Cpa32U)size) {
417	qcc_contig_mem_free((void **)&(qcc_op_mem[avail_inst].src_buff));
418	qcc_op_mem[avail_inst].buff_size = (Cpa32U)size;
419	stat = qcc_contig_mem_alloc((void **)&(qcc_op_mem[avail_inst].src_buff),
420	qcc_op_mem[avail_inst].buff_size);
421	if(stat != CPA_STATUS_SUCCESS) {
422	derr << "Unable to allocate contig memory" << dendl;
423	return false;
424	}
425	}
426
427	// Copy src & iv into the respective buffers
428	memcpy(qcc_op_mem[avail_inst].src_buff, in, size);
429	memcpy(qcc_op_mem[avail_inst].iv_buff, iv, AES_256_IV_LEN);
430
431	//Assign the reminder of the stuff
432	qcc_op_mem[avail_inst].src_buff_flat->dataLenInBytes = qcc_op_mem[avail_inst].buff_size;
433	qcc_op_mem[avail_inst].src_buff_flat->pData = qcc_op_mem[avail_inst].src_buff;
434
435	//OpData assignment
436	qcc_op_mem[avail_inst].sym_op_data->sessionCtx = qcc_sess[avail_inst].sess_ctx;
437	qcc_op_mem[avail_inst].sym_op_data->packetType = CPA_CY_SYM_PACKET_TYPE_FULL;
438	qcc_op_mem[avail_inst].sym_op_data->pIv = qcc_op_mem[avail_inst].iv_buff;
439	qcc_op_mem[avail_inst].sym_op_data->ivLenInBytes = AES_256_IV_LEN;
440	qcc_op_mem[avail_inst].sym_op_data->cryptoStartSrcOffsetInBytes = 0;
441	qcc_op_mem[avail_inst].sym_op_data->messageLenToCipherInBytes = qcc_op_mem[avail_inst].buff_size;
442
443	// Perform cipher operation in a thread
444	qcc_thread_args* thread_args = new qcc_thread_args();
445	thread_args->qccinstance = this;
446	thread_args->entry = avail_inst;
447
448	if (pthread_create(&cypollthreads[avail_inst], NULL, crypt_thread, (void *)thread_args) != 0) {
449	derr << "Unable to create thread for crypt operation" << dendl;
450	return false;
451	}
452	if (qcc_inst->is_polled[avail_inst] == CPA_TRUE) {
453	while (!qcc_op_mem[avail_inst].op_complete) {
454	icp_sal_CyPollInstance(qcc_inst->cy_inst_handles[avail_inst], 0);
455	}
456	}
457	pthread_join(cypollthreads[avail_inst], NULL);
458
459	if(qcc_op_mem[avail_inst].op_result != CPA_STATUS_SUCCESS) {
460	derr << "Unable to perform crypt operation" << dendl;
461	return false;
462	}
463
464	//Copy data back to out buffer
465	memcpy(out, qcc_op_mem[avail_inst].src_buff, size);
466	//Always cleanup memory holding user-data at the end
467	memset(qcc_op_mem[avail_inst].iv_buff, 0, AES_256_IV_LEN);
468	memset(qcc_op_mem[avail_inst].src_buff, 0, qcc_op_mem[avail_inst].buff_size);
469
470	return true;
471	}