ceph/src/crypto/isa-l/isa-l_crypto/sm3_mb/aarch64/sm3_mb_mgr_asimd_aarch64.c

   1 /**********************************************************************
   2   Copyright(c) 2020 Arm Corporation All rights reserved.
   3
   4   Redistribution and use in source and binary forms, with or without
   5   modification, are permitted provided that the following conditions
   6   are met:
   7     * Redistributions of source code must retain the above copyright
   8       notice, this list of conditions and the following disclaimer.
   9     * Redistributions in binary form must reproduce the above copyright
  10       notice, this list of conditions and the following disclaimer in
  11       the documentation and/or other materials provided with the
  12       distribution.
  13     * Neither the name of Arm Corporation nor the names of its
  14       contributors may be used to endorse or promote products derived
  15       from this software without specific prior written permission.
  16
  17   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  18   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  19   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  20   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  21   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  22   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  23   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  24   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  25   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  26   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  27   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  28 **********************************************************************/
  29 #include <stddef.h>
  30 #include <sm3_mb.h>
  31 #include <assert.h>
  32
  33 #ifndef max
  34 #define max(a,b)            (((a) > (b)) ? (a) : (b))
  35 #endif
  36
  37 #ifndef min
  38 #define min(a,b)            (((a) < (b)) ? (a) : (b))
  39 #endif
  40
  41 #define SM3_MB_CE_MAX_LANES     4
  42 void sm3_mb_asimd_x4(SM3_JOB *, SM3_JOB *, SM3_JOB *, SM3_JOB *, int);
  43 void sm3_mb_asimd_x1(SM3_JOB *, int);
  44
  45 #define LANE_IS_NOT_FINISHED(state,i)   \
  46         (((state->lens[i]&(~0xf))!=0) && state->ldata[i].job_in_lane!=NULL)
  47 #define LANE_IS_FINISHED(state,i)       \
  48         (((state->lens[i]&(~0xf))==0) && state->ldata[i].job_in_lane!=NULL)
  49 #define LANE_IS_FREE(state,i)           \
  50         (((state->lens[i]&(~0xf))==0) && state->ldata[i].job_in_lane==NULL)
  51 #define LANE_IS_INVALID(state,i)        \
  52         (((state->lens[i]&(~0xf))!=0) && state->ldata[i].job_in_lane==NULL)
  53 void sm3_mb_mgr_init_asimd(SM3_MB_JOB_MGR * state)
  54 {
  55         unsigned int i;
  56
  57         state->unused_lanes = 0xf;
  58         state->num_lanes_inuse = 0;
  59         for (i = 0; i < SM3_MB_CE_MAX_LANES; i++) {
  60                 state->unused_lanes <<= 4;
  61                 state->unused_lanes |= SM3_MB_CE_MAX_LANES - 1 - i;
  62                 state->lens[i] = i;
  63                 state->ldata[i].job_in_lane = 0;
  64         }
  65
  66         //lanes > SM3_MB_CE_MAX_LANES is invalid lane
  67         for (; i < SM3_MAX_LANES; i++) {
  68                 state->lens[i] = 0xf;
  69                 state->ldata[i].job_in_lane = 0;
  70         }
  71 }
  72
  73 static int sm3_mb_mgr_do_jobs(SM3_MB_JOB_MGR * state)
  74 {
  75         int lane_idx, len, i;
  76
  77         if (state->num_lanes_inuse == 0) {
  78                 return -1;
  79         }
  80         if (state->num_lanes_inuse == 4) {
  81                 len = min(min(state->lens[0], state->lens[1]),
  82                           min(state->lens[2], state->lens[3]));
  83                 lane_idx = len & 0xf;
  84                 len &= ~0xf;
  85                 sm3_mb_asimd_x4(state->ldata[0].job_in_lane,
  86                                 state->ldata[1].job_in_lane,
  87                                 state->ldata[2].job_in_lane,
  88                                 state->ldata[3].job_in_lane, len >> 4);
  89                 //only return the min length job
  90                 for (i = 0; i < SM3_MAX_LANES; i++) {
  91                         if (LANE_IS_NOT_FINISHED(state, i)) {
  92                                 state->lens[i] -= len;
  93                                 state->ldata[i].job_in_lane->len -= len;
  94                                 state->ldata[i].job_in_lane->buffer += len << 2;
  95                         }
  96                 }
  97
  98                 return lane_idx;
  99         } else {
 100                 for (i = 0; i < SM3_MAX_LANES; i++) {
 101                         if (LANE_IS_NOT_FINISHED(state, i)) {
 102                                 len = state->lens[i] & (~0xf);
 103                                 sm3_mb_asimd_x1(state->ldata[i].job_in_lane, len >> 4);
 104                                 state->lens[i] -= len;
 105                                 state->ldata[i].job_in_lane->len -= len;
 106                                 state->ldata[i].job_in_lane->buffer += len << 2;
 107                                 return i;
 108                         }
 109                 }
 110         }
 111         return -1;
 112
 113 }
 114
 115 static SM3_JOB *sm3_mb_mgr_free_lane(SM3_MB_JOB_MGR * state)
 116 {
 117         int i;
 118         SM3_JOB *ret = NULL;
 119
 120         for (i = 0; i < SM3_MB_CE_MAX_LANES; i++) {
 121                 if (LANE_IS_FINISHED(state, i)) {
 122
 123                         state->unused_lanes <<= 4;
 124                         state->unused_lanes |= i;
 125                         state->num_lanes_inuse--;
 126                         ret = state->ldata[i].job_in_lane;
 127                         ret->status = STS_COMPLETED;
 128                         state->ldata[i].job_in_lane = NULL;
 129                         break;
 130                 }
 131         }
 132         return ret;
 133 }
 134
 135 static void sm3_mb_mgr_insert_job(SM3_MB_JOB_MGR * state, SM3_JOB * job)
 136 {
 137         int lane_idx;
 138         //add job into lanes
 139         lane_idx = state->unused_lanes & 0xf;
 140         //fatal error
 141         assert(lane_idx < SM3_MB_CE_MAX_LANES);
 142         state->lens[lane_idx] = (job->len << 4) | lane_idx;
 143         state->ldata[lane_idx].job_in_lane = job;
 144         state->unused_lanes >>= 4;
 145         state->num_lanes_inuse++;
 146 }
 147
 148 SM3_JOB *sm3_mb_mgr_submit_asimd(SM3_MB_JOB_MGR * state, SM3_JOB * job)
 149 {
 150 #ifndef NDEBUG
 151         int lane_idx;
 152 #endif
 153         SM3_JOB *ret;
 154
 155         //add job into lanes
 156         sm3_mb_mgr_insert_job(state, job);
 157
 158         ret = sm3_mb_mgr_free_lane(state);
 159         if (ret != NULL) {
 160                 return ret;
 161         }
 162         //submit will wait all lane has data
 163         if (state->num_lanes_inuse < SM3_MB_CE_MAX_LANES)
 164                 return NULL;
 165 #ifndef NDEBUG
 166         lane_idx = sm3_mb_mgr_do_jobs(state);
 167         assert(lane_idx != -1);
 168 #else
 169         sm3_mb_mgr_do_jobs(state);
 170 #endif
 171
 172         //~ i = lane_idx;
 173         ret = sm3_mb_mgr_free_lane(state);
 174         return ret;
 175 }
 176
 177 SM3_JOB *sm3_mb_mgr_flush_asimd(SM3_MB_JOB_MGR * state)
 178 {
 179         SM3_JOB *ret;
 180         ret = sm3_mb_mgr_free_lane(state);
 181         if (ret) {
 182                 return ret;
 183         }
 184
 185         sm3_mb_mgr_do_jobs(state);
 186         return sm3_mb_mgr_free_lane(state);
 187
 188 }