ceph/src/seastar/dpdk/lib/librte_ipsec/esp_outb.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2018 Intel Corporation
   3  */
   4
   5 #include <rte_ipsec.h>
   6 #include <rte_esp.h>
   7 #include <rte_ip.h>
   8 #include <rte_errno.h>
   9 #include <rte_cryptodev.h>
  10
  11 #include "sa.h"
  12 #include "ipsec_sqn.h"
  13 #include "crypto.h"
  14 #include "iph.h"
  15 #include "misc.h"
  16 #include "pad.h"
  17
  18
  19 /*
  20  * helper function to fill crypto_sym op for cipher+auth algorithms.
  21  * used by outb_cop_prepare(), see below.
  22  */
  23 static inline void
  24 sop_ciph_auth_prepare(struct rte_crypto_sym_op *sop,
  25         const struct rte_ipsec_sa *sa, const union sym_op_data *icv,
  26         uint32_t pofs, uint32_t plen)
  27 {
  28         sop->cipher.data.offset = sa->ctp.cipher.offset + pofs;
  29         sop->cipher.data.length = sa->ctp.cipher.length + plen;
  30         sop->auth.data.offset = sa->ctp.auth.offset + pofs;
  31         sop->auth.data.length = sa->ctp.auth.length + plen;
  32         sop->auth.digest.data = icv->va;
  33         sop->auth.digest.phys_addr = icv->pa;
  34 }
  35
  36 /*
  37  * helper function to fill crypto_sym op for cipher+auth algorithms.
  38  * used by outb_cop_prepare(), see below.
  39  */
  40 static inline void
  41 sop_aead_prepare(struct rte_crypto_sym_op *sop,
  42         const struct rte_ipsec_sa *sa, const union sym_op_data *icv,
  43         uint32_t pofs, uint32_t plen)
  44 {
  45         sop->aead.data.offset = sa->ctp.cipher.offset + pofs;
  46         sop->aead.data.length = sa->ctp.cipher.length + plen;
  47         sop->aead.digest.data = icv->va;
  48         sop->aead.digest.phys_addr = icv->pa;
  49         sop->aead.aad.data = icv->va + sa->icv_len;
  50         sop->aead.aad.phys_addr = icv->pa + sa->icv_len;
  51 }
  52
  53 /*
  54  * setup crypto op and crypto sym op for ESP outbound packet.
  55  */
  56 static inline void
  57 outb_cop_prepare(struct rte_crypto_op *cop,
  58         const struct rte_ipsec_sa *sa, const uint64_t ivp[IPSEC_MAX_IV_QWORD],
  59         const union sym_op_data *icv, uint32_t hlen, uint32_t plen)
  60 {
  61         struct rte_crypto_sym_op *sop;
  62         struct aead_gcm_iv *gcm;
  63         struct aesctr_cnt_blk *ctr;
  64         uint32_t algo;
  65
  66         algo = sa->algo_type;
  67
  68         /* fill sym op fields */
  69         sop = cop->sym;
  70
  71         switch (algo) {
  72         case ALGO_TYPE_AES_CBC:
  73                 /* Cipher-Auth (AES-CBC *) case */
  74         case ALGO_TYPE_3DES_CBC:
  75                 /* Cipher-Auth (3DES-CBC *) case */
  76         case ALGO_TYPE_NULL:
  77                 /* NULL case */
  78                 sop_ciph_auth_prepare(sop, sa, icv, hlen, plen);
  79                 break;
  80         case ALGO_TYPE_AES_GCM:
  81                 /* AEAD (AES_GCM) case */
  82                 sop_aead_prepare(sop, sa, icv, hlen, plen);
  83
  84                 /* fill AAD IV (located inside crypto op) */
  85                 gcm = rte_crypto_op_ctod_offset(cop, struct aead_gcm_iv *,
  86                         sa->iv_ofs);
  87                 aead_gcm_iv_fill(gcm, ivp[0], sa->salt);
  88                 break;
  89         case ALGO_TYPE_AES_CTR:
  90                 /* Cipher-Auth (AES-CTR *) case */
  91                 sop_ciph_auth_prepare(sop, sa, icv, hlen, plen);
  92
  93                 /* fill CTR block (located inside crypto op) */
  94                 ctr = rte_crypto_op_ctod_offset(cop, struct aesctr_cnt_blk *,
  95                         sa->iv_ofs);
  96                 aes_ctr_cnt_blk_fill(ctr, ivp[0], sa->salt);
  97                 break;
  98         }
  99 }
 100
 101 /*
 102  * setup/update packet data and metadata for ESP outbound tunnel case.
 103  */
 104 static inline int32_t
 105 outb_tun_pkt_prepare(struct rte_ipsec_sa *sa, rte_be64_t sqc,
 106         const uint64_t ivp[IPSEC_MAX_IV_QWORD], struct rte_mbuf *mb,
 107         union sym_op_data *icv)
 108 {
 109         uint32_t clen, hlen, l2len, pdlen, pdofs, plen, tlen;
 110         struct rte_mbuf *ml;
 111         struct esp_hdr *esph;
 112         struct esp_tail *espt;
 113         char *ph, *pt;
 114         uint64_t *iv;
 115
 116         /* calculate extra header space required */
 117         hlen = sa->hdr_len + sa->iv_len + sizeof(*esph);
 118
 119         /* size of ipsec protected data */
 120         l2len = mb->l2_len;
 121         plen = mb->pkt_len - l2len;
 122
 123         /* number of bytes to encrypt */
 124         clen = plen + sizeof(*espt);
 125         clen = RTE_ALIGN_CEIL(clen, sa->pad_align);
 126
 127         /* pad length + esp tail */
 128         pdlen = clen - plen;
 129         tlen = pdlen + sa->icv_len;
 130
 131         /* do append and prepend */
 132         ml = rte_pktmbuf_lastseg(mb);
 133         if (tlen + sa->sqh_len + sa->aad_len > rte_pktmbuf_tailroom(ml))
 134                 return -ENOSPC;
 135
 136         /* prepend header */
 137         ph = rte_pktmbuf_prepend(mb, hlen - l2len);
 138         if (ph == NULL)
 139                 return -ENOSPC;
 140
 141         /* append tail */
 142         pdofs = ml->data_len;
 143         ml->data_len += tlen;
 144         mb->pkt_len += tlen;
 145         pt = rte_pktmbuf_mtod_offset(ml, typeof(pt), pdofs);
 146
 147         /* update pkt l2/l3 len */
 148         mb->tx_offload = (mb->tx_offload & sa->tx_offload.msk) |
 149                 sa->tx_offload.val;
 150
 151         /* copy tunnel pkt header */
 152         rte_memcpy(ph, sa->hdr, sa->hdr_len);
 153
 154         /* update original and new ip header fields */
 155         update_tun_l3hdr(sa, ph + sa->hdr_l3_off, mb->pkt_len, sa->hdr_l3_off,
 156                         sqn_low16(sqc));
 157
 158         /* update spi, seqn and iv */
 159         esph = (struct esp_hdr *)(ph + sa->hdr_len);
 160         iv = (uint64_t *)(esph + 1);
 161         copy_iv(iv, ivp, sa->iv_len);
 162
 163         esph->spi = sa->spi;
 164         esph->seq = sqn_low32(sqc);
 165
 166         /* offset for ICV */
 167         pdofs += pdlen + sa->sqh_len;
 168
 169         /* pad length */
 170         pdlen -= sizeof(*espt);
 171
 172         /* copy padding data */
 173         rte_memcpy(pt, esp_pad_bytes, pdlen);
 174
 175         /* update esp trailer */
 176         espt = (struct esp_tail *)(pt + pdlen);
 177         espt->pad_len = pdlen;
 178         espt->next_proto = sa->proto;
 179
 180         icv->va = rte_pktmbuf_mtod_offset(ml, void *, pdofs);
 181         icv->pa = rte_pktmbuf_iova_offset(ml, pdofs);
 182
 183         return clen;
 184 }
 185
 186 /*
 187  * for pure cryptodev (lookaside none) depending on SA settings,
 188  * we might have to write some extra data to the packet.
 189  */
 190 static inline void
 191 outb_pkt_xprepare(const struct rte_ipsec_sa *sa, rte_be64_t sqc,
 192         const union sym_op_data *icv)
 193 {
 194         uint32_t *psqh;
 195         struct aead_gcm_aad *aad;
 196
 197         /* insert SQN.hi between ESP trailer and ICV */
 198         if (sa->sqh_len != 0) {
 199                 psqh = (uint32_t *)(icv->va - sa->sqh_len);
 200                 psqh[0] = sqn_hi32(sqc);
 201         }
 202
 203         /*
 204          * fill IV and AAD fields, if any (aad fields are placed after icv),
 205          * right now we support only one AEAD algorithm: AES-GCM .
 206          */
 207         if (sa->aad_len != 0) {
 208                 aad = (struct aead_gcm_aad *)(icv->va + sa->icv_len);
 209                 aead_gcm_aad_fill(aad, sa->spi, sqc, IS_ESN(sa));
 210         }
 211 }
 212
 213 /*
 214  * setup/update packets and crypto ops for ESP outbound tunnel case.
 215  */
 216 uint16_t
 217 esp_outb_tun_prepare(const struct rte_ipsec_session *ss, struct rte_mbuf *mb[],
 218         struct rte_crypto_op *cop[], uint16_t num)
 219 {
 220         int32_t rc;
 221         uint32_t i, k, n;
 222         uint64_t sqn;
 223         rte_be64_t sqc;
 224         struct rte_ipsec_sa *sa;
 225         struct rte_cryptodev_sym_session *cs;
 226         union sym_op_data icv;
 227         uint64_t iv[IPSEC_MAX_IV_QWORD];
 228         uint32_t dr[num];
 229
 230         sa = ss->sa;
 231         cs = ss->crypto.ses;
 232
 233         n = num;
 234         sqn = esn_outb_update_sqn(sa, &n);
 235         if (n != num)
 236                 rte_errno = EOVERFLOW;
 237
 238         k = 0;
 239         for (i = 0; i != n; i++) {
 240
 241                 sqc = rte_cpu_to_be_64(sqn + i);
 242                 gen_iv(iv, sqc);
 243
 244                 /* try to update the packet itself */
 245                 rc = outb_tun_pkt_prepare(sa, sqc, iv, mb[i], &icv);
 246
 247                 /* success, setup crypto op */
 248                 if (rc >= 0) {
 249                         outb_pkt_xprepare(sa, sqc, &icv);
 250                         lksd_none_cop_prepare(cop[k], cs, mb[i]);
 251                         outb_cop_prepare(cop[k], sa, iv, &icv, 0, rc);
 252                         k++;
 253                 /* failure, put packet into the death-row */
 254                 } else {
 255                         dr[i - k] = i;
 256                         rte_errno = -rc;
 257                 }
 258         }
 259
 260          /* copy not prepared mbufs beyond good ones */
 261         if (k != n && k != 0)
 262                 move_bad_mbufs(mb, dr, n, n - k);
 263
 264         return k;
 265 }
 266
 267 /*
 268  * setup/update packet data and metadata for ESP outbound transport case.
 269  */
 270 static inline int32_t
 271 outb_trs_pkt_prepare(struct rte_ipsec_sa *sa, rte_be64_t sqc,
 272         const uint64_t ivp[IPSEC_MAX_IV_QWORD], struct rte_mbuf *mb,
 273         uint32_t l2len, uint32_t l3len, union sym_op_data *icv)
 274 {
 275         uint8_t np;
 276         uint32_t clen, hlen, pdlen, pdofs, plen, tlen, uhlen;
 277         struct rte_mbuf *ml;
 278         struct esp_hdr *esph;
 279         struct esp_tail *espt;
 280         char *ph, *pt;
 281         uint64_t *iv;
 282
 283         uhlen = l2len + l3len;
 284         plen = mb->pkt_len - uhlen;
 285
 286         /* calculate extra header space required */
 287         hlen = sa->iv_len + sizeof(*esph);
 288
 289         /* number of bytes to encrypt */
 290         clen = plen + sizeof(*espt);
 291         clen = RTE_ALIGN_CEIL(clen, sa->pad_align);
 292
 293         /* pad length + esp tail */
 294         pdlen = clen - plen;
 295         tlen = pdlen + sa->icv_len;
 296
 297         /* do append and insert */
 298         ml = rte_pktmbuf_lastseg(mb);
 299         if (tlen + sa->sqh_len + sa->aad_len > rte_pktmbuf_tailroom(ml))
 300                 return -ENOSPC;
 301
 302         /* prepend space for ESP header */
 303         ph = rte_pktmbuf_prepend(mb, hlen);
 304         if (ph == NULL)
 305                 return -ENOSPC;
 306
 307         /* append tail */
 308         pdofs = ml->data_len;
 309         ml->data_len += tlen;
 310         mb->pkt_len += tlen;
 311         pt = rte_pktmbuf_mtod_offset(ml, typeof(pt), pdofs);
 312
 313         /* shift L2/L3 headers */
 314         insert_esph(ph, ph + hlen, uhlen);
 315
 316         /* update ip  header fields */
 317         np = update_trs_l3hdr(sa, ph + l2len, mb->pkt_len, l2len, l3len,
 318                         IPPROTO_ESP);
 319
 320         /* update spi, seqn and iv */
 321         esph = (struct esp_hdr *)(ph + uhlen);
 322         iv = (uint64_t *)(esph + 1);
 323         copy_iv(iv, ivp, sa->iv_len);
 324
 325         esph->spi = sa->spi;
 326         esph->seq = sqn_low32(sqc);
 327
 328         /* offset for ICV */
 329         pdofs += pdlen + sa->sqh_len;
 330
 331         /* pad length */
 332         pdlen -= sizeof(*espt);
 333
 334         /* copy padding data */
 335         rte_memcpy(pt, esp_pad_bytes, pdlen);
 336
 337         /* update esp trailer */
 338         espt = (struct esp_tail *)(pt + pdlen);
 339         espt->pad_len = pdlen;
 340         espt->next_proto = np;
 341
 342         icv->va = rte_pktmbuf_mtod_offset(ml, void *, pdofs);
 343         icv->pa = rte_pktmbuf_iova_offset(ml, pdofs);
 344
 345         return clen;
 346 }
 347
 348 /*
 349  * setup/update packets and crypto ops for ESP outbound transport case.
 350  */
 351 uint16_t
 352 esp_outb_trs_prepare(const struct rte_ipsec_session *ss, struct rte_mbuf *mb[],
 353         struct rte_crypto_op *cop[], uint16_t num)
 354 {
 355         int32_t rc;
 356         uint32_t i, k, n, l2, l3;
 357         uint64_t sqn;
 358         rte_be64_t sqc;
 359         struct rte_ipsec_sa *sa;
 360         struct rte_cryptodev_sym_session *cs;
 361         union sym_op_data icv;
 362         uint64_t iv[IPSEC_MAX_IV_QWORD];
 363         uint32_t dr[num];
 364
 365         sa = ss->sa;
 366         cs = ss->crypto.ses;
 367
 368         n = num;
 369         sqn = esn_outb_update_sqn(sa, &n);
 370         if (n != num)
 371                 rte_errno = EOVERFLOW;
 372
 373         k = 0;
 374         for (i = 0; i != n; i++) {
 375
 376                 l2 = mb[i]->l2_len;
 377                 l3 = mb[i]->l3_len;
 378
 379                 sqc = rte_cpu_to_be_64(sqn + i);
 380                 gen_iv(iv, sqc);
 381
 382                 /* try to update the packet itself */
 383                 rc = outb_trs_pkt_prepare(sa, sqc, iv, mb[i], l2, l3, &icv);
 384
 385                 /* success, setup crypto op */
 386                 if (rc >= 0) {
 387                         outb_pkt_xprepare(sa, sqc, &icv);
 388                         lksd_none_cop_prepare(cop[k], cs, mb[i]);
 389                         outb_cop_prepare(cop[k], sa, iv, &icv, l2 + l3, rc);
 390                         k++;
 391                 /* failure, put packet into the death-row */
 392                 } else {
 393                         dr[i - k] = i;
 394                         rte_errno = -rc;
 395                 }
 396         }
 397
 398         /* copy not prepared mbufs beyond good ones */
 399         if (k != n && k != 0)
 400                 move_bad_mbufs(mb, dr, n, n - k);
 401
 402         return k;
 403 }
 404
 405 /*
 406  * process outbound packets for SA with ESN support,
 407  * for algorithms that require SQN.hibits to be implictly included
 408  * into digest computation.
 409  * In that case we have to move ICV bytes back to their proper place.
 410  */
 411 uint16_t
 412 esp_outb_sqh_process(const struct rte_ipsec_session *ss, struct rte_mbuf *mb[],
 413         uint16_t num)
 414 {
 415         uint32_t i, k, icv_len, *icv;
 416         struct rte_mbuf *ml;
 417         struct rte_ipsec_sa *sa;
 418         uint32_t dr[num];
 419
 420         sa = ss->sa;
 421
 422         k = 0;
 423         icv_len = sa->icv_len;
 424
 425         for (i = 0; i != num; i++) {
 426                 if ((mb[i]->ol_flags & PKT_RX_SEC_OFFLOAD_FAILED) == 0) {
 427                         ml = rte_pktmbuf_lastseg(mb[i]);
 428                         icv = rte_pktmbuf_mtod_offset(ml, void *,
 429                                 ml->data_len - icv_len);
 430                         remove_sqh(icv, icv_len);
 431                         k++;
 432                 } else
 433                         dr[i - k] = i;
 434         }
 435
 436         /* handle unprocessed mbufs */
 437         if (k != num) {
 438                 rte_errno = EBADMSG;
 439                 if (k != 0)
 440                         move_bad_mbufs(mb, dr, num, num - k);
 441         }
 442
 443         return k;
 444 }
 445
 446 /*
 447  * prepare packets for inline ipsec processing:
 448  * set ol_flags and attach metadata.
 449  */
 450 static inline void
 451 inline_outb_mbuf_prepare(const struct rte_ipsec_session *ss,
 452         struct rte_mbuf *mb[], uint16_t num)
 453 {
 454         uint32_t i, ol_flags;
 455
 456         ol_flags = ss->security.ol_flags & RTE_SECURITY_TX_OLOAD_NEED_MDATA;
 457         for (i = 0; i != num; i++) {
 458
 459                 mb[i]->ol_flags |= PKT_TX_SEC_OFFLOAD;
 460                 if (ol_flags != 0)
 461                         rte_security_set_pkt_metadata(ss->security.ctx,
 462                                 ss->security.ses, mb[i], NULL);
 463         }
 464 }
 465
 466 /*
 467  * process group of ESP outbound tunnel packets destined for
 468  * INLINE_CRYPTO type of device.
 469  */
 470 uint16_t
 471 inline_outb_tun_pkt_process(const struct rte_ipsec_session *ss,
 472         struct rte_mbuf *mb[], uint16_t num)
 473 {
 474         int32_t rc;
 475         uint32_t i, k, n;
 476         uint64_t sqn;
 477         rte_be64_t sqc;
 478         struct rte_ipsec_sa *sa;
 479         union sym_op_data icv;
 480         uint64_t iv[IPSEC_MAX_IV_QWORD];
 481         uint32_t dr[num];
 482
 483         sa = ss->sa;
 484
 485         n = num;
 486         sqn = esn_outb_update_sqn(sa, &n);
 487         if (n != num)
 488                 rte_errno = EOVERFLOW;
 489
 490         k = 0;
 491         for (i = 0; i != n; i++) {
 492
 493                 sqc = rte_cpu_to_be_64(sqn + i);
 494                 gen_iv(iv, sqc);
 495
 496                 /* try to update the packet itself */
 497                 rc = outb_tun_pkt_prepare(sa, sqc, iv, mb[i], &icv);
 498
 499                 k += (rc >= 0);
 500
 501                 /* failure, put packet into the death-row */
 502                 if (rc < 0) {
 503                         dr[i - k] = i;
 504                         rte_errno = -rc;
 505                 }
 506         }
 507
 508         /* copy not processed mbufs beyond good ones */
 509         if (k != n && k != 0)
 510                 move_bad_mbufs(mb, dr, n, n - k);
 511
 512         inline_outb_mbuf_prepare(ss, mb, k);
 513         return k;
 514 }
 515
 516 /*
 517  * process group of ESP outbound transport packets destined for
 518  * INLINE_CRYPTO type of device.
 519  */
 520 uint16_t
 521 inline_outb_trs_pkt_process(const struct rte_ipsec_session *ss,
 522         struct rte_mbuf *mb[], uint16_t num)
 523 {
 524         int32_t rc;
 525         uint32_t i, k, n, l2, l3;
 526         uint64_t sqn;
 527         rte_be64_t sqc;
 528         struct rte_ipsec_sa *sa;
 529         union sym_op_data icv;
 530         uint64_t iv[IPSEC_MAX_IV_QWORD];
 531         uint32_t dr[num];
 532
 533         sa = ss->sa;
 534
 535         n = num;
 536         sqn = esn_outb_update_sqn(sa, &n);
 537         if (n != num)
 538                 rte_errno = EOVERFLOW;
 539
 540         k = 0;
 541         for (i = 0; i != n; i++) {
 542
 543                 l2 = mb[i]->l2_len;
 544                 l3 = mb[i]->l3_len;
 545
 546                 sqc = rte_cpu_to_be_64(sqn + i);
 547                 gen_iv(iv, sqc);
 548
 549                 /* try to update the packet itself */
 550                 rc = outb_trs_pkt_prepare(sa, sqc, iv, mb[i],
 551                                 l2, l3, &icv);
 552
 553                 k += (rc >= 0);
 554
 555                 /* failure, put packet into the death-row */
 556                 if (rc < 0) {
 557                         dr[i - k] = i;
 558                         rte_errno = -rc;
 559                 }
 560         }
 561
 562         /* copy not processed mbufs beyond good ones */
 563         if (k != n && k != 0)
 564                 move_bad_mbufs(mb, dr, n, n - k);
 565
 566         inline_outb_mbuf_prepare(ss, mb, k);
 567         return k;
 568 }
 569
 570 /*
 571  * outbound for RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL:
 572  * actual processing is done by HW/PMD, just set flags and metadata.
 573  */
 574 uint16_t
 575 inline_proto_outb_pkt_process(const struct rte_ipsec_session *ss,
 576         struct rte_mbuf *mb[], uint16_t num)
 577 {
 578         inline_outb_mbuf_prepare(ss, mb, num);
 579         return num;
 580 }