lib/crypto/chacha20poly1305.c

   1 // SPDX-License-Identifier: GPL-2.0 OR MIT
   2 /*
   3  * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
   4  *
   5  * This is an implementation of the ChaCha20Poly1305 AEAD construction.
   6  *
   7  * Information: https://tools.ietf.org/html/rfc8439
   8  */
   9
  10 #include <crypto/algapi.h>
  11 #include <crypto/chacha20poly1305.h>
  12 #include <crypto/chacha.h>
  13 #include <crypto/poly1305.h>
  14 #include <crypto/scatterwalk.h>
  15
  16 #include <asm/unaligned.h>
  17 #include <linux/kernel.h>
  18 #include <linux/init.h>
  19 #include <linux/mm.h>
  20 #include <linux/module.h>
  21
  22 #define CHACHA_KEY_WORDS        (CHACHA_KEY_SIZE / sizeof(u32))
  23
  24 static void chacha_load_key(u32 *k, const u8 *in)
  25 {
  26         k[0] = get_unaligned_le32(in);
  27         k[1] = get_unaligned_le32(in + 4);
  28         k[2] = get_unaligned_le32(in + 8);
  29         k[3] = get_unaligned_le32(in + 12);
  30         k[4] = get_unaligned_le32(in + 16);
  31         k[5] = get_unaligned_le32(in + 20);
  32         k[6] = get_unaligned_le32(in + 24);
  33         k[7] = get_unaligned_le32(in + 28);
  34 }
  35
  36 static void xchacha_init(u32 *chacha_state, const u8 *key, const u8 *nonce)
  37 {
  38         u32 k[CHACHA_KEY_WORDS];
  39         u8 iv[CHACHA_IV_SIZE];
  40
  41         memset(iv, 0, 8);
  42         memcpy(iv + 8, nonce + 16, 8);
  43
  44         chacha_load_key(k, key);
  45
  46         /* Compute the subkey given the original key and first 128 nonce bits */
  47         chacha_init(chacha_state, k, nonce);
  48         hchacha_block(chacha_state, k, 20);
  49
  50         chacha_init(chacha_state, k, iv);
  51
  52         memzero_explicit(k, sizeof(k));
  53         memzero_explicit(iv, sizeof(iv));
  54 }
  55
  56 static void
  57 __chacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len,
  58                            const u8 *ad, const size_t ad_len, u32 *chacha_state)
  59 {
  60         const u8 *pad0 = page_address(ZERO_PAGE(0));
  61         struct poly1305_desc_ctx poly1305_state;
  62         union {
  63                 u8 block0[POLY1305_KEY_SIZE];
  64                 __le64 lens[2];
  65         } b;
  66
  67         chacha20_crypt(chacha_state, b.block0, pad0, sizeof(b.block0));
  68         poly1305_init(&poly1305_state, b.block0);
  69
  70         poly1305_update(&poly1305_state, ad, ad_len);
  71         if (ad_len & 0xf)
  72                 poly1305_update(&poly1305_state, pad0, 0x10 - (ad_len & 0xf));
  73
  74         chacha20_crypt(chacha_state, dst, src, src_len);
  75
  76         poly1305_update(&poly1305_state, dst, src_len);
  77         if (src_len & 0xf)
  78                 poly1305_update(&poly1305_state, pad0, 0x10 - (src_len & 0xf));
  79
  80         b.lens[0] = cpu_to_le64(ad_len);
  81         b.lens[1] = cpu_to_le64(src_len);
  82         poly1305_update(&poly1305_state, (u8 *)b.lens, sizeof(b.lens));
  83
  84         poly1305_final(&poly1305_state, dst + src_len);
  85
  86         memzero_explicit(chacha_state, CHACHA_STATE_WORDS * sizeof(u32));
  87         memzero_explicit(&b, sizeof(b));
  88 }
  89
  90 void chacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len,
  91                               const u8 *ad, const size_t ad_len,
  92                               const u64 nonce,
  93                               const u8 key[CHACHA20POLY1305_KEY_SIZE])
  94 {
  95         u32 chacha_state[CHACHA_STATE_WORDS];
  96         u32 k[CHACHA_KEY_WORDS];
  97         __le64 iv[2];
  98
  99         chacha_load_key(k, key);
 100
 101         iv[0] = 0;
 102         iv[1] = cpu_to_le64(nonce);
 103
 104         chacha_init(chacha_state, k, (u8 *)iv);
 105         __chacha20poly1305_encrypt(dst, src, src_len, ad, ad_len, chacha_state);
 106
 107         memzero_explicit(iv, sizeof(iv));
 108         memzero_explicit(k, sizeof(k));
 109 }
 110 EXPORT_SYMBOL(chacha20poly1305_encrypt);
 111
 112 void xchacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len,
 113                                const u8 *ad, const size_t ad_len,
 114                                const u8 nonce[XCHACHA20POLY1305_NONCE_SIZE],
 115                                const u8 key[CHACHA20POLY1305_KEY_SIZE])
 116 {
 117         u32 chacha_state[CHACHA_STATE_WORDS];
 118
 119         xchacha_init(chacha_state, key, nonce);
 120         __chacha20poly1305_encrypt(dst, src, src_len, ad, ad_len, chacha_state);
 121 }
 122 EXPORT_SYMBOL(xchacha20poly1305_encrypt);
 123
 124 static bool
 125 __chacha20poly1305_decrypt(u8 *dst, const u8 *src, const size_t src_len,
 126                            const u8 *ad, const size_t ad_len, u32 *chacha_state)
 127 {
 128         const u8 *pad0 = page_address(ZERO_PAGE(0));
 129         struct poly1305_desc_ctx poly1305_state;
 130         size_t dst_len;
 131         int ret;
 132         union {
 133                 u8 block0[POLY1305_KEY_SIZE];
 134                 u8 mac[POLY1305_DIGEST_SIZE];
 135                 __le64 lens[2];
 136         } b;
 137
 138         if (unlikely(src_len < POLY1305_DIGEST_SIZE))
 139                 return false;
 140
 141         chacha20_crypt(chacha_state, b.block0, pad0, sizeof(b.block0));
 142         poly1305_init(&poly1305_state, b.block0);
 143
 144         poly1305_update(&poly1305_state, ad, ad_len);
 145         if (ad_len & 0xf)
 146                 poly1305_update(&poly1305_state, pad0, 0x10 - (ad_len & 0xf));
 147
 148         dst_len = src_len - POLY1305_DIGEST_SIZE;
 149         poly1305_update(&poly1305_state, src, dst_len);
 150         if (dst_len & 0xf)
 151                 poly1305_update(&poly1305_state, pad0, 0x10 - (dst_len & 0xf));
 152
 153         b.lens[0] = cpu_to_le64(ad_len);
 154         b.lens[1] = cpu_to_le64(dst_len);
 155         poly1305_update(&poly1305_state, (u8 *)b.lens, sizeof(b.lens));
 156
 157         poly1305_final(&poly1305_state, b.mac);
 158
 159         ret = crypto_memneq(b.mac, src + dst_len, POLY1305_DIGEST_SIZE);
 160         if (likely(!ret))
 161                 chacha20_crypt(chacha_state, dst, src, dst_len);
 162
 163         memzero_explicit(&b, sizeof(b));
 164
 165         return !ret;
 166 }
 167
 168 bool chacha20poly1305_decrypt(u8 *dst, const u8 *src, const size_t src_len,
 169                               const u8 *ad, const size_t ad_len,
 170                               const u64 nonce,
 171                               const u8 key[CHACHA20POLY1305_KEY_SIZE])
 172 {
 173         u32 chacha_state[CHACHA_STATE_WORDS];
 174         u32 k[CHACHA_KEY_WORDS];
 175         __le64 iv[2];
 176         bool ret;
 177
 178         chacha_load_key(k, key);
 179
 180         iv[0] = 0;
 181         iv[1] = cpu_to_le64(nonce);
 182
 183         chacha_init(chacha_state, k, (u8 *)iv);
 184         ret = __chacha20poly1305_decrypt(dst, src, src_len, ad, ad_len,
 185                                          chacha_state);
 186
 187         memzero_explicit(chacha_state, sizeof(chacha_state));
 188         memzero_explicit(iv, sizeof(iv));
 189         memzero_explicit(k, sizeof(k));
 190         return ret;
 191 }
 192 EXPORT_SYMBOL(chacha20poly1305_decrypt);
 193
 194 bool xchacha20poly1305_decrypt(u8 *dst, const u8 *src, const size_t src_len,
 195                                const u8 *ad, const size_t ad_len,
 196                                const u8 nonce[XCHACHA20POLY1305_NONCE_SIZE],
 197                                const u8 key[CHACHA20POLY1305_KEY_SIZE])
 198 {
 199         u32 chacha_state[CHACHA_STATE_WORDS];
 200
 201         xchacha_init(chacha_state, key, nonce);
 202         return __chacha20poly1305_decrypt(dst, src, src_len, ad, ad_len,
 203                                           chacha_state);
 204 }
 205 EXPORT_SYMBOL(xchacha20poly1305_decrypt);
 206
 207 static
 208 bool chacha20poly1305_crypt_sg_inplace(struct scatterlist *src,
 209                                        const size_t src_len,
 210                                        const u8 *ad, const size_t ad_len,
 211                                        const u64 nonce,
 212                                        const u8 key[CHACHA20POLY1305_KEY_SIZE],
 213                                        int encrypt)
 214 {
 215         const u8 *pad0 = page_address(ZERO_PAGE(0));
 216         struct poly1305_desc_ctx poly1305_state;
 217         u32 chacha_state[CHACHA_STATE_WORDS];
 218         struct sg_mapping_iter miter;
 219         size_t partial = 0;
 220         unsigned int flags;
 221         bool ret = true;
 222         int sl;
 223         union {
 224                 struct {
 225                         u32 k[CHACHA_KEY_WORDS];
 226                         __le64 iv[2];
 227                 };
 228                 u8 block0[POLY1305_KEY_SIZE];
 229                 u8 chacha_stream[CHACHA_BLOCK_SIZE];
 230                 struct {
 231                         u8 mac[2][POLY1305_DIGEST_SIZE];
 232                 };
 233                 __le64 lens[2];
 234         } b __aligned(16);
 235
 236         if (WARN_ON(src_len > INT_MAX))
 237                 return false;
 238
 239         chacha_load_key(b.k, key);
 240
 241         b.iv[0] = 0;
 242         b.iv[1] = cpu_to_le64(nonce);
 243
 244         chacha_init(chacha_state, b.k, (u8 *)b.iv);
 245         chacha20_crypt(chacha_state, b.block0, pad0, sizeof(b.block0));
 246         poly1305_init(&poly1305_state, b.block0);
 247
 248         if (unlikely(ad_len)) {
 249                 poly1305_update(&poly1305_state, ad, ad_len);
 250                 if (ad_len & 0xf)
 251                         poly1305_update(&poly1305_state, pad0, 0x10 - (ad_len & 0xf));
 252         }
 253
 254         flags = SG_MITER_TO_SG;
 255         if (!preemptible())
 256                 flags |= SG_MITER_ATOMIC;
 257
 258         sg_miter_start(&miter, src, sg_nents(src), flags);
 259
 260         for (sl = src_len; sl > 0 && sg_miter_next(&miter); sl -= miter.length) {
 261                 u8 *addr = miter.addr;
 262                 size_t length = min_t(size_t, sl, miter.length);
 263
 264                 if (!encrypt)
 265                         poly1305_update(&poly1305_state, addr, length);
 266
 267                 if (unlikely(partial)) {
 268                         size_t l = min(length, CHACHA_BLOCK_SIZE - partial);
 269
 270                         crypto_xor(addr, b.chacha_stream + partial, l);
 271                         partial = (partial + l) & (CHACHA_BLOCK_SIZE - 1);
 272
 273                         addr += l;
 274                         length -= l;
 275                 }
 276
 277                 if (likely(length >= CHACHA_BLOCK_SIZE || length == sl)) {
 278                         size_t l = length;
 279
 280                         if (unlikely(length < sl))
 281                                 l &= ~(CHACHA_BLOCK_SIZE - 1);
 282                         chacha20_crypt(chacha_state, addr, addr, l);
 283                         addr += l;
 284                         length -= l;
 285                 }
 286
 287                 if (unlikely(length > 0)) {
 288                         chacha20_crypt(chacha_state, b.chacha_stream, pad0,
 289                                        CHACHA_BLOCK_SIZE);
 290                         crypto_xor(addr, b.chacha_stream, length);
 291                         partial = length;
 292                 }
 293
 294                 if (encrypt)
 295                         poly1305_update(&poly1305_state, miter.addr,
 296                                         min_t(size_t, sl, miter.length));
 297         }
 298
 299         if (src_len & 0xf)
 300                 poly1305_update(&poly1305_state, pad0, 0x10 - (src_len & 0xf));
 301
 302         b.lens[0] = cpu_to_le64(ad_len);
 303         b.lens[1] = cpu_to_le64(src_len);
 304         poly1305_update(&poly1305_state, (u8 *)b.lens, sizeof(b.lens));
 305
 306         if (likely(sl <= -POLY1305_DIGEST_SIZE)) {
 307                 if (encrypt) {
 308                         poly1305_final(&poly1305_state,
 309                                        miter.addr + miter.length + sl);
 310                         ret = true;
 311                 } else {
 312                         poly1305_final(&poly1305_state, b.mac[0]);
 313                         ret = !crypto_memneq(b.mac[0],
 314                                              miter.addr + miter.length + sl,
 315                                              POLY1305_DIGEST_SIZE);
 316                 }
 317         }
 318
 319         sg_miter_stop(&miter);
 320
 321         if (unlikely(sl > -POLY1305_DIGEST_SIZE)) {
 322                 poly1305_final(&poly1305_state, b.mac[1]);
 323                 scatterwalk_map_and_copy(b.mac[encrypt], src, src_len,
 324                                          sizeof(b.mac[1]), encrypt);
 325                 ret = encrypt ||
 326                       !crypto_memneq(b.mac[0], b.mac[1], POLY1305_DIGEST_SIZE);
 327         }
 328
 329         memzero_explicit(chacha_state, sizeof(chacha_state));
 330         memzero_explicit(&b, sizeof(b));
 331
 332         return ret;
 333 }
 334
 335 bool chacha20poly1305_encrypt_sg_inplace(struct scatterlist *src, size_t src_len,
 336                                          const u8 *ad, const size_t ad_len,
 337                                          const u64 nonce,
 338                                          const u8 key[CHACHA20POLY1305_KEY_SIZE])
 339 {
 340         return chacha20poly1305_crypt_sg_inplace(src, src_len, ad, ad_len,
 341                                                  nonce, key, 1);
 342 }
 343 EXPORT_SYMBOL(chacha20poly1305_encrypt_sg_inplace);
 344
 345 bool chacha20poly1305_decrypt_sg_inplace(struct scatterlist *src, size_t src_len,
 346                                          const u8 *ad, const size_t ad_len,
 347                                          const u64 nonce,
 348                                          const u8 key[CHACHA20POLY1305_KEY_SIZE])
 349 {
 350         if (unlikely(src_len < POLY1305_DIGEST_SIZE))
 351                 return false;
 352
 353         return chacha20poly1305_crypt_sg_inplace(src,
 354                                                  src_len - POLY1305_DIGEST_SIZE,
 355                                                  ad, ad_len, nonce, key, 0);
 356 }
 357 EXPORT_SYMBOL(chacha20poly1305_decrypt_sg_inplace);
 358
 359 static int __init mod_init(void)
 360 {
 361         if (!IS_ENABLED(CONFIG_CRYPTO_MANAGER_DISABLE_TESTS) &&
 362             WARN_ON(!chacha20poly1305_selftest()))
 363                 return -ENODEV;
 364         return 0;
 365 }
 366
 367 module_init(mod_init);
 368 MODULE_LICENSE("GPL v2");
 369 MODULE_DESCRIPTION("ChaCha20Poly1305 AEAD construction");
 370 MODULE_AUTHOR("Jason A. Donenfeld <Jason@zx2c4.com>");