ceph/src/arrow/go/parquet/internal/encryption/decryptor.go

   1 // Licensed to the Apache Software Foundation (ASF) under one
   2 // or more contributor license agreements.  See the NOTICE file
   3 // distributed with this work for additional information
   4 // regarding copyright ownership.  The ASF licenses this file
   5 // to you under the Apache License, Version 2.0 (the
   6 // "License"); you may not use this file except in compliance
   7 // with the License.  You may obtain a copy of the License at
   8 //
   9 // http://www.apache.org/licenses/LICENSE-2.0
  10 //
  11 // Unless required by applicable law or agreed to in writing, software
  12 // distributed under the License is distributed on an "AS IS" BASIS,
  13 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14 // See the License for the specific language governing permissions and
  15 // limitations under the License.
  16
  17 package encryption
  18
  19 import (
  20         "github.com/apache/arrow/go/v6/arrow/memory"
  21         "github.com/apache/arrow/go/v6/parquet"
  22 )
  23
  24 // FileDecryptor is an interface used by the filereader for decrypting an
  25 // entire parquet file as we go, usually constructed from the DecryptionProperties
  26 type FileDecryptor interface {
  27         // Returns the key for decrypting the footer if provided
  28         GetFooterKey() string
  29         // Provides the file level AAD security bytes
  30         FileAad() string
  31         // return which algorithm this decryptor was constructed for
  32         Algorithm() parquet.Cipher
  33         // return the FileDecryptionProperties that were used for this decryptor
  34         Properties() *parquet.FileDecryptionProperties
  35         // Clear out the decryption keys, this is automatically called after every
  36         // successfully decrypted file to ensure that keys aren't kept around.
  37         WipeOutDecryptionKeys()
  38         // GetFooterDecryptor returns a Decryptor interface for use to decrypt the footer
  39         // of a parquet file.
  40         GetFooterDecryptor() Decryptor
  41         // GetFooterDecryptorForColumnMeta returns a Decryptor interface for Column Metadata
  42         // in the file footer using the AAD bytes provided.
  43         GetFooterDecryptorForColumnMeta(aad string) Decryptor
  44         // GetFooterDecryptorForColumnData returns the decryptor that can be used for decrypting
  45         // actual column data footer bytes, not column metadata.
  46         GetFooterDecryptorForColumnData(aad string) Decryptor
  47         // GetColumnMetaDecryptor returns a decryptor for the requested column path, key and AAD bytes
  48         // but only for decrypting the row group level metadata
  49         GetColumnMetaDecryptor(columnPath, columnKeyMetadata, aad string) Decryptor
  50         // GetColumnDataDecryptor returns a decryptor for the requested column path, key, and AAD bytes
  51         // but only for the rowgroup column data.
  52         GetColumnDataDecryptor(columnPath, columnKeyMetadata, aad string) Decryptor
  53 }
  54
  55 type fileDecryptor struct {
  56         // the properties contains the key retriever for us to get keys
  57         // from the key metadata
  58         props *parquet.FileDecryptionProperties
  59         // concatenation of aad_prefix (if exists) and aad_file_unique
  60         fileAad                 string
  61         columnDataMap           map[string]Decryptor
  62         columnMetaDataMap       map[string]Decryptor
  63         footerMetadataDecryptor Decryptor
  64         footerDataDecryptor     Decryptor
  65         alg                     parquet.Cipher
  66         footerKeyMetadata       string
  67         metaDecryptor           *aesDecryptor
  68         dataDecryptor           *aesDecryptor
  69         mem                     memory.Allocator
  70 }
  71
  72 // NewFileDecryptor constructs a decryptor from the provided configuration of properties, cipher and key metadata. Using the provided memory allocator or
  73 // the default allocator if one isn't provided.
  74 func NewFileDecryptor(props *parquet.FileDecryptionProperties, fileAad string, alg parquet.Cipher, keymetadata string, mem memory.Allocator) FileDecryptor {
  75         if mem == nil {
  76                 mem = memory.DefaultAllocator
  77         }
  78         return &fileDecryptor{
  79                 fileAad:           fileAad,
  80                 props:             props,
  81                 alg:               alg,
  82                 footerKeyMetadata: keymetadata,
  83                 mem:               mem,
  84                 columnDataMap:     make(map[string]Decryptor),
  85                 columnMetaDataMap: make(map[string]Decryptor),
  86         }
  87 }
  88
  89 func (d *fileDecryptor) FileAad() string                               { return d.fileAad }
  90 func (d *fileDecryptor) Properties() *parquet.FileDecryptionProperties { return d.props }
  91 func (d *fileDecryptor) Algorithm() parquet.Cipher                     { return d.alg }
  92 func (d *fileDecryptor) GetFooterKey() string {
  93         footerKey := d.props.FooterKey()
  94         if footerKey == "" {
  95                 if d.footerKeyMetadata == "" {
  96                         panic("no footer key or key metadata")
  97                 }
  98                 if d.props.KeyRetriever == nil {
  99                         panic("no footer key or key retriever")
 100                 }
 101                 footerKey = d.props.KeyRetriever.GetKey([]byte(d.footerKeyMetadata))
 102         }
 103         if footerKey == "" {
 104                 panic("invalid footer encryption key. Could not parse footer metadata")
 105         }
 106         return footerKey
 107 }
 108
 109 func (d *fileDecryptor) GetFooterDecryptor() Decryptor {
 110         aad := CreateFooterAad(d.fileAad)
 111         return d.getFooterDecryptor(aad, true)
 112 }
 113
 114 func (d *fileDecryptor) GetFooterDecryptorForColumnMeta(aad string) Decryptor {
 115         return d.getFooterDecryptor(aad, true)
 116 }
 117
 118 func (d *fileDecryptor) GetFooterDecryptorForColumnData(aad string) Decryptor {
 119         return d.getFooterDecryptor(aad, false)
 120 }
 121
 122 func (d *fileDecryptor) GetColumnMetaDecryptor(columnPath, columnKeyMetadata, aad string) Decryptor {
 123         return d.getColumnDecryptor(columnPath, columnKeyMetadata, aad, true)
 124 }
 125
 126 func (d *fileDecryptor) GetColumnDataDecryptor(columnPath, columnKeyMetadata, aad string) Decryptor {
 127         return d.getColumnDecryptor(columnPath, columnKeyMetadata, aad, false)
 128 }
 129
 130 func (d *fileDecryptor) WipeOutDecryptionKeys() {
 131         d.props.WipeOutDecryptionKeys()
 132 }
 133
 134 func (d *fileDecryptor) getFooterDecryptor(aad string, metadata bool) Decryptor {
 135         if metadata {
 136                 if d.footerMetadataDecryptor != nil {
 137                         return d.footerMetadataDecryptor
 138                 }
 139         } else {
 140                 if d.footerDataDecryptor != nil {
 141                         return d.footerDataDecryptor
 142                 }
 143         }
 144
 145         footerKey := d.GetFooterKey()
 146
 147         // Create both data and metadata decryptors to avoid redundant retrieval of key
 148         // from the key_retriever.
 149         aesMetaDecrypt := d.getMetaAesDecryptor()
 150         aesDataDecrypt := d.getDataAesDecryptor()
 151
 152         d.footerMetadataDecryptor = &decryptor{
 153                 decryptor: aesMetaDecrypt,
 154                 key:       []byte(footerKey),
 155                 fileAad:   []byte(d.fileAad),
 156                 aad:       []byte(aad),
 157                 mem:       d.mem,
 158         }
 159         d.footerDataDecryptor = &decryptor{
 160                 decryptor: aesDataDecrypt,
 161                 key:       []byte(footerKey),
 162                 fileAad:   []byte(d.fileAad),
 163                 aad:       []byte(aad),
 164                 mem:       d.mem,
 165         }
 166
 167         if metadata {
 168                 return d.footerMetadataDecryptor
 169         }
 170         return d.footerDataDecryptor
 171 }
 172
 173 func (d *fileDecryptor) getColumnDecryptor(columnPath, columnMeta, aad string, metadata bool) Decryptor {
 174         if metadata {
 175                 if res, ok := d.columnMetaDataMap[columnPath]; ok {
 176                         res.UpdateAad(aad)
 177                         return res
 178                 }
 179         } else {
 180                 if res, ok := d.columnDataMap[columnPath]; ok {
 181                         res.UpdateAad(aad)
 182                         return res
 183                 }
 184         }
 185
 186         columnKey := d.props.ColumnKey(columnPath)
 187         // No explicit column key given via API. Retrieve via key metadata.
 188         if columnKey == "" && columnMeta != "" && d.props.KeyRetriever != nil {
 189                 columnKey = d.props.KeyRetriever.GetKey([]byte(columnMeta))
 190         }
 191         if columnKey == "" {
 192                 panic("hidden column exception, path=" + columnPath)
 193         }
 194
 195         aesDataDecrypt := d.getDataAesDecryptor()
 196         aesMetaDecrypt := d.getMetaAesDecryptor()
 197
 198         d.columnDataMap[columnPath] = &decryptor{
 199                 decryptor: aesDataDecrypt,
 200                 key:       []byte(columnKey),
 201                 fileAad:   []byte(d.fileAad),
 202                 aad:       []byte(aad),
 203                 mem:       d.mem,
 204         }
 205         d.columnMetaDataMap[columnPath] = &decryptor{
 206                 decryptor: aesMetaDecrypt,
 207                 key:       []byte(columnKey),
 208                 fileAad:   []byte(d.fileAad),
 209                 aad:       []byte(aad),
 210                 mem:       d.mem,
 211         }
 212
 213         if metadata {
 214                 return d.columnMetaDataMap[columnPath]
 215         }
 216         return d.columnDataMap[columnPath]
 217 }
 218
 219 func (d *fileDecryptor) getMetaAesDecryptor() *aesDecryptor {
 220         if d.metaDecryptor == nil {
 221                 d.metaDecryptor = newAesDecryptor(d.alg, true)
 222         }
 223         return d.metaDecryptor
 224 }
 225
 226 func (d *fileDecryptor) getDataAesDecryptor() *aesDecryptor {
 227         if d.dataDecryptor == nil {
 228                 d.dataDecryptor = newAesDecryptor(d.alg, false)
 229         }
 230         return d.dataDecryptor
 231 }
 232
 233 // Decryptor is the basic interface for any decryptor generated from a FileDecryptor
 234 type Decryptor interface {
 235         // returns the File Level AAD bytes
 236         FileAad() string
 237         // returns the current allocator that was used for any extra allocations of buffers
 238         Allocator() memory.Allocator
 239         // returns the CiphertextSizeDelta from the decryptor
 240         CiphertextSizeDelta() int
 241         // Decrypt just returns the decrypted plaintext from the src ciphertext
 242         Decrypt(src []byte) []byte
 243         // set the AAD bytes of the decryptor to the provided string
 244         UpdateAad(string)
 245 }
 246
 247 type decryptor struct {
 248         decryptor *aesDecryptor
 249         key       []byte
 250         fileAad   []byte
 251         aad       []byte
 252         mem       memory.Allocator
 253 }
 254
 255 func (d *decryptor) Allocator() memory.Allocator { return d.mem }
 256 func (d *decryptor) FileAad() string             { return string(d.fileAad) }
 257 func (d *decryptor) UpdateAad(aad string)        { d.aad = []byte(aad) }
 258 func (d *decryptor) CiphertextSizeDelta() int    { return d.decryptor.CiphertextSizeDelta() }
 259 func (d *decryptor) Decrypt(src []byte) []byte {
 260         return d.decryptor.Decrypt(src, d.key, d.aad)
 261 }