]>
git.proxmox.com Git - ceph.git/blob - ceph/src/rocksdb/env/env_encryption.cc
1 // Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
2 // This source code is licensed under both the GPLv2 (found in the
3 // COPYING file in the root directory) and Apache 2.0 License
4 // (found in the LICENSE.Apache file in the root directory).
8 #include "rocksdb/env_encryption.h"
15 #include "env/env_encryption_ctr.h"
16 #include "monitoring/perf_context_imp.h"
17 #include "rocksdb/convenience.h"
18 #include "util/aligned_buffer.h"
19 #include "util/coding.h"
20 #include "util/random.h"
21 #include "util/string_util.h"
25 namespace ROCKSDB_NAMESPACE
{
28 static constexpr char kROT13CipherName
[] = "ROT13";
29 static constexpr char kCTRProviderName
[] = "CTR";
31 Status
BlockCipher::CreateFromString(const ConfigOptions
& /*config_options*/,
32 const std::string
& value
,
33 std::shared_ptr
<BlockCipher
>* result
) {
34 std::string id
= value
;
35 size_t colon
= value
.find(':');
36 if (colon
!= std::string::npos
) {
37 id
= value
.substr(0, colon
);
39 if (id
== kROT13CipherName
) {
40 if (colon
!= std::string::npos
) {
41 size_t block_size
= ParseSizeT(value
.substr(colon
+ 1));
42 result
->reset(new ROT13BlockCipher(block_size
));
44 result
->reset(new ROT13BlockCipher(32));
48 return Status::NotSupported("Could not find cipher ", value
);
52 Status
EncryptionProvider::CreateFromString(
53 const ConfigOptions
& /*config_options*/, const std::string
& value
,
54 std::shared_ptr
<EncryptionProvider
>* result
) {
55 std::string id
= value
;
56 bool is_test
= StartsWith(value
, "test://");
57 Status status
= Status::OK();
59 id
= value
.substr(strlen("test://"));
61 if (id
== kCTRProviderName
) {
62 result
->reset(new CTREncryptionProvider());
64 result
->reset(new CTREncryptionProvider());
66 return Status::NotSupported("Could not find provider ", value
);
68 if (status
.ok() && is_test
) {
69 status
= result
->get()->TEST_Initialize();
74 std::shared_ptr
<EncryptionProvider
> EncryptionProvider::NewCTRProvider(
75 const std::shared_ptr
<BlockCipher
>& cipher
) {
76 return std::make_shared
<CTREncryptionProvider
>(cipher
);
79 // Read up to "n" bytes from the file. "scratch[0..n-1]" may be
80 // written by this routine. Sets "*result" to the data that was
81 // read (including if fewer than "n" bytes were successfully read).
82 // May set "*result" to point at data in "scratch[0..n-1]", so
83 // "scratch[0..n-1]" must be live when "*result" is used.
84 // If an error was encountered, returns a non-OK status.
86 // REQUIRES: External synchronization
87 Status
EncryptedSequentialFile::Read(size_t n
, Slice
* result
, char* scratch
) {
89 Status status
= file_
->Read(n
, result
, scratch
);
94 PERF_TIMER_GUARD(decrypt_data_nanos
);
95 status
= stream_
->Decrypt(offset_
, (char*)result
->data(), result
->size());
97 offset_
+= result
->size(); // We've already ready data from disk, so update
98 // offset_ even if decryption fails.
102 // Skip "n" bytes from the file. This is guaranteed to be no
103 // slower that reading the same data, but may be faster.
105 // If end of file is reached, skipping will stop at the end of the
106 // file, and Skip will return OK.
108 // REQUIRES: External synchronization
109 Status
EncryptedSequentialFile::Skip(uint64_t n
) {
110 auto status
= file_
->Skip(n
);
118 // Indicates the upper layers if the current SequentialFile implementation
120 bool EncryptedSequentialFile::use_direct_io() const {
121 return file_
->use_direct_io();
124 // Use the returned alignment value to allocate
125 // aligned buffer for Direct I/O
126 size_t EncryptedSequentialFile::GetRequiredBufferAlignment() const {
127 return file_
->GetRequiredBufferAlignment();
130 // Remove any kind of caching of data from the offset to offset+length
131 // of this file. If the length is 0, then it refers to the end of file.
132 // If the system is not caching the file contents, then this is a noop.
133 Status
EncryptedSequentialFile::InvalidateCache(size_t offset
, size_t length
) {
134 return file_
->InvalidateCache(offset
+ prefixLength_
, length
);
137 // Positioned Read for direct I/O
138 // If Direct I/O enabled, offset, n, and scratch should be properly aligned
139 Status
EncryptedSequentialFile::PositionedRead(uint64_t offset
, size_t n
,
140 Slice
* result
, char* scratch
) {
142 offset
+= prefixLength_
; // Skip prefix
143 auto status
= file_
->PositionedRead(offset
, n
, result
, scratch
);
147 offset_
= offset
+ result
->size();
149 PERF_TIMER_GUARD(decrypt_data_nanos
);
150 status
= stream_
->Decrypt(offset
, (char*)result
->data(), result
->size());
155 // Read up to "n" bytes from the file starting at "offset".
156 // "scratch[0..n-1]" may be written by this routine. Sets "*result"
157 // to the data that was read (including if fewer than "n" bytes were
158 // successfully read). May set "*result" to point at data in
159 // "scratch[0..n-1]", so "scratch[0..n-1]" must be live when
160 // "*result" is used. If an error was encountered, returns a non-OK
163 // Safe for concurrent use by multiple threads.
164 // If Direct I/O enabled, offset, n, and scratch should be aligned properly.
165 Status
EncryptedRandomAccessFile::Read(uint64_t offset
, size_t n
, Slice
* result
,
166 char* scratch
) const {
168 offset
+= prefixLength_
;
169 auto status
= file_
->Read(offset
, n
, result
, scratch
);
174 PERF_TIMER_GUARD(decrypt_data_nanos
);
175 status
= stream_
->Decrypt(offset
, (char*)result
->data(), result
->size());
180 // Readahead the file starting from offset by n bytes for caching.
181 Status
EncryptedRandomAccessFile::Prefetch(uint64_t offset
, size_t n
) {
182 // return Status::OK();
183 return file_
->Prefetch(offset
+ prefixLength_
, n
);
186 // Tries to get an unique ID for this file that will be the same each time
187 // the file is opened (and will stay the same while the file is open).
188 // Furthermore, it tries to make this ID at most "max_size" bytes. If such an
189 // ID can be created this function returns the length of the ID and places it
190 // in "id"; otherwise, this function returns 0, in which case "id"
191 // may not have been modified.
193 // This function guarantees, for IDs from a given environment, two unique ids
194 // cannot be made equal to each other by adding arbitrary bytes to one of
195 // them. That is, no unique ID is the prefix of another.
197 // This function guarantees that the returned ID will not be interpretable as
200 // Note: these IDs are only valid for the duration of the process.
201 size_t EncryptedRandomAccessFile::GetUniqueId(char* id
, size_t max_size
) const {
202 return file_
->GetUniqueId(id
, max_size
);
205 void EncryptedRandomAccessFile::Hint(AccessPattern pattern
) {
206 file_
->Hint(pattern
);
209 // Indicates the upper layers if the current RandomAccessFile implementation
211 bool EncryptedRandomAccessFile::use_direct_io() const {
212 return file_
->use_direct_io();
215 // Use the returned alignment value to allocate
216 // aligned buffer for Direct I/O
217 size_t EncryptedRandomAccessFile::GetRequiredBufferAlignment() const {
218 return file_
->GetRequiredBufferAlignment();
221 // Remove any kind of caching of data from the offset to offset+length
222 // of this file. If the length is 0, then it refers to the end of file.
223 // If the system is not caching the file contents, then this is a noop.
224 Status
EncryptedRandomAccessFile::InvalidateCache(size_t offset
,
226 return file_
->InvalidateCache(offset
+ prefixLength_
, length
);
229 // A file abstraction for sequential writing. The implementation
230 // must provide buffering since callers may append small fragments
231 // at a time to the file.
232 Status
EncryptedWritableFile::Append(const Slice
& data
) {
235 Slice
dataToAppend(data
);
236 if (data
.size() > 0) {
237 auto offset
= file_
->GetFileSize(); // size including prefix
238 // Encrypt in cloned buffer
239 buf
.Alignment(GetRequiredBufferAlignment());
240 buf
.AllocateNewBuffer(data
.size());
241 // TODO (sagar0): Modify AlignedBuffer.Append to allow doing a memmove
242 // so that the next two lines can be replaced with buf.Append().
243 memmove(buf
.BufferStart(), data
.data(), data
.size());
244 buf
.Size(data
.size());
246 PERF_TIMER_GUARD(encrypt_data_nanos
);
247 status
= stream_
->Encrypt(offset
, buf
.BufferStart(), buf
.CurrentSize());
252 dataToAppend
= Slice(buf
.BufferStart(), buf
.CurrentSize());
254 status
= file_
->Append(dataToAppend
);
261 Status
EncryptedWritableFile::PositionedAppend(const Slice
& data
,
265 Slice
dataToAppend(data
);
266 offset
+= prefixLength_
;
267 if (data
.size() > 0) {
268 // Encrypt in cloned buffer
269 buf
.Alignment(GetRequiredBufferAlignment());
270 buf
.AllocateNewBuffer(data
.size());
271 memmove(buf
.BufferStart(), data
.data(), data
.size());
272 buf
.Size(data
.size());
274 PERF_TIMER_GUARD(encrypt_data_nanos
);
275 status
= stream_
->Encrypt(offset
, buf
.BufferStart(), buf
.CurrentSize());
280 dataToAppend
= Slice(buf
.BufferStart(), buf
.CurrentSize());
282 status
= file_
->PositionedAppend(dataToAppend
, offset
);
289 // Indicates the upper layers if the current WritableFile implementation
291 bool EncryptedWritableFile::use_direct_io() const {
292 return file_
->use_direct_io();
295 // Use the returned alignment value to allocate
296 // aligned buffer for Direct I/O
297 size_t EncryptedWritableFile::GetRequiredBufferAlignment() const {
298 return file_
->GetRequiredBufferAlignment();
302 * Get the size of valid data in the file.
304 uint64_t EncryptedWritableFile::GetFileSize() {
305 return file_
->GetFileSize() - prefixLength_
;
308 // Truncate is necessary to trim the file to the correct size
309 // before closing. It is not always possible to keep track of the file
310 // size due to whole pages writes. The behavior is undefined if called
311 // with other writes to follow.
312 Status
EncryptedWritableFile::Truncate(uint64_t size
) {
313 return file_
->Truncate(size
+ prefixLength_
);
316 // Remove any kind of caching of data from the offset to offset+length
317 // of this file. If the length is 0, then it refers to the end of file.
318 // If the system is not caching the file contents, then this is a noop.
319 // This call has no effect on dirty pages in the cache.
320 Status
EncryptedWritableFile::InvalidateCache(size_t offset
, size_t length
) {
321 return file_
->InvalidateCache(offset
+ prefixLength_
, length
);
324 // Sync a file range with disk.
325 // offset is the starting byte of the file range to be synchronized.
326 // nbytes specifies the length of the range to be synchronized.
327 // This asks the OS to initiate flushing the cached data to disk,
328 // without waiting for completion.
329 // Default implementation does nothing.
330 Status
EncryptedWritableFile::RangeSync(uint64_t offset
, uint64_t nbytes
) {
331 return file_
->RangeSync(offset
+ prefixLength_
, nbytes
);
334 // PrepareWrite performs any necessary preparation for a write
335 // before the write actually occurs. This allows for pre-allocation
336 // of space on devices where it can result in less file
337 // fragmentation and/or less waste from over-zealous filesystem
339 void EncryptedWritableFile::PrepareWrite(size_t offset
, size_t len
) {
340 file_
->PrepareWrite(offset
+ prefixLength_
, len
);
343 // Pre-allocates space for a file.
344 Status
EncryptedWritableFile::Allocate(uint64_t offset
, uint64_t len
) {
345 return file_
->Allocate(offset
+ prefixLength_
, len
);
348 // A file abstraction for random reading and writing.
350 // Indicates if the class makes use of direct I/O
351 // If false you must pass aligned buffer to Write()
352 bool EncryptedRandomRWFile::use_direct_io() const {
353 return file_
->use_direct_io();
356 // Use the returned alignment value to allocate
357 // aligned buffer for Direct I/O
358 size_t EncryptedRandomRWFile::GetRequiredBufferAlignment() const {
359 return file_
->GetRequiredBufferAlignment();
362 // Write bytes in `data` at offset `offset`, Returns Status::OK() on success.
363 // Pass aligned buffer when use_direct_io() returns true.
364 Status
EncryptedRandomRWFile::Write(uint64_t offset
, const Slice
& data
) {
367 Slice
dataToWrite(data
);
368 offset
+= prefixLength_
;
369 if (data
.size() > 0) {
370 // Encrypt in cloned buffer
371 buf
.Alignment(GetRequiredBufferAlignment());
372 buf
.AllocateNewBuffer(data
.size());
373 memmove(buf
.BufferStart(), data
.data(), data
.size());
374 buf
.Size(data
.size());
376 PERF_TIMER_GUARD(encrypt_data_nanos
);
377 status
= stream_
->Encrypt(offset
, buf
.BufferStart(), buf
.CurrentSize());
382 dataToWrite
= Slice(buf
.BufferStart(), buf
.CurrentSize());
384 status
= file_
->Write(offset
, dataToWrite
);
388 // Read up to `n` bytes starting from offset `offset` and store them in
389 // result, provided `scratch` size should be at least `n`.
390 // Returns Status::OK() on success.
391 Status
EncryptedRandomRWFile::Read(uint64_t offset
, size_t n
, Slice
* result
,
392 char* scratch
) const {
394 offset
+= prefixLength_
;
395 auto status
= file_
->Read(offset
, n
, result
, scratch
);
400 PERF_TIMER_GUARD(decrypt_data_nanos
);
401 status
= stream_
->Decrypt(offset
, (char*)result
->data(), result
->size());
406 Status
EncryptedRandomRWFile::Flush() { return file_
->Flush(); }
408 Status
EncryptedRandomRWFile::Sync() { return file_
->Sync(); }
410 Status
EncryptedRandomRWFile::Fsync() { return file_
->Fsync(); }
412 Status
EncryptedRandomRWFile::Close() { return file_
->Close(); }
414 // EncryptedEnv implements an Env wrapper that adds encryption to files stored
416 class EncryptedEnvImpl
: public EnvWrapper
{
417 // Returns the raw encryption provider that should be used to write the input
418 // encrypted file. If there is no such provider, NotFound is returned.
419 Status
GetWritableProvider(const std::string
& /*fname*/,
420 EncryptionProvider
** result
) {
422 *result
= provider_
.get();
426 return Status::NotFound("No WriteProvider specified");
430 // Returns the raw encryption provider that should be used to read the input
431 // encrypted file. If there is no such provider, NotFound is returned.
432 Status
GetReadableProvider(const std::string
& /*fname*/,
433 EncryptionProvider
** result
) {
435 *result
= provider_
.get();
439 return Status::NotFound("No Provider specified");
443 // Creates a CipherStream for the underlying file/name using the options
444 // If a writable provider is found and encryption is enabled, uses
445 // this provider to create a cipher stream.
446 // @param fname Name of the writable file
447 // @param underlying The underlying "raw" file
448 // @param options Options for creating the file/cipher
449 // @param prefix_length Returns the length of the encryption prefix used for
451 // @param stream Returns the cipher stream to use for this file if it
452 // should be encrypted
453 // @return OK on success, non-OK on failure.
454 template <class TypeFile
>
455 Status
CreateWritableCipherStream(
456 const std::string
& fname
, const std::unique_ptr
<TypeFile
>& underlying
,
457 const EnvOptions
& options
, size_t* prefix_length
,
458 std::unique_ptr
<BlockAccessCipherStream
>* stream
) {
459 EncryptionProvider
* provider
= nullptr;
461 Status status
= GetWritableProvider(fname
, &provider
);
464 } else if (provider
!= nullptr) {
465 // Initialize & write prefix (if needed)
466 AlignedBuffer buffer
;
468 *prefix_length
= provider
->GetPrefixLength();
469 if (*prefix_length
> 0) {
471 buffer
.Alignment(underlying
->GetRequiredBufferAlignment());
472 buffer
.AllocateNewBuffer(*prefix_length
);
473 status
= provider
->CreateNewPrefix(fname
, buffer
.BufferStart(),
476 buffer
.Size(*prefix_length
);
477 prefix
= Slice(buffer
.BufferStart(), buffer
.CurrentSize());
479 status
= underlying
->Append(prefix
);
485 // Create cipher stream
486 status
= provider
->CreateCipherStream(fname
, options
, prefix
, stream
);
491 template <class TypeFile
>
492 Status
CreateWritableEncryptedFile(const std::string
& fname
,
493 std::unique_ptr
<TypeFile
>& underlying
,
494 const EnvOptions
& options
,
495 std::unique_ptr
<TypeFile
>* result
) {
496 // Create cipher stream
497 std::unique_ptr
<BlockAccessCipherStream
> stream
;
498 size_t prefix_length
;
499 Status status
= CreateWritableCipherStream(fname
, underlying
, options
,
500 &prefix_length
, &stream
);
503 result
->reset(new EncryptedWritableFile(
504 std::move(underlying
), std::move(stream
), prefix_length
));
506 result
->reset(underlying
.release());
512 // Creates a CipherStream for the underlying file/name using the options
513 // If a writable provider is found and encryption is enabled, uses
514 // this provider to create a cipher stream.
515 // @param fname Name of the writable file
516 // @param underlying The underlying "raw" file
517 // @param options Options for creating the file/cipher
518 // @param prefix_length Returns the length of the encryption prefix used for
520 // @param stream Returns the cipher stream to use for this file if it
521 // should be encrypted
522 // @return OK on success, non-OK on failure.
523 template <class TypeFile
>
524 Status
CreateRandomWriteCipherStream(
525 const std::string
& fname
, const std::unique_ptr
<TypeFile
>& underlying
,
526 const EnvOptions
& options
, size_t* prefix_length
,
527 std::unique_ptr
<BlockAccessCipherStream
>* stream
) {
528 EncryptionProvider
* provider
= nullptr;
530 Status status
= GetWritableProvider(fname
, &provider
);
533 } else if (provider
!= nullptr) {
534 // Initialize & write prefix (if needed)
535 AlignedBuffer buffer
;
537 *prefix_length
= provider
->GetPrefixLength();
538 if (*prefix_length
> 0) {
540 buffer
.Alignment(underlying
->GetRequiredBufferAlignment());
541 buffer
.AllocateNewBuffer(*prefix_length
);
542 status
= provider
->CreateNewPrefix(fname
, buffer
.BufferStart(),
545 buffer
.Size(*prefix_length
);
546 prefix
= Slice(buffer
.BufferStart(), buffer
.CurrentSize());
548 status
= underlying
->Write(0, prefix
);
554 // Create cipher stream
555 status
= provider
->CreateCipherStream(fname
, options
, prefix
, stream
);
560 // Creates a CipherStream for the underlying file/name using the options
561 // If a readable provider is found and the file is encrypted, uses
562 // this provider to create a cipher stream.
563 // @param fname Name of the writable file
564 // @param underlying The underlying "raw" file
565 // @param options Options for creating the file/cipher
566 // @param prefix_length Returns the length of the encryption prefix used for
568 // @param stream Returns the cipher stream to use for this file if it
570 // @return OK on success, non-OK on failure.
571 template <class TypeFile
>
572 Status
CreateSequentialCipherStream(
573 const std::string
& fname
, const std::unique_ptr
<TypeFile
>& underlying
,
574 const EnvOptions
& options
, size_t* prefix_length
,
575 std::unique_ptr
<BlockAccessCipherStream
>* stream
) {
576 // Read prefix (if needed)
577 AlignedBuffer buffer
;
579 *prefix_length
= provider_
->GetPrefixLength();
580 if (*prefix_length
> 0) {
582 buffer
.Alignment(underlying
->GetRequiredBufferAlignment());
583 buffer
.AllocateNewBuffer(*prefix_length
);
585 underlying
->Read(*prefix_length
, &prefix
, buffer
.BufferStart());
589 buffer
.Size(*prefix_length
);
591 return provider_
->CreateCipherStream(fname
, options
, prefix
, stream
);
594 // Creates a CipherStream for the underlying file/name using the options
595 // If a readable provider is found and the file is encrypted, uses
596 // this provider to create a cipher stream.
597 // @param fname Name of the writable file
598 // @param underlying The underlying "raw" file
599 // @param options Options for creating the file/cipher
600 // @param prefix_length Returns the length of the encryption prefix used for
602 // @param stream Returns the cipher stream to use for this file if it
604 // @return OK on success, non-OK on failure.
605 template <class TypeFile
>
606 Status
CreateRandomReadCipherStream(
607 const std::string
& fname
, const std::unique_ptr
<TypeFile
>& underlying
,
608 const EnvOptions
& options
, size_t* prefix_length
,
609 std::unique_ptr
<BlockAccessCipherStream
>* stream
) {
610 // Read prefix (if needed)
611 AlignedBuffer buffer
;
613 *prefix_length
= provider_
->GetPrefixLength();
614 if (*prefix_length
> 0) {
616 buffer
.Alignment(underlying
->GetRequiredBufferAlignment());
617 buffer
.AllocateNewBuffer(*prefix_length
);
619 underlying
->Read(0, *prefix_length
, &prefix
, buffer
.BufferStart());
623 buffer
.Size(*prefix_length
);
625 return provider_
->CreateCipherStream(fname
, options
, prefix
, stream
);
629 EncryptedEnvImpl(Env
* base_env
,
630 const std::shared_ptr
<EncryptionProvider
>& provider
)
631 : EnvWrapper(base_env
) {
632 provider_
= provider
;
635 // NewSequentialFile opens a file for sequential reading.
636 virtual Status
NewSequentialFile(const std::string
& fname
,
637 std::unique_ptr
<SequentialFile
>* result
,
638 const EnvOptions
& options
) override
{
640 if (options
.use_mmap_reads
) {
641 return Status::InvalidArgument();
643 // Open file using underlying Env implementation
644 std::unique_ptr
<SequentialFile
> underlying
;
645 auto status
= EnvWrapper::NewSequentialFile(fname
, &underlying
, options
);
649 // Create cipher stream
650 std::unique_ptr
<BlockAccessCipherStream
> stream
;
651 size_t prefix_length
;
652 status
= CreateSequentialCipherStream(fname
, underlying
, options
,
653 &prefix_length
, &stream
);
655 result
->reset(new EncryptedSequentialFile(
656 std::move(underlying
), std::move(stream
), prefix_length
));
661 // NewRandomAccessFile opens a file for random read access.
662 virtual Status
NewRandomAccessFile(const std::string
& fname
,
663 std::unique_ptr
<RandomAccessFile
>* result
,
664 const EnvOptions
& options
) override
{
666 if (options
.use_mmap_reads
) {
667 return Status::InvalidArgument();
669 // Open file using underlying Env implementation
670 std::unique_ptr
<RandomAccessFile
> underlying
;
671 auto status
= EnvWrapper::NewRandomAccessFile(fname
, &underlying
, options
);
675 std::unique_ptr
<BlockAccessCipherStream
> stream
;
676 size_t prefix_length
;
677 status
= CreateRandomReadCipherStream(fname
, underlying
, options
,
678 &prefix_length
, &stream
);
681 result
->reset(new EncryptedRandomAccessFile(
682 std::move(underlying
), std::move(stream
), prefix_length
));
684 result
->reset(underlying
.release());
690 // NewWritableFile opens a file for sequential writing.
691 virtual Status
NewWritableFile(const std::string
& fname
,
692 std::unique_ptr
<WritableFile
>* result
,
693 const EnvOptions
& options
) override
{
695 if (options
.use_mmap_writes
) {
696 return Status::InvalidArgument();
698 // Open file using underlying Env implementation
699 std::unique_ptr
<WritableFile
> underlying
;
700 Status status
= EnvWrapper::NewWritableFile(fname
, &underlying
, options
);
704 return CreateWritableEncryptedFile(fname
, underlying
, options
, result
);
707 // Create an object that writes to a new file with the specified
708 // name. Deletes any existing file with the same name and creates a
709 // new file. On success, stores a pointer to the new file in
710 // *result and returns OK. On failure stores nullptr in *result and
713 // The returned file will only be accessed by one thread at a time.
714 virtual Status
ReopenWritableFile(const std::string
& fname
,
715 std::unique_ptr
<WritableFile
>* result
,
716 const EnvOptions
& options
) override
{
718 if (options
.use_mmap_writes
) {
719 return Status::InvalidArgument();
721 // Open file using underlying Env implementation
722 std::unique_ptr
<WritableFile
> underlying
;
723 Status status
= EnvWrapper::ReopenWritableFile(fname
, &underlying
, options
);
727 return CreateWritableEncryptedFile(fname
, underlying
, options
, result
);
730 // Reuse an existing file by renaming it and opening it as writable.
731 virtual Status
ReuseWritableFile(const std::string
& fname
,
732 const std::string
& old_fname
,
733 std::unique_ptr
<WritableFile
>* result
,
734 const EnvOptions
& options
) override
{
736 if (options
.use_mmap_writes
) {
737 return Status::InvalidArgument();
739 // Open file using underlying Env implementation
740 std::unique_ptr
<WritableFile
> underlying
;
742 EnvWrapper::ReuseWritableFile(fname
, old_fname
, &underlying
, options
);
746 return CreateWritableEncryptedFile(fname
, underlying
, options
, result
);
749 // Open `fname` for random read and write, if file doesn't exist the file
750 // will be created. On success, stores a pointer to the new file in
751 // *result and returns OK. On failure returns non-OK.
753 // The returned file will only be accessed by one thread at a time.
754 virtual Status
NewRandomRWFile(const std::string
& fname
,
755 std::unique_ptr
<RandomRWFile
>* result
,
756 const EnvOptions
& options
) override
{
758 if (options
.use_mmap_reads
|| options
.use_mmap_writes
) {
759 return Status::InvalidArgument();
762 bool isNewFile
= !FileExists(fname
).ok();
764 // Open file using underlying Env implementation
765 std::unique_ptr
<RandomRWFile
> underlying
;
766 Status status
= EnvWrapper::NewRandomRWFile(fname
, &underlying
, options
);
770 // Create cipher stream
771 std::unique_ptr
<BlockAccessCipherStream
> stream
;
772 size_t prefix_length
= 0;
774 // File already exists, read prefix
775 status
= CreateRandomReadCipherStream(fname
, underlying
, options
,
776 &prefix_length
, &stream
);
778 status
= CreateRandomWriteCipherStream(fname
, underlying
, options
,
779 &prefix_length
, &stream
);
783 result
->reset(new EncryptedRandomRWFile(
784 std::move(underlying
), std::move(stream
), prefix_length
));
786 result
->reset(underlying
.release());
792 // Store in *result the attributes of the children of the specified
794 // In case the implementation lists the directory prior to iterating the
796 // and files are concurrently deleted, the deleted files will be omitted
799 // The name attributes are relative to "dir".
800 // Original contents of *results are dropped.
801 // Returns OK if "dir" exists and "*result" contains its children.
802 // NotFound if "dir" does not exist, the calling process does not
804 // permission to access "dir", or if "dir" is invalid.
805 // IOError if an IO Error was encountered
806 virtual Status
GetChildrenFileAttributes(
807 const std::string
& dir
, std::vector
<FileAttributes
>* result
) override
{
808 auto status
= EnvWrapper::GetChildrenFileAttributes(dir
, result
);
812 for (auto it
= std::begin(*result
); it
!= std::end(*result
); ++it
) {
813 // assert(it->size_bytes >= prefixLength);
814 // breaks env_basic_test when called on directory containing
816 // which makes subtraction of prefixLength worrisome since
817 // FileAttributes does not identify directories
818 EncryptionProvider
* provider
;
819 status
= GetReadableProvider(it
->name
, &provider
);
822 } else if (provider
!= nullptr) {
823 it
->size_bytes
-= provider
->GetPrefixLength();
829 // Store the size of fname in *file_size.
830 virtual Status
GetFileSize(const std::string
& fname
,
831 uint64_t* file_size
) override
{
832 auto status
= EnvWrapper::GetFileSize(fname
, file_size
);
836 EncryptionProvider
* provider
;
837 status
= GetReadableProvider(fname
, &provider
);
838 if (provider
!= nullptr && status
.ok()) {
839 size_t prefixLength
= provider
->GetPrefixLength();
840 assert(*file_size
>= prefixLength
);
841 *file_size
-= prefixLength
;
847 std::shared_ptr
<EncryptionProvider
> provider_
;
850 // Returns an Env that encrypts data when stored on disk and decrypts data when
852 Env
* NewEncryptedEnv(Env
* base_env
,
853 const std::shared_ptr
<EncryptionProvider
>& provider
) {
854 return new EncryptedEnvImpl(base_env
, provider
);
857 // Encrypt one or more (partial) blocks of data at the file offset.
858 // Length of data is given in dataSize.
859 Status
BlockAccessCipherStream::Encrypt(uint64_t fileOffset
, char *data
, size_t dataSize
) {
860 // Calculate block index
861 auto blockSize
= BlockSize();
862 uint64_t blockIndex
= fileOffset
/ blockSize
;
863 size_t blockOffset
= fileOffset
% blockSize
;
864 std::unique_ptr
<char[]> blockBuffer
;
867 AllocateScratch(scratch
);
869 // Encrypt individual blocks.
872 size_t n
= std::min(dataSize
, blockSize
- blockOffset
);
873 if (n
!= blockSize
) {
874 // We're not encrypting a full block.
875 // Copy data to blockBuffer
876 if (!blockBuffer
.get()) {
878 blockBuffer
= std::unique_ptr
<char[]>(new char[blockSize
]);
880 block
= blockBuffer
.get();
881 // Copy plain data to block buffer
882 memmove(block
+ blockOffset
, data
, n
);
884 auto status
= EncryptBlock(blockIndex
, block
, (char*)scratch
.data());
889 // Copy encrypted data back to `data`.
890 memmove(data
, block
+ blockOffset
, n
);
902 // Decrypt one or more (partial) blocks of data at the file offset.
903 // Length of data is given in dataSize.
904 Status
BlockAccessCipherStream::Decrypt(uint64_t fileOffset
, char *data
, size_t dataSize
) {
905 // Calculate block index
906 auto blockSize
= BlockSize();
907 uint64_t blockIndex
= fileOffset
/ blockSize
;
908 size_t blockOffset
= fileOffset
% blockSize
;
909 std::unique_ptr
<char[]> blockBuffer
;
912 AllocateScratch(scratch
);
914 // Decrypt individual blocks.
917 size_t n
= std::min(dataSize
, blockSize
- blockOffset
);
918 if (n
!= blockSize
) {
919 // We're not decrypting a full block.
920 // Copy data to blockBuffer
921 if (!blockBuffer
.get()) {
923 blockBuffer
= std::unique_ptr
<char[]>(new char[blockSize
]);
925 block
= blockBuffer
.get();
926 // Copy encrypted data to block buffer
927 memmove(block
+ blockOffset
, data
, n
);
929 auto status
= DecryptBlock(blockIndex
, block
, (char*)scratch
.data());
934 // Copy decrypted data back to `data`.
935 memmove(data
, block
+ blockOffset
, n
);
938 // Simply decrementing dataSize by n could cause it to underflow,
939 // which will very likely make it read over the original bounds later
940 assert(dataSize
>= n
);
942 return Status::Corruption("Cannot decrypt data at given offset");
955 const char* ROT13BlockCipher::Name() const { return kROT13CipherName
; }
957 // Encrypt a block of data.
958 // Length of data is equal to BlockSize().
959 Status
ROT13BlockCipher::Encrypt(char* data
) {
960 for (size_t i
= 0; i
< blockSize_
; ++i
) {
966 // Decrypt a block of data.
967 // Length of data is equal to BlockSize().
968 Status
ROT13BlockCipher::Decrypt(char* data
) { return Encrypt(data
); }
970 // Allocate scratch space which is passed to EncryptBlock/DecryptBlock.
971 void CTRCipherStream::AllocateScratch(std::string
& scratch
) {
972 auto blockSize
= cipher_
->BlockSize();
973 scratch
.reserve(blockSize
);
976 // Encrypt a block of data at the given block index.
977 // Length of data is equal to BlockSize();
978 Status
CTRCipherStream::EncryptBlock(uint64_t blockIndex
, char* data
,
980 // Create nonce + counter
981 auto blockSize
= cipher_
->BlockSize();
982 memmove(scratch
, iv_
.data(), blockSize
);
983 EncodeFixed64(scratch
, blockIndex
+ initialCounter_
);
985 // Encrypt nonce+counter
986 auto status
= cipher_
->Encrypt(scratch
);
991 // XOR data with ciphertext.
992 for (size_t i
= 0; i
< blockSize
; i
++) {
993 data
[i
] = data
[i
] ^ scratch
[i
];
998 // Decrypt a block of data at the given block index.
999 // Length of data is equal to BlockSize();
1000 Status
CTRCipherStream::DecryptBlock(uint64_t blockIndex
, char* data
,
1002 // For CTR decryption & encryption are the same
1003 return EncryptBlock(blockIndex
, data
, scratch
);
1006 const char* CTREncryptionProvider::Name() const { return kCTRProviderName
; }
1008 // GetPrefixLength returns the length of the prefix that is added to every file
1009 // and used for storing encryption options.
1010 // For optimal performance, the prefix length should be a multiple of
1012 size_t CTREncryptionProvider::GetPrefixLength() const {
1013 return defaultPrefixLength
;
1016 Status
CTREncryptionProvider::TEST_Initialize() {
1018 return BlockCipher::CreateFromString(
1019 ConfigOptions(), std::string(kROT13CipherName
) + ":32", &cipher_
);
1021 return Status::OK();
1024 Status
CTREncryptionProvider::AddCipher(const std::string
& /*descriptor*/,
1025 const char* cipher
, size_t len
,
1026 bool /*for_write*/) {
1028 return Status::NotSupported("Cannot add keys to CTREncryptionProvider");
1029 } else if (strcmp(kROT13CipherName
, cipher
) == 0) {
1030 cipher_
.reset(new ROT13BlockCipher(len
));
1031 return Status::OK();
1033 return BlockCipher::CreateFromString(ConfigOptions(), std::string(cipher
),
1038 // decodeCTRParameters decodes the initial counter & IV from the given
1039 // (plain text) prefix.
1040 static void decodeCTRParameters(const char* prefix
, size_t blockSize
,
1041 uint64_t& initialCounter
, Slice
& iv
) {
1042 // First block contains 64-bit initial counter
1043 initialCounter
= DecodeFixed64(prefix
);
1044 // Second block contains IV
1045 iv
= Slice(prefix
+ blockSize
, blockSize
);
1048 // CreateNewPrefix initialized an allocated block of prefix memory
1050 Status
CTREncryptionProvider::CreateNewPrefix(const std::string
& /*fname*/,
1052 size_t prefixLength
) const {
1054 return Status::InvalidArgument("Encryption Cipher is missing");
1056 // Create & seed rnd.
1057 Random
rnd((uint32_t)Env::Default()->NowMicros());
1058 // Fill entire prefix block with random values.
1059 for (size_t i
= 0; i
< prefixLength
; i
++) {
1060 prefix
[i
] = rnd
.Uniform(256) & 0xFF;
1062 // Take random data to extract initial counter & IV
1063 auto blockSize
= cipher_
->BlockSize();
1064 uint64_t initialCounter
;
1066 decodeCTRParameters(prefix
, blockSize
, initialCounter
, prefixIV
);
1068 // Now populate the rest of the prefix, starting from the third block.
1069 PopulateSecretPrefixPart(prefix
+ (2 * blockSize
),
1070 prefixLength
- (2 * blockSize
), blockSize
);
1072 // Encrypt the prefix, starting from block 2 (leave block 0, 1 with initial
1073 // counter & IV unencrypted)
1074 CTRCipherStream
cipherStream(cipher_
, prefixIV
.data(), initialCounter
);
1077 PERF_TIMER_GUARD(encrypt_data_nanos
);
1078 status
= cipherStream
.Encrypt(0, prefix
+ (2 * blockSize
),
1079 prefixLength
- (2 * blockSize
));
1084 return Status::OK();
1087 // PopulateSecretPrefixPart initializes the data into a new prefix block
1089 // Returns the amount of space (starting from the start of the prefix)
1090 // that has been initialized.
1091 size_t CTREncryptionProvider::PopulateSecretPrefixPart(
1092 char* /*prefix*/, size_t /*prefixLength*/, size_t /*blockSize*/) const {
1093 // Nothing to do here, put in custom data in override when needed.
1097 Status
CTREncryptionProvider::CreateCipherStream(
1098 const std::string
& fname
, const EnvOptions
& options
, Slice
& prefix
,
1099 std::unique_ptr
<BlockAccessCipherStream
>* result
) {
1101 return Status::InvalidArgument("Encryption Cipher is missing");
1103 // Read plain text part of prefix.
1104 auto blockSize
= cipher_
->BlockSize();
1105 uint64_t initialCounter
;
1107 decodeCTRParameters(prefix
.data(), blockSize
, initialCounter
, iv
);
1109 // If the prefix is smaller than twice the block size, we would below read a
1110 // very large chunk of the file (and very likely read over the bounds)
1111 assert(prefix
.size() >= 2 * blockSize
);
1112 if (prefix
.size() < 2 * blockSize
) {
1113 return Status::Corruption("Unable to read from file " + fname
+
1114 ": read attempt would read beyond file bounds");
1117 // Decrypt the encrypted part of the prefix, starting from block 2 (block 0, 1
1118 // with initial counter & IV are unencrypted)
1119 CTRCipherStream
cipherStream(cipher_
, iv
.data(), initialCounter
);
1122 PERF_TIMER_GUARD(decrypt_data_nanos
);
1123 status
= cipherStream
.Decrypt(0, (char*)prefix
.data() + (2 * blockSize
),
1124 prefix
.size() - (2 * blockSize
));
1130 // Create cipher stream
1131 return CreateCipherStreamFromPrefix(fname
, options
, initialCounter
, iv
,
1135 // CreateCipherStreamFromPrefix creates a block access cipher stream for a file
1136 // given given name and options. The given prefix is already decrypted.
1137 Status
CTREncryptionProvider::CreateCipherStreamFromPrefix(
1138 const std::string
& /*fname*/, const EnvOptions
& /*options*/,
1139 uint64_t initialCounter
, const Slice
& iv
, const Slice
& /*prefix*/,
1140 std::unique_ptr
<BlockAccessCipherStream
>* result
) {
1141 (*result
) = std::unique_ptr
<BlockAccessCipherStream
>(
1142 new CTRCipherStream(cipher_
, iv
.data(), initialCounter
));
1143 return Status::OK();
1146 #endif // ROCKSDB_LITE
1148 } // namespace ROCKSDB_NAMESPACE