Keith M Wesolowski <wesolows@foobazco.org>
Kevin Tanguy <kevin.tanguy@ovh.net>
KireinaHoro <i@jsteward.moe>
+ Kjeld Schouten-Lebbing <kjeld@schouten-lebbing.nl>
Kohsuke Kawaguchi <kk@kohsuke.org>
Kyle Blatter <kyleblatter@llnl.gov>
Kyle Fuller <inbox@kylefuller.co.uk>
Michael Gebetsroither <michael@mgeb.org>
Michael Kjorling <michael@kjorling.se>
Michael Martin <mgmartin.mgm@gmail.com>
+ Michael Niewöhner <foss@mniewoehner.de>
Mike Gerdts <mike.gerdts@joyent.com>
Mike Harsch <mike@harschsystems.com>
Mike Leddy <mike.leddy@gmail.com>
Saso Kiselkov <saso.kiselkov@nexenta.com>
Scot W. Stevenson <scot.stevenson@gmail.com>
Sean Eric Fagan <sef@ixsystems.com>
+ Sebastian Gottschall <s.gottschall@dd-wrt.com>
Sen Haerens <sen@senhaerens.be>
Serapheim Dimitropoulos <serapheim@delphix.com>
Seth Forshee <seth.forshee@canonical.com>
"ZIO_COMPRESS_GZIP_6", "ZIO_COMPRESS_GZIP_7",
"ZIO_COMPRESS_GZIP_8", "ZIO_COMPRESS_GZIP_9",
"ZIO_COMPRESS_ZLE", "ZIO_COMPRESS_LZ4",
- "ZIO_COMPRESS_FUNCTION"]
+ "ZIO_COMPRESS_ZSTD", "ZIO_COMPRESS_FUNCTION"]
# If "-rr" option is used, don't convert to string representation
if raw > 1:
* Copyright (c) 2017, 2018 Lawrence Livermore National Security, LLC.
* Copyright (c) 2015, 2017, Intel Corporation.
* Copyright (c) 2020 Datto Inc.
+ * Copyright (c) 2020, The FreeBSD Foundation [1]
+ *
+ * [1] Portions of this software were developed by Allan Jude
+ * under sponsorship from the FreeBSD Foundation.
*/
#include <stdio.h>
#include <sys/dsl_scan.h>
#include <sys/btree.h>
#include <zfs_comutil.h>
+#include <sys/zstd/zstd.h>
#include <libnvpair.h>
#include <libzutil.h>
"work with dataset)\n");
(void) fprintf(stderr, " -Y attempt all reconstruction "
"combinations for split blocks\n");
+ (void) fprintf(stderr, " -Z show ZSTD headers \n");
(void) fprintf(stderr, "Specify an option more than once (e.g. -bb) "
"to make only that option verbose\n");
(void) fprintf(stderr, "Default is to dump everything non-verbosely\n");
dnp->dn_datablkszsec << SPA_MINBLOCKSHIFT);
}
+static void
+snprintf_zstd_header(spa_t *spa, char *blkbuf, size_t buflen,
+ const blkptr_t *bp)
+{
+ abd_t *pabd;
+ void *buf;
+ zio_t *zio;
+ zfs_zstdhdr_t zstd_hdr;
+ int error;
+
+ if (BP_GET_COMPRESS(bp) != ZIO_COMPRESS_ZSTD)
+ return;
+
+ if (BP_IS_HOLE(bp))
+ return;
+
+ if (BP_IS_EMBEDDED(bp)) {
+ buf = malloc(SPA_MAXBLOCKSIZE);
+ if (buf == NULL) {
+ (void) fprintf(stderr, "out of memory\n");
+ exit(1);
+ }
+ decode_embedded_bp_compressed(bp, buf);
+ memcpy(&zstd_hdr, buf, sizeof (zstd_hdr));
+ free(buf);
+ zstd_hdr.c_len = BE_32(zstd_hdr.c_len);
+ zstd_hdr.raw_version_level = BE_32(zstd_hdr.raw_version_level);
+ (void) snprintf(blkbuf + strlen(blkbuf),
+ buflen - strlen(blkbuf),
+ " ZSTD:size=%u:version=%u:level=%u:EMBEDDED",
+ zstd_hdr.c_len, zstd_hdr.version, zstd_hdr.level);
+ return;
+ }
+
+ pabd = abd_alloc_for_io(SPA_MAXBLOCKSIZE, B_FALSE);
+ zio = zio_root(spa, NULL, NULL, 0);
+
+ /* Decrypt but don't decompress so we can read the compression header */
+ zio_nowait(zio_read(zio, spa, bp, pabd, BP_GET_PSIZE(bp), NULL, NULL,
+ ZIO_PRIORITY_SYNC_READ, ZIO_FLAG_CANFAIL | ZIO_FLAG_RAW_COMPRESS,
+ NULL));
+ error = zio_wait(zio);
+ if (error) {
+ (void) fprintf(stderr, "read failed: %d\n", error);
+ return;
+ }
+ buf = abd_borrow_buf_copy(pabd, BP_GET_LSIZE(bp));
+ memcpy(&zstd_hdr, buf, sizeof (zstd_hdr));
+ zstd_hdr.c_len = BE_32(zstd_hdr.c_len);
+ zstd_hdr.raw_version_level = BE_32(zstd_hdr.raw_version_level);
+
+ (void) snprintf(blkbuf + strlen(blkbuf),
+ buflen - strlen(blkbuf),
+ " ZSTD:size=%u:version=%u:level=%u:NORMAL",
+ zstd_hdr.c_len, zstd_hdr.version, zstd_hdr.level);
+
+ abd_return_buf_copy(pabd, buf, BP_GET_LSIZE(bp));
+}
+
static void
snprintf_blkptr_compact(char *blkbuf, size_t buflen, const blkptr_t *bp,
boolean_t bp_freed)
}
static void
-print_indirect(blkptr_t *bp, const zbookmark_phys_t *zb,
+print_indirect(spa_t *spa, blkptr_t *bp, const zbookmark_phys_t *zb,
const dnode_phys_t *dnp)
{
char blkbuf[BP_SPRINTF_LEN];
}
snprintf_blkptr_compact(blkbuf, sizeof (blkbuf), bp, B_FALSE);
+ if (dump_opt['Z'] && BP_GET_COMPRESS(bp) == ZIO_COMPRESS_ZSTD)
+ snprintf_zstd_header(spa, blkbuf, sizeof (blkbuf), bp);
(void) printf("%s\n", blkbuf);
}
if (bp->blk_birth == 0)
return (0);
- print_indirect(bp, zb, dnp);
+ print_indirect(spa, bp, zb, dnp);
if (BP_GET_LEVEL(bp) > 0 && !BP_IS_HOLE(bp)) {
arc_flags_t flags = ARC_FLAG_WAIT;
" (K=%s)", ZDB_CHECKSUM_NAME(doi.doi_checksum));
}
- if (doi.doi_compress != ZIO_COMPRESS_INHERIT || verbosity >= 6) {
+ if (doi.doi_compress == ZIO_COMPRESS_INHERIT &&
+ ZIO_COMPRESS_HASLEVEL(os->os_compress) && verbosity >= 6) {
+ const char *compname = NULL;
+ if (zfs_prop_index_to_string(ZFS_PROP_COMPRESSION,
+ ZIO_COMPRESS_RAW(os->os_compress, os->os_complevel),
+ &compname) == 0) {
+ (void) snprintf(aux + strlen(aux),
+ sizeof (aux) - strlen(aux), " (Z=inherit=%s)",
+ compname);
+ } else {
+ (void) snprintf(aux + strlen(aux),
+ sizeof (aux) - strlen(aux),
+ " (Z=inherit=%s-unknown)",
+ ZDB_COMPRESS_NAME(os->os_compress));
+ }
+ } else if (doi.doi_compress == ZIO_COMPRESS_INHERIT && verbosity >= 6) {
+ (void) snprintf(aux + strlen(aux), sizeof (aux) - strlen(aux),
+ " (Z=inherit=%s)", ZDB_COMPRESS_NAME(os->os_compress));
+ } else if (doi.doi_compress != ZIO_COMPRESS_INHERIT || verbosity >= 6) {
(void) snprintf(aux + strlen(aux), sizeof (aux) - strlen(aux),
" (Z=%s)", ZDB_COMPRESS_NAME(doi.doi_compress));
}
(u_longlong_t)L2BLK_GET_PSIZE((&le[j])->le_prop));
(void) printf("|\t\t\t\tcompr: %llu\n",
(u_longlong_t)L2BLK_GET_COMPRESS((&le[j])->le_prop));
+ (void) printf("|\t\t\t\tcomplevel: %llu\n",
+ (u_longlong_t)(&le[j])->le_complevel);
(void) printf("|\t\t\t\ttype: %llu\n",
(u_longlong_t)L2BLK_GET_TYPE((&le[j])->le_prop));
(void) printf("|\t\t\t\tprotected: %llu\n",
switch (L2BLK_GET_COMPRESS((&lbps[0])->lbp_prop)) {
case ZIO_COMPRESS_OFF:
break;
- case ZIO_COMPRESS_LZ4:
+ default:
abd = abd_alloc_for_io(asize, B_TRUE);
abd_copy_from_buf_off(abd, &this_lb, 0, asize);
zio_decompress_data(L2BLK_GET_COMPRESS(
(&lbps[0])->lbp_prop), abd, &this_lb,
- asize, sizeof (this_lb));
+ asize, sizeof (this_lb), NULL);
abd_free(abd);
break;
- default:
- break;
}
if (this_lb.lb_magic == BSWAP_64(L2ARC_LOG_BLK_MAGIC))
VERIFY0(random_get_pseudo_bytes(lbuf2, lsize));
if (zio_decompress_data(*cfuncp, pabd,
- lbuf, psize, lsize) == 0 &&
+ lbuf, psize, lsize, NULL) == 0 &&
zio_decompress_data(*cfuncp, pabd,
- lbuf2, psize, lsize) == 0 &&
+ lbuf2, psize, lsize, NULL) == 0 &&
bcmp(lbuf, lbuf2, lsize) == 0)
break;
}
zfs_btree_verify_intensity = 3;
while ((c = getopt(argc, argv,
- "AbcCdDeEFGhiI:klLmMo:Op:PqRsSt:uU:vVx:XYy")) != -1) {
+ "AbcCdDeEFGhiI:klLmMo:Op:PqRsSt:uU:vVx:XYyZ")) != -1) {
switch (c) {
case 'b':
case 'c':
case 'S':
case 'u':
case 'y':
+ case 'Z':
dump_opt[c]++;
dump_all = 0;
break;
include/sys/fs/Makefile
include/sys/lua/Makefile
include/sys/sysevent/Makefile
+ include/sys/zstd/Makefile
lib/Makefile
lib/libavl/Makefile
lib/libefi/Makefile
lib/libzfs_core/Makefile
lib/libzfs_core/libzfs_core.pc
lib/libzpool/Makefile
+ lib/libzstd/Makefile
lib/libzutil/Makefile
man/Makefile
man/man1/Makefile
module/unicode/Makefile
module/zcommon/Makefile
module/zfs/Makefile
+ module/zstd/Makefile
rpm/Makefile
rpm/generic/Makefile
rpm/generic/zfs-dkms.spec
-SUBDIRS = fm fs crypto lua sysevent
+SUBDIRS = fm fs crypto lua sysevent zstd
COMMON_H = \
abd.h \
* Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2012, 2016 by Delphix. All rights reserved.
* Copyright (c) 2013 by Saso Kiselkov. All rights reserved.
+ * Copyright (c) 2019, Allan Jude
+ * Copyright (c) 2019, Klara Inc.
*/
#ifndef _SYS_ARC_H
arc_buf_t *arc_alloc_buf(spa_t *spa, void *tag, arc_buf_contents_t type,
int32_t size);
arc_buf_t *arc_alloc_compressed_buf(spa_t *spa, void *tag,
- uint64_t psize, uint64_t lsize, enum zio_compress compression_type);
+ uint64_t psize, uint64_t lsize, enum zio_compress compression_type,
+ uint8_t complevel);
arc_buf_t *arc_alloc_raw_buf(spa_t *spa, void *tag, uint64_t dsobj,
boolean_t byteorder, const uint8_t *salt, const uint8_t *iv,
const uint8_t *mac, dmu_object_type_t ot, uint64_t psize, uint64_t lsize,
- enum zio_compress compression_type);
+ enum zio_compress compression_type, uint8_t complevel);
+uint8_t arc_get_complevel(arc_buf_t *buf);
arc_buf_t *arc_loan_buf(spa_t *spa, boolean_t is_metadata, int size);
arc_buf_t *arc_loan_compressed_buf(spa_t *spa, uint64_t psize, uint64_t lsize,
- enum zio_compress compression_type);
+ enum zio_compress compression_type, uint8_t complevel);
arc_buf_t *arc_loan_raw_buf(spa_t *spa, uint64_t dsobj, boolean_t byteorder,
const uint8_t *salt, const uint8_t *iv, const uint8_t *mac,
dmu_object_type_t ot, uint64_t psize, uint64_t lsize,
- enum zio_compress compression_type);
+ enum zio_compress compression_type, uint8_t complevel);
void arc_return_buf(arc_buf_t *buf, void *tag);
void arc_loan_inuse_buf(arc_buf_t *buf, void *tag);
void arc_buf_destroy(arc_buf_t *buf, void *tag);
*/
uint64_t le_prop;
uint64_t le_daddr; /* buf location on l2dev */
+ uint64_t le_complevel;
/*
* We pad the size of each entry to a power of 2 so that the size of
* l2arc_log_blk_phys_t is power-of-2 aligned with SPA_MINBLOCKSHIFT,
* because of the L2ARC_SET_*SIZE macros.
*/
- const uint64_t le_pad[3]; /* pad to 64 bytes */
+ const uint64_t le_pad[2]; /* pad to 64 bytes */
} l2arc_log_ent_phys_t;
#define L2ARC_LOG_BLK_MAX_ENTRIES (1022)
uint64_t b_birth;
arc_buf_contents_t b_type;
+ uint8_t b_complevel;
+ uint8_t b_reserved1; /* used for 4 byte alignment */
+ uint16_t b_reserved2; /* used for 4 byte alignment */
arc_buf_hdr_t *b_hash_next;
arc_flags_t b_flags;
uint64_t os_dnodesize; /* default dnode size for new objects */
enum zio_checksum os_checksum;
enum zio_compress os_compress;
+ uint8_t os_complevel;
uint8_t os_copies;
enum zio_checksum os_dedup_checksum;
boolean_t os_dedup_verify;
uint64_t quota);
int dsl_dataset_set_refreservation(const char *dsname, zprop_source_t source,
uint64_t reservation);
+int dsl_dataset_set_compression(const char *dsname, zprop_source_t source,
+ uint64_t compression);
boolean_t dsl_dataset_is_before(dsl_dataset_t *later, dsl_dataset_t *earlier,
uint64_t earlier_txg);
* Copyright 2017 Joyent, Inc.
* Copyright (c) 2017, 2019, Datto Inc. All rights reserved.
* Copyright (c) 2017, Intel Corporation.
+ * Copyright (c) 2019, Allan Jude
+ * Copyright (c) 2019, Klara Inc.
*/
#ifndef _SYS_SPA_H
#define SPA_COMPRESSBITS 7
#define SPA_VDEVBITS 24
+#define SPA_COMPRESSMASK ((1U << SPA_COMPRESSBITS) - 1)
/*
* All SPA data is represented by 128-bit data virtual addresses (DVAs).
#define ____cacheline_aligned
+/*
+ * Kernel modules
+ */
+#define __init
+#define __exit
+
#endif /* _KERNEL */
#ifdef __cplusplus
#define DMU_BACKUP_FEATURE_COMPRESSED (1 << 22)
#define DMU_BACKUP_FEATURE_LARGE_DNODE (1 << 23)
#define DMU_BACKUP_FEATURE_RAW (1 << 24)
-/* flag #25 is reserved for the ZSTD compression feature */
+#define DMU_BACKUP_FEATURE_ZSTD (1 << 25)
#define DMU_BACKUP_FEATURE_HOLDS (1 << 26)
/*
* The SWITCH_TO_LARGE_BLOCKS feature indicates that we can receive
DMU_BACKUP_FEATURE_RESUMING | DMU_BACKUP_FEATURE_LARGE_BLOCKS | \
DMU_BACKUP_FEATURE_COMPRESSED | DMU_BACKUP_FEATURE_LARGE_DNODE | \
DMU_BACKUP_FEATURE_RAW | DMU_BACKUP_FEATURE_HOLDS | \
- DMU_BACKUP_FEATURE_REDACTED | DMU_BACKUP_FEATURE_SWITCH_TO_LARGE_BLOCKS)
+ DMU_BACKUP_FEATURE_REDACTED | DMU_BACKUP_FEATURE_SWITCH_TO_LARGE_BLOCKS | \
+ DMU_BACKUP_FEATURE_ZSTD)
/* Are all features in the given flag word currently supported? */
#define DMU_STREAM_SUPPORTED(x) (!((x) & ~DMU_BACKUP_FEATURE_MASK))
* Copyright (c) 2013 by Saso Kiselkov. All rights reserved.
* Copyright (c) 2013, Joyent, Inc. All rights reserved.
* Copyright 2016 Toomas Soome <tsoome@me.com>
+ * Copyright (c) 2019, Allan Jude
+ * Copyright (c) 2019, Klara Inc.
+ * Copyright (c) 2019-2020, Michael Niewöhner
*/
#ifndef _ZIO_H
(compress) == ZIO_COMPRESS_GZIP_8 || \
(compress) == ZIO_COMPRESS_GZIP_9 || \
(compress) == ZIO_COMPRESS_ZLE || \
+ (compress) == ZIO_COMPRESS_ZSTD || \
(compress) == ZIO_COMPRESS_ON || \
(compress) == ZIO_COMPRESS_OFF)
+
+#define ZIO_COMPRESS_ALGO(x) (x & SPA_COMPRESSMASK)
+#define ZIO_COMPRESS_LEVEL(x) ((x & ~SPA_COMPRESSMASK) >> SPA_COMPRESSBITS)
+#define ZIO_COMPRESS_RAW(type, level) (type | ((level) << SPA_COMPRESSBITS))
+
+#define ZIO_COMPLEVEL_ZSTD(level) \
+ ZIO_COMPRESS_RAW(ZIO_COMPRESS_ZSTD, level)
+
#define ZIO_FAILURE_MODE_WAIT 0
#define ZIO_FAILURE_MODE_CONTINUE 1
#define ZIO_FAILURE_MODE_PANIC 2
typedef struct zio_prop {
enum zio_checksum zp_checksum;
enum zio_compress zp_compress;
+ uint8_t zp_complevel;
dmu_object_type_t zp_type;
uint8_t zp_level;
uint8_t zp_copies;
enum zio_checksum child, enum zio_checksum parent);
extern enum zio_compress zio_compress_select(spa_t *spa,
enum zio_compress child, enum zio_compress parent);
+extern uint8_t zio_complevel_select(spa_t *spa, enum zio_compress compress,
+ uint8_t child, uint8_t parent);
extern void zio_suspend(spa_t *spa, zio_t *zio, zio_suspend_reason_t);
extern int zio_resume(spa_t *spa);
/*
* Copyright 2009 Sun Microsystems, Inc. All rights reserved.
+ * Copyright (c) 2019, Allan Jude
+ * Copyright (c) 2019, Klara Inc.
* Use is subject to license terms.
* Copyright (c) 2015, 2016 by Delphix. All rights reserved.
*/
ZIO_COMPRESS_GZIP_9,
ZIO_COMPRESS_ZLE,
ZIO_COMPRESS_LZ4,
+ ZIO_COMPRESS_ZSTD,
ZIO_COMPRESS_FUNCTIONS
};
+/* Compression algorithms that have levels */
+#define ZIO_COMPRESS_HASLEVEL(compress) ((compress == ZIO_COMPRESS_ZSTD || \
+ (compress >= ZIO_COMPRESS_GZIP_1 && \
+ compress <= ZIO_COMPRESS_GZIP_9)))
+
+#define ZIO_COMPLEVEL_INHERIT 0
+#define ZIO_COMPLEVEL_DEFAULT 255
+
+enum zio_zstd_levels {
+ ZIO_ZSTD_LEVEL_INHERIT = 0,
+ ZIO_ZSTD_LEVEL_1,
+#define ZIO_ZSTD_LEVEL_MIN ZIO_ZSTD_LEVEL_1
+ ZIO_ZSTD_LEVEL_2,
+ ZIO_ZSTD_LEVEL_3,
+#define ZIO_ZSTD_LEVEL_DEFAULT ZIO_ZSTD_LEVEL_3
+ ZIO_ZSTD_LEVEL_4,
+ ZIO_ZSTD_LEVEL_5,
+ ZIO_ZSTD_LEVEL_6,
+ ZIO_ZSTD_LEVEL_7,
+ ZIO_ZSTD_LEVEL_8,
+ ZIO_ZSTD_LEVEL_9,
+ ZIO_ZSTD_LEVEL_10,
+ ZIO_ZSTD_LEVEL_11,
+ ZIO_ZSTD_LEVEL_12,
+ ZIO_ZSTD_LEVEL_13,
+ ZIO_ZSTD_LEVEL_14,
+ ZIO_ZSTD_LEVEL_15,
+ ZIO_ZSTD_LEVEL_16,
+ ZIO_ZSTD_LEVEL_17,
+ ZIO_ZSTD_LEVEL_18,
+ ZIO_ZSTD_LEVEL_19,
+#define ZIO_ZSTD_LEVEL_MAX ZIO_ZSTD_LEVEL_19
+ ZIO_ZSTD_LEVEL_RESERVE = 101, /* Leave room for new positive levels */
+ ZIO_ZSTD_LEVEL_FAST, /* Fast levels are negative */
+ ZIO_ZSTD_LEVEL_FAST_1,
+#define ZIO_ZSTD_LEVEL_FAST_DEFAULT ZIO_ZSTD_LEVEL_FAST_1
+ ZIO_ZSTD_LEVEL_FAST_2,
+ ZIO_ZSTD_LEVEL_FAST_3,
+ ZIO_ZSTD_LEVEL_FAST_4,
+ ZIO_ZSTD_LEVEL_FAST_5,
+ ZIO_ZSTD_LEVEL_FAST_6,
+ ZIO_ZSTD_LEVEL_FAST_7,
+ ZIO_ZSTD_LEVEL_FAST_8,
+ ZIO_ZSTD_LEVEL_FAST_9,
+ ZIO_ZSTD_LEVEL_FAST_10,
+ ZIO_ZSTD_LEVEL_FAST_20,
+ ZIO_ZSTD_LEVEL_FAST_30,
+ ZIO_ZSTD_LEVEL_FAST_40,
+ ZIO_ZSTD_LEVEL_FAST_50,
+ ZIO_ZSTD_LEVEL_FAST_60,
+ ZIO_ZSTD_LEVEL_FAST_70,
+ ZIO_ZSTD_LEVEL_FAST_80,
+ ZIO_ZSTD_LEVEL_FAST_90,
+ ZIO_ZSTD_LEVEL_FAST_100,
+ ZIO_ZSTD_LEVEL_FAST_500,
+ ZIO_ZSTD_LEVEL_FAST_1000,
+#define ZIO_ZSTD_LEVEL_FAST_MAX ZIO_ZSTD_LEVEL_FAST_1000
+ ZIO_ZSTD_LEVEL_AUTO = 251, /* Reserved for future use */
+ ZIO_ZSTD_LEVEL_LEVELS
+};
+
+/* Forward Declaration to avoid visibility problems */
+struct zio_prop;
+
/* Common signature for all zio compress functions. */
typedef size_t zio_compress_func_t(void *src, void *dst,
size_t s_len, size_t d_len, int);
/* Common signature for all zio decompress functions. */
typedef int zio_decompress_func_t(void *src, void *dst,
size_t s_len, size_t d_len, int);
+/* Common signature for all zio decompress and get level functions. */
+typedef int zio_decompresslevel_func_t(void *src, void *dst,
+ size_t s_len, size_t d_len, uint8_t *level);
+/* Common signature for all zio get-compression-level functions. */
+typedef int zio_getlevel_func_t(void *src, size_t s_len, uint8_t *level);
+
/*
* Common signature for all zio decompress functions using an ABD as input.
int ci_level;
zio_compress_func_t *ci_compress;
zio_decompress_func_t *ci_decompress;
+ zio_decompresslevel_func_t *ci_decompress_level;
} zio_compress_info_t;
extern zio_compress_info_t zio_compress_table[ZIO_COMPRESS_FUNCTIONS];
* Compress and decompress data if necessary.
*/
extern size_t zio_compress_data(enum zio_compress c, abd_t *src, void *dst,
- size_t s_len);
+ size_t s_len, uint8_t level);
extern int zio_decompress_data(enum zio_compress c, abd_t *src, void *dst,
- size_t s_len, size_t d_len);
+ size_t s_len, size_t d_len, uint8_t *level);
extern int zio_decompress_data_buf(enum zio_compress c, void *src, void *dst,
- size_t s_len, size_t d_len);
+ size_t s_len, size_t d_len, uint8_t *level);
+extern int zio_compress_to_feature(enum zio_compress comp);
#ifdef __cplusplus
}
* the supported transformations:
*
* Compression:
- * ZFS supports three different flavors of compression -- gzip, lzjb, and
- * zle. Compression occurs as part of the write pipeline and is performed
- * in the ZIO_STAGE_WRITE_BP_INIT stage.
+ * ZFS supports five different flavors of compression -- gzip, lzjb, lz4, zle,
+ * and zstd. Compression occurs as part of the write pipeline and is
+ * performed in the ZIO_STAGE_WRITE_BP_INIT stage.
*
* Dedup:
* Dedup reads are handled by the ZIO_STAGE_DDT_READ_START and
--- /dev/null
+COMMON_H = \
+ $(top_srcdir)/include/sys/zstd/zstd.h
+
+KERNEL_H =
+
+USER_H =
+
+EXTRA_DIST = $(COMMON_H) $(KERNEL_H) $(USER_H)
+
+if CONFIG_USER
+libzfsdir = $(includedir)/libzfs/sys/zstd
+libzfs_HEADERS = $(COMMON_H) $(USER_H)
+endif
+
+if CONFIG_KERNEL
+kerneldir = @prefix@/src/zfs-$(VERSION)/include/sys/zstd
+kernel_HEADERS = $(COMMON_H) $(KERNEL_H)
+endif
--- /dev/null
+/*
+ * BSD 3-Clause New License (https://spdx.org/licenses/BSD-3-Clause.html)
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ *
+ * 3. Neither the name of the copyright holder nor the names of its
+ * contributors may be used to endorse or promote products derived from this
+ * software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/*
+ * Copyright (c) 2016-2018, Klara Inc.
+ * Copyright (c) 2016-2018, Allan Jude
+ * Copyright (c) 2018-2020, Sebastian Gottschall
+ * Copyright (c) 2019-2020, Michael Niewöhner
+ * Copyright (c) 2020, The FreeBSD Foundation [1]
+ *
+ * [1] Portions of this software were developed by Allan Jude
+ * under sponsorship from the FreeBSD Foundation.
+ */
+
+#ifndef _ZFS_ZSTD_H
+#define _ZFS_ZSTD_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*
+ * ZSTD block header
+ * NOTE: all fields in this header are in big endian order.
+ */
+typedef struct zfs_zstd_header {
+ /* Compressed size of data */
+ uint32_t c_len;
+
+ /*
+ * Version and compression level
+ * We use a union to be able to big endian encode a single 32 bit
+ * unsigned integer, but still access the individual bitmasked
+ * components easily.
+ */
+ union {
+ uint32_t raw_version_level;
+ struct {
+ uint32_t version : 24;
+ uint8_t level;
+ };
+ };
+
+ char data[];
+} zfs_zstdhdr_t;
+
+/*
+ * kstat helper macros
+ */
+#define ZSTDSTAT(stat) (zstd_stats.stat.value.ui64)
+#define ZSTDSTAT_INCR(stat, val) \
+ atomic_add_64(&zstd_stats.stat.value.ui64, (val))
+#define ZSTDSTAT_BUMP(stat) ZSTDSTAT_INCR(stat, 1)
+
+/* (de)init for user space / kernel emulation */
+int zstd_init(void);
+void zstd_fini(void);
+
+size_t zstd_compress(void *s_start, void *d_start, size_t s_len, size_t d_len,
+ int level);
+int zstd_get_level(void *s_start, size_t s_len, uint8_t *level);
+int zstd_decompress_level(void *s_start, void *d_start, size_t s_len,
+ size_t d_len, uint8_t *level);
+int zstd_decompress(void *s_start, void *d_start, size_t s_len, size_t d_len,
+ int n);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _ZFS_ZSTD_H */
SPA_FEATURE_LOG_SPACEMAP,
SPA_FEATURE_LIVELIST,
SPA_FEATURE_DEVICE_REBUILD,
+ SPA_FEATURE_ZSTD_COMPRESS,
SPA_FEATURES
} spa_feature_t;
# NB: GNU Automake Manual, Chapter 8.3.5: Libtool Convenience Libraries
-# These eight libraries are intermediary build components.
-SUBDIRS = libavl libicp libshare libspl libtpool
+# These nine libraries are intermediary build components.
+SUBDIRS = libavl libicp libshare libspl libtpool libzstd
if BUILD_LINUX
SUBDIRS += libefi
$(abs_top_builddir)/lib/libicp/libicp.la \
$(abs_top_builddir)/lib/libunicode/libunicode.la \
$(abs_top_builddir)/lib/libzfs_core/libzfs_core.la \
- $(abs_top_builddir)/lib/libnvpair/libnvpair.la
+ $(abs_top_builddir)/lib/libnvpair/libnvpair.la \
+ $(abs_top_builddir)/lib/libzstd/libzstd.la
libzpool_la_LIBADD += $(LIBCLOCK_GETTIME) $(ZLIB_LIBS) -ldl
#include <sys/utsname.h>
#include <sys/zfs_context.h>
#include <sys/zfs_onexit.h>
+#include <sys/zstd/zstd.h>
#include <sys/zvol.h>
#include <zfs_fletcher.h>
#include <zlib.h>
system_taskq_init();
icp_init();
+ zstd_init();
+
spa_init((spa_mode_t)mode);
fletcher_4_init();
fletcher_4_fini();
spa_fini();
+ zstd_fini();
+
icp_fini();
system_taskq_fini();
--- /dev/null
+include $(top_srcdir)/config/Rules.am
+
+VPATH = $(top_srcdir)/module/zstd
+
+# Includes kernel code, generate warnings for large stack frames
+AM_CFLAGS += $(FRAME_LARGER_THAN)
+
+noinst_LTLIBRARIES = libzstd.la
+
+KERNEL_C = \
+ lib/zstd.c \
+ zfs_zstd.c
+
+nodist_libzstd_la_SOURCES = $(KERNEL_C)
+
+# -fno-tree-vectorize is set for gcc in zstd/common/compiler.h
+# Set it for other compilers, too.
+lib/zstd.$(OBJEXT): CFLAGS += -fno-tree-vectorize
+lib/zstd.l$(OBJEXT): CFLAGS += -fno-tree-vectorize
+
+# Quiet warnings about frame size due to unused code in unmodified zstd lib
+lib/zstd.$(OBJEXT): CFLAGS += -Wframe-larger-than=20480
+lib/zstd.l$(OBJEXT): CFLAGS += -Wframe-larger-than=20480
.\" CDDL HEADER, with the fields enclosed by brackets "[]" replaced with your
.\" own identifying information:
.\" Portions Copyright [yyyy] [name of copyright owner]
+.\" Copyright (c) 2019, Klara Inc.
+.\" Copyright (c) 2019, Allan Jude
.TH ZPOOL-FEATURES 5 "Jun 8, 2018"
.SH NAME
zpool\-features \- ZFS pool feature descriptions
is used to checkpoint the pool.
The feature will only return back to being \fBenabled\fR when the pool
is rewound or the checkpoint has been discarded.
+.RE
+
+.sp
+.ne 2
+.na
+\fBzstd_compress\fR
+.ad
+.RS 4n
+.TS
+l l .
+GUID org.freebsd:zstd_compress
+READ\-ONLY COMPATIBLE no
+DEPENDENCIES extensible_dataset
+.TE
+
+\fBzstd\fR is a high-performance compression algorithm that features a
+combination of high compression ratios and high speed. Compared to \fBgzip\fR,
+\fBzstd\fR offers slighty better compression at much higher speeds. Compared
+to \fBlz4\fR, \fBzstd\fR offers much better compression while being only
+modestly slower. Typically, \fBzstd\fR compression speed ranges from 250 to 500
+MB/s per thread and decompression speed is over 1 GB/s per thread.
+
+When the \fBzstd\fR feature is set to \fBenabled\fR, the administrator can turn
+on \fBzstd\fR compression of any dataset by running
+`zfs set compress=zstd <pool/fs>`.
+
+This feature becomes \fBactive\fR once a \fBcompress\fR property has been set to
+\fBzstd\fR, and will return to being \fBenabled\fR once all filesystems that
+have ever had their compress property set to \fBzstd\fR are destroyed.
+
+Booting off of \fBzstd\fR-compressed root pools is not yet supported.
+.RE
.SH "SEE ALSO"
zpool(8)
.\" Copyright 2019 Richard Laager. All rights reserved.
.\" Copyright 2018 Nexenta Systems, Inc.
.\" Copyright 2019 Joyent, Inc.
+.\" Copyright (c) 2019, Kjeld Schouten-Lebbing
.\"
.Dd January 30, 2020
.Dt ZFSPROPS 8
Changing this property affects only newly-written data.
.It Xo
.Sy compression Ns = Ns Sy on Ns | Ns Sy off Ns | Ns Sy gzip Ns | Ns
-.Sy gzip- Ns Em N Ns | Ns Sy lz4 Ns | Ns Sy lzjb Ns | Ns Sy zle
+.Sy gzip- Ns Em N Ns | Ns Sy lz4 Ns | Ns Sy lzjb Ns | Ns Sy zle Ns | Ns Sy zstd Ns | Ns
+.Sy zstd- Ns Em N Ns | Ns Sy zstd-fast Ns | Ns Sy zstd-fast- Ns Em N
.Xc
Controls the compression algorithm used for this dataset.
.Pp
.Pc .
.Pp
The
+.Sy zstd
+compression algorithm provides both high compression ratios and good
+performance. You can specify the
+.Sy zstd
+level by using the value
+.Sy zstd- Ns Em N ,
+where
+.Em N
+is an integer from 1
+.Pq fastest
+to 19
+.Pq best compression ratio .
+.Sy zstd
+is equivalent to
+.Sy zstd-3 .
+.Pp
+Faster speeds at the cost of the compression ratio can be requested by
+setting a negative
+.Sy zstd
+level. This is done using
+.Sy zstd-fast- Ns Em N ,
+where
+.Em N
+is an integer in [1-9,10,20,30,...,100,500,1000] which maps to a negative
+.Sy zstd
+level. The lower the level the faster the compression - 1000 provides
+the fastest compression and lowest compression ratio.
+.Sy zstd-fast
+is equivalent to
+.Sy zstd-fast-1 .
+.Pp
+The
.Sy zle
compression algorithm compresses runs of zeros.
.Pp
ZFS_MODULES += unicode/
ZFS_MODULES += zcommon/
ZFS_MODULES += zfs/
+ZFS_MODULES += zstd/
# The rest is only relevant when run by kbuild
ifneq ($(KERNELRELEASE),)
${SRCDIR}/os/freebsd/zfs \
${SRCDIR}/unicode \
${SRCDIR}/zcommon \
- ${SRCDIR}/zfs
+ ${SRCDIR}/zfs \
+ ${SRCDIR}/zstd \
+ ${SRCDIR}/zstd/lib
+
CFLAGS+= -I${.OBJDIR:H}/include
CFLAGS+= -I${INCDIR}/os/freebsd
CFLAGS+= -I${INCDIR}/os/freebsd/spl
CFLAGS+= -I${INCDIR}/os/freebsd/zfs
+CFLAGS+= -I${SRCDIR}/zstd/include
CFLAGS+= -include ${INCDIR}/os/freebsd/spl/sys/ccompile.h
CFLAGS+= -D__KERNEL__ -DFREEBSD_NAMECACHE -DBUILDING_ZFS -D__BSD_VISIBLE=1 \
zthr.c \
zvol.c
+#zstd
+SRCS+= zfs_zstd.c \
+ zstd.c
+
beforeinstall:
.if ${MK_DEBUG_FILES} != "no"
mtree -eu \
CFLAGS.zil.c= -Wno-cast-qual
CFLAGS.zio.c= -Wno-cast-qual
CFLAGS.zrlock.c= -Wno-cast-qual
+CFLAGS.zfs_zstd.c= -Wno-cast-qual -Wno-pointer-arith
+CFLAGS.zstd.c= -fno-tree-vectorize
INSTALL_MOD_DIR ?= extra
-SUBDIR_TARGETS = icp lua
+SUBDIR_TARGETS = icp lua zstd
all: modules
distclean maintainer-clean: clean
* Copyright (c) 2013, Joyent, Inc. All rights reserved.
* Copyright (c) 2014, Nexenta Systems, Inc. All rights reserved.
* Copyright (c) 2017, Intel Corporation.
+ * Copyright (c) 2019, Klara Inc.
+ * Copyright (c) 2019, Allan Jude
*/
#ifndef _KERNEL
"org.openzfs:device_rebuild", "device_rebuild",
"Support for sequential device rebuilds",
ZFEATURE_FLAG_READONLY_COMPAT, ZFEATURE_TYPE_BOOLEAN, NULL);
+
+ {
+ static const spa_feature_t zstd_deps[] = {
+ SPA_FEATURE_EXTENSIBLE_DATASET,
+ SPA_FEATURE_NONE
+ };
+ zfeature_register(SPA_FEATURE_ZSTD_COMPRESS,
+ "org.freebsd:zstd_compress", "zstd_compress",
+ "zstd compression algorithm support.",
+ ZFEATURE_FLAG_PER_DATASET, ZFEATURE_TYPE_BOOLEAN, zstd_deps);
+ }
}
#if defined(_KERNEL)
* Copyright (c) 2011, 2018 by Delphix. All rights reserved.
* Copyright (c) 2013 by Saso Kiselkov. All rights reserved.
* Copyright 2016, Joyent, Inc.
+ * Copyright (c) 2019, Klara Inc.
+ * Copyright (c) 2019, Allan Jude
*/
/* Portions Copyright 2010 Robert Milkowski */
{ "gzip-9", ZIO_COMPRESS_GZIP_9 },
{ "zle", ZIO_COMPRESS_ZLE },
{ "lz4", ZIO_COMPRESS_LZ4 },
+ { "zstd", ZIO_COMPRESS_ZSTD },
+ { "zstd-fast",
+ ZIO_COMPLEVEL_ZSTD(ZIO_ZSTD_LEVEL_FAST_DEFAULT) },
+
+ /*
+ * ZSTD 1-19 are synthetic. We store the compression level in a
+ * separate hidden property to avoid wasting a large amount of
+ * space in the ZIO_COMPRESS enum.
+ *
+ * The compression level is also stored within the header of the
+ * compressed block since we may need it for later recompression
+ * to avoid checksum errors (L2ARC).
+ *
+ * Note that the level here is defined as bit shifted mask on
+ * top of the method.
+ */
+ { "zstd-1", ZIO_COMPLEVEL_ZSTD(ZIO_ZSTD_LEVEL_1) },
+ { "zstd-2", ZIO_COMPLEVEL_ZSTD(ZIO_ZSTD_LEVEL_2) },
+ { "zstd-3", ZIO_COMPLEVEL_ZSTD(ZIO_ZSTD_LEVEL_3) },
+ { "zstd-4", ZIO_COMPLEVEL_ZSTD(ZIO_ZSTD_LEVEL_4) },
+ { "zstd-5", ZIO_COMPLEVEL_ZSTD(ZIO_ZSTD_LEVEL_5) },
+ { "zstd-6", ZIO_COMPLEVEL_ZSTD(ZIO_ZSTD_LEVEL_6) },
+ { "zstd-7", ZIO_COMPLEVEL_ZSTD(ZIO_ZSTD_LEVEL_7) },
+ { "zstd-8", ZIO_COMPLEVEL_ZSTD(ZIO_ZSTD_LEVEL_8) },
+ { "zstd-9", ZIO_COMPLEVEL_ZSTD(ZIO_ZSTD_LEVEL_9) },
+ { "zstd-10", ZIO_COMPLEVEL_ZSTD(ZIO_ZSTD_LEVEL_10) },
+ { "zstd-11", ZIO_COMPLEVEL_ZSTD(ZIO_ZSTD_LEVEL_11) },
+ { "zstd-12", ZIO_COMPLEVEL_ZSTD(ZIO_ZSTD_LEVEL_12) },
+ { "zstd-13", ZIO_COMPLEVEL_ZSTD(ZIO_ZSTD_LEVEL_13) },
+ { "zstd-14", ZIO_COMPLEVEL_ZSTD(ZIO_ZSTD_LEVEL_14) },
+ { "zstd-15", ZIO_COMPLEVEL_ZSTD(ZIO_ZSTD_LEVEL_15) },
+ { "zstd-16", ZIO_COMPLEVEL_ZSTD(ZIO_ZSTD_LEVEL_16) },
+ { "zstd-17", ZIO_COMPLEVEL_ZSTD(ZIO_ZSTD_LEVEL_17) },
+ { "zstd-18", ZIO_COMPLEVEL_ZSTD(ZIO_ZSTD_LEVEL_18) },
+ { "zstd-19", ZIO_COMPLEVEL_ZSTD(ZIO_ZSTD_LEVEL_19) },
+
+ /*
+ * The ZSTD-Fast levels are also synthetic.
+ */
+ { "zstd-fast-1",
+ ZIO_COMPLEVEL_ZSTD(ZIO_ZSTD_LEVEL_FAST_1) },
+ { "zstd-fast-2",
+ ZIO_COMPLEVEL_ZSTD(ZIO_ZSTD_LEVEL_FAST_2) },
+ { "zstd-fast-3",
+ ZIO_COMPLEVEL_ZSTD(ZIO_ZSTD_LEVEL_FAST_3) },
+ { "zstd-fast-4",
+ ZIO_COMPLEVEL_ZSTD(ZIO_ZSTD_LEVEL_FAST_4) },
+ { "zstd-fast-5",
+ ZIO_COMPLEVEL_ZSTD(ZIO_ZSTD_LEVEL_FAST_5) },
+ { "zstd-fast-6",
+ ZIO_COMPLEVEL_ZSTD(ZIO_ZSTD_LEVEL_FAST_6) },
+ { "zstd-fast-7",
+ ZIO_COMPLEVEL_ZSTD(ZIO_ZSTD_LEVEL_FAST_7) },
+ { "zstd-fast-8",
+ ZIO_COMPLEVEL_ZSTD(ZIO_ZSTD_LEVEL_FAST_8) },
+ { "zstd-fast-9",
+ ZIO_COMPLEVEL_ZSTD(ZIO_ZSTD_LEVEL_FAST_9) },
+ { "zstd-fast-10",
+ ZIO_COMPLEVEL_ZSTD(ZIO_ZSTD_LEVEL_FAST_10) },
+ { "zstd-fast-20",
+ ZIO_COMPLEVEL_ZSTD(ZIO_ZSTD_LEVEL_FAST_20) },
+ { "zstd-fast-30",
+ ZIO_COMPLEVEL_ZSTD(ZIO_ZSTD_LEVEL_FAST_30) },
+ { "zstd-fast-40",
+ ZIO_COMPLEVEL_ZSTD(ZIO_ZSTD_LEVEL_FAST_40) },
+ { "zstd-fast-50",
+ ZIO_COMPLEVEL_ZSTD(ZIO_ZSTD_LEVEL_FAST_50) },
+ { "zstd-fast-60",
+ ZIO_COMPLEVEL_ZSTD(ZIO_ZSTD_LEVEL_FAST_60) },
+ { "zstd-fast-70",
+ ZIO_COMPLEVEL_ZSTD(ZIO_ZSTD_LEVEL_FAST_70) },
+ { "zstd-fast-80",
+ ZIO_COMPLEVEL_ZSTD(ZIO_ZSTD_LEVEL_FAST_80) },
+ { "zstd-fast-90",
+ ZIO_COMPLEVEL_ZSTD(ZIO_ZSTD_LEVEL_FAST_90) },
+ { "zstd-fast-100",
+ ZIO_COMPLEVEL_ZSTD(ZIO_ZSTD_LEVEL_FAST_100) },
+ { "zstd-fast-500",
+ ZIO_COMPLEVEL_ZSTD(ZIO_ZSTD_LEVEL_FAST_500) },
+ { "zstd-fast-1000",
+ ZIO_COMPLEVEL_ZSTD(ZIO_ZSTD_LEVEL_FAST_1000) },
{ NULL }
};
zprop_register_index(ZFS_PROP_COMPRESSION, "compression",
ZIO_COMPRESS_DEFAULT, PROP_INHERIT,
ZFS_TYPE_FILESYSTEM | ZFS_TYPE_VOLUME,
- "on | off | lzjb | gzip | gzip-[1-9] | zle | lz4", "COMPRESS",
- compress_table);
+ "on | off | lzjb | gzip | gzip-[1-9] | zle | lz4 | "
+ "zstd | zstd-[1-19] | "
+ "zstd-fast-[1-10,20,30,40,50,60,70,80,90,100,500,1000]",
+ "COMPRESS", compress_table);
zprop_register_index(ZFS_PROP_SNAPDIR, "snapdir", ZFS_SNAPDIR_HIDDEN,
PROP_INHERIT, ZFS_TYPE_FILESYSTEM,
"hidden | visible", "SNAPDIR", snapdir_table);
* Copyright (c) 2017, Nexenta Systems, Inc. All rights reserved.
* Copyright (c) 2019, loli10K <ezomori.nozomu@gmail.com>. All rights reserved.
* Copyright (c) 2020, George Amanakis. All rights reserved.
+ * Copyright (c) 2019, Klara Inc.
+ * Copyright (c) 2019, Allan Jude
* Copyright (c) 2020, The FreeBSD Foundation [1]
*
* [1] Portions of this software were developed by Allan Jude
HDR_GET_COMPRESS(hdr) : ZIO_COMPRESS_OFF);
}
+uint8_t
+arc_get_complevel(arc_buf_t *buf)
+{
+ return (buf->b_hdr->b_complevel);
+}
+
static inline boolean_t
arc_buf_is_shared(arc_buf_t *buf)
{
static arc_buf_hdr_t *
arc_buf_alloc_l2only(size_t size, arc_buf_contents_t type, l2arc_dev_t *dev,
dva_t dva, uint64_t daddr, int32_t psize, uint64_t birth,
- enum zio_compress compress, boolean_t protected, boolean_t prefetch)
+ enum zio_compress compress, uint8_t complevel, boolean_t protected,
+ boolean_t prefetch)
{
arc_buf_hdr_t *hdr;
HDR_SET_LSIZE(hdr, size);
HDR_SET_PSIZE(hdr, psize);
arc_hdr_set_compress(hdr, compress);
+ hdr->b_complevel = complevel;
if (protected)
arc_hdr_set_flags(hdr, ARC_FLAG_PROTECTED);
if (prefetch)
tmpbuf = zio_buf_alloc(lsize);
abd = abd_get_from_buf(tmpbuf, lsize);
abd_take_ownership_of_buf(abd, B_TRUE);
-
csize = zio_compress_data(HDR_GET_COMPRESS(hdr),
- hdr->b_l1hdr.b_pabd, tmpbuf, lsize);
+ hdr->b_l1hdr.b_pabd, tmpbuf, lsize, hdr->b_complevel);
ASSERT3U(csize, <=, psize);
abd_zero_off(abd, csize, psize - csize);
}
ret = zio_decompress_data(HDR_GET_COMPRESS(hdr),
hdr->b_l1hdr.b_pabd, tmp, HDR_GET_PSIZE(hdr),
- HDR_GET_LSIZE(hdr));
+ HDR_GET_LSIZE(hdr), &hdr->b_complevel);
if (ret != 0) {
abd_return_buf(cabd, tmp, arc_hdr_size(hdr));
goto error;
} else {
error = zio_decompress_data(HDR_GET_COMPRESS(hdr),
hdr->b_l1hdr.b_pabd, buf->b_data,
- HDR_GET_PSIZE(hdr), HDR_GET_LSIZE(hdr));
+ HDR_GET_PSIZE(hdr), HDR_GET_LSIZE(hdr),
+ &hdr->b_complevel);
/*
* Absent hardware errors or software bugs, this should
arc_buf_t *
arc_loan_compressed_buf(spa_t *spa, uint64_t psize, uint64_t lsize,
- enum zio_compress compression_type)
+ enum zio_compress compression_type, uint8_t complevel)
{
arc_buf_t *buf = arc_alloc_compressed_buf(spa, arc_onloan_tag,
- psize, lsize, compression_type);
+ psize, lsize, compression_type, complevel);
arc_loaned_bytes_update(arc_buf_size(buf));
arc_loan_raw_buf(spa_t *spa, uint64_t dsobj, boolean_t byteorder,
const uint8_t *salt, const uint8_t *iv, const uint8_t *mac,
dmu_object_type_t ot, uint64_t psize, uint64_t lsize,
- enum zio_compress compression_type)
+ enum zio_compress compression_type, uint8_t complevel)
{
arc_buf_t *buf = arc_alloc_raw_buf(spa, arc_onloan_tag, dsobj,
- byteorder, salt, iv, mac, ot, psize, lsize, compression_type);
+ byteorder, salt, iv, mac, ot, psize, lsize, compression_type,
+ complevel);
atomic_add_64(&arc_loaned_bytes, psize);
return (buf);
static arc_buf_hdr_t *
arc_hdr_alloc(uint64_t spa, int32_t psize, int32_t lsize,
- boolean_t protected, enum zio_compress compression_type,
+ boolean_t protected, enum zio_compress compression_type, uint8_t complevel,
arc_buf_contents_t type, boolean_t alloc_rdata)
{
arc_buf_hdr_t *hdr;
hdr->b_flags = 0;
arc_hdr_set_flags(hdr, arc_bufc_to_flags(type) | ARC_FLAG_HAS_L1HDR);
arc_hdr_set_compress(hdr, compression_type);
+ hdr->b_complevel = complevel;
if (protected)
arc_hdr_set_flags(hdr, ARC_FLAG_PROTECTED);
arc_alloc_buf(spa_t *spa, void *tag, arc_buf_contents_t type, int32_t size)
{
arc_buf_hdr_t *hdr = arc_hdr_alloc(spa_load_guid(spa), size, size,
- B_FALSE, ZIO_COMPRESS_OFF, type, B_FALSE);
+ B_FALSE, ZIO_COMPRESS_OFF, 0, type, B_FALSE);
arc_buf_t *buf = NULL;
VERIFY0(arc_buf_alloc_impl(hdr, spa, NULL, tag, B_FALSE, B_FALSE,
*/
arc_buf_t *
arc_alloc_compressed_buf(spa_t *spa, void *tag, uint64_t psize, uint64_t lsize,
- enum zio_compress compression_type)
+ enum zio_compress compression_type, uint8_t complevel)
{
ASSERT3U(lsize, >, 0);
ASSERT3U(lsize, >=, psize);
ASSERT3U(compression_type, <, ZIO_COMPRESS_FUNCTIONS);
arc_buf_hdr_t *hdr = arc_hdr_alloc(spa_load_guid(spa), psize, lsize,
- B_FALSE, compression_type, ARC_BUFC_DATA, B_FALSE);
+ B_FALSE, compression_type, complevel, ARC_BUFC_DATA, B_FALSE);
arc_buf_t *buf = NULL;
VERIFY0(arc_buf_alloc_impl(hdr, spa, NULL, tag, B_FALSE,
arc_alloc_raw_buf(spa_t *spa, void *tag, uint64_t dsobj, boolean_t byteorder,
const uint8_t *salt, const uint8_t *iv, const uint8_t *mac,
dmu_object_type_t ot, uint64_t psize, uint64_t lsize,
- enum zio_compress compression_type)
+ enum zio_compress compression_type, uint8_t complevel)
{
arc_buf_hdr_t *hdr;
arc_buf_t *buf;
ASSERT3U(compression_type, <, ZIO_COMPRESS_FUNCTIONS);
hdr = arc_hdr_alloc(spa_load_guid(spa), psize, lsize, B_TRUE,
- compression_type, type, B_TRUE);
+ compression_type, complevel, type, B_TRUE);
hdr->b_crypt_hdr.b_dsobj = dsobj;
hdr->b_crypt_hdr.b_ot = ot;
} else {
hdr->b_l1hdr.b_byteswap = DMU_BSWAP_NUMFUNCS;
}
+ if (!HDR_L2_READING(hdr)) {
+ hdr->b_complevel = zio->io_prop.zp_complevel;
+ }
}
arc_hdr_clear_flags(hdr, ARC_FLAG_L2_EVICTED);
arc_buf_hdr_t *exists = NULL;
arc_buf_contents_t type = BP_GET_BUFC_TYPE(bp);
hdr = arc_hdr_alloc(spa_load_guid(spa), psize, lsize,
- BP_IS_PROTECTED(bp), BP_GET_COMPRESS(bp), type,
+ BP_IS_PROTECTED(bp), BP_GET_COMPRESS(bp), 0, type,
encrypted_read);
if (!embedded_bp) {
* buffer which will be freed in arc_write().
*/
nhdr = arc_hdr_alloc(spa, psize, lsize, protected,
- compress, type, HDR_HAS_RABD(hdr));
+ compress, hdr->b_complevel, type, HDR_HAS_RABD(hdr));
ASSERT3P(nhdr->b_l1hdr.b_buf, ==, NULL);
ASSERT0(nhdr->b_l1hdr.b_bufcnt);
ASSERT0(zfs_refcount_count(&nhdr->b_l1hdr.b_refcnt));
}
HDR_SET_PSIZE(hdr, psize);
arc_hdr_set_compress(hdr, compress);
+ hdr->b_complevel = zio->io_prop.zp_complevel;
if (zio->io_error != 0 || psize == 0)
goto out;
ASSERT(ARC_BUF_COMPRESSED(buf));
localprop.zp_encrypt = B_TRUE;
localprop.zp_compress = HDR_GET_COMPRESS(hdr);
+ localprop.zp_complevel = hdr->b_complevel;
localprop.zp_byteorder =
(hdr->b_l1hdr.b_byteswap == DMU_BSWAP_NUMFUNCS) ?
ZFS_HOST_BYTEORDER : !ZFS_HOST_BYTEORDER;
} else if (ARC_BUF_COMPRESSED(buf)) {
ASSERT3U(HDR_GET_LSIZE(hdr), !=, arc_buf_size(buf));
localprop.zp_compress = HDR_GET_COMPRESS(hdr);
+ localprop.zp_complevel = hdr->b_complevel;
zio_flags |= ZIO_FLAG_RAW_COMPRESS;
}
callback = kmem_zalloc(sizeof (arc_write_callback_t), KM_SLEEP);
ret = zio_decompress_data(HDR_GET_COMPRESS(hdr),
hdr->b_l1hdr.b_pabd, tmp, HDR_GET_PSIZE(hdr),
- HDR_GET_LSIZE(hdr));
+ HDR_GET_LSIZE(hdr), &hdr->b_complevel);
if (ret != 0) {
abd_return_buf_copy(cabd, tmp, arc_hdr_size(hdr));
arc_free_data_abd(hdr, cabd, arc_hdr_size(hdr), hdr);
(HDR_HAS_RABD(hdr) && zio->io_abd == hdr->b_crypt_hdr.b_rabd));
zio->io_bp_copy = cb->l2rcb_bp; /* XXX fix in L2ARC 2.0 */
zio->io_bp = &zio->io_bp_copy; /* XXX fix in L2ARC 2.0 */
+ zio->io_prop.zp_complevel = hdr->b_complevel;
valid_cksum = arc_cksum_is_equal(hdr, zio);
cabd = abd_alloc_for_io(asize, ismd);
tmp = abd_borrow_buf(cabd, asize);
- psize = zio_compress_data(compress, to_write, tmp, size);
+ psize = zio_compress_data(compress, to_write, tmp, size,
+ hdr->b_complevel);
+
+ if (psize >= size) {
+ abd_return_buf(cabd, tmp, asize);
+ HDR_SET_COMPRESS(hdr, ZIO_COMPRESS_OFF);
+ to_write = cabd;
+ abd_copy(to_write, hdr->b_l1hdr.b_pabd, size);
+ if (size != asize)
+ abd_zero_off(to_write, size, asize - size);
+ goto encrypt;
+ }
ASSERT3U(psize, <=, HDR_GET_PSIZE(hdr));
if (psize < asize)
bzero((char *)tmp + psize, asize - psize);
to_write = cabd;
}
+encrypt:
if (HDR_ENCRYPTED(hdr)) {
eabd = abd_alloc_for_io(asize, ismd);
abd_copy_from_buf_off(abd, this_lb, 0, asize);
if ((err = zio_decompress_data(
L2BLK_GET_COMPRESS((this_lbp)->lbp_prop),
- abd, this_lb, asize, sizeof (*this_lb))) != 0) {
+ abd, this_lb, asize, sizeof (*this_lb), NULL)) != 0) {
err = SET_ERROR(EINVAL);
goto cleanup;
}
hdr = arc_buf_alloc_l2only(L2BLK_GET_LSIZE((le)->le_prop), type,
dev, le->le_dva, le->le_daddr,
L2BLK_GET_PSIZE((le)->le_prop), le->le_birth,
- L2BLK_GET_COMPRESS((le)->le_prop),
+ L2BLK_GET_COMPRESS((le)->le_prop), le->le_complevel,
L2BLK_GET_PROTECTED((le)->le_prop),
L2BLK_GET_PREFETCH((le)->le_prop));
asize = vdev_psize_to_asize(dev->l2ad_vdev,
/* try to compress the buffer */
psize = zio_compress_data(ZIO_COMPRESS_LZ4,
- abd_buf->abd, tmpbuf, sizeof (*lb));
+ abd_buf->abd, tmpbuf, sizeof (*lb), 0);
/* a log block is never entirely zero */
ASSERT(psize != 0);
L2BLK_SET_LSIZE((le)->le_prop, HDR_GET_LSIZE(hdr));
L2BLK_SET_PSIZE((le)->le_prop, HDR_GET_PSIZE(hdr));
L2BLK_SET_COMPRESS((le)->le_prop, HDR_GET_COMPRESS(hdr));
+ le->le_complevel = hdr->b_complevel;
L2BLK_SET_TYPE((le)->le_prop, hdr->b_type);
L2BLK_SET_PROTECTED((le)->le_prop, !!(HDR_PROTECTED(hdr)));
L2BLK_SET_PREFETCH((le)->le_prop, !!(HDR_PREFETCH(hdr)));
uint8_t dstbuf[BPE_PAYLOAD_SIZE];
decode_embedded_bp_compressed(bp, dstbuf);
VERIFY0(zio_decompress_data_buf(BP_GET_COMPRESS(bp),
- dstbuf, buf, psize, buflen));
+ dstbuf, buf, psize, buflen, NULL));
} else {
ASSERT3U(lsize, ==, psize);
decode_embedded_bp_compressed(bp, buf);
* Copyright (c) 2012, 2019 by Delphix. All rights reserved.
* Copyright (c) 2013 by Saso Kiselkov. All rights reserved.
* Copyright (c) 2014 Spectra Logic Corporation, All rights reserved.
+ * Copyright (c) 2019, Klara Inc.
+ * Copyright (c) 2019, Allan Jude
*/
#include <sys/zfs_context.h>
spa_t *spa = os->os_spa;
arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db);
enum zio_compress compress_type;
+ uint8_t complevel;
int psize, lsize;
psize = arc_buf_size(data);
lsize = arc_buf_lsize(data);
compress_type = arc_get_compression(data);
+ complevel = arc_get_complevel(data);
if (arc_is_encrypted(data)) {
boolean_t byteorder;
arc_get_raw_params(data, &byteorder, salt, iv, mac);
data = arc_alloc_raw_buf(spa, db, dmu_objset_id(os),
byteorder, salt, iv, mac, dn->dn_type, psize, lsize,
- compress_type);
+ compress_type, complevel);
} else if (compress_type != ZIO_COMPRESS_OFF) {
ASSERT3U(type, ==, ARC_BUFC_DATA);
data = arc_alloc_compressed_buf(spa, db,
- psize, lsize, compress_type);
+ psize, lsize, compress_type, complevel);
} else {
data = arc_alloc_buf(spa, db, type, psize);
}
* Copyright (c) 2016, Nexenta Systems, Inc. All rights reserved.
* Copyright (c) 2015 by Chunwei Chen. All rights reserved.
* Copyright (c) 2019 Datto Inc.
+ * Copyright (c) 2019, Klara Inc.
+ * Copyright (c) 2019, Allan Jude
*/
#include <sys/dmu.h>
(wp & WP_SPILL));
enum zio_checksum checksum = os->os_checksum;
enum zio_compress compress = os->os_compress;
+ uint8_t complevel = os->os_complevel;
enum zio_checksum dedup_checksum = os->os_dedup_checksum;
boolean_t dedup = B_FALSE;
boolean_t nopwrite = B_FALSE;
} else {
compress = zio_compress_select(os->os_spa, dn->dn_compress,
compress);
+ complevel = zio_complevel_select(os->os_spa, compress,
+ complevel, complevel);
checksum = (dedup_checksum == ZIO_CHECKSUM_OFF) ?
zio_checksum_select(dn->dn_checksum, checksum) :
}
zp->zp_compress = compress;
+ zp->zp_complevel = complevel;
zp->zp_checksum = checksum;
zp->zp_type = (wp & WP_SPILL) ? dn->dn_bonustype : type;
zp->zp_level = level;
* Copyright 2017 Nexenta Systems, Inc.
* Copyright (c) 2017 Open-E, Inc. All Rights Reserved.
* Copyright (c) 2018, loli10K <ezomori.nozomu@gmail.com>. All rights reserved.
+ * Copyright (c) 2019, Klara Inc.
+ * Copyright (c) 2019, Allan Jude
*/
/* Portions Copyright 2010 Robert Milkowski */
*/
ASSERT(newval != ZIO_COMPRESS_INHERIT);
- os->os_compress = zio_compress_select(os->os_spa, newval,
- ZIO_COMPRESS_ON);
+ os->os_compress = zio_compress_select(os->os_spa,
+ ZIO_COMPRESS_ALGO(newval), ZIO_COMPRESS_ON);
+ os->os_complevel = zio_complevel_select(os->os_spa, os->os_compress,
+ ZIO_COMPRESS_LEVEL(newval), ZIO_COMPLEVEL_DEFAULT);
}
static void
/* It's the meta-objset. */
os->os_checksum = ZIO_CHECKSUM_FLETCHER_4;
os->os_compress = ZIO_COMPRESS_ON;
+ os->os_complevel = ZIO_COMPLEVEL_DEFAULT;
os->os_encrypted = B_FALSE;
os->os_copies = spa_max_replication(spa);
os->os_dedup_checksum = ZIO_CHECKSUM_OFF;
* Copyright (c) 2014, Joyent, Inc. All rights reserved.
* Copyright 2014 HybridCluster. All rights reserved.
* Copyright (c) 2018, loli10K <ezomori.nozomu@gmail.com>. All rights reserved.
+ * Copyright (c) 2019, Klara Inc.
+ * Copyright (c) 2019, Allan Jude
*/
#include <sys/dmu.h>
return (SET_ERROR(ENOTSUP));
/*
- * LZ4 compressed, embedded, mooched, large blocks, and large_dnodes
- * in the stream can only be used if those pool features are enabled
- * because we don't attempt to decompress / un-embed / un-mooch /
- * split up the blocks / dnodes during the receive process.
+ * LZ4 compressed, ZSTD compressed, embedded, mooched, large blocks,
+ * and large_dnodes in the stream can only be used if those pool
+ * features are enabled because we don't attempt to decompress /
+ * un-embed / un-mooch / split up the blocks / dnodes during the
+ * receive process.
*/
if ((featureflags & DMU_BACKUP_FEATURE_LZ4) &&
!spa_feature_is_enabled(spa, SPA_FEATURE_LZ4_COMPRESS))
return (SET_ERROR(ENOTSUP));
+ if ((featureflags & DMU_BACKUP_FEATURE_ZSTD) &&
+ !spa_feature_is_enabled(spa, SPA_FEATURE_ZSTD_COMPRESS))
+ return (SET_ERROR(ENOTSUP));
if ((featureflags & DMU_BACKUP_FEATURE_EMBED_DATA) &&
!spa_feature_is_enabled(spa, SPA_FEATURE_EMBEDDED_DATA))
return (SET_ERROR(ENOTSUP));
drrw->drr_object, byteorder, drrw->drr_salt,
drrw->drr_iv, drrw->drr_mac, drrw->drr_type,
drrw->drr_compressed_size, drrw->drr_logical_size,
- drrw->drr_compressiontype);
+ drrw->drr_compressiontype, 0);
} else if (DRR_WRITE_COMPRESSED(drrw)) {
ASSERT3U(drrw->drr_compressed_size, >, 0);
ASSERT3U(drrw->drr_logical_size, >=,
abuf = arc_loan_compressed_buf(
dmu_objset_spa(drc->drc_os),
drrw->drr_compressed_size, drrw->drr_logical_size,
- drrw->drr_compressiontype);
+ drrw->drr_compressiontype, 0);
} else {
abuf = arc_loan_buf(dmu_objset_spa(drc->drc_os),
is_meta, drrw->drr_logical_size);
drrs->drr_object, byteorder, drrs->drr_salt,
drrs->drr_iv, drrs->drr_mac, drrs->drr_type,
drrs->drr_compressed_size, drrs->drr_length,
- drrs->drr_compressiontype);
+ drrs->drr_compressiontype, 0);
} else {
abuf = arc_loan_buf(dmu_objset_spa(drc->drc_os),
DMU_OT_IS_METADATA(drrs->drr_type),
* Copyright 2014 HybridCluster. All rights reserved.
* Copyright 2016 RackTop Systems.
* Copyright (c) 2016 Actifio, Inc. All rights reserved.
+ * Copyright (c) 2019, Klara Inc.
+ * Copyright (c) 2019, Allan Jude
*/
#include <sys/dmu.h>
!(featureflags & DMU_BACKUP_FEATURE_LZ4)))
return (B_FALSE);
+ /*
+ * If we have not set the ZSTD feature flag, we can't send ZSTD
+ * compressed embedded blocks, as the receiver may not support them.
+ */
+ if ((BP_GET_COMPRESS(bp) == ZIO_COMPRESS_ZSTD &&
+ !(featureflags & DMU_BACKUP_FEATURE_ZSTD)))
+ return (B_FALSE);
+
/*
* Embed type must be explicitly enabled.
*/
/* raw send implies compressok */
if (dspp->compressok || dspp->rawok)
*featureflags |= DMU_BACKUP_FEATURE_COMPRESSED;
+
if (dspp->rawok && os->os_encrypted)
*featureflags |= DMU_BACKUP_FEATURE_RAW;
*featureflags |= DMU_BACKUP_FEATURE_LZ4;
}
+ /*
+ * We specifically do not include DMU_BACKUP_FEATURE_EMBED_DATA here to
+ * allow sending ZSTD compressed datasets to a receiver that does not
+ * support ZSTD
+ */
+ if ((*featureflags &
+ (DMU_BACKUP_FEATURE_COMPRESSED | DMU_BACKUP_FEATURE_RAW)) != 0 &&
+ dsl_dataset_feature_is_active(to_ds, SPA_FEATURE_ZSTD_COMPRESS)) {
+ *featureflags |= DMU_BACKUP_FEATURE_ZSTD;
+ }
+
if (dspp->resumeobj != 0 || dspp->resumeoff != 0) {
*featureflags |= DMU_BACKUP_FEATURE_RESUMING;
}
* Copyright (c) 2016 Actifio, Inc. All rights reserved.
* Copyright 2016, OmniTI Computer Consulting, Inc. All rights reserved.
* Copyright 2017 Nexenta Systems, Inc.
+ * Copyright (c) 2019, Klara Inc.
+ * Copyright (c) 2019, Allan Jude
+ * Copyright (c) 2020 The FreeBSD Foundation [1]
+ *
+ * [1] Portions of this software were developed by Allan Jude
+ * under sponsorship from the FreeBSD Foundation.
*/
#include <sys/dmu_objset.h>
int compressed = BP_GET_PSIZE(bp);
int uncompressed = BP_GET_UCSIZE(bp);
int64_t delta;
+ spa_feature_t f;
dprintf_bp(bp, "ds=%p", ds);
(void *)B_TRUE;
}
- spa_feature_t f = zio_checksum_to_feature(BP_GET_CHECKSUM(bp));
+
+ f = zio_checksum_to_feature(BP_GET_CHECKSUM(bp));
+ if (f != SPA_FEATURE_NONE) {
+ ASSERT3S(spa_feature_table[f].fi_type, ==,
+ ZFEATURE_TYPE_BOOLEAN);
+ ds->ds_feature_activation[f] = (void *)B_TRUE;
+ }
+
+ f = zio_compress_to_feature(BP_GET_COMPRESS(bp));
if (f != SPA_FEATURE_NONE) {
ASSERT3S(spa_feature_table[f].fi_type, ==,
ZFEATURE_TYPE_BOOLEAN);
ZFS_SPACE_CHECK_EXTRA_RESERVED));
}
+typedef struct dsl_dataset_set_compression_arg {
+ const char *ddsca_name;
+ zprop_source_t ddsca_source;
+ uint64_t ddsca_value;
+} dsl_dataset_set_compression_arg_t;
+
+/* ARGSUSED */
+static int
+dsl_dataset_set_compression_check(void *arg, dmu_tx_t *tx)
+{
+ dsl_dataset_set_compression_arg_t *ddsca = arg;
+ dsl_pool_t *dp = dmu_tx_pool(tx);
+
+ uint64_t compval = ZIO_COMPRESS_ALGO(ddsca->ddsca_value);
+ spa_feature_t f = zio_compress_to_feature(compval);
+
+ if (f == SPA_FEATURE_NONE)
+ return (SET_ERROR(EINVAL));
+
+ if (!spa_feature_is_enabled(dp->dp_spa, f))
+ return (SET_ERROR(ENOTSUP));
+
+ return (0);
+}
+
+static void
+dsl_dataset_set_compression_sync(void *arg, dmu_tx_t *tx)
+{
+ dsl_dataset_set_compression_arg_t *ddsca = arg;
+ dsl_pool_t *dp = dmu_tx_pool(tx);
+ dsl_dataset_t *ds = NULL;
+
+ uint64_t compval = ZIO_COMPRESS_ALGO(ddsca->ddsca_value);
+ spa_feature_t f = zio_compress_to_feature(compval);
+ ASSERT3S(spa_feature_table[f].fi_type, ==, ZFEATURE_TYPE_BOOLEAN);
+
+ VERIFY0(dsl_dataset_hold(dp, ddsca->ddsca_name, FTAG, &ds));
+ if (zfeature_active(f, ds->ds_feature[f]) != B_TRUE) {
+ ds->ds_feature_activation[f] = (void *)B_TRUE;
+ dsl_dataset_activate_feature(ds->ds_object, f,
+ ds->ds_feature_activation[f], tx);
+ ds->ds_feature[f] = ds->ds_feature_activation[f];
+ }
+ dsl_dataset_rele(ds, FTAG);
+}
+
+int
+dsl_dataset_set_compression(const char *dsname, zprop_source_t source,
+ uint64_t compression)
+{
+ dsl_dataset_set_compression_arg_t ddsca;
+
+ /*
+ * The sync task is only required for zstd in order to activate
+ * the feature flag when the property is first set.
+ */
+ if (ZIO_COMPRESS_ALGO(compression) != ZIO_COMPRESS_ZSTD)
+ return (0);
+
+ ddsca.ddsca_name = dsname;
+ ddsca.ddsca_source = source;
+ ddsca.ddsca_value = compression;
+
+ return (dsl_sync_task(dsname, dsl_dataset_set_compression_check,
+ dsl_dataset_set_compression_sync, &ddsca, 0,
+ ZFS_SPACE_CHECK_EXTRA_RESERVED));
+}
+
/*
* Return (in *usedp) the amount of space referenced by "new" that was not
* referenced at the time the bookmark corresponds to. "New" may be a
#include <sys/btree.h>
#include <sys/zfeature.h>
#include <sys/qat.h>
+#include <sys/zstd/zstd.h>
/*
* SPA locking
* Copyright (c) 2017 Open-E, Inc. All Rights Reserved.
* Copyright (c) 2019 Datto Inc.
* Copyright (c) 2019, 2020 by Christian Schwarz. All rights reserved.
+ * Copyright (c) 2019, Klara Inc.
+ * Copyright (c) 2019, Allan Jude
*/
/*
case ZFS_PROP_REFRESERVATION:
err = dsl_dataset_set_refreservation(dsname, source, intval);
break;
+ case ZFS_PROP_COMPRESSION:
+ err = dsl_dataset_set_compression(dsname, source, intval);
+ /*
+ * Set err to -1 to force the zfs_set_prop_nvlist code down the
+ * default path to set the value in the nvlist.
+ */
+ if (err == 0)
+ err = -1;
+ break;
case ZFS_PROP_VOLSIZE:
err = zvol_set_volsize(dsname, intval);
break;
const char *propname = nvpair_name(pair);
boolean_t issnap = (strchr(dsname, '@') != NULL);
zfs_prop_t prop = zfs_name_to_prop(propname);
- uint64_t intval;
+ uint64_t intval, compval;
int err;
if (prop == ZPROP_INVAL) {
* we'll catch them later.
*/
if (nvpair_value_uint64(pair, &intval) == 0) {
- if (intval >= ZIO_COMPRESS_GZIP_1 &&
- intval <= ZIO_COMPRESS_GZIP_9 &&
+ compval = ZIO_COMPRESS_ALGO(intval);
+ if (compval >= ZIO_COMPRESS_GZIP_1 &&
+ compval <= ZIO_COMPRESS_GZIP_9 &&
zfs_earlier_version(dsname,
SPA_VERSION_GZIP_COMPRESSION)) {
return (SET_ERROR(ENOTSUP));
}
- if (intval == ZIO_COMPRESS_ZLE &&
+ if (compval == ZIO_COMPRESS_ZLE &&
zfs_earlier_version(dsname,
SPA_VERSION_ZLE_COMPRESSION))
return (SET_ERROR(ENOTSUP));
- if (intval == ZIO_COMPRESS_LZ4) {
+ if (compval == ZIO_COMPRESS_LZ4) {
spa_t *spa;
if ((err = spa_open(dsname, &spa, FTAG)) != 0)
}
spa_close(spa, FTAG);
}
+
+ if (compval == ZIO_COMPRESS_ZSTD) {
+ spa_t *spa;
+
+ if ((err = spa_open(dsname, &spa, FTAG)) != 0)
+ return (err);
+
+ if (!spa_feature_is_enabled(spa,
+ SPA_FEATURE_ZSTD_COMPRESS)) {
+ spa_close(spa, FTAG);
+ return (SET_ERROR(ENOTSUP));
+ }
+ spa_close(spa, FTAG);
+ }
}
break;
* Copyright (c) 2011, 2019 by Delphix. All rights reserved.
* Copyright (c) 2011 Nexenta Systems, Inc. All rights reserved.
* Copyright (c) 2017, Intel Corporation.
+ * Copyright (c) 2019, Klara Inc.
+ * Copyright (c) 2019, Allan Jude
*/
#include <sys/sysmacros.h>
if (zio->io_error == 0) {
void *tmp = abd_borrow_buf(data, size);
int ret = zio_decompress_data(BP_GET_COMPRESS(zio->io_bp),
- zio->io_abd, tmp, zio->io_size, size);
+ zio->io_abd, tmp, zio->io_size, size,
+ &zio->io_prop.zp_complevel);
abd_return_buf_copy(data, tmp, size);
if (zio_injection_enabled && ret == 0)
*/
tmp = zio_buf_alloc(lsize);
ret = zio_decompress_data(BP_GET_COMPRESS(bp),
- zio->io_abd, tmp, zio->io_size, lsize);
+ zio->io_abd, tmp, zio->io_size, lsize,
+ &zio->io_prop.zp_complevel);
if (ret != 0) {
ret = SET_ERROR(EIO);
goto error;
if (compress != ZIO_COMPRESS_OFF &&
!(zio->io_flags & ZIO_FLAG_RAW_COMPRESS)) {
void *cbuf = zio_buf_alloc(lsize);
- psize = zio_compress_data(compress, zio->io_abd, cbuf, lsize);
- if (psize == 0 || psize == lsize) {
+ psize = zio_compress_data(compress, zio->io_abd, cbuf, lsize,
+ zp->zp_complevel);
+ if (psize == 0 || psize >= lsize) {
compress = ZIO_COMPRESS_OFF;
zio_buf_free(cbuf, lsize);
} else if (!zp->zp_dedup && !zp->zp_encrypt &&
* to a hole.
*/
psize = zio_compress_data(ZIO_COMPRESS_EMPTY,
- zio->io_abd, NULL, lsize);
- if (psize == 0)
+ zio->io_abd, NULL, lsize, zp->zp_complevel);
+ if (psize == 0 || psize >= lsize)
compress = ZIO_COMPRESS_OFF;
} else {
ASSERT3U(psize, !=, 0);
zp.zp_checksum = gio->io_prop.zp_checksum;
zp.zp_compress = ZIO_COMPRESS_OFF;
+ zp.zp_complevel = gio->io_prop.zp_complevel;
zp.zp_type = DMU_OT_NONE;
zp.zp_level = 0;
zp.zp_copies = gio->io_prop.zp_copies;
/*
* Copyright (c) 2013, 2018 by Delphix. All rights reserved.
+ * Copyright (c) 2019, Klara Inc.
+ * Copyright (c) 2019, Allan Jude
*/
#include <sys/zfs_context.h>
#include <sys/zfeature.h>
#include <sys/zio.h>
#include <sys/zio_compress.h>
+#include <sys/zstd/zstd.h>
/*
* If nonzero, every 1/X decompression attempts will fail, simulating
* Compression vectors.
*/
zio_compress_info_t zio_compress_table[ZIO_COMPRESS_FUNCTIONS] = {
- {"inherit", 0, NULL, NULL},
- {"on", 0, NULL, NULL},
- {"uncompressed", 0, NULL, NULL},
- {"lzjb", 0, lzjb_compress, lzjb_decompress},
- {"empty", 0, NULL, NULL},
- {"gzip-1", 1, gzip_compress, gzip_decompress},
- {"gzip-2", 2, gzip_compress, gzip_decompress},
- {"gzip-3", 3, gzip_compress, gzip_decompress},
- {"gzip-4", 4, gzip_compress, gzip_decompress},
- {"gzip-5", 5, gzip_compress, gzip_decompress},
- {"gzip-6", 6, gzip_compress, gzip_decompress},
- {"gzip-7", 7, gzip_compress, gzip_decompress},
- {"gzip-8", 8, gzip_compress, gzip_decompress},
- {"gzip-9", 9, gzip_compress, gzip_decompress},
- {"zle", 64, zle_compress, zle_decompress},
- {"lz4", 0, lz4_compress_zfs, lz4_decompress_zfs}
+ {"inherit", 0, NULL, NULL, NULL},
+ {"on", 0, NULL, NULL, NULL},
+ {"uncompressed", 0, NULL, NULL, NULL},
+ {"lzjb", 0, lzjb_compress, lzjb_decompress, NULL},
+ {"empty", 0, NULL, NULL, NULL},
+ {"gzip-1", 1, gzip_compress, gzip_decompress, NULL},
+ {"gzip-2", 2, gzip_compress, gzip_decompress, NULL},
+ {"gzip-3", 3, gzip_compress, gzip_decompress, NULL},
+ {"gzip-4", 4, gzip_compress, gzip_decompress, NULL},
+ {"gzip-5", 5, gzip_compress, gzip_decompress, NULL},
+ {"gzip-6", 6, gzip_compress, gzip_decompress, NULL},
+ {"gzip-7", 7, gzip_compress, gzip_decompress, NULL},
+ {"gzip-8", 8, gzip_compress, gzip_decompress, NULL},
+ {"gzip-9", 9, gzip_compress, gzip_decompress, NULL},
+ {"zle", 64, zle_compress, zle_decompress, NULL},
+ {"lz4", 0, lz4_compress_zfs, lz4_decompress_zfs, NULL},
+ {"zstd", ZIO_ZSTD_LEVEL_DEFAULT, zstd_compress, zstd_decompress,
+ zstd_decompress_level},
};
+uint8_t
+zio_complevel_select(spa_t *spa, enum zio_compress compress, uint8_t child,
+ uint8_t parent)
+{
+ uint8_t result;
+
+ if (!ZIO_COMPRESS_HASLEVEL(compress))
+ return (0);
+
+ result = child;
+ if (result == ZIO_COMPLEVEL_INHERIT)
+ result = parent;
+
+ return (result);
+}
+
enum zio_compress
zio_compress_select(spa_t *spa, enum zio_compress child,
enum zio_compress parent)
}
size_t
-zio_compress_data(enum zio_compress c, abd_t *src, void *dst, size_t s_len)
+zio_compress_data(enum zio_compress c, abd_t *src, void *dst, size_t s_len,
+ uint8_t level)
{
size_t c_len, d_len;
+ uint8_t complevel;
zio_compress_info_t *ci = &zio_compress_table[c];
ASSERT((uint_t)c < ZIO_COMPRESS_FUNCTIONS);
/* Compress at least 12.5% */
d_len = s_len - (s_len >> 3);
+ complevel = ci->ci_level;
+
+ if (c == ZIO_COMPRESS_ZSTD) {
+ /* If we don't know the level, we can't compress it */
+ if (level == ZIO_COMPLEVEL_INHERIT)
+ return (s_len);
+
+ if (level == ZIO_COMPLEVEL_DEFAULT)
+ complevel = ZIO_ZSTD_LEVEL_DEFAULT;
+ else
+ complevel = level;
+
+ ASSERT3U(complevel, !=, ZIO_COMPLEVEL_INHERIT);
+ }
+
/* No compression algorithms can read from ABDs directly */
void *tmp = abd_borrow_buf_copy(src, s_len);
- c_len = ci->ci_compress(tmp, dst, s_len, d_len, ci->ci_level);
+ c_len = ci->ci_compress(tmp, dst, s_len, d_len, complevel);
abd_return_buf(src, tmp, s_len);
if (c_len > d_len)
int
zio_decompress_data_buf(enum zio_compress c, void *src, void *dst,
- size_t s_len, size_t d_len)
+ size_t s_len, size_t d_len, uint8_t *level)
{
zio_compress_info_t *ci = &zio_compress_table[c];
if ((uint_t)c >= ZIO_COMPRESS_FUNCTIONS || ci->ci_decompress == NULL)
return (SET_ERROR(EINVAL));
+ if (ci->ci_decompress_level != NULL && level != NULL)
+ return (ci->ci_decompress_level(src, dst, s_len, d_len, level));
+
return (ci->ci_decompress(src, dst, s_len, d_len, ci->ci_level));
}
int
zio_decompress_data(enum zio_compress c, abd_t *src, void *dst,
- size_t s_len, size_t d_len)
+ size_t s_len, size_t d_len, uint8_t *level)
{
void *tmp = abd_borrow_buf_copy(src, s_len);
- int ret = zio_decompress_data_buf(c, tmp, dst, s_len, d_len);
+ int ret = zio_decompress_data_buf(c, tmp, dst, s_len, d_len, level);
abd_return_buf(src, tmp, s_len);
/*
return (ret);
}
+
+int
+zio_compress_to_feature(enum zio_compress comp)
+{
+ switch (comp) {
+ case ZIO_COMPRESS_ZSTD:
+ return (SPA_FEATURE_ZSTD_COMPRESS);
+ default:
+ /* fallthru */;
+ }
+ return (SPA_FEATURE_NONE);
+}
--- /dev/null
+ifneq ($(KBUILD_EXTMOD),)
+src = @abs_srcdir@
+obj = @abs_builddir@
+zstd_include = $(src)/include
+else
+zstd_include = $(srctree)/$(src)/include
+endif
+
+MODULE := zzstd
+
+obj-$(CONFIG_ZFS) := $(MODULE).o
+
+asflags-y := -I$(zstd_include)
+ccflags-y := -I$(zstd_include)
+
+# Zstd uses -O3 by default, so we should follow
+ccflags-y += -O3
+
+# -fno-tree-vectorize gets set for gcc in zstd/common/compiler.h
+# Set it for other compilers, too.
+$(obj)/lib/zstd.o: c_flags += -fno-tree-vectorize
+
+# Quiet warnings about frame size due to unused code in unmodified zstd lib
+$(obj)/lib/zstd.o: c_flags += -Wframe-larger-than=20480
+
+# Disable aarch64 neon SIMD instructions for kernel mode
+$(obj)/lib/zstd.o: c_flags += -include $(zstd_include)/aarch64_compat.h
+
+$(MODULE)-objs += zfs_zstd.o
+$(MODULE)-objs += lib/zstd.o
+
+all:
+ mkdir -p lib
--- /dev/null
+# ZSTD-On-ZFS Library Manual
+
+## Introduction
+
+This subtree contains the ZSTD library used in ZFS. It is heavily cut-down by
+dropping any unneeded files, and combined into a single file, but otherwise is
+intentionally unmodified. Please do not alter the file containing the zstd
+library, besides upgrading to a newer ZSTD release.
+
+Tree structure:
+
+* `zfs_zstd.c` is the actual `zzstd` kernel module.
+* `lib/` contains the the unmodified, [_"amalgamated"_](https://github.com/facebook/zstd/blob/dev/contrib/single_file_libs/README.md)
+ version of the `Zstandard` library, generated from our template file
+* `zstd-in.c` is our template file for generating the library
+* `include/`: This directory contains supplemental includes for platform
+ compatibility, which are not expected to be used by ZFS elsewhere in the
+ future. Thus we keep them private to ZSTD.
+
+## Updating ZSTD
+
+To update ZSTD the following steps need to be taken:
+
+1. Grab the latest release of [ZSTD](https://github.com/facebook/zstd/releases).
+2. Update `module/zstd/zstd-in.c` if required. (see
+ `zstd/contrib/single_file_libs/zstd-in.c` in the zstd repository)
+3. Generate the "single-file-library" and put it to `module/zstd/lib/`.
+4. Copy the following files to `module/zstd/lib/`:
+ - `zstd/lib/zstd.h`
+ - `zstd/lib/common/zstd_errors.h`
+
+This can be done using a few shell commands from inside the zfs repo:
+
+~~~sh
+cd PATH/TO/ZFS
+
+url="https://github.com/facebook/zstd"
+release="$(curl -s "${url}"/releases/latest | grep -oP '(?<=v)[\d\.]+')"
+zstd="/tmp/zstd-${release}/"
+
+wget -O /tmp/zstd.tar.gz \
+ "${url}/releases/download/v${release}/zstd-${release}.tar.gz"
+tar -C /tmp -xzf /tmp/zstd.tar.gz
+
+cp ${zstd}/lib/zstd.h module/zstd/lib/
+cp ${zstd}/lib/zstd_errors.h module/zstd/lib/
+${zstd}/contrib/single_file_libs/combine.sh \
+ -r ${zstd}/lib -o module/zstd/lib/zstd.c module/zstd/zstd-in.c
+~~~
+
+
+## Altering ZSTD and breaking changes
+
+If ZSTD made changes that break compatibility or you need to make breaking
+changes to the way we handle ZSTD, it is required to maintain backwards
+compatibility.
+
+We already save the ZSTD version number within the block header to be used
+to add future compatibility checks and/or fixes. However, currently it is
+not actually used in such a way.
--- /dev/null
+/*
+ * BSD 3-Clause New License (https://spdx.org/licenses/BSD-3-Clause.html)
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ *
+ * 3. Neither the name of the copyright holder nor the names of its
+ * contributors may be used to endorse or promote products derived from this
+ * software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/*
+ * Copyright (c) 2018-2020, Sebastian Gottschall
+ */
+
+#ifdef _KERNEL
+#undef __aarch64__
+#endif
--- /dev/null
+/*
+ * BSD 3-Clause New License (https://spdx.org/licenses/BSD-3-Clause.html)
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ *
+ * 3. Neither the name of the copyright holder nor the names of its
+ * contributors may be used to endorse or promote products derived from this
+ * software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/*
+ * Copyright (c) 2014-2019, Allan Jude
+ * Copyright (c) 2020, Brian Behlendorf
+ * Copyright (c) 2020, Michael Niewöhner
+ */
+
+#ifndef _ZSTD_LIMITS_H
+#define _ZSTD_LIMITS_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifdef _KERNEL
+
+#if defined(__FreeBSD__)
+#include <sys/limits.h>
+#elif defined(__linux__)
+#include <linux/limits.h>
+#include <linux/kernel.h>
+#else
+#error "Unsupported platform"
+#endif
+
+#else /* !_KERNEL */
+#include_next <limits.h>
+#endif /* _KERNEL */
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _ZSTD_LIMITS_H */
--- /dev/null
+/*
+ * BSD 3-Clause New License (https://spdx.org/licenses/BSD-3-Clause.html)
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ *
+ * 3. Neither the name of the copyright holder nor the names of its
+ * contributors may be used to endorse or promote products derived from this
+ * software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/*
+ * Copyright (c) 2014-2019, Allan Jude
+ * Copyright (c) 2020, Brian Behlendorf
+ * Copyright (c) 2020, Michael Niewöhner
+ */
+
+#ifndef _ZSTD_STDDEF_H
+#define _ZSTD_STDDEF_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifdef _KERNEL
+
+#if defined(__FreeBSD__)
+#include <sys/types.h>
+#elif defined(__linux__)
+#include <linux/types.h>
+#else
+#error "Unsupported platform"
+#endif
+
+#else /* !_KERNEL */
+#include_next <stddef.h>
+#endif /* _KERNEL */
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _ZSTD_STDDEF_H */
--- /dev/null
+/*
+ * BSD 3-Clause New License (https://spdx.org/licenses/BSD-3-Clause.html)
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ *
+ * 3. Neither the name of the copyright holder nor the names of its
+ * contributors may be used to endorse or promote products derived from this
+ * software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/*
+ * Copyright (c) 2014-2019, Allan Jude
+ * Copyright (c) 2020, Brian Behlendorf
+ * Copyright (c) 2020, Michael Niewöhner
+ */
+
+#ifndef _ZSTD_STDINT_H
+#define _ZSTD_STDINT_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifdef _KERNEL
+
+#if defined(__FreeBSD__)
+#include <sys/stdint.h>
+#elif defined(__linux__)
+#include <linux/types.h>
+#else
+#error "Unsupported platform"
+#endif
+
+#else /* !_KERNEL */
+#include_next <stdint.h>
+#endif /* _KERNEL */
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _ZSTD_STDINT_H */
--- /dev/null
+/*
+ * BSD 3-Clause New License (https://spdx.org/licenses/BSD-3-Clause.html)
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ *
+ * 3. Neither the name of the copyright holder nor the names of its
+ * contributors may be used to endorse or promote products derived from this
+ * software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/*
+ * Copyright (c) 2014-2019, Allan Jude
+ * Copyright (c) 2020, Brian Behlendorf
+ * Copyright (c) 2020, Michael Niewöhner
+ */
+
+#ifndef _ZSTD_STDIO_H
+#define _ZSTD_STDIO_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifndef _KERNEL
+
+#include_next <stdio.h>
+
+#endif /* _KERNEL */
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _ZSTD_STDIO_H */
--- /dev/null
+/*
+ * BSD 3-Clause New License (https://spdx.org/licenses/BSD-3-Clause.html)
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ *
+ * 3. Neither the name of the copyright holder nor the names of its
+ * contributors may be used to endorse or promote products derived from this
+ * software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/*
+ * Copyright (c) 2014-2019, Allan Jude
+ * Copyright (c) 2020, Brian Behlendorf
+ * Copyright (c) 2020, Michael Niewöhner
+ */
+
+#ifndef _ZSTD_STDLIB_H
+#define _ZSTD_STDLIB_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#undef GCC_VERSION
+
+/*
+ * Define calloc, malloc, free to make building work. They are never really used
+ * in zstdlib.c since allocation is done in zstd.c.
+ */
+#define calloc(n, sz) NULL
+#define malloc(sz) NULL
+#define free(ptr)
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _ZSTD_STDLIB_H */
--- /dev/null
+/*
+ * BSD 3-Clause New License (https://spdx.org/licenses/BSD-3-Clause.html)
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ *
+ * 3. Neither the name of the copyright holder nor the names of its
+ * contributors may be used to endorse or promote products derived from this
+ * software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/*
+ * Copyright (c) 2014-2019, Allan Jude
+ * Copyright (c) 2020, Brian Behlendorf
+ * Copyright (c) 2020, Michael Niewöhner
+ */
+
+#ifndef _ZSTD_STRING_H
+#define _ZSTD_STRING_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifdef _KERNEL
+
+#if defined(__FreeBSD__)
+#include <sys/systm.h> /* memcpy, memset */
+#elif defined(__linux__)
+#include <linux/string.h> /* memcpy, memset */
+#else
+#error "Unsupported platform"
+#endif
+
+#else /* !_KERNEL */
+#include_next <string.h>
+#endif /* _KERNEL */
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _ZSTD_STRING_H */
--- /dev/null
+/*
+ * BSD 3-Clause New License (https://spdx.org/licenses/BSD-3-Clause.html)
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ *
+ * 3. Neither the name of the copyright holder nor the names of its
+ * contributors may be used to endorse or promote products derived from this
+ * software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/*
+ * Copyright (c) 2016-2018, Klara Inc.
+ * Copyright (c) 2016-2018, Allan Jude
+ * Copyright (c) 2018-2020, Sebastian Gottschall
+ * Copyright (c) 2019-2020, Michael Niewöhner
+ * Copyright (c) 2020, The FreeBSD Foundation [1]
+ *
+ * [1] Portions of this software were developed by Allan Jude
+ * under sponsorship from the FreeBSD Foundation.
+ */
+
+#include <sys/param.h>
+#include <sys/sysmacros.h>
+#include <sys/zfs_context.h>
+#include <sys/zio_compress.h>
+#include <sys/spa.h>
+#include <sys/zstd/zstd.h>
+
+#define ZSTD_STATIC_LINKING_ONLY
+#include "lib/zstd.h"
+#include "lib/zstd_errors.h"
+
+kstat_t *zstd_ksp = NULL;
+
+typedef struct zstd_stats {
+ kstat_named_t zstd_stat_alloc_fail;
+ kstat_named_t zstd_stat_alloc_fallback;
+ kstat_named_t zstd_stat_com_alloc_fail;
+ kstat_named_t zstd_stat_dec_alloc_fail;
+ kstat_named_t zstd_stat_com_inval;
+ kstat_named_t zstd_stat_dec_inval;
+ kstat_named_t zstd_stat_dec_header_inval;
+ kstat_named_t zstd_stat_com_fail;
+ kstat_named_t zstd_stat_dec_fail;
+} zstd_stats_t;
+
+static zstd_stats_t zstd_stats = {
+ { "alloc_fail", KSTAT_DATA_UINT64 },
+ { "alloc_fallback", KSTAT_DATA_UINT64 },
+ { "compress_alloc_fail", KSTAT_DATA_UINT64 },
+ { "decompress_alloc_fail", KSTAT_DATA_UINT64 },
+ { "compress_level_invalid", KSTAT_DATA_UINT64 },
+ { "decompress_level_invalid", KSTAT_DATA_UINT64 },
+ { "decompress_header_invalid", KSTAT_DATA_UINT64 },
+ { "compress_failed", KSTAT_DATA_UINT64 },
+ { "decompress_failed", KSTAT_DATA_UINT64 },
+};
+
+/* Enums describing the allocator type specified by kmem_type in zstd_kmem */
+enum zstd_kmem_type {
+ ZSTD_KMEM_UNKNOWN = 0,
+ /* Allocation type using kmem_vmalloc */
+ ZSTD_KMEM_DEFAULT,
+ /* Pool based allocation using mempool_alloc */
+ ZSTD_KMEM_POOL,
+ /* Reserved fallback memory for decompression only */
+ ZSTD_KMEM_DCTX,
+ ZSTD_KMEM_COUNT,
+};
+
+/* Structure for pooled memory objects */
+struct zstd_pool {
+ void *mem;
+ size_t size;
+ kmutex_t barrier;
+ hrtime_t timeout;
+};
+
+/* Global structure for handling memory allocations */
+struct zstd_kmem {
+ enum zstd_kmem_type kmem_type;
+ size_t kmem_size;
+ struct zstd_pool *pool;
+};
+
+/* Fallback memory structure used for decompression only if memory runs out */
+struct zstd_fallback_mem {
+ size_t mem_size;
+ void *mem;
+ kmutex_t barrier;
+};
+
+struct zstd_levelmap {
+ int16_t zstd_level;
+ enum zio_zstd_levels level;
+};
+
+/*
+ * ZSTD memory handlers
+ *
+ * For decompression we use a different handler which also provides fallback
+ * memory allocation in case memory runs out.
+ *
+ * The ZSTD handlers were split up for the most simplified implementation.
+ */
+static void *zstd_alloc(void *opaque, size_t size);
+static void *zstd_dctx_alloc(void *opaque, size_t size);
+static void zstd_free(void *opaque, void *ptr);
+
+/* Compression memory handler */
+static const ZSTD_customMem zstd_malloc = {
+ zstd_alloc,
+ zstd_free,
+ NULL,
+};
+
+/* Decompression memory handler */
+static const ZSTD_customMem zstd_dctx_malloc = {
+ zstd_dctx_alloc,
+ zstd_free,
+ NULL,
+};
+
+/* Level map for converting ZFS internal levels to ZSTD levels and vice versa */
+static struct zstd_levelmap zstd_levels[] = {
+ {ZIO_ZSTD_LEVEL_1, ZIO_ZSTD_LEVEL_1},
+ {ZIO_ZSTD_LEVEL_2, ZIO_ZSTD_LEVEL_2},
+ {ZIO_ZSTD_LEVEL_3, ZIO_ZSTD_LEVEL_3},
+ {ZIO_ZSTD_LEVEL_4, ZIO_ZSTD_LEVEL_4},
+ {ZIO_ZSTD_LEVEL_5, ZIO_ZSTD_LEVEL_5},
+ {ZIO_ZSTD_LEVEL_6, ZIO_ZSTD_LEVEL_6},
+ {ZIO_ZSTD_LEVEL_7, ZIO_ZSTD_LEVEL_7},
+ {ZIO_ZSTD_LEVEL_8, ZIO_ZSTD_LEVEL_8},
+ {ZIO_ZSTD_LEVEL_9, ZIO_ZSTD_LEVEL_9},
+ {ZIO_ZSTD_LEVEL_10, ZIO_ZSTD_LEVEL_10},
+ {ZIO_ZSTD_LEVEL_11, ZIO_ZSTD_LEVEL_11},
+ {ZIO_ZSTD_LEVEL_12, ZIO_ZSTD_LEVEL_12},
+ {ZIO_ZSTD_LEVEL_13, ZIO_ZSTD_LEVEL_13},
+ {ZIO_ZSTD_LEVEL_14, ZIO_ZSTD_LEVEL_14},
+ {ZIO_ZSTD_LEVEL_15, ZIO_ZSTD_LEVEL_15},
+ {ZIO_ZSTD_LEVEL_16, ZIO_ZSTD_LEVEL_16},
+ {ZIO_ZSTD_LEVEL_17, ZIO_ZSTD_LEVEL_17},
+ {ZIO_ZSTD_LEVEL_18, ZIO_ZSTD_LEVEL_18},
+ {ZIO_ZSTD_LEVEL_19, ZIO_ZSTD_LEVEL_19},
+ {-1, ZIO_ZSTD_LEVEL_FAST_1},
+ {-2, ZIO_ZSTD_LEVEL_FAST_2},
+ {-3, ZIO_ZSTD_LEVEL_FAST_3},
+ {-4, ZIO_ZSTD_LEVEL_FAST_4},
+ {-5, ZIO_ZSTD_LEVEL_FAST_5},
+ {-6, ZIO_ZSTD_LEVEL_FAST_6},
+ {-7, ZIO_ZSTD_LEVEL_FAST_7},
+ {-8, ZIO_ZSTD_LEVEL_FAST_8},
+ {-9, ZIO_ZSTD_LEVEL_FAST_9},
+ {-10, ZIO_ZSTD_LEVEL_FAST_10},
+ {-20, ZIO_ZSTD_LEVEL_FAST_20},
+ {-30, ZIO_ZSTD_LEVEL_FAST_30},
+ {-40, ZIO_ZSTD_LEVEL_FAST_40},
+ {-50, ZIO_ZSTD_LEVEL_FAST_50},
+ {-60, ZIO_ZSTD_LEVEL_FAST_60},
+ {-70, ZIO_ZSTD_LEVEL_FAST_70},
+ {-80, ZIO_ZSTD_LEVEL_FAST_80},
+ {-90, ZIO_ZSTD_LEVEL_FAST_90},
+ {-100, ZIO_ZSTD_LEVEL_FAST_100},
+ {-500, ZIO_ZSTD_LEVEL_FAST_500},
+ {-1000, ZIO_ZSTD_LEVEL_FAST_1000},
+};
+
+/*
+ * This variable represents the maximum count of the pool based on the number
+ * of CPUs plus some buffer. We default to cpu count * 4, see init_zstd.
+ */
+static int pool_count = 16;
+
+#define ZSTD_POOL_MAX pool_count
+#define ZSTD_POOL_TIMEOUT 60 * 2
+
+static struct zstd_fallback_mem zstd_dctx_fallback;
+static struct zstd_pool *zstd_mempool_cctx;
+static struct zstd_pool *zstd_mempool_dctx;
+
+/*
+ * Try to get a cached allocated buffer from memory pool or allocate a new one
+ * if necessary. If a object is older than 2 minutes and does not fit the
+ * requested size, it will be released and a new cached entry will be allocated.
+ * If other pooled objects are detected without being used for 2 minutes, they
+ * will be released, too.
+ *
+ * The concept is that high frequency memory allocations of bigger objects are
+ * expensive. So if a lot of work is going on, allocations will be kept for a
+ * while and can be reused in that time frame.
+ *
+ * The scheduled release will be updated every time a object is reused.
+ */
+static void *
+zstd_mempool_alloc(struct zstd_pool *zstd_mempool, size_t size)
+{
+ struct zstd_pool *pool;
+ struct zstd_kmem *mem = NULL;
+
+ if (!zstd_mempool) {
+ return (NULL);
+ }
+
+ /* Seek for preallocated memory slot and free obsolete slots */
+ for (int i = 0; i < ZSTD_POOL_MAX; i++) {
+ pool = &zstd_mempool[i];
+ /*
+ * This lock is simply a marker for a pool object beeing in use.
+ * If it's already hold, it will be skipped.
+ *
+ * We need to create it before checking it to avoid race
+ * conditions caused by running in a threaded context.
+ *
+ * The lock is later released by zstd_mempool_free.
+ */
+ if (mutex_tryenter(&pool->barrier)) {
+ /*
+ * Check if objects fits the size, if so we take it and
+ * update the timestamp.
+ */
+ if (!mem && pool->mem && size <= pool->size) {
+ pool->timeout = gethrestime_sec() +
+ ZSTD_POOL_TIMEOUT;
+ mem = pool->mem;
+ continue;
+ }
+
+ /* Free memory if unused object older than 2 minutes */
+ if (pool->mem && gethrestime_sec() > pool->timeout) {
+ vmem_free(pool->mem, pool->size);
+ pool->mem = NULL;
+ pool->size = 0;
+ pool->timeout = 0;
+ }
+
+ mutex_exit(&pool->barrier);
+ }
+ }
+
+ if (mem) {
+ return (mem);
+ }
+
+ /*
+ * If no preallocated slot was found, try to fill in a new one.
+ *
+ * We run a similar algorithm twice here to avoid pool fragmentation.
+ * The first one may generate holes in the list if objects get released.
+ * We always make sure that these holes get filled instead of adding new
+ * allocations constantly at the end.
+ */
+ for (int i = 0; i < ZSTD_POOL_MAX; i++) {
+ pool = &zstd_mempool[i];
+ if (mutex_tryenter(&pool->barrier)) {
+ /* Object is free, try to allocate new one */
+ if (!pool->mem) {
+ mem = vmem_alloc(size, KM_SLEEP);
+ pool->mem = mem;
+
+ if (pool->mem) {
+ /* Keep track for later release */
+ mem->pool = pool;
+ pool->size = size;
+ mem->kmem_type = ZSTD_KMEM_POOL;
+ mem->kmem_size = size;
+ }
+ }
+
+ if (size <= pool->size) {
+ /* Update timestamp */
+ pool->timeout = gethrestime_sec() +
+ ZSTD_POOL_TIMEOUT;
+
+ return (pool->mem);
+ }
+
+ mutex_exit(&pool->barrier);
+ }
+ }
+
+ /*
+ * If the pool is full or the allocation failed, try lazy allocation
+ * instead.
+ */
+ if (!mem) {
+ mem = vmem_alloc(size, KM_NOSLEEP);
+ if (mem) {
+ mem->pool = NULL;
+ mem->kmem_type = ZSTD_KMEM_DEFAULT;
+ mem->kmem_size = size;
+ }
+ }
+
+ return (mem);
+}
+
+/* Mark object as released by releasing the barrier mutex */
+static void
+zstd_mempool_free(struct zstd_kmem *z)
+{
+ mutex_exit(&z->pool->barrier);
+}
+
+/* Convert ZFS internal enum to ZSTD level */
+static int
+zstd_enum_to_level(enum zio_zstd_levels level, int16_t *zstd_level)
+{
+ if (level > 0 && level <= ZIO_ZSTD_LEVEL_19) {
+ *zstd_level = zstd_levels[level - 1].zstd_level;
+ return (0);
+ }
+ if (level >= ZIO_ZSTD_LEVEL_FAST_1 &&
+ level <= ZIO_ZSTD_LEVEL_FAST_1000) {
+ *zstd_level = zstd_levels[level - ZIO_ZSTD_LEVEL_FAST_1
+ + ZIO_ZSTD_LEVEL_19].zstd_level;
+ return (0);
+ }
+
+ /* Invalid/unknown zfs compression enum - this should never happen. */
+ return (1);
+}
+
+/* Compress block using zstd */
+size_t
+zstd_compress(void *s_start, void *d_start, size_t s_len, size_t d_len,
+ int level)
+{
+ size_t c_len;
+ int16_t zstd_level;
+ zfs_zstdhdr_t *hdr;
+ ZSTD_CCtx *cctx;
+
+ hdr = (zfs_zstdhdr_t *)d_start;
+
+ /* Skip compression if the specified level is invalid */
+ if (zstd_enum_to_level(level, &zstd_level)) {
+ ZSTDSTAT_BUMP(zstd_stat_com_inval);
+ return (s_len);
+ }
+
+ ASSERT3U(d_len, >=, sizeof (*hdr));
+ ASSERT3U(d_len, <=, s_len);
+ ASSERT3U(zstd_level, !=, 0);
+
+ cctx = ZSTD_createCCtx_advanced(zstd_malloc);
+
+ /*
+ * Out of kernel memory, gently fall through - this will disable
+ * compression in zio_compress_data
+ */
+ if (!cctx) {
+ ZSTDSTAT_BUMP(zstd_stat_com_alloc_fail);
+ return (s_len);
+ }
+
+ /* Set the compression level */
+ ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, zstd_level);
+
+ /* Use the "magicless" zstd header which saves us 4 header bytes */
+ ZSTD_CCtx_setParameter(cctx, ZSTD_c_format, ZSTD_f_zstd1_magicless);
+
+ /*
+ * Disable redundant checksum calculation and content size storage since
+ * this is already done by ZFS itself.
+ */
+ ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 0);
+ ZSTD_CCtx_setParameter(cctx, ZSTD_c_contentSizeFlag, 0);
+
+ c_len = ZSTD_compress2(cctx,
+ hdr->data,
+ d_len - sizeof (*hdr),
+ s_start, s_len);
+
+ ZSTD_freeCCtx(cctx);
+
+ /* Error in the compression routine, disable compression. */
+ if (ZSTD_isError(c_len)) {
+ /*
+ * If we are aborting the compression because the saves are
+ * too small, that is not a failure. Everything else is a
+ * failure, so increment the compression failure counter.
+ */
+ if (ZSTD_getErrorCode(c_len) != ZSTD_error_dstSize_tooSmall) {
+ ZSTDSTAT_BUMP(zstd_stat_com_fail);
+ }
+ return (s_len);
+ }
+
+ /*
+ * Encode the compressed buffer size at the start. We'll need this in
+ * decompression to counter the effects of padding which might be added
+ * to the compressed buffer and which, if unhandled, would confuse the
+ * hell out of our decompression function.
+ */
+ hdr->c_len = BE_32(c_len);
+
+ /*
+ * Check version for overflow.
+ * The limit of 24 bits must not be exceeded. This allows a maximum
+ * version 1677.72.15 which we don't expect to be ever reached.
+ */
+ ASSERT3U(ZSTD_VERSION_NUMBER, <=, 0xFFFFFF);
+
+ /*
+ * Encode the compression level as well. We may need to know the
+ * original compression level if compressed_arc is disabled, to match
+ * the compression settings to write this block to the L2ARC.
+ *
+ * Encode the actual level, so if the enum changes in the future, we
+ * will be compatible.
+ *
+ * The upper 24 bits store the ZSTD version to be able to provide
+ * future compatibility, since new versions might enhance the
+ * compression algorithm in a way, where the compressed data will
+ * change.
+ *
+ * As soon as such incompatibility occurs, handling code needs to be
+ * added, differentiating between the versions.
+ */
+ hdr->version = ZSTD_VERSION_NUMBER;
+ hdr->level = level;
+ hdr->raw_version_level = BE_32(hdr->raw_version_level);
+
+ return (c_len + sizeof (*hdr));
+}
+
+/* Decompress block using zstd and return its stored level */
+int
+zstd_decompress_level(void *s_start, void *d_start, size_t s_len, size_t d_len,
+ uint8_t *level)
+{
+ ZSTD_DCtx *dctx;
+ size_t result;
+ int16_t zstd_level;
+ uint32_t c_len;
+ const zfs_zstdhdr_t *hdr;
+ zfs_zstdhdr_t hdr_copy;
+
+ hdr = (const zfs_zstdhdr_t *)s_start;
+ c_len = BE_32(hdr->c_len);
+
+ /*
+ * Make a copy instead of directly converting the header, since we must
+ * not modify the original data that may be used again later.
+ */
+ hdr_copy.raw_version_level = BE_32(hdr->raw_version_level);
+
+ /*
+ * NOTE: We ignore the ZSTD version for now. As soon as any
+ * incompatibility occurrs, it has to be handled accordingly.
+ * The version can be accessed via `hdr_copy.version`.
+ */
+
+ /*
+ * Convert and check the level
+ * An invalid level is a strong indicator for data corruption! In such
+ * case return an error so the upper layers can try to fix it.
+ */
+ if (zstd_enum_to_level(hdr_copy.level, &zstd_level)) {
+ ZSTDSTAT_BUMP(zstd_stat_dec_inval);
+ return (1);
+ }
+
+ ASSERT3U(d_len, >=, s_len);
+ ASSERT3U(hdr_copy.level, !=, ZIO_COMPLEVEL_INHERIT);
+
+ /* Invalid compressed buffer size encoded at start */
+ if (c_len + sizeof (*hdr) > s_len) {
+ ZSTDSTAT_BUMP(zstd_stat_dec_header_inval);
+ return (1);
+ }
+
+ dctx = ZSTD_createDCtx_advanced(zstd_dctx_malloc);
+ if (!dctx) {
+ ZSTDSTAT_BUMP(zstd_stat_dec_alloc_fail);
+ return (1);
+ }
+
+ /* Set header type to "magicless" */
+ ZSTD_DCtx_setParameter(dctx, ZSTD_d_format, ZSTD_f_zstd1_magicless);
+
+ /* Decompress the data and release the context */
+ result = ZSTD_decompressDCtx(dctx, d_start, d_len, hdr->data, c_len);
+ ZSTD_freeDCtx(dctx);
+
+ /*
+ * Returns 0 on success (decompression function returned non-negative)
+ * and non-zero on failure (decompression function returned negative.
+ */
+ if (ZSTD_isError(result)) {
+ ZSTDSTAT_BUMP(zstd_stat_dec_fail);
+ return (1);
+ }
+
+ if (level) {
+ *level = hdr_copy.level;
+ }
+
+ return (0);
+}
+
+/* Decompress datablock using zstd */
+int
+zstd_decompress(void *s_start, void *d_start, size_t s_len, size_t d_len,
+ int level __maybe_unused)
+{
+
+ return (zstd_decompress_level(s_start, d_start, s_len, d_len, NULL));
+}
+
+/* Allocator for zstd compression context using mempool_allocator */
+static void *
+zstd_alloc(void *opaque __maybe_unused, size_t size)
+{
+ size_t nbytes = sizeof (struct zstd_kmem) + size;
+ struct zstd_kmem *z = NULL;
+
+ z = (struct zstd_kmem *)zstd_mempool_alloc(zstd_mempool_cctx, nbytes);
+
+ if (!z) {
+ ZSTDSTAT_BUMP(zstd_stat_alloc_fail);
+ return (NULL);
+ }
+
+ return ((void*)z + (sizeof (struct zstd_kmem)));
+}
+
+/*
+ * Allocator for zstd decompression context using mempool_allocator with
+ * fallback to reserved memory if allocation fails
+ */
+static void *
+zstd_dctx_alloc(void *opaque __maybe_unused, size_t size)
+{
+ size_t nbytes = sizeof (struct zstd_kmem) + size;
+ struct zstd_kmem *z = NULL;
+ enum zstd_kmem_type type = ZSTD_KMEM_DEFAULT;
+
+ z = (struct zstd_kmem *)zstd_mempool_alloc(zstd_mempool_dctx, nbytes);
+ if (!z) {
+ /* Try harder, decompression shall not fail */
+ z = vmem_alloc(nbytes, KM_SLEEP);
+ if (z) {
+ z->pool = NULL;
+ }
+ ZSTDSTAT_BUMP(zstd_stat_alloc_fail);
+ } else {
+ return ((void*)z + (sizeof (struct zstd_kmem)));
+ }
+
+ /* Fallback if everything fails */
+ if (!z) {
+ /*
+ * Barrier since we only can handle it in a single thread. All
+ * other following threads need to wait here until decompression
+ * is completed. zstd_free will release this barrier later.
+ */
+ mutex_enter(&zstd_dctx_fallback.barrier);
+
+ z = zstd_dctx_fallback.mem;
+ type = ZSTD_KMEM_DCTX;
+ ZSTDSTAT_BUMP(zstd_stat_alloc_fallback);
+ }
+
+ /* Allocation should always be successful */
+ if (!z) {
+ return (NULL);
+ }
+
+ z->kmem_type = type;
+ z->kmem_size = nbytes;
+
+ return ((void*)z + (sizeof (struct zstd_kmem)));
+}
+
+/* Free allocated memory by its specific type */
+static void
+zstd_free(void *opaque __maybe_unused, void *ptr)
+{
+ struct zstd_kmem *z = (ptr - sizeof (struct zstd_kmem));
+ enum zstd_kmem_type type;
+
+ ASSERT3U(z->kmem_type, <, ZSTD_KMEM_COUNT);
+ ASSERT3U(z->kmem_type, >, ZSTD_KMEM_UNKNOWN);
+
+ type = z->kmem_type;
+ switch (type) {
+ case ZSTD_KMEM_DEFAULT:
+ vmem_free(z, z->kmem_size);
+ break;
+ case ZSTD_KMEM_POOL:
+ zstd_mempool_free(z);
+ break;
+ case ZSTD_KMEM_DCTX:
+ mutex_exit(&zstd_dctx_fallback.barrier);
+ break;
+ default:
+ break;
+ }
+}
+
+/* Allocate fallback memory to ensure safe decompression */
+static void __init
+create_fallback_mem(struct zstd_fallback_mem *mem, size_t size)
+{
+ mem->mem_size = size;
+ mem->mem = vmem_zalloc(mem->mem_size, KM_SLEEP);
+ mutex_init(&mem->barrier, NULL, MUTEX_DEFAULT, NULL);
+}
+
+/* Initialize memory pool barrier mutexes */
+static void __init
+zstd_mempool_init(void)
+{
+ zstd_mempool_cctx = (struct zstd_pool *)
+ kmem_zalloc(ZSTD_POOL_MAX * sizeof (struct zstd_pool), KM_SLEEP);
+ zstd_mempool_dctx = (struct zstd_pool *)
+ kmem_zalloc(ZSTD_POOL_MAX * sizeof (struct zstd_pool), KM_SLEEP);
+
+ for (int i = 0; i < ZSTD_POOL_MAX; i++) {
+ mutex_init(&zstd_mempool_cctx[i].barrier, NULL,
+ MUTEX_DEFAULT, NULL);
+ mutex_init(&zstd_mempool_dctx[i].barrier, NULL,
+ MUTEX_DEFAULT, NULL);
+ }
+}
+
+/* Initialize zstd-related memory handling */
+static int __init
+zstd_meminit(void)
+{
+ zstd_mempool_init();
+
+ /*
+ * Estimate the size of the fallback decompression context.
+ * The expected size on x64 with current ZSTD should be about 160 KB.
+ */
+ create_fallback_mem(&zstd_dctx_fallback,
+ P2ROUNDUP(ZSTD_estimateDCtxSize() + sizeof (struct zstd_kmem),
+ PAGESIZE));
+
+ return (0);
+}
+
+/* Release object from pool and free memory */
+static void __exit
+release_pool(struct zstd_pool *pool)
+{
+ mutex_destroy(&pool->barrier);
+ vmem_free(pool->mem, pool->size);
+ pool->mem = NULL;
+ pool->size = 0;
+}
+
+/* Release memory pool objects */
+static void __exit
+zstd_mempool_deinit(void)
+{
+ for (int i = 0; i < ZSTD_POOL_MAX; i++) {
+ release_pool(&zstd_mempool_cctx[i]);
+ release_pool(&zstd_mempool_dctx[i]);
+ }
+
+ kmem_free(zstd_mempool_dctx, ZSTD_POOL_MAX * sizeof (struct zstd_pool));
+ kmem_free(zstd_mempool_cctx, ZSTD_POOL_MAX * sizeof (struct zstd_pool));
+ zstd_mempool_dctx = NULL;
+ zstd_mempool_cctx = NULL;
+}
+
+extern int __init
+zstd_init(void)
+{
+ /* Set pool size by using maximum sane thread count * 4 */
+ pool_count = (boot_ncpus * 4);
+ zstd_meminit();
+
+ /* Initialize kstat */
+ zstd_ksp = kstat_create("zfs", 0, "zstd", "misc",
+ KSTAT_TYPE_NAMED, sizeof (zstd_stats) / sizeof (kstat_named_t),
+ KSTAT_FLAG_VIRTUAL);
+ if (zstd_ksp != NULL) {
+ zstd_ksp->ks_data = &zstd_stats;
+ kstat_install(zstd_ksp);
+ }
+
+ return (0);
+}
+
+extern void __exit
+zstd_fini(void)
+{
+ /* Deinitialize kstat */
+ if (zstd_ksp != NULL) {
+ kstat_delete(zstd_ksp);
+ zstd_ksp = NULL;
+ }
+
+ /* Release fallback memory */
+ vmem_free(zstd_dctx_fallback.mem, zstd_dctx_fallback.mem_size);
+ mutex_destroy(&zstd_dctx_fallback.barrier);
+
+ /* Deinit memory pool */
+ zstd_mempool_deinit();
+}
+
+#if defined(_KERNEL)
+module_init(zstd_init);
+module_exit(zstd_fini);
+
+ZFS_MODULE_DESCRIPTION("ZSTD Compression for ZFS");
+ZFS_MODULE_LICENSE("BSD");
+ZFS_MODULE_VERSION(ZSTD_VERSION_STRING);
+
+EXPORT_SYMBOL(zstd_compress);
+EXPORT_SYMBOL(zstd_decompress_level);
+EXPORT_SYMBOL(zstd_decompress);
+#endif
export KMOD_ICP=@abs_top_builddir@/module/icp/icp.ko
export KMOD_ZFS=@abs_top_builddir@/module/zfs/zfs.ko
export KMOD_FREEBSD=@abs_top_builddir@/module/openzfs.ko
+export KMOD_ZZSTD=@abs_top_builddir@/module/zstd/zzstd.ko
endef
export EXTRA_ENVIRONMENT
STRIP[5]="\${STRIP[0]}"
STRIP[6]="\${STRIP[0]}"
STRIP[7]="\${STRIP[0]}"
+STRIP[8]="\${STRIP[0]}"
BUILT_MODULE_NAME[0]="zavl"
BUILT_MODULE_LOCATION[0]="module/avl/"
DEST_MODULE_LOCATION[0]="/extra/avl/avl"
BUILT_MODULE_NAME[7]="spl"
BUILT_MODULE_LOCATION[7]="module/spl/"
DEST_MODULE_LOCATION[7]="/extra/spl/spl"
+BUILT_MODULE_NAME[8]="zzstd"
+BUILT_MODULE_LOCATION[8]="module/zstd/"
+DEST_MODULE_LOCATION[8]="/extra/zstd/zzstd"
EOF
KMOD_ICP=${KMOD_ICP:-icp}
KMOD_ZFS=${KMOD_ZFS:-zfs}
KMOD_FREEBSD=${KMOD_FREEBSD:-openzfs}
+KMOD_ZZSTD=${KMOD_ZZSTD:-zzstd}
usage() {
LOADED_MODULES=""
MISSING_MODULES=""
- for KMOD in $KMOD_SPL $KMOD_ZAVL $KMOD_ZNVPAIR \
- $KMOD_ZUNICODE $KMOD_ZCOMMON $KMOD_ZLUA $KMOD_ICP $KMOD_ZFS; do
+ for KMOD in $KMOD_SPL $KMOD_ZAVL $KMOD_ZNVPAIR $KMOD_ZUNICODE $KMOD_ZCOMMON \
+ $KMOD_ZLUA $KMOD_ZZSTD $KMOD_ICP $KMOD_ZFS; do
NAME=$(basename "$KMOD" .ko)
if lsmod | grep -E -q "^${NAME}"; then
fi
for KMOD in $KMOD_SPL $KMOD_ZAVL $KMOD_ZNVPAIR \
- $KMOD_ZUNICODE $KMOD_ZCOMMON $KMOD_ZLUA $KMOD_ICP $KMOD_ZFS; do
+ $KMOD_ZUNICODE $KMOD_ZCOMMON $KMOD_ZLUA $KMOD_ZZSTD \
+ $KMOD_ICP $KMOD_ZFS; do
load_module_linux "$KMOD" || return 1
done
}
unload_modules_linux() {
- for KMOD in $KMOD_ZFS $KMOD_ICP $KMOD_ZLUA $KMOD_ZCOMMON $KMOD_ZUNICODE \
- $KMOD_ZNVPAIR $KMOD_ZAVL $KMOD_SPL; do
+ for KMOD in $KMOD_ZFS $KMOD_ICP $KMOD_ZZSTD $KMOD_ZLUA $KMOD_ZCOMMON \
+ $KMOD_ZUNICODE $KMOD_ZNVPAIR $KMOD_ZAVL $KMOD_SPL; do
NAME=$(basename "$KMOD" .ko)
USE_COUNT=$(lsmod | grep -E "^${NAME} " | awk '{print $3}')
'zdb_006_pos', 'zdb_args_neg', 'zdb_args_pos',
'zdb_block_size_histogram', 'zdb_checksum', 'zdb_decompress',
'zdb_display_block', 'zdb_object_range_neg', 'zdb_object_range_pos',
- 'zdb_objset_id']
+ 'zdb_objset_id', 'zdb_decompress_zstd']
pre =
post =
tags = ['functional', 'cli_root', 'zdb']
'zfs_receive_016_pos', 'receive-o-x_props_override',
'zfs_receive_from_encrypted', 'zfs_receive_to_encrypted',
'zfs_receive_raw', 'zfs_receive_raw_incremental', 'zfs_receive_-e',
- 'zfs_receive_raw_-d']
+ 'zfs_receive_raw_-d', 'zfs_receive_from_zstd']
tags = ['functional', 'cli_root', 'zfs_receive']
[tests/functional/cli_root/zfs_rename]
'user_property_001_pos', 'user_property_003_neg', 'readonly_001_pos',
'user_property_004_pos', 'version_001_neg', 'zfs_set_001_neg',
'zfs_set_002_neg', 'zfs_set_003_neg', 'property_alias_001_pos',
- 'mountpoint_003_pos', 'ro_props_001_pos', 'zfs_set_keylocation']
+ 'mountpoint_003_pos', 'ro_props_001_pos', 'zfs_set_keylocation',
+ 'zfs_set_feature_activation']
tags = ['functional', 'cli_root', 'zfs_set']
[tests/functional/cli_root/zfs_share]
[tests/functional/compression]
tests = ['compress_001_pos', 'compress_002_pos', 'compress_003_pos',
- 'l2arc_compressed_arc', 'l2arc_compressed_arc_disabled']
+ 'l2arc_compressed_arc', 'l2arc_compressed_arc_disabled',
+ 'l2arc_encrypted', 'l2arc_encrypted_no_compressed_arc']
tags = ['functional', 'compression']
[tests/functional/cp_files]
. $STF_SUITE/include/libtest.shlib
-typeset -a compress_prop_vals=('off' 'lzjb' 'lz4' 'gzip' 'zle')
+typeset -a compress_prop_vals=('off' 'lzjb' 'lz4' 'gzip' 'zle' 'zstd')
typeset -a checksum_prop_vals=('on' 'off' 'fletcher2' 'fletcher4' 'sha256'
'noparity' 'sha512' 'skein')
if ! is_freebsd; then
zdb_block_size_histogram.ksh \
zdb_checksum.ksh \
zdb_decompress.ksh \
+ zdb_decompress_zstd.ksh \
zdb_object_range_neg.ksh \
zdb_object_range_pos.ksh \
zdb_display_block.ksh \
"setvprop" "blah blah" "-%" "--?" "-*" "-=" \
"-a" "-f" "-g" "-j" "-n" "-o" "-p" "-p /tmp" "-r" \
"-t" "-w" "-z" "-E" "-H" "-I" "-J" "-K" \
- "-N" "-Q" "-R" "-T" "-W" "-Z"
+ "-N" "-Q" "-R" "-T" "-W"
log_assert "Execute zdb using invalid parameters."
--- /dev/null
+#!/bin/ksh
+
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source. A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+
+#
+# Copyright (c) 2020 The FreeBSD Foundation [1]
+#
+# [1] Portions of this software were developed by Allan Jude
+# under sponsorship from the FreeBSD Foundation.
+
+. $STF_SUITE/include/libtest.shlib
+
+#
+# Description:
+# zdb -Z pool <objid> will display the ZSTD compression header
+# This will contain the actual length of the compressed data, as well as
+# the version of ZSTD used to compress the block, and the compression level
+#
+# Strategy:
+# 1. Create a pool, set compression to zstd-<random level>
+# 2. Write some identifiable data to a file
+# 3. Run zdb -Zddddddbbbbbb against the file
+# 4. Record the DVA, lsize, and psize, and ZSTD header of L0 block 0
+# 5. Check that the ZSTD length is less than psize
+# 6. Check that the ZSTD level matches the level we requested
+# 7. Run zdb -R with :dr flags and confirm the size and content match
+#
+
+function cleanup
+{
+ datasetexists $TESTPOOL && destroy_pool $TESTPOOL
+}
+
+log_assert "Verify zdb -Z (read ZSTD header) works as expected"
+log_onexit cleanup
+src_data="$STF_SUITE/tests/functional/cli_root/zfs_receive/zstd_test_data.txt"
+init_data=$TESTDIR/file1
+write_count=128
+blksize=131072
+verify_runnable "global"
+verify_disk_count "$DISKS" 2
+random_level=$((RANDOM%19 + 1))
+
+default_mirror_setup_noexit $DISKS
+log_must zfs set recordsize=$blksize $TESTPOOL/$TESTFS
+log_must zfs set compression=zstd-$random_level $TESTPOOL/$TESTFS
+
+# write the 1k of text 128 times
+for i in {1..$write_count}
+do
+ cat $src_data >> $init_data
+done
+
+sync_pool $TESTPOOL true
+
+# get object number of file
+listing=$(ls -i $init_data)
+set -A array $listing
+obj=${array[0]}
+log_note "file $init_data has object number $obj"
+
+output=$(zdb -Zddddddbbbbbb $TESTPOOL/$TESTFS $obj 2> /dev/null \
+ |grep -m 1 "L0 DVA" |head -n1)
+dva=$(sed -Ene 's/^.+DVA\[0\]=<([^>]+)>.*$/\1/p' <<< "$output")
+log_note "block 0 of $init_data has a DVA of $dva"
+
+# use the length reported by zdb -ddddddbbbbbb
+size_str=$(sed -Ene 's/^.+ size=([^ ]+) .*$/\1/p' <<< "$output")
+# convert sizes to decimal
+lsize=$(echo $size_str |awk '{split($0,array,"/")} END{print array[1]}')
+lsize_orig=$lsize
+lsize=${lsize%?}
+lsize_bytes=$((16#$lsize))
+psize=$(echo $size_str |awk '{split($0,array,"/")} END{print array[2]}')
+psize_orig=$psize
+psize=${psize%?}
+psize_bytes=$((16#$psize))
+log_note "block size $size_str"
+
+# Get the ZSTD header reported by zdb -Z
+zstd_str=$(sed -Ene 's/^.+ ZSTD:size=([^:]+):version=([^:]+):level=([^:]+):.*$/\1:\2:\3/p' <<< "$output")
+zstd_size=$(echo "$zstd_str" |awk '{split($0,array,":")} END{print array[1]}')
+log_note "ZSTD compressed size $zstd_size"
+(( $psize_bytes < $zstd_size )) && log_fail \
+"zdb -Z failed: physical block size was less than header content length ($psize_bytes < $zstd_size)"
+
+zstd_version=$(echo "$zstd_str" |awk '{split($0,array,":")} END{print array[2]}')
+log_note "ZSTD version $zstd_version"
+
+zstd_level=$(echo "$zstd_str" |awk '{split($0,array,":")} END{print array[3]}')
+log_note "ZSTD level $zstd_level"
+(( $zstd_level != $random_level )) && log_fail \
+"zdb -Z failed: compression level did not match header level ($zstd_level < $random_level)"
+
+vdev=$(echo "$dva" |awk '{split($0,array,":")} END{print array[1]}')
+offset=$(echo "$dva" |awk '{split($0,array,":")} END{print array[2]}')
+# Check the first 1024 bytes
+output=$(ZDB_NO_ZLE="true" zdb -R $TESTPOOL $vdev:$offset:$size_str:dr 2> /dev/null)
+outsize=$(wc -c <<< "$output")
+(( $outsize != $blksize )) && log_fail \
+"zdb -Z failed to decompress the data to the expected length ($outsize != $lsize_bytes)"
+cmp $init_data - <<< "$output"
+(( $? != 0 )) && log_fail "zdb -R :dr failed to decompress the data properly"
+
+log_pass "zdb -Z flag (ZSTD compression header) works as expected"
zfs_receive_016_pos.ksh \
receive-o-x_props_override.ksh \
zfs_receive_from_encrypted.ksh \
+ zfs_receive_from_zstd.ksh \
zfs_receive_to_encrypted.ksh \
zfs_receive_raw.ksh \
zfs_receive_raw_incremental.ksh \
zfs_receive_raw_-d.ksh \
zfs_receive_-e.ksh
+
+dist_pkgdata_DATA = \
+ zstd_test_data.txt
--- /dev/null
+#!/bin/ksh -p
+#
+# CDDL HEADER START
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source. A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2020 The FreeBSD Foundation [1]
+#
+# [1] Portions of this software were developed by Allan Jude
+# under sponsorship from the FreeBSD Foundation.
+
+. $STF_SUITE/include/libtest.shlib
+
+#
+# DESCRIPTION:
+# ZFS should receive a ZSTD compressed block and be able to determine the level
+#
+# STRATEGY:
+# 1. Create a ZSTD compressed dataset (random level)
+# 2. Create and checksum a file on the compressed dataset
+# 3. Snapshot the compressed dataset
+# 4. Attempt to receive the snapshot into a new dataset
+# 5. Verify the checksum of the file is the same as the original
+# 6. Verify the compression level is correctly stored
+#
+
+verify_runnable "both"
+
+function cleanup
+{
+ datasetexists $TESTPOOL/$TESTFS1 && \
+ log_must zfs destroy -r $TESTPOOL/$TESTFS1
+
+ datasetexists $TESTPOOL/$TESTFS2 && \
+ log_must zfs destroy -r $TESTPOOL/$TESTFS2
+}
+
+log_onexit cleanup
+
+log_assert "ZFS should track compression level when receiving a ZSTD stream"
+
+typeset src_data="$STF_SUITE/tests/functional/cli_root/zfs_receive/zstd_test_data.txt"
+typeset snap="$TESTPOOL/$TESTFS1@snap"
+
+random_level=$((RANDOM%19 + 1))
+log_note "Randomly selected ZSTD level: $random_level"
+
+log_must zfs create -o compress=zstd-$random_level $TESTPOOL/$TESTFS1
+# Make a 5kb compressible file
+log_must cat $src_data $src_data $src_data $src_data $src_data \
+ > /$TESTPOOL/$TESTFS1/$TESTFILE0
+typeset checksum=$(md5digest /$TESTPOOL/$TESTFS1/$TESTFILE0)
+
+log_must zfs snapshot $snap
+
+# get object number of file
+listing=$(ls -i /$TESTPOOL/$TESTFS1/$TESTFILE0)
+set -A array $listing
+obj=${array[0]}
+log_note "file /$TESTPOOL/$TESTFS1/$TESTFILE0 has object number $obj"
+
+output=$(zdb -Zddddddbbbbbb $TESTPOOL/$TESTFS1 $obj 2> /dev/null \
+ |grep -m 1 "L0 DVA" |head -n1)
+dva=$(sed -Ene 's/^.+DVA\[0\]=<([^>]+)>.*$/\1/p' <<< "$output")
+log_note "block 0 of /$TESTPOOL/$TESTFS1/$TESTFILE0 has a DVA of $dva"
+
+zstd_str=$(sed -Ene 's/^.+ ZSTD:size=([^:]+):version=([^:]+):level=([^:]+):.*$/\1:\2:\3/p' <<< "$output")
+zstd_size1=$(echo "$zstd_str" |awk '{split($0,array,":")} END{print array[1]}')
+zstd_version1=$(echo "$zstd_str" |awk '{split($0,array,":")} END{print array[2]}')
+zstd_level1=$(echo "$zstd_str" |awk '{split($0,array,":")} END{print array[3]}')
+log_note "ZSTD src: size=$zstd_size1 version=$zstd_version1 level=$zstd_level1"
+
+log_note "Verify ZFS can receive the ZSTD compressed stream"
+log_must eval "zfs send -ec $snap | zfs receive $TESTPOOL/$TESTFS2"
+
+typeset cksum1=$(md5digest /$TESTPOOL/$TESTFS2/$TESTFILE0)
+[[ "$cksum1" == "$checksum" ]] || \
+ log_fail "Checksums differ ($cksum1 != $checksum)"
+
+# get object number of file
+listing=$(ls -i /$TESTPOOL/$TESTFS2/$TESTFILE0)
+set -A array $listing
+obj=${array[0]}
+log_note "file /$TESTPOOL/$TESTFS2/$TESTFILE0 has object number $obj"
+
+output=$(zdb -Zddddddbbbbbb $TESTPOOL/$TESTFS2 $obj 2> /dev/null \
+ |grep -m 1 "L0 DVA" |head -n1)
+dva=$(sed -Ene 's/^.+DVA\[0\]=<([^>]+)>.*$/\1/p' <<< "$output")
+log_note "block 0 of /$TESTPOOL/$TESTFS2/$TESTFILE0 has a DVA of $dva"
+
+zstd_str=$(sed -Ene 's/^.+ ZSTD:size=([^:]+):version=([^:]+):level=([^:]+):.*$/\1:\2:\3/p' <<< "$output")
+zstd_size2=$(echo "$zstd_str" |awk '{split($0,array,":")} END{print array[1]}')
+(( $zstd_size2 != $zstd_size1 )) && log_fail \
+"ZFS recv failed: compressed size differs ($zstd_size2 != $zstd_size1)"
+zstd_version2=$(echo "$zstd_str" |awk '{split($0,array,":")} END{print array[2]}')
+zstd_level2=$(echo "$zstd_str" |awk '{split($0,array,":")} END{print array[3]}')
+log_note "ZSTD dest: size=$zstd_size2 version=$zstd_version2 level=$zstd_level2"
+(( $zstd_level2 != $zstd_level1 )) && log_fail \
+"ZFS recv failed: compression level did not match header level ($zstd_level2 != $zstd_level1)"
+
+log_pass "ZFS can receive a ZSTD stream and determine the compression level"
--- /dev/null
+Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum. Sed ut perspiciatis unde omnis iste natus error sit voluptatem accusantium doloremque laudantium, totam rem aperiam, eaque ipsa quae ab illo inventore veritatis et quasi architecto beatae vitae dicta sunt explicabo. Nemo enim ipsam voluptatem quia voluptas sit aspernatur aut odit aut fugit, sed quia consequuntur magni dolores eos qui ratione voluptatem sequi nesciunt. Neque porro quisquam est, qui dolorem ipsum quia dolor sit amet, consectetur, adipisci velit, sed quia non numquam eius modi tempora incidunt ut labore et dolore magnam aliquam quaerat voluptatem. Ut enim..
zfs_set_001_neg.ksh \
zfs_set_002_neg.ksh \
zfs_set_003_neg.ksh \
- zfs_set_keylocation.ksh
+ zfs_set_keylocation.ksh \
+ zfs_set_feature_activation.ksh
dist_pkgdata_DATA = \
zfs_set_common.kshlib
--- /dev/null
+#!/bin/ksh -p
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or http://www.opensolaris.org/os/licensing.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2020 The FreeBSD Foundation [1]
+#
+# [1] Portions of this software were developed by Allan Jude
+# under sponsorship from the FreeBSD Foundation.
+
+. $STF_SUITE/include/libtest.shlib
+
+#
+# DESCRIPTION:
+# Setting the compression property to any of the zstd levels should activate
+# the zstd feature flag. Destroying the last dataset using the zstd feature flag
+# should revert the feature to the 'enabled' state.
+#
+# STRATEGY:
+# 1. Create pool, then create a file system within it.
+# 2. Check that the zstd feature flag is 'enabled'.
+# 3. Setting the compression property to zstd.
+# 4. Check that the zstd feature flag is now 'active'.
+# 5. Destroy the dataset
+# 6. Confirm that the feature flag reverts to the 'enabled' state.
+#
+
+verify_runnable "both"
+
+log_assert "Setting compression=zstd should activate the"\
+ "org.freebsd:zstd_compress feature flag, and destroying the last"\
+ "dataset using that property, should revert the feature flag to"\
+ "the enabled state."
+
+export VDEV_ZSTD="$TEST_BASE_DIR/vdev-zstd"
+
+function cleanup
+{
+ if poolexists $TESTPOOL-zstd ; then
+ destroy_pool $TESTPOOL-zstd
+ fi
+
+ rm $VDEV_ZSTD
+}
+log_onexit cleanup
+
+log_must truncate -s $SPA_MINDEVSIZE $VDEV_ZSTD
+log_must zpool create $TESTPOOL-zstd $VDEV_ZSTD
+
+featureval="$(get_pool_prop feature@zstd_compress $TESTPOOL-zstd)"
+
+[[ "$featureval" == "disabled" ]] && \
+ log_unsupported "ZSTD feature flag unsupposed"
+
+[[ "$featureval" == "active" ]] && \
+ log_unsupported "ZSTD feature already active before test"
+
+random_level=$((RANDOM%19 + 1))
+log_note "Randomly selected ZSTD level: $random_level"
+
+log_must zfs create -o compress=zstd-$random_level $TESTPOOL-zstd/$TESTFS-zstd
+
+featureval="$(get_pool_prop feature@zstd_compress $TESTPOOL-zstd)"
+
+log_note "After zfs set, feature flag value is: $featureval"
+
+[[ "$featureval" == "active" ]] ||
+ log_fail "ZSTD feature flag not activated"
+
+log_must zfs destroy $TESTPOOL-zstd/$TESTFS-zstd
+
+featureval="$(get_pool_prop feature@zstd_compress $TESTPOOL-zstd)"
+
+log_note "After zfs destroy, feature flag value is: $featureval"
+
+[[ "$featureval" == "enabled" ]] ||
+ log_fail "ZSTD feature flag not deactivated"
+
+log_pass "Setting compression=zstd activated the feature flag, and"\
+ "destroying the dataset deactivated it."
"feature@resilver_defer"
"feature@bookmark_v2"
"feature@livelist"
+ "feature@zstd_compress"
)
fi
compress_003_pos.ksh \
compress_004_pos.ksh \
l2arc_compressed_arc.ksh \
- l2arc_compressed_arc_disabled.ksh
+ l2arc_compressed_arc_disabled.ksh \
+ l2arc_encrypted.ksh \
+ l2arc_encrypted_no_compressed_arc.ksh
dist_pkgdata_DATA = \
compress.cfg
--- /dev/null
+#!/bin/ksh -p
+#
+# CDDL HEADER START
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source. A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2020 The FreeBSD Foundation [1]
+#
+# [1] Portions of this software were developed by Allan Jude
+# under sponsorship from the FreeBSD Foundation.
+
+. $STF_SUITE/include/libtest.shlib
+
+export SIZE=1G
+export VDIR=$TESTDIR/disk.persist_l2arc
+export VDEV="$VDIR/a"
+export VDEV_CACHE="$VDIR/b"
+export PASSPHRASE="password"
+
+# fio options
+export DIRECTORY=/$TESTPOOL-l2arc/encrypted
+export NUMJOBS=4
+export RUNTIME=30
+export PERF_RANDSEED=1234
+export PERF_COMPPERCENT=66
+export PERF_COMPCHUNK=0
+export BLOCKSIZE=128K
+export SYNC_TYPE=0
+export DIRECT=1
+
+#
+# DESCRIPTION:
+# System with compressed_arc disabled succeeds at reading from L2ARC
+#
+# STRATEGY:
+# 1. Enable compressed_arc.
+# 2. Create pool with a cache device, encryption, and compression enabled.
+# 3. Read the number of L2ARC checksum failures.
+# 4. Create a random file in that pool and random read for 30 sec.
+# 5. Read the number of L2ARC checksum failures.
+#
+
+verify_runnable "global"
+
+log_assert "L2ARC with encryption enabled succeeds."
+
+origin_carc_setting=$(get_tunable COMPRESSED_ARC_ENABLED)
+
+function cleanup
+{
+ if poolexists $TESTPOOL-l2arc ; then
+ destroy_pool $TESTPOOL-l2arc
+ fi
+
+ log_must set_tunable64 COMPRESSED_ARC_ENABLED $origin_carc_setting
+}
+log_onexit cleanup
+
+# Enable Compressed ARC so that in-ARC and on-disk will match
+log_must set_tunable64 COMPRESSED_ARC_ENABLED 1
+
+log_must rm -rf $VDIR
+log_must mkdir -p $VDIR
+log_must mkfile $SIZE $VDEV
+
+typeset fill_mb=800
+typeset cache_sz=$(( floor($fill_mb / 2) ))
+export FILE_SIZE=$(( floor($fill_mb / $NUMJOBS) ))M
+
+log_must truncate -s ${cache_sz}M $VDEV_CACHE
+
+log_must zpool create -O compression=zstd -f $TESTPOOL-l2arc $VDEV cache $VDEV_CACHE
+
+log_must eval "echo $PASSPHRASE | zfs create -o compression=zstd " \
+ "-o encryption=on -o keyformat=passphrase -o keylocation=prompt " \
+ "$TESTPOOL-l2arc/encrypted"
+
+l2_cksum_bad_start=$(get_arcstat l2_cksum_bad)
+
+log_must fio $FIO_SCRIPTS/mkfiles.fio
+log_must fio $FIO_SCRIPTS/random_reads.fio
+
+l2_cksum_bad_end=$(get_arcstat l2_cksum_bad)
+
+log_note "L2ARC Failed Checksums before: $l2_cksum_bad_start After:"\
+ "$l2_cksum_bad_end"
+log_must test $(( $l2_cksum_bad_end - $l2_cksum_bad_start )) -eq 0
+
+log_must zpool destroy -f $TESTPOOL-l2arc
+
+log_pass "L2ARC with encryption and compressed_arc enabled does not result in"\
+ "checksum errors."
--- /dev/null
+#!/bin/ksh -p
+#
+# CDDL HEADER START
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source. A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2020 The FreeBSD Foundation [1]
+#
+# [1] Portions of this software were developed by Allan Jude
+# under sponsorship from the FreeBSD Foundation.
+
+. $STF_SUITE/include/libtest.shlib
+
+export SIZE=1G
+export VDIR=$TESTDIR/disk.persist_l2arc
+export VDEV="$VDIR/a"
+export VDEV_CACHE="$VDIR/b"
+export PASSPHRASE="password"
+
+# fio options
+export DIRECTORY=/$TESTPOOL-l2arc/encrypted
+export NUMJOBS=4
+export RUNTIME=30
+export PERF_RANDSEED=1234
+export PERF_COMPPERCENT=66
+export PERF_COMPCHUNK=0
+export BLOCKSIZE=128K
+export SYNC_TYPE=0
+export DIRECT=1
+
+#
+# DESCRIPTION:
+# System with compressed_arc disabled succeeds at reading from L2ARC
+#
+# STRATEGY:
+# 1. Disable compressed_arc.
+# 2. Create pool with a cache device, encryption, and compression enabled.
+# 3. Read the number of L2ARC checksum failures.
+# 4. Create a random file in that pool and random read for 30 sec.
+# 5. Read the number of L2ARC checksum failures.
+#
+
+verify_runnable "global"
+
+log_assert "L2ARC with compressed_arc disabled succeeds."
+
+origin_carc_setting=$(get_tunable COMPRESSED_ARC_ENABLED)
+
+function cleanup
+{
+ if poolexists $TESTPOOL-l2arc ; then
+ destroy_pool $TESTPOOL-l2arc
+ fi
+
+ log_must set_tunable64 COMPRESSED_ARC_ENABLED $origin_carc_setting
+}
+log_onexit cleanup
+
+log_must rm -rf $VDIR
+log_must mkdir -p $VDIR
+log_must mkfile $SIZE $VDEV
+
+# Disable Compressed ARC so that in-ARC and on-disk will not match
+log_must set_tunable64 COMPRESSED_ARC_ENABLED 0
+
+typeset fill_mb=800
+typeset cache_sz=$(( floor($fill_mb / 2) ))
+export FILE_SIZE=$(( floor($fill_mb / $NUMJOBS) ))M
+
+log_must truncate -s ${cache_sz}M $VDEV_CACHE
+
+log_must zpool create -O compression=zstd -f $TESTPOOL-l2arc $VDEV cache $VDEV_CACHE
+
+log_must eval "echo $PASSPHRASE | zfs create -o compression=zstd " \
+ "-o encryption=on -o keyformat=passphrase -o keylocation=prompt " \
+ "$TESTPOOL-l2arc/encrypted"
+
+l2_cksum_bad_start=$(get_arcstat l2_cksum_bad)
+
+log_must fio $FIO_SCRIPTS/mkfiles.fio
+log_must fio $FIO_SCRIPTS/random_reads.fio
+
+l2_cksum_bad_end=$(get_arcstat l2_cksum_bad)
+
+log_note "L2ARC Failed Checksums before: $l2_cksum_bad_start After:"\
+ "$l2_cksum_bad_end"
+log_must test $(( $l2_cksum_bad_end - $l2_cksum_bad_start )) -eq 0
+
+log_must zpool destroy -f $TESTPOOL-l2arc
+
+log_pass "L2ARC with encryption enabled and compressed_arc disabled does not"\
+ "result in checksum errors."
canmount off canmount on
xattr on xattr off
compression gzip compression gzip-$((RANDOM%9 + 1))
- copies $((RANDOM%3 + 1))
+ compression zstd compression zstd-$((RANDOM%9 + 1))
+ compression zstd-fast copies $((RANDOM%3 + 1))
+ compression zstd-fast-$((RANDOM%9 + 1))
)
elif is_freebsd; then
# property value property value
aclinherit secure aclinherit passthrough
canmount off canmount on
compression gzip compression gzip-$((RANDOM%9 + 1))
- copies $((RANDOM%3 + 1))
+ compression zstd compression zstd-$((RANDOM%9 + 1))
+ compression zstd-fast copies $((RANDOM%3 + 1))
+ compression zstd-fast-$((RANDOM%9 + 1))
)
else
# property value property value
projects / mirror_zfs.git / commitdiff