]> git.proxmox.com Git - mirror_zfs.git/blobdiff - cmd/zfs/zfs_main.c
OpenZFS 7614, 9064 - zfs device evacuation/removal
[mirror_zfs.git] / cmd / zfs / zfs_main.c
index 0f00688b359856d3c0e0d7fbb15460ab859594d7..45d9a661b0d5c125dd8e8cc19201042692d9baf9 100644 (file)
 
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright 2011 Nexenta Systems, Inc. All rights reserved.
+ * Copyright (c) 2011, 2017 by Delphix. All rights reserved.
+ * Copyright 2012 Milan Jurik. All rights reserved.
+ * Copyright (c) 2012, Joyent, Inc. All rights reserved.
+ * Copyright (c) 2013 Steven Hartland.  All rights reserved.
+ * Copyright 2016 Igor Kozhukhov <ikozhukhov@gmail.com>.
+ * Copyright 2016 Nexenta Systems, Inc.
+ * Copyright (c) 2018 Datto Inc.
  */
 
 #include <assert.h>
 #include <ctype.h>
 #include <errno.h>
+#include <getopt.h>
 #include <libgen.h>
 #include <libintl.h>
 #include <libuutil.h>
@@ -42,6 +49,7 @@
 #include <grp.h>
 #include <pwd.h>
 #include <signal.h>
+#include <sys/debug.h>
 #include <sys/list.h>
 #include <sys/mkdev.h>
 #include <sys/mntent.h>
 #include <sys/mount.h>
 #include <sys/stat.h>
 #include <sys/fs/zfs.h>
+#include <sys/systeminfo.h>
 #include <sys/types.h>
 #include <time.h>
+#include <sys/zfs_project.h>
 
 #include <libzfs.h>
+#include <libzfs_core.h>
 #include <zfs_prop.h>
 #include <zfs_deleg.h>
 #include <libuutil.h>
 #include "zfs_iter.h"
 #include "zfs_util.h"
 #include "zfs_comutil.h"
+#include "libzfs_impl.h"
+#include "zfs_projectutil.h"
 
 libzfs_handle_t *g_zfs;
 
 static FILE *mnttab_file;
 static char history_str[HIS_MAX_RECORD_LEN];
+static boolean_t log_history = B_TRUE;
 
 static int zfs_do_clone(int argc, char **argv);
 static int zfs_do_create(int argc, char **argv);
@@ -95,6 +109,13 @@ static int zfs_do_hold(int argc, char **argv);
 static int zfs_do_holds(int argc, char **argv);
 static int zfs_do_release(int argc, char **argv);
 static int zfs_do_diff(int argc, char **argv);
+static int zfs_do_bookmark(int argc, char **argv);
+static int zfs_do_channel_program(int argc, char **argv);
+static int zfs_do_remap(int argc, char **argv);
+static int zfs_do_load_key(int argc, char **argv);
+static int zfs_do_unload_key(int argc, char **argv);
+static int zfs_do_change_key(int argc, char **argv);
+static int zfs_do_project(int argc, char **argv);
 
 /*
  * Enable a reasonable set of defaults for libumem debugging on DEBUG builds.
@@ -137,10 +158,18 @@ typedef enum {
        HELP_UNALLOW,
        HELP_USERSPACE,
        HELP_GROUPSPACE,
+       HELP_PROJECTSPACE,
+       HELP_PROJECT,
        HELP_HOLD,
        HELP_HOLDS,
        HELP_RELEASE,
-       HELP_DIFF
+       HELP_DIFF,
+       HELP_REMAP,
+       HELP_BOOKMARK,
+       HELP_CHANNEL_PROGRAM,
+       HELP_LOAD_KEY,
+       HELP_UNLOAD_KEY,
+       HELP_CHANGE_KEY,
 } zfs_help_t;
 
 typedef struct zfs_command {
@@ -167,6 +196,8 @@ static zfs_command_t command_table[] = {
        { "clone",      zfs_do_clone,           HELP_CLONE              },
        { "promote",    zfs_do_promote,         HELP_PROMOTE            },
        { "rename",     zfs_do_rename,          HELP_RENAME             },
+       { "bookmark",   zfs_do_bookmark,        HELP_BOOKMARK           },
+       { "program",    zfs_do_channel_program, HELP_CHANNEL_PROGRAM    },
        { NULL },
        { "list",       zfs_do_list,            HELP_LIST               },
        { NULL },
@@ -174,8 +205,12 @@ static zfs_command_t command_table[] = {
        { "get",        zfs_do_get,             HELP_GET                },
        { "inherit",    zfs_do_inherit,         HELP_INHERIT            },
        { "upgrade",    zfs_do_upgrade,         HELP_UPGRADE            },
+       { NULL },
        { "userspace",  zfs_do_userspace,       HELP_USERSPACE          },
        { "groupspace", zfs_do_userspace,       HELP_GROUPSPACE         },
+       { "projectspace", zfs_do_userspace,     HELP_PROJECTSPACE       },
+       { NULL },
+       { "project",    zfs_do_project,         HELP_PROJECT            },
        { NULL },
        { "mount",      zfs_do_mount,           HELP_MOUNT              },
        { "unmount",    zfs_do_unmount,         HELP_UNMOUNT            },
@@ -193,6 +228,10 @@ static zfs_command_t command_table[] = {
        { "holds",      zfs_do_holds,           HELP_HOLDS              },
        { "release",    zfs_do_release,         HELP_RELEASE            },
        { "diff",       zfs_do_diff,            HELP_DIFF               },
+       { "remap",      zfs_do_remap,           HELP_REMAP              },
+       { "load-key",   zfs_do_load_key,        HELP_LOAD_KEY           },
+       { "unload-key", zfs_do_unload_key,      HELP_UNLOAD_KEY         },
+       { "change-key", zfs_do_change_key,      HELP_CHANGE_KEY         },
 };
 
 #define        NCOMMAND        (sizeof (command_table) / sizeof (command_table[0]))
@@ -212,13 +251,16 @@ get_usage(zfs_help_t idx)
                    "\tcreate [-ps] [-b blocksize] [-o property=value] ... "
                    "-V <size> <volume>\n"));
        case HELP_DESTROY:
-               return (gettext("\tdestroy [-rRf] <filesystem|volume>\n"
-                   "\tdestroy [-rRd] <snapshot>\n"));
+               return (gettext("\tdestroy [-fnpRrv] <filesystem|volume>\n"
+                   "\tdestroy [-dnpRrv] "
+                   "<filesystem|volume>@<snap>[%<snap>][,...]\n"
+                   "\tdestroy <filesystem|volume>#<bookmark>\n"));
        case HELP_GET:
                return (gettext("\tget [-rHp] [-d max] "
-                   "[-o \"all\" | field[,...]] [-s source[,...]]\n"
+                   "[-o \"all\" | field[,...]]\n"
+                   "\t    [-t type[,...]] [-s source[,...]]\n"
                    "\t    <\"all\" | property[,...]> "
-                   "[filesystem|volume|snapshot] ...\n"));
+                   "[filesystem|volume|snapshot|bookmark] ...\n"));
        case HELP_INHERIT:
                return (gettext("\tinherit [-rS] <property> "
                    "<filesystem|volume|snapshot> ...\n"));
@@ -226,42 +268,49 @@ get_usage(zfs_help_t idx)
                return (gettext("\tupgrade [-v]\n"
                    "\tupgrade [-r] [-V version] <-a | filesystem ...>\n"));
        case HELP_LIST:
-               return (gettext("\tlist [-rH][-d max] "
-                   "[-o property[,...]] [-t type[,...]] [-s property] ...\n"
-                   "\t    [-S property] ... "
-                   "[filesystem|volume|snapshot|snap] ...\n"));
+               return (gettext("\tlist [-Hp] [-r|-d max] [-o property[,...]] "
+                   "[-s property]...\n\t    [-S property]... [-t type[,...]] "
+                   "[filesystem|volume|snapshot] ...\n"));
        case HELP_MOUNT:
                return (gettext("\tmount\n"
-                   "\tmount [-vO] [-o opts] <-a | filesystem>\n"));
+                   "\tmount [-lvO] [-o opts] <-a | filesystem>\n"));
        case HELP_PROMOTE:
                return (gettext("\tpromote <clone-filesystem>\n"));
        case HELP_RECEIVE:
-               return (gettext("\treceive [-vnFu] <filesystem|volume|"
-               "snapshot>\n"
-               "\treceive [-vnFu] [-d | -e] <filesystem>\n"));
+               return (gettext("\treceive [-vnsFu] "
+                   "[-o <property>=<value>] ... [-x <property>] ...\n"
+                   "\t    <filesystem|volume|snapshot>\n"
+                   "\treceive [-vnsFu] [-o <property>=<value>] ... "
+                   "[-x <property>] ... \n"
+                   "\t    [-d | -e] <filesystem>\n"
+                   "\treceive -A <filesystem|volume>\n"));
        case HELP_RENAME:
-               return (gettext("\trename <filesystem|volume|snapshot> "
+               return (gettext("\trename [-f] <filesystem|volume|snapshot> "
                    "<filesystem|volume|snapshot>\n"
-                   "\trename -p <filesystem|volume> <filesystem|volume>\n"
-                   "\trename -r <snapshot> <snapshot>"));
+                   "\trename [-f] -p <filesystem|volume> <filesystem|volume>\n"
+                   "\trename -r <snapshot> <snapshot>\n"));
        case HELP_ROLLBACK:
                return (gettext("\trollback [-rRf] <snapshot>\n"));
        case HELP_SEND:
-               return (gettext("\tsend [-vRDp] [-[iI] snapshot] <snapshot>\n"));
+               return (gettext("\tsend [-DnPpRvLecwb] [-[i|I] snapshot] "
+                   "<snapshot>\n"
+                   "\tsend [-nvPLecw] [-i snapshot|bookmark] "
+                   "<filesystem|volume|snapshot>\n"
+                   "\tsend [-nvPe] -t <receive_resume_token>\n"));
        case HELP_SET:
-               return (gettext("\tset <property=value> "
+               return (gettext("\tset <property=value> ... "
                    "<filesystem|volume|snapshot> ...\n"));
        case HELP_SHARE:
-               return (gettext("\tshare <-a | filesystem>\n"));
+               return (gettext("\tshare [-l] <-a [nfs|smb] | filesystem>\n"));
        case HELP_SNAPSHOT:
-               return (gettext("\tsnapshot|snap [-r] [-o property=value] ... "
-                   "<filesystem@snapname|volume@snapname>\n"));
+               return (gettext("\tsnapshot [-r] [-o property=value] ... "
+                   "<filesystem|volume>@<snap> ...\n"));
        case HELP_UNMOUNT:
                return (gettext("\tunmount [-f] "
                    "<-a | filesystem|mountpoint>\n"));
        case HELP_UNSHARE:
                return (gettext("\tunshare "
-                   "<-a | filesystem|mountpoint>\n"));
+                   "<-a [nfs|smb] | filesystem|mountpoint>\n"));
        case HELP_ALLOW:
                return (gettext("\tallow <filesystem|volume>\n"
                    "\tallow [-ldug] "
@@ -283,13 +332,24 @@ get_usage(zfs_help_t idx)
                    "\tunallow [-r] -s @setname [<perm|@setname>[,...]] "
                    "<filesystem|volume>\n"));
        case HELP_USERSPACE:
-               return (gettext("\tuserspace [-hniHp] [-o field[,...]] "
-                   "[-sS field] ... [-t type[,...]]\n"
-                   "\t    <filesystem|snapshot>\n"));
+               return (gettext("\tuserspace [-Hinp] [-o field[,...]] "
+                   "[-s field] ...\n"
+                   "\t    [-S field] ... [-t type[,...]] "
+                   "<filesystem|snapshot>\n"));
        case HELP_GROUPSPACE:
-               return (gettext("\tgroupspace [-hniHpU] [-o field[,...]] "
-                   "[-sS field] ... [-t type[,...]]\n"
-                   "\t    <filesystem|snapshot>\n"));
+               return (gettext("\tgroupspace [-Hinp] [-o field[,...]] "
+                   "[-s field] ...\n"
+                   "\t    [-S field] ... [-t type[,...]] "
+                   "<filesystem|snapshot>\n"));
+       case HELP_PROJECTSPACE:
+               return (gettext("\tprojectspace [-Hp] [-o field[,...]] "
+                   "[-s field] ... \n"
+                   "\t    [-S field] ... <filesystem|snapshot>\n"));
+       case HELP_PROJECT:
+               return (gettext("\tproject [-d|-r] <directory|file ...>\n"
+                   "\tproject -c [-0] [-d|-r] [-p id] <directory|file ...>\n"
+                   "\tproject -C [-k] [-r] <directory ...>\n"
+                   "\tproject [-p id] [-r] [-s] <directory ...>\n"));
        case HELP_HOLD:
                return (gettext("\thold [-r] <tag> <snapshot> ...\n"));
        case HELP_HOLDS:
@@ -299,6 +359,25 @@ get_usage(zfs_help_t idx)
        case HELP_DIFF:
                return (gettext("\tdiff [-FHt] <snapshot> "
                    "[snapshot|filesystem]\n"));
+       case HELP_REMAP:
+               return (gettext("\tremap <filesystem | volume>\n"));
+       case HELP_BOOKMARK:
+               return (gettext("\tbookmark <snapshot> <bookmark>\n"));
+       case HELP_CHANNEL_PROGRAM:
+               return (gettext("\tprogram [-jn] [-t <instruction limit>] "
+                   "[-m <memory limit (b)>] <pool> <program file> "
+                   "[lua args...]\n"));
+       case HELP_LOAD_KEY:
+               return (gettext("\tload-key [-rn] [-L <keylocation>] "
+                   "<-a | filesystem|volume>\n"));
+       case HELP_UNLOAD_KEY:
+               return (gettext("\tunload-key [-r] "
+                   "<-a | filesystem|volume>\n"));
+       case HELP_CHANGE_KEY:
+               return (gettext("\tchange-key [-l] [-o keyformat=<value>]\n"
+                   "\t    [-o keylocation=<value>] [-o pbkfd2iters=<value>]\n"
+                   "\t    <filesystem|volume>\n"
+                   "\tchange-key -i [-l] <filesystem|volume>\n"));
        }
 
        abort();
@@ -327,6 +406,18 @@ safe_malloc(size_t size)
        return (data);
 }
 
+void *
+safe_realloc(void *data, size_t size)
+{
+       void *newp;
+       if ((newp = realloc(data, size)) == NULL) {
+               free(data);
+               nomem();
+       }
+
+       return (newp);
+}
+
 static char *
 safe_strdup(char *str)
 {
@@ -422,18 +513,36 @@ usage(boolean_t requested)
                (void) fprintf(fp, " NO       NO   <size>\n");
                (void) fprintf(fp, "\t%-15s ", "groupused@...");
                (void) fprintf(fp, " NO       NO   <size>\n");
+               (void) fprintf(fp, "\t%-15s ", "projectused@...");
+               (void) fprintf(fp, " NO       NO   <size>\n");
+               (void) fprintf(fp, "\t%-15s ", "userobjused@...");
+               (void) fprintf(fp, " NO       NO   <size>\n");
+               (void) fprintf(fp, "\t%-15s ", "groupobjused@...");
+               (void) fprintf(fp, " NO       NO   <size>\n");
+               (void) fprintf(fp, "\t%-15s ", "projectobjused@...");
+               (void) fprintf(fp, " NO       NO   <size>\n");
                (void) fprintf(fp, "\t%-15s ", "userquota@...");
                (void) fprintf(fp, "YES       NO   <size> | none\n");
                (void) fprintf(fp, "\t%-15s ", "groupquota@...");
                (void) fprintf(fp, "YES       NO   <size> | none\n");
+               (void) fprintf(fp, "\t%-15s ", "projectquota@...");
+               (void) fprintf(fp, "YES       NO   <size> | none\n");
+               (void) fprintf(fp, "\t%-15s ", "userobjquota@...");
+               (void) fprintf(fp, "YES       NO   <size> | none\n");
+               (void) fprintf(fp, "\t%-15s ", "groupobjquota@...");
+               (void) fprintf(fp, "YES       NO   <size> | none\n");
+               (void) fprintf(fp, "\t%-15s ", "projectobjquota@...");
+               (void) fprintf(fp, "YES       NO   <size> | none\n");
+               (void) fprintf(fp, "\t%-15s ", "written@<snap>");
+               (void) fprintf(fp, " NO       NO   <size>\n");
 
                (void) fprintf(fp, gettext("\nSizes are specified in bytes "
                    "with standard units such as K, M, G, etc.\n"));
                (void) fprintf(fp, gettext("\nUser-defined properties can "
                    "be specified by using a name containing a colon (:).\n"));
-               (void) fprintf(fp, gettext("\nThe {user|group}{used|quota}@ "
-                   "properties must be appended with\n"
-                   "a user or group specifier of one of these forms:\n"
+               (void) fprintf(fp, gettext("\nThe {user|group|project}"
+                   "[obj]{used|quota}@ properties must be appended with\n"
+                   "a user|group|project specifier of one of these forms:\n"
                    "    POSIX name      (eg: \"matt\")\n"
                    "    POSIX id        (eg: \"126829\")\n"
                    "    SMB name@domain (eg: \"matt@sun\")\n"
@@ -458,27 +567,52 @@ usage(boolean_t requested)
        exit(requested ? 0 : 2);
 }
 
-static int
-parseprop(nvlist_t *props)
+/*
+ * Take a property=value argument string and add it to the given nvlist.
+ * Modifies the argument inplace.
+ */
+static boolean_t
+parseprop(nvlist_t *props, char *propname)
 {
-       char *propname = optarg;
-       char *propval, *strval;
+       char *propval;
 
        if ((propval = strchr(propname, '=')) == NULL) {
                (void) fprintf(stderr, gettext("missing "
-                   "'=' for -o option\n"));
-               return (-1);
+                   "'=' for property=value argument\n"));
+               return (B_FALSE);
        }
        *propval = '\0';
        propval++;
-       if (nvlist_lookup_string(props, propname, &strval) == 0) {
+       if (nvlist_exists(props, propname)) {
                (void) fprintf(stderr, gettext("property '%s' "
                    "specified multiple times\n"), propname);
-               return (-1);
+               return (B_FALSE);
        }
        if (nvlist_add_string(props, propname, propval) != 0)
                nomem();
-       return (0);
+       return (B_TRUE);
+}
+
+/*
+ * Take a property name argument and add it to the given nvlist.
+ * Modifies the argument inplace.
+ */
+static boolean_t
+parsepropname(nvlist_t *props, char *propname)
+{
+       if (strchr(propname, '=') != NULL) {
+               (void) fprintf(stderr, gettext("invalid character "
+                   "'=' in property argument\n"));
+               return (B_FALSE);
+       }
+       if (nvlist_exists(props, propname)) {
+               (void) fprintf(stderr, gettext("property '%s' "
+                   "specified multiple times\n"), propname);
+               return (B_FALSE);
+       }
+       if (nvlist_add_boolean(props, propname) != 0)
+               nomem();
+       return (B_TRUE);
 }
 
 static int
@@ -556,6 +690,56 @@ finish_progress(char *done)
        pt_header = NULL;
 }
 
+static int
+zfs_mount_and_share(libzfs_handle_t *hdl, const char *dataset, zfs_type_t type)
+{
+       zfs_handle_t *zhp = NULL;
+       int ret = 0;
+
+       zhp = zfs_open(hdl, dataset, type);
+       if (zhp == NULL)
+               return (1);
+
+       /*
+        * Volumes may neither be mounted or shared.  Potentially in the
+        * future filesystems detected on these volumes could be mounted.
+        */
+       if (zfs_get_type(zhp) == ZFS_TYPE_VOLUME) {
+               zfs_close(zhp);
+               return (0);
+       }
+
+       /*
+        * Mount and/or share the new filesystem as appropriate.  We provide a
+        * verbose error message to let the user know that their filesystem was
+        * in fact created, even if we failed to mount or share it.
+        *
+        * If the user doesn't want the dataset automatically mounted, then
+        * skip the mount/share step
+        */
+       if (zfs_prop_valid_for_type(ZFS_PROP_CANMOUNT, type, B_FALSE) &&
+           zfs_prop_get_int(zhp, ZFS_PROP_CANMOUNT) == ZFS_CANMOUNT_ON) {
+               if (geteuid() != 0) {
+                       (void) fprintf(stderr, gettext("filesystem "
+                           "successfully created, but it may only be "
+                           "mounted by root\n"));
+                       ret = 1;
+               } else if (zfs_mount(zhp, NULL, 0) != 0) {
+                       (void) fprintf(stderr, gettext("filesystem "
+                           "successfully created, but not mounted\n"));
+                       ret = 1;
+               } else if (zfs_share(zhp) != 0) {
+                       (void) fprintf(stderr, gettext("filesystem "
+                           "successfully created, but not shared\n"));
+                       ret = 1;
+               }
+       }
+
+       zfs_close(zhp);
+
+       return (ret);
+}
+
 /*
  * zfs clone [-p] [-o prop=value] ... <snap> <fs | vol>
  *
@@ -582,8 +766,10 @@ zfs_do_clone(int argc, char **argv)
        while ((c = getopt(argc, argv, "o:p")) != -1) {
                switch (c) {
                case 'o':
-                       if (parseprop(props))
+                       if (!parseprop(props, optarg)) {
+                               nvlist_free(props);
                                return (1);
+                       }
                        break;
                case 'p':
                        parents = B_TRUE;
@@ -615,8 +801,10 @@ zfs_do_clone(int argc, char **argv)
        }
 
        /* open the source dataset */
-       if ((zhp = zfs_open(g_zfs, argv[0], ZFS_TYPE_SNAPSHOT)) == NULL)
+       if ((zhp = zfs_open(g_zfs, argv[0], ZFS_TYPE_SNAPSHOT)) == NULL) {
+               nvlist_free(props);
                return (1);
+       }
 
        if (parents && zfs_name_valid(argv[1], ZFS_TYPE_FILESYSTEM |
            ZFS_TYPE_VOLUME)) {
@@ -626,10 +814,16 @@ zfs_do_clone(int argc, char **argv)
                 * complain.
                 */
                if (zfs_dataset_exists(g_zfs, argv[1], ZFS_TYPE_FILESYSTEM |
-                   ZFS_TYPE_VOLUME))
+                   ZFS_TYPE_VOLUME)) {
+                       zfs_close(zhp);
+                       nvlist_free(props);
                        return (0);
-               if (zfs_create_ancestors(g_zfs, argv[1]) != 0)
+               }
+               if (zfs_create_ancestors(g_zfs, argv[1]) != 0) {
+                       zfs_close(zhp);
+                       nvlist_free(props);
                        return (1);
+               }
        }
 
        /* pass to libzfs */
@@ -637,15 +831,12 @@ zfs_do_clone(int argc, char **argv)
 
        /* create the mountpoint if necessary */
        if (ret == 0) {
-               zfs_handle_t *clone;
-
-               clone = zfs_open(g_zfs, argv[1], ZFS_TYPE_DATASET);
-               if (clone != NULL) {
-                       if (zfs_get_type(clone) != ZFS_TYPE_VOLUME)
-                               if ((ret = zfs_mount(clone, NULL, 0)) == 0)
-                                       ret = zfs_share(clone);
-                       zfs_close(clone);
+               if (log_history) {
+                       (void) zpool_log_history(g_zfs, history_str);
+                       log_history = B_FALSE;
                }
+
+               ret = zfs_mount_and_share(g_zfs, argv[1], ZFS_TYPE_DATASET);
        }
 
        zfs_close(zhp);
@@ -654,8 +845,7 @@ zfs_do_clone(int argc, char **argv)
        return (!!ret);
 
 usage:
-       if (zhp)
-               zfs_close(zhp);
+       ASSERT3P(zhp, ==, NULL);
        nvlist_free(props);
        usage(B_FALSE);
        return (-1);
@@ -680,7 +870,6 @@ static int
 zfs_do_create(int argc, char **argv)
 {
        zfs_type_t type = ZFS_TYPE_FILESYSTEM;
-       zfs_handle_t *zhp = NULL;
        uint64_t volsize = 0;
        int c;
        boolean_t noreserve = B_FALSE;
@@ -689,7 +878,6 @@ zfs_do_create(int argc, char **argv)
        int ret = 1;
        nvlist_t *props;
        uint64_t intval;
-       int canmount = ZFS_CANMOUNT_OFF;
 
        if (nvlist_alloc(&props, NV_UNIQUE_NAME, 0) != 0)
                nomem();
@@ -729,7 +917,7 @@ zfs_do_create(int argc, char **argv)
                                nomem();
                        break;
                case 'o':
-                       if (parseprop(props))
+                       if (!parseprop(props, optarg))
                                goto error;
                        break;
                case 's':
@@ -739,7 +927,6 @@ zfs_do_create(int argc, char **argv)
                        (void) fprintf(stderr, gettext("missing size "
                            "argument\n"));
                        goto badusage;
-                       break;
                case '?':
                        (void) fprintf(stderr, gettext("invalid option '%c'\n"),
                            optopt);
@@ -769,12 +956,14 @@ zfs_do_create(int argc, char **argv)
 
        if (type == ZFS_TYPE_VOLUME && !noreserve) {
                zpool_handle_t *zpool_handle;
+               nvlist_t *real_props = NULL;
                uint64_t spa_version;
                char *p;
                zfs_prop_t resv_prop;
                char *strval;
+               char msg[1024];
 
-               if ((p = strchr(argv[0], '/')))
+               if ((p = strchr(argv[0], '/')) != NULL)
                        *p = '\0';
                zpool_handle = zpool_open(g_zfs, argv[0]);
                if (p != NULL)
@@ -783,12 +972,22 @@ zfs_do_create(int argc, char **argv)
                        goto error;
                spa_version = zpool_get_prop_int(zpool_handle,
                    ZPOOL_PROP_VERSION, NULL);
-               zpool_close(zpool_handle);
                if (spa_version >= SPA_VERSION_REFRESERVATION)
                        resv_prop = ZFS_PROP_REFRESERVATION;
                else
                        resv_prop = ZFS_PROP_RESERVATION;
-               volsize = zvol_volsize_to_reservation(volsize, props);
+
+               (void) snprintf(msg, sizeof (msg),
+                   gettext("cannot create '%s'"), argv[0]);
+               if (props && (real_props = zfs_valid_proplist(g_zfs, type,
+                   props, 0, NULL, zpool_handle, B_TRUE, msg)) == NULL) {
+                       zpool_close(zpool_handle);
+                       goto error;
+               }
+               zpool_close(zpool_handle);
+
+               volsize = zvol_volsize_to_reservation(volsize, real_props);
+               nvlist_free(real_props);
 
                if (nvlist_lookup_string(props, zfs_prop_to_name(resv_prop),
                    &strval) != 0) {
@@ -818,37 +1017,13 @@ zfs_do_create(int argc, char **argv)
        if (zfs_create(g_zfs, argv[0], type, props) != 0)
                goto error;
 
-       if ((zhp = zfs_open(g_zfs, argv[0], ZFS_TYPE_DATASET)) == NULL)
-               goto error;
-
-       ret = 0;
-       /*
-        * if the user doesn't want the dataset automatically mounted,
-        * then skip the mount/share step
-        */
-       if (zfs_prop_valid_for_type(ZFS_PROP_CANMOUNT, type))
-               canmount = zfs_prop_get_int(zhp, ZFS_PROP_CANMOUNT);
-
-       /*
-        * Mount and/or share the new filesystem as appropriate.  We provide a
-        * verbose error message to let the user know that their filesystem was
-        * in fact created, even if we failed to mount or share it.
-        */
-       if (canmount == ZFS_CANMOUNT_ON) {
-               if (zfs_mount(zhp, NULL, 0) != 0) {
-                       (void) fprintf(stderr, gettext("filesystem "
-                           "successfully created, but not mounted\n"));
-                       ret = 1;
-               } else if (zfs_share(zhp) != 0) {
-                       (void) fprintf(stderr, gettext("filesystem "
-                           "successfully created, but not shared\n"));
-                       ret = 1;
-               }
+       if (log_history) {
+               (void) zpool_log_history(g_zfs, history_str);
+               log_history = B_FALSE;
        }
 
+       ret = zfs_mount_and_share(g_zfs, argv[0], ZFS_TYPE_DATASET);
 error:
-       if (zhp)
-               zfs_close(zhp);
        nvlist_free(props);
        return (ret);
 badusage:
@@ -872,15 +1047,26 @@ badusage:
  */
 typedef struct destroy_cbdata {
        boolean_t       cb_first;
-       int             cb_force;
-       int             cb_recurse;
-       int             cb_error;
-       int             cb_needforce;
-       int             cb_doclones;
-       boolean_t       cb_closezhp;
+       boolean_t       cb_force;
+       boolean_t       cb_recurse;
+       boolean_t       cb_error;
+       boolean_t       cb_doclones;
        zfs_handle_t    *cb_target;
-       char            *cb_snapname;
        boolean_t       cb_defer_destroy;
+       boolean_t       cb_verbose;
+       boolean_t       cb_parsable;
+       boolean_t       cb_dryrun;
+       nvlist_t        *cb_nvl;
+       nvlist_t        *cb_batchedsnaps;
+
+       /* first snap in contiguous run */
+       char            *cb_firstsnap;
+       /* previous snap in contiguous run */
+       char            *cb_prevsnap;
+       int64_t         cb_snapused;
+       char            *cb_snapspec;
+       char            *cb_bookmark;
+       uint64_t        cb_snap_count;
 } destroy_cbdata_t;
 
 /*
@@ -910,7 +1096,7 @@ destroy_check_dependent(zfs_handle_t *zhp, void *data)
                        (void) fprintf(stderr, gettext("use '-r' to destroy "
                            "the following datasets:\n"));
                        cbp->cb_first = B_FALSE;
-                       cbp->cb_error = 1;
+                       cbp->cb_error = B_TRUE;
                }
 
                (void) fprintf(stderr, "%s\n", zfs_get_name(zhp));
@@ -931,7 +1117,8 @@ destroy_check_dependent(zfs_handle_t *zhp, void *data)
                        (void) fprintf(stderr, gettext("use '-R' to destroy "
                            "the following datasets:\n"));
                        cbp->cb_first = B_FALSE;
-                       cbp->cb_error = 1;
+                       cbp->cb_error = B_TRUE;
+                       cbp->cb_dryrun = B_TRUE;
                }
 
                (void) fprintf(stderr, "%s\n", zfs_get_name(zhp));
@@ -942,10 +1129,34 @@ out:
        return (0);
 }
 
+static int
+destroy_batched(destroy_cbdata_t *cb)
+{
+       int error = zfs_destroy_snaps_nvl(g_zfs,
+           cb->cb_batchedsnaps, B_FALSE);
+       fnvlist_free(cb->cb_batchedsnaps);
+       cb->cb_batchedsnaps = fnvlist_alloc();
+       return (error);
+}
+
 static int
 destroy_callback(zfs_handle_t *zhp, void *data)
 {
-       destroy_cbdata_t *cbp = data;
+       destroy_cbdata_t *cb = data;
+       const char *name = zfs_get_name(zhp);
+       int error;
+
+       if (cb->cb_verbose) {
+               if (cb->cb_parsable) {
+                       (void) printf("destroy\t%s\n", name);
+               } else if (cb->cb_dryrun) {
+                       (void) printf(gettext("would destroy %s\n"),
+                           name);
+               } else {
+                       (void) printf(gettext("will destroy %s\n"),
+                           name);
+               }
+       }
 
        /*
         * Ignore pools (which we've already flagged as an error before getting
@@ -956,14 +1167,30 @@ destroy_callback(zfs_handle_t *zhp, void *data)
                zfs_close(zhp);
                return (0);
        }
+       if (cb->cb_dryrun) {
+               zfs_close(zhp);
+               return (0);
+       }
 
        /*
-        * Bail out on the first error.
+        * We batch up all contiguous snapshots (even of different
+        * filesystems) and destroy them with one ioctl.  We can't
+        * simply do all snap deletions and then all fs deletions,
+        * because we must delete a clone before its origin.
         */
-       if (zfs_unmount(zhp, NULL, cbp->cb_force ? MS_FORCE : 0) != 0 ||
-           zfs_destroy(zhp, cbp->cb_defer_destroy) != 0) {
-               zfs_close(zhp);
-               return (-1);
+       if (zfs_get_type(zhp) == ZFS_TYPE_SNAPSHOT) {
+               cb->cb_snap_count++;
+               fnvlist_add_boolean(cb->cb_batchedsnaps, name);
+               if (cb->cb_snap_count % 10 == 0 && cb->cb_defer_destroy)
+                       error = destroy_batched(cb);
+       } else {
+               error = destroy_batched(cb);
+               if (error != 0 ||
+                   zfs_unmount(zhp, NULL, cb->cb_force ? MS_FORCE : 0) != 0 ||
+                   zfs_destroy(zhp, cb->cb_defer_destroy) != 0) {
+                       zfs_close(zhp);
+                       return (-1);
+               }
        }
 
        zfs_close(zhp);
@@ -971,66 +1198,183 @@ destroy_callback(zfs_handle_t *zhp, void *data)
 }
 
 static int
-destroy_snap_clones(zfs_handle_t *zhp, void *arg)
+destroy_print_cb(zfs_handle_t *zhp, void *arg)
 {
-       destroy_cbdata_t *cbp = arg;
-       char thissnap[MAXPATHLEN];
-       zfs_handle_t *szhp;
-       boolean_t closezhp = cbp->cb_closezhp;
-       int rv;
+       destroy_cbdata_t *cb = arg;
+       const char *name = zfs_get_name(zhp);
+       int err = 0;
 
-       (void) snprintf(thissnap, sizeof (thissnap),
-           "%s@%s", zfs_get_name(zhp), cbp->cb_snapname);
+       if (nvlist_exists(cb->cb_nvl, name)) {
+               if (cb->cb_firstsnap == NULL)
+                       cb->cb_firstsnap = strdup(name);
+               if (cb->cb_prevsnap != NULL)
+                       free(cb->cb_prevsnap);
+               /* this snap continues the current range */
+               cb->cb_prevsnap = strdup(name);
+               if (cb->cb_firstsnap == NULL || cb->cb_prevsnap == NULL)
+                       nomem();
+               if (cb->cb_verbose) {
+                       if (cb->cb_parsable) {
+                               (void) printf("destroy\t%s\n", name);
+                       } else if (cb->cb_dryrun) {
+                               (void) printf(gettext("would destroy %s\n"),
+                                   name);
+                       } else {
+                               (void) printf(gettext("will destroy %s\n"),
+                                   name);
+                       }
+               }
+       } else if (cb->cb_firstsnap != NULL) {
+               /* end of this range */
+               uint64_t used = 0;
+               err = lzc_snaprange_space(cb->cb_firstsnap,
+                   cb->cb_prevsnap, &used);
+               cb->cb_snapused += used;
+               free(cb->cb_firstsnap);
+               cb->cb_firstsnap = NULL;
+               free(cb->cb_prevsnap);
+               cb->cb_prevsnap = NULL;
+       }
+       zfs_close(zhp);
+       return (err);
+}
 
-       libzfs_print_on_error(g_zfs, B_FALSE);
-       szhp = zfs_open(g_zfs, thissnap, ZFS_TYPE_SNAPSHOT);
-       libzfs_print_on_error(g_zfs, B_TRUE);
-       if (szhp) {
-               /*
-                * Destroy any clones of this snapshot
-                */
-               if (zfs_iter_dependents(szhp, B_FALSE, destroy_callback,
-                   cbp) != 0) {
-                       zfs_close(szhp);
-                       if (closezhp)
-                               zfs_close(zhp);
-                       return (-1);
+static int
+destroy_print_snapshots(zfs_handle_t *fs_zhp, destroy_cbdata_t *cb)
+{
+       int err;
+       assert(cb->cb_firstsnap == NULL);
+       assert(cb->cb_prevsnap == NULL);
+       err = zfs_iter_snapshots_sorted(fs_zhp, destroy_print_cb, cb);
+       if (cb->cb_firstsnap != NULL) {
+               uint64_t used = 0;
+               if (err == 0) {
+                       err = lzc_snaprange_space(cb->cb_firstsnap,
+                           cb->cb_prevsnap, &used);
                }
-               zfs_close(szhp);
+               cb->cb_snapused += used;
+               free(cb->cb_firstsnap);
+               cb->cb_firstsnap = NULL;
+               free(cb->cb_prevsnap);
+               cb->cb_prevsnap = NULL;
        }
+       return (err);
+}
 
-       cbp->cb_closezhp = B_TRUE;
-       rv = zfs_iter_filesystems(zhp, destroy_snap_clones, arg);
-       if (closezhp)
-               zfs_close(zhp);
-       return (rv);
+static int
+snapshot_to_nvl_cb(zfs_handle_t *zhp, void *arg)
+{
+       destroy_cbdata_t *cb = arg;
+       int err = 0;
+
+       /* Check for clones. */
+       if (!cb->cb_doclones && !cb->cb_defer_destroy) {
+               cb->cb_target = zhp;
+               cb->cb_first = B_TRUE;
+               err = zfs_iter_dependents(zhp, B_TRUE,
+                   destroy_check_dependent, cb);
+       }
+
+       if (err == 0) {
+               if (nvlist_add_boolean(cb->cb_nvl, zfs_get_name(zhp)))
+                       nomem();
+       }
+       zfs_close(zhp);
+       return (err);
+}
+
+static int
+gather_snapshots(zfs_handle_t *zhp, void *arg)
+{
+       destroy_cbdata_t *cb = arg;
+       int err = 0;
+
+       err = zfs_iter_snapspec(zhp, cb->cb_snapspec, snapshot_to_nvl_cb, cb);
+       if (err == ENOENT)
+               err = 0;
+       if (err != 0)
+               goto out;
+
+       if (cb->cb_verbose) {
+               err = destroy_print_snapshots(zhp, cb);
+               if (err != 0)
+                       goto out;
+       }
+
+       if (cb->cb_recurse)
+               err = zfs_iter_filesystems(zhp, gather_snapshots, cb);
+
+out:
+       zfs_close(zhp);
+       return (err);
+}
+
+static int
+destroy_clones(destroy_cbdata_t *cb)
+{
+       nvpair_t *pair;
+       for (pair = nvlist_next_nvpair(cb->cb_nvl, NULL);
+           pair != NULL;
+           pair = nvlist_next_nvpair(cb->cb_nvl, pair)) {
+               zfs_handle_t *zhp = zfs_open(g_zfs, nvpair_name(pair),
+                   ZFS_TYPE_SNAPSHOT);
+               if (zhp != NULL) {
+                       boolean_t defer = cb->cb_defer_destroy;
+                       int err;
+
+                       /*
+                        * We can't defer destroy non-snapshots, so set it to
+                        * false while destroying the clones.
+                        */
+                       cb->cb_defer_destroy = B_FALSE;
+                       err = zfs_iter_dependents(zhp, B_FALSE,
+                           destroy_callback, cb);
+                       cb->cb_defer_destroy = defer;
+                       zfs_close(zhp);
+                       if (err != 0)
+                               return (err);
+               }
+       }
+       return (0);
 }
 
 static int
 zfs_do_destroy(int argc, char **argv)
 {
        destroy_cbdata_t cb = { 0 };
+       int rv = 0;
+       int err = 0;
        int c;
-       zfs_handle_t *zhp;
-       char *cp;
+       zfs_handle_t *zhp = NULL;
+       char *at, *pound;
        zfs_type_t type = ZFS_TYPE_DATASET;
 
        /* check options */
-       while ((c = getopt(argc, argv, "dfrR")) != -1) {
+       while ((c = getopt(argc, argv, "vpndfrR")) != -1) {
                switch (c) {
+               case 'v':
+                       cb.cb_verbose = B_TRUE;
+                       break;
+               case 'p':
+                       cb.cb_verbose = B_TRUE;
+                       cb.cb_parsable = B_TRUE;
+                       break;
+               case 'n':
+                       cb.cb_dryrun = B_TRUE;
+                       break;
                case 'd':
                        cb.cb_defer_destroy = B_TRUE;
                        type = ZFS_TYPE_SNAPSHOT;
                        break;
                case 'f':
-                       cb.cb_force = 1;
+                       cb.cb_force = B_TRUE;
                        break;
                case 'r':
-                       cb.cb_recurse = 1;
+                       cb.cb_recurse = B_TRUE;
                        break;
                case 'R':
-                       cb.cb_recurse = 1;
-                       cb.cb_doclones = 1;
+                       cb.cb_recurse = B_TRUE;
+                       cb.cb_doclones = B_TRUE;
                        break;
                case '?':
                default:
@@ -1045,7 +1389,7 @@ zfs_do_destroy(int argc, char **argv)
 
        /* check number of arguments */
        if (argc == 0) {
-               (void) fprintf(stderr, gettext("missing path argument\n"));
+               (void) fprintf(stderr, gettext("missing dataset argument\n"));
                usage(B_FALSE);
        }
        if (argc > 1) {
@@ -1053,93 +1397,177 @@ zfs_do_destroy(int argc, char **argv)
                usage(B_FALSE);
        }
 
-       /*
-        * If we are doing recursive destroy of a snapshot, then the
-        * named snapshot may not exist.  Go straight to libzfs.
-        */
-       if (cb.cb_recurse && (cp = strchr(argv[0], '@'))) {
-               int ret = 0;
+       at = strchr(argv[0], '@');
+       pound = strchr(argv[0], '#');
+       if (at != NULL) {
 
-               *cp = '\0';
-               if ((zhp = zfs_open(g_zfs, argv[0], ZFS_TYPE_DATASET)) == NULL)
+               /* Build the list of snaps to destroy in cb_nvl. */
+               cb.cb_nvl = fnvlist_alloc();
+
+               *at = '\0';
+               zhp = zfs_open(g_zfs, argv[0],
+                   ZFS_TYPE_FILESYSTEM | ZFS_TYPE_VOLUME);
+               if (zhp == NULL) {
+                       nvlist_free(cb.cb_nvl);
                        return (1);
-               *cp = '@';
-               cp++;
+               }
 
-               if (cb.cb_doclones) {
-                       boolean_t defer = cb.cb_defer_destroy;
+               cb.cb_snapspec = at + 1;
+               if (gather_snapshots(zfs_handle_dup(zhp), &cb) != 0 ||
+                   cb.cb_error) {
+                       rv = 1;
+                       goto out;
+               }
 
-                       /*
-                        * Temporarily ignore the defer_destroy setting since
-                        * it's not supported for clones.
-                        */
-                       cb.cb_defer_destroy = B_FALSE;
-                       cb.cb_snapname = cp;
-                       if (destroy_snap_clones(zhp, &cb) != 0) {
-                               zfs_close(zhp);
-                               return (1);
+               if (nvlist_empty(cb.cb_nvl)) {
+                       (void) fprintf(stderr, gettext("could not find any "
+                           "snapshots to destroy; check snapshot names.\n"));
+                       rv = 1;
+                       goto out;
+               }
+
+               if (cb.cb_verbose) {
+                       char buf[16];
+                       zfs_nicebytes(cb.cb_snapused, buf, sizeof (buf));
+                       if (cb.cb_parsable) {
+                               (void) printf("reclaim\t%llu\n",
+                                   (u_longlong_t)cb.cb_snapused);
+                       } else if (cb.cb_dryrun) {
+                               (void) printf(gettext("would reclaim %s\n"),
+                                   buf);
+                       } else {
+                               (void) printf(gettext("will reclaim %s\n"),
+                                   buf);
                        }
-                       cb.cb_defer_destroy = defer;
                }
 
-               ret = zfs_destroy_snaps(zhp, cp, cb.cb_defer_destroy);
-               zfs_close(zhp);
-               if (ret) {
+               if (!cb.cb_dryrun) {
+                       if (cb.cb_doclones) {
+                               cb.cb_batchedsnaps = fnvlist_alloc();
+                               err = destroy_clones(&cb);
+                               if (err == 0) {
+                                       err = zfs_destroy_snaps_nvl(g_zfs,
+                                           cb.cb_batchedsnaps, B_FALSE);
+                               }
+                               if (err != 0) {
+                                       rv = 1;
+                                       goto out;
+                               }
+                       }
+                       if (err == 0) {
+                               err = zfs_destroy_snaps_nvl(g_zfs, cb.cb_nvl,
+                                   cb.cb_defer_destroy);
+                       }
+               }
+
+               if (err != 0)
+                       rv = 1;
+       } else if (pound != NULL) {
+               int err;
+               nvlist_t *nvl;
+
+               if (cb.cb_dryrun) {
                        (void) fprintf(stderr,
-                           gettext("no snapshots destroyed\n"));
+                           "dryrun is not supported with bookmark\n");
+                       return (-1);
                }
-               return (ret != 0);
-       }
 
-       /* Open the given dataset */
-       if ((zhp = zfs_open(g_zfs, argv[0], type)) == NULL)
-               return (1);
+               if (cb.cb_defer_destroy) {
+                       (void) fprintf(stderr,
+                           "defer destroy is not supported with bookmark\n");
+                       return (-1);
+               }
 
-       cb.cb_target = zhp;
+               if (cb.cb_recurse) {
+                       (void) fprintf(stderr,
+                           "recursive is not supported with bookmark\n");
+                       return (-1);
+               }
 
-       /*
-        * Perform an explicit check for pools before going any further.
-        */
-       if (!cb.cb_recurse && strchr(zfs_get_name(zhp), '/') == NULL &&
-           zfs_get_type(zhp) == ZFS_TYPE_FILESYSTEM) {
-               (void) fprintf(stderr, gettext("cannot destroy '%s': "
-                   "operation does not apply to pools\n"),
-                   zfs_get_name(zhp));
-               (void) fprintf(stderr, gettext("use 'zfs destroy -r "
-                   "%s' to destroy all datasets in the pool\n"),
-                   zfs_get_name(zhp));
-               (void) fprintf(stderr, gettext("use 'zpool destroy %s' "
-                   "to destroy the pool itself\n"), zfs_get_name(zhp));
-               zfs_close(zhp);
-               return (1);
-       }
+               if (!zfs_bookmark_exists(argv[0])) {
+                       (void) fprintf(stderr, gettext("bookmark '%s' "
+                           "does not exist.\n"), argv[0]);
+                       return (1);
+               }
 
-       /*
-        * Check for any dependents and/or clones.
-        */
-       cb.cb_first = B_TRUE;
-       if (!cb.cb_doclones && !cb.cb_defer_destroy &&
-           zfs_iter_dependents(zhp, B_TRUE, destroy_check_dependent,
-           &cb) != 0) {
-               zfs_close(zhp);
-               return (1);
-       }
+               nvl = fnvlist_alloc();
+               fnvlist_add_boolean(nvl, argv[0]);
 
-       if (cb.cb_error || (!cb.cb_defer_destroy &&
-           (zfs_iter_dependents(zhp, B_FALSE, destroy_callback, &cb) != 0))) {
-               zfs_close(zhp);
-               return (1);
-       }
+               err = lzc_destroy_bookmarks(nvl, NULL);
+               if (err != 0) {
+                       (void) zfs_standard_error(g_zfs, err,
+                           "cannot destroy bookmark");
+               }
 
-       /*
-        * Do the real thing.  The callback will close the handle regardless of
-        * whether it succeeds or not.
-        */
+               nvlist_free(nvl);
 
-       if (destroy_callback(zhp, &cb) != 0)
-               return (1);
+               return (err);
+       } else {
+               /* Open the given dataset */
+               if ((zhp = zfs_open(g_zfs, argv[0], type)) == NULL)
+                       return (1);
 
-       return (0);
+               cb.cb_target = zhp;
+
+               /*
+                * Perform an explicit check for pools before going any further.
+                */
+               if (!cb.cb_recurse && strchr(zfs_get_name(zhp), '/') == NULL &&
+                   zfs_get_type(zhp) == ZFS_TYPE_FILESYSTEM) {
+                       (void) fprintf(stderr, gettext("cannot destroy '%s': "
+                           "operation does not apply to pools\n"),
+                           zfs_get_name(zhp));
+                       (void) fprintf(stderr, gettext("use 'zfs destroy -r "
+                           "%s' to destroy all datasets in the pool\n"),
+                           zfs_get_name(zhp));
+                       (void) fprintf(stderr, gettext("use 'zpool destroy %s' "
+                           "to destroy the pool itself\n"), zfs_get_name(zhp));
+                       rv = 1;
+                       goto out;
+               }
+
+               /*
+                * Check for any dependents and/or clones.
+                */
+               cb.cb_first = B_TRUE;
+               if (!cb.cb_doclones &&
+                   zfs_iter_dependents(zhp, B_TRUE, destroy_check_dependent,
+                   &cb) != 0) {
+                       rv = 1;
+                       goto out;
+               }
+
+               if (cb.cb_error) {
+                       rv = 1;
+                       goto out;
+               }
+               cb.cb_batchedsnaps = fnvlist_alloc();
+               if (zfs_iter_dependents(zhp, B_FALSE, destroy_callback,
+                   &cb) != 0) {
+                       rv = 1;
+                       goto out;
+               }
+
+               /*
+                * Do the real thing.  The callback will close the
+                * handle regardless of whether it succeeds or not.
+                */
+               err = destroy_callback(zhp, &cb);
+               zhp = NULL;
+               if (err == 0) {
+                       err = zfs_destroy_snaps_nvl(g_zfs,
+                           cb.cb_batchedsnaps, cb.cb_defer_destroy);
+               }
+               if (err != 0)
+                       rv = 1;
+       }
+
+out:
+       fnvlist_free(cb.cb_batchedsnaps);
+       fnvlist_free(cb.cb_nvl);
+       if (zhp != NULL)
+               zfs_close(zhp);
+       return (rv);
 }
 
 static boolean_t
@@ -1183,7 +1611,7 @@ get_callback(zfs_handle_t *zhp, void *data)
        char buf[ZFS_MAXPROPLEN];
        char rbuf[ZFS_MAXPROPLEN];
        zprop_source_t sourcetype;
-       char source[ZFS_MAXNAMELEN];
+       char source[ZFS_MAX_DATASET_NAME_LEN];
        zprop_get_cbdata_t *cbp = data;
        nvlist_t *user_props = zfs_get_user_props(zhp);
        zprop_list_t *pl = cbp->cb_proplist;
@@ -1210,7 +1638,7 @@ get_callback(zfs_handle_t *zhp, void *data)
                                if (pl->pl_all)
                                        continue;
                                if (!zfs_prop_valid_for_type(pl->pl_prop,
-                                   ZFS_TYPE_DATASET)) {
+                                   ZFS_TYPE_DATASET, B_FALSE)) {
                                        (void) fprintf(stderr,
                                            gettext("No such property '%s'\n"),
                                            zfs_prop_to_name(pl->pl_prop));
@@ -1237,6 +1665,17 @@ get_callback(zfs_handle_t *zhp, void *data)
                                (void) strlcpy(buf, "-", sizeof (buf));
                        }
 
+                       zprop_print_one_property(zfs_get_name(zhp), cbp,
+                           pl->pl_user_prop, buf, sourcetype, source, NULL);
+               } else if (zfs_prop_written(pl->pl_user_prop)) {
+                       sourcetype = ZPROP_SRC_LOCAL;
+
+                       if (zfs_prop_get_written(zhp, pl->pl_user_prop,
+                           buf, sizeof (buf), cbp->cb_literal) != 0) {
+                               sourcetype = ZPROP_SRC_NONE;
+                               (void) strlcpy(buf, "-", sizeof (buf));
+                       }
+
                        zprop_print_one_property(zfs_get_name(zhp), cbp,
                            pl->pl_user_prop, buf, sourcetype, source, NULL);
                } else {
@@ -1284,6 +1723,7 @@ zfs_do_get(int argc, char **argv)
 {
        zprop_get_cbdata_t cb = { 0 };
        int i, c, flags = ZFS_ITER_ARGS_CAN_BE_PATHS;
+       int types = ZFS_TYPE_DATASET | ZFS_TYPE_BOOKMARK;
        char *value, *fields;
        int ret = 0;
        int limit = 0;
@@ -1300,7 +1740,7 @@ zfs_do_get(int argc, char **argv)
        cb.cb_type = ZFS_TYPE_DATASET;
 
        /* check options */
-       while ((c = getopt(argc, argv, ":d:o:s:rHp")) != -1) {
+       while ((c = getopt(argc, argv, ":d:o:s:rt:Hp")) != -1) {
                switch (c) {
                case 'p':
                        cb.cb_literal = B_TRUE;
@@ -1418,6 +1858,42 @@ zfs_do_get(int argc, char **argv)
                        }
                        break;
 
+               case 't':
+                       types = 0;
+                       flags &= ~ZFS_ITER_PROP_LISTSNAPS;
+                       while (*optarg != '\0') {
+                               static char *type_subopts[] = { "filesystem",
+                                   "volume", "snapshot", "bookmark",
+                                   "all", NULL };
+
+                               switch (getsubopt(&optarg, type_subopts,
+                                   &value)) {
+                               case 0:
+                                       types |= ZFS_TYPE_FILESYSTEM;
+                                       break;
+                               case 1:
+                                       types |= ZFS_TYPE_VOLUME;
+                                       break;
+                               case 2:
+                                       types |= ZFS_TYPE_SNAPSHOT;
+                                       break;
+                               case 3:
+                                       types |= ZFS_TYPE_BOOKMARK;
+                                       break;
+                               case 4:
+                                       types = ZFS_TYPE_DATASET |
+                                           ZFS_TYPE_BOOKMARK;
+                                       break;
+
+                               default:
+                                       (void) fprintf(stderr,
+                                           gettext("invalid type '%s'\n"),
+                                           value);
+                                       usage(B_FALSE);
+                               }
+                       }
+                       break;
+
                case '?':
                        (void) fprintf(stderr, gettext("invalid option '%c'\n"),
                            optopt);
@@ -1461,7 +1937,7 @@ zfs_do_get(int argc, char **argv)
        cb.cb_first = B_TRUE;
 
        /* run for each object */
-       ret = zfs_for_each(argc, argv, flags, ZFS_TYPE_DATASET, NULL,
+       ret = zfs_for_each(argc, argv, flags, types, NULL,
            &cb.cb_proplist, limit, get_callback, &cb);
 
        if (cb.cb_proplist == &fake_name)
@@ -1501,7 +1977,7 @@ inherit_recurse_cb(zfs_handle_t *zhp, void *data)
         * are not valid for this type of dataset.
         */
        if (prop != ZPROP_INVAL &&
-           !zfs_prop_valid_for_type(prop, zfs_get_type(zhp)))
+           !zfs_prop_valid_for_type(prop, zfs_get_type(zhp), B_FALSE))
                return (0);
 
        return (zfs_prop_inherit(zhp, cb->cb_propname, cb->cb_received) != 0);
@@ -1573,9 +2049,13 @@ zfs_do_inherit(int argc, char **argv)
                        if (prop == ZFS_PROP_QUOTA ||
                            prop == ZFS_PROP_RESERVATION ||
                            prop == ZFS_PROP_REFQUOTA ||
-                           prop == ZFS_PROP_REFRESERVATION)
+                           prop == ZFS_PROP_REFRESERVATION) {
                                (void) fprintf(stderr, gettext("use 'zfs set "
                                    "%s=none' to clear\n"), propname);
+                               (void) fprintf(stderr, gettext("use 'zfs "
+                                   "inherit -S %s' to revert to received "
+                                   "value\n"), propname);
+                       }
                        return (1);
                }
                if (received && (prop == ZFS_PROP_VOLSIZE ||
@@ -1611,7 +2091,7 @@ typedef struct upgrade_cbdata {
        uint64_t cb_version;
        boolean_t cb_newer;
        boolean_t cb_foundone;
-       char cb_lastfs[ZFS_MAXNAMELEN];
+       char cb_lastfs[ZFS_MAX_DATASET_NAME_LEN];
 } upgrade_cbdata_t;
 
 static int
@@ -1697,9 +2177,11 @@ upgrade_set_callback(zfs_handle_t *zhp, void *data)
                        /*
                         * If they did "zfs upgrade -a", then we could
                         * be doing ioctls to different pools.  We need
-                        * to log this history once to each pool.
+                        * to log this history once to each pool, and bypass
+                        * the normal history logging that happens in main().
                         */
-                       verify(zpool_stage_history(g_zfs, history_str) == 0);
+                       (void) zpool_log_history(g_zfs, history_str);
+                       log_history = B_FALSE;
                }
                if (zfs_prop_set(zhp, "version", verstr) == 0)
                        cb->cb_numupgraded++;
@@ -1783,8 +2265,8 @@ zfs_do_upgrade(int argc, char **argv)
                    "---------------\n");
                (void) printf(gettext(" 1   Initial ZFS filesystem version\n"));
                (void) printf(gettext(" 2   Enhanced directory entries\n"));
-               (void) printf(gettext(" 3   Case insensitive and File system "
-                   "unique identifier (FUID)\n"));
+               (void) printf(gettext(" 3   Case insensitive and filesystem "
+                   "user identifier (FUID)\n"));
                (void) printf(gettext(" 4   userquota, groupquota "
                    "properties\n"));
                (void) printf(gettext(" 5   System attributes\n"));
@@ -1808,7 +2290,7 @@ zfs_do_upgrade(int argc, char **argv)
                if (cb.cb_numfailed != 0)
                        ret = 1;
        } else {
-               /* List old-version filesytems */
+               /* List old-version filesystems */
                boolean_t found;
                (void) printf(gettext("This system is currently running "
                    "ZFS filesystem version %llu.\n\n"), ZPL_VERSION);
@@ -1833,30 +2315,60 @@ zfs_do_upgrade(int argc, char **argv)
        return (ret);
 }
 
-#define        USTYPE_USR_BIT (0)
-#define        USTYPE_GRP_BIT (1)
-#define        USTYPE_PSX_BIT (2)
-#define        USTYPE_SMB_BIT (3)
-
-#define        USTYPE_USR (1 << USTYPE_USR_BIT)
-#define        USTYPE_GRP (1 << USTYPE_GRP_BIT)
-
-#define        USTYPE_PSX (1 << USTYPE_PSX_BIT)
-#define        USTYPE_SMB (1 << USTYPE_SMB_BIT)
-
-#define        USTYPE_PSX_USR (USTYPE_PSX | USTYPE_USR)
-#define        USTYPE_SMB_USR (USTYPE_SMB | USTYPE_USR)
-#define        USTYPE_PSX_GRP (USTYPE_PSX | USTYPE_GRP)
-#define        USTYPE_SMB_GRP (USTYPE_SMB | USTYPE_GRP)
-#define        USTYPE_ALL (USTYPE_PSX_USR | USTYPE_SMB_USR \
-               | USTYPE_PSX_GRP | USTYPE_SMB_GRP)
-
-
-#define        USPROP_USED_BIT (0)
-#define        USPROP_QUOTA_BIT (1)
+/*
+ * zfs userspace [-Hinp] [-o field[,...]] [-s field [-s field]...]
+ *               [-S field [-S field]...] [-t type[,...]] filesystem | snapshot
+ * zfs groupspace [-Hinp] [-o field[,...]] [-s field [-s field]...]
+ *                [-S field [-S field]...] [-t type[,...]] filesystem | snapshot
+ * zfs projectspace [-Hp] [-o field[,...]] [-s field [-s field]...]
+ *                [-S field [-S field]...] filesystem | snapshot
+ *
+ *     -H      Scripted mode; elide headers and separate columns by tabs.
+ *     -i      Translate SID to POSIX ID.
+ *     -n      Print numeric ID instead of user/group name.
+ *     -o      Control which fields to display.
+ *     -p      Use exact (parsable) numeric output.
+ *     -s      Specify sort columns, descending order.
+ *     -S      Specify sort columns, ascending order.
+ *     -t      Control which object types to display.
+ *
+ *     Displays space consumed by, and quotas on, each user in the specified
+ *     filesystem or snapshot.
+ */
 
-#define        USPROP_USED (1 << USPROP_USED_BIT)
-#define        USPROP_QUOTA (1 << USPROP_QUOTA_BIT)
+/* us_field_types, us_field_hdr and us_field_names should be kept in sync */
+enum us_field_types {
+       USFIELD_TYPE,
+       USFIELD_NAME,
+       USFIELD_USED,
+       USFIELD_QUOTA,
+       USFIELD_OBJUSED,
+       USFIELD_OBJQUOTA
+};
+static char *us_field_hdr[] = { "TYPE", "NAME", "USED", "QUOTA",
+                                   "OBJUSED", "OBJQUOTA" };
+static char *us_field_names[] = { "type", "name", "used", "quota",
+                                   "objused", "objquota" };
+#define        USFIELD_LAST    (sizeof (us_field_names) / sizeof (char *))
+
+#define        USTYPE_PSX_GRP  (1 << 0)
+#define        USTYPE_PSX_USR  (1 << 1)
+#define        USTYPE_SMB_GRP  (1 << 2)
+#define        USTYPE_SMB_USR  (1 << 3)
+#define        USTYPE_PROJ     (1 << 4)
+#define        USTYPE_ALL      \
+       (USTYPE_PSX_GRP | USTYPE_PSX_USR | USTYPE_SMB_GRP | USTYPE_SMB_USR | \
+           USTYPE_PROJ)
+
+static int us_type_bits[] = {
+       USTYPE_PSX_GRP,
+       USTYPE_PSX_USR,
+       USTYPE_SMB_GRP,
+       USTYPE_SMB_USR,
+       USTYPE_ALL
+};
+static char *us_type_names[] = { "posixgroup", "posixuser", "smbgroup",
+       "smbuser", "all" };
 
 typedef struct us_node {
        nvlist_t        *usn_nvl;
@@ -1865,37 +2377,49 @@ typedef struct us_node {
 } us_node_t;
 
 typedef struct us_cbdata {
-       nvlist_t                **cb_nvlp;
-       uu_avl_pool_t           *cb_avl_pool;
-       uu_avl_t                *cb_avl;
-       boolean_t               cb_numname;
-       boolean_t               cb_nicenum;
-       boolean_t               cb_sid2posix;
-       zfs_userquota_prop_t    cb_prop;
-       zfs_sort_column_t       *cb_sortcol;
-       size_t                  cb_max_typelen;
-       size_t                  cb_max_namelen;
-       size_t                  cb_max_usedlen;
-       size_t                  cb_max_quotalen;
+       nvlist_t        **cb_nvlp;
+       uu_avl_pool_t   *cb_avl_pool;
+       uu_avl_t        *cb_avl;
+       boolean_t       cb_numname;
+       boolean_t       cb_nicenum;
+       boolean_t       cb_sid2posix;
+       zfs_userquota_prop_t cb_prop;
+       zfs_sort_column_t *cb_sortcol;
+       size_t          cb_width[USFIELD_LAST];
 } us_cbdata_t;
 
+static boolean_t us_populated = B_FALSE;
+
 typedef struct {
        zfs_sort_column_t *si_sortcol;
-       boolean_t si_num_name;
-       boolean_t si_parsable;
+       boolean_t       si_numname;
 } us_sort_info_t;
 
+static int
+us_field_index(char *field)
+{
+       int i;
+
+       for (i = 0; i < USFIELD_LAST; i++) {
+               if (strcmp(field, us_field_names[i]) == 0)
+                       return (i);
+       }
+
+       return (-1);
+}
+
 static int
 us_compare(const void *larg, const void *rarg, void *unused)
 {
        const us_node_t *l = larg;
        const us_node_t *r = rarg;
-       int rc = 0;
        us_sort_info_t *si = (us_sort_info_t *)unused;
        zfs_sort_column_t *sortcol = si->si_sortcol;
-       boolean_t num_name = si->si_num_name;
+       boolean_t numname = si->si_numname;
        nvlist_t *lnvl = l->usn_nvl;
        nvlist_t *rnvl = r->usn_nvl;
+       int rc = 0;
+       boolean_t lvb, rvb;
 
        for (; sortcol != NULL; sortcol = sortcol->sc_next) {
                char *lvstr = "";
@@ -1914,41 +2438,47 @@ us_compare(const void *larg, const void *rarg, void *unused)
                        (void) nvlist_lookup_uint32(lnvl, propname, &lv32);
                        (void) nvlist_lookup_uint32(rnvl, propname, &rv32);
                        if (rv32 != lv32)
-                               rc = (rv32 > lv32) ? 1 : -1;
+                               rc = (rv32 < lv32) ? 1 : -1;
                        break;
                case ZFS_PROP_NAME:
                        propname = "name";
-                       if (num_name) {
-                               (void) nvlist_lookup_uint32(lnvl, propname,
-                                   &lv32);
-                               (void) nvlist_lookup_uint32(rnvl, propname,
-                                   &rv32);
-                               if (rv32 != lv32)
-                                       rc = (rv32 > lv32) ? 1 : -1;
+                       if (numname) {
+compare_nums:
+                               (void) nvlist_lookup_uint64(lnvl, propname,
+                                   &lv64);
+                               (void) nvlist_lookup_uint64(rnvl, propname,
+                                   &rv64);
+                               if (rv64 != lv64)
+                                       rc = (rv64 < lv64) ? 1 : -1;
                        } else {
-                               (void) nvlist_lookup_string(lnvl, propname,
-                                   &lvstr);
-                               (void) nvlist_lookup_string(rnvl, propname,
-                                   &rvstr);
+                               if ((nvlist_lookup_string(lnvl, propname,
+                                   &lvstr) == ENOENT) ||
+                                   (nvlist_lookup_string(rnvl, propname,
+                                   &rvstr) == ENOENT)) {
+                                       goto compare_nums;
+                               }
                                rc = strcmp(lvstr, rvstr);
                        }
                        break;
-
                case ZFS_PROP_USED:
                case ZFS_PROP_QUOTA:
-                       if (ZFS_PROP_USED == prop)
+                       if (!us_populated)
+                               break;
+                       if (prop == ZFS_PROP_USED)
                                propname = "used";
                        else
                                propname = "quota";
                        (void) nvlist_lookup_uint64(lnvl, propname, &lv64);
                        (void) nvlist_lookup_uint64(rnvl, propname, &rv64);
                        if (rv64 != lv64)
-                               rc = (rv64 > lv64) ? 1 : -1;
+                               rc = (rv64 < lv64) ? 1 : -1;
+                       break;
+
                default:
                        break;
                }
 
-               if (rc) {
+               if (rc != 0) {
                        if (rc < 0)
                                return (reverse ? 1 : -1);
                        else
@@ -1956,7 +2486,38 @@ us_compare(const void *larg, const void *rarg, void *unused)
                }
        }
 
-       return (rc);
+       /*
+        * If entries still seem to be the same, check if they are of the same
+        * type (smbentity is added only if we are doing SID to POSIX ID
+        * translation where we can have duplicate type/name combinations).
+        */
+       if (nvlist_lookup_boolean_value(lnvl, "smbentity", &lvb) == 0 &&
+           nvlist_lookup_boolean_value(rnvl, "smbentity", &rvb) == 0 &&
+           lvb != rvb)
+               return (lvb < rvb ? -1 : 1);
+
+       return (0);
+}
+
+static boolean_t
+zfs_prop_is_user(unsigned p)
+{
+       return (p == ZFS_PROP_USERUSED || p == ZFS_PROP_USERQUOTA ||
+           p == ZFS_PROP_USEROBJUSED || p == ZFS_PROP_USEROBJQUOTA);
+}
+
+static boolean_t
+zfs_prop_is_group(unsigned p)
+{
+       return (p == ZFS_PROP_GROUPUSED || p == ZFS_PROP_GROUPQUOTA ||
+           p == ZFS_PROP_GROUPOBJUSED || p == ZFS_PROP_GROUPOBJQUOTA);
+}
+
+static boolean_t
+zfs_prop_is_project(unsigned p)
+{
+       return (p == ZFS_PROP_PROJECTUSED || p == ZFS_PROP_PROJECTQUOTA ||
+           p == ZFS_PROP_PROJECTOBJUSED || p == ZFS_PROP_PROJECTOBJQUOTA);
 }
 
 static inline const char *
@@ -1971,14 +2532,13 @@ us_type2str(unsigned field_type)
                return ("SMB User");
        case USTYPE_SMB_GRP:
                return ("SMB Group");
+       case USTYPE_PROJ:
+               return ("Project");
        default:
                return ("Undefined");
        }
 }
 
-/*
- * zfs userspace
- */
 static int
 userspace_cb(void *arg, const char *domain, uid_t rid, uint64_t space)
 {
@@ -1986,7 +2546,6 @@ userspace_cb(void *arg, const char *domain, uid_t rid, uint64_t space)
        zfs_userquota_prop_t prop = cb->cb_prop;
        char *name = NULL;
        char *propname;
-       char namebuf[32];
        char sizebuf[32];
        us_node_t *node;
        uu_avl_pool_t *avl_pool = cb->cb_avl_pool;
@@ -1995,36 +2554,34 @@ userspace_cb(void *arg, const char *domain, uid_t rid, uint64_t space)
        nvlist_t *props;
        us_node_t *n;
        zfs_sort_column_t *sortcol = cb->cb_sortcol;
-       unsigned type;
+       unsigned type = 0;
        const char *typestr;
        size_t namelen;
        size_t typelen;
        size_t sizelen;
+       int typeidx, nameidx, sizeidx;
        us_sort_info_t sortinfo = { sortcol, cb->cb_numname };
+       boolean_t smbentity = B_FALSE;
 
-       if (domain == NULL || domain[0] == '\0') {
-               /* POSIX */
-               if (prop == ZFS_PROP_GROUPUSED || prop == ZFS_PROP_GROUPQUOTA) {
-                       type = USTYPE_PSX_GRP;
-                       struct group *g = getgrgid(rid);
-                       if (g)
-                               name = g->gr_name;
-               } else {
-                       type = USTYPE_PSX_USR;
-                       struct passwd *p = getpwuid(rid);
-                       if (p)
-                               name = p->pw_name;
-               }
-       } else {
+       if (nvlist_alloc(&props, NV_UNIQUE_NAME, 0) != 0)
+               nomem();
+       node = safe_malloc(sizeof (us_node_t));
+       uu_avl_node_init(node, &node->usn_avlnode, avl_pool);
+       node->usn_nvl = props;
+
+       if (domain != NULL && domain[0] != '\0') {
 #ifdef HAVE_IDMAP
-               char sid[ZFS_MAXNAMELEN+32];
+               /* SMB */
+               char sid[MAXNAMELEN + 32];
                uid_t id;
                uint64_t classes;
-               int err = 0;
+               int err;
                directory_error_t e;
 
+               smbentity = B_TRUE;
+
                (void) snprintf(sid, sizeof (sid), "%s-%u", domain, rid);
-               /* SMB */
+
                if (prop == ZFS_PROP_GROUPUSED || prop == ZFS_PROP_GROUPQUOTA) {
                        type = USTYPE_SMB_GRP;
                        err = sid_to_id(sid, B_FALSE, &id);
@@ -2035,326 +2592,283 @@ userspace_cb(void *arg, const char *domain, uid_t rid, uint64_t space)
 
                if (err == 0) {
                        rid = id;
-
-                       e = directory_name_from_sid(NULL, sid, &name, &classes);
-                       if (e != NULL) {
-                               directory_error_free(e);
-                               return (NULL);
+                       if (!cb->cb_sid2posix) {
+                               e = directory_name_from_sid(NULL, sid, &name,
+                                   &classes);
+                               if (e != NULL)
+                                       directory_error_free(e);
+                               if (name == NULL)
+                                       name = sid;
                        }
-
-                       if (name == NULL)
-                               name = sid;
                }
 #else
+               nvlist_free(props);
+               free(node);
+
                return (-1);
 #endif /* HAVE_IDMAP */
        }
 
-/*
- *     if (prop == ZFS_PROP_GROUPUSED || prop == ZFS_PROP_GROUPQUOTA)
- *             ug = "group";
- *     else
- *             ug = "user";
- */
-
-       if (prop == ZFS_PROP_USERUSED || prop == ZFS_PROP_GROUPUSED)
-               propname = "used";
-       else
-               propname = "quota";
-
-       (void) snprintf(namebuf, sizeof (namebuf), "%u", rid);
-       if (name == NULL)
-               name = namebuf;
-
-       if (cb->cb_nicenum)
-               zfs_nicenum(space, sizebuf, sizeof (sizebuf));
-       else
-               (void) sprintf(sizebuf, "%llu", (u_longlong_t)space);
+       if (cb->cb_sid2posix || domain == NULL || domain[0] == '\0') {
+               /* POSIX or -i */
+               if (zfs_prop_is_group(prop)) {
+                       type = USTYPE_PSX_GRP;
+                       if (!cb->cb_numname) {
+                               struct group *g;
 
-       node = safe_malloc(sizeof (us_node_t));
-       uu_avl_node_init(node, &node->usn_avlnode, avl_pool);
+                               if ((g = getgrgid(rid)) != NULL)
+                                       name = g->gr_name;
+                       }
+               } else if (zfs_prop_is_user(prop)) {
+                       type = USTYPE_PSX_USR;
+                       if (!cb->cb_numname) {
+                               struct passwd *p;
 
-       if (nvlist_alloc(&props, NV_UNIQUE_NAME, 0) != 0) {
-               free(node);
-               return (-1);
+                               if ((p = getpwuid(rid)) != NULL)
+                                       name = p->pw_name;
+                       }
+               } else {
+                       type = USTYPE_PROJ;
+               }
        }
 
+       /*
+        * Make sure that the type/name combination is unique when doing
+        * SID to POSIX ID translation (hence changing the type from SMB to
+        * POSIX).
+        */
+       if (cb->cb_sid2posix &&
+           nvlist_add_boolean_value(props, "smbentity", smbentity) != 0)
+               nomem();
+
+       /* Calculate/update width of TYPE field */
+       typestr = us_type2str(type);
+       typelen = strlen(gettext(typestr));
+       typeidx = us_field_index("type");
+       if (typelen > cb->cb_width[typeidx])
+               cb->cb_width[typeidx] = typelen;
        if (nvlist_add_uint32(props, "type", type) != 0)
                nomem();
 
-       if (cb->cb_numname) {
-               if (nvlist_add_uint32(props, "name", rid) != 0)
+       /* Calculate/update width of NAME field */
+       if ((cb->cb_numname && cb->cb_sid2posix) || name == NULL) {
+               if (nvlist_add_uint64(props, "name", rid) != 0)
                        nomem();
-               namelen = strlen(namebuf);
+               namelen = snprintf(NULL, 0, "%u", rid);
        } else {
                if (nvlist_add_string(props, "name", name) != 0)
                        nomem();
                namelen = strlen(name);
        }
+       nameidx = us_field_index("name");
+       if (nameidx >= 0 && namelen > cb->cb_width[nameidx])
+               cb->cb_width[nameidx] = namelen;
 
-       typestr = us_type2str(type);
-       typelen = strlen(gettext(typestr));
-       if (typelen > cb->cb_max_typelen)
-               cb->cb_max_typelen  = typelen;
-
-       if (namelen > cb->cb_max_namelen)
-               cb->cb_max_namelen  = namelen;
-
-       sizelen = strlen(sizebuf);
-       if (0 == strcmp(propname, "used")) {
-               if (sizelen > cb->cb_max_usedlen)
-                       cb->cb_max_usedlen  = sizelen;
-       } else {
-               if (sizelen > cb->cb_max_quotalen)
-                       cb->cb_max_quotalen  = sizelen;
-       }
-
-       node->usn_nvl = props;
-
-       n = uu_avl_find(avl, node, &sortinfo, &idx);
-       if (n == NULL)
+       /*
+        * Check if this type/name combination is in the list and update it;
+        * otherwise add new node to the list.
+        */
+       if ((n = uu_avl_find(avl, node, &sortinfo, &idx)) == NULL) {
                uu_avl_insert(avl, node, idx);
-       else {
+       else {
                nvlist_free(props);
                free(node);
                node = n;
                props = node->usn_nvl;
        }
 
+       /* Calculate/update width of USED/QUOTA fields */
+       if (cb->cb_nicenum) {
+               if (prop == ZFS_PROP_USERUSED || prop == ZFS_PROP_GROUPUSED ||
+                   prop == ZFS_PROP_USERQUOTA || prop == ZFS_PROP_GROUPQUOTA ||
+                   prop == ZFS_PROP_PROJECTUSED ||
+                   prop == ZFS_PROP_PROJECTQUOTA) {
+                       zfs_nicebytes(space, sizebuf, sizeof (sizebuf));
+               } else {
+                       zfs_nicenum(space, sizebuf, sizeof (sizebuf));
+               }
+       } else {
+               (void) snprintf(sizebuf, sizeof (sizebuf), "%llu",
+                   (u_longlong_t)space);
+       }
+       sizelen = strlen(sizebuf);
+       if (prop == ZFS_PROP_USERUSED || prop == ZFS_PROP_GROUPUSED ||
+           prop == ZFS_PROP_PROJECTUSED) {
+               propname = "used";
+               if (!nvlist_exists(props, "quota"))
+                       (void) nvlist_add_uint64(props, "quota", 0);
+       } else if (prop == ZFS_PROP_USERQUOTA || prop == ZFS_PROP_GROUPQUOTA ||
+           prop == ZFS_PROP_PROJECTQUOTA) {
+               propname = "quota";
+               if (!nvlist_exists(props, "used"))
+                       (void) nvlist_add_uint64(props, "used", 0);
+       } else if (prop == ZFS_PROP_USEROBJUSED ||
+           prop == ZFS_PROP_GROUPOBJUSED || prop == ZFS_PROP_PROJECTOBJUSED) {
+               propname = "objused";
+               if (!nvlist_exists(props, "objquota"))
+                       (void) nvlist_add_uint64(props, "objquota", 0);
+       } else if (prop == ZFS_PROP_USEROBJQUOTA ||
+           prop == ZFS_PROP_GROUPOBJQUOTA ||
+           prop == ZFS_PROP_PROJECTOBJQUOTA) {
+               propname = "objquota";
+               if (!nvlist_exists(props, "objused"))
+                       (void) nvlist_add_uint64(props, "objused", 0);
+       } else {
+               return (-1);
+       }
+       sizeidx = us_field_index(propname);
+       if (sizeidx >= 0 && sizelen > cb->cb_width[sizeidx])
+               cb->cb_width[sizeidx] = sizelen;
+
        if (nvlist_add_uint64(props, propname, space) != 0)
                nomem();
 
        return (0);
 }
 
-static inline boolean_t
-usprop_check(zfs_userquota_prop_t p, unsigned types, unsigned props)
+static void
+print_us_node(boolean_t scripted, boolean_t parsable, int *fields, int types,
+    size_t *width, us_node_t *node)
 {
-       unsigned type;
-       unsigned prop;
-
-       switch (p) {
-       case ZFS_PROP_USERUSED:
-               type = USTYPE_USR;
-               prop = USPROP_USED;
-               break;
-       case ZFS_PROP_USERQUOTA:
-               type = USTYPE_USR;
-               prop = USPROP_QUOTA;
-               break;
-       case ZFS_PROP_GROUPUSED:
-               type = USTYPE_GRP;
-               prop = USPROP_USED;
-               break;
-       case ZFS_PROP_GROUPQUOTA:
-               type = USTYPE_GRP;
-               prop = USPROP_QUOTA;
-               break;
-       default: /* ALL */
-               return (B_TRUE);
-       };
-
-       return (type & types && prop & props);
-}
-
-#define        USFIELD_TYPE (1 << 0)
-#define        USFIELD_NAME (1 << 1)
-#define        USFIELD_USED (1 << 2)
-#define        USFIELD_QUOTA (1 << 3)
-#define        USFIELD_ALL (USFIELD_TYPE | USFIELD_NAME | USFIELD_USED | USFIELD_QUOTA)
+       nvlist_t *nvl = node->usn_nvl;
+       char valstr[MAXNAMELEN];
+       boolean_t first = B_TRUE;
+       int cfield = 0;
+       int field;
+       uint32_t ustype;
 
-static int
-parsefields(unsigned *fieldsp, char **names, unsigned *bits, size_t len)
-{
-       char *field = optarg;
-       char *delim;
+       /* Check type */
+       (void) nvlist_lookup_uint32(nvl, "type", &ustype);
+       if (!(ustype & types))
+               return;
 
-       do {
-               int i;
-               boolean_t found = B_FALSE;
-               delim = strchr(field, ',');
-               if (delim != NULL)
-                       *delim = '\0';
+       while ((field = fields[cfield]) != USFIELD_LAST) {
+               nvpair_t *nvp = NULL;
+               data_type_t type;
+               uint32_t val32;
+               uint64_t val64;
+               char *strval = "-";
 
-               for (i = 0; i < len; i++)
-                       if (0 == strcmp(field, names[i])) {
-                               found = B_TRUE;
-                               *fieldsp |= bits[i];
+               while ((nvp = nvlist_next_nvpair(nvl, nvp)) != NULL) {
+                       if (strcmp(nvpair_name(nvp),
+                           us_field_names[field]) == 0)
                                break;
-                       }
-
-               if (!found) {
-                       (void) fprintf(stderr, gettext("invalid type '%s'"
-                           "for -t option\n"), field);
-                       return (-1);
                }
 
-               field = delim + 1;
-       } while (delim);
-
-       return (0);
-}
-
-
-static char *type_names[] = { "posixuser", "smbuser", "posixgroup", "smbgroup",
-       "all" };
-static unsigned type_bits[] = {
-       USTYPE_PSX_USR,
-       USTYPE_SMB_USR,
-       USTYPE_PSX_GRP,
-       USTYPE_SMB_GRP,
-       USTYPE_ALL
-};
-
-static char *us_field_names[] = { "type", "name", "used", "quota" };
-static unsigned us_field_bits[] = {
-       USFIELD_TYPE,
-       USFIELD_NAME,
-       USFIELD_USED,
-       USFIELD_QUOTA
-};
-
-static void
-print_us_node(boolean_t scripted, boolean_t parseable, unsigned fields,
-               size_t type_width, size_t name_width, size_t used_width,
-               size_t quota_width, us_node_t *node)
-{
-       nvlist_t *nvl = node->usn_nvl;
-       nvpair_t *nvp = NULL;
-       char valstr[ZFS_MAXNAMELEN];
-       boolean_t first = B_TRUE;
-       boolean_t quota_found = B_FALSE;
-
-       if (fields & USFIELD_QUOTA && !nvlist_exists(nvl, "quota"))
-               if (nvlist_add_string(nvl, "quota", "none") != 0)
-                       nomem();
-
-       while ((nvp = nvlist_next_nvpair(nvl, nvp)) != NULL) {
-               char *pname = nvpair_name(nvp);
-               data_type_t type = nvpair_type(nvp);
-               uint32_t val32 = 0;
-               uint64_t val64 = 0;
-               char *strval = NULL;
-               unsigned field = 0;
-               unsigned width = 0;
-               int i;
-               for (i = 0; i < 4; i++) {
-                       if (0 == strcmp(pname, us_field_names[i])) {
-                               field = us_field_bits[i];
-                               break;
-                       }
-               }
-
-               if (!(field & fields))
-                       continue;
-
-               switch (type) {
-               case DATA_TYPE_UINT32:
-                       (void) nvpair_value_uint32(nvp, &val32);
-                       break;
-               case DATA_TYPE_UINT64:
-                       (void) nvpair_value_uint64(nvp, &val64);
-                       break;
-               case DATA_TYPE_STRING:
-                       (void) nvpair_value_string(nvp, &strval);
-                       break;
-               default:
-                       (void) fprintf(stderr, "Invalid data type\n");
-               }
-
-               if (!first) {
-                       if (scripted)
-                               (void) printf("\t");
-                       else
-                               (void) printf("  ");
-               }
+               type = nvp == NULL ? DATA_TYPE_UNKNOWN : nvpair_type(nvp);
+               switch (type) {
+               case DATA_TYPE_UINT32:
+                       (void) nvpair_value_uint32(nvp, &val32);
+                       break;
+               case DATA_TYPE_UINT64:
+                       (void) nvpair_value_uint64(nvp, &val64);
+                       break;
+               case DATA_TYPE_STRING:
+                       (void) nvpair_value_string(nvp, &strval);
+                       break;
+               case DATA_TYPE_UNKNOWN:
+                       break;
+               default:
+                       (void) fprintf(stderr, "invalid data type\n");
+               }
 
                switch (field) {
                case USFIELD_TYPE:
-                       strval = (char *)us_type2str(val32);
-                       width = type_width;
+                       if (type == DATA_TYPE_UINT32)
+                               strval = (char *)us_type2str(val32);
                        break;
                case USFIELD_NAME:
                        if (type == DATA_TYPE_UINT64) {
                                (void) sprintf(valstr, "%llu",
-                                   (u_longlong_t) val64);
+                                   (u_longlong_t)val64);
                                strval = valstr;
                        }
-                       width = name_width;
                        break;
                case USFIELD_USED:
                case USFIELD_QUOTA:
                        if (type == DATA_TYPE_UINT64) {
-                               (void) nvpair_value_uint64(nvp, &val64);
-                               if (parseable)
+                               if (parsable) {
                                        (void) sprintf(valstr, "%llu",
-                                           (u_longlong_t) val64);
-                               else
-                                       zfs_nicenum(val64, valstr,
+                                           (u_longlong_t)val64);
+                                       strval = valstr;
+                               } else if (field == USFIELD_QUOTA &&
+                                   val64 == 0) {
+                                       strval = "none";
+                               } else {
+                                       zfs_nicebytes(val64, valstr,
                                            sizeof (valstr));
-                               strval = valstr;
+                                       strval = valstr;
+                               }
                        }
-
-                       if (field == USFIELD_USED)
-                               width = used_width;
-                       else {
-                               quota_found = B_FALSE;
-                               width = quota_width;
+                       break;
+               case USFIELD_OBJUSED:
+               case USFIELD_OBJQUOTA:
+                       if (type == DATA_TYPE_UINT64) {
+                               if (parsable) {
+                                       (void) sprintf(valstr, "%llu",
+                                           (u_longlong_t)val64);
+                                       strval = valstr;
+                               } else if (field == USFIELD_OBJQUOTA &&
+                                   val64 == 0) {
+                                       strval = "none";
+                               } else {
+                                       zfs_nicenum(val64, valstr,
+                                           sizeof (valstr));
+                                       strval = valstr;
+                               }
                        }
-
                        break;
                }
 
-               if (field == USFIELD_QUOTA && !quota_found)
-                       (void) printf("%*s", width, strval);
-               else {
-                       if (type == DATA_TYPE_STRING)
-                               (void) printf("%-*s", width, strval);
+               if (!first) {
+                       if (scripted)
+                               (void) printf("\t");
                        else
-                               (void) printf("%*s", width, strval);
+                               (void) printf("  ");
                }
+               if (scripted)
+                       (void) printf("%s", strval);
+               else if (field == USFIELD_TYPE || field == USFIELD_NAME)
+                       (void) printf("%-*s", (int)width[field], strval);
+               else
+                       (void) printf("%*s", (int)width[field], strval);
 
                first = B_FALSE;
-
+               cfield++;
        }
 
        (void) printf("\n");
 }
 
 static void
-print_us(boolean_t scripted, boolean_t parsable, unsigned fields,
-               unsigned type_width, unsigned name_width, unsigned used_width,
-               unsigned quota_width, boolean_t rmnode, uu_avl_t *avl)
+print_us(boolean_t scripted, boolean_t parsable, int *fields, int types,
+    size_t *width, boolean_t rmnode, uu_avl_t *avl)
 {
-       static char *us_field_hdr[] = { "TYPE", "NAME", "USED", "QUOTA" };
        us_node_t *node;
        const char *col;
-       int i;
-       int width[4] = { type_width, name_width, used_width, quota_width };
+       int cfield = 0;
+       int field;
 
        if (!scripted) {
                boolean_t first = B_TRUE;
-               for (i = 0; i < 4; i++) {
-                       unsigned field = us_field_bits[i];
-                       if (!(field & fields))
-                               continue;
 
-                       col = gettext(us_field_hdr[i]);
-                       if (field == USFIELD_TYPE || field == USFIELD_NAME)
-                               (void) printf(first?"%-*s":"  %-*s", width[i],
-                                   col);
-                       else
-                               (void) printf(first?"%*s":"  %*s", width[i],
-                                   col);
+               while ((field = fields[cfield]) != USFIELD_LAST) {
+                       col = gettext(us_field_hdr[field]);
+                       if (field == USFIELD_TYPE || field == USFIELD_NAME) {
+                               (void) printf(first ? "%-*s" : "  %-*s",
+                                   (int)width[field], col);
+                       } else {
+                               (void) printf(first ? "%*s" : "  %*s",
+                                   (int)width[field], col);
+                       }
                        first = B_FALSE;
+                       cfield++;
                }
                (void) printf("\n");
        }
 
-       for (node = uu_avl_first(avl); node != NULL;
-           node = uu_avl_next(avl, node)) {
-               print_us_node(scripted, parsable, fields, type_width,
-                   name_width, used_width, used_width, node);
+       for (node = uu_avl_first(avl); node; node = uu_avl_next(avl, node)) {
+               print_us_node(scripted, parsable, fields, types, width, node);
                if (rmnode)
                        nvlist_free(node->usn_nvl);
        }
@@ -2368,69 +2882,82 @@ zfs_do_userspace(int argc, char **argv)
        uu_avl_pool_t *avl_pool;
        uu_avl_t *avl_tree;
        uu_avl_walk_t *walk;
-
-       char *cmd;
+       char *delim;
+       char deffields[] = "type,name,used,quota,objused,objquota";
+       char *ofield = NULL;
+       char *tfield = NULL;
+       int cfield = 0;
+       int fields[256];
+       int i;
        boolean_t scripted = B_FALSE;
        boolean_t prtnum = B_FALSE;
-       boolean_t parseable = B_FALSE;
+       boolean_t parsable = B_FALSE;
        boolean_t sid2posix = B_FALSE;
-       int error = 0;
+       int ret = 0;
        int c;
-       zfs_sort_column_t *default_sortcol = NULL;
        zfs_sort_column_t *sortcol = NULL;
-       unsigned types = USTYPE_PSX_USR | USTYPE_SMB_USR;
-       unsigned fields = 0;
-       unsigned props = USPROP_USED | USPROP_QUOTA;
+       int types = USTYPE_PSX_USR | USTYPE_SMB_USR;
        us_cbdata_t cb;
        us_node_t *node;
-       boolean_t resort_avl = B_FALSE;
+       us_node_t *rmnode;
+       uu_list_pool_t *listpool;
+       uu_list_t *list;
+       uu_avl_index_t idx = 0;
+       uu_list_index_t idx2 = 0;
 
        if (argc < 2)
                usage(B_FALSE);
 
-       cmd = argv[0];
-       if (0 == strcmp(cmd, "groupspace"))
-               /* toggle default group types */
+       if (strcmp(argv[0], "groupspace") == 0) {
+               /* Toggle default group types */
                types = USTYPE_PSX_GRP | USTYPE_SMB_GRP;
+       } else if (strcmp(argv[0], "projectspace") == 0) {
+               types = USTYPE_PROJ;
+               prtnum = B_TRUE;
+       }
 
-       /* check options */
        while ((c = getopt(argc, argv, "nHpo:s:S:t:i")) != -1) {
                switch (c) {
                case 'n':
+                       if (types == USTYPE_PROJ) {
+                               (void) fprintf(stderr,
+                                   gettext("invalid option 'n'\n"));
+                               usage(B_FALSE);
+                       }
                        prtnum = B_TRUE;
                        break;
                case 'H':
                        scripted = B_TRUE;
                        break;
                case 'p':
-                       parseable = B_TRUE;
+                       parsable = B_TRUE;
                        break;
                case 'o':
-                       if (parsefields(&fields, us_field_names, us_field_bits,
-                           4) != 0)
-                               return (1);
+                       ofield = optarg;
                        break;
                case 's':
+               case 'S':
                        if (zfs_add_sort_column(&sortcol, optarg,
-                           B_FALSE) != 0) {
+                           c == 's' ? B_FALSE : B_TRUE) != 0) {
                                (void) fprintf(stderr,
-                                   gettext("invalid property '%s'\n"), optarg);
+                                   gettext("invalid field '%s'\n"), optarg);
                                usage(B_FALSE);
                        }
                        break;
-               case 'S':
-                       if (zfs_add_sort_column(&sortcol, optarg,
-                           B_TRUE) != 0) {
+               case 't':
+                       if (types == USTYPE_PROJ) {
                                (void) fprintf(stderr,
-                                   gettext("invalid property '%s'\n"), optarg);
+                                   gettext("invalid option 't'\n"));
                                usage(B_FALSE);
                        }
-                       break;
-               case 't':
-                       if (parsefields(&types, type_names, type_bits, 5))
-                               return (1);
+                       tfield = optarg;
                        break;
                case 'i':
+                       if (types == USTYPE_PROJ) {
+                               (void) fprintf(stderr,
+                                   gettext("invalid option 'i'\n"));
+                               usage(B_FALSE);
+                       }
                        sid2posix = B_TRUE;
                        break;
                case ':':
@@ -2448,104 +2975,131 @@ zfs_do_userspace(int argc, char **argv)
        argc -= optind;
        argv += optind;
 
-       /* ok, now we have sorted by default colums (type,name) avl tree */
-       if (sortcol) {
-               zfs_sort_column_t *sc;
-               for (sc = sortcol; sc; sc = sc->sc_next) {
-                       if (sc->sc_prop == ZFS_PROP_QUOTA) {
-                               resort_avl = B_TRUE;
-                               break;
-                       }
-               }
+       if (argc < 1) {
+               (void) fprintf(stderr, gettext("missing dataset name\n"));
+               usage(B_FALSE);
+       }
+       if (argc > 1) {
+               (void) fprintf(stderr, gettext("too many arguments\n"));
+               usage(B_FALSE);
        }
 
-       if (!fields)
-               fields = USFIELD_ALL;
+       /* Use default output fields if not specified using -o */
+       if (ofield == NULL)
+               ofield = deffields;
+       do {
+               if ((delim = strchr(ofield, ',')) != NULL)
+                       *delim = '\0';
+               if ((fields[cfield++] = us_field_index(ofield)) == -1) {
+                       (void) fprintf(stderr, gettext("invalid type '%s' "
+                           "for -o option\n"), ofield);
+                       return (-1);
+               }
+               if (delim != NULL)
+                       ofield = delim + 1;
+       } while (delim != NULL);
+       fields[cfield] = USFIELD_LAST;
+
+       /* Override output types (-t option) */
+       if (tfield != NULL) {
+               types = 0;
+
+               do {
+                       boolean_t found = B_FALSE;
+
+                       if ((delim = strchr(tfield, ',')) != NULL)
+                               *delim = '\0';
+                       for (i = 0; i < sizeof (us_type_bits) / sizeof (int);
+                           i++) {
+                               if (strcmp(tfield, us_type_names[i]) == 0) {
+                                       found = B_TRUE;
+                                       types |= us_type_bits[i];
+                                       break;
+                               }
+                       }
+                       if (!found) {
+                               (void) fprintf(stderr, gettext("invalid type "
+                                   "'%s' for -t option\n"), tfield);
+                               return (-1);
+                       }
+                       if (delim != NULL)
+                               tfield = delim + 1;
+               } while (delim != NULL);
+       }
 
-       if ((zhp = zfs_open(g_zfs, argv[argc-1], ZFS_TYPE_DATASET)) == NULL)
+       if ((zhp = zfs_open(g_zfs, argv[0], ZFS_TYPE_DATASET)) == NULL)
                return (1);
 
        if ((avl_pool = uu_avl_pool_create("us_avl_pool", sizeof (us_node_t),
-           offsetof(us_node_t, usn_avlnode),
-           us_compare, UU_DEFAULT)) == NULL)
+           offsetof(us_node_t, usn_avlnode), us_compare, UU_DEFAULT)) == NULL)
                nomem();
        if ((avl_tree = uu_avl_create(avl_pool, NULL, UU_DEFAULT)) == NULL)
                nomem();
 
-       if (sortcol && !resort_avl)
-               cb.cb_sortcol = sortcol;
-       else {
-               (void) zfs_add_sort_column(&default_sortcol, "type", B_FALSE);
-               (void) zfs_add_sort_column(&default_sortcol, "name", B_FALSE);
-               cb.cb_sortcol = default_sortcol;
-       }
+       /* Always add default sorting columns */
+       (void) zfs_add_sort_column(&sortcol, "type", B_FALSE);
+       (void) zfs_add_sort_column(&sortcol, "name", B_FALSE);
+
+       cb.cb_sortcol = sortcol;
        cb.cb_numname = prtnum;
-       cb.cb_nicenum = !parseable;
+       cb.cb_nicenum = !parsable;
        cb.cb_avl_pool = avl_pool;
        cb.cb_avl = avl_tree;
        cb.cb_sid2posix = sid2posix;
-       cb.cb_max_typelen = strlen(gettext("TYPE"));
-       cb.cb_max_namelen = strlen(gettext("NAME"));
-       cb.cb_max_usedlen = strlen(gettext("USED"));
-       cb.cb_max_quotalen = strlen(gettext("QUOTA"));
+
+       for (i = 0; i < USFIELD_LAST; i++)
+               cb.cb_width[i] = strlen(gettext(us_field_hdr[i]));
 
        for (p = 0; p < ZFS_NUM_USERQUOTA_PROPS; p++) {
-               if (!usprop_check(p, types, props))
+               if ((zfs_prop_is_user(p) &&
+                   !(types & (USTYPE_PSX_USR | USTYPE_SMB_USR))) ||
+                   (zfs_prop_is_group(p) &&
+                   !(types & (USTYPE_PSX_GRP | USTYPE_SMB_GRP))) ||
+                   (zfs_prop_is_project(p) && types != USTYPE_PROJ))
                        continue;
 
                cb.cb_prop = p;
-               error = zfs_userspace(zhp, p, userspace_cb, &cb);
-               if (error)
-                       break;
-       }
-
-
-       if (resort_avl) {
-               us_node_t *node;
-               us_node_t *rmnode;
-               uu_list_pool_t *listpool;
-               uu_list_t *list;
-               uu_avl_index_t idx = 0;
-               uu_list_index_t idx2 = 0;
-               listpool = uu_list_pool_create("tmplist", sizeof (us_node_t),
-                   offsetof(us_node_t, usn_listnode), NULL,
-                   UU_DEFAULT);
-               list = uu_list_create(listpool, NULL, UU_DEFAULT);
-
-               node = uu_avl_first(avl_tree);
-               uu_list_node_init(node, &node->usn_listnode, listpool);
-               while (node != NULL) {
-                       rmnode = node;
-                       node = uu_avl_next(avl_tree, node);
-                       uu_avl_remove(avl_tree, rmnode);
-                       if (uu_list_find(list, rmnode, NULL, &idx2) == NULL) {
-                               uu_list_insert(list, rmnode, idx2);
-                       }
-               }
+               if ((ret = zfs_userspace(zhp, p, userspace_cb, &cb)) != 0)
+                       return (ret);
+       }
 
-               for (node = uu_list_first(list); node != NULL;
-                   node = uu_list_next(list, node)) {
-                       us_sort_info_t sortinfo = { sortcol, cb.cb_numname };
-                       if (uu_avl_find(avl_tree, node, &sortinfo, &idx) ==
-                           NULL)
-                       uu_avl_insert(avl_tree, node, idx);
-               }
+       /* Sort the list */
+       if ((node = uu_avl_first(avl_tree)) == NULL)
+               return (0);
+
+       us_populated = B_TRUE;
 
-               uu_list_destroy(list);
+       listpool = uu_list_pool_create("tmplist", sizeof (us_node_t),
+           offsetof(us_node_t, usn_listnode), NULL, UU_DEFAULT);
+       list = uu_list_create(listpool, NULL, UU_DEFAULT);
+       uu_list_node_init(node, &node->usn_listnode, listpool);
+
+       while (node != NULL) {
+               rmnode = node;
+               node = uu_avl_next(avl_tree, node);
+               uu_avl_remove(avl_tree, rmnode);
+               if (uu_list_find(list, rmnode, NULL, &idx2) == NULL)
+                       uu_list_insert(list, rmnode, idx2);
        }
 
-       /* print & free node`s nvlist memory */
-       print_us(scripted, parseable, fields, cb.cb_max_typelen,
-           cb.cb_max_namelen, cb.cb_max_usedlen,
-           cb.cb_max_quotalen, B_TRUE, cb.cb_avl);
+       for (node = uu_list_first(list); node != NULL;
+           node = uu_list_next(list, node)) {
+               us_sort_info_t sortinfo = { sortcol, cb.cb_numname };
 
-       if (sortcol)
-               zfs_free_sort_columns(sortcol);
-       zfs_free_sort_columns(default_sortcol);
+               if (uu_avl_find(avl_tree, node, &sortinfo, &idx) == NULL)
+                       uu_avl_insert(avl_tree, node, idx);
+       }
 
-       /*
-        * Finally, clean up the AVL tree.
-        */
+       uu_list_destroy(list);
+       uu_list_pool_destroy(listpool);
+
+       /* Print and free node nvlist memory */
+       print_us(scripted, parsable, fields, types, cb.cb_width, B_TRUE,
+           cb.cb_avl);
+
+       zfs_free_sort_columns(sortcol);
+
+       /* Clean up the AVL tree */
        if ((walk = uu_avl_walk_start(cb.cb_avl, UU_WALK_ROBUST)) == NULL)
                nomem();
 
@@ -2558,28 +3112,29 @@ zfs_do_userspace(int argc, char **argv)
        uu_avl_destroy(avl_tree);
        uu_avl_pool_destroy(avl_pool);
 
-       return (error);
+       return (ret);
 }
 
 /*
- * list [-r][-d max] [-H] [-o property[,property]...] [-t type[,type]...]
- *      [-s property [-s property]...] [-S property [-S property]...]
- *      <dataset> ...
+ * list [-Hp][-r|-d max] [-o property[,...]] [-s property] ... [-S property]
+ *      [-t type[,...]] [filesystem|volume|snapshot] ...
  *
+ *     -H      Scripted mode; elide headers and separate columns by tabs
+ *     -p      Display values in parsable (literal) format.
  *     -r      Recurse over all children
  *     -d      Limit recursion by depth.
- *     -H      Scripted mode; elide headers and separate columns by tabs
  *     -o      Control which fields to display.
- *     -t      Control which object types to display.
  *     -s      Specify sort columns, descending order.
  *     -S      Specify sort columns, ascending order.
+ *     -t      Control which object types to display.
  *
- * When given no arguments, lists all filesystems in the system.
+ * When given no arguments, list all filesystems in the system.
  * Otherwise, list the specified datasets, optionally recursing down them if
  * '-r' is specified.
  */
 typedef struct list_cbdata {
        boolean_t       cb_first;
+       boolean_t       cb_literal;
        boolean_t       cb_scripted;
        zprop_list_t    *cb_proplist;
 } list_cbdata_t;
@@ -2588,8 +3143,9 @@ typedef struct list_cbdata {
  * Given a list of columns to display, output appropriate headers for each one.
  */
 static void
-print_header(zprop_list_t *pl)
+print_header(list_cbdata_t *cb)
 {
+       zprop_list_t *pl = cb->cb_proplist;
        char headerbuf[ZFS_MAXPROPLEN];
        const char *header;
        int i;
@@ -2630,19 +3186,19 @@ print_header(zprop_list_t *pl)
  * to the described layout.
  */
 static void
-print_dataset(zfs_handle_t *zhp, zprop_list_t *pl, boolean_t scripted)
+print_dataset(zfs_handle_t *zhp, list_cbdata_t *cb)
 {
+       zprop_list_t *pl = cb->cb_proplist;
        boolean_t first = B_TRUE;
        char property[ZFS_MAXPROPLEN];
        nvlist_t *userprops = zfs_get_user_props(zhp);
        nvlist_t *propval;
        char *propstr;
        boolean_t right_justify;
-       int width;
 
        for (; pl != NULL; pl = pl->pl_next) {
                if (!first) {
-                       if (scripted)
+                       if (cb->cb_scripted)
                                (void) printf("\t");
                        else
                                (void) printf("  ");
@@ -2652,20 +3208,27 @@ print_dataset(zfs_handle_t *zhp, zprop_list_t *pl, boolean_t scripted)
 
                if (pl->pl_prop == ZFS_PROP_NAME) {
                        (void) strlcpy(property, zfs_get_name(zhp),
-                           sizeof(property));
+                           sizeof (property));
                        propstr = property;
                        right_justify = zfs_prop_align_right(pl->pl_prop);
                } else if (pl->pl_prop != ZPROP_INVAL) {
                        if (zfs_prop_get(zhp, pl->pl_prop, property,
-                           sizeof (property), NULL, NULL, 0, B_FALSE) != 0)
+                           sizeof (property), NULL, NULL, 0,
+                           cb->cb_literal) != 0)
                                propstr = "-";
                        else
                                propstr = property;
-
                        right_justify = zfs_prop_align_right(pl->pl_prop);
                } else if (zfs_prop_userquota(pl->pl_user_prop)) {
                        if (zfs_prop_get_userquota(zhp, pl->pl_user_prop,
-                           property, sizeof (property), B_FALSE) != 0)
+                           property, sizeof (property), cb->cb_literal) != 0)
+                               propstr = "-";
+                       else
+                               propstr = property;
+                       right_justify = B_TRUE;
+               } else if (zfs_prop_written(pl->pl_user_prop)) {
+                       if (zfs_prop_get_written(zhp, pl->pl_user_prop,
+                           property, sizeof (property), cb->cb_literal) != 0)
                                propstr = "-";
                        else
                                propstr = property;
@@ -2680,19 +3243,17 @@ print_dataset(zfs_handle_t *zhp, zprop_list_t *pl, boolean_t scripted)
                        right_justify = B_FALSE;
                }
 
-               width = pl->pl_width;
-
                /*
                 * If this is being called in scripted mode, or if this is the
                 * last column and it is left-justified, don't include a width
                 * format specifier.
                 */
-               if (scripted || (pl->pl_next == NULL && !right_justify))
+               if (cb->cb_scripted || (pl->pl_next == NULL && !right_justify))
                        (void) printf("%s", propstr);
                else if (right_justify)
-                       (void) printf("%*s", width, propstr);
+                       (void) printf("%*s", (int)pl->pl_width, propstr);
                else
-                       (void) printf("%-*s", width, propstr);
+                       (void) printf("%-*s", (int)pl->pl_width, propstr);
        }
 
        (void) printf("\n");
@@ -2708,11 +3269,11 @@ list_callback(zfs_handle_t *zhp, void *data)
 
        if (cbp->cb_first) {
                if (!cbp->cb_scripted)
-                       print_header(cbp->cb_proplist);
+                       print_header(cbp);
                cbp->cb_first = B_FALSE;
        }
 
-       print_dataset(zhp, cbp->cb_proplist, cbp->cb_scripted);
+       print_dataset(zhp, cbp);
 
        return (0);
 }
@@ -2721,7 +3282,6 @@ static int
 zfs_do_list(int argc, char **argv)
 {
        int c;
-       boolean_t scripted = B_FALSE;
        static char default_fields[] =
            "name,used,available,referenced,mountpoint";
        int types = ZFS_TYPE_DATASET;
@@ -2735,11 +3295,15 @@ zfs_do_list(int argc, char **argv)
        int flags = ZFS_ITER_PROP_LISTSNAPS | ZFS_ITER_ARGS_CAN_BE_PATHS;
 
        /* check options */
-       while ((c = getopt(argc, argv, ":d:o:rt:Hs:S:")) != -1) {
+       while ((c = getopt(argc, argv, "HS:d:o:prs:t:")) != -1) {
                switch (c) {
                case 'o':
                        fields = optarg;
                        break;
+               case 'p':
+                       cb.cb_literal = B_TRUE;
+                       flags |= ZFS_ITER_LITERAL_PROPS;
+                       break;
                case 'd':
                        limit = parse_depth(optarg, &flags);
                        break;
@@ -2747,7 +3311,7 @@ zfs_do_list(int argc, char **argv)
                        flags |= ZFS_ITER_RECURSE;
                        break;
                case 'H':
-                       scripted = B_TRUE;
+                       cb.cb_scripted = B_TRUE;
                        break;
                case 's':
                        if (zfs_add_sort_column(&sortcol, optarg,
@@ -2771,7 +3335,8 @@ zfs_do_list(int argc, char **argv)
                        flags &= ~ZFS_ITER_PROP_LISTSNAPS;
                        while (*optarg != '\0') {
                                static char *type_subopts[] = { "filesystem",
-                                   "volume", "snapshot", "snap", "all", NULL };
+                                   "volume", "snapshot", "snap", "bookmark",
+                                   "all", NULL };
 
                                switch (getsubopt(&optarg, type_subopts,
                                    &value)) {
@@ -2786,9 +3351,12 @@ zfs_do_list(int argc, char **argv)
                                        types |= ZFS_TYPE_SNAPSHOT;
                                        break;
                                case 4:
-                                       types = ZFS_TYPE_DATASET;
+                                       types |= ZFS_TYPE_BOOKMARK;
+                                       break;
+                               case 5:
+                                       types = ZFS_TYPE_DATASET |
+                                           ZFS_TYPE_BOOKMARK;
                                        break;
-
                                default:
                                        (void) fprintf(stderr,
                                            gettext("invalid type '%s'\n"),
@@ -2837,7 +3405,6 @@ zfs_do_list(int argc, char **argv)
            != 0)
                usage(B_FALSE);
 
-       cb.cb_scripted = scripted;
        cb.cb_first = B_TRUE;
 
        ret = zfs_for_each(argc, argv, flags, types, sortcol, &cb.cb_proplist,
@@ -2853,8 +3420,8 @@ zfs_do_list(int argc, char **argv)
 }
 
 /*
- * zfs rename <fs | snap | vol> <fs | snap | vol>
- * zfs rename -p <fs | vol> <fs | vol>
+ * zfs rename [-f] <fs | snap | vol> <fs | snap | vol>
+ * zfs rename [-f] -p <fs | vol> <fs | vol>
  * zfs rename -r <snap> <snap>
  *
  * Renames the given dataset to another of the same type.
@@ -2870,9 +3437,10 @@ zfs_do_rename(int argc, char **argv)
        int ret = 0;
        boolean_t recurse = B_FALSE;
        boolean_t parents = B_FALSE;
+       boolean_t force_unmount = B_FALSE;
 
        /* check options */
-       while ((c = getopt(argc, argv, "pr")) != -1) {
+       while ((c = getopt(argc, argv, "prf")) != -1) {
                switch (c) {
                case 'p':
                        parents = B_TRUE;
@@ -2880,6 +3448,9 @@ zfs_do_rename(int argc, char **argv)
                case 'r':
                        recurse = B_TRUE;
                        break;
+               case 'f':
+                       force_unmount = B_TRUE;
+                       break;
                case '?':
                default:
                        (void) fprintf(stderr, gettext("invalid option '%c'\n"),
@@ -2930,7 +3501,7 @@ zfs_do_rename(int argc, char **argv)
                return (1);
        }
 
-       ret = (zfs_rename(zhp, argv[1], recurse) != 0);
+       ret = (zfs_rename(zhp, argv[1], recurse, force_unmount) != 0);
 
        zfs_close(zhp);
        return (ret);
@@ -2982,7 +3553,7 @@ zfs_do_promote(int argc, char **argv)
  *
  *     -r      Delete any intervening snapshots before doing rollback
  *     -R      Delete any snapshots and their clones
- *     -f      ignored for backwards compatability
+ *     -f      ignored for backwards compatibility
  *
  * Given a filesystem, rollback to a specific snapshot, discarding any changes
  * since then and making it the active dataset.  If more recent snapshots exist,
@@ -2995,9 +3566,31 @@ typedef struct rollback_cbdata {
        char            *cb_target;
        int             cb_error;
        boolean_t       cb_recurse;
-       boolean_t       cb_dependent;
 } rollback_cbdata_t;
 
+static int
+rollback_check_dependent(zfs_handle_t *zhp, void *data)
+{
+       rollback_cbdata_t *cbp = data;
+
+       if (cbp->cb_first && cbp->cb_recurse) {
+               (void) fprintf(stderr, gettext("cannot rollback to "
+                   "'%s': clones of previous snapshots exist\n"),
+                   cbp->cb_target);
+               (void) fprintf(stderr, gettext("use '-R' to "
+                   "force deletion of the following clones and "
+                   "dependents:\n"));
+               cbp->cb_first = 0;
+               cbp->cb_error = 1;
+       }
+
+       (void) fprintf(stderr, "%s\n", zfs_get_name(zhp));
+
+       zfs_close(zhp);
+       return (0);
+}
+
+
 /*
  * Report any snapshots more recent than the one specified.  Used when '-r' is
  * not specified.  We reuse this same callback for the snapshot dependents - if
@@ -3014,52 +3607,30 @@ rollback_check(zfs_handle_t *zhp, void *data)
                return (0);
        }
 
-       if (!cbp->cb_dependent) {
-               if (strcmp(zfs_get_name(zhp), cbp->cb_target) != 0 &&
-                   zfs_get_type(zhp) == ZFS_TYPE_SNAPSHOT &&
-                   zfs_prop_get_int(zhp, ZFS_PROP_CREATETXG) >
-                   cbp->cb_create) {
-
-                       if (cbp->cb_first && !cbp->cb_recurse) {
-                               (void) fprintf(stderr, gettext("cannot "
-                                   "rollback to '%s': more recent snapshots "
-                                   "exist\n"),
-                                   cbp->cb_target);
-                               (void) fprintf(stderr, gettext("use '-r' to "
-                                   "force deletion of the following "
-                                   "snapshots:\n"));
-                               cbp->cb_first = 0;
-                               cbp->cb_error = 1;
-                       }
-
-                       if (cbp->cb_recurse) {
-                               cbp->cb_dependent = B_TRUE;
-                               if (zfs_iter_dependents(zhp, B_TRUE,
-                                   rollback_check, cbp) != 0) {
-                                       zfs_close(zhp);
-                                       return (-1);
-                               }
-                               cbp->cb_dependent = B_FALSE;
-                       } else {
-                               (void) fprintf(stderr, "%s\n",
-                                   zfs_get_name(zhp));
-                       }
-               }
-       } else {
-               if (cbp->cb_first && cbp->cb_recurse) {
-                       (void) fprintf(stderr, gettext("cannot rollback to "
-                           "'%s': clones of previous snapshots exist\n"),
+       if (zfs_prop_get_int(zhp, ZFS_PROP_CREATETXG) > cbp->cb_create) {
+               if (cbp->cb_first && !cbp->cb_recurse) {
+                       (void) fprintf(stderr, gettext("cannot "
+                           "rollback to '%s': more recent snapshots "
+                           "or bookmarks exist\n"),
                            cbp->cb_target);
-                       (void) fprintf(stderr, gettext("use '-R' to "
-                           "force deletion of the following clones and "
-                           "dependents:\n"));
+                       (void) fprintf(stderr, gettext("use '-r' to "
+                           "force deletion of the following "
+                           "snapshots and bookmarks:\n"));
                        cbp->cb_first = 0;
                        cbp->cb_error = 1;
                }
 
-               (void) fprintf(stderr, "%s\n", zfs_get_name(zhp));
+               if (cbp->cb_recurse) {
+                       if (zfs_iter_dependents(zhp, B_TRUE,
+                           rollback_check_dependent, cbp) != 0) {
+                               zfs_close(zhp);
+                               return (-1);
+                       }
+               } else {
+                       (void) fprintf(stderr, "%s\n",
+                           zfs_get_name(zhp));
+               }
        }
-
        zfs_close(zhp);
        return (0);
 }
@@ -3072,7 +3643,7 @@ zfs_do_rollback(int argc, char **argv)
        boolean_t force = B_FALSE;
        rollback_cbdata_t cb = { 0 };
        zfs_handle_t *zhp, *snap;
-       char parentname[ZFS_MAXNAMELEN];
+       char parentname[ZFS_MAX_DATASET_NAME_LEN];
        char *delim;
 
        /* check options */
@@ -3129,7 +3700,9 @@ zfs_do_rollback(int argc, char **argv)
        cb.cb_create = zfs_prop_get_int(snap, ZFS_PROP_CREATETXG);
        cb.cb_first = B_TRUE;
        cb.cb_error = 0;
-       if ((ret = zfs_iter_children(zhp, rollback_check, &cb)) != 0)
+       if ((ret = zfs_iter_snapshots(zhp, B_FALSE, rollback_check, &cb)) != 0)
+               goto out;
+       if ((ret = zfs_iter_bookmarks(zhp, rollback_check, &cb)) != 0)
                goto out;
 
        if ((ret = cb.cb_error) != 0)
@@ -3151,21 +3724,17 @@ out:
 }
 
 /*
- * zfs set property=value { fs | snap | vol } ...
+ * zfs set property=value ... { fs | snap | vol } ...
  *
- * Sets the given property for all datasets specified on the command line.
+ * Sets the given properties for all datasets specified on the command line.
  */
-typedef struct set_cbdata {
-       char            *cb_propname;
-       char            *cb_value;
-} set_cbdata_t;
 
 static int
 set_callback(zfs_handle_t *zhp, void *data)
 {
-       set_cbdata_t *cbp = data;
+       nvlist_t *props = data;
 
-       if (zfs_prop_set(zhp, cbp->cb_propname, cbp->cb_value) != 0) {
+       if (zfs_prop_set_list(zhp, props) != 0) {
                switch (libzfs_errno(g_zfs)) {
                case EZFS_MOUNTFAILED:
                        (void) fprintf(stderr, gettext("property may be set "
@@ -3184,8 +3753,10 @@ set_callback(zfs_handle_t *zhp, void *data)
 static int
 zfs_do_set(int argc, char **argv)
 {
-       set_cbdata_t cb;
+       nvlist_t *props = NULL;
+       int ds_start = -1; /* argv idx of first dataset arg */
        int ret = 0;
+       int i;
 
        /* check for options */
        if (argc > 1 && argv[1][0] == '-') {
@@ -3196,39 +3767,88 @@ zfs_do_set(int argc, char **argv)
 
        /* check number of arguments */
        if (argc < 2) {
-               (void) fprintf(stderr, gettext("missing property=value "
-                   "argument\n"));
+               (void) fprintf(stderr, gettext("missing arguments\n"));
                usage(B_FALSE);
        }
        if (argc < 3) {
-               (void) fprintf(stderr, gettext("missing dataset name\n"));
+               if (strchr(argv[1], '=') == NULL) {
+                       (void) fprintf(stderr, gettext("missing property=value "
+                           "argument(s)\n"));
+               } else {
+                       (void) fprintf(stderr, gettext("missing dataset "
+                           "name(s)\n"));
+               }
                usage(B_FALSE);
        }
 
-       /* validate property=value argument */
-       cb.cb_propname = argv[1];
-       if (((cb.cb_value = strchr(cb.cb_propname, '=')) == NULL) ||
-           (cb.cb_value[1] == '\0')) {
-               (void) fprintf(stderr, gettext("missing value in "
-                   "property=value argument\n"));
+       /* validate argument order:  prop=val args followed by dataset args */
+       for (i = 1; i < argc; i++) {
+               if (strchr(argv[i], '=') != NULL) {
+                       if (ds_start > 0) {
+                               /* out-of-order prop=val argument */
+                               (void) fprintf(stderr, gettext("invalid "
+                                   "argument order\n"));
+                               usage(B_FALSE);
+                       }
+               } else if (ds_start < 0) {
+                       ds_start = i;
+               }
+       }
+       if (ds_start < 0) {
+               (void) fprintf(stderr, gettext("missing dataset name(s)\n"));
                usage(B_FALSE);
        }
 
-       *cb.cb_value = '\0';
-       cb.cb_value++;
-
-       if (*cb.cb_propname == '\0') {
-               (void) fprintf(stderr,
-                   gettext("missing property in property=value argument\n"));
-               usage(B_FALSE);
+       /* Populate a list of property settings */
+       if (nvlist_alloc(&props, NV_UNIQUE_NAME, 0) != 0)
+               nomem();
+       for (i = 1; i < ds_start; i++) {
+               if (!parseprop(props, argv[i])) {
+                       ret = -1;
+                       goto error;
+               }
        }
 
-       ret = zfs_for_each(argc - 2, argv + 2, 0,
-           ZFS_TYPE_DATASET, NULL, NULL, 0, set_callback, &cb);
+       ret = zfs_for_each(argc - ds_start, argv + ds_start, 0,
+           ZFS_TYPE_DATASET, NULL, NULL, 0, set_callback, props);
 
+error:
+       nvlist_free(props);
        return (ret);
 }
 
+typedef struct snap_cbdata {
+       nvlist_t *sd_nvl;
+       boolean_t sd_recursive;
+       const char *sd_snapname;
+} snap_cbdata_t;
+
+static int
+zfs_snapshot_cb(zfs_handle_t *zhp, void *arg)
+{
+       snap_cbdata_t *sd = arg;
+       char *name;
+       int rv = 0;
+       int error;
+
+       if (sd->sd_recursive &&
+           zfs_prop_get_int(zhp, ZFS_PROP_INCONSISTENT) != 0) {
+               zfs_close(zhp);
+               return (0);
+       }
+
+       error = asprintf(&name, "%s@%s", zfs_get_name(zhp), sd->sd_snapname);
+       if (error == -1)
+               nomem();
+       fnvlist_add_boolean(sd->sd_nvl, name);
+       free(name);
+
+       if (sd->sd_recursive)
+               rv = zfs_iter_filesystems(zhp, zfs_snapshot_cb, sd);
+       zfs_close(zhp);
+       return (rv);
+}
+
 /*
  * zfs snapshot [-r] [-o prop=value] ... <fs@snap>
  *
@@ -3238,23 +3858,30 @@ zfs_do_set(int argc, char **argv)
 static int
 zfs_do_snapshot(int argc, char **argv)
 {
-       boolean_t recursive = B_FALSE;
        int ret = 0;
        signed char c;
        nvlist_t *props;
+       snap_cbdata_t sd = { 0 };
+       boolean_t multiple_snaps = B_FALSE;
 
        if (nvlist_alloc(&props, NV_UNIQUE_NAME, 0) != 0)
                nomem();
+       if (nvlist_alloc(&sd.sd_nvl, NV_UNIQUE_NAME, 0) != 0)
+               nomem();
 
        /* check options */
        while ((c = getopt(argc, argv, "ro:")) != -1) {
                switch (c) {
                case 'o':
-                       if (parseprop(props))
+                       if (!parseprop(props, optarg)) {
+                               nvlist_free(sd.sd_nvl);
+                               nvlist_free(props);
                                return (1);
+                       }
                        break;
                case 'r':
-                       recursive = B_TRUE;
+                       sd.sd_recursive = B_TRUE;
+                       multiple_snaps = B_TRUE;
                        break;
                case '?':
                        (void) fprintf(stderr, gettext("invalid option '%c'\n"),
@@ -3271,27 +3898,41 @@ zfs_do_snapshot(int argc, char **argv)
                (void) fprintf(stderr, gettext("missing snapshot argument\n"));
                goto usage;
        }
-       if (argc > 1) {
-               (void) fprintf(stderr, gettext("too many arguments\n"));
-               goto usage;
+
+       if (argc > 1)
+               multiple_snaps = B_TRUE;
+       for (; argc > 0; argc--, argv++) {
+               char *atp;
+               zfs_handle_t *zhp;
+
+               atp = strchr(argv[0], '@');
+               if (atp == NULL)
+                       goto usage;
+               *atp = '\0';
+               sd.sd_snapname = atp + 1;
+               zhp = zfs_open(g_zfs, argv[0],
+                   ZFS_TYPE_FILESYSTEM | ZFS_TYPE_VOLUME);
+               if (zhp == NULL)
+                       goto usage;
+               if (zfs_snapshot_cb(zhp, &sd) != 0)
+                       goto usage;
        }
 
-       ret = zfs_snapshot(g_zfs, argv[0], recursive, props);
+       ret = zfs_snapshot_nvl(g_zfs, sd.sd_nvl, props);
+       nvlist_free(sd.sd_nvl);
        nvlist_free(props);
-       if (ret && recursive)
+       if (ret != 0 && multiple_snaps)
                (void) fprintf(stderr, gettext("no snapshots were created\n"));
        return (ret != 0);
 
 usage:
+       nvlist_free(sd.sd_nvl);
        nvlist_free(props);
        usage(B_FALSE);
        return (-1);
 }
 
 /*
- * zfs send [-vDp] -R [-i|-I <@snap>] <fs@snap>
- * zfs send [-vDp] [-i|-I <@snap>] <fs@snap>
- *
  * Send a backup stream to stdout.
  */
 static int
@@ -3299,15 +3940,33 @@ zfs_do_send(int argc, char **argv)
 {
        char *fromname = NULL;
        char *toname = NULL;
+       char *resume_token = NULL;
        char *cp;
        zfs_handle_t *zhp;
        sendflags_t flags = { 0 };
        int c, err;
-       nvlist_t *dbgnv;
+       nvlist_t *dbgnv = NULL;
        boolean_t extraverbose = B_FALSE;
 
+       struct option long_options[] = {
+               {"replicate",   no_argument,            NULL, 'R'},
+               {"props",       no_argument,            NULL, 'p'},
+               {"parsable",    no_argument,            NULL, 'P'},
+               {"dedup",       no_argument,            NULL, 'D'},
+               {"verbose",     no_argument,            NULL, 'v'},
+               {"dryrun",      no_argument,            NULL, 'n'},
+               {"large-block", no_argument,            NULL, 'L'},
+               {"embed",       no_argument,            NULL, 'e'},
+               {"resume",      required_argument,      NULL, 't'},
+               {"compressed",  no_argument,            NULL, 'c'},
+               {"raw",         no_argument,            NULL, 'w'},
+               {"backup",      no_argument,            NULL, 'b'},
+               {0, 0, 0, 0}
+       };
+
        /* check options */
-       while ((c = getopt(argc, argv, ":i:I:RDpv")) != -1) {
+       while ((c = getopt_long(argc, argv, ":i:I:RDpvnPLet:cwb", long_options,
+           NULL)) != -1) {
                switch (c) {
                case 'i':
                        if (fromname)
@@ -3326,22 +3985,80 @@ zfs_do_send(int argc, char **argv)
                case 'p':
                        flags.props = B_TRUE;
                        break;
+               case 'b':
+                       flags.backup = B_TRUE;
+                       break;
+               case 'P':
+                       flags.parsable = B_TRUE;
+                       flags.verbose = B_TRUE;
+                       break;
                case 'v':
                        if (flags.verbose)
                                extraverbose = B_TRUE;
                        flags.verbose = B_TRUE;
+                       flags.progress = B_TRUE;
                        break;
                case 'D':
                        flags.dedup = B_TRUE;
                        break;
-               case ':':
-                       (void) fprintf(stderr, gettext("missing argument for "
-                           "'%c' option\n"), optopt);
-                       usage(B_FALSE);
+               case 'n':
+                       flags.dryrun = B_TRUE;
+                       break;
+               case 'L':
+                       flags.largeblock = B_TRUE;
+                       break;
+               case 'e':
+                       flags.embed_data = B_TRUE;
+                       break;
+               case 't':
+                       resume_token = optarg;
+                       break;
+               case 'c':
+                       flags.compress = B_TRUE;
+                       break;
+               case 'w':
+                       flags.raw = B_TRUE;
+                       flags.compress = B_TRUE;
+                       flags.embed_data = B_TRUE;
+                       flags.largeblock = B_TRUE;
+                       break;
+               case ':':
+                       /*
+                        * If a parameter was not passed, optopt contains the
+                        * value that would normally lead us into the
+                        * appropriate case statement.  If it's > 256, then this
+                        * must be a longopt and we should look at argv to get
+                        * the string.  Otherwise it's just the character, so we
+                        * should use it directly.
+                        */
+                       if (optopt <= UINT8_MAX) {
+                               (void) fprintf(stderr,
+                                   gettext("missing argument for '%c' "
+                                   "option\n"), optopt);
+                       } else {
+                               (void) fprintf(stderr,
+                                   gettext("missing argument for '%s' "
+                                   "option\n"), argv[optind - 1]);
+                       }
+                       usage(B_FALSE);
                        break;
                case '?':
-                       (void) fprintf(stderr, gettext("invalid option '%c'\n"),
-                           optopt);
+                       /*FALLTHROUGH*/
+               default:
+                       /*
+                        * If an invalid flag was passed, optopt contains the
+                        * character if it was a short flag, or 0 if it was a
+                        * longopt.
+                        */
+                       if (optopt != 0) {
+                               (void) fprintf(stderr,
+                                   gettext("invalid option '%c'\n"), optopt);
+                       } else {
+                               (void) fprintf(stderr,
+                                   gettext("invalid option '%s'\n"),
+                                   argv[optind - 1]);
+
+                       }
                        usage(B_FALSE);
                }
        }
@@ -3349,29 +4066,81 @@ zfs_do_send(int argc, char **argv)
        argc -= optind;
        argv += optind;
 
-       /* check number of arguments */
-       if (argc < 1) {
-               (void) fprintf(stderr, gettext("missing snapshot argument\n"));
-               usage(B_FALSE);
-       }
-       if (argc > 1) {
-               (void) fprintf(stderr, gettext("too many arguments\n"));
-               usage(B_FALSE);
+       if (resume_token != NULL) {
+               if (fromname != NULL || flags.replicate || flags.props ||
+                   flags.backup || flags.dedup) {
+                       (void) fprintf(stderr,
+                           gettext("invalid flags combined with -t\n"));
+                       usage(B_FALSE);
+               }
+               if (argc != 0) {
+                       (void) fprintf(stderr, gettext("no additional "
+                           "arguments are permitted with -t\n"));
+                       usage(B_FALSE);
+               }
+       } else {
+               if (argc < 1) {
+                       (void) fprintf(stderr,
+                           gettext("missing snapshot argument\n"));
+                       usage(B_FALSE);
+               }
+               if (argc > 1) {
+                       (void) fprintf(stderr, gettext("too many arguments\n"));
+                       usage(B_FALSE);
+               }
        }
 
-       if (isatty(STDOUT_FILENO)) {
+       if (!flags.dryrun && isatty(STDOUT_FILENO)) {
                (void) fprintf(stderr,
                    gettext("Error: Stream can not be written to a terminal.\n"
                    "You must redirect standard output.\n"));
                return (1);
        }
 
-       cp = strchr(argv[0], '@');
-       if (cp == NULL) {
-               (void) fprintf(stderr,
-                   gettext("argument must be a snapshot\n"));
-               usage(B_FALSE);
+       if (resume_token != NULL) {
+               return (zfs_send_resume(g_zfs, &flags, STDOUT_FILENO,
+                   resume_token));
+       }
+
+       /*
+        * Special case sending a filesystem, or from a bookmark.
+        */
+       if (strchr(argv[0], '@') == NULL ||
+           (fromname && strchr(fromname, '#') != NULL)) {
+               char frombuf[ZFS_MAX_DATASET_NAME_LEN];
+
+               if (flags.replicate || flags.doall || flags.props ||
+                   flags.backup || flags.dedup ||
+                   (strchr(argv[0], '@') == NULL &&
+                   (flags.dryrun || flags.verbose || flags.progress))) {
+                       (void) fprintf(stderr, gettext("Error: "
+                           "Unsupported flag with filesystem or bookmark.\n"));
+                       return (1);
+               }
+
+               zhp = zfs_open(g_zfs, argv[0], ZFS_TYPE_DATASET);
+               if (zhp == NULL)
+                       return (1);
+
+               if (fromname != NULL &&
+                   (fromname[0] == '#' || fromname[0] == '@')) {
+                       /*
+                        * Incremental source name begins with # or @.
+                        * Default to same fs as target.
+                        */
+                       (void) strlcpy(frombuf, argv[0], sizeof (frombuf));
+                       cp = strchr(frombuf, '@');
+                       if (cp != NULL)
+                               *cp = '\0';
+                       (void) strlcat(frombuf, fromname, sizeof (frombuf));
+                       fromname = frombuf;
+               }
+               err = zfs_send_one(zhp, fromname, STDOUT_FILENO, flags);
+               zfs_close(zhp);
+               return (err != 0);
        }
+
+       cp = strchr(argv[0], '@');
        *cp = '\0';
        toname = cp + 1;
        zhp = zfs_open(g_zfs, argv[0], ZFS_TYPE_FILESYSTEM | ZFS_TYPE_VOLUME);
@@ -3384,7 +4153,7 @@ zfs_do_send(int argc, char **argv)
         * case if they specify the origin.
         */
        if (fromname && (cp = strchr(fromname, '@')) != NULL) {
-               char origin[ZFS_MAXNAMELEN];
+               char origin[ZFS_MAX_DATASET_NAME_LEN];
                zprop_source_t src;
 
                (void) zfs_prop_get(zhp, ZFS_PROP_ORIGIN,
@@ -3413,10 +4182,10 @@ zfs_do_send(int argc, char **argv)
        if (flags.replicate && fromname == NULL)
                flags.doall = B_TRUE;
 
-       err = zfs_send(zhp, fromname, toname, flags, STDOUT_FILENO, NULL, 0,
+       err = zfs_send(zhp, fromname, toname, &flags, STDOUT_FILENO, NULL, 0,
            extraverbose ? &dbgnv : NULL);
 
-       if (extraverbose) {
+       if (extraverbose && dbgnv != NULL) {
                /*
                 * dump_nvlist prints to stdout, but that's been
                 * redirected to a file.  Make it print to stderr
@@ -3432,19 +4201,34 @@ zfs_do_send(int argc, char **argv)
 }
 
 /*
- * zfs receive [-vnFu] [-d | -e] <fs@snap>
- *
  * Restore a backup stream from stdin.
  */
 static int
 zfs_do_receive(int argc, char **argv)
 {
-       int c, err;
+       int c, err = 0;
        recvflags_t flags = { 0 };
+       boolean_t abort_resumable = B_FALSE;
+       nvlist_t *props;
+
+       if (nvlist_alloc(&props, NV_UNIQUE_NAME, 0) != 0)
+               nomem();
 
        /* check options */
-       while ((c = getopt(argc, argv, ":denuvF")) != -1) {
+       while ((c = getopt(argc, argv, ":o:x:denuvFsA")) != -1) {
                switch (c) {
+               case 'o':
+                       if (!parseprop(props, optarg)) {
+                               nvlist_free(props);
+                               usage(B_FALSE);
+                       }
+                       break;
+               case 'x':
+                       if (!parsepropname(props, optarg)) {
+                               nvlist_free(props);
+                               usage(B_FALSE);
+                       }
+                       break;
                case 'd':
                        flags.isprefix = B_TRUE;
                        break;
@@ -3461,9 +4245,15 @@ zfs_do_receive(int argc, char **argv)
                case 'v':
                        flags.verbose = B_TRUE;
                        break;
+               case 's':
+                       flags.resumable = B_TRUE;
+                       break;
                case 'F':
                        flags.force = B_TRUE;
                        break;
+               case 'A':
+                       abort_resumable = B_TRUE;
+                       break;
                case ':':
                        (void) fprintf(stderr, gettext("missing argument for "
                            "'%c' option\n"), optopt);
@@ -3489,15 +4279,60 @@ zfs_do_receive(int argc, char **argv)
                usage(B_FALSE);
        }
 
+       if (abort_resumable) {
+               if (flags.isprefix || flags.istail || flags.dryrun ||
+                   flags.resumable || flags.nomount) {
+                       (void) fprintf(stderr, gettext("invalid option\n"));
+                       usage(B_FALSE);
+               }
+
+               char namebuf[ZFS_MAX_DATASET_NAME_LEN];
+               (void) snprintf(namebuf, sizeof (namebuf),
+                   "%s/%%recv", argv[0]);
+
+               if (zfs_dataset_exists(g_zfs, namebuf,
+                   ZFS_TYPE_FILESYSTEM | ZFS_TYPE_VOLUME)) {
+                       zfs_handle_t *zhp = zfs_open(g_zfs,
+                           namebuf, ZFS_TYPE_FILESYSTEM | ZFS_TYPE_VOLUME);
+                       if (zhp == NULL) {
+                               nvlist_free(props);
+                               return (1);
+                       }
+                       err = zfs_destroy(zhp, B_FALSE);
+                       zfs_close(zhp);
+               } else {
+                       zfs_handle_t *zhp = zfs_open(g_zfs,
+                           argv[0], ZFS_TYPE_FILESYSTEM | ZFS_TYPE_VOLUME);
+                       if (zhp == NULL)
+                               usage(B_FALSE);
+                       if (!zfs_prop_get_int(zhp, ZFS_PROP_INCONSISTENT) ||
+                           zfs_prop_get(zhp, ZFS_PROP_RECEIVE_RESUME_TOKEN,
+                           NULL, 0, NULL, NULL, 0, B_TRUE) == -1) {
+                               (void) fprintf(stderr,
+                                   gettext("'%s' does not have any "
+                                   "resumable receive state to abort\n"),
+                                   argv[0]);
+                               nvlist_free(props);
+                               zfs_close(zhp);
+                               return (1);
+                       }
+                       err = zfs_destroy(zhp, B_FALSE);
+                       zfs_close(zhp);
+               }
+               nvlist_free(props);
+               return (err != 0);
+       }
+
        if (isatty(STDIN_FILENO)) {
                (void) fprintf(stderr,
                    gettext("Error: Backup stream can not be read "
                    "from a terminal.\n"
                    "You must redirect standard input.\n"));
+               nvlist_free(props);
                return (1);
        }
-
-       err = zfs_receive(g_zfs, argv[0], flags, STDIN_FILENO, NULL);
+       err = zfs_receive(g_zfs, argv[0], props, &flags, STDIN_FILENO, NULL);
+       nvlist_free(props);
 
        return (err != 0);
 }
@@ -3524,9 +4359,23 @@ zfs_do_receive(int argc, char **argv)
 #define        ZFS_DELEG_PERM_GROUPQUOTA       "groupquota"
 #define        ZFS_DELEG_PERM_USERUSED         "userused"
 #define        ZFS_DELEG_PERM_GROUPUSED        "groupused"
+#define        ZFS_DELEG_PERM_USEROBJQUOTA     "userobjquota"
+#define        ZFS_DELEG_PERM_GROUPOBJQUOTA    "groupobjquota"
+#define        ZFS_DELEG_PERM_USEROBJUSED      "userobjused"
+#define        ZFS_DELEG_PERM_GROUPOBJUSED     "groupobjused"
+
 #define        ZFS_DELEG_PERM_HOLD             "hold"
 #define        ZFS_DELEG_PERM_RELEASE          "release"
 #define        ZFS_DELEG_PERM_DIFF             "diff"
+#define        ZFS_DELEG_PERM_BOOKMARK         "bookmark"
+#define        ZFS_DELEG_PERM_REMAP            "remap"
+#define        ZFS_DELEG_PERM_LOAD_KEY         "load-key"
+#define        ZFS_DELEG_PERM_CHANGE_KEY       "change-key"
+
+#define        ZFS_DELEG_PERM_PROJECTUSED      "projectused"
+#define        ZFS_DELEG_PERM_PROJECTQUOTA     "projectquota"
+#define        ZFS_DELEG_PERM_PROJECTOBJUSED   "projectobjused"
+#define        ZFS_DELEG_PERM_PROJECTOBJQUOTA  "projectobjquota"
 
 #define        ZFS_NUM_DELEG_NOTES ZFS_DELEG_NOTE_NONE
 
@@ -3546,12 +4395,24 @@ static zfs_deleg_perm_tab_t zfs_deleg_perm_tbl[] = {
        { ZFS_DELEG_PERM_SEND, ZFS_DELEG_NOTE_SEND },
        { ZFS_DELEG_PERM_SHARE, ZFS_DELEG_NOTE_SHARE },
        { ZFS_DELEG_PERM_SNAPSHOT, ZFS_DELEG_NOTE_SNAPSHOT },
+       { ZFS_DELEG_PERM_BOOKMARK, ZFS_DELEG_NOTE_BOOKMARK },
+       { ZFS_DELEG_PERM_REMAP, ZFS_DELEG_NOTE_REMAP },
+       { ZFS_DELEG_PERM_LOAD_KEY, ZFS_DELEG_NOTE_LOAD_KEY },
+       { ZFS_DELEG_PERM_CHANGE_KEY, ZFS_DELEG_NOTE_CHANGE_KEY },
 
        { ZFS_DELEG_PERM_GROUPQUOTA, ZFS_DELEG_NOTE_GROUPQUOTA },
        { ZFS_DELEG_PERM_GROUPUSED, ZFS_DELEG_NOTE_GROUPUSED },
        { ZFS_DELEG_PERM_USERPROP, ZFS_DELEG_NOTE_USERPROP },
        { ZFS_DELEG_PERM_USERQUOTA, ZFS_DELEG_NOTE_USERQUOTA },
        { ZFS_DELEG_PERM_USERUSED, ZFS_DELEG_NOTE_USERUSED },
+       { ZFS_DELEG_PERM_USEROBJQUOTA, ZFS_DELEG_NOTE_USEROBJQUOTA },
+       { ZFS_DELEG_PERM_USEROBJUSED, ZFS_DELEG_NOTE_USEROBJUSED },
+       { ZFS_DELEG_PERM_GROUPOBJQUOTA, ZFS_DELEG_NOTE_GROUPOBJQUOTA },
+       { ZFS_DELEG_PERM_GROUPOBJUSED, ZFS_DELEG_NOTE_GROUPOBJUSED },
+       { ZFS_DELEG_PERM_PROJECTUSED, ZFS_DELEG_NOTE_PROJECTUSED },
+       { ZFS_DELEG_PERM_PROJECTQUOTA, ZFS_DELEG_NOTE_PROJECTQUOTA },
+       { ZFS_DELEG_PERM_PROJECTOBJUSED, ZFS_DELEG_NOTE_PROJECTOBJUSED },
+       { ZFS_DELEG_PERM_PROJECTOBJQUOTA, ZFS_DELEG_NOTE_PROJECTOBJQUOTA },
        { NULL, ZFS_DELEG_NOTE_NONE }
 };
 
@@ -3629,6 +4490,14 @@ deleg_perm_type(zfs_deleg_note_t note)
        case ZFS_DELEG_NOTE_USERPROP:
        case ZFS_DELEG_NOTE_USERQUOTA:
        case ZFS_DELEG_NOTE_USERUSED:
+       case ZFS_DELEG_NOTE_USEROBJQUOTA:
+       case ZFS_DELEG_NOTE_USEROBJUSED:
+       case ZFS_DELEG_NOTE_GROUPOBJQUOTA:
+       case ZFS_DELEG_NOTE_GROUPOBJUSED:
+       case ZFS_DELEG_NOTE_PROJECTUSED:
+       case ZFS_DELEG_NOTE_PROJECTQUOTA:
+       case ZFS_DELEG_NOTE_PROJECTOBJUSED:
+       case ZFS_DELEG_NOTE_PROJECTOBJQUOTA:
                /* other */
                return (gettext("other"));
        default:
@@ -3636,7 +4505,7 @@ deleg_perm_type(zfs_deleg_note_t note)
        }
 }
 
-static int inline
+static int
 who_type2weight(zfs_deleg_who_type_t who_type)
 {
        int res;
@@ -3856,7 +4725,7 @@ fs_perm_fini(fs_perm_t *fsperm)
        uu_avl_destroy(fsperm->fsp_uge_avl);
 }
 
-static void inline
+static void
 set_deleg_perm_node(uu_avl_t *avl, deleg_perm_node_t *node,
     zfs_deleg_who_type_t who_type, const char *name, char locality)
 {
@@ -3953,8 +4822,9 @@ parse_fs_perm(fs_perm_t *fsperm, nvlist_t *nvl)
                        avl_pool = fspset->fsps_who_perm_avl_pool;
                        avl = fsperm->fsp_uge_avl;
                        break;
+
                default:
-                       break;
+                       assert(!"unhandled zfs_deleg_who_type_t");
                }
 
                if (is_set) {
@@ -3990,6 +4860,7 @@ parse_fs_perm(fs_perm_t *fsperm, nvlist_t *nvl)
                                                if (g)
                                                        nice_name = g->gr_name;
                                                break;
+
                                        default:
                                                break;
                                        }
@@ -4006,7 +4877,7 @@ parse_fs_perm(fs_perm_t *fsperm, nvlist_t *nvl)
                                who_perm = &node->who_perm;
                        }
                }
-
+               VERIFY3P(who_perm, !=, NULL);
                (void) parse_who_perm(who_perm, nvl2, perm_locality);
        }
 
@@ -4111,6 +4982,12 @@ deleg_perm_comment(zfs_deleg_note_t note)
        case ZFS_DELEG_NOTE_SNAPSHOT:
                str = gettext("");
                break;
+       case ZFS_DELEG_NOTE_LOAD_KEY:
+               str = gettext("Allows loading or unloading an encryption key");
+               break;
+       case ZFS_DELEG_NOTE_CHANGE_KEY:
+               str = gettext("Allows changing or adding an encryption key");
+               break;
 /*
  *     case ZFS_DELEG_NOTE_VSCAN:
  *             str = gettext("");
@@ -4132,6 +5009,33 @@ deleg_perm_comment(zfs_deleg_note_t note)
        case ZFS_DELEG_NOTE_USERUSED:
                str = gettext("Allows reading any userused@... property");
                break;
+       case ZFS_DELEG_NOTE_USEROBJQUOTA:
+               str = gettext("Allows accessing any userobjquota@... property");
+               break;
+       case ZFS_DELEG_NOTE_GROUPOBJQUOTA:
+               str = gettext("Allows accessing any \n\t\t\t\t"
+                   "groupobjquota@... property");
+               break;
+       case ZFS_DELEG_NOTE_GROUPOBJUSED:
+               str = gettext("Allows reading any groupobjused@... property");
+               break;
+       case ZFS_DELEG_NOTE_USEROBJUSED:
+               str = gettext("Allows reading any userobjused@... property");
+               break;
+       case ZFS_DELEG_NOTE_PROJECTQUOTA:
+               str = gettext("Allows accessing any projectquota@... property");
+               break;
+       case ZFS_DELEG_NOTE_PROJECTOBJQUOTA:
+               str = gettext("Allows accessing any \n\t\t\t\t"
+                   "projectobjquota@... property");
+               break;
+       case ZFS_DELEG_NOTE_PROJECTUSED:
+               str = gettext("Allows reading any projectused@... property");
+               break;
+       case ZFS_DELEG_NOTE_PROJECTOBJUSED:
+               str = gettext("Allows accessing any \n\t\t\t\t"
+                   "projectobjused@... property");
+               break;
                /* other */
        default:
                str = "";
@@ -4192,9 +5096,9 @@ allow_usage(boolean_t un, boolean_t requested, const char *msg)
        (void) fprintf(fp, gettext("Usage: %s\n"), get_usage(un ? HELP_UNALLOW :
            HELP_ALLOW));
        (void) fprintf(fp, gettext("Options:\n"));
-       for (i = 0; i < (un ? unallow_size : allow_size); i++) {
-               const char *opt = opt_desc[i++];
-               const char *optdsc = opt_desc[i];
+       for (i = 0; i < (un ? unallow_size : allow_size); i += 2) {
+               const char *opt = opt_desc[i];
+               const char *optdsc = opt_desc[i + 1];
                (void) fprintf(fp, gettext("  %-10s  %s\n"), opt, optdsc);
        }
 
@@ -4293,7 +5197,7 @@ parse_allow_args(int argc, char **argv, boolean_t un, struct allow_opts *opts)
                argc--;
                argv++;
                opts->dataset = munge_args(argc, argv, un, 2, &opts->perms);
-       } else if (argc == 1) {
+       } else if (argc == 1 && !un) {
                opts->prt_perms = B_TRUE;
                opts->dataset = argv[argc-1];
        } else {
@@ -4313,9 +5217,9 @@ store_allow_perm(zfs_deleg_who_type_t type, boolean_t local, boolean_t descend,
 {
        int i;
        char ld[2] = { '\0', '\0' };
-       char who_buf[ZFS_MAXNAMELEN+32];
-       char base_type = ZFS_DELEG_WHO_UNKNOWN;
-       char set_type = ZFS_DELEG_WHO_UNKNOWN;
+       char who_buf[MAXNAMELEN + 32];
+       char base_type = '\0';
+       char set_type = '\0';
        nvlist_t *base_nvl = NULL;
        nvlist_t *set_nvl = NULL;
        nvlist_t *nvl;
@@ -4364,8 +5268,10 @@ store_allow_perm(zfs_deleg_who_type_t type, boolean_t local, boolean_t descend,
                        ld[0] = ZFS_DELEG_LOCAL;
                if (descend)
                        ld[1] = ZFS_DELEG_DESCENDENT;
-       default:
                break;
+
+       default:
+               assert(set_type != '\0' && base_type != '\0');
        }
 
        if (perms != NULL) {
@@ -4599,7 +5505,7 @@ print_set_creat_perms(uu_avl_t *who_avl)
        }
 }
 
-static void inline
+static void
 print_uge_deleg_perms(uu_avl_t *who_avl, boolean_t local, boolean_t descend,
     const char *title)
 {
@@ -4650,8 +5556,10 @@ print_uge_deleg_perms(uu_avl_t *who_avl, boolean_t local, boolean_t descend,
                                case ZFS_DELEG_EVERYONE:
                                        who = gettext("everyone");
                                        who_name = NULL;
-                               default:
                                        break;
+
+                               default:
+                                       assert(who != NULL);
                                }
 
                                prt_who = B_FALSE;
@@ -4677,7 +5585,7 @@ static void
 print_fs_perms(fs_perm_set_t *fspset)
 {
        fs_perm_node_t *node = NULL;
-       char buf[ZFS_MAXNAMELEN+32];
+       char buf[MAXNAMELEN + 32];
        const char *dsname = buf;
 
        for (node = uu_list_first(fspset->fsps_list); node != NULL;
@@ -4686,7 +5594,7 @@ print_fs_perms(fs_perm_set_t *fspset)
                uu_avl_t *uge_avl = node->fspn_fsperm.fsp_uge_avl;
                int left = 0;
 
-               (void) snprintf(buf, ZFS_MAXNAMELEN+32,
+               (void) snprintf(buf, sizeof (buf),
                    gettext("---- Permissions on %s "),
                    node->fspn_fsperm.fsp_name);
                (void) printf("%s", dsname);
@@ -4785,9 +5693,9 @@ zfs_do_allow_unallow_impl(int argc, char **argv, boolean_t un)
        parse_allow_args(argc, argv, un, &opts);
 
        /* try to open the dataset */
-       if ((zhp = zfs_open(g_zfs, opts.dataset, ZFS_TYPE_FILESYSTEM))
-           == NULL) {
-               (void) fprintf(stderr, "Failed to open Dataset *%s*\n",
+       if ((zhp = zfs_open(g_zfs, opts.dataset, ZFS_TYPE_FILESYSTEM |
+           ZFS_TYPE_VOLUME)) == NULL) {
+               (void) fprintf(stderr, "Failed to open dataset: %s\n",
                    opts.dataset);
                return (-1);
        }
@@ -4797,7 +5705,7 @@ zfs_do_allow_unallow_impl(int argc, char **argv, boolean_t un)
 
        fs_perm_set_init(&fs_perm_set);
        if (parse_fs_perm_set(&fs_perm_set, perm_nvl) != 0) {
-               (void) fprintf(stderr, "Failed to parse fsacl permissionsn");
+               (void) fprintf(stderr, "Failed to parse fsacl permissions\n");
                goto cleanup1;
        }
 
@@ -4820,8 +5728,7 @@ zfs_do_allow_unallow_impl(int argc, char **argv, boolean_t un)
 
 cleanup0:
        nvlist_free(perm_nvl);
-       if (update_perm_nvl != NULL)
-               nvlist_free(update_perm_nvl);
+       nvlist_free(update_perm_nvl);
 cleanup1:
        fs_perm_set_fini(&fs_perm_set);
 cleanup2:
@@ -4830,28 +5737,12 @@ cleanup2:
        return (error);
 }
 
-/*
- * zfs allow [-r] [-t] <tag> <snap> ...
- *
- *     -r      Recursively hold
- *     -t      Temporary hold (hidden option)
- *
- * Apply a user-hold with the given tag to the list of snapshots.
- */
 static int
 zfs_do_allow(int argc, char **argv)
 {
        return (zfs_do_allow_unallow_impl(argc, argv, B_FALSE));
 }
 
-/*
- * zfs unallow [-r] [-t] <tag> <snap> ...
- *
- *     -r      Recursively hold
- *     -t      Temporary hold (hidden option)
- *
- * Apply a user-hold with the given tag to the list of snapshots.
- */
 static int
 zfs_do_unallow(int argc, char **argv)
 {
@@ -4865,7 +5756,6 @@ zfs_do_hold_rele_impl(int argc, char **argv, boolean_t holding)
        int i;
        const char *tag;
        boolean_t recursive = B_FALSE;
-       boolean_t temphold = B_FALSE;
        const char *opts = holding ? "rt" : "r";
        int c;
 
@@ -4875,9 +5765,6 @@ zfs_do_hold_rele_impl(int argc, char **argv, boolean_t holding)
                case 'r':
                        recursive = B_TRUE;
                        break;
-               case 't':
-                       temphold = B_TRUE;
-                       break;
                case '?':
                        (void) fprintf(stderr, gettext("invalid option '%c'\n"),
                            optopt);
@@ -4904,7 +5791,7 @@ zfs_do_hold_rele_impl(int argc, char **argv, boolean_t holding)
 
        for (i = 0; i < argc; ++i) {
                zfs_handle_t *zhp;
-               char parent[ZFS_MAXNAMELEN];
+               char parent[ZFS_MAX_DATASET_NAME_LEN];
                const char *delim;
                char *path = argv[i];
 
@@ -4925,8 +5812,7 @@ zfs_do_hold_rele_impl(int argc, char **argv, boolean_t holding)
                        continue;
                }
                if (holding) {
-                       if (zfs_hold(zhp, delim+1, tag, recursive,
-                           temphold, B_FALSE, -1, 0, 0) != 0)
+                       if (zfs_hold(zhp, delim+1, tag, recursive, -1) != 0)
                                ++errors;
                } else {
                        if (zfs_release(zhp, delim+1, tag, recursive) != 0)
@@ -4942,7 +5828,6 @@ zfs_do_hold_rele_impl(int argc, char **argv, boolean_t holding)
  * zfs hold [-r] [-t] <tag> <snap> ...
  *
  *     -r      Recursively hold
- *     -t      Temporary hold (hidden option)
  *
  * Apply a user-hold with the given tag to the list of snapshots.
  */
@@ -5008,8 +5893,6 @@ print_holds(boolean_t scripted, int nwidth, int tagwidth, nvlist_t *nvl)
                        uint64_t val = 0;
                        time_t time;
                        struct tm t;
-                       char sep = scripted ? '\t' : ' ';
-                       int sepnum = scripted ? 1 : 2;
 
                        (void) nvpair_value_uint64(nvp2, &val);
                        time = (time_t)val;
@@ -5017,8 +5900,13 @@ print_holds(boolean_t scripted, int nwidth, int tagwidth, nvlist_t *nvl)
                        (void) strftime(tsbuf, DATETIME_BUF_LEN,
                            gettext(STRFTIME_FMT_STR), &t);
 
-                       (void) printf("%-*s%*c%-*s%*c%s\n", nwidth, zname,
-                           sepnum, sep, tagwidth, tagname, sepnum, sep, tsbuf);
+                       if (scripted) {
+                               (void) printf("%s\t%s\t%s\n", zname,
+                                   tagname, tsbuf);
+                       } else {
+                               (void) printf("%-*s  %-*s  %s\n", nwidth,
+                                   zname, tagwidth, tagname, tsbuf);
+                       }
                }
        }
 }
@@ -5034,7 +5922,7 @@ holds_callback(zfs_handle_t *zhp, void *data)
        nvlist_t *nvl = NULL;
        nvpair_t *nvp = NULL;
        const char *zname = zfs_get_name(zhp);
-       size_t znamelen = strnlen(zname, ZFS_MAXNAMELEN);
+       size_t znamelen = strlen(zname);
 
        if (cbp->cb_recursive) {
                const char *snapname;
@@ -5055,7 +5943,7 @@ holds_callback(zfs_handle_t *zhp, void *data)
 
        while ((nvp = nvlist_next_nvpair(nvl, nvp)) != NULL) {
                const char *tag = nvpair_name(nvp);
-               size_t taglen = strnlen(tag, MAXNAMELEN);
+               size_t taglen = strlen(tag);
                if (taglen > cbp->cb_max_taglen)
                        cbp->cb_max_taglen  = taglen;
        }
@@ -5185,7 +6073,7 @@ get_one_dataset(zfs_handle_t *zhp, void *data)
        }
 
        /*
-        * Interate over any nested datasets.
+        * Iterate over any nested datasets.
         */
        if (zfs_iter_filesystems(zhp, get_one_dataset, data) != 0) {
                zfs_close(zhp);
@@ -5347,6 +6235,40 @@ share_mount_one(zfs_handle_t *zhp, int op, int flags, char *protocol,
                return (0);
        }
 
+       /*
+        * If this filesystem is encrypted and does not have
+        * a loaded key, we can not mount it.
+        */
+       if ((flags & MS_CRYPT) == 0 &&
+           zfs_prop_get_int(zhp, ZFS_PROP_ENCRYPTION) != ZIO_CRYPT_OFF &&
+           zfs_prop_get_int(zhp, ZFS_PROP_KEYSTATUS) ==
+           ZFS_KEYSTATUS_UNAVAILABLE) {
+               if (!explicit)
+                       return (0);
+
+               (void) fprintf(stderr, gettext("cannot %s '%s': "
+                   "encryption key not loaded\n"), cmdname, zfs_get_name(zhp));
+               return (1);
+       }
+
+       /*
+        * If this filesystem is inconsistent and has a receive resume
+        * token, we can not mount it.
+        */
+       if (zfs_prop_get_int(zhp, ZFS_PROP_INCONSISTENT) &&
+           zfs_prop_get(zhp, ZFS_PROP_RECEIVE_RESUME_TOKEN,
+           NULL, 0, NULL, NULL, 0, B_TRUE) == 0) {
+               if (!explicit)
+                       return (0);
+
+               (void) fprintf(stderr, gettext("cannot %s '%s': "
+                   "Contains partially-completed state from "
+                   "\"zfs receive -s\", which can be resumed with "
+                   "\"zfs send -t\"\n"),
+                   cmdname, zfs_get_name(zhp));
+               return (1);
+       }
+
        /*
         * At this point, we have verified that the mountpoint and/or
         * shareopts are appropriate for auto management. If the
@@ -5361,10 +6283,10 @@ share_mount_one(zfs_handle_t *zhp, int op, int flags, char *protocol,
                shared_smb = zfs_is_shared_smb(zhp, NULL);
 
                if ((shared_nfs && shared_smb) ||
-                   ((shared_nfs && strcmp(shareopts, "on") == 0) &&
-                   (strcmp(smbshareopts, "off") == 0)) ||
-                   ((shared_smb && strcmp(smbshareopts, "on") == 0) &&
-                   (strcmp(shareopts, "off") == 0))) {
+                   (shared_nfs && strcmp(shareopts, "on") == 0 &&
+                   strcmp(smbshareopts, "off") == 0) ||
+                   (shared_smb && strcmp(smbshareopts, "on") == 0 &&
+                   strcmp(shareopts, "off") == 0)) {
                        if (!explicit)
                                return (0);
 
@@ -5375,7 +6297,7 @@ share_mount_one(zfs_handle_t *zhp, int op, int flags, char *protocol,
                }
 
                if (!zfs_is_mounted(zhp, NULL) &&
-                   zfs_mount(zhp, NULL, 0) != 0)
+                   zfs_mount(zhp, NULL, flags) != 0)
                        return (1);
 
                if (protocol == NULL) {
@@ -5482,7 +6404,7 @@ share_mount(int op, int argc, char **argv)
        int flags = 0;
 
        /* check options */
-       while ((c = getopt(argc, argv, op == OP_MOUNT ? ":avo:O" : "a"))
+       while ((c = getopt(argc, argv, op == OP_MOUNT ? ":alvo:O" : "al"))
            != -1) {
                switch (c) {
                case 'a':
@@ -5491,6 +6413,9 @@ share_mount(int op, int argc, char **argv)
                case 'v':
                        verbose = B_TRUE;
                        break;
+               case 'l':
+                       flags |= MS_CRYPT;
+                       break;
                case 'o':
                        if (*optarg == '\0') {
                                (void) fprintf(stderr, gettext("empty mount "
@@ -5548,8 +6473,11 @@ share_mount(int op, int argc, char **argv)
                start_progress_timer();
                get_all_datasets(&dslist, &count, verbose);
 
-               if (count == 0)
+               if (count == 0) {
+                       if (options != NULL)
+                               free(options);
                        return (0);
+               }
 
                qsort(dslist, count, sizeof (void *), libzfs_dataset_cmp);
 
@@ -5574,11 +6502,19 @@ share_mount(int op, int argc, char **argv)
                }
 
                /*
-                * When mount is given no arguments, go through /etc/mtab and
-                * display any active ZFS mounts.  We hide any snapshots, since
-                * they are controlled automatically.
+                * When mount is given no arguments, go through
+                * /proc/self/mounts and display any active ZFS mounts.
+                * We hide any snapshots, since they are controlled
+                * automatically.
                 */
-               rewind(mnttab_file);
+
+               /* Reopen MNTTAB to prevent reading stale data from open file */
+               if (freopen(MNTTAB, "r", mnttab_file) == NULL) {
+                       if (options != NULL)
+                               free(options);
+                       return (ENOENT);
+               }
+
                while (getmntent(mnttab_file, &entry) == 0) {
                        if (strcmp(entry.mnt_fstype, MNTTYPE_ZFS) != 0 ||
                            strchr(entry.mnt_special, '@') != NULL)
@@ -5607,6 +6543,9 @@ share_mount(int op, int argc, char **argv)
                }
        }
 
+       if (options != NULL)
+               free(options);
+
        return (ret);
 }
 
@@ -5652,8 +6591,8 @@ unshare_unmount_compare(const void *larg, const void *rarg, void *unused)
 
 /*
  * Convenience routine used by zfs_do_umount() and manual_unmount().  Given an
- * absolute path, find the entry /etc/mtab, verify that its a ZFS filesystem,
- * and unmount it appropriately.
+ * absolute path, find the entry /proc/self/mounts, verify that its a
+ * ZFS filesystems, and unmount it appropriately.
  */
 static int
 unshare_unmount_path(int op, char *path, int flags, boolean_t is_manual)
@@ -5666,7 +6605,7 @@ unshare_unmount_path(int op, char *path, int flags, boolean_t is_manual)
        ino_t path_inode;
 
        /*
-        * Search for the path in /etc/mtab.  Rather than looking for the
+        * Search for the path in /proc/self/mounts. Rather than looking for the
         * specific path, which can be fooled by non-standard paths (i.e. ".."
         * or "//"), we stat() the path and search for the corresponding
         * (major,minor) device pair.
@@ -5681,7 +6620,11 @@ unshare_unmount_path(int op, char *path, int flags, boolean_t is_manual)
        /*
         * Search for the given (major,minor) pair in the mount table.
         */
-       rewind(mnttab_file);
+
+       /* Reopen MNTTAB to prevent reading stale data from open file */
+       if (freopen(MNTTAB, "r", mnttab_file) == NULL)
+               return (ENOENT);
+
        while ((ret = getextmntent(mnttab_file, &entry, 0)) == 0) {
                if (entry.mnt_major == major(statbuf.st_dev) &&
                    entry.mnt_minor == minor(statbuf.st_dev))
@@ -5693,8 +6636,8 @@ unshare_unmount_path(int op, char *path, int flags, boolean_t is_manual)
                            "currently mounted\n"), cmdname, path);
                        return (1);
                }
-               (void) fprintf(stderr, gettext("warning: %s not in mtab\n"),
-                   path);
+               (void) fprintf(stderr, gettext("warning: %s not in"
+                   "/proc/self/mounts\n"), path);
                if ((ret = umount2(path, flags)) != 0)
                        (void) fprintf(stderr, gettext("%s: %s\n"), path,
                            strerror(errno));
@@ -5783,7 +6726,7 @@ unshare_unmount(int op, int argc, char **argv)
        char sharesmb[ZFS_MAXPROPLEN];
 
        /* check options */
-       while ((c = getopt(argc, argv, op == OP_SHARE ? "a" : "af")) != -1) {
+       while ((c = getopt(argc, argv, op == OP_SHARE ? ":a" : "af")) != -1) {
                switch (c) {
                case 'a':
                        do_all = 1;
@@ -5791,6 +6734,11 @@ unshare_unmount(int op, int argc, char **argv)
                case 'f':
                        flags = MS_FORCE;
                        break;
+               case ':':
+                       (void) fprintf(stderr, gettext("missing argument for "
+                           "'%c' option\n"), optopt);
+                       usage(B_FALSE);
+                       break;
                case '?':
                        (void) fprintf(stderr, gettext("invalid option '%c'\n"),
                            optopt);
@@ -5805,9 +6753,9 @@ unshare_unmount(int op, int argc, char **argv)
                /*
                 * We could make use of zfs_for_each() to walk all datasets in
                 * the system, but this would be very inefficient, especially
-                * since we would have to linearly search /etc/mtab for each
-                * one.  Instead, do one pass through /etc/mtab looking for
-                * zfs entries and call zfs_unmount() for each one.
+                * since we would have to linearly search /proc/self/mounts for
+                * each one. Instead, do one pass through /proc/self/mounts
+                * looking for zfs entries and call zfs_unmount() for each one.
                 *
                 * Things get a little tricky if the administrator has created
                 * mountpoints beneath other ZFS filesystems.  In this case, we
@@ -5822,6 +6770,19 @@ unshare_unmount(int op, int argc, char **argv)
                unshare_unmount_node_t *node;
                uu_avl_index_t idx;
                uu_avl_walk_t *walk;
+               char *protocol = NULL;
+
+               if (op == OP_SHARE && argc > 0) {
+                       if (strcmp(argv[0], "nfs") != 0 &&
+                           strcmp(argv[0], "smb") != 0) {
+                               (void) fprintf(stderr, gettext("share type "
+                                   "must be 'nfs' or 'smb'\n"));
+                               usage(B_FALSE);
+                       }
+                       protocol = argv[0];
+                       argc--;
+                       argv++;
+               }
 
                if (argc != 0) {
                        (void) fprintf(stderr, gettext("too many arguments\n"));
@@ -5835,7 +6796,10 @@ unshare_unmount(int op, int argc, char **argv)
                    ((tree = uu_avl_create(pool, NULL, UU_DEFAULT)) == NULL))
                        nomem();
 
-               rewind(mnttab_file);
+               /* Reopen MNTTAB to prevent reading stale data from open file */
+               if (freopen(MNTTAB, "r", mnttab_file) == NULL)
+                       return (ENOENT);
+
                while (getmntent(mnttab_file, &entry) == 0) {
 
                        /* ignore non-ZFS entries */
@@ -5852,6 +6816,15 @@ unshare_unmount(int op, int argc, char **argv)
                                continue;
                        }
 
+                       /*
+                        * Ignore datasets that are excluded/restricted by
+                        * parent pool name.
+                        */
+                       if (zpool_skip_pool(zfs_get_pool_name(zhp))) {
+                               zfs_close(zhp);
+                               continue;
+                       }
+
                        switch (op) {
                        case OP_SHARE:
                                verify(zfs_prop_get(zhp, ZFS_PROP_SHARENFS,
@@ -5911,14 +6884,14 @@ unshare_unmount(int op, int argc, char **argv)
 
                        switch (op) {
                        case OP_SHARE:
-                               if (zfs_unshareall_bypath(node->un_zhp,
-                                   node->un_mountp) != 0)
+                               if (zfs_unshareall_bytype(node->un_zhp,
+                                   node->un_mountp, protocol) != 0)
                                        ret = 1;
                                break;
 
                        case OP_MOUNT:
                                if (zfs_unmount(node->un_zhp,
-                                   node->un_mountp, flags) != 0)
+                                   node->un_zhp->zfs_name, flags) != 0)
                                        ret = 1;
                                break;
                        }
@@ -6093,7 +7066,7 @@ zfs_do_diff(int argc, char **argv)
 
        if (argc < 1) {
                (void) fprintf(stderr,
-               gettext("must provide at least one snapshot name\n"));
+                   gettext("must provide at least one snapshot name\n"));
                usage(B_FALSE);
        }
 
@@ -6113,12 +7086,13 @@ zfs_do_diff(int argc, char **argv)
        if (copy == NULL)
                usage(B_FALSE);
 
-       if ((atp = strchr(copy, '@')))
+       if ((atp = strchr(copy, '@')) != NULL)
                *atp = '\0';
 
-       if ((zhp = zfs_open(g_zfs, copy, ZFS_TYPE_FILESYSTEM)) == NULL)
+       if ((zhp = zfs_open(g_zfs, copy, ZFS_TYPE_FILESYSTEM)) == NULL) {
+               free(copy);
                return (1);
-
+       }
        free(copy);
 
        /*
@@ -6134,35 +7108,811 @@ zfs_do_diff(int argc, char **argv)
        return (err != 0);
 }
 
-int
-main(int argc, char **argv)
+static int
+zfs_do_remap(int argc, char **argv)
 {
-       int ret = 0;
-       int i = 0;
-       char *cmdname;
+       const char *fsname;
+       int err = 0;
+       if (argc != 2) {
+               (void) fprintf(stderr, gettext("wrong number of arguments\n"));
+               usage(B_FALSE);
+       }
 
-       (void) setlocale(LC_ALL, "");
-       (void) textdomain(TEXT_DOMAIN);
+       fsname = argv[1];
+       err = zfs_remap_indirects(g_zfs, fsname);
 
-       opterr = 0;
+       return (err);
+}
 
-       if ((mnttab_file = fopen(MNTTAB, "r")) == NULL) {
-               (void) fprintf(stderr, gettext("internal error: unable to "
-                   "open %s\n"), MNTTAB);
-               return (1);
+/*
+ * zfs bookmark <fs@snap> <fs#bmark>
+ *
+ * Creates a bookmark with the given name from the given snapshot.
+ */
+static int
+zfs_do_bookmark(int argc, char **argv)
+{
+       char snapname[ZFS_MAX_DATASET_NAME_LEN];
+       char bookname[ZFS_MAX_DATASET_NAME_LEN];
+       zfs_handle_t *zhp;
+       nvlist_t *nvl;
+       int ret = 0;
+       int c;
+
+       /* check options */
+       while ((c = getopt(argc, argv, "")) != -1) {
+               switch (c) {
+               case '?':
+                       (void) fprintf(stderr,
+                           gettext("invalid option '%c'\n"), optopt);
+                       goto usage;
+               }
        }
 
-       /*
-        * Make sure the user has specified some command.
-        */
+       argc -= optind;
+       argv += optind;
+
+       /* check number of arguments */
+       if (argc < 1) {
+               (void) fprintf(stderr, gettext("missing snapshot argument\n"));
+               goto usage;
+       }
        if (argc < 2) {
-               (void) fprintf(stderr, gettext("missing command\n"));
-               usage(B_FALSE);
+               (void) fprintf(stderr, gettext("missing bookmark argument\n"));
+               goto usage;
        }
 
-       cmdname = argv[1];
+       if (strchr(argv[0], '@') == NULL) {
+               (void) fprintf(stderr,
+                   gettext("invalid snapshot name '%s': "
+                   "must contain a '@'\n"), argv[0]);
+               goto usage;
+       }
+       if (strchr(argv[1], '#') == NULL) {
+               (void) fprintf(stderr,
+                   gettext("invalid bookmark name '%s': "
+                   "must contain a '#'\n"), argv[1]);
+               goto usage;
+       }
 
-       /*
+       if (argv[0][0] == '@') {
+               /*
+                * Snapshot name begins with @.
+                * Default to same fs as bookmark.
+                */
+               (void) strlcpy(snapname, argv[1], sizeof (snapname));
+               *strchr(snapname, '#') = '\0';
+               (void) strlcat(snapname, argv[0], sizeof (snapname));
+       } else {
+               (void) strlcpy(snapname, argv[0], sizeof (snapname));
+       }
+       if (argv[1][0] == '#') {
+               /*
+                * Bookmark name begins with #.
+                * Default to same fs as snapshot.
+                */
+               (void) strlcpy(bookname, argv[0], sizeof (bookname));
+               *strchr(bookname, '@') = '\0';
+               (void) strlcat(bookname, argv[1], sizeof (bookname));
+       } else {
+               (void) strlcpy(bookname, argv[1], sizeof (bookname));
+       }
+
+       zhp = zfs_open(g_zfs, snapname, ZFS_TYPE_SNAPSHOT);
+       if (zhp == NULL)
+               goto usage;
+       zfs_close(zhp);
+
+
+       nvl = fnvlist_alloc();
+       fnvlist_add_string(nvl, bookname, snapname);
+       ret = lzc_bookmark(nvl, NULL);
+       fnvlist_free(nvl);
+
+       if (ret != 0) {
+               const char *err_msg;
+               char errbuf[1024];
+
+               (void) snprintf(errbuf, sizeof (errbuf),
+                   dgettext(TEXT_DOMAIN,
+                   "cannot create bookmark '%s'"), bookname);
+
+               switch (ret) {
+               case EXDEV:
+                       err_msg = "bookmark is in a different pool";
+                       break;
+               case EEXIST:
+                       err_msg = "bookmark exists";
+                       break;
+               case EINVAL:
+                       err_msg = "invalid argument";
+                       break;
+               case ENOTSUP:
+                       err_msg = "bookmark feature not enabled";
+                       break;
+               case ENOSPC:
+                       err_msg = "out of space";
+                       break;
+               case ENOENT:
+                       err_msg = "dataset does not exist";
+                       break;
+               default:
+                       err_msg = "unknown error";
+                       break;
+               }
+               (void) fprintf(stderr, "%s: %s\n", errbuf,
+                   dgettext(TEXT_DOMAIN, err_msg));
+       }
+
+       return (ret != 0);
+
+usage:
+       usage(B_FALSE);
+       return (-1);
+}
+
+static int
+zfs_do_channel_program(int argc, char **argv)
+{
+       int ret, fd, c;
+       char *progbuf, *filename, *poolname;
+       size_t progsize, progread;
+       nvlist_t *outnvl;
+       uint64_t instrlimit = ZCP_DEFAULT_INSTRLIMIT;
+       uint64_t memlimit = ZCP_DEFAULT_MEMLIMIT;
+       boolean_t sync_flag = B_TRUE, json_output = B_FALSE;
+       zpool_handle_t *zhp;
+
+       /* check options */
+       while ((c = getopt(argc, argv, "nt:m:j")) != -1) {
+               switch (c) {
+               case 't':
+               case 'm': {
+                       uint64_t arg;
+                       char *endp;
+
+                       errno = 0;
+                       arg = strtoull(optarg, &endp, 0);
+                       if (errno != 0 || *endp != '\0') {
+                               (void) fprintf(stderr, gettext(
+                                   "invalid argument "
+                                   "'%s': expected integer\n"), optarg);
+                               goto usage;
+                       }
+
+                       if (c == 't') {
+                               if (arg > ZCP_MAX_INSTRLIMIT || arg == 0) {
+                                       (void) fprintf(stderr, gettext(
+                                           "Invalid instruction limit: "
+                                           "%s\n"), optarg);
+                                       return (1);
+                               } else {
+                                       instrlimit = arg;
+                               }
+                       } else {
+                               ASSERT3U(c, ==, 'm');
+                               if (arg > ZCP_MAX_MEMLIMIT || arg == 0) {
+                                       (void) fprintf(stderr, gettext(
+                                           "Invalid memory limit: "
+                                           "%s\n"), optarg);
+                                       return (1);
+                               } else {
+                                       memlimit = arg;
+                               }
+                       }
+                       break;
+               }
+               case 'n': {
+                       sync_flag = B_FALSE;
+                       break;
+               }
+               case 'j': {
+                       json_output = B_TRUE;
+                       break;
+               }
+               case '?':
+                       (void) fprintf(stderr, gettext("invalid option '%c'\n"),
+                           optopt);
+                       goto usage;
+               }
+       }
+
+       argc -= optind;
+       argv += optind;
+
+       if (argc < 2) {
+               (void) fprintf(stderr,
+                   gettext("invalid number of arguments\n"));
+               goto usage;
+       }
+
+       poolname = argv[0];
+       filename = argv[1];
+       if (strcmp(filename, "-") == 0) {
+               fd = 0;
+               filename = "standard input";
+       } else if ((fd = open(filename, O_RDONLY)) < 0) {
+               (void) fprintf(stderr, gettext("cannot open '%s': %s\n"),
+                   filename, strerror(errno));
+               return (1);
+       }
+
+       if ((zhp = zpool_open(g_zfs, poolname)) == NULL) {
+               (void) fprintf(stderr, gettext("cannot open pool '%s'"),
+                   poolname);
+               if (fd != 0)
+                       (void) close(fd);
+               return (1);
+       }
+       zpool_close(zhp);
+
+       /*
+        * Read in the channel program, expanding the program buffer as
+        * necessary.
+        */
+       progread = 0;
+       progsize = 1024;
+       progbuf = safe_malloc(progsize);
+       do {
+               ret = read(fd, progbuf + progread, progsize - progread);
+               progread += ret;
+               if (progread == progsize && ret > 0) {
+                       progsize *= 2;
+                       progbuf = safe_realloc(progbuf, progsize);
+               }
+       } while (ret > 0);
+
+       if (fd != 0)
+               (void) close(fd);
+       if (ret < 0) {
+               free(progbuf);
+               (void) fprintf(stderr,
+                   gettext("cannot read '%s': %s\n"),
+                   filename, strerror(errno));
+               return (1);
+       }
+       progbuf[progread] = '\0';
+
+       /*
+        * Any remaining arguments are passed as arguments to the lua script as
+        * a string array:
+        * {
+        *      "argv" -> [ "arg 1", ... "arg n" ],
+        * }
+        */
+       nvlist_t *argnvl = fnvlist_alloc();
+       fnvlist_add_string_array(argnvl, ZCP_ARG_CLIARGV, argv + 2, argc - 2);
+
+       if (sync_flag) {
+               ret = lzc_channel_program(poolname, progbuf,
+                   instrlimit, memlimit, argnvl, &outnvl);
+       } else {
+               ret = lzc_channel_program_nosync(poolname, progbuf,
+                   instrlimit, memlimit, argnvl, &outnvl);
+       }
+
+       if (ret != 0) {
+               /*
+                * On error, report the error message handed back by lua if one
+                * exists.  Otherwise, generate an appropriate error message,
+                * falling back on strerror() for an unexpected return code.
+                */
+               char *errstring = NULL;
+               uint64_t instructions = 0;
+               if (nvlist_exists(outnvl, ZCP_RET_ERROR)) {
+                       (void) nvlist_lookup_string(outnvl,
+                           ZCP_RET_ERROR, &errstring);
+                       if (errstring == NULL)
+                               errstring = strerror(ret);
+                       if (ret == ETIME) {
+                               (void) nvlist_lookup_uint64(outnvl,
+                                   ZCP_ARG_INSTRLIMIT, &instructions);
+                       }
+               } else {
+                       switch (ret) {
+                       case EINVAL:
+                               errstring =
+                                   "Invalid instruction or memory limit.";
+                               break;
+                       case ENOMEM:
+                               errstring = "Return value too large.";
+                               break;
+                       case ENOSPC:
+                               errstring = "Memory limit exhausted.";
+                               break;
+                       case ETIME:
+                               errstring = "Timed out.";
+                               break;
+                       case EPERM:
+                               errstring = "Permission denied. Channel "
+                                   "programs must be run as root.";
+                               break;
+                       default:
+                               errstring = strerror(ret);
+                       }
+               }
+               (void) fprintf(stderr,
+                   gettext("Channel program execution failed:\n%s\n"),
+                   errstring);
+               if (ret == ETIME && instructions != 0)
+                       (void) fprintf(stderr,
+                           gettext("%llu Lua instructions\n"),
+                           (u_longlong_t)instructions);
+       } else {
+               if (json_output) {
+                       (void) nvlist_print_json(stdout, outnvl);
+               } else if (nvlist_empty(outnvl)) {
+                       (void) fprintf(stdout, gettext("Channel program fully "
+                           "executed and did not produce output.\n"));
+               } else {
+                       (void) fprintf(stdout, gettext("Channel program fully "
+                           "executed and produced output:\n"));
+                       dump_nvlist(outnvl, 4);
+               }
+       }
+
+       free(progbuf);
+       fnvlist_free(outnvl);
+       fnvlist_free(argnvl);
+       return (ret != 0);
+
+usage:
+       usage(B_FALSE);
+       return (-1);
+}
+
+
+typedef struct loadkey_cbdata {
+       boolean_t cb_loadkey;
+       boolean_t cb_recursive;
+       boolean_t cb_noop;
+       char *cb_keylocation;
+       uint64_t cb_numfailed;
+       uint64_t cb_numattempted;
+} loadkey_cbdata_t;
+
+static int
+load_key_callback(zfs_handle_t *zhp, void *data)
+{
+       int ret;
+       boolean_t is_encroot;
+       loadkey_cbdata_t *cb = data;
+       uint64_t keystatus = zfs_prop_get_int(zhp, ZFS_PROP_KEYSTATUS);
+
+       /*
+        * If we are working recursively, we want to skip loading / unloading
+        * keys for non-encryption roots and datasets whose keys are already
+        * in the desired end-state.
+        */
+       if (cb->cb_recursive) {
+               ret = zfs_crypto_get_encryption_root(zhp, &is_encroot, NULL);
+               if (ret != 0)
+                       return (ret);
+               if (!is_encroot)
+                       return (0);
+
+               if ((cb->cb_loadkey && keystatus == ZFS_KEYSTATUS_AVAILABLE) ||
+                   (!cb->cb_loadkey && keystatus == ZFS_KEYSTATUS_UNAVAILABLE))
+                       return (0);
+       }
+
+       cb->cb_numattempted++;
+
+       if (cb->cb_loadkey)
+               ret = zfs_crypto_load_key(zhp, cb->cb_noop, cb->cb_keylocation);
+       else
+               ret = zfs_crypto_unload_key(zhp);
+
+       if (ret != 0) {
+               cb->cb_numfailed++;
+               return (ret);
+       }
+
+       return (0);
+}
+
+static int
+load_unload_keys(int argc, char **argv, boolean_t loadkey)
+{
+       int c, ret = 0, flags = 0;
+       boolean_t do_all = B_FALSE;
+       loadkey_cbdata_t cb = { 0 };
+
+       cb.cb_loadkey = loadkey;
+
+       while ((c = getopt(argc, argv, "anrL:")) != -1) {
+               /* noop and alternate keylocations only apply to zfs load-key */
+               if (loadkey) {
+                       switch (c) {
+                       case 'n':
+                               cb.cb_noop = B_TRUE;
+                               continue;
+                       case 'L':
+                               cb.cb_keylocation = optarg;
+                               continue;
+                       default:
+                               break;
+                       }
+               }
+
+               switch (c) {
+               case 'a':
+                       do_all = B_TRUE;
+                       cb.cb_recursive = B_TRUE;
+                       break;
+               case 'r':
+                       flags |= ZFS_ITER_RECURSE;
+                       cb.cb_recursive = B_TRUE;
+                       break;
+               default:
+                       (void) fprintf(stderr,
+                           gettext("invalid option '%c'\n"), optopt);
+                       usage(B_FALSE);
+               }
+       }
+
+       argc -= optind;
+       argv += optind;
+
+       if (!do_all && argc == 0) {
+               (void) fprintf(stderr,
+                   gettext("Missing dataset argument or -a option\n"));
+               usage(B_FALSE);
+       }
+
+       if (do_all && argc != 0) {
+               (void) fprintf(stderr,
+                   gettext("Cannot specify dataset with -a option\n"));
+               usage(B_FALSE);
+       }
+
+       if (cb.cb_recursive && cb.cb_keylocation != NULL &&
+           strcmp(cb.cb_keylocation, "prompt") != 0) {
+               (void) fprintf(stderr, gettext("alternate keylocation may only "
+                   "be 'prompt' with -r or -a\n"));
+               usage(B_FALSE);
+       }
+
+       ret = zfs_for_each(argc, argv, flags,
+           ZFS_TYPE_FILESYSTEM | ZFS_TYPE_VOLUME, NULL, NULL, 0,
+           load_key_callback, &cb);
+
+       if (cb.cb_noop || (cb.cb_recursive && cb.cb_numattempted != 0)) {
+               (void) printf(gettext("%llu / %llu key(s) successfully %s\n"),
+                   (u_longlong_t)(cb.cb_numattempted - cb.cb_numfailed),
+                   (u_longlong_t)cb.cb_numattempted,
+                   loadkey ? (cb.cb_noop ? "verified" : "loaded") :
+                   "unloaded");
+       }
+
+       if (cb.cb_numfailed != 0)
+               ret = -1;
+
+       return (ret);
+}
+
+static int
+zfs_do_load_key(int argc, char **argv)
+{
+       return (load_unload_keys(argc, argv, B_TRUE));
+}
+
+
+static int
+zfs_do_unload_key(int argc, char **argv)
+{
+       return (load_unload_keys(argc, argv, B_FALSE));
+}
+
+static int
+zfs_do_change_key(int argc, char **argv)
+{
+       int c, ret;
+       uint64_t keystatus;
+       boolean_t loadkey = B_FALSE, inheritkey = B_FALSE;
+       zfs_handle_t *zhp = NULL;
+       nvlist_t *props = fnvlist_alloc();
+
+       while ((c = getopt(argc, argv, "lio:")) != -1) {
+               switch (c) {
+               case 'l':
+                       loadkey = B_TRUE;
+                       break;
+               case 'i':
+                       inheritkey = B_TRUE;
+                       break;
+               case 'o':
+                       if (!parseprop(props, optarg)) {
+                               nvlist_free(props);
+                               return (1);
+                       }
+                       break;
+               default:
+                       (void) fprintf(stderr,
+                           gettext("invalid option '%c'\n"), optopt);
+                       usage(B_FALSE);
+               }
+       }
+
+       if (inheritkey && !nvlist_empty(props)) {
+               (void) fprintf(stderr,
+                   gettext("Properties not allowed for inheriting\n"));
+               usage(B_FALSE);
+       }
+
+       argc -= optind;
+       argv += optind;
+
+       if (argc < 1) {
+               (void) fprintf(stderr, gettext("Missing dataset argument\n"));
+               usage(B_FALSE);
+       }
+
+       if (argc > 1) {
+               (void) fprintf(stderr, gettext("Too many arguments\n"));
+               usage(B_FALSE);
+       }
+
+       zhp = zfs_open(g_zfs, argv[argc - 1],
+           ZFS_TYPE_FILESYSTEM | ZFS_TYPE_VOLUME);
+       if (zhp == NULL)
+               usage(B_FALSE);
+
+       if (loadkey) {
+               keystatus = zfs_prop_get_int(zhp, ZFS_PROP_KEYSTATUS);
+               if (keystatus != ZFS_KEYSTATUS_AVAILABLE) {
+                       ret = zfs_crypto_load_key(zhp, B_FALSE, NULL);
+                       if (ret != 0) {
+                               nvlist_free(props);
+                               zfs_close(zhp);
+                               return (-1);
+                       }
+               }
+
+               /* refresh the properties so the new keystatus is visible */
+               zfs_refresh_properties(zhp);
+       }
+
+       ret = zfs_crypto_rewrap(zhp, props, inheritkey);
+       if (ret != 0) {
+               nvlist_free(props);
+               zfs_close(zhp);
+               return (-1);
+       }
+
+       nvlist_free(props);
+       zfs_close(zhp);
+       return (0);
+}
+
+/*
+ * 1) zfs project [-d|-r] <file|directory ...>
+ *    List project ID and inherit flag of file(s) or directories.
+ *    -d: List the directory itself, not its children.
+ *    -r: List subdirectories recursively.
+ *
+ * 2) zfs project -C [-k] [-r] <file|directory ...>
+ *    Clear project inherit flag and/or ID on the file(s) or directories.
+ *    -k: Keep the project ID unchanged. If not specified, the project ID
+ *       will be reset as zero.
+ *    -r: Clear on subdirectories recursively.
+ *
+ * 3) zfs project -c [-0] [-d|-r] [-p id] <file|directory ...>
+ *    Check project ID and inherit flag on the file(s) or directories,
+ *    report the outliers.
+ *    -0: Print file name followed by a NUL instead of newline.
+ *    -d: Check the directory itself, not its children.
+ *    -p: Specify the referenced ID for comparing with the target file(s)
+ *       or directories' project IDs. If not specified, the target (top)
+ *       directory's project ID will be used as the referenced one.
+ *    -r: Check subdirectories recursively.
+ *
+ * 4) zfs project [-p id] [-r] [-s] <file|directory ...>
+ *    Set project ID and/or inherit flag on the file(s) or directories.
+ *    -p: Set the project ID as the given id.
+ *    -r: Set on subdirectorie recursively. If not specify "-p" option,
+ *       it will use top-level directory's project ID as the given id,
+ *       then set both project ID and inherit flag on all descendants
+ *       of the top-level directory.
+ *    -s: Set project inherit flag.
+ */
+static int
+zfs_do_project(int argc, char **argv)
+{
+       zfs_project_control_t zpc = {
+               .zpc_expected_projid = ZFS_INVALID_PROJID,
+               .zpc_op = ZFS_PROJECT_OP_DEFAULT,
+               .zpc_dironly = B_FALSE,
+               .zpc_keep_projid = B_FALSE,
+               .zpc_newline = B_TRUE,
+               .zpc_recursive = B_FALSE,
+               .zpc_set_flag = B_FALSE,
+       };
+       int ret = 0, c;
+
+       if (argc < 2)
+               usage(B_FALSE);
+
+       while ((c = getopt(argc, argv, "0Ccdkp:rs")) != -1) {
+               switch (c) {
+               case '0':
+                       zpc.zpc_newline = B_FALSE;
+                       break;
+               case 'C':
+                       if (zpc.zpc_op != ZFS_PROJECT_OP_DEFAULT) {
+                               (void) fprintf(stderr, gettext("cannot "
+                                   "specify '-C' '-c' '-s' together\n"));
+                               usage(B_FALSE);
+                       }
+
+                       zpc.zpc_op = ZFS_PROJECT_OP_CLEAR;
+                       break;
+               case 'c':
+                       if (zpc.zpc_op != ZFS_PROJECT_OP_DEFAULT) {
+                               (void) fprintf(stderr, gettext("cannot "
+                                   "specify '-C' '-c' '-s' together\n"));
+                               usage(B_FALSE);
+                       }
+
+                       zpc.zpc_op = ZFS_PROJECT_OP_CHECK;
+                       break;
+               case 'd':
+                       zpc.zpc_dironly = B_TRUE;
+                       /* overwrite "-r" option */
+                       zpc.zpc_recursive = B_FALSE;
+                       break;
+               case 'k':
+                       zpc.zpc_keep_projid = B_TRUE;
+                       break;
+               case 'p': {
+                       char *endptr;
+
+                       errno = 0;
+                       zpc.zpc_expected_projid = strtoull(optarg, &endptr, 0);
+                       if (errno != 0 || *endptr != '\0') {
+                               (void) fprintf(stderr,
+                                   gettext("project ID must be less than "
+                                   "%u\n"), UINT32_MAX);
+                               usage(B_FALSE);
+                       }
+                       if (zpc.zpc_expected_projid >= UINT32_MAX) {
+                               (void) fprintf(stderr,
+                                   gettext("invalid project ID\n"));
+                               usage(B_FALSE);
+                       }
+                       break;
+               }
+               case 'r':
+                       zpc.zpc_recursive = B_TRUE;
+                       /* overwrite "-d" option */
+                       zpc.zpc_dironly = B_FALSE;
+                       break;
+               case 's':
+                       if (zpc.zpc_op != ZFS_PROJECT_OP_DEFAULT) {
+                               (void) fprintf(stderr, gettext("cannot "
+                                   "specify '-C' '-c' '-s' together\n"));
+                               usage(B_FALSE);
+                       }
+
+                       zpc.zpc_set_flag = B_TRUE;
+                       zpc.zpc_op = ZFS_PROJECT_OP_SET;
+                       break;
+               default:
+                       (void) fprintf(stderr, gettext("invalid option '%c'\n"),
+                           optopt);
+                       usage(B_FALSE);
+               }
+       }
+
+       if (zpc.zpc_op == ZFS_PROJECT_OP_DEFAULT) {
+               if (zpc.zpc_expected_projid != ZFS_INVALID_PROJID)
+                       zpc.zpc_op = ZFS_PROJECT_OP_SET;
+               else
+                       zpc.zpc_op = ZFS_PROJECT_OP_LIST;
+       }
+
+       switch (zpc.zpc_op) {
+       case ZFS_PROJECT_OP_LIST:
+               if (zpc.zpc_keep_projid) {
+                       (void) fprintf(stderr,
+                           gettext("'-k' is only valid together with '-C'\n"));
+                       usage(B_FALSE);
+               }
+               if (!zpc.zpc_newline) {
+                       (void) fprintf(stderr,
+                           gettext("'-0' is only valid together with '-c'\n"));
+                       usage(B_FALSE);
+               }
+               break;
+       case ZFS_PROJECT_OP_CHECK:
+               if (zpc.zpc_keep_projid) {
+                       (void) fprintf(stderr,
+                           gettext("'-k' is only valid together with '-C'\n"));
+                       usage(B_FALSE);
+               }
+               break;
+       case ZFS_PROJECT_OP_CLEAR:
+               if (zpc.zpc_dironly) {
+                       (void) fprintf(stderr,
+                           gettext("'-d' is useless together with '-C'\n"));
+                       usage(B_FALSE);
+               }
+               if (!zpc.zpc_newline) {
+                       (void) fprintf(stderr,
+                           gettext("'-0' is only valid together with '-c'\n"));
+                       usage(B_FALSE);
+               }
+               if (zpc.zpc_expected_projid != ZFS_INVALID_PROJID) {
+                       (void) fprintf(stderr,
+                           gettext("'-p' is useless together with '-C'\n"));
+                       usage(B_FALSE);
+               }
+               break;
+       case ZFS_PROJECT_OP_SET:
+               if (zpc.zpc_dironly) {
+                       (void) fprintf(stderr,
+                           gettext("'-d' is useless for set project ID and/or "
+                           "inherit flag\n"));
+                       usage(B_FALSE);
+               }
+               if (zpc.zpc_keep_projid) {
+                       (void) fprintf(stderr,
+                           gettext("'-k' is only valid together with '-C'\n"));
+                       usage(B_FALSE);
+               }
+               if (!zpc.zpc_newline) {
+                       (void) fprintf(stderr,
+                           gettext("'-0' is only valid together with '-c'\n"));
+                       usage(B_FALSE);
+               }
+               break;
+       default:
+               ASSERT(0);
+               break;
+       }
+
+       argv += optind;
+       argc -= optind;
+       if (argc == 0) {
+               (void) fprintf(stderr,
+                   gettext("missing file or directory target(s)\n"));
+               usage(B_FALSE);
+       }
+
+       for (int i = 0; i < argc; i++) {
+               int err;
+
+               err = zfs_project_handle(argv[i], &zpc);
+               if (err && !ret)
+                       ret = err;
+       }
+
+       return (ret);
+}
+
+int
+main(int argc, char **argv)
+{
+       int ret = 0;
+       int i = 0;
+       char *cmdname;
+
+       (void) setlocale(LC_ALL, "");
+       (void) textdomain(TEXT_DOMAIN);
+
+       opterr = 0;
+
+       /*
+        * Make sure the user has specified some command.
+        */
+       if (argc < 2) {
+               (void) fprintf(stderr, gettext("missing command\n"));
+               usage(B_FALSE);
+       }
+
+       cmdname = argv[1];
+
+       /*
         * The 'umount' command is an alias for 'unmount'
         */
        if (strcmp(cmdname, "umount") == 0)
@@ -6187,18 +7937,21 @@ main(int argc, char **argv)
            (strcmp(cmdname, "--help") == 0))
                usage(B_TRUE);
 
-       if ((g_zfs = libzfs_init()) == NULL)
+       if ((g_zfs = libzfs_init()) == NULL) {
+               (void) fprintf(stderr, "%s", libzfs_error_init(errno));
                return (1);
+       }
+
+       mnttab_file = g_zfs->libzfs_mnttab;
 
-       zpool_set_history_str("zfs", argc, argv, history_str);
-       verify(zpool_stage_history(g_zfs, history_str) == 0);
+       zfs_save_arguments(argc, argv, history_str, sizeof (history_str));
 
        libzfs_print_on_error(g_zfs, B_TRUE);
 
        /*
         * Run the appropriate command.
         */
-       libzfs_mnttab_cache(g_zfs, B_FALSE);
+       libzfs_mnttab_cache(g_zfs, B_TRUE);
        if (find_command_idx(cmdname, &i) == 0) {
                current_command = &command_table[i];
                ret = command_table[i].func(argc - 1, argv + 1);
@@ -6212,9 +7965,11 @@ main(int argc, char **argv)
                usage(B_FALSE);
                ret = 1;
        }
-       libzfs_fini(g_zfs);
 
-       (void) fclose(mnttab_file);
+       if (ret == 0 && log_history)
+               (void) zpool_log_history(g_zfs, history_str);
+
+       libzfs_fini(g_zfs);
 
        /*
         * The 'ZFS_ABORT' environment variable causes us to dump core on exit