]> git.proxmox.com Git - mirror_zfs.git/blobdiff - cmd/zpool/zpool_main.c
Fix locale-specific time
[mirror_zfs.git] / cmd / zpool / zpool_main.c
index 6d0dae8d8b059d321a926757f5d33999a2a33120..d670cd1afeb1bf53de1e83fe3b0ea4ff712c4d96 100644 (file)
@@ -22,7 +22,7 @@
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright 2011 Nexenta Systems, Inc. All rights reserved.
- * Copyright (c) 2011, 2020 by Delphix. All rights reserved.
+ * Copyright (c) 2011, 2024 by Delphix. All rights reserved.
  * Copyright (c) 2012 by Frederik Wessels. All rights reserved.
  * Copyright (c) 2012 by Cyril Plisko. All rights reserved.
  * Copyright (c) 2013 by Prasad Joshi (sTec). All rights reserved.
@@ -126,9 +126,18 @@ static int zpool_do_version(int, char **);
 
 static int zpool_do_wait(int, char **);
 
+static int zpool_do_help(int argc, char **argv);
+
 static zpool_compat_status_t zpool_do_load_compat(
     const char *, boolean_t *);
 
+enum zpool_options {
+       ZPOOL_OPTION_POWER = 1024,
+       ZPOOL_OPTION_ALLOW_INUSE,
+       ZPOOL_OPTION_ALLOW_REPLICATION_MISMATCH,
+       ZPOOL_OPTION_ALLOW_ASHIFT_MISMATCH
+};
+
 /*
  * These libumem hooks provide a reasonable set of defaults for the allocator's
  * debugging facilities.
@@ -345,13 +354,13 @@ get_usage(zpool_help_t idx)
 {
        switch (idx) {
        case HELP_ADD:
-               return (gettext("\tadd [-fgLnP] [-o property=value] "
+               return (gettext("\tadd [-afgLnP] [-o property=value] "
                    "<pool> <vdev> ...\n"));
        case HELP_ATTACH:
                return (gettext("\tattach [-fsw] [-o property=value] "
                    "<pool> <device> <new-device>\n"));
        case HELP_CLEAR:
-               return (gettext("\tclear [-nF] <pool> [device]\n"));
+               return (gettext("\tclear [[--power]|[-nF]] <pool> [device]\n"));
        case HELP_CREATE:
                return (gettext("\tcreate [-fnd] [-o property=value] ... \n"
                    "\t    [-O file-system-property=value] ... \n"
@@ -387,9 +396,11 @@ get_usage(zpool_help_t idx)
                    "[-T d|u] [pool] ... \n"
                    "\t    [interval [count]]\n"));
        case HELP_OFFLINE:
-               return (gettext("\toffline [-f] [-t] <pool> <device> ...\n"));
+               return (gettext("\toffline [--power]|[[-f][-t]] <pool> "
+                   "<device> ...\n"));
        case HELP_ONLINE:
-               return (gettext("\tonline [-e] <pool> <device> ...\n"));
+               return (gettext("\tonline [--power][-e] <pool> <device> "
+                   "...\n"));
        case HELP_REPLACE:
                return (gettext("\treplace [-fsw] [-o property=value] "
                    "<pool> <device> [new-device]\n"));
@@ -408,8 +419,8 @@ get_usage(zpool_help_t idx)
                return (gettext("\ttrim [-dw] [-r <rate>] [-c | -s] <pool> "
                    "[<device> ...]\n"));
        case HELP_STATUS:
-               return (gettext("\tstatus [-c [script1,script2,...]] "
-                   "[-igLpPstvxD]  [-T d|u] [pool] ... \n"
+               return (gettext("\tstatus [--power] [-c [script1,script2,...]] "
+                   "[-DegiLpPstvx] [-T d|u] [pool] ...\n"
                    "\t    [interval [count]]\n"));
        case HELP_UPGRADE:
                return (gettext("\tupgrade\n"
@@ -514,6 +525,77 @@ print_vdev_prop_cb(int prop, void *cb)
        return (ZPROP_CONT);
 }
 
+/*
+ * Given a leaf vdev name like 'L5' return its VDEV_CONFIG_PATH like
+ * '/dev/disk/by-vdev/L5'.
+ */
+static const char *
+vdev_name_to_path(zpool_handle_t *zhp, char *vdev)
+{
+       nvlist_t *vdev_nv = zpool_find_vdev(zhp, vdev, NULL, NULL, NULL);
+       if (vdev_nv == NULL) {
+               return (NULL);
+       }
+       return (fnvlist_lookup_string(vdev_nv, ZPOOL_CONFIG_PATH));
+}
+
+static int
+zpool_power_on(zpool_handle_t *zhp, char *vdev)
+{
+       return (zpool_power(zhp, vdev, B_TRUE));
+}
+
+static int
+zpool_power_on_and_disk_wait(zpool_handle_t *zhp, char *vdev)
+{
+       int rc;
+
+       rc = zpool_power_on(zhp, vdev);
+       if (rc != 0)
+               return (rc);
+
+       zpool_disk_wait(vdev_name_to_path(zhp, vdev));
+
+       return (0);
+}
+
+static int
+zpool_power_on_pool_and_wait_for_devices(zpool_handle_t *zhp)
+{
+       nvlist_t *nv;
+       const char *path = NULL;
+       int rc;
+
+       /* Power up all the devices first */
+       FOR_EACH_REAL_LEAF_VDEV(zhp, nv) {
+               path = fnvlist_lookup_string(nv, ZPOOL_CONFIG_PATH);
+               if (path != NULL) {
+                       rc = zpool_power_on(zhp, (char *)path);
+                       if (rc != 0) {
+                               return (rc);
+                       }
+               }
+       }
+
+       /*
+        * Wait for their devices to show up.  Since we powered them on
+        * at roughly the same time, they should all come online around
+        * the same time.
+        */
+       FOR_EACH_REAL_LEAF_VDEV(zhp, nv) {
+               path = fnvlist_lookup_string(nv, ZPOOL_CONFIG_PATH);
+               zpool_disk_wait(path);
+       }
+
+       return (0);
+}
+
+static int
+zpool_power_off(zpool_handle_t *zhp, char *vdev)
+{
+       return (zpool_power(zhp, vdev, B_FALSE));
+}
+
 /*
  * Display usage message.  If we're inside a command, display only the usage for
  * that command.  Otherwise, iterate over the entire command table and display
@@ -538,6 +620,10 @@ usage(boolean_t requested)
                                (void) fprintf(fp, "%s",
                                    get_usage(command_table[i].usage));
                }
+
+               (void) fprintf(fp,
+                   gettext("\nFor further help on a command or topic, "
+                   "run: %s\n"), "zpool help [<topic>]");
        } else {
                (void) fprintf(fp, gettext("usage:\n"));
                (void) fprintf(fp, "%s", get_usage(current_command->usage));
@@ -930,8 +1016,9 @@ add_prop_list_default(const char *propname, const char *propval,
 }
 
 /*
- * zpool add [-fgLnP] [-o property=value] <pool> <vdev> ...
+ * zpool add [-afgLnP] [-o property=value] <pool> <vdev> ...
  *
+ *     -a      Disable the ashift validation checks
  *     -f      Force addition of devices, even if they appear in use
  *     -g      Display guid for individual vdev name.
  *     -L      Follow links when resolving vdev path name.
@@ -947,8 +1034,11 @@ add_prop_list_default(const char *propname, const char *propval,
 int
 zpool_do_add(int argc, char **argv)
 {
-       boolean_t force = B_FALSE;
+       boolean_t check_replication = B_TRUE;
+       boolean_t check_inuse = B_TRUE;
        boolean_t dryrun = B_FALSE;
+       boolean_t check_ashift = B_TRUE;
+       boolean_t force = B_FALSE;
        int name_flags = 0;
        int c;
        nvlist_t *nvroot;
@@ -959,8 +1049,18 @@ zpool_do_add(int argc, char **argv)
        nvlist_t *props = NULL;
        char *propval;
 
+       struct option long_options[] = {
+               {"allow-in-use", no_argument, NULL, ZPOOL_OPTION_ALLOW_INUSE},
+               {"allow-replication-mismatch", no_argument, NULL,
+                   ZPOOL_OPTION_ALLOW_REPLICATION_MISMATCH},
+               {"allow-ashift-mismatch", no_argument, NULL,
+                   ZPOOL_OPTION_ALLOW_ASHIFT_MISMATCH},
+               {0, 0, 0, 0}
+       };
+
        /* check options */
-       while ((c = getopt(argc, argv, "fgLno:P")) != -1) {
+       while ((c = getopt_long(argc, argv, "fgLno:P", long_options, NULL))
+           != -1) {
                switch (c) {
                case 'f':
                        force = B_TRUE;
@@ -990,6 +1090,15 @@ zpool_do_add(int argc, char **argv)
                case 'P':
                        name_flags |= VDEV_NAME_PATH;
                        break;
+               case ZPOOL_OPTION_ALLOW_INUSE:
+                       check_inuse = B_FALSE;
+                       break;
+               case ZPOOL_OPTION_ALLOW_REPLICATION_MISMATCH:
+                       check_replication = B_FALSE;
+                       break;
+               case ZPOOL_OPTION_ALLOW_ASHIFT_MISMATCH:
+                       check_ashift = B_FALSE;
+                       break;
                case '?':
                        (void) fprintf(stderr, gettext("invalid option '%c'\n"),
                            optopt);
@@ -1010,6 +1119,19 @@ zpool_do_add(int argc, char **argv)
                usage(B_FALSE);
        }
 
+       if (force) {
+               if (!check_inuse || !check_replication || !check_ashift) {
+                       (void) fprintf(stderr, gettext("'-f' option is not "
+                           "allowed with '--allow-replication-mismatch', "
+                           "'--allow-ashift-mismatch', or "
+                           "'--allow-in-use'\n"));
+                       usage(B_FALSE);
+               }
+               check_inuse = B_FALSE;
+               check_replication = B_FALSE;
+               check_ashift = B_FALSE;
+       }
+
        poolname = argv[0];
 
        argc--;
@@ -1040,8 +1162,8 @@ zpool_do_add(int argc, char **argv)
        }
 
        /* pass off to make_root_vdev for processing */
-       nvroot = make_root_vdev(zhp, props, force, !force, B_FALSE, dryrun,
-           argc, argv);
+       nvroot = make_root_vdev(zhp, props, !check_inuse,
+           check_replication, B_FALSE, dryrun, argc, argv);
        if (nvroot == NULL) {
                zpool_close(zhp);
                return (1);
@@ -1145,7 +1267,7 @@ zpool_do_add(int argc, char **argv)
 
                ret = 0;
        } else {
-               ret = (zpool_add(zhp, nvroot) != 0);
+               ret = (zpool_add(zhp, nvroot, check_ashift) != 0);
        }
 
        nvlist_free(props);
@@ -2082,11 +2204,13 @@ typedef struct status_cbdata {
        boolean_t       cb_explain;
        boolean_t       cb_first;
        boolean_t       cb_dedup_stats;
+       boolean_t       cb_print_unhealthy;
        boolean_t       cb_print_status;
        boolean_t       cb_print_slow_ios;
        boolean_t       cb_print_vdev_init;
        boolean_t       cb_print_vdev_trim;
        vdev_cmd_data_list_t    *vcdl;
+       boolean_t       cb_print_power;
 } status_cbdata_t;
 
 /* Return 1 if string is NULL, empty, or whitespace; return 0 otherwise. */
@@ -2165,7 +2289,6 @@ print_status_initialize(vdev_stat_t *vs, boolean_t verbose)
                    !vs->vs_scan_removing) {
                        char zbuf[1024];
                        char tbuf[256];
-                       struct tm zaction_ts;
 
                        time_t t = vs->vs_initialize_action_time;
                        int initialize_pct = 100;
@@ -2175,8 +2298,8 @@ print_status_initialize(vdev_stat_t *vs, boolean_t verbose)
                                    100 / (vs->vs_initialize_bytes_est + 1));
                        }
 
-                       (void) localtime_r(&t, &zaction_ts);
-                       (void) strftime(tbuf, sizeof (tbuf), "%c", &zaction_ts);
+                       (void) ctime_r(&t, tbuf);
+                       tbuf[24] = 0;
 
                        switch (vs->vs_initialize_state) {
                        case VDEV_INITIALIZE_SUSPENDED:
@@ -2216,7 +2339,6 @@ print_status_trim(vdev_stat_t *vs, boolean_t verbose)
                    !vs->vs_scan_removing) {
                        char zbuf[1024];
                        char tbuf[256];
-                       struct tm zaction_ts;
 
                        time_t t = vs->vs_trim_action_time;
                        int trim_pct = 100;
@@ -2225,8 +2347,8 @@ print_status_trim(vdev_stat_t *vs, boolean_t verbose)
                                    100 / (vs->vs_trim_bytes_est + 1));
                        }
 
-                       (void) localtime_r(&t, &zaction_ts);
-                       (void) strftime(tbuf, sizeof (tbuf), "%c", &zaction_ts);
+                       (void) ctime_r(&t, tbuf);
+                       tbuf[24] = 0;
 
                        switch (vs->vs_trim_state) {
                        case VDEV_TRIM_SUSPENDED:
@@ -2277,6 +2399,35 @@ health_str_to_color(const char *health)
        return (NULL);
 }
 
+/*
+ * Called for each leaf vdev.  Returns 0 if the vdev is healthy.
+ * A vdev is unhealthy if any of the following are true:
+ * 1) there are read, write, or checksum errors,
+ * 2) its state is not ONLINE, or
+ * 3) slow IO reporting was requested (-s) and there are slow IOs.
+ */
+static int
+vdev_health_check_cb(void *hdl_data, nvlist_t *nv, void *data)
+{
+       status_cbdata_t *cb = data;
+       vdev_stat_t *vs;
+       uint_t vsc;
+       (void) hdl_data;
+
+       if (nvlist_lookup_uint64_array(nv, ZPOOL_CONFIG_VDEV_STATS,
+           (uint64_t **)&vs, &vsc) != 0)
+               return (1);
+
+       if (vs->vs_checksum_errors || vs->vs_read_errors ||
+           vs->vs_write_errors || vs->vs_state != VDEV_STATE_HEALTHY)
+               return (1);
+
+       if (cb->cb_print_slow_ios && vs->vs_slow_ios)
+               return (1);
+
+       return (0);
+}
+
 /*
  * Print out configuration state as requested by status_callback.
  */
@@ -2295,7 +2446,8 @@ print_status_config(zpool_handle_t *zhp, status_cbdata_t *cb, const char *name,
        const char *state;
        const char *type;
        const char *path = NULL;
-       const char *rcolor = NULL, *wcolor = NULL, *ccolor = NULL;
+       const char *rcolor = NULL, *wcolor = NULL, *ccolor = NULL,
+           *scolor = NULL;
 
        if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN,
            &child, &children) != 0)
@@ -2322,6 +2474,15 @@ print_status_config(zpool_handle_t *zhp, status_cbdata_t *cb, const char *name,
                        state = gettext("AVAIL");
        }
 
+       /*
+        * If '-e' is specified then top-level vdevs and their children
+        * can be pruned if all of their leaves are healthy.
+        */
+       if (cb->cb_print_unhealthy && depth > 0 &&
+           for_each_vdev_in_nvlist(nv, vdev_health_check_cb, cb) == 0) {
+               return;
+       }
+
        printf_color(health_str_to_color(state),
            "\t%*s%-*s  %-8s", depth, "", cb->cb_namewidth - depth,
            name, state);
@@ -2336,6 +2497,9 @@ print_status_config(zpool_handle_t *zhp, status_cbdata_t *cb, const char *name,
                if (vs->vs_checksum_errors)
                        ccolor = ANSI_RED;
 
+               if (vs->vs_slow_ios)
+                       scolor = ANSI_BLUE;
+
                if (cb->cb_literal) {
                        fputc(' ', stdout);
                        printf_color(rcolor, "%5llu",
@@ -2368,9 +2532,30 @@ print_status_config(zpool_handle_t *zhp, status_cbdata_t *cb, const char *name,
                        }
 
                        if (cb->cb_literal)
-                               printf(" %5llu", (u_longlong_t)vs->vs_slow_ios);
+                               printf_color(scolor, " %5llu",
+                                   (u_longlong_t)vs->vs_slow_ios);
                        else
-                               printf(" %5s", rbuf);
+                               printf_color(scolor, " %5s", rbuf);
+               }
+               if (cb->cb_print_power) {
+                       if (children == 0)  {
+                               /* Only leaf vdevs have physical slots */
+                               switch (zpool_power_current_state(zhp, (char *)
+                                   fnvlist_lookup_string(nv,
+                                   ZPOOL_CONFIG_PATH))) {
+                               case 0:
+                                       printf_color(ANSI_RED, " %5s",
+                                           gettext("off"));
+                                       break;
+                               case 1:
+                                       printf(" %5s", gettext("on"));
+                                       break;
+                               default:
+                                       printf(" %5s", "-");
+                               }
+                       } else {
+                               printf(" %5s", "-");
+                       }
                }
        }
 
@@ -2425,7 +2610,13 @@ print_status_config(zpool_handle_t *zhp, status_cbdata_t *cb, const char *name,
                        break;
 
                case VDEV_AUX_ERR_EXCEEDED:
-                       (void) printf(gettext("too many errors"));
+                       if (vs->vs_read_errors + vs->vs_write_errors +
+                           vs->vs_checksum_errors == 0 && children == 0 &&
+                           vs->vs_slow_ios > 0) {
+                               (void) printf(gettext("too many slow I/Os"));
+                       } else {
+                               (void) printf(gettext("too many errors"));
+                       }
                        break;
 
                case VDEV_AUX_IO_FAILURE:
@@ -3116,12 +3307,22 @@ zfs_force_import_required(nvlist_t *config)
        nvlist_t *nvinfo;
 
        state = fnvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_STATE);
-       (void) nvlist_lookup_uint64(config, ZPOOL_CONFIG_HOSTID, &hostid);
+       nvinfo = fnvlist_lookup_nvlist(config, ZPOOL_CONFIG_LOAD_INFO);
+
+       /*
+        * The hostid on LOAD_INFO comes from the MOS label via
+        * spa_tryimport(). If its not there then we're likely talking to an
+        * older kernel, so use the top one, which will be from the label
+        * discovered in zpool_find_import(), or if a cachefile is in use, the
+        * local hostid.
+        */
+       if (nvlist_lookup_uint64(nvinfo, ZPOOL_CONFIG_HOSTID, &hostid) != 0)
+               (void) nvlist_lookup_uint64(config, ZPOOL_CONFIG_HOSTID,
+                   &hostid);
 
        if (state != POOL_STATE_EXPORTED && hostid != get_system_hostid())
                return (B_TRUE);
 
-       nvinfo = fnvlist_lookup_nvlist(config, ZPOOL_CONFIG_LOAD_INFO);
        if (nvlist_exists(nvinfo, ZPOOL_CONFIG_MMP_STATE)) {
                mmp_state_t mmp_state = fnvlist_lookup_uint64(nvinfo,
                    ZPOOL_CONFIG_MMP_STATE);
@@ -3192,7 +3393,10 @@ do_import(nvlist_t *config, const char *newname, const char *mntopts,
                        time_t timestamp = 0;
                        uint64_t hostid = 0;
 
-                       if (nvlist_exists(config, ZPOOL_CONFIG_HOSTNAME))
+                       if (nvlist_exists(nvinfo, ZPOOL_CONFIG_HOSTNAME))
+                               hostname = fnvlist_lookup_string(nvinfo,
+                                   ZPOOL_CONFIG_HOSTNAME);
+                       else if (nvlist_exists(config, ZPOOL_CONFIG_HOSTNAME))
                                hostname = fnvlist_lookup_string(config,
                                    ZPOOL_CONFIG_HOSTNAME);
 
@@ -3200,7 +3404,10 @@ do_import(nvlist_t *config, const char *newname, const char *mntopts,
                                timestamp = fnvlist_lookup_uint64(config,
                                    ZPOOL_CONFIG_TIMESTAMP);
 
-                       if (nvlist_exists(config, ZPOOL_CONFIG_HOSTID))
+                       if (nvlist_exists(nvinfo, ZPOOL_CONFIG_HOSTID))
+                               hostid = fnvlist_lookup_uint64(nvinfo,
+                                   ZPOOL_CONFIG_HOSTID);
+                       else if (nvlist_exists(config, ZPOOL_CONFIG_HOSTID))
                                hostid = fnvlist_lookup_uint64(config,
                                    ZPOOL_CONFIG_HOSTID);
 
@@ -5407,19 +5614,6 @@ get_interval_count_filter_guids(int *argc, char **argv, float *interval,
            interval, count);
 }
 
-/*
- * Floating point sleep().  Allows you to pass in a floating point value for
- * seconds.
- */
-static void
-fsleep(float sec)
-{
-       struct timespec req;
-       req.tv_sec = floor(sec);
-       req.tv_nsec = (sec - (float)req.tv_sec) * NANOSEC;
-       nanosleep(&req, NULL);
-}
-
 /*
  * Terminal height, in rows. Returns -1 if stdout is not connected to a TTY or
  * if we were unable to determine its size.
@@ -5929,6 +6123,7 @@ zpool_do_iostat(int argc, char **argv)
                                print_iostat_header(&cb);
 
                        if (skip) {
+                               (void) fflush(stdout);
                                (void) fsleep(interval);
                                continue;
                        }
@@ -5959,18 +6154,13 @@ zpool_do_iostat(int argc, char **argv)
 
                }
 
-               /*
-                * Flush the output so that redirection to a file isn't buffered
-                * indefinitely.
-                */
-               (void) fflush(stdout);
-
                if (interval == 0)
                        break;
 
                if (count != 0 && --count == 0)
                        break;
 
+               (void) fflush(stdout);
                (void) fsleep(interval);
        }
 
@@ -6493,6 +6683,8 @@ zpool_do_list(int argc, char **argv)
                        break;
 
                pool_list_free(list);
+
+               (void) fflush(stdout);
                (void) fsleep(interval);
        }
 
@@ -6629,9 +6821,17 @@ zpool_do_attach_or_replace(int argc, char **argv, int replacing)
        ret = zpool_vdev_attach(zhp, old_disk, new_disk, nvroot, replacing,
            rebuild);
 
-       if (ret == 0 && wait)
-               ret = zpool_wait(zhp,
-                   replacing ? ZPOOL_WAIT_REPLACE : ZPOOL_WAIT_RESILVER);
+       if (ret == 0 && wait) {
+               zpool_wait_activity_t activity = ZPOOL_WAIT_RESILVER;
+               char raidz_prefix[] = "raidz";
+               if (replacing) {
+                       activity = ZPOOL_WAIT_REPLACE;
+               } else if (strncmp(old_disk,
+                   raidz_prefix, strlen(raidz_prefix)) == 0) {
+                       activity = ZPOOL_WAIT_RAIDZ_EXPAND;
+               }
+               ret = zpool_wait(zhp, activity);
+       }
 
        nvlist_free(props);
        nvlist_free(nvroot);
@@ -6657,17 +6857,21 @@ zpool_do_replace(int argc, char **argv)
 }
 
 /*
- * zpool attach [-fsw] [-o property=value] <pool> <device> <new_device>
+ * zpool attach [-fsw] [-o property=value] <pool> <device>|<vdev> <new_device>
  *
  *     -f      Force attach, even if <new_device> appears to be in use.
  *     -s      Use sequential instead of healing reconstruction for resilver.
  *     -o      Set property=value.
- *     -w      Wait for resilvering to complete before returning
+ *     -w      Wait for resilvering (mirror) or expansion (raidz) to complete
+ *             before returning.
  *
- * Attach <new_device> to the mirror containing <device>.  If <device> is not
- * part of a mirror, then <device> will be transformed into a mirror of
- * <device> and <new_device>.  In either case, <new_device> will begin life
- * with a DTL of [0, now], and will immediately begin to resilver itself.
+ * Attach <new_device> to a <device> or <vdev>, where the vdev can be of type
+ * mirror or raidz. If <device> is not part of a mirror, then <device> will
+ * be transformed into a mirror of <device> and <new_device>. When a mirror
+ * is involved, <new_device> will begin life with a DTL of [0, now], and will
+ * immediately begin to resilver itself. For the raidz case, a expansion will
+ * commence and reflow the raidz data across all the disks including the
+ * <new_device>.
  */
 int
 zpool_do_attach(int argc, char **argv)
@@ -6919,9 +7123,10 @@ zpool_do_split(int argc, char **argv)
 }
 
 
-
 /*
- * zpool online <pool> <device> ...
+ * zpool online [--power] <pool> <device> ...
+ *
+ * --power: Power on the enclosure slot to the drive (if possible)
  */
 int
 zpool_do_online(int argc, char **argv)
@@ -6932,13 +7137,21 @@ zpool_do_online(int argc, char **argv)
        int ret = 0;
        vdev_state_t newstate;
        int flags = 0;
+       boolean_t is_power_on = B_FALSE;
+       struct option long_options[] = {
+               {"power", no_argument, NULL, ZPOOL_OPTION_POWER},
+               {0, 0, 0, 0}
+       };
 
        /* check options */
-       while ((c = getopt(argc, argv, "e")) != -1) {
+       while ((c = getopt_long(argc, argv, "e", long_options, NULL)) != -1) {
                switch (c) {
                case 'e':
                        flags |= ZFS_ONLINE_EXPAND;
                        break;
+               case ZPOOL_OPTION_POWER:
+                       is_power_on = B_TRUE;
+                       break;
                case '?':
                        (void) fprintf(stderr, gettext("invalid option '%c'\n"),
                            optopt);
@@ -6946,6 +7159,9 @@ zpool_do_online(int argc, char **argv)
                }
        }
 
+       if (libzfs_envvar_is_set("ZPOOL_AUTO_POWER_ON_SLOT"))
+               is_power_on = B_TRUE;
+
        argc -= optind;
        argv += optind;
 
@@ -6967,6 +7183,18 @@ zpool_do_online(int argc, char **argv)
        for (i = 1; i < argc; i++) {
                vdev_state_t oldstate;
                boolean_t avail_spare, l2cache;
+               int rc;
+
+               if (is_power_on) {
+                       rc = zpool_power_on_and_disk_wait(zhp, argv[i]);
+                       if (rc == ENOTSUP) {
+                               (void) fprintf(stderr,
+                                   gettext("Power control not supported\n"));
+                       }
+                       if (rc != 0)
+                               return (rc);
+               }
+
                nvlist_t *tgt = zpool_find_vdev(zhp, argv[i], &avail_spare,
                    &l2cache, NULL);
                if (tgt == NULL) {
@@ -7012,12 +7240,15 @@ zpool_do_online(int argc, char **argv)
 }
 
 /*
- * zpool offline [-ft] <pool> <device> ...
+ * zpool offline [-ft]|[--power] <pool> <device> ...
+ *
  *
  *     -f      Force the device into a faulted state.
  *
  *     -t      Only take the device off-line temporarily.  The offline/faulted
  *             state will not be persistent across reboots.
+ *
+ *     --power Power off the enclosure slot to the drive (if possible)
  */
 int
 zpool_do_offline(int argc, char **argv)
@@ -7028,9 +7259,15 @@ zpool_do_offline(int argc, char **argv)
        int ret = 0;
        boolean_t istmp = B_FALSE;
        boolean_t fault = B_FALSE;
+       boolean_t is_power_off = B_FALSE;
+
+       struct option long_options[] = {
+               {"power", no_argument, NULL, ZPOOL_OPTION_POWER},
+               {0, 0, 0, 0}
+       };
 
        /* check options */
-       while ((c = getopt(argc, argv, "ft")) != -1) {
+       while ((c = getopt_long(argc, argv, "ft", long_options, NULL)) != -1) {
                switch (c) {
                case 'f':
                        fault = B_TRUE;
@@ -7038,6 +7275,9 @@ zpool_do_offline(int argc, char **argv)
                case 't':
                        istmp = B_TRUE;
                        break;
+               case ZPOOL_OPTION_POWER:
+                       is_power_off = B_TRUE;
+                       break;
                case '?':
                        (void) fprintf(stderr, gettext("invalid option '%c'\n"),
                            optopt);
@@ -7045,6 +7285,20 @@ zpool_do_offline(int argc, char **argv)
                }
        }
 
+       if (is_power_off && fault) {
+               (void) fprintf(stderr,
+                   gettext("-0 and -f cannot be used together\n"));
+               usage(B_FALSE);
+               return (1);
+       }
+
+       if (is_power_off && istmp) {
+               (void) fprintf(stderr,
+                   gettext("-0 and -t cannot be used together\n"));
+               usage(B_FALSE);
+               return (1);
+       }
+
        argc -= optind;
        argv += optind;
 
@@ -7064,8 +7318,22 @@ zpool_do_offline(int argc, char **argv)
                return (1);
 
        for (i = 1; i < argc; i++) {
-               if (fault) {
-                       uint64_t guid = zpool_vdev_path_to_guid(zhp, argv[i]);
+               uint64_t guid = zpool_vdev_path_to_guid(zhp, argv[i]);
+               if (is_power_off) {
+                       /*
+                        * Note: we have to power off first, then set REMOVED,
+                        * or else zpool_vdev_set_removed_state() returns
+                        * EAGAIN.
+                        */
+                       ret = zpool_power_off(zhp, argv[i]);
+                       if (ret != 0) {
+                               (void) fprintf(stderr, "%s %s %d\n",
+                                   gettext("unable to power off slot for"),
+                                   argv[i], ret);
+                       }
+                       zpool_vdev_set_removed_state(zhp, guid, VDEV_AUX_NONE);
+
+               } else if (fault) {
                        vdev_aux_t aux;
                        if (istmp == B_FALSE) {
                                /* Force the fault to persist across imports */
@@ -7088,7 +7356,7 @@ zpool_do_offline(int argc, char **argv)
 }
 
 /*
- * zpool clear <pool> [device]
+ * zpool clear [-nF]|[--power] <pool> [device]
  *
  * Clear all errors associated with a pool or a particular device.
  */
@@ -7100,13 +7368,20 @@ zpool_do_clear(int argc, char **argv)
        boolean_t dryrun = B_FALSE;
        boolean_t do_rewind = B_FALSE;
        boolean_t xtreme_rewind = B_FALSE;
+       boolean_t is_power_on = B_FALSE;
        uint32_t rewind_policy = ZPOOL_NO_REWIND;
        nvlist_t *policy = NULL;
        zpool_handle_t *zhp;
        char *pool, *device;
 
+       struct option long_options[] = {
+               {"power", no_argument, NULL, ZPOOL_OPTION_POWER},
+               {0, 0, 0, 0}
+       };
+
        /* check options */
-       while ((c = getopt(argc, argv, "FnX")) != -1) {
+       while ((c = getopt_long(argc, argv, "FnX", long_options,
+           NULL)) != -1) {
                switch (c) {
                case 'F':
                        do_rewind = B_TRUE;
@@ -7117,6 +7392,9 @@ zpool_do_clear(int argc, char **argv)
                case 'X':
                        xtreme_rewind = B_TRUE;
                        break;
+               case ZPOOL_OPTION_POWER:
+                       is_power_on = B_TRUE;
+                       break;
                case '?':
                        (void) fprintf(stderr, gettext("invalid option '%c'\n"),
                            optopt);
@@ -7124,6 +7402,9 @@ zpool_do_clear(int argc, char **argv)
                }
        }
 
+       if (libzfs_envvar_is_set("ZPOOL_AUTO_POWER_ON_SLOT"))
+               is_power_on = B_TRUE;
+
        argc -= optind;
        argv += optind;
 
@@ -7164,6 +7445,14 @@ zpool_do_clear(int argc, char **argv)
                return (1);
        }
 
+       if (is_power_on) {
+               if (device == NULL) {
+                       zpool_power_on_pool_and_wait_for_devices(zhp);
+               } else {
+                       zpool_power_on_and_disk_wait(zhp, device);
+               }
+       }
+
        if (zpool_clear(zhp, device, policy) != 0)
                ret = 1;
 
@@ -8174,6 +8463,97 @@ print_removal_status(zpool_handle_t *zhp, pool_removal_stat_t *prs)
        }
 }
 
+/*
+ * Print out detailed raidz expansion status.
+ */
+static void
+print_raidz_expand_status(zpool_handle_t *zhp, pool_raidz_expand_stat_t *pres)
+{
+       char copied_buf[7];
+
+       if (pres == NULL || pres->pres_state == DSS_NONE)
+               return;
+
+       /*
+        * Determine name of vdev.
+        */
+       nvlist_t *config = zpool_get_config(zhp, NULL);
+       nvlist_t *nvroot = fnvlist_lookup_nvlist(config,
+           ZPOOL_CONFIG_VDEV_TREE);
+       nvlist_t **child;
+       uint_t children;
+       verify(nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_CHILDREN,
+           &child, &children) == 0);
+       assert(pres->pres_expanding_vdev < children);
+
+       printf_color(ANSI_BOLD, gettext("expand: "));
+
+       time_t start = pres->pres_start_time;
+       time_t end = pres->pres_end_time;
+       char *vname =
+           zpool_vdev_name(g_zfs, zhp, child[pres->pres_expanding_vdev], 0);
+       zfs_nicenum(pres->pres_reflowed, copied_buf, sizeof (copied_buf));
+
+       /*
+        * Expansion is finished or canceled.
+        */
+       if (pres->pres_state == DSS_FINISHED) {
+               char time_buf[32];
+               secs_to_dhms(end - start, time_buf);
+
+               (void) printf(gettext("expanded %s-%u copied %s in %s, "
+                   "on %s"), vname, (int)pres->pres_expanding_vdev,
+                   copied_buf, time_buf, ctime((time_t *)&end));
+       } else {
+               char examined_buf[7], total_buf[7], rate_buf[7];
+               uint64_t copied, total, elapsed, secs_left;
+               double fraction_done;
+               uint_t rate;
+
+               assert(pres->pres_state == DSS_SCANNING);
+
+               /*
+                * Expansion is in progress.
+                */
+               (void) printf(gettext(
+                   "expansion of %s-%u in progress since %s"),
+                   vname, (int)pres->pres_expanding_vdev, ctime(&start));
+
+               copied = pres->pres_reflowed > 0 ? pres->pres_reflowed : 1;
+               total = pres->pres_to_reflow;
+               fraction_done = (double)copied / total;
+
+               /* elapsed time for this pass */
+               elapsed = time(NULL) - pres->pres_start_time;
+               elapsed = elapsed > 0 ? elapsed : 1;
+               rate = copied / elapsed;
+               rate = rate > 0 ? rate : 1;
+               secs_left = (total - copied) / rate;
+
+               zfs_nicenum(copied, examined_buf, sizeof (examined_buf));
+               zfs_nicenum(total, total_buf, sizeof (total_buf));
+               zfs_nicenum(rate, rate_buf, sizeof (rate_buf));
+
+               /*
+                * do not print estimated time if hours_left is more than
+                * 30 days
+                */
+               (void) printf(gettext("\t%s / %s copied at %s/s, %.2f%% done"),
+                   examined_buf, total_buf, rate_buf, 100 * fraction_done);
+               if (pres->pres_waiting_for_resilver) {
+                       (void) printf(gettext(", paused for resilver or "
+                           "clear\n"));
+               } else if (secs_left < (30 * 24 * 3600)) {
+                       char time_buf[32];
+                       secs_to_dhms(secs_left, time_buf);
+                       (void) printf(gettext(", %s to go\n"), time_buf);
+               } else {
+                       (void) printf(gettext(
+                           ", (copy is slow, no estimated time)\n"));
+               }
+       }
+       free(vname);
+}
 static void
 print_checkpoint_status(pool_checkpoint_stat_t *pcs)
 {
@@ -8751,19 +9131,24 @@ status_callback(zpool_handle_t *zhp, void *data)
                uint64_t nerr;
                nvlist_t **spares, **l2cache;
                uint_t nspares, nl2cache;
-               pool_checkpoint_stat_t *pcs = NULL;
-               pool_removal_stat_t *prs = NULL;
 
                print_scan_status(zhp, nvroot);
 
+               pool_removal_stat_t *prs = NULL;
                (void) nvlist_lookup_uint64_array(nvroot,
                    ZPOOL_CONFIG_REMOVAL_STATS, (uint64_t **)&prs, &c);
                print_removal_status(zhp, prs);
 
+               pool_checkpoint_stat_t *pcs = NULL;
                (void) nvlist_lookup_uint64_array(nvroot,
                    ZPOOL_CONFIG_CHECKPOINT_STATS, (uint64_t **)&pcs, &c);
                print_checkpoint_status(pcs);
 
+               pool_raidz_expand_stat_t *pres = NULL;
+               (void) nvlist_lookup_uint64_array(nvroot,
+                   ZPOOL_CONFIG_RAIDZ_EXPAND_STATS, (uint64_t **)&pres, &c);
+               print_raidz_expand_status(zhp, pres);
+
                cbp->cb_namewidth = max_width(zhp, nvroot, 0, 0,
                    cbp->cb_name_flags | VDEV_NAME_TYPE_ID);
                if (cbp->cb_namewidth < 10)
@@ -8780,6 +9165,10 @@ status_callback(zpool_handle_t *zhp, void *data)
                        printf_color(ANSI_BOLD, " %5s", gettext("SLOW"));
                }
 
+               if (cbp->cb_print_power) {
+                       printf_color(ANSI_BOLD, " %5s", gettext("POWER"));
+               }
+
                if (cbp->vcdl != NULL)
                        print_cmd_columns(cbp->vcdl, 0);
 
@@ -8807,9 +9196,11 @@ status_callback(zpool_handle_t *zhp, void *data)
                                (void) printf(gettext(
                                    "errors: No known data errors\n"));
                        } else if (!cbp->cb_verbose) {
+                               color_start(ANSI_RED);
                                (void) printf(gettext("errors: %llu data "
                                    "errors, use '-v' for a list\n"),
                                    (u_longlong_t)nerr);
+                               color_end();
                        } else {
                                print_error_log(zhp);
                        }
@@ -8826,21 +9217,23 @@ status_callback(zpool_handle_t *zhp, void *data)
 }
 
 /*
- * zpool status [-c [script1,script2,...]] [-igLpPstvx] [-T d|u] [pool] ...
- *              [interval [count]]
+ * zpool status [-c [script1,script2,...]] [-DegiLpPstvx] [--power] [-T d|u] ...
+ *              [pool] [interval [count]]
  *
  *     -c CMD  For each vdev, run command CMD
- *     -i      Display vdev initialization status.
+ *     -D      Display dedup status (undocumented)
+ *     -e      Display only unhealthy vdevs
  *     -g      Display guid for individual vdev name.
+ *     -i      Display vdev initialization status.
  *     -L      Follow links when resolving vdev path name.
  *     -p      Display values in parsable (exact) format.
  *     -P      Display full path for vdev name.
  *     -s      Display slow IOs column.
- *     -v      Display complete error logs
- *     -x      Display only pools with potential problems
- *     -D      Display dedup status (undocumented)
  *     -t      Display vdev TRIM status.
  *     -T      Display a timestamp in date(1) or Unix format
+ *     -v      Display complete error logs
+ *     -x      Display only pools with potential problems
+ *     --power Display vdev enclosure slot power status
  *
  * Describes the health status of all pools or some subset.
  */
@@ -8854,8 +9247,14 @@ zpool_do_status(int argc, char **argv)
        status_cbdata_t cb = { 0 };
        char *cmd = NULL;
 
+       struct option long_options[] = {
+               {"power", no_argument, NULL, ZPOOL_OPTION_POWER},
+               {0, 0, 0, 0}
+       };
+
        /* check options */
-       while ((c = getopt(argc, argv, "c:igLpPsvxDtT:")) != -1) {
+       while ((c = getopt_long(argc, argv, "c:DegiLpPstT:vx", long_options,
+           NULL)) != -1) {
                switch (c) {
                case 'c':
                        if (cmd != NULL) {
@@ -8881,12 +9280,18 @@ zpool_do_status(int argc, char **argv)
                        }
                        cmd = optarg;
                        break;
-               case 'i':
-                       cb.cb_print_vdev_init = B_TRUE;
+               case 'D':
+                       cb.cb_dedup_stats = B_TRUE;
+                       break;
+               case 'e':
+                       cb.cb_print_unhealthy = B_TRUE;
                        break;
                case 'g':
                        cb.cb_name_flags |= VDEV_NAME_GUID;
                        break;
+               case 'i':
+                       cb.cb_print_vdev_init = B_TRUE;
+                       break;
                case 'L':
                        cb.cb_name_flags |= VDEV_NAME_FOLLOW_LINKS;
                        break;
@@ -8899,20 +9304,20 @@ zpool_do_status(int argc, char **argv)
                case 's':
                        cb.cb_print_slow_ios = B_TRUE;
                        break;
+               case 't':
+                       cb.cb_print_vdev_trim = B_TRUE;
+                       break;
+               case 'T':
+                       get_timestamp_arg(*optarg);
+                       break;
                case 'v':
                        cb.cb_verbose = B_TRUE;
                        break;
                case 'x':
                        cb.cb_explain = B_TRUE;
                        break;
-               case 'D':
-                       cb.cb_dedup_stats = B_TRUE;
-                       break;
-               case 't':
-                       cb.cb_print_vdev_trim = B_TRUE;
-                       break;
-               case 'T':
-                       get_timestamp_arg(*optarg);
+               case ZPOOL_OPTION_POWER:
+                       cb.cb_print_power = B_TRUE;
                        break;
                case '?':
                        if (optopt == 'c') {
@@ -8950,7 +9355,6 @@ zpool_do_status(int argc, char **argv)
 
                if (cb.vcdl != NULL)
                        free_vdev_cmd_data_list(cb.vcdl);
-
                if (argc == 0 && cb.cb_count == 0)
                        (void) fprintf(stderr, gettext("no pools available\n"));
                else if (cb.cb_explain && cb.cb_first && cb.cb_allpools)
@@ -8965,6 +9369,7 @@ zpool_do_status(int argc, char **argv)
                if (count != 0 && --count == 0)
                        break;
 
+               (void) fflush(stdout);
                (void) fsleep(interval);
        }
 
@@ -10386,11 +10791,10 @@ found:
                }
        } else {
                /*
-                * The first arg isn't a pool name,
+                * The first arg isn't the name of a valid pool.
                 */
-               fprintf(stderr, gettext("missing pool name.\n"));
-               fprintf(stderr, "\n");
-               usage(B_FALSE);
+               fprintf(stderr, gettext("Cannot get properties of %s: "
+                   "no such pool available.\n"), argv[0]);
                return (1);
        }
 
@@ -10717,8 +11121,9 @@ print_wait_status_row(wait_data_t *wd, zpool_handle_t *zhp, int row)
        pool_checkpoint_stat_t *pcs = NULL;
        pool_scan_stat_t *pss = NULL;
        pool_removal_stat_t *prs = NULL;
+       pool_raidz_expand_stat_t *pres = NULL;
        const char *const headers[] = {"DISCARD", "FREE", "INITIALIZE",
-           "REPLACE", "REMOVE", "RESILVER", "SCRUB", "TRIM"};
+           "REPLACE", "REMOVE", "RESILVER", "SCRUB", "TRIM", "RAIDZ_EXPAND"};
        int col_widths[ZPOOL_WAIT_NUM_ACTIVITIES];
 
        /* Calculate the width of each column */
@@ -10731,6 +11136,9 @@ print_wait_status_row(wait_data_t *wd, zpool_handle_t *zhp, int row)
                col_widths[i] = MAX(strlen(headers[i]), 6) + 2;
        }
 
+       if (timestamp_fmt != NODATE)
+               print_timestamp(timestamp_fmt);
+
        /* Print header if appropriate */
        int term_height = terminal_height();
        boolean_t reprint_header = (!wd->wd_headers_once && term_height > 0 &&
@@ -10777,6 +11185,13 @@ print_wait_status_row(wait_data_t *wd, zpool_handle_t *zhp, int row)
                    vdev_activity_top_remaining(nvroot);
        }
 
+       (void) nvlist_lookup_uint64_array(nvroot,
+           ZPOOL_CONFIG_RAIDZ_EXPAND_STATS, (uint64_t **)&pres, &c);
+       if (pres != NULL && pres->pres_state == DSS_SCANNING) {
+               int64_t rem = pres->pres_to_reflow - pres->pres_reflowed;
+               bytes_rem[ZPOOL_WAIT_RAIDZ_EXPAND] = rem;
+       }
+
        bytes_rem[ZPOOL_WAIT_INITIALIZE] =
            vdev_activity_remaining(nvroot, ZPOOL_WAIT_INITIALIZE);
        bytes_rem[ZPOOL_WAIT_TRIM] =
@@ -10798,19 +11213,17 @@ print_wait_status_row(wait_data_t *wd, zpool_handle_t *zhp, int row)
        if (vdev_any_spare_replacing(nvroot))
                bytes_rem[ZPOOL_WAIT_REPLACE] =  bytes_rem[ZPOOL_WAIT_RESILVER];
 
-       if (timestamp_fmt != NODATE)
-               print_timestamp(timestamp_fmt);
-
        for (i = 0; i < ZPOOL_WAIT_NUM_ACTIVITIES; i++) {
                char buf[64];
                if (!wd->wd_enabled[i])
                        continue;
 
-               if (wd->wd_exact)
+               if (wd->wd_exact) {
                        (void) snprintf(buf, sizeof (buf), "%" PRIi64,
                            bytes_rem[i]);
-               else
+               } else {
                        zfs_nicenum(bytes_rem[i], buf, sizeof (buf));
+               }
 
                if (wd->wd_scripted)
                        (void) printf(i == 0 ? "%s" : "\t%s", buf);
@@ -10916,7 +11329,8 @@ zpool_do_wait(int argc, char **argv)
                        for (char *tok; (tok = strsep(&optarg, ",")); ) {
                                static const char *const col_opts[] = {
                                    "discard", "free", "initialize", "replace",
-                                   "remove", "resilver", "scrub", "trim" };
+                                   "remove", "resilver", "scrub", "trim",
+                                   "raidz_expand" };
 
                                for (i = 0; i < ARRAY_SIZE(col_opts); ++i)
                                        if (strcmp(tok, col_opts[i]) == 0) {
@@ -11051,6 +11465,25 @@ zpool_do_version(int argc, char **argv)
        return (zfs_version_print() != 0);
 }
 
+/* Display documentation */
+static int
+zpool_do_help(int argc, char **argv)
+{
+       char page[MAXNAMELEN];
+       if (argc < 3 || strcmp(argv[2], "zpool") == 0)
+               strcpy(page, "zpool");
+       else if (strcmp(argv[2], "concepts") == 0 ||
+           strcmp(argv[2], "props") == 0)
+               snprintf(page, sizeof (page), "zpool%s", argv[2]);
+       else
+               snprintf(page, sizeof (page), "zpool-%s", argv[2]);
+
+       execlp("man", "man", page, NULL);
+
+       fprintf(stderr, "couldn't run man program: %s", strerror(errno));
+       return (-1);
+}
+
 /*
  * Do zpool_load_compat() and print error message on failure
  */
@@ -11118,6 +11551,12 @@ main(int argc, char **argv)
        if ((strcmp(cmdname, "-V") == 0) || (strcmp(cmdname, "--version") == 0))
                return (zpool_do_version(argc, argv));
 
+       /*
+        * Special case 'help'
+        */
+       if (strcmp(cmdname, "help") == 0)
+               return (zpool_do_help(argc, argv));
+
        if ((g_zfs = libzfs_init()) == NULL) {
                (void) fprintf(stderr, "%s\n", libzfs_error_init(errno));
                return (1);