lib/libzfs/libzfs_status.c

   1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21
  22 /*
  23  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  24  * Copyright (c) 2012 by Delphix. All rights reserved.
  25  * Copyright (c) 2013 Steven Hartland. All rights reserved.
  26  * Copyright (c) 2021, Colm Buckley <colm@tuatha.org>
  27  */
  28
  29 /*
  30  * This file contains the functions which analyze the status of a pool.  This
  31  * include both the status of an active pool, as well as the status exported
  32  * pools.  Returns one of the ZPOOL_STATUS_* defines describing the status of
  33  * the pool.  This status is independent (to a certain degree) from the state of
  34  * the pool.  A pool's state describes only whether or not it is capable of
  35  * providing the necessary fault tolerance for data.  The status describes the
  36  * overall status of devices.  A pool that is online can still have a device
  37  * that is experiencing errors.
  38  *
  39  * Only a subset of the possible faults can be detected using 'zpool status',
  40  * and not all possible errors correspond to a FMA message ID.  The explanation
  41  * is left up to the caller, depending on whether it is a live pool or an
  42  * import.
  43  */
  44
  45 #include <libzfs.h>
  46 #include <libzutil.h>
  47 #include <stdlib.h>
  48 #include <string.h>
  49 #include <unistd.h>
  50 #include <sys/systeminfo.h>
  51 #include "libzfs_impl.h"
  52 #include "zfeature_common.h"
  53
  54 /*
  55  * Message ID table.  This must be kept in sync with the ZPOOL_STATUS_* defines
  56  * in include/libzfs.h.  Note that there are some status results which go past
  57  * the end of this table, and hence have no associated message ID.
  58  */
  59 static char *zfs_msgid_table[] = {
  60         "ZFS-8000-14", /* ZPOOL_STATUS_CORRUPT_CACHE */
  61         "ZFS-8000-2Q", /* ZPOOL_STATUS_MISSING_DEV_R */
  62         "ZFS-8000-3C", /* ZPOOL_STATUS_MISSING_DEV_NR */
  63         "ZFS-8000-4J", /* ZPOOL_STATUS_CORRUPT_LABEL_R */
  64         "ZFS-8000-5E", /* ZPOOL_STATUS_CORRUPT_LABEL_NR */
  65         "ZFS-8000-6X", /* ZPOOL_STATUS_BAD_GUID_SUM */
  66         "ZFS-8000-72", /* ZPOOL_STATUS_CORRUPT_POOL */
  67         "ZFS-8000-8A", /* ZPOOL_STATUS_CORRUPT_DATA */
  68         "ZFS-8000-9P", /* ZPOOL_STATUS_FAILING_DEV */
  69         "ZFS-8000-A5", /* ZPOOL_STATUS_VERSION_NEWER */
  70         "ZFS-8000-EY", /* ZPOOL_STATUS_HOSTID_MISMATCH */
  71         "ZFS-8000-EY", /* ZPOOL_STATUS_HOSTID_ACTIVE */
  72         "ZFS-8000-EY", /* ZPOOL_STATUS_HOSTID_REQUIRED */
  73         "ZFS-8000-HC", /* ZPOOL_STATUS_IO_FAILURE_WAIT */
  74         "ZFS-8000-JQ", /* ZPOOL_STATUS_IO_FAILURE_CONTINUE */
  75         "ZFS-8000-MM", /* ZPOOL_STATUS_IO_FAILURE_MMP */
  76         "ZFS-8000-K4", /* ZPOOL_STATUS_BAD_LOG */
  77         "ZFS-8000-ER", /* ZPOOL_STATUS_ERRATA */
  78         /*
  79          * The following results have no message ID.
  80          *      ZPOOL_STATUS_UNSUP_FEAT_READ
  81          *      ZPOOL_STATUS_UNSUP_FEAT_WRITE
  82          *      ZPOOL_STATUS_FAULTED_DEV_R
  83          *      ZPOOL_STATUS_FAULTED_DEV_NR
  84          *      ZPOOL_STATUS_VERSION_OLDER
  85          *      ZPOOL_STATUS_FEAT_DISABLED
  86          *      ZPOOL_STATUS_RESILVERING
  87          *      ZPOOL_STATUS_OFFLINE_DEV
  88          *      ZPOOL_STATUS_REMOVED_DEV
  89          *      ZPOOL_STATUS_REBUILDING
  90          *      ZPOOL_STATUS_REBUILD_SCRUB
  91          *      ZPOOL_STATUS_COMPATIBILITY_ERR
  92          *      ZPOOL_STATUS_OK
  93          */
  94 };
  95
  96 #define NMSGID  (sizeof (zfs_msgid_table) / sizeof (zfs_msgid_table[0]))
  97
  98 /* ARGSUSED */
  99 static int
 100 vdev_missing(vdev_stat_t *vs, uint_t vsc)
 101 {
 102         return (vs->vs_state == VDEV_STATE_CANT_OPEN &&
 103             vs->vs_aux == VDEV_AUX_OPEN_FAILED);
 104 }
 105
 106 /* ARGSUSED */
 107 static int
 108 vdev_faulted(vdev_stat_t *vs, uint_t vsc)
 109 {
 110         return (vs->vs_state == VDEV_STATE_FAULTED);
 111 }
 112
 113 /* ARGSUSED */
 114 static int
 115 vdev_errors(vdev_stat_t *vs, uint_t vsc)
 116 {
 117         return (vs->vs_state == VDEV_STATE_DEGRADED ||
 118             vs->vs_read_errors != 0 || vs->vs_write_errors != 0 ||
 119             vs->vs_checksum_errors != 0);
 120 }
 121
 122 /* ARGSUSED */
 123 static int
 124 vdev_broken(vdev_stat_t *vs, uint_t vsc)
 125 {
 126         return (vs->vs_state == VDEV_STATE_CANT_OPEN);
 127 }
 128
 129 /* ARGSUSED */
 130 static int
 131 vdev_offlined(vdev_stat_t *vs, uint_t vsc)
 132 {
 133         return (vs->vs_state == VDEV_STATE_OFFLINE);
 134 }
 135
 136 /* ARGSUSED */
 137 static int
 138 vdev_removed(vdev_stat_t *vs, uint_t vsc)
 139 {
 140         return (vs->vs_state == VDEV_STATE_REMOVED);
 141 }
 142
 143 static int
 144 vdev_non_native_ashift(vdev_stat_t *vs, uint_t vsc)
 145 {
 146         if (getenv("ZPOOL_STATUS_NON_NATIVE_ASHIFT_IGNORE") != NULL)
 147                 return (0);
 148
 149         return (VDEV_STAT_VALID(vs_physical_ashift, vsc) &&
 150             vs->vs_configured_ashift < vs->vs_physical_ashift);
 151 }
 152
 153 /*
 154  * Detect if any leaf devices that have seen errors or could not be opened.
 155  */
 156 static boolean_t
 157 find_vdev_problem(nvlist_t *vdev, int (*func)(vdev_stat_t *, uint_t),
 158     boolean_t ignore_replacing)
 159 {
 160         nvlist_t **child;
 161         vdev_stat_t *vs;
 162         uint_t c, vsc, children;
 163
 164         /*
 165          * Ignore problems within a 'replacing' vdev, since we're presumably in
 166          * the process of repairing any such errors, and don't want to call them
 167          * out again.  We'll pick up the fact that a resilver is happening
 168          * later.
 169          */
 170         if (ignore_replacing == B_TRUE) {
 171                 char *type;
 172
 173                 verify(nvlist_lookup_string(vdev, ZPOOL_CONFIG_TYPE,
 174                     &type) == 0);
 175                 if (strcmp(type, VDEV_TYPE_REPLACING) == 0)
 176                         return (B_FALSE);
 177         }
 178
 179         if (nvlist_lookup_nvlist_array(vdev, ZPOOL_CONFIG_CHILDREN, &child,
 180             &children) == 0) {
 181                 for (c = 0; c < children; c++)
 182                         if (find_vdev_problem(child[c], func, ignore_replacing))
 183                                 return (B_TRUE);
 184         } else {
 185                 verify(nvlist_lookup_uint64_array(vdev, ZPOOL_CONFIG_VDEV_STATS,
 186                     (uint64_t **)&vs, &vsc) == 0);
 187
 188                 if (func(vs, vsc) != 0)
 189                         return (B_TRUE);
 190         }
 191
 192         /*
 193          * Check any L2 cache devs
 194          */
 195         if (nvlist_lookup_nvlist_array(vdev, ZPOOL_CONFIG_L2CACHE, &child,
 196             &children) == 0) {
 197                 for (c = 0; c < children; c++)
 198                         if (find_vdev_problem(child[c], func, ignore_replacing))
 199                                 return (B_TRUE);
 200         }
 201
 202         return (B_FALSE);
 203 }
 204
 205 /*
 206  * Active pool health status.
 207  *
 208  * To determine the status for a pool, we make several passes over the config,
 209  * picking the most egregious error we find.  In order of importance, we do the
 210  * following:
 211  *
 212  *      - Check for a complete and valid configuration
 213  *      - Look for any faulted or missing devices in a non-replicated config
 214  *      - Check for any data errors
 215  *      - Check for any faulted or missing devices in a replicated config
 216  *      - Look for any devices showing errors
 217  *      - Check for any resilvering or rebuilding devices
 218  *
 219  * There can obviously be multiple errors within a single pool, so this routine
 220  * only picks the most damaging of all the current errors to report.
 221  */
 222 static zpool_status_t
 223 check_status(nvlist_t *config, boolean_t isimport,
 224     zpool_errata_t *erratap, const char *compat)
 225 {
 226         nvlist_t *nvroot;
 227         vdev_stat_t *vs;
 228         pool_scan_stat_t *ps = NULL;
 229         uint_t vsc, psc;
 230         uint64_t nerr;
 231         uint64_t version;
 232         uint64_t stateval;
 233         uint64_t suspended;
 234         uint64_t hostid = 0;
 235         uint64_t errata = 0;
 236         unsigned long system_hostid = get_system_hostid();
 237
 238         verify(nvlist_lookup_uint64(config, ZPOOL_CONFIG_VERSION,
 239             &version) == 0);
 240         verify(nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE,
 241             &nvroot) == 0);
 242         verify(nvlist_lookup_uint64_array(nvroot, ZPOOL_CONFIG_VDEV_STATS,
 243             (uint64_t **)&vs, &vsc) == 0);
 244         verify(nvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_STATE,
 245             &stateval) == 0);
 246
 247         /*
 248          * Currently resilvering a vdev
 249          */
 250         (void) nvlist_lookup_uint64_array(nvroot, ZPOOL_CONFIG_SCAN_STATS,
 251             (uint64_t **)&ps, &psc);
 252         if (ps != NULL && ps->pss_func == POOL_SCAN_RESILVER &&
 253             ps->pss_state == DSS_SCANNING)
 254                 return (ZPOOL_STATUS_RESILVERING);
 255
 256         /*
 257          * Currently rebuilding a vdev, check top-level vdevs.
 258          */
 259         vdev_rebuild_stat_t *vrs = NULL;
 260         nvlist_t **child;
 261         uint_t c, i, children;
 262         uint64_t rebuild_end_time = 0;
 263         if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_CHILDREN,
 264             &child, &children) == 0) {
 265                 for (c = 0; c < children; c++) {
 266                         if ((nvlist_lookup_uint64_array(child[c],
 267                             ZPOOL_CONFIG_REBUILD_STATS,
 268                             (uint64_t **)&vrs, &i) == 0) && (vrs != NULL)) {
 269                                 uint64_t state = vrs->vrs_state;
 270
 271                                 if (state == VDEV_REBUILD_ACTIVE) {
 272                                         return (ZPOOL_STATUS_REBUILDING);
 273                                 } else if (state == VDEV_REBUILD_COMPLETE &&
 274                                     vrs->vrs_end_time > rebuild_end_time) {
 275                                         rebuild_end_time = vrs->vrs_end_time;
 276                                 }
 277                         }
 278                 }
 279
 280                 /*
 281                  * If we can determine when the last scrub was run, and it
 282                  * was before the last rebuild completed, then recommend
 283                  * that the pool be scrubbed to verify all checksums.  When
 284                  * ps is NULL we can infer the pool has never been scrubbed.
 285                  */
 286                 if (rebuild_end_time > 0) {
 287                         if (ps != NULL) {
 288                                 if ((ps->pss_state == DSS_FINISHED &&
 289                                     ps->pss_func == POOL_SCAN_SCRUB &&
 290                                     rebuild_end_time > ps->pss_end_time) ||
 291                                     ps->pss_state == DSS_NONE)
 292                                         return (ZPOOL_STATUS_REBUILD_SCRUB);
 293                         } else {
 294                                 return (ZPOOL_STATUS_REBUILD_SCRUB);
 295                         }
 296                 }
 297         }
 298
 299         /*
 300          * The multihost property is set and the pool may be active.
 301          */
 302         if (vs->vs_state == VDEV_STATE_CANT_OPEN &&
 303             vs->vs_aux == VDEV_AUX_ACTIVE) {
 304                 mmp_state_t mmp_state;
 305                 nvlist_t *nvinfo;
 306
 307                 nvinfo = fnvlist_lookup_nvlist(config, ZPOOL_CONFIG_LOAD_INFO);
 308                 mmp_state = fnvlist_lookup_uint64(nvinfo,
 309                     ZPOOL_CONFIG_MMP_STATE);
 310
 311                 if (mmp_state == MMP_STATE_ACTIVE)
 312                         return (ZPOOL_STATUS_HOSTID_ACTIVE);
 313                 else if (mmp_state == MMP_STATE_NO_HOSTID)
 314                         return (ZPOOL_STATUS_HOSTID_REQUIRED);
 315                 else
 316                         return (ZPOOL_STATUS_HOSTID_MISMATCH);
 317         }
 318
 319         /*
 320          * Pool last accessed by another system.
 321          */
 322         (void) nvlist_lookup_uint64(config, ZPOOL_CONFIG_HOSTID, &hostid);
 323         if (hostid != 0 && (unsigned long)hostid != system_hostid &&
 324             stateval == POOL_STATE_ACTIVE)
 325                 return (ZPOOL_STATUS_HOSTID_MISMATCH);
 326
 327         /*
 328          * Newer on-disk version.
 329          */
 330         if (vs->vs_state == VDEV_STATE_CANT_OPEN &&
 331             vs->vs_aux == VDEV_AUX_VERSION_NEWER)
 332                 return (ZPOOL_STATUS_VERSION_NEWER);
 333
 334         /*
 335          * Unsupported feature(s).
 336          */
 337         if (vs->vs_state == VDEV_STATE_CANT_OPEN &&
 338             vs->vs_aux == VDEV_AUX_UNSUP_FEAT) {
 339                 nvlist_t *nvinfo;
 340
 341                 verify(nvlist_lookup_nvlist(config, ZPOOL_CONFIG_LOAD_INFO,
 342                     &nvinfo) == 0);
 343                 if (nvlist_exists(nvinfo, ZPOOL_CONFIG_CAN_RDONLY))
 344                         return (ZPOOL_STATUS_UNSUP_FEAT_WRITE);
 345                 return (ZPOOL_STATUS_UNSUP_FEAT_READ);
 346         }
 347
 348         /*
 349          * Check that the config is complete.
 350          */
 351         if (vs->vs_state == VDEV_STATE_CANT_OPEN &&
 352             vs->vs_aux == VDEV_AUX_BAD_GUID_SUM)
 353                 return (ZPOOL_STATUS_BAD_GUID_SUM);
 354
 355         /*
 356          * Check whether the pool has suspended.
 357          */
 358         if (nvlist_lookup_uint64(config, ZPOOL_CONFIG_SUSPENDED,
 359             &suspended) == 0) {
 360                 uint64_t reason;
 361
 362                 if (nvlist_lookup_uint64(config, ZPOOL_CONFIG_SUSPENDED_REASON,
 363                     &reason) == 0 && reason == ZIO_SUSPEND_MMP)
 364                         return (ZPOOL_STATUS_IO_FAILURE_MMP);
 365
 366                 if (suspended == ZIO_FAILURE_MODE_CONTINUE)
 367                         return (ZPOOL_STATUS_IO_FAILURE_CONTINUE);
 368                 return (ZPOOL_STATUS_IO_FAILURE_WAIT);
 369         }
 370
 371         /*
 372          * Could not read a log.
 373          */
 374         if (vs->vs_state == VDEV_STATE_CANT_OPEN &&
 375             vs->vs_aux == VDEV_AUX_BAD_LOG) {
 376                 return (ZPOOL_STATUS_BAD_LOG);
 377         }
 378
 379         /*
 380          * Bad devices in non-replicated config.
 381          */
 382         if (vs->vs_state == VDEV_STATE_CANT_OPEN &&
 383             find_vdev_problem(nvroot, vdev_faulted, B_TRUE))
 384                 return (ZPOOL_STATUS_FAULTED_DEV_NR);
 385
 386         if (vs->vs_state == VDEV_STATE_CANT_OPEN &&
 387             find_vdev_problem(nvroot, vdev_missing, B_TRUE))
 388                 return (ZPOOL_STATUS_MISSING_DEV_NR);
 389
 390         if (vs->vs_state == VDEV_STATE_CANT_OPEN &&
 391             find_vdev_problem(nvroot, vdev_broken, B_TRUE))
 392                 return (ZPOOL_STATUS_CORRUPT_LABEL_NR);
 393
 394         /*
 395          * Corrupted pool metadata
 396          */
 397         if (vs->vs_state == VDEV_STATE_CANT_OPEN &&
 398             vs->vs_aux == VDEV_AUX_CORRUPT_DATA)
 399                 return (ZPOOL_STATUS_CORRUPT_POOL);
 400
 401         /*
 402          * Persistent data errors.
 403          */
 404         if (!isimport) {
 405                 if (nvlist_lookup_uint64(config, ZPOOL_CONFIG_ERRCOUNT,
 406                     &nerr) == 0 && nerr != 0)
 407                         return (ZPOOL_STATUS_CORRUPT_DATA);
 408         }
 409
 410         /*
 411          * Missing devices in a replicated config.
 412          */
 413         if (find_vdev_problem(nvroot, vdev_faulted, B_TRUE))
 414                 return (ZPOOL_STATUS_FAULTED_DEV_R);
 415         if (find_vdev_problem(nvroot, vdev_missing, B_TRUE))
 416                 return (ZPOOL_STATUS_MISSING_DEV_R);
 417         if (find_vdev_problem(nvroot, vdev_broken, B_TRUE))
 418                 return (ZPOOL_STATUS_CORRUPT_LABEL_R);
 419
 420         /*
 421          * Devices with errors
 422          */
 423         if (!isimport && find_vdev_problem(nvroot, vdev_errors, B_TRUE))
 424                 return (ZPOOL_STATUS_FAILING_DEV);
 425
 426         /*
 427          * Offlined devices
 428          */
 429         if (find_vdev_problem(nvroot, vdev_offlined, B_TRUE))
 430                 return (ZPOOL_STATUS_OFFLINE_DEV);
 431
 432         /*
 433          * Removed device
 434          */
 435         if (find_vdev_problem(nvroot, vdev_removed, B_TRUE))
 436                 return (ZPOOL_STATUS_REMOVED_DEV);
 437
 438         /*
 439          * Suboptimal, but usable, ashift configuration.
 440          */
 441         if (find_vdev_problem(nvroot, vdev_non_native_ashift, B_FALSE))
 442                 return (ZPOOL_STATUS_NON_NATIVE_ASHIFT);
 443
 444         /*
 445          * Informational errata available.
 446          */
 447         (void) nvlist_lookup_uint64(config, ZPOOL_CONFIG_ERRATA, &errata);
 448         if (errata) {
 449                 *erratap = errata;
 450                 return (ZPOOL_STATUS_ERRATA);
 451         }
 452
 453         /*
 454          * Outdated, but usable, version
 455          */
 456         if (SPA_VERSION_IS_SUPPORTED(version) && version != SPA_VERSION)
 457                 return (ZPOOL_STATUS_VERSION_OLDER);
 458
 459         /*
 460          * Usable pool with disabled features
 461          */
 462         if (version >= SPA_VERSION_FEATURES) {
 463                 int i;
 464                 nvlist_t *feat;
 465
 466                 if (isimport) {
 467                         feat = fnvlist_lookup_nvlist(config,
 468                             ZPOOL_CONFIG_LOAD_INFO);
 469                         if (nvlist_exists(feat, ZPOOL_CONFIG_ENABLED_FEAT))
 470                                 feat = fnvlist_lookup_nvlist(feat,
 471                                     ZPOOL_CONFIG_ENABLED_FEAT);
 472                 } else {
 473                         feat = fnvlist_lookup_nvlist(config,
 474                             ZPOOL_CONFIG_FEATURE_STATS);
 475                 }
 476
 477                 /* check against all features, or limited set? */
 478                 boolean_t pool_features[SPA_FEATURES];
 479
 480                 if (zpool_load_compat(compat, pool_features, NULL, NULL) !=
 481                     ZPOOL_COMPATIBILITY_OK)
 482                         return (ZPOOL_STATUS_COMPATIBILITY_ERR);
 483                 for (i = 0; i < SPA_FEATURES; i++) {
 484                         zfeature_info_t *fi = &spa_feature_table[i];
 485                         if (!fi->fi_zfs_mod_supported)
 486                                 continue;
 487                         if (pool_features[i] &&
 488                             !nvlist_exists(feat, fi->fi_guid))
 489                                 return (ZPOOL_STATUS_FEAT_DISABLED);
 490                 }
 491         }
 492
 493         return (ZPOOL_STATUS_OK);
 494 }
 495
 496 zpool_status_t
 497 zpool_get_status(zpool_handle_t *zhp, char **msgid, zpool_errata_t *errata)
 498 {
 499         /*
 500          * pass in the desired feature set, as
 501          * it affects check for disabled features
 502          */
 503         char compatibility[ZFS_MAXPROPLEN];
 504         if (zpool_get_prop(zhp, ZPOOL_PROP_COMPATIBILITY, compatibility,
 505             ZFS_MAXPROPLEN, NULL, B_FALSE) != 0)
 506                 compatibility[0] = '\0';
 507
 508         zpool_status_t ret = check_status(zhp->zpool_config, B_FALSE, errata,
 509             compatibility);
 510
 511         if (msgid != NULL) {
 512                 if (ret >= NMSGID)
 513                         *msgid = NULL;
 514                 else
 515                         *msgid = zfs_msgid_table[ret];
 516         }
 517         return (ret);
 518 }
 519
 520 zpool_status_t
 521 zpool_import_status(nvlist_t *config, char **msgid, zpool_errata_t *errata)
 522 {
 523         zpool_status_t ret = check_status(config, B_TRUE, errata, NULL);
 524
 525         if (ret >= NMSGID)
 526                 *msgid = NULL;
 527         else
 528                 *msgid = zfs_msgid_table[ret];
 529
 530         return (ret);
 531 }