4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or https://opensource.org/licenses/CDDL-1.0.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
22 * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved.
23 * Copyright (c) 2012 by Delphix. All rights reserved.
24 * Copyright 2014 Nexenta Systems, Inc. All rights reserved.
25 * Copyright (c) 2016, 2017, Intel Corporation.
26 * Copyright (c) 2017 Open-E, Inc. All Rights Reserved.
30 * ZFS syseventd module.
32 * file origin: openzfs/usr/src/cmd/syseventd/modules/zfs_mod/zfs_mod.c
34 * The purpose of this module is to identify when devices are added to the
35 * system, and appropriately online or replace the affected vdevs.
37 * When a device is added to the system:
39 * 1. Search for any vdevs whose devid matches that of the newly added
42 * 2. If no vdevs are found, then search for any vdevs whose udev path
43 * matches that of the new device.
45 * 3. If no vdevs match by either method, then ignore the event.
47 * 4. Attempt to online the device with a flag to indicate that it should
48 * be unspared when resilvering completes. If this succeeds, then the
49 * same device was inserted and we should continue normally.
51 * 5. If the pool does not have the 'autoreplace' property set, attempt to
52 * online the device again without the unspare flag, which will
53 * generate a FMA fault.
55 * 6. If the pool has the 'autoreplace' property set, and the matching vdev
56 * is a whole disk, then label the new disk and attempt a 'zpool
59 * The module responds to EC_DEV_ADD events. The special ESC_ZFS_VDEV_CHECK
60 * event indicates that a device failed to open during pool load, but the
61 * autoreplace property was set. In this case, we deferred the associated
62 * FMA fault until our module had a chance to process the autoreplace logic.
63 * If the device could not be replaced, then the second online attempt will
64 * trigger the FMA fault that we skipped earlier.
66 * On Linux udev provides a disk insert for both the disk and the partition.
71 #include <libnvpair.h>
80 #include <sys/sunddi.h>
81 #include <sys/sysevent/eventdefs.h>
82 #include <sys/sysevent/dev.h>
83 #include <thread_pool.h>
87 #include "zfs_agents.h"
88 #include "../zed_log.h"
90 #define DEV_BYID_PATH "/dev/disk/by-id/"
91 #define DEV_BYPATH_PATH "/dev/disk/by-path/"
92 #define DEV_BYVDEV_PATH "/dev/disk/by-vdev/"
94 typedef void (*zfs_process_func_t
)(zpool_handle_t
*, nvlist_t
*, boolean_t
);
96 libzfs_handle_t
*g_zfshdl
;
97 list_t g_pool_list
; /* list of unavailable pools at initialization */
98 list_t g_device_list
; /* list of disks with asynchronous label request */
100 boolean_t g_enumeration_done
;
101 pthread_t g_zfs_tid
; /* zfs_enum_pools() thread */
103 typedef struct unavailpool
{
104 zpool_handle_t
*uap_zhp
;
105 list_node_t uap_node
;
108 typedef struct pendingdev
{
109 char pd_physpath
[128];
114 zfs_toplevel_state(zpool_handle_t
*zhp
)
120 verify(nvlist_lookup_nvlist(zpool_get_config(zhp
, NULL
),
121 ZPOOL_CONFIG_VDEV_TREE
, &nvroot
) == 0);
122 verify(nvlist_lookup_uint64_array(nvroot
, ZPOOL_CONFIG_VDEV_STATS
,
123 (uint64_t **)&vs
, &c
) == 0);
124 return (vs
->vs_state
);
128 zfs_unavail_pool(zpool_handle_t
*zhp
, void *data
)
130 zed_log_msg(LOG_INFO
, "zfs_unavail_pool: examining '%s' (state %d)",
131 zpool_get_name(zhp
), (int)zfs_toplevel_state(zhp
));
133 if (zfs_toplevel_state(zhp
) < VDEV_STATE_DEGRADED
) {
135 uap
= malloc(sizeof (unavailpool_t
));
142 list_insert_tail((list_t
*)data
, uap
);
150 * Write an array of strings to the zed log
152 static void lines_to_zed_log_msg(char **lines
, int lines_cnt
)
155 for (i
= 0; i
< lines_cnt
; i
++) {
156 zed_log_msg(LOG_INFO
, "%s", lines
[i
]);
161 * Two stage replace on Linux
162 * since we get disk notifications
163 * we can wait for partitioned disk slice to show up!
165 * First stage tags the disk, initiates async partitioning, and returns
166 * Second stage finds the tag and proceeds to ZFS labeling/replace
168 * disk-add --> label-disk + tag-disk --> partition-add --> zpool_vdev_attach
170 * 1. physical match with no fs, no partition
171 * tag it top, partition disk
173 * 2. physical match again, see partition and tag
178 * The device associated with the given vdev (either by devid or physical path)
179 * has been added to the system. If 'isdisk' is set, then we only attempt a
180 * replacement if it's a whole disk. This also implies that we should label the
183 * First, we attempt to online the device (making sure to undo any spare
184 * operation when finished). If this succeeds, then we're done. If it fails,
185 * and the new state is VDEV_CANT_OPEN, it indicates that the device was opened,
186 * but that the label was not what we expected. If the 'autoreplace' property
187 * is enabled, then we relabel the disk (if specified), and attempt a 'zpool
188 * replace'. If the online is successful, but the new state is something else
189 * (REMOVED or FAULTED), it indicates that we're out of sync or in some sort of
190 * race, and we should avoid attempting to relabel the disk.
192 * Also can arrive here from a ESC_ZFS_VDEV_CHECK event
195 zfs_process_add(zpool_handle_t
*zhp
, nvlist_t
*vdev
, boolean_t labeled
)
198 vdev_state_t newstate
;
199 nvlist_t
*nvroot
, *newvd
;
200 pendingdev_t
*device
;
201 uint64_t wholedisk
= 0ULL;
202 uint64_t offline
= 0ULL, faulted
= 0ULL;
203 uint64_t guid
= 0ULL;
204 uint64_t is_spare
= 0;
205 const char *physpath
= NULL
, *new_devid
= NULL
, *enc_sysfs_path
= NULL
;
206 char rawpath
[PATH_MAX
], fullpath
[PATH_MAX
];
207 char devpath
[PATH_MAX
];
209 int online_flag
= ZFS_ONLINE_CHECKREMOVE
| ZFS_ONLINE_UNSPARE
;
210 boolean_t is_sd
= B_FALSE
;
211 boolean_t is_mpath_wholedisk
= B_FALSE
;
217 if (nvlist_lookup_string(vdev
, ZPOOL_CONFIG_PATH
, &path
) != 0)
220 /* Skip healthy disks */
221 verify(nvlist_lookup_uint64_array(vdev
, ZPOOL_CONFIG_VDEV_STATS
,
222 (uint64_t **)&vs
, &c
) == 0);
223 if (vs
->vs_state
== VDEV_STATE_HEALTHY
) {
224 zed_log_msg(LOG_INFO
, "%s: %s is already healthy, skip it.",
229 (void) nvlist_lookup_string(vdev
, ZPOOL_CONFIG_PHYS_PATH
, &physpath
);
230 (void) nvlist_lookup_string(vdev
, ZPOOL_CONFIG_VDEV_ENC_SYSFS_PATH
,
232 (void) nvlist_lookup_uint64(vdev
, ZPOOL_CONFIG_WHOLE_DISK
, &wholedisk
);
233 (void) nvlist_lookup_uint64(vdev
, ZPOOL_CONFIG_OFFLINE
, &offline
);
234 (void) nvlist_lookup_uint64(vdev
, ZPOOL_CONFIG_FAULTED
, &faulted
);
236 (void) nvlist_lookup_uint64(vdev
, ZPOOL_CONFIG_GUID
, &guid
);
237 (void) nvlist_lookup_uint64(vdev
, ZPOOL_CONFIG_IS_SPARE
, &is_spare
);
242 * We've seen times where a disk won't have a ZPOOL_CONFIG_PHYS_PATH
243 * entry in their config. For example, on this force-faulted disk:
248 * guid: 14309659774640089719
249 * path: '/dev/disk/by-vdev/L28'
253 * com.delphix:vdev_zap_leaf: 1161
255 * aux_state: 'external'
259 * guid: 16002508084177980912
260 * path: '/dev/disk/by-vdev/L29'
261 * devid: 'dm-uuid-mpath-35000c500a61d68a3'
263 * vdev_enc_sysfs_path: '/sys/class/enclosure/0:0:1:0/SLOT 30 32'
267 * com.delphix:vdev_zap_leaf: 131
269 * If the disk's path is a /dev/disk/by-vdev/ path, then we can infer
270 * the ZPOOL_CONFIG_PHYS_PATH from the by-vdev disk name.
272 if (physpath
== NULL
&& path
!= NULL
) {
273 /* If path begins with "/dev/disk/by-vdev/" ... */
274 if (strncmp(path
, DEV_BYVDEV_PATH
,
275 strlen(DEV_BYVDEV_PATH
)) == 0) {
276 /* Set physpath to the char after "/dev/disk/by-vdev" */
277 physpath
= &path
[strlen(DEV_BYVDEV_PATH
)];
282 * We don't want to autoreplace offlined disks. However, we do want to
283 * replace force-faulted disks (`zpool offline -f`). Force-faulted
284 * disks have both offline=1 and faulted=1 in the nvlist.
286 if (offline
&& !faulted
) {
287 zed_log_msg(LOG_INFO
, "%s: %s is offline, skip autoreplace",
292 is_mpath_wholedisk
= is_mpath_whole_disk(path
);
293 zed_log_msg(LOG_INFO
, "zfs_process_add: pool '%s' vdev '%s', phys '%s'"
294 " %s blank disk, %s mpath blank disk, %s labeled, enc sysfs '%s', "
296 zpool_get_name(zhp
), path
,
297 physpath
? physpath
: "NULL",
298 wholedisk
? "is" : "not",
299 is_mpath_wholedisk
? "is" : "not",
300 labeled
? "is" : "not",
302 (long long unsigned int)guid
);
305 * The VDEV guid is preferred for identification (gets passed in path)
308 (void) snprintf(fullpath
, sizeof (fullpath
), "%llu",
309 (long long unsigned int)guid
);
312 * otherwise use path sans partition suffix for whole disks
314 (void) strlcpy(fullpath
, path
, sizeof (fullpath
));
316 char *spath
= zfs_strip_partition(fullpath
);
318 zed_log_msg(LOG_INFO
, "%s: Can't alloc",
323 (void) strlcpy(fullpath
, spath
, sizeof (fullpath
));
329 online_flag
|= ZFS_ONLINE_SPARE
;
332 * Attempt to online the device.
334 if (zpool_vdev_online(zhp
, fullpath
, online_flag
, &newstate
) == 0 &&
335 (newstate
== VDEV_STATE_HEALTHY
||
336 newstate
== VDEV_STATE_DEGRADED
)) {
337 zed_log_msg(LOG_INFO
,
338 " zpool_vdev_online: vdev '%s' ('%s') is "
339 "%s", fullpath
, physpath
, (newstate
== VDEV_STATE_HEALTHY
) ?
340 "HEALTHY" : "DEGRADED");
345 * vdev_id alias rule for using scsi_debug devices (FMA automated
348 if (physpath
!= NULL
&& strcmp("scsidebug", physpath
) == 0)
352 * If the pool doesn't have the autoreplace property set, then use
353 * vdev online to trigger a FMA fault by posting an ereport.
355 if (!zpool_get_prop_int(zhp
, ZPOOL_PROP_AUTOREPLACE
, NULL
) ||
356 !(wholedisk
|| is_mpath_wholedisk
) || (physpath
== NULL
)) {
357 (void) zpool_vdev_online(zhp
, fullpath
, ZFS_ONLINE_FORCEFAULT
,
359 zed_log_msg(LOG_INFO
, "Pool's autoreplace is not enabled or "
360 "not a blank disk for '%s' ('%s')", fullpath
,
366 * Convert physical path into its current device node. Rawpath
367 * needs to be /dev/disk/by-vdev for a scsi_debug device since
368 * /dev/disk/by-path will not be present.
370 (void) snprintf(rawpath
, sizeof (rawpath
), "%s%s",
371 is_sd
? DEV_BYVDEV_PATH
: DEV_BYPATH_PATH
, physpath
);
373 if (realpath(rawpath
, devpath
) == NULL
&& !is_mpath_wholedisk
) {
374 zed_log_msg(LOG_INFO
, " realpath: %s failed (%s)",
375 rawpath
, strerror(errno
));
377 (void) zpool_vdev_online(zhp
, fullpath
, ZFS_ONLINE_FORCEFAULT
,
380 zed_log_msg(LOG_INFO
, " zpool_vdev_online: %s FORCEFAULT (%s)",
381 fullpath
, libzfs_error_description(g_zfshdl
));
385 /* Only autoreplace bad disks */
386 if ((vs
->vs_state
!= VDEV_STATE_DEGRADED
) &&
387 (vs
->vs_state
!= VDEV_STATE_FAULTED
) &&
388 (vs
->vs_state
!= VDEV_STATE_REMOVED
) &&
389 (vs
->vs_state
!= VDEV_STATE_CANT_OPEN
)) {
390 zed_log_msg(LOG_INFO
, " not autoreplacing since disk isn't in "
391 "a bad state (currently %llu)", vs
->vs_state
);
395 nvlist_lookup_string(vdev
, "new_devid", &new_devid
);
397 if (is_mpath_wholedisk
) {
398 /* Don't label device mapper or multipath disks. */
399 zed_log_msg(LOG_INFO
,
400 " it's a multipath wholedisk, don't label");
401 if (zpool_prepare_disk(zhp
, vdev
, "autoreplace", &lines
,
403 zed_log_msg(LOG_INFO
,
404 " zpool_prepare_disk: could not "
405 "prepare '%s' (%s)", fullpath
,
406 libzfs_error_description(g_zfshdl
));
408 zed_log_msg(LOG_INFO
,
409 " zfs_prepare_disk output:");
410 lines_to_zed_log_msg(lines
, lines_cnt
);
412 libzfs_free_str_array(lines
, lines_cnt
);
415 } else if (!labeled
) {
417 * we're auto-replacing a raw disk, so label it first
422 * If this is a request to label a whole disk, then attempt to
423 * write out the label. Before we can label the disk, we need
424 * to map the physical string that was matched on to the under
427 * If any part of this process fails, then do a force online
428 * to trigger a ZFS fault for the device (and any hot spare
431 leafname
= strrchr(devpath
, '/') + 1;
434 * If this is a request to label a whole disk, then attempt to
435 * write out the label.
437 if (zpool_prepare_and_label_disk(g_zfshdl
, zhp
, leafname
,
438 vdev
, "autoreplace", &lines
, &lines_cnt
) != 0) {
439 zed_log_msg(LOG_INFO
,
440 " zpool_prepare_and_label_disk: could not "
441 "label '%s' (%s)", leafname
,
442 libzfs_error_description(g_zfshdl
));
444 zed_log_msg(LOG_INFO
,
445 " zfs_prepare_disk output:");
446 lines_to_zed_log_msg(lines
, lines_cnt
);
448 libzfs_free_str_array(lines
, lines_cnt
);
450 (void) zpool_vdev_online(zhp
, fullpath
,
451 ZFS_ONLINE_FORCEFAULT
, &newstate
);
456 * The disk labeling is asynchronous on Linux. Just record
457 * this label request and return as there will be another
458 * disk add event for the partition after the labeling is
461 device
= malloc(sizeof (pendingdev_t
));
462 if (device
== NULL
) {
467 (void) strlcpy(device
->pd_physpath
, physpath
,
468 sizeof (device
->pd_physpath
));
469 list_insert_tail(&g_device_list
, device
);
471 zed_log_msg(LOG_INFO
, " zpool_label_disk: async '%s' (%llu)",
472 leafname
, (u_longlong_t
)guid
);
474 return; /* resumes at EC_DEV_ADD.ESC_DISK for partition */
476 } else /* labeled */ {
477 boolean_t found
= B_FALSE
;
479 * match up with request above to label the disk
481 for (device
= list_head(&g_device_list
); device
!= NULL
;
482 device
= list_next(&g_device_list
, device
)) {
483 if (strcmp(physpath
, device
->pd_physpath
) == 0) {
484 list_remove(&g_device_list
, device
);
489 zed_log_msg(LOG_INFO
, "zpool_label_disk: %s != %s",
490 physpath
, device
->pd_physpath
);
493 /* unexpected partition slice encountered */
494 zed_log_msg(LOG_INFO
, "labeled disk %s unexpected here",
496 (void) zpool_vdev_online(zhp
, fullpath
,
497 ZFS_ONLINE_FORCEFAULT
, &newstate
);
501 zed_log_msg(LOG_INFO
, " zpool_label_disk: resume '%s' (%llu)",
502 physpath
, (u_longlong_t
)guid
);
504 (void) snprintf(devpath
, sizeof (devpath
), "%s%s",
505 DEV_BYID_PATH
, new_devid
);
508 libzfs_free_str_array(lines
, lines_cnt
);
511 * Construct the root vdev to pass to zpool_vdev_attach(). While adding
512 * the entire vdev structure is harmless, we construct a reduced set of
513 * path/physpath/wholedisk to keep it simple.
515 if (nvlist_alloc(&nvroot
, NV_UNIQUE_NAME
, 0) != 0) {
516 zed_log_msg(LOG_WARNING
, "zfs_mod: nvlist_alloc out of memory");
519 if (nvlist_alloc(&newvd
, NV_UNIQUE_NAME
, 0) != 0) {
520 zed_log_msg(LOG_WARNING
, "zfs_mod: nvlist_alloc out of memory");
525 if (nvlist_add_string(newvd
, ZPOOL_CONFIG_TYPE
, VDEV_TYPE_DISK
) != 0 ||
526 nvlist_add_string(newvd
, ZPOOL_CONFIG_PATH
, path
) != 0 ||
527 nvlist_add_string(newvd
, ZPOOL_CONFIG_DEVID
, new_devid
) != 0 ||
528 (physpath
!= NULL
&& nvlist_add_string(newvd
,
529 ZPOOL_CONFIG_PHYS_PATH
, physpath
) != 0) ||
530 (enc_sysfs_path
!= NULL
&& nvlist_add_string(newvd
,
531 ZPOOL_CONFIG_VDEV_ENC_SYSFS_PATH
, enc_sysfs_path
) != 0) ||
532 nvlist_add_uint64(newvd
, ZPOOL_CONFIG_WHOLE_DISK
, wholedisk
) != 0 ||
533 nvlist_add_string(nvroot
, ZPOOL_CONFIG_TYPE
, VDEV_TYPE_ROOT
) != 0 ||
534 nvlist_add_nvlist_array(nvroot
, ZPOOL_CONFIG_CHILDREN
,
535 (const nvlist_t
**)&newvd
, 1) != 0) {
536 zed_log_msg(LOG_WARNING
, "zfs_mod: unable to add nvlist pairs");
545 * Wait for udev to verify the links exist, then auto-replace
546 * the leaf disk at same physical location.
548 if (zpool_label_disk_wait(path
, 3000) != 0) {
549 zed_log_msg(LOG_WARNING
, "zfs_mod: expected replacement "
550 "disk %s is missing", path
);
556 * Prefer sequential resilvering when supported (mirrors and dRAID),
557 * otherwise fallback to a traditional healing resilver.
559 ret
= zpool_vdev_attach(zhp
, fullpath
, path
, nvroot
, B_TRUE
, B_TRUE
);
561 ret
= zpool_vdev_attach(zhp
, fullpath
, path
, nvroot
,
565 zed_log_msg(LOG_INFO
, " zpool_vdev_replace: %s with %s (%s)",
566 fullpath
, path
, (ret
== 0) ? "no errors" :
567 libzfs_error_description(g_zfshdl
));
573 * Utility functions to find a vdev matching given criteria.
575 typedef struct dev_data
{
576 const char *dd_compare
;
578 zfs_process_func_t dd_func
;
580 boolean_t dd_islabeled
;
581 uint64_t dd_pool_guid
;
582 uint64_t dd_vdev_guid
;
583 uint64_t dd_new_vdev_guid
;
584 const char *dd_new_devid
;
585 uint64_t dd_num_spares
;
589 zfs_iter_vdev(zpool_handle_t
*zhp
, nvlist_t
*nvl
, void *data
)
591 dev_data_t
*dp
= data
;
592 const char *path
= NULL
;
596 uint64_t isspare
= 0;
599 * First iterate over any children.
601 if (nvlist_lookup_nvlist_array(nvl
, ZPOOL_CONFIG_CHILDREN
,
602 &child
, &children
) == 0) {
603 for (c
= 0; c
< children
; c
++)
604 zfs_iter_vdev(zhp
, child
[c
], data
);
608 * Iterate over any spares and cache devices
610 if (nvlist_lookup_nvlist_array(nvl
, ZPOOL_CONFIG_SPARES
,
611 &child
, &children
) == 0) {
612 for (c
= 0; c
< children
; c
++)
613 zfs_iter_vdev(zhp
, child
[c
], data
);
615 if (nvlist_lookup_nvlist_array(nvl
, ZPOOL_CONFIG_L2CACHE
,
616 &child
, &children
) == 0) {
617 for (c
= 0; c
< children
; c
++)
618 zfs_iter_vdev(zhp
, child
[c
], data
);
621 /* once a vdev was matched and processed there is nothing left to do */
622 if (dp
->dd_found
&& dp
->dd_num_spares
== 0)
624 (void) nvlist_lookup_uint64(nvl
, ZPOOL_CONFIG_GUID
, &guid
);
627 * Match by GUID if available otherwise fallback to devid or physical
629 if (dp
->dd_vdev_guid
!= 0) {
630 if (guid
!= dp
->dd_vdev_guid
)
632 zed_log_msg(LOG_INFO
, " zfs_iter_vdev: matched on %llu", guid
);
633 dp
->dd_found
= B_TRUE
;
635 } else if (dp
->dd_compare
!= NULL
) {
637 * NOTE: On Linux there is an event for partition, so unlike
638 * illumos, substring matching is not required to accommodate
639 * the partition suffix. An exact match will be present in
640 * the dp->dd_compare value.
641 * If the attached disk already contains a vdev GUID, it means
642 * the disk is not clean. In such a scenario, the physical path
643 * would be a match that makes the disk faulted when trying to
644 * online it. So, we would only want to proceed if either GUID
645 * matches with the last attached disk or the disk is in clean
648 if (nvlist_lookup_string(nvl
, dp
->dd_prop
, &path
) != 0 ||
649 strcmp(dp
->dd_compare
, path
) != 0) {
652 if (dp
->dd_new_vdev_guid
!= 0 && dp
->dd_new_vdev_guid
!= guid
) {
653 zed_log_msg(LOG_INFO
, " %s: no match (GUID:%llu"
654 " != vdev GUID:%llu)", __func__
,
655 dp
->dd_new_vdev_guid
, guid
);
659 zed_log_msg(LOG_INFO
, " zfs_iter_vdev: matched %s on %s",
661 dp
->dd_found
= B_TRUE
;
663 /* pass the new devid for use by replacing code */
664 if (dp
->dd_new_devid
!= NULL
) {
665 (void) nvlist_add_string(nvl
, "new_devid",
670 if (dp
->dd_found
== B_TRUE
&& nvlist_lookup_uint64(nvl
,
671 ZPOOL_CONFIG_IS_SPARE
, &isspare
) == 0 && isspare
)
674 (dp
->dd_func
)(zhp
, nvl
, dp
->dd_islabeled
);
678 zfs_enable_ds(void *arg
)
680 unavailpool_t
*pool
= (unavailpool_t
*)arg
;
682 (void) zpool_enable_datasets(pool
->uap_zhp
, NULL
, 0);
683 zpool_close(pool
->uap_zhp
);
688 zfs_iter_pool(zpool_handle_t
*zhp
, void *data
)
690 nvlist_t
*config
, *nvl
;
691 dev_data_t
*dp
= data
;
695 zed_log_msg(LOG_INFO
, "zfs_iter_pool: evaluating vdevs on %s (by %s)",
696 zpool_get_name(zhp
), dp
->dd_vdev_guid
? "GUID" : dp
->dd_prop
);
699 * For each vdev in this pool, look for a match to apply dd_func
701 if ((config
= zpool_get_config(zhp
, NULL
)) != NULL
) {
702 if (dp
->dd_pool_guid
== 0 ||
703 (nvlist_lookup_uint64(config
, ZPOOL_CONFIG_POOL_GUID
,
704 &pool_guid
) == 0 && pool_guid
== dp
->dd_pool_guid
)) {
705 (void) nvlist_lookup_nvlist(config
,
706 ZPOOL_CONFIG_VDEV_TREE
, &nvl
);
707 zfs_iter_vdev(zhp
, nvl
, data
);
710 zed_log_msg(LOG_INFO
, "%s: no config\n", __func__
);
714 * if this pool was originally unavailable,
715 * then enable its datasets asynchronously
717 if (g_enumeration_done
) {
718 for (pool
= list_head(&g_pool_list
); pool
!= NULL
;
719 pool
= list_next(&g_pool_list
, pool
)) {
721 if (strcmp(zpool_get_name(zhp
),
722 zpool_get_name(pool
->uap_zhp
)))
724 if (zfs_toplevel_state(zhp
) >= VDEV_STATE_DEGRADED
) {
725 list_remove(&g_pool_list
, pool
);
726 (void) tpool_dispatch(g_tpool
, zfs_enable_ds
,
735 /* cease iteration after a match */
736 return (dp
->dd_found
&& dp
->dd_num_spares
== 0);
740 * Given a physical device location, iterate over all
741 * (pool, vdev) pairs which correspond to that location.
744 devphys_iter(const char *physical
, const char *devid
, zfs_process_func_t func
,
745 boolean_t is_slice
, uint64_t new_vdev_guid
)
747 dev_data_t data
= { 0 };
749 data
.dd_compare
= physical
;
751 data
.dd_prop
= ZPOOL_CONFIG_PHYS_PATH
;
752 data
.dd_found
= B_FALSE
;
753 data
.dd_islabeled
= is_slice
;
754 data
.dd_new_devid
= devid
; /* used by auto replace code */
755 data
.dd_new_vdev_guid
= new_vdev_guid
;
757 (void) zpool_iter(g_zfshdl
, zfs_iter_pool
, &data
);
759 return (data
.dd_found
);
763 * Given a device identifier, find any vdevs with a matching by-vdev
764 * path. Normally we shouldn't need this as the comparison would be
765 * made earlier in the devphys_iter(). For example, if we were replacing
766 * /dev/disk/by-vdev/L28, normally devphys_iter() would match the
767 * ZPOOL_CONFIG_PHYS_PATH of "L28" from the old disk config to "L28"
768 * of the new disk config. However, we've seen cases where
769 * ZPOOL_CONFIG_PHYS_PATH was not in the config for the old disk. Here's
770 * an example of a real 2-disk mirror pool where one disk was force
773 * com.delphix:vdev_zap_top: 129
777 * guid: 14309659774640089719
778 * path: '/dev/disk/by-vdev/L28'
782 * com.delphix:vdev_zap_leaf: 1161
784 * aux_state: 'external'
788 * guid: 16002508084177980912
789 * path: '/dev/disk/by-vdev/L29'
790 * devid: 'dm-uuid-mpath-35000c500a61d68a3'
792 * vdev_enc_sysfs_path: '/sys/class/enclosure/0:0:1:0/SLOT 30 32'
796 * com.delphix:vdev_zap_leaf: 131
798 * So in the case above, the only thing we could compare is the path.
800 * We can do this because we assume by-vdev paths are authoritative as physical
801 * paths. We could not assume this for normal paths like /dev/sda since the
802 * physical location /dev/sda points to could change over time.
805 by_vdev_path_iter(const char *by_vdev_path
, const char *devid
,
806 zfs_process_func_t func
, boolean_t is_slice
)
808 dev_data_t data
= { 0 };
810 data
.dd_compare
= by_vdev_path
;
812 data
.dd_prop
= ZPOOL_CONFIG_PATH
;
813 data
.dd_found
= B_FALSE
;
814 data
.dd_islabeled
= is_slice
;
815 data
.dd_new_devid
= devid
;
817 if (strncmp(by_vdev_path
, DEV_BYVDEV_PATH
,
818 strlen(DEV_BYVDEV_PATH
)) != 0) {
819 /* by_vdev_path doesn't start with "/dev/disk/by-vdev/" */
823 (void) zpool_iter(g_zfshdl
, zfs_iter_pool
, &data
);
825 return (data
.dd_found
);
829 * Given a device identifier, find any vdevs with a matching devid.
830 * On Linux we can match devid directly which is always a whole disk.
833 devid_iter(const char *devid
, zfs_process_func_t func
, boolean_t is_slice
)
835 dev_data_t data
= { 0 };
837 data
.dd_compare
= devid
;
839 data
.dd_prop
= ZPOOL_CONFIG_DEVID
;
840 data
.dd_found
= B_FALSE
;
841 data
.dd_islabeled
= is_slice
;
842 data
.dd_new_devid
= devid
;
844 (void) zpool_iter(g_zfshdl
, zfs_iter_pool
, &data
);
846 return (data
.dd_found
);
850 * Given a device guid, find any vdevs with a matching guid.
853 guid_iter(uint64_t pool_guid
, uint64_t vdev_guid
, const char *devid
,
854 zfs_process_func_t func
, boolean_t is_slice
)
856 dev_data_t data
= { 0 };
859 data
.dd_found
= B_FALSE
;
860 data
.dd_pool_guid
= pool_guid
;
861 data
.dd_vdev_guid
= vdev_guid
;
862 data
.dd_islabeled
= is_slice
;
863 data
.dd_new_devid
= devid
;
865 (void) zpool_iter(g_zfshdl
, zfs_iter_pool
, &data
);
867 return (data
.dd_found
);
871 * Handle a EC_DEV_ADD.ESC_DISK event.
874 * Expects: DEV_PHYS_PATH string in schema
875 * Matches: vdev's ZPOOL_CONFIG_PHYS_PATH or ZPOOL_CONFIG_DEVID
877 * path: '/dev/dsk/c0t1d0s0' (persistent)
878 * devid: 'id1,sd@SATA_____Hitachi_HDS72101______JP2940HZ3H74MC/a'
879 * phys_path: '/pci@0,0/pci103c,1609@11/disk@1,0:a'
882 * provides: DEV_PHYS_PATH and DEV_IDENTIFIER strings in schema
883 * Matches: vdev's ZPOOL_CONFIG_PHYS_PATH or ZPOOL_CONFIG_DEVID
885 * path: '/dev/sdc1' (not persistent)
886 * devid: 'ata-SAMSUNG_HD204UI_S2HGJD2Z805891-part1'
887 * phys_path: 'pci-0000:04:00.0-sas-0x4433221106000000-lun-0'
890 zfs_deliver_add(nvlist_t
*nvl
)
892 const char *devpath
= NULL
, *devid
= NULL
;
893 uint64_t pool_guid
= 0, vdev_guid
= 0;
897 * Expecting a devid string and an optional physical location and guid
899 if (nvlist_lookup_string(nvl
, DEV_IDENTIFIER
, &devid
) != 0) {
900 zed_log_msg(LOG_INFO
, "%s: no dev identifier\n", __func__
);
904 (void) nvlist_lookup_string(nvl
, DEV_PHYS_PATH
, &devpath
);
905 (void) nvlist_lookup_uint64(nvl
, ZFS_EV_POOL_GUID
, &pool_guid
);
906 (void) nvlist_lookup_uint64(nvl
, ZFS_EV_VDEV_GUID
, &vdev_guid
);
908 is_slice
= (nvlist_lookup_boolean(nvl
, DEV_IS_PART
) == 0);
910 zed_log_msg(LOG_INFO
, "zfs_deliver_add: adding %s (%s) (is_slice %d)",
911 devid
, devpath
? devpath
: "NULL", is_slice
);
914 * Iterate over all vdevs looking for a match in the following order:
915 * 1. ZPOOL_CONFIG_DEVID (identifies the unique disk)
916 * 2. ZPOOL_CONFIG_PHYS_PATH (identifies disk physical location).
917 * 3. ZPOOL_CONFIG_GUID (identifies unique vdev).
918 * 4. ZPOOL_CONFIG_PATH for /dev/disk/by-vdev devices only (since
919 * by-vdev paths represent physical paths).
921 if (devid_iter(devid
, zfs_process_add
, is_slice
))
923 if (devpath
!= NULL
&& devphys_iter(devpath
, devid
, zfs_process_add
,
924 is_slice
, vdev_guid
))
927 (void) guid_iter(pool_guid
, vdev_guid
, devid
, zfs_process_add
,
930 if (devpath
!= NULL
) {
931 /* Can we match a /dev/disk/by-vdev/ path? */
932 char by_vdev_path
[MAXPATHLEN
];
933 snprintf(by_vdev_path
, sizeof (by_vdev_path
),
934 "/dev/disk/by-vdev/%s", devpath
);
935 if (by_vdev_path_iter(by_vdev_path
, devid
, zfs_process_add
,
944 * Called when we receive a VDEV_CHECK event, which indicates a device could not
945 * be opened during initial pool open, but the autoreplace property was set on
946 * the pool. In this case, we treat it as if it were an add event.
949 zfs_deliver_check(nvlist_t
*nvl
)
951 dev_data_t data
= { 0 };
953 if (nvlist_lookup_uint64(nvl
, ZFS_EV_POOL_GUID
,
954 &data
.dd_pool_guid
) != 0 ||
955 nvlist_lookup_uint64(nvl
, ZFS_EV_VDEV_GUID
,
956 &data
.dd_vdev_guid
) != 0 ||
957 data
.dd_vdev_guid
== 0)
960 zed_log_msg(LOG_INFO
, "zfs_deliver_check: pool '%llu', vdev %llu",
961 data
.dd_pool_guid
, data
.dd_vdev_guid
);
963 data
.dd_func
= zfs_process_add
;
965 (void) zpool_iter(g_zfshdl
, zfs_iter_pool
, &data
);
971 * Given a path to a vdev, lookup the vdev's physical size from its
974 * Returns the vdev's physical size in bytes on success, 0 on error.
977 vdev_size_from_config(zpool_handle_t
*zhp
, const char *vdev_path
)
979 nvlist_t
*nvl
= NULL
;
980 boolean_t avail_spare
, l2cache
, log
;
981 vdev_stat_t
*vs
= NULL
;
984 nvl
= zpool_find_vdev(zhp
, vdev_path
, &avail_spare
, &l2cache
, &log
);
988 verify(nvlist_lookup_uint64_array(nvl
, ZPOOL_CONFIG_VDEV_STATS
,
989 (uint64_t **)&vs
, &c
) == 0);
991 zed_log_msg(LOG_INFO
, "%s: no nvlist for '%s'", __func__
,
996 return (vs
->vs_pspace
);
1000 * Given a path to a vdev, lookup if the vdev is a "whole disk" in the
1001 * config nvlist. "whole disk" means that ZFS was passed a whole disk
1002 * at pool creation time, which it partitioned up and has full control over.
1003 * Thus a partition with wholedisk=1 set tells us that zfs created the
1004 * partition at creation time. A partition without whole disk set would have
1005 * been created by externally (like with fdisk) and passed to ZFS.
1007 * Returns the whole disk value (either 0 or 1).
1010 vdev_whole_disk_from_config(zpool_handle_t
*zhp
, const char *vdev_path
)
1012 nvlist_t
*nvl
= NULL
;
1013 boolean_t avail_spare
, l2cache
, log
;
1014 uint64_t wholedisk
= 0;
1016 nvl
= zpool_find_vdev(zhp
, vdev_path
, &avail_spare
, &l2cache
, &log
);
1020 (void) nvlist_lookup_uint64(nvl
, ZPOOL_CONFIG_WHOLE_DISK
, &wholedisk
);
1026 * If the device size grew more than 1% then return true.
1028 #define DEVICE_GREW(oldsize, newsize) \
1029 ((newsize > oldsize) && \
1030 ((newsize / (newsize - oldsize)) <= 100))
1033 zfsdle_vdev_online(zpool_handle_t
*zhp
, void *data
)
1035 boolean_t avail_spare
, l2cache
;
1036 nvlist_t
*udev_nvl
= data
;
1040 const char *tmp_devname
;
1041 char devname
[MAXPATHLEN
] = "";
1044 if (nvlist_lookup_uint64(udev_nvl
, ZFS_EV_VDEV_GUID
, &guid
) == 0) {
1045 sprintf(devname
, "%llu", (u_longlong_t
)guid
);
1046 } else if (nvlist_lookup_string(udev_nvl
, DEV_PHYS_PATH
,
1047 &tmp_devname
) == 0) {
1048 strlcpy(devname
, tmp_devname
, MAXPATHLEN
);
1049 zfs_append_partition(devname
, MAXPATHLEN
);
1051 zed_log_msg(LOG_INFO
, "%s: no guid or physpath", __func__
);
1054 zed_log_msg(LOG_INFO
, "zfsdle_vdev_online: searching for '%s' in '%s'",
1055 devname
, zpool_get_name(zhp
));
1057 if ((tgt
= zpool_find_vdev_by_physpath(zhp
, devname
,
1058 &avail_spare
, &l2cache
, NULL
)) != NULL
) {
1060 char fullpath
[MAXPATHLEN
];
1061 uint64_t wholedisk
= 0;
1063 error
= nvlist_lookup_string(tgt
, ZPOOL_CONFIG_PATH
, &path
);
1069 (void) nvlist_lookup_uint64(tgt
, ZPOOL_CONFIG_WHOLE_DISK
,
1074 path
= strrchr(path
, '/');
1076 tmp
= zfs_strip_partition(path
+ 1);
1086 (void) strlcpy(fullpath
, tmp
, sizeof (fullpath
));
1090 * We need to reopen the pool associated with this
1091 * device so that the kernel can update the size of
1092 * the expanded device. When expanding there is no
1093 * need to restart the scrub from the beginning.
1095 boolean_t scrub_restart
= B_FALSE
;
1096 (void) zpool_reopen_one(zhp
, &scrub_restart
);
1098 (void) strlcpy(fullpath
, path
, sizeof (fullpath
));
1101 if (zpool_get_prop_int(zhp
, ZPOOL_PROP_AUTOEXPAND
, NULL
)) {
1102 vdev_state_t newstate
;
1104 if (zpool_get_state(zhp
) != POOL_STATE_UNAVAIL
) {
1106 * If this disk size has not changed, then
1107 * there's no need to do an autoexpand. To
1108 * check we look at the disk's size in its
1109 * config, and compare it to the disk size
1110 * that udev is reporting.
1112 uint64_t udev_size
= 0, conf_size
= 0,
1113 wholedisk
= 0, udev_parent_size
= 0;
1116 * Get the size of our disk that udev is
1119 if (nvlist_lookup_uint64(udev_nvl
, DEV_SIZE
,
1125 * Get the size of our disk's parent device
1126 * from udev (where sda1's parent is sda).
1128 if (nvlist_lookup_uint64(udev_nvl
,
1129 DEV_PARENT_SIZE
, &udev_parent_size
) != 0) {
1130 udev_parent_size
= 0;
1133 conf_size
= vdev_size_from_config(zhp
,
1136 wholedisk
= vdev_whole_disk_from_config(zhp
,
1140 * Only attempt an autoexpand if the vdev size
1141 * changed. There are two different cases
1145 * If you do a 'zpool create' on a whole disk
1146 * (like /dev/sda), then zfs will create
1147 * partitions on the disk (like /dev/sda1). In
1148 * that case, wholedisk=1 will be set in the
1149 * partition's nvlist config. So zed will need
1150 * to see if your parent device (/dev/sda)
1151 * expanded in size, and if so, then attempt
1155 * If you do a 'zpool create' on an existing
1156 * partition, or a device that doesn't allow
1157 * partitions, then wholedisk=0, and you will
1158 * simply need to check if the device itself
1161 if (DEVICE_GREW(conf_size
, udev_size
) ||
1162 (wholedisk
&& DEVICE_GREW(conf_size
,
1163 udev_parent_size
))) {
1164 error
= zpool_vdev_online(zhp
, fullpath
,
1167 zed_log_msg(LOG_INFO
,
1168 "%s: autoexpanding '%s' from %llu"
1169 " to %llu bytes in pool '%s': %d",
1170 __func__
, fullpath
, conf_size
,
1171 MAX(udev_size
, udev_parent_size
),
1172 zpool_get_name(zhp
), error
);
1184 * This function handles the ESC_DEV_DLE device change event. Use the
1185 * provided vdev guid when looking up a disk or partition, when the guid
1186 * is not present assume the entire disk is owned by ZFS and append the
1187 * expected -part1 partition information then lookup by physical path.
1190 zfs_deliver_dle(nvlist_t
*nvl
)
1192 const char *devname
;
1193 char name
[MAXPATHLEN
];
1196 if (nvlist_lookup_uint64(nvl
, ZFS_EV_VDEV_GUID
, &guid
) == 0) {
1197 sprintf(name
, "%llu", (u_longlong_t
)guid
);
1198 } else if (nvlist_lookup_string(nvl
, DEV_PHYS_PATH
, &devname
) == 0) {
1199 strlcpy(name
, devname
, MAXPATHLEN
);
1200 zfs_append_partition(name
, MAXPATHLEN
);
1202 sprintf(name
, "unknown");
1203 zed_log_msg(LOG_INFO
, "zfs_deliver_dle: no guid or physpath");
1206 if (zpool_iter(g_zfshdl
, zfsdle_vdev_online
, nvl
) != 1) {
1207 zed_log_msg(LOG_INFO
, "zfs_deliver_dle: device '%s' not "
1216 * syseventd daemon module event handler
1218 * Handles syseventd daemon zfs device related events:
1220 * EC_DEV_ADD.ESC_DISK
1221 * EC_DEV_STATUS.ESC_DEV_DLE
1222 * EC_ZFS.ESC_ZFS_VDEV_CHECK
1224 * Note: assumes only one thread active at a time (not thread safe)
1227 zfs_slm_deliver_event(const char *class, const char *subclass
, nvlist_t
*nvl
)
1230 boolean_t is_check
= B_FALSE
, is_dle
= B_FALSE
;
1232 if (strcmp(class, EC_DEV_ADD
) == 0) {
1234 * We're mainly interested in disk additions, but we also listen
1235 * for new loop devices, to allow for simplified testing.
1237 if (strcmp(subclass
, ESC_DISK
) != 0 &&
1238 strcmp(subclass
, ESC_LOFI
) != 0)
1242 } else if (strcmp(class, EC_ZFS
) == 0 &&
1243 strcmp(subclass
, ESC_ZFS_VDEV_CHECK
) == 0) {
1245 * This event signifies that a device failed to open
1246 * during pool load, but the 'autoreplace' property was
1247 * set, so we should pretend it's just been added.
1250 } else if (strcmp(class, EC_DEV_STATUS
) == 0 &&
1251 strcmp(subclass
, ESC_DEV_DLE
) == 0) {
1258 ret
= zfs_deliver_dle(nvl
);
1260 ret
= zfs_deliver_check(nvl
);
1262 ret
= zfs_deliver_add(nvl
);
1268 zfs_enum_pools(void *arg
)
1272 (void) zpool_iter(g_zfshdl
, zfs_unavail_pool
, (void *)&g_pool_list
);
1274 * Linux - instead of using a thread pool, each list entry
1275 * will spawn a thread when an unavailable pool transitions
1276 * to available. zfs_slm_fini will wait for these threads.
1278 g_enumeration_done
= B_TRUE
;
1283 * called from zed daemon at startup
1285 * sent messages from zevents or udev monitor
1287 * For now, each agent has its own libzfs instance
1292 if ((g_zfshdl
= libzfs_init()) == NULL
)
1296 * collect a list of unavailable pools (asynchronously,
1297 * since this can take a while)
1299 list_create(&g_pool_list
, sizeof (struct unavailpool
),
1300 offsetof(struct unavailpool
, uap_node
));
1302 if (pthread_create(&g_zfs_tid
, NULL
, zfs_enum_pools
, NULL
) != 0) {
1303 list_destroy(&g_pool_list
);
1304 libzfs_fini(g_zfshdl
);
1308 pthread_setname_np(g_zfs_tid
, "enum-pools");
1309 list_create(&g_device_list
, sizeof (struct pendingdev
),
1310 offsetof(struct pendingdev
, pd_node
));
1318 unavailpool_t
*pool
;
1319 pendingdev_t
*device
;
1321 /* wait for zfs_enum_pools thread to complete */
1322 (void) pthread_join(g_zfs_tid
, NULL
);
1323 /* destroy the thread pool */
1324 if (g_tpool
!= NULL
) {
1325 tpool_wait(g_tpool
);
1326 tpool_destroy(g_tpool
);
1329 while ((pool
= list_remove_head(&g_pool_list
)) != NULL
) {
1330 zpool_close(pool
->uap_zhp
);
1333 list_destroy(&g_pool_list
);
1335 while ((device
= list_remove_head(&g_device_list
)) != NULL
)
1337 list_destroy(&g_device_list
);
1339 libzfs_fini(g_zfshdl
);
1343 zfs_slm_event(const char *class, const char *subclass
, nvlist_t
*nvl
)
1345 zed_log_msg(LOG_INFO
, "zfs_slm_event: %s.%s", class, subclass
);
1346 (void) zfs_slm_deliver_event(class, subclass
, nvl
);