cmd/zed/agents/fmd_api.c

   1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or https://opensource.org/licenses/CDDL-1.0.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
  23  *
  24  * Copyright (c) 2016, Intel Corporation.
  25  * Copyright (c) 2023, Klara Inc.
  26  */
  27
  28 /*
  29  * This file implements the minimal FMD module API required to support the
  30  * fault logic modules in ZED. This support includes module registration,
  31  * memory allocation, module property accessors, basic case management,
  32  * one-shot timers and SERD engines.
  33  *
  34  * In the ZED runtime, the modules are called from a single thread so no
  35  * locking is required in this emulated FMD environment.
  36  */
  37
  38 #include <sys/types.h>
  39 #include <sys/fm/protocol.h>
  40 #include <uuid/uuid.h>
  41 #include <signal.h>
  42 #include <string.h>
  43 #include <time.h>
  44
  45 #include "fmd_api.h"
  46 #include "fmd_serd.h"
  47
  48 #include "zfs_agents.h"
  49 #include "../zed_log.h"
  50
  51 typedef struct fmd_modstat {
  52         fmd_stat_t      ms_accepted;    /* total events accepted by module */
  53         fmd_stat_t      ms_caseopen;    /* cases currently open */
  54         fmd_stat_t      ms_casesolved;  /* total cases solved by module */
  55         fmd_stat_t      ms_caseclosed;  /* total cases closed by module */
  56 } fmd_modstat_t;
  57
  58 typedef struct fmd_module {
  59         const char      *mod_name;      /* basename of module (ro) */
  60         const fmd_hdl_info_t *mod_info; /* module info registered with handle */
  61         void            *mod_spec;      /* fmd_hdl_get/setspecific data value */
  62         fmd_stat_t      *mod_ustat;     /* module specific custom stats */
  63         uint_t          mod_ustat_cnt;  /* count of ustat stats */
  64         fmd_modstat_t   mod_stats;      /* fmd built-in per-module statistics */
  65         fmd_serd_hash_t mod_serds;      /* hash of serd engs owned by module */
  66         char            *mod_vers;      /* a copy of module version string */
  67 } fmd_module_t;
  68
  69 /*
  70  * ZED has two FMD hardwired module instances
  71  */
  72 fmd_module_t    zfs_retire_module;
  73 fmd_module_t    zfs_diagnosis_module;
  74
  75 /*
  76  * Enable a reasonable set of defaults for libumem debugging on DEBUG builds.
  77  */
  78
  79 #ifdef DEBUG
  80 const char *
  81 _umem_debug_init(void)
  82 {
  83         return ("default,verbose"); /* $UMEM_DEBUG setting */
  84 }
  85
  86 const char *
  87 _umem_logging_init(void)
  88 {
  89         return ("fail,contents"); /* $UMEM_LOGGING setting */
  90 }
  91 #endif
  92
  93 /*
  94  * Register a module with fmd and finish module initialization.
  95  * Returns an integer indicating whether it succeeded (zero) or
  96  * failed (non-zero).
  97  */
  98 int
  99 fmd_hdl_register(fmd_hdl_t *hdl, int version, const fmd_hdl_info_t *mip)
 100 {
 101         (void) version;
 102         fmd_module_t *mp = (fmd_module_t *)hdl;
 103
 104         mp->mod_info = mip;
 105         mp->mod_name = mip->fmdi_desc + 4;      /* drop 'ZFS ' prefix */
 106         mp->mod_spec = NULL;
 107
 108         /* bare minimum module stats */
 109         (void) strcpy(mp->mod_stats.ms_accepted.fmds_name, "fmd.accepted");
 110         (void) strcpy(mp->mod_stats.ms_caseopen.fmds_name, "fmd.caseopen");
 111         (void) strcpy(mp->mod_stats.ms_casesolved.fmds_name, "fmd.casesolved");
 112         (void) strcpy(mp->mod_stats.ms_caseclosed.fmds_name, "fmd.caseclosed");
 113
 114         fmd_serd_hash_create(&mp->mod_serds);
 115
 116         fmd_hdl_debug(hdl, "register module");
 117
 118         return (0);
 119 }
 120
 121 void
 122 fmd_hdl_unregister(fmd_hdl_t *hdl)
 123 {
 124         fmd_module_t *mp = (fmd_module_t *)hdl;
 125         fmd_modstat_t *msp = &mp->mod_stats;
 126         const fmd_hdl_ops_t *ops = mp->mod_info->fmdi_ops;
 127
 128         /* dump generic module stats */
 129         fmd_hdl_debug(hdl, "%s: %llu", msp->ms_accepted.fmds_name,
 130             msp->ms_accepted.fmds_value.ui64);
 131         if (ops->fmdo_close != NULL) {
 132                 fmd_hdl_debug(hdl, "%s: %llu", msp->ms_caseopen.fmds_name,
 133                     msp->ms_caseopen.fmds_value.ui64);
 134                 fmd_hdl_debug(hdl, "%s: %llu", msp->ms_casesolved.fmds_name,
 135                     msp->ms_casesolved.fmds_value.ui64);
 136                 fmd_hdl_debug(hdl, "%s: %llu", msp->ms_caseclosed.fmds_name,
 137                     msp->ms_caseclosed.fmds_value.ui64);
 138         }
 139
 140         /* dump module specific stats */
 141         if (mp->mod_ustat != NULL) {
 142                 int i;
 143
 144                 for (i = 0; i < mp->mod_ustat_cnt; i++) {
 145                         fmd_hdl_debug(hdl, "%s: %llu",
 146                             mp->mod_ustat[i].fmds_name,
 147                             mp->mod_ustat[i].fmds_value.ui64);
 148                 }
 149         }
 150
 151         fmd_serd_hash_destroy(&mp->mod_serds);
 152
 153         fmd_hdl_debug(hdl, "unregister module");
 154 }
 155
 156 /*
 157  * fmd_hdl_setspecific() is used to associate a data pointer with
 158  * the specified handle for the duration of the module's lifetime.
 159  * This pointer can be retrieved using fmd_hdl_getspecific().
 160  */
 161 void
 162 fmd_hdl_setspecific(fmd_hdl_t *hdl, void *spec)
 163 {
 164         fmd_module_t *mp = (fmd_module_t *)hdl;
 165
 166         mp->mod_spec = spec;
 167 }
 168
 169 /*
 170  * Return the module-specific data pointer previously associated
 171  * with the handle using fmd_hdl_setspecific().
 172  */
 173 void *
 174 fmd_hdl_getspecific(fmd_hdl_t *hdl)
 175 {
 176         fmd_module_t *mp = (fmd_module_t *)hdl;
 177
 178         return (mp->mod_spec);
 179 }
 180
 181 void *
 182 fmd_hdl_alloc(fmd_hdl_t *hdl, size_t size, int flags)
 183 {
 184         (void) hdl;
 185         return (umem_alloc(size, flags));
 186 }
 187
 188 void *
 189 fmd_hdl_zalloc(fmd_hdl_t *hdl, size_t size, int flags)
 190 {
 191         (void) hdl;
 192         return (umem_zalloc(size, flags));
 193 }
 194
 195 void
 196 fmd_hdl_free(fmd_hdl_t *hdl, void *data, size_t size)
 197 {
 198         (void) hdl;
 199         umem_free(data, size);
 200 }
 201
 202 /*
 203  * Record a module debug message using the specified format.
 204  */
 205 void
 206 fmd_hdl_debug(fmd_hdl_t *hdl, const char *format, ...)
 207 {
 208         char message[256];
 209         va_list vargs;
 210         fmd_module_t *mp = (fmd_module_t *)hdl;
 211
 212         va_start(vargs, format);
 213         (void) vsnprintf(message, sizeof (message), format, vargs);
 214         va_end(vargs);
 215
 216         /* prefix message with module name */
 217         zed_log_msg(LOG_INFO, "%s: %s", mp->mod_name, message);
 218 }
 219
 220 /* Property Retrieval */
 221
 222 int32_t
 223 fmd_prop_get_int32(fmd_hdl_t *hdl, const char *name)
 224 {
 225         (void) hdl;
 226
 227         /*
 228          * These can be looked up in mp->modinfo->fmdi_props
 229          * For now we just hard code for phase 2. In the
 230          * future, there can be a ZED based override.
 231          */
 232         if (strcmp(name, "spare_on_remove") == 0)
 233                 return (1);
 234
 235         return (0);
 236 }
 237
 238 /* FMD Statistics */
 239
 240 fmd_stat_t *
 241 fmd_stat_create(fmd_hdl_t *hdl, uint_t flags, uint_t nstats, fmd_stat_t *statv)
 242 {
 243         fmd_module_t *mp = (fmd_module_t *)hdl;
 244
 245         if (flags == FMD_STAT_NOALLOC) {
 246                 mp->mod_ustat = statv;
 247                 mp->mod_ustat_cnt = nstats;
 248         }
 249
 250         return (statv);
 251 }
 252
 253 /* Case Management */
 254
 255 fmd_case_t *
 256 fmd_case_open(fmd_hdl_t *hdl, void *data)
 257 {
 258         fmd_module_t *mp = (fmd_module_t *)hdl;
 259         uuid_t uuid;
 260
 261         fmd_case_t *cp;
 262
 263         cp = fmd_hdl_zalloc(hdl, sizeof (fmd_case_t), FMD_SLEEP);
 264         cp->ci_mod = hdl;
 265         cp->ci_state = FMD_CASE_UNSOLVED;
 266         cp->ci_flags = FMD_CF_DIRTY;
 267         cp->ci_data = data;
 268         cp->ci_bufptr = NULL;
 269         cp->ci_bufsiz = 0;
 270
 271         uuid_generate(uuid);
 272         uuid_unparse(uuid, cp->ci_uuid);
 273
 274         fmd_hdl_debug(hdl, "case opened (%s)", cp->ci_uuid);
 275         mp->mod_stats.ms_caseopen.fmds_value.ui64++;
 276
 277         return (cp);
 278 }
 279
 280 void
 281 fmd_case_solve(fmd_hdl_t *hdl, fmd_case_t *cp)
 282 {
 283         fmd_module_t *mp = (fmd_module_t *)hdl;
 284
 285         /*
 286          * For ZED, the event was already sent from fmd_case_add_suspect()
 287          */
 288
 289         if (cp->ci_state >= FMD_CASE_SOLVED)
 290                 fmd_hdl_debug(hdl, "case is already solved or closed");
 291
 292         cp->ci_state = FMD_CASE_SOLVED;
 293
 294         fmd_hdl_debug(hdl, "case solved (%s)", cp->ci_uuid);
 295         mp->mod_stats.ms_casesolved.fmds_value.ui64++;
 296 }
 297
 298 void
 299 fmd_case_close(fmd_hdl_t *hdl, fmd_case_t *cp)
 300 {
 301         fmd_module_t *mp = (fmd_module_t *)hdl;
 302         const fmd_hdl_ops_t *ops = mp->mod_info->fmdi_ops;
 303
 304         fmd_hdl_debug(hdl, "case closed (%s)", cp->ci_uuid);
 305
 306         if (ops->fmdo_close != NULL)
 307                 ops->fmdo_close(hdl, cp);
 308
 309         mp->mod_stats.ms_caseopen.fmds_value.ui64--;
 310         mp->mod_stats.ms_caseclosed.fmds_value.ui64++;
 311
 312         if (cp->ci_bufptr != NULL && cp->ci_bufsiz > 0)
 313                 fmd_hdl_free(hdl, cp->ci_bufptr, cp->ci_bufsiz);
 314
 315         fmd_hdl_free(hdl, cp, sizeof (fmd_case_t));
 316 }
 317
 318 void
 319 fmd_case_uuresolved(fmd_hdl_t *hdl, const char *uuid)
 320 {
 321         fmd_hdl_debug(hdl, "case resolved by uuid (%s)", uuid);
 322 }
 323
 324 boolean_t
 325 fmd_case_solved(fmd_hdl_t *hdl, fmd_case_t *cp)
 326 {
 327         (void) hdl;
 328         return (cp->ci_state >= FMD_CASE_SOLVED);
 329 }
 330
 331 void
 332 fmd_case_add_ereport(fmd_hdl_t *hdl, fmd_case_t *cp, fmd_event_t *ep)
 333 {
 334         (void) hdl, (void) cp, (void) ep;
 335 }
 336
 337 static void
 338 zed_log_fault(nvlist_t *nvl, const char *uuid, const char *code)
 339 {
 340         nvlist_t *rsrc;
 341         const char *strval;
 342         uint64_t guid;
 343         uint8_t byte;
 344
 345         zed_log_msg(LOG_INFO, "\nzed_fault_event:");
 346
 347         if (uuid != NULL)
 348                 zed_log_msg(LOG_INFO, "\t%s: %s", FM_SUSPECT_UUID, uuid);
 349         if (nvlist_lookup_string(nvl, FM_CLASS, &strval) == 0)
 350                 zed_log_msg(LOG_INFO, "\t%s: %s", FM_CLASS, strval);
 351         if (code != NULL)
 352                 zed_log_msg(LOG_INFO, "\t%s: %s", FM_SUSPECT_DIAG_CODE, code);
 353         if (nvlist_lookup_uint8(nvl, FM_FAULT_CERTAINTY, &byte) == 0)
 354                 zed_log_msg(LOG_INFO, "\t%s: %hhu", FM_FAULT_CERTAINTY, byte);
 355         if (nvlist_lookup_nvlist(nvl, FM_FAULT_RESOURCE, &rsrc) == 0) {
 356                 if (nvlist_lookup_string(rsrc, FM_FMRI_SCHEME, &strval) == 0)
 357                         zed_log_msg(LOG_INFO, "\t%s: %s", FM_FMRI_SCHEME,
 358                             strval);
 359                 if (nvlist_lookup_uint64(rsrc, FM_FMRI_ZFS_POOL, &guid) == 0)
 360                         zed_log_msg(LOG_INFO, "\t%s: %llu", FM_FMRI_ZFS_POOL,
 361                             guid);
 362                 if (nvlist_lookup_uint64(rsrc, FM_FMRI_ZFS_VDEV, &guid) == 0)
 363                         zed_log_msg(LOG_INFO, "\t%s: %llu \n", FM_FMRI_ZFS_VDEV,
 364                             guid);
 365         }
 366 }
 367
 368 static const char *
 369 fmd_fault_mkcode(nvlist_t *fault)
 370 {
 371         const char *class;
 372         const char *code = "-";
 373
 374         /*
 375          * Note: message codes come from: openzfs/usr/src/cmd/fm/dicts/ZFS.po
 376          */
 377         if (nvlist_lookup_string(fault, FM_CLASS, &class) == 0) {
 378                 if (strcmp(class, "fault.fs.zfs.vdev.io") == 0)
 379                         code = "ZFS-8000-FD";
 380                 else if (strcmp(class, "fault.fs.zfs.vdev.checksum") == 0)
 381                         code = "ZFS-8000-GH";
 382                 else if (strcmp(class, "fault.fs.zfs.io_failure_wait") == 0)
 383                         code = "ZFS-8000-HC";
 384                 else if (strcmp(class, "fault.fs.zfs.io_failure_continue") == 0)
 385                         code = "ZFS-8000-JQ";
 386                 else if (strcmp(class, "fault.fs.zfs.log_replay") == 0)
 387                         code = "ZFS-8000-K4";
 388                 else if (strcmp(class, "fault.fs.zfs.pool") == 0)
 389                         code = "ZFS-8000-CS";
 390                 else if (strcmp(class, "fault.fs.zfs.device") == 0)
 391                         code = "ZFS-8000-D3";
 392
 393         }
 394         return (code);
 395 }
 396
 397 void
 398 fmd_case_add_suspect(fmd_hdl_t *hdl, fmd_case_t *cp, nvlist_t *fault)
 399 {
 400         nvlist_t *nvl;
 401         const char *code = fmd_fault_mkcode(fault);
 402         int64_t tod[2];
 403         int err = 0;
 404
 405         /*
 406          * payload derived from fmd_protocol_list()
 407          */
 408
 409         (void) gettimeofday(&cp->ci_tv, NULL);
 410         tod[0] = cp->ci_tv.tv_sec;
 411         tod[1] = cp->ci_tv.tv_usec;
 412
 413         nvl = fmd_nvl_alloc(hdl, FMD_SLEEP);
 414
 415         err |= nvlist_add_uint8(nvl, FM_VERSION, FM_SUSPECT_VERSION);
 416         err |= nvlist_add_string(nvl, FM_CLASS, FM_LIST_SUSPECT_CLASS);
 417         err |= nvlist_add_string(nvl, FM_SUSPECT_UUID, cp->ci_uuid);
 418         err |= nvlist_add_string(nvl, FM_SUSPECT_DIAG_CODE, code);
 419         err |= nvlist_add_int64_array(nvl, FM_SUSPECT_DIAG_TIME, tod, 2);
 420         err |= nvlist_add_uint32(nvl, FM_SUSPECT_FAULT_SZ, 1);
 421         err |= nvlist_add_nvlist_array(nvl, FM_SUSPECT_FAULT_LIST,
 422             (const nvlist_t **)&fault, 1);
 423
 424         if (err)
 425                 zed_log_die("failed to populate nvlist");
 426
 427         zed_log_fault(fault, cp->ci_uuid, code);
 428         zfs_agent_post_event(FM_LIST_SUSPECT_CLASS, NULL, nvl);
 429
 430         nvlist_free(nvl);
 431         nvlist_free(fault);
 432 }
 433
 434 void
 435 fmd_case_setspecific(fmd_hdl_t *hdl, fmd_case_t *cp, void *data)
 436 {
 437         (void) hdl;
 438         cp->ci_data = data;
 439 }
 440
 441 void *
 442 fmd_case_getspecific(fmd_hdl_t *hdl, fmd_case_t *cp)
 443 {
 444         (void) hdl;
 445         return (cp->ci_data);
 446 }
 447
 448 void
 449 fmd_buf_create(fmd_hdl_t *hdl, fmd_case_t *cp, const char *name, size_t size)
 450 {
 451         assert(strcmp(name, "data") == 0), (void) name;
 452         assert(cp->ci_bufptr == NULL);
 453         assert(size < (1024 * 1024));
 454
 455         cp->ci_bufptr = fmd_hdl_alloc(hdl, size, FMD_SLEEP);
 456         cp->ci_bufsiz = size;
 457 }
 458
 459 void
 460 fmd_buf_read(fmd_hdl_t *hdl, fmd_case_t *cp,
 461     const char *name, void *buf, size_t size)
 462 {
 463         (void) hdl;
 464         assert(strcmp(name, "data") == 0), (void) name;
 465         assert(cp->ci_bufptr != NULL);
 466         assert(size <= cp->ci_bufsiz);
 467
 468         memcpy(buf, cp->ci_bufptr, size);
 469 }
 470
 471 void
 472 fmd_buf_write(fmd_hdl_t *hdl, fmd_case_t *cp,
 473     const char *name, const void *buf, size_t size)
 474 {
 475         (void) hdl;
 476         assert(strcmp(name, "data") == 0), (void) name;
 477         assert(cp->ci_bufptr != NULL);
 478         assert(cp->ci_bufsiz >= size);
 479
 480         memcpy(cp->ci_bufptr, buf, size);
 481 }
 482
 483 /* SERD Engines */
 484
 485 void
 486 fmd_serd_create(fmd_hdl_t *hdl, const char *name, uint_t n, hrtime_t t)
 487 {
 488         fmd_module_t *mp = (fmd_module_t *)hdl;
 489
 490         if (fmd_serd_eng_lookup(&mp->mod_serds, name) != NULL) {
 491                 zed_log_msg(LOG_ERR, "failed to create SERD engine '%s': "
 492                     " name already exists", name);
 493                 return;
 494         }
 495
 496         (void) fmd_serd_eng_insert(&mp->mod_serds, name, n, t);
 497 }
 498
 499 void
 500 fmd_serd_destroy(fmd_hdl_t *hdl, const char *name)
 501 {
 502         fmd_module_t *mp = (fmd_module_t *)hdl;
 503
 504         fmd_serd_eng_delete(&mp->mod_serds, name);
 505
 506         fmd_hdl_debug(hdl, "serd_destroy %s", name);
 507 }
 508
 509 int
 510 fmd_serd_exists(fmd_hdl_t *hdl, const char *name)
 511 {
 512         fmd_module_t *mp = (fmd_module_t *)hdl;
 513
 514         return (fmd_serd_eng_lookup(&mp->mod_serds, name) != NULL);
 515 }
 516
 517 int
 518 fmd_serd_active(fmd_hdl_t *hdl, const char *name)
 519 {
 520         fmd_module_t *mp = (fmd_module_t *)hdl;
 521         fmd_serd_eng_t *sgp;
 522
 523         if ((sgp = fmd_serd_eng_lookup(&mp->mod_serds, name)) == NULL) {
 524                 zed_log_msg(LOG_ERR, "serd engine '%s' does not exist", name);
 525                 return (0);
 526         }
 527         return (fmd_serd_eng_fired(sgp) || !fmd_serd_eng_empty(sgp));
 528 }
 529
 530 void
 531 fmd_serd_reset(fmd_hdl_t *hdl, const char *name)
 532 {
 533         fmd_module_t *mp = (fmd_module_t *)hdl;
 534         fmd_serd_eng_t *sgp;
 535
 536         if ((sgp = fmd_serd_eng_lookup(&mp->mod_serds, name)) == NULL) {
 537                 zed_log_msg(LOG_ERR, "serd engine '%s' does not exist", name);
 538         } else {
 539                 fmd_serd_eng_reset(sgp);
 540                 fmd_hdl_debug(hdl, "serd_reset %s", name);
 541         }
 542 }
 543
 544 int
 545 fmd_serd_record(fmd_hdl_t *hdl, const char *name, fmd_event_t *ep)
 546 {
 547         fmd_module_t *mp = (fmd_module_t *)hdl;
 548         fmd_serd_eng_t *sgp;
 549
 550         if ((sgp = fmd_serd_eng_lookup(&mp->mod_serds, name)) == NULL) {
 551                 zed_log_msg(LOG_ERR, "failed to add record to SERD engine '%s'",
 552                     name);
 553                 return (0);
 554         }
 555         return (fmd_serd_eng_record(sgp, ep->ev_hrt));
 556 }
 557
 558 void
 559 fmd_serd_gc(fmd_hdl_t *hdl)
 560 {
 561         fmd_module_t *mp = (fmd_module_t *)hdl;
 562
 563         fmd_serd_hash_apply(&mp->mod_serds, fmd_serd_eng_gc, NULL);
 564 }
 565
 566 /* FMD Timers */
 567
 568 static void
 569 _timer_notify(union sigval sv)
 570 {
 571         fmd_timer_t *ftp = sv.sival_ptr;
 572         fmd_hdl_t *hdl = ftp->ft_hdl;
 573         fmd_module_t *mp = (fmd_module_t *)hdl;
 574         const fmd_hdl_ops_t *ops = mp->mod_info->fmdi_ops;
 575         struct itimerspec its;
 576
 577         fmd_hdl_debug(hdl, "%s timer fired (%p)", mp->mod_name, ftp->ft_tid);
 578
 579         /* disarm the timer */
 580         memset(&its, 0, sizeof (struct itimerspec));
 581         timer_settime(ftp->ft_tid, 0, &its, NULL);
 582
 583         /* Note that the fmdo_timeout can remove this timer */
 584         if (ops->fmdo_timeout != NULL)
 585                 ops->fmdo_timeout(hdl, ftp, ftp->ft_arg);
 586 }
 587
 588 /*
 589  * Install a new timer which will fire at least delta nanoseconds after the
 590  * current time. After the timeout has expired, the module's fmdo_timeout
 591  * entry point is called.
 592  */
 593 fmd_timer_t *
 594 fmd_timer_install(fmd_hdl_t *hdl, void *arg, fmd_event_t *ep, hrtime_t delta)
 595 {
 596         (void) ep;
 597         struct sigevent sev;
 598         struct itimerspec its;
 599         fmd_timer_t *ftp;
 600
 601         ftp = fmd_hdl_alloc(hdl, sizeof (fmd_timer_t), FMD_SLEEP);
 602         ftp->ft_arg = arg;
 603         ftp->ft_hdl = hdl;
 604
 605         its.it_value.tv_sec = delta / 1000000000;
 606         its.it_value.tv_nsec = delta % 1000000000;
 607         its.it_interval.tv_sec = its.it_value.tv_sec;
 608         its.it_interval.tv_nsec = its.it_value.tv_nsec;
 609
 610         sev.sigev_notify = SIGEV_THREAD;
 611         sev.sigev_notify_function = _timer_notify;
 612         sev.sigev_notify_attributes = NULL;
 613         sev.sigev_value.sival_ptr = ftp;
 614         sev.sigev_signo = 0;
 615
 616         timer_create(CLOCK_REALTIME, &sev, &ftp->ft_tid);
 617         timer_settime(ftp->ft_tid, 0, &its, NULL);
 618
 619         fmd_hdl_debug(hdl, "installing timer for %d secs (%p)",
 620             (int)its.it_value.tv_sec, ftp->ft_tid);
 621
 622         return (ftp);
 623 }
 624
 625 void
 626 fmd_timer_remove(fmd_hdl_t *hdl, fmd_timer_t *ftp)
 627 {
 628         fmd_hdl_debug(hdl, "removing timer (%p)", ftp->ft_tid);
 629
 630         timer_delete(ftp->ft_tid);
 631
 632         fmd_hdl_free(hdl, ftp, sizeof (fmd_timer_t));
 633 }
 634
 635 /* Name-Value Pair Lists */
 636
 637 nvlist_t *
 638 fmd_nvl_create_fault(fmd_hdl_t *hdl, const char *class, uint8_t certainty,
 639     nvlist_t *asru, nvlist_t *fru, nvlist_t *resource)
 640 {
 641         (void) hdl;
 642         nvlist_t *nvl;
 643         int err = 0;
 644
 645         if (nvlist_alloc(&nvl, NV_UNIQUE_NAME, 0) != 0)
 646                 zed_log_die("failed to xalloc fault nvlist");
 647
 648         err |= nvlist_add_uint8(nvl, FM_VERSION, FM_FAULT_VERSION);
 649         err |= nvlist_add_string(nvl, FM_CLASS, class);
 650         err |= nvlist_add_uint8(nvl, FM_FAULT_CERTAINTY, certainty);
 651
 652         if (asru != NULL)
 653                 err |= nvlist_add_nvlist(nvl, FM_FAULT_ASRU, asru);
 654         if (fru != NULL)
 655                 err |= nvlist_add_nvlist(nvl, FM_FAULT_FRU, fru);
 656         if (resource != NULL)
 657                 err |= nvlist_add_nvlist(nvl, FM_FAULT_RESOURCE, resource);
 658
 659         if (err)
 660                 zed_log_die("failed to populate nvlist: %s\n", strerror(err));
 661
 662         return (nvl);
 663 }
 664
 665 /*
 666  * sourced from fmd_string.c
 667  */
 668 static int
 669 fmd_strmatch(const char *s, const char *p)
 670 {
 671         char c;
 672
 673         if (p == NULL)
 674                 return (0);
 675
 676         if (s == NULL)
 677                 s = ""; /* treat NULL string as the empty string */
 678
 679         do {
 680                 if ((c = *p++) == '\0')
 681                         return (*s == '\0');
 682
 683                 if (c == '*') {
 684                         while (*p == '*')
 685                                 p++; /* consecutive *'s can be collapsed */
 686
 687                         if (*p == '\0')
 688                                 return (1);
 689
 690                         while (*s != '\0') {
 691                                 if (fmd_strmatch(s++, p) != 0)
 692                                         return (1);
 693                         }
 694
 695                         return (0);
 696                 }
 697         } while (c == *s++);
 698
 699         return (0);
 700 }
 701
 702 int
 703 fmd_nvl_class_match(fmd_hdl_t *hdl, nvlist_t *nvl, const char *pattern)
 704 {
 705         (void) hdl;
 706         const char *class;
 707
 708         return (nvl != NULL &&
 709             nvlist_lookup_string(nvl, FM_CLASS, &class) == 0 &&
 710             fmd_strmatch(class, pattern));
 711 }
 712
 713 nvlist_t *
 714 fmd_nvl_alloc(fmd_hdl_t *hdl, int flags)
 715 {
 716         (void) hdl, (void) flags;
 717         nvlist_t *nvl = NULL;
 718
 719         if (nvlist_alloc(&nvl, NV_UNIQUE_NAME, 0) != 0)
 720                 return (NULL);
 721
 722         return (nvl);
 723 }
 724
 725
 726 /*
 727  * ZED Agent specific APIs
 728  */
 729
 730 fmd_hdl_t *
 731 fmd_module_hdl(const char *name)
 732 {
 733         if (strcmp(name, "zfs-retire") == 0)
 734                 return ((fmd_hdl_t *)&zfs_retire_module);
 735         if (strcmp(name, "zfs-diagnosis") == 0)
 736                 return ((fmd_hdl_t *)&zfs_diagnosis_module);
 737
 738         return (NULL);
 739 }
 740
 741 boolean_t
 742 fmd_module_initialized(fmd_hdl_t *hdl)
 743 {
 744         fmd_module_t *mp = (fmd_module_t *)hdl;
 745
 746         return (mp->mod_info != NULL);
 747 }
 748
 749 /*
 750  * fmd_module_recv is called for each event that is received by
 751  * the fault manager that has a class that matches one of the
 752  * module's subscriptions.
 753  */
 754 void
 755 fmd_module_recv(fmd_hdl_t *hdl, nvlist_t *nvl, const char *class)
 756 {
 757         fmd_module_t *mp = (fmd_module_t *)hdl;
 758         const fmd_hdl_ops_t *ops = mp->mod_info->fmdi_ops;
 759         fmd_event_t faux_event = {0};
 760         int64_t *tv;
 761         uint_t n;
 762
 763         /*
 764          * Will need to normalized this if we persistently store the case data
 765          */
 766         if (nvlist_lookup_int64_array(nvl, FM_EREPORT_TIME, &tv, &n) == 0)
 767                 faux_event.ev_hrt = tv[0] * NANOSEC + tv[1];
 768         else
 769                 faux_event.ev_hrt = 0;
 770
 771         ops->fmdo_recv(hdl, &faux_event, nvl, class);
 772
 773         mp->mod_stats.ms_accepted.fmds_value.ui64++;
 774
 775         /* TBD - should we initiate fm_module_gc() periodically? */
 776 }