lib/libzpool/kernel.c

   1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  23  * Copyright (c) 2016 Actifio, Inc. All rights reserved.
  24  */
  25
  26 #include <assert.h>
  27 #include <fcntl.h>
  28 #include <poll.h>
  29 #include <stdio.h>
  30 #include <stdlib.h>
  31 #include <string.h>
  32 #include <zlib.h>
  33 #include <libgen.h>
  34 #include <sys/signal.h>
  35 #include <sys/spa.h>
  36 #include <sys/stat.h>
  37 #include <sys/processor.h>
  38 #include <sys/zfs_context.h>
  39 #include <sys/rrwlock.h>
  40 #include <sys/utsname.h>
  41 #include <sys/time.h>
  42 #include <sys/systeminfo.h>
  43 #include <zfs_fletcher.h>
  44
  45 /*
  46  * Emulation of kernel services in userland.
  47  */
  48
  49 int aok;
  50 uint64_t physmem;
  51 vnode_t *rootdir = (vnode_t *)0xabcd1234;
  52 char hw_serial[HW_HOSTID_LEN];
  53 struct utsname hw_utsname;
  54 vmem_t *zio_arena = NULL;
  55
  56 /* If set, all blocks read will be copied to the specified directory. */
  57 char *vn_dumpdir = NULL;
  58
  59 /* this only exists to have its address taken */
  60 struct proc p0;
  61
  62 /*
  63  * =========================================================================
  64  * threads
  65  * =========================================================================
  66  */
  67
  68 pthread_cond_t kthread_cond = PTHREAD_COND_INITIALIZER;
  69 pthread_mutex_t kthread_lock = PTHREAD_MUTEX_INITIALIZER;
  70 pthread_key_t kthread_key;
  71 int kthread_nr = 0;
  72
  73 void
  74 thread_init(void)
  75 {
  76         kthread_t *kt;
  77
  78         VERIFY3S(pthread_key_create(&kthread_key, NULL), ==, 0);
  79
  80         /* Create entry for primary kthread */
  81         kt = umem_zalloc(sizeof (kthread_t), UMEM_NOFAIL);
  82         kt->t_tid = pthread_self();
  83         kt->t_func = NULL;
  84
  85         VERIFY3S(pthread_setspecific(kthread_key, kt), ==, 0);
  86
  87         /* Only the main thread should be running at the moment */
  88         ASSERT3S(kthread_nr, ==, 0);
  89         kthread_nr = 1;
  90 }
  91
  92 void
  93 thread_fini(void)
  94 {
  95         kthread_t *kt = curthread;
  96
  97         ASSERT(pthread_equal(kt->t_tid, pthread_self()));
  98         ASSERT3P(kt->t_func, ==, NULL);
  99
 100         umem_free(kt, sizeof (kthread_t));
 101
 102         /* Wait for all threads to exit via thread_exit() */
 103         VERIFY3S(pthread_mutex_lock(&kthread_lock), ==, 0);
 104
 105         kthread_nr--; /* Main thread is exiting */
 106
 107         while (kthread_nr > 0)
 108                 VERIFY0(pthread_cond_wait(&kthread_cond, &kthread_lock));
 109
 110         ASSERT3S(kthread_nr, ==, 0);
 111         VERIFY3S(pthread_mutex_unlock(&kthread_lock), ==, 0);
 112
 113         VERIFY3S(pthread_key_delete(kthread_key), ==, 0);
 114 }
 115
 116 kthread_t *
 117 zk_thread_current(void)
 118 {
 119         kthread_t *kt = pthread_getspecific(kthread_key);
 120
 121         ASSERT3P(kt, !=, NULL);
 122
 123         return (kt);
 124 }
 125
 126 void *
 127 zk_thread_helper(void *arg)
 128 {
 129         kthread_t *kt = (kthread_t *) arg;
 130
 131         VERIFY3S(pthread_setspecific(kthread_key, kt), ==, 0);
 132
 133         VERIFY3S(pthread_mutex_lock(&kthread_lock), ==, 0);
 134         kthread_nr++;
 135         VERIFY3S(pthread_mutex_unlock(&kthread_lock), ==, 0);
 136         (void) setpriority(PRIO_PROCESS, 0, kt->t_pri);
 137
 138         kt->t_tid = pthread_self();
 139         ((thread_func_arg_t) kt->t_func)(kt->t_arg);
 140
 141         /* Unreachable, thread must exit with thread_exit() */
 142         abort();
 143
 144         return (NULL);
 145 }
 146
 147 kthread_t *
 148 zk_thread_create(caddr_t stk, size_t stksize, thread_func_t func, void *arg,
 149     size_t len, proc_t *pp, int state, pri_t pri, int detachstate)
 150 {
 151         kthread_t *kt;
 152         pthread_attr_t attr;
 153         char *stkstr;
 154
 155         ASSERT0(state & ~TS_RUN);
 156
 157         kt = umem_zalloc(sizeof (kthread_t), UMEM_NOFAIL);
 158         kt->t_func = func;
 159         kt->t_arg = arg;
 160         kt->t_pri = pri;
 161
 162         VERIFY0(pthread_attr_init(&attr));
 163         VERIFY0(pthread_attr_setdetachstate(&attr, detachstate));
 164
 165         /*
 166          * We allow the default stack size in user space to be specified by
 167          * setting the ZFS_STACK_SIZE environment variable.  This allows us
 168          * the convenience of observing and debugging stack overruns in
 169          * user space.  Explicitly specified stack sizes will be honored.
 170          * The usage of ZFS_STACK_SIZE is discussed further in the
 171          * ENVIRONMENT VARIABLES sections of the ztest(1) man page.
 172          */
 173         if (stksize == 0) {
 174                 stkstr = getenv("ZFS_STACK_SIZE");
 175
 176                 if (stkstr == NULL)
 177                         stksize = TS_STACK_MAX;
 178                 else
 179                         stksize = MAX(atoi(stkstr), TS_STACK_MIN);
 180         }
 181
 182         VERIFY3S(stksize, >, 0);
 183         stksize = P2ROUNDUP(MAX(stksize, TS_STACK_MIN), PAGESIZE);
 184         /*
 185          * If this ever fails, it may be because the stack size is not a
 186          * multiple of system page size.
 187          */
 188         VERIFY0(pthread_attr_setstacksize(&attr, stksize));
 189         VERIFY0(pthread_attr_setguardsize(&attr, PAGESIZE));
 190
 191         VERIFY0(pthread_create(&kt->t_tid, &attr, &zk_thread_helper, kt));
 192         VERIFY0(pthread_attr_destroy(&attr));
 193
 194         return (kt);
 195 }
 196
 197 void
 198 zk_thread_exit(void)
 199 {
 200         kthread_t *kt = curthread;
 201
 202         ASSERT(pthread_equal(kt->t_tid, pthread_self()));
 203
 204         umem_free(kt, sizeof (kthread_t));
 205
 206         VERIFY0(pthread_mutex_lock(&kthread_lock));
 207         kthread_nr--;
 208         VERIFY0(pthread_mutex_unlock(&kthread_lock));
 209
 210         VERIFY0(pthread_cond_broadcast(&kthread_cond));
 211         pthread_exit((void *)TS_MAGIC);
 212 }
 213
 214 void
 215 zk_thread_join(kt_did_t tid)
 216 {
 217         void *ret;
 218
 219         pthread_join((pthread_t)tid, &ret);
 220         VERIFY3P(ret, ==, (void *)TS_MAGIC);
 221 }
 222
 223 /*
 224  * =========================================================================
 225  * kstats
 226  * =========================================================================
 227  */
 228 /*ARGSUSED*/
 229 kstat_t *
 230 kstat_create(const char *module, int instance, const char *name,
 231     const char *class, uchar_t type, ulong_t ndata, uchar_t ks_flag)
 232 {
 233         return (NULL);
 234 }
 235
 236 /*ARGSUSED*/
 237 void
 238 kstat_install(kstat_t *ksp)
 239 {}
 240
 241 /*ARGSUSED*/
 242 void
 243 kstat_delete(kstat_t *ksp)
 244 {}
 245
 246 /*ARGSUSED*/
 247 void
 248 kstat_waitq_enter(kstat_io_t *kiop)
 249 {}
 250
 251 /*ARGSUSED*/
 252 void
 253 kstat_waitq_exit(kstat_io_t *kiop)
 254 {}
 255
 256 /*ARGSUSED*/
 257 void
 258 kstat_runq_enter(kstat_io_t *kiop)
 259 {}
 260
 261 /*ARGSUSED*/
 262 void
 263 kstat_runq_exit(kstat_io_t *kiop)
 264 {}
 265
 266 /*ARGSUSED*/
 267 void
 268 kstat_waitq_to_runq(kstat_io_t *kiop)
 269 {}
 270
 271 /*ARGSUSED*/
 272 void
 273 kstat_runq_back_to_waitq(kstat_io_t *kiop)
 274 {}
 275
 276 void
 277 kstat_set_raw_ops(kstat_t *ksp,
 278     int (*headers)(char *buf, size_t size),
 279     int (*data)(char *buf, size_t size, void *data),
 280     void *(*addr)(kstat_t *ksp, loff_t index))
 281 {}
 282
 283 /*
 284  * =========================================================================
 285  * mutexes
 286  * =========================================================================
 287  */
 288
 289 void
 290 mutex_init(kmutex_t *mp, char *name, int type, void *cookie)
 291 {
 292         ASSERT3S(type, ==, MUTEX_DEFAULT);
 293         ASSERT3P(cookie, ==, NULL);
 294         mp->m_owner = MTX_INIT;
 295         mp->m_magic = MTX_MAGIC;
 296         VERIFY3S(pthread_mutex_init(&mp->m_lock, NULL), ==, 0);
 297 }
 298
 299 void
 300 mutex_destroy(kmutex_t *mp)
 301 {
 302         ASSERT3U(mp->m_magic, ==, MTX_MAGIC);
 303         ASSERT3P(mp->m_owner, ==, MTX_INIT);
 304         ASSERT0(pthread_mutex_destroy(&(mp)->m_lock));
 305         mp->m_owner = MTX_DEST;
 306         mp->m_magic = 0;
 307 }
 308
 309 void
 310 mutex_enter(kmutex_t *mp)
 311 {
 312         ASSERT3U(mp->m_magic, ==, MTX_MAGIC);
 313         ASSERT3P(mp->m_owner, !=, MTX_DEST);
 314         ASSERT3P(mp->m_owner, !=, curthread);
 315         VERIFY3S(pthread_mutex_lock(&mp->m_lock), ==, 0);
 316         ASSERT3P(mp->m_owner, ==, MTX_INIT);
 317         mp->m_owner = curthread;
 318 }
 319
 320 int
 321 mutex_tryenter(kmutex_t *mp)
 322 {
 323         int err;
 324         ASSERT3U(mp->m_magic, ==, MTX_MAGIC);
 325         ASSERT3P(mp->m_owner, !=, MTX_DEST);
 326         if (0 == (err = pthread_mutex_trylock(&mp->m_lock))) {
 327                 ASSERT3P(mp->m_owner, ==, MTX_INIT);
 328                 mp->m_owner = curthread;
 329                 return (1);
 330         } else {
 331                 VERIFY3S(err, ==, EBUSY);
 332                 return (0);
 333         }
 334 }
 335
 336 void
 337 mutex_exit(kmutex_t *mp)
 338 {
 339         ASSERT3U(mp->m_magic, ==, MTX_MAGIC);
 340         ASSERT3P(mutex_owner(mp), ==, curthread);
 341         mp->m_owner = MTX_INIT;
 342         VERIFY3S(pthread_mutex_unlock(&mp->m_lock), ==, 0);
 343 }
 344
 345 void *
 346 mutex_owner(kmutex_t *mp)
 347 {
 348         ASSERT3U(mp->m_magic, ==, MTX_MAGIC);
 349         return (mp->m_owner);
 350 }
 351
 352 int
 353 mutex_held(kmutex_t *mp)
 354 {
 355         return (mp->m_owner == curthread);
 356 }
 357
 358 /*
 359  * =========================================================================
 360  * rwlocks
 361  * =========================================================================
 362  */
 363
 364 void
 365 rw_init(krwlock_t *rwlp, char *name, int type, void *arg)
 366 {
 367         ASSERT3S(type, ==, RW_DEFAULT);
 368         ASSERT3P(arg, ==, NULL);
 369         VERIFY3S(pthread_rwlock_init(&rwlp->rw_lock, NULL), ==, 0);
 370         rwlp->rw_owner = RW_INIT;
 371         rwlp->rw_wr_owner = RW_INIT;
 372         rwlp->rw_readers = 0;
 373         rwlp->rw_magic = RW_MAGIC;
 374 }
 375
 376 void
 377 rw_destroy(krwlock_t *rwlp)
 378 {
 379         ASSERT3U(rwlp->rw_magic, ==, RW_MAGIC);
 380         ASSERT(rwlp->rw_readers == 0 && rwlp->rw_wr_owner == RW_INIT);
 381         VERIFY3S(pthread_rwlock_destroy(&rwlp->rw_lock), ==, 0);
 382         rwlp->rw_magic = 0;
 383 }
 384
 385 void
 386 rw_enter(krwlock_t *rwlp, krw_t rw)
 387 {
 388         ASSERT3U(rwlp->rw_magic, ==, RW_MAGIC);
 389         ASSERT3P(rwlp->rw_owner, !=, curthread);
 390         ASSERT3P(rwlp->rw_wr_owner, !=, curthread);
 391
 392         if (rw == RW_READER) {
 393                 VERIFY3S(pthread_rwlock_rdlock(&rwlp->rw_lock), ==, 0);
 394                 ASSERT3P(rwlp->rw_wr_owner, ==, RW_INIT);
 395
 396                 atomic_inc_uint(&rwlp->rw_readers);
 397         } else {
 398                 VERIFY3S(pthread_rwlock_wrlock(&rwlp->rw_lock), ==, 0);
 399                 ASSERT3P(rwlp->rw_wr_owner, ==, RW_INIT);
 400                 ASSERT3U(rwlp->rw_readers, ==, 0);
 401
 402                 rwlp->rw_wr_owner = curthread;
 403         }
 404
 405         rwlp->rw_owner = curthread;
 406 }
 407
 408 void
 409 rw_exit(krwlock_t *rwlp)
 410 {
 411         ASSERT3U(rwlp->rw_magic, ==, RW_MAGIC);
 412         ASSERT(RW_LOCK_HELD(rwlp));
 413
 414         if (RW_READ_HELD(rwlp))
 415                 atomic_dec_uint(&rwlp->rw_readers);
 416         else
 417                 rwlp->rw_wr_owner = RW_INIT;
 418
 419         rwlp->rw_owner = RW_INIT;
 420         VERIFY3S(pthread_rwlock_unlock(&rwlp->rw_lock), ==, 0);
 421 }
 422
 423 int
 424 rw_tryenter(krwlock_t *rwlp, krw_t rw)
 425 {
 426         int rv;
 427
 428         ASSERT3U(rwlp->rw_magic, ==, RW_MAGIC);
 429
 430         if (rw == RW_READER)
 431                 rv = pthread_rwlock_tryrdlock(&rwlp->rw_lock);
 432         else
 433                 rv = pthread_rwlock_trywrlock(&rwlp->rw_lock);
 434
 435         if (rv == 0) {
 436                 ASSERT3P(rwlp->rw_wr_owner, ==, RW_INIT);
 437
 438                 if (rw == RW_READER)
 439                         atomic_inc_uint(&rwlp->rw_readers);
 440                 else {
 441                         ASSERT3U(rwlp->rw_readers, ==, 0);
 442                         rwlp->rw_wr_owner = curthread;
 443                 }
 444
 445                 rwlp->rw_owner = curthread;
 446                 return (1);
 447         }
 448
 449         VERIFY3S(rv, ==, EBUSY);
 450
 451         return (0);
 452 }
 453
 454 int
 455 rw_tryupgrade(krwlock_t *rwlp)
 456 {
 457         ASSERT3U(rwlp->rw_magic, ==, RW_MAGIC);
 458
 459         return (0);
 460 }
 461
 462 /*
 463  * =========================================================================
 464  * condition variables
 465  * =========================================================================
 466  */
 467
 468 void
 469 cv_init(kcondvar_t *cv, char *name, int type, void *arg)
 470 {
 471         ASSERT3S(type, ==, CV_DEFAULT);
 472         cv->cv_magic = CV_MAGIC;
 473         VERIFY0(pthread_cond_init(&cv->cv, NULL));
 474 }
 475
 476 void
 477 cv_destroy(kcondvar_t *cv)
 478 {
 479         ASSERT3U(cv->cv_magic, ==, CV_MAGIC);
 480         VERIFY0(pthread_cond_destroy(&cv->cv));
 481         cv->cv_magic = 0;
 482 }
 483
 484 void
 485 cv_wait(kcondvar_t *cv, kmutex_t *mp)
 486 {
 487         ASSERT3U(cv->cv_magic, ==, CV_MAGIC);
 488         ASSERT3P(mutex_owner(mp), ==, curthread);
 489         mp->m_owner = MTX_INIT;
 490         VERIFY0(pthread_cond_wait(&cv->cv, &mp->m_lock));
 491         mp->m_owner = curthread;
 492 }
 493
 494 clock_t
 495 cv_timedwait(kcondvar_t *cv, kmutex_t *mp, clock_t abstime)
 496 {
 497         int error;
 498         struct timeval tv;
 499         timestruc_t ts;
 500         clock_t delta;
 501
 502         ASSERT3U(cv->cv_magic, ==, CV_MAGIC);
 503
 504         delta = abstime - ddi_get_lbolt();
 505         if (delta <= 0)
 506                 return (-1);
 507
 508         VERIFY(gettimeofday(&tv, NULL) == 0);
 509
 510         ts.tv_sec = tv.tv_sec + delta / hz;
 511         ts.tv_nsec = tv.tv_usec * 1000 + (delta % hz) * (NANOSEC / hz);
 512         if (ts.tv_nsec >= NANOSEC) {
 513                 ts.tv_sec++;
 514                 ts.tv_nsec -= NANOSEC;
 515         }
 516
 517         ASSERT3P(mutex_owner(mp), ==, curthread);
 518         mp->m_owner = MTX_INIT;
 519         error = pthread_cond_timedwait(&cv->cv, &mp->m_lock, &ts);
 520         mp->m_owner = curthread;
 521
 522         if (error == ETIMEDOUT)
 523                 return (-1);
 524
 525         VERIFY0(error);
 526
 527         return (1);
 528 }
 529
 530 /*ARGSUSED*/
 531 clock_t
 532 cv_timedwait_hires(kcondvar_t *cv, kmutex_t *mp, hrtime_t tim, hrtime_t res,
 533     int flag)
 534 {
 535         int error;
 536         timestruc_t ts;
 537         hrtime_t delta;
 538
 539         ASSERT(flag == 0 || flag == CALLOUT_FLAG_ABSOLUTE);
 540
 541         delta = tim;
 542         if (flag & CALLOUT_FLAG_ABSOLUTE)
 543                 delta -= gethrtime();
 544
 545         if (delta <= 0)
 546                 return (-1);
 547
 548         ts.tv_sec = delta / NANOSEC;
 549         ts.tv_nsec = delta % NANOSEC;
 550
 551         ASSERT(mutex_owner(mp) == curthread);
 552         mp->m_owner = NULL;
 553         error = pthread_cond_timedwait(&cv->cv, &mp->m_lock, &ts);
 554         mp->m_owner = curthread;
 555
 556         if (error == ETIMEDOUT)
 557                 return (-1);
 558
 559         VERIFY0(error);
 560
 561         return (1);
 562 }
 563
 564 void
 565 cv_signal(kcondvar_t *cv)
 566 {
 567         ASSERT3U(cv->cv_magic, ==, CV_MAGIC);
 568         VERIFY0(pthread_cond_signal(&cv->cv));
 569 }
 570
 571 void
 572 cv_broadcast(kcondvar_t *cv)
 573 {
 574         ASSERT3U(cv->cv_magic, ==, CV_MAGIC);
 575         VERIFY0(pthread_cond_broadcast(&cv->cv));
 576 }
 577
 578 /*
 579  * =========================================================================
 580  * vnode operations
 581  * =========================================================================
 582  */
 583 /*
 584  * Note: for the xxxat() versions of these functions, we assume that the
 585  * starting vp is always rootdir (which is true for spa_directory.c, the only
 586  * ZFS consumer of these interfaces).  We assert this is true, and then emulate
 587  * them by adding '/' in front of the path.
 588  */
 589
 590 /*ARGSUSED*/
 591 int
 592 vn_open(char *path, int x1, int flags, int mode, vnode_t **vpp, int x2, int x3)
 593 {
 594         int fd;
 595         int dump_fd;
 596         vnode_t *vp;
 597         int old_umask = 0;
 598         char *realpath;
 599         struct stat64 st;
 600         int err;
 601
 602         realpath = umem_alloc(MAXPATHLEN, UMEM_NOFAIL);
 603
 604         /*
 605          * If we're accessing a real disk from userland, we need to use
 606          * the character interface to avoid caching.  This is particularly
 607          * important if we're trying to look at a real in-kernel storage
 608          * pool from userland, e.g. via zdb, because otherwise we won't
 609          * see the changes occurring under the segmap cache.
 610          * On the other hand, the stupid character device returns zero
 611          * for its size.  So -- gag -- we open the block device to get
 612          * its size, and remember it for subsequent VOP_GETATTR().
 613          */
 614 #if defined(__sun__) || defined(__sun)
 615         if (strncmp(path, "/dev/", 5) == 0) {
 616 #else
 617         if (0) {
 618 #endif
 619                 char *dsk;
 620                 fd = open64(path, O_RDONLY);
 621                 if (fd == -1) {
 622                         err = errno;
 623                         free(realpath);
 624                         return (err);
 625                 }
 626                 if (fstat64(fd, &st) == -1) {
 627                         err = errno;
 628                         close(fd);
 629                         free(realpath);
 630                         return (err);
 631                 }
 632                 close(fd);
 633                 (void) sprintf(realpath, "%s", path);
 634                 dsk = strstr(path, "/dsk/");
 635                 if (dsk != NULL)
 636                         (void) sprintf(realpath + (dsk - path) + 1, "r%s",
 637                             dsk + 1);
 638         } else {
 639                 (void) sprintf(realpath, "%s", path);
 640                 if (!(flags & FCREAT) && stat64(realpath, &st) == -1) {
 641                         err = errno;
 642                         free(realpath);
 643                         return (err);
 644                 }
 645         }
 646
 647         if (!(flags & FCREAT) && S_ISBLK(st.st_mode)) {
 648 #ifdef __linux__
 649                 flags |= O_DIRECT;
 650 #endif
 651                 /* We shouldn't be writing to block devices in userspace */
 652                 VERIFY(!(flags & FWRITE));
 653         }
 654
 655         if (flags & FCREAT)
 656                 old_umask = umask(0);
 657
 658         /*
 659          * The construct 'flags - FREAD' conveniently maps combinations of
 660          * FREAD and FWRITE to the corresponding O_RDONLY, O_WRONLY, and O_RDWR.
 661          */
 662         fd = open64(realpath, flags - FREAD, mode);
 663         err = errno;
 664
 665         if (flags & FCREAT)
 666                 (void) umask(old_umask);
 667
 668         if (vn_dumpdir != NULL) {
 669                 char *dumppath = umem_zalloc(MAXPATHLEN, UMEM_NOFAIL);
 670                 (void) snprintf(dumppath, MAXPATHLEN,
 671                     "%s/%s", vn_dumpdir, basename(realpath));
 672                 dump_fd = open64(dumppath, O_CREAT | O_WRONLY, 0666);
 673                 umem_free(dumppath, MAXPATHLEN);
 674                 if (dump_fd == -1) {
 675                         err = errno;
 676                         free(realpath);
 677                         close(fd);
 678                         return (err);
 679                 }
 680         } else {
 681                 dump_fd = -1;
 682         }
 683
 684         free(realpath);
 685
 686         if (fd == -1)
 687                 return (err);
 688
 689         if (fstat64_blk(fd, &st) == -1) {
 690                 err = errno;
 691                 close(fd);
 692                 return (err);
 693         }
 694
 695         (void) fcntl(fd, F_SETFD, FD_CLOEXEC);
 696
 697         *vpp = vp = umem_zalloc(sizeof (vnode_t), UMEM_NOFAIL);
 698
 699         vp->v_fd = fd;
 700         vp->v_size = st.st_size;
 701         vp->v_path = spa_strdup(path);
 702         vp->v_dump_fd = dump_fd;
 703
 704         return (0);
 705 }
 706
 707 /*ARGSUSED*/
 708 int
 709 vn_openat(char *path, int x1, int flags, int mode, vnode_t **vpp, int x2,
 710     int x3, vnode_t *startvp, int fd)
 711 {
 712         char *realpath = umem_alloc(strlen(path) + 2, UMEM_NOFAIL);
 713         int ret;
 714
 715         ASSERT(startvp == rootdir);
 716         (void) sprintf(realpath, "/%s", path);
 717
 718         /* fd ignored for now, need if want to simulate nbmand support */
 719         ret = vn_open(realpath, x1, flags, mode, vpp, x2, x3);
 720
 721         umem_free(realpath, strlen(path) + 2);
 722
 723         return (ret);
 724 }
 725
 726 /*ARGSUSED*/
 727 int
 728 vn_rdwr(int uio, vnode_t *vp, void *addr, ssize_t len, offset_t offset,
 729         int x1, int x2, rlim64_t x3, void *x4, ssize_t *residp)
 730 {
 731         ssize_t rc, done = 0, split;
 732
 733         if (uio == UIO_READ) {
 734                 rc = pread64(vp->v_fd, addr, len, offset);
 735                 if (vp->v_dump_fd != -1) {
 736                         int status;
 737                         status = pwrite64(vp->v_dump_fd, addr, rc, offset);
 738                         ASSERT(status != -1);
 739                 }
 740         } else {
 741                 /*
 742                  * To simulate partial disk writes, we split writes into two
 743                  * system calls so that the process can be killed in between.
 744                  */
 745                 int sectors = len >> SPA_MINBLOCKSHIFT;
 746                 split = (sectors > 0 ? rand() % sectors : 0) <<
 747                     SPA_MINBLOCKSHIFT;
 748                 rc = pwrite64(vp->v_fd, addr, split, offset);
 749                 if (rc != -1) {
 750                         done = rc;
 751                         rc = pwrite64(vp->v_fd, (char *)addr + split,
 752                             len - split, offset + split);
 753                 }
 754         }
 755
 756 #ifdef __linux__
 757         if (rc == -1 && errno == EINVAL) {
 758                 /*
 759                  * Under Linux, this most likely means an alignment issue
 760                  * (memory or disk) due to O_DIRECT, so we abort() in order to
 761                  * catch the offender.
 762                  */
 763                 abort();
 764         }
 765 #endif
 766         if (rc == -1)
 767                 return (errno);
 768
 769         done += rc;
 770
 771         if (residp)
 772                 *residp = len - done;
 773         else if (done != len)
 774                 return (EIO);
 775         return (0);
 776 }
 777
 778 void
 779 vn_close(vnode_t *vp)
 780 {
 781         close(vp->v_fd);
 782         if (vp->v_dump_fd != -1)
 783                 close(vp->v_dump_fd);
 784         spa_strfree(vp->v_path);
 785         umem_free(vp, sizeof (vnode_t));
 786 }
 787
 788 /*
 789  * At a minimum we need to update the size since vdev_reopen()
 790  * will no longer call vn_openat().
 791  */
 792 int
 793 fop_getattr(vnode_t *vp, vattr_t *vap)
 794 {
 795         struct stat64 st;
 796         int err;
 797
 798         if (fstat64_blk(vp->v_fd, &st) == -1) {
 799                 err = errno;
 800                 close(vp->v_fd);
 801                 return (err);
 802         }
 803
 804         vap->va_size = st.st_size;
 805         return (0);
 806 }
 807
 808 /*
 809  * =========================================================================
 810  * Figure out which debugging statements to print
 811  * =========================================================================
 812  */
 813
 814 static char *dprintf_string;
 815 static int dprintf_print_all;
 816
 817 int
 818 dprintf_find_string(const char *string)
 819 {
 820         char *tmp_str = dprintf_string;
 821         int len = strlen(string);
 822
 823         /*
 824          * Find out if this is a string we want to print.
 825          * String format: file1.c,function_name1,file2.c,file3.c
 826          */
 827
 828         while (tmp_str != NULL) {
 829                 if (strncmp(tmp_str, string, len) == 0 &&
 830                     (tmp_str[len] == ',' || tmp_str[len] == '\0'))
 831                         return (1);
 832                 tmp_str = strchr(tmp_str, ',');
 833                 if (tmp_str != NULL)
 834                         tmp_str++; /* Get rid of , */
 835         }
 836         return (0);
 837 }
 838
 839 void
 840 dprintf_setup(int *argc, char **argv)
 841 {
 842         int i, j;
 843
 844         /*
 845          * Debugging can be specified two ways: by setting the
 846          * environment variable ZFS_DEBUG, or by including a
 847          * "debug=..."  argument on the command line.  The command
 848          * line setting overrides the environment variable.
 849          */
 850
 851         for (i = 1; i < *argc; i++) {
 852                 int len = strlen("debug=");
 853                 /* First look for a command line argument */
 854                 if (strncmp("debug=", argv[i], len) == 0) {
 855                         dprintf_string = argv[i] + len;
 856                         /* Remove from args */
 857                         for (j = i; j < *argc; j++)
 858                                 argv[j] = argv[j+1];
 859                         argv[j] = NULL;
 860                         (*argc)--;
 861                 }
 862         }
 863
 864         if (dprintf_string == NULL) {
 865                 /* Look for ZFS_DEBUG environment variable */
 866                 dprintf_string = getenv("ZFS_DEBUG");
 867         }
 868
 869         /*
 870          * Are we just turning on all debugging?
 871          */
 872         if (dprintf_find_string("on"))
 873                 dprintf_print_all = 1;
 874
 875         if (dprintf_string != NULL)
 876                 zfs_flags |= ZFS_DEBUG_DPRINTF;
 877 }
 878
 879 /*
 880  * =========================================================================
 881  * debug printfs
 882  * =========================================================================
 883  */
 884 void
 885 __dprintf(const char *file, const char *func, int line, const char *fmt, ...)
 886 {
 887         const char *newfile;
 888         va_list adx;
 889
 890         /*
 891          * Get rid of annoying "../common/" prefix to filename.
 892          */
 893         newfile = strrchr(file, '/');
 894         if (newfile != NULL) {
 895                 newfile = newfile + 1; /* Get rid of leading / */
 896         } else {
 897                 newfile = file;
 898         }
 899
 900         if (dprintf_print_all ||
 901             dprintf_find_string(newfile) ||
 902             dprintf_find_string(func)) {
 903                 /* Print out just the function name if requested */
 904                 flockfile(stdout);
 905                 if (dprintf_find_string("pid"))
 906                         (void) printf("%d ", getpid());
 907                 if (dprintf_find_string("tid"))
 908                         (void) printf("%u ", (uint_t) pthread_self());
 909                 if (dprintf_find_string("cpu"))
 910                         (void) printf("%u ", getcpuid());
 911                 if (dprintf_find_string("time"))
 912                         (void) printf("%llu ", gethrtime());
 913                 if (dprintf_find_string("long"))
 914                         (void) printf("%s, line %d: ", newfile, line);
 915                 (void) printf("%s: ", func);
 916                 va_start(adx, fmt);
 917                 (void) vprintf(fmt, adx);
 918                 va_end(adx);
 919                 funlockfile(stdout);
 920         }
 921 }
 922
 923 /*
 924  * =========================================================================
 925  * cmn_err() and panic()
 926  * =========================================================================
 927  */
 928 static char ce_prefix[CE_IGNORE][10] = { "", "NOTICE: ", "WARNING: ", "" };
 929 static char ce_suffix[CE_IGNORE][2] = { "", "\n", "\n", "" };
 930
 931 void
 932 vpanic(const char *fmt, va_list adx)
 933 {
 934         (void) fprintf(stderr, "error: ");
 935         (void) vfprintf(stderr, fmt, adx);
 936         (void) fprintf(stderr, "\n");
 937
 938         abort();        /* think of it as a "user-level crash dump" */
 939 }
 940
 941 void
 942 panic(const char *fmt, ...)
 943 {
 944         va_list adx;
 945
 946         va_start(adx, fmt);
 947         vpanic(fmt, adx);
 948         va_end(adx);
 949 }
 950
 951 void
 952 vcmn_err(int ce, const char *fmt, va_list adx)
 953 {
 954         if (ce == CE_PANIC)
 955                 vpanic(fmt, adx);
 956         if (ce != CE_NOTE) {    /* suppress noise in userland stress testing */
 957                 (void) fprintf(stderr, "%s", ce_prefix[ce]);
 958                 (void) vfprintf(stderr, fmt, adx);
 959                 (void) fprintf(stderr, "%s", ce_suffix[ce]);
 960         }
 961 }
 962
 963 /*PRINTFLIKE2*/
 964 void
 965 cmn_err(int ce, const char *fmt, ...)
 966 {
 967         va_list adx;
 968
 969         va_start(adx, fmt);
 970         vcmn_err(ce, fmt, adx);
 971         va_end(adx);
 972 }
 973
 974 /*
 975  * =========================================================================
 976  * kobj interfaces
 977  * =========================================================================
 978  */
 979 struct _buf *
 980 kobj_open_file(char *name)
 981 {
 982         struct _buf *file;
 983         vnode_t *vp;
 984
 985         /* set vp as the _fd field of the file */
 986         if (vn_openat(name, UIO_SYSSPACE, FREAD, 0, &vp, 0, 0, rootdir,
 987             -1) != 0)
 988                 return ((void *)-1UL);
 989
 990         file = umem_zalloc(sizeof (struct _buf), UMEM_NOFAIL);
 991         file->_fd = (intptr_t)vp;
 992         return (file);
 993 }
 994
 995 int
 996 kobj_read_file(struct _buf *file, char *buf, unsigned size, unsigned off)
 997 {
 998         ssize_t resid;
 999
1000         if (vn_rdwr(UIO_READ, (vnode_t *)file->_fd, buf, size, (offset_t)off,
1001             UIO_SYSSPACE, 0, 0, 0, &resid) != 0)
1002                 return (-1);
1003
1004         return (size - resid);
1005 }
1006
1007 void
1008 kobj_close_file(struct _buf *file)
1009 {
1010         vn_close((vnode_t *)file->_fd);
1011         umem_free(file, sizeof (struct _buf));
1012 }
1013
1014 int
1015 kobj_get_filesize(struct _buf *file, uint64_t *size)
1016 {
1017         struct stat64 st;
1018         vnode_t *vp = (vnode_t *)file->_fd;
1019
1020         if (fstat64(vp->v_fd, &st) == -1) {
1021                 vn_close(vp);
1022                 return (errno);
1023         }
1024         *size = st.st_size;
1025         return (0);
1026 }
1027
1028 /*
1029  * =========================================================================
1030  * misc routines
1031  * =========================================================================
1032  */
1033
1034 void
1035 delay(clock_t ticks)
1036 {
1037         poll(0, 0, ticks * (1000 / hz));
1038 }
1039
1040 /*
1041  * Find highest one bit set.
1042  *      Returns bit number + 1 of highest bit that is set, otherwise returns 0.
1043  * High order bit is 31 (or 63 in _LP64 kernel).
1044  */
1045 int
1046 highbit64(uint64_t i)
1047 {
1048         register int h = 1;
1049
1050         if (i == 0)
1051                 return (0);
1052         if (i & 0xffffffff00000000ULL) {
1053                 h += 32; i >>= 32;
1054         }
1055         if (i & 0xffff0000) {
1056                 h += 16; i >>= 16;
1057         }
1058         if (i & 0xff00) {
1059                 h += 8; i >>= 8;
1060         }
1061         if (i & 0xf0) {
1062                 h += 4; i >>= 4;
1063         }
1064         if (i & 0xc) {
1065                 h += 2; i >>= 2;
1066         }
1067         if (i & 0x2) {
1068                 h += 1;
1069         }
1070         return (h);
1071 }
1072
1073 /*
1074  * Find lowest one bit set.
1075  * Returns bit number + 1 of lowest bit that is set, otherwise returns 0.
1076  * This is basically a reimplementation of ffsll(), which is GNU specific.
1077  */
1078 int
1079 lowbit64(uint64_t i)
1080 {
1081         register int h = 64;
1082         if (i == 0)
1083                 return (0);
1084
1085         if (i & 0x00000000ffffffffULL)
1086                 h -= 32;
1087         else
1088                 i >>= 32;
1089
1090         if (i & 0x0000ffff)
1091                 h -= 16;
1092         else
1093                 i >>= 16;
1094
1095         if (i & 0x00ff)
1096                 h -= 8;
1097         else
1098                 i >>= 8;
1099
1100         if (i & 0x0f)
1101                 h -= 4;
1102         else
1103                 i >>= 4;
1104
1105         if (i & 0x3)
1106                 h -= 2;
1107         else
1108                 i >>= 2;
1109
1110         if (i & 0x1)
1111                 h -= 1;
1112
1113         return (h);
1114 }
1115
1116
1117 static int random_fd = -1, urandom_fd = -1;
1118
1119 static int
1120 random_get_bytes_common(uint8_t *ptr, size_t len, int fd)
1121 {
1122         size_t resid = len;
1123         ssize_t bytes;
1124
1125         ASSERT(fd != -1);
1126
1127         while (resid != 0) {
1128                 bytes = read(fd, ptr, resid);
1129                 ASSERT3S(bytes, >=, 0);
1130                 ptr += bytes;
1131                 resid -= bytes;
1132         }
1133
1134         return (0);
1135 }
1136
1137 int
1138 random_get_bytes(uint8_t *ptr, size_t len)
1139 {
1140         return (random_get_bytes_common(ptr, len, random_fd));
1141 }
1142
1143 int
1144 random_get_pseudo_bytes(uint8_t *ptr, size_t len)
1145 {
1146         return (random_get_bytes_common(ptr, len, urandom_fd));
1147 }
1148
1149 int
1150 ddi_strtoul(const char *hw_serial, char **nptr, int base, unsigned long *result)
1151 {
1152         char *end;
1153
1154         *result = strtoul(hw_serial, &end, base);
1155         if (*result == 0)
1156                 return (errno);
1157         return (0);
1158 }
1159
1160 int
1161 ddi_strtoull(const char *str, char **nptr, int base, u_longlong_t *result)
1162 {
1163         char *end;
1164
1165         *result = strtoull(str, &end, base);
1166         if (*result == 0)
1167                 return (errno);
1168         return (0);
1169 }
1170
1171 utsname_t *
1172 utsname(void)
1173 {
1174         return (&hw_utsname);
1175 }
1176
1177 /*
1178  * =========================================================================
1179  * kernel emulation setup & teardown
1180  * =========================================================================
1181  */
1182 static int
1183 umem_out_of_memory(void)
1184 {
1185         char errmsg[] = "out of memory -- generating core dump\n";
1186
1187         (void) fprintf(stderr, "%s", errmsg);
1188         abort();
1189         return (0);
1190 }
1191
1192 static unsigned long
1193 get_spl_hostid(void)
1194 {
1195         FILE *f;
1196         unsigned long hostid;
1197
1198         f = fopen("/sys/module/spl/parameters/spl_hostid", "r");
1199         if (!f)
1200                 return (0);
1201         if (fscanf(f, "%lu", &hostid) != 1)
1202                 hostid = 0;
1203         fclose(f);
1204         return (hostid & 0xffffffff);
1205 }
1206
1207 unsigned long
1208 get_system_hostid(void)
1209 {
1210         unsigned long system_hostid = get_spl_hostid();
1211         if (system_hostid == 0)
1212                 system_hostid = gethostid() & 0xffffffff;
1213         return (system_hostid);
1214 }
1215
1216 void
1217 kernel_init(int mode)
1218 {
1219         extern uint_t rrw_tsd_key;
1220
1221         umem_nofail_callback(umem_out_of_memory);
1222
1223         physmem = sysconf(_SC_PHYS_PAGES);
1224
1225         dprintf("physmem = %llu pages (%.2f GB)\n", physmem,
1226             (double)physmem * sysconf(_SC_PAGE_SIZE) / (1ULL << 30));
1227
1228         (void) snprintf(hw_serial, sizeof (hw_serial), "%ld",
1229             (mode & FWRITE) ? get_system_hostid() : 0);
1230
1231         VERIFY((random_fd = open("/dev/random", O_RDONLY)) != -1);
1232         VERIFY((urandom_fd = open("/dev/urandom", O_RDONLY)) != -1);
1233         VERIFY0(uname(&hw_utsname));
1234
1235         thread_init();
1236         system_taskq_init();
1237
1238         spa_init(mode);
1239
1240         fletcher_4_init();
1241
1242         tsd_create(&rrw_tsd_key, rrw_tsd_destroy);
1243 }
1244
1245 void
1246 kernel_fini(void)
1247 {
1248         fletcher_4_fini();
1249         spa_fini();
1250
1251         system_taskq_fini();
1252         thread_fini();
1253
1254         close(random_fd);
1255         close(urandom_fd);
1256
1257         random_fd = -1;
1258         urandom_fd = -1;
1259 }
1260
1261 uid_t
1262 crgetuid(cred_t *cr)
1263 {
1264         return (0);
1265 }
1266
1267 uid_t
1268 crgetruid(cred_t *cr)
1269 {
1270         return (0);
1271 }
1272
1273 gid_t
1274 crgetgid(cred_t *cr)
1275 {
1276         return (0);
1277 }
1278
1279 int
1280 crgetngroups(cred_t *cr)
1281 {
1282         return (0);
1283 }
1284
1285 gid_t *
1286 crgetgroups(cred_t *cr)
1287 {
1288         return (NULL);
1289 }
1290
1291 int
1292 zfs_secpolicy_snapshot_perms(const char *name, cred_t *cr)
1293 {
1294         return (0);
1295 }
1296
1297 int
1298 zfs_secpolicy_rename_perms(const char *from, const char *to, cred_t *cr)
1299 {
1300         return (0);
1301 }
1302
1303 int
1304 zfs_secpolicy_destroy_perms(const char *name, cred_t *cr)
1305 {
1306         return (0);
1307 }
1308
1309 ksiddomain_t *
1310 ksid_lookupdomain(const char *dom)
1311 {
1312         ksiddomain_t *kd;
1313
1314         kd = umem_zalloc(sizeof (ksiddomain_t), UMEM_NOFAIL);
1315         kd->kd_name = spa_strdup(dom);
1316         return (kd);
1317 }
1318
1319 void
1320 ksiddomain_rele(ksiddomain_t *ksid)
1321 {
1322         spa_strfree(ksid->kd_name);
1323         umem_free(ksid, sizeof (ksiddomain_t));
1324 }
1325
1326 char *
1327 kmem_vasprintf(const char *fmt, va_list adx)
1328 {
1329         char *buf = NULL;
1330         va_list adx_copy;
1331
1332         va_copy(adx_copy, adx);
1333         VERIFY(vasprintf(&buf, fmt, adx_copy) != -1);
1334         va_end(adx_copy);
1335
1336         return (buf);
1337 }
1338
1339 char *
1340 kmem_asprintf(const char *fmt, ...)
1341 {
1342         char *buf = NULL;
1343         va_list adx;
1344
1345         va_start(adx, fmt);
1346         VERIFY(vasprintf(&buf, fmt, adx) != -1);
1347         va_end(adx);
1348
1349         return (buf);
1350 }
1351
1352 /* ARGSUSED */
1353 int
1354 zfs_onexit_fd_hold(int fd, minor_t *minorp)
1355 {
1356         *minorp = 0;
1357         return (0);
1358 }
1359
1360 /* ARGSUSED */
1361 void
1362 zfs_onexit_fd_rele(int fd)
1363 {
1364 }
1365
1366 /* ARGSUSED */
1367 int
1368 zfs_onexit_add_cb(minor_t minor, void (*func)(void *), void *data,
1369     uint64_t *action_handle)
1370 {
1371         return (0);
1372 }
1373
1374 /* ARGSUSED */
1375 int
1376 zfs_onexit_del_cb(minor_t minor, uint64_t action_handle, boolean_t fire)
1377 {
1378         return (0);
1379 }
1380
1381 /* ARGSUSED */
1382 int
1383 zfs_onexit_cb_data(minor_t minor, uint64_t action_handle, void **data)
1384 {
1385         return (0);
1386 }
1387
1388 fstrans_cookie_t
1389 spl_fstrans_mark(void)
1390 {
1391         return ((fstrans_cookie_t) 0);
1392 }
1393
1394 void
1395 spl_fstrans_unmark(fstrans_cookie_t cookie)
1396 {
1397 }
1398
1399 int
1400 spl_fstrans_check(void)
1401 {
1402         return (0);
1403 }
1404
1405 void
1406 zvol_create_minors(spa_t *spa, const char *name, boolean_t async)
1407 {
1408 }
1409
1410 void
1411 zvol_remove_minor(spa_t *spa, const char *name, boolean_t async)
1412 {
1413 }
1414
1415 void
1416 zvol_remove_minors(spa_t *spa, const char *name, boolean_t async)
1417 {
1418 }
1419
1420 void
1421 zvol_rename_minors(spa_t *spa, const char *oldname, const char *newname,
1422     boolean_t async)
1423 {
1424 }