lib/libzpool/kernel.c

   1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  23  */
  24
  25 #include <assert.h>
  26 #include <fcntl.h>
  27 #include <poll.h>
  28 #include <stdio.h>
  29 #include <stdlib.h>
  30 #include <string.h>
  31 #include <zlib.h>
  32 #include <sys/signal.h>
  33 #include <sys/spa.h>
  34 #include <sys/stat.h>
  35 #include <sys/processor.h>
  36 #include <sys/zfs_context.h>
  37 #include <sys/rrwlock.h>
  38 #include <sys/utsname.h>
  39 #include <sys/time.h>
  40 #include <sys/systeminfo.h>
  41
  42 /*
  43  * Emulation of kernel services in userland.
  44  */
  45
  46 int aok;
  47 uint64_t physmem;
  48 vnode_t *rootdir = (vnode_t *)0xabcd1234;
  49 char hw_serial[HW_HOSTID_LEN];
  50 struct utsname hw_utsname;
  51 vmem_t *zio_arena = NULL;
  52
  53 /* this only exists to have its address taken */
  54 struct proc p0;
  55
  56 /*
  57  * =========================================================================
  58  * threads
  59  * =========================================================================
  60  */
  61
  62 pthread_cond_t kthread_cond = PTHREAD_COND_INITIALIZER;
  63 pthread_mutex_t kthread_lock = PTHREAD_MUTEX_INITIALIZER;
  64 pthread_key_t kthread_key;
  65 int kthread_nr = 0;
  66
  67 static void
  68 thread_init(void)
  69 {
  70         kthread_t *kt;
  71
  72         VERIFY3S(pthread_key_create(&kthread_key, NULL), ==, 0);
  73
  74         /* Create entry for primary kthread */
  75         kt = umem_zalloc(sizeof (kthread_t), UMEM_NOFAIL);
  76         kt->t_tid = pthread_self();
  77         kt->t_func = NULL;
  78
  79         VERIFY3S(pthread_setspecific(kthread_key, kt), ==, 0);
  80
  81         /* Only the main thread should be running at the moment */
  82         ASSERT3S(kthread_nr, ==, 0);
  83         kthread_nr = 1;
  84 }
  85
  86 static void
  87 thread_fini(void)
  88 {
  89         kthread_t *kt = curthread;
  90
  91         ASSERT(pthread_equal(kt->t_tid, pthread_self()));
  92         ASSERT3P(kt->t_func, ==, NULL);
  93
  94         umem_free(kt, sizeof (kthread_t));
  95
  96         /* Wait for all threads to exit via thread_exit() */
  97         VERIFY3S(pthread_mutex_lock(&kthread_lock), ==, 0);
  98
  99         kthread_nr--; /* Main thread is exiting */
 100
 101         while (kthread_nr > 0)
 102                 VERIFY3S(pthread_cond_wait(&kthread_cond, &kthread_lock), ==,
 103                     0);
 104
 105         ASSERT3S(kthread_nr, ==, 0);
 106         VERIFY3S(pthread_mutex_unlock(&kthread_lock), ==, 0);
 107
 108         VERIFY3S(pthread_key_delete(kthread_key), ==, 0);
 109 }
 110
 111 kthread_t *
 112 zk_thread_current(void)
 113 {
 114         kthread_t *kt = pthread_getspecific(kthread_key);
 115
 116         ASSERT3P(kt, !=, NULL);
 117
 118         return (kt);
 119 }
 120
 121 void *
 122 zk_thread_helper(void *arg)
 123 {
 124         kthread_t *kt = (kthread_t *) arg;
 125
 126         VERIFY3S(pthread_setspecific(kthread_key, kt), ==, 0);
 127
 128         VERIFY3S(pthread_mutex_lock(&kthread_lock), ==, 0);
 129         kthread_nr++;
 130         VERIFY3S(pthread_mutex_unlock(&kthread_lock), ==, 0);
 131         (void) setpriority(PRIO_PROCESS, 0, kt->t_pri);
 132
 133         kt->t_tid = pthread_self();
 134         ((thread_func_arg_t) kt->t_func)(kt->t_arg);
 135
 136         /* Unreachable, thread must exit with thread_exit() */
 137         abort();
 138
 139         return (NULL);
 140 }
 141
 142 kthread_t *
 143 zk_thread_create(caddr_t stk, size_t stksize, thread_func_t func, void *arg,
 144     size_t len, proc_t *pp, int state, pri_t pri, int detachstate)
 145 {
 146         kthread_t *kt;
 147         pthread_attr_t attr;
 148         char *stkstr;
 149
 150         ASSERT0(state & ~TS_RUN);
 151
 152         kt = umem_zalloc(sizeof (kthread_t), UMEM_NOFAIL);
 153         kt->t_func = func;
 154         kt->t_arg = arg;
 155         kt->t_pri = pri;
 156
 157         VERIFY0(pthread_attr_init(&attr));
 158         VERIFY0(pthread_attr_setdetachstate(&attr, detachstate));
 159
 160         /*
 161          * We allow the default stack size in user space to be specified by
 162          * setting the ZFS_STACK_SIZE environment variable.  This allows us
 163          * the convenience of observing and debugging stack overruns in
 164          * user space.  Explicitly specified stack sizes will be honored.
 165          * The usage of ZFS_STACK_SIZE is discussed further in the
 166          * ENVIRONMENT VARIABLES sections of the ztest(1) man page.
 167          */
 168         if (stksize == 0) {
 169                 stkstr = getenv("ZFS_STACK_SIZE");
 170
 171                 if (stkstr == NULL)
 172                         stksize = TS_STACK_MAX;
 173                 else
 174                         stksize = MAX(atoi(stkstr), TS_STACK_MIN);
 175         }
 176
 177         VERIFY3S(stksize, >, 0);
 178         stksize = P2ROUNDUP(MAX(stksize, TS_STACK_MIN), PAGESIZE);
 179         VERIFY0(pthread_attr_setstacksize(&attr, stksize));
 180         VERIFY0(pthread_attr_setguardsize(&attr, PAGESIZE));
 181
 182         VERIFY0(pthread_create(&kt->t_tid, &attr, &zk_thread_helper, kt));
 183         VERIFY0(pthread_attr_destroy(&attr));
 184
 185         return (kt);
 186 }
 187
 188 void
 189 zk_thread_exit(void)
 190 {
 191         kthread_t *kt = curthread;
 192
 193         ASSERT(pthread_equal(kt->t_tid, pthread_self()));
 194
 195         umem_free(kt, sizeof (kthread_t));
 196
 197         pthread_mutex_lock(&kthread_lock);
 198         kthread_nr--;
 199         pthread_mutex_unlock(&kthread_lock);
 200
 201         pthread_cond_broadcast(&kthread_cond);
 202         pthread_exit((void *)TS_MAGIC);
 203 }
 204
 205 void
 206 zk_thread_join(kt_did_t tid)
 207 {
 208         void *ret;
 209
 210         pthread_join((pthread_t)tid, &ret);
 211         VERIFY3P(ret, ==, (void *)TS_MAGIC);
 212 }
 213
 214 /*
 215  * =========================================================================
 216  * kstats
 217  * =========================================================================
 218  */
 219 /*ARGSUSED*/
 220 kstat_t *
 221 kstat_create(const char *module, int instance, const char *name,
 222     const char *class, uchar_t type, ulong_t ndata, uchar_t ks_flag)
 223 {
 224         return (NULL);
 225 }
 226
 227 /*ARGSUSED*/
 228 void
 229 kstat_install(kstat_t *ksp)
 230 {}
 231
 232 /*ARGSUSED*/
 233 void
 234 kstat_delete(kstat_t *ksp)
 235 {}
 236
 237 /*ARGSUSED*/
 238 void
 239 kstat_waitq_enter(kstat_io_t *kiop)
 240 {}
 241
 242 /*ARGSUSED*/
 243 void
 244 kstat_waitq_exit(kstat_io_t *kiop)
 245 {}
 246
 247 /*ARGSUSED*/
 248 void
 249 kstat_runq_enter(kstat_io_t *kiop)
 250 {}
 251
 252 /*ARGSUSED*/
 253 void
 254 kstat_runq_exit(kstat_io_t *kiop)
 255 {}
 256
 257 /*ARGSUSED*/
 258 void
 259 kstat_waitq_to_runq(kstat_io_t *kiop)
 260 {}
 261
 262 /*ARGSUSED*/
 263 void
 264 kstat_runq_back_to_waitq(kstat_io_t *kiop)
 265 {}
 266
 267 void
 268 kstat_set_raw_ops(kstat_t *ksp,
 269     int (*headers)(char *buf, size_t size),
 270     int (*data)(char *buf, size_t size, void *data),
 271     void *(*addr)(kstat_t *ksp, loff_t index))
 272 {}
 273
 274 /*
 275  * =========================================================================
 276  * mutexes
 277  * =========================================================================
 278  */
 279
 280 void
 281 mutex_init(kmutex_t *mp, char *name, int type, void *cookie)
 282 {
 283         ASSERT3S(type, ==, MUTEX_DEFAULT);
 284         ASSERT3P(cookie, ==, NULL);
 285         mp->m_owner = MTX_INIT;
 286         mp->m_magic = MTX_MAGIC;
 287         VERIFY3S(pthread_mutex_init(&mp->m_lock, NULL), ==, 0);
 288 }
 289
 290 void
 291 mutex_destroy(kmutex_t *mp)
 292 {
 293         ASSERT3U(mp->m_magic, ==, MTX_MAGIC);
 294         ASSERT3P(mp->m_owner, ==, MTX_INIT);
 295         ASSERT0(pthread_mutex_destroy(&(mp)->m_lock));
 296         mp->m_owner = MTX_DEST;
 297         mp->m_magic = 0;
 298 }
 299
 300 void
 301 mutex_enter(kmutex_t *mp)
 302 {
 303         ASSERT3U(mp->m_magic, ==, MTX_MAGIC);
 304         ASSERT3P(mp->m_owner, !=, MTX_DEST);
 305         ASSERT3P(mp->m_owner, !=, curthread);
 306         VERIFY3S(pthread_mutex_lock(&mp->m_lock), ==, 0);
 307         ASSERT3P(mp->m_owner, ==, MTX_INIT);
 308         mp->m_owner = curthread;
 309 }
 310
 311 int
 312 mutex_tryenter(kmutex_t *mp)
 313 {
 314         ASSERT3U(mp->m_magic, ==, MTX_MAGIC);
 315         ASSERT3P(mp->m_owner, !=, MTX_DEST);
 316         if (0 == pthread_mutex_trylock(&mp->m_lock)) {
 317                 ASSERT3P(mp->m_owner, ==, MTX_INIT);
 318                 mp->m_owner = curthread;
 319                 return (1);
 320         } else {
 321                 return (0);
 322         }
 323 }
 324
 325 void
 326 mutex_exit(kmutex_t *mp)
 327 {
 328         ASSERT3U(mp->m_magic, ==, MTX_MAGIC);
 329         ASSERT3P(mutex_owner(mp), ==, curthread);
 330         mp->m_owner = MTX_INIT;
 331         VERIFY3S(pthread_mutex_unlock(&mp->m_lock), ==, 0);
 332 }
 333
 334 void *
 335 mutex_owner(kmutex_t *mp)
 336 {
 337         ASSERT3U(mp->m_magic, ==, MTX_MAGIC);
 338         return (mp->m_owner);
 339 }
 340
 341 int
 342 mutex_held(kmutex_t *mp)
 343 {
 344         return (mp->m_owner == curthread);
 345 }
 346
 347 /*
 348  * =========================================================================
 349  * rwlocks
 350  * =========================================================================
 351  */
 352
 353 void
 354 rw_init(krwlock_t *rwlp, char *name, int type, void *arg)
 355 {
 356         ASSERT3S(type, ==, RW_DEFAULT);
 357         ASSERT3P(arg, ==, NULL);
 358         VERIFY3S(pthread_rwlock_init(&rwlp->rw_lock, NULL), ==, 0);
 359         rwlp->rw_owner = RW_INIT;
 360         rwlp->rw_wr_owner = RW_INIT;
 361         rwlp->rw_readers = 0;
 362         rwlp->rw_magic = RW_MAGIC;
 363 }
 364
 365 void
 366 rw_destroy(krwlock_t *rwlp)
 367 {
 368         ASSERT3U(rwlp->rw_magic, ==, RW_MAGIC);
 369         ASSERT(rwlp->rw_readers == 0 && rwlp->rw_wr_owner == RW_INIT);
 370         VERIFY3S(pthread_rwlock_destroy(&rwlp->rw_lock), ==, 0);
 371         rwlp->rw_magic = 0;
 372 }
 373
 374 void
 375 rw_enter(krwlock_t *rwlp, krw_t rw)
 376 {
 377         ASSERT3U(rwlp->rw_magic, ==, RW_MAGIC);
 378         ASSERT3P(rwlp->rw_owner, !=, curthread);
 379         ASSERT3P(rwlp->rw_wr_owner, !=, curthread);
 380
 381         if (rw == RW_READER) {
 382                 VERIFY3S(pthread_rwlock_rdlock(&rwlp->rw_lock), ==, 0);
 383                 ASSERT3P(rwlp->rw_wr_owner, ==, RW_INIT);
 384
 385                 atomic_inc_uint(&rwlp->rw_readers);
 386         } else {
 387                 VERIFY3S(pthread_rwlock_wrlock(&rwlp->rw_lock), ==, 0);
 388                 ASSERT3P(rwlp->rw_wr_owner, ==, RW_INIT);
 389                 ASSERT3U(rwlp->rw_readers, ==, 0);
 390
 391                 rwlp->rw_wr_owner = curthread;
 392         }
 393
 394         rwlp->rw_owner = curthread;
 395 }
 396
 397 void
 398 rw_exit(krwlock_t *rwlp)
 399 {
 400         ASSERT3U(rwlp->rw_magic, ==, RW_MAGIC);
 401         ASSERT(RW_LOCK_HELD(rwlp));
 402
 403         if (RW_READ_HELD(rwlp))
 404                 atomic_dec_uint(&rwlp->rw_readers);
 405         else
 406                 rwlp->rw_wr_owner = RW_INIT;
 407
 408         rwlp->rw_owner = RW_INIT;
 409         VERIFY3S(pthread_rwlock_unlock(&rwlp->rw_lock), ==, 0);
 410 }
 411
 412 int
 413 rw_tryenter(krwlock_t *rwlp, krw_t rw)
 414 {
 415         int rv;
 416
 417         ASSERT3U(rwlp->rw_magic, ==, RW_MAGIC);
 418
 419         if (rw == RW_READER)
 420                 rv = pthread_rwlock_tryrdlock(&rwlp->rw_lock);
 421         else
 422                 rv = pthread_rwlock_trywrlock(&rwlp->rw_lock);
 423
 424         if (rv == 0) {
 425                 ASSERT3P(rwlp->rw_wr_owner, ==, RW_INIT);
 426
 427                 if (rw == RW_READER)
 428                         atomic_inc_uint(&rwlp->rw_readers);
 429                 else {
 430                         ASSERT3U(rwlp->rw_readers, ==, 0);
 431                         rwlp->rw_wr_owner = curthread;
 432                 }
 433
 434                 rwlp->rw_owner = curthread;
 435                 return (1);
 436         }
 437
 438         VERIFY3S(rv, ==, EBUSY);
 439
 440         return (0);
 441 }
 442
 443 int
 444 rw_tryupgrade(krwlock_t *rwlp)
 445 {
 446         ASSERT3U(rwlp->rw_magic, ==, RW_MAGIC);
 447
 448         return (0);
 449 }
 450
 451 /*
 452  * =========================================================================
 453  * condition variables
 454  * =========================================================================
 455  */
 456
 457 void
 458 cv_init(kcondvar_t *cv, char *name, int type, void *arg)
 459 {
 460         ASSERT3S(type, ==, CV_DEFAULT);
 461         cv->cv_magic = CV_MAGIC;
 462         VERIFY3S(pthread_cond_init(&cv->cv, NULL), ==, 0);
 463 }
 464
 465 void
 466 cv_destroy(kcondvar_t *cv)
 467 {
 468         ASSERT3U(cv->cv_magic, ==, CV_MAGIC);
 469         VERIFY3S(pthread_cond_destroy(&cv->cv), ==, 0);
 470         cv->cv_magic = 0;
 471 }
 472
 473 void
 474 cv_wait(kcondvar_t *cv, kmutex_t *mp)
 475 {
 476         ASSERT3U(cv->cv_magic, ==, CV_MAGIC);
 477         ASSERT3P(mutex_owner(mp), ==, curthread);
 478         mp->m_owner = MTX_INIT;
 479         int ret = pthread_cond_wait(&cv->cv, &mp->m_lock);
 480         if (ret != 0)
 481                 VERIFY3S(ret, ==, EINTR);
 482         mp->m_owner = curthread;
 483 }
 484
 485 clock_t
 486 cv_timedwait(kcondvar_t *cv, kmutex_t *mp, clock_t abstime)
 487 {
 488         int error;
 489         struct timeval tv;
 490         timestruc_t ts;
 491         clock_t delta;
 492
 493         ASSERT3U(cv->cv_magic, ==, CV_MAGIC);
 494
 495 top:
 496         delta = abstime - ddi_get_lbolt();
 497         if (delta <= 0)
 498                 return (-1);
 499
 500         VERIFY(gettimeofday(&tv, NULL) == 0);
 501
 502         ts.tv_sec = tv.tv_sec + delta / hz;
 503         ts.tv_nsec = tv.tv_usec * 1000 + (delta % hz) * (NANOSEC / hz);
 504         if (ts.tv_nsec >= NANOSEC) {
 505                 ts.tv_sec++;
 506                 ts.tv_nsec -= NANOSEC;
 507         }
 508
 509         ASSERT3P(mutex_owner(mp), ==, curthread);
 510         mp->m_owner = MTX_INIT;
 511         error = pthread_cond_timedwait(&cv->cv, &mp->m_lock, &ts);
 512         mp->m_owner = curthread;
 513
 514         if (error == ETIMEDOUT)
 515                 return (-1);
 516
 517         if (error == EINTR)
 518                 goto top;
 519
 520         VERIFY3S(error, ==, 0);
 521
 522         return (1);
 523 }
 524
 525 /*ARGSUSED*/
 526 clock_t
 527 cv_timedwait_hires(kcondvar_t *cv, kmutex_t *mp, hrtime_t tim, hrtime_t res,
 528     int flag)
 529 {
 530         int error;
 531         timestruc_t ts;
 532         hrtime_t delta;
 533
 534         ASSERT(flag == 0);
 535
 536 top:
 537         delta = tim - gethrtime();
 538         if (delta <= 0)
 539                 return (-1);
 540
 541         ts.tv_sec = delta / NANOSEC;
 542         ts.tv_nsec = delta % NANOSEC;
 543
 544         ASSERT(mutex_owner(mp) == curthread);
 545         mp->m_owner = NULL;
 546         error = pthread_cond_timedwait(&cv->cv, &mp->m_lock, &ts);
 547         mp->m_owner = curthread;
 548
 549         if (error == ETIME)
 550                 return (-1);
 551
 552         if (error == EINTR)
 553                 goto top;
 554
 555         ASSERT(error == 0);
 556
 557         return (1);
 558 }
 559
 560 void
 561 cv_signal(kcondvar_t *cv)
 562 {
 563         ASSERT3U(cv->cv_magic, ==, CV_MAGIC);
 564         VERIFY3S(pthread_cond_signal(&cv->cv), ==, 0);
 565 }
 566
 567 void
 568 cv_broadcast(kcondvar_t *cv)
 569 {
 570         ASSERT3U(cv->cv_magic, ==, CV_MAGIC);
 571         VERIFY3S(pthread_cond_broadcast(&cv->cv), ==, 0);
 572 }
 573
 574 /*
 575  * =========================================================================
 576  * vnode operations
 577  * =========================================================================
 578  */
 579 /*
 580  * Note: for the xxxat() versions of these functions, we assume that the
 581  * starting vp is always rootdir (which is true for spa_directory.c, the only
 582  * ZFS consumer of these interfaces).  We assert this is true, and then emulate
 583  * them by adding '/' in front of the path.
 584  */
 585
 586 /*ARGSUSED*/
 587 int
 588 vn_open(char *path, int x1, int flags, int mode, vnode_t **vpp, int x2, int x3)
 589 {
 590         int fd;
 591         vnode_t *vp;
 592         int old_umask = 0;
 593         char *realpath;
 594         struct stat64 st;
 595         int err;
 596
 597         realpath = umem_alloc(MAXPATHLEN, UMEM_NOFAIL);
 598
 599         /*
 600          * If we're accessing a real disk from userland, we need to use
 601          * the character interface to avoid caching.  This is particularly
 602          * important if we're trying to look at a real in-kernel storage
 603          * pool from userland, e.g. via zdb, because otherwise we won't
 604          * see the changes occurring under the segmap cache.
 605          * On the other hand, the stupid character device returns zero
 606          * for its size.  So -- gag -- we open the block device to get
 607          * its size, and remember it for subsequent VOP_GETATTR().
 608          */
 609 #if defined(__sun__) || defined(__sun)
 610         if (strncmp(path, "/dev/", 5) == 0) {
 611 #else
 612         if (0) {
 613 #endif
 614                 char *dsk;
 615                 fd = open64(path, O_RDONLY);
 616                 if (fd == -1) {
 617                         err = errno;
 618                         free(realpath);
 619                         return (err);
 620                 }
 621                 if (fstat64(fd, &st) == -1) {
 622                         err = errno;
 623                         close(fd);
 624                         free(realpath);
 625                         return (err);
 626                 }
 627                 close(fd);
 628                 (void) sprintf(realpath, "%s", path);
 629                 dsk = strstr(path, "/dsk/");
 630                 if (dsk != NULL)
 631                         (void) sprintf(realpath + (dsk - path) + 1, "r%s",
 632                             dsk + 1);
 633         } else {
 634                 (void) sprintf(realpath, "%s", path);
 635                 if (!(flags & FCREAT) && stat64(realpath, &st) == -1) {
 636                         err = errno;
 637                         free(realpath);
 638                         return (err);
 639                 }
 640         }
 641
 642         if (!(flags & FCREAT) && S_ISBLK(st.st_mode)) {
 643 #ifdef __linux__
 644                 flags |= O_DIRECT;
 645 #endif
 646                 /* We shouldn't be writing to block devices in userspace */
 647                 VERIFY(!(flags & FWRITE));
 648         }
 649
 650         if (flags & FCREAT)
 651                 old_umask = umask(0);
 652
 653         /*
 654          * The construct 'flags - FREAD' conveniently maps combinations of
 655          * FREAD and FWRITE to the corresponding O_RDONLY, O_WRONLY, and O_RDWR.
 656          */
 657         fd = open64(realpath, flags - FREAD, mode);
 658         free(realpath);
 659
 660         if (flags & FCREAT)
 661                 (void) umask(old_umask);
 662
 663         if (fd == -1)
 664                 return (errno);
 665
 666         if (fstat64_blk(fd, &st) == -1) {
 667                 err = errno;
 668                 close(fd);
 669                 return (err);
 670         }
 671
 672         (void) fcntl(fd, F_SETFD, FD_CLOEXEC);
 673
 674         *vpp = vp = umem_zalloc(sizeof (vnode_t), UMEM_NOFAIL);
 675
 676         vp->v_fd = fd;
 677         vp->v_size = st.st_size;
 678         vp->v_path = spa_strdup(path);
 679
 680         return (0);
 681 }
 682
 683 /*ARGSUSED*/
 684 int
 685 vn_openat(char *path, int x1, int flags, int mode, vnode_t **vpp, int x2,
 686     int x3, vnode_t *startvp, int fd)
 687 {
 688         char *realpath = umem_alloc(strlen(path) + 2, UMEM_NOFAIL);
 689         int ret;
 690
 691         ASSERT(startvp == rootdir);
 692         (void) sprintf(realpath, "/%s", path);
 693
 694         /* fd ignored for now, need if want to simulate nbmand support */
 695         ret = vn_open(realpath, x1, flags, mode, vpp, x2, x3);
 696
 697         umem_free(realpath, strlen(path) + 2);
 698
 699         return (ret);
 700 }
 701
 702 /*ARGSUSED*/
 703 int
 704 vn_rdwr(int uio, vnode_t *vp, void *addr, ssize_t len, offset_t offset,
 705         int x1, int x2, rlim64_t x3, void *x4, ssize_t *residp)
 706 {
 707         ssize_t rc, done = 0, split;
 708
 709         if (uio == UIO_READ) {
 710                 rc = pread64(vp->v_fd, addr, len, offset);
 711         } else {
 712                 /*
 713                  * To simulate partial disk writes, we split writes into two
 714                  * system calls so that the process can be killed in between.
 715                  */
 716                 int sectors = len >> SPA_MINBLOCKSHIFT;
 717                 split = (sectors > 0 ? rand() % sectors : 0) <<
 718                     SPA_MINBLOCKSHIFT;
 719                 rc = pwrite64(vp->v_fd, addr, split, offset);
 720                 if (rc != -1) {
 721                         done = rc;
 722                         rc = pwrite64(vp->v_fd, (char *)addr + split,
 723                             len - split, offset + split);
 724                 }
 725         }
 726
 727 #ifdef __linux__
 728         if (rc == -1 && errno == EINVAL) {
 729                 /*
 730                  * Under Linux, this most likely means an alignment issue
 731                  * (memory or disk) due to O_DIRECT, so we abort() in order to
 732                  * catch the offender.
 733                  */
 734                 abort();
 735         }
 736 #endif
 737         if (rc == -1)
 738                 return (errno);
 739
 740         done += rc;
 741
 742         if (residp)
 743                 *residp = len - done;
 744         else if (done != len)
 745                 return (EIO);
 746         return (0);
 747 }
 748
 749 void
 750 vn_close(vnode_t *vp)
 751 {
 752         close(vp->v_fd);
 753         spa_strfree(vp->v_path);
 754         umem_free(vp, sizeof (vnode_t));
 755 }
 756
 757 /*
 758  * At a minimum we need to update the size since vdev_reopen()
 759  * will no longer call vn_openat().
 760  */
 761 int
 762 fop_getattr(vnode_t *vp, vattr_t *vap)
 763 {
 764         struct stat64 st;
 765         int err;
 766
 767         if (fstat64_blk(vp->v_fd, &st) == -1) {
 768                 err = errno;
 769                 close(vp->v_fd);
 770                 return (err);
 771         }
 772
 773         vap->va_size = st.st_size;
 774         return (0);
 775 }
 776
 777 /*
 778  * =========================================================================
 779  * Figure out which debugging statements to print
 780  * =========================================================================
 781  */
 782
 783 static char *dprintf_string;
 784 static int dprintf_print_all;
 785
 786 int
 787 dprintf_find_string(const char *string)
 788 {
 789         char *tmp_str = dprintf_string;
 790         int len = strlen(string);
 791
 792         /*
 793          * Find out if this is a string we want to print.
 794          * String format: file1.c,function_name1,file2.c,file3.c
 795          */
 796
 797         while (tmp_str != NULL) {
 798                 if (strncmp(tmp_str, string, len) == 0 &&
 799                     (tmp_str[len] == ',' || tmp_str[len] == '\0'))
 800                         return (1);
 801                 tmp_str = strchr(tmp_str, ',');
 802                 if (tmp_str != NULL)
 803                         tmp_str++; /* Get rid of , */
 804         }
 805         return (0);
 806 }
 807
 808 void
 809 dprintf_setup(int *argc, char **argv)
 810 {
 811         int i, j;
 812
 813         /*
 814          * Debugging can be specified two ways: by setting the
 815          * environment variable ZFS_DEBUG, or by including a
 816          * "debug=..."  argument on the command line.  The command
 817          * line setting overrides the environment variable.
 818          */
 819
 820         for (i = 1; i < *argc; i++) {
 821                 int len = strlen("debug=");
 822                 /* First look for a command line argument */
 823                 if (strncmp("debug=", argv[i], len) == 0) {
 824                         dprintf_string = argv[i] + len;
 825                         /* Remove from args */
 826                         for (j = i; j < *argc; j++)
 827                                 argv[j] = argv[j+1];
 828                         argv[j] = NULL;
 829                         (*argc)--;
 830                 }
 831         }
 832
 833         if (dprintf_string == NULL) {
 834                 /* Look for ZFS_DEBUG environment variable */
 835                 dprintf_string = getenv("ZFS_DEBUG");
 836         }
 837
 838         /*
 839          * Are we just turning on all debugging?
 840          */
 841         if (dprintf_find_string("on"))
 842                 dprintf_print_all = 1;
 843
 844         if (dprintf_string != NULL)
 845                 zfs_flags |= ZFS_DEBUG_DPRINTF;
 846 }
 847
 848 /*
 849  * =========================================================================
 850  * debug printfs
 851  * =========================================================================
 852  */
 853 void
 854 __dprintf(const char *file, const char *func, int line, const char *fmt, ...)
 855 {
 856         const char *newfile;
 857         va_list adx;
 858
 859         /*
 860          * Get rid of annoying "../common/" prefix to filename.
 861          */
 862         newfile = strrchr(file, '/');
 863         if (newfile != NULL) {
 864                 newfile = newfile + 1; /* Get rid of leading / */
 865         } else {
 866                 newfile = file;
 867         }
 868
 869         if (dprintf_print_all ||
 870             dprintf_find_string(newfile) ||
 871             dprintf_find_string(func)) {
 872                 /* Print out just the function name if requested */
 873                 flockfile(stdout);
 874                 if (dprintf_find_string("pid"))
 875                         (void) printf("%d ", getpid());
 876                 if (dprintf_find_string("tid"))
 877                         (void) printf("%u ", (uint_t) pthread_self());
 878                 if (dprintf_find_string("cpu"))
 879                         (void) printf("%u ", getcpuid());
 880                 if (dprintf_find_string("time"))
 881                         (void) printf("%llu ", gethrtime());
 882                 if (dprintf_find_string("long"))
 883                         (void) printf("%s, line %d: ", newfile, line);
 884                 (void) printf("%s: ", func);
 885                 va_start(adx, fmt);
 886                 (void) vprintf(fmt, adx);
 887                 va_end(adx);
 888                 funlockfile(stdout);
 889         }
 890 }
 891
 892 /*
 893  * =========================================================================
 894  * cmn_err() and panic()
 895  * =========================================================================
 896  */
 897 static char ce_prefix[CE_IGNORE][10] = { "", "NOTICE: ", "WARNING: ", "" };
 898 static char ce_suffix[CE_IGNORE][2] = { "", "\n", "\n", "" };
 899
 900 void
 901 vpanic(const char *fmt, va_list adx)
 902 {
 903         (void) fprintf(stderr, "error: ");
 904         (void) vfprintf(stderr, fmt, adx);
 905         (void) fprintf(stderr, "\n");
 906
 907         abort();        /* think of it as a "user-level crash dump" */
 908 }
 909
 910 void
 911 panic(const char *fmt, ...)
 912 {
 913         va_list adx;
 914
 915         va_start(adx, fmt);
 916         vpanic(fmt, adx);
 917         va_end(adx);
 918 }
 919
 920 void
 921 vcmn_err(int ce, const char *fmt, va_list adx)
 922 {
 923         if (ce == CE_PANIC)
 924                 vpanic(fmt, adx);
 925         if (ce != CE_NOTE) {    /* suppress noise in userland stress testing */
 926                 (void) fprintf(stderr, "%s", ce_prefix[ce]);
 927                 (void) vfprintf(stderr, fmt, adx);
 928                 (void) fprintf(stderr, "%s", ce_suffix[ce]);
 929         }
 930 }
 931
 932 /*PRINTFLIKE2*/
 933 void
 934 cmn_err(int ce, const char *fmt, ...)
 935 {
 936         va_list adx;
 937
 938         va_start(adx, fmt);
 939         vcmn_err(ce, fmt, adx);
 940         va_end(adx);
 941 }
 942
 943 /*
 944  * =========================================================================
 945  * kobj interfaces
 946  * =========================================================================
 947  */
 948 struct _buf *
 949 kobj_open_file(char *name)
 950 {
 951         struct _buf *file;
 952         vnode_t *vp;
 953
 954         /* set vp as the _fd field of the file */
 955         if (vn_openat(name, UIO_SYSSPACE, FREAD, 0, &vp, 0, 0, rootdir,
 956             -1) != 0)
 957                 return ((void *)-1UL);
 958
 959         file = umem_zalloc(sizeof (struct _buf), UMEM_NOFAIL);
 960         file->_fd = (intptr_t)vp;
 961         return (file);
 962 }
 963
 964 int
 965 kobj_read_file(struct _buf *file, char *buf, unsigned size, unsigned off)
 966 {
 967         ssize_t resid;
 968
 969         vn_rdwr(UIO_READ, (vnode_t *)file->_fd, buf, size, (offset_t)off,
 970             UIO_SYSSPACE, 0, 0, 0, &resid);
 971
 972         return (size - resid);
 973 }
 974
 975 void
 976 kobj_close_file(struct _buf *file)
 977 {
 978         vn_close((vnode_t *)file->_fd);
 979         umem_free(file, sizeof (struct _buf));
 980 }
 981
 982 int
 983 kobj_get_filesize(struct _buf *file, uint64_t *size)
 984 {
 985         struct stat64 st;
 986         vnode_t *vp = (vnode_t *)file->_fd;
 987
 988         if (fstat64(vp->v_fd, &st) == -1) {
 989                 vn_close(vp);
 990                 return (errno);
 991         }
 992         *size = st.st_size;
 993         return (0);
 994 }
 995
 996 /*
 997  * =========================================================================
 998  * misc routines
 999  * =========================================================================
1000  */
1001
1002 void
1003 delay(clock_t ticks)
1004 {
1005         poll(0, 0, ticks * (1000 / hz));
1006 }
1007
1008 /*
1009  * Find highest one bit set.
1010  *      Returns bit number + 1 of highest bit that is set, otherwise returns 0.
1011  * High order bit is 31 (or 63 in _LP64 kernel).
1012  */
1013 int
1014 highbit64(uint64_t i)
1015 {
1016         register int h = 1;
1017
1018         if (i == 0)
1019                 return (0);
1020         if (i & 0xffffffff00000000ULL) {
1021                 h += 32; i >>= 32;
1022         }
1023         if (i & 0xffff0000) {
1024                 h += 16; i >>= 16;
1025         }
1026         if (i & 0xff00) {
1027                 h += 8; i >>= 8;
1028         }
1029         if (i & 0xf0) {
1030                 h += 4; i >>= 4;
1031         }
1032         if (i & 0xc) {
1033                 h += 2; i >>= 2;
1034         }
1035         if (i & 0x2) {
1036                 h += 1;
1037         }
1038         return (h);
1039 }
1040
1041 static int random_fd = -1, urandom_fd = -1;
1042
1043 static int
1044 random_get_bytes_common(uint8_t *ptr, size_t len, int fd)
1045 {
1046         size_t resid = len;
1047         ssize_t bytes;
1048
1049         ASSERT(fd != -1);
1050
1051         while (resid != 0) {
1052                 bytes = read(fd, ptr, resid);
1053                 ASSERT3S(bytes, >=, 0);
1054                 ptr += bytes;
1055                 resid -= bytes;
1056         }
1057
1058         return (0);
1059 }
1060
1061 int
1062 random_get_bytes(uint8_t *ptr, size_t len)
1063 {
1064         return (random_get_bytes_common(ptr, len, random_fd));
1065 }
1066
1067 int
1068 random_get_pseudo_bytes(uint8_t *ptr, size_t len)
1069 {
1070         return (random_get_bytes_common(ptr, len, urandom_fd));
1071 }
1072
1073 int
1074 ddi_strtoul(const char *hw_serial, char **nptr, int base, unsigned long *result)
1075 {
1076         char *end;
1077
1078         *result = strtoul(hw_serial, &end, base);
1079         if (*result == 0)
1080                 return (errno);
1081         return (0);
1082 }
1083
1084 int
1085 ddi_strtoull(const char *str, char **nptr, int base, u_longlong_t *result)
1086 {
1087         char *end;
1088
1089         *result = strtoull(str, &end, base);
1090         if (*result == 0)
1091                 return (errno);
1092         return (0);
1093 }
1094
1095 utsname_t *
1096 utsname(void)
1097 {
1098         return (&hw_utsname);
1099 }
1100
1101 /*
1102  * =========================================================================
1103  * kernel emulation setup & teardown
1104  * =========================================================================
1105  */
1106 static int
1107 umem_out_of_memory(void)
1108 {
1109         char errmsg[] = "out of memory -- generating core dump\n";
1110
1111         (void) fprintf(stderr, "%s", errmsg);
1112         abort();
1113         return (0);
1114 }
1115
1116 static unsigned long
1117 get_spl_hostid(void)
1118 {
1119         FILE *f;
1120         unsigned long hostid;
1121
1122         f = fopen("/sys/module/spl/parameters/spl_hostid", "r");
1123         if (!f)
1124                 return (0);
1125         if (fscanf(f, "%lu", &hostid) != 1)
1126                 hostid = 0;
1127         fclose(f);
1128         return (hostid & 0xffffffff);
1129 }
1130
1131 unsigned long
1132 get_system_hostid(void)
1133 {
1134         unsigned long system_hostid = get_spl_hostid();
1135         if (system_hostid == 0)
1136                 system_hostid = gethostid() & 0xffffffff;
1137         return (system_hostid);
1138 }
1139
1140 void
1141 kernel_init(int mode)
1142 {
1143         extern uint_t rrw_tsd_key;
1144
1145         umem_nofail_callback(umem_out_of_memory);
1146
1147         physmem = sysconf(_SC_PHYS_PAGES);
1148
1149         dprintf("physmem = %llu pages (%.2f GB)\n", physmem,
1150             (double)physmem * sysconf(_SC_PAGE_SIZE) / (1ULL << 30));
1151
1152         (void) snprintf(hw_serial, sizeof (hw_serial), "%ld",
1153             (mode & FWRITE) ? get_system_hostid() : 0);
1154
1155         VERIFY((random_fd = open("/dev/random", O_RDONLY)) != -1);
1156         VERIFY((urandom_fd = open("/dev/urandom", O_RDONLY)) != -1);
1157         VERIFY0(uname(&hw_utsname));
1158
1159         thread_init();
1160         system_taskq_init();
1161
1162         spa_init(mode);
1163
1164         tsd_create(&rrw_tsd_key, rrw_tsd_destroy);
1165 }
1166
1167 void
1168 kernel_fini(void)
1169 {
1170         spa_fini();
1171
1172         system_taskq_fini();
1173         thread_fini();
1174
1175         close(random_fd);
1176         close(urandom_fd);
1177
1178         random_fd = -1;
1179         urandom_fd = -1;
1180 }
1181
1182 uid_t
1183 crgetuid(cred_t *cr)
1184 {
1185         return (0);
1186 }
1187
1188 uid_t
1189 crgetruid(cred_t *cr)
1190 {
1191         return (0);
1192 }
1193
1194 gid_t
1195 crgetgid(cred_t *cr)
1196 {
1197         return (0);
1198 }
1199
1200 int
1201 crgetngroups(cred_t *cr)
1202 {
1203         return (0);
1204 }
1205
1206 gid_t *
1207 crgetgroups(cred_t *cr)
1208 {
1209         return (NULL);
1210 }
1211
1212 int
1213 zfs_secpolicy_snapshot_perms(const char *name, cred_t *cr)
1214 {
1215         return (0);
1216 }
1217
1218 int
1219 zfs_secpolicy_rename_perms(const char *from, const char *to, cred_t *cr)
1220 {
1221         return (0);
1222 }
1223
1224 int
1225 zfs_secpolicy_destroy_perms(const char *name, cred_t *cr)
1226 {
1227         return (0);
1228 }
1229
1230 ksiddomain_t *
1231 ksid_lookupdomain(const char *dom)
1232 {
1233         ksiddomain_t *kd;
1234
1235         kd = umem_zalloc(sizeof (ksiddomain_t), UMEM_NOFAIL);
1236         kd->kd_name = spa_strdup(dom);
1237         return (kd);
1238 }
1239
1240 void
1241 ksiddomain_rele(ksiddomain_t *ksid)
1242 {
1243         spa_strfree(ksid->kd_name);
1244         umem_free(ksid, sizeof (ksiddomain_t));
1245 }
1246
1247 char *
1248 kmem_vasprintf(const char *fmt, va_list adx)
1249 {
1250         char *buf = NULL;
1251         va_list adx_copy;
1252
1253         va_copy(adx_copy, adx);
1254         VERIFY(vasprintf(&buf, fmt, adx_copy) != -1);
1255         va_end(adx_copy);
1256
1257         return (buf);
1258 }
1259
1260 char *
1261 kmem_asprintf(const char *fmt, ...)
1262 {
1263         char *buf = NULL;
1264         va_list adx;
1265
1266         va_start(adx, fmt);
1267         VERIFY(vasprintf(&buf, fmt, adx) != -1);
1268         va_end(adx);
1269
1270         return (buf);
1271 }
1272
1273 /* ARGSUSED */
1274 int
1275 zfs_onexit_fd_hold(int fd, minor_t *minorp)
1276 {
1277         *minorp = 0;
1278         return (0);
1279 }
1280
1281 /* ARGSUSED */
1282 void
1283 zfs_onexit_fd_rele(int fd)
1284 {
1285 }
1286
1287 /* ARGSUSED */
1288 int
1289 zfs_onexit_add_cb(minor_t minor, void (*func)(void *), void *data,
1290     uint64_t *action_handle)
1291 {
1292         return (0);
1293 }
1294
1295 /* ARGSUSED */
1296 int
1297 zfs_onexit_del_cb(minor_t minor, uint64_t action_handle, boolean_t fire)
1298 {
1299         return (0);
1300 }
1301
1302 /* ARGSUSED */
1303 int
1304 zfs_onexit_cb_data(minor_t minor, uint64_t action_handle, void **data)
1305 {
1306         return (0);
1307 }
1308
1309 fstrans_cookie_t
1310 spl_fstrans_mark(void)
1311 {
1312         return ((fstrans_cookie_t) 0);
1313 }
1314
1315 void
1316 spl_fstrans_unmark(fstrans_cookie_t cookie)
1317 {
1318 }
1319
1320 int
1321 spl_fstrans_check(void)
1322 {
1323         return (0);
1324 }