lib/libzpool/kernel.c

   1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  23  */
  24
  25 #include <assert.h>
  26 #include <fcntl.h>
  27 #include <poll.h>
  28 #include <stdio.h>
  29 #include <stdlib.h>
  30 #include <string.h>
  31 #include <zlib.h>
  32 #include <sys/signal.h>
  33 #include <sys/spa.h>
  34 #include <sys/stat.h>
  35 #include <sys/processor.h>
  36 #include <sys/zfs_context.h>
  37 #include <sys/rrwlock.h>
  38 #include <sys/utsname.h>
  39 #include <sys/time.h>
  40 #include <sys/systeminfo.h>
  41
  42 /*
  43  * Emulation of kernel services in userland.
  44  */
  45
  46 int aok;
  47 uint64_t physmem;
  48 vnode_t *rootdir = (vnode_t *)0xabcd1234;
  49 char hw_serial[HW_HOSTID_LEN];
  50
  51 struct utsname utsname = {
  52         "userland", "libzpool", "1", "1", "na"
  53 };
  54
  55 /* this only exists to have its address taken */
  56 struct proc p0;
  57
  58 /*
  59  * =========================================================================
  60  * threads
  61  * =========================================================================
  62  */
  63
  64 pthread_cond_t kthread_cond = PTHREAD_COND_INITIALIZER;
  65 pthread_mutex_t kthread_lock = PTHREAD_MUTEX_INITIALIZER;
  66 pthread_key_t kthread_key;
  67 int kthread_nr = 0;
  68
  69 static void
  70 thread_init(void)
  71 {
  72         kthread_t *kt;
  73
  74         VERIFY3S(pthread_key_create(&kthread_key, NULL), ==, 0);
  75
  76         /* Create entry for primary kthread */
  77         kt = umem_zalloc(sizeof (kthread_t), UMEM_NOFAIL);
  78         kt->t_tid = pthread_self();
  79         kt->t_func = NULL;
  80
  81         VERIFY3S(pthread_setspecific(kthread_key, kt), ==, 0);
  82
  83         /* Only the main thread should be running at the moment */
  84         ASSERT3S(kthread_nr, ==, 0);
  85         kthread_nr = 1;
  86 }
  87
  88 static void
  89 thread_fini(void)
  90 {
  91         kthread_t *kt = curthread;
  92
  93         ASSERT(pthread_equal(kt->t_tid, pthread_self()));
  94         ASSERT3P(kt->t_func, ==, NULL);
  95
  96         umem_free(kt, sizeof (kthread_t));
  97
  98         /* Wait for all threads to exit via thread_exit() */
  99         VERIFY3S(pthread_mutex_lock(&kthread_lock), ==, 0);
 100
 101         kthread_nr--; /* Main thread is exiting */
 102
 103         while (kthread_nr > 0)
 104                 VERIFY3S(pthread_cond_wait(&kthread_cond, &kthread_lock), ==,
 105                     0);
 106
 107         ASSERT3S(kthread_nr, ==, 0);
 108         VERIFY3S(pthread_mutex_unlock(&kthread_lock), ==, 0);
 109
 110         VERIFY3S(pthread_key_delete(kthread_key), ==, 0);
 111 }
 112
 113 kthread_t *
 114 zk_thread_current(void)
 115 {
 116         kthread_t *kt = pthread_getspecific(kthread_key);
 117
 118         ASSERT3P(kt, !=, NULL);
 119
 120         return (kt);
 121 }
 122
 123 void *
 124 zk_thread_helper(void *arg)
 125 {
 126         kthread_t *kt = (kthread_t *) arg;
 127
 128         VERIFY3S(pthread_setspecific(kthread_key, kt), ==, 0);
 129
 130         VERIFY3S(pthread_mutex_lock(&kthread_lock), ==, 0);
 131         kthread_nr++;
 132         VERIFY3S(pthread_mutex_unlock(&kthread_lock), ==, 0);
 133
 134         kt->t_tid = pthread_self();
 135         ((thread_func_arg_t) kt->t_func)(kt->t_arg);
 136
 137         /* Unreachable, thread must exit with thread_exit() */
 138         abort();
 139
 140         return (NULL);
 141 }
 142
 143 kthread_t *
 144 zk_thread_create(caddr_t stk, size_t stksize, thread_func_t func, void *arg,
 145     size_t len, proc_t *pp, int state, pri_t pri, int detachstate)
 146 {
 147         kthread_t *kt;
 148         pthread_attr_t attr;
 149         size_t stack;
 150
 151         ASSERT3S(state & ~TS_RUN, ==, 0);
 152
 153         kt = umem_zalloc(sizeof (kthread_t), UMEM_NOFAIL);
 154         kt->t_func = func;
 155         kt->t_arg = arg;
 156
 157         /*
 158          * The Solaris kernel stack size is 24k for x86/x86_64.
 159          * The Linux kernel stack size is 8k for x86/x86_64.
 160          *
 161          * We reduce the default stack size in userspace, to ensure
 162          * we observe stack overruns in user space as well as in
 163          * kernel space. In practice we can't set the userspace stack
 164          * size to 8k because differences in stack usage between kernel
 165          * space and userspace could lead to spurious stack overflows
 166          * (especially when debugging is enabled). Nevertheless, we try
 167          * to set it to the lowest value that works (currently 8k*4).
 168          * PTHREAD_STACK_MIN is the minimum stack required for a NULL
 169          * procedure in user space and is added in to the stack
 170          * requirements.
 171          */
 172
 173         stack = PTHREAD_STACK_MIN + MAX(stksize, STACK_SIZE) * 4;
 174
 175         VERIFY3S(pthread_attr_init(&attr), ==, 0);
 176         VERIFY3S(pthread_attr_setstacksize(&attr, stack), ==, 0);
 177         VERIFY3S(pthread_attr_setguardsize(&attr, PAGESIZE), ==, 0);
 178         VERIFY3S(pthread_attr_setdetachstate(&attr, detachstate), ==, 0);
 179
 180         VERIFY3S(pthread_create(&kt->t_tid, &attr, &zk_thread_helper, kt),
 181             ==, 0);
 182
 183         VERIFY3S(pthread_attr_destroy(&attr), ==, 0);
 184
 185         return (kt);
 186 }
 187
 188 void
 189 zk_thread_exit(void)
 190 {
 191         kthread_t *kt = curthread;
 192
 193         ASSERT(pthread_equal(kt->t_tid, pthread_self()));
 194
 195         umem_free(kt, sizeof (kthread_t));
 196
 197         pthread_mutex_lock(&kthread_lock);
 198         kthread_nr--;
 199         pthread_mutex_unlock(&kthread_lock);
 200
 201         pthread_cond_broadcast(&kthread_cond);
 202         pthread_exit((void *)TS_MAGIC);
 203 }
 204
 205 void
 206 zk_thread_join(kt_did_t tid)
 207 {
 208         void *ret;
 209
 210         pthread_join((pthread_t)tid, &ret);
 211         VERIFY3P(ret, ==, (void *)TS_MAGIC);
 212 }
 213
 214 /*
 215  * =========================================================================
 216  * kstats
 217  * =========================================================================
 218  */
 219 /*ARGSUSED*/
 220 kstat_t *
 221 kstat_create(const char *module, int instance, const char *name,
 222     const char *class, uchar_t type, ulong_t ndata, uchar_t ks_flag)
 223 {
 224         return (NULL);
 225 }
 226
 227 /*ARGSUSED*/
 228 void
 229 kstat_install(kstat_t *ksp)
 230 {}
 231
 232 /*ARGSUSED*/
 233 void
 234 kstat_delete(kstat_t *ksp)
 235 {}
 236
 237 /*ARGSUSED*/
 238 void
 239 kstat_waitq_enter(kstat_io_t *kiop)
 240 {}
 241
 242 /*ARGSUSED*/
 243 void
 244 kstat_waitq_exit(kstat_io_t *kiop)
 245 {}
 246
 247 /*ARGSUSED*/
 248 void
 249 kstat_runq_enter(kstat_io_t *kiop)
 250 {}
 251
 252 /*ARGSUSED*/
 253 void
 254 kstat_runq_exit(kstat_io_t *kiop)
 255 {}
 256
 257 /*ARGSUSED*/
 258 void
 259 kstat_waitq_to_runq(kstat_io_t *kiop)
 260 {}
 261
 262 /*ARGSUSED*/
 263 void
 264 kstat_runq_back_to_waitq(kstat_io_t *kiop)
 265 {}
 266
 267 void
 268 kstat_set_raw_ops(kstat_t *ksp,
 269     int (*headers)(char *buf, size_t size),
 270     int (*data)(char *buf, size_t size, void *data),
 271     void *(*addr)(kstat_t *ksp, loff_t index))
 272 {}
 273
 274 /*
 275  * =========================================================================
 276  * mutexes
 277  * =========================================================================
 278  */
 279
 280 void
 281 mutex_init(kmutex_t *mp, char *name, int type, void *cookie)
 282 {
 283         ASSERT3S(type, ==, MUTEX_DEFAULT);
 284         ASSERT3P(cookie, ==, NULL);
 285         mp->m_owner = MTX_INIT;
 286         mp->m_magic = MTX_MAGIC;
 287         VERIFY3S(pthread_mutex_init(&mp->m_lock, NULL), ==, 0);
 288 }
 289
 290 void
 291 mutex_destroy(kmutex_t *mp)
 292 {
 293         ASSERT3U(mp->m_magic, ==, MTX_MAGIC);
 294         ASSERT3P(mp->m_owner, ==, MTX_INIT);
 295         VERIFY3S(pthread_mutex_destroy(&(mp)->m_lock), ==, 0);
 296         mp->m_owner = MTX_DEST;
 297         mp->m_magic = 0;
 298 }
 299
 300 void
 301 mutex_enter(kmutex_t *mp)
 302 {
 303         ASSERT3U(mp->m_magic, ==, MTX_MAGIC);
 304         ASSERT3P(mp->m_owner, !=, MTX_DEST);
 305         ASSERT3P(mp->m_owner, !=, curthread);
 306         VERIFY3S(pthread_mutex_lock(&mp->m_lock), ==, 0);
 307         ASSERT3P(mp->m_owner, ==, MTX_INIT);
 308         mp->m_owner = curthread;
 309 }
 310
 311 int
 312 mutex_tryenter(kmutex_t *mp)
 313 {
 314         ASSERT3U(mp->m_magic, ==, MTX_MAGIC);
 315         ASSERT3P(mp->m_owner, !=, MTX_DEST);
 316         if (0 == pthread_mutex_trylock(&mp->m_lock)) {
 317                 ASSERT3P(mp->m_owner, ==, MTX_INIT);
 318                 mp->m_owner = curthread;
 319                 return (1);
 320         } else {
 321                 return (0);
 322         }
 323 }
 324
 325 void
 326 mutex_exit(kmutex_t *mp)
 327 {
 328         ASSERT3U(mp->m_magic, ==, MTX_MAGIC);
 329         ASSERT3P(mutex_owner(mp), ==, curthread);
 330         mp->m_owner = MTX_INIT;
 331         VERIFY3S(pthread_mutex_unlock(&mp->m_lock), ==, 0);
 332 }
 333
 334 void *
 335 mutex_owner(kmutex_t *mp)
 336 {
 337         ASSERT3U(mp->m_magic, ==, MTX_MAGIC);
 338         return (mp->m_owner);
 339 }
 340
 341 int
 342 mutex_held(kmutex_t *mp)
 343 {
 344         return (mp->m_owner == curthread);
 345 }
 346
 347 /*
 348  * =========================================================================
 349  * rwlocks
 350  * =========================================================================
 351  */
 352
 353 void
 354 rw_init(krwlock_t *rwlp, char *name, int type, void *arg)
 355 {
 356         ASSERT3S(type, ==, RW_DEFAULT);
 357         ASSERT3P(arg, ==, NULL);
 358         VERIFY3S(pthread_rwlock_init(&rwlp->rw_lock, NULL), ==, 0);
 359         rwlp->rw_owner = RW_INIT;
 360         rwlp->rw_wr_owner = RW_INIT;
 361         rwlp->rw_readers = 0;
 362         rwlp->rw_magic = RW_MAGIC;
 363 }
 364
 365 void
 366 rw_destroy(krwlock_t *rwlp)
 367 {
 368         ASSERT3U(rwlp->rw_magic, ==, RW_MAGIC);
 369         ASSERT(rwlp->rw_readers == 0 && rwlp->rw_wr_owner == RW_INIT);
 370         VERIFY3S(pthread_rwlock_destroy(&rwlp->rw_lock), ==, 0);
 371         rwlp->rw_magic = 0;
 372 }
 373
 374 void
 375 rw_enter(krwlock_t *rwlp, krw_t rw)
 376 {
 377         ASSERT3U(rwlp->rw_magic, ==, RW_MAGIC);
 378         ASSERT3P(rwlp->rw_owner, !=, curthread);
 379         ASSERT3P(rwlp->rw_wr_owner, !=, curthread);
 380
 381         if (rw == RW_READER) {
 382                 VERIFY3S(pthread_rwlock_rdlock(&rwlp->rw_lock), ==, 0);
 383                 ASSERT3P(rwlp->rw_wr_owner, ==, RW_INIT);
 384
 385                 atomic_inc_uint(&rwlp->rw_readers);
 386         } else {
 387                 VERIFY3S(pthread_rwlock_wrlock(&rwlp->rw_lock), ==, 0);
 388                 ASSERT3P(rwlp->rw_wr_owner, ==, RW_INIT);
 389                 ASSERT3U(rwlp->rw_readers, ==, 0);
 390
 391                 rwlp->rw_wr_owner = curthread;
 392         }
 393
 394         rwlp->rw_owner = curthread;
 395 }
 396
 397 void
 398 rw_exit(krwlock_t *rwlp)
 399 {
 400         ASSERT3U(rwlp->rw_magic, ==, RW_MAGIC);
 401         ASSERT(RW_LOCK_HELD(rwlp));
 402
 403         if (RW_READ_HELD(rwlp))
 404                 atomic_dec_uint(&rwlp->rw_readers);
 405         else
 406                 rwlp->rw_wr_owner = RW_INIT;
 407
 408         rwlp->rw_owner = RW_INIT;
 409         VERIFY3S(pthread_rwlock_unlock(&rwlp->rw_lock), ==, 0);
 410 }
 411
 412 int
 413 rw_tryenter(krwlock_t *rwlp, krw_t rw)
 414 {
 415         int rv;
 416
 417         ASSERT3U(rwlp->rw_magic, ==, RW_MAGIC);
 418
 419         if (rw == RW_READER)
 420                 rv = pthread_rwlock_tryrdlock(&rwlp->rw_lock);
 421         else
 422                 rv = pthread_rwlock_trywrlock(&rwlp->rw_lock);
 423
 424         if (rv == 0) {
 425                 ASSERT3P(rwlp->rw_wr_owner, ==, RW_INIT);
 426
 427                 if (rw == RW_READER)
 428                         atomic_inc_uint(&rwlp->rw_readers);
 429                 else {
 430                         ASSERT3U(rwlp->rw_readers, ==, 0);
 431                         rwlp->rw_wr_owner = curthread;
 432                 }
 433
 434                 rwlp->rw_owner = curthread;
 435                 return (1);
 436         }
 437
 438         VERIFY3S(rv, ==, EBUSY);
 439
 440         return (0);
 441 }
 442
 443 int
 444 rw_tryupgrade(krwlock_t *rwlp)
 445 {
 446         ASSERT3U(rwlp->rw_magic, ==, RW_MAGIC);
 447
 448         return (0);
 449 }
 450
 451 /*
 452  * =========================================================================
 453  * condition variables
 454  * =========================================================================
 455  */
 456
 457 void
 458 cv_init(kcondvar_t *cv, char *name, int type, void *arg)
 459 {
 460         ASSERT3S(type, ==, CV_DEFAULT);
 461         cv->cv_magic = CV_MAGIC;
 462         VERIFY3S(pthread_cond_init(&cv->cv, NULL), ==, 0);
 463 }
 464
 465 void
 466 cv_destroy(kcondvar_t *cv)
 467 {
 468         ASSERT3U(cv->cv_magic, ==, CV_MAGIC);
 469         VERIFY3S(pthread_cond_destroy(&cv->cv), ==, 0);
 470         cv->cv_magic = 0;
 471 }
 472
 473 void
 474 cv_wait(kcondvar_t *cv, kmutex_t *mp)
 475 {
 476         ASSERT3U(cv->cv_magic, ==, CV_MAGIC);
 477         ASSERT3P(mutex_owner(mp), ==, curthread);
 478         mp->m_owner = MTX_INIT;
 479         int ret = pthread_cond_wait(&cv->cv, &mp->m_lock);
 480         if (ret != 0)
 481                 VERIFY3S(ret, ==, EINTR);
 482         mp->m_owner = curthread;
 483 }
 484
 485 clock_t
 486 cv_timedwait(kcondvar_t *cv, kmutex_t *mp, clock_t abstime)
 487 {
 488         int error;
 489         struct timeval tv;
 490         timestruc_t ts;
 491         clock_t delta;
 492
 493         ASSERT3U(cv->cv_magic, ==, CV_MAGIC);
 494
 495 top:
 496         delta = abstime - ddi_get_lbolt();
 497         if (delta <= 0)
 498                 return (-1);
 499
 500         VERIFY(gettimeofday(&tv, NULL) == 0);
 501
 502         ts.tv_sec = tv.tv_sec + delta / hz;
 503         ts.tv_nsec = tv.tv_usec * 1000 + (delta % hz) * (NANOSEC / hz);
 504         if (ts.tv_nsec >= NANOSEC) {
 505                 ts.tv_sec++;
 506                 ts.tv_nsec -= NANOSEC;
 507         }
 508
 509         ASSERT3P(mutex_owner(mp), ==, curthread);
 510         mp->m_owner = MTX_INIT;
 511         error = pthread_cond_timedwait(&cv->cv, &mp->m_lock, &ts);
 512         mp->m_owner = curthread;
 513
 514         if (error == ETIMEDOUT)
 515                 return (-1);
 516
 517         if (error == EINTR)
 518                 goto top;
 519
 520         VERIFY3S(error, ==, 0);
 521
 522         return (1);
 523 }
 524
 525 /*ARGSUSED*/
 526 clock_t
 527 cv_timedwait_hires(kcondvar_t *cv, kmutex_t *mp, hrtime_t tim, hrtime_t res,
 528     int flag)
 529 {
 530         int error;
 531         timestruc_t ts;
 532         hrtime_t delta;
 533
 534         ASSERT(flag == 0);
 535
 536 top:
 537         delta = tim - gethrtime();
 538         if (delta <= 0)
 539                 return (-1);
 540
 541         ts.tv_sec = delta / NANOSEC;
 542         ts.tv_nsec = delta % NANOSEC;
 543
 544         ASSERT(mutex_owner(mp) == curthread);
 545         mp->m_owner = NULL;
 546         error = pthread_cond_timedwait(&cv->cv, &mp->m_lock, &ts);
 547         mp->m_owner = curthread;
 548
 549         if (error == ETIME)
 550                 return (-1);
 551
 552         if (error == EINTR)
 553                 goto top;
 554
 555         ASSERT(error == 0);
 556
 557         return (1);
 558 }
 559
 560 void
 561 cv_signal(kcondvar_t *cv)
 562 {
 563         ASSERT3U(cv->cv_magic, ==, CV_MAGIC);
 564         VERIFY3S(pthread_cond_signal(&cv->cv), ==, 0);
 565 }
 566
 567 void
 568 cv_broadcast(kcondvar_t *cv)
 569 {
 570         ASSERT3U(cv->cv_magic, ==, CV_MAGIC);
 571         VERIFY3S(pthread_cond_broadcast(&cv->cv), ==, 0);
 572 }
 573
 574 /*
 575  * =========================================================================
 576  * vnode operations
 577  * =========================================================================
 578  */
 579 /*
 580  * Note: for the xxxat() versions of these functions, we assume that the
 581  * starting vp is always rootdir (which is true for spa_directory.c, the only
 582  * ZFS consumer of these interfaces).  We assert this is true, and then emulate
 583  * them by adding '/' in front of the path.
 584  */
 585
 586 /*ARGSUSED*/
 587 int
 588 vn_open(char *path, int x1, int flags, int mode, vnode_t **vpp, int x2, int x3)
 589 {
 590         int fd;
 591         vnode_t *vp;
 592         int old_umask = 0;
 593         char *realpath;
 594         struct stat64 st;
 595         int err;
 596
 597         realpath = umem_alloc(MAXPATHLEN, UMEM_NOFAIL);
 598
 599         /*
 600          * If we're accessing a real disk from userland, we need to use
 601          * the character interface to avoid caching.  This is particularly
 602          * important if we're trying to look at a real in-kernel storage
 603          * pool from userland, e.g. via zdb, because otherwise we won't
 604          * see the changes occurring under the segmap cache.
 605          * On the other hand, the stupid character device returns zero
 606          * for its size.  So -- gag -- we open the block device to get
 607          * its size, and remember it for subsequent VOP_GETATTR().
 608          */
 609 #if defined(__sun__) || defined(__sun)
 610         if (strncmp(path, "/dev/", 5) == 0) {
 611 #else
 612         if (0) {
 613 #endif
 614                 char *dsk;
 615                 fd = open64(path, O_RDONLY);
 616                 if (fd == -1) {
 617                         err = errno;
 618                         free(realpath);
 619                         return (err);
 620                 }
 621                 if (fstat64(fd, &st) == -1) {
 622                         err = errno;
 623                         close(fd);
 624                         free(realpath);
 625                         return (err);
 626                 }
 627                 close(fd);
 628                 (void) sprintf(realpath, "%s", path);
 629                 dsk = strstr(path, "/dsk/");
 630                 if (dsk != NULL)
 631                         (void) sprintf(realpath + (dsk - path) + 1, "r%s",
 632                             dsk + 1);
 633         } else {
 634                 (void) sprintf(realpath, "%s", path);
 635                 if (!(flags & FCREAT) && stat64(realpath, &st) == -1) {
 636                         err = errno;
 637                         free(realpath);
 638                         return (err);
 639                 }
 640         }
 641
 642         if (!(flags & FCREAT) && S_ISBLK(st.st_mode)) {
 643 #ifdef __linux__
 644                 flags |= O_DIRECT;
 645 #endif
 646                 /* We shouldn't be writing to block devices in userspace */
 647                 VERIFY(!(flags & FWRITE));
 648         }
 649
 650         if (flags & FCREAT)
 651                 old_umask = umask(0);
 652
 653         /*
 654          * The construct 'flags - FREAD' conveniently maps combinations of
 655          * FREAD and FWRITE to the corresponding O_RDONLY, O_WRONLY, and O_RDWR.
 656          */
 657         fd = open64(realpath, flags - FREAD, mode);
 658         free(realpath);
 659
 660         if (flags & FCREAT)
 661                 (void) umask(old_umask);
 662
 663         if (fd == -1)
 664                 return (errno);
 665
 666         if (fstat64_blk(fd, &st) == -1) {
 667                 err = errno;
 668                 close(fd);
 669                 return (err);
 670         }
 671
 672         (void) fcntl(fd, F_SETFD, FD_CLOEXEC);
 673
 674         *vpp = vp = umem_zalloc(sizeof (vnode_t), UMEM_NOFAIL);
 675
 676         vp->v_fd = fd;
 677         vp->v_size = st.st_size;
 678         vp->v_path = spa_strdup(path);
 679
 680         return (0);
 681 }
 682
 683 /*ARGSUSED*/
 684 int
 685 vn_openat(char *path, int x1, int flags, int mode, vnode_t **vpp, int x2,
 686     int x3, vnode_t *startvp, int fd)
 687 {
 688         char *realpath = umem_alloc(strlen(path) + 2, UMEM_NOFAIL);
 689         int ret;
 690
 691         ASSERT(startvp == rootdir);
 692         (void) sprintf(realpath, "/%s", path);
 693
 694         /* fd ignored for now, need if want to simulate nbmand support */
 695         ret = vn_open(realpath, x1, flags, mode, vpp, x2, x3);
 696
 697         umem_free(realpath, strlen(path) + 2);
 698
 699         return (ret);
 700 }
 701
 702 /*ARGSUSED*/
 703 int
 704 vn_rdwr(int uio, vnode_t *vp, void *addr, ssize_t len, offset_t offset,
 705         int x1, int x2, rlim64_t x3, void *x4, ssize_t *residp)
 706 {
 707         ssize_t rc, done = 0, split;
 708
 709         if (uio == UIO_READ) {
 710                 rc = pread64(vp->v_fd, addr, len, offset);
 711         } else {
 712                 /*
 713                  * To simulate partial disk writes, we split writes into two
 714                  * system calls so that the process can be killed in between.
 715                  */
 716                 int sectors = len >> SPA_MINBLOCKSHIFT;
 717                 split = (sectors > 0 ? rand() % sectors : 0) <<
 718                     SPA_MINBLOCKSHIFT;
 719                 rc = pwrite64(vp->v_fd, addr, split, offset);
 720                 if (rc != -1) {
 721                         done = rc;
 722                         rc = pwrite64(vp->v_fd, (char *)addr + split,
 723                             len - split, offset + split);
 724                 }
 725         }
 726
 727 #ifdef __linux__
 728         if (rc == -1 && errno == EINVAL) {
 729                 /*
 730                  * Under Linux, this most likely means an alignment issue
 731                  * (memory or disk) due to O_DIRECT, so we abort() in order to
 732                  * catch the offender.
 733                  */
 734                 abort();
 735         }
 736 #endif
 737         if (rc == -1)
 738                 return (errno);
 739
 740         done += rc;
 741
 742         if (residp)
 743                 *residp = len - done;
 744         else if (done != len)
 745                 return (EIO);
 746         return (0);
 747 }
 748
 749 void
 750 vn_close(vnode_t *vp)
 751 {
 752         close(vp->v_fd);
 753         spa_strfree(vp->v_path);
 754         umem_free(vp, sizeof (vnode_t));
 755 }
 756
 757 /*
 758  * At a minimum we need to update the size since vdev_reopen()
 759  * will no longer call vn_openat().
 760  */
 761 int
 762 fop_getattr(vnode_t *vp, vattr_t *vap)
 763 {
 764         struct stat64 st;
 765         int err;
 766
 767         if (fstat64_blk(vp->v_fd, &st) == -1) {
 768                 err = errno;
 769                 close(vp->v_fd);
 770                 return (err);
 771         }
 772
 773         vap->va_size = st.st_size;
 774         return (0);
 775 }
 776
 777 /*
 778  * =========================================================================
 779  * Figure out which debugging statements to print
 780  * =========================================================================
 781  */
 782
 783 static char *dprintf_string;
 784 static int dprintf_print_all;
 785
 786 int
 787 dprintf_find_string(const char *string)
 788 {
 789         char *tmp_str = dprintf_string;
 790         int len = strlen(string);
 791
 792         /*
 793          * Find out if this is a string we want to print.
 794          * String format: file1.c,function_name1,file2.c,file3.c
 795          */
 796
 797         while (tmp_str != NULL) {
 798                 if (strncmp(tmp_str, string, len) == 0 &&
 799                     (tmp_str[len] == ',' || tmp_str[len] == '\0'))
 800                         return (1);
 801                 tmp_str = strchr(tmp_str, ',');
 802                 if (tmp_str != NULL)
 803                         tmp_str++; /* Get rid of , */
 804         }
 805         return (0);
 806 }
 807
 808 void
 809 dprintf_setup(int *argc, char **argv)
 810 {
 811         int i, j;
 812
 813         /*
 814          * Debugging can be specified two ways: by setting the
 815          * environment variable ZFS_DEBUG, or by including a
 816          * "debug=..."  argument on the command line.  The command
 817          * line setting overrides the environment variable.
 818          */
 819
 820         for (i = 1; i < *argc; i++) {
 821                 int len = strlen("debug=");
 822                 /* First look for a command line argument */
 823                 if (strncmp("debug=", argv[i], len) == 0) {
 824                         dprintf_string = argv[i] + len;
 825                         /* Remove from args */
 826                         for (j = i; j < *argc; j++)
 827                                 argv[j] = argv[j+1];
 828                         argv[j] = NULL;
 829                         (*argc)--;
 830                 }
 831         }
 832
 833         if (dprintf_string == NULL) {
 834                 /* Look for ZFS_DEBUG environment variable */
 835                 dprintf_string = getenv("ZFS_DEBUG");
 836         }
 837
 838         /*
 839          * Are we just turning on all debugging?
 840          */
 841         if (dprintf_find_string("on"))
 842                 dprintf_print_all = 1;
 843 }
 844
 845 /*
 846  * =========================================================================
 847  * debug printfs
 848  * =========================================================================
 849  */
 850 void
 851 __dprintf(const char *file, const char *func, int line, const char *fmt, ...)
 852 {
 853         const char *newfile;
 854         va_list adx;
 855
 856         /*
 857          * Get rid of annoying "../common/" prefix to filename.
 858          */
 859         newfile = strrchr(file, '/');
 860         if (newfile != NULL) {
 861                 newfile = newfile + 1; /* Get rid of leading / */
 862         } else {
 863                 newfile = file;
 864         }
 865
 866         if (dprintf_print_all ||
 867             dprintf_find_string(newfile) ||
 868             dprintf_find_string(func)) {
 869                 /* Print out just the function name if requested */
 870                 flockfile(stdout);
 871                 if (dprintf_find_string("pid"))
 872                         (void) printf("%d ", getpid());
 873                 if (dprintf_find_string("tid"))
 874                         (void) printf("%u ", (uint_t) pthread_self());
 875                 if (dprintf_find_string("cpu"))
 876                         (void) printf("%u ", getcpuid());
 877                 if (dprintf_find_string("time"))
 878                         (void) printf("%llu ", gethrtime());
 879                 if (dprintf_find_string("long"))
 880                         (void) printf("%s, line %d: ", newfile, line);
 881                 (void) printf("%s: ", func);
 882                 va_start(adx, fmt);
 883                 (void) vprintf(fmt, adx);
 884                 va_end(adx);
 885                 funlockfile(stdout);
 886         }
 887 }
 888
 889 /*
 890  * =========================================================================
 891  * cmn_err() and panic()
 892  * =========================================================================
 893  */
 894 static char ce_prefix[CE_IGNORE][10] = { "", "NOTICE: ", "WARNING: ", "" };
 895 static char ce_suffix[CE_IGNORE][2] = { "", "\n", "\n", "" };
 896
 897 void
 898 vpanic(const char *fmt, va_list adx)
 899 {
 900         (void) fprintf(stderr, "error: ");
 901         (void) vfprintf(stderr, fmt, adx);
 902         (void) fprintf(stderr, "\n");
 903
 904         abort();        /* think of it as a "user-level crash dump" */
 905 }
 906
 907 void
 908 panic(const char *fmt, ...)
 909 {
 910         va_list adx;
 911
 912         va_start(adx, fmt);
 913         vpanic(fmt, adx);
 914         va_end(adx);
 915 }
 916
 917 void
 918 vcmn_err(int ce, const char *fmt, va_list adx)
 919 {
 920         if (ce == CE_PANIC)
 921                 vpanic(fmt, adx);
 922         if (ce != CE_NOTE) {    /* suppress noise in userland stress testing */
 923                 (void) fprintf(stderr, "%s", ce_prefix[ce]);
 924                 (void) vfprintf(stderr, fmt, adx);
 925                 (void) fprintf(stderr, "%s", ce_suffix[ce]);
 926         }
 927 }
 928
 929 /*PRINTFLIKE2*/
 930 void
 931 cmn_err(int ce, const char *fmt, ...)
 932 {
 933         va_list adx;
 934
 935         va_start(adx, fmt);
 936         vcmn_err(ce, fmt, adx);
 937         va_end(adx);
 938 }
 939
 940 /*
 941  * =========================================================================
 942  * kobj interfaces
 943  * =========================================================================
 944  */
 945 struct _buf *
 946 kobj_open_file(char *name)
 947 {
 948         struct _buf *file;
 949         vnode_t *vp;
 950
 951         /* set vp as the _fd field of the file */
 952         if (vn_openat(name, UIO_SYSSPACE, FREAD, 0, &vp, 0, 0, rootdir,
 953             -1) != 0)
 954                 return ((void *)-1UL);
 955
 956         file = umem_zalloc(sizeof (struct _buf), UMEM_NOFAIL);
 957         file->_fd = (intptr_t)vp;
 958         return (file);
 959 }
 960
 961 int
 962 kobj_read_file(struct _buf *file, char *buf, unsigned size, unsigned off)
 963 {
 964         ssize_t resid;
 965
 966         vn_rdwr(UIO_READ, (vnode_t *)file->_fd, buf, size, (offset_t)off,
 967             UIO_SYSSPACE, 0, 0, 0, &resid);
 968
 969         return (size - resid);
 970 }
 971
 972 void
 973 kobj_close_file(struct _buf *file)
 974 {
 975         vn_close((vnode_t *)file->_fd);
 976         umem_free(file, sizeof (struct _buf));
 977 }
 978
 979 int
 980 kobj_get_filesize(struct _buf *file, uint64_t *size)
 981 {
 982         struct stat64 st;
 983         vnode_t *vp = (vnode_t *)file->_fd;
 984
 985         if (fstat64(vp->v_fd, &st) == -1) {
 986                 vn_close(vp);
 987                 return (errno);
 988         }
 989         *size = st.st_size;
 990         return (0);
 991 }
 992
 993 /*
 994  * =========================================================================
 995  * misc routines
 996  * =========================================================================
 997  */
 998
 999 void
1000 delay(clock_t ticks)
1001 {
1002         poll(0, 0, ticks * (1000 / hz));
1003 }
1004
1005 /*
1006  * Find highest one bit set.
1007  *      Returns bit number + 1 of highest bit that is set, otherwise returns 0.
1008  * High order bit is 31 (or 63 in _LP64 kernel).
1009  */
1010 int
1011 highbit64(uint64_t i)
1012 {
1013         register int h = 1;
1014
1015         if (i == 0)
1016                 return (0);
1017         if (i & 0xffffffff00000000ULL) {
1018                 h += 32; i >>= 32;
1019         }
1020         if (i & 0xffff0000) {
1021                 h += 16; i >>= 16;
1022         }
1023         if (i & 0xff00) {
1024                 h += 8; i >>= 8;
1025         }
1026         if (i & 0xf0) {
1027                 h += 4; i >>= 4;
1028         }
1029         if (i & 0xc) {
1030                 h += 2; i >>= 2;
1031         }
1032         if (i & 0x2) {
1033                 h += 1;
1034         }
1035         return (h);
1036 }
1037
1038 static int random_fd = -1, urandom_fd = -1;
1039
1040 static int
1041 random_get_bytes_common(uint8_t *ptr, size_t len, int fd)
1042 {
1043         size_t resid = len;
1044         ssize_t bytes;
1045
1046         ASSERT(fd != -1);
1047
1048         while (resid != 0) {
1049                 bytes = read(fd, ptr, resid);
1050                 ASSERT3S(bytes, >=, 0);
1051                 ptr += bytes;
1052                 resid -= bytes;
1053         }
1054
1055         return (0);
1056 }
1057
1058 int
1059 random_get_bytes(uint8_t *ptr, size_t len)
1060 {
1061         return (random_get_bytes_common(ptr, len, random_fd));
1062 }
1063
1064 int
1065 random_get_pseudo_bytes(uint8_t *ptr, size_t len)
1066 {
1067         return (random_get_bytes_common(ptr, len, urandom_fd));
1068 }
1069
1070 int
1071 ddi_strtoul(const char *hw_serial, char **nptr, int base, unsigned long *result)
1072 {
1073         char *end;
1074
1075         *result = strtoul(hw_serial, &end, base);
1076         if (*result == 0)
1077                 return (errno);
1078         return (0);
1079 }
1080
1081 int
1082 ddi_strtoull(const char *str, char **nptr, int base, u_longlong_t *result)
1083 {
1084         char *end;
1085
1086         *result = strtoull(str, &end, base);
1087         if (*result == 0)
1088                 return (errno);
1089         return (0);
1090 }
1091
1092 /*
1093  * =========================================================================
1094  * kernel emulation setup & teardown
1095  * =========================================================================
1096  */
1097 static int
1098 umem_out_of_memory(void)
1099 {
1100         char errmsg[] = "out of memory -- generating core dump\n";
1101
1102         (void) fprintf(stderr, "%s", errmsg);
1103         abort();
1104         return (0);
1105 }
1106
1107 void
1108 kernel_init(int mode)
1109 {
1110         extern uint_t rrw_tsd_key;
1111
1112         umem_nofail_callback(umem_out_of_memory);
1113
1114         physmem = sysconf(_SC_PHYS_PAGES);
1115
1116         dprintf("physmem = %llu pages (%.2f GB)\n", physmem,
1117             (double)physmem * sysconf(_SC_PAGE_SIZE) / (1ULL << 30));
1118
1119         (void) snprintf(hw_serial, sizeof (hw_serial), "%ld",
1120             (mode & FWRITE) ? gethostid() : 0);
1121
1122         VERIFY((random_fd = open("/dev/random", O_RDONLY)) != -1);
1123         VERIFY((urandom_fd = open("/dev/urandom", O_RDONLY)) != -1);
1124
1125         thread_init();
1126         system_taskq_init();
1127
1128         spa_init(mode);
1129
1130         tsd_create(&rrw_tsd_key, rrw_tsd_destroy);
1131 }
1132
1133 void
1134 kernel_fini(void)
1135 {
1136         spa_fini();
1137
1138         system_taskq_fini();
1139         thread_fini();
1140
1141         close(random_fd);
1142         close(urandom_fd);
1143
1144         random_fd = -1;
1145         urandom_fd = -1;
1146 }
1147
1148 uid_t
1149 crgetuid(cred_t *cr)
1150 {
1151         return (0);
1152 }
1153
1154 uid_t
1155 crgetruid(cred_t *cr)
1156 {
1157         return (0);
1158 }
1159
1160 gid_t
1161 crgetgid(cred_t *cr)
1162 {
1163         return (0);
1164 }
1165
1166 int
1167 crgetngroups(cred_t *cr)
1168 {
1169         return (0);
1170 }
1171
1172 gid_t *
1173 crgetgroups(cred_t *cr)
1174 {
1175         return (NULL);
1176 }
1177
1178 int
1179 zfs_secpolicy_snapshot_perms(const char *name, cred_t *cr)
1180 {
1181         return (0);
1182 }
1183
1184 int
1185 zfs_secpolicy_rename_perms(const char *from, const char *to, cred_t *cr)
1186 {
1187         return (0);
1188 }
1189
1190 int
1191 zfs_secpolicy_destroy_perms(const char *name, cred_t *cr)
1192 {
1193         return (0);
1194 }
1195
1196 ksiddomain_t *
1197 ksid_lookupdomain(const char *dom)
1198 {
1199         ksiddomain_t *kd;
1200
1201         kd = umem_zalloc(sizeof (ksiddomain_t), UMEM_NOFAIL);
1202         kd->kd_name = spa_strdup(dom);
1203         return (kd);
1204 }
1205
1206 void
1207 ksiddomain_rele(ksiddomain_t *ksid)
1208 {
1209         spa_strfree(ksid->kd_name);
1210         umem_free(ksid, sizeof (ksiddomain_t));
1211 }
1212
1213 char *
1214 kmem_vasprintf(const char *fmt, va_list adx)
1215 {
1216         char *buf = NULL;
1217         va_list adx_copy;
1218
1219         va_copy(adx_copy, adx);
1220         VERIFY(vasprintf(&buf, fmt, adx_copy) != -1);
1221         va_end(adx_copy);
1222
1223         return (buf);
1224 }
1225
1226 char *
1227 kmem_asprintf(const char *fmt, ...)
1228 {
1229         char *buf = NULL;
1230         va_list adx;
1231
1232         va_start(adx, fmt);
1233         VERIFY(vasprintf(&buf, fmt, adx) != -1);
1234         va_end(adx);
1235
1236         return (buf);
1237 }
1238
1239 /* ARGSUSED */
1240 int
1241 zfs_onexit_fd_hold(int fd, minor_t *minorp)
1242 {
1243         *minorp = 0;
1244         return (0);
1245 }
1246
1247 /* ARGSUSED */
1248 void
1249 zfs_onexit_fd_rele(int fd)
1250 {
1251 }
1252
1253 /* ARGSUSED */
1254 int
1255 zfs_onexit_add_cb(minor_t minor, void (*func)(void *), void *data,
1256     uint64_t *action_handle)
1257 {
1258         return (0);
1259 }
1260
1261 /* ARGSUSED */
1262 int
1263 zfs_onexit_del_cb(minor_t minor, uint64_t action_handle, boolean_t fire)
1264 {
1265         return (0);
1266 }
1267
1268 /* ARGSUSED */
1269 int
1270 zfs_onexit_cb_data(minor_t minor, uint64_t action_handle, void **data)
1271 {
1272         return (0);
1273 }