lib/libzpool/kernel.c

   1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  23  */
  24
  25 #include <assert.h>
  26 #include <fcntl.h>
  27 #include <poll.h>
  28 #include <stdio.h>
  29 #include <stdlib.h>
  30 #include <string.h>
  31 #include <zlib.h>
  32 #include <sys/signal.h>
  33 #include <sys/spa.h>
  34 #include <sys/stat.h>
  35 #include <sys/processor.h>
  36 #include <sys/zfs_context.h>
  37 #include <sys/rrwlock.h>
  38 #include <sys/utsname.h>
  39 #include <sys/time.h>
  40 #include <sys/systeminfo.h>
  41
  42 /*
  43  * Emulation of kernel services in userland.
  44  */
  45
  46 int aok;
  47 uint64_t physmem;
  48 vnode_t *rootdir = (vnode_t *)0xabcd1234;
  49 char hw_serial[HW_HOSTID_LEN];
  50 struct utsname hw_utsname;
  51
  52 /* this only exists to have its address taken */
  53 struct proc p0;
  54
  55 /*
  56  * =========================================================================
  57  * threads
  58  * =========================================================================
  59  */
  60
  61 pthread_cond_t kthread_cond = PTHREAD_COND_INITIALIZER;
  62 pthread_mutex_t kthread_lock = PTHREAD_MUTEX_INITIALIZER;
  63 pthread_key_t kthread_key;
  64 int kthread_nr = 0;
  65
  66 static void
  67 thread_init(void)
  68 {
  69         kthread_t *kt;
  70
  71         VERIFY3S(pthread_key_create(&kthread_key, NULL), ==, 0);
  72
  73         /* Create entry for primary kthread */
  74         kt = umem_zalloc(sizeof (kthread_t), UMEM_NOFAIL);
  75         kt->t_tid = pthread_self();
  76         kt->t_func = NULL;
  77
  78         VERIFY3S(pthread_setspecific(kthread_key, kt), ==, 0);
  79
  80         /* Only the main thread should be running at the moment */
  81         ASSERT3S(kthread_nr, ==, 0);
  82         kthread_nr = 1;
  83 }
  84
  85 static void
  86 thread_fini(void)
  87 {
  88         kthread_t *kt = curthread;
  89
  90         ASSERT(pthread_equal(kt->t_tid, pthread_self()));
  91         ASSERT3P(kt->t_func, ==, NULL);
  92
  93         umem_free(kt, sizeof (kthread_t));
  94
  95         /* Wait for all threads to exit via thread_exit() */
  96         VERIFY3S(pthread_mutex_lock(&kthread_lock), ==, 0);
  97
  98         kthread_nr--; /* Main thread is exiting */
  99
 100         while (kthread_nr > 0)
 101                 VERIFY3S(pthread_cond_wait(&kthread_cond, &kthread_lock), ==,
 102                     0);
 103
 104         ASSERT3S(kthread_nr, ==, 0);
 105         VERIFY3S(pthread_mutex_unlock(&kthread_lock), ==, 0);
 106
 107         VERIFY3S(pthread_key_delete(kthread_key), ==, 0);
 108 }
 109
 110 kthread_t *
 111 zk_thread_current(void)
 112 {
 113         kthread_t *kt = pthread_getspecific(kthread_key);
 114
 115         ASSERT3P(kt, !=, NULL);
 116
 117         return (kt);
 118 }
 119
 120 void *
 121 zk_thread_helper(void *arg)
 122 {
 123         kthread_t *kt = (kthread_t *) arg;
 124
 125         VERIFY3S(pthread_setspecific(kthread_key, kt), ==, 0);
 126
 127         VERIFY3S(pthread_mutex_lock(&kthread_lock), ==, 0);
 128         kthread_nr++;
 129         VERIFY3S(pthread_mutex_unlock(&kthread_lock), ==, 0);
 130
 131         kt->t_tid = pthread_self();
 132         ((thread_func_arg_t) kt->t_func)(kt->t_arg);
 133
 134         /* Unreachable, thread must exit with thread_exit() */
 135         abort();
 136
 137         return (NULL);
 138 }
 139
 140 kthread_t *
 141 zk_thread_create(caddr_t stk, size_t stksize, thread_func_t func, void *arg,
 142     size_t len, proc_t *pp, int state, pri_t pri, int detachstate)
 143 {
 144         kthread_t *kt;
 145         pthread_attr_t attr;
 146         char *stkstr;
 147
 148         ASSERT0(state & ~TS_RUN);
 149
 150         kt = umem_zalloc(sizeof (kthread_t), UMEM_NOFAIL);
 151         kt->t_func = func;
 152         kt->t_arg = arg;
 153
 154         VERIFY0(pthread_attr_init(&attr));
 155         VERIFY0(pthread_attr_setdetachstate(&attr, detachstate));
 156
 157         /*
 158          * We allow the default stack size in user space to be specified by
 159          * setting the ZFS_STACK_SIZE environment variable.  This allows us
 160          * the convenience of observing and debugging stack overruns in
 161          * user space.  Explicitly specified stack sizes will be honored.
 162          * The usage of ZFS_STACK_SIZE is discussed further in the
 163          * ENVIRONMENT VARIABLES sections of the ztest(1) man page.
 164          */
 165         if (stksize == 0) {
 166                 stkstr = getenv("ZFS_STACK_SIZE");
 167
 168                 if (stkstr == NULL)
 169                         stksize = TS_STACK_MAX;
 170                 else
 171                         stksize = MAX(atoi(stkstr), TS_STACK_MIN);
 172         }
 173
 174         VERIFY3S(stksize, >, 0);
 175         stksize = P2ROUNDUP(MAX(stksize, TS_STACK_MIN), PAGESIZE);
 176         VERIFY0(pthread_attr_setstacksize(&attr, stksize));
 177         VERIFY0(pthread_attr_setguardsize(&attr, PAGESIZE));
 178
 179         VERIFY0(pthread_create(&kt->t_tid, &attr, &zk_thread_helper, kt));
 180         VERIFY0(pthread_attr_destroy(&attr));
 181
 182         return (kt);
 183 }
 184
 185 void
 186 zk_thread_exit(void)
 187 {
 188         kthread_t *kt = curthread;
 189
 190         ASSERT(pthread_equal(kt->t_tid, pthread_self()));
 191
 192         umem_free(kt, sizeof (kthread_t));
 193
 194         pthread_mutex_lock(&kthread_lock);
 195         kthread_nr--;
 196         pthread_mutex_unlock(&kthread_lock);
 197
 198         pthread_cond_broadcast(&kthread_cond);
 199         pthread_exit((void *)TS_MAGIC);
 200 }
 201
 202 void
 203 zk_thread_join(kt_did_t tid)
 204 {
 205         void *ret;
 206
 207         pthread_join((pthread_t)tid, &ret);
 208         VERIFY3P(ret, ==, (void *)TS_MAGIC);
 209 }
 210
 211 /*
 212  * =========================================================================
 213  * kstats
 214  * =========================================================================
 215  */
 216 /*ARGSUSED*/
 217 kstat_t *
 218 kstat_create(const char *module, int instance, const char *name,
 219     const char *class, uchar_t type, ulong_t ndata, uchar_t ks_flag)
 220 {
 221         return (NULL);
 222 }
 223
 224 /*ARGSUSED*/
 225 void
 226 kstat_install(kstat_t *ksp)
 227 {}
 228
 229 /*ARGSUSED*/
 230 void
 231 kstat_delete(kstat_t *ksp)
 232 {}
 233
 234 /*ARGSUSED*/
 235 void
 236 kstat_waitq_enter(kstat_io_t *kiop)
 237 {}
 238
 239 /*ARGSUSED*/
 240 void
 241 kstat_waitq_exit(kstat_io_t *kiop)
 242 {}
 243
 244 /*ARGSUSED*/
 245 void
 246 kstat_runq_enter(kstat_io_t *kiop)
 247 {}
 248
 249 /*ARGSUSED*/
 250 void
 251 kstat_runq_exit(kstat_io_t *kiop)
 252 {}
 253
 254 /*ARGSUSED*/
 255 void
 256 kstat_waitq_to_runq(kstat_io_t *kiop)
 257 {}
 258
 259 /*ARGSUSED*/
 260 void
 261 kstat_runq_back_to_waitq(kstat_io_t *kiop)
 262 {}
 263
 264 void
 265 kstat_set_raw_ops(kstat_t *ksp,
 266     int (*headers)(char *buf, size_t size),
 267     int (*data)(char *buf, size_t size, void *data),
 268     void *(*addr)(kstat_t *ksp, loff_t index))
 269 {}
 270
 271 /*
 272  * =========================================================================
 273  * mutexes
 274  * =========================================================================
 275  */
 276
 277 void
 278 mutex_init(kmutex_t *mp, char *name, int type, void *cookie)
 279 {
 280         ASSERT3S(type, ==, MUTEX_DEFAULT);
 281         ASSERT3P(cookie, ==, NULL);
 282         mp->m_owner = MTX_INIT;
 283         mp->m_magic = MTX_MAGIC;
 284         VERIFY3S(pthread_mutex_init(&mp->m_lock, NULL), ==, 0);
 285 }
 286
 287 void
 288 mutex_destroy(kmutex_t *mp)
 289 {
 290         ASSERT3U(mp->m_magic, ==, MTX_MAGIC);
 291         ASSERT3P(mp->m_owner, ==, MTX_INIT);
 292         ASSERT0(pthread_mutex_destroy(&(mp)->m_lock));
 293         mp->m_owner = MTX_DEST;
 294         mp->m_magic = 0;
 295 }
 296
 297 void
 298 mutex_enter(kmutex_t *mp)
 299 {
 300         ASSERT3U(mp->m_magic, ==, MTX_MAGIC);
 301         ASSERT3P(mp->m_owner, !=, MTX_DEST);
 302         ASSERT3P(mp->m_owner, !=, curthread);
 303         VERIFY3S(pthread_mutex_lock(&mp->m_lock), ==, 0);
 304         ASSERT3P(mp->m_owner, ==, MTX_INIT);
 305         mp->m_owner = curthread;
 306 }
 307
 308 int
 309 mutex_tryenter(kmutex_t *mp)
 310 {
 311         ASSERT3U(mp->m_magic, ==, MTX_MAGIC);
 312         ASSERT3P(mp->m_owner, !=, MTX_DEST);
 313         if (0 == pthread_mutex_trylock(&mp->m_lock)) {
 314                 ASSERT3P(mp->m_owner, ==, MTX_INIT);
 315                 mp->m_owner = curthread;
 316                 return (1);
 317         } else {
 318                 return (0);
 319         }
 320 }
 321
 322 void
 323 mutex_exit(kmutex_t *mp)
 324 {
 325         ASSERT3U(mp->m_magic, ==, MTX_MAGIC);
 326         ASSERT3P(mutex_owner(mp), ==, curthread);
 327         mp->m_owner = MTX_INIT;
 328         VERIFY3S(pthread_mutex_unlock(&mp->m_lock), ==, 0);
 329 }
 330
 331 void *
 332 mutex_owner(kmutex_t *mp)
 333 {
 334         ASSERT3U(mp->m_magic, ==, MTX_MAGIC);
 335         return (mp->m_owner);
 336 }
 337
 338 int
 339 mutex_held(kmutex_t *mp)
 340 {
 341         return (mp->m_owner == curthread);
 342 }
 343
 344 /*
 345  * =========================================================================
 346  * rwlocks
 347  * =========================================================================
 348  */
 349
 350 void
 351 rw_init(krwlock_t *rwlp, char *name, int type, void *arg)
 352 {
 353         ASSERT3S(type, ==, RW_DEFAULT);
 354         ASSERT3P(arg, ==, NULL);
 355         VERIFY3S(pthread_rwlock_init(&rwlp->rw_lock, NULL), ==, 0);
 356         rwlp->rw_owner = RW_INIT;
 357         rwlp->rw_wr_owner = RW_INIT;
 358         rwlp->rw_readers = 0;
 359         rwlp->rw_magic = RW_MAGIC;
 360 }
 361
 362 void
 363 rw_destroy(krwlock_t *rwlp)
 364 {
 365         ASSERT3U(rwlp->rw_magic, ==, RW_MAGIC);
 366         ASSERT(rwlp->rw_readers == 0 && rwlp->rw_wr_owner == RW_INIT);
 367         VERIFY3S(pthread_rwlock_destroy(&rwlp->rw_lock), ==, 0);
 368         rwlp->rw_magic = 0;
 369 }
 370
 371 void
 372 rw_enter(krwlock_t *rwlp, krw_t rw)
 373 {
 374         ASSERT3U(rwlp->rw_magic, ==, RW_MAGIC);
 375         ASSERT3P(rwlp->rw_owner, !=, curthread);
 376         ASSERT3P(rwlp->rw_wr_owner, !=, curthread);
 377
 378         if (rw == RW_READER) {
 379                 VERIFY3S(pthread_rwlock_rdlock(&rwlp->rw_lock), ==, 0);
 380                 ASSERT3P(rwlp->rw_wr_owner, ==, RW_INIT);
 381
 382                 atomic_inc_uint(&rwlp->rw_readers);
 383         } else {
 384                 VERIFY3S(pthread_rwlock_wrlock(&rwlp->rw_lock), ==, 0);
 385                 ASSERT3P(rwlp->rw_wr_owner, ==, RW_INIT);
 386                 ASSERT3U(rwlp->rw_readers, ==, 0);
 387
 388                 rwlp->rw_wr_owner = curthread;
 389         }
 390
 391         rwlp->rw_owner = curthread;
 392 }
 393
 394 void
 395 rw_exit(krwlock_t *rwlp)
 396 {
 397         ASSERT3U(rwlp->rw_magic, ==, RW_MAGIC);
 398         ASSERT(RW_LOCK_HELD(rwlp));
 399
 400         if (RW_READ_HELD(rwlp))
 401                 atomic_dec_uint(&rwlp->rw_readers);
 402         else
 403                 rwlp->rw_wr_owner = RW_INIT;
 404
 405         rwlp->rw_owner = RW_INIT;
 406         VERIFY3S(pthread_rwlock_unlock(&rwlp->rw_lock), ==, 0);
 407 }
 408
 409 int
 410 rw_tryenter(krwlock_t *rwlp, krw_t rw)
 411 {
 412         int rv;
 413
 414         ASSERT3U(rwlp->rw_magic, ==, RW_MAGIC);
 415
 416         if (rw == RW_READER)
 417                 rv = pthread_rwlock_tryrdlock(&rwlp->rw_lock);
 418         else
 419                 rv = pthread_rwlock_trywrlock(&rwlp->rw_lock);
 420
 421         if (rv == 0) {
 422                 ASSERT3P(rwlp->rw_wr_owner, ==, RW_INIT);
 423
 424                 if (rw == RW_READER)
 425                         atomic_inc_uint(&rwlp->rw_readers);
 426                 else {
 427                         ASSERT3U(rwlp->rw_readers, ==, 0);
 428                         rwlp->rw_wr_owner = curthread;
 429                 }
 430
 431                 rwlp->rw_owner = curthread;
 432                 return (1);
 433         }
 434
 435         VERIFY3S(rv, ==, EBUSY);
 436
 437         return (0);
 438 }
 439
 440 int
 441 rw_tryupgrade(krwlock_t *rwlp)
 442 {
 443         ASSERT3U(rwlp->rw_magic, ==, RW_MAGIC);
 444
 445         return (0);
 446 }
 447
 448 /*
 449  * =========================================================================
 450  * condition variables
 451  * =========================================================================
 452  */
 453
 454 void
 455 cv_init(kcondvar_t *cv, char *name, int type, void *arg)
 456 {
 457         ASSERT3S(type, ==, CV_DEFAULT);
 458         cv->cv_magic = CV_MAGIC;
 459         VERIFY3S(pthread_cond_init(&cv->cv, NULL), ==, 0);
 460 }
 461
 462 void
 463 cv_destroy(kcondvar_t *cv)
 464 {
 465         ASSERT3U(cv->cv_magic, ==, CV_MAGIC);
 466         VERIFY3S(pthread_cond_destroy(&cv->cv), ==, 0);
 467         cv->cv_magic = 0;
 468 }
 469
 470 void
 471 cv_wait(kcondvar_t *cv, kmutex_t *mp)
 472 {
 473         ASSERT3U(cv->cv_magic, ==, CV_MAGIC);
 474         ASSERT3P(mutex_owner(mp), ==, curthread);
 475         mp->m_owner = MTX_INIT;
 476         int ret = pthread_cond_wait(&cv->cv, &mp->m_lock);
 477         if (ret != 0)
 478                 VERIFY3S(ret, ==, EINTR);
 479         mp->m_owner = curthread;
 480 }
 481
 482 clock_t
 483 cv_timedwait(kcondvar_t *cv, kmutex_t *mp, clock_t abstime)
 484 {
 485         int error;
 486         struct timeval tv;
 487         timestruc_t ts;
 488         clock_t delta;
 489
 490         ASSERT3U(cv->cv_magic, ==, CV_MAGIC);
 491
 492 top:
 493         delta = abstime - ddi_get_lbolt();
 494         if (delta <= 0)
 495                 return (-1);
 496
 497         VERIFY(gettimeofday(&tv, NULL) == 0);
 498
 499         ts.tv_sec = tv.tv_sec + delta / hz;
 500         ts.tv_nsec = tv.tv_usec * 1000 + (delta % hz) * (NANOSEC / hz);
 501         if (ts.tv_nsec >= NANOSEC) {
 502                 ts.tv_sec++;
 503                 ts.tv_nsec -= NANOSEC;
 504         }
 505
 506         ASSERT3P(mutex_owner(mp), ==, curthread);
 507         mp->m_owner = MTX_INIT;
 508         error = pthread_cond_timedwait(&cv->cv, &mp->m_lock, &ts);
 509         mp->m_owner = curthread;
 510
 511         if (error == ETIMEDOUT)
 512                 return (-1);
 513
 514         if (error == EINTR)
 515                 goto top;
 516
 517         VERIFY3S(error, ==, 0);
 518
 519         return (1);
 520 }
 521
 522 /*ARGSUSED*/
 523 clock_t
 524 cv_timedwait_hires(kcondvar_t *cv, kmutex_t *mp, hrtime_t tim, hrtime_t res,
 525     int flag)
 526 {
 527         int error;
 528         timestruc_t ts;
 529         hrtime_t delta;
 530
 531         ASSERT(flag == 0);
 532
 533 top:
 534         delta = tim - gethrtime();
 535         if (delta <= 0)
 536                 return (-1);
 537
 538         ts.tv_sec = delta / NANOSEC;
 539         ts.tv_nsec = delta % NANOSEC;
 540
 541         ASSERT(mutex_owner(mp) == curthread);
 542         mp->m_owner = NULL;
 543         error = pthread_cond_timedwait(&cv->cv, &mp->m_lock, &ts);
 544         mp->m_owner = curthread;
 545
 546         if (error == ETIME)
 547                 return (-1);
 548
 549         if (error == EINTR)
 550                 goto top;
 551
 552         ASSERT(error == 0);
 553
 554         return (1);
 555 }
 556
 557 void
 558 cv_signal(kcondvar_t *cv)
 559 {
 560         ASSERT3U(cv->cv_magic, ==, CV_MAGIC);
 561         VERIFY3S(pthread_cond_signal(&cv->cv), ==, 0);
 562 }
 563
 564 void
 565 cv_broadcast(kcondvar_t *cv)
 566 {
 567         ASSERT3U(cv->cv_magic, ==, CV_MAGIC);
 568         VERIFY3S(pthread_cond_broadcast(&cv->cv), ==, 0);
 569 }
 570
 571 /*
 572  * =========================================================================
 573  * vnode operations
 574  * =========================================================================
 575  */
 576 /*
 577  * Note: for the xxxat() versions of these functions, we assume that the
 578  * starting vp is always rootdir (which is true for spa_directory.c, the only
 579  * ZFS consumer of these interfaces).  We assert this is true, and then emulate
 580  * them by adding '/' in front of the path.
 581  */
 582
 583 /*ARGSUSED*/
 584 int
 585 vn_open(char *path, int x1, int flags, int mode, vnode_t **vpp, int x2, int x3)
 586 {
 587         int fd;
 588         vnode_t *vp;
 589         int old_umask = 0;
 590         char *realpath;
 591         struct stat64 st;
 592         int err;
 593
 594         realpath = umem_alloc(MAXPATHLEN, UMEM_NOFAIL);
 595
 596         /*
 597          * If we're accessing a real disk from userland, we need to use
 598          * the character interface to avoid caching.  This is particularly
 599          * important if we're trying to look at a real in-kernel storage
 600          * pool from userland, e.g. via zdb, because otherwise we won't
 601          * see the changes occurring under the segmap cache.
 602          * On the other hand, the stupid character device returns zero
 603          * for its size.  So -- gag -- we open the block device to get
 604          * its size, and remember it for subsequent VOP_GETATTR().
 605          */
 606 #if defined(__sun__) || defined(__sun)
 607         if (strncmp(path, "/dev/", 5) == 0) {
 608 #else
 609         if (0) {
 610 #endif
 611                 char *dsk;
 612                 fd = open64(path, O_RDONLY);
 613                 if (fd == -1) {
 614                         err = errno;
 615                         free(realpath);
 616                         return (err);
 617                 }
 618                 if (fstat64(fd, &st) == -1) {
 619                         err = errno;
 620                         close(fd);
 621                         free(realpath);
 622                         return (err);
 623                 }
 624                 close(fd);
 625                 (void) sprintf(realpath, "%s", path);
 626                 dsk = strstr(path, "/dsk/");
 627                 if (dsk != NULL)
 628                         (void) sprintf(realpath + (dsk - path) + 1, "r%s",
 629                             dsk + 1);
 630         } else {
 631                 (void) sprintf(realpath, "%s", path);
 632                 if (!(flags & FCREAT) && stat64(realpath, &st) == -1) {
 633                         err = errno;
 634                         free(realpath);
 635                         return (err);
 636                 }
 637         }
 638
 639         if (!(flags & FCREAT) && S_ISBLK(st.st_mode)) {
 640 #ifdef __linux__
 641                 flags |= O_DIRECT;
 642 #endif
 643                 /* We shouldn't be writing to block devices in userspace */
 644                 VERIFY(!(flags & FWRITE));
 645         }
 646
 647         if (flags & FCREAT)
 648                 old_umask = umask(0);
 649
 650         /*
 651          * The construct 'flags - FREAD' conveniently maps combinations of
 652          * FREAD and FWRITE to the corresponding O_RDONLY, O_WRONLY, and O_RDWR.
 653          */
 654         fd = open64(realpath, flags - FREAD, mode);
 655         free(realpath);
 656
 657         if (flags & FCREAT)
 658                 (void) umask(old_umask);
 659
 660         if (fd == -1)
 661                 return (errno);
 662
 663         if (fstat64_blk(fd, &st) == -1) {
 664                 err = errno;
 665                 close(fd);
 666                 return (err);
 667         }
 668
 669         (void) fcntl(fd, F_SETFD, FD_CLOEXEC);
 670
 671         *vpp = vp = umem_zalloc(sizeof (vnode_t), UMEM_NOFAIL);
 672
 673         vp->v_fd = fd;
 674         vp->v_size = st.st_size;
 675         vp->v_path = spa_strdup(path);
 676
 677         return (0);
 678 }
 679
 680 /*ARGSUSED*/
 681 int
 682 vn_openat(char *path, int x1, int flags, int mode, vnode_t **vpp, int x2,
 683     int x3, vnode_t *startvp, int fd)
 684 {
 685         char *realpath = umem_alloc(strlen(path) + 2, UMEM_NOFAIL);
 686         int ret;
 687
 688         ASSERT(startvp == rootdir);
 689         (void) sprintf(realpath, "/%s", path);
 690
 691         /* fd ignored for now, need if want to simulate nbmand support */
 692         ret = vn_open(realpath, x1, flags, mode, vpp, x2, x3);
 693
 694         umem_free(realpath, strlen(path) + 2);
 695
 696         return (ret);
 697 }
 698
 699 /*ARGSUSED*/
 700 int
 701 vn_rdwr(int uio, vnode_t *vp, void *addr, ssize_t len, offset_t offset,
 702         int x1, int x2, rlim64_t x3, void *x4, ssize_t *residp)
 703 {
 704         ssize_t rc, done = 0, split;
 705
 706         if (uio == UIO_READ) {
 707                 rc = pread64(vp->v_fd, addr, len, offset);
 708         } else {
 709                 /*
 710                  * To simulate partial disk writes, we split writes into two
 711                  * system calls so that the process can be killed in between.
 712                  */
 713                 int sectors = len >> SPA_MINBLOCKSHIFT;
 714                 split = (sectors > 0 ? rand() % sectors : 0) <<
 715                     SPA_MINBLOCKSHIFT;
 716                 rc = pwrite64(vp->v_fd, addr, split, offset);
 717                 if (rc != -1) {
 718                         done = rc;
 719                         rc = pwrite64(vp->v_fd, (char *)addr + split,
 720                             len - split, offset + split);
 721                 }
 722         }
 723
 724 #ifdef __linux__
 725         if (rc == -1 && errno == EINVAL) {
 726                 /*
 727                  * Under Linux, this most likely means an alignment issue
 728                  * (memory or disk) due to O_DIRECT, so we abort() in order to
 729                  * catch the offender.
 730                  */
 731                 abort();
 732         }
 733 #endif
 734         if (rc == -1)
 735                 return (errno);
 736
 737         done += rc;
 738
 739         if (residp)
 740                 *residp = len - done;
 741         else if (done != len)
 742                 return (EIO);
 743         return (0);
 744 }
 745
 746 void
 747 vn_close(vnode_t *vp)
 748 {
 749         close(vp->v_fd);
 750         spa_strfree(vp->v_path);
 751         umem_free(vp, sizeof (vnode_t));
 752 }
 753
 754 /*
 755  * At a minimum we need to update the size since vdev_reopen()
 756  * will no longer call vn_openat().
 757  */
 758 int
 759 fop_getattr(vnode_t *vp, vattr_t *vap)
 760 {
 761         struct stat64 st;
 762         int err;
 763
 764         if (fstat64_blk(vp->v_fd, &st) == -1) {
 765                 err = errno;
 766                 close(vp->v_fd);
 767                 return (err);
 768         }
 769
 770         vap->va_size = st.st_size;
 771         return (0);
 772 }
 773
 774 /*
 775  * =========================================================================
 776  * Figure out which debugging statements to print
 777  * =========================================================================
 778  */
 779
 780 static char *dprintf_string;
 781 static int dprintf_print_all;
 782
 783 int
 784 dprintf_find_string(const char *string)
 785 {
 786         char *tmp_str = dprintf_string;
 787         int len = strlen(string);
 788
 789         /*
 790          * Find out if this is a string we want to print.
 791          * String format: file1.c,function_name1,file2.c,file3.c
 792          */
 793
 794         while (tmp_str != NULL) {
 795                 if (strncmp(tmp_str, string, len) == 0 &&
 796                     (tmp_str[len] == ',' || tmp_str[len] == '\0'))
 797                         return (1);
 798                 tmp_str = strchr(tmp_str, ',');
 799                 if (tmp_str != NULL)
 800                         tmp_str++; /* Get rid of , */
 801         }
 802         return (0);
 803 }
 804
 805 void
 806 dprintf_setup(int *argc, char **argv)
 807 {
 808         int i, j;
 809
 810         /*
 811          * Debugging can be specified two ways: by setting the
 812          * environment variable ZFS_DEBUG, or by including a
 813          * "debug=..."  argument on the command line.  The command
 814          * line setting overrides the environment variable.
 815          */
 816
 817         for (i = 1; i < *argc; i++) {
 818                 int len = strlen("debug=");
 819                 /* First look for a command line argument */
 820                 if (strncmp("debug=", argv[i], len) == 0) {
 821                         dprintf_string = argv[i] + len;
 822                         /* Remove from args */
 823                         for (j = i; j < *argc; j++)
 824                                 argv[j] = argv[j+1];
 825                         argv[j] = NULL;
 826                         (*argc)--;
 827                 }
 828         }
 829
 830         if (dprintf_string == NULL) {
 831                 /* Look for ZFS_DEBUG environment variable */
 832                 dprintf_string = getenv("ZFS_DEBUG");
 833         }
 834
 835         /*
 836          * Are we just turning on all debugging?
 837          */
 838         if (dprintf_find_string("on"))
 839                 dprintf_print_all = 1;
 840 }
 841
 842 /*
 843  * =========================================================================
 844  * debug printfs
 845  * =========================================================================
 846  */
 847 void
 848 __dprintf(const char *file, const char *func, int line, const char *fmt, ...)
 849 {
 850         const char *newfile;
 851         va_list adx;
 852
 853         /*
 854          * Get rid of annoying "../common/" prefix to filename.
 855          */
 856         newfile = strrchr(file, '/');
 857         if (newfile != NULL) {
 858                 newfile = newfile + 1; /* Get rid of leading / */
 859         } else {
 860                 newfile = file;
 861         }
 862
 863         if (dprintf_print_all ||
 864             dprintf_find_string(newfile) ||
 865             dprintf_find_string(func)) {
 866                 /* Print out just the function name if requested */
 867                 flockfile(stdout);
 868                 if (dprintf_find_string("pid"))
 869                         (void) printf("%d ", getpid());
 870                 if (dprintf_find_string("tid"))
 871                         (void) printf("%u ", (uint_t) pthread_self());
 872                 if (dprintf_find_string("cpu"))
 873                         (void) printf("%u ", getcpuid());
 874                 if (dprintf_find_string("time"))
 875                         (void) printf("%llu ", gethrtime());
 876                 if (dprintf_find_string("long"))
 877                         (void) printf("%s, line %d: ", newfile, line);
 878                 (void) printf("%s: ", func);
 879                 va_start(adx, fmt);
 880                 (void) vprintf(fmt, adx);
 881                 va_end(adx);
 882                 funlockfile(stdout);
 883         }
 884 }
 885
 886 /*
 887  * =========================================================================
 888  * cmn_err() and panic()
 889  * =========================================================================
 890  */
 891 static char ce_prefix[CE_IGNORE][10] = { "", "NOTICE: ", "WARNING: ", "" };
 892 static char ce_suffix[CE_IGNORE][2] = { "", "\n", "\n", "" };
 893
 894 void
 895 vpanic(const char *fmt, va_list adx)
 896 {
 897         (void) fprintf(stderr, "error: ");
 898         (void) vfprintf(stderr, fmt, adx);
 899         (void) fprintf(stderr, "\n");
 900
 901         abort();        /* think of it as a "user-level crash dump" */
 902 }
 903
 904 void
 905 panic(const char *fmt, ...)
 906 {
 907         va_list adx;
 908
 909         va_start(adx, fmt);
 910         vpanic(fmt, adx);
 911         va_end(adx);
 912 }
 913
 914 void
 915 vcmn_err(int ce, const char *fmt, va_list adx)
 916 {
 917         if (ce == CE_PANIC)
 918                 vpanic(fmt, adx);
 919         if (ce != CE_NOTE) {    /* suppress noise in userland stress testing */
 920                 (void) fprintf(stderr, "%s", ce_prefix[ce]);
 921                 (void) vfprintf(stderr, fmt, adx);
 922                 (void) fprintf(stderr, "%s", ce_suffix[ce]);
 923         }
 924 }
 925
 926 /*PRINTFLIKE2*/
 927 void
 928 cmn_err(int ce, const char *fmt, ...)
 929 {
 930         va_list adx;
 931
 932         va_start(adx, fmt);
 933         vcmn_err(ce, fmt, adx);
 934         va_end(adx);
 935 }
 936
 937 /*
 938  * =========================================================================
 939  * kobj interfaces
 940  * =========================================================================
 941  */
 942 struct _buf *
 943 kobj_open_file(char *name)
 944 {
 945         struct _buf *file;
 946         vnode_t *vp;
 947
 948         /* set vp as the _fd field of the file */
 949         if (vn_openat(name, UIO_SYSSPACE, FREAD, 0, &vp, 0, 0, rootdir,
 950             -1) != 0)
 951                 return ((void *)-1UL);
 952
 953         file = umem_zalloc(sizeof (struct _buf), UMEM_NOFAIL);
 954         file->_fd = (intptr_t)vp;
 955         return (file);
 956 }
 957
 958 int
 959 kobj_read_file(struct _buf *file, char *buf, unsigned size, unsigned off)
 960 {
 961         ssize_t resid;
 962
 963         vn_rdwr(UIO_READ, (vnode_t *)file->_fd, buf, size, (offset_t)off,
 964             UIO_SYSSPACE, 0, 0, 0, &resid);
 965
 966         return (size - resid);
 967 }
 968
 969 void
 970 kobj_close_file(struct _buf *file)
 971 {
 972         vn_close((vnode_t *)file->_fd);
 973         umem_free(file, sizeof (struct _buf));
 974 }
 975
 976 int
 977 kobj_get_filesize(struct _buf *file, uint64_t *size)
 978 {
 979         struct stat64 st;
 980         vnode_t *vp = (vnode_t *)file->_fd;
 981
 982         if (fstat64(vp->v_fd, &st) == -1) {
 983                 vn_close(vp);
 984                 return (errno);
 985         }
 986         *size = st.st_size;
 987         return (0);
 988 }
 989
 990 /*
 991  * =========================================================================
 992  * misc routines
 993  * =========================================================================
 994  */
 995
 996 void
 997 delay(clock_t ticks)
 998 {
 999         poll(0, 0, ticks * (1000 / hz));
1000 }
1001
1002 /*
1003  * Find highest one bit set.
1004  *      Returns bit number + 1 of highest bit that is set, otherwise returns 0.
1005  * High order bit is 31 (or 63 in _LP64 kernel).
1006  */
1007 int
1008 highbit64(uint64_t i)
1009 {
1010         register int h = 1;
1011
1012         if (i == 0)
1013                 return (0);
1014         if (i & 0xffffffff00000000ULL) {
1015                 h += 32; i >>= 32;
1016         }
1017         if (i & 0xffff0000) {
1018                 h += 16; i >>= 16;
1019         }
1020         if (i & 0xff00) {
1021                 h += 8; i >>= 8;
1022         }
1023         if (i & 0xf0) {
1024                 h += 4; i >>= 4;
1025         }
1026         if (i & 0xc) {
1027                 h += 2; i >>= 2;
1028         }
1029         if (i & 0x2) {
1030                 h += 1;
1031         }
1032         return (h);
1033 }
1034
1035 static int random_fd = -1, urandom_fd = -1;
1036
1037 static int
1038 random_get_bytes_common(uint8_t *ptr, size_t len, int fd)
1039 {
1040         size_t resid = len;
1041         ssize_t bytes;
1042
1043         ASSERT(fd != -1);
1044
1045         while (resid != 0) {
1046                 bytes = read(fd, ptr, resid);
1047                 ASSERT3S(bytes, >=, 0);
1048                 ptr += bytes;
1049                 resid -= bytes;
1050         }
1051
1052         return (0);
1053 }
1054
1055 int
1056 random_get_bytes(uint8_t *ptr, size_t len)
1057 {
1058         return (random_get_bytes_common(ptr, len, random_fd));
1059 }
1060
1061 int
1062 random_get_pseudo_bytes(uint8_t *ptr, size_t len)
1063 {
1064         return (random_get_bytes_common(ptr, len, urandom_fd));
1065 }
1066
1067 int
1068 ddi_strtoul(const char *hw_serial, char **nptr, int base, unsigned long *result)
1069 {
1070         char *end;
1071
1072         *result = strtoul(hw_serial, &end, base);
1073         if (*result == 0)
1074                 return (errno);
1075         return (0);
1076 }
1077
1078 int
1079 ddi_strtoull(const char *str, char **nptr, int base, u_longlong_t *result)
1080 {
1081         char *end;
1082
1083         *result = strtoull(str, &end, base);
1084         if (*result == 0)
1085                 return (errno);
1086         return (0);
1087 }
1088
1089 utsname_t *
1090 utsname(void)
1091 {
1092         return (&hw_utsname);
1093 }
1094
1095 /*
1096  * =========================================================================
1097  * kernel emulation setup & teardown
1098  * =========================================================================
1099  */
1100 static int
1101 umem_out_of_memory(void)
1102 {
1103         char errmsg[] = "out of memory -- generating core dump\n";
1104
1105         (void) fprintf(stderr, "%s", errmsg);
1106         abort();
1107         return (0);
1108 }
1109
1110 static unsigned long
1111 get_spl_hostid(void)
1112 {
1113         FILE *f;
1114         unsigned long hostid;
1115
1116         f = fopen("/sys/module/spl/parameters/spl_hostid", "r");
1117         if (!f)
1118                 return (0);
1119         if (fscanf(f, "%lu", &hostid) != 1)
1120                 hostid = 0;
1121         fclose(f);
1122         return (hostid & 0xffffffff);
1123 }
1124
1125 unsigned long
1126 get_system_hostid(void)
1127 {
1128         unsigned long system_hostid = get_spl_hostid();
1129         if (system_hostid == 0)
1130                 system_hostid = gethostid() & 0xffffffff;
1131         return (system_hostid);
1132 }
1133
1134 void
1135 kernel_init(int mode)
1136 {
1137         extern uint_t rrw_tsd_key;
1138
1139         umem_nofail_callback(umem_out_of_memory);
1140
1141         physmem = sysconf(_SC_PHYS_PAGES);
1142
1143         dprintf("physmem = %llu pages (%.2f GB)\n", physmem,
1144             (double)physmem * sysconf(_SC_PAGE_SIZE) / (1ULL << 30));
1145
1146         (void) snprintf(hw_serial, sizeof (hw_serial), "%ld",
1147             (mode & FWRITE) ? get_system_hostid() : 0);
1148
1149         VERIFY((random_fd = open("/dev/random", O_RDONLY)) != -1);
1150         VERIFY((urandom_fd = open("/dev/urandom", O_RDONLY)) != -1);
1151         VERIFY0(uname(&hw_utsname));
1152
1153         thread_init();
1154         system_taskq_init();
1155
1156         spa_init(mode);
1157
1158         tsd_create(&rrw_tsd_key, rrw_tsd_destroy);
1159 }
1160
1161 void
1162 kernel_fini(void)
1163 {
1164         spa_fini();
1165
1166         system_taskq_fini();
1167         thread_fini();
1168
1169         close(random_fd);
1170         close(urandom_fd);
1171
1172         random_fd = -1;
1173         urandom_fd = -1;
1174 }
1175
1176 uid_t
1177 crgetuid(cred_t *cr)
1178 {
1179         return (0);
1180 }
1181
1182 uid_t
1183 crgetruid(cred_t *cr)
1184 {
1185         return (0);
1186 }
1187
1188 gid_t
1189 crgetgid(cred_t *cr)
1190 {
1191         return (0);
1192 }
1193
1194 int
1195 crgetngroups(cred_t *cr)
1196 {
1197         return (0);
1198 }
1199
1200 gid_t *
1201 crgetgroups(cred_t *cr)
1202 {
1203         return (NULL);
1204 }
1205
1206 int
1207 zfs_secpolicy_snapshot_perms(const char *name, cred_t *cr)
1208 {
1209         return (0);
1210 }
1211
1212 int
1213 zfs_secpolicy_rename_perms(const char *from, const char *to, cred_t *cr)
1214 {
1215         return (0);
1216 }
1217
1218 int
1219 zfs_secpolicy_destroy_perms(const char *name, cred_t *cr)
1220 {
1221         return (0);
1222 }
1223
1224 ksiddomain_t *
1225 ksid_lookupdomain(const char *dom)
1226 {
1227         ksiddomain_t *kd;
1228
1229         kd = umem_zalloc(sizeof (ksiddomain_t), UMEM_NOFAIL);
1230         kd->kd_name = spa_strdup(dom);
1231         return (kd);
1232 }
1233
1234 void
1235 ksiddomain_rele(ksiddomain_t *ksid)
1236 {
1237         spa_strfree(ksid->kd_name);
1238         umem_free(ksid, sizeof (ksiddomain_t));
1239 }
1240
1241 char *
1242 kmem_vasprintf(const char *fmt, va_list adx)
1243 {
1244         char *buf = NULL;
1245         va_list adx_copy;
1246
1247         va_copy(adx_copy, adx);
1248         VERIFY(vasprintf(&buf, fmt, adx_copy) != -1);
1249         va_end(adx_copy);
1250
1251         return (buf);
1252 }
1253
1254 char *
1255 kmem_asprintf(const char *fmt, ...)
1256 {
1257         char *buf = NULL;
1258         va_list adx;
1259
1260         va_start(adx, fmt);
1261         VERIFY(vasprintf(&buf, fmt, adx) != -1);
1262         va_end(adx);
1263
1264         return (buf);
1265 }
1266
1267 /* ARGSUSED */
1268 int
1269 zfs_onexit_fd_hold(int fd, minor_t *minorp)
1270 {
1271         *minorp = 0;
1272         return (0);
1273 }
1274
1275 /* ARGSUSED */
1276 void
1277 zfs_onexit_fd_rele(int fd)
1278 {
1279 }
1280
1281 /* ARGSUSED */
1282 int
1283 zfs_onexit_add_cb(minor_t minor, void (*func)(void *), void *data,
1284     uint64_t *action_handle)
1285 {
1286         return (0);
1287 }
1288
1289 /* ARGSUSED */
1290 int
1291 zfs_onexit_del_cb(minor_t minor, uint64_t action_handle, boolean_t fire)
1292 {
1293         return (0);
1294 }
1295
1296 /* ARGSUSED */
1297 int
1298 zfs_onexit_cb_data(minor_t minor, uint64_t action_handle, void **data)
1299 {
1300         return (0);
1301 }
1302
1303 fstrans_cookie_t
1304 spl_fstrans_mark(void)
1305 {
1306         return ((fstrans_cookie_t) 0);
1307 }
1308
1309 void
1310 spl_fstrans_unmark(fstrans_cookie_t cookie)
1311 {
1312 }
1313
1314 int
1315 spl_fstrans_check(void)
1316 {
1317         return (0);
1318 }