lib/libzpool/kernel.c

   1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  23  */
  24
  25 #include <assert.h>
  26 #include <fcntl.h>
  27 #include <poll.h>
  28 #include <stdio.h>
  29 #include <stdlib.h>
  30 #include <string.h>
  31 #include <zlib.h>
  32 #include <libgen.h>
  33 #include <sys/signal.h>
  34 #include <sys/spa.h>
  35 #include <sys/stat.h>
  36 #include <sys/processor.h>
  37 #include <sys/zfs_context.h>
  38 #include <sys/rrwlock.h>
  39 #include <sys/utsname.h>
  40 #include <sys/time.h>
  41 #include <sys/systeminfo.h>
  42
  43 /*
  44  * Emulation of kernel services in userland.
  45  */
  46
  47 int aok;
  48 uint64_t physmem;
  49 vnode_t *rootdir = (vnode_t *)0xabcd1234;
  50 char hw_serial[HW_HOSTID_LEN];
  51 struct utsname hw_utsname;
  52 vmem_t *zio_arena = NULL;
  53
  54 /* If set, all blocks read will be copied to the specified directory. */
  55 char *vn_dumpdir = NULL;
  56
  57 /* this only exists to have its address taken */
  58 struct proc p0;
  59
  60 /*
  61  * =========================================================================
  62  * threads
  63  * =========================================================================
  64  */
  65
  66 pthread_cond_t kthread_cond = PTHREAD_COND_INITIALIZER;
  67 pthread_mutex_t kthread_lock = PTHREAD_MUTEX_INITIALIZER;
  68 pthread_key_t kthread_key;
  69 int kthread_nr = 0;
  70
  71 static void
  72 thread_init(void)
  73 {
  74         kthread_t *kt;
  75
  76         VERIFY3S(pthread_key_create(&kthread_key, NULL), ==, 0);
  77
  78         /* Create entry for primary kthread */
  79         kt = umem_zalloc(sizeof (kthread_t), UMEM_NOFAIL);
  80         kt->t_tid = pthread_self();
  81         kt->t_func = NULL;
  82
  83         VERIFY3S(pthread_setspecific(kthread_key, kt), ==, 0);
  84
  85         /* Only the main thread should be running at the moment */
  86         ASSERT3S(kthread_nr, ==, 0);
  87         kthread_nr = 1;
  88 }
  89
  90 static void
  91 thread_fini(void)
  92 {
  93         kthread_t *kt = curthread;
  94
  95         ASSERT(pthread_equal(kt->t_tid, pthread_self()));
  96         ASSERT3P(kt->t_func, ==, NULL);
  97
  98         umem_free(kt, sizeof (kthread_t));
  99
 100         /* Wait for all threads to exit via thread_exit() */
 101         VERIFY3S(pthread_mutex_lock(&kthread_lock), ==, 0);
 102
 103         kthread_nr--; /* Main thread is exiting */
 104
 105         while (kthread_nr > 0)
 106                 VERIFY3S(pthread_cond_wait(&kthread_cond, &kthread_lock), ==,
 107                     0);
 108
 109         ASSERT3S(kthread_nr, ==, 0);
 110         VERIFY3S(pthread_mutex_unlock(&kthread_lock), ==, 0);
 111
 112         VERIFY3S(pthread_key_delete(kthread_key), ==, 0);
 113 }
 114
 115 kthread_t *
 116 zk_thread_current(void)
 117 {
 118         kthread_t *kt = pthread_getspecific(kthread_key);
 119
 120         ASSERT3P(kt, !=, NULL);
 121
 122         return (kt);
 123 }
 124
 125 void *
 126 zk_thread_helper(void *arg)
 127 {
 128         kthread_t *kt = (kthread_t *) arg;
 129
 130         VERIFY3S(pthread_setspecific(kthread_key, kt), ==, 0);
 131
 132         VERIFY3S(pthread_mutex_lock(&kthread_lock), ==, 0);
 133         kthread_nr++;
 134         VERIFY3S(pthread_mutex_unlock(&kthread_lock), ==, 0);
 135         (void) setpriority(PRIO_PROCESS, 0, kt->t_pri);
 136
 137         kt->t_tid = pthread_self();
 138         ((thread_func_arg_t) kt->t_func)(kt->t_arg);
 139
 140         /* Unreachable, thread must exit with thread_exit() */
 141         abort();
 142
 143         return (NULL);
 144 }
 145
 146 kthread_t *
 147 zk_thread_create(caddr_t stk, size_t stksize, thread_func_t func, void *arg,
 148     size_t len, proc_t *pp, int state, pri_t pri, int detachstate)
 149 {
 150         kthread_t *kt;
 151         pthread_attr_t attr;
 152         char *stkstr;
 153
 154         ASSERT0(state & ~TS_RUN);
 155
 156         kt = umem_zalloc(sizeof (kthread_t), UMEM_NOFAIL);
 157         kt->t_func = func;
 158         kt->t_arg = arg;
 159         kt->t_pri = pri;
 160
 161         VERIFY0(pthread_attr_init(&attr));
 162         VERIFY0(pthread_attr_setdetachstate(&attr, detachstate));
 163
 164         /*
 165          * We allow the default stack size in user space to be specified by
 166          * setting the ZFS_STACK_SIZE environment variable.  This allows us
 167          * the convenience of observing and debugging stack overruns in
 168          * user space.  Explicitly specified stack sizes will be honored.
 169          * The usage of ZFS_STACK_SIZE is discussed further in the
 170          * ENVIRONMENT VARIABLES sections of the ztest(1) man page.
 171          */
 172         if (stksize == 0) {
 173                 stkstr = getenv("ZFS_STACK_SIZE");
 174
 175                 if (stkstr == NULL)
 176                         stksize = TS_STACK_MAX;
 177                 else
 178                         stksize = MAX(atoi(stkstr), TS_STACK_MIN);
 179         }
 180
 181         VERIFY3S(stksize, >, 0);
 182         stksize = P2ROUNDUP(MAX(stksize, TS_STACK_MIN), PAGESIZE);
 183         VERIFY0(pthread_attr_setstacksize(&attr, stksize));
 184         VERIFY0(pthread_attr_setguardsize(&attr, PAGESIZE));
 185
 186         VERIFY0(pthread_create(&kt->t_tid, &attr, &zk_thread_helper, kt));
 187         VERIFY0(pthread_attr_destroy(&attr));
 188
 189         return (kt);
 190 }
 191
 192 void
 193 zk_thread_exit(void)
 194 {
 195         kthread_t *kt = curthread;
 196
 197         ASSERT(pthread_equal(kt->t_tid, pthread_self()));
 198
 199         umem_free(kt, sizeof (kthread_t));
 200
 201         pthread_mutex_lock(&kthread_lock);
 202         kthread_nr--;
 203         pthread_mutex_unlock(&kthread_lock);
 204
 205         pthread_cond_broadcast(&kthread_cond);
 206         pthread_exit((void *)TS_MAGIC);
 207 }
 208
 209 void
 210 zk_thread_join(kt_did_t tid)
 211 {
 212         void *ret;
 213
 214         pthread_join((pthread_t)tid, &ret);
 215         VERIFY3P(ret, ==, (void *)TS_MAGIC);
 216 }
 217
 218 /*
 219  * =========================================================================
 220  * kstats
 221  * =========================================================================
 222  */
 223 /*ARGSUSED*/
 224 kstat_t *
 225 kstat_create(const char *module, int instance, const char *name,
 226     const char *class, uchar_t type, ulong_t ndata, uchar_t ks_flag)
 227 {
 228         return (NULL);
 229 }
 230
 231 /*ARGSUSED*/
 232 void
 233 kstat_install(kstat_t *ksp)
 234 {}
 235
 236 /*ARGSUSED*/
 237 void
 238 kstat_delete(kstat_t *ksp)
 239 {}
 240
 241 /*ARGSUSED*/
 242 void
 243 kstat_waitq_enter(kstat_io_t *kiop)
 244 {}
 245
 246 /*ARGSUSED*/
 247 void
 248 kstat_waitq_exit(kstat_io_t *kiop)
 249 {}
 250
 251 /*ARGSUSED*/
 252 void
 253 kstat_runq_enter(kstat_io_t *kiop)
 254 {}
 255
 256 /*ARGSUSED*/
 257 void
 258 kstat_runq_exit(kstat_io_t *kiop)
 259 {}
 260
 261 /*ARGSUSED*/
 262 void
 263 kstat_waitq_to_runq(kstat_io_t *kiop)
 264 {}
 265
 266 /*ARGSUSED*/
 267 void
 268 kstat_runq_back_to_waitq(kstat_io_t *kiop)
 269 {}
 270
 271 void
 272 kstat_set_raw_ops(kstat_t *ksp,
 273     int (*headers)(char *buf, size_t size),
 274     int (*data)(char *buf, size_t size, void *data),
 275     void *(*addr)(kstat_t *ksp, loff_t index))
 276 {}
 277
 278 /*
 279  * =========================================================================
 280  * mutexes
 281  * =========================================================================
 282  */
 283
 284 void
 285 mutex_init(kmutex_t *mp, char *name, int type, void *cookie)
 286 {
 287         ASSERT3S(type, ==, MUTEX_DEFAULT);
 288         ASSERT3P(cookie, ==, NULL);
 289         mp->m_owner = MTX_INIT;
 290         mp->m_magic = MTX_MAGIC;
 291         VERIFY3S(pthread_mutex_init(&mp->m_lock, NULL), ==, 0);
 292 }
 293
 294 void
 295 mutex_destroy(kmutex_t *mp)
 296 {
 297         ASSERT3U(mp->m_magic, ==, MTX_MAGIC);
 298         ASSERT3P(mp->m_owner, ==, MTX_INIT);
 299         ASSERT0(pthread_mutex_destroy(&(mp)->m_lock));
 300         mp->m_owner = MTX_DEST;
 301         mp->m_magic = 0;
 302 }
 303
 304 void
 305 mutex_enter(kmutex_t *mp)
 306 {
 307         ASSERT3U(mp->m_magic, ==, MTX_MAGIC);
 308         ASSERT3P(mp->m_owner, !=, MTX_DEST);
 309         ASSERT3P(mp->m_owner, !=, curthread);
 310         VERIFY3S(pthread_mutex_lock(&mp->m_lock), ==, 0);
 311         ASSERT3P(mp->m_owner, ==, MTX_INIT);
 312         mp->m_owner = curthread;
 313 }
 314
 315 int
 316 mutex_tryenter(kmutex_t *mp)
 317 {
 318         ASSERT3U(mp->m_magic, ==, MTX_MAGIC);
 319         ASSERT3P(mp->m_owner, !=, MTX_DEST);
 320         if (0 == pthread_mutex_trylock(&mp->m_lock)) {
 321                 ASSERT3P(mp->m_owner, ==, MTX_INIT);
 322                 mp->m_owner = curthread;
 323                 return (1);
 324         } else {
 325                 return (0);
 326         }
 327 }
 328
 329 void
 330 mutex_exit(kmutex_t *mp)
 331 {
 332         ASSERT3U(mp->m_magic, ==, MTX_MAGIC);
 333         ASSERT3P(mutex_owner(mp), ==, curthread);
 334         mp->m_owner = MTX_INIT;
 335         VERIFY3S(pthread_mutex_unlock(&mp->m_lock), ==, 0);
 336 }
 337
 338 void *
 339 mutex_owner(kmutex_t *mp)
 340 {
 341         ASSERT3U(mp->m_magic, ==, MTX_MAGIC);
 342         return (mp->m_owner);
 343 }
 344
 345 int
 346 mutex_held(kmutex_t *mp)
 347 {
 348         return (mp->m_owner == curthread);
 349 }
 350
 351 /*
 352  * =========================================================================
 353  * rwlocks
 354  * =========================================================================
 355  */
 356
 357 void
 358 rw_init(krwlock_t *rwlp, char *name, int type, void *arg)
 359 {
 360         ASSERT3S(type, ==, RW_DEFAULT);
 361         ASSERT3P(arg, ==, NULL);
 362         VERIFY3S(pthread_rwlock_init(&rwlp->rw_lock, NULL), ==, 0);
 363         rwlp->rw_owner = RW_INIT;
 364         rwlp->rw_wr_owner = RW_INIT;
 365         rwlp->rw_readers = 0;
 366         rwlp->rw_magic = RW_MAGIC;
 367 }
 368
 369 void
 370 rw_destroy(krwlock_t *rwlp)
 371 {
 372         ASSERT3U(rwlp->rw_magic, ==, RW_MAGIC);
 373         ASSERT(rwlp->rw_readers == 0 && rwlp->rw_wr_owner == RW_INIT);
 374         VERIFY3S(pthread_rwlock_destroy(&rwlp->rw_lock), ==, 0);
 375         rwlp->rw_magic = 0;
 376 }
 377
 378 void
 379 rw_enter(krwlock_t *rwlp, krw_t rw)
 380 {
 381         ASSERT3U(rwlp->rw_magic, ==, RW_MAGIC);
 382         ASSERT3P(rwlp->rw_owner, !=, curthread);
 383         ASSERT3P(rwlp->rw_wr_owner, !=, curthread);
 384
 385         if (rw == RW_READER) {
 386                 VERIFY3S(pthread_rwlock_rdlock(&rwlp->rw_lock), ==, 0);
 387                 ASSERT3P(rwlp->rw_wr_owner, ==, RW_INIT);
 388
 389                 atomic_inc_uint(&rwlp->rw_readers);
 390         } else {
 391                 VERIFY3S(pthread_rwlock_wrlock(&rwlp->rw_lock), ==, 0);
 392                 ASSERT3P(rwlp->rw_wr_owner, ==, RW_INIT);
 393                 ASSERT3U(rwlp->rw_readers, ==, 0);
 394
 395                 rwlp->rw_wr_owner = curthread;
 396         }
 397
 398         rwlp->rw_owner = curthread;
 399 }
 400
 401 void
 402 rw_exit(krwlock_t *rwlp)
 403 {
 404         ASSERT3U(rwlp->rw_magic, ==, RW_MAGIC);
 405         ASSERT(RW_LOCK_HELD(rwlp));
 406
 407         if (RW_READ_HELD(rwlp))
 408                 atomic_dec_uint(&rwlp->rw_readers);
 409         else
 410                 rwlp->rw_wr_owner = RW_INIT;
 411
 412         rwlp->rw_owner = RW_INIT;
 413         VERIFY3S(pthread_rwlock_unlock(&rwlp->rw_lock), ==, 0);
 414 }
 415
 416 int
 417 rw_tryenter(krwlock_t *rwlp, krw_t rw)
 418 {
 419         int rv;
 420
 421         ASSERT3U(rwlp->rw_magic, ==, RW_MAGIC);
 422
 423         if (rw == RW_READER)
 424                 rv = pthread_rwlock_tryrdlock(&rwlp->rw_lock);
 425         else
 426                 rv = pthread_rwlock_trywrlock(&rwlp->rw_lock);
 427
 428         if (rv == 0) {
 429                 ASSERT3P(rwlp->rw_wr_owner, ==, RW_INIT);
 430
 431                 if (rw == RW_READER)
 432                         atomic_inc_uint(&rwlp->rw_readers);
 433                 else {
 434                         ASSERT3U(rwlp->rw_readers, ==, 0);
 435                         rwlp->rw_wr_owner = curthread;
 436                 }
 437
 438                 rwlp->rw_owner = curthread;
 439                 return (1);
 440         }
 441
 442         VERIFY3S(rv, ==, EBUSY);
 443
 444         return (0);
 445 }
 446
 447 int
 448 rw_tryupgrade(krwlock_t *rwlp)
 449 {
 450         ASSERT3U(rwlp->rw_magic, ==, RW_MAGIC);
 451
 452         return (0);
 453 }
 454
 455 /*
 456  * =========================================================================
 457  * condition variables
 458  * =========================================================================
 459  */
 460
 461 void
 462 cv_init(kcondvar_t *cv, char *name, int type, void *arg)
 463 {
 464         ASSERT3S(type, ==, CV_DEFAULT);
 465         cv->cv_magic = CV_MAGIC;
 466         VERIFY3S(pthread_cond_init(&cv->cv, NULL), ==, 0);
 467 }
 468
 469 void
 470 cv_destroy(kcondvar_t *cv)
 471 {
 472         ASSERT3U(cv->cv_magic, ==, CV_MAGIC);
 473         VERIFY3S(pthread_cond_destroy(&cv->cv), ==, 0);
 474         cv->cv_magic = 0;
 475 }
 476
 477 void
 478 cv_wait(kcondvar_t *cv, kmutex_t *mp)
 479 {
 480         ASSERT3U(cv->cv_magic, ==, CV_MAGIC);
 481         ASSERT3P(mutex_owner(mp), ==, curthread);
 482         mp->m_owner = MTX_INIT;
 483         int ret = pthread_cond_wait(&cv->cv, &mp->m_lock);
 484         if (ret != 0)
 485                 VERIFY3S(ret, ==, EINTR);
 486         mp->m_owner = curthread;
 487 }
 488
 489 clock_t
 490 cv_timedwait(kcondvar_t *cv, kmutex_t *mp, clock_t abstime)
 491 {
 492         int error;
 493         struct timeval tv;
 494         timestruc_t ts;
 495         clock_t delta;
 496
 497         ASSERT3U(cv->cv_magic, ==, CV_MAGIC);
 498
 499 top:
 500         delta = abstime - ddi_get_lbolt();
 501         if (delta <= 0)
 502                 return (-1);
 503
 504         VERIFY(gettimeofday(&tv, NULL) == 0);
 505
 506         ts.tv_sec = tv.tv_sec + delta / hz;
 507         ts.tv_nsec = tv.tv_usec * 1000 + (delta % hz) * (NANOSEC / hz);
 508         if (ts.tv_nsec >= NANOSEC) {
 509                 ts.tv_sec++;
 510                 ts.tv_nsec -= NANOSEC;
 511         }
 512
 513         ASSERT3P(mutex_owner(mp), ==, curthread);
 514         mp->m_owner = MTX_INIT;
 515         error = pthread_cond_timedwait(&cv->cv, &mp->m_lock, &ts);
 516         mp->m_owner = curthread;
 517
 518         if (error == ETIMEDOUT)
 519                 return (-1);
 520
 521         if (error == EINTR)
 522                 goto top;
 523
 524         VERIFY3S(error, ==, 0);
 525
 526         return (1);
 527 }
 528
 529 /*ARGSUSED*/
 530 clock_t
 531 cv_timedwait_hires(kcondvar_t *cv, kmutex_t *mp, hrtime_t tim, hrtime_t res,
 532     int flag)
 533 {
 534         int error;
 535         timestruc_t ts;
 536         hrtime_t delta;
 537
 538         ASSERT(flag == 0);
 539
 540 top:
 541         delta = tim - gethrtime();
 542         if (delta <= 0)
 543                 return (-1);
 544
 545         ts.tv_sec = delta / NANOSEC;
 546         ts.tv_nsec = delta % NANOSEC;
 547
 548         ASSERT(mutex_owner(mp) == curthread);
 549         mp->m_owner = NULL;
 550         error = pthread_cond_timedwait(&cv->cv, &mp->m_lock, &ts);
 551         mp->m_owner = curthread;
 552
 553         if (error == ETIME)
 554                 return (-1);
 555
 556         if (error == EINTR)
 557                 goto top;
 558
 559         ASSERT(error == 0);
 560
 561         return (1);
 562 }
 563
 564 void
 565 cv_signal(kcondvar_t *cv)
 566 {
 567         ASSERT3U(cv->cv_magic, ==, CV_MAGIC);
 568         VERIFY3S(pthread_cond_signal(&cv->cv), ==, 0);
 569 }
 570
 571 void
 572 cv_broadcast(kcondvar_t *cv)
 573 {
 574         ASSERT3U(cv->cv_magic, ==, CV_MAGIC);
 575         VERIFY3S(pthread_cond_broadcast(&cv->cv), ==, 0);
 576 }
 577
 578 /*
 579  * =========================================================================
 580  * vnode operations
 581  * =========================================================================
 582  */
 583 /*
 584  * Note: for the xxxat() versions of these functions, we assume that the
 585  * starting vp is always rootdir (which is true for spa_directory.c, the only
 586  * ZFS consumer of these interfaces).  We assert this is true, and then emulate
 587  * them by adding '/' in front of the path.
 588  */
 589
 590 /*ARGSUSED*/
 591 int
 592 vn_open(char *path, int x1, int flags, int mode, vnode_t **vpp, int x2, int x3)
 593 {
 594         int fd;
 595         int dump_fd;
 596         vnode_t *vp;
 597         int old_umask = 0;
 598         char *realpath;
 599         struct stat64 st;
 600         int err;
 601
 602         realpath = umem_alloc(MAXPATHLEN, UMEM_NOFAIL);
 603
 604         /*
 605          * If we're accessing a real disk from userland, we need to use
 606          * the character interface to avoid caching.  This is particularly
 607          * important if we're trying to look at a real in-kernel storage
 608          * pool from userland, e.g. via zdb, because otherwise we won't
 609          * see the changes occurring under the segmap cache.
 610          * On the other hand, the stupid character device returns zero
 611          * for its size.  So -- gag -- we open the block device to get
 612          * its size, and remember it for subsequent VOP_GETATTR().
 613          */
 614 #if defined(__sun__) || defined(__sun)
 615         if (strncmp(path, "/dev/", 5) == 0) {
 616 #else
 617         if (0) {
 618 #endif
 619                 char *dsk;
 620                 fd = open64(path, O_RDONLY);
 621                 if (fd == -1) {
 622                         err = errno;
 623                         free(realpath);
 624                         return (err);
 625                 }
 626                 if (fstat64(fd, &st) == -1) {
 627                         err = errno;
 628                         close(fd);
 629                         free(realpath);
 630                         return (err);
 631                 }
 632                 close(fd);
 633                 (void) sprintf(realpath, "%s", path);
 634                 dsk = strstr(path, "/dsk/");
 635                 if (dsk != NULL)
 636                         (void) sprintf(realpath + (dsk - path) + 1, "r%s",
 637                             dsk + 1);
 638         } else {
 639                 (void) sprintf(realpath, "%s", path);
 640                 if (!(flags & FCREAT) && stat64(realpath, &st) == -1) {
 641                         err = errno;
 642                         free(realpath);
 643                         return (err);
 644                 }
 645         }
 646
 647         if (!(flags & FCREAT) && S_ISBLK(st.st_mode)) {
 648 #ifdef __linux__
 649                 flags |= O_DIRECT;
 650 #endif
 651                 /* We shouldn't be writing to block devices in userspace */
 652                 VERIFY(!(flags & FWRITE));
 653         }
 654
 655         if (flags & FCREAT)
 656                 old_umask = umask(0);
 657
 658         /*
 659          * The construct 'flags - FREAD' conveniently maps combinations of
 660          * FREAD and FWRITE to the corresponding O_RDONLY, O_WRONLY, and O_RDWR.
 661          */
 662         fd = open64(realpath, flags - FREAD, mode);
 663         err = errno;
 664
 665         if (flags & FCREAT)
 666                 (void) umask(old_umask);
 667
 668         if (vn_dumpdir != NULL) {
 669                 char *dumppath = umem_zalloc(MAXPATHLEN, UMEM_NOFAIL);
 670                 (void) snprintf(dumppath, MAXPATHLEN,
 671                     "%s/%s", vn_dumpdir, basename(realpath));
 672                 dump_fd = open64(dumppath, O_CREAT | O_WRONLY, 0666);
 673                 umem_free(dumppath, MAXPATHLEN);
 674                 if (dump_fd == -1) {
 675                         err = errno;
 676                         free(realpath);
 677                         close(fd);
 678                         return (err);
 679                 }
 680         } else {
 681                 dump_fd = -1;
 682         }
 683
 684         free(realpath);
 685
 686         if (fd == -1)
 687                 return (err);
 688
 689         if (fstat64_blk(fd, &st) == -1) {
 690                 err = errno;
 691                 close(fd);
 692                 return (err);
 693         }
 694
 695         (void) fcntl(fd, F_SETFD, FD_CLOEXEC);
 696
 697         *vpp = vp = umem_zalloc(sizeof (vnode_t), UMEM_NOFAIL);
 698
 699         vp->v_fd = fd;
 700         vp->v_size = st.st_size;
 701         vp->v_path = spa_strdup(path);
 702         vp->v_dump_fd = dump_fd;
 703
 704         return (0);
 705 }
 706
 707 /*ARGSUSED*/
 708 int
 709 vn_openat(char *path, int x1, int flags, int mode, vnode_t **vpp, int x2,
 710     int x3, vnode_t *startvp, int fd)
 711 {
 712         char *realpath = umem_alloc(strlen(path) + 2, UMEM_NOFAIL);
 713         int ret;
 714
 715         ASSERT(startvp == rootdir);
 716         (void) sprintf(realpath, "/%s", path);
 717
 718         /* fd ignored for now, need if want to simulate nbmand support */
 719         ret = vn_open(realpath, x1, flags, mode, vpp, x2, x3);
 720
 721         umem_free(realpath, strlen(path) + 2);
 722
 723         return (ret);
 724 }
 725
 726 /*ARGSUSED*/
 727 int
 728 vn_rdwr(int uio, vnode_t *vp, void *addr, ssize_t len, offset_t offset,
 729         int x1, int x2, rlim64_t x3, void *x4, ssize_t *residp)
 730 {
 731         ssize_t rc, done = 0, split;
 732
 733         if (uio == UIO_READ) {
 734                 rc = pread64(vp->v_fd, addr, len, offset);
 735                 if (vp->v_dump_fd != -1) {
 736                         int status =
 737                             pwrite64(vp->v_dump_fd, addr, rc, offset);
 738                         ASSERT(status != -1);
 739                 }
 740         } else {
 741                 /*
 742                  * To simulate partial disk writes, we split writes into two
 743                  * system calls so that the process can be killed in between.
 744                  */
 745                 int sectors = len >> SPA_MINBLOCKSHIFT;
 746                 split = (sectors > 0 ? rand() % sectors : 0) <<
 747                     SPA_MINBLOCKSHIFT;
 748                 rc = pwrite64(vp->v_fd, addr, split, offset);
 749                 if (rc != -1) {
 750                         done = rc;
 751                         rc = pwrite64(vp->v_fd, (char *)addr + split,
 752                             len - split, offset + split);
 753                 }
 754         }
 755
 756 #ifdef __linux__
 757         if (rc == -1 && errno == EINVAL) {
 758                 /*
 759                  * Under Linux, this most likely means an alignment issue
 760                  * (memory or disk) due to O_DIRECT, so we abort() in order to
 761                  * catch the offender.
 762                  */
 763                 abort();
 764         }
 765 #endif
 766         if (rc == -1)
 767                 return (errno);
 768
 769         done += rc;
 770
 771         if (residp)
 772                 *residp = len - done;
 773         else if (done != len)
 774                 return (EIO);
 775         return (0);
 776 }
 777
 778 void
 779 vn_close(vnode_t *vp)
 780 {
 781         close(vp->v_fd);
 782         if (vp->v_dump_fd != -1)
 783                 close(vp->v_dump_fd);
 784         spa_strfree(vp->v_path);
 785         umem_free(vp, sizeof (vnode_t));
 786 }
 787
 788 /*
 789  * At a minimum we need to update the size since vdev_reopen()
 790  * will no longer call vn_openat().
 791  */
 792 int
 793 fop_getattr(vnode_t *vp, vattr_t *vap)
 794 {
 795         struct stat64 st;
 796         int err;
 797
 798         if (fstat64_blk(vp->v_fd, &st) == -1) {
 799                 err = errno;
 800                 close(vp->v_fd);
 801                 return (err);
 802         }
 803
 804         vap->va_size = st.st_size;
 805         return (0);
 806 }
 807
 808 /*
 809  * =========================================================================
 810  * Figure out which debugging statements to print
 811  * =========================================================================
 812  */
 813
 814 static char *dprintf_string;
 815 static int dprintf_print_all;
 816
 817 int
 818 dprintf_find_string(const char *string)
 819 {
 820         char *tmp_str = dprintf_string;
 821         int len = strlen(string);
 822
 823         /*
 824          * Find out if this is a string we want to print.
 825          * String format: file1.c,function_name1,file2.c,file3.c
 826          */
 827
 828         while (tmp_str != NULL) {
 829                 if (strncmp(tmp_str, string, len) == 0 &&
 830                     (tmp_str[len] == ',' || tmp_str[len] == '\0'))
 831                         return (1);
 832                 tmp_str = strchr(tmp_str, ',');
 833                 if (tmp_str != NULL)
 834                         tmp_str++; /* Get rid of , */
 835         }
 836         return (0);
 837 }
 838
 839 void
 840 dprintf_setup(int *argc, char **argv)
 841 {
 842         int i, j;
 843
 844         /*
 845          * Debugging can be specified two ways: by setting the
 846          * environment variable ZFS_DEBUG, or by including a
 847          * "debug=..."  argument on the command line.  The command
 848          * line setting overrides the environment variable.
 849          */
 850
 851         for (i = 1; i < *argc; i++) {
 852                 int len = strlen("debug=");
 853                 /* First look for a command line argument */
 854                 if (strncmp("debug=", argv[i], len) == 0) {
 855                         dprintf_string = argv[i] + len;
 856                         /* Remove from args */
 857                         for (j = i; j < *argc; j++)
 858                                 argv[j] = argv[j+1];
 859                         argv[j] = NULL;
 860                         (*argc)--;
 861                 }
 862         }
 863
 864         if (dprintf_string == NULL) {
 865                 /* Look for ZFS_DEBUG environment variable */
 866                 dprintf_string = getenv("ZFS_DEBUG");
 867         }
 868
 869         /*
 870          * Are we just turning on all debugging?
 871          */
 872         if (dprintf_find_string("on"))
 873                 dprintf_print_all = 1;
 874
 875         if (dprintf_string != NULL)
 876                 zfs_flags |= ZFS_DEBUG_DPRINTF;
 877 }
 878
 879 /*
 880  * =========================================================================
 881  * debug printfs
 882  * =========================================================================
 883  */
 884 void
 885 __dprintf(const char *file, const char *func, int line, const char *fmt, ...)
 886 {
 887         const char *newfile;
 888         va_list adx;
 889
 890         /*
 891          * Get rid of annoying "../common/" prefix to filename.
 892          */
 893         newfile = strrchr(file, '/');
 894         if (newfile != NULL) {
 895                 newfile = newfile + 1; /* Get rid of leading / */
 896         } else {
 897                 newfile = file;
 898         }
 899
 900         if (dprintf_print_all ||
 901             dprintf_find_string(newfile) ||
 902             dprintf_find_string(func)) {
 903                 /* Print out just the function name if requested */
 904                 flockfile(stdout);
 905                 if (dprintf_find_string("pid"))
 906                         (void) printf("%d ", getpid());
 907                 if (dprintf_find_string("tid"))
 908                         (void) printf("%u ", (uint_t) pthread_self());
 909                 if (dprintf_find_string("cpu"))
 910                         (void) printf("%u ", getcpuid());
 911                 if (dprintf_find_string("time"))
 912                         (void) printf("%llu ", gethrtime());
 913                 if (dprintf_find_string("long"))
 914                         (void) printf("%s, line %d: ", newfile, line);
 915                 (void) printf("%s: ", func);
 916                 va_start(adx, fmt);
 917                 (void) vprintf(fmt, adx);
 918                 va_end(adx);
 919                 funlockfile(stdout);
 920         }
 921 }
 922
 923 /*
 924  * =========================================================================
 925  * cmn_err() and panic()
 926  * =========================================================================
 927  */
 928 static char ce_prefix[CE_IGNORE][10] = { "", "NOTICE: ", "WARNING: ", "" };
 929 static char ce_suffix[CE_IGNORE][2] = { "", "\n", "\n", "" };
 930
 931 void
 932 vpanic(const char *fmt, va_list adx)
 933 {
 934         (void) fprintf(stderr, "error: ");
 935         (void) vfprintf(stderr, fmt, adx);
 936         (void) fprintf(stderr, "\n");
 937
 938         abort();        /* think of it as a "user-level crash dump" */
 939 }
 940
 941 void
 942 panic(const char *fmt, ...)
 943 {
 944         va_list adx;
 945
 946         va_start(adx, fmt);
 947         vpanic(fmt, adx);
 948         va_end(adx);
 949 }
 950
 951 void
 952 vcmn_err(int ce, const char *fmt, va_list adx)
 953 {
 954         if (ce == CE_PANIC)
 955                 vpanic(fmt, adx);
 956         if (ce != CE_NOTE) {    /* suppress noise in userland stress testing */
 957                 (void) fprintf(stderr, "%s", ce_prefix[ce]);
 958                 (void) vfprintf(stderr, fmt, adx);
 959                 (void) fprintf(stderr, "%s", ce_suffix[ce]);
 960         }
 961 }
 962
 963 /*PRINTFLIKE2*/
 964 void
 965 cmn_err(int ce, const char *fmt, ...)
 966 {
 967         va_list adx;
 968
 969         va_start(adx, fmt);
 970         vcmn_err(ce, fmt, adx);
 971         va_end(adx);
 972 }
 973
 974 /*
 975  * =========================================================================
 976  * kobj interfaces
 977  * =========================================================================
 978  */
 979 struct _buf *
 980 kobj_open_file(char *name)
 981 {
 982         struct _buf *file;
 983         vnode_t *vp;
 984
 985         /* set vp as the _fd field of the file */
 986         if (vn_openat(name, UIO_SYSSPACE, FREAD, 0, &vp, 0, 0, rootdir,
 987             -1) != 0)
 988                 return ((void *)-1UL);
 989
 990         file = umem_zalloc(sizeof (struct _buf), UMEM_NOFAIL);
 991         file->_fd = (intptr_t)vp;
 992         return (file);
 993 }
 994
 995 int
 996 kobj_read_file(struct _buf *file, char *buf, unsigned size, unsigned off)
 997 {
 998         ssize_t resid;
 999
1000         vn_rdwr(UIO_READ, (vnode_t *)file->_fd, buf, size, (offset_t)off,
1001             UIO_SYSSPACE, 0, 0, 0, &resid);
1002
1003         return (size - resid);
1004 }
1005
1006 void
1007 kobj_close_file(struct _buf *file)
1008 {
1009         vn_close((vnode_t *)file->_fd);
1010         umem_free(file, sizeof (struct _buf));
1011 }
1012
1013 int
1014 kobj_get_filesize(struct _buf *file, uint64_t *size)
1015 {
1016         struct stat64 st;
1017         vnode_t *vp = (vnode_t *)file->_fd;
1018
1019         if (fstat64(vp->v_fd, &st) == -1) {
1020                 vn_close(vp);
1021                 return (errno);
1022         }
1023         *size = st.st_size;
1024         return (0);
1025 }
1026
1027 /*
1028  * =========================================================================
1029  * misc routines
1030  * =========================================================================
1031  */
1032
1033 void
1034 delay(clock_t ticks)
1035 {
1036         poll(0, 0, ticks * (1000 / hz));
1037 }
1038
1039 /*
1040  * Find highest one bit set.
1041  *      Returns bit number + 1 of highest bit that is set, otherwise returns 0.
1042  * High order bit is 31 (or 63 in _LP64 kernel).
1043  */
1044 int
1045 highbit64(uint64_t i)
1046 {
1047         register int h = 1;
1048
1049         if (i == 0)
1050                 return (0);
1051         if (i & 0xffffffff00000000ULL) {
1052                 h += 32; i >>= 32;
1053         }
1054         if (i & 0xffff0000) {
1055                 h += 16; i >>= 16;
1056         }
1057         if (i & 0xff00) {
1058                 h += 8; i >>= 8;
1059         }
1060         if (i & 0xf0) {
1061                 h += 4; i >>= 4;
1062         }
1063         if (i & 0xc) {
1064                 h += 2; i >>= 2;
1065         }
1066         if (i & 0x2) {
1067                 h += 1;
1068         }
1069         return (h);
1070 }
1071
1072 static int random_fd = -1, urandom_fd = -1;
1073
1074 static int
1075 random_get_bytes_common(uint8_t *ptr, size_t len, int fd)
1076 {
1077         size_t resid = len;
1078         ssize_t bytes;
1079
1080         ASSERT(fd != -1);
1081
1082         while (resid != 0) {
1083                 bytes = read(fd, ptr, resid);
1084                 ASSERT3S(bytes, >=, 0);
1085                 ptr += bytes;
1086                 resid -= bytes;
1087         }
1088
1089         return (0);
1090 }
1091
1092 int
1093 random_get_bytes(uint8_t *ptr, size_t len)
1094 {
1095         return (random_get_bytes_common(ptr, len, random_fd));
1096 }
1097
1098 int
1099 random_get_pseudo_bytes(uint8_t *ptr, size_t len)
1100 {
1101         return (random_get_bytes_common(ptr, len, urandom_fd));
1102 }
1103
1104 int
1105 ddi_strtoul(const char *hw_serial, char **nptr, int base, unsigned long *result)
1106 {
1107         char *end;
1108
1109         *result = strtoul(hw_serial, &end, base);
1110         if (*result == 0)
1111                 return (errno);
1112         return (0);
1113 }
1114
1115 int
1116 ddi_strtoull(const char *str, char **nptr, int base, u_longlong_t *result)
1117 {
1118         char *end;
1119
1120         *result = strtoull(str, &end, base);
1121         if (*result == 0)
1122                 return (errno);
1123         return (0);
1124 }
1125
1126 utsname_t *
1127 utsname(void)
1128 {
1129         return (&hw_utsname);
1130 }
1131
1132 /*
1133  * =========================================================================
1134  * kernel emulation setup & teardown
1135  * =========================================================================
1136  */
1137 static int
1138 umem_out_of_memory(void)
1139 {
1140         char errmsg[] = "out of memory -- generating core dump\n";
1141
1142         (void) fprintf(stderr, "%s", errmsg);
1143         abort();
1144         return (0);
1145 }
1146
1147 static unsigned long
1148 get_spl_hostid(void)
1149 {
1150         FILE *f;
1151         unsigned long hostid;
1152
1153         f = fopen("/sys/module/spl/parameters/spl_hostid", "r");
1154         if (!f)
1155                 return (0);
1156         if (fscanf(f, "%lu", &hostid) != 1)
1157                 hostid = 0;
1158         fclose(f);
1159         return (hostid & 0xffffffff);
1160 }
1161
1162 unsigned long
1163 get_system_hostid(void)
1164 {
1165         unsigned long system_hostid = get_spl_hostid();
1166         if (system_hostid == 0)
1167                 system_hostid = gethostid() & 0xffffffff;
1168         return (system_hostid);
1169 }
1170
1171 void
1172 kernel_init(int mode)
1173 {
1174         extern uint_t rrw_tsd_key;
1175
1176         umem_nofail_callback(umem_out_of_memory);
1177
1178         physmem = sysconf(_SC_PHYS_PAGES);
1179
1180         dprintf("physmem = %llu pages (%.2f GB)\n", physmem,
1181             (double)physmem * sysconf(_SC_PAGE_SIZE) / (1ULL << 30));
1182
1183         (void) snprintf(hw_serial, sizeof (hw_serial), "%ld",
1184             (mode & FWRITE) ? get_system_hostid() : 0);
1185
1186         VERIFY((random_fd = open("/dev/random", O_RDONLY)) != -1);
1187         VERIFY((urandom_fd = open("/dev/urandom", O_RDONLY)) != -1);
1188         VERIFY0(uname(&hw_utsname));
1189
1190         thread_init();
1191         system_taskq_init();
1192
1193         spa_init(mode);
1194
1195         tsd_create(&rrw_tsd_key, rrw_tsd_destroy);
1196 }
1197
1198 void
1199 kernel_fini(void)
1200 {
1201         spa_fini();
1202
1203         system_taskq_fini();
1204         thread_fini();
1205
1206         close(random_fd);
1207         close(urandom_fd);
1208
1209         random_fd = -1;
1210         urandom_fd = -1;
1211 }
1212
1213 uid_t
1214 crgetuid(cred_t *cr)
1215 {
1216         return (0);
1217 }
1218
1219 uid_t
1220 crgetruid(cred_t *cr)
1221 {
1222         return (0);
1223 }
1224
1225 gid_t
1226 crgetgid(cred_t *cr)
1227 {
1228         return (0);
1229 }
1230
1231 int
1232 crgetngroups(cred_t *cr)
1233 {
1234         return (0);
1235 }
1236
1237 gid_t *
1238 crgetgroups(cred_t *cr)
1239 {
1240         return (NULL);
1241 }
1242
1243 int
1244 zfs_secpolicy_snapshot_perms(const char *name, cred_t *cr)
1245 {
1246         return (0);
1247 }
1248
1249 int
1250 zfs_secpolicy_rename_perms(const char *from, const char *to, cred_t *cr)
1251 {
1252         return (0);
1253 }
1254
1255 int
1256 zfs_secpolicy_destroy_perms(const char *name, cred_t *cr)
1257 {
1258         return (0);
1259 }
1260
1261 ksiddomain_t *
1262 ksid_lookupdomain(const char *dom)
1263 {
1264         ksiddomain_t *kd;
1265
1266         kd = umem_zalloc(sizeof (ksiddomain_t), UMEM_NOFAIL);
1267         kd->kd_name = spa_strdup(dom);
1268         return (kd);
1269 }
1270
1271 void
1272 ksiddomain_rele(ksiddomain_t *ksid)
1273 {
1274         spa_strfree(ksid->kd_name);
1275         umem_free(ksid, sizeof (ksiddomain_t));
1276 }
1277
1278 char *
1279 kmem_vasprintf(const char *fmt, va_list adx)
1280 {
1281         char *buf = NULL;
1282         va_list adx_copy;
1283
1284         va_copy(adx_copy, adx);
1285         VERIFY(vasprintf(&buf, fmt, adx_copy) != -1);
1286         va_end(adx_copy);
1287
1288         return (buf);
1289 }
1290
1291 char *
1292 kmem_asprintf(const char *fmt, ...)
1293 {
1294         char *buf = NULL;
1295         va_list adx;
1296
1297         va_start(adx, fmt);
1298         VERIFY(vasprintf(&buf, fmt, adx) != -1);
1299         va_end(adx);
1300
1301         return (buf);
1302 }
1303
1304 /* ARGSUSED */
1305 int
1306 zfs_onexit_fd_hold(int fd, minor_t *minorp)
1307 {
1308         *minorp = 0;
1309         return (0);
1310 }
1311
1312 /* ARGSUSED */
1313 void
1314 zfs_onexit_fd_rele(int fd)
1315 {
1316 }
1317
1318 /* ARGSUSED */
1319 int
1320 zfs_onexit_add_cb(minor_t minor, void (*func)(void *), void *data,
1321     uint64_t *action_handle)
1322 {
1323         return (0);
1324 }
1325
1326 /* ARGSUSED */
1327 int
1328 zfs_onexit_del_cb(minor_t minor, uint64_t action_handle, boolean_t fire)
1329 {
1330         return (0);
1331 }
1332
1333 /* ARGSUSED */
1334 int
1335 zfs_onexit_cb_data(minor_t minor, uint64_t action_handle, void **data)
1336 {
1337         return (0);
1338 }
1339
1340 fstrans_cookie_t
1341 spl_fstrans_mark(void)
1342 {
1343         return ((fstrans_cookie_t) 0);
1344 }
1345
1346 void
1347 spl_fstrans_unmark(fstrans_cookie_t cookie)
1348 {
1349 }
1350
1351 int
1352 spl_fstrans_check(void)
1353 {
1354         return (0);
1355 }