]> git.proxmox.com Git - mirror_zfs.git/blame - module/zfs/dmu_send.c
projects / mirror_zfs.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Add missing -V option to zdb
[mirror_zfs.git] / module / zfs / dmu_send.c
CommitLineData
34dc7c2f
BB		 1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21/*
428870ff 22 * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
241b5415 23 * Copyright (c) 2011, 2015 by Delphix. All rights reserved.
8d35c149 24 * Copyright 2011 Nexenta Systems, Inc. All rights reserved.
788eb90c 25 * Copyright (c) 2014, Joyent, Inc. All rights reserved.
5dbd68a3 26 * Copyright (c) 2011, 2014 by Delphix. All rights reserved.
8d35c149 27 */
34dc7c2f 28
34dc7c2f
BB		 29#include <sys/dmu.h>
30#include <sys/dmu_impl.h>
31#include <sys/dmu_tx.h>
32#include <sys/dbuf.h>
33#include <sys/dnode.h>
34#include <sys/zfs_context.h>
35#include <sys/dmu_objset.h>
36#include <sys/dmu_traverse.h>
37#include <sys/dsl_dataset.h>
38#include <sys/dsl_dir.h>
428870ff 39#include <sys/dsl_prop.h>
34dc7c2f
BB		 40#include <sys/dsl_pool.h>
41#include <sys/dsl_synctask.h>
044baf00 42#include <sys/spa_impl.h>
34dc7c2f
BB		 43#include <sys/zfs_ioctl.h>
44#include <sys/zap.h>
45#include <sys/zio_checksum.h>
428870ff
BB		 46#include <sys/zfs_znode.h>
47#include <zfs_fletcher.h>
48#include <sys/avl.h>
49#include <sys/ddt.h>
572e2857 50#include <sys/zfs_onexit.h>
13fe0198
MA		 51#include <sys/dmu_send.h>
52#include <sys/dsl_destroy.h>
9b67f605 53#include <sys/blkptr.h>
da536844 54#include <sys/dsl_bookmark.h>
9b67f605 55#include <sys/zfeature.h>
34dc7c2f 56
330d06f9
MA		 57/* Set this tunable to TRUE to replace corrupt data with 0x2f5baddb10c */
58int zfs_send_corrupt_data = B_FALSE;
59
34dc7c2f 60static char *dmu_recv_tag = "dmu_recv_tag";
13fe0198 61static const char *recv_clone_name = "%recv";
34dc7c2f 62
044baf00
BB		 63typedef struct dump_bytes_io {
64 dmu_sendarg_t *dbi_dsp;
65 void *dbi_buf;
66 int dbi_len;
67} dump_bytes_io_t;
68
69static void
b58986ee 70dump_bytes_cb(void *arg)
34dc7c2f 71{
044baf00
BB		 72 dump_bytes_io_t *dbi = (dump_bytes_io_t *)arg;
73 dmu_sendarg_t *dsp = dbi->dbi_dsp;
37abac6d 74 dsl_dataset_t *ds = dsp->dsa_os->os_dsl_dataset;
34dc7c2f 75 ssize_t resid; /* have to get resid to get detailed errno */
c99c9001 76 ASSERT0(dbi->dbi_len % 8);
34dc7c2f 77
37abac6d 78 dsp->dsa_err = vn_rdwr(UIO_WRITE, dsp->dsa_vp,
044baf00 79 (caddr_t)dbi->dbi_buf, dbi->dbi_len,
34dc7c2f 80 0, UIO_SYSSPACE, FAPPEND, RLIM64_INFINITY, CRED(), &resid);
37abac6d
BP		 81
82 mutex_enter(&ds->ds_sendstream_lock);
044baf00 83 *dsp->dsa_off += dbi->dbi_len;
37abac6d 84 mutex_exit(&ds->ds_sendstream_lock);
044baf00
BB		 85}
86
87static int
88dump_bytes(dmu_sendarg_t *dsp, void *buf, int len)
89{
90 dump_bytes_io_t dbi;
91
92 dbi.dbi_dsp = dsp;
93 dbi.dbi_buf = buf;
94 dbi.dbi_len = len;
95
b58986ee
BB		 96#if defined(HAVE_LARGE_STACKS)
97 dump_bytes_cb(&dbi);
98#else
044baf00
BB		 99 /*
100 * The vn_rdwr() call is performed in a taskq to ensure that there is
101 * always enough stack space to write safely to the target filesystem.
102 * The ZIO_TYPE_FREE threads are used because there can be a lot of
103 * them and they are used in vdev_file.c for a similar purpose.
104 */
105 spa_taskq_dispatch_sync(dmu_objset_spa(dsp->dsa_os), ZIO_TYPE_FREE,
b58986ee
BB		 106 ZIO_TASKQ_ISSUE, dump_bytes_cb, &dbi, TQ_SLEEP);
107#endif /* HAVE_LARGE_STACKS */
37abac6d
BP		 108
109 return (dsp->dsa_err);
34dc7c2f
BB		 110}
111
37f8a883
MA		 112/*
113 * For all record types except BEGIN, fill in the checksum (overlaid in
114 * drr_u.drr_checksum.drr_checksum). The checksum verifies everything
115 * up to the start of the checksum itself.
116 */
117static int
118dump_record(dmu_sendarg_t *dsp, void *payload, int payload_len)
119{
120 ASSERT3U(offsetof(dmu_replay_record_t, drr_u.drr_checksum.drr_checksum),
121 ==, sizeof (dmu_replay_record_t) - sizeof (zio_cksum_t));
122 fletcher_4_incremental_native(dsp->dsa_drr,
123 offsetof(dmu_replay_record_t, drr_u.drr_checksum.drr_checksum),
124 &dsp->dsa_zc);
125 if (dsp->dsa_drr->drr_type != DRR_BEGIN) {
126 ASSERT(ZIO_CHECKSUM_IS_ZERO(&dsp->dsa_drr->drr_u.
127 drr_checksum.drr_checksum));
128 dsp->dsa_drr->drr_u.drr_checksum.drr_checksum = dsp->dsa_zc;
129 }
130 fletcher_4_incremental_native(&dsp->dsa_drr->
131 drr_u.drr_checksum.drr_checksum,
132 sizeof (zio_cksum_t), &dsp->dsa_zc);
133 if (dump_bytes(dsp, dsp->dsa_drr, sizeof (dmu_replay_record_t)) != 0)
134 return (SET_ERROR(EINTR));
135 if (payload_len != 0) {
136 fletcher_4_incremental_native(payload, payload_len,
137 &dsp->dsa_zc);
138 if (dump_bytes(dsp, payload, payload_len) != 0)
139 return (SET_ERROR(EINTR));
140 }
141 return (0);
142}
143
34dc7c2f 144static int
37abac6d 145dump_free(dmu_sendarg_t *dsp, uint64_t object, uint64_t offset,
34dc7c2f
BB		 146 uint64_t length)
147{
37abac6d 148 struct drr_free *drrf = &(dsp->dsa_drr->drr_u.drr_free);
428870ff 149
ea97f8ce
MA		 150 /*
151 * When we receive a free record, dbuf_free_range() assumes
152 * that the receiving system doesn't have any dbufs in the range
153 * being freed. This is always true because there is a one-record
154 * constraint: we only send one WRITE record for any given
155 * object+offset. We know that the one-record constraint is
156 * true because we always send data in increasing order by
157 * object,offset.
158 *
159 * If the increasing-order constraint ever changes, we should find
160 * another way to assert that the one-record constraint is still
161 * satisfied.
162 */
163 ASSERT(object > dsp->dsa_last_data_object ||
164 (object == dsp->dsa_last_data_object &&
165 offset > dsp->dsa_last_data_offset));
166
167 /*
168 * If we are doing a non-incremental send, then there can't
169 * be any data in the dataset we're receiving into. Therefore
170 * a free record would simply be a no-op. Save space by not
171 * sending it to begin with.
172 */
173 if (!dsp->dsa_incremental)
174 return (0);
175
c578f007
SK		 176 if (length != -1ULL && offset + length < offset)
177 length = -1ULL;
178
428870ff
BB		 179 /*
180 * If there is a pending op, but it's not PENDING_FREE, push it out,
181 * since free block aggregation can only be done for blocks of the
182 * same type (i.e., DRR_FREE records can only be aggregated with
183 * other DRR_FREE records. DRR_FREEOBJECTS records can only be
184 * aggregated with other DRR_FREEOBJECTS records.
185 */
37abac6d
BP		 186 if (dsp->dsa_pending_op != PENDING_NONE &&
187 dsp->dsa_pending_op != PENDING_FREE) {
37f8a883 188 if (dump_record(dsp, NULL, 0) != 0)
2e528b49 189 return (SET_ERROR(EINTR));
37abac6d 190 dsp->dsa_pending_op = PENDING_NONE;
428870ff
BB		 191 }
192
37abac6d 193 if (dsp->dsa_pending_op == PENDING_FREE) {
428870ff
BB		 194 /*
195 * There should never be a PENDING_FREE if length is -1
196 * (because dump_dnode is the only place where this
197 * function is called with a -1, and only after flushing
198 * any pending record).
199 */
200 ASSERT(length != -1ULL);
201 /*
202 * Check to see whether this free block can be aggregated
203 * with pending one.
204 */
205 if (drrf->drr_object == object && drrf->drr_offset +
206 drrf->drr_length == offset) {
207 drrf->drr_length += length;
208 return (0);
209 } else {
210 /* not a continuation. Push out pending record */
37f8a883 211 if (dump_record(dsp, NULL, 0) != 0)
2e528b49 212 return (SET_ERROR(EINTR));
37abac6d 213 dsp->dsa_pending_op = PENDING_NONE;
428870ff
BB		 214 }
215 }
216 /* create a FREE record and make it pending */
37abac6d
BP		 217 bzero(dsp->dsa_drr, sizeof (dmu_replay_record_t));
218 dsp->dsa_drr->drr_type = DRR_FREE;
428870ff
BB		 219 drrf->drr_object = object;
220 drrf->drr_offset = offset;
221 drrf->drr_length = length;
37abac6d 222 drrf->drr_toguid = dsp->dsa_toguid;
428870ff 223 if (length == -1ULL) {
37f8a883 224 if (dump_record(dsp, NULL, 0) != 0)
2e528b49 225 return (SET_ERROR(EINTR));
428870ff 226 } else {
37abac6d 227 dsp->dsa_pending_op = PENDING_FREE;
428870ff 228 }
34dc7c2f 229
34dc7c2f
BB		 230 return (0);
231}
232
233static int
9b67f605 234dump_write(dmu_sendarg_t *dsp, dmu_object_type_t type,
428870ff 235 uint64_t object, uint64_t offset, int blksz, const blkptr_t *bp, void *data)
34dc7c2f 236{
37abac6d 237 struct drr_write *drrw = &(dsp->dsa_drr->drr_u.drr_write);
428870ff 238
ea97f8ce
MA		 239 /*
240 * We send data in increasing object, offset order.
241 * See comment in dump_free() for details.
242 */
243 ASSERT(object > dsp->dsa_last_data_object ||
244 (object == dsp->dsa_last_data_object &&
245 offset > dsp->dsa_last_data_offset));
246 dsp->dsa_last_data_object = object;
247 dsp->dsa_last_data_offset = offset + blksz - 1;
428870ff
BB		 248
249 /*
250 * If there is any kind of pending aggregation (currently either
251 * a grouping of free objects or free blocks), push it out to
252 * the stream, since aggregation can't be done across operations
253 * of different types.
254 */
37abac6d 255 if (dsp->dsa_pending_op != PENDING_NONE) {
37f8a883 256 if (dump_record(dsp, NULL, 0) != 0)
2e528b49 257 return (SET_ERROR(EINTR));
37abac6d 258 dsp->dsa_pending_op = PENDING_NONE;
428870ff 259 }
37f8a883 260 /* write a WRITE record */
37abac6d
BP		 261 bzero(dsp->dsa_drr, sizeof (dmu_replay_record_t));
262 dsp->dsa_drr->drr_type = DRR_WRITE;
428870ff
BB		 263 drrw->drr_object = object;
264 drrw->drr_type = type;
265 drrw->drr_offset = offset;
266 drrw->drr_length = blksz;
37abac6d 267 drrw->drr_toguid = dsp->dsa_toguid;
f1512ee6 268 if (bp == NULL || BP_IS_EMBEDDED(bp)) {
9b67f605 269 /*
f1512ee6
MA		 270 * There's no pre-computed checksum for partial-block
271 * writes or embedded BP's, so (like
272 * fletcher4-checkummed blocks) userland will have to
273 * compute a dedup-capable checksum itself.
9b67f605
MA		 274 */
275 drrw->drr_checksumtype = ZIO_CHECKSUM_OFF;
276 } else {
277 drrw->drr_checksumtype = BP_GET_CHECKSUM(bp);
278 if (zio_checksum_table[drrw->drr_checksumtype].ci_dedup)
279 drrw->drr_checksumflags |= DRR_CHECKSUM_DEDUP;
280 DDK_SET_LSIZE(&drrw->drr_key, BP_GET_LSIZE(bp));
281 DDK_SET_PSIZE(&drrw->drr_key, BP_GET_PSIZE(bp));
282 DDK_SET_COMPRESS(&drrw->drr_key, BP_GET_COMPRESS(bp));
283 drrw->drr_key.ddk_cksum = bp->blk_cksum;
284 }
428870ff 285
37f8a883 286 if (dump_record(dsp, data, blksz) != 0)
2e528b49 287 return (SET_ERROR(EINTR));
428870ff
BB		 288 return (0);
289}
290
9b67f605
MA		 291static int
292dump_write_embedded(dmu_sendarg_t *dsp, uint64_t object, uint64_t offset,
293 int blksz, const blkptr_t *bp)
294{
295 char buf[BPE_PAYLOAD_SIZE];
296 struct drr_write_embedded *drrw =
297 &(dsp->dsa_drr->drr_u.drr_write_embedded);
298
299 if (dsp->dsa_pending_op != PENDING_NONE) {
37f8a883 300 if (dump_record(dsp, NULL, 0) != 0)
9b67f605
MA		 301 return (EINTR);
302 dsp->dsa_pending_op = PENDING_NONE;
303 }
304
305 ASSERT(BP_IS_EMBEDDED(bp));
306
307 bzero(dsp->dsa_drr, sizeof (dmu_replay_record_t));
308 dsp->dsa_drr->drr_type = DRR_WRITE_EMBEDDED;
309 drrw->drr_object = object;
310 drrw->drr_offset = offset;
311 drrw->drr_length = blksz;
312 drrw->drr_toguid = dsp->dsa_toguid;
313 drrw->drr_compression = BP_GET_COMPRESS(bp);
314 drrw->drr_etype = BPE_GET_ETYPE(bp);
315 drrw->drr_lsize = BPE_GET_LSIZE(bp);
316 drrw->drr_psize = BPE_GET_PSIZE(bp);
317
318 decode_embedded_bp_compressed(bp, buf);
319
37f8a883 320 if (dump_record(dsp, buf, P2ROUNDUP(drrw->drr_psize, 8)) != 0)
9b67f605
MA		 321 return (EINTR);
322 return (0);
323}
324
428870ff 325static int
37abac6d 326dump_spill(dmu_sendarg_t *dsp, uint64_t object, int blksz, void *data)
428870ff 327{
37abac6d 328 struct drr_spill *drrs = &(dsp->dsa_drr->drr_u.drr_spill);
428870ff 329
37abac6d 330 if (dsp->dsa_pending_op != PENDING_NONE) {
37f8a883 331 if (dump_record(dsp, NULL, 0) != 0)
2e528b49 332 return (SET_ERROR(EINTR));
37abac6d 333 dsp->dsa_pending_op = PENDING_NONE;
428870ff
BB		 334 }
335
336 /* write a SPILL record */
37abac6d
BP		 337 bzero(dsp->dsa_drr, sizeof (dmu_replay_record_t));
338 dsp->dsa_drr->drr_type = DRR_SPILL;
428870ff
BB		 339 drrs->drr_object = object;
340 drrs->drr_length = blksz;
37abac6d 341 drrs->drr_toguid = dsp->dsa_toguid;
34dc7c2f 342
37f8a883 343 if (dump_record(dsp, data, blksz) != 0)
2e528b49 344 return (SET_ERROR(EINTR));
34dc7c2f
BB		 345 return (0);
346}
347
348static int
37abac6d 349dump_freeobjects(dmu_sendarg_t *dsp, uint64_t firstobj, uint64_t numobjs)
34dc7c2f 350{
37abac6d 351 struct drr_freeobjects *drrfo = &(dsp->dsa_drr->drr_u.drr_freeobjects);
428870ff 352
ea97f8ce
MA		 353 /* See comment in dump_free(). */
354 if (!dsp->dsa_incremental)
355 return (0);
356
428870ff
BB		 357 /*
358 * If there is a pending op, but it's not PENDING_FREEOBJECTS,
359 * push it out, since free block aggregation can only be done for
360 * blocks of the same type (i.e., DRR_FREE records can only be
361 * aggregated with other DRR_FREE records. DRR_FREEOBJECTS records
362 * can only be aggregated with other DRR_FREEOBJECTS records.
363 */
37abac6d
BP		 364 if (dsp->dsa_pending_op != PENDING_NONE &&
365 dsp->dsa_pending_op != PENDING_FREEOBJECTS) {
37f8a883 366 if (dump_record(dsp, NULL, 0) != 0)
2e528b49 367 return (SET_ERROR(EINTR));
37abac6d 368 dsp->dsa_pending_op = PENDING_NONE;
428870ff 369 }
37abac6d 370 if (dsp->dsa_pending_op == PENDING_FREEOBJECTS) {
428870ff
BB		 371 /*
372 * See whether this free object array can be aggregated
373 * with pending one
374 */
375 if (drrfo->drr_firstobj + drrfo->drr_numobjs == firstobj) {
376 drrfo->drr_numobjs += numobjs;
377 return (0);
378 } else {
379 /* can't be aggregated. Push out pending record */
37f8a883 380 if (dump_record(dsp, NULL, 0) != 0)
2e528b49 381 return (SET_ERROR(EINTR));
37abac6d 382 dsp->dsa_pending_op = PENDING_NONE;
428870ff
BB		 383 }
384 }
385
34dc7c2f 386 /* write a FREEOBJECTS record */
37abac6d
BP		 387 bzero(dsp->dsa_drr, sizeof (dmu_replay_record_t));
388 dsp->dsa_drr->drr_type = DRR_FREEOBJECTS;
428870ff
BB		 389 drrfo->drr_firstobj = firstobj;
390 drrfo->drr_numobjs = numobjs;
37abac6d 391 drrfo->drr_toguid = dsp->dsa_toguid;
428870ff 392
37abac6d 393 dsp->dsa_pending_op = PENDING_FREEOBJECTS;
34dc7c2f 394
34dc7c2f
BB		 395 return (0);
396}
397
398static int
37abac6d 399dump_dnode(dmu_sendarg_t *dsp, uint64_t object, dnode_phys_t *dnp)
34dc7c2f 400{
37abac6d 401 struct drr_object *drro = &(dsp->dsa_drr->drr_u.drr_object);
428870ff 402
34dc7c2f 403 if (dnp == NULL || dnp->dn_type == DMU_OT_NONE)
37abac6d 404 return (dump_freeobjects(dsp, object, 1));
34dc7c2f 405
37abac6d 406 if (dsp->dsa_pending_op != PENDING_NONE) {
37f8a883 407 if (dump_record(dsp, NULL, 0) != 0)
2e528b49 408 return (SET_ERROR(EINTR));
37abac6d 409 dsp->dsa_pending_op = PENDING_NONE;
428870ff
BB		 410 }
411
34dc7c2f 412 /* write an OBJECT record */
37abac6d
BP		 413 bzero(dsp->dsa_drr, sizeof (dmu_replay_record_t));
414 dsp->dsa_drr->drr_type = DRR_OBJECT;
428870ff
BB		 415 drro->drr_object = object;
416 drro->drr_type = dnp->dn_type;
417 drro->drr_bonustype = dnp->dn_bonustype;
418 drro->drr_blksz = dnp->dn_datablkszsec << SPA_MINBLOCKSHIFT;
419 drro->drr_bonuslen = dnp->dn_bonuslen;
420 drro->drr_checksumtype = dnp->dn_checksum;
421 drro->drr_compress = dnp->dn_compress;
37abac6d 422 drro->drr_toguid = dsp->dsa_toguid;
428870ff 423
f1512ee6
MA		 424 if (!(dsp->dsa_featureflags & DMU_BACKUP_FEATURE_LARGE_BLOCKS) &&
425 drro->drr_blksz > SPA_OLD_MAXBLOCKSIZE)
426 drro->drr_blksz = SPA_OLD_MAXBLOCKSIZE;
427
37f8a883
MA		 428 if (dump_record(dsp, DN_BONUS(dnp),
429 P2ROUNDUP(dnp->dn_bonuslen, 8)) != 0) {
2e528b49 430 return (SET_ERROR(EINTR));
37f8a883 431 }
34dc7c2f 432
ea97f8ce 433 /* Free anything past the end of the file. */
37abac6d 434 if (dump_free(dsp, object, (dnp->dn_maxblkid + 1) *
ea97f8ce 435 (dnp->dn_datablkszsec << SPA_MINBLOCKSHIFT), -1ULL) != 0)
2e528b49 436 return (SET_ERROR(EINTR));
13fe0198 437 if (dsp->dsa_err != 0)
2e528b49 438 return (SET_ERROR(EINTR));
34dc7c2f
BB		 439 return (0);
440}
441
9b67f605
MA		 442static boolean_t
443backup_do_embed(dmu_sendarg_t *dsp, const blkptr_t *bp)
444{
445 if (!BP_IS_EMBEDDED(bp))
446 return (B_FALSE);
447
448 /*
449 * Compression function must be legacy, or explicitly enabled.
450 */
451 if ((BP_GET_COMPRESS(bp) >= ZIO_COMPRESS_LEGACY_FUNCTIONS &&
452 !(dsp->dsa_featureflags & DMU_BACKUP_FEATURE_EMBED_DATA_LZ4)))
453 return (B_FALSE);
454
455 /*
456 * Embed type must be explicitly enabled.
457 */
458 switch (BPE_GET_ETYPE(bp)) {
459 case BP_EMBEDDED_TYPE_DATA:
460 if (dsp->dsa_featureflags & DMU_BACKUP_FEATURE_EMBED_DATA)
461 return (B_TRUE);
462 break;
463 default:
464 return (B_FALSE);
465 }
466 return (B_FALSE);
467}
468
34dc7c2f
BB		 469#define BP_SPAN(dnp, level) \
470 (((uint64_t)dnp->dn_datablkszsec) << (SPA_MINBLOCKSHIFT + \
471 (level) * (dnp->dn_indblkshift - SPA_BLKPTRSHIFT)))
472
428870ff 473/* ARGSUSED */
34dc7c2f 474static int
294f6806 475backup_cb(spa_t *spa, zilog_t *zilog, const blkptr_t *bp,
5dbd68a3 476 const zbookmark_phys_t *zb, const dnode_phys_t *dnp, void *arg)
34dc7c2f 477{
37abac6d 478 dmu_sendarg_t *dsp = arg;
34dc7c2f 479 dmu_object_type_t type = bp ? BP_GET_TYPE(bp) : DMU_OT_NONE;
34dc7c2f
BB		 480 int err = 0;
481
482 if (issig(JUSTLOOKING) && issig(FORREAL))
2e528b49 483 return (SET_ERROR(EINTR));
34dc7c2f 484
428870ff
BB		 485 if (zb->zb_object != DMU_META_DNODE_OBJECT &&
486 DMU_OBJECT_IS_SPECIAL(zb->zb_object)) {
9babb374 487 return (0);
da536844
MA		 488 } else if (zb->zb_level == ZB_ZIL_LEVEL) {
489 /*
490 * If we are sending a non-snapshot (which is allowed on
491 * read-only pools), it may have a ZIL, which must be ignored.
492 */
493 return (0);
b0bc7a84
MG		 494 } else if (BP_IS_HOLE(bp) &&
495 zb->zb_object == DMU_META_DNODE_OBJECT) {
b128c09f
BB		 496 uint64_t span = BP_SPAN(dnp, zb->zb_level);
497 uint64_t dnobj = (zb->zb_blkid * span) >> DNODE_SHIFT;
37abac6d 498 err = dump_freeobjects(dsp, dnobj, span >> DNODE_SHIFT);
b0bc7a84 499 } else if (BP_IS_HOLE(bp)) {
b128c09f 500 uint64_t span = BP_SPAN(dnp, zb->zb_level);
37abac6d 501 err = dump_free(dsp, zb->zb_object, zb->zb_blkid * span, span);
b128c09f
BB		 502 } else if (zb->zb_level > 0 || type == DMU_OT_OBJSET) {
503 return (0);
504 } else if (type == DMU_OT_DNODE) {
505 dnode_phys_t *blk;
34dc7c2f
BB		 506 int i;
507 int blksz = BP_GET_LSIZE(bp);
2a432414 508 arc_flags_t aflags = ARC_FLAG_WAIT;
b128c09f
BB		 509 arc_buf_t *abuf;
510
294f6806
GW		 511 if (arc_read(NULL, spa, bp, arc_getbuf_func, &abuf,
512 ZIO_PRIORITY_ASYNC_READ, ZIO_FLAG_CANFAIL,
513 &aflags, zb) != 0)
2e528b49 514 return (SET_ERROR(EIO));
34dc7c2f 515
b128c09f 516 blk = abuf->b_data;
34dc7c2f 517 for (i = 0; i < blksz >> DNODE_SHIFT; i++) {
b128c09f
BB		 518 uint64_t dnobj = (zb->zb_blkid <<
519 (DNODE_BLOCK_SHIFT - DNODE_SHIFT)) + i;
37abac6d 520 err = dump_dnode(dsp, dnobj, blk+i);
13fe0198 521 if (err != 0)
34dc7c2f
BB		 522 break;
523 }
b128c09f 524 (void) arc_buf_remove_ref(abuf, &abuf);
428870ff 525 } else if (type == DMU_OT_SA) {
2a432414 526 arc_flags_t aflags = ARC_FLAG_WAIT;
b128c09f 527 arc_buf_t *abuf;
34dc7c2f 528 int blksz = BP_GET_LSIZE(bp);
b128c09f 529
294f6806
GW		 530 if (arc_read(NULL, spa, bp, arc_getbuf_func, &abuf,
531 ZIO_PRIORITY_ASYNC_READ, ZIO_FLAG_CANFAIL,
532 &aflags, zb) != 0)
2e528b49 533 return (SET_ERROR(EIO));
b128c09f 534
37abac6d 535 err = dump_spill(dsp, zb->zb_object, blksz, abuf->b_data);
428870ff 536 (void) arc_buf_remove_ref(abuf, &abuf);
9b67f605
MA		 537 } else if (backup_do_embed(dsp, bp)) {
538 /* it's an embedded level-0 block of a regular object */
539 int blksz = dnp->dn_datablkszsec << SPA_MINBLOCKSHIFT;
540 err = dump_write_embedded(dsp, zb->zb_object,
541 zb->zb_blkid * blksz, blksz, bp);
428870ff 542 } else { /* it's a level-0 block of a regular object */
f1512ee6 543 uint64_t offset;
2a432414 544 arc_flags_t aflags = ARC_FLAG_WAIT;
428870ff
BB		 545 arc_buf_t *abuf;
546 int blksz = BP_GET_LSIZE(bp);
547
9b67f605 548 ASSERT3U(blksz, ==, dnp->dn_datablkszsec << SPA_MINBLOCKSHIFT);
da536844 549 ASSERT0(zb->zb_level);
294f6806
GW		 550 if (arc_read(NULL, spa, bp, arc_getbuf_func, &abuf,
551 ZIO_PRIORITY_ASYNC_READ, ZIO_FLAG_CANFAIL,
552 &aflags, zb) != 0) {
330d06f9
MA		 553 if (zfs_send_corrupt_data) {
554 uint64_t *ptr;
555 /* Send a block filled with 0x"zfs badd bloc" */
556 abuf = arc_buf_alloc(spa, blksz, &abuf,
557 ARC_BUFC_DATA);
558 for (ptr = abuf->b_data;
559 (char *)ptr < (char *)abuf->b_data + blksz;
560 ptr++)
dd26aa53 561 *ptr = 0x2f5baddb10cULL;
330d06f9 562 } else {
2e528b49 563 return (SET_ERROR(EIO));
330d06f9
MA		 564 }
565 }
428870ff 566
f1512ee6
MA		 567 offset = zb->zb_blkid * blksz;
568
569 if (!(dsp->dsa_featureflags &
570 DMU_BACKUP_FEATURE_LARGE_BLOCKS) &&
571 blksz > SPA_OLD_MAXBLOCKSIZE) {
572 char *buf = abuf->b_data;
573 while (blksz > 0 && err == 0) {
574 int n = MIN(blksz, SPA_OLD_MAXBLOCKSIZE);
575 err = dump_write(dsp, type, zb->zb_object,
576 offset, n, NULL, buf);
577 offset += n;
578 buf += n;
579 blksz -= n;
580 }
581 } else {
582 err = dump_write(dsp, type, zb->zb_object,
583 offset, blksz, bp, abuf->b_data);
584 }
b128c09f 585 (void) arc_buf_remove_ref(abuf, &abuf);
34dc7c2f
BB		 586 }
587
588 ASSERT(err == 0 || err == EINTR);
589 return (err);
590}
591
6f1ffb06 592/*
da536844 593 * Releases dp using the specified tag.
6f1ffb06 594 */
13fe0198
MA		 595static int
596dmu_send_impl(void *tag, dsl_pool_t *dp, dsl_dataset_t *ds,
9b67f605 597 zfs_bookmark_phys_t *fromzb, boolean_t is_clone, boolean_t embedok,
f1512ee6 598 boolean_t large_block_ok, int outfd, vnode_t *vp, offset_t *off)
34dc7c2f 599{
13fe0198 600 objset_t *os;
34dc7c2f 601 dmu_replay_record_t *drr;
37abac6d 602 dmu_sendarg_t *dsp;
34dc7c2f
BB		 603 int err;
604 uint64_t fromtxg = 0;
9b67f605 605 uint64_t featureflags = 0;
34dc7c2f 606
13fe0198
MA		 607 err = dmu_objset_from_ds(ds, &os);
608 if (err != 0) {
13fe0198
MA		 609 dsl_pool_rele(dp, tag);
610 return (err);
611 }
34dc7c2f 612
34dc7c2f
BB		 613 drr = kmem_zalloc(sizeof (dmu_replay_record_t), KM_SLEEP);
614 drr->drr_type = DRR_BEGIN;
615 drr->drr_u.drr_begin.drr_magic = DMU_BACKUP_MAGIC;
428870ff
BB		 616 DMU_SET_STREAM_HDRTYPE(drr->drr_u.drr_begin.drr_versioninfo,
617 DMU_SUBSTREAM);
618
619#ifdef _KERNEL
13fe0198 620 if (dmu_objset_type(os) == DMU_OST_ZFS) {
428870ff 621 uint64_t version;
13fe0198 622 if (zfs_get_zplprop(os, ZFS_PROP_VERSION, &version) != 0) {
37abac6d 623 kmem_free(drr, sizeof (dmu_replay_record_t));
13fe0198 624 dsl_pool_rele(dp, tag);
2e528b49 625 return (SET_ERROR(EINVAL));
37abac6d 626 }
13fe0198 627 if (version >= ZPL_VERSION_SA) {
9b67f605 628 featureflags |= DMU_BACKUP_FEATURE_SA_SPILL;
428870ff
BB		 629 }
630 }
631#endif
632
241b5415 633 if (large_block_ok && ds->ds_feature_inuse[SPA_FEATURE_LARGE_BLOCKS])
f1512ee6 634 featureflags |= DMU_BACKUP_FEATURE_LARGE_BLOCKS;
9b67f605
MA		 635 if (embedok &&
636 spa_feature_is_active(dp->dp_spa, SPA_FEATURE_EMBEDDED_DATA)) {
637 featureflags |= DMU_BACKUP_FEATURE_EMBED_DATA;
638 if (spa_feature_is_active(dp->dp_spa, SPA_FEATURE_LZ4_COMPRESS))
639 featureflags |= DMU_BACKUP_FEATURE_EMBED_DATA_LZ4;
640 } else {
641 embedok = B_FALSE;
642 }
643
644 DMU_SET_FEATUREFLAGS(drr->drr_u.drr_begin.drr_versioninfo,
645 featureflags);
646
34dc7c2f 647 drr->drr_u.drr_begin.drr_creation_time =
d683ddbb 648 dsl_dataset_phys(ds)->ds_creation_time;
13fe0198 649 drr->drr_u.drr_begin.drr_type = dmu_objset_type(os);
da536844 650 if (is_clone)
34dc7c2f 651 drr->drr_u.drr_begin.drr_flags |= DRR_FLAG_CLONE;
d683ddbb
JG		 652 drr->drr_u.drr_begin.drr_toguid = dsl_dataset_phys(ds)->ds_guid;
653 if (dsl_dataset_phys(ds)->ds_flags & DS_FLAG_CI_DATASET)
34dc7c2f
BB		 654 drr->drr_u.drr_begin.drr_flags |= DRR_FLAG_CI_DATA;
655
da536844
MA		 656 if (fromzb != NULL) {
657 drr->drr_u.drr_begin.drr_fromguid = fromzb->zbm_guid;
658 fromtxg = fromzb->zbm_creation_txg;
659 }
34dc7c2f 660 dsl_dataset_name(ds, drr->drr_u.drr_begin.drr_toname);
0c66c32d 661 if (!ds->ds_is_snapshot) {
da536844
MA		 662 (void) strlcat(drr->drr_u.drr_begin.drr_toname, "@--head--",
663 sizeof (drr->drr_u.drr_begin.drr_toname));
13fe0198 664 }
34dc7c2f 665
37abac6d
BP		 666 dsp = kmem_zalloc(sizeof (dmu_sendarg_t), KM_SLEEP);
667
668 dsp->dsa_drr = drr;
669 dsp->dsa_vp = vp;
670 dsp->dsa_outfd = outfd;
671 dsp->dsa_proc = curproc;
13fe0198 672 dsp->dsa_os = os;
37abac6d 673 dsp->dsa_off = off;
d683ddbb 674 dsp->dsa_toguid = dsl_dataset_phys(ds)->ds_guid;
37abac6d 675 dsp->dsa_pending_op = PENDING_NONE;
da536844 676 dsp->dsa_incremental = (fromzb != NULL);
9b67f605 677 dsp->dsa_featureflags = featureflags;
37abac6d
BP		 678
679 mutex_enter(&ds->ds_sendstream_lock);
680 list_insert_head(&ds->ds_sendstreams, dsp);
681 mutex_exit(&ds->ds_sendstream_lock);
682
7ec09286
MA		 683 dsl_dataset_long_hold(ds, FTAG);
684 dsl_pool_rele(dp, tag);
685
37f8a883 686 if (dump_record(dsp, NULL, 0) != 0) {
37abac6d
BP		 687 err = dsp->dsa_err;
688 goto out;
34dc7c2f
BB		 689 }
690
b128c09f 691 err = traverse_dataset(ds, fromtxg, TRAVERSE_PRE | TRAVERSE_PREFETCH,
37abac6d 692 backup_cb, dsp);
34dc7c2f 693
37abac6d 694 if (dsp->dsa_pending_op != PENDING_NONE)
37f8a883 695 if (dump_record(dsp, NULL, 0) != 0)
2e528b49 696 err = SET_ERROR(EINTR);
428870ff 697
13fe0198
MA		 698 if (err != 0) {
699 if (err == EINTR && dsp->dsa_err != 0)
37abac6d
BP		 700 err = dsp->dsa_err;
701 goto out;
34dc7c2f
BB		 702 }
703
704 bzero(drr, sizeof (dmu_replay_record_t));
705 drr->drr_type = DRR_END;
37abac6d
BP		 706 drr->drr_u.drr_end.drr_checksum = dsp->dsa_zc;
707 drr->drr_u.drr_end.drr_toguid = dsp->dsa_toguid;
34dc7c2f 708
37f8a883 709 if (dump_record(dsp, NULL, 0) != 0) {
37abac6d
BP		 710 err = dsp->dsa_err;
711 goto out;
34dc7c2f
BB		 712 }
713
37abac6d
BP		 714out:
715 mutex_enter(&ds->ds_sendstream_lock);
716 list_remove(&ds->ds_sendstreams, dsp);
717 mutex_exit(&ds->ds_sendstream_lock);
718
34dc7c2f 719 kmem_free(drr, sizeof (dmu_replay_record_t));
37abac6d 720 kmem_free(dsp, sizeof (dmu_sendarg_t));
34dc7c2f 721
13fe0198 722 dsl_dataset_long_rele(ds, FTAG);
13fe0198 723
37abac6d 724 return (err);
34dc7c2f
BB		 725}
726
330d06f9 727int
13fe0198 728dmu_send_obj(const char *pool, uint64_t tosnap, uint64_t fromsnap,
f1512ee6
MA		 729 boolean_t embedok, boolean_t large_block_ok,
730 int outfd, vnode_t *vp, offset_t *off)
13fe0198
MA		 731{
732 dsl_pool_t *dp;
733 dsl_dataset_t *ds;
734 dsl_dataset_t *fromds = NULL;
735 int err;
736
737 err = dsl_pool_hold(pool, FTAG, &dp);
738 if (err != 0)
739 return (err);
740
741 err = dsl_dataset_hold_obj(dp, tosnap, FTAG, &ds);
742 if (err != 0) {
743 dsl_pool_rele(dp, FTAG);
744 return (err);
745 }
746
747 if (fromsnap != 0) {
da536844
MA		 748 zfs_bookmark_phys_t zb;
749 boolean_t is_clone;
750
13fe0198
MA		 751 err = dsl_dataset_hold_obj(dp, fromsnap, FTAG, &fromds);
752 if (err != 0) {
753 dsl_dataset_rele(ds, FTAG);
754 dsl_pool_rele(dp, FTAG);
755 return (err);
756 }
da536844
MA		 757 if (!dsl_dataset_is_before(ds, fromds, 0))
758 err = SET_ERROR(EXDEV);
d683ddbb
JG		 759 zb.zbm_creation_time =
760 dsl_dataset_phys(fromds)->ds_creation_time;
761 zb.zbm_creation_txg = dsl_dataset_phys(fromds)->ds_creation_txg;
762 zb.zbm_guid = dsl_dataset_phys(fromds)->ds_guid;
da536844
MA		 763 is_clone = (fromds->ds_dir != ds->ds_dir);
764 dsl_dataset_rele(fromds, FTAG);
f1512ee6
MA		 765 err = dmu_send_impl(FTAG, dp, ds, &zb, is_clone,
766 embedok, large_block_ok, outfd, vp, off);
da536844 767 } else {
f1512ee6
MA		 768 err = dmu_send_impl(FTAG, dp, ds, NULL, B_FALSE,
769 embedok, large_block_ok, outfd, vp, off);
13fe0198 770 }
da536844
MA		 771 dsl_dataset_rele(ds, FTAG);
772 return (err);
13fe0198
MA		 773}
774
775int
f1512ee6
MA		 776dmu_send(const char *tosnap, const char *fromsnap,
777 boolean_t embedok, boolean_t large_block_ok,
13fe0198
MA		 778 int outfd, vnode_t *vp, offset_t *off)
779{
780 dsl_pool_t *dp;
781 dsl_dataset_t *ds;
13fe0198 782 int err;
da536844 783 boolean_t owned = B_FALSE;
13fe0198 784
da536844 785 if (fromsnap != NULL && strpbrk(fromsnap, "@#") == NULL)
2e528b49 786 return (SET_ERROR(EINVAL));
13fe0198
MA		 787
788 err = dsl_pool_hold(tosnap, FTAG, &dp);
789 if (err != 0)
790 return (err);
791
da536844
MA		 792 if (strchr(tosnap, '@') == NULL && spa_writeable(dp->dp_spa)) {
793 /*
794 * We are sending a filesystem or volume. Ensure
795 * that it doesn't change by owning the dataset.
796 */
797 err = dsl_dataset_own(dp, tosnap, FTAG, &ds);
798 owned = B_TRUE;
799 } else {
800 err = dsl_dataset_hold(dp, tosnap, FTAG, &ds);
801 }
13fe0198
MA		 802 if (err != 0) {
803 dsl_pool_rele(dp, FTAG);
804 return (err);
805 }
806
807 if (fromsnap != NULL) {
da536844
MA		 808 zfs_bookmark_phys_t zb;
809 boolean_t is_clone = B_FALSE;
810 int fsnamelen = strchr(tosnap, '@') - tosnap;
811
812 /*
813 * If the fromsnap is in a different filesystem, then
814 * mark the send stream as a clone.
815 */
816 if (strncmp(tosnap, fromsnap, fsnamelen) != 0 ||
817 (fromsnap[fsnamelen] != '@' &&
818 fromsnap[fsnamelen] != '#')) {
819 is_clone = B_TRUE;
820 }
821
822 if (strchr(fromsnap, '@')) {
823 dsl_dataset_t *fromds;
824 err = dsl_dataset_hold(dp, fromsnap, FTAG, &fromds);
825 if (err == 0) {
826 if (!dsl_dataset_is_before(ds, fromds, 0))
827 err = SET_ERROR(EXDEV);
828 zb.zbm_creation_time =
d683ddbb 829 dsl_dataset_phys(fromds)->ds_creation_time;
da536844 830 zb.zbm_creation_txg =
d683ddbb
JG		 831 dsl_dataset_phys(fromds)->ds_creation_txg;
832 zb.zbm_guid = dsl_dataset_phys(fromds)->ds_guid;
da536844
MA		 833 is_clone = (ds->ds_dir != fromds->ds_dir);
834 dsl_dataset_rele(fromds, FTAG);
835 }
836 } else {
837 err = dsl_bookmark_lookup(dp, fromsnap, ds, &zb);
838 }
13fe0198
MA		 839 if (err != 0) {
840 dsl_dataset_rele(ds, FTAG);
841 dsl_pool_rele(dp, FTAG);
842 return (err);
843 }
f1512ee6
MA		 844 err = dmu_send_impl(FTAG, dp, ds, &zb, is_clone,
845 embedok, large_block_ok, outfd, vp, off);
da536844 846 } else {
f1512ee6
MA		 847 err = dmu_send_impl(FTAG, dp, ds, NULL, B_FALSE,
848 embedok, large_block_ok, outfd, vp, off);
13fe0198 849 }
da536844
MA		 850 if (owned)
851 dsl_dataset_disown(ds, FTAG);
852 else
853 dsl_dataset_rele(ds, FTAG);
854 return (err);
13fe0198
MA		 855}
856
5dc8b736
MG		 857static int
858dmu_adjust_send_estimate_for_indirects(dsl_dataset_t *ds, uint64_t size,
859 uint64_t *sizep)
860{
861 int err;
862 /*
863 * Assume that space (both on-disk and in-stream) is dominated by
864 * data. We will adjust for indirect blocks and the copies property,
865 * but ignore per-object space used (eg, dnodes and DRR_OBJECT records).
866 */
867
868 /*
869 * Subtract out approximate space used by indirect blocks.
870 * Assume most space is used by data blocks (non-indirect, non-dnode).
871 * Assume all blocks are recordsize. Assume ditto blocks and
872 * internal fragmentation counter out compression.
873 *
874 * Therefore, space used by indirect blocks is sizeof(blkptr_t) per
875 * block, which we observe in practice.
876 */
877 uint64_t recordsize;
878 err = dsl_prop_get_int_ds(ds, "recordsize", &recordsize);
879 if (err != 0)
880 return (err);
881 size -= size / recordsize * sizeof (blkptr_t);
882
883 /* Add in the space for the record associated with each block. */
884 size += size / recordsize * sizeof (dmu_replay_record_t);
885
886 *sizep = size;
887
888 return (0);
889}
890
13fe0198
MA		 891int
892dmu_send_estimate(dsl_dataset_t *ds, dsl_dataset_t *fromds, uint64_t *sizep)
330d06f9 893{
330d06f9 894 int err;
5dc8b736 895 uint64_t size;
13fe0198 896
fd0fd646 897 ASSERT(dsl_pool_config_held(ds->ds_dir->dd_pool));
330d06f9
MA		 898
899 /* tosnap must be a snapshot */
0c66c32d 900 if (!ds->ds_is_snapshot)
2e528b49 901 return (SET_ERROR(EINVAL));
330d06f9 902
71e2fe41
AG		 903 /* fromsnap, if provided, must be a snapshot */
904 if (fromds != NULL && !fromds->ds_is_snapshot)
905 return (SET_ERROR(EINVAL));
906
6f1ffb06
MA		 907 /*
908 * fromsnap must be an earlier snapshot from the same fs as tosnap,
909 * or the origin's fs.
910 */
da536844 911 if (fromds != NULL && !dsl_dataset_is_before(ds, fromds, 0))
2e528b49 912 return (SET_ERROR(EXDEV));
330d06f9 913
330d06f9
MA		 914 /* Get uncompressed size estimate of changed data. */
915 if (fromds == NULL) {
d683ddbb 916 size = dsl_dataset_phys(ds)->ds_uncompressed_bytes;
330d06f9
MA		 917 } else {
918 uint64_t used, comp;
919 err = dsl_dataset_space_written(fromds, ds,
920 &used, &comp, &size);
13fe0198 921 if (err != 0)
330d06f9
MA		 922 return (err);
923 }
924
5dc8b736
MG		 925 err = dmu_adjust_send_estimate_for_indirects(ds, size, sizep);
926 return (err);
927}
330d06f9 928
5dc8b736
MG		 929/*
930 * Simple callback used to traverse the blocks of a snapshot and sum their
931 * uncompressed size
932 */
933/* ARGSUSED */
934static int
935dmu_calculate_send_traversal(spa_t *spa, zilog_t *zilog, const blkptr_t *bp,
936 const zbookmark_phys_t *zb, const dnode_phys_t *dnp, void *arg)
937{
938 uint64_t *spaceptr = arg;
939 if (bp != NULL && !BP_IS_HOLE(bp)) {
940 *spaceptr += BP_GET_UCSIZE(bp);
941 }
942 return (0);
943}
944
945/*
946 * Given a desination snapshot and a TXG, calculate the approximate size of a
947 * send stream sent from that TXG. from_txg may be zero, indicating that the
948 * whole snapshot will be sent.
949 */
950int
951dmu_send_estimate_from_txg(dsl_dataset_t *ds, uint64_t from_txg,
952 uint64_t *sizep)
953{
5dc8b736
MG		 954 int err;
955 uint64_t size = 0;
956
fd0fd646 957 ASSERT(dsl_pool_config_held(ds->ds_dir->dd_pool));
5dc8b736
MG		 958
959 /* tosnap must be a snapshot */
960 if (!dsl_dataset_is_snapshot(ds))
961 return (SET_ERROR(EINVAL));
962
963 /* verify that from_txg is before the provided snapshot was taken */
964 if (from_txg >= dsl_dataset_phys(ds)->ds_creation_txg) {
965 return (SET_ERROR(EXDEV));
966 }
330d06f9 967 /*
5dc8b736
MG		 968 * traverse the blocks of the snapshot with birth times after
969 * from_txg, summing their uncompressed size
330d06f9 970 */
5dc8b736
MG		 971 err = traverse_dataset(ds, from_txg, TRAVERSE_POST,
972 dmu_calculate_send_traversal, &size);
973 if (err)
330d06f9 974 return (err);
330d06f9 975
5dc8b736
MG		 976 err = dmu_adjust_send_estimate_for_indirects(ds, size, sizep);
977 return (err);
330d06f9
MA		 978}
979
13fe0198
MA		 980typedef struct dmu_recv_begin_arg {
981 const char *drba_origin;
982 dmu_recv_cookie_t *drba_cookie;
983 cred_t *drba_cred;
19580676 984 uint64_t drba_snapobj;
13fe0198 985} dmu_recv_begin_arg_t;
34dc7c2f 986
34dc7c2f 987static int
13fe0198
MA		 988recv_begin_check_existing_impl(dmu_recv_begin_arg_t *drba, dsl_dataset_t *ds,
989 uint64_t fromguid)
34dc7c2f 990{
34dc7c2f 991 uint64_t val;
13fe0198
MA		 992 int error;
993 dsl_pool_t *dp = ds->ds_dir->dd_pool;
34dc7c2f 994
13fe0198
MA		 995 /* temporary clone name must not exist */
996 error = zap_lookup(dp->dp_meta_objset,
d683ddbb 997 dsl_dir_phys(ds->ds_dir)->dd_child_dir_zapobj, recv_clone_name,
13fe0198
MA		 998 8, 1, &val);
999 if (error != ENOENT)
1000 return (error == 0 ? EBUSY : error);
1001
572e2857 1002 /* new snapshot name must not exist */
13fe0198 1003 error = zap_lookup(dp->dp_meta_objset,
d683ddbb
JG		 1004 dsl_dataset_phys(ds)->ds_snapnames_zapobj,
1005 drba->drba_cookie->drc_tosnap, 8, 1, &val);
13fe0198
MA		 1006 if (error != ENOENT)
1007 return (error == 0 ? EEXIST : error);
572e2857 1008
788eb90c
JJ		 1009 /*
1010 * Check snapshot limit before receiving. We'll recheck again at the
1011 * end, but might as well abort before receiving if we're already over
1012 * the limit.
1013 *
1014 * Note that we do not check the file system limit with
1015 * dsl_dir_fscount_check because the temporary %clones don't count
1016 * against that limit.
1017 */
1018 error = dsl_fs_ss_limit_check(ds->ds_dir, 1, ZFS_PROP_SNAPSHOT_LIMIT,
1019 NULL, drba->drba_cred);
1020 if (error != 0)
1021 return (error);
1022
13fe0198 1023 if (fromguid != 0) {
19580676 1024 dsl_dataset_t *snap;
d683ddbb 1025 uint64_t obj = dsl_dataset_phys(ds)->ds_prev_snap_obj;
19580676
MA		 1026
1027 /* Find snapshot in this dir that matches fromguid. */
1028 while (obj != 0) {
1029 error = dsl_dataset_hold_obj(dp, obj, FTAG,
1030 &snap);
1031 if (error != 0)
1032 return (SET_ERROR(ENODEV));
1033 if (snap->ds_dir != ds->ds_dir) {
1034 dsl_dataset_rele(snap, FTAG);
1035 return (SET_ERROR(ENODEV));
1036 }
d683ddbb 1037 if (dsl_dataset_phys(snap)->ds_guid == fromguid)
19580676 1038 break;
d683ddbb 1039 obj = dsl_dataset_phys(snap)->ds_prev_snap_obj;
19580676
MA		 1040 dsl_dataset_rele(snap, FTAG);
1041 }
1042 if (obj == 0)
2e528b49 1043 return (SET_ERROR(ENODEV));
34dc7c2f 1044
19580676
MA		 1045 if (drba->drba_cookie->drc_force) {
1046 drba->drba_snapobj = obj;
1047 } else {
1048 /*
1049 * If we are not forcing, there must be no
1050 * changes since fromsnap.
1051 */
1052 if (dsl_dataset_modified_since_snap(ds, snap)) {
428870ff 1053 dsl_dataset_rele(snap, FTAG);
19580676 1054 return (SET_ERROR(ETXTBSY));
428870ff 1055 }
19580676 1056 drba->drba_snapobj = ds->ds_prev->ds_object;
428870ff 1057 }
19580676
MA		 1058
1059 dsl_dataset_rele(snap, FTAG);
428870ff 1060 } else {
cf50a2b0
AG		 1061 /* if full, then must be forced */
1062 if (!drba->drba_cookie->drc_force)
1063 return (SET_ERROR(EEXIST));
1064 /* start from $ORIGIN@$ORIGIN, if supported */
1065 drba->drba_snapobj = dp->dp_origin_snap != NULL ?
1066 dp->dp_origin_snap->ds_object : 0;
428870ff 1067 }
34dc7c2f 1068
34dc7c2f 1069 return (0);
13fe0198
MA		 1070
1071}
1072
1073static int
1074dmu_recv_begin_check(void *arg, dmu_tx_t *tx)
1075{
1076 dmu_recv_begin_arg_t *drba = arg;
1077 dsl_pool_t *dp = dmu_tx_pool(tx);
1078 struct drr_begin *drrb = drba->drba_cookie->drc_drrb;
1079 uint64_t fromguid = drrb->drr_fromguid;
1080 int flags = drrb->drr_flags;
1081 int error;
9b67f605 1082 uint64_t featureflags = DMU_GET_FEATUREFLAGS(drrb->drr_versioninfo);
13fe0198
MA		 1083 dsl_dataset_t *ds;
1084 const char *tofs = drba->drba_cookie->drc_tofs;
1085
1086 /* already checked */
1087 ASSERT3U(drrb->drr_magic, ==, DMU_BACKUP_MAGIC);
1088
1089 if (DMU_GET_STREAM_HDRTYPE(drrb->drr_versioninfo) ==
1090 DMU_COMPOUNDSTREAM ||
1091 drrb->drr_type >= DMU_OST_NUMTYPES ||
1092 ((flags & DRR_FLAG_CLONE) && drba->drba_origin == NULL))
2e528b49 1093 return (SET_ERROR(EINVAL));
13fe0198
MA		 1094
1095 /* Verify pool version supports SA if SA_SPILL feature set */
9b67f605
MA		 1096 if ((featureflags & DMU_BACKUP_FEATURE_SA_SPILL) &&
1097 spa_version(dp->dp_spa) < SPA_VERSION_SA)
1098 return (SET_ERROR(ENOTSUP));
1099
1100 /*
1101 * The receiving code doesn't know how to translate a WRITE_EMBEDDED
1102 * record to a plan WRITE record, so the pool must have the
1103 * EMBEDDED_DATA feature enabled if the stream has WRITE_EMBEDDED
1104 * records. Same with WRITE_EMBEDDED records that use LZ4 compression.
1105 */
1106 if ((featureflags & DMU_BACKUP_FEATURE_EMBED_DATA) &&
1107 !spa_feature_is_enabled(dp->dp_spa, SPA_FEATURE_EMBEDDED_DATA))
1108 return (SET_ERROR(ENOTSUP));
1109 if ((featureflags & DMU_BACKUP_FEATURE_EMBED_DATA_LZ4) &&
1110 !spa_feature_is_enabled(dp->dp_spa, SPA_FEATURE_LZ4_COMPRESS))
2e528b49 1111 return (SET_ERROR(ENOTSUP));
13fe0198 1112
f1512ee6
MA		 1113 /*
1114 * The receiving code doesn't know how to translate large blocks
1115 * to smaller ones, so the pool must have the LARGE_BLOCKS
1116 * feature enabled if the stream has LARGE_BLOCKS.
1117 */
1118 if ((featureflags & DMU_BACKUP_FEATURE_LARGE_BLOCKS) &&
1119 !spa_feature_is_enabled(dp->dp_spa, SPA_FEATURE_LARGE_BLOCKS))
1120 return (SET_ERROR(ENOTSUP));
1121
13fe0198
MA		 1122 error = dsl_dataset_hold(dp, tofs, FTAG, &ds);
1123 if (error == 0) {
1124 /* target fs already exists; recv into temp clone */
1125
1126 /* Can't recv a clone into an existing fs */
1127 if (flags & DRR_FLAG_CLONE) {
1128 dsl_dataset_rele(ds, FTAG);
2e528b49 1129 return (SET_ERROR(EINVAL));
13fe0198
MA		 1130 }
1131
1132 error = recv_begin_check_existing_impl(drba, ds, fromguid);
1133 dsl_dataset_rele(ds, FTAG);
1134 } else if (error == ENOENT) {
1135 /* target fs does not exist; must be a full backup or clone */
1136 char buf[MAXNAMELEN];
1137
1138 /*
1139 * If it's a non-clone incremental, we are missing the
1140 * target fs, so fail the recv.
1141 */
1142 if (fromguid != 0 && !(flags & DRR_FLAG_CLONE))
2e528b49 1143 return (SET_ERROR(ENOENT));
13fe0198
MA		 1144
1145 /* Open the parent of tofs */
1146 ASSERT3U(strlen(tofs), <, MAXNAMELEN);
1147 (void) strlcpy(buf, tofs, strrchr(tofs, '/') - tofs + 1);
1148 error = dsl_dataset_hold(dp, buf, FTAG, &ds);
1149 if (error != 0)
1150 return (error);
1151
788eb90c
JJ		 1152 /*
1153 * Check filesystem and snapshot limits before receiving. We'll
1154 * recheck snapshot limits again at the end (we create the
1155 * filesystems and increment those counts during begin_sync).
1156 */
1157 error = dsl_fs_ss_limit_check(ds->ds_dir, 1,
1158 ZFS_PROP_FILESYSTEM_LIMIT, NULL, drba->drba_cred);
1159 if (error != 0) {
1160 dsl_dataset_rele(ds, FTAG);
1161 return (error);
1162 }
1163
1164 error = dsl_fs_ss_limit_check(ds->ds_dir, 1,
1165 ZFS_PROP_SNAPSHOT_LIMIT, NULL, drba->drba_cred);
1166 if (error != 0) {
1167 dsl_dataset_rele(ds, FTAG);
1168 return (error);
1169 }
1170
13fe0198
MA		 1171 if (drba->drba_origin != NULL) {
1172 dsl_dataset_t *origin;
1173 error = dsl_dataset_hold(dp, drba->drba_origin,
1174 FTAG, &origin);
1175 if (error != 0) {
1176 dsl_dataset_rele(ds, FTAG);
1177 return (error);
1178 }
0c66c32d 1179 if (!origin->ds_is_snapshot) {
13fe0198
MA		 1180 dsl_dataset_rele(origin, FTAG);
1181 dsl_dataset_rele(ds, FTAG);
2e528b49 1182 return (SET_ERROR(EINVAL));
13fe0198 1183 }
d683ddbb 1184 if (dsl_dataset_phys(origin)->ds_guid != fromguid) {
13fe0198
MA		 1185 dsl_dataset_rele(origin, FTAG);
1186 dsl_dataset_rele(ds, FTAG);
2e528b49 1187 return (SET_ERROR(ENODEV));
13fe0198
MA		 1188 }
1189 dsl_dataset_rele(origin, FTAG);
1190 }
1191 dsl_dataset_rele(ds, FTAG);
1192 error = 0;
1193 }
1194 return (error);
34dc7c2f
BB		 1195}
1196
34dc7c2f 1197static void
13fe0198 1198dmu_recv_begin_sync(void *arg, dmu_tx_t *tx)
34dc7c2f 1199{
13fe0198
MA		 1200 dmu_recv_begin_arg_t *drba = arg;
1201 dsl_pool_t *dp = dmu_tx_pool(tx);
1202 struct drr_begin *drrb = drba->drba_cookie->drc_drrb;
1203 const char *tofs = drba->drba_cookie->drc_tofs;
1204 dsl_dataset_t *ds, *newds;
34dc7c2f 1205 uint64_t dsobj;
13fe0198
MA		 1206 int error;
1207 uint64_t crflags;
1208
1209 crflags = (drrb->drr_flags & DRR_FLAG_CI_DATA) ?
1210 DS_FLAG_CI_DATASET : 0;
34dc7c2f 1211
13fe0198
MA		 1212 error = dsl_dataset_hold(dp, tofs, FTAG, &ds);
1213 if (error == 0) {
1214 /* create temporary clone */
19580676
MA		 1215 dsl_dataset_t *snap = NULL;
1216 if (drba->drba_snapobj != 0) {
1217 VERIFY0(dsl_dataset_hold_obj(dp,
1218 drba->drba_snapobj, FTAG, &snap));
1219 }
13fe0198 1220 dsobj = dsl_dataset_create_sync(ds->ds_dir, recv_clone_name,
19580676
MA		 1221 snap, crflags, drba->drba_cred, tx);
1222 dsl_dataset_rele(snap, FTAG);
13fe0198
MA		 1223 dsl_dataset_rele(ds, FTAG);
1224 } else {
1225 dsl_dir_t *dd;
1226 const char *tail;
1227 dsl_dataset_t *origin = NULL;
1228
1229 VERIFY0(dsl_dir_hold(dp, tofs, FTAG, &dd, &tail));
1230
1231 if (drba->drba_origin != NULL) {
1232 VERIFY0(dsl_dataset_hold(dp, drba->drba_origin,
1233 FTAG, &origin));
1234 }
1235
1236 /* Create new dataset. */
1237 dsobj = dsl_dataset_create_sync(dd,
1238 strrchr(tofs, '/') + 1,
1239 origin, crflags, drba->drba_cred, tx);
1240 if (origin != NULL)
1241 dsl_dataset_rele(origin, FTAG);
1242 dsl_dir_rele(dd, FTAG);
1243 drba->drba_cookie->drc_newfs = B_TRUE;
1244 }
1245 VERIFY0(dsl_dataset_own_obj(dp, dsobj, dmu_recv_tag, &newds));
1246
1247 dmu_buf_will_dirty(newds->ds_dbuf, tx);
d683ddbb 1248 dsl_dataset_phys(newds)->ds_flags |= DS_FLAG_INCONSISTENT;
34dc7c2f 1249
428870ff
BB		 1250 /*
1251 * If we actually created a non-clone, we need to create the
1252 * objset in our new dataset.
1253 */
13fe0198 1254 if (BP_IS_HOLE(dsl_dataset_get_blkptr(newds))) {
428870ff 1255 (void) dmu_objset_create_impl(dp->dp_spa,
13fe0198 1256 newds, dsl_dataset_get_blkptr(newds), drrb->drr_type, tx);
428870ff 1257 }
34dc7c2f 1258
13fe0198 1259 drba->drba_cookie->drc_ds = newds;
428870ff 1260
13fe0198 1261 spa_history_log_internal_ds(newds, "receive", tx, "");
34dc7c2f
BB		 1262}
1263
34dc7c2f
BB		 1264/*
1265 * NB: callers *MUST* call dmu_recv_stream() if dmu_recv_begin()
1266 * succeeds; otherwise we will leak the holds on the datasets.
1267 */
1268int
13fe0198
MA		 1269dmu_recv_begin(char *tofs, char *tosnap, struct drr_begin *drrb,
1270 boolean_t force, char *origin, dmu_recv_cookie_t *drc)
34dc7c2f 1271{
13fe0198
MA		 1272 dmu_recv_begin_arg_t drba = { 0 };
1273 dmu_replay_record_t *drr;
34dc7c2f
BB		 1274
1275 bzero(drc, sizeof (dmu_recv_cookie_t));
1276 drc->drc_drrb = drrb;
1277 drc->drc_tosnap = tosnap;
13fe0198 1278 drc->drc_tofs = tofs;
34dc7c2f 1279 drc->drc_force = force;
788eb90c 1280 drc->drc_cred = CRED();
34dc7c2f 1281
13fe0198
MA		 1282 if (drrb->drr_magic == BSWAP_64(DMU_BACKUP_MAGIC))
1283 drc->drc_byteswap = B_TRUE;
1284 else if (drrb->drr_magic != DMU_BACKUP_MAGIC)
2e528b49 1285 return (SET_ERROR(EINVAL));
34dc7c2f 1286
13fe0198
MA		 1287 drr = kmem_zalloc(sizeof (dmu_replay_record_t), KM_SLEEP);
1288 drr->drr_type = DRR_BEGIN;
1289 drr->drr_u.drr_begin = *drc->drc_drrb;
1290 if (drc->drc_byteswap) {
1291 fletcher_4_incremental_byteswap(drr,
1292 sizeof (dmu_replay_record_t), &drc->drc_cksum);
1293 } else {
1294 fletcher_4_incremental_native(drr,
1295 sizeof (dmu_replay_record_t), &drc->drc_cksum);
1296 }
1297 kmem_free(drr, sizeof (dmu_replay_record_t));
428870ff 1298
13fe0198
MA		 1299 if (drc->drc_byteswap) {
1300 drrb->drr_magic = BSWAP_64(drrb->drr_magic);
1301 drrb->drr_versioninfo = BSWAP_64(drrb->drr_versioninfo);
1302 drrb->drr_creation_time = BSWAP_64(drrb->drr_creation_time);
1303 drrb->drr_type = BSWAP_32(drrb->drr_type);
1304 drrb->drr_toguid = BSWAP_64(drrb->drr_toguid);
1305 drrb->drr_fromguid = BSWAP_64(drrb->drr_fromguid);
34dc7c2f
BB		 1306 }
1307
13fe0198
MA		 1308 drba.drba_origin = origin;
1309 drba.drba_cookie = drc;
1310 drba.drba_cred = CRED();
1311
1312 return (dsl_sync_task(tofs, dmu_recv_begin_check, dmu_recv_begin_sync,
3d45fdd6 1313 &drba, 5, ZFS_SPACE_CHECK_NORMAL));
34dc7c2f
BB		 1314}
1315
1316struct restorearg {
37f8a883 1317 objset_t *os;
34dc7c2f 1318 int err;
13fe0198 1319 boolean_t byteswap;
34dc7c2f 1320 vnode_t *vp;
34dc7c2f
BB		 1321 uint64_t voff;
1322 int bufsize; /* amount of memory allocated for buf */
37f8a883
MA		 1323
1324 dmu_replay_record_t *drr;
1325 dmu_replay_record_t *next_drr;
1326 char *buf;
34dc7c2f 1327 zio_cksum_t cksum;
37f8a883
MA		 1328 zio_cksum_t prev_cksum;
1329
572e2857 1330 avl_tree_t *guid_to_ds_map;
34dc7c2f
BB		 1331};
1332
428870ff
BB		 1333typedef struct guid_map_entry {
1334 uint64_t guid;
1335 dsl_dataset_t *gme_ds;
1336 avl_node_t avlnode;
1337} guid_map_entry_t;
1338
1339static int
1340guid_compare(const void *arg1, const void *arg2)
1341{
1342 const guid_map_entry_t *gmep1 = arg1;
1343 const guid_map_entry_t *gmep2 = arg2;
1344
1345 if (gmep1->guid < gmep2->guid)
1346 return (-1);
1347 else if (gmep1->guid > gmep2->guid)
1348 return (1);
1349 return (0);
1350}
1351
572e2857
BB		 1352static void
1353free_guid_map_onexit(void *arg)
1354{
1355 avl_tree_t *ca = arg;
1356 void *cookie = NULL;
1357 guid_map_entry_t *gmep;
1358
1359 while ((gmep = avl_destroy_nodes(ca, &cookie)) != NULL) {
13fe0198 1360 dsl_dataset_long_rele(gmep->gme_ds, gmep);
7ec09286 1361 dsl_dataset_rele(gmep->gme_ds, gmep);
572e2857
BB		 1362 kmem_free(gmep, sizeof (guid_map_entry_t));
1363 }
1364 avl_destroy(ca);
1365 kmem_free(ca, sizeof (avl_tree_t));
1366}
1367
37f8a883
MA		 1368static int
1369restore_read(struct restorearg *ra, int len, void *buf)
34dc7c2f 1370{
34dc7c2f
BB		 1371 int done = 0;
1372
1373 /* some things will require 8-byte alignment, so everything must */
c99c9001 1374 ASSERT0(len % 8);
f1512ee6 1375 ASSERT3U(len, <=, ra->bufsize);
34dc7c2f
BB		 1376
1377 while (done < len) {
1378 ssize_t resid;
1379
1380 ra->err = vn_rdwr(UIO_READ, ra->vp,
37f8a883 1381 (char *)buf + done, len - done,
34dc7c2f
BB		 1382 ra->voff, UIO_SYSSPACE, FAPPEND,
1383 RLIM64_INFINITY, CRED(), &resid);
1384
1385 if (resid == len - done)
2e528b49 1386 ra->err = SET_ERROR(EINVAL);
34dc7c2f
BB		 1387 ra->voff += len - done - resid;
1388 done = len - resid;
13fe0198 1389 if (ra->err != 0)
37f8a883 1390 return (ra->err);
34dc7c2f
BB		 1391 }
1392
1393 ASSERT3U(done, ==, len);
37f8a883 1394 return (0);
34dc7c2f
BB		 1395}
1396
60948de1 1397noinline static void
37f8a883 1398byteswap_record(dmu_replay_record_t *drr)
34dc7c2f
BB		 1399{
1400#define DO64(X) (drr->drr_u.X = BSWAP_64(drr->drr_u.X))
1401#define DO32(X) (drr->drr_u.X = BSWAP_32(drr->drr_u.X))
1402 drr->drr_type = BSWAP_32(drr->drr_type);
1403 drr->drr_payloadlen = BSWAP_32(drr->drr_payloadlen);
37f8a883 1404
34dc7c2f
BB		 1405 switch (drr->drr_type) {
1406 case DRR_BEGIN:
1407 DO64(drr_begin.drr_magic);
428870ff 1408 DO64(drr_begin.drr_versioninfo);
34dc7c2f
BB		 1409 DO64(drr_begin.drr_creation_time);
1410 DO32(drr_begin.drr_type);
1411 DO32(drr_begin.drr_flags);
1412 DO64(drr_begin.drr_toguid);
1413 DO64(drr_begin.drr_fromguid);
1414 break;
1415 case DRR_OBJECT:
1416 DO64(drr_object.drr_object);
34dc7c2f
BB		 1417 DO32(drr_object.drr_type);
1418 DO32(drr_object.drr_bonustype);
1419 DO32(drr_object.drr_blksz);
1420 DO32(drr_object.drr_bonuslen);
428870ff 1421 DO64(drr_object.drr_toguid);
34dc7c2f
BB		 1422 break;
1423 case DRR_FREEOBJECTS:
1424 DO64(drr_freeobjects.drr_firstobj);
1425 DO64(drr_freeobjects.drr_numobjs);
428870ff 1426 DO64(drr_freeobjects.drr_toguid);
34dc7c2f
BB		 1427 break;
1428 case DRR_WRITE:
1429 DO64(drr_write.drr_object);
1430 DO32(drr_write.drr_type);
1431 DO64(drr_write.drr_offset);
1432 DO64(drr_write.drr_length);
428870ff 1433 DO64(drr_write.drr_toguid);
37f8a883 1434 ZIO_CHECKSUM_BSWAP(&drr->drr_u.drr_write.drr_key.ddk_cksum);
428870ff
BB		 1435 DO64(drr_write.drr_key.ddk_prop);
1436 break;
1437 case DRR_WRITE_BYREF:
1438 DO64(drr_write_byref.drr_object);
1439 DO64(drr_write_byref.drr_offset);
1440 DO64(drr_write_byref.drr_length);
1441 DO64(drr_write_byref.drr_toguid);
1442 DO64(drr_write_byref.drr_refguid);
1443 DO64(drr_write_byref.drr_refobject);
1444 DO64(drr_write_byref.drr_refoffset);
37f8a883
MA		 1445 ZIO_CHECKSUM_BSWAP(&drr->drr_u.drr_write_byref.
1446 drr_key.ddk_cksum);
428870ff 1447 DO64(drr_write_byref.drr_key.ddk_prop);
34dc7c2f 1448 break;
9b67f605
MA		 1449 case DRR_WRITE_EMBEDDED:
1450 DO64(drr_write_embedded.drr_object);
1451 DO64(drr_write_embedded.drr_offset);
1452 DO64(drr_write_embedded.drr_length);
1453 DO64(drr_write_embedded.drr_toguid);
1454 DO32(drr_write_embedded.drr_lsize);
1455 DO32(drr_write_embedded.drr_psize);
1456 break;
34dc7c2f
BB		 1457 case DRR_FREE:
1458 DO64(drr_free.drr_object);
1459 DO64(drr_free.drr_offset);
1460 DO64(drr_free.drr_length);
428870ff
BB		 1461 DO64(drr_free.drr_toguid);
1462 break;
1463 case DRR_SPILL:
1464 DO64(drr_spill.drr_object);
1465 DO64(drr_spill.drr_length);
1466 DO64(drr_spill.drr_toguid);
34dc7c2f
BB		 1467 break;
1468 case DRR_END:
428870ff 1469 DO64(drr_end.drr_toguid);
37f8a883 1470 ZIO_CHECKSUM_BSWAP(&drr->drr_u.drr_end.drr_checksum);
34dc7c2f 1471 break;
e75c13c3
BB		 1472 default:
1473 break;
34dc7c2f 1474 }
37f8a883
MA		 1475
1476 if (drr->drr_type != DRR_BEGIN) {
1477 ZIO_CHECKSUM_BSWAP(&drr->drr_u.drr_checksum.drr_checksum);
1478 }
1479
34dc7c2f
BB		 1480#undef DO64
1481#undef DO32
1482}
1483
6c59307a
MA		 1484static inline uint8_t
1485deduce_nblkptr(dmu_object_type_t bonus_type, uint64_t bonus_size)
1486{
1487 if (bonus_type == DMU_OT_SA) {
1488 return (1);
1489 } else {
1490 return (1 +
1491 ((DN_MAX_BONUSLEN - bonus_size) >> SPA_BLKPTRSHIFT));
1492 }
1493}
1494
60948de1 1495noinline static int
37f8a883 1496restore_object(struct restorearg *ra, struct drr_object *drro, void *data)
34dc7c2f 1497{
6c59307a 1498 dmu_object_info_t doi;
34dc7c2f 1499 dmu_tx_t *tx;
6c59307a
MA		 1500 uint64_t object;
1501 int err;
34dc7c2f 1502
34dc7c2f 1503 if (drro->drr_type == DMU_OT_NONE ||
9ae529ec
CS		 1504 !DMU_OT_IS_VALID(drro->drr_type) ||
1505 !DMU_OT_IS_VALID(drro->drr_bonustype) ||
428870ff 1506 drro->drr_checksumtype >= ZIO_CHECKSUM_FUNCTIONS ||
34dc7c2f
BB		 1507 drro->drr_compress >= ZIO_COMPRESS_FUNCTIONS ||
1508 P2PHASE(drro->drr_blksz, SPA_MINBLOCKSIZE) ||
1509 drro->drr_blksz < SPA_MINBLOCKSIZE ||
37f8a883 1510 drro->drr_blksz > spa_maxblocksize(dmu_objset_spa(ra->os)) ||
34dc7c2f 1511 drro->drr_bonuslen > DN_MAX_BONUSLEN) {
2e528b49 1512 return (SET_ERROR(EINVAL));
34dc7c2f
BB		 1513 }
1514
37f8a883 1515 err = dmu_object_info(ra->os, drro->drr_object, &doi);
9babb374
BB		 1516
1517 if (err != 0 && err != ENOENT)
2e528b49 1518 return (SET_ERROR(EINVAL));
6c59307a 1519 object = err == 0 ? drro->drr_object : DMU_NEW_OBJECT;
9babb374 1520
6c59307a
MA		 1521 /*
1522 * If we are losing blkptrs or changing the block size this must
1523 * be a new file instance. We must clear out the previous file
1524 * contents before we can change this type of metadata in the dnode.
1525 */
1526 if (err == 0) {
1527 int nblkptr;
1528
1529 nblkptr = deduce_nblkptr(drro->drr_bonustype,
1530 drro->drr_bonuslen);
1531
1532 if (drro->drr_blksz != doi.doi_data_block_size ||
1533 nblkptr < doi.doi_nblkptr) {
37f8a883 1534 err = dmu_free_long_range(ra->os, drro->drr_object,
6c59307a
MA		 1535 0, DMU_OBJECT_END);
1536 if (err != 0)
1537 return (SET_ERROR(EINVAL));
34dc7c2f 1538 }
6c59307a
MA		 1539 }
1540
37f8a883 1541 tx = dmu_tx_create(ra->os);
6c59307a
MA		 1542 dmu_tx_hold_bonus(tx, object);
1543 err = dmu_tx_assign(tx, TXG_WAIT);
1544 if (err != 0) {
1545 dmu_tx_abort(tx);
1546 return (err);
1547 }
1548
1549 if (object == DMU_NEW_OBJECT) {
1550 /* currently free, want to be allocated */
37f8a883 1551 err = dmu_object_claim(ra->os, drro->drr_object,
34dc7c2f
BB		 1552 drro->drr_type, drro->drr_blksz,
1553 drro->drr_bonustype, drro->drr_bonuslen, tx);
6c59307a
MA		 1554 } else if (drro->drr_type != doi.doi_type ||
1555 drro->drr_blksz != doi.doi_data_block_size ||
1556 drro->drr_bonustype != doi.doi_bonus_type ||
1557 drro->drr_bonuslen != doi.doi_bonus_size) {
1558 /* currently allocated, but with different properties */
37f8a883 1559 err = dmu_object_reclaim(ra->os, drro->drr_object,
34dc7c2f 1560 drro->drr_type, drro->drr_blksz,
6c59307a 1561 drro->drr_bonustype, drro->drr_bonuslen, tx);
34dc7c2f 1562 }
13fe0198 1563 if (err != 0) {
6c59307a 1564 dmu_tx_commit(tx);
2e528b49 1565 return (SET_ERROR(EINVAL));
428870ff 1566 }
9babb374 1567
37f8a883
MA		 1568 dmu_object_set_checksum(ra->os, drro->drr_object,
1569 drro->drr_checksumtype, tx);
1570 dmu_object_set_compress(ra->os, drro->drr_object,
1571 drro->drr_compress, tx);
34dc7c2f 1572
b128c09f 1573 if (data != NULL) {
34dc7c2f 1574 dmu_buf_t *db;
b128c09f 1575
37f8a883 1576 VERIFY0(dmu_bonus_hold(ra->os, drro->drr_object, FTAG, &db));
34dc7c2f
BB		 1577 dmu_buf_will_dirty(db, tx);
1578
1579 ASSERT3U(db->db_size, >=, drro->drr_bonuslen);
34dc7c2f
BB		 1580 bcopy(data, db->db_data, drro->drr_bonuslen);
1581 if (ra->byteswap) {
9ae529ec
CS		 1582 dmu_object_byteswap_t byteswap =
1583 DMU_OT_BYTESWAP(drro->drr_bonustype);
1584 dmu_ot_byteswap[byteswap].ob_func(db->db_data,
34dc7c2f
BB		 1585 drro->drr_bonuslen);
1586 }
1587 dmu_buf_rele(db, FTAG);
1588 }
1589 dmu_tx_commit(tx);
1590 return (0);
1591}
1592
1593/* ARGSUSED */
60948de1 1594noinline static int
37f8a883 1595restore_freeobjects(struct restorearg *ra,
34dc7c2f
BB		 1596 struct drr_freeobjects *drrfo)
1597{
1598 uint64_t obj;
1599
1600 if (drrfo->drr_firstobj + drrfo->drr_numobjs < drrfo->drr_firstobj)
2e528b49 1601 return (SET_ERROR(EINVAL));
34dc7c2f
BB		 1602
1603 for (obj = drrfo->drr_firstobj;
1604 obj < drrfo->drr_firstobj + drrfo->drr_numobjs;
37f8a883 1605 (void) dmu_object_next(ra->os, &obj, FALSE, 0)) {
34dc7c2f
BB		 1606 int err;
1607
37f8a883 1608 if (dmu_object_info(ra->os, obj, NULL) != 0)
34dc7c2f
BB		 1609 continue;
1610
37f8a883 1611 err = dmu_free_long_object(ra->os, obj);
13fe0198 1612 if (err != 0)
34dc7c2f 1613 return (err);
34dc7c2f
BB		 1614 }
1615 return (0);
1616}
1617
60948de1 1618noinline static int
37f8a883 1619restore_write(struct restorearg *ra, struct drr_write *drrw, arc_buf_t *abuf)
34dc7c2f
BB		 1620{
1621 dmu_tx_t *tx;
88904bb3 1622 dmu_buf_t *bonus;
34dc7c2f
BB		 1623 int err;
1624
1625 if (drrw->drr_offset + drrw->drr_length < drrw->drr_offset ||
9ae529ec 1626 !DMU_OT_IS_VALID(drrw->drr_type))
2e528b49 1627 return (SET_ERROR(EINVAL));
34dc7c2f 1628
37f8a883 1629 if (dmu_object_info(ra->os, drrw->drr_object, NULL) != 0)
88904bb3
MA		 1630 return (SET_ERROR(EINVAL));
1631
37f8a883 1632 tx = dmu_tx_create(ra->os);
34dc7c2f
BB		 1633
1634 dmu_tx_hold_write(tx, drrw->drr_object,
1635 drrw->drr_offset, drrw->drr_length);
1636 err = dmu_tx_assign(tx, TXG_WAIT);
13fe0198 1637 if (err != 0) {
34dc7c2f
BB		 1638 dmu_tx_abort(tx);
1639 return (err);
1640 }
9ae529ec
CS		 1641 if (ra->byteswap) {
1642 dmu_object_byteswap_t byteswap =
1643 DMU_OT_BYTESWAP(drrw->drr_type);
37f8a883
MA		 1644 dmu_ot_byteswap[byteswap].ob_func(abuf->b_data,
1645 drrw->drr_length);
9ae529ec 1646 }
37f8a883
MA		 1647
1648 if (dmu_bonus_hold(ra->os, drrw->drr_object, FTAG, &bonus) != 0)
1649 return (SET_ERROR(EINVAL));
88904bb3 1650 dmu_assign_arcbuf(bonus, drrw->drr_offset, abuf, tx);
34dc7c2f 1651 dmu_tx_commit(tx);
88904bb3 1652 dmu_buf_rele(bonus, FTAG);
34dc7c2f
BB		 1653 return (0);
1654}
1655
428870ff
BB		 1656/*
1657 * Handle a DRR_WRITE_BYREF record. This record is used in dedup'ed
1658 * streams to refer to a copy of the data that is already on the
1659 * system because it came in earlier in the stream. This function
1660 * finds the earlier copy of the data, and uses that copy instead of
1661 * data from the stream to fulfill this write.
1662 */
1663static int
37f8a883 1664restore_write_byref(struct restorearg *ra, struct drr_write_byref *drrwbr)
428870ff
BB		 1665{
1666 dmu_tx_t *tx;
1667 int err;
1668 guid_map_entry_t gmesrch;
1669 guid_map_entry_t *gmep;
9b67f605 1670 avl_index_t where;
428870ff
BB		 1671 objset_t *ref_os = NULL;
1672 dmu_buf_t *dbp;
1673
1674 if (drrwbr->drr_offset + drrwbr->drr_length < drrwbr->drr_offset)
2e528b49 1675 return (SET_ERROR(EINVAL));
428870ff
BB		 1676
1677 /*
1678 * If the GUID of the referenced dataset is different from the
1679 * GUID of the target dataset, find the referenced dataset.
1680 */
1681 if (drrwbr->drr_toguid != drrwbr->drr_refguid) {
1682 gmesrch.guid = drrwbr->drr_refguid;
572e2857 1683 if ((gmep = avl_find(ra->guid_to_ds_map, &gmesrch,
428870ff 1684 &where)) == NULL) {
2e528b49 1685 return (SET_ERROR(EINVAL));
428870ff
BB		 1686 }
1687 if (dmu_objset_from_ds(gmep->gme_ds, &ref_os))
2e528b49 1688 return (SET_ERROR(EINVAL));
428870ff 1689 } else {
37f8a883 1690 ref_os = ra->os;
428870ff
BB		 1691 }
1692
c65aa5b2
BB		 1693 err = dmu_buf_hold(ref_os, drrwbr->drr_refobject,
1694 drrwbr->drr_refoffset, FTAG, &dbp, DMU_READ_PREFETCH);
9b67f605 1695 if (err != 0)
428870ff
BB		 1696 return (err);
1697
37f8a883 1698 tx = dmu_tx_create(ra->os);
428870ff
BB		 1699
1700 dmu_tx_hold_write(tx, drrwbr->drr_object,
1701 drrwbr->drr_offset, drrwbr->drr_length);
1702 err = dmu_tx_assign(tx, TXG_WAIT);
13fe0198 1703 if (err != 0) {
428870ff
BB		 1704 dmu_tx_abort(tx);
1705 return (err);
1706 }
37f8a883 1707 dmu_write(ra->os, drrwbr->drr_object,
428870ff
BB		 1708 drrwbr->drr_offset, drrwbr->drr_length, dbp->db_data, tx);
1709 dmu_buf_rele(dbp, FTAG);
1710 dmu_tx_commit(tx);
1711 return (0);
1712}
1713
9b67f605 1714static int
37f8a883
MA		 1715restore_write_embedded(struct restorearg *ra,
1716 struct drr_write_embedded *drrwnp, void *data)
9b67f605
MA		 1717{
1718 dmu_tx_t *tx;
1719 int err;
9b67f605
MA		 1720
1721 if (drrwnp->drr_offset + drrwnp->drr_length < drrwnp->drr_offset)
1722 return (EINVAL);
1723
1724 if (drrwnp->drr_psize > BPE_PAYLOAD_SIZE)
1725 return (EINVAL);
1726
1727 if (drrwnp->drr_etype >= NUM_BP_EMBEDDED_TYPES)
1728 return (EINVAL);
1729 if (drrwnp->drr_compression >= ZIO_COMPRESS_FUNCTIONS)
1730 return (EINVAL);
1731
37f8a883 1732 tx = dmu_tx_create(ra->os);
9b67f605
MA		 1733
1734 dmu_tx_hold_write(tx, drrwnp->drr_object,
1735 drrwnp->drr_offset, drrwnp->drr_length);
1736 err = dmu_tx_assign(tx, TXG_WAIT);
1737 if (err != 0) {
1738 dmu_tx_abort(tx);
1739 return (err);
1740 }
1741
37f8a883 1742 dmu_write_embedded(ra->os, drrwnp->drr_object,
9b67f605
MA		 1743 drrwnp->drr_offset, data, drrwnp->drr_etype,
1744 drrwnp->drr_compression, drrwnp->drr_lsize, drrwnp->drr_psize,
1745 ra->byteswap ^ ZFS_HOST_BYTEORDER, tx);
1746
1747 dmu_tx_commit(tx);
1748 return (0);
1749}
1750
428870ff 1751static int
37f8a883 1752restore_spill(struct restorearg *ra, struct drr_spill *drrs, void *data)
428870ff
BB		 1753{
1754 dmu_tx_t *tx;
428870ff
BB		 1755 dmu_buf_t *db, *db_spill;
1756 int err;
1757
1758 if (drrs->drr_length < SPA_MINBLOCKSIZE ||
37f8a883 1759 drrs->drr_length > spa_maxblocksize(dmu_objset_spa(ra->os)))
2e528b49 1760 return (SET_ERROR(EINVAL));
428870ff 1761
37f8a883 1762 if (dmu_object_info(ra->os, drrs->drr_object, NULL) != 0)
2e528b49 1763 return (SET_ERROR(EINVAL));
428870ff 1764
37f8a883 1765 VERIFY0(dmu_bonus_hold(ra->os, drrs->drr_object, FTAG, &db));
428870ff
BB		 1766 if ((err = dmu_spill_hold_by_bonus(db, FTAG, &db_spill)) != 0) {
1767 dmu_buf_rele(db, FTAG);
1768 return (err);
1769 }
1770
37f8a883 1771 tx = dmu_tx_create(ra->os);
428870ff
BB		 1772
1773 dmu_tx_hold_spill(tx, db->db_object);
1774
1775 err = dmu_tx_assign(tx, TXG_WAIT);
13fe0198 1776 if (err != 0) {
428870ff
BB		 1777 dmu_buf_rele(db, FTAG);
1778 dmu_buf_rele(db_spill, FTAG);
1779 dmu_tx_abort(tx);
1780 return (err);
1781 }
1782 dmu_buf_will_dirty(db_spill, tx);
1783
1784 if (db_spill->db_size < drrs->drr_length)
1785 VERIFY(0 == dbuf_spill_set_blksz(db_spill,
1786 drrs->drr_length, tx));
1787 bcopy(data, db_spill->db_data, drrs->drr_length);
1788
1789 dmu_buf_rele(db, FTAG);
1790 dmu_buf_rele(db_spill, FTAG);
1791
1792 dmu_tx_commit(tx);
1793 return (0);
1794}
1795
34dc7c2f 1796/* ARGSUSED */
60948de1 1797noinline static int
37f8a883 1798restore_free(struct restorearg *ra, struct drr_free *drrf)
34dc7c2f 1799{
34dc7c2f
BB		 1800 int err;
1801
1802 if (drrf->drr_length != -1ULL &&
1803 drrf->drr_offset + drrf->drr_length < drrf->drr_offset)
2e528b49 1804 return (SET_ERROR(EINVAL));
34dc7c2f 1805
37f8a883 1806 if (dmu_object_info(ra->os, drrf->drr_object, NULL) != 0)
2e528b49 1807 return (SET_ERROR(EINVAL));
34dc7c2f 1808
37f8a883 1809 err = dmu_free_long_range(ra->os, drrf->drr_object,
34dc7c2f 1810 drrf->drr_offset, drrf->drr_length);
34dc7c2f
BB		 1811 return (err);
1812}
1813
13fe0198
MA		 1814/* used to destroy the drc_ds on error */
1815static void
1816dmu_recv_cleanup_ds(dmu_recv_cookie_t *drc)
1817{
1818 char name[MAXNAMELEN];
1819 dsl_dataset_name(drc->drc_ds, name);
1820 dsl_dataset_disown(drc->drc_ds, dmu_recv_tag);
1821 (void) dsl_destroy_head(name);
1822}
1823
37f8a883
MA		 1824static void
1825restore_cksum(struct restorearg *ra, int len, void *buf)
1826{
1827 if (ra->byteswap) {
1828 fletcher_4_incremental_byteswap(buf, len, &ra->cksum);
1829 } else {
1830 fletcher_4_incremental_native(buf, len, &ra->cksum);
1831 }
1832}
1833
1834/*
1835 * If len != 0, read payload into buf.
1836 * Read next record's header into ra->next_drr.
1837 * Verify checksum of payload and next record.
1838 */
1839static int
1840restore_read_payload_and_next_header(struct restorearg *ra, int len, void *buf)
1841{
1842 int err;
1843 zio_cksum_t cksum_orig;
1844 zio_cksum_t *cksump;
1845
1846 if (len != 0) {
1847 ASSERT3U(len, <=, ra->bufsize);
1848 err = restore_read(ra, len, buf);
1849 if (err != 0)
1850 return (err);
1851 restore_cksum(ra, len, buf);
1852 }
1853
1854 ra->prev_cksum = ra->cksum;
1855
1856 err = restore_read(ra, sizeof (*ra->next_drr), ra->next_drr);
1857 if (err != 0)
1858 return (err);
1859 if (ra->next_drr->drr_type == DRR_BEGIN)
1860 return (SET_ERROR(EINVAL));
1861
1862 /*
1863 * Note: checksum is of everything up to but not including the
1864 * checksum itself.
1865 */
1866 ASSERT3U(offsetof(dmu_replay_record_t, drr_u.drr_checksum.drr_checksum),
1867 ==, sizeof (dmu_replay_record_t) - sizeof (zio_cksum_t));
1868 restore_cksum(ra,
1869 offsetof(dmu_replay_record_t, drr_u.drr_checksum.drr_checksum),
1870 ra->next_drr);
1871
1872 cksum_orig = ra->next_drr->drr_u.drr_checksum.drr_checksum;
1873 cksump = &ra->next_drr->drr_u.drr_checksum.drr_checksum;
1874
1875 if (ra->byteswap)
1876 byteswap_record(ra->next_drr);
1877
1878 if ((!ZIO_CHECKSUM_IS_ZERO(cksump)) &&
1879 !ZIO_CHECKSUM_EQUAL(ra->cksum, *cksump))
1880 return (SET_ERROR(ECKSUM));
1881
1882 restore_cksum(ra, sizeof (cksum_orig), &cksum_orig);
1883
1884 return (0);
1885}
1886
1887static int
1888restore_process_record(struct restorearg *ra)
1889{
1890 int err;
1891
1892 switch (ra->drr->drr_type) {
1893 case DRR_OBJECT:
1894 {
1895 struct drr_object *drro = &ra->drr->drr_u.drr_object;
1896 err = restore_read_payload_and_next_header(ra,
1897 P2ROUNDUP(drro->drr_bonuslen, 8), ra->buf);
1898 if (err != 0)
1899 return (err);
1900 return (restore_object(ra, drro, ra->buf));
1901 }
1902 case DRR_FREEOBJECTS:
1903 {
1904 struct drr_freeobjects *drrfo =
1905 &ra->drr->drr_u.drr_freeobjects;
1906 err = restore_read_payload_and_next_header(ra, 0, NULL);
1907 if (err != 0)
1908 return (err);
1909 return (restore_freeobjects(ra, drrfo));
1910 }
1911 case DRR_WRITE:
1912 {
1913 struct drr_write *drrw = &ra->drr->drr_u.drr_write;
1914 arc_buf_t *abuf = arc_loan_buf(dmu_objset_spa(ra->os),
1915 drrw->drr_length);
1916
1917 err = restore_read_payload_and_next_header(ra,
1918 drrw->drr_length, abuf->b_data);
1919 if (err != 0)
1920 return (err);
1921 err = restore_write(ra, drrw, abuf);
1922 /* if restore_write() is successful, it consumes the arc_buf */
1923 if (err != 0)
1924 dmu_return_arcbuf(abuf);
1925 return (err);
1926 }
1927 case DRR_WRITE_BYREF:
1928 {
1929 struct drr_write_byref *drrwbr =
1930 &ra->drr->drr_u.drr_write_byref;
1931 err = restore_read_payload_and_next_header(ra, 0, NULL);
1932 if (err != 0)
1933 return (err);
1934 return (restore_write_byref(ra, drrwbr));
1935 }
1936 case DRR_WRITE_EMBEDDED:
1937 {
1938 struct drr_write_embedded *drrwe =
1939 &ra->drr->drr_u.drr_write_embedded;
1940 err = restore_read_payload_and_next_header(ra,
1941 P2ROUNDUP(drrwe->drr_psize, 8), ra->buf);
1942 if (err != 0)
1943 return (err);
1944 return (restore_write_embedded(ra, drrwe, ra->buf));
1945 }
1946 case DRR_FREE:
1947 {
1948 struct drr_free *drrf = &ra->drr->drr_u.drr_free;
1949 err = restore_read_payload_and_next_header(ra, 0, NULL);
1950 if (err != 0)
1951 return (err);
1952 return (restore_free(ra, drrf));
1953 }
1954 case DRR_END:
1955 {
1956 struct drr_end *drre = &ra->drr->drr_u.drr_end;
1957 if (!ZIO_CHECKSUM_EQUAL(ra->prev_cksum, drre->drr_checksum))
1958 return (SET_ERROR(EINVAL));
1959 return (0);
1960 }
1961 case DRR_SPILL:
1962 {
1963 struct drr_spill *drrs = &ra->drr->drr_u.drr_spill;
1964 err = restore_read_payload_and_next_header(ra,
1965 drrs->drr_length, ra->buf);
1966 if (err != 0)
1967 return (err);
1968 return (restore_spill(ra, drrs, ra->buf));
1969 }
1970 default:
1971 return (SET_ERROR(EINVAL));
1972 }
1973}
1974
34dc7c2f
BB		 1975/*
1976 * NB: callers *must* call dmu_recv_end() if this succeeds.
1977 */
1978int
572e2857
BB		 1979dmu_recv_stream(dmu_recv_cookie_t *drc, vnode_t *vp, offset_t *voffp,
1980 int cleanup_fd, uint64_t *action_handlep)
34dc7c2f 1981{
37f8a883 1982 int err = 0;
34dc7c2f 1983 struct restorearg ra = { 0 };
428870ff 1984 int featureflags;
34dc7c2f 1985
13fe0198
MA		 1986 ra.byteswap = drc->drc_byteswap;
1987 ra.cksum = drc->drc_cksum;
34dc7c2f
BB		 1988 ra.vp = vp;
1989 ra.voff = *voffp;
f1512ee6 1990 ra.bufsize = SPA_MAXBLOCKSIZE;
37f8a883 1991 ra.drr = kmem_alloc(sizeof (*ra.drr), KM_SLEEP);
00b46022 1992 ra.buf = vmem_alloc(ra.bufsize, KM_SLEEP);
37f8a883 1993 ra.next_drr = kmem_alloc(sizeof (*ra.next_drr), KM_SLEEP);
34dc7c2f
BB		 1994
1995 /* these were verified in dmu_recv_begin */
13fe0198 1996 ASSERT3U(DMU_GET_STREAM_HDRTYPE(drc->drc_drrb->drr_versioninfo), ==,
428870ff 1997 DMU_SUBSTREAM);
13fe0198 1998 ASSERT3U(drc->drc_drrb->drr_type, <, DMU_OST_NUMTYPES);
34dc7c2f
BB		 1999
2000 /*
2001 * Open the objset we are modifying.
2002 */
37f8a883 2003 VERIFY0(dmu_objset_from_ds(drc->drc_ds, &ra.os));
34dc7c2f 2004
d683ddbb 2005 ASSERT(dsl_dataset_phys(drc->drc_ds)->ds_flags & DS_FLAG_INCONSISTENT);
34dc7c2f 2006
428870ff
BB		 2007 featureflags = DMU_GET_FEATUREFLAGS(drc->drc_drrb->drr_versioninfo);
2008
2009 /* if this stream is dedup'ed, set up the avl tree for guid mapping */
2010 if (featureflags & DMU_BACKUP_FEATURE_DEDUP) {
572e2857
BB		 2011 minor_t minor;
2012
2013 if (cleanup_fd == -1) {
2e528b49 2014 ra.err = SET_ERROR(EBADF);
572e2857
BB		 2015 goto out;
2016 }
2017 ra.err = zfs_onexit_fd_hold(cleanup_fd, &minor);
13fe0198 2018 if (ra.err != 0) {
572e2857
BB		 2019 cleanup_fd = -1;
2020 goto out;
2021 }
2022
2023 if (*action_handlep == 0) {
2024 ra.guid_to_ds_map =
2025 kmem_alloc(sizeof (avl_tree_t), KM_SLEEP);
2026 avl_create(ra.guid_to_ds_map, guid_compare,
2027 sizeof (guid_map_entry_t),
2028 offsetof(guid_map_entry_t, avlnode));
37f8a883 2029 err = zfs_onexit_add_cb(minor,
572e2857
BB		 2030 free_guid_map_onexit, ra.guid_to_ds_map,
2031 action_handlep);
13fe0198 2032 if (ra.err != 0)
572e2857
BB		 2033 goto out;
2034 } else {
37f8a883 2035 err = zfs_onexit_cb_data(minor, *action_handlep,
572e2857 2036 (void **)&ra.guid_to_ds_map);
13fe0198 2037 if (ra.err != 0)
572e2857
BB		 2038 goto out;
2039 }
8d35c149
AS		 2040
2041 drc->drc_guid_to_ds_map = ra.guid_to_ds_map;
428870ff
BB		 2042 }
2043
37f8a883
MA		 2044 err = restore_read_payload_and_next_header(&ra, 0, NULL);
2045 if (err != 0)
2046 goto out;
2047 for (;;) {
2048 void *tmp;
2049
34dc7c2f 2050 if (issig(JUSTLOOKING) && issig(FORREAL)) {
37f8a883
MA		 2051 err = SET_ERROR(EINTR);
2052 break;
34dc7c2f
BB		 2053 }
2054
37f8a883
MA		 2055 tmp = ra.next_drr;
2056 ra.next_drr = ra.drr;
2057 ra.drr = tmp;
34dc7c2f 2058
37f8a883
MA		 2059 /* process ra.drr, read in ra.next_drr */
2060 err = restore_process_record(&ra);
2061 if (err != 0)
34dc7c2f 2062 break;
37f8a883 2063 if (ra.drr->drr_type == DRR_END)
428870ff 2064 break;
34dc7c2f 2065 }
34dc7c2f
BB		 2066
2067out:
572e2857
BB		 2068 if ((featureflags & DMU_BACKUP_FEATURE_DEDUP) && (cleanup_fd != -1))
2069 zfs_onexit_fd_rele(cleanup_fd);
2070
37f8a883 2071 if (err != 0) {
34dc7c2f 2072 /*
45d1cae3
BB		 2073 * destroy what we created, so we don't leave it in the
2074 * inconsistent restoring state.
34dc7c2f 2075 */
13fe0198 2076 dmu_recv_cleanup_ds(drc);
34dc7c2f
BB		 2077 }
2078
37f8a883 2079 kmem_free(ra.drr, sizeof (*ra.drr));
00b46022 2080 vmem_free(ra.buf, ra.bufsize);
37f8a883 2081 kmem_free(ra.next_drr, sizeof (*ra.next_drr));
34dc7c2f 2082 *voffp = ra.voff;
37f8a883 2083 return (err);
34dc7c2f
BB		 2084}
2085
34dc7c2f 2086static int
13fe0198 2087dmu_recv_end_check(void *arg, dmu_tx_t *tx)
34dc7c2f 2088{
13fe0198
MA		 2089 dmu_recv_cookie_t *drc = arg;
2090 dsl_pool_t *dp = dmu_tx_pool(tx);
2091 int error;
2092
2093 ASSERT3P(drc->drc_ds->ds_owner, ==, dmu_recv_tag);
2094
2095 if (!drc->drc_newfs) {
2096 dsl_dataset_t *origin_head;
2097
2098 error = dsl_dataset_hold(dp, drc->drc_tofs, FTAG, &origin_head);
2099 if (error != 0)
2100 return (error);
19580676
MA		 2101 if (drc->drc_force) {
2102 /*
2103 * We will destroy any snapshots in tofs (i.e. before
2104 * origin_head) that are after the origin (which is
2105 * the snap before drc_ds, because drc_ds can not
2106 * have any snaps of its own).
2107 */
d683ddbb
JG		 2108 uint64_t obj;
2109
2110 obj = dsl_dataset_phys(origin_head)->ds_prev_snap_obj;
2111 while (obj !=
2112 dsl_dataset_phys(drc->drc_ds)->ds_prev_snap_obj) {
19580676
MA		 2113 dsl_dataset_t *snap;
2114 error = dsl_dataset_hold_obj(dp, obj, FTAG,
2115 &snap);
2116 if (error != 0)
b6640117 2117 break;
19580676
MA		 2118 if (snap->ds_dir != origin_head->ds_dir)
2119 error = SET_ERROR(EINVAL);
2120 if (error == 0) {
2121 error = dsl_destroy_snapshot_check_impl(
2122 snap, B_FALSE);
2123 }
d683ddbb 2124 obj = dsl_dataset_phys(snap)->ds_prev_snap_obj;
19580676
MA		 2125 dsl_dataset_rele(snap, FTAG);
2126 if (error != 0)
b6640117
AG		 2127 break;
2128 }
2129 if (error != 0) {
2130 dsl_dataset_rele(origin_head, FTAG);
2131 return (error);
19580676
MA		 2132 }
2133 }
13fe0198 2134 error = dsl_dataset_clone_swap_check_impl(drc->drc_ds,
831baf06 2135 origin_head, drc->drc_force, drc->drc_owner, tx);
13fe0198
MA		 2136 if (error != 0) {
2137 dsl_dataset_rele(origin_head, FTAG);
2138 return (error);
2139 }
2140 error = dsl_dataset_snapshot_check_impl(origin_head,
788eb90c 2141 drc->drc_tosnap, tx, B_TRUE, 1, drc->drc_cred);
13fe0198
MA		 2142 dsl_dataset_rele(origin_head, FTAG);
2143 if (error != 0)
2144 return (error);
34dc7c2f 2145
13fe0198
MA		 2146 error = dsl_destroy_head_check_impl(drc->drc_ds, 1);
2147 } else {
2148 error = dsl_dataset_snapshot_check_impl(drc->drc_ds,
788eb90c 2149 drc->drc_tosnap, tx, B_TRUE, 1, drc->drc_cred);
13fe0198
MA		 2150 }
2151 return (error);
34dc7c2f
BB		 2152}
2153
2154static void
13fe0198 2155dmu_recv_end_sync(void *arg, dmu_tx_t *tx)
34dc7c2f 2156{
13fe0198
MA		 2157 dmu_recv_cookie_t *drc = arg;
2158 dsl_pool_t *dp = dmu_tx_pool(tx);
2159
2160 spa_history_log_internal_ds(drc->drc_ds, "finish receiving",
2161 tx, "snap=%s", drc->drc_tosnap);
2162
2163 if (!drc->drc_newfs) {
2164 dsl_dataset_t *origin_head;
2165
2166 VERIFY0(dsl_dataset_hold(dp, drc->drc_tofs, FTAG,
2167 &origin_head));
19580676
MA		 2168
2169 if (drc->drc_force) {
2170 /*
2171 * Destroy any snapshots of drc_tofs (origin_head)
2172 * after the origin (the snap before drc_ds).
2173 */
d683ddbb
JG		 2174 uint64_t obj;
2175
2176 obj = dsl_dataset_phys(origin_head)->ds_prev_snap_obj;
2177 while (obj !=
2178 dsl_dataset_phys(drc->drc_ds)->ds_prev_snap_obj) {
19580676
MA		 2179 dsl_dataset_t *snap;
2180 VERIFY0(dsl_dataset_hold_obj(dp, obj, FTAG,
2181 &snap));
2182 ASSERT3P(snap->ds_dir, ==, origin_head->ds_dir);
d683ddbb 2183 obj = dsl_dataset_phys(snap)->ds_prev_snap_obj;
19580676
MA		 2184 dsl_destroy_snapshot_sync_impl(snap,
2185 B_FALSE, tx);
2186 dsl_dataset_rele(snap, FTAG);
2187 }
2188 }
2189 VERIFY3P(drc->drc_ds->ds_prev, ==,
2190 origin_head->ds_prev);
2191
13fe0198
MA		 2192 dsl_dataset_clone_swap_sync_impl(drc->drc_ds,
2193 origin_head, tx);
2194 dsl_dataset_snapshot_sync_impl(origin_head,
2195 drc->drc_tosnap, tx);
2196
2197 /* set snapshot's creation time and guid */
2198 dmu_buf_will_dirty(origin_head->ds_prev->ds_dbuf, tx);
d683ddbb 2199 dsl_dataset_phys(origin_head->ds_prev)->ds_creation_time =
13fe0198 2200 drc->drc_drrb->drr_creation_time;
d683ddbb 2201 dsl_dataset_phys(origin_head->ds_prev)->ds_guid =
13fe0198 2202 drc->drc_drrb->drr_toguid;
d683ddbb 2203 dsl_dataset_phys(origin_head->ds_prev)->ds_flags &=
13fe0198
MA		 2204 ~DS_FLAG_INCONSISTENT;
2205
2206 dmu_buf_will_dirty(origin_head->ds_dbuf, tx);
d683ddbb
JG		 2207 dsl_dataset_phys(origin_head)->ds_flags &=
2208 ~DS_FLAG_INCONSISTENT;
13fe0198
MA		 2209
2210 dsl_dataset_rele(origin_head, FTAG);
2211 dsl_destroy_head_sync_impl(drc->drc_ds, tx);
831baf06
KW		 2212
2213 if (drc->drc_owner != NULL)
2214 VERIFY3P(origin_head->ds_owner, ==, drc->drc_owner);
13fe0198
MA		 2215 } else {
2216 dsl_dataset_t *ds = drc->drc_ds;
34dc7c2f 2217
13fe0198 2218 dsl_dataset_snapshot_sync_impl(ds, drc->drc_tosnap, tx);
34dc7c2f 2219
13fe0198
MA		 2220 /* set snapshot's creation time and guid */
2221 dmu_buf_will_dirty(ds->ds_prev->ds_dbuf, tx);
d683ddbb 2222 dsl_dataset_phys(ds->ds_prev)->ds_creation_time =
13fe0198 2223 drc->drc_drrb->drr_creation_time;
d683ddbb
JG		 2224 dsl_dataset_phys(ds->ds_prev)->ds_guid =
2225 drc->drc_drrb->drr_toguid;
2226 dsl_dataset_phys(ds->ds_prev)->ds_flags &=
2227 ~DS_FLAG_INCONSISTENT;
34dc7c2f 2228
13fe0198 2229 dmu_buf_will_dirty(ds->ds_dbuf, tx);
d683ddbb 2230 dsl_dataset_phys(ds)->ds_flags &= ~DS_FLAG_INCONSISTENT;
13fe0198 2231 }
d683ddbb 2232 drc->drc_newsnapobj = dsl_dataset_phys(drc->drc_ds)->ds_prev_snap_obj;
13fe0198
MA		 2233 /*
2234 * Release the hold from dmu_recv_begin. This must be done before
2235 * we return to open context, so that when we free the dataset's dnode,
2236 * we can evict its bonus buffer.
2237 */
2238 dsl_dataset_disown(drc->drc_ds, dmu_recv_tag);
2239 drc->drc_ds = NULL;
34dc7c2f
BB		 2240}
2241
8d35c149 2242static int
13fe0198 2243add_ds_to_guidmap(const char *name, avl_tree_t *guid_map, uint64_t snapobj)
8d35c149 2244{
13fe0198 2245 dsl_pool_t *dp;
8d35c149
AS		 2246 dsl_dataset_t *snapds;
2247 guid_map_entry_t *gmep;
2248 int err;
2249
2250 ASSERT(guid_map != NULL);
2251
13fe0198
MA		 2252 err = dsl_pool_hold(name, FTAG, &dp);
2253 if (err != 0)
2254 return (err);
7ec09286
MA		 2255 gmep = kmem_alloc(sizeof (*gmep), KM_SLEEP);
2256 err = dsl_dataset_hold_obj(dp, snapobj, gmep, &snapds);
8d35c149 2257 if (err == 0) {
d683ddbb 2258 gmep->guid = dsl_dataset_phys(snapds)->ds_guid;
8d35c149
AS		 2259 gmep->gme_ds = snapds;
2260 avl_add(guid_map, gmep);
13fe0198 2261 dsl_dataset_long_hold(snapds, gmep);
7ec09286
MA		 2262 } else {
2263 kmem_free(gmep, sizeof (*gmep));
8d35c149
AS		 2264 }
2265
13fe0198 2266 dsl_pool_rele(dp, FTAG);
8d35c149
AS		 2267 return (err);
2268}
2269
13fe0198
MA		 2270static int dmu_recv_end_modified_blocks = 3;
2271
428870ff
BB		 2272static int
2273dmu_recv_existing_end(dmu_recv_cookie_t *drc)
34dc7c2f 2274{
13fe0198 2275 int error;
34dc7c2f 2276
13fe0198
MA		 2277#ifdef _KERNEL
2278 char *name;
34dc7c2f 2279
13fe0198
MA		 2280 /*
2281 * We will be destroying the ds; make sure its origin is unmounted if
2282 * necessary.
2283 */
2284 name = kmem_alloc(MAXNAMELEN, KM_SLEEP);
2285 dsl_dataset_name(drc->drc_ds, name);
2286 zfs_destroy_unmount_origin(name);
2287 kmem_free(name, MAXNAMELEN);
2288#endif
34dc7c2f 2289
13fe0198
MA		 2290 error = dsl_sync_task(drc->drc_tofs,
2291 dmu_recv_end_check, dmu_recv_end_sync, drc,
3d45fdd6 2292 dmu_recv_end_modified_blocks, ZFS_SPACE_CHECK_NORMAL);
13fe0198
MA		 2293
2294 if (error != 0)
2295 dmu_recv_cleanup_ds(drc);
2296 return (error);
34dc7c2f 2297}
428870ff
BB		 2298
2299static int
2300dmu_recv_new_end(dmu_recv_cookie_t *drc)
2301{
13fe0198 2302 int error;
428870ff 2303
13fe0198
MA		 2304 error = dsl_sync_task(drc->drc_tofs,
2305 dmu_recv_end_check, dmu_recv_end_sync, drc,
3d45fdd6 2306 dmu_recv_end_modified_blocks, ZFS_SPACE_CHECK_NORMAL);
428870ff 2307
13fe0198
MA		 2308 if (error != 0) {
2309 dmu_recv_cleanup_ds(drc);
2310 } else if (drc->drc_guid_to_ds_map != NULL) {
2311 (void) add_ds_to_guidmap(drc->drc_tofs,
2312 drc->drc_guid_to_ds_map,
2313 drc->drc_newsnapobj);
428870ff 2314 }
13fe0198 2315 return (error);
428870ff
BB		 2316}
2317
2318int
831baf06 2319dmu_recv_end(dmu_recv_cookie_t *drc, void *owner)
428870ff 2320{
831baf06
KW		 2321 drc->drc_owner = owner;
2322
13fe0198 2323 if (drc->drc_newfs)
428870ff 2324 return (dmu_recv_new_end(drc));
13fe0198
MA		 2325 else
2326 return (dmu_recv_existing_end(drc));
428870ff 2327}
ea97f8ce
MA		 2328
2329/*
2330 * Return TRUE if this objset is currently being received into.
2331 */
2332boolean_t
2333dmu_objset_is_receiving(objset_t *os)
2334{
2335 return (os->os_dsl_dataset != NULL &&
2336 os->os_dsl_dataset->ds_owner == dmu_recv_tag);
2337}
fd8febbd
TF		 2338
2339#if defined(_KERNEL)
2340module_param(zfs_send_corrupt_data, int, 0644);
2341MODULE_PARM_DESC(zfs_send_corrupt_data, "Allow sending corrupt data");
2342#endif
ZFS On Linux mirror for PVE