zfs-patches/0011-include-upstream-PR-6616.patch

   1 From 27d60535a612beaf31c27f5d8969abffac514841 Mon Sep 17 00:00:00 2001
   2 From: =?UTF-8?q?Fabian=20Gr=C3=BCnbichler?= <f.gruenbichler@proxmox.com>
   3 Date: Thu, 12 Oct 2017 12:29:01 +0200
   4 Subject: [PATCH 11/11] include upstream PR #6616
   5 MIME-Version: 1.0
   6 Content-Type: text/plain; charset=UTF-8
   7 Content-Transfer-Encoding: 8bit
   8
   9 send/recv compatibility with 0.6.5.x
  10
  11 Signed-off-by: Fabian Grünbichler <f.gruenbichler@proxmox.com>
  12 ---
  13  ...FREEOBJECTS-for-objects-which-can-t-exist.patch |  54 +++++++
  14  ...jects-when-receiving-full-stream-as-clone.patch | 167 +++++++++++++++++++++
  15  debian/patches/series                              |   2 +
  16  3 files changed, 223 insertions(+)
  17  create mode 100644 debian/patches/0011-Skip-FREEOBJECTS-for-objects-which-can-t-exist.patch
  18  create mode 100644 debian/patches/0012-Free-objects-when-receiving-full-stream-as-clone.patch
  19
  20 diff --git a/debian/patches/0011-Skip-FREEOBJECTS-for-objects-which-can-t-exist.patch b/debian/patches/0011-Skip-FREEOBJECTS-for-objects-which-can-t-exist.patch
  21 new file mode 100644
  22 index 000000000..e2f0a7cdd
  23 --- /dev/null
  24 +++ b/debian/patches/0011-Skip-FREEOBJECTS-for-objects-which-can-t-exist.patch
  25 @@ -0,0 +1,54 @@
  26 +From: =?utf-8?q?Fabian_Gr=C3=BCnbichler?= <f.gruenbichler@proxmox.com>
  27 +Date: Tue, 26 Sep 2017 14:03:21 +0200
  28 +Subject: Skip FREEOBJECTS for objects which can't exist
  29 +MIME-Version: 1.0
  30 +Content-Type: text/plain; charset="utf-8"
  31 +Content-Transfer-Encoding: 8bit
  32 +
  33 +When sending an incremental stream based on a snapshot, the receiving
  34 +side must have the same base snapshot.  Thus we do not need to send
  35 +FREEOBJECTS records for any objects past the maximum one which exists
  36 +locally.
  37 +
  38 +This allows us to send incremental streams (again) to older ZFS
  39 +implementations (e.g. ZoL < 0.7) which actually try to free all objects
  40 +in a FREEOBJECTS record, instead of bailing out early.
  41 +
  42 +Reviewed by: Paul Dagnelie <pcd@delphix.com>
  43 +Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
  44 +Signed-off-by: Fabian Grünbichler <f.gruenbichler@proxmox.com>
  45 +Closes #5699
  46 +Closes #6507
  47 +Closes #6616
  48 +(cherry picked from commit 829e95c4dc74d7d6d31d01af9c39e03752499b15)
  49 +---
  50 + module/zfs/dmu_send.c | 16 ++++++++++++++++
  51 + 1 file changed, 16 insertions(+)
  52 +
  53 +diff --git a/module/zfs/dmu_send.c b/module/zfs/dmu_send.c
  54 +index 0c53ced..a81580d 100644
  55 +--- a/module/zfs/dmu_send.c
  56 ++++ b/module/zfs/dmu_send.c
  57 +@@ -421,6 +421,22 @@ static int
  58 + dump_freeobjects(dmu_sendarg_t *dsp, uint64_t firstobj, uint64_t numobjs)
  59 + {
  60 +       struct drr_freeobjects *drrfo = &(dsp->dsa_drr->drr_u.drr_freeobjects);
  61 ++      uint64_t maxobj = DNODES_PER_BLOCK *
  62 ++          (DMU_META_DNODE(dsp->dsa_os)->dn_maxblkid + 1);
  63 ++
  64 ++      /*
  65 ++       * ZoL < 0.7 does not handle large FREEOBJECTS records correctly,
  66 ++       * leading to zfs recv never completing. to avoid this issue, don't
  67 ++       * send FREEOBJECTS records for object IDs which cannot exist on the
  68 ++       * receiving side.
  69 ++       */
  70 ++      if (maxobj > 0) {
  71 ++              if (maxobj < firstobj)
  72 ++                      return (0);
  73 ++
  74 ++              if (maxobj < firstobj + numobjs)
  75 ++                      numobjs = maxobj - firstobj;
  76 ++      }
  77 +
  78 +       /*
  79 +        * If there is a pending op, but it's not PENDING_FREEOBJECTS,
  80 diff --git a/debian/patches/0012-Free-objects-when-receiving-full-stream-as-clone.patch b/debian/patches/0012-Free-objects-when-receiving-full-stream-as-clone.patch
  81 new file mode 100644
  82 index 000000000..75f4be84c
  83 --- /dev/null
  84 +++ b/debian/patches/0012-Free-objects-when-receiving-full-stream-as-clone.patch
  85 @@ -0,0 +1,167 @@
  86 +From: =?utf-8?q?Fabian_Gr=C3=BCnbichler?= <f.gruenbichler@proxmox.com>
  87 +Date: Fri, 29 Sep 2017 12:00:29 +0200
  88 +Subject: Free objects when receiving full stream as clone
  89 +MIME-Version: 1.0
  90 +Content-Type: text/plain; charset="utf-8"
  91 +Content-Transfer-Encoding: 8bit
  92 +
  93 +All objects after the last written or freed object are not supposed to
  94 +exist after receiving the stream.  Free them accordingly, as if a
  95 +freeobjects record for them had been included in the stream.
  96 +
  97 +Reviewed by: Paul Dagnelie <pcd@delphix.com>
  98 +Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
  99 +Signed-off-by: Fabian Grünbichler <f.gruenbichler@proxmox.com>
 100 +Closes #5699
 101 +Closes #6507
 102 +Closes #6616
 103 +(backported from commit 48fbb9ddbf2281911560dfbc2821aa8b74127315)
 104 +
 105 + Conflicts:
 106 +       include/sys/dmu_send.h
 107 +       module/zfs/dmu_send.c
 108 +
 109 +modified for code paths missing in 0.7
 110 +---
 111 + include/sys/dmu_send.h |  1 +
 112 + module/zfs/dmu_send.c  | 52 +++++++++++++++++++++++++++++++++++++++++++++++++-
 113 + 2 files changed, 52 insertions(+), 1 deletion(-)
 114 +
 115 +diff --git a/include/sys/dmu_send.h b/include/sys/dmu_send.h
 116 +index e9bef8b..5cf67a6 100644
 117 +--- a/include/sys/dmu_send.h
 118 ++++ b/include/sys/dmu_send.h
 119 +@@ -61,6 +61,7 @@ typedef struct dmu_recv_cookie {
 120 +       boolean_t drc_byteswap;
 121 +       boolean_t drc_force;
 122 +       boolean_t drc_resumable;
 123 ++      boolean_t drc_clone;
 124 +       struct avl_tree *drc_guid_to_ds_map;
 125 +       zio_cksum_t drc_cksum;
 126 +       uint64_t drc_newsnapobj;
 127 +diff --git a/module/zfs/dmu_send.c b/module/zfs/dmu_send.c
 128 +index a81580d..344e420 100644
 129 +--- a/module/zfs/dmu_send.c
 130 ++++ b/module/zfs/dmu_send.c
 131 +@@ -1823,6 +1823,7 @@ dmu_recv_begin(char *tofs, char *tosnap, dmu_replay_record_t *drr_begin,
 132 +       drc->drc_force = force;
 133 +       drc->drc_resumable = resumable;
 134 +       drc->drc_cred = CRED();
 135 ++      drc->drc_clone = (origin != NULL);
 136 +
 137 +       if (drc->drc_drrb->drr_magic == BSWAP_64(DMU_BACKUP_MAGIC)) {
 138 +               drc->drc_byteswap = B_TRUE;
 139 +@@ -1883,7 +1884,9 @@ struct receive_writer_arg {
 140 +       /* A map from guid to dataset to help handle dedup'd streams. */
 141 +       avl_tree_t *guid_to_ds_map;
 142 +       boolean_t resumable;
 143 +-      uint64_t last_object, last_offset;
 144 ++      uint64_t last_object;
 145 ++      uint64_t last_offset;
 146 ++      uint64_t max_object; /* highest object ID referenced in stream */
 147 +       uint64_t bytes_read; /* bytes read when current record created */
 148 + };
 149 +
 150 +@@ -2157,6 +2160,9 @@ receive_object(struct receive_writer_arg *rwa, struct drr_object *drro,
 151 +               return (SET_ERROR(EINVAL));
 152 +       object = err == 0 ? drro->drr_object : DMU_NEW_OBJECT;
 153 +
 154 ++      if (drro->drr_object > rwa->max_object)
 155 ++              rwa->max_object = drro->drr_object;
 156 ++
 157 +       /*
 158 +        * If we are losing blkptrs or changing the block size this must
 159 +        * be a new file instance.  We must clear out the previous file
 160 +@@ -2257,6 +2263,9 @@ receive_freeobjects(struct receive_writer_arg *rwa,
 161 +               err = dmu_free_long_object(rwa->os, obj);
 162 +               if (err != 0)
 163 +                       return (err);
 164 ++
 165 ++              if (obj > rwa->max_object)
 166 ++                      rwa->max_object = obj;
 167 +       }
 168 +       if (next_err != ESRCH)
 169 +               return (next_err);
 170 +@@ -2287,6 +2296,9 @@ receive_write(struct receive_writer_arg *rwa, struct drr_write *drrw,
 171 +       rwa->last_object = drrw->drr_object;
 172 +       rwa->last_offset = drrw->drr_offset;
 173 +
 174 ++      if (rwa->last_object > rwa->max_object)
 175 ++              rwa->max_object = rwa->last_object;
 176 ++
 177 +       if (dmu_object_info(rwa->os, drrw->drr_object, NULL) != 0)
 178 +               return (SET_ERROR(EINVAL));
 179 +
 180 +@@ -2362,6 +2374,9 @@ receive_write_byref(struct receive_writer_arg *rwa,
 181 +               ref_os = rwa->os;
 182 +       }
 183 +
 184 ++      if (drrwbr->drr_object > rwa->max_object)
 185 ++              rwa->max_object = drrwbr->drr_object;
 186 ++
 187 +       err = dmu_buf_hold(ref_os, drrwbr->drr_refobject,
 188 +           drrwbr->drr_refoffset, FTAG, &dbp, DMU_READ_PREFETCH);
 189 +       if (err != 0)
 190 +@@ -2404,6 +2419,9 @@ receive_write_embedded(struct receive_writer_arg *rwa,
 191 +       if (drrwe->drr_compression >= ZIO_COMPRESS_FUNCTIONS)
 192 +               return (EINVAL);
 193 +
 194 ++      if (drrwe->drr_object > rwa->max_object)
 195 ++              rwa->max_object = drrwe->drr_object;
 196 ++
 197 +       tx = dmu_tx_create(rwa->os);
 198 +
 199 +       dmu_tx_hold_write(tx, drrwe->drr_object,
 200 +@@ -2440,6 +2458,9 @@ receive_spill(struct receive_writer_arg *rwa, struct drr_spill *drrs,
 201 +       if (dmu_object_info(rwa->os, drrs->drr_object, NULL) != 0)
 202 +               return (SET_ERROR(EINVAL));
 203 +
 204 ++      if (drrs->drr_object > rwa->max_object)
 205 ++              rwa->max_object = drrs->drr_object;
 206 ++
 207 +       VERIFY0(dmu_bonus_hold(rwa->os, drrs->drr_object, FTAG, &db));
 208 +       if ((err = dmu_spill_hold_by_bonus(db, FTAG, &db_spill)) != 0) {
 209 +               dmu_buf_rele(db, FTAG);
 210 +@@ -2484,6 +2505,9 @@ receive_free(struct receive_writer_arg *rwa, struct drr_free *drrf)
 211 +       if (dmu_object_info(rwa->os, drrf->drr_object, NULL) != 0)
 212 +               return (SET_ERROR(EINVAL));
 213 +
 214 ++      if (drrf->drr_object > rwa->max_object)
 215 ++              rwa->max_object = drrf->drr_object;
 216 ++
 217 +       err = dmu_free_long_range(rwa->os, drrf->drr_object,
 218 +           drrf->drr_offset, drrf->drr_length);
 219 +
 220 +@@ -3207,6 +3231,32 @@ dmu_recv_stream(dmu_recv_cookie_t *drc, vnode_t *vp, offset_t *voffp,
 221 +       }
 222 +       mutex_exit(&rwa->mutex);
 223 +
 224 ++      /*
 225 ++       * If we are receiving a full stream as a clone, all object IDs which
 226 ++       * are greater than the maximum ID referenced in the stream are
 227 ++       * by definition unused and must be freed.
 228 ++       */
 229 ++      if (drc->drc_clone && drc->drc_drrb->drr_fromguid == 0) {
 230 ++              uint64_t obj = rwa->max_object + 1;
 231 ++              int free_err = 0;
 232 ++              int next_err = 0;
 233 ++
 234 ++              while (next_err == 0) {
 235 ++                      free_err = dmu_free_long_object(rwa->os, obj);
 236 ++                      if (free_err != 0 && free_err != ENOENT)
 237 ++                              break;
 238 ++
 239 ++                      next_err = dmu_object_next(rwa->os, &obj, FALSE, 0);
 240 ++              }
 241 ++
 242 ++              if (err == 0) {
 243 ++                      if (free_err != 0 && free_err != ENOENT)
 244 ++                              err = free_err;
 245 ++                      else if (next_err != ESRCH)
 246 ++                              err = next_err;
 247 ++              }
 248 ++      }
 249 ++
 250 +       cv_destroy(&rwa->cv);
 251 +       mutex_destroy(&rwa->mutex);
 252 +       bqueue_destroy(&rwa->q);
 253 diff --git a/debian/patches/series b/debian/patches/series
 254 index 3c8f56483..8bd73161c 100644
 255 --- a/debian/patches/series
 256 +++ b/debian/patches/series
 257 @@ -8,3 +8,5 @@ enable-zed.patch
 258  0009-add-man-page-reference-to-systemd-units.patch
 259  0010-fix-install-path-of-zpool.d-scripts.patch
 260  0010-receive_freeobjects-skips-freeing-some-objects.patch
 261 +0011-Skip-FREEOBJECTS-for-objects-which-can-t-exist.patch
 262 +0012-Free-objects-when-receiving-full-stream-as-clone.patch
 263 --
 264 2.14.1
 265