fs/aufs/cpup.c

   1 /*
   2  * Copyright (C) 2005-2017 Junjiro R. Okajima
   3  *
   4  * This program, aufs is free software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License as published by
   6  * the Free Software Foundation; either version 2 of the License, or
   7  * (at your option) any later version.
   8  *
   9  * This program is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  * GNU General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU General Public License
  15  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  16  */
  17
  18 /*
  19  * copy-up functions, see wbr_policy.c for copy-down
  20  */
  21
  22 #include <linux/fs_stack.h>
  23 #include <linux/mm.h>
  24 #include <linux/task_work.h>
  25 #include "aufs.h"
  26
  27 void au_cpup_attr_flags(struct inode *dst, unsigned int iflags)
  28 {
  29         const unsigned int mask = S_DEAD | S_SWAPFILE | S_PRIVATE
  30                 | S_NOATIME | S_NOCMTIME | S_AUTOMOUNT;
  31
  32         BUILD_BUG_ON(sizeof(iflags) != sizeof(dst->i_flags));
  33
  34         dst->i_flags |= iflags & ~mask;
  35         if (au_test_fs_notime(dst->i_sb))
  36                 dst->i_flags |= S_NOATIME | S_NOCMTIME;
  37 }
  38
  39 void au_cpup_attr_timesizes(struct inode *inode)
  40 {
  41         struct inode *h_inode;
  42
  43         h_inode = au_h_iptr(inode, au_ibtop(inode));
  44         fsstack_copy_attr_times(inode, h_inode);
  45         fsstack_copy_inode_size(inode, h_inode);
  46 }
  47
  48 void au_cpup_attr_nlink(struct inode *inode, int force)
  49 {
  50         struct inode *h_inode;
  51         struct super_block *sb;
  52         aufs_bindex_t bindex, bbot;
  53
  54         sb = inode->i_sb;
  55         bindex = au_ibtop(inode);
  56         h_inode = au_h_iptr(inode, bindex);
  57         if (!force
  58             && !S_ISDIR(h_inode->i_mode)
  59             && au_opt_test(au_mntflags(sb), PLINK)
  60             && au_plink_test(inode))
  61                 return;
  62
  63         /*
  64          * 0 can happen in revalidating.
  65          * h_inode->i_mutex may not be held here, but it is harmless since once
  66          * i_nlink reaches 0, it will never become positive except O_TMPFILE
  67          * case.
  68          * todo: O_TMPFILE+linkat(AT_SYMLINK_FOLLOW) bypassing aufs may cause
  69          *       the incorrect link count.
  70          */
  71         set_nlink(inode, h_inode->i_nlink);
  72
  73         /*
  74          * fewer nlink makes find(1) noisy, but larger nlink doesn't.
  75          * it may includes whplink directory.
  76          */
  77         if (S_ISDIR(h_inode->i_mode)) {
  78                 bbot = au_ibbot(inode);
  79                 for (bindex++; bindex <= bbot; bindex++) {
  80                         h_inode = au_h_iptr(inode, bindex);
  81                         if (h_inode)
  82                                 au_add_nlink(inode, h_inode);
  83                 }
  84         }
  85 }
  86
  87 void au_cpup_attr_changeable(struct inode *inode)
  88 {
  89         struct inode *h_inode;
  90
  91         h_inode = au_h_iptr(inode, au_ibtop(inode));
  92         inode->i_mode = h_inode->i_mode;
  93         inode->i_uid = h_inode->i_uid;
  94         inode->i_gid = h_inode->i_gid;
  95         au_cpup_attr_timesizes(inode);
  96         au_cpup_attr_flags(inode, h_inode->i_flags);
  97 }
  98
  99 void au_cpup_igen(struct inode *inode, struct inode *h_inode)
 100 {
 101         struct au_iinfo *iinfo = au_ii(inode);
 102
 103         IiMustWriteLock(inode);
 104
 105         iinfo->ii_higen = h_inode->i_generation;
 106         iinfo->ii_hsb1 = h_inode->i_sb;
 107 }
 108
 109 void au_cpup_attr_all(struct inode *inode, int force)
 110 {
 111         struct inode *h_inode;
 112
 113         h_inode = au_h_iptr(inode, au_ibtop(inode));
 114         au_cpup_attr_changeable(inode);
 115         if (inode->i_nlink > 0)
 116                 au_cpup_attr_nlink(inode, force);
 117         inode->i_rdev = h_inode->i_rdev;
 118         inode->i_blkbits = h_inode->i_blkbits;
 119         au_cpup_igen(inode, h_inode);
 120 }
 121
 122 /* ---------------------------------------------------------------------- */
 123
 124 /* Note: dt_dentry and dt_h_dentry are not dget/dput-ed */
 125
 126 /* keep the timestamps of the parent dir when cpup */
 127 void au_dtime_store(struct au_dtime *dt, struct dentry *dentry,
 128                     struct path *h_path)
 129 {
 130         struct inode *h_inode;
 131
 132         dt->dt_dentry = dentry;
 133         dt->dt_h_path = *h_path;
 134         h_inode = d_inode(h_path->dentry);
 135         dt->dt_atime = h_inode->i_atime;
 136         dt->dt_mtime = h_inode->i_mtime;
 137         /* smp_mb(); */
 138 }
 139
 140 void au_dtime_revert(struct au_dtime *dt)
 141 {
 142         struct iattr attr;
 143         int err;
 144
 145         attr.ia_atime = dt->dt_atime;
 146         attr.ia_mtime = dt->dt_mtime;
 147         attr.ia_valid = ATTR_FORCE | ATTR_MTIME | ATTR_MTIME_SET
 148                 | ATTR_ATIME | ATTR_ATIME_SET;
 149
 150         /* no delegation since this is a directory */
 151         err = vfsub_notify_change(&dt->dt_h_path, &attr, /*delegated*/NULL);
 152         if (unlikely(err))
 153                 pr_warn("restoring timestamps failed(%d). ignored\n", err);
 154 }
 155
 156 /* ---------------------------------------------------------------------- */
 157
 158 /* internal use only */
 159 struct au_cpup_reg_attr {
 160         int             valid;
 161         struct kstat    st;
 162         unsigned int    iflags; /* inode->i_flags */
 163 };
 164
 165 static noinline_for_stack
 166 int cpup_iattr(struct dentry *dst, aufs_bindex_t bindex, struct dentry *h_src,
 167                struct au_cpup_reg_attr *h_src_attr)
 168 {
 169         int err, sbits, icex;
 170         unsigned int mnt_flags;
 171         unsigned char verbose;
 172         struct iattr ia;
 173         struct path h_path;
 174         struct inode *h_isrc, *h_idst;
 175         struct kstat *h_st;
 176         struct au_branch *br;
 177
 178         h_path.dentry = au_h_dptr(dst, bindex);
 179         h_idst = d_inode(h_path.dentry);
 180         br = au_sbr(dst->d_sb, bindex);
 181         h_path.mnt = au_br_mnt(br);
 182         h_isrc = d_inode(h_src);
 183         ia.ia_valid = ATTR_FORCE | ATTR_UID | ATTR_GID
 184                 | ATTR_ATIME | ATTR_MTIME
 185                 | ATTR_ATIME_SET | ATTR_MTIME_SET;
 186         if (h_src_attr && h_src_attr->valid) {
 187                 h_st = &h_src_attr->st;
 188                 ia.ia_uid = h_st->uid;
 189                 ia.ia_gid = h_st->gid;
 190                 ia.ia_atime = h_st->atime;
 191                 ia.ia_mtime = h_st->mtime;
 192                 if (h_idst->i_mode != h_st->mode
 193                     && !S_ISLNK(h_idst->i_mode)) {
 194                         ia.ia_valid |= ATTR_MODE;
 195                         ia.ia_mode = h_st->mode;
 196                 }
 197                 sbits = !!(h_st->mode & (S_ISUID | S_ISGID));
 198                 au_cpup_attr_flags(h_idst, h_src_attr->iflags);
 199         } else {
 200                 ia.ia_uid = h_isrc->i_uid;
 201                 ia.ia_gid = h_isrc->i_gid;
 202                 ia.ia_atime = h_isrc->i_atime;
 203                 ia.ia_mtime = h_isrc->i_mtime;
 204                 if (h_idst->i_mode != h_isrc->i_mode
 205                     && !S_ISLNK(h_idst->i_mode)) {
 206                         ia.ia_valid |= ATTR_MODE;
 207                         ia.ia_mode = h_isrc->i_mode;
 208                 }
 209                 sbits = !!(h_isrc->i_mode & (S_ISUID | S_ISGID));
 210                 au_cpup_attr_flags(h_idst, h_isrc->i_flags);
 211         }
 212         /* no delegation since it is just created */
 213         err = vfsub_notify_change(&h_path, &ia, /*delegated*/NULL);
 214
 215         /* is this nfs only? */
 216         if (!err && sbits && au_test_nfs(h_path.dentry->d_sb)) {
 217                 ia.ia_valid = ATTR_FORCE | ATTR_MODE;
 218                 ia.ia_mode = h_isrc->i_mode;
 219                 err = vfsub_notify_change(&h_path, &ia, /*delegated*/NULL);
 220         }
 221
 222         icex = br->br_perm & AuBrAttr_ICEX;
 223         if (!err) {
 224                 mnt_flags = au_mntflags(dst->d_sb);
 225                 verbose = !!au_opt_test(mnt_flags, VERBOSE);
 226                 err = au_cpup_xattr(h_path.dentry, h_src, icex, verbose);
 227         }
 228
 229         return err;
 230 }
 231
 232 /* ---------------------------------------------------------------------- */
 233
 234 static int au_do_copy_file(struct file *dst, struct file *src, loff_t len,
 235                            char *buf, unsigned long blksize)
 236 {
 237         int err;
 238         size_t sz, rbytes, wbytes;
 239         unsigned char all_zero;
 240         char *p, *zp;
 241         struct inode *h_inode;
 242         /* reduce stack usage */
 243         struct iattr *ia;
 244
 245         zp = page_address(ZERO_PAGE(0));
 246         if (unlikely(!zp))
 247                 return -ENOMEM; /* possible? */
 248
 249         err = 0;
 250         all_zero = 0;
 251         while (len) {
 252                 AuDbg("len %lld\n", len);
 253                 sz = blksize;
 254                 if (len < blksize)
 255                         sz = len;
 256
 257                 rbytes = 0;
 258                 /* todo: signal_pending? */
 259                 while (!rbytes || err == -EAGAIN || err == -EINTR) {
 260                         rbytes = vfsub_read_k(src, buf, sz, &src->f_pos);
 261                         err = rbytes;
 262                 }
 263                 if (unlikely(err < 0))
 264                         break;
 265
 266                 all_zero = 0;
 267                 if (len >= rbytes && rbytes == blksize)
 268                         all_zero = !memcmp(buf, zp, rbytes);
 269                 if (!all_zero) {
 270                         wbytes = rbytes;
 271                         p = buf;
 272                         while (wbytes) {
 273                                 size_t b;
 274
 275                                 b = vfsub_write_k(dst, p, wbytes, &dst->f_pos);
 276                                 err = b;
 277                                 /* todo: signal_pending? */
 278                                 if (unlikely(err == -EAGAIN || err == -EINTR))
 279                                         continue;
 280                                 if (unlikely(err < 0))
 281                                         break;
 282                                 wbytes -= b;
 283                                 p += b;
 284                         }
 285                         if (unlikely(err < 0))
 286                                 break;
 287                 } else {
 288                         loff_t res;
 289
 290                         AuLabel(hole);
 291                         res = vfsub_llseek(dst, rbytes, SEEK_CUR);
 292                         err = res;
 293                         if (unlikely(res < 0))
 294                                 break;
 295                 }
 296                 len -= rbytes;
 297                 err = 0;
 298         }
 299
 300         /* the last block may be a hole */
 301         if (!err && all_zero) {
 302                 AuLabel(last hole);
 303
 304                 err = 1;
 305                 if (au_test_nfs(dst->f_path.dentry->d_sb)) {
 306                         /* nfs requires this step to make last hole */
 307                         /* is this only nfs? */
 308                         do {
 309                                 /* todo: signal_pending? */
 310                                 err = vfsub_write_k(dst, "\0", 1, &dst->f_pos);
 311                         } while (err == -EAGAIN || err == -EINTR);
 312                         if (err == 1)
 313                                 dst->f_pos--;
 314                 }
 315
 316                 if (err == 1) {
 317                         ia = (void *)buf;
 318                         ia->ia_size = dst->f_pos;
 319                         ia->ia_valid = ATTR_SIZE | ATTR_FILE;
 320                         ia->ia_file = dst;
 321                         h_inode = file_inode(dst);
 322                         inode_lock_nested(h_inode, AuLsc_I_CHILD2);
 323                         /* no delegation since it is just created */
 324                         err = vfsub_notify_change(&dst->f_path, ia,
 325                                                   /*delegated*/NULL);
 326                         inode_unlock(h_inode);
 327                 }
 328         }
 329
 330         return err;
 331 }
 332
 333 int au_copy_file(struct file *dst, struct file *src, loff_t len)
 334 {
 335         int err;
 336         unsigned long blksize;
 337         unsigned char do_kfree;
 338         char *buf;
 339
 340         err = -ENOMEM;
 341         blksize = dst->f_path.dentry->d_sb->s_blocksize;
 342         if (!blksize || PAGE_SIZE < blksize)
 343                 blksize = PAGE_SIZE;
 344         AuDbg("blksize %lu\n", blksize);
 345         do_kfree = (blksize != PAGE_SIZE && blksize >= sizeof(struct iattr *));
 346         if (do_kfree)
 347                 buf = kmalloc(blksize, GFP_NOFS);
 348         else
 349                 buf = (void *)__get_free_page(GFP_NOFS);
 350         if (unlikely(!buf))
 351                 goto out;
 352
 353         if (len > (1 << 22))
 354                 AuDbg("copying a large file %lld\n", (long long)len);
 355
 356         src->f_pos = 0;
 357         dst->f_pos = 0;
 358         err = au_do_copy_file(dst, src, len, buf, blksize);
 359         if (do_kfree)
 360                 kfree(buf);
 361         else
 362                 free_page((unsigned long)buf);
 363
 364 out:
 365         return err;
 366 }
 367
 368 static int au_do_copy(struct file *dst, struct file *src, loff_t len)
 369 {
 370         int err;
 371         struct super_block *h_src_sb;
 372         struct inode *h_src_inode;
 373
 374         h_src_inode = file_inode(src);
 375         h_src_sb = h_src_inode->i_sb;
 376
 377         /* XFS acquires inode_lock */
 378         if (!au_test_xfs(h_src_sb))
 379                 err = au_copy_file(dst, src, len);
 380         else {
 381                 inode_unlock_shared(h_src_inode);
 382                 err = au_copy_file(dst, src, len);
 383                 vfsub_inode_lock_shared_nested(h_src_inode, AuLsc_I_CHILD);
 384         }
 385
 386         return err;
 387 }
 388
 389 static int au_clone_or_copy(struct file *dst, struct file *src, loff_t len)
 390 {
 391         int err;
 392         struct super_block *h_src_sb;
 393         struct inode *h_src_inode;
 394
 395         h_src_inode = file_inode(src);
 396         h_src_sb = h_src_inode->i_sb;
 397         if (h_src_sb != file_inode(dst)->i_sb
 398             || !dst->f_op->clone_file_range) {
 399                 err = au_do_copy(dst, src, len);
 400                 goto out;
 401         }
 402
 403         if (!au_test_nfs(h_src_sb)) {
 404                 inode_unlock_shared(h_src_inode);
 405                 err = vfsub_clone_file_range(src, dst, len);
 406                 vfsub_inode_lock_shared_nested(h_src_inode, AuLsc_I_CHILD);
 407         } else
 408                 err = vfsub_clone_file_range(src, dst, len);
 409         /* older XFS has a condition in cloning */
 410         if (unlikely(err != -EOPNOTSUPP))
 411                 goto out;
 412
 413         /* the backend fs on NFS may not support cloning */
 414         err = au_do_copy(dst, src, len);
 415
 416 out:
 417         AuTraceErr(err);
 418         return err;
 419 }
 420
 421 /*
 422  * to support a sparse file which is opened with O_APPEND,
 423  * we need to close the file.
 424  */
 425 static int au_cp_regular(struct au_cp_generic *cpg)
 426 {
 427         int err, i;
 428         enum { SRC, DST };
 429         struct {
 430                 aufs_bindex_t bindex;
 431                 unsigned int flags;
 432                 struct dentry *dentry;
 433                 int force_wr;
 434                 struct file *file;
 435                 void *label;
 436         } *f, file[] = {
 437                 {
 438                         .bindex = cpg->bsrc,
 439                         .flags = O_RDONLY | O_NOATIME | O_LARGEFILE,
 440                         .label = &&out
 441                 },
 442                 {
 443                         .bindex = cpg->bdst,
 444                         .flags = O_WRONLY | O_NOATIME | O_LARGEFILE,
 445                         .force_wr = !!au_ftest_cpup(cpg->flags, RWDST),
 446                         .label = &&out_src
 447                 }
 448         };
 449         struct super_block *sb, *h_src_sb;
 450         struct inode *h_src_inode;
 451         struct task_struct *tsk = current;
 452
 453         /* bsrc branch can be ro/rw. */
 454         sb = cpg->dentry->d_sb;
 455         f = file;
 456         for (i = 0; i < 2; i++, f++) {
 457                 f->dentry = au_h_dptr(cpg->dentry, f->bindex);
 458                 f->file = au_h_open(cpg->dentry, f->bindex, f->flags,
 459                                     /*file*/NULL, f->force_wr);
 460                 err = PTR_ERR(f->file);
 461                 if (IS_ERR(f->file))
 462                         goto *f->label;
 463         }
 464
 465         /* try stopping to update while we copyup */
 466         h_src_inode = d_inode(file[SRC].dentry);
 467         h_src_sb = h_src_inode->i_sb;
 468         if (!au_test_nfs(h_src_sb))
 469                 IMustLock(h_src_inode);
 470         err = au_clone_or_copy(file[DST].file, file[SRC].file, cpg->len);
 471
 472         /* i wonder if we had O_NO_DELAY_FPUT flag */
 473         if (tsk->flags & PF_KTHREAD)
 474                 __fput_sync(file[DST].file);
 475         else {
 476                 WARN(1, "%pD\nPlease report this warning to aufs-users ML",
 477                      file[DST].file);
 478                 fput(file[DST].file);
 479                 /*
 480                  * too bad.
 481                  * we have to call both since we don't know which place the file
 482                  * was added to.
 483                  */
 484                 task_work_run();
 485                 flush_delayed_fput();
 486         }
 487         au_sbr_put(sb, file[DST].bindex);
 488
 489 out_src:
 490         fput(file[SRC].file);
 491         au_sbr_put(sb, file[SRC].bindex);
 492 out:
 493         return err;
 494 }
 495
 496 static int au_do_cpup_regular(struct au_cp_generic *cpg,
 497                               struct au_cpup_reg_attr *h_src_attr)
 498 {
 499         int err, rerr;
 500         loff_t l;
 501         struct path h_path;
 502         struct inode *h_src_inode, *h_dst_inode;
 503
 504         err = 0;
 505         h_src_inode = au_h_iptr(d_inode(cpg->dentry), cpg->bsrc);
 506         l = i_size_read(h_src_inode);
 507         if (cpg->len == -1 || l < cpg->len)
 508                 cpg->len = l;
 509         if (cpg->len) {
 510                 /* try stopping to update while we are referencing */
 511                 vfsub_inode_lock_shared_nested(h_src_inode, AuLsc_I_CHILD);
 512                 au_pin_hdir_unlock(cpg->pin);
 513
 514                 h_path.dentry = au_h_dptr(cpg->dentry, cpg->bsrc);
 515                 h_path.mnt = au_sbr_mnt(cpg->dentry->d_sb, cpg->bsrc);
 516                 h_src_attr->iflags = h_src_inode->i_flags;
 517                 if (!au_test_nfs(h_src_inode->i_sb))
 518                         err = vfsub_getattr(&h_path, &h_src_attr->st);
 519                 else {
 520                         inode_unlock_shared(h_src_inode);
 521                         err = vfsub_getattr(&h_path, &h_src_attr->st);
 522                         vfsub_inode_lock_shared_nested(h_src_inode,
 523                                                        AuLsc_I_CHILD);
 524                 }
 525                 if (unlikely(err)) {
 526                         inode_unlock_shared(h_src_inode);
 527                         goto out;
 528                 }
 529                 h_src_attr->valid = 1;
 530                 if (!au_test_nfs(h_src_inode->i_sb)) {
 531                         err = au_cp_regular(cpg);
 532                         inode_unlock_shared(h_src_inode);
 533                 } else {
 534                         inode_unlock_shared(h_src_inode);
 535                         err = au_cp_regular(cpg);
 536                 }
 537                 rerr = au_pin_hdir_relock(cpg->pin);
 538                 if (!err && rerr)
 539                         err = rerr;
 540         }
 541         if (!err && (h_src_inode->i_state & I_LINKABLE)) {
 542                 h_path.dentry = au_h_dptr(cpg->dentry, cpg->bdst);
 543                 h_dst_inode = d_inode(h_path.dentry);
 544                 spin_lock(&h_dst_inode->i_lock);
 545                 h_dst_inode->i_state |= I_LINKABLE;
 546                 spin_unlock(&h_dst_inode->i_lock);
 547         }
 548
 549 out:
 550         return err;
 551 }
 552
 553 static int au_do_cpup_symlink(struct path *h_path, struct dentry *h_src,
 554                               struct inode *h_dir)
 555 {
 556         int err, symlen;
 557         mm_segment_t old_fs;
 558         union {
 559                 char *k;
 560                 char __user *u;
 561         } sym;
 562
 563         err = -ENOMEM;
 564         sym.k = (void *)__get_free_page(GFP_NOFS);
 565         if (unlikely(!sym.k))
 566                 goto out;
 567
 568         /* unnecessary to support mmap_sem since symlink is not mmap-able */
 569         old_fs = get_fs();
 570         set_fs(KERNEL_DS);
 571         symlen = vfs_readlink(h_src, sym.u, PATH_MAX);
 572         err = symlen;
 573         set_fs(old_fs);
 574
 575         if (symlen > 0) {
 576                 sym.k[symlen] = 0;
 577                 err = vfsub_symlink(h_dir, h_path, sym.k);
 578         }
 579         free_page((unsigned long)sym.k);
 580
 581 out:
 582         return err;
 583 }
 584
 585 /*
 586  * regardless 'acl' option, reset all ACL.
 587  * All ACL will be copied up later from the original entry on the lower branch.
 588  */
 589 static int au_reset_acl(struct inode *h_dir, struct path *h_path, umode_t mode)
 590 {
 591         int err;
 592         struct dentry *h_dentry;
 593         struct inode *h_inode;
 594
 595         h_dentry = h_path->dentry;
 596         h_inode = d_inode(h_dentry);
 597         /* forget_all_cached_acls(h_inode)); */
 598         err = vfsub_removexattr(h_dentry, XATTR_NAME_POSIX_ACL_ACCESS);
 599         AuTraceErr(err);
 600         if (err == -EOPNOTSUPP)
 601                 err = 0;
 602         if (!err)
 603                 err = vfsub_acl_chmod(h_inode, mode);
 604
 605         AuTraceErr(err);
 606         return err;
 607 }
 608
 609 static int au_do_cpup_dir(struct au_cp_generic *cpg, struct dentry *dst_parent,
 610                           struct inode *h_dir, struct path *h_path)
 611 {
 612         int err;
 613         struct inode *dir, *inode;
 614
 615         err = vfsub_removexattr(h_path->dentry, XATTR_NAME_POSIX_ACL_DEFAULT);
 616         AuTraceErr(err);
 617         if (err == -EOPNOTSUPP)
 618                 err = 0;
 619         if (unlikely(err))
 620                 goto out;
 621
 622         /*
 623          * strange behaviour from the users view,
 624          * particularry setattr case
 625          */
 626         dir = d_inode(dst_parent);
 627         if (au_ibtop(dir) == cpg->bdst)
 628                 au_cpup_attr_nlink(dir, /*force*/1);
 629         inode = d_inode(cpg->dentry);
 630         au_cpup_attr_nlink(inode, /*force*/1);
 631
 632 out:
 633         return err;
 634 }
 635
 636 static noinline_for_stack
 637 int cpup_entry(struct au_cp_generic *cpg, struct dentry *dst_parent,
 638                struct au_cpup_reg_attr *h_src_attr)
 639 {
 640         int err;
 641         umode_t mode;
 642         unsigned int mnt_flags;
 643         unsigned char isdir, isreg, force;
 644         const unsigned char do_dt = !!au_ftest_cpup(cpg->flags, DTIME);
 645         struct au_dtime dt;
 646         struct path h_path;
 647         struct dentry *h_src, *h_dst, *h_parent;
 648         struct inode *h_inode, *h_dir;
 649         struct super_block *sb;
 650
 651         /* bsrc branch can be ro/rw. */
 652         h_src = au_h_dptr(cpg->dentry, cpg->bsrc);
 653         h_inode = d_inode(h_src);
 654         AuDebugOn(h_inode != au_h_iptr(d_inode(cpg->dentry), cpg->bsrc));
 655
 656         /* try stopping to be referenced while we are creating */
 657         h_dst = au_h_dptr(cpg->dentry, cpg->bdst);
 658         if (au_ftest_cpup(cpg->flags, RENAME))
 659                 AuDebugOn(strncmp(h_dst->d_name.name, AUFS_WH_PFX,
 660                                   AUFS_WH_PFX_LEN));
 661         h_parent = h_dst->d_parent; /* dir inode is locked */
 662         h_dir = d_inode(h_parent);
 663         IMustLock(h_dir);
 664         AuDebugOn(h_parent != h_dst->d_parent);
 665
 666         sb = cpg->dentry->d_sb;
 667         h_path.mnt = au_sbr_mnt(sb, cpg->bdst);
 668         if (do_dt) {
 669                 h_path.dentry = h_parent;
 670                 au_dtime_store(&dt, dst_parent, &h_path);
 671         }
 672         h_path.dentry = h_dst;
 673
 674         isreg = 0;
 675         isdir = 0;
 676         mode = h_inode->i_mode;
 677         switch (mode & S_IFMT) {
 678         case S_IFREG:
 679                 isreg = 1;
 680                 err = vfsub_create(h_dir, &h_path, S_IRUSR | S_IWUSR,
 681                                    /*want_excl*/true);
 682                 if (!err)
 683                         err = au_do_cpup_regular(cpg, h_src_attr);
 684                 break;
 685         case S_IFDIR:
 686                 isdir = 1;
 687                 err = vfsub_mkdir(h_dir, &h_path, mode);
 688                 if (!err)
 689                         err = au_do_cpup_dir(cpg, dst_parent, h_dir, &h_path);
 690                 break;
 691         case S_IFLNK:
 692                 err = au_do_cpup_symlink(&h_path, h_src, h_dir);
 693                 break;
 694         case S_IFCHR:
 695         case S_IFBLK:
 696                 AuDebugOn(!capable(CAP_MKNOD));
 697                 /*FALLTHROUGH*/
 698         case S_IFIFO:
 699         case S_IFSOCK:
 700                 err = vfsub_mknod(h_dir, &h_path, mode, h_inode->i_rdev);
 701                 break;
 702         default:
 703                 AuIOErr("Unknown inode type 0%o\n", mode);
 704                 err = -EIO;
 705         }
 706         if (!err)
 707                 err = au_reset_acl(h_dir, &h_path, mode);
 708
 709         mnt_flags = au_mntflags(sb);
 710         if (!au_opt_test(mnt_flags, UDBA_NONE)
 711             && !isdir
 712             && au_opt_test(mnt_flags, XINO)
 713             && (h_inode->i_nlink == 1
 714                 || (h_inode->i_state & I_LINKABLE))
 715             /* todo: unnecessary? */
 716             /* && d_inode(cpg->dentry)->i_nlink == 1 */
 717             && cpg->bdst < cpg->bsrc
 718             && !au_ftest_cpup(cpg->flags, KEEPLINO))
 719                 au_xino_write(sb, cpg->bsrc, h_inode->i_ino, /*ino*/0);
 720                 /* ignore this error */
 721
 722         if (!err) {
 723                 force = 0;
 724                 if (isreg) {
 725                         force = !!cpg->len;
 726                         if (cpg->len == -1)
 727                                 force = !!i_size_read(h_inode);
 728                 }
 729                 au_fhsm_wrote(sb, cpg->bdst, force);
 730         }
 731
 732         if (do_dt)
 733                 au_dtime_revert(&dt);
 734         return err;
 735 }
 736
 737 static int au_do_ren_after_cpup(struct au_cp_generic *cpg, struct path *h_path)
 738 {
 739         int err;
 740         struct dentry *dentry, *h_dentry, *h_parent, *parent;
 741         struct inode *h_dir;
 742         aufs_bindex_t bdst;
 743
 744         dentry = cpg->dentry;
 745         bdst = cpg->bdst;
 746         h_dentry = au_h_dptr(dentry, bdst);
 747         if (!au_ftest_cpup(cpg->flags, OVERWRITE)) {
 748                 dget(h_dentry);
 749                 au_set_h_dptr(dentry, bdst, NULL);
 750                 err = au_lkup_neg(dentry, bdst, /*wh*/0);
 751                 if (!err)
 752                         h_path->dentry = dget(au_h_dptr(dentry, bdst));
 753                 au_set_h_dptr(dentry, bdst, h_dentry);
 754         } else {
 755                 err = 0;
 756                 parent = dget_parent(dentry);
 757                 h_parent = au_h_dptr(parent, bdst);
 758                 dput(parent);
 759                 h_path->dentry = vfsub_lkup_one(&dentry->d_name, h_parent);
 760                 if (IS_ERR(h_path->dentry))
 761                         err = PTR_ERR(h_path->dentry);
 762         }
 763         if (unlikely(err))
 764                 goto out;
 765
 766         h_parent = h_dentry->d_parent; /* dir inode is locked */
 767         h_dir = d_inode(h_parent);
 768         IMustLock(h_dir);
 769         AuDbg("%pd %pd\n", h_dentry, h_path->dentry);
 770         /* no delegation since it is just created */
 771         err = vfsub_rename(h_dir, h_dentry, h_dir, h_path, /*delegated*/NULL,
 772                            /*flags*/0);
 773         dput(h_path->dentry);
 774
 775 out:
 776         return err;
 777 }
 778
 779 /*
 780  * copyup the @dentry from @bsrc to @bdst.
 781  * the caller must set the both of lower dentries.
 782  * @len is for truncating when it is -1 copyup the entire file.
 783  * in link/rename cases, @dst_parent may be different from the real one.
 784  * basic->bsrc can be larger than basic->bdst.
 785  * aufs doesn't touch the credential so
 786  * security_inode_copy_up{,_xattr}() are unnecrssary.
 787  */
 788 static int au_cpup_single(struct au_cp_generic *cpg, struct dentry *dst_parent)
 789 {
 790         int err, rerr;
 791         aufs_bindex_t old_ibtop;
 792         unsigned char isdir, plink;
 793         struct dentry *h_src, *h_dst, *h_parent;
 794         struct inode *dst_inode, *h_dir, *inode, *delegated, *src_inode;
 795         struct super_block *sb;
 796         struct au_branch *br;
 797         /* to reuduce stack size */
 798         struct {
 799                 struct au_dtime dt;
 800                 struct path h_path;
 801                 struct au_cpup_reg_attr h_src_attr;
 802         } *a;
 803
 804         err = -ENOMEM;
 805         a = kmalloc(sizeof(*a), GFP_NOFS);
 806         if (unlikely(!a))
 807                 goto out;
 808         a->h_src_attr.valid = 0;
 809
 810         sb = cpg->dentry->d_sb;
 811         br = au_sbr(sb, cpg->bdst);
 812         a->h_path.mnt = au_br_mnt(br);
 813         h_dst = au_h_dptr(cpg->dentry, cpg->bdst);
 814         h_parent = h_dst->d_parent; /* dir inode is locked */
 815         h_dir = d_inode(h_parent);
 816         IMustLock(h_dir);
 817
 818         h_src = au_h_dptr(cpg->dentry, cpg->bsrc);
 819         inode = d_inode(cpg->dentry);
 820
 821         if (!dst_parent)
 822                 dst_parent = dget_parent(cpg->dentry);
 823         else
 824                 dget(dst_parent);
 825
 826         plink = !!au_opt_test(au_mntflags(sb), PLINK);
 827         dst_inode = au_h_iptr(inode, cpg->bdst);
 828         if (dst_inode) {
 829                 if (unlikely(!plink)) {
 830                         err = -EIO;
 831                         AuIOErr("hi%lu(i%lu) exists on b%d "
 832                                 "but plink is disabled\n",
 833                                 dst_inode->i_ino, inode->i_ino, cpg->bdst);
 834                         goto out_parent;
 835                 }
 836
 837                 if (dst_inode->i_nlink) {
 838                         const int do_dt = au_ftest_cpup(cpg->flags, DTIME);
 839
 840                         h_src = au_plink_lkup(inode, cpg->bdst);
 841                         err = PTR_ERR(h_src);
 842                         if (IS_ERR(h_src))
 843                                 goto out_parent;
 844                         if (unlikely(d_is_negative(h_src))) {
 845                                 err = -EIO;
 846                                 AuIOErr("i%lu exists on b%d "
 847                                         "but not pseudo-linked\n",
 848                                         inode->i_ino, cpg->bdst);
 849                                 dput(h_src);
 850                                 goto out_parent;
 851                         }
 852
 853                         if (do_dt) {
 854                                 a->h_path.dentry = h_parent;
 855                                 au_dtime_store(&a->dt, dst_parent, &a->h_path);
 856                         }
 857
 858                         a->h_path.dentry = h_dst;
 859                         delegated = NULL;
 860                         err = vfsub_link(h_src, h_dir, &a->h_path, &delegated);
 861                         if (!err && au_ftest_cpup(cpg->flags, RENAME))
 862                                 err = au_do_ren_after_cpup(cpg, &a->h_path);
 863                         if (do_dt)
 864                                 au_dtime_revert(&a->dt);
 865                         if (unlikely(err == -EWOULDBLOCK)) {
 866                                 pr_warn("cannot retry for NFSv4 delegation"
 867                                         " for an internal link\n");
 868                                 iput(delegated);
 869                         }
 870                         dput(h_src);
 871                         goto out_parent;
 872                 } else
 873                         /* todo: cpup_wh_file? */
 874                         /* udba work */
 875                         au_update_ibrange(inode, /*do_put_zero*/1);
 876         }
 877
 878         isdir = S_ISDIR(inode->i_mode);
 879         old_ibtop = au_ibtop(inode);
 880         err = cpup_entry(cpg, dst_parent, &a->h_src_attr);
 881         if (unlikely(err))
 882                 goto out_rev;
 883         dst_inode = d_inode(h_dst);
 884         inode_lock_nested(dst_inode, AuLsc_I_CHILD2);
 885         /* todo: necessary? */
 886         /* au_pin_hdir_unlock(cpg->pin); */
 887
 888         err = cpup_iattr(cpg->dentry, cpg->bdst, h_src, &a->h_src_attr);
 889         if (unlikely(err)) {
 890                 /* todo: necessary? */
 891                 /* au_pin_hdir_relock(cpg->pin); */ /* ignore an error */
 892                 inode_unlock(dst_inode);
 893                 goto out_rev;
 894         }
 895
 896         if (cpg->bdst < old_ibtop) {
 897                 if (S_ISREG(inode->i_mode)) {
 898                         err = au_dy_iaop(inode, cpg->bdst, dst_inode);
 899                         if (unlikely(err)) {
 900                                 /* ignore an error */
 901                                 /* au_pin_hdir_relock(cpg->pin); */
 902                                 inode_unlock(dst_inode);
 903                                 goto out_rev;
 904                         }
 905                 }
 906                 au_set_ibtop(inode, cpg->bdst);
 907         } else
 908                 au_set_ibbot(inode, cpg->bdst);
 909         au_set_h_iptr(inode, cpg->bdst, au_igrab(dst_inode),
 910                       au_hi_flags(inode, isdir));
 911
 912         /* todo: necessary? */
 913         /* err = au_pin_hdir_relock(cpg->pin); */
 914         inode_unlock(dst_inode);
 915         if (unlikely(err))
 916                 goto out_rev;
 917
 918         src_inode = d_inode(h_src);
 919         if (!isdir
 920             && (src_inode->i_nlink > 1
 921                 || src_inode->i_state & I_LINKABLE)
 922             && plink)
 923                 au_plink_append(inode, cpg->bdst, h_dst);
 924
 925         if (au_ftest_cpup(cpg->flags, RENAME)) {
 926                 a->h_path.dentry = h_dst;
 927                 err = au_do_ren_after_cpup(cpg, &a->h_path);
 928         }
 929         if (!err)
 930                 goto out_parent; /* success */
 931
 932         /* revert */
 933 out_rev:
 934         a->h_path.dentry = h_parent;
 935         au_dtime_store(&a->dt, dst_parent, &a->h_path);
 936         a->h_path.dentry = h_dst;
 937         rerr = 0;
 938         if (d_is_positive(h_dst)) {
 939                 if (!isdir) {
 940                         /* no delegation since it is just created */
 941                         rerr = vfsub_unlink(h_dir, &a->h_path,
 942                                             /*delegated*/NULL, /*force*/0);
 943                 } else
 944                         rerr = vfsub_rmdir(h_dir, &a->h_path);
 945         }
 946         au_dtime_revert(&a->dt);
 947         if (rerr) {
 948                 AuIOErr("failed removing broken entry(%d, %d)\n", err, rerr);
 949                 err = -EIO;
 950         }
 951 out_parent:
 952         dput(dst_parent);
 953         kfree(a);
 954 out:
 955         return err;
 956 }
 957
 958 #if 0 /* reserved */
 959 struct au_cpup_single_args {
 960         int *errp;
 961         struct au_cp_generic *cpg;
 962         struct dentry *dst_parent;
 963 };
 964
 965 static void au_call_cpup_single(void *args)
 966 {
 967         struct au_cpup_single_args *a = args;
 968
 969         au_pin_hdir_acquire_nest(a->cpg->pin);
 970         *a->errp = au_cpup_single(a->cpg, a->dst_parent);
 971         au_pin_hdir_release(a->cpg->pin);
 972 }
 973 #endif
 974
 975 /*
 976  * prevent SIGXFSZ in copy-up.
 977  * testing CAP_MKNOD is for generic fs,
 978  * but CAP_FSETID is for xfs only, currently.
 979  */
 980 static int au_cpup_sio_test(struct au_pin *pin, umode_t mode)
 981 {
 982         int do_sio;
 983         struct super_block *sb;
 984         struct inode *h_dir;
 985
 986         do_sio = 0;
 987         sb = au_pinned_parent(pin)->d_sb;
 988         if (!au_wkq_test()
 989             && (!au_sbi(sb)->si_plink_maint_pid
 990                 || au_plink_maint(sb, AuLock_NOPLM))) {
 991                 switch (mode & S_IFMT) {
 992                 case S_IFREG:
 993                         /* no condition about RLIMIT_FSIZE and the file size */
 994                         do_sio = 1;
 995                         break;
 996                 case S_IFCHR:
 997                 case S_IFBLK:
 998                         do_sio = !capable(CAP_MKNOD);
 999                         break;
1000                 }
1001                 if (!do_sio)
1002                         do_sio = ((mode & (S_ISUID | S_ISGID))
1003                                   && !capable(CAP_FSETID));
1004                 /* this workaround may be removed in the future */
1005                 if (!do_sio) {
1006                         h_dir = au_pinned_h_dir(pin);
1007                         do_sio = h_dir->i_mode & S_ISVTX;
1008                 }
1009         }
1010
1011         return do_sio;
1012 }
1013
1014 #if 0 /* reserved */
1015 int au_sio_cpup_single(struct au_cp_generic *cpg, struct dentry *dst_parent)
1016 {
1017         int err, wkq_err;
1018         struct dentry *h_dentry;
1019
1020         h_dentry = au_h_dptr(cpg->dentry, cpg->bsrc);
1021         if (!au_cpup_sio_test(pin, d_inode(h_dentry)->i_mode))
1022                 err = au_cpup_single(cpg, dst_parent);
1023         else {
1024                 struct au_cpup_single_args args = {
1025                         .errp           = &err,
1026                         .cpg            = cpg,
1027                         .dst_parent     = dst_parent
1028                 };
1029                 wkq_err = au_wkq_wait(au_call_cpup_single, &args);
1030                 if (unlikely(wkq_err))
1031                         err = wkq_err;
1032         }
1033
1034         return err;
1035 }
1036 #endif
1037
1038 /*
1039  * copyup the @dentry from the first active lower branch to @bdst,
1040  * using au_cpup_single().
1041  */
1042 static int au_cpup_simple(struct au_cp_generic *cpg)
1043 {
1044         int err;
1045         unsigned int flags_orig;
1046         struct dentry *dentry;
1047
1048         AuDebugOn(cpg->bsrc < 0);
1049
1050         dentry = cpg->dentry;
1051         DiMustWriteLock(dentry);
1052
1053         err = au_lkup_neg(dentry, cpg->bdst, /*wh*/1);
1054         if (!err) {
1055                 flags_orig = cpg->flags;
1056                 au_fset_cpup(cpg->flags, RENAME);
1057                 err = au_cpup_single(cpg, NULL);
1058                 cpg->flags = flags_orig;
1059                 if (!err)
1060                         return 0; /* success */
1061
1062                 /* revert */
1063                 au_set_h_dptr(dentry, cpg->bdst, NULL);
1064                 au_set_dbtop(dentry, cpg->bsrc);
1065         }
1066
1067         return err;
1068 }
1069
1070 struct au_cpup_simple_args {
1071         int *errp;
1072         struct au_cp_generic *cpg;
1073 };
1074
1075 static void au_call_cpup_simple(void *args)
1076 {
1077         struct au_cpup_simple_args *a = args;
1078
1079         au_pin_hdir_acquire_nest(a->cpg->pin);
1080         *a->errp = au_cpup_simple(a->cpg);
1081         au_pin_hdir_release(a->cpg->pin);
1082 }
1083
1084 static int au_do_sio_cpup_simple(struct au_cp_generic *cpg)
1085 {
1086         int err, wkq_err;
1087         struct dentry *dentry, *parent;
1088         struct file *h_file;
1089         struct inode *h_dir;
1090
1091         dentry = cpg->dentry;
1092         h_file = NULL;
1093         if (au_ftest_cpup(cpg->flags, HOPEN)) {
1094                 AuDebugOn(cpg->bsrc < 0);
1095                 h_file = au_h_open_pre(dentry, cpg->bsrc, /*force_wr*/0);
1096                 err = PTR_ERR(h_file);
1097                 if (IS_ERR(h_file))
1098                         goto out;
1099         }
1100
1101         parent = dget_parent(dentry);
1102         h_dir = au_h_iptr(d_inode(parent), cpg->bdst);
1103         if (!au_test_h_perm_sio(h_dir, MAY_EXEC | MAY_WRITE)
1104             && !au_cpup_sio_test(cpg->pin, d_inode(dentry)->i_mode))
1105                 err = au_cpup_simple(cpg);
1106         else {
1107                 struct au_cpup_simple_args args = {
1108                         .errp           = &err,
1109                         .cpg            = cpg
1110                 };
1111                 wkq_err = au_wkq_wait(au_call_cpup_simple, &args);
1112                 if (unlikely(wkq_err))
1113                         err = wkq_err;
1114         }
1115
1116         dput(parent);
1117         if (h_file)
1118                 au_h_open_post(dentry, cpg->bsrc, h_file);
1119
1120 out:
1121         return err;
1122 }
1123
1124 int au_sio_cpup_simple(struct au_cp_generic *cpg)
1125 {
1126         aufs_bindex_t bsrc, bbot;
1127         struct dentry *dentry, *h_dentry;
1128
1129         if (cpg->bsrc < 0) {
1130                 dentry = cpg->dentry;
1131                 bbot = au_dbbot(dentry);
1132                 for (bsrc = cpg->bdst + 1; bsrc <= bbot; bsrc++) {
1133                         h_dentry = au_h_dptr(dentry, bsrc);
1134                         if (h_dentry) {
1135                                 AuDebugOn(d_is_negative(h_dentry));
1136                                 break;
1137                         }
1138                 }
1139                 AuDebugOn(bsrc > bbot);
1140                 cpg->bsrc = bsrc;
1141         }
1142         AuDebugOn(cpg->bsrc <= cpg->bdst);
1143         return au_do_sio_cpup_simple(cpg);
1144 }
1145
1146 int au_sio_cpdown_simple(struct au_cp_generic *cpg)
1147 {
1148         AuDebugOn(cpg->bdst <= cpg->bsrc);
1149         return au_do_sio_cpup_simple(cpg);
1150 }
1151
1152 /* ---------------------------------------------------------------------- */
1153
1154 /*
1155  * copyup the deleted file for writing.
1156  */
1157 static int au_do_cpup_wh(struct au_cp_generic *cpg, struct dentry *wh_dentry,
1158                          struct file *file)
1159 {
1160         int err;
1161         unsigned int flags_orig;
1162         aufs_bindex_t bsrc_orig;
1163         struct au_dinfo *dinfo;
1164         struct {
1165                 struct au_hdentry *hd;
1166                 struct dentry *h_dentry;
1167         } hdst, hsrc;
1168
1169         dinfo = au_di(cpg->dentry);
1170         AuRwMustWriteLock(&dinfo->di_rwsem);
1171
1172         bsrc_orig = cpg->bsrc;
1173         cpg->bsrc = dinfo->di_btop;
1174         hdst.hd = au_hdentry(dinfo, cpg->bdst);
1175         hdst.h_dentry = hdst.hd->hd_dentry;
1176         hdst.hd->hd_dentry = wh_dentry;
1177         dinfo->di_btop = cpg->bdst;
1178
1179         hsrc.h_dentry = NULL;
1180         if (file) {
1181                 hsrc.hd = au_hdentry(dinfo, cpg->bsrc);
1182                 hsrc.h_dentry = hsrc.hd->hd_dentry;
1183                 hsrc.hd->hd_dentry = au_hf_top(file)->f_path.dentry;
1184         }
1185         flags_orig = cpg->flags;
1186         cpg->flags = !AuCpup_DTIME;
1187         err = au_cpup_single(cpg, /*h_parent*/NULL);
1188         cpg->flags = flags_orig;
1189         if (file) {
1190                 if (!err)
1191                         err = au_reopen_nondir(file);
1192                 hsrc.hd->hd_dentry = hsrc.h_dentry;
1193         }
1194         hdst.hd->hd_dentry = hdst.h_dentry;
1195         dinfo->di_btop = cpg->bsrc;
1196         cpg->bsrc = bsrc_orig;
1197
1198         return err;
1199 }
1200
1201 static int au_cpup_wh(struct au_cp_generic *cpg, struct file *file)
1202 {
1203         int err;
1204         aufs_bindex_t bdst;
1205         struct au_dtime dt;
1206         struct dentry *dentry, *parent, *h_parent, *wh_dentry;
1207         struct au_branch *br;
1208         struct path h_path;
1209
1210         dentry = cpg->dentry;
1211         bdst = cpg->bdst;
1212         br = au_sbr(dentry->d_sb, bdst);
1213         parent = dget_parent(dentry);
1214         h_parent = au_h_dptr(parent, bdst);
1215         wh_dentry = au_whtmp_lkup(h_parent, br, &dentry->d_name);
1216         err = PTR_ERR(wh_dentry);
1217         if (IS_ERR(wh_dentry))
1218                 goto out;
1219
1220         h_path.dentry = h_parent;
1221         h_path.mnt = au_br_mnt(br);
1222         au_dtime_store(&dt, parent, &h_path);
1223         err = au_do_cpup_wh(cpg, wh_dentry, file);
1224         if (unlikely(err))
1225                 goto out_wh;
1226
1227         dget(wh_dentry);
1228         h_path.dentry = wh_dentry;
1229         if (!d_is_dir(wh_dentry)) {
1230                 /* no delegation since it is just created */
1231                 err = vfsub_unlink(d_inode(h_parent), &h_path,
1232                                    /*delegated*/NULL, /*force*/0);
1233         } else
1234                 err = vfsub_rmdir(d_inode(h_parent), &h_path);
1235         if (unlikely(err)) {
1236                 AuIOErr("failed remove copied-up tmp file %pd(%d)\n",
1237                         wh_dentry, err);
1238                 err = -EIO;
1239         }
1240         au_dtime_revert(&dt);
1241         au_set_hi_wh(d_inode(dentry), bdst, wh_dentry);
1242
1243 out_wh:
1244         dput(wh_dentry);
1245 out:
1246         dput(parent);
1247         return err;
1248 }
1249
1250 struct au_cpup_wh_args {
1251         int *errp;
1252         struct au_cp_generic *cpg;
1253         struct file *file;
1254 };
1255
1256 static void au_call_cpup_wh(void *args)
1257 {
1258         struct au_cpup_wh_args *a = args;
1259
1260         au_pin_hdir_acquire_nest(a->cpg->pin);
1261         *a->errp = au_cpup_wh(a->cpg, a->file);
1262         au_pin_hdir_release(a->cpg->pin);
1263 }
1264
1265 int au_sio_cpup_wh(struct au_cp_generic *cpg, struct file *file)
1266 {
1267         int err, wkq_err;
1268         aufs_bindex_t bdst;
1269         struct dentry *dentry, *parent, *h_orph, *h_parent;
1270         struct inode *dir, *h_dir, *h_tmpdir;
1271         struct au_wbr *wbr;
1272         struct au_pin wh_pin, *pin_orig;
1273
1274         dentry = cpg->dentry;
1275         bdst = cpg->bdst;
1276         parent = dget_parent(dentry);
1277         dir = d_inode(parent);
1278         h_orph = NULL;
1279         h_parent = NULL;
1280         h_dir = au_igrab(au_h_iptr(dir, bdst));
1281         h_tmpdir = h_dir;
1282         pin_orig = NULL;
1283         if (!h_dir->i_nlink) {
1284                 wbr = au_sbr(dentry->d_sb, bdst)->br_wbr;
1285                 h_orph = wbr->wbr_orph;
1286
1287                 h_parent = dget(au_h_dptr(parent, bdst));
1288                 au_set_h_dptr(parent, bdst, dget(h_orph));
1289                 h_tmpdir = d_inode(h_orph);
1290                 au_set_h_iptr(dir, bdst, au_igrab(h_tmpdir), /*flags*/0);
1291
1292                 inode_lock_nested(h_tmpdir, AuLsc_I_PARENT3);
1293                 /* todo: au_h_open_pre()? */
1294
1295                 pin_orig = cpg->pin;
1296                 au_pin_init(&wh_pin, dentry, bdst, AuLsc_DI_PARENT,
1297                             AuLsc_I_PARENT3, cpg->pin->udba, AuPin_DI_LOCKED);
1298                 cpg->pin = &wh_pin;
1299         }
1300
1301         if (!au_test_h_perm_sio(h_tmpdir, MAY_EXEC | MAY_WRITE)
1302             && !au_cpup_sio_test(cpg->pin, d_inode(dentry)->i_mode))
1303                 err = au_cpup_wh(cpg, file);
1304         else {
1305                 struct au_cpup_wh_args args = {
1306                         .errp   = &err,
1307                         .cpg    = cpg,
1308                         .file   = file
1309                 };
1310                 wkq_err = au_wkq_wait(au_call_cpup_wh, &args);
1311                 if (unlikely(wkq_err))
1312                         err = wkq_err;
1313         }
1314
1315         if (h_orph) {
1316                 inode_unlock(h_tmpdir);
1317                 /* todo: au_h_open_post()? */
1318                 au_set_h_iptr(dir, bdst, au_igrab(h_dir), /*flags*/0);
1319                 au_set_h_dptr(parent, bdst, h_parent);
1320                 AuDebugOn(!pin_orig);
1321                 cpg->pin = pin_orig;
1322         }
1323         iput(h_dir);
1324         dput(parent);
1325
1326         return err;
1327 }
1328
1329 /* ---------------------------------------------------------------------- */
1330
1331 /*
1332  * generic routine for both of copy-up and copy-down.
1333  */
1334 /* cf. revalidate function in file.c */
1335 int au_cp_dirs(struct dentry *dentry, aufs_bindex_t bdst,
1336                int (*cp)(struct dentry *dentry, aufs_bindex_t bdst,
1337                          struct au_pin *pin,
1338                          struct dentry *h_parent, void *arg),
1339                void *arg)
1340 {
1341         int err;
1342         struct au_pin pin;
1343         struct dentry *d, *parent, *h_parent, *real_parent, *h_dentry;
1344
1345         err = 0;
1346         parent = dget_parent(dentry);
1347         if (IS_ROOT(parent))
1348                 goto out;
1349
1350         au_pin_init(&pin, dentry, bdst, AuLsc_DI_PARENT2, AuLsc_I_PARENT2,
1351                     au_opt_udba(dentry->d_sb), AuPin_MNT_WRITE);
1352
1353         /* do not use au_dpage */
1354         real_parent = parent;
1355         while (1) {
1356                 dput(parent);
1357                 parent = dget_parent(dentry);
1358                 h_parent = au_h_dptr(parent, bdst);
1359                 if (h_parent)
1360                         goto out; /* success */
1361
1362                 /* find top dir which is necessary to cpup */
1363                 do {
1364                         d = parent;
1365                         dput(parent);
1366                         parent = dget_parent(d);
1367                         di_read_lock_parent3(parent, !AuLock_IR);
1368                         h_parent = au_h_dptr(parent, bdst);
1369                         di_read_unlock(parent, !AuLock_IR);
1370                 } while (!h_parent);
1371
1372                 if (d != real_parent)
1373                         di_write_lock_child3(d);
1374
1375                 /* somebody else might create while we were sleeping */
1376                 h_dentry = au_h_dptr(d, bdst);
1377                 if (!h_dentry || d_is_negative(h_dentry)) {
1378                         if (h_dentry)
1379                                 au_update_dbtop(d);
1380
1381                         au_pin_set_dentry(&pin, d);
1382                         err = au_do_pin(&pin);
1383                         if (!err) {
1384                                 err = cp(d, bdst, &pin, h_parent, arg);
1385                                 au_unpin(&pin);
1386                         }
1387                 }
1388
1389                 if (d != real_parent)
1390                         di_write_unlock(d);
1391                 if (unlikely(err))
1392                         break;
1393         }
1394
1395 out:
1396         dput(parent);
1397         return err;
1398 }
1399
1400 static int au_cpup_dir(struct dentry *dentry, aufs_bindex_t bdst,
1401                        struct au_pin *pin,
1402                        struct dentry *h_parent __maybe_unused,
1403                        void *arg __maybe_unused)
1404 {
1405         struct au_cp_generic cpg = {
1406                 .dentry = dentry,
1407                 .bdst   = bdst,
1408                 .bsrc   = -1,
1409                 .len    = 0,
1410                 .pin    = pin,
1411                 .flags  = AuCpup_DTIME
1412         };
1413         return au_sio_cpup_simple(&cpg);
1414 }
1415
1416 int au_cpup_dirs(struct dentry *dentry, aufs_bindex_t bdst)
1417 {
1418         return au_cp_dirs(dentry, bdst, au_cpup_dir, NULL);
1419 }
1420
1421 int au_test_and_cpup_dirs(struct dentry *dentry, aufs_bindex_t bdst)
1422 {
1423         int err;
1424         struct dentry *parent;
1425         struct inode *dir;
1426
1427         parent = dget_parent(dentry);
1428         dir = d_inode(parent);
1429         err = 0;
1430         if (au_h_iptr(dir, bdst))
1431                 goto out;
1432
1433         di_read_unlock(parent, AuLock_IR);
1434         di_write_lock_parent(parent);
1435         /* someone else might change our inode while we were sleeping */
1436         if (!au_h_iptr(dir, bdst))
1437                 err = au_cpup_dirs(dentry, bdst);
1438         di_downgrade_lock(parent, AuLock_IR);
1439
1440 out:
1441         dput(parent);
1442         return err;
1443 }