fs/aufs/dynop.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright (C) 2010-2021 Junjiro R. Okajima
   4  *
   5  * This program, aufs is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation; either version 2 of the License, or
   8  * (at your option) any later version.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13  * GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License
  16  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  17  */
  18
  19 /*
  20  * dynamically customizable operations for regular files
  21  */
  22
  23 #include "aufs.h"
  24
  25 #define DyPrSym(key)    AuDbgSym(key->dk_op.dy_hop)
  26
  27 /*
  28  * How large will these lists be?
  29  * Usually just a few elements, 20-30 at most for each, I guess.
  30  */
  31 static struct hlist_bl_head dynop[AuDyLast];
  32
  33 static struct au_dykey *dy_gfind_get(struct hlist_bl_head *hbl,
  34                                      const void *h_op)
  35 {
  36         struct au_dykey *key, *tmp;
  37         struct hlist_bl_node *pos;
  38
  39         key = NULL;
  40         hlist_bl_lock(hbl);
  41         hlist_bl_for_each_entry(tmp, pos, hbl, dk_hnode)
  42                 if (tmp->dk_op.dy_hop == h_op) {
  43                         if (kref_get_unless_zero(&tmp->dk_kref))
  44                                 key = tmp;
  45                         break;
  46                 }
  47         hlist_bl_unlock(hbl);
  48
  49         return key;
  50 }
  51
  52 static struct au_dykey *dy_bradd(struct au_branch *br, struct au_dykey *key)
  53 {
  54         struct au_dykey **k, *found;
  55         const void *h_op = key->dk_op.dy_hop;
  56         int i;
  57
  58         found = NULL;
  59         k = br->br_dykey;
  60         for (i = 0; i < AuBrDynOp; i++)
  61                 if (k[i]) {
  62                         if (k[i]->dk_op.dy_hop == h_op) {
  63                                 found = k[i];
  64                                 break;
  65                         }
  66                 } else
  67                         break;
  68         if (!found) {
  69                 spin_lock(&br->br_dykey_lock);
  70                 for (; i < AuBrDynOp; i++)
  71                         if (k[i]) {
  72                                 if (k[i]->dk_op.dy_hop == h_op) {
  73                                         found = k[i];
  74                                         break;
  75                                 }
  76                         } else {
  77                                 k[i] = key;
  78                                 break;
  79                         }
  80                 spin_unlock(&br->br_dykey_lock);
  81                 BUG_ON(i == AuBrDynOp); /* expand the array */
  82         }
  83
  84         return found;
  85 }
  86
  87 /* kref_get() if @key is already added */
  88 static struct au_dykey *dy_gadd(struct hlist_bl_head *hbl, struct au_dykey *key)
  89 {
  90         struct au_dykey *tmp, *found;
  91         struct hlist_bl_node *pos;
  92         const void *h_op = key->dk_op.dy_hop;
  93
  94         found = NULL;
  95         hlist_bl_lock(hbl);
  96         hlist_bl_for_each_entry(tmp, pos, hbl, dk_hnode)
  97                 if (tmp->dk_op.dy_hop == h_op) {
  98                         if (kref_get_unless_zero(&tmp->dk_kref))
  99                                 found = tmp;
 100                         break;
 101                 }
 102         if (!found)
 103                 hlist_bl_add_head(&key->dk_hnode, hbl);
 104         hlist_bl_unlock(hbl);
 105
 106         if (!found)
 107                 DyPrSym(key);
 108         return found;
 109 }
 110
 111 static void dy_free_rcu(struct rcu_head *rcu)
 112 {
 113         struct au_dykey *key;
 114
 115         key = container_of(rcu, struct au_dykey, dk_rcu);
 116         DyPrSym(key);
 117         kfree(key);
 118 }
 119
 120 static void dy_free(struct kref *kref)
 121 {
 122         struct au_dykey *key;
 123         struct hlist_bl_head *hbl;
 124
 125         key = container_of(kref, struct au_dykey, dk_kref);
 126         hbl = dynop + key->dk_op.dy_type;
 127         au_hbl_del(&key->dk_hnode, hbl);
 128         call_rcu(&key->dk_rcu, dy_free_rcu);
 129 }
 130
 131 void au_dy_put(struct au_dykey *key)
 132 {
 133         kref_put(&key->dk_kref, dy_free);
 134 }
 135
 136 /* ---------------------------------------------------------------------- */
 137
 138 #define DyDbgSize(cnt, op)      AuDebugOn(cnt != sizeof(op)/sizeof(void *))
 139
 140 #ifdef CONFIG_AUFS_DEBUG
 141 #define DyDbgDeclare(cnt)       unsigned int cnt = 0
 142 #define DyDbgInc(cnt)           do { cnt++; } while (0)
 143 #else
 144 #define DyDbgDeclare(cnt)       do {} while (0)
 145 #define DyDbgInc(cnt)           do {} while (0)
 146 #endif
 147
 148 #define DySet(func, dst, src, h_op, h_sb) do {                          \
 149         DyDbgInc(cnt);                                                  \
 150         if (h_op->func) {                                               \
 151                 if (src.func)                                           \
 152                         dst.func = src.func;                            \
 153                 else                                                    \
 154                         AuDbg("%s %s\n", au_sbtype(h_sb), #func);       \
 155         }                                                               \
 156 } while (0)
 157
 158 #define DySetForce(func, dst, src) do {         \
 159         AuDebugOn(!src.func);                   \
 160         DyDbgInc(cnt);                          \
 161         dst.func = src.func;                    \
 162 } while (0)
 163
 164 #define DySetAop(func) \
 165         DySet(func, dyaop->da_op, aufs_aop, h_aop, h_sb)
 166 #define DySetAopForce(func) \
 167         DySetForce(func, dyaop->da_op, aufs_aop)
 168
 169 static void dy_aop(struct au_dykey *key, const void *h_op,
 170                    struct super_block *h_sb __maybe_unused)
 171 {
 172         struct au_dyaop *dyaop = (void *)key;
 173         const struct address_space_operations *h_aop = h_op;
 174         DyDbgDeclare(cnt);
 175
 176         AuDbg("%s\n", au_sbtype(h_sb));
 177
 178         DySetAop(writepage);
 179         DySetAopForce(readpage);        /* force */
 180         DySetAop(writepages);
 181         DySetAop(set_page_dirty);
 182         DySetAop(readpages);
 183         DySetAop(readahead);
 184         DySetAop(write_begin);
 185         DySetAop(write_end);
 186         DySetAop(bmap);
 187         DySetAop(invalidatepage);
 188         DySetAop(releasepage);
 189         DySetAop(freepage);
 190         /* this one will be changed according to an aufs mount option */
 191         DySetAop(direct_IO);
 192         DySetAop(migratepage);
 193         DySetAop(isolate_page);
 194         DySetAop(putback_page);
 195         DySetAop(launder_page);
 196         DySetAop(is_partially_uptodate);
 197         DySetAop(is_dirty_writeback);
 198         DySetAop(error_remove_page);
 199         DySetAop(swap_activate);
 200         DySetAop(swap_deactivate);
 201
 202         DyDbgSize(cnt, *h_aop);
 203 }
 204
 205 /* ---------------------------------------------------------------------- */
 206
 207 static void dy_bug(struct kref *kref)
 208 {
 209         BUG();
 210 }
 211
 212 static struct au_dykey *dy_get(struct au_dynop *op, struct au_branch *br)
 213 {
 214         struct au_dykey *key, *old;
 215         struct hlist_bl_head *hbl;
 216         struct op {
 217                 unsigned int sz;
 218                 void (*set)(struct au_dykey *key, const void *h_op,
 219                             struct super_block *h_sb __maybe_unused);
 220         };
 221         static const struct op a[] = {
 222                 [AuDy_AOP] = {
 223                         .sz     = sizeof(struct au_dyaop),
 224                         .set    = dy_aop
 225                 }
 226         };
 227         const struct op *p;
 228
 229         hbl = dynop + op->dy_type;
 230         key = dy_gfind_get(hbl, op->dy_hop);
 231         if (key)
 232                 goto out_add; /* success */
 233
 234         p = a + op->dy_type;
 235         key = kzalloc(p->sz, GFP_NOFS);
 236         if (unlikely(!key)) {
 237                 key = ERR_PTR(-ENOMEM);
 238                 goto out;
 239         }
 240
 241         key->dk_op.dy_hop = op->dy_hop;
 242         kref_init(&key->dk_kref);
 243         p->set(key, op->dy_hop, au_br_sb(br));
 244         old = dy_gadd(hbl, key);
 245         if (old) {
 246                 au_kfree_rcu(key);
 247                 key = old;
 248         }
 249
 250 out_add:
 251         old = dy_bradd(br, key);
 252         if (old)
 253                 /* its ref-count should never be zero here */
 254                 kref_put(&key->dk_kref, dy_bug);
 255 out:
 256         return key;
 257 }
 258
 259 /* ---------------------------------------------------------------------- */
 260 /*
 261  * Aufs prohibits O_DIRECT by default even if the branch supports it.
 262  * This behaviour is necessary to return an error from open(O_DIRECT) instead
 263  * of the succeeding I/O. The dio mount option enables O_DIRECT and makes
 264  * open(O_DIRECT) always succeed, but the succeeding I/O may return an error.
 265  * See the aufs manual in detail.
 266  */
 267 static void dy_adx(struct au_dyaop *dyaop, int do_dx)
 268 {
 269         if (!do_dx)
 270                 dyaop->da_op.direct_IO = NULL;
 271         else
 272                 dyaop->da_op.direct_IO = aufs_aop.direct_IO;
 273 }
 274
 275 static struct au_dyaop *dy_aget(struct au_branch *br,
 276                                 const struct address_space_operations *h_aop,
 277                                 int do_dx)
 278 {
 279         struct au_dyaop *dyaop;
 280         struct au_dynop op;
 281
 282         op.dy_type = AuDy_AOP;
 283         op.dy_haop = h_aop;
 284         dyaop = (void *)dy_get(&op, br);
 285         if (IS_ERR(dyaop))
 286                 goto out;
 287         dy_adx(dyaop, do_dx);
 288
 289 out:
 290         return dyaop;
 291 }
 292
 293 int au_dy_iaop(struct inode *inode, aufs_bindex_t bindex,
 294                 struct inode *h_inode)
 295 {
 296         int err, do_dx;
 297         struct super_block *sb;
 298         struct au_branch *br;
 299         struct au_dyaop *dyaop;
 300
 301         AuDebugOn(!S_ISREG(h_inode->i_mode));
 302         IiMustWriteLock(inode);
 303
 304         sb = inode->i_sb;
 305         br = au_sbr(sb, bindex);
 306         do_dx = !!au_opt_test(au_mntflags(sb), DIO);
 307         dyaop = dy_aget(br, h_inode->i_mapping->a_ops, do_dx);
 308         err = PTR_ERR(dyaop);
 309         if (IS_ERR(dyaop))
 310                 /* unnecessary to call dy_fput() */
 311                 goto out;
 312
 313         err = 0;
 314         inode->i_mapping->a_ops = &dyaop->da_op;
 315
 316 out:
 317         return err;
 318 }
 319
 320 /*
 321  * Is it safe to replace a_ops during the inode/file is in operation?
 322  * Yes, I hope so.
 323  */
 324 int au_dy_irefresh(struct inode *inode)
 325 {
 326         int err;
 327         aufs_bindex_t btop;
 328         struct inode *h_inode;
 329
 330         err = 0;
 331         if (S_ISREG(inode->i_mode)) {
 332                 btop = au_ibtop(inode);
 333                 h_inode = au_h_iptr(inode, btop);
 334                 err = au_dy_iaop(inode, btop, h_inode);
 335         }
 336         return err;
 337 }
 338
 339 void au_dy_arefresh(int do_dx)
 340 {
 341         struct hlist_bl_head *hbl;
 342         struct hlist_bl_node *pos;
 343         struct au_dykey *key;
 344
 345         hbl = dynop + AuDy_AOP;
 346         hlist_bl_lock(hbl);
 347         hlist_bl_for_each_entry(key, pos, hbl, dk_hnode)
 348                 dy_adx((void *)key, do_dx);
 349         hlist_bl_unlock(hbl);
 350 }
 351
 352 /* ---------------------------------------------------------------------- */
 353
 354 void __init au_dy_init(void)
 355 {
 356         int i;
 357
 358         for (i = 0; i < AuDyLast; i++)
 359                 INIT_HLIST_BL_HEAD(dynop + i);
 360 }
 361
 362 void au_dy_fin(void)
 363 {
 364         int i;
 365
 366         for (i = 0; i < AuDyLast; i++)
 367                 WARN_ON(!hlist_bl_empty(dynop + i));
 368 }