dm-mirror-y += dm-raid1.o
dm-log-userspace-y \
+= dm-log-userspace-base.o dm-log-userspace-transfer.o
+dm-bio-prison-y += dm-bio-prison-v1.o dm-bio-prison-v2.o
dm-thin-pool-y += dm-thin.o dm-thin-metadata.o
dm-cache-y += dm-cache-target.o dm-cache-metadata.o dm-cache-policy.o
dm-cache-smq-y += dm-cache-policy-smq.o
--- /dev/null
+/*
+ * Copyright (C) 2012 Red Hat, Inc.
+ *
+ * This file is released under the GPL.
+ */
+
+#include "dm.h"
+#include "dm-bio-prison-v1.h"
+#include "dm-bio-prison-v2.h"
+
+#include <linux/spinlock.h>
+#include <linux/mempool.h>
+#include <linux/module.h>
+#include <linux/slab.h>
+
+/*----------------------------------------------------------------*/
+
+#define MIN_CELLS 1024
+
+struct dm_bio_prison {
+ spinlock_t lock;
+ mempool_t *cell_pool;
+ struct rb_root cells;
+};
+
+static struct kmem_cache *_cell_cache;
+
+/*----------------------------------------------------------------*/
+
+/*
+ * @nr_cells should be the number of cells you want in use _concurrently_.
+ * Don't confuse it with the number of distinct keys.
+ */
+struct dm_bio_prison *dm_bio_prison_create(void)
+{
+ struct dm_bio_prison *prison = kmalloc(sizeof(*prison), GFP_KERNEL);
+
+ if (!prison)
+ return NULL;
+
+ spin_lock_init(&prison->lock);
+
+ prison->cell_pool = mempool_create_slab_pool(MIN_CELLS, _cell_cache);
+ if (!prison->cell_pool) {
+ kfree(prison);
+ return NULL;
+ }
+
+ prison->cells = RB_ROOT;
+
+ return prison;
+}
+EXPORT_SYMBOL_GPL(dm_bio_prison_create);
+
+void dm_bio_prison_destroy(struct dm_bio_prison *prison)
+{
+ mempool_destroy(prison->cell_pool);
+ kfree(prison);
+}
+EXPORT_SYMBOL_GPL(dm_bio_prison_destroy);
+
+struct dm_bio_prison_cell *dm_bio_prison_alloc_cell(struct dm_bio_prison *prison, gfp_t gfp)
+{
+ return mempool_alloc(prison->cell_pool, gfp);
+}
+EXPORT_SYMBOL_GPL(dm_bio_prison_alloc_cell);
+
+void dm_bio_prison_free_cell(struct dm_bio_prison *prison,
+ struct dm_bio_prison_cell *cell)
+{
+ mempool_free(cell, prison->cell_pool);
+}
+EXPORT_SYMBOL_GPL(dm_bio_prison_free_cell);
+
+static void __setup_new_cell(struct dm_cell_key *key,
+ struct bio *holder,
+ struct dm_bio_prison_cell *cell)
+{
+ memcpy(&cell->key, key, sizeof(cell->key));
+ cell->holder = holder;
+ bio_list_init(&cell->bios);
+}
+
+static int cmp_keys(struct dm_cell_key *lhs,
+ struct dm_cell_key *rhs)
+{
+ if (lhs->virtual < rhs->virtual)
+ return -1;
+
+ if (lhs->virtual > rhs->virtual)
+ return 1;
+
+ if (lhs->dev < rhs->dev)
+ return -1;
+
+ if (lhs->dev > rhs->dev)
+ return 1;
+
+ if (lhs->block_end <= rhs->block_begin)
+ return -1;
+
+ if (lhs->block_begin >= rhs->block_end)
+ return 1;
+
+ return 0;
+}
+
+static int __bio_detain(struct dm_bio_prison *prison,
+ struct dm_cell_key *key,
+ struct bio *inmate,
+ struct dm_bio_prison_cell *cell_prealloc,
+ struct dm_bio_prison_cell **cell_result)
+{
+ int r;
+ struct rb_node **new = &prison->cells.rb_node, *parent = NULL;
+
+ while (*new) {
+ struct dm_bio_prison_cell *cell =
+ container_of(*new, struct dm_bio_prison_cell, node);
+
+ r = cmp_keys(key, &cell->key);
+
+ parent = *new;
+ if (r < 0)
+ new = &((*new)->rb_left);
+ else if (r > 0)
+ new = &((*new)->rb_right);
+ else {
+ if (inmate)
+ bio_list_add(&cell->bios, inmate);
+ *cell_result = cell;
+ return 1;
+ }
+ }
+
+ __setup_new_cell(key, inmate, cell_prealloc);
+ *cell_result = cell_prealloc;
+
+ rb_link_node(&cell_prealloc->node, parent, new);
+ rb_insert_color(&cell_prealloc->node, &prison->cells);
+
+ return 0;
+}
+
+static int bio_detain(struct dm_bio_prison *prison,
+ struct dm_cell_key *key,
+ struct bio *inmate,
+ struct dm_bio_prison_cell *cell_prealloc,
+ struct dm_bio_prison_cell **cell_result)
+{
+ int r;
+ unsigned long flags;
+
+ spin_lock_irqsave(&prison->lock, flags);
+ r = __bio_detain(prison, key, inmate, cell_prealloc, cell_result);
+ spin_unlock_irqrestore(&prison->lock, flags);
+
+ return r;
+}
+
+int dm_bio_detain(struct dm_bio_prison *prison,
+ struct dm_cell_key *key,
+ struct bio *inmate,
+ struct dm_bio_prison_cell *cell_prealloc,
+ struct dm_bio_prison_cell **cell_result)
+{
+ return bio_detain(prison, key, inmate, cell_prealloc, cell_result);
+}
+EXPORT_SYMBOL_GPL(dm_bio_detain);
+
+int dm_get_cell(struct dm_bio_prison *prison,
+ struct dm_cell_key *key,
+ struct dm_bio_prison_cell *cell_prealloc,
+ struct dm_bio_prison_cell **cell_result)
+{
+ return bio_detain(prison, key, NULL, cell_prealloc, cell_result);
+}
+EXPORT_SYMBOL_GPL(dm_get_cell);
+
+/*
+ * @inmates must have been initialised prior to this call
+ */
+static void __cell_release(struct dm_bio_prison *prison,
+ struct dm_bio_prison_cell *cell,
+ struct bio_list *inmates)
+{
+ rb_erase(&cell->node, &prison->cells);
+
+ if (inmates) {
+ if (cell->holder)
+ bio_list_add(inmates, cell->holder);
+ bio_list_merge(inmates, &cell->bios);
+ }
+}
+
+void dm_cell_release(struct dm_bio_prison *prison,
+ struct dm_bio_prison_cell *cell,
+ struct bio_list *bios)
+{
+ unsigned long flags;
+
+ spin_lock_irqsave(&prison->lock, flags);
+ __cell_release(prison, cell, bios);
+ spin_unlock_irqrestore(&prison->lock, flags);
+}
+EXPORT_SYMBOL_GPL(dm_cell_release);
+
+/*
+ * Sometimes we don't want the holder, just the additional bios.
+ */
+static void __cell_release_no_holder(struct dm_bio_prison *prison,
+ struct dm_bio_prison_cell *cell,
+ struct bio_list *inmates)
+{
+ rb_erase(&cell->node, &prison->cells);
+ bio_list_merge(inmates, &cell->bios);
+}
+
+void dm_cell_release_no_holder(struct dm_bio_prison *prison,
+ struct dm_bio_prison_cell *cell,
+ struct bio_list *inmates)
+{
+ unsigned long flags;
+
+ spin_lock_irqsave(&prison->lock, flags);
+ __cell_release_no_holder(prison, cell, inmates);
+ spin_unlock_irqrestore(&prison->lock, flags);
+}
+EXPORT_SYMBOL_GPL(dm_cell_release_no_holder);
+
+void dm_cell_error(struct dm_bio_prison *prison,
+ struct dm_bio_prison_cell *cell, int error)
+{
+ struct bio_list bios;
+ struct bio *bio;
+
+ bio_list_init(&bios);
+ dm_cell_release(prison, cell, &bios);
+
+ while ((bio = bio_list_pop(&bios))) {
+ bio->bi_error = error;
+ bio_endio(bio);
+ }
+}
+EXPORT_SYMBOL_GPL(dm_cell_error);
+
+void dm_cell_visit_release(struct dm_bio_prison *prison,
+ void (*visit_fn)(void *, struct dm_bio_prison_cell *),
+ void *context,
+ struct dm_bio_prison_cell *cell)
+{
+ unsigned long flags;
+
+ spin_lock_irqsave(&prison->lock, flags);
+ visit_fn(context, cell);
+ rb_erase(&cell->node, &prison->cells);
+ spin_unlock_irqrestore(&prison->lock, flags);
+}
+EXPORT_SYMBOL_GPL(dm_cell_visit_release);
+
+static int __promote_or_release(struct dm_bio_prison *prison,
+ struct dm_bio_prison_cell *cell)
+{
+ if (bio_list_empty(&cell->bios)) {
+ rb_erase(&cell->node, &prison->cells);
+ return 1;
+ }
+
+ cell->holder = bio_list_pop(&cell->bios);
+ return 0;
+}
+
+int dm_cell_promote_or_release(struct dm_bio_prison *prison,
+ struct dm_bio_prison_cell *cell)
+{
+ int r;
+ unsigned long flags;
+
+ spin_lock_irqsave(&prison->lock, flags);
+ r = __promote_or_release(prison, cell);
+ spin_unlock_irqrestore(&prison->lock, flags);
+
+ return r;
+}
+EXPORT_SYMBOL_GPL(dm_cell_promote_or_release);
+
+/*----------------------------------------------------------------*/
+
+#define DEFERRED_SET_SIZE 64
+
+struct dm_deferred_entry {
+ struct dm_deferred_set *ds;
+ unsigned count;
+ struct list_head work_items;
+};
+
+struct dm_deferred_set {
+ spinlock_t lock;
+ unsigned current_entry;
+ unsigned sweeper;
+ struct dm_deferred_entry entries[DEFERRED_SET_SIZE];
+};
+
+struct dm_deferred_set *dm_deferred_set_create(void)
+{
+ int i;
+ struct dm_deferred_set *ds;
+
+ ds = kmalloc(sizeof(*ds), GFP_KERNEL);
+ if (!ds)
+ return NULL;
+
+ spin_lock_init(&ds->lock);
+ ds->current_entry = 0;
+ ds->sweeper = 0;
+ for (i = 0; i < DEFERRED_SET_SIZE; i++) {
+ ds->entries[i].ds = ds;
+ ds->entries[i].count = 0;
+ INIT_LIST_HEAD(&ds->entries[i].work_items);
+ }
+
+ return ds;
+}
+EXPORT_SYMBOL_GPL(dm_deferred_set_create);
+
+void dm_deferred_set_destroy(struct dm_deferred_set *ds)
+{
+ kfree(ds);
+}
+EXPORT_SYMBOL_GPL(dm_deferred_set_destroy);
+
+struct dm_deferred_entry *dm_deferred_entry_inc(struct dm_deferred_set *ds)
+{
+ unsigned long flags;
+ struct dm_deferred_entry *entry;
+
+ spin_lock_irqsave(&ds->lock, flags);
+ entry = ds->entries + ds->current_entry;
+ entry->count++;
+ spin_unlock_irqrestore(&ds->lock, flags);
+
+ return entry;
+}
+EXPORT_SYMBOL_GPL(dm_deferred_entry_inc);
+
+static unsigned ds_next(unsigned index)
+{
+ return (index + 1) % DEFERRED_SET_SIZE;
+}
+
+static void __sweep(struct dm_deferred_set *ds, struct list_head *head)
+{
+ while ((ds->sweeper != ds->current_entry) &&
+ !ds->entries[ds->sweeper].count) {
+ list_splice_init(&ds->entries[ds->sweeper].work_items, head);
+ ds->sweeper = ds_next(ds->sweeper);
+ }
+
+ if ((ds->sweeper == ds->current_entry) && !ds->entries[ds->sweeper].count)
+ list_splice_init(&ds->entries[ds->sweeper].work_items, head);
+}
+
+void dm_deferred_entry_dec(struct dm_deferred_entry *entry, struct list_head *head)
+{
+ unsigned long flags;
+
+ spin_lock_irqsave(&entry->ds->lock, flags);
+ BUG_ON(!entry->count);
+ --entry->count;
+ __sweep(entry->ds, head);
+ spin_unlock_irqrestore(&entry->ds->lock, flags);
+}
+EXPORT_SYMBOL_GPL(dm_deferred_entry_dec);
+
+/*
+ * Returns 1 if deferred or 0 if no pending items to delay job.
+ */
+int dm_deferred_set_add_work(struct dm_deferred_set *ds, struct list_head *work)
+{
+ int r = 1;
+ unsigned long flags;
+ unsigned next_entry;
+
+ spin_lock_irqsave(&ds->lock, flags);
+ if ((ds->sweeper == ds->current_entry) &&
+ !ds->entries[ds->current_entry].count)
+ r = 0;
+ else {
+ list_add(work, &ds->entries[ds->current_entry].work_items);
+ next_entry = ds_next(ds->current_entry);
+ if (!ds->entries[next_entry].count)
+ ds->current_entry = next_entry;
+ }
+ spin_unlock_irqrestore(&ds->lock, flags);
+
+ return r;
+}
+EXPORT_SYMBOL_GPL(dm_deferred_set_add_work);
+
+/*----------------------------------------------------------------*/
+
+static int __init dm_bio_prison_init_v1(void)
+{
+ _cell_cache = KMEM_CACHE(dm_bio_prison_cell, 0);
+ if (!_cell_cache)
+ return -ENOMEM;
+
+ return 0;
+}
+
+static void dm_bio_prison_exit_v1(void)
+{
+ kmem_cache_destroy(_cell_cache);
+ _cell_cache = NULL;
+}
+
+static int (*_inits[])(void) __initdata = {
+ dm_bio_prison_init_v1,
+ dm_bio_prison_init_v2,
+};
+
+static void (*_exits[])(void) = {
+ dm_bio_prison_exit_v1,
+ dm_bio_prison_exit_v2,
+};
+
+static int __init dm_bio_prison_init(void)
+{
+ const int count = ARRAY_SIZE(_inits);
+
+ int r, i;
+
+ for (i = 0; i < count; i++) {
+ r = _inits[i]();
+ if (r)
+ goto bad;
+ }
+
+ return 0;
+
+ bad:
+ while (i--)
+ _exits[i]();
+
+ return r;
+}
+
+static void __exit dm_bio_prison_exit(void)
+{
+ int i = ARRAY_SIZE(_exits);
+
+ while (i--)
+ _exits[i]();
+}
+
+/*
+ * module hooks
+ */
+module_init(dm_bio_prison_init);
+module_exit(dm_bio_prison_exit);
+
+MODULE_DESCRIPTION(DM_NAME " bio prison");
+MODULE_AUTHOR("Joe Thornber <dm-devel@redhat.com>");
+MODULE_LICENSE("GPL");
--- /dev/null
+/*
+ * Copyright (C) 2011-2017 Red Hat, Inc.
+ *
+ * This file is released under the GPL.
+ */
+
+#ifndef DM_BIO_PRISON_H
+#define DM_BIO_PRISON_H
+
+#include "persistent-data/dm-block-manager.h" /* FIXME: for dm_block_t */
+#include "dm-thin-metadata.h" /* FIXME: for dm_thin_id */
+
+#include <linux/bio.h>
+#include <linux/rbtree.h>
+
+/*----------------------------------------------------------------*/
+
+/*
+ * Sometimes we can't deal with a bio straight away. We put them in prison
+ * where they can't cause any mischief. Bios are put in a cell identified
+ * by a key, multiple bios can be in the same cell. When the cell is
+ * subsequently unlocked the bios become available.
+ */
+struct dm_bio_prison;
+
+/*
+ * Keys define a range of blocks within either a virtual or physical
+ * device.
+ */
+struct dm_cell_key {
+ int virtual;
+ dm_thin_id dev;
+ dm_block_t block_begin, block_end;
+};
+
+/*
+ * Treat this as opaque, only in header so callers can manage allocation
+ * themselves.
+ */
+struct dm_bio_prison_cell {
+ struct list_head user_list; /* for client use */
+ struct rb_node node;
+
+ struct dm_cell_key key;
+ struct bio *holder;
+ struct bio_list bios;
+};
+
+struct dm_bio_prison *dm_bio_prison_create(void);
+void dm_bio_prison_destroy(struct dm_bio_prison *prison);
+
+/*
+ * These two functions just wrap a mempool. This is a transitory step:
+ * Eventually all bio prison clients should manage their own cell memory.
+ *
+ * Like mempool_alloc(), dm_bio_prison_alloc_cell() can only fail if called
+ * in interrupt context or passed GFP_NOWAIT.
+ */
+struct dm_bio_prison_cell *dm_bio_prison_alloc_cell(struct dm_bio_prison *prison,
+ gfp_t gfp);
+void dm_bio_prison_free_cell(struct dm_bio_prison *prison,
+ struct dm_bio_prison_cell *cell);
+
+/*
+ * Creates, or retrieves a cell that overlaps the given key.
+ *
+ * Returns 1 if pre-existing cell returned, zero if new cell created using
+ * @cell_prealloc.
+ */
+int dm_get_cell(struct dm_bio_prison *prison,
+ struct dm_cell_key *key,
+ struct dm_bio_prison_cell *cell_prealloc,
+ struct dm_bio_prison_cell **cell_result);
+
+/*
+ * An atomic op that combines retrieving or creating a cell, and adding a
+ * bio to it.
+ *
+ * Returns 1 if the cell was already held, 0 if @inmate is the new holder.
+ */
+int dm_bio_detain(struct dm_bio_prison *prison,
+ struct dm_cell_key *key,
+ struct bio *inmate,
+ struct dm_bio_prison_cell *cell_prealloc,
+ struct dm_bio_prison_cell **cell_result);
+
+void dm_cell_release(struct dm_bio_prison *prison,
+ struct dm_bio_prison_cell *cell,
+ struct bio_list *bios);
+void dm_cell_release_no_holder(struct dm_bio_prison *prison,
+ struct dm_bio_prison_cell *cell,
+ struct bio_list *inmates);
+void dm_cell_error(struct dm_bio_prison *prison,
+ struct dm_bio_prison_cell *cell, int error);
+
+/*
+ * Visits the cell and then releases. Guarantees no new inmates are
+ * inserted between the visit and release.
+ */
+void dm_cell_visit_release(struct dm_bio_prison *prison,
+ void (*visit_fn)(void *, struct dm_bio_prison_cell *),
+ void *context, struct dm_bio_prison_cell *cell);
+
+/*
+ * Rather than always releasing the prisoners in a cell, the client may
+ * want to promote one of them to be the new holder. There is a race here
+ * though between releasing an empty cell, and other threads adding new
+ * inmates. So this function makes the decision with its lock held.
+ *
+ * This function can have two outcomes:
+ * i) An inmate is promoted to be the holder of the cell (return value of 0).
+ * ii) The cell has no inmate for promotion and is released (return value of 1).
+ */
+int dm_cell_promote_or_release(struct dm_bio_prison *prison,
+ struct dm_bio_prison_cell *cell);
+
+/*----------------------------------------------------------------*/
+
+/*
+ * We use the deferred set to keep track of pending reads to shared blocks.
+ * We do this to ensure the new mapping caused by a write isn't performed
+ * until these prior reads have completed. Otherwise the insertion of the
+ * new mapping could free the old block that the read bios are mapped to.
+ */
+
+struct dm_deferred_set;
+struct dm_deferred_entry;
+
+struct dm_deferred_set *dm_deferred_set_create(void);
+void dm_deferred_set_destroy(struct dm_deferred_set *ds);
+
+struct dm_deferred_entry *dm_deferred_entry_inc(struct dm_deferred_set *ds);
+void dm_deferred_entry_dec(struct dm_deferred_entry *entry, struct list_head *head);
+int dm_deferred_set_add_work(struct dm_deferred_set *ds, struct list_head *work);
+
+/*----------------------------------------------------------------*/
+
+#endif
--- /dev/null
+/*
+ * Copyright (C) 2012-2017 Red Hat, Inc.
+ *
+ * This file is released under the GPL.
+ */
+
+#include "dm.h"
+#include "dm-bio-prison-v2.h"
+
+#include <linux/spinlock.h>
+#include <linux/mempool.h>
+#include <linux/module.h>
+#include <linux/slab.h>
+#include <linux/rwsem.h>
+
+/*----------------------------------------------------------------*/
+
+#define MIN_CELLS 1024
+
+struct dm_bio_prison_v2 {
+ struct workqueue_struct *wq;
+
+ spinlock_t lock;
+ mempool_t *cell_pool;
+ struct rb_root cells;
+};
+
+static struct kmem_cache *_cell_cache;
+
+/*----------------------------------------------------------------*/
+
+/*
+ * @nr_cells should be the number of cells you want in use _concurrently_.
+ * Don't confuse it with the number of distinct keys.
+ */
+struct dm_bio_prison_v2 *dm_bio_prison_create_v2(struct workqueue_struct *wq)
+{
+ struct dm_bio_prison_v2 *prison = kmalloc(sizeof(*prison), GFP_KERNEL);
+
+ if (!prison)
+ return NULL;
+
+ prison->wq = wq;
+ spin_lock_init(&prison->lock);
+
+ prison->cell_pool = mempool_create_slab_pool(MIN_CELLS, _cell_cache);
+ if (!prison->cell_pool) {
+ kfree(prison);
+ return NULL;
+ }
+
+ prison->cells = RB_ROOT;
+
+ return prison;
+}
+EXPORT_SYMBOL_GPL(dm_bio_prison_create_v2);
+
+void dm_bio_prison_destroy_v2(struct dm_bio_prison_v2 *prison)
+{
+ mempool_destroy(prison->cell_pool);
+ kfree(prison);
+}
+EXPORT_SYMBOL_GPL(dm_bio_prison_destroy_v2);
+
+struct dm_bio_prison_cell_v2 *dm_bio_prison_alloc_cell_v2(struct dm_bio_prison_v2 *prison, gfp_t gfp)
+{
+ return mempool_alloc(prison->cell_pool, gfp);
+}
+EXPORT_SYMBOL_GPL(dm_bio_prison_alloc_cell_v2);
+
+void dm_bio_prison_free_cell_v2(struct dm_bio_prison_v2 *prison,
+ struct dm_bio_prison_cell_v2 *cell)
+{
+ mempool_free(cell, prison->cell_pool);
+}
+EXPORT_SYMBOL_GPL(dm_bio_prison_free_cell_v2);
+
+static void __setup_new_cell(struct dm_cell_key_v2 *key,
+ struct dm_bio_prison_cell_v2 *cell)
+{
+ memset(cell, 0, sizeof(*cell));
+ memcpy(&cell->key, key, sizeof(cell->key));
+ bio_list_init(&cell->bios);
+}
+
+static int cmp_keys(struct dm_cell_key_v2 *lhs,
+ struct dm_cell_key_v2 *rhs)
+{
+ if (lhs->virtual < rhs->virtual)
+ return -1;
+
+ if (lhs->virtual > rhs->virtual)
+ return 1;
+
+ if (lhs->dev < rhs->dev)
+ return -1;
+
+ if (lhs->dev > rhs->dev)
+ return 1;
+
+ if (lhs->block_end <= rhs->block_begin)
+ return -1;
+
+ if (lhs->block_begin >= rhs->block_end)
+ return 1;
+
+ return 0;
+}
+
+/*
+ * Returns true if node found, otherwise it inserts a new one.
+ */
+static bool __find_or_insert(struct dm_bio_prison_v2 *prison,
+ struct dm_cell_key_v2 *key,
+ struct dm_bio_prison_cell_v2 *cell_prealloc,
+ struct dm_bio_prison_cell_v2 **result)
+{
+ int r;
+ struct rb_node **new = &prison->cells.rb_node, *parent = NULL;
+
+ while (*new) {
+ struct dm_bio_prison_cell_v2 *cell =
+ container_of(*new, struct dm_bio_prison_cell_v2, node);
+
+ r = cmp_keys(key, &cell->key);
+
+ parent = *new;
+ if (r < 0)
+ new = &((*new)->rb_left);
+
+ else if (r > 0)
+ new = &((*new)->rb_right);
+
+ else {
+ *result = cell;
+ return true;
+ }
+ }
+
+ __setup_new_cell(key, cell_prealloc);
+ *result = cell_prealloc;
+ rb_link_node(&cell_prealloc->node, parent, new);
+ rb_insert_color(&cell_prealloc->node, &prison->cells);
+
+ return false;
+}
+
+static bool __get(struct dm_bio_prison_v2 *prison,
+ struct dm_cell_key_v2 *key,
+ unsigned lock_level,
+ struct bio *inmate,
+ struct dm_bio_prison_cell_v2 *cell_prealloc,
+ struct dm_bio_prison_cell_v2 **cell)
+{
+ if (__find_or_insert(prison, key, cell_prealloc, cell)) {
+ if ((*cell)->exclusive_lock) {
+ if (lock_level <= (*cell)->exclusive_level) {
+ bio_list_add(&(*cell)->bios, inmate);
+ return false;
+ }
+ }
+
+ (*cell)->shared_count++;
+
+ } else
+ (*cell)->shared_count = 1;
+
+ return true;
+}
+
+bool dm_cell_get_v2(struct dm_bio_prison_v2 *prison,
+ struct dm_cell_key_v2 *key,
+ unsigned lock_level,
+ struct bio *inmate,
+ struct dm_bio_prison_cell_v2 *cell_prealloc,
+ struct dm_bio_prison_cell_v2 **cell_result)
+{
+ int r;
+ unsigned long flags;
+
+ spin_lock_irqsave(&prison->lock, flags);
+ r = __get(prison, key, lock_level, inmate, cell_prealloc, cell_result);
+ spin_unlock_irqrestore(&prison->lock, flags);
+
+ return r;
+}
+EXPORT_SYMBOL_GPL(dm_cell_get_v2);
+
+static bool __put(struct dm_bio_prison_v2 *prison,
+ struct dm_bio_prison_cell_v2 *cell)
+{
+ BUG_ON(!cell->shared_count);
+ cell->shared_count--;
+
+ // FIXME: shared locks granted above the lock level could starve this
+ if (!cell->shared_count) {
+ if (cell->exclusive_lock){
+ if (cell->quiesce_continuation) {
+ queue_work(prison->wq, cell->quiesce_continuation);
+ cell->quiesce_continuation = NULL;
+ }
+ } else {
+ rb_erase(&cell->node, &prison->cells);
+ return true;
+ }
+ }
+
+ return false;
+}
+
+bool dm_cell_put_v2(struct dm_bio_prison_v2 *prison,
+ struct dm_bio_prison_cell_v2 *cell)
+{
+ bool r;
+ unsigned long flags;
+
+ spin_lock_irqsave(&prison->lock, flags);
+ r = __put(prison, cell);
+ spin_unlock_irqrestore(&prison->lock, flags);
+
+ return r;
+}
+EXPORT_SYMBOL_GPL(dm_cell_put_v2);
+
+static int __lock(struct dm_bio_prison_v2 *prison,
+ struct dm_cell_key_v2 *key,
+ unsigned lock_level,
+ struct dm_bio_prison_cell_v2 *cell_prealloc,
+ struct dm_bio_prison_cell_v2 **cell_result)
+{
+ struct dm_bio_prison_cell_v2 *cell;
+
+ if (__find_or_insert(prison, key, cell_prealloc, &cell)) {
+ if (cell->exclusive_lock)
+ return -EBUSY;
+
+ cell->exclusive_lock = true;
+ cell->exclusive_level = lock_level;
+ *cell_result = cell;
+
+ // FIXME: we don't yet know what level these shared locks
+ // were taken at, so have to quiesce them all.
+ return cell->shared_count > 0;
+
+ } else {
+ cell = cell_prealloc;
+ cell->shared_count = 0;
+ cell->exclusive_lock = true;
+ cell->exclusive_level = lock_level;
+ *cell_result = cell;
+ }
+
+ return 0;
+}
+
+int dm_cell_lock_v2(struct dm_bio_prison_v2 *prison,
+ struct dm_cell_key_v2 *key,
+ unsigned lock_level,
+ struct dm_bio_prison_cell_v2 *cell_prealloc,
+ struct dm_bio_prison_cell_v2 **cell_result)
+{
+ int r;
+ unsigned long flags;
+
+ spin_lock_irqsave(&prison->lock, flags);
+ r = __lock(prison, key, lock_level, cell_prealloc, cell_result);
+ spin_unlock_irqrestore(&prison->lock, flags);
+
+ return r;
+}
+EXPORT_SYMBOL_GPL(dm_cell_lock_v2);
+
+static void __quiesce(struct dm_bio_prison_v2 *prison,
+ struct dm_bio_prison_cell_v2 *cell,
+ struct work_struct *continuation)
+{
+ if (!cell->shared_count)
+ queue_work(prison->wq, continuation);
+ else
+ cell->quiesce_continuation = continuation;
+}
+
+void dm_cell_quiesce_v2(struct dm_bio_prison_v2 *prison,
+ struct dm_bio_prison_cell_v2 *cell,
+ struct work_struct *continuation)
+{
+ unsigned long flags;
+
+ spin_lock_irqsave(&prison->lock, flags);
+ __quiesce(prison, cell, continuation);
+ spin_unlock_irqrestore(&prison->lock, flags);
+}
+EXPORT_SYMBOL_GPL(dm_cell_quiesce_v2);
+
+static int __promote(struct dm_bio_prison_v2 *prison,
+ struct dm_bio_prison_cell_v2 *cell,
+ unsigned new_lock_level)
+{
+ if (!cell->exclusive_lock)
+ return -EINVAL;
+
+ cell->exclusive_level = new_lock_level;
+ return cell->shared_count > 0;
+}
+
+int dm_cell_lock_promote_v2(struct dm_bio_prison_v2 *prison,
+ struct dm_bio_prison_cell_v2 *cell,
+ unsigned new_lock_level)
+{
+ int r;
+ unsigned long flags;
+
+ spin_lock_irqsave(&prison->lock, flags);
+ r = __promote(prison, cell, new_lock_level);
+ spin_unlock_irqrestore(&prison->lock, flags);
+
+ return r;
+}
+EXPORT_SYMBOL_GPL(dm_cell_lock_promote_v2);
+
+static bool __unlock(struct dm_bio_prison_v2 *prison,
+ struct dm_bio_prison_cell_v2 *cell,
+ struct bio_list *bios)
+{
+ BUG_ON(!cell->exclusive_lock);
+
+ bio_list_merge(bios, &cell->bios);
+ bio_list_init(&cell->bios);
+
+ if (cell->shared_count) {
+ cell->exclusive_lock = 0;
+ return false;
+ }
+
+ rb_erase(&cell->node, &prison->cells);
+ return true;
+}
+
+bool dm_cell_unlock_v2(struct dm_bio_prison_v2 *prison,
+ struct dm_bio_prison_cell_v2 *cell,
+ struct bio_list *bios)
+{
+ bool r;
+ unsigned long flags;
+
+ spin_lock_irqsave(&prison->lock, flags);
+ r = __unlock(prison, cell, bios);
+ spin_unlock_irqrestore(&prison->lock, flags);
+
+ return r;
+}
+EXPORT_SYMBOL_GPL(dm_cell_unlock_v2);
+
+/*----------------------------------------------------------------*/
+
+int __init dm_bio_prison_init_v2(void)
+{
+ _cell_cache = KMEM_CACHE(dm_bio_prison_cell_v2, 0);
+ if (!_cell_cache)
+ return -ENOMEM;
+
+ return 0;
+}
+
+void dm_bio_prison_exit_v2(void)
+{
+ kmem_cache_destroy(_cell_cache);
+ _cell_cache = NULL;
+}
--- /dev/null
+/*
+ * Copyright (C) 2011-2017 Red Hat, Inc.
+ *
+ * This file is released under the GPL.
+ */
+
+#ifndef DM_BIO_PRISON_V2_H
+#define DM_BIO_PRISON_V2_H
+
+#include "persistent-data/dm-block-manager.h" /* FIXME: for dm_block_t */
+#include "dm-thin-metadata.h" /* FIXME: for dm_thin_id */
+
+#include <linux/bio.h>
+#include <linux/rbtree.h>
+#include <linux/workqueue.h>
+
+/*----------------------------------------------------------------*/
+
+int dm_bio_prison_init_v2(void);
+void dm_bio_prison_exit_v2(void);
+
+/*
+ * Sometimes we can't deal with a bio straight away. We put them in prison
+ * where they can't cause any mischief. Bios are put in a cell identified
+ * by a key, multiple bios can be in the same cell. When the cell is
+ * subsequently unlocked the bios become available.
+ */
+struct dm_bio_prison_v2;
+
+/*
+ * Keys define a range of blocks within either a virtual or physical
+ * device.
+ */
+struct dm_cell_key_v2 {
+ int virtual;
+ dm_thin_id dev;
+ dm_block_t block_begin, block_end;
+};
+
+/*
+ * Treat this as opaque, only in header so callers can manage allocation
+ * themselves.
+ */
+struct dm_bio_prison_cell_v2 {
+ // FIXME: pack these
+ bool exclusive_lock;
+ unsigned exclusive_level;
+ unsigned shared_count;
+ struct work_struct *quiesce_continuation;
+
+ struct rb_node node;
+ struct dm_cell_key_v2 key;
+ struct bio_list bios;
+};
+
+struct dm_bio_prison_v2 *dm_bio_prison_create_v2(struct workqueue_struct *wq);
+void dm_bio_prison_destroy_v2(struct dm_bio_prison_v2 *prison);
+
+/*
+ * These two functions just wrap a mempool. This is a transitory step:
+ * Eventually all bio prison clients should manage their own cell memory.
+ *
+ * Like mempool_alloc(), dm_bio_prison_alloc_cell_v2() can only fail if called
+ * in interrupt context or passed GFP_NOWAIT.
+ */
+struct dm_bio_prison_cell_v2 *dm_bio_prison_alloc_cell_v2(struct dm_bio_prison_v2 *prison,
+ gfp_t gfp);
+void dm_bio_prison_free_cell_v2(struct dm_bio_prison_v2 *prison,
+ struct dm_bio_prison_cell_v2 *cell);
+
+/*
+ * Shared locks have a bio associated with them.
+ *
+ * If the lock is granted the caller can continue to use the bio, and must
+ * call dm_cell_put_v2() to drop the reference count when finished using it.
+ *
+ * If the lock cannot be granted then the bio will be tracked within the
+ * cell, and later given to the holder of the exclusive lock.
+ *
+ * See dm_cell_lock_v2() for discussion of the lock_level parameter.
+ *
+ * Compare *cell_result with cell_prealloc to see if the prealloc was used.
+ * If cell_prealloc was used then inmate wasn't added to it.
+ *
+ * Returns true if the lock is granted.
+ */
+bool dm_cell_get_v2(struct dm_bio_prison_v2 *prison,
+ struct dm_cell_key_v2 *key,
+ unsigned lock_level,
+ struct bio *inmate,
+ struct dm_bio_prison_cell_v2 *cell_prealloc,
+ struct dm_bio_prison_cell_v2 **cell_result);
+
+/*
+ * Decrement the shared reference count for the lock. Returns true if
+ * returning ownership of the cell (ie. you should free it).
+ */
+bool dm_cell_put_v2(struct dm_bio_prison_v2 *prison,
+ struct dm_bio_prison_cell_v2 *cell);
+
+/*
+ * Locks a cell. No associated bio. Exclusive locks get priority. These
+ * locks constrain whether the io locks are granted according to level.
+ *
+ * Shared locks will still be granted if the lock_level is > (not = to) the
+ * exclusive lock level.
+ *
+ * If an _exclusive_ lock is already held then -EBUSY is returned.
+ *
+ * Return values:
+ * < 0 - error
+ * 0 - locked; no quiescing needed
+ * 1 - locked; quiescing needed
+ */
+int dm_cell_lock_v2(struct dm_bio_prison_v2 *prison,
+ struct dm_cell_key_v2 *key,
+ unsigned lock_level,
+ struct dm_bio_prison_cell_v2 *cell_prealloc,
+ struct dm_bio_prison_cell_v2 **cell_result);
+
+void dm_cell_quiesce_v2(struct dm_bio_prison_v2 *prison,
+ struct dm_bio_prison_cell_v2 *cell,
+ struct work_struct *continuation);
+
+/*
+ * Promotes an _exclusive_ lock to a higher lock level.
+ *
+ * Return values:
+ * < 0 - error
+ * 0 - promoted; no quiescing needed
+ * 1 - promoted; quiescing needed
+ */
+int dm_cell_lock_promote_v2(struct dm_bio_prison_v2 *prison,
+ struct dm_bio_prison_cell_v2 *cell,
+ unsigned new_lock_level);
+
+/*
+ * Adds any held bios to the bio list.
+ *
+ * There may be shared locks still held at this point even if you quiesced
+ * (ie. different lock levels).
+ *
+ * Returns true if returning ownership of the cell (ie. you should free
+ * it).
+ */
+bool dm_cell_unlock_v2(struct dm_bio_prison_v2 *prison,
+ struct dm_bio_prison_cell_v2 *cell,
+ struct bio_list *bios);
+
+/*----------------------------------------------------------------*/
+
+#endif
+++ /dev/null
-/*
- * Copyright (C) 2012 Red Hat, Inc.
- *
- * This file is released under the GPL.
- */
-
-#include "dm.h"
-#include "dm-bio-prison.h"
-
-#include <linux/spinlock.h>
-#include <linux/mempool.h>
-#include <linux/module.h>
-#include <linux/slab.h>
-
-/*----------------------------------------------------------------*/
-
-#define MIN_CELLS 1024
-
-struct dm_bio_prison {
- spinlock_t lock;
- mempool_t *cell_pool;
- struct rb_root cells;
-};
-
-static struct kmem_cache *_cell_cache;
-
-/*----------------------------------------------------------------*/
-
-/*
- * @nr_cells should be the number of cells you want in use _concurrently_.
- * Don't confuse it with the number of distinct keys.
- */
-struct dm_bio_prison *dm_bio_prison_create(void)
-{
- struct dm_bio_prison *prison = kmalloc(sizeof(*prison), GFP_KERNEL);
-
- if (!prison)
- return NULL;
-
- spin_lock_init(&prison->lock);
-
- prison->cell_pool = mempool_create_slab_pool(MIN_CELLS, _cell_cache);
- if (!prison->cell_pool) {
- kfree(prison);
- return NULL;
- }
-
- prison->cells = RB_ROOT;
-
- return prison;
-}
-EXPORT_SYMBOL_GPL(dm_bio_prison_create);
-
-void dm_bio_prison_destroy(struct dm_bio_prison *prison)
-{
- mempool_destroy(prison->cell_pool);
- kfree(prison);
-}
-EXPORT_SYMBOL_GPL(dm_bio_prison_destroy);
-
-struct dm_bio_prison_cell *dm_bio_prison_alloc_cell(struct dm_bio_prison *prison, gfp_t gfp)
-{
- return mempool_alloc(prison->cell_pool, gfp);
-}
-EXPORT_SYMBOL_GPL(dm_bio_prison_alloc_cell);
-
-void dm_bio_prison_free_cell(struct dm_bio_prison *prison,
- struct dm_bio_prison_cell *cell)
-{
- mempool_free(cell, prison->cell_pool);
-}
-EXPORT_SYMBOL_GPL(dm_bio_prison_free_cell);
-
-static void __setup_new_cell(struct dm_cell_key *key,
- struct bio *holder,
- struct dm_bio_prison_cell *cell)
-{
- memcpy(&cell->key, key, sizeof(cell->key));
- cell->holder = holder;
- bio_list_init(&cell->bios);
-}
-
-static int cmp_keys(struct dm_cell_key *lhs,
- struct dm_cell_key *rhs)
-{
- if (lhs->virtual < rhs->virtual)
- return -1;
-
- if (lhs->virtual > rhs->virtual)
- return 1;
-
- if (lhs->dev < rhs->dev)
- return -1;
-
- if (lhs->dev > rhs->dev)
- return 1;
-
- if (lhs->block_end <= rhs->block_begin)
- return -1;
-
- if (lhs->block_begin >= rhs->block_end)
- return 1;
-
- return 0;
-}
-
-static int __bio_detain(struct dm_bio_prison *prison,
- struct dm_cell_key *key,
- struct bio *inmate,
- struct dm_bio_prison_cell *cell_prealloc,
- struct dm_bio_prison_cell **cell_result)
-{
- int r;
- struct rb_node **new = &prison->cells.rb_node, *parent = NULL;
-
- while (*new) {
- struct dm_bio_prison_cell *cell =
- container_of(*new, struct dm_bio_prison_cell, node);
-
- r = cmp_keys(key, &cell->key);
-
- parent = *new;
- if (r < 0)
- new = &((*new)->rb_left);
- else if (r > 0)
- new = &((*new)->rb_right);
- else {
- if (inmate)
- bio_list_add(&cell->bios, inmate);
- *cell_result = cell;
- return 1;
- }
- }
-
- __setup_new_cell(key, inmate, cell_prealloc);
- *cell_result = cell_prealloc;
-
- rb_link_node(&cell_prealloc->node, parent, new);
- rb_insert_color(&cell_prealloc->node, &prison->cells);
-
- return 0;
-}
-
-static int bio_detain(struct dm_bio_prison *prison,
- struct dm_cell_key *key,
- struct bio *inmate,
- struct dm_bio_prison_cell *cell_prealloc,
- struct dm_bio_prison_cell **cell_result)
-{
- int r;
- unsigned long flags;
-
- spin_lock_irqsave(&prison->lock, flags);
- r = __bio_detain(prison, key, inmate, cell_prealloc, cell_result);
- spin_unlock_irqrestore(&prison->lock, flags);
-
- return r;
-}
-
-int dm_bio_detain(struct dm_bio_prison *prison,
- struct dm_cell_key *key,
- struct bio *inmate,
- struct dm_bio_prison_cell *cell_prealloc,
- struct dm_bio_prison_cell **cell_result)
-{
- return bio_detain(prison, key, inmate, cell_prealloc, cell_result);
-}
-EXPORT_SYMBOL_GPL(dm_bio_detain);
-
-int dm_get_cell(struct dm_bio_prison *prison,
- struct dm_cell_key *key,
- struct dm_bio_prison_cell *cell_prealloc,
- struct dm_bio_prison_cell **cell_result)
-{
- return bio_detain(prison, key, NULL, cell_prealloc, cell_result);
-}
-EXPORT_SYMBOL_GPL(dm_get_cell);
-
-/*
- * @inmates must have been initialised prior to this call
- */
-static void __cell_release(struct dm_bio_prison *prison,
- struct dm_bio_prison_cell *cell,
- struct bio_list *inmates)
-{
- rb_erase(&cell->node, &prison->cells);
-
- if (inmates) {
- if (cell->holder)
- bio_list_add(inmates, cell->holder);
- bio_list_merge(inmates, &cell->bios);
- }
-}
-
-void dm_cell_release(struct dm_bio_prison *prison,
- struct dm_bio_prison_cell *cell,
- struct bio_list *bios)
-{
- unsigned long flags;
-
- spin_lock_irqsave(&prison->lock, flags);
- __cell_release(prison, cell, bios);
- spin_unlock_irqrestore(&prison->lock, flags);
-}
-EXPORT_SYMBOL_GPL(dm_cell_release);
-
-/*
- * Sometimes we don't want the holder, just the additional bios.
- */
-static void __cell_release_no_holder(struct dm_bio_prison *prison,
- struct dm_bio_prison_cell *cell,
- struct bio_list *inmates)
-{
- rb_erase(&cell->node, &prison->cells);
- bio_list_merge(inmates, &cell->bios);
-}
-
-void dm_cell_release_no_holder(struct dm_bio_prison *prison,
- struct dm_bio_prison_cell *cell,
- struct bio_list *inmates)
-{
- unsigned long flags;
-
- spin_lock_irqsave(&prison->lock, flags);
- __cell_release_no_holder(prison, cell, inmates);
- spin_unlock_irqrestore(&prison->lock, flags);
-}
-EXPORT_SYMBOL_GPL(dm_cell_release_no_holder);
-
-void dm_cell_error(struct dm_bio_prison *prison,
- struct dm_bio_prison_cell *cell, int error)
-{
- struct bio_list bios;
- struct bio *bio;
-
- bio_list_init(&bios);
- dm_cell_release(prison, cell, &bios);
-
- while ((bio = bio_list_pop(&bios))) {
- bio->bi_error = error;
- bio_endio(bio);
- }
-}
-EXPORT_SYMBOL_GPL(dm_cell_error);
-
-void dm_cell_visit_release(struct dm_bio_prison *prison,
- void (*visit_fn)(void *, struct dm_bio_prison_cell *),
- void *context,
- struct dm_bio_prison_cell *cell)
-{
- unsigned long flags;
-
- spin_lock_irqsave(&prison->lock, flags);
- visit_fn(context, cell);
- rb_erase(&cell->node, &prison->cells);
- spin_unlock_irqrestore(&prison->lock, flags);
-}
-EXPORT_SYMBOL_GPL(dm_cell_visit_release);
-
-static int __promote_or_release(struct dm_bio_prison *prison,
- struct dm_bio_prison_cell *cell)
-{
- if (bio_list_empty(&cell->bios)) {
- rb_erase(&cell->node, &prison->cells);
- return 1;
- }
-
- cell->holder = bio_list_pop(&cell->bios);
- return 0;
-}
-
-int dm_cell_promote_or_release(struct dm_bio_prison *prison,
- struct dm_bio_prison_cell *cell)
-{
- int r;
- unsigned long flags;
-
- spin_lock_irqsave(&prison->lock, flags);
- r = __promote_or_release(prison, cell);
- spin_unlock_irqrestore(&prison->lock, flags);
-
- return r;
-}
-EXPORT_SYMBOL_GPL(dm_cell_promote_or_release);
-
-/*----------------------------------------------------------------*/
-
-#define DEFERRED_SET_SIZE 64
-
-struct dm_deferred_entry {
- struct dm_deferred_set *ds;
- unsigned count;
- struct list_head work_items;
-};
-
-struct dm_deferred_set {
- spinlock_t lock;
- unsigned current_entry;
- unsigned sweeper;
- struct dm_deferred_entry entries[DEFERRED_SET_SIZE];
-};
-
-struct dm_deferred_set *dm_deferred_set_create(void)
-{
- int i;
- struct dm_deferred_set *ds;
-
- ds = kmalloc(sizeof(*ds), GFP_KERNEL);
- if (!ds)
- return NULL;
-
- spin_lock_init(&ds->lock);
- ds->current_entry = 0;
- ds->sweeper = 0;
- for (i = 0; i < DEFERRED_SET_SIZE; i++) {
- ds->entries[i].ds = ds;
- ds->entries[i].count = 0;
- INIT_LIST_HEAD(&ds->entries[i].work_items);
- }
-
- return ds;
-}
-EXPORT_SYMBOL_GPL(dm_deferred_set_create);
-
-void dm_deferred_set_destroy(struct dm_deferred_set *ds)
-{
- kfree(ds);
-}
-EXPORT_SYMBOL_GPL(dm_deferred_set_destroy);
-
-struct dm_deferred_entry *dm_deferred_entry_inc(struct dm_deferred_set *ds)
-{
- unsigned long flags;
- struct dm_deferred_entry *entry;
-
- spin_lock_irqsave(&ds->lock, flags);
- entry = ds->entries + ds->current_entry;
- entry->count++;
- spin_unlock_irqrestore(&ds->lock, flags);
-
- return entry;
-}
-EXPORT_SYMBOL_GPL(dm_deferred_entry_inc);
-
-static unsigned ds_next(unsigned index)
-{
- return (index + 1) % DEFERRED_SET_SIZE;
-}
-
-static void __sweep(struct dm_deferred_set *ds, struct list_head *head)
-{
- while ((ds->sweeper != ds->current_entry) &&
- !ds->entries[ds->sweeper].count) {
- list_splice_init(&ds->entries[ds->sweeper].work_items, head);
- ds->sweeper = ds_next(ds->sweeper);
- }
-
- if ((ds->sweeper == ds->current_entry) && !ds->entries[ds->sweeper].count)
- list_splice_init(&ds->entries[ds->sweeper].work_items, head);
-}
-
-void dm_deferred_entry_dec(struct dm_deferred_entry *entry, struct list_head *head)
-{
- unsigned long flags;
-
- spin_lock_irqsave(&entry->ds->lock, flags);
- BUG_ON(!entry->count);
- --entry->count;
- __sweep(entry->ds, head);
- spin_unlock_irqrestore(&entry->ds->lock, flags);
-}
-EXPORT_SYMBOL_GPL(dm_deferred_entry_dec);
-
-/*
- * Returns 1 if deferred or 0 if no pending items to delay job.
- */
-int dm_deferred_set_add_work(struct dm_deferred_set *ds, struct list_head *work)
-{
- int r = 1;
- unsigned long flags;
- unsigned next_entry;
-
- spin_lock_irqsave(&ds->lock, flags);
- if ((ds->sweeper == ds->current_entry) &&
- !ds->entries[ds->current_entry].count)
- r = 0;
- else {
- list_add(work, &ds->entries[ds->current_entry].work_items);
- next_entry = ds_next(ds->current_entry);
- if (!ds->entries[next_entry].count)
- ds->current_entry = next_entry;
- }
- spin_unlock_irqrestore(&ds->lock, flags);
-
- return r;
-}
-EXPORT_SYMBOL_GPL(dm_deferred_set_add_work);
-
-/*----------------------------------------------------------------*/
-
-static int __init dm_bio_prison_init(void)
-{
- _cell_cache = KMEM_CACHE(dm_bio_prison_cell, 0);
- if (!_cell_cache)
- return -ENOMEM;
-
- return 0;
-}
-
-static void __exit dm_bio_prison_exit(void)
-{
- kmem_cache_destroy(_cell_cache);
- _cell_cache = NULL;
-}
-
-/*
- * module hooks
- */
-module_init(dm_bio_prison_init);
-module_exit(dm_bio_prison_exit);
-
-MODULE_DESCRIPTION(DM_NAME " bio prison");
-MODULE_AUTHOR("Joe Thornber <dm-devel@redhat.com>");
-MODULE_LICENSE("GPL");
+++ /dev/null
-/*
- * Copyright (C) 2011-2012 Red Hat, Inc.
- *
- * This file is released under the GPL.
- */
-
-#ifndef DM_BIO_PRISON_H
-#define DM_BIO_PRISON_H
-
-#include "persistent-data/dm-block-manager.h" /* FIXME: for dm_block_t */
-#include "dm-thin-metadata.h" /* FIXME: for dm_thin_id */
-
-#include <linux/bio.h>
-#include <linux/rbtree.h>
-
-/*----------------------------------------------------------------*/
-
-/*
- * Sometimes we can't deal with a bio straight away. We put them in prison
- * where they can't cause any mischief. Bios are put in a cell identified
- * by a key, multiple bios can be in the same cell. When the cell is
- * subsequently unlocked the bios become available.
- */
-struct dm_bio_prison;
-
-/*
- * Keys define a range of blocks within either a virtual or physical
- * device.
- */
-struct dm_cell_key {
- int virtual;
- dm_thin_id dev;
- dm_block_t block_begin, block_end;
-};
-
-/*
- * Treat this as opaque, only in header so callers can manage allocation
- * themselves.
- */
-struct dm_bio_prison_cell {
- struct list_head user_list; /* for client use */
- struct rb_node node;
-
- struct dm_cell_key key;
- struct bio *holder;
- struct bio_list bios;
-};
-
-struct dm_bio_prison *dm_bio_prison_create(void);
-void dm_bio_prison_destroy(struct dm_bio_prison *prison);
-
-/*
- * These two functions just wrap a mempool. This is a transitory step:
- * Eventually all bio prison clients should manage their own cell memory.
- *
- * Like mempool_alloc(), dm_bio_prison_alloc_cell() can only fail if called
- * in interrupt context or passed GFP_NOWAIT.
- */
-struct dm_bio_prison_cell *dm_bio_prison_alloc_cell(struct dm_bio_prison *prison,
- gfp_t gfp);
-void dm_bio_prison_free_cell(struct dm_bio_prison *prison,
- struct dm_bio_prison_cell *cell);
-
-/*
- * Creates, or retrieves a cell that overlaps the given key.
- *
- * Returns 1 if pre-existing cell returned, zero if new cell created using
- * @cell_prealloc.
- */
-int dm_get_cell(struct dm_bio_prison *prison,
- struct dm_cell_key *key,
- struct dm_bio_prison_cell *cell_prealloc,
- struct dm_bio_prison_cell **cell_result);
-
-/*
- * An atomic op that combines retrieving or creating a cell, and adding a
- * bio to it.
- *
- * Returns 1 if the cell was already held, 0 if @inmate is the new holder.
- */
-int dm_bio_detain(struct dm_bio_prison *prison,
- struct dm_cell_key *key,
- struct bio *inmate,
- struct dm_bio_prison_cell *cell_prealloc,
- struct dm_bio_prison_cell **cell_result);
-
-void dm_cell_release(struct dm_bio_prison *prison,
- struct dm_bio_prison_cell *cell,
- struct bio_list *bios);
-void dm_cell_release_no_holder(struct dm_bio_prison *prison,
- struct dm_bio_prison_cell *cell,
- struct bio_list *inmates);
-void dm_cell_error(struct dm_bio_prison *prison,
- struct dm_bio_prison_cell *cell, int error);
-
-/*
- * Visits the cell and then releases. Guarantees no new inmates are
- * inserted between the visit and release.
- */
-void dm_cell_visit_release(struct dm_bio_prison *prison,
- void (*visit_fn)(void *, struct dm_bio_prison_cell *),
- void *context, struct dm_bio_prison_cell *cell);
-
-/*
- * Rather than always releasing the prisoners in a cell, the client may
- * want to promote one of them to be the new holder. There is a race here
- * though between releasing an empty cell, and other threads adding new
- * inmates. So this function makes the decision with its lock held.
- *
- * This function can have two outcomes:
- * i) An inmate is promoted to be the holder of the cell (return value of 0).
- * ii) The cell has no inmate for promotion and is released (return value of 1).
- */
-int dm_cell_promote_or_release(struct dm_bio_prison *prison,
- struct dm_bio_prison_cell *cell);
-
-/*----------------------------------------------------------------*/
-
-/*
- * We use the deferred set to keep track of pending reads to shared blocks.
- * We do this to ensure the new mapping caused by a write isn't performed
- * until these prior reads have completed. Otherwise the insertion of the
- * new mapping could free the old block that the read bios are mapped to.
- */
-
-struct dm_deferred_set;
-struct dm_deferred_entry;
-
-struct dm_deferred_set *dm_deferred_set_create(void);
-void dm_deferred_set_destroy(struct dm_deferred_set *ds);
-
-struct dm_deferred_entry *dm_deferred_entry_inc(struct dm_deferred_set *ds);
-void dm_deferred_entry_dec(struct dm_deferred_entry *entry, struct list_head *head);
-int dm_deferred_set_add_work(struct dm_deferred_set *ds, struct list_head *work);
-
-/*----------------------------------------------------------------*/
-
-#endif
*/
#include "dm.h"
-#include "dm-bio-prison.h"
+#include "dm-bio-prison-v1.h"
#include "dm-bio-record.h"
#include "dm-cache-metadata.h"
*/
#include "dm-thin-metadata.h"
-#include "dm-bio-prison.h"
+#include "dm-bio-prison-v1.h"
#include "dm.h"
#include <linux/device-mapper.h>
projects / mirror_ubuntu-bionic-kernel.git / commitdiff