*
* CDDL HEADER END
*/
+
/*
* Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2013, 2017 by Delphix. All rights reserved.
+ * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved.
*/
#include <sys/zfs_context.h>
#include <sys/types.h>
#include <sys/param.h>
-#include <sys/systm.h>
#include <sys/sysmacros.h>
#include <sys/dmu.h>
#include <sys/dmu_impl.h>
#include <sys/dmu_objset.h>
+#include <sys/dmu_tx.h>
#include <sys/dbuf.h>
#include <sys/dnode.h>
#include <sys/zap.h>
#include <sys/errno.h>
#include <sys/zfs_context.h>
+#ifdef _KERNEL
+#include <sys/zfs_znode.h>
+#endif
+
/*
* ZFS System attributes:
*
* location.
*
* Byteswap implications:
+ *
* Since the SA attributes are not entirely self describing we can't do
* the normal byteswap processing. The special ZAP layout attribute and
* attribute registration attributes define the byteswap function and the
static int sa_build_index(sa_handle_t *hdl, sa_buf_type_t buftype);
static void sa_idx_tab_hold(objset_t *os, sa_idx_tab_t *idx_tab);
-static void *sa_find_idx_tab(objset_t *os, dmu_object_type_t bonustype,
- void *data);
+static sa_idx_tab_t *sa_find_idx_tab(objset_t *os, dmu_object_type_t bonustype,
+ sa_hdr_phys_t *hdr);
static void sa_idx_tab_rele(objset_t *os, void *arg);
static void sa_copy_data(sa_data_locator_t *func, void *start, void *target,
int buflen);
sa_data_op_t action, sa_data_locator_t *locator, void *datastart,
uint16_t buflen, dmu_tx_t *tx);
-arc_byteswap_func_t *sa_bswap_table[] = {
+arc_byteswap_func_t sa_bswap_table[] = {
byteswap_uint64_array,
byteswap_uint32_array,
byteswap_uint16_array,
zfs_acl_byteswap,
};
-#define SA_COPY_DATA(f, s, t, l) \
- { \
- if (f == NULL) { \
- if (l == 8) { \
- *(uint64_t *)t = *(uint64_t *)s; \
- } else if (l == 16) { \
- *(uint64_t *)t = *(uint64_t *)s; \
- *(uint64_t *)((uintptr_t)t + 8) = \
- *(uint64_t *)((uintptr_t)s + 8); \
- } else { \
- bcopy(s, t, l); \
- } \
- } else \
- sa_copy_data(f, s, t, l); \
- }
+#ifdef HAVE_EFFICIENT_UNALIGNED_ACCESS
+#define SA_COPY_DATA(f, s, t, l) \
+do { \
+ if (f == NULL) { \
+ if (l == 8) { \
+ *(uint64_t *)t = *(uint64_t *)s; \
+ } else if (l == 16) { \
+ *(uint64_t *)t = *(uint64_t *)s; \
+ *(uint64_t *)((uintptr_t)t + 8) = \
+ *(uint64_t *)((uintptr_t)s + 8); \
+ } else { \
+ bcopy(s, t, l); \
+ } \
+ } else { \
+ sa_copy_data(f, s, t, l); \
+ } \
+} while (0)
+#else
+#define SA_COPY_DATA(f, s, t, l) sa_copy_data(f, s, t, l)
+#endif
/*
* This table is fixed and cannot be changed. Its purpose is to
};
/*
- * ZPL legacy layout
* This is only used for objects of type DMU_OT_ZNODE
*/
sa_attr_type_t sa_legacy_zpl_layout[] = {
/*
* Special dummy layout used for buffers with no attributes.
*/
-
sa_attr_type_t sa_dummy_zpl_layout[] = { 0 };
-static int sa_legacy_attr_count = 16;
+static int sa_legacy_attr_count = ARRAY_SIZE(sa_legacy_attrs);
static kmem_cache_t *sa_cache = NULL;
-static kmem_cache_t *spill_cache = NULL;
/*ARGSUSED*/
static int
{
sa_handle_t *hdl = buf;
- hdl->sa_bonus_tab = NULL;
- hdl->sa_spill_tab = NULL;
- hdl->sa_os = NULL;
- hdl->sa_userp = NULL;
- hdl->sa_bonus = NULL;
- hdl->sa_spill = NULL;
mutex_init(&hdl->sa_lock, NULL, MUTEX_DEFAULT, NULL);
return (0);
}
sa_cache = kmem_cache_create("sa_cache",
sizeof (sa_handle_t), 0, sa_cache_constructor,
sa_cache_destructor, NULL, NULL, NULL, 0);
- spill_cache = kmem_cache_create("spill_cache",
- SPA_MAXBLOCKSIZE, 0, NULL, NULL, NULL, NULL, NULL, 0);
}
void
{
if (sa_cache)
kmem_cache_destroy(sa_cache);
-
- if (spill_cache)
- kmem_cache_destroy(spill_cache);
-}
-
-void *
-sa_spill_alloc(int flags)
-{
- return kmem_cache_alloc(spill_cache, flags);
-}
-
-void
-sa_spill_free(void *obj)
-{
- kmem_cache_free(spill_cache, obj);
}
static int
layout_num_compare(const void *arg1, const void *arg2)
{
- const sa_lot_t *node1 = arg1;
- const sa_lot_t *node2 = arg2;
+ const sa_lot_t *node1 = (const sa_lot_t *)arg1;
+ const sa_lot_t *node2 = (const sa_lot_t *)arg2;
- if (node1->lot_num > node2->lot_num)
- return (1);
- else if (node1->lot_num < node2->lot_num)
- return (-1);
- return (0);
+ return (AVL_CMP(node1->lot_num, node2->lot_num));
}
static int
layout_hash_compare(const void *arg1, const void *arg2)
{
- const sa_lot_t *node1 = arg1;
- const sa_lot_t *node2 = arg2;
+ const sa_lot_t *node1 = (const sa_lot_t *)arg1;
+ const sa_lot_t *node2 = (const sa_lot_t *)arg2;
- if (node1->lot_hash > node2->lot_hash)
- return (1);
- if (node1->lot_hash < node2->lot_hash)
- return (-1);
- if (node1->lot_instance > node2->lot_instance)
- return (1);
- if (node1->lot_instance < node2->lot_instance)
- return (-1);
- return (0);
+ int cmp = AVL_CMP(node1->lot_hash, node2->lot_hash);
+ if (likely(cmp))
+ return (cmp);
+
+ return (AVL_CMP(node1->lot_instance, node2->lot_instance));
}
boolean_t
switch (data_op) {
case SA_LOOKUP:
if (bulk[i].sa_addr == NULL)
- return (ENOENT);
+ return (SET_ERROR(ENOENT));
if (bulk[i].sa_data) {
SA_COPY_DATA(bulk[i].sa_data_func,
bulk[i].sa_addr, bulk[i].sa_data,
avl_index_t loc;
ASSERT(MUTEX_HELD(&sa->sa_lock));
- tb = kmem_zalloc(sizeof (sa_lot_t), KM_PUSHPAGE);
+ tb = kmem_zalloc(sizeof (sa_lot_t), KM_SLEEP);
tb->lot_attr_count = attr_count;
tb->lot_attrs = kmem_alloc(sizeof (sa_attr_type_t) * attr_count,
- KM_PUSHPAGE);
+ KM_SLEEP);
bcopy(attrs, tb->lot_attrs, sizeof (sa_attr_type_t) * attr_count);
tb->lot_num = lot_num;
tb->lot_hash = hash;
char attr_name[8];
if (sa->sa_layout_attr_obj == 0) {
- sa->sa_layout_attr_obj = zap_create(os,
- DMU_OT_SA_ATTR_LAYOUTS, DMU_OT_NONE, 0, tx);
- VERIFY(zap_add(os, sa->sa_master_obj, SA_LAYOUTS, 8, 1,
- &sa->sa_layout_attr_obj, tx) == 0);
+ sa->sa_layout_attr_obj = zap_create_link(os,
+ DMU_OT_SA_ATTR_LAYOUTS,
+ sa->sa_master_obj, SA_LAYOUTS, tx);
}
(void) snprintf(attr_name, sizeof (attr_name),
if (size == 0) {
blocksize = SPA_MINBLOCKSIZE;
- } else if (size > SPA_MAXBLOCKSIZE) {
+ } else if (size > SPA_OLD_MAXBLOCKSIZE) {
ASSERT(0);
- return (EFBIG);
+ return (SET_ERROR(EFBIG));
} else {
blocksize = P2ROUNDUP_TYPED(size, SPA_MINBLOCKSIZE, uint32_t);
}
}
/*
- * Determine several different sizes
- * first the sa header size
- * the number of bytes to be stored
- * if spill would occur the index in the attribute array is returned
+ * Determine several different values pertaining to system attribute
+ * buffers.
+ *
+ * Return the size of the sa_hdr_phys_t header for the buffer. Each
+ * variable length attribute except the first contributes two bytes to
+ * the header size, which is then rounded up to an 8-byte boundary.
*
- * the boolean will_spill will be set when spilling is necessary. It
- * is only set when the buftype is SA_BONUS
+ * The following output parameters are also computed.
+ *
+ * index - The index of the first attribute in attr_desc that will
+ * spill over. Only valid if will_spill is set.
+ *
+ * total - The total number of bytes of all system attributes described
+ * in attr_desc.
+ *
+ * will_spill - Set when spilling is necessary. It is only set when
+ * the buftype is SA_BONUS.
*/
static int
sa_find_sizes(sa_os_t *sa, sa_bulk_attr_t *attr_desc, int attr_count,
- dmu_buf_t *db, sa_buf_type_t buftype, int *index, int *total,
- boolean_t *will_spill)
+ dmu_buf_t *db, sa_buf_type_t buftype, int full_space, int *index,
+ int *total, boolean_t *will_spill)
{
- int var_size = 0;
+ int var_size_count = 0;
int i;
- int full_space;
int hdrsize;
- boolean_t done = B_FALSE;
+ int extra_hdrsize;
if (buftype == SA_BONUS && sa->sa_force_spill) {
*total = 0;
*index = -1;
*total = 0;
+ *will_spill = B_FALSE;
- if (buftype == SA_BONUS)
- *will_spill = B_FALSE;
-
+ extra_hdrsize = 0;
hdrsize = (SA_BONUSTYPE_FROM_DB(db) == DMU_OT_ZNODE) ? 0 :
sizeof (sa_hdr_phys_t);
- full_space = (buftype == SA_BONUS) ? DN_MAX_BONUSLEN : db->db_size;
+ ASSERT(IS_P2ALIGNED(full_space, 8));
for (i = 0; i != attr_count; i++) {
- boolean_t is_var_sz;
+ boolean_t is_var_sz, might_spill_here;
+ int tmp_hdrsize;
+ *total = P2ROUNDUP(*total, 8);
*total += attr_desc[i].sa_length;
- if (done)
- goto next;
+ if (*will_spill)
+ continue;
is_var_sz = (SA_REGISTERED_LEN(sa, attr_desc[i].sa_attr) == 0);
- if (is_var_sz) {
- var_size++;
- }
+ if (is_var_sz)
+ var_size_count++;
- if (is_var_sz && var_size > 1) {
- if (P2ROUNDUP(hdrsize + sizeof (uint16_t), 8) +
- *total < full_space) {
- hdrsize += sizeof (uint16_t);
+ /*
+ * Calculate what the SA header size would be if this
+ * attribute doesn't spill.
+ */
+ tmp_hdrsize = hdrsize + ((is_var_sz && var_size_count > 1) ?
+ sizeof (uint16_t) : 0);
+
+ /*
+ * Check whether this attribute spans into the space
+ * that would be used by the spill block pointer should
+ * a spill block be needed.
+ */
+ might_spill_here =
+ buftype == SA_BONUS && *index == -1 &&
+ (*total + P2ROUNDUP(tmp_hdrsize, 8)) >
+ (full_space - sizeof (blkptr_t));
+
+ if (is_var_sz && var_size_count > 1) {
+ if (buftype == SA_SPILL ||
+ tmp_hdrsize + *total < full_space) {
+ /*
+ * Record the extra header size in case this
+ * increase needs to be reversed due to
+ * spill-over.
+ */
+ hdrsize = tmp_hdrsize;
+ if (*index != -1 || might_spill_here)
+ extra_hdrsize += sizeof (uint16_t);
} else {
- done = B_TRUE;
- *index = i;
- if (buftype == SA_BONUS)
- *will_spill = B_TRUE;
+ ASSERT(buftype == SA_BONUS);
+ if (*index == -1)
+ *index = i;
+ *will_spill = B_TRUE;
continue;
}
}
/*
- * find index of where spill *could* occur.
- * Then continue to count of remainder attribute
- * space. The sum is used later for sizing bonus
- * and spill buffer.
+ * Store index of where spill *could* occur. Then
+ * continue to count the remaining attribute sizes. The
+ * sum is used later for sizing bonus and spill buffer.
*/
- if (buftype == SA_BONUS && *index == -1 &&
- (*total + P2ROUNDUP(hdrsize, 8)) >
- (full_space - sizeof (blkptr_t))) {
+ if (might_spill_here)
*index = i;
- done = B_TRUE;
- }
-next:
if ((*total + P2ROUNDUP(hdrsize, 8)) > full_space &&
buftype == SA_BONUS)
*will_spill = B_TRUE;
}
+ if (*will_spill)
+ hdrsize -= extra_hdrsize;
+
hdrsize = P2ROUNDUP(hdrsize, 8);
return (hdrsize);
}
sa_buf_type_t buftype;
sa_hdr_phys_t *sahdr;
void *data_start;
- int buf_space;
sa_attr_type_t *attrs, *attrs_start;
int i, lot_count;
- int hdrsize, spillhdrsize = 0;
+ int dnodesize;
+ int spill_idx;
+ int hdrsize;
+ int spillhdrsize = 0;
int used;
dmu_object_type_t bonustype;
sa_lot_t *lot;
int len_idx;
int spill_used;
+ int bonuslen;
boolean_t spilling;
dmu_buf_will_dirty(hdl->sa_bonus, tx);
bonustype = SA_BONUSTYPE_FROM_DB(hdl->sa_bonus);
+ dmu_object_dnsize_from_db(hdl->sa_bonus, &dnodesize);
+ bonuslen = DN_BONUS_SIZE(dnodesize);
/* first determine bonus header size and sum of all attributes */
hdrsize = sa_find_sizes(sa, attr_desc, attr_count, hdl->sa_bonus,
- SA_BONUS, &i, &used, &spilling);
+ SA_BONUS, bonuslen, &spill_idx, &used, &spilling);
- if (used > SPA_MAXBLOCKSIZE)
- return (EFBIG);
+ if (used > SPA_OLD_MAXBLOCKSIZE)
+ return (SET_ERROR(EFBIG));
- VERIFY(0 == dmu_set_bonus(hdl->sa_bonus, spilling ?
- MIN(DN_MAX_BONUSLEN - sizeof (blkptr_t), used + hdrsize) :
+ VERIFY0(dmu_set_bonus(hdl->sa_bonus, spilling ?
+ MIN(bonuslen - sizeof (blkptr_t), used + hdrsize) :
used + hdrsize, tx));
ASSERT((bonustype == DMU_OT_ZNODE && spilling == 0) ||
boolean_t dummy;
if (hdl->sa_spill == NULL) {
- VERIFY(dmu_spill_hold_by_bonus(hdl->sa_bonus, NULL,
+ VERIFY(dmu_spill_hold_by_bonus(hdl->sa_bonus, 0, NULL,
&hdl->sa_spill) == 0);
}
dmu_buf_will_dirty(hdl->sa_spill, tx);
- spillhdrsize = sa_find_sizes(sa, &attr_desc[i],
- attr_count - i, hdl->sa_spill, SA_SPILL, &i,
- &spill_used, &dummy);
+ spillhdrsize = sa_find_sizes(sa, &attr_desc[spill_idx],
+ attr_count - spill_idx, hdl->sa_spill, SA_SPILL,
+ hdl->sa_spill->db_size, &i, &spill_used, &dummy);
- if (spill_used > SPA_MAXBLOCKSIZE)
- return (EFBIG);
+ if (spill_used > SPA_OLD_MAXBLOCKSIZE)
+ return (SET_ERROR(EFBIG));
- buf_space = hdl->sa_spill->db_size - spillhdrsize;
if (BUF_SPACE_NEEDED(spill_used, spillhdrsize) >
hdl->sa_spill->db_size)
VERIFY(0 == sa_resize_spill(hdl,
sahdr = (sa_hdr_phys_t *)hdl->sa_bonus->db_data;
buftype = SA_BONUS;
- if (spilling)
- buf_space = (sa->sa_force_spill) ?
- 0 : SA_BLKPTR_SPACE - hdrsize;
- else
- buf_space = hdl->sa_bonus->db_size - hdrsize;
-
attrs_start = attrs = kmem_alloc(sizeof (sa_attr_type_t) * attr_count,
- KM_PUSHPAGE);
+ KM_SLEEP);
lot_count = 0;
for (i = 0, len_idx = 0, hash = -1ULL; i != attr_count; i++) {
uint16_t length;
+ ASSERT(IS_P2ALIGNED(data_start, 8));
attrs[i] = attr_desc[i].sa_attr;
length = SA_REGISTERED_LEN(sa, attrs[i]);
if (length == 0)
length = attr_desc[i].sa_length;
- if (buf_space < length) { /* switch to spill buffer */
+ if (spilling && i == spill_idx) { /* switch to spill buffer */
VERIFY(bonustype == DMU_OT_SA);
if (buftype == SA_BONUS && !sa->sa_force_spill) {
sa_find_layout(hdl->sa_os, hash, attrs_start,
data_start = (void *)((uintptr_t)sahdr +
spillhdrsize);
attrs_start = &attrs[i];
- buf_space = hdl->sa_spill->db_size - spillhdrsize;
lot_count = 0;
}
hash ^= SA_ATTR_HASH(attrs[i]);
}
data_start = (void *)P2ROUNDUP(((uintptr_t)data_start +
length), 8);
- buf_space -= P2ROUNDUP(length, 8);
lot_count++;
}
dmu_objset_type_t ostype = dmu_objset_type(os);
sa->sa_user_table =
- kmem_zalloc(count * sizeof (sa_attr_type_t), KM_PUSHPAGE);
+ kmem_zalloc(count * sizeof (sa_attr_type_t), KM_SLEEP);
sa->sa_user_table_sz = count * sizeof (sa_attr_type_t);
if (sa->sa_reg_attr_obj != 0) {
*/
if (error || (error == 0 && sa_attr_count == 0)) {
if (error == 0)
- error = EINVAL;
+ error = SET_ERROR(EINVAL);
goto bail;
}
sa_reg_count = sa_attr_count;
error = zap_lookup(os, sa->sa_reg_attr_obj,
reg_attrs[i].sa_name, 8, 1, &attr_value);
else
- error = ENOENT;
+ error = SET_ERROR(ENOENT);
switch (error) {
case ENOENT:
sa->sa_user_table[i] = (sa_attr_type_t)sa_attr_count;
sa->sa_num_attrs = sa_attr_count;
tb = sa->sa_attr_table =
- kmem_zalloc(sizeof (sa_attr_table_t) * sa_attr_count, KM_PUSHPAGE);
+ kmem_zalloc(sizeof (sa_attr_table_t) * sa_attr_count, KM_SLEEP);
/*
* Attribute table is constructed from requested attribute list,
continue;
}
tb[ATTR_NUM(value)].sa_name =
- kmem_zalloc(strlen(za.za_name) +1, KM_PUSHPAGE);
+ kmem_zalloc(strlen(za.za_name) +1, KM_SLEEP);
(void) strlcpy(tb[ATTR_NUM(value)].sa_name, za.za_name,
strlen(za.za_name) +1);
}
tb[i].sa_registered = B_FALSE;
tb[i].sa_name =
kmem_zalloc(strlen(sa_legacy_attrs[i].sa_name) +1,
- KM_PUSHPAGE);
+ KM_SLEEP);
(void) strlcpy(tb[i].sa_name,
sa_legacy_attrs[i].sa_name,
strlen(sa_legacy_attrs[i].sa_name) + 1);
tb[attr_id].sa_byteswap = reg_attrs[i].sa_byteswap;
tb[attr_id].sa_attr = attr_id;
tb[attr_id].sa_name =
- kmem_zalloc(strlen(reg_attrs[i].sa_name) + 1, KM_PUSHPAGE);
+ kmem_zalloc(strlen(reg_attrs[i].sa_name) + 1, KM_SLEEP);
(void) strlcpy(tb[attr_id].sa_name, reg_attrs[i].sa_name,
strlen(reg_attrs[i].sa_name) + 1);
}
kmem_free(sa->sa_user_table, count * sizeof (sa_attr_type_t));
sa->sa_user_table = NULL;
sa_free_attr_table(sa);
- return ((error != 0) ? error : EINVAL);
+ ASSERT(error != 0);
+ return (error);
}
int
sa_attr_type_t *tb;
int error;
- mutex_enter(&os->os_lock);
+ mutex_enter(&os->os_user_ptr_lock);
if (os->os_sa) {
mutex_enter(&os->os_sa->sa_lock);
- mutex_exit(&os->os_lock);
+ mutex_exit(&os->os_user_ptr_lock);
tb = os->os_sa->sa_user_table;
mutex_exit(&os->os_sa->sa_lock);
*user_table = tb;
return (0);
}
- sa = kmem_zalloc(sizeof (sa_os_t), KM_PUSHPAGE);
+ sa = kmem_zalloc(sizeof (sa_os_t), KM_SLEEP);
mutex_init(&sa->sa_lock, NULL, MUTEX_DEFAULT, NULL);
sa->sa_master_obj = sa_obj;
os->os_sa = sa;
mutex_enter(&sa->sa_lock);
- mutex_exit(&os->os_lock);
+ mutex_exit(&os->os_user_ptr_lock);
avl_create(&sa->sa_layout_num_tree, layout_num_compare,
sizeof (sa_lot_t), offsetof(sa_lot_t, lot_num_node));
avl_create(&sa->sa_layout_hash_tree, layout_hash_compare,
*/
if (error || (error == 0 && layout_count == 0)) {
if (error == 0)
- error = EINVAL;
+ error = SET_ERROR(EINVAL);
goto fail;
}
uint64_t lot_num;
lot_attrs = kmem_zalloc(sizeof (sa_attr_type_t) *
- za.za_num_integers, KM_PUSHPAGE);
+ za.za_num_integers, KM_SLEEP);
if ((error = (zap_lookup(os, sa->sa_layout_attr_obj,
za.za_name, 2, za.za_num_integers,
if (sa->sa_user_table)
kmem_free(sa->sa_user_table, sa->sa_user_table_sz);
mutex_exit(&sa->sa_lock);
+ avl_destroy(&sa->sa_layout_hash_tree);
+ avl_destroy(&sa->sa_layout_num_tree);
+ mutex_destroy(&sa->sa_lock);
kmem_free(sa, sizeof (sa_os_t));
return ((error == ECKSUM) ? EIO : error);
}
sa_free_attr_table(sa);
cookie = NULL;
- while ((layout = avl_destroy_nodes(&sa->sa_layout_hash_tree, &cookie))){
+ while ((layout =
+ avl_destroy_nodes(&sa->sa_layout_hash_tree, &cookie))) {
sa_idx_tab_t *tab;
while ((tab = list_head(&layout->lot_idx_tab))) {
- ASSERT(refcount_count(&tab->sa_refcount));
+ ASSERT(zfs_refcount_count(&tab->sa_refcount));
sa_idx_tab_rele(os, tab);
}
}
cookie = NULL;
- while ((layout = avl_destroy_nodes(&sa->sa_layout_num_tree, &cookie))){
+ while ((layout = avl_destroy_nodes(&sa->sa_layout_num_tree, &cookie))) {
kmem_free(layout->lot_attrs,
sizeof (sa_attr_type_t) * layout->lot_attr_count);
kmem_free(layout, sizeof (sa_lot_t));
avl_destroy(&sa->sa_layout_hash_tree);
avl_destroy(&sa->sa_layout_num_tree);
+ mutex_destroy(&sa->sa_lock);
kmem_free(sa, sizeof (sa_os_t));
os->os_sa = NULL;
sa_hdr_phys->sa_layout_info = BSWAP_16(sa_hdr_phys->sa_layout_info);
/*
- * Determine number of variable lenghts in header
+ * Determine number of variable lengths in header
* The standard 8 byte header has one for free and a
* 16 byte header would have 4 + 1;
*/
/* only check if not old znode */
if (IS_SA_BONUSTYPE(bonustype) && sa_hdr_phys->sa_magic != SA_MAGIC &&
sa_hdr_phys->sa_magic != 0) {
- VERIFY(BSWAP_32(sa_hdr_phys->sa_magic) == SA_MAGIC);
+ if (BSWAP_32(sa_hdr_phys->sa_magic) != SA_MAGIC) {
+ mutex_exit(&sa->sa_lock);
+ zfs_dbgmsg("Buffer Header: %x != SA_MAGIC:%x "
+ "object=%#llx\n", sa_hdr_phys->sa_magic, SA_MAGIC,
+ db->db.db_object);
+ return (SET_ERROR(EIO));
+ }
sa_byteswap(hdl, buftype);
}
}
/*ARGSUSED*/
-void
-sa_evict(dmu_buf_t *db, void *sap)
+static void
+sa_evict_sync(void *dbu)
{
- panic("evicting sa dbuf %p\n", (void *)db);
+ panic("evicting sa dbuf\n");
}
static void
return;
mutex_enter(&sa->sa_lock);
- if (refcount_remove(&idx_tab->sa_refcount, NULL) == 0) {
+ if (zfs_refcount_remove(&idx_tab->sa_refcount, NULL) == 0) {
list_remove(&idx_tab->sa_layout->lot_idx_tab, idx_tab);
if (idx_tab->sa_variable_lengths)
kmem_free(idx_tab->sa_variable_lengths,
sizeof (uint16_t) *
idx_tab->sa_layout->lot_var_sizes);
- refcount_destroy(&idx_tab->sa_refcount);
+ zfs_refcount_destroy(&idx_tab->sa_refcount);
kmem_free(idx_tab->sa_idx_tab,
sizeof (uint32_t) * sa->sa_num_attrs);
kmem_free(idx_tab, sizeof (sa_idx_tab_t));
ASSERTV(sa_os_t *sa = os->os_sa);
ASSERT(MUTEX_HELD(&sa->sa_lock));
- (void) refcount_add(&idx_tab->sa_refcount, NULL);
+ (void) zfs_refcount_add(&idx_tab->sa_refcount, NULL);
}
void
void
sa_handle_destroy(sa_handle_t *hdl)
{
+ dmu_buf_t *db = hdl->sa_bonus;
+
mutex_enter(&hdl->sa_lock);
- (void) dmu_buf_update_user((dmu_buf_t *)hdl->sa_bonus, hdl,
- NULL, NULL, NULL);
+ (void) dmu_buf_remove_user(db, &hdl->sa_dbu);
- if (hdl->sa_bonus_tab) {
+ if (hdl->sa_bonus_tab)
sa_idx_tab_rele(hdl->sa_os, hdl->sa_bonus_tab);
- hdl->sa_bonus_tab = NULL;
- }
- if (hdl->sa_spill_tab) {
+
+ if (hdl->sa_spill_tab)
sa_idx_tab_rele(hdl->sa_os, hdl->sa_spill_tab);
- hdl->sa_spill_tab = NULL;
- }
dmu_buf_rele(hdl->sa_bonus, NULL);
sa_handle_type_t hdl_type, sa_handle_t **handlepp)
{
int error = 0;
- sa_handle_t *handle;
+ sa_handle_t *handle = NULL;
#ifdef ZFS_DEBUG
dmu_object_info_t doi;
/* find handle, if it exists */
/* if one doesn't exist then create a new one, and initialize it */
- handle = (hdl_type == SA_HDL_SHARED) ? dmu_buf_get_user(db) : NULL;
+ if (hdl_type == SA_HDL_SHARED)
+ handle = dmu_buf_get_user(db);
+
if (handle == NULL) {
- sa_handle_t *newhandle;
+ sa_handle_t *winner = NULL;
+
handle = kmem_cache_alloc(sa_cache, KM_SLEEP);
+ handle->sa_dbu.dbu_evict_func_sync = NULL;
+ handle->sa_dbu.dbu_evict_func_async = NULL;
handle->sa_userp = userp;
handle->sa_bonus = db;
handle->sa_os = os;
handle->sa_spill = NULL;
+ handle->sa_bonus_tab = NULL;
+ handle->sa_spill_tab = NULL;
error = sa_build_index(handle, SA_BONUS);
- newhandle = (hdl_type == SA_HDL_SHARED) ?
- dmu_buf_set_user_ie(db, handle,
- NULL, sa_evict) : NULL;
- if (newhandle != NULL) {
+ if (hdl_type == SA_HDL_SHARED) {
+ dmu_buf_init_user(&handle->sa_dbu, sa_evict_sync, NULL,
+ NULL);
+ winner = dmu_buf_set_user_ie(db, &handle->sa_dbu);
+ }
+
+ if (winner != NULL) {
kmem_cache_free(sa_cache, handle);
- handle = newhandle;
+ handle = winner;
}
}
*handlepp = handle;
return (sa_attr_op(hdl, bulk, count, SA_LOOKUP, NULL));
}
-int
-sa_lookup(sa_handle_t *hdl, sa_attr_type_t attr, void *buf, uint32_t buflen)
+static int
+sa_lookup_locked(sa_handle_t *hdl, sa_attr_type_t attr, void *buf,
+ uint32_t buflen)
{
int error;
sa_bulk_attr_t bulk;
+ VERIFY3U(buflen, <=, SA_ATTR_MAX_LEN);
+
bulk.sa_attr = attr;
bulk.sa_data = buf;
bulk.sa_length = buflen;
bulk.sa_data_func = NULL;
ASSERT(hdl);
- mutex_enter(&hdl->sa_lock);
error = sa_lookup_impl(hdl, &bulk, 1);
+ return (error);
+}
+
+int
+sa_lookup(sa_handle_t *hdl, sa_attr_type_t attr, void *buf, uint32_t buflen)
+{
+ int error;
+
+ mutex_enter(&hdl->sa_lock);
+ error = sa_lookup_locked(hdl, attr, buf, buflen);
mutex_exit(&hdl->sa_lock);
+
return (error);
}
mutex_exit(&hdl->sa_lock);
return (error);
}
+
+/*
+ * For the existed object that is upgraded from old system, its ondisk layout
+ * has no slot for the project ID attribute. But quota accounting logic needs
+ * to access related slots by offset directly. So we need to adjust these old
+ * objects' layout to make the project ID to some unified and fixed offset.
+ */
+int
+sa_add_projid(sa_handle_t *hdl, dmu_tx_t *tx, uint64_t projid)
+{
+ znode_t *zp = sa_get_userdata(hdl);
+ dmu_buf_t *db = sa_get_db(hdl);
+ zfsvfs_t *zfsvfs = ZTOZSB(zp);
+ int count = 0, err = 0;
+ sa_bulk_attr_t *bulk, *attrs;
+ zfs_acl_locator_cb_t locate = { 0 };
+ uint64_t uid, gid, mode, rdev, xattr = 0, parent, gen, links;
+ uint64_t crtime[2], mtime[2], ctime[2], atime[2];
+ zfs_acl_phys_t znode_acl = { 0 };
+ char scanstamp[AV_SCANSTAMP_SZ];
+
+ if (zp->z_acl_cached == NULL) {
+ zfs_acl_t *aclp;
+
+ mutex_enter(&zp->z_acl_lock);
+ err = zfs_acl_node_read(zp, B_FALSE, &aclp, B_FALSE);
+ mutex_exit(&zp->z_acl_lock);
+ if (err != 0 && err != ENOENT)
+ return (err);
+ }
+
+ bulk = kmem_zalloc(sizeof (sa_bulk_attr_t) * ZPL_END, KM_SLEEP);
+ attrs = kmem_zalloc(sizeof (sa_bulk_attr_t) * ZPL_END, KM_SLEEP);
+ mutex_enter(&hdl->sa_lock);
+ mutex_enter(&zp->z_lock);
+
+ err = sa_lookup_locked(hdl, SA_ZPL_PROJID(zfsvfs), &projid,
+ sizeof (uint64_t));
+ if (unlikely(err == 0))
+ /* Someone has added project ID attr by race. */
+ err = EEXIST;
+ if (err != ENOENT)
+ goto out;
+
+ /* First do a bulk query of the attributes that aren't cached */
+ if (zp->z_is_sa) {
+ SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_MODE(zfsvfs), NULL,
+ &mode, 8);
+ SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_GEN(zfsvfs), NULL,
+ &gen, 8);
+ SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_UID(zfsvfs), NULL,
+ &uid, 8);
+ SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_GID(zfsvfs), NULL,
+ &gid, 8);
+ SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_PARENT(zfsvfs), NULL,
+ &parent, 8);
+ SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_ATIME(zfsvfs), NULL,
+ &atime, 16);
+ SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_MTIME(zfsvfs), NULL,
+ &mtime, 16);
+ SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_CTIME(zfsvfs), NULL,
+ &ctime, 16);
+ SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_CRTIME(zfsvfs), NULL,
+ &crtime, 16);
+ if (S_ISBLK(ZTOI(zp)->i_mode) || S_ISCHR(ZTOI(zp)->i_mode))
+ SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_RDEV(zfsvfs), NULL,
+ &rdev, 8);
+ } else {
+ SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_ATIME(zfsvfs), NULL,
+ &atime, 16);
+ SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_MTIME(zfsvfs), NULL,
+ &mtime, 16);
+ SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_CTIME(zfsvfs), NULL,
+ &ctime, 16);
+ SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_CRTIME(zfsvfs), NULL,
+ &crtime, 16);
+ SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_GEN(zfsvfs), NULL,
+ &gen, 8);
+ SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_MODE(zfsvfs), NULL,
+ &mode, 8);
+ SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_PARENT(zfsvfs), NULL,
+ &parent, 8);
+ SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_XATTR(zfsvfs), NULL,
+ &xattr, 8);
+ SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_RDEV(zfsvfs), NULL,
+ &rdev, 8);
+ SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_UID(zfsvfs), NULL,
+ &uid, 8);
+ SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_GID(zfsvfs), NULL,
+ &gid, 8);
+ SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_ZNODE_ACL(zfsvfs), NULL,
+ &znode_acl, 88);
+ }
+ err = sa_bulk_lookup_locked(hdl, bulk, count);
+ if (err != 0)
+ goto out;
+
+ err = sa_lookup_locked(hdl, SA_ZPL_XATTR(zfsvfs), &xattr, 8);
+ if (err != 0 && err != ENOENT)
+ goto out;
+
+ zp->z_projid = projid;
+ zp->z_pflags |= ZFS_PROJID;
+ links = ZTOI(zp)->i_nlink;
+ count = 0;
+ err = 0;
+
+ SA_ADD_BULK_ATTR(attrs, count, SA_ZPL_MODE(zfsvfs), NULL, &mode, 8);
+ SA_ADD_BULK_ATTR(attrs, count, SA_ZPL_SIZE(zfsvfs), NULL,
+ &zp->z_size, 8);
+ SA_ADD_BULK_ATTR(attrs, count, SA_ZPL_GEN(zfsvfs), NULL, &gen, 8);
+ SA_ADD_BULK_ATTR(attrs, count, SA_ZPL_UID(zfsvfs), NULL, &uid, 8);
+ SA_ADD_BULK_ATTR(attrs, count, SA_ZPL_GID(zfsvfs), NULL, &gid, 8);
+ SA_ADD_BULK_ATTR(attrs, count, SA_ZPL_PARENT(zfsvfs), NULL, &parent, 8);
+ SA_ADD_BULK_ATTR(attrs, count, SA_ZPL_FLAGS(zfsvfs), NULL,
+ &zp->z_pflags, 8);
+ SA_ADD_BULK_ATTR(attrs, count, SA_ZPL_ATIME(zfsvfs), NULL, &atime, 16);
+ SA_ADD_BULK_ATTR(attrs, count, SA_ZPL_MTIME(zfsvfs), NULL, &mtime, 16);
+ SA_ADD_BULK_ATTR(attrs, count, SA_ZPL_CTIME(zfsvfs), NULL, &ctime, 16);
+ SA_ADD_BULK_ATTR(attrs, count, SA_ZPL_CRTIME(zfsvfs), NULL,
+ &crtime, 16);
+ SA_ADD_BULK_ATTR(attrs, count, SA_ZPL_LINKS(zfsvfs), NULL, &links, 8);
+ SA_ADD_BULK_ATTR(attrs, count, SA_ZPL_PROJID(zfsvfs), NULL, &projid, 8);
+
+ if (S_ISBLK(ZTOI(zp)->i_mode) || S_ISCHR(ZTOI(zp)->i_mode))
+ SA_ADD_BULK_ATTR(attrs, count, SA_ZPL_RDEV(zfsvfs), NULL,
+ &rdev, 8);
+
+ if (zp->z_acl_cached != NULL) {
+ SA_ADD_BULK_ATTR(attrs, count, SA_ZPL_DACL_COUNT(zfsvfs), NULL,
+ &zp->z_acl_cached->z_acl_count, 8);
+ if (zp->z_acl_cached->z_version < ZFS_ACL_VERSION_FUID)
+ zfs_acl_xform(zp, zp->z_acl_cached, CRED());
+ locate.cb_aclp = zp->z_acl_cached;
+ SA_ADD_BULK_ATTR(attrs, count, SA_ZPL_DACL_ACES(zfsvfs),
+ zfs_acl_data_locator, &locate,
+ zp->z_acl_cached->z_acl_bytes);
+ }
+
+ if (xattr)
+ SA_ADD_BULK_ATTR(attrs, count, SA_ZPL_XATTR(zfsvfs), NULL,
+ &xattr, 8);
+
+ if (zp->z_pflags & ZFS_BONUS_SCANSTAMP) {
+ bcopy((caddr_t)db->db_data + ZFS_OLD_ZNODE_PHYS_SIZE,
+ scanstamp, AV_SCANSTAMP_SZ);
+ SA_ADD_BULK_ATTR(attrs, count, SA_ZPL_SCANSTAMP(zfsvfs), NULL,
+ scanstamp, AV_SCANSTAMP_SZ);
+ zp->z_pflags &= ~ZFS_BONUS_SCANSTAMP;
+ }
+
+ VERIFY(dmu_set_bonustype(db, DMU_OT_SA, tx) == 0);
+ VERIFY(sa_replace_all_by_template_locked(hdl, attrs, count, tx) == 0);
+ if (znode_acl.z_acl_extern_obj) {
+ VERIFY(0 == dmu_object_free(zfsvfs->z_os,
+ znode_acl.z_acl_extern_obj, tx));
+ }
+
+ zp->z_is_sa = B_TRUE;
+
+out:
+ mutex_exit(&zp->z_lock);
+ mutex_exit(&hdl->sa_lock);
+ kmem_free(attrs, sizeof (sa_bulk_attr_t) * ZPL_END);
+ kmem_free(bulk, sizeof (sa_bulk_attr_t) * ZPL_END);
+ return (err);
+}
#endif
-void *
-sa_find_idx_tab(objset_t *os, dmu_object_type_t bonustype, void *data)
+static sa_idx_tab_t *
+sa_find_idx_tab(objset_t *os, dmu_object_type_t bonustype, sa_hdr_phys_t *hdr)
{
sa_idx_tab_t *idx_tab;
- sa_hdr_phys_t *hdr = (sa_hdr_phys_t *)data;
sa_os_t *sa = os->os_sa;
sa_lot_t *tb, search;
avl_index_t loc;
}
/* No such luck, create a new entry */
- idx_tab = kmem_zalloc(sizeof (sa_idx_tab_t), KM_PUSHPAGE);
+ idx_tab = kmem_zalloc(sizeof (sa_idx_tab_t), KM_SLEEP);
idx_tab->sa_idx_tab =
- kmem_zalloc(sizeof (uint32_t) * sa->sa_num_attrs, KM_PUSHPAGE);
+ kmem_zalloc(sizeof (uint32_t) * sa->sa_num_attrs, KM_SLEEP);
idx_tab->sa_layout = tb;
- refcount_create(&idx_tab->sa_refcount);
+ zfs_refcount_create(&idx_tab->sa_refcount);
if (tb->lot_var_sizes)
idx_tab->sa_variable_lengths = kmem_alloc(sizeof (uint16_t) *
- tb->lot_var_sizes, KM_PUSHPAGE);
+ tb->lot_var_sizes, KM_SLEEP);
sa_attr_iter(os, hdr, bonustype, sa_build_idx_tab,
tb, idx_tab);
}
if (sa->sa_reg_attr_obj == 0) {
- sa->sa_reg_attr_obj = zap_create(hdl->sa_os,
- DMU_OT_SA_ATTR_REGISTRATION, DMU_OT_NONE, 0, tx);
- VERIFY(zap_add(hdl->sa_os, sa->sa_master_obj,
- SA_REGISTRY, 8, 1, &sa->sa_reg_attr_obj, tx) == 0);
+ sa->sa_reg_attr_obj = zap_create_link(hdl->sa_os,
+ DMU_OT_SA_ATTR_REGISTRATION,
+ sa->sa_master_obj, SA_REGISTRY, tx);
}
for (i = 0; i != sa->sa_num_attrs; i++) {
if (sa->sa_attr_table[i].sa_registered)
}
/*
- * add/remove/replace a single attribute and then rewrite the entire set
+ * Add/remove a single attribute or replace a variable-sized attribute value
+ * with a value of a different size, and then rewrite the entire set
* of attributes.
+ * Same-length attribute value replacement (including fixed-length attributes)
+ * is handled more efficiently by the upper layers.
*/
static int
sa_modify_attrs(sa_handle_t *hdl, sa_attr_type_t newattr,
void *old_data[2];
int bonus_attr_count = 0;
int bonus_data_size = 0;
+ int spill_data_size = 0;
int spill_attr_count = 0;
int error;
- uint16_t length;
+ uint16_t length, reg_length;
int i, j, k, length_idx;
sa_hdr_phys_t *hdr;
sa_idx_tab_t *idx_tab;
/* Bring spill buffer online if it isn't currently */
if ((error = sa_get_spill(hdl)) == 0) {
- ASSERT3U(hdl->sa_spill->db_size, <=, SPA_MAXBLOCKSIZE);
- old_data[1] = sa_spill_alloc(KM_SLEEP);
+ spill_data_size = hdl->sa_spill->db_size;
+ old_data[1] = vmem_alloc(spill_data_size, KM_SLEEP);
bcopy(hdl->sa_spill->db_data, old_data[1],
hdl->sa_spill->db_size);
spill_attr_count =
hdr = SA_GET_HDR(hdl, SA_BONUS);
idx_tab = SA_IDX_TAB_GET(hdl, SA_BONUS);
for (; k != 2; k++) {
- /* iterate over each attribute in layout */
+ /*
+ * Iterate over each attribute in layout. Fetch the
+ * size of variable-length attributes needing rewrite
+ * from sa_lengths[].
+ */
for (i = 0, length_idx = 0; i != count; i++) {
sa_attr_type_t attr;
attr = idx_tab->sa_layout->lot_attrs[i];
+ reg_length = SA_REGISTERED_LEN(sa, attr);
+ if (reg_length == 0) {
+ length = hdr->sa_lengths[length_idx];
+ length_idx++;
+ } else {
+ length = reg_length;
+ }
if (attr == newattr) {
- if (action == SA_REMOVE) {
- j++;
+ /*
+ * There is nothing to do for SA_REMOVE,
+ * so it is just skipped.
+ */
+ if (action == SA_REMOVE)
continue;
- }
- ASSERT(SA_REGISTERED_LEN(sa, attr) == 0);
- ASSERT(action == SA_REPLACE);
+
+ /*
+ * Duplicate attributes are not allowed, so the
+ * action can not be SA_ADD here.
+ */
+ ASSERT3S(action, ==, SA_REPLACE);
+
+ /*
+ * Only a variable-sized attribute can be
+ * replaced here, and its size must be changing.
+ */
+ ASSERT3U(reg_length, ==, 0);
+ ASSERT3U(length, !=, buflen);
SA_ADD_BULK_ATTR(attr_desc, j, attr,
locator, datastart, buflen);
} else {
- length = SA_REGISTERED_LEN(sa, attr);
- if (length == 0) {
- length = hdr->sa_lengths[length_idx++];
- }
-
SA_ADD_BULK_ATTR(attr_desc, j, attr,
NULL, (void *)
(TOC_OFF(idx_tab->sa_idx_tab[attr]) +
}
}
if (action == SA_ADD) {
- length = SA_REGISTERED_LEN(sa, newattr);
- if (length == 0) {
- length = buflen;
- }
+ reg_length = SA_REGISTERED_LEN(sa, newattr);
+ IMPLY(reg_length != 0, reg_length == buflen);
SA_ADD_BULK_ATTR(attr_desc, j, newattr, locator,
datastart, buflen);
}
+ ASSERT3U(j, ==, attr_count);
error = sa_build_layouts(hdl, attr_desc, attr_count, tx);
if (old_data[0])
kmem_free(old_data[0], bonus_data_size);
if (old_data[1])
- sa_spill_free(old_data[1]);
+ vmem_free(old_data[1], spill_data_size);
kmem_free(attr_desc, sizeof (sa_bulk_attr_t) * attr_count);
return (error);
int error;
sa_bulk_attr_t bulk;
+ VERIFY3U(buflen, <=, SA_ATTR_MAX_LEN);
+
bulk.sa_attr = type;
bulk.sa_data_func = NULL;
bulk.sa_length = buflen;
return (error);
}
-int
-sa_update_from_cb(sa_handle_t *hdl, sa_attr_type_t attr,
- uint32_t buflen, sa_data_locator_t *locator, void *userdata, dmu_tx_t *tx)
-{
- int error;
- sa_bulk_attr_t bulk;
-
- bulk.sa_attr = attr;
- bulk.sa_data = userdata;
- bulk.sa_data_func = locator;
- bulk.sa_length = buflen;
-
- mutex_enter(&hdl->sa_lock);
- error = sa_bulk_update_impl(hdl, &bulk, 1, tx);
- mutex_exit(&hdl->sa_lock);
- return (error);
-}
-
/*
* Return size of an attribute
*/
blksize, nblocks);
}
-void
-sa_update_user(sa_handle_t *newhdl, sa_handle_t *oldhdl)
-{
- (void) dmu_buf_update_user((dmu_buf_t *)newhdl->sa_bonus,
- oldhdl, newhdl, NULL, sa_evict);
- oldhdl->sa_bonus = NULL;
-}
-
void
sa_set_userp(sa_handle_t *hdl, void *ptr)
{
EXPORT_SYMBOL(sa_bulk_lookup_locked);
EXPORT_SYMBOL(sa_bulk_update);
EXPORT_SYMBOL(sa_size);
-EXPORT_SYMBOL(sa_update_from_cb);
EXPORT_SYMBOL(sa_object_info);
EXPORT_SYMBOL(sa_object_size);
-EXPORT_SYMBOL(sa_update_user);
EXPORT_SYMBOL(sa_get_userdata);
EXPORT_SYMBOL(sa_set_userp);
EXPORT_SYMBOL(sa_get_db);
EXPORT_SYMBOL(sa_enabled);
EXPORT_SYMBOL(sa_cache_init);
EXPORT_SYMBOL(sa_cache_fini);
-EXPORT_SYMBOL(sa_spill_alloc);
-EXPORT_SYMBOL(sa_spill_free);
EXPORT_SYMBOL(sa_set_sa_object);
EXPORT_SYMBOL(sa_hdrsize);
EXPORT_SYMBOL(sa_handle_lock);
EXPORT_SYMBOL(sa_handle_unlock);
EXPORT_SYMBOL(sa_lookup_uio);
+EXPORT_SYMBOL(sa_add_projid);
#endif /* _KERNEL */
projects / mirror_zfs.git / blobdiff