pNFS/flexfiles: Refactor encoding of the layoutreturn payload

Add the layout error payload to the flexfiles layoutreturn private
data, and set up the encoding mechanisms. This is a refactoring in
preparation for adding the layout iostats payload.

Signed-off-by: Trond Myklebust <trond.myklebust@primarydata.com>

diff --git a/fs/nfs/flexfilelayout/flexfilelayout.c b/fs/nfs/flexfilelayout/flexfilelayout.c
index 540813c30b00b57b792cd909ec9be42444a662c3..7f31c359a28a8284d6d6f5ca9e163eb171fe9c16 100644 (file)
--- a/fs/nfs/flexfilelayout/flexfilelayout.c
+++ b/fs/nfs/flexfilelayout/flexfilelayout.c
@@ -25,6 +25,8 @@
 #define NFSDBG_FACILITY         NFSDBG_PNFS_LD
 
 #define FF_LAYOUT_POLL_RETRY_MAX     (15*HZ)
+#define FF_LAYOUTRETURN_MAXERR 20
+
 
 static struct group_info       *ff_zero_group;
 
@@ -1971,24 +1973,18 @@ ff_layout_free_deviceid_node(struct nfs4_deviceid_node *d)
 
 static int ff_layout_encode_ioerr(struct nfs4_flexfile_layout *flo,
                                  struct xdr_stream *xdr,
-                                 const struct nfs4_layoutreturn_args *args)
+                                 const struct nfs4_layoutreturn_args *args,
+                                 const struct nfs4_flexfile_layoutreturn_args *ff_args)
 {
-       struct pnfs_layout_hdr *hdr = &flo->generic_hdr;
        __be32 *start;
-       int count = 0, ret = 0;
 
        start = xdr_reserve_space(xdr, 4);
        if (unlikely(!start))
                return -E2BIG;
 
+       *start = cpu_to_be32(ff_args->num_errors);
        /* This assume we always return _ALL_ layouts */
-       spin_lock(&hdr->plh_inode->i_lock);
-       ret = ff_layout_encode_ds_ioerr(flo, xdr, &count, &args->range);
-       spin_unlock(&hdr->plh_inode->i_lock);
-
-       *start = cpu_to_be32(count);
-
-       return ret;
+       return ff_layout_encode_ds_ioerr(xdr, &ff_args->errors);
 }
 
 /* report nothing for now */
@@ -2017,8 +2013,10 @@ ff_layout_alloc_deviceid_node(struct nfs_server *server,
 
 static void
 ff_layout_encode_layoutreturn(struct xdr_stream *xdr,
-                             const struct nfs4_layoutreturn_args *args)
+               const void *voidargs,
+               const struct nfs4_xdr_opaque_data *ff_opaque)
 {
+       const struct nfs4_layoutreturn_args *args = voidargs;
        struct pnfs_layout_hdr *lo = args->layout;
        struct nfs4_flexfile_layout *flo = FF_LAYOUT_FROM_HDR(lo);
        __be32 *start;
@@ -2027,13 +2025,52 @@ ff_layout_encode_layoutreturn(struct xdr_stream *xdr,
        start = xdr_reserve_space(xdr, 4);
        BUG_ON(!start);
 
-       ff_layout_encode_ioerr(flo, xdr, args);
+       ff_layout_encode_ioerr(flo, xdr, args, ff_opaque->data);
        ff_layout_encode_iostats(flo, xdr, args);
 
        *start = cpu_to_be32((xdr->p - start - 1) * 4);
        dprintk("%s: Return\n", __func__);
 }
 
+static void
+ff_layout_free_layoutreturn(struct nfs4_xdr_opaque_data *args)
+{
+       struct nfs4_flexfile_layoutreturn_args *ff_args;
+
+       if (!args->data)
+               return;
+       ff_args = args->data;
+       args->data = NULL;
+
+       ff_layout_free_ds_ioerr(&ff_args->errors);
+
+       kfree(ff_args);
+}
+
+const struct nfs4_xdr_opaque_ops layoutreturn_ops = {
+       .encode = ff_layout_encode_layoutreturn,
+       .free = ff_layout_free_layoutreturn,
+};
+
+static int
+ff_layout_prepare_layoutreturn(struct nfs4_layoutreturn_args *args)
+{
+       struct nfs4_flexfile_layoutreturn_args *ff_args;
+
+       ff_args = kmalloc(sizeof(*ff_args), GFP_KERNEL);
+       if (!ff_args)
+               return -ENOMEM;
+
+       INIT_LIST_HEAD(&ff_args->errors);
+       ff_args->num_errors = ff_layout_fetch_ds_ioerr(args->layout,
+                       &args->range, &ff_args->errors,
+                       FF_LAYOUTRETURN_MAXERR);
+
+       args->ld_private->ops = &layoutreturn_ops;
+       args->ld_private->data = ff_args;
+       return 0;
+}
+
 static int
 ff_layout_ntop4(const struct sockaddr *sap, char *buf, const size_t buflen)
 {
@@ -2299,7 +2336,7 @@ static struct pnfs_layoutdriver_type flexfilelayout_type = {
        .read_pagelist          = ff_layout_read_pagelist,
        .write_pagelist         = ff_layout_write_pagelist,
        .alloc_deviceid_node    = ff_layout_alloc_deviceid_node,
-       .encode_layoutreturn    = ff_layout_encode_layoutreturn,
+       .prepare_layoutreturn   = ff_layout_prepare_layoutreturn,
        .sync                   = pnfs_nfs_generic_sync,
        .prepare_layoutstats    = ff_layout_prepare_layoutstats,
        .cleanup_layoutstats    = ff_layout_cleanup_layoutstats,

diff --git a/fs/nfs/flexfilelayout/flexfilelayout.h b/fs/nfs/flexfilelayout/flexfilelayout.h
index 09f292e3a4ada64561ef0775a24e88bf89b02c6f..560c837995fcca16b8ad807d4f69806302ba8637 100644 (file)
--- a/fs/nfs/flexfilelayout/flexfilelayout.h
+++ b/fs/nfs/flexfilelayout/flexfilelayout.h
@@ -106,6 +106,11 @@ struct nfs4_flexfile_layout {
        ktime_t                 last_report_time; /* Layoutstat report times */
 };
 
+struct nfs4_flexfile_layoutreturn_args {
+       struct list_head errors;
+       unsigned int num_errors;
+};
+
 static inline struct nfs4_flexfile_layout *
 FF_LAYOUT_FROM_HDR(struct pnfs_layout_hdr *lo)
 {
@@ -183,9 +188,12 @@ int ff_layout_track_ds_error(struct nfs4_flexfile_layout *flo,
                             struct nfs4_ff_layout_mirror *mirror, u64 offset,
                             u64 length, int status, enum nfs_opnum4 opnum,
                             gfp_t gfp_flags);
-int ff_layout_encode_ds_ioerr(struct nfs4_flexfile_layout *flo,
-                             struct xdr_stream *xdr, int *count,
-                             const struct pnfs_layout_range *range);
+int ff_layout_encode_ds_ioerr(struct xdr_stream *xdr, const struct list_head *head);
+void ff_layout_free_ds_ioerr(struct list_head *head);
+unsigned int ff_layout_fetch_ds_ioerr(struct pnfs_layout_hdr *lo,
+               const struct pnfs_layout_range *range,
+               struct list_head *head,
+               unsigned int maxnum);
 struct nfs_fh *
 nfs4_ff_layout_select_ds_fh(struct pnfs_layout_segment *lseg, u32 mirror_idx);
 

diff --git a/fs/nfs/flexfilelayout/flexfilelayoutdev.c b/fs/nfs/flexfilelayout/flexfilelayoutdev.c
index 75767aa3845546895c1525805bfce2c9db3d8928..eb98395c3651c0720506f2cc40f7986092438b7b 100644 (file)
--- a/fs/nfs/flexfilelayout/flexfilelayoutdev.c
+++ b/fs/nfs/flexfilelayout/flexfilelayoutdev.c
@@ -447,20 +447,26 @@ nfs4_ff_find_or_create_ds_client(struct pnfs_layout_segment *lseg, u32 ds_idx,
        }
 }
 
+void ff_layout_free_ds_ioerr(struct list_head *head)
+{
+       struct nfs4_ff_layout_ds_err *err;
+
+       while (!list_empty(head)) {
+               err = list_first_entry(head,
+                               struct nfs4_ff_layout_ds_err,
+                               list);
+               list_del(&err->list);
+               kfree(err);
+       }
+}
+
 /* called with inode i_lock held */
-int ff_layout_encode_ds_ioerr(struct nfs4_flexfile_layout *flo,
-                             struct xdr_stream *xdr, int *count,
-                             const struct pnfs_layout_range *range)
+int ff_layout_encode_ds_ioerr(struct xdr_stream *xdr, const struct list_head *head)
 {
-       struct nfs4_ff_layout_ds_err *err, *n;
+       struct nfs4_ff_layout_ds_err *err;
        __be32 *p;
 
-       list_for_each_entry_safe(err, n, &flo->error_list, list) {
-               if (!pnfs_is_range_intersecting(err->offset,
-                               pnfs_end_offset(err->offset, err->length),
-                               range->offset,
-                               pnfs_end_offset(range->offset, range->length)))
-                       continue;
+       list_for_each_entry(err, head, list) {
                /* offset(8) + length(8) + stateid(NFS4_STATEID_SIZE)
                 * + array length + deviceid(NFS4_DEVICEID4_SIZE)
                 * + status(4) + opnum(4)
@@ -479,17 +485,59 @@ int ff_layout_encode_ds_ioerr(struct nfs4_flexfile_layout *flo,
                                            NFS4_DEVICEID4_SIZE);
                *p++ = cpu_to_be32(err->status);
                *p++ = cpu_to_be32(err->opnum);
-               *count += 1;
-               list_del(&err->list);
-               dprintk("%s: offset %llu length %llu status %d op %d count %d\n",
+               dprintk("%s: offset %llu length %llu status %d op %d\n",
                        __func__, err->offset, err->length, err->status,
-                       err->opnum, *count);
-               kfree(err);
+                       err->opnum);
        }
 
        return 0;
 }
 
+static
+unsigned int do_layout_fetch_ds_ioerr(struct pnfs_layout_hdr *lo,
+                                     const struct pnfs_layout_range *range,
+                                     struct list_head *head,
+                                     unsigned int maxnum)
+{
+       struct nfs4_flexfile_layout *flo = FF_LAYOUT_FROM_HDR(lo);
+       struct inode *inode = lo->plh_inode;
+       struct nfs4_ff_layout_ds_err *err, *n;
+       unsigned int ret = 0;
+
+       spin_lock(&inode->i_lock);
+       list_for_each_entry_safe(err, n, &flo->error_list, list) {
+               if (!pnfs_is_range_intersecting(err->offset,
+                               pnfs_end_offset(err->offset, err->length),
+                               range->offset,
+                               pnfs_end_offset(range->offset, range->length)))
+                       continue;
+               if (!maxnum)
+                       break;
+               list_move(&err->list, head);
+               maxnum--;
+               ret++;
+       }
+       spin_unlock(&inode->i_lock);
+       return ret;
+}
+
+unsigned int ff_layout_fetch_ds_ioerr(struct pnfs_layout_hdr *lo,
+                                     const struct pnfs_layout_range *range,
+                                     struct list_head *head,
+                                     unsigned int maxnum)
+{
+       unsigned int ret;
+
+       ret = do_layout_fetch_ds_ioerr(lo, range, head, maxnum);
+       /* If we're over the max, discard all remaining entries */
+       if (ret == maxnum) {
+               LIST_HEAD(discard);
+               do_layout_fetch_ds_ioerr(lo, range, &discard, -1);
+               ff_layout_free_ds_ioerr(&discard);
+       }
+       return ret;
+}
+
 static bool ff_read_layout_has_available_ds(struct pnfs_layout_segment *lseg)
 {
        struct nfs4_ff_layout_mirror *mirror;