libceph: fix pg_temp mapping calculation

We need to apply the modulo pg_num calculation before looking up a pgid in
the pg_temp mapping rbtree.  This fixes pg_temp mappings, and fixes
(some) misdirected requests that result in messages like

[WRN] client4104 10.0.1.219:0/275025290 misdirected client4104.1:129 0.1 to osd0 not [1,0] in e11/11

on the server and stall make the client block without getting a reply (at
least until the pg_temp mapping goes way, but that can take a long long
time).

Reorder calc_pg_raw() a bit to make more sense.

Signed-off-by: Sage Weil <sage@newdream.net>

diff --git a/net/ceph/osdmap.c b/net/ceph/osdmap.c
index e97c3588c3ec00e719f7cb400784c36b5cb2fad6..eceb8d56703c807f9c830677840d29ccab8e13fd 100644 (file)
--- a/net/ceph/osdmap.c
+++ b/net/ceph/osdmap.c
@@ -1046,10 +1046,25 @@ static int *calc_pg_raw(struct ceph_osdmap *osdmap, struct ceph_pg pgid,
        struct ceph_pg_mapping *pg;
        struct ceph_pg_pool_info *pool;
        int ruleno;
-       unsigned poolid, ps, pps;
+       unsigned poolid, ps, pps, t;
        int preferred;
 
+       poolid = le32_to_cpu(pgid.pool);
+       ps = le16_to_cpu(pgid.ps);
+       preferred = (s16)le16_to_cpu(pgid.preferred);
+
+       pool = __lookup_pg_pool(&osdmap->pg_pools, poolid);
+       if (!pool)
+               return NULL;
+
        /* pg_temp? */
+       if (preferred >= 0)
+               t = ceph_stable_mod(ps, le32_to_cpu(pool->v.lpg_num),
+                                   pool->lpgp_num_mask);
+       else
+               t = ceph_stable_mod(ps, le32_to_cpu(pool->v.pg_num),
+                                   pool->pgp_num_mask);
+       pgid.ps = cpu_to_le16(t);
        pg = __lookup_pg_mapping(&osdmap->pg_temp, pgid);
        if (pg) {
                *num = pg->len;
@@ -1057,18 +1072,6 @@ static int *calc_pg_raw(struct ceph_osdmap *osdmap, struct ceph_pg pgid,
        }
 
        /* crush */
-       poolid = le32_to_cpu(pgid.pool);
-       ps = le16_to_cpu(pgid.ps);
-       preferred = (s16)le16_to_cpu(pgid.preferred);
-
-       /* don't forcefeed bad device ids to crush */
-       if (preferred >= osdmap->max_osd ||
-           preferred >= osdmap->crush->max_devices)
-               preferred = -1;
-
-       pool = __lookup_pg_pool(&osdmap->pg_pools, poolid);
-       if (!pool)
-               return NULL;
        ruleno = crush_find_rule(osdmap->crush, pool->v.crush_ruleset,
                                 pool->v.type, pool->v.size);
        if (ruleno < 0) {
@@ -1078,6 +1081,11 @@ static int *calc_pg_raw(struct ceph_osdmap *osdmap, struct ceph_pg pgid,
                return NULL;
        }
 
+       /* don't forcefeed bad device ids to crush */
+       if (preferred >= osdmap->max_osd ||
+           preferred >= osdmap->crush->max_devices)
+               preferred = -1;
+
        if (preferred >= 0)
                pps = ceph_stable_mod(ps,
                                      le32_to_cpu(pool->v.lpgp_num),