ceph/src/test/rgw/rgw_multi/tests_es.py

   1 import json
   2 import logging
   3
   4 import boto
   5 import boto.s3.connection
   6
   7 import datetime
   8 import dateutil
   9
  10 from nose.tools import eq_ as eq
  11
  12 from rgw_multi.multisite import *
  13 from rgw_multi.tests import *
  14 from rgw_multi.zone_es import *
  15
  16 log = logging.getLogger(__name__)
  17
  18
  19 def check_es_configured():
  20     realm = get_realm()
  21     zonegroup = realm.master_zonegroup()
  22
  23     es_zones = zonegroup.zones_by_type.get("elasticsearch")
  24     if not es_zones:
  25         raise SkipTest("Requires at least one ES zone")
  26
  27 def is_es_zone(zone_conn):
  28     if not zone_conn:
  29         return False
  30
  31     return zone_conn.zone.tier_type() == "elasticsearch"
  32
  33 def verify_search(bucket_name, src_keys, result_keys, f):
  34     check_keys = []
  35     for k in src_keys:
  36         if bucket_name:
  37             if bucket_name != k.bucket.name:
  38                 continue
  39         if f(k):
  40             check_keys.append(k)
  41     check_keys.sort(key = lambda l: (l.bucket.name, l.name, l.version_id))
  42
  43     log.debug('check keys:' + dump_json(check_keys))
  44     log.debug('result keys:' + dump_json(result_keys))
  45
  46     for k1, k2 in zip_longest(check_keys, result_keys):
  47         assert k1
  48         assert k2
  49         check_object_eq(k1, k2)
  50
  51 def do_check_mdsearch(conn, bucket, src_keys, req_str, src_filter):
  52     if bucket:
  53         bucket_name = bucket.name
  54     else:
  55         bucket_name = ''
  56     req = MDSearch(conn, bucket_name, req_str)
  57     result_keys = req.search(sort_key = lambda k: (k.bucket.name, k.name, k.version_id))
  58     verify_search(bucket_name, src_keys, result_keys, src_filter)
  59
  60 def init_env(create_obj, num_keys = 5, buckets_per_zone = 1, bucket_init_cb = None):
  61     check_es_configured()
  62
  63     realm = get_realm()
  64     zonegroup = realm.master_zonegroup()
  65     zonegroup_conns = ZonegroupConns(zonegroup)
  66     buckets, zone_bucket = create_bucket_per_zone(zonegroup_conns, buckets_per_zone = buckets_per_zone)
  67
  68     if bucket_init_cb:
  69         for zone_conn, bucket in zone_bucket:
  70             bucket_init_cb(zone_conn, bucket)
  71
  72     src_keys = []
  73
  74     owner = None
  75
  76     obj_prefix=''.join(random.choice(string.ascii_lowercase) for _ in range(6))
  77
  78     # don't wait for meta sync just yet
  79     for zone, bucket in zone_bucket:
  80         for count in xrange(0, num_keys):
  81             objname = obj_prefix + str(count)
  82             k = new_key(zone, bucket.name, objname)
  83             # k.set_contents_from_string(content + 'x' * count)
  84             if not create_obj:
  85                 continue
  86
  87             create_obj(k, count)
  88
  89             if not owner:
  90                 for list_key in bucket.list_versions():
  91                     owner = list_key.owner
  92                     break
  93
  94             k = bucket.get_key(k.name, version_id = k.version_id)
  95             k.owner = owner # owner is not set when doing get_key()
  96
  97             src_keys.append(k)
  98
  99     zonegroup_meta_checkpoint(zonegroup)
 100
 101     sources = []
 102     targets = []
 103     for target_conn in zonegroup_conns.zones:
 104         if not is_es_zone(target_conn):
 105             sources.append(target_conn)
 106             continue
 107
 108         targets.append(target_conn)
 109
 110     buckets = []
 111     # make sure all targets are synced
 112     for source_conn, bucket in zone_bucket:
 113         buckets.append(bucket)
 114         for target_conn in targets:
 115             zone_bucket_checkpoint(target_conn.zone, source_conn.zone, bucket.name)
 116
 117     return targets, sources, buckets, src_keys
 118
 119 def test_es_object_search():
 120     min_size = 10
 121     content = 'a' * min_size
 122
 123     def create_obj(k, i):
 124         k.set_contents_from_string(content + 'x' * i)
 125
 126     targets, _, buckets, src_keys = init_env(create_obj, num_keys = 5, buckets_per_zone = 2)
 127
 128     for target_conn in targets:
 129
 130         # bucket checks
 131         for bucket in buckets:
 132             # check name
 133             do_check_mdsearch(target_conn.conn, None, src_keys , 'bucket == ' + bucket.name, lambda k: k.bucket.name == bucket.name)
 134             do_check_mdsearch(target_conn.conn, bucket, src_keys , 'bucket == ' + bucket.name, lambda k: k.bucket.name == bucket.name)
 135
 136         # check on all buckets
 137         for key in src_keys:
 138             # limiting to checking specific key name, otherwise could get results from
 139             # other runs / tests
 140             do_check_mdsearch(target_conn.conn, None, src_keys , 'name == ' + key.name, lambda k: k.name == key.name)
 141
 142         # check on specific bucket
 143         for bucket in buckets:
 144             for key in src_keys:
 145                 do_check_mdsearch(target_conn.conn, bucket, src_keys , 'name < ' + key.name, lambda k: k.name < key.name)
 146                 do_check_mdsearch(target_conn.conn, bucket, src_keys , 'name <= ' + key.name, lambda k: k.name <= key.name)
 147                 do_check_mdsearch(target_conn.conn, bucket, src_keys , 'name == ' + key.name, lambda k: k.name == key.name)
 148                 do_check_mdsearch(target_conn.conn, bucket, src_keys , 'name >= ' + key.name, lambda k: k.name >= key.name)
 149                 do_check_mdsearch(target_conn.conn, bucket, src_keys , 'name > ' + key.name, lambda k: k.name > key.name)
 150
 151             do_check_mdsearch(target_conn.conn, bucket, src_keys , 'name == ' + src_keys[0].name + ' or name >= ' + src_keys[2].name,
 152                               lambda k: k.name == src_keys[0].name or k.name >= src_keys[2].name)
 153
 154             # check etag
 155             for key in src_keys:
 156                 do_check_mdsearch(target_conn.conn, bucket, src_keys , 'etag < ' + key.etag[1:-1], lambda k: k.etag < key.etag)
 157             for key in src_keys:
 158                 do_check_mdsearch(target_conn.conn, bucket, src_keys , 'etag == ' + key.etag[1:-1], lambda k: k.etag == key.etag)
 159             for key in src_keys:
 160                 do_check_mdsearch(target_conn.conn, bucket, src_keys , 'etag > ' + key.etag[1:-1], lambda k: k.etag > key.etag)
 161
 162             # check size
 163             for key in src_keys:
 164                 do_check_mdsearch(target_conn.conn, bucket, src_keys , 'size < ' + str(key.size), lambda k: k.size < key.size)
 165             for key in src_keys:
 166                 do_check_mdsearch(target_conn.conn, bucket, src_keys , 'size <= ' + str(key.size), lambda k: k.size <= key.size)
 167             for key in src_keys:
 168                 do_check_mdsearch(target_conn.conn, bucket, src_keys , 'size == ' + str(key.size), lambda k: k.size == key.size)
 169             for key in src_keys:
 170                 do_check_mdsearch(target_conn.conn, bucket, src_keys , 'size >= ' + str(key.size), lambda k: k.size >= key.size)
 171             for key in src_keys:
 172                 do_check_mdsearch(target_conn.conn, bucket, src_keys , 'size > ' + str(key.size), lambda k: k.size > key.size)
 173
 174 def date_from_str(s):
 175     return dateutil.parser.parse(s)
 176
 177 def test_es_object_search_custom():
 178     min_size = 10
 179     content = 'a' * min_size
 180
 181     def bucket_init(zone_conn, bucket):
 182         req = MDSearchConfig(zone_conn.conn, bucket.name)
 183         req.set_config('x-amz-meta-foo-str; string, x-amz-meta-foo-int; int, x-amz-meta-foo-date; date')
 184
 185     def create_obj(k, i):
 186         date = datetime.datetime.now() + datetime.timedelta(seconds=1) * i
 187         date_str = date.strftime('%Y-%m-%dT%H:%M:%S.%f')[:-3] + 'Z'
 188         k.set_contents_from_string(content + 'x' * i, headers = { 'X-Amz-Meta-Foo-Str': str(i * 5),
 189                                                                   'X-Amz-Meta-Foo-Int': str(i * 5),
 190                                                                   'X-Amz-Meta-Foo-Date': date_str})
 191
 192     targets, _, buckets, src_keys = init_env(create_obj, num_keys = 5, buckets_per_zone = 1, bucket_init_cb = bucket_init)
 193
 194
 195     for target_conn in targets:
 196
 197         # bucket checks
 198         for bucket in buckets:
 199             str_vals = []
 200             for key in src_keys:
 201                 # check string values
 202                 val = key.get_metadata('foo-str')
 203                 str_vals.append(val)
 204                 do_check_mdsearch(target_conn.conn, bucket, src_keys , 'x-amz-meta-foo-str < ' + val, lambda k: k.get_metadata('foo-str') < val)
 205                 do_check_mdsearch(target_conn.conn, bucket, src_keys , 'x-amz-meta-foo-str <= ' + val, lambda k: k.get_metadata('foo-str') <= val)
 206                 do_check_mdsearch(target_conn.conn, bucket, src_keys , 'x-amz-meta-foo-str == ' + val, lambda k: k.get_metadata('foo-str') == val)
 207                 do_check_mdsearch(target_conn.conn, bucket, src_keys , 'x-amz-meta-foo-str >= ' + val, lambda k: k.get_metadata('foo-str') >= val)
 208                 do_check_mdsearch(target_conn.conn, bucket, src_keys , 'x-amz-meta-foo-str > ' + val, lambda k: k.get_metadata('foo-str') > val)
 209
 210                 # check int values
 211                 sval = key.get_metadata('foo-int')
 212                 val = int(sval)
 213                 do_check_mdsearch(target_conn.conn, bucket, src_keys , 'x-amz-meta-foo-int < ' + sval, lambda k: int(k.get_metadata('foo-int')) < val)
 214                 do_check_mdsearch(target_conn.conn, bucket, src_keys , 'x-amz-meta-foo-int <= ' + sval, lambda k: int(k.get_metadata('foo-int')) <= val)
 215                 do_check_mdsearch(target_conn.conn, bucket, src_keys , 'x-amz-meta-foo-int == ' + sval, lambda k: int(k.get_metadata('foo-int')) == val)
 216                 do_check_mdsearch(target_conn.conn, bucket, src_keys , 'x-amz-meta-foo-int >= ' + sval, lambda k: int(k.get_metadata('foo-int')) >= val)
 217                 do_check_mdsearch(target_conn.conn, bucket, src_keys , 'x-amz-meta-foo-int > ' + sval, lambda k: int(k.get_metadata('foo-int')) > val)
 218
 219                 # check int values
 220                 sval = key.get_metadata('foo-date')
 221                 val = date_from_str(sval)
 222                 do_check_mdsearch(target_conn.conn, bucket, src_keys , 'x-amz-meta-foo-date < ' + sval, lambda k: date_from_str(k.get_metadata('foo-date')) < val)
 223                 do_check_mdsearch(target_conn.conn, bucket, src_keys , 'x-amz-meta-foo-date <= ' + sval, lambda k: date_from_str(k.get_metadata('foo-date')) <= val)
 224                 do_check_mdsearch(target_conn.conn, bucket, src_keys , 'x-amz-meta-foo-date == ' + sval, lambda k: date_from_str(k.get_metadata('foo-date')) == val)
 225                 do_check_mdsearch(target_conn.conn, bucket, src_keys , 'x-amz-meta-foo-date >= ' + sval, lambda k: date_from_str(k.get_metadata('foo-date')) >= val)
 226                 do_check_mdsearch(target_conn.conn, bucket, src_keys , 'x-amz-meta-foo-date > ' + sval, lambda k: date_from_str(k.get_metadata('foo-date')) > val)
 227
 228             # 'or' query
 229             for i in xrange(len(src_keys) / 2):
 230                 do_check_mdsearch(target_conn.conn, bucket, src_keys , 'x-amz-meta-foo-str <= ' + str_vals[i] + ' or x-amz-meta-foo-str >= ' + str_vals[-i],
 231                         lambda k: k.get_metadata('foo-str') <= str_vals[i] or k.get_metadata('foo-str') >= str_vals[-i] )
 232
 233             # 'and' query
 234             for i in xrange(len(src_keys) / 2):
 235                 do_check_mdsearch(target_conn.conn, bucket, src_keys , 'x-amz-meta-foo-str >= ' + str_vals[i] + ' and x-amz-meta-foo-str <= ' + str_vals[i + 1],
 236                         lambda k: k.get_metadata('foo-str') >= str_vals[i] and k.get_metadata('foo-str') <= str_vals[i + 1] )
 237             # more complicated query
 238             for i in xrange(len(src_keys) / 2):
 239                 do_check_mdsearch(target_conn.conn, None, src_keys , 'bucket == ' + bucket.name + ' and x-amz-meta-foo-str >= ' + str_vals[i] +
 240                                                                      ' and (x-amz-meta-foo-str <= ' + str_vals[i + 1] + ')',
 241                         lambda k: k.bucket.name == bucket.name and (k.get_metadata('foo-str') >= str_vals[i] and
 242                                                                     k.get_metadata('foo-str') <= str_vals[i + 1]) )
 243
 244 def test_es_bucket_conf():
 245     min_size = 0
 246
 247     def bucket_init(zone_conn, bucket):
 248         req = MDSearchConfig(zone_conn.conn, bucket.name)
 249         req.set_config('x-amz-meta-foo-str; string, x-amz-meta-foo-int; int, x-amz-meta-foo-date; date')
 250
 251     targets, sources, buckets, _ = init_env(None, num_keys = 5, buckets_per_zone = 1, bucket_init_cb = bucket_init)
 252
 253     for source_conn in sources:
 254         for bucket in buckets:
 255             req = MDSearchConfig(source_conn.conn, bucket.name)
 256             conf = req.get_config()
 257
 258             d = {}
 259
 260             for entry in conf:
 261               d[entry['Key']] = entry['Type']
 262
 263             eq(len(d), 3)
 264             eq(d['x-amz-meta-foo-str'], 'str')
 265             eq(d['x-amz-meta-foo-int'], 'int')
 266             eq(d['x-amz-meta-foo-date'], 'date')
 267
 268             req.del_config()
 269
 270             conf = req.get_config()
 271
 272             eq(len(conf), 0)
 273
 274         break # no need to iterate over all zones