]>
Commit | Line | Data |
---|---|---|
31f18b77 FG |
1 | import json |
2 | import logging | |
3 | ||
4 | import boto | |
5 | import boto.s3.connection | |
6 | ||
7 | import datetime | |
8 | import dateutil | |
9 | ||
f67539c2 TL |
10 | from itertools import zip_longest # type: ignore |
11 | ||
31f18b77 FG |
12 | from nose.tools import eq_ as eq |
13 | ||
e306af50 TL |
14 | from .multisite import * |
15 | from .tests import * | |
16 | from .zone_es import * | |
31f18b77 FG |
17 | |
18 | log = logging.getLogger(__name__) | |
19 | ||
20 | ||
21 | def check_es_configured(): | |
22 | realm = get_realm() | |
23 | zonegroup = realm.master_zonegroup() | |
24 | ||
25 | es_zones = zonegroup.zones_by_type.get("elasticsearch") | |
26 | if not es_zones: | |
27 | raise SkipTest("Requires at least one ES zone") | |
28 | ||
29 | def is_es_zone(zone_conn): | |
30 | if not zone_conn: | |
31 | return False | |
32 | ||
33 | return zone_conn.zone.tier_type() == "elasticsearch" | |
34 | ||
35 | def verify_search(bucket_name, src_keys, result_keys, f): | |
36 | check_keys = [] | |
37 | for k in src_keys: | |
38 | if bucket_name: | |
39 | if bucket_name != k.bucket.name: | |
40 | continue | |
41 | if f(k): | |
42 | check_keys.append(k) | |
43 | check_keys.sort(key = lambda l: (l.bucket.name, l.name, l.version_id)) | |
44 | ||
45 | log.debug('check keys:' + dump_json(check_keys)) | |
46 | log.debug('result keys:' + dump_json(result_keys)) | |
47 | ||
48 | for k1, k2 in zip_longest(check_keys, result_keys): | |
49 | assert k1 | |
50 | assert k2 | |
51 | check_object_eq(k1, k2) | |
52 | ||
53 | def do_check_mdsearch(conn, bucket, src_keys, req_str, src_filter): | |
54 | if bucket: | |
55 | bucket_name = bucket.name | |
56 | else: | |
57 | bucket_name = '' | |
58 | req = MDSearch(conn, bucket_name, req_str) | |
59 | result_keys = req.search(sort_key = lambda k: (k.bucket.name, k.name, k.version_id)) | |
60 | verify_search(bucket_name, src_keys, result_keys, src_filter) | |
61 | ||
62 | def init_env(create_obj, num_keys = 5, buckets_per_zone = 1, bucket_init_cb = None): | |
63 | check_es_configured() | |
64 | ||
65 | realm = get_realm() | |
66 | zonegroup = realm.master_zonegroup() | |
67 | zonegroup_conns = ZonegroupConns(zonegroup) | |
68 | buckets, zone_bucket = create_bucket_per_zone(zonegroup_conns, buckets_per_zone = buckets_per_zone) | |
69 | ||
70 | if bucket_init_cb: | |
71 | for zone_conn, bucket in zone_bucket: | |
72 | bucket_init_cb(zone_conn, bucket) | |
73 | ||
74 | src_keys = [] | |
75 | ||
76 | owner = None | |
77 | ||
78 | obj_prefix=''.join(random.choice(string.ascii_lowercase) for _ in range(6)) | |
79 | ||
80 | # don't wait for meta sync just yet | |
81 | for zone, bucket in zone_bucket: | |
e306af50 | 82 | for count in range(num_keys): |
31f18b77 FG |
83 | objname = obj_prefix + str(count) |
84 | k = new_key(zone, bucket.name, objname) | |
85 | # k.set_contents_from_string(content + 'x' * count) | |
86 | if not create_obj: | |
87 | continue | |
88 | ||
89 | create_obj(k, count) | |
90 | ||
91 | if not owner: | |
92 | for list_key in bucket.list_versions(): | |
93 | owner = list_key.owner | |
94 | break | |
95 | ||
96 | k = bucket.get_key(k.name, version_id = k.version_id) | |
97 | k.owner = owner # owner is not set when doing get_key() | |
98 | ||
99 | src_keys.append(k) | |
100 | ||
101 | zonegroup_meta_checkpoint(zonegroup) | |
102 | ||
103 | sources = [] | |
104 | targets = [] | |
105 | for target_conn in zonegroup_conns.zones: | |
106 | if not is_es_zone(target_conn): | |
107 | sources.append(target_conn) | |
108 | continue | |
109 | ||
110 | targets.append(target_conn) | |
111 | ||
112 | buckets = [] | |
113 | # make sure all targets are synced | |
114 | for source_conn, bucket in zone_bucket: | |
115 | buckets.append(bucket) | |
116 | for target_conn in targets: | |
117 | zone_bucket_checkpoint(target_conn.zone, source_conn.zone, bucket.name) | |
118 | ||
119 | return targets, sources, buckets, src_keys | |
120 | ||
121 | def test_es_object_search(): | |
122 | min_size = 10 | |
123 | content = 'a' * min_size | |
124 | ||
125 | def create_obj(k, i): | |
126 | k.set_contents_from_string(content + 'x' * i) | |
127 | ||
128 | targets, _, buckets, src_keys = init_env(create_obj, num_keys = 5, buckets_per_zone = 2) | |
129 | ||
130 | for target_conn in targets: | |
131 | ||
132 | # bucket checks | |
133 | for bucket in buckets: | |
134 | # check name | |
135 | do_check_mdsearch(target_conn.conn, None, src_keys , 'bucket == ' + bucket.name, lambda k: k.bucket.name == bucket.name) | |
136 | do_check_mdsearch(target_conn.conn, bucket, src_keys , 'bucket == ' + bucket.name, lambda k: k.bucket.name == bucket.name) | |
137 | ||
138 | # check on all buckets | |
139 | for key in src_keys: | |
140 | # limiting to checking specific key name, otherwise could get results from | |
141 | # other runs / tests | |
142 | do_check_mdsearch(target_conn.conn, None, src_keys , 'name == ' + key.name, lambda k: k.name == key.name) | |
143 | ||
144 | # check on specific bucket | |
145 | for bucket in buckets: | |
146 | for key in src_keys: | |
147 | do_check_mdsearch(target_conn.conn, bucket, src_keys , 'name < ' + key.name, lambda k: k.name < key.name) | |
148 | do_check_mdsearch(target_conn.conn, bucket, src_keys , 'name <= ' + key.name, lambda k: k.name <= key.name) | |
149 | do_check_mdsearch(target_conn.conn, bucket, src_keys , 'name == ' + key.name, lambda k: k.name == key.name) | |
150 | do_check_mdsearch(target_conn.conn, bucket, src_keys , 'name >= ' + key.name, lambda k: k.name >= key.name) | |
151 | do_check_mdsearch(target_conn.conn, bucket, src_keys , 'name > ' + key.name, lambda k: k.name > key.name) | |
152 | ||
153 | do_check_mdsearch(target_conn.conn, bucket, src_keys , 'name == ' + src_keys[0].name + ' or name >= ' + src_keys[2].name, | |
154 | lambda k: k.name == src_keys[0].name or k.name >= src_keys[2].name) | |
155 | ||
156 | # check etag | |
157 | for key in src_keys: | |
158 | do_check_mdsearch(target_conn.conn, bucket, src_keys , 'etag < ' + key.etag[1:-1], lambda k: k.etag < key.etag) | |
159 | for key in src_keys: | |
160 | do_check_mdsearch(target_conn.conn, bucket, src_keys , 'etag == ' + key.etag[1:-1], lambda k: k.etag == key.etag) | |
161 | for key in src_keys: | |
162 | do_check_mdsearch(target_conn.conn, bucket, src_keys , 'etag > ' + key.etag[1:-1], lambda k: k.etag > key.etag) | |
163 | ||
164 | # check size | |
165 | for key in src_keys: | |
166 | do_check_mdsearch(target_conn.conn, bucket, src_keys , 'size < ' + str(key.size), lambda k: k.size < key.size) | |
167 | for key in src_keys: | |
168 | do_check_mdsearch(target_conn.conn, bucket, src_keys , 'size <= ' + str(key.size), lambda k: k.size <= key.size) | |
169 | for key in src_keys: | |
170 | do_check_mdsearch(target_conn.conn, bucket, src_keys , 'size == ' + str(key.size), lambda k: k.size == key.size) | |
171 | for key in src_keys: | |
172 | do_check_mdsearch(target_conn.conn, bucket, src_keys , 'size >= ' + str(key.size), lambda k: k.size >= key.size) | |
173 | for key in src_keys: | |
174 | do_check_mdsearch(target_conn.conn, bucket, src_keys , 'size > ' + str(key.size), lambda k: k.size > key.size) | |
175 | ||
176 | def date_from_str(s): | |
177 | return dateutil.parser.parse(s) | |
178 | ||
179 | def test_es_object_search_custom(): | |
180 | min_size = 10 | |
181 | content = 'a' * min_size | |
182 | ||
183 | def bucket_init(zone_conn, bucket): | |
184 | req = MDSearchConfig(zone_conn.conn, bucket.name) | |
185 | req.set_config('x-amz-meta-foo-str; string, x-amz-meta-foo-int; int, x-amz-meta-foo-date; date') | |
186 | ||
187 | def create_obj(k, i): | |
188 | date = datetime.datetime.now() + datetime.timedelta(seconds=1) * i | |
189 | date_str = date.strftime('%Y-%m-%dT%H:%M:%S.%f')[:-3] + 'Z' | |
190 | k.set_contents_from_string(content + 'x' * i, headers = { 'X-Amz-Meta-Foo-Str': str(i * 5), | |
191 | 'X-Amz-Meta-Foo-Int': str(i * 5), | |
192 | 'X-Amz-Meta-Foo-Date': date_str}) | |
193 | ||
194 | targets, _, buckets, src_keys = init_env(create_obj, num_keys = 5, buckets_per_zone = 1, bucket_init_cb = bucket_init) | |
195 | ||
196 | ||
197 | for target_conn in targets: | |
198 | ||
199 | # bucket checks | |
200 | for bucket in buckets: | |
201 | str_vals = [] | |
202 | for key in src_keys: | |
203 | # check string values | |
204 | val = key.get_metadata('foo-str') | |
205 | str_vals.append(val) | |
206 | do_check_mdsearch(target_conn.conn, bucket, src_keys , 'x-amz-meta-foo-str < ' + val, lambda k: k.get_metadata('foo-str') < val) | |
207 | do_check_mdsearch(target_conn.conn, bucket, src_keys , 'x-amz-meta-foo-str <= ' + val, lambda k: k.get_metadata('foo-str') <= val) | |
208 | do_check_mdsearch(target_conn.conn, bucket, src_keys , 'x-amz-meta-foo-str == ' + val, lambda k: k.get_metadata('foo-str') == val) | |
209 | do_check_mdsearch(target_conn.conn, bucket, src_keys , 'x-amz-meta-foo-str >= ' + val, lambda k: k.get_metadata('foo-str') >= val) | |
210 | do_check_mdsearch(target_conn.conn, bucket, src_keys , 'x-amz-meta-foo-str > ' + val, lambda k: k.get_metadata('foo-str') > val) | |
211 | ||
212 | # check int values | |
213 | sval = key.get_metadata('foo-int') | |
214 | val = int(sval) | |
215 | do_check_mdsearch(target_conn.conn, bucket, src_keys , 'x-amz-meta-foo-int < ' + sval, lambda k: int(k.get_metadata('foo-int')) < val) | |
216 | do_check_mdsearch(target_conn.conn, bucket, src_keys , 'x-amz-meta-foo-int <= ' + sval, lambda k: int(k.get_metadata('foo-int')) <= val) | |
217 | do_check_mdsearch(target_conn.conn, bucket, src_keys , 'x-amz-meta-foo-int == ' + sval, lambda k: int(k.get_metadata('foo-int')) == val) | |
218 | do_check_mdsearch(target_conn.conn, bucket, src_keys , 'x-amz-meta-foo-int >= ' + sval, lambda k: int(k.get_metadata('foo-int')) >= val) | |
219 | do_check_mdsearch(target_conn.conn, bucket, src_keys , 'x-amz-meta-foo-int > ' + sval, lambda k: int(k.get_metadata('foo-int')) > val) | |
220 | ||
221 | # check int values | |
222 | sval = key.get_metadata('foo-date') | |
223 | val = date_from_str(sval) | |
224 | do_check_mdsearch(target_conn.conn, bucket, src_keys , 'x-amz-meta-foo-date < ' + sval, lambda k: date_from_str(k.get_metadata('foo-date')) < val) | |
225 | do_check_mdsearch(target_conn.conn, bucket, src_keys , 'x-amz-meta-foo-date <= ' + sval, lambda k: date_from_str(k.get_metadata('foo-date')) <= val) | |
226 | do_check_mdsearch(target_conn.conn, bucket, src_keys , 'x-amz-meta-foo-date == ' + sval, lambda k: date_from_str(k.get_metadata('foo-date')) == val) | |
227 | do_check_mdsearch(target_conn.conn, bucket, src_keys , 'x-amz-meta-foo-date >= ' + sval, lambda k: date_from_str(k.get_metadata('foo-date')) >= val) | |
228 | do_check_mdsearch(target_conn.conn, bucket, src_keys , 'x-amz-meta-foo-date > ' + sval, lambda k: date_from_str(k.get_metadata('foo-date')) > val) | |
229 | ||
230 | # 'or' query | |
e306af50 | 231 | for i in range(len(src_keys) // 2): |
31f18b77 FG |
232 | do_check_mdsearch(target_conn.conn, bucket, src_keys , 'x-amz-meta-foo-str <= ' + str_vals[i] + ' or x-amz-meta-foo-str >= ' + str_vals[-i], |
233 | lambda k: k.get_metadata('foo-str') <= str_vals[i] or k.get_metadata('foo-str') >= str_vals[-i] ) | |
234 | ||
235 | # 'and' query | |
e306af50 | 236 | for i in range(len(src_keys) // 2): |
31f18b77 FG |
237 | do_check_mdsearch(target_conn.conn, bucket, src_keys , 'x-amz-meta-foo-str >= ' + str_vals[i] + ' and x-amz-meta-foo-str <= ' + str_vals[i + 1], |
238 | lambda k: k.get_metadata('foo-str') >= str_vals[i] and k.get_metadata('foo-str') <= str_vals[i + 1] ) | |
239 | # more complicated query | |
e306af50 | 240 | for i in range(len(src_keys) // 2): |
31f18b77 FG |
241 | do_check_mdsearch(target_conn.conn, None, src_keys , 'bucket == ' + bucket.name + ' and x-amz-meta-foo-str >= ' + str_vals[i] + |
242 | ' and (x-amz-meta-foo-str <= ' + str_vals[i + 1] + ')', | |
243 | lambda k: k.bucket.name == bucket.name and (k.get_metadata('foo-str') >= str_vals[i] and | |
244 | k.get_metadata('foo-str') <= str_vals[i + 1]) ) | |
245 | ||
246 | def test_es_bucket_conf(): | |
247 | min_size = 0 | |
248 | ||
249 | def bucket_init(zone_conn, bucket): | |
250 | req = MDSearchConfig(zone_conn.conn, bucket.name) | |
251 | req.set_config('x-amz-meta-foo-str; string, x-amz-meta-foo-int; int, x-amz-meta-foo-date; date') | |
252 | ||
253 | targets, sources, buckets, _ = init_env(None, num_keys = 5, buckets_per_zone = 1, bucket_init_cb = bucket_init) | |
254 | ||
255 | for source_conn in sources: | |
256 | for bucket in buckets: | |
257 | req = MDSearchConfig(source_conn.conn, bucket.name) | |
258 | conf = req.get_config() | |
259 | ||
260 | d = {} | |
261 | ||
262 | for entry in conf: | |
263 | d[entry['Key']] = entry['Type'] | |
264 | ||
265 | eq(len(d), 3) | |
266 | eq(d['x-amz-meta-foo-str'], 'str') | |
267 | eq(d['x-amz-meta-foo-int'], 'int') | |
268 | eq(d['x-amz-meta-foo-date'], 'date') | |
269 | ||
270 | req.del_config() | |
271 | ||
272 | conf = req.get_config() | |
273 | ||
274 | eq(len(conf), 0) | |
275 | ||
276 | break # no need to iterate over all zones |