]>
Commit | Line | Data |
---|---|---|
05a536ef TL |
1 | #!/usr/bin/env python3 |
2 | ||
3 | import errno | |
4 | import subprocess | |
5 | import logging as log | |
6 | import boto3 | |
7 | import botocore.exceptions | |
aee94f69 TL |
8 | import random |
9 | import json | |
10 | from time import sleep | |
05a536ef TL |
11 | |
12 | log.basicConfig(format = '%(message)s', level=log.DEBUG) | |
13 | log.getLogger('botocore').setLevel(log.CRITICAL) | |
14 | log.getLogger('boto3').setLevel(log.CRITICAL) | |
15 | log.getLogger('urllib3').setLevel(log.CRITICAL) | |
16 | ||
17 | def exec_cmd(cmd, wait = True, **kwargs): | |
18 | check_retcode = kwargs.pop('check_retcode', True) | |
19 | kwargs['shell'] = True | |
20 | kwargs['stdout'] = subprocess.PIPE | |
21 | proc = subprocess.Popen(cmd, **kwargs) | |
22 | log.info(proc.args) | |
23 | if wait: | |
24 | out, _ = proc.communicate() | |
25 | if check_retcode: | |
26 | assert(proc.returncode == 0) | |
27 | return out | |
28 | return (out, proc.returncode) | |
29 | return '' | |
30 | ||
31 | def create_user(uid, display_name, access_key, secret_key): | |
32 | _, ret = exec_cmd(f'radosgw-admin user create --uid {uid} --display-name "{display_name}" --access-key {access_key} --secret {secret_key}', check_retcode=False) | |
33 | assert(ret == 0 or errno.EEXIST) | |
34 | ||
35 | def boto_connect(access_key, secret_key, config=None): | |
36 | def try_connect(portnum, ssl, proto): | |
37 | endpoint = proto + '://localhost:' + portnum | |
38 | conn = boto3.resource('s3', | |
39 | aws_access_key_id=access_key, | |
40 | aws_secret_access_key=secret_key, | |
41 | use_ssl=ssl, | |
42 | endpoint_url=endpoint, | |
43 | verify=False, | |
44 | config=config, | |
45 | ) | |
46 | try: | |
47 | list(conn.buckets.limit(1)) # just verify we can list buckets | |
48 | except botocore.exceptions.ConnectionError as e: | |
49 | print(e) | |
50 | raise | |
51 | print('connected to', endpoint) | |
52 | return conn | |
53 | try: | |
54 | return try_connect('80', False, 'http') | |
55 | except botocore.exceptions.ConnectionError: | |
56 | try: # retry on non-privileged http port | |
57 | return try_connect('8000', False, 'http') | |
58 | except botocore.exceptions.ConnectionError: | |
59 | # retry with ssl | |
60 | return try_connect('443', True, 'https') | |
aee94f69 TL |
61 | |
62 | def put_objects(bucket, key_list): | |
63 | objs = [] | |
64 | for key in key_list: | |
65 | o = bucket.put_object(Key=key, Body=b"some_data") | |
66 | objs.append((o.key, o.version_id)) | |
67 | return objs | |
68 | ||
69 | def create_unlinked_objects(conn, bucket, key_list): | |
70 | # creates an unlinked/unlistable object for each key in key_list | |
71 | ||
72 | object_versions = [] | |
73 | try: | |
74 | exec_cmd('ceph config set client rgw_debug_inject_set_olh_err 2') | |
75 | exec_cmd('ceph config set client rgw_debug_inject_olh_cancel_modification_err true') | |
76 | sleep(1) | |
77 | for key in key_list: | |
78 | tag = str(random.randint(0, 1_000_000)) | |
79 | try: | |
80 | bucket.put_object(Key=key, Body=b"some_data", Metadata = { | |
81 | 'tag': tag, | |
82 | }) | |
83 | except Exception as e: | |
84 | log.debug(e) | |
85 | out = exec_cmd(f'radosgw-admin bi list --bucket {bucket.name} --object {key}') | |
86 | instance_entries = filter( | |
87 | lambda x: x['type'] == 'instance', | |
88 | json.loads(out.replace(b'\x80', b'0x80'))) | |
89 | found = False | |
90 | for ie in instance_entries: | |
91 | instance_id = ie['entry']['instance'] | |
92 | ov = conn.ObjectVersion(bucket.name, key, instance_id).head() | |
93 | if ov['Metadata'] and ov['Metadata']['tag'] == tag: | |
94 | object_versions.append((key, instance_id)) | |
95 | found = True | |
96 | break | |
97 | if not found: | |
98 | raise Exception(f'failed to create unlinked object for key={key}') | |
99 | finally: | |
100 | exec_cmd('ceph config rm client rgw_debug_inject_set_olh_err') | |
101 | exec_cmd('ceph config rm client rgw_debug_inject_olh_cancel_modification_err') | |
102 | return object_versions | |
103 |